updates 5.1.0

This commit is contained in:
VineFeeder
2026-06-04 13:12:55 +01:00
parent 74782f0f3f
commit 7e10493609
82 changed files with 13340 additions and 2370 deletions
+13 -16
View File
@@ -4,7 +4,7 @@ build-backend = "uv_build"
[project]
name = "envied"
version = "4.0.0"
version = "5.1.0"
description = "Modular Movie, TV, and Music Archival Software."
authors = [{ name = "rlaphoenix and others" }]
requires-python = ">=3.10,<=3.14"
@@ -26,7 +26,6 @@ classifiers = [
"Topic :: Security :: Cryptography",
]
dependencies = [
"appdirs>=1.4.4,<2",
"Brotli>=1.1.0,<2",
"click>=8.1.8,<9",
@@ -36,19 +35,18 @@ dependencies = [
"fonttools>=4.60.2,<5",
"jsonpickle>=3.0.4,<5",
"langcodes>=3.4.0,<4",
"lxml>=6.1.0,<7",
"pproxy>=2.7.9,<3",
"lxml>=5.2.1,<7",
"protobuf>=4.25.3,<7",
"pycaption>=2.2.6,<3",
"pycryptodomex>=3.20.0,<4",
"pyjwt>=2.8.0,<3",
"pyjwt>=2.12.0,<3",
"pymediainfo>=6.1.0,<8",
"pymp4>=1.4.0,<2",
"pymysql>=1.1.0,<2",
"pywidevine[serve]>=1.8.0,<2",
"PyYAML>=6.0.1,<7",
"requests[socks]>=2.32.5,<3",
"rich>=13.7.1,<15",
"rich>=13.7.1,<15.1",
"rlaphoenix.m3u8>=3.4.0,<4",
"ruamel.yaml>=0.18.6,<0.19",
"sortedcontainers>=2.4.0,<3",
@@ -56,12 +54,10 @@ dependencies = [
"Unidecode>=1.3.8,<2",
"urllib3>=2.6.3,<3",
"chardet>=5.2.0,<6",
"curl-cffi>=0.7.0b4,<0.14",
"pyplayready>=0.8.3,<0.9",
"httpx>=0.28.1,<0.29",
"cryptography>=45.0.0,<47",
"subby",
"aiohttp>=3.13.3,<4",
"aiohttp>=3.13.4,<4",
"aiohttp-swagger3>=0.9.0,<1",
"pysubs2>=1.7.0,<2",
"PyExecJS>=1.5.1,<2",
@@ -69,13 +65,14 @@ dependencies = [
"language-data>=1.4.0",
"wasmtime>=41.0.0",
"animeapi-py>=0.6.0",
# envied specifix
"webvtt-py>=0.5.1,<0.6",
"isodate>=0.7.2,<0.8",
"scrapy>=2.14.0,<3",
"mypy>=1.19.1,<2",
"rnet>=2.4.2",
"bandit>=1.9.4",
# envied specific dependencies
"webvtt-py>=0.5.1,<0.6",
"isodate>=0.7.2,<0.8",
"scrapy>=2.14.0,<3",
"mypy>=1.19.1,<2",
"httpx>=0.28.0,<0.35",
# Keeping your existing cap style here; latest visible is 0.10.0.
File diff suppressed because it is too large Load Diff
@@ -68,15 +68,6 @@ def check() -> None:
"desc": "HDR10+ metadata",
"cat": "HDR",
},
# Downloaders
{"name": "aria2c", "binary": binaries.Aria2, "required": False, "desc": "Multi-thread DL", "cat": "Download"},
{
"name": "N_m3u8DL-RE",
"binary": binaries.N_m3u8DL_RE,
"required": False,
"desc": "HLS/DASH/ISM",
"cat": "Download",
},
# Subtitle Tools
{
"name": "SubtitleEdit",
@@ -0,0 +1,54 @@
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
import click
from envied.commands.dl import dl
from envied.core.constants import context_settings
class ImportCommand:
@staticmethod
@click.command(
name="import",
short_help="Reconstruct a download (download, decrypt, mux) from an --export JSON file.",
context_settings={**context_settings, "ignore_unknown_options": True},
)
@click.argument("export_file", type=Path)
@click.argument("dl_args", nargs=-1, type=click.UNPROCESSED)
@click.pass_context
def cli(ctx: click.Context, export_file: Path, dl_args: tuple[str, ...]) -> None:
"""
Reconstruct an exported download without re-contacting the service.
Re-fetches the manifest, injects the stored keys, then downloads/decrypts/muxes as a
normal `dl` run. Any `dl` options after the file are forwarded verbatim:
unshackle import export.json -r HDR10 --proxy US
"""
if not export_file.is_file():
raise click.ClickException(f"Export file not found: {export_file}")
try:
data: dict[str, Any] = json.loads(export_file.read_text(encoding="utf8"))
except json.JSONDecodeError as e:
raise click.ClickException(f"Export file is not valid JSON: {e}")
if data.get("version") != 2:
raise click.ClickException(
f"Unsupported export version {data.get('version')!r}. "
"Re-create the export with a current build of envied."
)
service_tag = data.get("service")
if not service_tag:
raise click.ClickException("Export file is missing the 'service' tag.")
args = [*dl_args, "--import", str(export_file), service_tag]
dl.cli.main(args=args, prog_name="unshackle dl", standalone_mode=False)
globals()["import"] = ImportCommand
+126 -5
View File
@@ -4,8 +4,12 @@ from pathlib import Path
from typing import Optional
import click
from rich.padding import Padding
from rich.text import Text
from rich.tree import Tree
from envied.core.config import config
from envied.core.console import console
from envied.core.constants import context_settings
from envied.core.services import Services
from envied.core.vault import Vault
@@ -77,7 +81,19 @@ def kv() -> None:
@click.argument("to_vault_name", type=str)
@click.argument("from_vault_names", nargs=-1, type=click.UNPROCESSED)
@click.option("-s", "--service", type=str, default=None, help="Only copy data to and from a specific service.")
def copy(to_vault_name: str, from_vault_names: list[str], service: Optional[str] = None) -> None:
@click.option(
"-l",
"--local-only",
is_flag=True,
default=False,
help="Only copy data for services installed locally (skip vault tables for services not present in the configured services path).",
)
def copy(
to_vault_name: str,
from_vault_names: list[str],
service: Optional[str] = None,
local_only: bool = False,
) -> None:
"""
Copy data from multiple Key Vaults into a single Key Vault.
Rows with matching KIDs are skipped unless there's no KEY set.
@@ -103,13 +119,28 @@ def copy(to_vault_name: str, from_vault_names: list[str], service: Optional[str]
vault_names = ", ".join([v.name for v in from_vaults])
log.info(f"Copying data from {vault_names}{to_vault.name}")
if service and local_only:
raise click.UsageError("--service and --local-only are mutually exclusive.")
if service:
service = Services.get_tag(service)
log.info(f"Filtering by service: {service}")
installed: Optional[set[str]] = None
if local_only:
installed = {t.upper() for t in Services.get_tags()}
log.info(f"Filtering by locally installed services ({len(installed)} found)")
total_added = 0
for from_vault in from_vaults:
services_to_copy = [service] if service else from_vault.get_services()
services_to_copy = [service] if service else list(from_vault.get_services())
if installed is not None:
before = len(services_to_copy)
services_to_copy = [s for s in services_to_copy if s and s.upper() in installed]
skipped = before - len(services_to_copy)
if skipped:
log.info(f"{from_vault.name}: skipping {skipped} service(s) not installed locally")
for service_tag in services_to_copy:
added = copy_service_data(to_vault, from_vault, service_tag, log)
@@ -124,8 +155,20 @@ def copy(to_vault_name: str, from_vault_names: list[str], service: Optional[str]
@kv.command()
@click.argument("vaults", nargs=-1, type=click.UNPROCESSED)
@click.option("-s", "--service", type=str, default=None, help="Only sync data to and from a specific service.")
@click.option(
"-l",
"--local-only",
is_flag=True,
default=False,
help="Only sync data for services installed locally (skip vault tables for services not present in the configured services path).",
)
@click.pass_context
def sync(ctx: click.Context, vaults: list[str], service: Optional[str] = None) -> None:
def sync(
ctx: click.Context,
vaults: list[str],
service: Optional[str] = None,
local_only: bool = False,
) -> None:
"""
Ensure multiple Key Vaults copies of all keys as each other.
It's essentially just a bi-way copy between each vault.
@@ -135,9 +178,21 @@ def sync(ctx: click.Context, vaults: list[str], service: Optional[str] = None) -
if not len(vaults) > 1:
raise click.ClickException("You must provide more than one Vault to sync.")
ctx.invoke(copy, to_vault_name=vaults[0], from_vault_names=vaults[1:], service=service)
ctx.invoke(
copy,
to_vault_name=vaults[0],
from_vault_names=vaults[1:],
service=service,
local_only=local_only,
)
for i in range(1, len(vaults)):
ctx.invoke(copy, to_vault_name=vaults[i], from_vault_names=[vaults[i - 1]], service=service)
ctx.invoke(
copy,
to_vault_name=vaults[i],
from_vault_names=[vaults[i - 1]],
service=service,
local_only=local_only,
)
@kv.command()
@@ -188,6 +243,72 @@ def add(file: Path, service: str, vaults: list[str]) -> None:
log.info("Done!")
@kv.command()
@click.argument("kid", type=str)
@click.option("-s", "--service", type=str, default=None, help="Limit search to a specific service tag.")
@click.option(
"-v", "--vault", "vault_name", type=str, default=None, help="Limit search to a specific configured vault by name."
)
def search(kid: str, service: Optional[str], vault_name: Optional[str]) -> None:
"""
Search configured Key Vault(s) for a KID and report any matching KEY.
KID must be 32 hex characters (no dashes). If --service is omitted, every
service table in each vault is scanned. If --vault is omitted, every
vault in the config is searched.
"""
log = logging.getLogger("kv")
kid_norm = kid.replace("-", "").lower()
if not re.fullmatch(r"[0-9a-f]{32}", kid_norm):
raise click.ClickException(f"KID '{kid}' is not 32 hex characters.")
if vault_name:
vault_names = [vault_name]
else:
vault_names = [v["name"] for v in config.key_vaults]
if not vault_names:
raise click.ClickException("No Key Vaults are configured.")
vaults_ = load_vaults(vault_names)
service_tag = Services.get_tag(service) if service else None
hit: Optional[tuple[str, str, str]] = None
for vault in vaults_:
if service_tag:
services_to_check: list[str] = [service_tag]
else:
try:
services_to_check = list(vault.get_services())
except Exception as e:
log.debug(f"{vault}: get_services() failed ({e})")
services_to_check = []
if not services_to_check:
log.warning(f"{vault}: cannot search without a service (remote vault requires --service). Skipping.")
continue
for svc in services_to_check:
try:
key = vault.get_key(kid_norm, svc)
except Exception as e:
log.debug(f"{vault} [{svc}]: lookup error ({e})")
continue
if key and key.count("0") != len(key):
hit = (vault.name, svc, key)
break
if hit:
break
if hit:
vname, svc, key = hit
tree = Tree(Text.assemble((svc, "cyan"), (f"({vname})", "text"), overflow="fold"))
tree.add(f"[text2]{kid_norm}:{key}")
console.print(Padding(tree, (1, 5)))
else:
log.info(f"KID {kid_norm} not found in {len(vaults_)} vault(s).")
@kv.command()
@click.argument("vaults", nargs=-1, type=click.UNPROCESSED)
def prepare(vaults: list[str]) -> None:
@@ -86,7 +86,9 @@ def search(ctx: click.Context, no_proxy: bool, profile: Optional[str] = None, pr
# requesting proxy from a specific proxy provider
requested_provider, proxy = proxy.split(":", maxsplit=1)
# Match simple region codes (us, ca, uk1) or provider:region format (nordvpn:ca, windscribe:us)
if re.match(r"^[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE) or re.match(r"^[a-z]+:[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE):
if re.match(r"^[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE) or re.match(
r"^[a-z]+:[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE
):
proxy = proxy.lower()
with console.status(f"Getting a Proxy to {proxy}...", spinner="dots"):
if requested_provider:
+89 -13
View File
@@ -6,6 +6,7 @@ from aiohttp import web
from envied.core import binaries
from envied.core.api import cors_middleware, setup_routes, setup_swagger
from envied.core.api.compression import compression_middleware
from envied.core.config import config
from envied.core.constants import context_settings
@@ -35,6 +36,12 @@ from envied.core.constants import context_settings
default=False,
help="Enable debug logging for API operations.",
)
@click.option(
"--remote-only",
is_flag=True,
default=False,
help="Only expose remote service session endpoints (health, services, search, session).",
)
def serve(
host: str,
port: int,
@@ -45,6 +52,7 @@ def serve(
no_key: bool,
debug_api: bool,
debug: bool,
remote_only: bool,
) -> None:
"""
Serve your Local Widevine and PlayReady Devices and REST API for Remote Access.
@@ -119,19 +127,64 @@ def serve(
config.serve["playready_devices"] = []
config.serve["playready_devices"].extend(list(config.directories.prds.glob("*.prd")))
@web.middleware
async def api_key_authentication(request: web.Request, handler) -> web.Response:
"""Authenticate API requests using X-Secret-Key header."""
if request.path == "/api/health":
return await handler(request)
secret_key = request.headers.get("X-Secret-Key")
if not secret_key:
return web.json_response({"status": 401, "message": "Secret Key is Empty."}, status=401)
if secret_key not in request.app["config"]["users"]:
return web.json_response({"status": 401, "message": "Secret Key is Invalid."}, status=401)
return await handler(request)
remote_only = remote_only or config.serve.get("remote_only", False)
if remote_only:
api_only = True
global_services = config.serve.get("services")
if global_services:
log.info(f"Global service allowlist: {', '.join(global_services)}")
users = config.serve.get("users", {})
for user_key, user_cfg in users.items() if isinstance(users, dict) else []:
user_services = user_cfg.get("services") if isinstance(user_cfg, dict) else None
if user_services:
username = user_cfg.get("username", user_key[:8] + "...")
log.info(f"User '{username}' restricted to services: {', '.join(user_services)}")
if api_only:
log.info("Starting REST API server (pywidevine/pyplayready CDM disabled)")
if no_key:
app = web.Application(middlewares=[cors_middleware])
app = web.Application(middlewares=[cors_middleware, compression_middleware])
app["config"] = {"users": {}}
else:
app = web.Application(middlewares=[cors_middleware, pywidevine_serve.authentication])
app = web.Application(middlewares=[cors_middleware, api_key_authentication, compression_middleware])
app["config"] = {"users": {api_secret: {"devices": [], "username": "api_user"}}}
app["debug_api"] = debug_api
setup_routes(app)
setup_swagger(app)
log.info(f"REST API endpoints available at http://{host}:{port}/api/")
log.info(f"Swagger UI available at http://{host}:{port}/api/docs/")
# Start session cleanup loop for remote-dl sessions
from envied.core.api.session_store import get_session_store
session_store = get_session_store()
async def start_session_cleanup(_app: web.Application) -> None:
await session_store.start_cleanup_loop()
async def stop_session_cleanup(_app: web.Application) -> None:
await session_store.cancel_all_bridges()
await session_store.stop_cleanup_loop()
app.on_startup.append(start_session_cleanup)
app.on_cleanup.append(stop_session_cleanup)
setup_routes(app, remote_only=remote_only)
if not remote_only:
setup_swagger(app)
log.info(f"REST API endpoints available at http://{host}:{port}/api/")
log.info(f"Swagger UI available at http://{host}:{port}/api/docs/")
else:
log.info(f"Remote service endpoints available at http://{host}:{port}/api/session/")
log.info("(Press CTRL+C to quit)")
web.run_app(app, host=host, port=port, print=None)
else:
@@ -181,20 +234,39 @@ def serve(
if serve_playready_flag and request.path.startswith("/playready"):
from pyplayready import __version__ as pyplayready_version
response.headers["Server"] = f"https://git.gay/ready-dl/pyplayready serve v{pyplayready_version}"
response.headers["Server"] = (
f"https://git.gay/ready-dl/pyplayready serve v{pyplayready_version}"
)
return response
return serve_authentication
if no_key:
app = web.Application(middlewares=[cors_middleware])
app = web.Application(middlewares=[cors_middleware, compression_middleware])
else:
serve_auth = create_serve_authentication(serve_playready and bool(prd_devices))
app = web.Application(middlewares=[cors_middleware, serve_auth])
app = web.Application(middlewares=[cors_middleware, serve_auth, compression_middleware])
app["config"] = serve_config
app["debug_api"] = debug_api
# Start session cleanup loop for remote-dl sessions
from envied.core.api.session_store import get_session_store
session_store = get_session_store()
async def start_session_cleanup(_app: web.Application) -> None:
await session_store.start_cleanup_loop()
async def stop_session_cleanup(_app: web.Application) -> None:
await session_store.cancel_all_bridges()
await session_store.stop_cleanup_loop()
app.on_startup.append(start_session_cleanup)
app.on_cleanup.append(stop_session_cleanup)
if serve_widevine:
app.on_startup.append(pywidevine_serve._startup)
app.on_cleanup.append(pywidevine_serve._cleanup)
@@ -226,6 +298,7 @@ def serve(
async def playready_ping(_: web.Request) -> web.Response:
from pyplayready import __version__ as pyplayready_version
response = web.json_response({"message": "OK"})
response.headers["Server"] = f"https://git.gay/ready-dl/pyplayready serve v{pyplayready_version}"
return response
@@ -237,13 +310,16 @@ def serve(
elif serve_playready:
log.info("No PlayReady devices found, skipping PlayReady CDM endpoints")
setup_routes(app)
setup_swagger(app)
setup_routes(app, remote_only=remote_only)
if serve_widevine:
log.info(f"Widevine CDM endpoints available at http://{host}:{port}/{{device}}/open")
log.info(f"REST API endpoints available at http://{host}:{port}/api/")
log.info(f"Swagger UI available at http://{host}:{port}/api/docs/")
if remote_only:
log.info(f"Remote service endpoints available at http://{host}:{port}/api/session/")
else:
setup_swagger(app)
log.info(f"REST API endpoints available at http://{host}:{port}/api/")
log.info(f"Swagger UI available at http://{host}:{port}/api/docs/")
log.info("(Press CTRL+C to quit)")
web.run_app(app, host=host, port=port, print=None)
finally:
+1 -1
View File
@@ -1 +1 @@
__version__ = "4.0.0"
__version__ = "5.1.0"
+15 -14
View File
@@ -49,22 +49,23 @@ def main(version: bool, debug: bool) -> None:
traceback.install(console=console, width=80, suppress=[click])
console.print(
Padding(
Group(
Text(
r"░█▀▀░█▀█░█░█░▀█▀░█▀▀░█▀▄" + "\n"
r"░█▀▀░█░█░▀▄▀░░█░░█▀▀░█░█" + "\n"
r"░▀▀▀░▀░▀░░▀░░▀▀▀░▀▀▀░▀▀░" + "\n" ,
style="ascii.art",
Padding(
Group(
Text(
r"░█▀▀░█▀█░█░█░▀█▀░█▀▀░█▀▄" + "\n"
r"░█▀▀░█░█░▀▄▀░░█░░█▀▀░█░█" + "\n"
r"░▀▀▀░▀░▀░░▀░░▀▀▀░▀▀▀░▀▀░" + "\n" ,
style="ascii.art",
),
Text(" and more than unshackled...", style = "ascii.art"),
f"\nv [repr.number]{__version__}[/] - https://github.com/vinefeeder/envied",
),
Text(" and more than unshackled...", style = "ascii.art"),
f"\nv [repr.number]{__version__}[/] - https://github.com/vinefeeder/envied",
(1, 11, 1, 10),
expand=True,
),
(1, 11, 1, 10),
expand=True,
),
justify="center",
)
justify="center",
)
@atexit.register
@@ -0,0 +1,40 @@
"""aiohttp middleware for gzip transport compression."""
from __future__ import annotations
import gzip
from aiohttp import web
@web.middleware
async def compression_middleware(request: web.Request, handler) -> web.Response:
"""Compress JSON responses with gzip when the client supports it."""
response = await handler(request)
accept_encoding = request.headers.get("Accept-Encoding", "")
if "gzip" not in accept_encoding:
return response
if response.content_type and "json" not in response.content_type:
return response
body = response.body
if body is None or len(body) < 256:
return response
from envied.core.config import config
level = config.serve.get("compression_level", 1)
if not level:
return response
compressed = gzip.compress(body, compresslevel=level)
headers = dict(response.headers)
headers["Content-Encoding"] = "gzip"
headers["Content-Length"] = str(len(compressed))
return web.Response(
body=compressed,
status=response.status,
headers=headers,
)
@@ -2,6 +2,7 @@ import asyncio
import json
import logging
import os
import re
import sys
import tempfile
import threading
@@ -16,6 +17,11 @@ from typing import Any, Callable, Dict, List, Optional
log = logging.getLogger("download_manager")
def _sanitize_log(value: object) -> str:
"""Sanitize a value for safe logging by removing newlines and control characters."""
return str(value).replace("\n", "").replace("\r", "").replace("\x00", "")
class JobStatus(Enum):
QUEUED = "queued"
DOWNLOADING = "downloading"
@@ -141,8 +147,37 @@ def _perform_download(
sub_map.update({c.value.upper(): c for c in Subtitle.Codec})
params["sub_format"] = sub_map.get(sub_format_raw.upper())
if params.get("export") and isinstance(params["export"], str):
params["export"] = Path(params["export"])
if params.get("export"):
params["export"] = bool(params["export"])
# Normalize slow: accept string "MIN-MAX", list/tuple, or True (default 60-120)
slow_raw = params.get("slow")
if slow_raw is not None and not isinstance(slow_raw, tuple):
if isinstance(slow_raw, bool):
params["slow"] = (60, 120) if slow_raw else None
elif isinstance(slow_raw, list) and len(slow_raw) == 2:
params["slow"] = (int(slow_raw[0]), int(slow_raw[1]))
elif isinstance(slow_raw, str):
from envied.core.utils.click_types import SLOW_DELAY_RANGE
try:
params["slow"] = SLOW_DELAY_RANGE.convert(slow_raw, None, None)
except click.BadParameter as exc:
raise Exception(f"Invalid slow parameter: {exc}")
# Convert wanted episode strings to internal "SxE" format
# Accepts: "S01E01", "S01-S03", "s1e1", "1x1", or already-parsed format
wanted_raw = params.get("wanted")
if wanted_raw:
from envied.core.utils.click_types import SeasonRange
if isinstance(wanted_raw, str):
wanted_raw = [wanted_raw]
# Only convert if not already in internal "SxE" format
needs_conversion = any(not re.match(r"^\d+x\d+$", w) for w in wanted_raw)
if needs_conversion:
season_range = SeasonRange()
params["wanted"] = season_range.parse_tokens(*wanted_raw)
# Load service configuration
service_config_path = Services.get_path(service) / config.filenames.config
@@ -156,10 +191,14 @@ def _perform_download(
ctx = click.Context(dl_command.cli)
ctx.invoked_subcommand = service
ctx.obj = ContextData(config=service_config, cdm=None, proxy_providers=[], profile=params.get("profile"))
from envied.core.api.handlers import load_full_cdm
cdm = load_full_cdm(service, params.get("profile"), params.get("cdm_type"))
ctx.obj = ContextData(config=service_config, cdm=cdm, proxy_providers=[], profile=params.get("profile"))
ctx.params = {
"proxy": params.get("proxy"),
"no_proxy": params.get("no_proxy", False),
"no_proxy_download": params.get("no_proxy_download", False),
"profile": params.get("profile"),
"repack": params.get("repack", False),
"tag": params.get("tag"),
@@ -266,6 +305,8 @@ def _perform_download(
acodec=params.get("acodec"),
vbitrate=params.get("vbitrate"),
abitrate=params.get("abitrate"),
vbitrate_range=params.get("vbitrate_range"),
abitrate_range=params.get("abitrate_range"),
range_=params.get("range", ["SDR"]),
channels=params.get("channels"),
no_atmos=params.get("no_atmos", False),
@@ -289,18 +330,20 @@ def _perform_download(
no_chapters=params.get("no_chapters", False),
no_video=params.get("no_video", False),
audio_description=params.get("audio_description", False),
slow=params.get("slow", False),
slow=params.get("slow", None),
list_=False,
list_titles=False,
skip_dl=params.get("skip_dl", False),
export=params.get("export"),
cdm_only=params.get("cdm_only"),
no_proxy=params.get("no_proxy", False),
no_proxy_download=params.get("no_proxy_download", False),
no_folder=params.get("no_folder", False),
no_source=params.get("no_source", False),
no_mux=params.get("no_mux", False),
workers=params.get("workers"),
downloads=params.get("downloads", 1),
worst=params.get("worst", False),
best_available=params.get("best_available", False),
split_audio=params.get("split_audio"),
)
@@ -322,9 +365,10 @@ def _perform_download(
log.error(f"Stderr: {stderr_str}")
raise
log.info(f"Download completed for job {job_id}, files in {original_download_dir}")
output_files = [str(p) for p in dl_instance.completed_files]
log.info(f"Download completed for job {job_id}, {len(output_files)} file(s) in {original_download_dir}")
return []
return output_files
class DownloadQueueManager:
@@ -361,7 +405,7 @@ class DownloadQueueManager:
self._jobs[job_id] = job
self._job_queue.put_nowait(job)
log.info(f"Created download job {job_id} for {service}:{title_id}")
log.info(f"Created download job {job_id} for {_sanitize_log(service)}:{_sanitize_log(title_id)}")
return job
def get_job(self, job_id: str) -> Optional[DownloadJob]:
@@ -381,27 +425,27 @@ class DownloadQueueManager:
if job.status == JobStatus.QUEUED:
job.status = JobStatus.CANCELLED
job.cancel_event.set() # Signal cancellation
log.info(f"Cancelled queued job {job_id}")
log.info(f"Cancelled queued job {_sanitize_log(job_id)}")
return True
elif job.status == JobStatus.DOWNLOADING:
# Set the cancellation event first - this will be checked by the download thread
job.cancel_event.set()
job.status = JobStatus.CANCELLED
log.info(f"Signaled cancellation for downloading job {job_id}")
log.info(f"Signaled cancellation for downloading job {_sanitize_log(job_id)}")
# Cancel the active download task
task = self._active_downloads.get(job_id)
if task:
task.cancel()
log.info(f"Cancelled download task for job {job_id}")
log.info(f"Cancelled download task for job {_sanitize_log(job_id)}")
process = self._download_processes.get(job_id)
if process:
try:
process.terminate()
log.info(f"Terminated worker process for job {job_id}")
log.info(f"Terminated worker process for job {_sanitize_log(job_id)}")
except ProcessLookupError:
log.debug(f"Worker process for job {job_id} already exited")
log.debug(f"Worker process for job {_sanitize_log(job_id)} already exited")
return True
@@ -629,8 +673,11 @@ class DownloadQueueManager:
if stdout.strip():
log.debug(f"Worker stdout for job {job.job_id}: {stdout.strip()}")
if stderr.strip():
log.warning(f"Worker stderr for job {job.job_id}: {stderr.strip()}")
job.worker_stderr = stderr.strip()
if returncode != 0:
log.warning(f"Worker stderr for job {job.job_id}: {stderr.strip()}")
else:
log.debug(f"Worker stderr for job {job.job_id}: {stderr.strip()}")
result_data: Optional[Dict[str, Any]] = None
try:
@@ -35,6 +35,8 @@ class APIErrorCode(str, Enum):
NOT_FOUND = "NOT_FOUND" # Resource not found (title, job, etc.)
NO_CONTENT = "NO_CONTENT" # No titles/tracks/episodes found
JOB_NOT_FOUND = "JOB_NOT_FOUND" # Download job doesn't exist
SESSION_NOT_FOUND = "SESSION_NOT_FOUND" # Remote-dl session doesn't exist or expired
TRACK_NOT_FOUND = "TRACK_NOT_FOUND" # Track ID not found in session
RATE_LIMITED = "RATE_LIMITED" # Service rate limiting
@@ -97,6 +99,8 @@ class APIError(Exception):
APIErrorCode.NOT_FOUND: 404,
APIErrorCode.NO_CONTENT: 404,
APIErrorCode.JOB_NOT_FOUND: 404,
APIErrorCode.SESSION_NOT_FOUND: 404,
APIErrorCode.TRACK_NOT_FOUND: 404,
# 429 Too Many Requests
APIErrorCode.RATE_LIMITED: 429,
# 500 Internal Server Error
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,139 @@
"""Thread-safe bridge for interactive input during remote authentication.
When a service calls ``request_input()`` during ``authenticate()`` on the
server, the InputBridge pauses the auth thread and exposes the prompt to
the HTTP layer so a remote client can poll for it, collect the user's
response, and submit it back. The auth thread then resumes with the
response value.
"""
from __future__ import annotations
import threading
import time
from dataclasses import dataclass, field
from enum import Enum
from typing import Optional
class AuthStatus(Enum):
"""Authentication lifecycle states for a remote session."""
AUTHENTICATING = "authenticating"
PENDING_INPUT = "pending_input"
AUTHENTICATED = "authenticated"
FAILED = "failed"
class BridgeCancelledError(Exception):
"""Raised when the bridge is cancelled (client disconnected, session deleted)."""
@dataclass
class InputBridge:
"""Thread-safe bridge between a sync auth thread and the async HTTP layer.
The auth thread calls :meth:`request_input` which blocks until the
remote client submits a response via the HTTP prompt endpoints.
"""
_prompt: Optional[str] = field(default=None, init=False, repr=False)
_response: Optional[str] = field(default=None, init=False, repr=False)
_status: AuthStatus = field(default=AuthStatus.AUTHENTICATING, init=False)
_error: Optional[str] = field(default=None, init=False, repr=False)
_cancelled: bool = field(default=False, init=False, repr=False)
_response_ready: threading.Event = field(default_factory=threading.Event, init=False, repr=False)
_lock: threading.Lock = field(default_factory=threading.Lock, init=False, repr=False)
def request_input(self, prompt: str, timeout: float = 600.0) -> str:
"""Block until the remote client submits a response for *prompt*.
Args:
prompt: The message to display to the remote user.
timeout: Maximum seconds to wait for a response.
Returns:
The string response from the remote client.
Raises:
TimeoutError: If no response is received within *timeout*.
BridgeCancelledError: If the bridge was cancelled.
"""
with self._lock:
if self._cancelled:
raise BridgeCancelledError("Session was cancelled")
self._prompt = prompt
self._response = None
self._status = AuthStatus.PENDING_INPUT
self._response_ready.clear()
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if self._response_ready.wait(timeout=0.5):
break
with self._lock:
if self._cancelled:
raise BridgeCancelledError("Session was cancelled")
else:
with self._lock:
self._status = AuthStatus.FAILED
self._error = "Input request timed out waiting for client response"
raise TimeoutError(f"No client response for prompt within {timeout}s")
with self._lock:
if self._cancelled:
raise BridgeCancelledError("Session was cancelled")
response = self._response or ""
self._prompt = None
self._response = None
self._status = AuthStatus.AUTHENTICATING
return response
def get_pending_prompt(self) -> Optional[str]:
"""Return the current prompt if the auth thread is waiting for input."""
with self._lock:
if self._status == AuthStatus.PENDING_INPUT:
return self._prompt
return None
def submit_response(self, response: str) -> bool:
"""Deliver the client's response and unblock the auth thread.
Returns:
``True`` if a prompt was pending and the response was accepted,
``False`` otherwise.
"""
with self._lock:
if self._status != AuthStatus.PENDING_INPUT:
return False
self._response = response
self._response_ready.set()
return True
def cancel(self) -> None:
"""Cancel the bridge, unblocking any waiting auth thread."""
with self._lock:
self._cancelled = True
self._status = AuthStatus.FAILED
self._error = "Session cancelled"
self._response_ready.set()
@property
def status(self) -> AuthStatus:
with self._lock:
return self._status
@status.setter
def status(self, value: AuthStatus) -> None:
with self._lock:
self._status = value
@property
def error(self) -> Optional[str]:
with self._lock:
return self._error
@error.setter
def error(self, value: Optional[str]) -> None:
with self._lock:
self._error = value
+494 -11
View File
@@ -1,14 +1,18 @@
import logging
import re
import click
from aiohttp import web
from aiohttp_swagger3 import SwaggerDocs, SwaggerInfo, SwaggerUiSettings
from envied.core import __version__
from envied.core.api.errors import APIError, APIErrorCode, build_error_response, handle_api_exception
from envied.core.api.handlers import (cancel_download_job_handler, download_handler, get_download_job_handler,
list_download_jobs_handler, list_titles_handler, list_tracks_handler,
search_handler)
from envied.core.api.handlers import (cancel_download_job_handler, download_handler, get_allowed_services,
get_download_job_handler, list_download_jobs_handler, list_titles_handler,
list_tracks_handler, search_handler, session_create_handler,
session_delete_handler, session_info_handler, session_license_handler,
session_prompt_get_handler, session_prompt_post_handler,
session_segments_handler, session_titles_handler, session_tracks_handler)
from envied.core.services import Services
from envied.core.update_checker import UpdateChecker
@@ -25,7 +29,7 @@ async def cors_middleware(request: web.Request, handler):
# Add CORS headers
response.headers["Access-Control-Allow-Origin"] = "*"
response.headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, DELETE, OPTIONS"
response.headers["Access-Control-Allow-Headers"] = "Content-Type, X-API-Key, Authorization"
response.headers["Access-Control-Allow-Headers"] = "Content-Type, X-Secret-Key, Authorization"
response.headers["Access-Control-Max-Age"] = "3600"
return response
@@ -151,6 +155,9 @@ async def services(request: web.Request) -> web.Response:
"""
try:
service_tags = Services.get_tags()
allowed = get_allowed_services(request)
if allowed is not None:
service_tags = [t for t in service_tags if t in allowed]
services_info = []
for tag in service_tags:
@@ -185,6 +192,32 @@ async def services(request: web.Request) -> web.Response:
if hasattr(service_module, "cli") and hasattr(service_module.cli, "short_help"):
service_data["url"] = service_module.cli.short_help
if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"):
cli_params = []
for param in service_module.cli.params:
param_info: dict = {"name": getattr(param, "name", None)}
if isinstance(param, click.Argument):
param_info["kind"] = "argument"
param_info["required"] = param.required
else:
param_info["kind"] = "option"
param_info["opts"] = list(param.opts) if hasattr(param, "opts") else []
param_info["is_flag"] = getattr(param, "is_flag", False)
default = param.default
if default is None:
pass
elif callable(default) or type(default).__name__ == "Sentinel":
default = None
elif hasattr(default, "name"):
default = default.name
elif not isinstance(default, (str, int, float, bool, list)):
default = str(default)
param_info["default"] = default
param_info["help"] = getattr(param, "help", None)
param_info["type"] = param.type.name if hasattr(param.type, "name") else str(param.type)
cli_params.append(param_info)
service_data["cli_params"] = cli_params
if service_module.__doc__:
service_data["help"] = service_module.__doc__.strip()
@@ -218,7 +251,7 @@ async def search(request: web.Request) -> web.Response:
properties:
service:
type: string
description: Service tag (e.g., NF, AMZN, ATV)
description: Service tag
query:
type: string
description: Search query string
@@ -597,8 +630,10 @@ async def download(request: web.Request) -> web.Response:
type: boolean
description: Download audio description tracks (default - false)
slow:
type: boolean
description: Add 60-120s delay between downloads (default - false)
oneOf:
- type: boolean
- type: string
description: Add randomized delay between downloads. `true` for default 60-120s, or `"MIN-MAX"` string (e.g., `"20-40"`). Min must be >= 20 (default - null)
split_audio:
type: boolean
description: Create separate output files per audio codec instead of merging all audio (default - null)
@@ -606,8 +641,8 @@ async def download(request: web.Request) -> web.Response:
type: boolean
description: Skip downloading, only retrieve decryption keys (default - false)
export:
type: string
description: Path to export decryption keys as JSON (default - None)
type: boolean
description: Export manifest, track URLs, keys, and subtitles to JSON in the exports directory (default - false)
cdm_only:
type: boolean
description: Only use CDM for key retrieval (true) or only vaults (false) (default - None)
@@ -617,6 +652,9 @@ async def download(request: web.Request) -> web.Response:
no_proxy:
type: boolean
description: Force disable all proxy use (default - false)
no_proxy_download:
type: boolean
description: Bypass proxy for segment downloads only. Manifest, license, and auth still use proxy (default - false)
tag:
type: string
description: Set the group tag to be used (default - None)
@@ -647,6 +685,9 @@ async def download(request: web.Request) -> web.Response:
best_available:
type: boolean
description: Continue with best available if requested quality unavailable (default - false)
worst:
type: boolean
description: Select the lowest bitrate track within the specified quality. Requires `quality` (default - false)
repack:
type: boolean
description: Add REPACK tag to the output filename (default - false)
@@ -836,8 +877,429 @@ async def cancel_download_job(request: web.Request) -> web.Response:
return build_error_response(e, debug_mode)
def setup_routes(app: web.Application) -> None:
"""Setup all API routes."""
async def session_create(request: web.Request) -> web.Response:
"""
Create a remote-dl session.
---
summary: Create session
description: Authenticate with a service, get titles, tracks, and chapters in one call
requestBody:
required: true
content:
application/json:
schema:
type: object
additionalProperties: true
required:
- service
- title_id
properties:
service:
type: string
title_id:
type: string
credentials:
type: object
additionalProperties: true
cookies:
type: string
proxy:
type: string
no_proxy:
type: boolean
profile:
type: string
cache:
type: object
additionalProperties: true
responses:
'200':
description: Session created with titles, tracks, and chapters
'400':
description: Invalid request
'401':
description: Authentication failed
"""
try:
data = await request.json()
except Exception as e:
return build_error_response(
APIError(APIErrorCode.INVALID_INPUT, "Invalid JSON request body", details={"error": str(e)}),
request.app.get("debug_api", False),
)
try:
return await session_create_handler(data, request)
except APIError as e:
return build_error_response(e, request.app.get("debug_api", False))
except Exception as e:
log.exception("Error in session create")
return handle_api_exception(
e, context={"operation": "session_create"}, debug_mode=request.app.get("debug_api", False)
)
async def session_titles(request: web.Request) -> web.Response:
"""
Get titles for an authenticated session.
---
summary: Get titles
description: Fetch titles from the authenticated service session
parameters:
- name: session_id
in: path
required: true
schema:
type: string
responses:
'200':
description: List of titles
'404':
description: Session not found
"""
session_id = request.match_info["session_id"]
try:
return await session_titles_handler(session_id, request)
except APIError as e:
return build_error_response(e, request.app.get("debug_api", False))
except Exception as e:
log.exception("Error in session titles")
return handle_api_exception(
e, context={"operation": "session_titles"}, debug_mode=request.app.get("debug_api", False)
)
async def session_tracks(request: web.Request) -> web.Response:
"""
Get tracks and chapters for a specific title.
---
summary: Get tracks
description: Fetch tracks and chapters for a title in the session
parameters:
- name: session_id
in: path
required: true
schema:
type: string
requestBody:
required: true
content:
application/json:
schema:
type: object
required:
- title_id
properties:
title_id:
type: string
description: ID of the title to get tracks for
responses:
'200':
description: Tracks and chapters for the title
'404':
description: Session or title not found
"""
session_id = request.match_info["session_id"]
try:
data = await request.json()
except Exception as e:
return build_error_response(
APIError(APIErrorCode.INVALID_INPUT, "Invalid JSON request body", details={"error": str(e)}),
request.app.get("debug_api", False),
)
try:
return await session_tracks_handler(data, session_id, request)
except APIError as e:
return build_error_response(e, request.app.get("debug_api", False))
except Exception as e:
log.exception("Error in session tracks")
return handle_api_exception(
e, context={"operation": "session_tracks"}, debug_mode=request.app.get("debug_api", False)
)
async def session_segments(request: web.Request) -> web.Response:
"""
Resolve segment URLs for selected tracks.
---
summary: Resolve segments
description: Get download URLs, DRM info, and headers for selected tracks
parameters:
- name: session_id
in: path
required: true
schema:
type: string
requestBody:
required: true
content:
application/json:
schema:
type: object
required:
- track_ids
properties:
track_ids:
type: array
items:
type: string
description: List of track IDs to resolve
responses:
'200':
description: Segment URLs and DRM info for each track
'404':
description: Session or track not found
"""
session_id = request.match_info["session_id"]
try:
data = await request.json()
except Exception as e:
return build_error_response(
APIError(APIErrorCode.INVALID_INPUT, "Invalid JSON request body", details={"error": str(e)}),
request.app.get("debug_api", False),
)
try:
return await session_segments_handler(data, session_id, request)
except APIError as e:
return build_error_response(e, request.app.get("debug_api", False))
except Exception as e:
log.exception("Error in session segments")
return handle_api_exception(
e, context={"operation": "session_segments"}, debug_mode=request.app.get("debug_api", False)
)
async def session_license(request: web.Request) -> web.Response:
"""
Proxy DRM license through authenticated service.
---
summary: Proxy license
description: Forward a CDM challenge to the service's license endpoint
parameters:
- name: session_id
in: path
required: true
schema:
type: string
requestBody:
required: true
content:
application/json:
schema:
type: object
required:
- track_id
- challenge
properties:
track_id:
type: string
description: Track ID this license is for
challenge:
type: string
description: Base64-encoded CDM challenge
drm_type:
type: string
enum: [widevine, playready]
description: DRM type (default widevine)
responses:
'200':
description: License response
'404':
description: Session or track not found
"""
session_id = request.match_info["session_id"]
try:
data = await request.json()
except Exception as e:
return build_error_response(
APIError(APIErrorCode.INVALID_INPUT, "Invalid JSON request body", details={"error": str(e)}),
request.app.get("debug_api", False),
)
try:
return await session_license_handler(data, session_id, request)
except APIError as e:
return build_error_response(e, request.app.get("debug_api", False))
except Exception as e:
log.exception("Error in session license")
return handle_api_exception(
e, context={"operation": "session_license"}, debug_mode=request.app.get("debug_api", False)
)
async def session_info(request: web.Request) -> web.Response:
"""
Get session info.
---
summary: Session info
description: Check session validity and get metadata
parameters:
- name: session_id
in: path
required: true
schema:
type: string
responses:
'200':
description: Session info
'404':
description: Session not found
"""
session_id = request.match_info["session_id"]
try:
return await session_info_handler(session_id, request)
except APIError as e:
return build_error_response(e, request.app.get("debug_api", False))
async def session_delete(request: web.Request) -> web.Response:
"""
Delete a session.
---
summary: Delete session
description: Clean up a remote-dl session
parameters:
- name: session_id
in: path
required: true
schema:
type: string
responses:
'200':
description: Session deleted
'404':
description: Session not found
"""
session_id = request.match_info["session_id"]
try:
return await session_delete_handler(session_id, request)
except APIError as e:
return build_error_response(e, request.app.get("debug_api", False))
async def session_prompt_get(request: web.Request) -> web.Response:
"""
Poll for pending interactive prompts during authentication.
---
summary: Get auth prompt
description: Poll for pending interactive prompts (OTP, device code, PIN) during session authentication
parameters:
- name: session_id
in: path
required: true
schema:
type: string
responses:
'200':
description: Auth status and optional prompt
content:
application/json:
schema:
type: object
properties:
status:
type: string
enum: [authenticating, pending_input, authenticated, failed]
prompt:
type: string
description: Prompt to display to the user (only when status is pending_input)
error:
type: string
description: Error message (only when status is failed)
'404':
description: Session not found
"""
session_id = request.match_info["session_id"]
try:
return await session_prompt_get_handler(session_id, request)
except APIError as e:
return build_error_response(e, request.app.get("debug_api", False))
except Exception as e:
log.exception("Error in session prompt get")
return handle_api_exception(
e, context={"operation": "session_prompt_get"}, debug_mode=request.app.get("debug_api", False)
)
async def session_prompt_submit(request: web.Request) -> web.Response:
"""
Submit a response to a pending interactive prompt.
---
summary: Submit prompt response
description: Submit user input (OTP code, PIN, device code confirmation) to unblock server authentication
parameters:
- name: session_id
in: path
required: true
schema:
type: string
requestBody:
required: true
content:
application/json:
schema:
type: object
required:
- response
properties:
response:
type: string
description: User's response to the prompt
responses:
'200':
description: Response accepted
content:
application/json:
schema:
type: object
properties:
status:
type: string
example: accepted
'400':
description: No prompt pending or invalid request
'404':
description: Session not found
"""
session_id = request.match_info["session_id"]
try:
data = await request.json()
except Exception as e:
return build_error_response(
APIError(APIErrorCode.INVALID_INPUT, "Invalid JSON request body", details={"error": str(e)}),
request.app.get("debug_api", False),
)
try:
return await session_prompt_post_handler(data, session_id, request)
except APIError as e:
return build_error_response(e, request.app.get("debug_api", False))
except Exception as e:
log.exception("Error in session prompt submit")
return handle_api_exception(
e, context={"operation": "session_prompt_submit"}, debug_mode=request.app.get("debug_api", False)
)
def _setup_remote_session_routes(app: web.Application) -> None:
"""Setup remote-DL session endpoints only."""
app.router.add_get("/api/health", health)
app.router.add_get("/api/services", services)
app.router.add_post("/api/search", search)
app.router.add_post("/api/session/create", session_create)
app.router.add_get("/api/session/{session_id}/titles", session_titles)
app.router.add_post("/api/session/{session_id}/tracks", session_tracks)
app.router.add_post("/api/session/{session_id}/segments", session_segments)
app.router.add_post("/api/session/{session_id}/license", session_license)
app.router.add_get("/api/session/{session_id}/prompt", session_prompt_get)
app.router.add_post("/api/session/{session_id}/prompt", session_prompt_submit)
app.router.add_get("/api/session/{session_id}", session_info)
app.router.add_delete("/api/session/{session_id}", session_delete)
def setup_routes(app: web.Application, remote_only: bool = False) -> None:
"""Setup API routes. When remote_only=True, only expose remote session endpoints."""
if remote_only:
_setup_remote_session_routes(app)
return
app.router.add_get("/api/health", health)
app.router.add_get("/api/services", services)
app.router.add_post("/api/search", search)
@@ -848,6 +1310,17 @@ def setup_routes(app: web.Application) -> None:
app.router.add_get("/api/download/jobs/{job_id}", download_job_detail)
app.router.add_delete("/api/download/jobs/{job_id}", cancel_download_job)
# Remote-DL session endpoints
app.router.add_post("/api/session/create", session_create)
app.router.add_get("/api/session/{session_id}/titles", session_titles)
app.router.add_post("/api/session/{session_id}/tracks", session_tracks)
app.router.add_post("/api/session/{session_id}/segments", session_segments)
app.router.add_post("/api/session/{session_id}/license", session_license)
app.router.add_get("/api/session/{session_id}/prompt", session_prompt_get)
app.router.add_post("/api/session/{session_id}/prompt", session_prompt_submit)
app.router.add_get("/api/session/{session_id}", session_info)
app.router.add_delete("/api/session/{session_id}", session_delete)
def setup_swagger(app: web.Application) -> None:
"""Setup Swagger UI documentation."""
@@ -873,5 +1346,15 @@ def setup_swagger(app: web.Application) -> None:
web.get("/api/download/jobs", download_jobs),
web.get("/api/download/jobs/{job_id}", download_job_detail),
web.delete("/api/download/jobs/{job_id}", cancel_download_job),
# Remote-DL session endpoints
web.post("/api/session/create", session_create),
web.get("/api/session/{session_id}/titles", session_titles),
web.post("/api/session/{session_id}/tracks", session_tracks),
web.post("/api/session/{session_id}/segments", session_segments),
web.post("/api/session/{session_id}/license", session_license),
web.get("/api/session/{session_id}/prompt", session_prompt_get),
web.post("/api/session/{session_id}/prompt", session_prompt_submit),
web.get("/api/session/{session_id}", session_info),
web.delete("/api/session/{session_id}", session_delete),
]
)
@@ -0,0 +1,209 @@
"""Server-side session store for remote-dl client-server architecture.
Maintains authenticated service instances between API calls so that
a client can authenticate once and then make multiple requests (list tracks,
resolve segments, proxy license) using the same session.
"""
from __future__ import annotations
import asyncio
import logging
import uuid
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
from envied.core.api.input_bridge import AuthStatus, InputBridge
from envied.core.config import config
from envied.core.tracks import Track
log = logging.getLogger("api.session")
@dataclass
class SessionEntry:
"""A single authenticated session with a service."""
session_id: str
service_tag: str
service_instance: Any # Service instance (authenticated)
titles: Any = None # Titles_T from get_titles()
title_map: Dict[str, Any] = field(default_factory=dict) # title_id -> Title object
tracks: Dict[str, Track] = field(default_factory=dict) # track_id -> Track object
tracks_by_title: Dict[str, Dict[str, Track]] = field(default_factory=dict) # title_key -> {track_id -> Track}
chapters_by_title: Dict[str, List[Any]] = field(default_factory=dict) # title_key -> [Chapter]
creator_ip: Optional[str] = None
cache_tag: Optional[str] = None # per-session cache directory tag
input_bridge: Optional[InputBridge] = None
auth_status: AuthStatus = AuthStatus.AUTHENTICATED
auth_error: Optional[str] = None
created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
last_accessed: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
def touch(self) -> None:
"""Update last_accessed timestamp."""
self.last_accessed = datetime.now(timezone.utc)
class SessionStore:
"""Thread-safe session store with TTL-based expiration."""
def __init__(self) -> None:
self._sessions: Dict[str, SessionEntry] = {}
self._lock = asyncio.Lock()
self._cleanup_task: Optional[asyncio.Task] = None
@property
def _ttl(self) -> int:
"""Session TTL in seconds from config."""
return config.serve.get("session_ttl", 300) # 5 min default
@property
def _max_sessions(self) -> int:
"""Max concurrent sessions from config."""
return config.serve.get("max_sessions", 100)
async def create(
self,
service_tag: str,
service_instance: Any,
session_id: Optional[str] = None,
) -> SessionEntry:
"""Create a new session with an authenticated service instance."""
async with self._lock:
if len(self._sessions) >= self._max_sessions:
oldest_id = min(self._sessions, key=lambda k: self._sessions[k].last_accessed)
log.warning(f"Max sessions reached ({self._max_sessions}), evicting oldest: {oldest_id}")
del self._sessions[oldest_id]
session_id = session_id or str(uuid.uuid4())
entry = SessionEntry(
session_id=session_id,
service_tag=service_tag,
service_instance=service_instance,
)
self._sessions[session_id] = entry
log.info(f"Created session {session_id} for service {service_tag}")
return entry
async def get(self, session_id: str) -> Optional[SessionEntry]:
"""Get a session by ID, returns None if not found or expired."""
async with self._lock:
entry = self._sessions.get(session_id)
if entry is None:
return None
if entry.auth_status not in (AuthStatus.AUTHENTICATING, AuthStatus.PENDING_INPUT):
elapsed = (datetime.now(timezone.utc) - entry.last_accessed).total_seconds()
if elapsed > self._ttl:
log.info(f"Session {session_id} expired (elapsed={elapsed:.0f}s, ttl={self._ttl}s)")
del self._sessions[session_id]
return None
entry.touch()
return entry
async def delete(self, session_id: str) -> bool:
"""Delete a session. Returns True if it existed."""
async with self._lock:
entry = self._sessions.pop(session_id, None)
if entry:
if entry.input_bridge:
entry.input_bridge.cancel()
self._cleanup_cache_dir(entry.cache_tag)
log.info(f"Deleted session {session_id}")
return True
return False
async def cleanup_expired(self) -> int:
"""Remove all expired sessions. Returns count of removed sessions."""
async with self._lock:
now = datetime.now(timezone.utc)
expired = []
for sid, entry in self._sessions.items():
elapsed = (now - entry.last_accessed).total_seconds()
if entry.auth_status in (AuthStatus.AUTHENTICATING, AuthStatus.PENDING_INPUT):
if elapsed > 600:
expired.append(sid)
elif elapsed > self._ttl:
expired.append(sid)
for sid in expired:
entry = self._sessions.pop(sid)
if entry.input_bridge:
entry.input_bridge.cancel()
self._cleanup_cache_dir(entry.cache_tag)
if expired:
log.info(f"Cleaned up {len(expired)} expired sessions")
return len(expired)
async def start_cleanup_loop(self) -> None:
"""Start periodic cleanup of expired sessions."""
if self._cleanup_task is not None:
return
async def _loop() -> None:
while True:
await asyncio.sleep(60) # Check every minute
try:
await self.cleanup_expired()
except Exception:
log.exception("Error during session cleanup")
self._cleanup_task = asyncio.create_task(_loop())
log.info("Session cleanup loop started")
async def stop_cleanup_loop(self) -> None:
"""Stop the periodic cleanup task."""
if self._cleanup_task is not None:
self._cleanup_task.cancel()
self._cleanup_task = None
async def cancel_all_bridges(self) -> None:
"""Cancel all active input bridges (called on server shutdown)."""
async with self._lock:
for entry in self._sessions.values():
if entry.input_bridge:
entry.input_bridge.cancel()
count = len(self._sessions)
if count:
log.info(f"Cancelled bridges for {count} active session(s)")
@staticmethod
def _cleanup_cache_dir(cache_tag: Optional[str]) -> None:
"""Remove session cache directory and empty parents."""
if not cache_tag:
return
import shutil
cache_dir = config.directories.cache / cache_tag
if cache_dir.is_dir():
try:
shutil.rmtree(cache_dir)
except Exception as e:
log.warning(f"Failed to remove session cache {cache_dir}: {e}")
for parent in cache_dir.parents:
if parent == config.directories.cache:
break
try:
if parent.is_dir() and not any(parent.iterdir()):
parent.rmdir()
except Exception:
break
@property
def session_count(self) -> int:
"""Number of active sessions."""
return len(self._sessions)
# Singleton instance
_session_store: Optional[SessionStore] = None
def get_session_store() -> SessionStore:
"""Get or create the global session store singleton."""
global _session_store
if _session_store is None:
_session_store = SessionStore()
return _session_store
@@ -45,12 +45,10 @@ ShakaPackager = find(
f"packager-{__shaka_platform}-arm64",
f"packager-{__shaka_platform}-x64",
)
Aria2 = find("aria2c", "aria2")
CCExtractor = find("ccextractor", "ccextractorwin", "ccextractorwinfull")
HolaProxy = find("hola-proxy")
MPV = find("mpv")
Caddy = find("caddy")
N_m3u8DL_RE = find("N_m3u8DL-RE", "n-m3u8dl-re")
MKVToolNix = find("mkvmerge")
Mkvpropedit = find("mkvpropedit")
DoviTool = find("dovi_tool")
@@ -66,12 +64,10 @@ __all__ = (
"FFPlay",
"SubtitleEdit",
"ShakaPackager",
"Aria2",
"CCExtractor",
"HolaProxy",
"MPV",
"Caddy",
"N_m3u8DL_RE",
"MKVToolNix",
"Mkvpropedit",
"DoviTool",
@@ -15,6 +15,7 @@ __all__ = [
"DecryptLabsRemoteCDM",
"CustomRemoteCDM",
"MonaLisaCDM",
"load_cdm",
"is_remote_cdm",
"is_local_cdm",
"cdm_location",
@@ -36,6 +37,10 @@ def __getattr__(name: str) -> Any:
from .monalisa import MonaLisaCDM
return MonaLisaCDM
if name == "load_cdm":
from .loader import load_cdm
return load_cdm
if name in {
"is_remote_cdm",
@@ -0,0 +1,128 @@
"""Shared CDM loading utility.
Instantiates a CDM object (local or remote) given a resolved device name.
Name resolution (quality-based, profile-based, DRM-type) is the caller's
responsibility this module only handles the instantiation step.
"""
from __future__ import annotations
import logging
from typing import Any, Optional
log = logging.getLogger("cdm")
def load_cdm(
cdm_name: str,
*,
service_name: str = "",
vaults: Optional[Any] = None,
) -> Any:
"""Instantiate a CDM by device name.
Looks up the name in config.remote_cdm first (for remote/API CDMs),
then falls back to local .prd / .wvd files.
Returns a CDM object (WidevineCdm, PlayReadyCdm, RemoteCdm,
DecryptLabsRemoteCDM, CustomRemoteCDM, or PlayReadyRemoteCdm).
Raises ValueError if the device cannot be found or loaded.
"""
from envied.core.config import config
cdm_api = next(iter(x.copy() for x in config.remote_cdm if x["name"] == cdm_name), None)
if cdm_api:
return _load_remote_cdm(cdm_api, cdm_name, service_name, vaults)
return _load_local_cdm(cdm_name)
def _load_remote_cdm(
cdm_api: dict,
cdm_name: str,
service_name: str,
vaults: Optional[Any],
) -> Any:
"""Instantiate a remote CDM from a config.remote_cdm entry."""
from envied.core.config import config
cdm_type = cdm_api.get("type")
if cdm_type == "decrypt_labs":
from envied.core.cdm.decrypt_labs_remote_cdm import DecryptLabsRemoteCDM
del cdm_api["name"]
del cdm_api["type"]
if "secret" not in cdm_api or not cdm_api["secret"]:
if config.decrypt_labs_api_key:
cdm_api["secret"] = config.decrypt_labs_api_key
else:
raise ValueError(
f"No secret provided for DecryptLabs CDM '{cdm_name}' and no global decrypt_labs_api_key configured"
)
return DecryptLabsRemoteCDM(service_name=service_name, vaults=vaults, **cdm_api)
if cdm_type == "custom_api":
from envied.core.cdm.custom_remote_cdm import CustomRemoteCDM
del cdm_api["name"]
del cdm_api["type"]
return CustomRemoteCDM(service_name=service_name, vaults=vaults, **cdm_api)
device_type = cdm_api.get("Device Type", cdm_api.get("device_type", ""))
if str(device_type).upper() == "PLAYREADY":
from pyplayready.remote.remotecdm import RemoteCdm as PlayReadyRemoteCdm
return PlayReadyRemoteCdm(
security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)),
host=cdm_api.get("Host", cdm_api.get("host")),
secret=cdm_api.get("Secret", cdm_api.get("secret")),
device_name=cdm_api.get("Device Name", cdm_api.get("device_name")),
)
from pywidevine.remotecdm import RemoteCdm
return RemoteCdm(
device_type=cdm_api.get("Device Type", cdm_api.get("device_type", "")),
system_id=cdm_api.get("System ID", cdm_api.get("system_id", "")),
security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)),
host=cdm_api.get("Host", cdm_api.get("host")),
secret=cdm_api.get("Secret", cdm_api.get("secret")),
device_name=cdm_api.get("Device Name", cdm_api.get("device_name")),
)
def _load_local_cdm(cdm_name: str) -> Any:
"""Instantiate a local CDM from a .prd or .wvd file."""
from envied.core.config import config
prd_path = config.directories.prds / f"{cdm_name}.prd"
if not prd_path.is_file():
prd_path = config.directories.wvds / f"{cdm_name}.prd"
if prd_path.is_file():
from pyplayready.cdm import Cdm as PlayReadyCdm
from pyplayready.device import Device as PlayReadyDevice
return PlayReadyCdm.from_device(PlayReadyDevice.load(prd_path))
cdm_path = config.directories.wvds / f"{cdm_name}.wvd"
if not cdm_path.is_file():
raise ValueError(f"{cdm_name} does not exist or is not a file")
from construct import ConstError
from pywidevine.cdm import Cdm as WidevineCdm
from pywidevine.device import Device
try:
device = Device.load(cdm_path)
except ConstError as e:
if "expected 2 but parsed 1" in str(e):
raise ValueError(
f"{cdm_name}.wvd seems to be a v1 WVD file, use `pywidevine migrate --help` to migrate it to v2."
)
raise ValueError(f"{cdm_name}.wvd is an invalid or corrupt Widevine Device file, {e}")
return WidevineCdm.from_device(device)
+52 -3
View File
@@ -1,3 +1,5 @@
import logging
from pathlib import Path
from typing import Optional
import click
@@ -5,11 +7,56 @@ import click
from envied.core.config import config
from envied.core.utilities import import_module_by_path
log = logging.getLogger("commands")
_COMMANDS = sorted(
(path for path in config.directories.commands.glob("*.py") if path.stem.lower() != "__init__"), key=lambda x: x.stem
)
_MODULES = {path.stem: getattr(import_module_by_path(path), path.stem) for path in _COMMANDS}
def load_command(path: Path) -> object:
"""Load one command module, returning its stem-named attribute.
Raises a concise, single-line error naming the command and the real cause so
a broken command never surfaces as a raw traceback pointing at the loader.
"""
try:
module = import_module_by_path(path)
except Exception as e:
raise RuntimeError(f"{path.stem}: failed to import — {type(e).__name__}: {e} ({path})") from e
try:
return getattr(module, path.stem)
except AttributeError as e:
raise RuntimeError(
f"{path.stem}: no object named '{path.stem}' found in {path} — it must match the filename"
) from e
def load_commands(paths: list[Path]) -> tuple[dict[str, object], list[str]]:
"""Load every command, returning the good ones plus a list of load errors.
Importing this module must never raise (it runs at CLI startup, before Rich
is installed, so a raise here prints an ugly pre-setup traceback). Instead we
collect failures and surface them once, cleanly, when the CLI is used.
"""
modules: dict[str, object] = {}
errors: list[str] = []
for path in paths:
try:
modules[path.stem] = load_command(path)
except Exception as e:
errors.append(str(e))
return modules, errors
_MODULES, LOAD_ERRORS = load_commands(_COMMANDS)
def check_load_errors() -> None:
"""Raise a single clean error if any command failed to load."""
if LOAD_ERRORS:
joined = "\n".join(f" - {err}" for err in LOAD_ERRORS)
raise click.ClickException(f"Failed to load {len(LOAD_ERRORS)} command(s):\n{joined}")
class Commands(click.MultiCommand):
@@ -17,11 +64,13 @@ class Commands(click.MultiCommand):
def list_commands(self, ctx: click.Context) -> list[str]:
"""Returns a list of command names from the command filenames."""
return [x.stem for x in _COMMANDS]
check_load_errors()
return [x.stem.replace("_", "-") for x in _COMMANDS]
def get_command(self, ctx: click.Context, name: str) -> Optional[click.Command]:
"""Load the command code and return the main click command function."""
module = _MODULES.get(name)
check_load_errors()
module = _MODULES.get(name) or _MODULES.get(name.replace("-", "_"))
if not module:
raise click.ClickException(f"Unable to find command by the name '{name}'")
+42 -18
View File
@@ -26,6 +26,7 @@ class Config:
cache = data / "cache"
cookies = data / "cookies"
logs = data / "logs"
exports = data / "exports"
wvds = data / "WVDs"
prds = data / "PRDs"
dcsl = data / "DCSL"
@@ -41,8 +42,6 @@ class Config:
def __init__(self, **kwargs: Any):
self.dl: dict = kwargs.get("dl") or {}
self.aria2c: dict = kwargs.get("aria2c") or {}
self.n_m3u8dl_re: dict = kwargs.get("n_m3u8dl_re") or {}
self.cdm: dict = kwargs.get("cdm") or {}
self.chapter_fallback_name: str = kwargs.get("chapter_fallback_name") or ""
self.curl_impersonate: dict = kwargs.get("curl_impersonate") or {}
@@ -60,22 +59,24 @@ class Config:
else:
setattr(self.directories, name, Path(path).expanduser())
downloader_cfg = kwargs.get("downloader") or "requests"
if isinstance(downloader_cfg, dict):
self.downloader_map = {k.upper(): v for k, v in downloader_cfg.items()}
self.downloader = self.downloader_map.get("DEFAULT", "requests")
else:
self.downloader_map = {}
self.downloader = downloader_cfg
downloader_cfg = kwargs.get("downloader")
if downloader_cfg and downloader_cfg != "requests":
warnings.warn(
f"downloader '{downloader_cfg}' is deprecated. The unified requests downloader is now used.",
DeprecationWarning,
stacklevel=2,
)
self.filenames = self._Filenames()
for name, filename in (kwargs.get("filenames") or {}).items():
setattr(self.filenames, name, filename)
self.audio: dict = kwargs.get("audio") or {}
self.headers: dict = kwargs.get("headers") or {}
self.key_vaults: list[dict[str, Any]] = kwargs.get("key_vaults", [])
self.muxing: dict = kwargs.get("muxing") or {}
self.proxy_providers: dict = kwargs.get("proxy_providers") or {}
self.remote_services: dict = kwargs.get("remote_services") or {}
self.serve: dict = kwargs.get("serve") or {}
self.services: dict = kwargs.get("services") or {}
decryption_cfg = kwargs.get("decryption") or {}
@@ -93,11 +94,19 @@ class Config:
self.tmdb_api_key: str = kwargs.get("tmdb_api_key") or ""
self.simkl_client_id: str = kwargs.get("simkl_client_id") or ""
self.decrypt_labs_api_key: str = kwargs.get("decrypt_labs_api_key") or ""
self.ipinfo_api_key: str = kwargs.get("ipinfo_api_key") or ""
self.update_checks: bool = kwargs.get("update_checks", True)
self.update_check_interval: int = kwargs.get("update_check_interval", 24)
self.language_tags: dict = kwargs.get("language_tags") or {}
self.output_template: dict = kwargs.get("output_template") or {}
folder_cfg = self.output_template.pop("folder", "")
self.folder_template: str = ""
self.folder_templates: dict = {}
if isinstance(folder_cfg, dict):
self.folder_templates = {k: v for k, v in folder_cfg.items() if isinstance(v, str) and v}
elif isinstance(folder_cfg, str):
self.folder_template = folder_cfg or ""
if kwargs.get("scene_naming") is not None:
raise SystemExit(
@@ -106,14 +115,8 @@ class Config:
"See unshackle-example.yaml for examples."
)
if not self.output_template:
raise SystemExit(
"ERROR: No 'output_template' configured in your envied.yaml.\n"
"Please add an 'output_template' section with movies, series, and songs templates.\n"
"See unshackle-example.yaml for examples."
)
self._validate_output_templates()
if self.output_template:
self._validate_output_templates()
self.unicode_filenames: bool = kwargs.get("unicode_filenames", False)
@@ -160,7 +163,16 @@ class Config:
unsafe_chars = r'[<>:"/\\|?*]'
for template_type, template_str in self.output_template.items():
all_templates = dict(self.output_template)
if self.folder_template:
all_templates["folder"] = self.folder_template
for kind, tmpl in self.folder_templates.items():
if kind not in {"movies", "series", "songs"}:
warnings.warn(f"Unknown folder template kind '{kind}' (expected movies/series/songs)")
continue
all_templates[f"folder.{kind}"] = tmpl
for template_type, template_str in all_templates.items():
if not isinstance(template_str, str):
warnings.warn(f"Template '{template_type}' must be a string, got {type(template_str).__name__}")
continue
@@ -179,6 +191,18 @@ class Config:
if not template_str.strip():
warnings.warn(f"Template '{template_type}' is empty")
def get_folder_template(self, kind: str) -> str:
"""Resolve the folder template for the given title kind.
kind: one of "movies", "series", "songs".
Falls back to the legacy single-string folder template, then "".
"""
if self.folder_templates:
tmpl = self.folder_templates.get(kind)
if tmpl:
return tmpl
return self.folder_template or ""
def get_template_separator(self, template_type: str = "movies") -> str:
"""Get the filename separator for the given template type.
@@ -1,6 +1,3 @@
from .aria2c import aria2c
from .curl_impersonate import curl_impersonate
from .n_m3u8dl_re import n_m3u8dl_re
from .requests import requests
__all__ = ("aria2c", "curl_impersonate", "requests", "n_m3u8dl_re")
__all__ = ("requests",)
@@ -1,10 +1,11 @@
import math
import os
import time
from concurrent.futures import as_completed
from concurrent.futures import FIRST_COMPLETED, wait
from concurrent.futures.thread import ThreadPoolExecutor
from http.cookiejar import CookieJar
from pathlib import Path
from queue import Empty, Queue
from typing import Any, Generator, MutableMapping, Optional, Union
from requests import Session
@@ -16,19 +17,175 @@ from envied.core.utilities import get_debug_logger, get_extension
MAX_ATTEMPTS = 5
RETRY_WAIT = 2
CHUNK_SIZE = 1024
PROGRESS_WINDOW = 5
PROGRESS_WINDOW = 2
DOWNLOAD_SIZES = []
LAST_SPEED_REFRESH = time.time()
# Adaptive chunk sizing — benchmarked optimal range
MIN_CHUNK = 524_288 # 512KB
MAX_CHUNK = 4_194_304 # 4MB
DEFAULT_CHUNK = 524_288 # 512KB
SPEED_ROLLING_WINDOW = 10 # seconds of history to keep for speed calculation
RANGE_PARALLEL_MIN_SIZE = 64 * 1024 * 1024
RANGE_PARALLEL_PART_SIZE = 16 * 1024 * 1024
def _adaptive_chunk_size(content_length: int) -> int:
"""Pick chunk size based on content length. Benchmarked sweet spot: 512KB-4MB."""
if content_length <= 0:
return DEFAULT_CHUNK
return min(MAX_CHUNK, max(MIN_CHUNK, content_length // 4))
def _is_requests_session(session: Any) -> bool:
"""Check if the session is a standard requests.Session (supports resp.raw)."""
return isinstance(session, Session)
def _is_rnet_session(session: Any) -> bool:
"""Check if the session is an RnetSession (uses resp.stream())."""
from envied.core.session import RnetSession
return isinstance(session, RnetSession)
def _probe_ranged(url: str, session: Any, **kwargs: Any) -> tuple[int, bool]:
headers = {**(kwargs.get("headers") or {}), "Range": "bytes=0-0"}
rest = {k: v for k, v in kwargs.items() if k != "headers"}
try:
resp = session.get(url, stream=True, headers=headers, **rest)
except Exception:
return 0, False
try:
if resp.status_code != 206:
return 0, False
ce = (resp.headers.get("Content-Encoding") or resp.headers.get("content-encoding") or "").lower()
if ce in ("gzip", "deflate", "br"):
return 0, False
content_range = resp.headers.get("Content-Range") or resp.headers.get("content-range") or ""
total = content_range.rsplit("/", 1)[-1].strip()
return (int(total), True) if total.isdigit() else (0, False)
finally:
try:
resp.close()
except Exception:
pass
def _dispatch_parts(
url: str,
save_path: Path,
session: Any,
total_size: int,
max_workers: int,
**kwargs: Any,
) -> Generator[dict[str, Any], None, None]:
save_path.parent.mkdir(parents=True, exist_ok=True)
control_file = save_path.with_name(f"{save_path.name}.!dev")
n_parts = max(1, min(max_workers, math.ceil(total_size / RANGE_PARALLEL_PART_SIZE)))
part_size = math.ceil(total_size / n_parts)
parts = [
(s, e)
for i in range(n_parts)
for s, e in [(i * part_size, min(total_size - 1, (i + 1) * part_size - 1))]
if s <= e
]
control_file.write_bytes(b"")
with open(save_path, "wb") as f:
f.truncate(total_size)
events: Queue[dict[str, Any]] = Queue()
def _worker(start: int, end: int) -> None:
for ev in download(
url=url,
save_path=save_path,
session=session,
part_offset=start,
part_end=end,
**kwargs,
):
events.put(ev)
pool = ThreadPoolExecutor(max_workers=len(parts))
futures = [pool.submit(_worker, s, e) for s, e in parts]
pending = set(futures)
yield {"total": total_size}
total_bytes = 0
start_time = last_report = time.time()
completed = False
worker_error = False
try:
while pending:
advance = 0
while not events.empty():
try:
ev = events.get_nowait()
except Empty:
break
a = ev.get("advance")
if a:
advance += a
if advance:
total_bytes += advance
yield {"advance": advance}
now = time.time()
if now - last_report > 0.5 and total_bytes > 0:
yield {"downloaded": f"{filesize.decimal(math.ceil(total_bytes / (now - start_time)))}/s"}
last_report = now
done, pending = wait(pending, timeout=0.1, return_when=FIRST_COMPLETED)
for fut in done:
exc = fut.exception()
if exc:
worker_error = True
DOWNLOAD_CANCELLED.set()
raise exc
advance = 0
while not events.empty():
try:
ev = events.get_nowait()
except Empty:
break
a = ev.get("advance")
if a:
advance += a
if advance:
total_bytes += advance
yield {"advance": advance}
yield {"file_downloaded": save_path, "written": total_size}
completed = True
except KeyboardInterrupt:
DOWNLOAD_CANCELLED.set()
pool.shutdown(wait=False, cancel_futures=True)
raise
finally:
pool.shutdown(wait=worker_error, cancel_futures=True)
if completed:
control_file.unlink(missing_ok=True)
def download(
url: str, save_path: Path, session: Optional[Session] = None, segmented: bool = False, **kwargs: Any
url: str,
save_path: Path,
session: Optional[Any] = None,
segmented: bool = False,
part_offset: Optional[int] = None,
part_end: Optional[int] = None,
**kwargs: Any,
) -> Generator[dict[str, Any], None, None]:
"""
Download a file using Python Requests.
https://requests.readthedocs.io
Download a file with optimized I/O.
Supports both requests.Session and RnetSession for TLS fingerprinting.
Uses raw socket reads for requests.Session and native rnet streaming for RnetSession.
Yields the following download status updates while chunks are downloading:
@@ -38,116 +195,180 @@ def download(
- {downloaded: "10.1 MB/s"} (currently downloading at a rate of 10.1 MB/s)
- {file_downloaded: Path(...), written: 1024} (download finished, has the save path and size)
The data is in the same format accepted by rich's progress.update() function. The
`downloaded` key is custom and is not natively accepted by all rich progress bars.
Parameters:
url: Web URL of a file to download.
save_path: The path to save the file to. If the save path's directory does not
exist then it will be made automatically.
session: The Requests Session to make HTTP requests with. Useful to set Header,
Cookie, and Proxy data. Connections are saved and re-used with the session
so long as the server keeps the connection alive.
session: A requests.Session or RnetSession to make HTTP requests with.
RnetSession preserves TLS fingerprinting for services that need it.
segmented: If downloads are segments or parts of one bigger file.
part_offset: Byte offset to write at within a pre-allocated file. When set
(with `part_end`), enables part mode for parallel ranged downloads
no truncate, no skip-if-exists, no control file; emits only `advance`
events; retries resume mid-part via Range.
part_end: Inclusive end byte of the part. Required when `part_offset` is set.
kwargs: Any extra keyword arguments to pass to the session.get() call. Use this
for one-time request changes like a header, cookie, or proxy. For example,
to request Byte-ranges use e.g., `headers={"Range": "bytes=0-128"}`.
"""
global LAST_SPEED_REFRESH
session = session or Session()
part_mode = part_offset is not None and part_end is not None
save_dir = save_path.parent
control_file = save_path.with_name(f"{save_path.name}.!dev")
save_dir.mkdir(parents=True, exist_ok=True)
if control_file.exists():
# consider the file corrupt if the control file exists
save_path.unlink(missing_ok=True)
control_file.unlink()
elif save_path.exists():
# if it exists, and no control file, then it should be safe
yield dict(file_downloaded=save_path, written=save_path.stat().st_size)
# TODO: This should return, potential recovery bug
resume_offset = 0
if not part_mode:
if control_file.exists() and save_path.exists():
resume_offset = save_path.stat().st_size
elif control_file.exists():
control_file.unlink()
elif save_path.exists():
yield dict(file_downloaded=save_path, written=save_path.stat().st_size)
return
control_file.write_bytes(b"")
# TODO: Design a control file format so we know how much of the file is missing
control_file.write_bytes(b"")
_time = time.time
use_raw = _is_requests_session(session)
attempts = 1
completed = False
written = 0
try:
while True:
written = 0
# these are for single-url speed calcs only
download_sizes = []
last_speed_refresh = time.time()
if not part_mode:
written = 0
last_speed_refresh = _time()
try:
stream = session.get(url, stream=True, **kwargs)
use_rnet = _is_rnet_session(session)
request_kwargs = dict(kwargs)
if part_mode:
req_headers = dict(request_kwargs.get("headers", {}) or {})
req_headers["Range"] = f"bytes={part_offset + written}-{part_end}"
request_kwargs["headers"] = req_headers
elif resume_offset > 0:
req_headers = dict(request_kwargs.get("headers", {}) or {})
req_headers["Range"] = f"bytes={resume_offset}-"
request_kwargs["headers"] = req_headers
stream = session.get(url, stream=True, **request_kwargs)
stream.raise_for_status()
if not segmented:
resumed = (not part_mode) and resume_offset > 0 and stream.status_code == 206
if (not part_mode) and resume_offset > 0 and not resumed:
resume_offset = 0
if part_mode and stream.status_code != 206:
raise IOError(f"expected 206 for ranged part, got {stream.status_code}")
if use_rnet:
content_length = stream.content_length or 0
else:
try:
content_length = int(stream.headers.get("Content-Length", "0"))
# Skip Content-Length validation for compressed responses since
# requests automatically decompresses but Content-Length shows compressed size
if stream.headers.get("Content-Encoding", "").lower() in ["gzip", "deflate", "br"]:
content_length = 0
except ValueError:
content_length = 0
if content_length > 0:
yield dict(total=math.ceil(content_length / CHUNK_SIZE))
else:
# we have no data to calculate total chunks
yield dict(total=None) # indeterminate mode
chunk_size = _adaptive_chunk_size(content_length)
total_size = (resume_offset + content_length) if resumed and content_length > 0 else content_length
with open(save_path, "wb") as f:
for chunk in stream.iter_content(chunk_size=CHUNK_SIZE):
if not segmented and not part_mode:
if total_size > 0:
yield dict(total=total_size)
else:
yield dict(total=None)
if resumed and resume_offset > 0:
yield dict(advance=resume_offset)
if part_mode:
file_mode = "r+b"
file_buffering = 0
else:
file_mode = "ab" if resumed else "wb"
file_buffering = 1_048_576
with open(save_path, file_mode, buffering=file_buffering) as f:
if part_mode:
f.seek(part_offset + written)
elif not resumed and content_length > 0:
f.truncate(content_length)
f.seek(0)
_write = f.write
if use_rnet:
chunks = stream.stream()
elif use_raw:
chunks = iter(lambda: stream.raw.read(chunk_size), b"")
else:
chunks = stream.iter_content(chunk_size=chunk_size)
_data_accumulated = 0
_bytes_since_yield = 0
emit_progress = (not segmented) or part_mode
for chunk in chunks:
if DOWNLOAD_CANCELLED.is_set():
break
_write(chunk)
download_size = len(chunk)
f.write(chunk)
written += download_size
if not segmented:
yield dict(advance=1)
now = time.time()
if emit_progress:
_bytes_since_yield += download_size
_data_accumulated += download_size
now = _time()
time_since = now - last_speed_refresh
download_sizes.append(download_size)
if time_since > PROGRESS_WINDOW or download_size < CHUNK_SIZE:
data_size = sum(download_sizes)
download_speed = math.ceil(data_size / (time_since or 1))
yield dict(downloaded=f"{filesize.decimal(download_speed)}/s")
if time_since > PROGRESS_WINDOW:
yield dict(advance=_bytes_since_yield)
_bytes_since_yield = 0
if not part_mode:
download_speed = math.ceil(_data_accumulated / (time_since or 1))
yield dict(downloaded=f"{filesize.decimal(download_speed)}/s")
last_speed_refresh = now
download_sizes.clear()
_data_accumulated = 0
if not segmented and content_length and written < content_length:
if emit_progress and _bytes_since_yield > 0:
yield dict(advance=_bytes_since_yield)
try:
stream.close()
except Exception:
pass
if not part_mode and not resumed and content_length > 0 and written != content_length:
f.truncate(written)
if part_mode:
expected = part_end - part_offset + 1
if written < expected:
raise IOError(f"Failed to read part {part_offset}-{part_end}: got {written}/{expected}")
elif not segmented and content_length and written < content_length:
raise IOError(f"Failed to read {content_length} bytes from the track URI.")
yield dict(file_downloaded=save_path, written=written)
if segmented:
yield dict(advance=1)
now = time.time()
time_since = now - LAST_SPEED_REFRESH
if written: # no size == skipped dl
DOWNLOAD_SIZES.append(written)
if DOWNLOAD_SIZES and time_since > PROGRESS_WINDOW:
data_size = sum(DOWNLOAD_SIZES)
download_speed = math.ceil(data_size / (time_since or 1))
yield dict(downloaded=f"{filesize.decimal(download_speed)}/s")
LAST_SPEED_REFRESH = now
DOWNLOAD_SIZES.clear()
if not part_mode:
yield dict(file_downloaded=save_path, written=resume_offset + written)
if segmented:
yield dict(advance=1)
completed = True
break
except Exception as e:
save_path.unlink(missing_ok=True)
except Exception:
try:
stream.close()
except Exception:
pass
if DOWNLOAD_CANCELLED.is_set() or attempts == MAX_ATTEMPTS:
raise e
if part_mode and not DOWNLOAD_CANCELLED.is_set():
raise
return
if not part_mode and save_path.exists():
resume_offset = save_path.stat().st_size
time.sleep(RETRY_WAIT)
attempts += 1
finally:
control_file.unlink()
if completed and not part_mode:
control_file.unlink(missing_ok=True)
def requests(
@@ -158,10 +379,14 @@ def requests(
cookies: Optional[Union[MutableMapping[str, str], CookieJar]] = None,
proxy: Optional[str] = None,
max_workers: Optional[int] = None,
session: Optional[Any] = None,
) -> Generator[dict[str, Any], None, None]:
"""
Download a file using Python Requests.
https://requests.readthedocs.io
Download files with optimized I/O and adaptive chunk sizing.
Supports both requests.Session and RnetSession. When a RnetSession is
provided (e.g. from a service's get_session()), TLS fingerprinting is preserved
on all segment downloads.
Yields the following download status updates while chunks are downloading:
@@ -186,7 +411,10 @@ def requests(
cookies: A mapping of Cookie Key/Values or a Cookie Jar to use for all downloads.
proxy: An optional proxy URI to route connections through for all downloads.
max_workers: The maximum amount of threads to use for downloads. Defaults to
min(32,(cpu_count+4)).
min(12,(cpu_count+4)).
session: An optional requests.Session or RnetSession to use. If provided,
it will be used directly (preserving TLS fingerprinting). If None, a new
requests.Session with HTTPAdapter connection pooling will be created.
"""
if not urls:
raise ValueError("urls must be provided and not empty")
@@ -221,7 +449,7 @@ def requests(
urls = [urls]
if not max_workers:
max_workers = min(32, (os.cpu_count() or 1) + 4)
max_workers = min(16, (os.cpu_count() or 1) + 4)
urls = [
dict(save_path=save_path, **url) if isinstance(url, dict) else dict(url=url, save_path=save_path)
@@ -231,25 +459,33 @@ def requests(
]
]
session = Session()
session.mount("https://", HTTPAdapter(pool_connections=max_workers, pool_maxsize=max_workers, pool_block=True))
session.mount("http://", session.adapters["https://"])
# Use provided session or create a new optimized requests.Session
# When a session is provided (e.g., service's RnetSession), don't mutate headers/cookies/proxy —
# they're already set and the session may be shared across tracks.
if session is None:
session = Session()
if headers:
headers = {k: v for k, v in headers.items() if k.lower() != "accept-encoding"}
session.headers.update(headers)
if cookies:
session.cookies.update(cookies)
if proxy:
session.proxies.update({"all": proxy})
if headers:
headers = {k: v for k, v in headers.items() if k.lower() != "accept-encoding"}
session.headers.update(headers)
if cookies:
session.cookies.update(cookies)
if proxy:
session.proxies.update({"all": proxy})
# Mount HTTPAdapter with connection pooling sized to worker count.
# Safe to do on any requests.Session — improves connection reuse for parallel downloads.
if _is_requests_session(session):
adapter = HTTPAdapter(pool_connections=max_workers, pool_maxsize=max_workers, pool_block=True)
session.mount("https://", adapter)
session.mount("http://", adapter)
if debug_logger:
first_url = urls[0].get("url", "") if urls else ""
url_display = first_url[:200] + "..." if len(first_url) > 200 else first_url
debug_logger.log(
level="DEBUG",
operation="downloader_requests_start",
message="Starting requests download",
operation="downloader_start",
message="Starting download",
context={
"url_count": len(urls),
"first_url": url_display,
@@ -257,64 +493,171 @@ def requests(
"filename": filename,
"max_workers": max_workers,
"has_proxy": bool(proxy),
"session_type": type(session).__name__,
},
)
# If we're downloading more than one URL, treat them as "segments" for progress purposes.
# For single-URL downloads we want per-chunk progress (and the inner `download()` will yield
# a chunk-based `total`), so don't set a segment total of 1 here.
segmented_batch = len(urls) > 1
if segmented_batch:
yield dict(total=len(urls))
try:
with ThreadPoolExecutor(max_workers=max_workers) as pool:
for future in as_completed(
pool.submit(download, session=session, segmented=segmented_batch, **url) for url in urls
):
try:
yield from future.result()
except KeyboardInterrupt:
DOWNLOAD_CANCELLED.set() # skip pending track downloads
yield dict(downloaded="[yellow]CANCELLING")
pool.shutdown(wait=True, cancel_futures=True)
yield dict(downloaded="[yellow]CANCELLED")
# tell dl that it was cancelled
# the pool is already shut down, so exiting loop is fine
raise
except Exception as e:
DOWNLOAD_CANCELLED.set() # skip pending track downloads
yield dict(downloaded="[red]FAILING")
pool.shutdown(wait=True, cancel_futures=True)
yield dict(downloaded="[red]FAILED")
if debug_logger:
debug_logger.log(
level="ERROR",
operation="downloader_requests_failed",
message=f"Requests download failed: {e}",
error=e,
context={
"url_count": len(urls),
"output_dir": str(output_dir),
},
if len(urls) == 1:
url_item = urls[0]
try:
ranged_used = False
if max_workers > 1:
total_size, supports_ranges = _probe_ranged(url_item["url"], session)
if supports_ranges and total_size >= RANGE_PARALLEL_MIN_SIZE:
try:
yield from _dispatch_parts(
session=session,
total_size=total_size,
max_workers=max_workers,
**url_item,
)
# tell dl that it failed
# the pool is already shut down, so exiting loop is fine
raise
ranged_used = True
except KeyboardInterrupt:
raise
except Exception:
save_path = url_item.get("save_path")
if save_path:
sp = Path(save_path)
for target in (sp, sp.with_name(f"{sp.name}.!dev")):
try:
target.unlink(missing_ok=True)
except OSError:
pass
if not ranged_used:
yield from download(
session=session,
segmented=segmented_batch,
**url_item,
)
except KeyboardInterrupt:
DOWNLOAD_CANCELLED.set()
yield dict(downloaded="[yellow]CANCELLED")
raise
else:
# Segmented download with thread pool
# Speed is tracked here on the main thread, not in workers
total_bytes = 0
start_time = time.time()
last_speed_report = start_time
if debug_logger:
debug_logger.log(
level="DEBUG",
operation="downloader_requests_complete",
message="Requests download completed successfully",
context={
"url_count": len(urls),
"output_dir": str(output_dir),
"filename": filename,
},
)
finally:
DOWNLOAD_SIZES.clear()
pool = ThreadPoolExecutor(max_workers=max_workers)
event_queue: Queue[dict[str, Any]] = Queue()
def _download_worker(url_item: dict[str, Any]) -> None:
for event in download(
session=session,
segmented=segmented_batch,
**url_item,
):
event_queue.put(event)
futures = [pool.submit(_download_worker, url) for url in urls]
pending = set(futures)
pending_advance = 0
try:
while pending:
# Drain queued events — batch advances, track bytes for speed
while True:
try:
event = event_queue.get_nowait()
except Empty:
break
# Accumulate advance events for batched yield
advance = event.get("advance")
if advance:
pending_advance += advance
continue
# Track bytes from completed segments for speed calculation
written = event.get("written")
if written:
total_bytes += written
# Pass through other events (file_downloaded, total, etc.)
yield event
# Yield batched advances every drain cycle for responsive progress bar
if pending_advance > 0:
yield dict(advance=pending_advance)
pending_advance = 0
# Yield speed every 0.5s (throttled to avoid spamming Rich)
now = time.time()
if now - last_speed_report > 0.5 and total_bytes > 0:
elapsed = now - start_time
if elapsed > 0:
download_speed = math.ceil(total_bytes / elapsed)
yield dict(downloaded=f"{filesize.decimal(download_speed)}/s")
last_speed_report = now
# Wait efficiently for next future completion (OS condition variable)
completed, pending = wait(pending, timeout=0.1, return_when=FIRST_COMPLETED)
for future in completed:
exc = future.exception()
if isinstance(exc, KeyboardInterrupt):
raise KeyboardInterrupt()
elif exc:
DOWNLOAD_CANCELLED.set()
yield dict(downloaded="[red]FAILING")
pool.shutdown(wait=False, cancel_futures=True)
yield dict(downloaded="[red]FAILED")
if debug_logger:
debug_logger.log(
level="ERROR",
operation="downloader_failed",
message=f"Download failed: {exc}",
error=exc,
context={
"url_count": len(urls),
"output_dir": str(output_dir),
},
)
raise exc
except KeyboardInterrupt:
DOWNLOAD_CANCELLED.set()
yield dict(downloaded="[yellow]CANCELLING")
pool.shutdown(wait=False, cancel_futures=True)
yield dict(downloaded="[yellow]CANCELLED")
raise
finally:
pool.shutdown(wait=False, cancel_futures=True)
# Drain remaining events
while True:
try:
event = event_queue.get_nowait()
except Empty:
break
advance = event.get("advance")
if advance:
pending_advance += advance
continue
written = event.get("written")
if written:
total_bytes += written
yield event
# Flush remaining advances and final speed
if pending_advance > 0:
yield dict(advance=pending_advance)
elapsed = time.time() - start_time
if elapsed > 0 and total_bytes > 0:
download_speed = math.ceil(total_bytes / elapsed)
yield dict(downloaded=f"{filesize.decimal(download_speed)}/s")
if debug_logger:
debug_logger.log(
level="DEBUG",
operation="downloader_complete",
message="Download completed successfully",
context={
"url_count": len(urls),
"output_dir": str(output_dir),
"filename": filename,
},
)
__all__ = ("requests",)
@@ -1,4 +1,6 @@
from typing import Union
import base64
from typing import Any, Union
from uuid import UUID
from envied.core.drm.clearkey import ClearKey
from envied.core.drm.monalisa import MonaLisa
@@ -8,4 +10,35 @@ from envied.core.drm.widevine import Widevine
DRM_T = Union[ClearKey, Widevine, PlayReady, MonaLisa]
__all__ = ("ClearKey", "Widevine", "PlayReady", "MonaLisa", "DRM_T")
def drm_from_dict(data: dict[str, Any]) -> Union[Widevine, PlayReady]:
"""Reconstruct a Widevine/PlayReady DRM instance from its ``to_dict()`` form.
Rebuilds the PSSH from the stored base64 and re-injects any saved content keys
so the resulting object can decrypt without contacting a license server.
"""
system = data.get("system")
pssh_b64 = data.get("pssh_b64")
kids = data.get("kids") or []
content_keys = data.get("content_keys") or {}
if not pssh_b64:
raise ValueError("Cannot reconstruct DRM without a stored PSSH.")
if system == "PlayReady":
from pyplayready.system.pssh import PSSH as PlayReadyPSSH
drm: Union[Widevine, PlayReady] = PlayReady(pssh=PlayReadyPSSH(base64.b64decode(pssh_b64)), pssh_b64=pssh_b64)
elif system == "Widevine":
from pywidevine.pssh import PSSH as WidevinePSSH
drm = Widevine(pssh=WidevinePSSH(pssh_b64), kid=kids[0] if kids else None)
else:
raise ValueError(f"Unsupported DRM system for reconstruction: {system!r}")
for kid_hex, key in content_keys.items():
drm.content_keys[UUID(hex=kid_hex)] = key
return drm
__all__ = ("ClearKey", "Widevine", "PlayReady", "MonaLisa", "DRM_T", "drm_from_dict")
@@ -8,10 +8,11 @@ from urllib.parse import urljoin
from Cryptodome.Cipher import AES
from Cryptodome.Util.Padding import unpad
from curl_cffi.requests import Session as CurlSession
from m3u8.model import Key
from requests import Session
from envied.core.session import RnetSession
class ClearKey:
"""AES Clear Key DRM System."""
@@ -70,8 +71,8 @@ class ClearKey:
"""
if not isinstance(m3u_key, Key):
raise ValueError(f"Provided M3U Key is in an unexpected type {m3u_key!r}")
if not isinstance(session, (Session, CurlSession, type(None))):
raise TypeError(f"Expected session to be a {Session} or {CurlSession}, not a {type(session)}")
if not isinstance(session, (Session, RnetSession, type(None))):
raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not a {type(session)}")
if not m3u_key.method.startswith("AES"):
raise ValueError(f"Provided M3U Key is not an AES Clear Key, {m3u_key.method}")
@@ -179,9 +179,9 @@ class PlayReady:
pssh_boxes.extend(
Box.parse(base64.b64decode(x.uri.split(",")[-1]))
for x in (master.session_keys or master.keys)
if x and x.keyformat and x.keyformat.lower() in {
f"urn:uuid:{PSSH.SYSTEM_ID}", "com.microsoft.playready"
}
if x
and x.keyformat
and x.keyformat.lower() in {f"urn:uuid:{PSSH.SYSTEM_ID}", "com.microsoft.playready"}
)
init_data = track.get_init_segment(session=session)
@@ -247,6 +247,17 @@ class PlayReady:
def kid(self) -> Optional[UUID]:
return next(iter(self.kids), None)
def to_dict(self) -> dict[str, Any]:
"""Serialise this DRM instance for export/import (PSSH + KIDs).
Content keys are stored once at the export's track level, not duplicated here.
"""
return {
"system": "PlayReady",
"pssh_b64": self.pssh_b64,
"kids": [kid.hex for kid in self.kids],
}
@property
def kids(self) -> list[UUID]:
return self._kids
@@ -295,7 +306,10 @@ class PlayReady:
if challenge:
try:
license_res = licence(challenge=challenge)
try:
license_res = licence(challenge=challenge, pssh_b64=self.pssh_b64)
except TypeError:
license_res = licence(challenge=challenge)
if isinstance(license_res, bytes):
license_str = license_res.decode(errors="ignore")
else:
@@ -356,6 +370,15 @@ class PlayReady:
key_hex = key if isinstance(key, str) else key.hex()
key_args.extend(["--key", f"{kid_hex}:{key_hex}"])
# Fallback for tracks whose tenc default_KID is all-zero and whose real
# KID is signalled out-of-band: emit a zero-KID entry per content key.
zero_kid = "00" * 16
existing_kids = {kid.hex if hasattr(kid, "hex") else str(kid).replace("-", "") for kid in self.content_keys}
if zero_kid not in existing_kids:
for key in self.content_keys.values():
key_hex = key if isinstance(key, str) else key.hex()
key_args.extend(["--key", f"{zero_kid}:{key_hex}"])
cmd = [
str(binaries.Mp4decrypt),
"--show-progress",
@@ -365,7 +388,7 @@ class PlayReady:
]
try:
subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8')
subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8")
except subprocess.CalledProcessError as e:
error_msg = e.stderr if e.stderr else f"mp4decrypt failed with exit code {e.returncode}"
raise subprocess.CalledProcessError(e.returncode, cmd, output=e.stdout, stderr=error_msg)
@@ -411,7 +434,9 @@ class PlayReady:
[binaries.ShakaPackager, *arguments],
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
universal_newlines=True,
text=True,
encoding="utf-8",
errors="replace",
)
stream_skipped = False
@@ -27,6 +27,12 @@ from envied.core.utils.subprocess import ffprobe
class Widevine:
"""Widevine DRM System."""
PLACEHOLDER_KIDS = {
UUID("00000000-0000-0000-0000-000000000000"), # All zeros (key rotation default)
UUID("00010203-0405-0607-0809-0a0b0c0d0e0f"), # Sequential 0x00-0x0f
UUID("00010203-0405-0607-0809-101112131415"), # Shaka Packager test pattern
}
def __init__(self, pssh: PSSH, kid: Union[UUID, str, bytes, None] = None, **kwargs: Any):
if not pssh:
raise ValueError("Provided PSSH is empty.")
@@ -36,6 +42,7 @@ class Widevine:
if pssh.system_id == PSSH.SystemId.PlayReady:
pssh.to_widevine()
self._kid: Optional[UUID] = None
if kid:
if isinstance(kid, str):
kid = UUID(hex=kid)
@@ -43,7 +50,11 @@ class Widevine:
kid = UUID(bytes=kid)
if not isinstance(kid, UUID):
raise ValueError(f"Expected kid to be a {UUID}, str, or bytes, not {kid!r}")
pssh.set_key_ids([kid])
self._kid = kid
if pssh.key_ids and all(k in self.PLACEHOLDER_KIDS for k in pssh.key_ids):
pssh.set_key_ids([kid])
elif kid not in (pssh.key_ids or []):
pssh.set_key_ids([*(pssh.key_ids or []), kid])
self._pssh = pssh
@@ -161,8 +172,24 @@ class Widevine:
@property
def kids(self) -> list[UUID]:
"""Get all Key IDs."""
return self._pssh.key_ids
"""Get all Key IDs from PSSH, falling back to the externally provided KID."""
pssh_kids = self._pssh.key_ids
if pssh_kids:
return pssh_kids
if self._kid:
return [self._kid]
return []
def to_dict(self) -> dict[str, Any]:
"""Serialise this DRM instance for export/import (PSSH + KIDs).
Content keys are stored once at the export's track level, not duplicated here.
"""
return {
"system": "Widevine",
"pssh_b64": self.pssh.dumps(),
"kids": [kid.hex for kid in self.kids],
}
def get_content_keys(self, cdm: WidevineCdm, certificate: Callable, licence: Callable) -> None:
"""
@@ -189,7 +216,11 @@ class Widevine:
if hasattr(cdm, "has_cached_keys") and cdm.has_cached_keys(session_id):
pass
else:
cdm.parse_license(session_id, licence(challenge=challenge))
try:
license_res = licence(challenge=challenge, pssh=self.pssh)
except TypeError:
license_res = licence(challenge=challenge)
cdm.parse_license(session_id, license_res)
self.content_keys = {key.kid: key.key.hex() for key in cdm.get_keys(session_id, "CONTENT")}
if not self.content_keys:
@@ -276,6 +307,15 @@ class Widevine:
key_hex = key if isinstance(key, str) else key.hex()
key_args.extend(["--key", f"{kid_hex}:{key_hex}"])
# Fallback for tracks whose tenc default_KID is all-zero and whose real
# KID is signalled out-of-band: emit a zero-KID entry per content key.
zero_kid = "00" * 16
existing_kids = {kid.hex if hasattr(kid, "hex") else str(kid).replace("-", "") for kid in self.content_keys}
if zero_kid not in existing_kids:
for key in self.content_keys.values():
key_hex = key if isinstance(key, str) else key.hex()
key_args.extend(["--key", f"{zero_kid}:{key_hex}"])
cmd = [
str(binaries.Mp4decrypt),
"--show-progress",
@@ -285,7 +325,7 @@ class Widevine:
]
try:
subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8')
subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8")
except subprocess.CalledProcessError as e:
error_msg = e.stderr if e.stderr else f"mp4decrypt failed with exit code {e.returncode}"
raise subprocess.CalledProcessError(e.returncode, cmd, output=e.stdout, stderr=error_msg)
@@ -0,0 +1,304 @@
from __future__ import annotations
import json
import logging
from http.cookiejar import CookieJar
from pathlib import Path
from typing import Any, Optional, Union
from uuid import UUID
import click
import requests
from envied.core.config import config
from envied.core.constants import AnyTrack
from envied.core.credential import Credential
from envied.core.drm import drm_from_dict
from envied.core.manifests import DASH, HLS, ISM
from envied.core.remote_service import RemoteService, _build_title, _resolve_proxy
from envied.core.titles import Episode, Movies, Series, Title_T, Titles_T, remap_titles
from envied.core.tracks import Audio, Chapter, Chapters, Tracks, Video
from envied.core.tracks.attachment import Attachment
from envied.core.tracks.track import Track
log = logging.getLogger("import")
PARSERS = {"DASH": DASH, "HLS": HLS, "ISM": ISM}
class ImportService:
"""Reconstructs a download from an export JSON.
Auth and licensing are skipped; tracks are rebuilt from the export and keys injected
directly. ``_server_cdm``/``_server_cdm_type`` keep their underscores: dl.py reads them
via getattr as the server-CDM contract that skips client licensing.
"""
ALIASES: tuple[str, ...] = ()
GEOFENCE: tuple[str, ...] = ()
NO_SUBTITLES: bool = False
def __init__(self, ctx: click.Context, service_tag: str, title: str, import_file: Optional[str]) -> None:
self.__class__.__name__ = service_tag
self.service_tag = service_tag
self.title_id = title
self.ctx = ctx
self.log = logging.getLogger(service_tag)
self.credential: Optional[Credential] = None
self.current_region: Optional[str] = None
self.title_cache = None
if not import_file:
raise click.ClickException("No export file was provided to import from.")
export_path = Path(import_file)
if not export_path.is_file():
raise click.ClickException(f"Export file not found: {export_path}")
self.data: dict[str, Any] = json.loads(export_path.read_text(encoding="utf8"))
version = self.data.get("version")
if version != 2:
raise click.ClickException(
f"Unsupported export version {version!r}. Re-create the export with a current build."
)
self.titles_data: dict[str, Any] = self.data.get("titles", {})
self.region: Optional[str] = self.data.get("region")
self.titles: Optional[Titles_T] = None
self.tracks_by_title: dict[str, Tracks] = {}
self._server_cdm = True
self._server_cdm_type = "widevine"
self.session = self.build_session(ctx, self.region)
@staticmethod
def build_session(ctx: click.Context, region: Optional[str] = None) -> requests.Session:
"""Session for re-fetching the manifest.
Honours the importer's ``--proxy``; otherwise falls back to the export region as a
geofence. An explicit proxy that fails to resolve raises; a region fallback warns.
"""
session = requests.Session()
session.headers.update(config.headers)
params = ctx.parent.params if ctx.parent else {}
if params.get("no_proxy"):
return session
explicit = params.get("proxy")
proxy_query = explicit or region
if not proxy_query:
return session
try:
proxy = _resolve_proxy(proxy_query)
except Exception as e:
if explicit:
raise click.ClickException(f"Failed to resolve proxy '{proxy_query}': {e}")
log.warning(f"Could not auto-select a proxy for export region '{region}': {e}. Continuing without proxy.")
proxy = None
if proxy:
session.proxies.update({"all": proxy})
if not explicit:
log.info(f"No --proxy given; using export region '{region}' via your proxy provider.")
return session
@property
def title(self) -> str:
return self.title_id
def authenticate(self, cookies: Optional[CookieJar] = None, credential: Optional[Credential] = None) -> None:
self.credential = credential
def get_titles(self) -> Titles_T:
if self.titles is not None:
return self.titles
titles_list = [
_build_title(entry.get("meta", {}), self.service_tag, fallback_id=title_id)
for title_id, entry in self.titles_data.items()
]
self.titles = (
Series(titles_list) if titles_list and isinstance(titles_list[0], Episode) else Movies(titles_list)
)
return self.titles
def get_titles_cached(self, title_id: Optional[str] = None) -> Titles_T:
"""Apply the service's title_map to titles reconstructed from the export sidecar."""
title_map = (config.services.get(self.service_tag) or {}).get("title_map") or {}
return remap_titles(self.get_titles(), title_map)
def get_tracks(self, title: Title_T) -> Tracks:
"""Reconstruct the title's tracks from the export.
DASH/ISM: re-fetch and re-parse the manifest and return the full ladder (the importer
picks quality with normal dl flags; keys are injected by KID later). HLS/URL: rebuild
from the stored per-track dicts, since the variant is re-fetched from track.url at
download time and ATV-style master playlists carry unstable per-fetch tokens.
"""
title_id = str(title.id)
if title_id in self.tracks_by_title:
return self.tracks_by_title[title_id]
entry = self.titles_data.get(title_id, {})
tracks_map: dict[str, Any] = entry.get("tracks") or {}
manifest_url = entry.get("manifest_url")
manifest_type = entry.get("manifest_type")
tracks = Tracks()
tracks.manifest_url = manifest_url
parser = PARSERS.get(manifest_type or "")
if manifest_url and parser is not None and manifest_type in ("DASH", "ISM"):
try:
parsed = parser.from_url(url=manifest_url, session=self.session).to_tracks(language=title.language)
except Exception as e:
raise click.ClickException(
f"Failed to re-fetch/parse the {manifest_type} manifest for '{title}'. "
f"The manifest URL may have expired since export. ({e})"
)
for track in parsed:
tracks.add(track)
else:
for track_dict in tracks_map.values():
track = Track.from_dict(track_dict)
drm = self.rebuild_drm(track_dict)
if drm:
track.drm = drm
tracks.add(track)
for attachment in entry.get("attachments") or []:
url = attachment.get("url")
if not url:
continue
try:
tracks.attachments.append(
Attachment.from_url(
url,
name=attachment.get("name"),
mime_type=attachment.get("mime_type"),
description=attachment.get("description"),
session=self.session,
)
)
except Exception as e:
self.log.warning(f"Skipping attachment '{attachment.get('name')}': {e}")
self.tracks_by_title[title_id] = tracks
return tracks
def key_pool(self) -> dict[UUID, str]:
"""All exported KID:KEY pairs across every title, as {UUID: key_hex}."""
pool: dict[UUID, str] = {}
for entry in self.titles_data.values():
for track_dict in (entry.get("tracks") or {}).values():
for kid_hex, key in (track_dict.get("keys") or {}).items():
pool[UUID(hex=kid_hex)] = key
return pool
def rebuild_drm(self, track_dict: dict[str, Any]) -> Optional[list[Any]]:
"""Rebuild a DRM object (from stored PSSH, falling back to a stub) with the exported keys."""
keys = track_dict.get("keys") or {}
drm_dicts = track_dict.get("drm") or []
if not drm_dicts and not keys:
return None
drm_obj = None
if drm_dicts:
try:
drm_obj = drm_from_dict(drm_dicts[0])
except Exception as e:
self.log.debug(f"Falling back to DRM stub (PSSH rebuild failed: {e})")
if drm_obj is None and keys:
drm_type = (drm_dicts[0].get("system", "Widevine").lower()) if drm_dicts else "widevine"
drm_obj = RemoteService._create_drm_stub(drm_type, list(keys.keys()))
if drm_obj is None:
return None
for kid_hex, key in keys.items():
drm_obj.content_keys[UUID(hex=kid_hex)] = key
return [drm_obj]
def resolve_server_keys(self, title: Title_T) -> None:
"""Inject exported keys into the selected encrypted tracks by KID (no network).
Called by dl.py after selection. Only encrypted video/audio are touched; encrypted
DASH tracks (no DRM at parse time) get a stub holding the keys, which
DASH.download_track preserves. decrypt() applies the key whose KID matches the media.
"""
pool = self.key_pool()
if not pool:
return
system = self.exported_drm_system()
kid_hexes = [kid.hex for kid in pool]
for track in title.tracks:
if not isinstance(track, (Video, Audio)) or not self.track_is_encrypted(track):
continue
drm_obj = track.drm[0] if track.drm else RemoteService._create_drm_stub(system, kid_hexes)
for kid, key in pool.items():
drm_obj.content_keys[kid] = key
track.drm = [drm_obj]
self._server_cdm_type = drm_obj.__class__.__name__.lower()
@staticmethod
def track_is_encrypted(track: Any) -> bool:
"""True if the track carries DRM or its DASH manifest declares ContentProtection."""
if track.drm:
return True
dash = track.data.get("dash") if getattr(track, "data", None) else None
if dash:
for element in (dash.get("representation"), dash.get("adaptation_set")):
if element is not None and element.findall("ContentProtection"):
return True
return False
def exported_drm_system(self) -> str:
"""The DRM system the exporter licensed (e.g. 'playready'), defaulting to widevine."""
for entry in self.titles_data.values():
for track_dict in (entry.get("tracks") or {}).values():
for drm_dict in track_dict.get("drm") or []:
if drm_dict.get("system"):
return drm_dict["system"].lower()
return "widevine"
def get_chapters(self, title: Title_T) -> Chapters:
entry = self.titles_data.get(str(title.id), {})
return Chapters(
[Chapter(ch["timestamp"], ch.get("name")) for ch in (entry.get("chapters") or []) if ch.get("timestamp")]
)
def get_widevine_service_certificate(self, **_: Any) -> Optional[str]:
return None
def get_widevine_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[Union[bytes, str]]:
raise RuntimeError("ImportService should not request a license; keys come from the export.")
def get_playready_license(
self, *, challenge: bytes, title: Title_T, track: AnyTrack
) -> Optional[Union[bytes, str]]:
raise RuntimeError("ImportService should not request a license; keys come from the export.")
def on_segment_downloaded(self, track: AnyTrack, segment: Any) -> None:
pass
def on_track_downloaded(self, track: AnyTrack) -> None:
pass
def on_track_decrypted(self, track: AnyTrack, drm: Any, segment: Any = None) -> None:
pass
def on_track_repacked(self, track: AnyTrack) -> None:
pass
def on_track_multiplex(self, track: AnyTrack) -> None:
pass
def close(self) -> None:
try:
self.session.close()
except Exception:
pass
+352 -273
View File
@@ -7,7 +7,7 @@ import math
import re
import shutil
import sys
from copy import copy, deepcopy
from copy import deepcopy
from functools import partial
from pathlib import Path
from typing import Any, Callable, Optional, Union
@@ -16,9 +16,7 @@ from uuid import UUID
from zlib import crc32
import requests
from curl_cffi.requests import Session as CurlSession
from langcodes import Language, tag_is_valid
from lxml import etree
from lxml.etree import Element, ElementTree
from pyplayready.system.pssh import PSSH as PR_PSSH
from pywidevine.cdm import Cdm as WidevineCdm
@@ -27,9 +25,9 @@ from requests import Session
from envied.core.cdm.detect import is_playready_cdm
from envied.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack
from envied.core.downloaders import requests as requests_downloader
from envied.core.drm import DRM_T, PlayReady, Widevine
from envied.core.events import events
from envied.core.session import RnetSession
from envied.core.tracks import Audio, Subtitle, Tracks, Video
from envied.core.utilities import get_debug_logger, is_close_match, try_ensure_utf8
from envied.core.utils.xml import load_xml
@@ -51,7 +49,7 @@ class DASH:
self.url = url
@classmethod
def from_url(cls, url: str, session: Optional[Union[Session, CurlSession]] = None, **args: Any) -> DASH:
def from_url(cls, url: str, session: Optional[Union[Session, RnetSession]] = None, **args: Any) -> DASH:
if not url:
raise requests.URLRequired("DASH manifest URL must be provided for relative path computations.")
if not isinstance(url, str):
@@ -59,8 +57,8 @@ class DASH:
if not session:
session = Session()
elif not isinstance(session, (Session, CurlSession)):
raise TypeError(f"Expected session to be a {Session} or {CurlSession}, not {session!r}")
elif not isinstance(session, (Session, RnetSession)):
raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {session!r}")
res = session.get(url, **args)
if res.url != url:
@@ -109,13 +107,7 @@ class DASH:
if period_id := period.get("id"):
filtered_period_ids.append(period_id)
continue
if next(iter(period.xpath("SegmentType/@value")), "content") != "content":
if period_id := period.get("id"):
filtered_period_ids.append(period_id)
continue
if "urn:amazon:primevideo:cachingBreadth" in [
x.get("schemeIdUri") for x in period.findall("SupplementalProperty")
]:
if not DASH._is_content_period(period, []):
if period_id := period.get("id"):
filtered_period_ids.append(period_id)
continue
@@ -244,6 +236,7 @@ class DASH:
"period": period,
"adaptation_set": adaptation_set,
"representation": rep,
"representation_id": rep.get("id"),
"filtered_period_ids": filtered_period_ids,
}
},
@@ -254,6 +247,7 @@ class DASH:
# only get tracks from the first main-content period
break
tracks.manifest_url = self.url
return tracks
@staticmethod
@@ -271,8 +265,8 @@ class DASH:
):
if not session:
session = Session()
elif not isinstance(session, (Session, CurlSession)):
raise TypeError(f"Expected session to be a {Session} or {CurlSession}, not {session!r}")
elif not isinstance(session, (Session, RnetSession)):
raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {session!r}")
if proxy:
session.proxies.update({"all": proxy})
@@ -280,9 +274,10 @@ class DASH:
log = logging.getLogger("DASH")
manifest: ElementTree = track.data["dash"]["manifest"]
period: Element = track.data["dash"]["period"]
adaptation_set: Element = track.data["dash"]["adaptation_set"]
representation: Element = track.data["dash"]["representation"]
rep_id: Optional[str] = track.data["dash"].get("representation_id") or representation.get("id")
filtered_period_ids: list[str] = track.data["dash"].get("filtered_period_ids", [])
# Preserve existing DRM if it was set by the service, especially when service set Widevine
# but manifest only contains PlayReady protection (common scenario for some services)
@@ -305,16 +300,346 @@ class DASH:
or (existing_drm and not any(isinstance(drm, Widevine) for drm in existing_drm))
)
pre_existing_keys = {}
if existing_drm:
for drm_obj in existing_drm:
if hasattr(drm_obj, "content_keys") and drm_obj.content_keys:
pre_existing_keys.update(drm_obj.content_keys)
if should_override_drm:
track.drm = manifest_drm
else:
track.drm = existing_drm
if pre_existing_keys and track.drm:
for drm_obj in track.drm:
if hasattr(drm_obj, "content_keys"):
for kid, key in pre_existing_keys.items():
if kid not in drm_obj.content_keys:
drm_obj.content_keys[kid] = key
# Collect segments from all content periods in the manifest
all_periods = manifest.findall("Period")
segments: list[tuple[str, Optional[str]]] = []
segment_durations: list[int] = []
segment_timescale: float = 0
init_data: Optional[bytes] = None
track_kid: Optional[UUID] = None
content_periods = [p for p in all_periods if DASH._is_content_period(p, filtered_period_ids)]
period_count = len(content_periods)
if period_count > 1:
log.debug(f"Multi-period manifest detected with {period_count} content periods")
for period_idx, content_period in enumerate(content_periods):
# Find the matching representation in this period
matched_rep = None
matched_as = None
for as_ in content_period.findall("AdaptationSet"):
if DASH.is_trick_mode(as_):
continue
for rep in as_.findall("Representation"):
if rep.get("id") == rep_id:
matched_rep = rep
matched_as = as_
break
if matched_rep is not None:
break
if matched_rep is None or matched_as is None:
period_id = content_period.get("id", period_idx)
log.warning(f"Representation '{rep_id}' not found in period '{period_id}', skipping")
continue
p_init, p_segments, p_timescale, p_durations, p_kid = DASH._get_period_segments(
period=content_period,
adaptation_set=matched_as,
representation=matched_rep,
manifest=manifest,
track=track,
track_url=track.url,
session=session,
)
if period_idx == 0:
# First period: use its init data and KID for DRM licensing
init_data = p_init
track_kid = p_kid
segment_timescale = p_timescale
else:
if p_kid and track_kid and p_kid != track_kid:
log.debug(f"Period {content_period.get('id', period_idx)} has different KID: {p_kid}")
for seg in p_segments:
if seg not in segments:
segments.append(seg)
segment_durations.extend(p_durations)
if not segments:
log.error("Could not find a way to get segments from this MPD manifest.")
log.debug(track.url)
sys.exit(1)
# TODO: Should we floor/ceil/round, or is int() ok?
track.data["dash"]["timescale"] = int(segment_timescale)
track.data["dash"]["segment_durations"] = segment_durations
if not track.drm and init_data and isinstance(track, (Video, Audio)):
prefers_playready = is_playready_cdm(cdm)
if prefers_playready:
try:
track.drm = [PlayReady.from_init_data(init_data)]
except PlayReady.Exceptions.PSSHNotFound:
try:
track.drm = [Widevine.from_init_data(init_data)]
except Widevine.Exceptions.PSSHNotFound:
log.warning("No PlayReady or Widevine PSSH was found for this track, is it DRM free?")
else:
try:
track.drm = [Widevine.from_init_data(init_data)]
except Widevine.Exceptions.PSSHNotFound:
try:
track.drm = [PlayReady.from_init_data(init_data)]
except PlayReady.Exceptions.PSSHNotFound:
log.warning("No Widevine or PlayReady PSSH was found for this track, is it DRM free?")
if track.drm:
track_kid = track_kid or track.get_key_id(url=segments[0][0], session=session)
drm = track.get_drm_for_cdm(cdm)
if isinstance(drm, (Widevine, PlayReady)):
# license and grab content keys
try:
if not license_widevine:
raise ValueError("license_widevine func must be supplied to use DRM")
progress(downloaded="LICENSING")
license_widevine(drm, track_kid=track_kid)
progress(downloaded="[yellow]LICENSED")
except Exception: # noqa
DOWNLOAD_CANCELLED.set() # skip pending track downloads
progress(downloaded="[red]FAILED")
raise
else:
drm = None
if DOWNLOAD_LICENCE_ONLY.is_set():
progress(downloaded="[yellow]SKIPPED")
return
progress(total=len(segments))
downloader = track.downloader
downloader_args = dict(
urls=[
{"url": url, "headers": {"Range": f"bytes={bytes_range}"} if bytes_range else {}}
for url, bytes_range in segments
],
output_dir=save_dir,
filename="{i:0%d}.mp4" % (len(str(len(segments)))),
headers=session.headers,
cookies=session.cookies,
proxy=proxy,
max_workers=max_workers,
session=session,
)
debug_logger = get_debug_logger()
if debug_logger:
debug_logger.log(
level="DEBUG",
operation="manifest_dash_download_start",
message="Starting DASH manifest download",
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"total_segments": len(segments),
"has_drm": bool(track.drm),
"drm_types": [drm.__class__.__name__ for drm in (track.drm or [])],
"save_path": str(save_path),
"has_init_data": bool(init_data),
},
)
for status_update in downloader(**downloader_args):
file_downloaded = status_update.get("file_downloaded")
if file_downloaded:
events.emit(events.Types.SEGMENT_DOWNLOADED, track=track, segment=file_downloaded)
else:
downloaded = status_update.get("downloaded")
if downloaded and downloaded.endswith("/s"):
status_update["downloaded"] = f"DASH {downloaded}"
progress(**status_update)
# Verify output directory exists and contains files
if not save_dir.exists():
error_msg = f"Output directory does not exist: {save_dir}"
if debug_logger:
debug_logger.log(
level="ERROR",
operation="manifest_dash_download_output_missing",
message=error_msg,
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"save_dir": str(save_dir),
"save_path": str(save_path),
"downloader": "requests",
},
)
raise FileNotFoundError(error_msg)
for control_file in save_dir.glob("*.!dev"):
control_file.unlink(missing_ok=True)
segments_to_merge = [x for x in sorted(save_dir.iterdir()) if x.is_file()]
if debug_logger:
debug_logger.log(
level="DEBUG",
operation="manifest_dash_download_complete",
message="DASH download complete, preparing to merge",
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"save_dir": str(save_dir),
"save_dir_exists": save_dir.exists(),
"segments_found": len(segments_to_merge),
"segment_files": [f.name for f in segments_to_merge[:10]], # Limit to first 10
"downloader": "requests",
},
)
if not segments_to_merge:
error_msg = f"No segment files found in output directory: {save_dir}"
if debug_logger:
# List all contents of the directory for debugging
all_contents = list(save_dir.iterdir()) if save_dir.exists() else []
debug_logger.log(
level="ERROR",
operation="manifest_dash_download_no_segments",
message=error_msg,
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"save_dir": str(save_dir),
"directory_contents": [str(p) for p in all_contents],
"downloader": "requests",
},
)
raise FileNotFoundError(error_msg)
with open(save_path, "wb") as f:
if init_data:
f.write(init_data)
if len(segments_to_merge) > 1:
progress(downloaded="Merging", completed=0, total=len(segments_to_merge))
for segment_file in segments_to_merge:
segment_data = segment_file.read_bytes()
if (
not drm
and isinstance(track, Subtitle)
and track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML)
):
segment_data = try_ensure_utf8(segment_data)
segment_data = (
segment_data.decode("utf8")
.replace("&lrm;", html.unescape("&lrm;"))
.replace("&rlm;", html.unescape("&rlm;"))
.encode("utf8")
)
f.write(segment_data)
f.flush()
segment_file.unlink()
progress(advance=1)
track.path = save_path
events.emit(events.Types.TRACK_DOWNLOADED, track=track)
if drm:
progress(downloaded="Decrypting", completed=0, total=100)
drm.decrypt(save_path)
track.drm = None
events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=drm, segment=None)
progress(downloaded="Decrypting", advance=100)
# Clean up empty segment directory
if save_dir.exists() and save_dir.name.endswith("_segments"):
try:
save_dir.rmdir()
except OSError:
# Directory might not be empty, try removing recursively
shutil.rmtree(save_dir, ignore_errors=True)
progress(downloaded="Downloaded")
@staticmethod
def _is_content_period(period: Element, filtered_period_ids: list[str]) -> bool:
"""Check if a period is a valid content period (not an ad, not filtered, not trick mode)."""
period_id = period.get("id")
if period_id and period_id in filtered_period_ids:
return False
if next(iter(period.xpath("SegmentType/@value")), "content") != "content":
return False
if "urn:amazon:primevideo:cachingBreadth" in [
x.get("schemeIdUri") for x in period.findall("SupplementalProperty")
]:
return False
return True
@staticmethod
def _merge_segment_templates(adaptation_set: Element, representation: Element) -> Optional[Element]:
"""
Build the effective SegmentTemplate for a Representation by cascading the
AdaptationSet > Representation levels (ISO/IEC 23009-1 5.3.9.1).
The Representation-level node, when present, is the base; attributes and the
SegmentTimeline child it does not declare are inherited from the AdaptationSet-level
node. Returns None if no SegmentTemplate exists at either level.
"""
levels = [node.find("SegmentTemplate") for node in (adaptation_set, representation)]
present = [node for node in levels if node is not None]
if not present:
return None
merged = deepcopy(present[-1])
for ancestor in reversed(present[:-1]):
for attr, value in ancestor.attrib.items():
if merged.get(attr) is None:
merged.set(attr, value)
if merged.find("SegmentTimeline") is None:
timeline = ancestor.find("SegmentTimeline")
if timeline is not None:
merged.append(deepcopy(timeline))
return merged
@staticmethod
def _get_period_segments(
period: Element,
adaptation_set: Element,
representation: Element,
manifest: ElementTree,
track: AnyTrack,
track_url: str,
session: Union[Session, RnetSession],
) -> tuple[
Optional[bytes],
list[tuple[str, Optional[str]]],
float,
list[int],
Optional[UUID],
]:
"""
Extract segments from a single period's representation.
Returns:
A tuple of (init_data, segments, segment_timescale, segment_durations, track_kid).
"""
manifest_base_url = manifest.findtext("BaseURL")
if not manifest_base_url:
manifest_base_url = track.url
manifest_base_url = track_url
elif not re.match("^https?://", manifest_base_url, re.IGNORECASE):
manifest_base_url = urljoin(track.url, f"./{manifest_base_url}")
manifest_base_url = urljoin(track_url, f"./{manifest_base_url}")
period_base_url = urljoin(manifest_base_url, period.findtext("BaseURL") or "")
adaptation_set_base_url = urljoin(period_base_url, adaptation_set.findtext("BaseURL") or "")
rep_base_url = urljoin(adaptation_set_base_url, representation.findtext("BaseURL") or "")
@@ -322,9 +647,7 @@ class DASH:
period_duration = period.get("duration") or manifest.get("mediaPresentationDuration")
init_data: Optional[bytes] = None
segment_template = representation.find("SegmentTemplate")
if segment_template is None:
segment_template = adaptation_set.find("SegmentTemplate")
segment_template = DASH._merge_segment_templates(adaptation_set, representation)
segment_list = representation.find("SegmentList")
if segment_list is None:
@@ -340,7 +663,6 @@ class DASH:
track_kid: Optional[UUID] = None
if segment_template is not None:
segment_template = copy(segment_template)
start_number = int(segment_template.get("startNumber") or 1)
end_number = int(segment_template.get("endNumber") or 0) or None
segment_timeline = segment_template.find("SegmentTimeline")
@@ -355,7 +677,7 @@ class DASH:
raise ValueError("Resolved Segment URL is not absolute, and no Base URL is available.")
value = urljoin(rep_base_url, value)
if not urlparse(value).query:
manifest_url_query = urlparse(track.url).query
manifest_url_query = urlparse(track_url).query
if manifest_url_query:
value += f"?{manifest_url_query}"
segment_template.set(item, value)
@@ -477,255 +799,8 @@ class DASH:
segments.append((rep_base_url, media_range))
elif rep_base_url:
segments.append((rep_base_url, None))
else:
log.error("Could not find a way to get segments from this MPD manifest.")
log.debug(track.url)
sys.exit(1)
# TODO: Should we floor/ceil/round, or is int() ok?
track.data["dash"]["timescale"] = int(segment_timescale)
track.data["dash"]["segment_durations"] = segment_durations
if not track.drm and init_data and isinstance(track, (Video, Audio)):
prefers_playready = is_playready_cdm(cdm)
if prefers_playready:
try:
track.drm = [PlayReady.from_init_data(init_data)]
except PlayReady.Exceptions.PSSHNotFound:
try:
track.drm = [Widevine.from_init_data(init_data)]
except Widevine.Exceptions.PSSHNotFound:
log.warning("No PlayReady or Widevine PSSH was found for this track, is it DRM free?")
else:
try:
track.drm = [Widevine.from_init_data(init_data)]
except Widevine.Exceptions.PSSHNotFound:
try:
track.drm = [PlayReady.from_init_data(init_data)]
except PlayReady.Exceptions.PSSHNotFound:
log.warning("No Widevine or PlayReady PSSH was found for this track, is it DRM free?")
if track.drm:
track_kid = track_kid or track.get_key_id(url=segments[0][0], session=session)
drm = track.get_drm_for_cdm(cdm)
if isinstance(drm, (Widevine, PlayReady)):
# license and grab content keys
try:
if not license_widevine:
raise ValueError("license_widevine func must be supplied to use DRM")
progress(downloaded="LICENSING")
license_widevine(drm, track_kid=track_kid)
progress(downloaded="[yellow]LICENSED")
except Exception: # noqa
DOWNLOAD_CANCELLED.set() # skip pending track downloads
progress(downloaded="[red]FAILED")
raise
else:
drm = None
if DOWNLOAD_LICENCE_ONLY.is_set():
progress(downloaded="[yellow]SKIPPED")
return
progress(total=len(segments))
downloader = track.downloader
if downloader.__name__ == "aria2c" and any(bytes_range is not None for url, bytes_range in segments):
# aria2(c) is shit and doesn't support the Range header, fallback to the requests downloader
downloader = requests_downloader
log.warning("Falling back to the requests downloader as aria2(c) doesn't support the Range header")
downloader_args = dict(
urls=[
{"url": url, "headers": {"Range": f"bytes={bytes_range}"} if bytes_range else {}}
for url, bytes_range in segments
],
output_dir=save_dir,
filename="{i:0%d}.mp4" % (len(str(len(segments)))),
headers=session.headers,
cookies=session.cookies,
proxy=proxy,
max_workers=max_workers,
)
skip_merge = False
if downloader.__name__ == "n_m3u8dl_re":
skip_merge = True
# When periods were filtered out during to_tracks(), n_m3u8dl_re will re-parse
# the raw MPD and download ALL periods (including ads/pre-rolls). Write a filtered
# MPD with the rejected periods removed so n_m3u8dl_re downloads the correct content.
filtered_period_ids = track.data.get("dash", {}).get("filtered_period_ids", [])
if filtered_period_ids:
filtered_manifest = deepcopy(manifest)
for child in list(filtered_manifest):
if not hasattr(child.tag, "find"):
continue
if child.tag == "Period" and child.get("id") in filtered_period_ids:
filtered_manifest.remove(child)
filtered_mpd_path = save_dir / f".{track.id}_filtered.mpd"
filtered_mpd_path.parent.mkdir(parents=True, exist_ok=True)
etree.ElementTree(filtered_manifest).write(
str(filtered_mpd_path), xml_declaration=True, encoding="utf-8"
)
track.from_file = filtered_mpd_path
downloader_args.update(
{
"filename": track.id,
"track": track,
"content_keys": drm.content_keys if drm else None,
}
)
debug_logger = get_debug_logger()
if debug_logger:
debug_logger.log(
level="DEBUG",
operation="manifest_dash_download_start",
message="Starting DASH manifest download",
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"total_segments": len(segments),
"downloader": downloader.__name__,
"has_drm": bool(track.drm),
"drm_types": [drm.__class__.__name__ for drm in (track.drm or [])],
"skip_merge": skip_merge,
"save_path": str(save_path),
"has_init_data": bool(init_data),
},
)
for status_update in downloader(**downloader_args):
file_downloaded = status_update.get("file_downloaded")
if file_downloaded:
events.emit(events.Types.SEGMENT_DOWNLOADED, track=track, segment=file_downloaded)
else:
downloaded = status_update.get("downloaded")
if downloaded and downloaded.endswith("/s"):
status_update["downloaded"] = f"DASH {downloaded}"
progress(**status_update)
# Clean up filtered MPD temp file before enumerating segments
filtered_mpd_path = save_dir / f".{track.id}_filtered.mpd"
if filtered_mpd_path.exists():
filtered_mpd_path.unlink()
# see https://github.com/devine-dl/devine/issues/71
for control_file in save_dir.glob("*.aria2__temp"):
control_file.unlink()
# Verify output directory exists and contains files
if not save_dir.exists():
error_msg = f"Output directory does not exist: {save_dir}"
if debug_logger:
debug_logger.log(
level="ERROR",
operation="manifest_dash_download_output_missing",
message=error_msg,
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"save_dir": str(save_dir),
"save_path": str(save_path),
"downloader": downloader.__name__,
"skip_merge": skip_merge,
},
)
raise FileNotFoundError(error_msg)
segments_to_merge = [x for x in sorted(save_dir.iterdir()) if x.is_file()]
if debug_logger:
debug_logger.log(
level="DEBUG",
operation="manifest_dash_download_complete",
message="DASH download complete, preparing to merge",
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"save_dir": str(save_dir),
"save_dir_exists": save_dir.exists(),
"segments_found": len(segments_to_merge),
"segment_files": [f.name for f in segments_to_merge[:10]], # Limit to first 10
"downloader": downloader.__name__,
"skip_merge": skip_merge,
},
)
if not segments_to_merge:
error_msg = f"No segment files found in output directory: {save_dir}"
if debug_logger:
# List all contents of the directory for debugging
all_contents = list(save_dir.iterdir()) if save_dir.exists() else []
debug_logger.log(
level="ERROR",
operation="manifest_dash_download_no_segments",
message=error_msg,
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"save_dir": str(save_dir),
"directory_contents": [str(p) for p in all_contents],
"downloader": downloader.__name__,
"skip_merge": skip_merge,
},
)
raise FileNotFoundError(error_msg)
if skip_merge:
# N_m3u8DL-RE handles merging and decryption internally
shutil.move(segments_to_merge[0], save_path)
if drm:
track.drm = None
events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=drm, segment=None)
else:
with open(save_path, "wb") as f:
if init_data:
f.write(init_data)
if len(segments_to_merge) > 1:
progress(downloaded="Merging", completed=0, total=len(segments_to_merge))
for segment_file in segments_to_merge:
segment_data = segment_file.read_bytes()
# TODO: fix encoding after decryption?
if (
not drm
and isinstance(track, Subtitle)
and track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML)
):
segment_data = try_ensure_utf8(segment_data)
segment_data = (
segment_data.decode("utf8")
.replace("&lrm;", html.unescape("&lrm;"))
.replace("&rlm;", html.unescape("&rlm;"))
.encode("utf8")
)
f.write(segment_data)
f.flush()
segment_file.unlink()
progress(advance=1)
track.path = save_path
events.emit(events.Types.TRACK_DOWNLOADED, track=track)
if not skip_merge and drm:
progress(downloaded="Decrypting", completed=0, total=100)
drm.decrypt(save_path)
track.drm = None
events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=drm, segment=None)
progress(downloaded="Decrypting", advance=100)
# Clean up empty segment directory
if save_dir.exists() and save_dir.name.endswith("_segments"):
try:
save_dir.rmdir()
except OSError:
# Directory might not be empty, try removing recursively
shutil.rmtree(save_dir, ignore_errors=True)
progress(downloaded="Downloaded")
return init_data, segments, segment_timescale, segment_durations, track_kid
@staticmethod
def _get(item: str, adaptation_set: Element, representation: Optional[Element] = None) -> Optional[Any]:
@@ -801,6 +876,10 @@ class DASH:
def get_video_range(
codecs: str, all_supplemental_props: list[Element], all_essential_props: list[Element]
) -> Video.Range:
# TODO: Detect Dolby Vision composite streams in DASH manifests (DV RPU embedded but
# primary codec is plain hvc1, signaled via a separate AdaptationSet/Representation
# with DV codec strings or DolbyVisionConfigurationBox). When found, mark the track
# with dv_compatible_bitstream=True so DVFixup runs pre-mux. No DASH samples seen yet.
if codecs.startswith(("dva1", "dvav", "dvhe", "dvh1")):
return Video.Range.DV
@@ -924,7 +1003,7 @@ class DASH:
None,
)
if kid and (not pssh.key_ids or all(k.int == 0 or k in PLACEHOLDER_KIDS for k in pssh.key_ids)):
if kid and pssh.key_ids and all(k.int == 0 or k in PLACEHOLDER_KIDS for k in pssh.key_ids):
pssh.set_key_ids([kid])
drm.append(Widevine(pssh=pssh, kid=kid))
+299 -100
View File
@@ -5,6 +5,7 @@ import html
import json
import logging
import os
import re
import shutil
import subprocess
import sys
@@ -17,8 +18,6 @@ from zlib import crc32
import m3u8
import requests
from curl_cffi.requests import Response as CurlResponse
from curl_cffi.requests import Session as CurlSession
from langcodes import Language, tag_is_valid
from m3u8 import M3U8
from pyplayready.cdm import Cdm as PlayReadyCdm
@@ -30,15 +29,23 @@ from requests import Session
from envied.core import binaries
from envied.core.cdm.detect import is_playready_cdm, is_widevine_cdm
from envied.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack
from envied.core.downloaders import requests as requests_downloader
from envied.core.drm import DRM_T, ClearKey, MonaLisa, PlayReady, Widevine
from envied.core.events import events
from envied.core.session import RnetResponse, RnetSession
from envied.core.tracks import Audio, Subtitle, Tracks, Video
from envied.core.utilities import get_debug_logger, get_extension, is_close_match, try_ensure_utf8
class HLS:
def __init__(self, manifest: M3U8, session: Optional[Union[Session, CurlSession]] = None):
SUPP_CODECS_RE = re.compile(r'SUPPLEMENTAL-CODECS="([^"]+)"', re.IGNORECASE)
def __init__(
self,
manifest: M3U8,
session: Optional[Union[Session, RnetSession]] = None,
url: Optional[str] = None,
raw_text: Optional[str] = None,
):
if not manifest:
raise ValueError("HLS manifest must be provided.")
if not isinstance(manifest, M3U8):
@@ -48,9 +55,11 @@ class HLS:
self.manifest = manifest
self.session = session or Session()
self.url = url
self.raw_text = raw_text
@classmethod
def from_url(cls, url: str, session: Optional[Union[Session, CurlSession]] = None, **args: Any) -> HLS:
def from_url(cls, url: str, session: Optional[Union[Session, RnetSession]] = None, **args: Any) -> HLS:
if not url:
raise requests.URLRequired("HLS manifest URL must be provided.")
if not isinstance(url, str):
@@ -58,26 +67,26 @@ class HLS:
if not session:
session = Session()
elif not isinstance(session, (Session, CurlSession)):
raise TypeError(f"Expected session to be a {Session} or {CurlSession}, not {session!r}")
elif not isinstance(session, (Session, RnetSession)):
raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {session!r}")
res = session.get(url, **args)
# Handle requests and curl_cffi response objects
# Handle requests and rnet response objects
if isinstance(res, requests.Response):
if not res.ok:
raise requests.ConnectionError("Failed to request the M3U(8) document.", response=res)
content = res.text
elif isinstance(res, CurlResponse):
elif isinstance(res, RnetResponse):
if not res.ok:
raise requests.ConnectionError("Failed to request the M3U(8) document.", response=res)
content = res.text
else:
raise TypeError(f"Expected response to be a requests.Response or curl_cffi.Response, not {type(res)}")
raise TypeError(f"Expected response to be a requests.Response or rnet.Response, not {type(res)}")
master = m3u8.loads(content, uri=url)
return cls(master, session)
return cls(master, session, url=url, raw_text=content)
@classmethod
def from_text(cls, text: str, url: str) -> HLS:
@@ -93,7 +102,31 @@ class HLS:
master = m3u8.loads(text, uri=url)
return cls(master)
return cls(master, raw_text=text)
def supplemental_codecs_by_uri(self) -> dict[str, str]:
"""Map each variant URI to its SUPPLEMENTAL-CODECS value.
python-m3u8 drops this attribute, so we re-parse the raw text to recover it for
Dolby Vision composite detection (dvh1.08.x advertised only in SUPPLEMENTAL-CODECS
while primary CODECS stays plain hvc1).
"""
if not self.raw_text:
return {}
out: dict[str, str] = {}
lines = self.raw_text.splitlines()
for i, line in enumerate(lines):
if not line.startswith("#EXT-X-STREAM-INF"):
continue
supp_match = self.SUPP_CODECS_RE.search(line)
if not supp_match:
continue
for j in range(i + 1, len(lines)):
uri = lines[j].strip()
if uri and not uri.startswith("#"):
out[uri] = supp_match.group(1)
break
return out
def to_tracks(self, language: Union[str, Language]) -> Tracks:
"""
@@ -115,14 +148,19 @@ class HLS:
cc_by_group_id: dict[str, list[dict[str, Any]]] = {}
for media in self.manifest.media:
if media.type == "CLOSED-CAPTIONS":
cc_by_group_id.setdefault(media.group_id, []).append({
"language": media.language,
"name": media.name,
"instream_id": media.instream_id,
"characteristics": media.characteristics,
})
cc_by_group_id.setdefault(media.group_id, []).append(
{
"language": media.language,
"name": media.name,
"instream_id": media.instream_id,
"characteristics": media.characteristics,
}
)
tracks = Tracks()
supplemental_codecs = self.supplemental_codecs_by_uri()
dv_supp_prefixes = ("dva1", "dvav", "dvhe", "dvh1")
for playlist in self.manifest.playlists:
audio_group = playlist.stream_info.audio
audio_codec: Optional[Audio.Codec] = None
@@ -143,6 +181,26 @@ class HLS:
else:
primary_track_type = Video
primary_codecs = (playlist.stream_info.codecs or "").lower()
primary_has_dv = any(codec.split(".")[0] in dv_supp_prefixes for codec in primary_codecs.split(","))
supp_codecs_str = supplemental_codecs.get(playlist.uri, "")
supp_dv_codec: Optional[str] = None
for codec in supp_codecs_str.lower().split(","):
token = codec.strip().split("/")[0]
if token.split(".")[0] in dv_supp_prefixes:
supp_dv_codec = token
break
video_range = (
Video.Range.DV if primary_has_dv else Video.Range.from_m3u_range_tag(playlist.stream_info.video_range)
)
# DV-composite track: primary codec is plain HEVC but SUPPLEMENTAL-CODECS advertises
# a DV codec. Range stays whatever VIDEO-RANGE signaled (HDR10/HLG/SDR); DVFixup will
# restore DV signaling post-download. Services that know their encoder embeds HDR10+
# SEI must override `range` themselves (see services/ATV).
dv_compatible_bitstream = primary_track_type is Video and not primary_has_dv and supp_dv_codec is not None
tracks.add(
primary_track_type(
id_=hex(crc32(str(playlist).encode()))[2:],
@@ -161,18 +219,14 @@ class HLS:
# video track args
**(
dict(
range_=Video.Range.DV
if any(
codec.split(".")[0] in ("dva1", "dvav", "dvhe", "dvh1")
for codec in (playlist.stream_info.codecs or "").lower().split(",")
)
else Video.Range.from_m3u_range_tag(playlist.stream_info.video_range),
range_=video_range,
width=playlist.stream_info.resolution[0] if playlist.stream_info.resolution else None,
height=playlist.stream_info.resolution[1] if playlist.stream_info.resolution else None,
fps=playlist.stream_info.frame_rate,
closed_captions=cc_by_group_id.get(
(playlist.stream_info.closed_captions or "").strip('"'), []
),
dv_compatible_bitstream=dv_compatible_bitstream,
)
if primary_track_type is Video
else {}
@@ -241,40 +295,200 @@ class HLS:
)
)
for video in tracks.videos:
has_resolution = video.width and video.height
has_codec = video.codec is not None
if has_resolution and has_codec:
continue
try:
probe = HLS._probe_ts_info(video.url, self.session)
if probe:
width, height, codec = probe
if not has_resolution:
video.width, video.height = width, height
if not has_codec:
video.codec = codec
except Exception:
pass
if self.url:
tracks.manifest_url = self.url
return tracks
@staticmethod
def _finalize_n_m3u8dl_re_output(*, track: AnyTrack, save_dir: Path, save_path: Path) -> Path:
"""
Finalize output from N_m3u8DL-RE.
def _probe_ts_info(
variant_url: str, session: Optional[Union[Session, RnetSession]] = None
) -> Optional[tuple[int, int, Video.Codec]]:
"""Probe the first TS segment of a variant playlist to extract resolution and codec."""
if not session:
session = Session()
We call N_m3u8DL-RE with `--save-name track.id`, so the final file should be `{track.id}.*` under `save_dir`.
This moves that output to `save_path` (preserving the real suffix) and, for subtitles, updates `track.codec`
to match the produced file extension.
"""
matches = [p for p in save_dir.rglob(f"{track.id}.*") if p.is_file()]
if not matches:
raise FileNotFoundError(f"No output files produced by N_m3u8DL-RE for save-name={track.id} in: {save_dir}")
res = session.get(variant_url)
variant = m3u8.loads(res.text if hasattr(res, "text") else res.text, uri=variant_url)
if not variant.segments:
return None
primary = max(matches, key=lambda p: p.stat().st_size)
seg_uri = urljoin(variant_url, variant.segments[0].uri)
final_save_path = save_path.with_suffix(primary.suffix) if primary.suffix else save_path
# Download only the first 8KB — SPS is always near the start of the first TS packet
res = session.get(seg_uri, headers={"Range": "bytes=0-8191"})
data = res.content
final_save_path.parent.mkdir(parents=True, exist_ok=True)
if primary.absolute() != final_save_path.absolute():
final_save_path.unlink(missing_ok=True)
shutil.move(str(primary), str(final_save_path))
return HLS._parse_ts_video_info(data)
if isinstance(track, Subtitle):
ext = final_save_path.suffix.lower().lstrip(".")
try:
track.codec = Subtitle.Codec.from_mime(ext)
except ValueError:
pass
@staticmethod
def _parse_ts_video_info(data: bytes) -> Optional[tuple[int, int, Video.Codec]]:
"""Parse H.264/H.265 NAL units from TS segment data to extract resolution and codec."""
shutil.rmtree(save_dir, ignore_errors=True)
class _BitReader:
def __init__(self, buf: bytes) -> None:
self.data = buf
self.pos = 0
return final_save_path
def bits(self, n: int) -> int:
val = 0
for _ in range(n):
val = (val << 1) | ((self.data[self.pos >> 3] >> (7 - (self.pos & 7))) & 1)
self.pos += 1
return val
def ue(self) -> int:
zeros = 0
while self.bits(1) == 0:
zeros += 1
return (1 << zeros) - 1 + self.bits(zeros) if zeros else 0
def se(self) -> int:
val = self.ue()
return (val + 1) // 2 if val & 1 else -(val // 2)
# Find SPS NAL unit via start code
# H.264: NAL type 7 (SPS), identified by byte & 0x1F == 7
# H.265: NAL type 33 (SPS), identified by (byte >> 1) & 0x3F == 33
for i in range(len(data) - 4):
start3 = data[i : i + 3] == b"\x00\x00\x01"
start4 = data[i : i + 4] == b"\x00\x00\x00\x01"
if not start3 and not start4:
continue
offset = i + (4 if start4 else 3)
if offset >= len(data):
continue
nal_byte = data[offset]
h264_type = nal_byte & 0x1F
h265_type = (nal_byte >> 1) & 0x3F
# H.264 SPS (NAL type 7)
if h264_type == 7:
sps = data[offset : offset + 64]
if len(sps) < 5:
continue
try:
r = _BitReader(sps[1:]) # skip NAL header byte
profile = r.bits(8)
r.bits(8) # constraint flags
r.bits(8) # level
r.ue() # sps_id
if profile in (100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134):
chroma = r.ue()
if chroma == 3:
r.bits(1)
r.ue() # bit_depth_luma
r.ue() # bit_depth_chroma
r.bits(1) # qpprime_y_zero_transform_bypass
if r.bits(1): # scaling_matrix_present
for j in range(6 if chroma != 3 else 12):
if r.bits(1):
last = 8
for _ in range(16 if j < 6 else 64):
if last != 0:
last = (last + r.se()) & 0xFF
r.ue() # log2_max_frame_num
poc_type = r.ue()
if poc_type == 0:
r.ue()
elif poc_type == 1:
r.bits(1)
r.se()
r.se()
for _ in range(r.ue()):
r.se()
r.ue() # max_num_ref_frames
r.bits(1) # gaps_in_frame_num
w_mbs = r.ue() + 1
h_map = r.ue() + 1
frame_mbs_only = r.bits(1)
if not frame_mbs_only:
r.bits(1)
r.bits(1) # direct_8x8_inference
cl = cr = ct = cb = 0
if r.bits(1): # crop
cl, cr, ct, cb = r.ue(), r.ue(), r.ue(), r.ue()
width = w_mbs * 16 - (cl + cr) * 2
height = (2 - frame_mbs_only) * h_map * 16 - (ct + cb) * 2
return (width, height, Video.Codec.AVC)
except (IndexError, ValueError):
continue
# H.265 SPS (NAL type 33)
elif h265_type == 33:
sps = data[offset : offset + 128]
if len(sps) < 10:
continue
try:
r = _BitReader(sps[2:]) # skip 2-byte NAL header
r.bits(4) # sps_video_parameter_set_id
max_sub_layers = r.bits(3)
r.bits(1) # sps_temporal_id_nesting
# profile_tier_level
r.bits(2) # general_profile_space
r.bits(1) # general_tier
r.bits(5) # general_profile_idc
r.bits(32) # general_profile_compatibility_flags
r.bits(48) # general_constraint_indicator_flags
r.bits(8) # general_level_idc
sub_layer_flags = []
for _ in range(max_sub_layers - 1):
sub_layer_flags.append((r.bits(1), r.bits(1)))
if max_sub_layers - 1 > 0:
for _ in range(8 - (max_sub_layers - 1)):
r.bits(2)
for profile_present, level_present in sub_layer_flags:
if profile_present:
r.bits(2 + 1 + 5 + 32 + 48 + 8)
if level_present:
r.bits(8)
r.ue() # sps_seq_parameter_set_id
chroma = r.ue()
if chroma == 3:
r.bits(1)
width = r.ue()
height = r.ue()
if r.bits(1): # conformance_window
cl = r.ue()
cr = r.ue()
ct = r.ue()
cb = r.ue()
sub_w = 2 if chroma in (1, 2) else 1
sub_h = 2 if chroma == 1 else 1
width -= (cl + cr) * sub_w
height -= (ct + cb) * sub_h
return (width, height, Video.Codec.HEVC)
except (IndexError, ValueError):
continue
return None
@staticmethod
def download_track(
@@ -282,7 +496,7 @@ class HLS:
save_path: Path,
save_dir: Path,
progress: partial,
session: Optional[Union[Session, CurlSession]] = None,
session: Optional[Union[Session, RnetSession]] = None,
proxy: Optional[str] = None,
max_workers: Optional[int] = None,
license_widevine: Optional[Callable] = None,
@@ -291,8 +505,8 @@ class HLS:
) -> None:
if not session:
session = Session()
elif not isinstance(session, (Session, CurlSession)):
raise TypeError(f"Expected session to be a {Session} or {CurlSession}, not {session!r}")
elif not isinstance(session, (Session, RnetSession)):
raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {session!r}")
if proxy:
# Handle proxies differently based on session type
@@ -306,15 +520,13 @@ class HLS:
else:
# Get the playlist text and handle both session types
response = session.get(track.url)
if isinstance(response, requests.Response) or isinstance(response, CurlResponse):
if isinstance(response, requests.Response) or isinstance(response, RnetResponse):
if not response.ok:
log.error(f"Failed to request the invariant M3U8 playlist: {response.status_code}")
sys.exit(1)
playlist_text = response.text
else:
raise TypeError(
f"Expected response to be a requests.Response or curl_cffi.Response, not {type(response)}"
)
raise TypeError(f"Expected response to be a requests.Response or rnet.Response, not {type(response)}")
master = m3u8.loads(playlist_text, uri=track.url)
@@ -339,6 +551,12 @@ class HLS:
media_drm = HLS.get_drm(media_playlist_key, session)
if isinstance(media_drm, (Widevine, PlayReady)):
track_kid = HLS.get_track_kid_from_init(master, track, session) or media_drm.kid
# Preserve pre-existing keys (e.g. from server_cdm)
if track.drm:
for existing_drm in track.drm:
if hasattr(existing_drm, "content_keys") and existing_drm.content_keys:
media_drm.content_keys.update(existing_drm.content_keys)
track.drm = [media_drm]
try:
if not license_widevine:
raise ValueError("license_widevine func must be supplied to use DRM")
@@ -347,7 +565,6 @@ class HLS:
progress(downloaded="[yellow]LICENSED")
initial_drm_licensed = True
initial_drm_key = media_playlist_key
track.drm = [media_drm]
session_drm = media_drm
except Exception: # noqa
DOWNLOAD_CANCELLED.set() # skip pending track downloads
@@ -359,8 +576,9 @@ class HLS:
try:
if not license_widevine:
raise ValueError("license_widevine func must be supplied to use DRM")
track_kid = HLS.get_track_kid_from_init(master, track, session) or session_drm.kid
progress(downloaded="LICENSING")
license_widevine(session_drm)
license_widevine(session_drm, track_kid=track_kid)
progress(downloaded="[yellow]LICENSED")
except Exception: # noqa
DOWNLOAD_CANCELLED.set() # skip pending track downloads
@@ -391,9 +609,6 @@ class HLS:
progress(total=total_segments)
downloader = track.downloader
if downloader.__name__ == "aria2c" and any(x.byterange for x in master.segments if x not in unwanted_segments):
downloader = requests_downloader
log.warning("Falling back to the requests downloader as aria2(c) doesn't support the Range header")
urls: list[dict[str, Any]] = []
segment_durations: list[int] = []
@@ -422,7 +637,6 @@ class HLS:
segment_save_dir = save_dir / "segments"
skip_merge = False
downloader_args = dict(
urls=urls,
output_dir=segment_save_dir,
@@ -431,22 +645,9 @@ class HLS:
cookies=session.cookies,
proxy=proxy,
max_workers=max_workers,
session=session,
)
if downloader.__name__ == "n_m3u8dl_re":
skip_merge = True
# session_drm already has correct content_keys from initial licensing above
n_m3u8dl_content_keys = session_drm.content_keys if session_drm else None
downloader_args.update(
{
"output_dir": save_dir,
"filename": track.id,
"track": track,
"content_keys": n_m3u8dl_content_keys,
}
)
debug_logger = get_debug_logger()
if debug_logger:
debug_logger.log(
@@ -457,10 +658,8 @@ class HLS:
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"total_segments": total_segments,
"downloader": downloader.__name__,
"has_drm": bool(session_drm),
"drm_type": session_drm.__class__.__name__ if session_drm else None,
"skip_merge": skip_merge,
"save_path": str(save_path),
},
)
@@ -475,16 +674,8 @@ class HLS:
status_update["downloaded"] = f"HLS {downloaded}"
progress(**status_update)
# see https://github.com/devine-dl/devine/issues/71
for control_file in segment_save_dir.glob("*.aria2__temp"):
control_file.unlink()
if skip_merge:
final_save_path = HLS._finalize_n_m3u8dl_re_output(track=track, save_dir=save_dir, save_path=save_path)
progress(downloaded="Downloaded")
track.path = final_save_path
events.emit(events.Types.TRACK_DOWNLOADED, track=track)
return
for control_file in segment_save_dir.glob("*.!dev"):
control_file.unlink(missing_ok=True)
progress(total=total_segments, completed=0, downloaded="Merging")
@@ -644,13 +835,11 @@ class HLS:
)
# Check response based on session type
if isinstance(res, requests.Response) or isinstance(res, CurlResponse):
if isinstance(res, requests.Response) or isinstance(res, RnetResponse):
res.raise_for_status()
init_content = res.content
else:
raise TypeError(
f"Expected response to be requests.Response or curl_cffi.Response, not {type(res)}"
)
raise TypeError(f"Expected response to be requests.Response or rnet.Response, not {type(res)}")
map_data = (segment.init_section, init_content)
@@ -687,6 +876,14 @@ class HLS:
DOWNLOAD_CANCELLED.set() # skip pending track downloads
progress(downloaded="[red]FAILED")
raise
if (
encryption_data
and isinstance(drm, (Widevine, PlayReady))
and isinstance(encryption_data[1], type(drm))
and getattr(encryption_data[1], "content_keys", None)
):
for prev_kid, prev_key in encryption_data[1].content_keys.items():
drm.content_keys.setdefault(prev_kid, prev_key)
encryption_data = (key, drm)
if DOWNLOAD_LICENCE_ONLY.is_set():
@@ -736,8 +933,7 @@ class HLS:
"save_dir_exists": save_dir.exists(),
"segments_found": len(segments_to_merge),
"segment_files": [f.name for f in segments_to_merge[:10]], # Limit to first 10
"downloader": downloader.__name__,
"skip_merge": skip_merge,
"downloader": "requests",
},
)
@@ -755,8 +951,7 @@ class HLS:
"save_dir": str(save_dir),
"save_dir_exists": save_dir.exists(),
"directory_contents": [str(p) for p in all_contents],
"downloader": downloader.__name__,
"skip_merge": skip_merge,
"downloader": "requests",
},
)
raise FileNotFoundError(error_msg)
@@ -787,6 +982,10 @@ class HLS:
progress(downloaded="Downloaded")
track.path = save_path
if session_drm:
track.drm = None
events.emit(events.Types.TRACK_DOWNLOADED, track=track)
@staticmethod
@@ -865,7 +1064,7 @@ class HLS:
@staticmethod
def parse_session_data_keys(
manifest: M3U8, session: Optional[Union[Session, CurlSession]] = None
manifest: M3U8, session: Optional[Union[Session, RnetSession]] = None
) -> list[m3u8.model.Key]:
"""Parse `com.apple.hls.keys` session data and return Key objects."""
keys: list[m3u8.model.Key] = []
@@ -940,7 +1139,7 @@ class HLS:
def get_track_kid_from_init(
master: M3U8,
track: AnyTrack,
session: Union[Session, CurlSession],
session: Union[Session, RnetSession],
) -> Optional[UUID]:
"""
Extract the track's Key ID from its init segment (EXT-X-MAP).
@@ -1007,7 +1206,7 @@ class HLS:
@staticmethod
def get_drm(
key: Union[m3u8.model.SessionKey, m3u8.model.Key],
session: Optional[Union[Session, CurlSession]] = None,
session: Optional[Union[Session, RnetSession]] = None,
) -> DRM_T:
"""
Convert HLS EXT-X-KEY data to an initialized DRM object.
@@ -1019,8 +1218,8 @@ class HLS:
Raises a NotImplementedError if the key system is not supported.
"""
if not isinstance(session, (Session, CurlSession, type(None))):
raise TypeError(f"Expected session to be a {Session} or {CurlSession}, not {type(session)}")
if not isinstance(session, (Session, RnetSession, type(None))):
raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {type(session)}")
if not session:
session = Session()
@@ -3,14 +3,12 @@ from __future__ import annotations
import base64
import hashlib
import html
import shutil
import urllib.parse
from functools import partial
from pathlib import Path
from typing import Any, Callable, Optional, Union
import requests
from curl_cffi.requests import Session as CurlSession
from langcodes import Language, tag_is_valid
from lxml.etree import Element
from pyplayready.system.pssh import PSSH as PR_PSSH
@@ -20,6 +18,7 @@ from requests import Session
from envied.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack
from envied.core.drm import DRM_T, PlayReady, Widevine
from envied.core.events import events
from envied.core.session import RnetSession
from envied.core.tracks import Audio, Subtitle, Track, Tracks, Video
from envied.core.utilities import get_debug_logger, try_ensure_utf8
from envied.core.utils.xml import load_xml
@@ -35,13 +34,13 @@ class ISM:
self.url = url
@classmethod
def from_url(cls, url: str, session: Optional[Union[Session, CurlSession]] = None, **kwargs: Any) -> "ISM":
def from_url(cls, url: str, session: Optional[Union[Session, RnetSession]] = None, **kwargs: Any) -> "ISM":
if not url:
raise requests.URLRequired("ISM manifest URL must be provided")
if not session:
session = Session()
elif not isinstance(session, (Session, CurlSession)):
raise TypeError(f"Expected session to be a {Session} or {CurlSession}, not {session!r}")
elif not isinstance(session, (Session, RnetSession)):
raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {session!r}")
res = session.get(url, **kwargs)
if res.url != url:
url = res.url
@@ -220,6 +219,7 @@ class ISM:
data=data,
)
)
tracks.manifest_url = self.url
return tracks
@staticmethod
@@ -269,7 +269,6 @@ class ISM:
progress(total=len(segments))
downloader = track.downloader
skip_merge = False
downloader_args = dict(
urls=[{"url": url} for url in segments],
output_dir=save_dir,
@@ -278,18 +277,9 @@ class ISM:
cookies=session.cookies,
proxy=proxy,
max_workers=max_workers,
session=session,
)
if downloader.__name__ == "n_m3u8dl_re":
skip_merge = True
downloader_args.update(
{
"filename": track.id,
"track": track,
"content_keys": session_drm.content_keys if session_drm else None,
}
)
debug_logger = get_debug_logger()
if debug_logger:
debug_logger.log(
@@ -300,10 +290,9 @@ class ISM:
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"total_segments": len(segments),
"downloader": downloader.__name__,
"downloader": "requests",
"has_drm": bool(session_drm),
"drm_type": session_drm.__class__.__name__ if session_drm else None,
"skip_merge": skip_merge,
"save_path": str(save_path),
},
)
@@ -318,9 +307,6 @@ class ISM:
status_update["downloaded"] = f"ISM {downloaded}"
progress(**status_update)
for control_file in save_dir.glob("*.aria2__temp"):
control_file.unlink()
# Verify output directory exists and contains files
if not save_dir.exists():
error_msg = f"Output directory does not exist: {save_dir}"
@@ -334,12 +320,14 @@ class ISM:
"track_type": track.__class__.__name__,
"save_dir": str(save_dir),
"save_path": str(save_path),
"downloader": downloader.__name__,
"skip_merge": skip_merge,
"downloader": "requests",
},
)
raise FileNotFoundError(error_msg)
for control_file in save_dir.glob("*.!dev"):
control_file.unlink(missing_ok=True)
segments_to_merge = [x for x in sorted(save_dir.iterdir()) if x.is_file()]
if debug_logger:
@@ -354,8 +342,7 @@ class ISM:
"save_dir_exists": save_dir.exists(),
"segments_found": len(segments_to_merge),
"segment_files": [f.name for f in segments_to_merge[:10]], # Limit to first 10
"downloader": downloader.__name__,
"skip_merge": skip_merge,
"downloader": "requests",
},
)
@@ -372,39 +359,35 @@ class ISM:
"track_type": track.__class__.__name__,
"save_dir": str(save_dir),
"directory_contents": [str(p) for p in all_contents],
"downloader": downloader.__name__,
"skip_merge": skip_merge,
"downloader": "requests",
},
)
raise FileNotFoundError(error_msg)
if skip_merge:
shutil.move(segments_to_merge[0], save_path)
else:
with open(save_path, "wb") as f:
for segment_file in segments_to_merge:
segment_data = segment_file.read_bytes()
if (
not session_drm
and isinstance(track, Subtitle)
and track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML)
):
segment_data = try_ensure_utf8(segment_data)
segment_data = (
segment_data.decode("utf8")
.replace("&lrm;", html.unescape("&lrm;"))
.replace("&rlm;", html.unescape("&rlm;"))
.encode("utf8")
)
f.write(segment_data)
f.flush()
segment_file.unlink()
progress(advance=1)
with open(save_path, "wb") as f:
for segment_file in segments_to_merge:
segment_data = segment_file.read_bytes()
if (
not session_drm
and isinstance(track, Subtitle)
and track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML)
):
segment_data = try_ensure_utf8(segment_data)
segment_data = (
segment_data.decode("utf8")
.replace("&lrm;", html.unescape("&lrm;"))
.replace("&rlm;", html.unescape("&rlm;"))
.encode("utf8")
)
f.write(segment_data)
f.flush()
segment_file.unlink()
progress(advance=1)
track.path = save_path
events.emit(events.Types.TRACK_DOWNLOADED, track=track)
if not skip_merge and session_drm:
if session_drm:
progress(downloaded="Decrypting", completed=0, total=100)
session_drm.decrypt(save_path)
track.drm = None
@@ -5,10 +5,10 @@ from __future__ import annotations
from typing import Optional, Union
import m3u8
from curl_cffi.requests import Session as CurlSession
from requests import Session
from envied.core.manifests.hls import HLS
from envied.core.session import RnetSession
from envied.core.tracks import Tracks
@@ -16,10 +16,11 @@ def parse(
master: m3u8.M3U8,
language: str,
*,
session: Optional[Union[Session, CurlSession]] = None,
session: Optional[Union[Session, RnetSession]] = None,
url: Optional[str] = None,
) -> Tracks:
"""Parse a variant playlist to ``Tracks`` with basic information, defer DRM loading."""
tracks = HLS(master, session=session).to_tracks(language)
tracks = HLS(master, session=session, url=url).to_tracks(language)
bool(master.session_keys or HLS.parse_session_data_keys(master, session or Session()))
@@ -13,7 +13,8 @@ import requests
from envied.core import binaries
from envied.core.proxies.proxy import Proxy
from envied.core.utilities import get_country_code, get_country_name, get_debug_logger, get_ip_info
from envied.core.utilities import get_country_code, get_country_name, get_debug_logger
from envied.core.utils.ip_info import get_ip_info
# Global registry for cleanup on exit
_gluetun_instances: list["Gluetun"] = []
@@ -1052,7 +1053,7 @@ class Gluetun(Proxy):
# Gluetun needs both proxy listening AND VPN connected
# The proxy starts before VPN is ready, so we need to wait for VPN
proxy_ready = "[http proxy] listening" in all_logs
vpn_ready = "initialization sequence completed" in all_logs
vpn_ready = "initialization sequence completed" in all_logs or "public ip address is" in all_logs
if proxy_ready and vpn_ready:
# Give a brief moment for the proxy to fully initialize
@@ -1235,10 +1236,10 @@ class Gluetun(Proxy):
duration_ms=duration_ms,
)
raise RuntimeError(
f"Region mismatch for {container['provider']}:{container['region']}: "
f"Expected '{expected_code}' but got '{actual_country}' "
f"(IP: {ip_info.get('ip')}, City: {ip_info.get('city')})"
)
f"Region mismatch for {container['provider']}:{container['region']}: "
f"Expected '{expected_code}' but got '{actual_country}' "
f"(IP: {ip_info.get('ip')}, City: {ip_info.get('city')})"
)
# Verification successful - store IP info in container record
if query_key in self.active_containers:
@@ -64,7 +64,7 @@ class NordVPN(Proxy):
if re.match(r"^[a-z]{2}\d+$", query):
# country and nordvpn server id, e.g., us1, fr1234
hostname = f"{query}.nordvpn.com"
hostname = f"{query}.proxy.nordvpn.com"
else:
if query.isdigit():
# country id
@@ -86,7 +86,7 @@ class NordVPN(Proxy):
if server_mapping:
# country was set to a specific server ID in config
hostname = f"{country['code'].lower()}{server_mapping}.nordvpn.com"
hostname = f"{country['code'].lower()}{server_mapping}.proxy.nordvpn.com"
else:
# get the recommended server ID
recommended_servers = self.get_recommended_servers(country["id"])
@@ -113,6 +113,9 @@ class NordVPN(Proxy):
# NordVPN uses the alpha2 of 'GB' in API responses, but 'UK' in the hostname
hostname = f"gb{hostname[2:]}"
if hostname.endswith(".nordvpn.com") and not hostname.endswith(".proxy.nordvpn.com"):
hostname = hostname[: -len(".nordvpn.com")] + ".proxy.nordvpn.com"
return f"https://{self.username}:{self.password}@{hostname}:89"
def get_country(self, by_id: Optional[int] = None, by_code: Optional[str] = None) -> Optional[dict]:
@@ -0,0 +1,88 @@
"""Shared proxy provider initialization and resolution.
Used by both the REST API handlers and the remote service client.
"""
from __future__ import annotations
import logging
import re
from typing import Any, List, Optional
log = logging.getLogger("proxies")
def initialize_proxy_providers() -> List[Any]:
"""Initialize and return available proxy providers from config."""
proxy_providers: list = []
try:
from envied.core import binaries
from envied.core.config import config as main_config
from envied.core.proxies.basic import Basic
from envied.core.proxies.hola import Hola
from envied.core.proxies.nordvpn import NordVPN
from envied.core.proxies.surfsharkvpn import SurfsharkVPN
proxy_config = getattr(main_config, "proxy_providers", {})
if proxy_config.get("basic"):
proxy_providers.append(Basic(**proxy_config["basic"]))
if proxy_config.get("nordvpn"):
proxy_providers.append(NordVPN(**proxy_config["nordvpn"]))
if proxy_config.get("surfsharkvpn"):
proxy_providers.append(SurfsharkVPN(**proxy_config["surfsharkvpn"]))
if hasattr(binaries, "HolaProxy") and binaries.HolaProxy:
proxy_providers.append(Hola())
for provider in proxy_providers:
log.info(f"Loaded {provider.__class__.__name__}: {provider}")
if not proxy_providers:
log.warning("No proxy providers were loaded. Check your proxy provider configuration in envied.yaml")
except Exception as e:
log.warning(f"Failed to initialize some proxy providers: {e}")
return proxy_providers
def resolve_proxy(proxy: str, proxy_providers: List[Any]) -> Optional[str]:
"""Resolve a proxy parameter to an actual proxy URI.
Accepts:
- Direct URI: "https://...", "socks5://..."
- Country code: "us", "uk"
- Provider:country: "nordvpn:us"
"""
if not proxy:
return None
if re.match(r"^(https?://|socks)", proxy):
return proxy
requested_provider = None
query = proxy
if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE):
requested_provider, query = proxy.split(":", maxsplit=1)
if requested_provider:
provider = next(
(x for x in proxy_providers if x.__class__.__name__.lower() == requested_provider.lower()),
None,
)
if not provider:
available = [x.__class__.__name__ for x in proxy_providers]
raise ValueError(f"Proxy provider '{requested_provider}' not found. Available: {available}")
proxy_uri = provider.get_proxy(query)
if not proxy_uri:
raise ValueError(f"Proxy provider {requested_provider} had no proxy for {query}")
log.info(f"Using {provider.__class__.__name__} Proxy: {proxy_uri}")
return proxy_uri
for provider in proxy_providers:
proxy_uri = provider.get_proxy(query)
if proxy_uri:
log.info(f"Using {provider.__class__.__name__} Proxy: {proxy_uri}")
return proxy_uri
raise ValueError(f"No proxy provider had a proxy for {proxy}")
@@ -83,7 +83,7 @@ class WindscribeVPN(Proxy):
if not hostname:
return None
hostname = hostname.split(':')[0]
hostname = hostname.split(":")[0]
return f"https://{self.username}:{self.password}@{hostname}:443"
def get_specific_server(self, country_code: str, server_num: str) -> Optional[str]:
@@ -0,0 +1,853 @@
"""Remote service adapter for envied.
Implements the Service interface by proxying authenticate, get_titles,
get_tracks, get_chapters, and license methods to a remote unshackle server.
Everything else (track selection, download, decrypt, mux) runs locally.
"""
from __future__ import annotations
import base64
import logging
import time
from enum import Enum
from http.cookiejar import CookieJar
from typing import Any, Dict, Optional, Union
import click
import requests
from langcodes import Language
from requests.adapters import HTTPAdapter, Retry
from rich.padding import Padding
from rich.rule import Rule
from envied.core.config import config
from envied.core.console import console
from envied.core.constants import AnyTrack
from envied.core.credential import Credential
from envied.core.titles import Title_T, Titles_T, remap_titles
from envied.core.titles.episode import Episode, Series
from envied.core.titles.movie import Movie, Movies
from envied.core.tracks import Audio, Chapter, Chapters, Subtitle, Tracks, Video
from envied.core.tracks.attachment import Attachment
from envied.core.tracks.track import Track
log = logging.getLogger("remote_service")
class RemoteClient:
"""HTTP client for the unshackle serve API."""
def __init__(self, server_url: str, api_key: str) -> None:
self.server_url = server_url.rstrip("/")
self.api_key = api_key
self._session: Optional[requests.Session] = None
@property
def session(self) -> requests.Session:
if self._session is None:
from envied.core import __version__
self._session = requests.Session()
self._session.headers["User-Agent"] = f"unshackle/{__version__}"
if self.api_key:
self._session.headers["X-Secret-Key"] = self.api_key
return self._session
def _request(self, method: str, endpoint: str, data: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
url = f"{self.server_url}{endpoint}"
try:
resp = getattr(self.session, method)(url, json=data, timeout=120 if method == "post" else 30)
except requests.ConnectionError:
log.error(f"Could not connect to remote server at {self.server_url}. Is it running? (unshackle serve)")
raise SystemExit(1)
except requests.Timeout:
log.error(f"Request to remote server timed out: {endpoint}")
raise SystemExit(1)
result = resp.json()
if resp.status_code >= 400:
error_msg = result.get("message", resp.text)
error_code = result.get("error_code", "UNKNOWN")
log.error(f"Server error [{error_code}]: {error_msg}")
raise SystemExit(1)
return result
def post(self, endpoint: str, data: Dict[str, Any]) -> Dict[str, Any]:
return self._request("post", endpoint, data)
def get(self, endpoint: str) -> Dict[str, Any]:
return self._request("get", endpoint)
def delete(self, endpoint: str) -> Dict[str, Any]:
return self._request("delete", endpoint)
def _enum_get(enum_cls: type[Enum], name: Optional[str], default: Any = None) -> Any:
"""Safely get an enum value by name."""
if not name:
return default
try:
return enum_cls[name]
except KeyError:
return default
def _deserialize_video(data: Dict[str, Any]) -> Video:
v = Video(
url=data.get("url") or "https://placeholder",
language=Language.get(data.get("language") or "und"),
descriptor=_enum_get(Track.Descriptor, data.get("descriptor"), Track.Descriptor.URL),
codec=_enum_get(Video.Codec, data.get("codec")),
range_=_enum_get(Video.Range, data.get("range"), Video.Range.SDR),
bitrate=data["bitrate"] * 1000 if data.get("bitrate") else 0,
width=data.get("width") or 0,
height=data.get("height") or 0,
fps=data.get("fps"),
id_=data.get("id"),
)
return v
def _deserialize_audio(data: Dict[str, Any]) -> Audio:
a = Audio(
url=data.get("url") or "https://placeholder",
language=Language.get(data.get("language") or "und"),
descriptor=_enum_get(Track.Descriptor, data.get("descriptor"), Track.Descriptor.URL),
codec=_enum_get(Audio.Codec, data.get("codec")),
bitrate=data["bitrate"] * 1000 if data.get("bitrate") else 0,
channels=data.get("channels"),
joc=1 if data.get("atmos") else 0,
descriptive=data.get("descriptive", False),
id_=data.get("id"),
)
return a
def _deserialize_subtitle(data: Dict[str, Any]) -> Subtitle:
return Subtitle(
url=data.get("url") or "https://placeholder",
language=Language.get(data.get("language") or "und"),
descriptor=_enum_get(Track.Descriptor, data.get("descriptor"), Track.Descriptor.URL),
codec=_enum_get(Subtitle.Codec, data.get("codec")),
cc=data.get("cc", False),
sdh=data.get("sdh", False),
forced=data.get("forced", False),
id_=data.get("id"),
)
def _reconstruct_drm(drm_list: Optional[list]) -> list:
"""Reconstruct DRM objects from serialized API data."""
if not drm_list:
return []
result = []
for drm_info in drm_list:
drm_type = drm_info.get("type", "")
pssh_str = drm_info.get("pssh")
if not pssh_str:
continue
try:
if drm_type == "widevine":
from pywidevine.pssh import PSSH as WidevinePSSH
from envied.core.drm import Widevine
wv_pssh = WidevinePSSH(pssh_str)
result.append(Widevine(pssh=wv_pssh))
elif drm_type == "playready":
import base64 as b64
from pyplayready.system.pssh import PSSH as PlayReadyPSSH
from envied.core.drm import PlayReady
pr_pssh = PlayReadyPSSH(b64.b64decode(pssh_str))
result.append(PlayReady(pssh=pr_pssh, pssh_b64=pssh_str))
except Exception:
continue
return result
def _build_tracks(data: Dict[str, Any]) -> Tracks:
tracks = Tracks()
tracks.videos = [_deserialize_video(v) for v in data.get("video", [])]
tracks.audio = [_deserialize_audio(a) for a in data.get("audio", [])]
tracks.subtitles = [_deserialize_subtitle(s) for s in data.get("subtitles", [])]
for track_data, track_obj in [
*zip(data.get("video", []), tracks.videos),
*zip(data.get("audio", []), tracks.audio),
]:
drm_objs = _reconstruct_drm(track_data.get("drm"))
if drm_objs:
track_obj.drm = drm_objs
tracks.attachments = [
Attachment(url=a["url"], name=a.get("name"), mime_type=a.get("mime_type"), description=a.get("description"))
for a in data.get("attachments", [])
]
return tracks
def _resolve_manifest_data(tracks: Tracks, manifests: list, session: Any) -> None:
"""Re-parse serialized manifests and populate track.data for downloading.
The server serializes DASH and ISM manifest XML as zlib-compressed base64.
We decode and decompress locally, re-parse with the appropriate manifest
parser, then match each remote track to the locally-parsed track by ID
to copy track.data. HLS is skipped as it re-fetches from track.url.
"""
import base64 as b64
import zlib
if not manifests:
return
log_m = logging.getLogger("remote_service")
all_tracks = list(tracks.videos) + list(tracks.audio) + list(tracks.subtitles)
for manifest_info in manifests:
m_type = manifest_info.get("type")
m_url = manifest_info.get("url")
m_data = manifest_info.get("data")
if not m_data or not m_url:
continue
try:
raw = zlib.decompress(b64.b64decode(m_data))
if m_type == "dash":
from lxml import etree
from envied.core.manifests import DASH
xml_tree = etree.fromstring(raw)
fallback_lang = next(
(t.language for t in all_tracks if t.language and str(t.language) != "und"),
None,
)
local_tracks = DASH(xml_tree, m_url).to_tracks(language=fallback_lang)
elif m_type == "ism":
from lxml import etree
from envied.core.manifests import ISM
local_tracks = ISM(etree.fromstring(raw), m_url).to_tracks()
else:
continue
local_all = list(local_tracks.videos) + list(local_tracks.audio) + list(local_tracks.subtitles)
for remote_track in all_tracks:
if remote_track.data.get(m_type):
continue
matched = _match_track(remote_track, local_all)
if matched and matched.data.get(m_type):
remote_track.data.update(matched.data)
remote_track.descriptor = matched.descriptor
if matched.drm and not remote_track.drm:
remote_track.drm = matched.drm
except Exception as e:
log_m.warning("Failed to re-parse %s manifest from %s: %s", m_type, m_url, e)
def _match_track(remote_track: Track, local_tracks: list) -> Optional[Track]:
"""Match a remote track to a locally-parsed track by ID or attributes."""
remote_id = str(remote_track.id)
for lt in local_tracks:
if str(lt.id) == remote_id:
return lt
for lt in local_tracks:
if type(lt).__name__ != type(remote_track).__name__:
continue
if lt.codec != remote_track.codec or str(lt.language) != str(remote_track.language):
continue
if hasattr(lt, "width") and hasattr(remote_track, "width"):
if lt.width == remote_track.width and lt.height == remote_track.height:
return lt
elif hasattr(lt, "channels") and hasattr(remote_track, "channels"):
if lt.bitrate == remote_track.bitrate:
return lt
elif hasattr(lt, "forced"):
if lt.forced == remote_track.forced and lt.sdh == remote_track.sdh:
return lt
return None
def _build_title(info: Dict[str, Any], service_tag: str, fallback_id: str) -> Union[Episode, Movie]:
svc_class = type(service_tag, (), {})
lang = Language.get(info["language"]) if info.get("language") else None
if info.get("type") == "episode":
return Episode(
id_=info.get("id", fallback_id),
service=svc_class,
title=info.get("series_title", "Unknown"),
season=info.get("season", 0),
number=info.get("number", 0),
name=info.get("name"),
year=info.get("year"),
language=lang,
)
return Movie(
id_=info.get("id", fallback_id),
service=svc_class,
name=info.get("name", "Unknown"),
year=info.get("year"),
language=lang,
)
def resolve_server(server_name: Optional[str]) -> tuple[str, str, dict]:
"""Resolve server URL, API key, and per-service config from remote_services."""
remote_services = config.remote_services
if not remote_services:
raise click.ClickException(
"No remote services configured. Add 'remote_services' to your envied.yaml:\n\n"
" remote_services:\n"
" my_server:\n"
' url: "https://server:8080"\n'
' api_key: "your-api-key"'
)
if server_name:
svc = remote_services.get(server_name)
if not svc:
available = ", ".join(remote_services.keys())
raise click.ClickException(f"Remote service '{server_name}' not found. Available: {available}")
services = svc.get("services", {})
services["_server_cdm"] = svc.get("server_cdm", False)
return svc["url"], svc.get("api_key", ""), services
if len(remote_services) == 1:
name, svc = next(iter(remote_services.items()))
log.info(f"Using remote service: {name}")
services = svc.get("services", {})
services["_server_cdm"] = svc.get("server_cdm", False)
return svc["url"], svc.get("api_key", ""), services
available = ", ".join(remote_services.keys())
raise click.ClickException(f"Multiple remote services configured. Use --server to select one: {available}")
def _load_credentials_for_transport(service_tag: str, profile: Optional[str]) -> Optional[Dict[str, str]]:
from envied.commands.dl import dl
credential = dl.get_credentials(service_tag, profile)
if credential:
result: Dict[str, str] = {"username": credential.username, "password": credential.password}
if credential.extra:
result["extra"] = credential.extra
return result
return None
def _load_cookies_for_transport(service_tag: str, profile: Optional[str]) -> Optional[str]:
import zlib
from envied.commands.dl import dl
cookie_path = dl.get_cookie_path(service_tag, profile)
if cookie_path and cookie_path.exists():
return base64.b64encode(zlib.compress(cookie_path.read_bytes())).decode("ascii")
return None
def _resolve_proxy(proxy_arg: Optional[str]) -> Optional[str]:
if not proxy_arg:
return None
from envied.core.proxies.resolve import initialize_proxy_providers, resolve_proxy
try:
providers = initialize_proxy_providers()
return resolve_proxy(proxy_arg, providers)
except ValueError as e:
raise click.ClickException(str(e))
class RemoteService:
"""Service adapter that proxies to a remote unshackle server.
Implements the same interface dl.py's result() expects without
subclassing Service (avoids proxy/geofence setup in __init__).
"""
ALIASES: tuple[str, ...] = ()
GEOFENCE: tuple[str, ...] = ()
NO_SUBTITLES: bool = False
def __init__(
self,
ctx: click.Context,
service_tag: str,
title_id: str,
server_url: str,
api_key: str,
services_config: dict,
service_params: Optional[Dict[str, Any]] = None,
) -> None:
self.__class__.__name__ = service_tag
console.print(Padding(Rule(f"[rule.text]Service: {service_tag} (Remote)"), (1, 2)))
self.service_tag = service_tag
self.title_id = title_id
self.client = RemoteClient(server_url, api_key)
self.ctx = ctx
self._service_params = service_params or {}
self.log = logging.getLogger(service_tag)
self.credential: Optional[Credential] = None
self.current_region: Optional[str] = None
self.title_cache = None
self._titles: Optional[Titles_T] = None
self._tracks_by_title: Dict[str, Tracks] = {}
self._chapters_by_title: Dict[str, list] = {}
self._session_id: Optional[str] = None
self._server_cdm_type: str = "widevine"
self._session = requests.Session()
self._session.headers.update(config.headers)
self._session.mount(
"https://",
HTTPAdapter(
max_retries=Retry(total=5, backoff_factor=0.2, status_forcelist=[429, 500, 502, 503, 504]),
pool_block=True,
),
)
self._session.mount("http://", self._session.adapters["https://"])
svc_config = services_config.get(service_tag, {})
self._server_cdm = services_config.get("_server_cdm", False)
self._apply_service_config(svc_config)
def _apply_service_config(self, svc_config: dict) -> None:
if not svc_config:
return
config_maps = {
"cdm": ("cdm", self.service_tag),
"decryption": ("decryption_map", self.service_tag),
"downloader": ("downloader_map", self.service_tag),
}
for key, (attr, tag) in config_maps.items():
if svc_config.get(key):
target = getattr(config, attr, None)
if target is None:
setattr(config, attr, {})
target = getattr(config, attr)
target[tag] = svc_config[key]
if svc_config.get("downloader"):
config.downloader = svc_config["downloader"]
if svc_config.get("decryption"):
config.decryption = svc_config["decryption"]
extra = {k: v for k, v in svc_config.items() if k not in config_maps}
if extra:
existing = config.services.get(self.service_tag, {})
for key, value in extra.items():
if key in existing and isinstance(existing[key], dict) and isinstance(value, dict):
existing[key].update(value)
else:
existing[key] = value
config.services[self.service_tag] = existing
@property
def session(self) -> requests.Session:
return self._session
@property
def title(self) -> str:
return self.title_id
def authenticate(self, cookies: Optional[CookieJar] = None, credential: Optional[Credential] = None) -> None:
self.credential = credential
profile = self.ctx.parent.params.get("profile") if self.ctx.parent else None
proxy = self.ctx.parent.params.get("proxy") if self.ctx.parent else None
no_proxy = self.ctx.parent.params.get("no_proxy", False) if self.ctx.parent else False
create_data: Dict[str, Any] = {"service": self.service_tag, "title_id": self.title_id}
credentials = _load_credentials_for_transport(self.service_tag, profile)
if credentials:
create_data["credentials"] = credentials
cookies_text = _load_cookies_for_transport(self.service_tag, profile)
if cookies_text:
create_data["cookies"] = cookies_text
if not no_proxy and proxy:
resolved_proxy = _resolve_proxy(proxy)
if resolved_proxy:
create_data["proxy"] = resolved_proxy
if not no_proxy and not proxy:
try:
from envied.core.utils.ip_info import get_ip_info
ip_info = get_ip_info(self._session, cached=True)
if ip_info and ip_info.get("country"):
create_data["client_region"] = ip_info["country"].lower()
except Exception:
pass
if profile:
create_data["profile"] = profile
if no_proxy:
create_data["no_proxy"] = True
# Forward track selection params so the server fetches the right manifests
if self.ctx.parent:
range_ = self.ctx.parent.params.get("range_")
if range_:
create_data["range_"] = [r.name for r in range_]
vcodec = self.ctx.parent.params.get("vcodec")
if vcodec:
create_data["vcodec"] = [c.name for c in vcodec]
quality = self.ctx.parent.params.get("quality")
if quality:
create_data["quality"] = list(quality)
if self.ctx.parent.params.get("best_available"):
create_data["best_available"] = True
if self._service_params:
create_data.update(self._service_params)
cdm = self.ctx.obj.cdm if self.ctx.obj else None
if cdm is not None:
from envied.core.cdm.detect import is_playready_cdm
create_data["cdm_type"] = "playready" if is_playready_cdm(cdm) else "widevine"
cache_data = self._load_cache_files()
if cache_data:
create_data["cache"] = cache_data
result = self.client.post("/api/session/create", create_data)
self._session_id = result["session_id"]
status = result.get("status", "authenticated")
if status == "authenticating":
self._poll_auth_completion()
def _poll_auth_completion(self, poll_interval: float = 2.0, timeout: float = 600.0) -> None:
"""Poll the server until authentication completes, handling interactive prompts.
When the server needs user input (OTP, device code, PIN), it returns
``pending_input`` with a prompt. We display it locally, collect the
response, and POST it back. The server resumes its auth flow.
"""
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
resp = self.client.get(f"/api/session/{self._session_id}/prompt")
status = resp.get("status")
if status == "authenticated":
return
if status == "failed":
error = resp.get("error", "Authentication failed on server")
log.error(f"Remote auth failed: {error}")
raise SystemExit(1)
if status == "pending_input":
prompt = resp.get("prompt", "Enter input: ")
user_response = click.prompt(prompt.rstrip("\n "), default="", show_default=False)
self.client.post(
f"/api/session/{self._session_id}/prompt",
{"response": user_response},
)
continue
time.sleep(poll_interval)
log.error("Remote authentication timed out")
raise SystemExit(1)
def get_titles(self) -> Titles_T:
if self._titles is not None:
return self._titles
result = self.client.get(f"/api/session/{self._session_id}/titles")
titles_list = [_build_title(t, self.service_tag, self.title_id) for t in result.get("titles", [])]
self._titles = (
Series(titles_list) if titles_list and isinstance(titles_list[0], Episode) else Movies(titles_list)
)
return self._titles
def get_titles_cached(self, title_id: str = None) -> Titles_T:
"""Apply the client's local title_map to titles fetched from the remote server.
Lets users rename titles for remote services they don't have installed locally.
The server sends raw titles; the client's own ``services.<TAG>.title_map`` wins.
"""
title_map = (config.services.get(self.service_tag) or {}).get("title_map") or {}
return remap_titles(self.get_titles(), title_map)
def get_tracks(self, title: Title_T) -> Tracks:
title_id = str(title.id)
if title_id in self._tracks_by_title:
return self._tracks_by_title[title_id]
result = self.client.post(f"/api/session/{self._session_id}/tracks", {"title_id": title_id})
tracks = _build_tracks(result)
for k, v in result.get("session_headers", {}).items():
if k.lower() not in ("host", "content-length", "content-type"):
self._session.headers[k] = v
for k, v in result.get("session_cookies", {}).items():
self._session.cookies.set(k, v)
_resolve_manifest_data(tracks, result.get("manifests", []), self._session)
self._server_cdm_type = result.get("server_cdm_type", "widevine")
self._tracks_by_title[title_id] = tracks
self._chapters_by_title[title_id] = result.get("chapters", [])
return tracks
def resolve_server_keys(self, title: Title_T) -> None:
"""Resolve DRM keys via server CDM for all tracks on a title.
Called by dl.py between track selection and download. The server
decides which CDM device to use and tells the client via
server_cdm_type. We send track IDs and the server does the full
CDM flow, returning KID:KEY pairs.
"""
if not self._server_cdm:
return
from uuid import UUID
track_ids = [str(t.id) for t in title.tracks.videos + title.tracks.audio]
if not track_ids:
return
drm_type = getattr(self, "_server_cdm_type", "widevine")
self.log.debug(f"Requesting server CDM keys (server_cdm_type={drm_type})")
try:
with console.status("Retrieving Remote License...", spinner="dots"):
resp = self.client.post(
f"/api/session/{self._session_id}/license",
{
"track_ids": track_ids,
"mode": "server_cdm",
"drm_type": drm_type,
},
)
keys_by_track = resp.get("keys", {})
server_drm_type = resp.get("drm_type", drm_type)
self._server_cdm_type = server_drm_type
self.log.debug(f"Server responded with drm_type={server_drm_type}, keys for {len(keys_by_track)} track(s)")
for track in title.tracks:
track_keys = keys_by_track.get(str(track.id), {})
if not track_keys:
continue
kid_list = list(track_keys.keys())
drm_obj = self._create_drm_stub(server_drm_type, kid_list)
for kid_hex, key_hex in track_keys.items():
drm_obj.content_keys[UUID(hex=kid_hex)] = key_hex
track.drm = [drm_obj]
self.log.debug(
f"Track {track.id}: set DRM to {drm_obj.__class__.__name__} with {len(track_keys)} key(s)"
)
key_count = sum(len(v) for v in keys_by_track.values())
if key_count:
self.log.debug(f"Server CDM resolved {key_count} key(s) using {server_drm_type.upper()}")
except Exception as e:
self.log.warning("Failed to resolve server CDM keys: %s", e)
@staticmethod
def _create_drm_stub(drm_type: str, kid_hexes: list[str]) -> Any:
"""Create a DRM object stub matching the type the server actually used.
For server_cdm mode, this is only used for display keys are already
resolved. We build a minimal DRM object that holds content_keys.
"""
from uuid import UUID
if drm_type == "playready":
import base64 as b64
import struct
from pyplayready.system.pssh import PSSH as PlayReadyPSSH
from envied.core.drm import PlayReady
kid_uuids = [UUID(hex=k) for k in kid_hexes]
kid_b64 = b64.b64encode(kid_uuids[0].bytes_le).decode()
wrm_xml = (
'<WRMHEADER xmlns="http://schemas.microsoft.com/DRM/2007/03/PlayReadyHeader" version="4.0.0.0">'
f"<DATA><PROTECTINFO><KEYLEN>16</KEYLEN><ALGID>AESCTR</ALGID></PROTECTINFO>"
f"<KID>{kid_b64}</KID></DATA></WRMHEADER>"
)
wrm_bytes = wrm_xml.encode("utf-16-le")
record_length = len(wrm_bytes)
obj_length = 4 + 2 + 2 + 2 + record_length
pr_obj = struct.pack("<IHH", obj_length, 1, 1) + struct.pack("<H", record_length) + wrm_bytes
pr_pssh = PlayReadyPSSH(pr_obj)
pssh_b64 = b64.b64encode(pr_obj).decode("ascii")
drm = PlayReady(pssh=pr_pssh, pssh_b64=pssh_b64)
for kid_uuid in kid_uuids:
if kid_uuid not in drm.kids:
drm.kids.append(kid_uuid)
return drm
else:
from pywidevine.pssh import PSSH as WvPSSH
from envied.core.drm import Widevine
kid_uuids = [UUID(hex=k) for k in kid_hexes]
WIDEVINE_SYSTEM_ID = UUID("edef8ba9-79d6-4ace-a3c8-27dcd51d21ed")
dummy_pssh = WvPSSH.new(system_id=WIDEVINE_SYSTEM_ID, key_ids=kid_uuids)
return Widevine(pssh=dummy_pssh, kid=kid_hexes[0])
def get_chapters(self, title: Title_T) -> Chapters:
title_id = str(title.id)
if title_id not in self._chapters_by_title:
self.get_tracks(title)
raw = self._chapters_by_title.get(title_id, [])
return Chapters([Chapter(ch["timestamp"], ch.get("name")) for ch in raw])
def get_widevine_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[Union[bytes, str]]:
return self._proxy_license(challenge, track, "widevine")
def get_playready_license(
self, *, challenge: bytes, title: Title_T, track: AnyTrack
) -> Optional[Union[bytes, str]]:
return self._proxy_license(challenge, track, "playready")
def get_widevine_service_certificate(
self,
*,
challenge: bytes,
title: Title_T,
track: AnyTrack,
) -> Union[bytes, str]:
try:
resp = self.client.post(
f"/api/session/{self._session_id}/license",
{
"track_id": str(track.id),
"challenge": base64.b64encode(challenge).decode("ascii"),
"drm_type": "widevine",
"is_certificate": True,
},
)
return base64.b64decode(resp["license"])
except Exception:
return None
def _proxy_license(self, challenge: Union[bytes, str], track: AnyTrack, drm_type: str) -> bytes:
if isinstance(challenge, str):
challenge = challenge.encode("utf-8")
pssh_b64 = None
if track.drm:
for drm_obj in track.drm:
drm_class = drm_obj.__class__.__name__
if drm_type == "playready" and drm_class == "PlayReady":
pssh_b64 = drm_obj.data["pssh_b64"]
break
elif drm_type == "widevine" and drm_class == "Widevine":
pssh_b64 = drm_obj.pssh.dumps()
break
if self._server_cdm:
from uuid import UUID
if pssh_b64:
try:
resp = self.client.post(
f"/api/session/{self._session_id}/license",
{
"track_id": str(track.id),
"drm_type": drm_type,
"mode": "server_cdm",
"pssh": pssh_b64,
},
)
keys = resp.get("keys", {})
if keys and track.drm:
for drm_obj in track.drm:
if hasattr(drm_obj, "content_keys"):
for kid_hex, key_hex in keys.items():
drm_obj.content_keys[UUID(hex=kid_hex)] = key_hex
return challenge
except Exception as e:
self.log.warning("server_cdm license failed: %s", e)
return challenge
payload = {
"track_id": str(track.id),
"challenge": base64.b64encode(challenge).decode("ascii"),
"drm_type": drm_type,
}
if pssh_b64:
payload["pssh"] = pssh_b64
resp = self.client.post(f"/api/session/{self._session_id}/license", payload)
return base64.b64decode(resp["license"])
def on_segment_downloaded(self, track: AnyTrack, segment: Any) -> None:
pass
def on_track_downloaded(self, track: AnyTrack) -> None:
pass
def on_track_decrypted(self, track: AnyTrack, drm: Any, segment: Any = None) -> None:
pass
def on_track_repacked(self, track: AnyTrack) -> None:
pass
def on_track_multiplex(self, track: AnyTrack) -> None:
pass
def close(self) -> None:
if self._session_id:
try:
result = self.client.delete(f"/api/session/{self._session_id}")
self._save_returned_cache(result.get("cache", {}))
except Exception as e:
self.log.warning(f"Failed to clean up remote session: {e}")
self._session_id = None
def _save_returned_cache(self, cache_data: Dict[str, str]) -> None:
"""Save cache files returned by the server to the local cache directory.
The server returns updated cache files (e.g. refreshed tokens) on
session close. Writing them locally means the next remote session
can forward them back, skipping interactive auth.
"""
if not cache_data:
return
import zlib
cache_dir = config.directories.cache / self.service_tag
cache_dir.mkdir(parents=True, exist_ok=True)
for key, content in cache_data.items():
try:
decompressed = zlib.decompress(base64.b64decode(content))
(cache_dir / key).with_suffix(".json").write_bytes(decompressed)
except Exception as e:
self.log.warning(f"Failed to save returned cache file '{key}': {e}")
self.log.info(f"Saved {len(cache_data)} cache file(s) from server")
def _load_cache_files(self) -> Dict[str, str]:
import zlib
cache_dir = config.directories.cache / self.service_tag
if not cache_dir.is_dir():
return {}
return {
f.stem: base64.b64encode(zlib.compress(f.read_bytes())).decode("ascii")
for f in cache_dir.glob("*.json")
if not f.stem.startswith("titles_")
}
__all__ = ("RemoteClient", "RemoteService", "resolve_server")
+49 -20
View File
@@ -1,19 +1,22 @@
import base64
import logging
from abc import ABCMeta, abstractmethod
from collections.abc import Callable, Generator
from dataclasses import dataclass, field
from http.cookiejar import CookieJar
from pathlib import Path
from typing import Optional, Union
from typing import TYPE_CHECKING, Optional, Union
from urllib.parse import urlparse, urlunparse
if TYPE_CHECKING:
from envied.core.api.input_bridge import InputBridge
import click
import m3u8
import requests
from requests.adapters import HTTPAdapter, Retry
from rich.padding import Padding
from rich.rule import Rule
from rich.text import Text
from envied.core.cacher import Cacher
from envied.core.config import config
@@ -23,10 +26,10 @@ from envied.core.credential import Credential
from envied.core.drm import DRM_T
from envied.core.search_result import SearchResult
from envied.core.title_cacher import TitleCacher, get_account_hash, get_region_from_proxy
from envied.core.titles import Title_T, Titles_T
from envied.core.titles import Title_T, Titles_T, remap_titles
from envied.core.tracks import Chapters, Tracks
from envied.core.tracks.video import Video
from envied.core.utilities import get_cached_ip_info, get_ip_info
from envied.core.utils.ip_info import get_ip_info
@dataclass
@@ -101,11 +104,13 @@ class Service(metaclass=ABCMeta):
self.session = self.get_session()
self.cache = Cacher(self.__class__.__name__)
self.title_cache = TitleCacher(self.__class__.__name__)
self.cache_dir = config.directories.cache / self.__class__.__name__
# Store context for cache control flags and credential
self.ctx = ctx
self.credential = None # Will be set in authenticate()
self.current_region = None # Will be set based on proxy/geolocation
self._input_bridge: Optional[InputBridge] = None
# Set track request from CLI params - services can read/override in their __init__
vcodec = ctx.parent.params.get("vcodec") if ctx.parent else None
@@ -207,15 +212,10 @@ class Service(metaclass=ABCMeta):
if proxy:
self.session.proxies.update({"all": proxy})
proxy_parse = urlparse(proxy)
if proxy_parse.username and proxy_parse.password:
self.session.headers.update(
{
"Proxy-Authorization": base64.b64encode(
f"{proxy_parse.username}:{proxy_parse.password}".encode("utf8")
).decode()
}
)
# Don't set Proxy-Authorization manually: both rnet (Proxy.all) and
# requests authenticate from the credentials embedded in the proxy URL.
# A manual header here was malformed (no "Basic " scheme) and broke
# plaintext-http forward-proxy requests with HTTP 407.
# Always verify proxy IP - proxies can change exit nodes
try:
proxy_ip_info = get_ip_info(self.session)
@@ -227,7 +227,7 @@ class Service(metaclass=ABCMeta):
else:
# No proxy, use cached IP info for title caching (non-critical)
try:
ip_info = get_cached_ip_info(self.session)
ip_info = get_ip_info(self.session, cached=True)
self.current_region = ip_info.get("country", "").lower() if ip_info else None
except Exception as e:
self.log.debug(f"Failed to get cached IP info: {e}")
@@ -271,6 +271,8 @@ class Service(metaclass=ABCMeta):
raise
if first:
all_tracks.add(hdr_tracks, warn_only=True)
if hdr_tracks.manifest_url and not all_tracks.manifest_url:
all_tracks.manifest_url = hdr_tracks.manifest_url
first = False
else:
for video in hdr_tracks.videos:
@@ -289,13 +291,13 @@ class Service(metaclass=ABCMeta):
except (ValueError, SystemExit) as e:
if self.track_request.best_available:
codec_name = codec_val.name if codec_val else "default"
self.log.warning(
f" - {range_val.name}/{codec_name} not available, skipping ({e})"
)
self.log.warning(f" - {range_val.name}/{codec_name} not available, skipping ({e})")
continue
raise
if first:
all_tracks.add(tracks, warn_only=True)
if tracks.manifest_url and not all_tracks.manifest_url:
all_tracks.manifest_url = tracks.manifest_url
first = False
else:
for video in tracks.videos:
@@ -349,6 +351,20 @@ class Service(metaclass=ABCMeta):
# Store credential for cache key generation
self.credential = credential
def request_input(self, prompt: str) -> str:
"""Request interactive input from the user.
When running locally (CLI), prompts via the shared rich console so the
prompt renders correctly alongside Live progress / log handlers.
When running in serve mode with an :class:`InputBridge` attached,
delegates to the bridge which relays the prompt to the remote client.
"""
if self._input_bridge is not None:
return self._input_bridge.request_input(prompt)
indent = " " * 5
padded = indent + prompt.replace("\n", "\n" + indent)
return console.input(Text(padded, style="text"))
def search(self) -> Generator[SearchResult, None, None]:
"""
Search by query for titles from the Service.
@@ -394,7 +410,9 @@ class Service(metaclass=ABCMeta):
Decode the data, return as is to reduce unnecessary computations.
"""
def get_playready_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[Union[bytes, str]]:
def get_playready_license(
self, *, challenge: bytes, title: Title_T, track: AnyTrack
) -> Optional[Union[bytes, str]]:
"""
Get a PlayReady License message by sending a License Request (challenge).
@@ -458,7 +476,7 @@ class Service(metaclass=ABCMeta):
else:
# If we can't determine title_id, just call get_titles directly
self.log.debug("Cannot determine title_id for caching, bypassing cache")
return self.get_titles()
return self.apply_title_map(self.get_titles())
# Get cache control flags from context
no_cache = False
@@ -471,7 +489,7 @@ class Service(metaclass=ABCMeta):
account_hash = get_account_hash(self.credential)
# Use title cache to get titles with fallback support
return self.title_cache.get_cached_titles(
titles = self.title_cache.get_cached_titles(
title_id=str(title_id),
fetch_function=self.get_titles,
region=self.current_region,
@@ -479,6 +497,17 @@ class Service(metaclass=ABCMeta):
no_cache=no_cache,
reset_cache=reset_cache,
)
return self.apply_title_map(titles)
def apply_title_map(self, titles: Titles_T) -> Titles_T:
"""
Rewrite service-provided titles using the per-service ``title_map`` config.
``title_map`` lives under ``services.<TAG>`` in envied.yaml. Applied after the
title cache so config edits take effect without a cache reset, and before any
``--enrich`` override so enrich wins. See ``remap_titles`` for the match rules.
"""
return remap_titles(titles, (self.config or {}).get("title_map") or {})
@abstractmethod
def get_tracks(self, title: Title_T) -> Tracks:
+187 -3
View File
@@ -1,3 +1,7 @@
from __future__ import annotations
import logging
import re
from pathlib import Path
import click
@@ -6,6 +10,8 @@ from envied.core.config import config
from envied.core.service import Service
from envied.core.utilities import import_module_by_path
log = logging.getLogger("services")
_service_dirs = config.directories.services
if not isinstance(_service_dirs, list):
_service_dirs = [_service_dirs]
@@ -15,23 +21,118 @@ _SERVICES = sorted(
key=lambda x: x.parent.stem,
)
_MODULES = {path.parent.stem: getattr(import_module_by_path(path), path.parent.stem) for path in _SERVICES}
_ALIASES = {tag: getattr(module, "ALIASES") for tag, module in _MODULES.items()}
def load_service(path: Path) -> object:
"""Load one Service module, returning its tag-named class.
Raises a concise, single-line error naming the Service and the real cause so
a broken Service never surfaces as a raw traceback pointing at the loader.
"""
tag = path.parent.stem
try:
module = import_module_by_path(path)
except Exception as e:
raise RuntimeError(f"{tag}: failed to import — {type(e).__name__}: {e} ({path})") from e
try:
return getattr(module, tag)
except AttributeError as e:
raise RuntimeError(
f"{tag}: no class named '{tag}' found in {path} — the class name must match the directory name"
) from e
def load_services(paths: list[Path]) -> tuple[dict[str, object], list[str]]:
"""Load every Service, returning the good ones plus a list of load errors.
Importing this module must never raise: it is imported by several commands,
and a failed import is not cached by Python, so raising here would re-run and
re-report for every command. Instead we collect failures and let the caller
surface them once, cleanly, at the point services are actually used.
"""
modules: dict[str, object] = {}
errors: list[str] = []
for path in paths:
try:
modules[path.parent.stem] = load_service(path)
except Exception as e:
errors.append(str(e))
return modules, errors
_MODULES, LOAD_ERRORS = load_services(_SERVICES)
_ALIASES = {tag: getattr(module, "ALIASES", ()) for tag, module in _MODULES.items()}
def check_load_errors() -> None:
"""Raise a single clean error if any Service failed to load.
Called when services are actually needed (listing/resolving) so the message
is rendered once by Click, without a traceback and without cascading through
every command that imports this module.
"""
if LOAD_ERRORS:
joined = "\n".join(f" - {err}" for err in LOAD_ERRORS)
raise click.ClickException(f"Failed to load {len(LOAD_ERRORS)} service(s):\n{joined}")
class Services(click.MultiCommand):
"""Lazy-loaded command group of project services."""
_remote_services_cache: list[dict] | None = None
# Click-specific methods
def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]:
"""Preprocess --slow to support optional range value before Click parses args."""
processed = []
i = 0
while i < len(args):
if args[i] == "--slow":
if i + 1 < len(args) and re.match(r"^\d+-\d+$", args[i + 1]):
processed.append(f"--slow={args[i + 1]}")
i += 2
else:
processed.append("--slow=60-120")
i += 1
else:
processed.append(args[i])
i += 1
return super().parse_args(ctx, processed)
def list_commands(self, ctx: click.Context) -> list[str]:
"""Returns a list of all available Services as command names for Click."""
"""Returns a list of all available Services as command names for Click.
In remote mode, fetches the service list from the remote server
so the user sees exactly what's available remotely.
"""
remote = ctx.params.get("remote") or (ctx.parent and ctx.parent.params.get("remote"))
if remote:
remote_services = Services._fetch_remote_services(ctx)
if remote_services is not None:
return [s["tag"] for s in remote_services]
tags = Services.get_tags()
for svc_cfg in config.remote_services.values():
for remote_tag in svc_cfg.get("services", {}).keys():
if remote_tag not in tags:
tags.append(remote_tag)
return tags
check_load_errors()
return Services.get_tags()
def get_command(self, ctx: click.Context, name: str) -> click.Command:
"""Load the Service and return the Click CLI method."""
check_load_errors()
tag = Services.get_tag(name)
import_file = ctx.params.get("import_file") or (ctx.parent and ctx.parent.params.get("import_file"))
if import_file:
return Services._make_import_command(tag, ctx)
remote = ctx.params.get("remote") or (ctx.parent and ctx.parent.params.get("remote"))
if remote:
return Services._make_remote_command(tag, ctx)
try:
service = Services.load(tag)
except KeyError as e:
@@ -47,6 +148,89 @@ class Services(click.MultiCommand):
raise click.ClickException(f"Service '{tag}' has no 'cli' method configured.")
@staticmethod
def _fetch_remote_services(ctx: click.Context) -> list[dict] | None:
"""Fetch the service list from the remote server (cached per process)."""
if Services._remote_services_cache is not None:
return Services._remote_services_cache
try:
from envied.core.remote_service import RemoteClient, resolve_server
server_name = ctx.params.get("server")
server_url, api_key, _ = resolve_server(server_name)
client = RemoteClient(server_url, api_key)
result = client.get("/api/services")
Services._remote_services_cache = result.get("services", [])
return Services._remote_services_cache
except Exception:
return None
@staticmethod
def _make_remote_command(tag: str, ctx: click.Context) -> click.Command:
"""Create a Click command for a remote service with server-provided options."""
svc_info = Services._fetch_remote_service_info(tag, ctx)
short_help = svc_info.get("url") if svc_info else None
cli_params = svc_info.get("cli_params") if svc_info else None
@click.command(name=tag, short_help=short_help)
@click.argument("title", type=str)
@click.pass_context
def remote_cli(ctx: click.Context, title: str, **kwargs: object) -> object:
from envied.core.remote_service import RemoteService, resolve_server
server_name = ctx.parent.params.get("server") if ctx.parent else None
server_url, api_key, services_config = resolve_server(server_name)
service_params = {k: v for k, v in kwargs.items() if v is not None and v is not False}
return RemoteService(ctx, tag, title, server_url, api_key, services_config, service_params=service_params)
if cli_params:
for param in cli_params:
if param.get("kind") == "option":
opts = param.get("opts", [f"--{param['name']}"])
kwargs: dict = {}
if param.get("is_flag"):
kwargs["is_flag"] = True
kwargs["default"] = param.get("default", False)
else:
kwargs["default"] = param.get("default")
kwargs["type"] = str
if param.get("help"):
kwargs["help"] = param["help"]
remote_cli = click.option(*opts, **kwargs)(remote_cli)
return remote_cli
@staticmethod
def _make_import_command(tag: str, ctx: click.Context) -> click.Command:
"""Create a synthetic command that yields an ImportService from an export JSON.
Mirrors how remote services are wired so dl.py's result() runs unchanged.
"""
@click.command(name=tag, short_help="Reconstruct a download from an export JSON.")
@click.argument("title", type=str, required=False, default="")
@click.pass_context
def import_cli(ctx: click.Context, title: str, **kwargs: object) -> object:
from envied.core.import_service import ImportService
import_file = ctx.params.get("import_file") or (ctx.parent and ctx.parent.params.get("import_file"))
return ImportService(ctx, tag, title, import_file)
return import_cli
@staticmethod
def _fetch_remote_service_info(tag: str, ctx: click.Context) -> dict | None:
"""Fetch service info for a specific service from the remote server."""
try:
services = Services._fetch_remote_services(ctx)
if services:
for svc in services:
if svc.get("tag") == tag:
return svc
except Exception:
pass
return None
# Methods intended to be used anywhere
@staticmethod
+638 -167
View File
@@ -1,96 +1,549 @@
"""Session utilities for creating HTTP sessions with different backends."""
"""Session utilities for creating HTTP sessions with TLS fingerprinting via rnet (Rust/BoringSSL)."""
from __future__ import annotations
import http
import logging
import random
import time
import warnings
from collections.abc import Iterator, MutableMapping
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from typing import Any
from urllib.parse import urlparse
from http.cookiejar import CookieJar
from typing import Any, Optional
from urllib.parse import urlencode, urlparse, urlunparse
from curl_cffi.requests import Response, Session, exceptions
import rnet
from requests import HTTPError, Request
from requests.structures import CaseInsensitiveDict
from envied.core.config import config
# Globally suppress curl_cffi HTTPS proxy warnings since some proxy providers
# (like NordVPN) require HTTPS URLs but curl_cffi expects HTTP format
warnings.filterwarnings(
"ignore", message="Make sure you are using https over https proxy.*", category=RuntimeWarning, module="curl_cffi.*"
)
# ---------------------------------------------------------------------------
# Impersonate preset mapping — rnet uses named presets (no custom JA3/Akamai)
# ---------------------------------------------------------------------------
FINGERPRINT_PRESETS = {
"okhttp4": {
"ja3": (
"771," # TLS 1.2
"4865-4866-4867-49195-49196-52393-49199-49200-52392-49171-49172-156-157-47-53," # Ciphers
"0-23-65281-10-11-35-16-5-13-51-45-43," # Extensions
"29-23-24," # Named groups (x25519, secp256r1, secp384r1)
"0" # EC point formats
),
"akamai": "4:16777216|16711681|0|m,p,a,s",
"description": "OkHttp 3.x/4.x (BoringSSL TLS stack)",
},
"okhttp5": {
"ja3": (
"771," # TLS 1.2
"4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53," # Ciphers
"0-23-65281-10-11-35-16-5-13-51-45-43," # Extensions
"29-23-24," # Named groups (x25519, secp256r1, secp384r1)
"0" # EC point formats
),
"akamai": "4:16777216|16711681|0|m,p,a,s",
"description": "OkHttp 5.x (BoringSSL TLS stack)",
},
"shield_okhttp": {
"ja3": (
"771," # TLS 1.2
"4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53," # Ciphers (OkHttp 4.11)
"0-23-65281-10-11-35-16-5-13-51-45-43-21," # Extensions (incl padding ext 21)
"29-23-24," # Named groups (x25519, secp256r1, secp384r1)
"0" # EC point formats
),
"akamai": "4:16777216|16711681|0|m,p,a,s",
"description": "NVIDIA SHIELD Android TV OkHttp 4.11 (captured JA3)",
},
DEFAULT_IMPERSONATE = rnet.Impersonate.Chrome131
def _resolve_impersonate(browser: str) -> rnet.Impersonate:
"""Resolve a browser string to an rnet.Impersonate preset.
Accepts exact rnet preset names (e.g. "Chrome131", "OkHttp4_12", "Edge101").
See https://github.com/0x676e67/rnet for the full list of available presets.
"""
preset = getattr(rnet.Impersonate, browser, None)
if preset is not None:
return preset
raise ValueError(
f"Unknown impersonate preset: {browser!r}. "
f"Use exact rnet preset names like 'Chrome131', 'OkHttp4_12', 'Edge101'. "
f"See rnet.Impersonate for all available presets."
)
# Map string method names to rnet.Method enum
_METHOD_MAP: dict[str, rnet.Method] = {
"GET": rnet.Method.GET,
"POST": rnet.Method.POST,
"PUT": rnet.Method.PUT,
"DELETE": rnet.Method.DELETE,
"HEAD": rnet.Method.HEAD,
"OPTIONS": rnet.Method.OPTIONS,
"PATCH": rnet.Method.PATCH,
"TRACE": rnet.Method.TRACE,
}
class MaxRetriesError(exceptions.RequestException):
def __init__(self, message, cause=None):
# ---------------------------------------------------------------------------
# Response headers adapter — bytes → str
# ---------------------------------------------------------------------------
class RnetResponseHeaders(MutableMapping):
"""Read-only str-based view over rnet's bytes-based HeaderMap."""
def __init__(self, header_map: Any) -> None:
self._map = header_map
def _decode(self, val: Any) -> str:
return val.decode("utf-8", errors="replace") if isinstance(val, (bytes, bytearray)) else str(val)
def __getitem__(self, key: str) -> str:
val = self._map[key]
return self._decode(val)
def __setitem__(self, key: str, value: str) -> None:
raise TypeError("Response headers are read-only")
def __delitem__(self, key: str) -> None:
raise TypeError("Response headers are read-only")
def __contains__(self, key: object) -> bool:
if not isinstance(key, str):
return False
return self._map.contains_key(key)
def __iter__(self) -> Iterator[str]:
seen: set[str] = set()
for k, _ in self._map.items():
dk = self._decode(k)
if dk not in seen:
seen.add(dk)
yield dk
def __len__(self) -> int:
return self._map.keys_len()
def get(self, key: str, default: Optional[str] = None) -> Optional[str]:
val = self._map.get(key)
if val is None:
return default
return self._decode(val)
def items(self) -> list[tuple[str, str]]:
return [(self._decode(k), self._decode(v)) for k, v in self._map.items()]
# ---------------------------------------------------------------------------
# Response wrapper — requests-compatible interface
# ---------------------------------------------------------------------------
class RnetResponse:
"""Wraps rnet.BlockingResponse with a requests-compatible API."""
def __init__(self, resp: Any) -> None:
self._resp = resp
self._headers: Optional[RnetResponseHeaders] = None
self._content: Optional[bytes] = None
self._text: Optional[str] = None
self._streamed = False
@property
def status_code(self) -> int:
return int(str(self._resp.status_code))
@property
def ok(self) -> bool:
return self._resp.ok
@property
def headers(self) -> RnetResponseHeaders:
if self._headers is None:
self._headers = RnetResponseHeaders(self._resp.headers)
return self._headers
@property
def url(self) -> str:
return str(self._resp.url)
@property
def content_length(self) -> Optional[int]:
return self._resp.content_length
@property
def content(self) -> bytes:
if self._content is None:
self._content = self._resp.bytes()
return self._content
@property
def text(self) -> str:
if self._text is None:
encoding = self._resp.encoding or "utf-8"
self._text = self.content.decode(encoding, errors="replace")
return self._text
@property
def reason(self) -> str:
try:
return http.HTTPStatus(self.status_code).phrase
except ValueError:
return "Unknown"
@property
def cookies(self) -> Any:
return self._resp.cookies
def json(self, **kwargs: Any) -> Any:
import json as _json
return _json.loads(self.content)
def raise_for_status(self) -> None:
if not self.ok:
raise HTTPError(
f"{self.status_code} {self.reason}: {self.url}",
response=self,
)
def iter_content(self, chunk_size: Optional[int] = None) -> Iterator[bytes]:
"""Re-chunk rnet's variable-size stream into fixed-size pieces."""
self._streamed = True
if chunk_size is None or chunk_size <= 0:
yield from self._resp.stream()
return
buf = bytearray()
for chunk in self._resp.stream():
buf.extend(chunk)
while len(buf) >= chunk_size:
yield bytes(buf[:chunk_size])
buf = buf[chunk_size:]
if buf:
yield bytes(buf)
def stream(self) -> Iterator[bytes]:
"""Direct pass-through of rnet's native stream iterator."""
self._streamed = True
yield from self._resp.stream()
def close(self) -> None:
try:
self._resp.close()
except Exception:
pass
# ---------------------------------------------------------------------------
# Session headers adapter — persists via client.update()
# ---------------------------------------------------------------------------
class RnetSessionHeaders(CaseInsensitiveDict):
"""Dict-like headers that persist to the rnet client via update()."""
def __init__(self, client: Any) -> None:
self._client = client
super().__init__()
def _sync(self) -> None:
"""Push current headers to the rnet client."""
if self._client is not None and hasattr(self, "_store") and self._store:
self._client.update(headers={k: v for k, v in self.items()})
def __setitem__(self, key: str, value: str) -> None:
super().__setitem__(key, value)
self._sync()
def update(self, __m: Any = None, **kwargs: Any) -> None:
if __m:
if hasattr(__m, "items"):
for k, v in __m.items():
super().__setitem__(k, v)
else:
for k, v in __m:
super().__setitem__(k, v)
for k, v in kwargs.items():
super().__setitem__(k, v)
self._sync()
def pop(self, key: str, *args: Any) -> Any:
result = super().pop(key, *args)
# rnet doesn't support removing individual headers, but we track locally
# and always send the full set on next update
return result
def __delitem__(self, key: str) -> None:
super().__delitem__(key)
# ---------------------------------------------------------------------------
# Session cookies adapter
# ---------------------------------------------------------------------------
class RnetCookieAdapter(MutableMapping):
"""Cookie adapter that bridges requests-style cookie access to rnet."""
def __init__(self, client: Any) -> None:
self._client = client
self._cookies: dict[str, dict[str, str]] = {}
self._flat: dict[str, str] = {}
self._original_cookies: list[Any] = []
def _set_cookie_on_client(self, url: str, name: str, value: str) -> None:
"""Set a cookie on the rnet client, or buffer locally if the client is not yet created."""
if self._client is not None:
try:
self._client.set_cookie(url, rnet.Cookie(name, value))
except Exception:
pass
def _flush_to_client(self) -> None:
"""Push all buffered cookies to the rnet client once it is created."""
if self._client is None:
return
for domain, cookies in self._cookies.items():
url = f"https://{domain.lstrip('.')}" if domain else "https://localhost"
for name, value in cookies.items():
try:
self._client.set_cookie(url, rnet.Cookie(name, value))
except Exception:
pass
@property
def jar(self) -> CookieJar:
"""Return a CookieJar with original Cookie objects (requests compat).
Used by ``save_cookies`` in dl.py to persist cookies back to disk.
"""
jar = CookieJar()
for cookie in self._original_cookies:
jar.set_cookie(cookie)
return jar
def update(self, other: Any = None, **kwargs: Any) -> None:
if other is None:
other = {}
if isinstance(other, CookieJar):
for cookie in other:
domain = cookie.domain or ""
name = cookie.name
value = cookie.value or ""
self._flat[name] = value
self._cookies.setdefault(domain, {})[name] = value
self._original_cookies.append(cookie)
url = f"https://{domain.lstrip('.')}" if domain else "https://localhost"
self._set_cookie_on_client(url, name, value)
elif isinstance(other, dict):
for name, value in other.items():
self._flat[name] = value
self._set_cookie_on_client("https://localhost", name, str(value))
self._flat.update(other)
elif hasattr(other, "items"):
for name, value in other.items():
self._flat[name] = str(value)
self._set_cookie_on_client("https://localhost", name, str(value))
for name, value in kwargs.items():
self._flat[name] = value
self._set_cookie_on_client("https://localhost", name, value)
def get(
self, name: str, default: Optional[str] = None, domain: Optional[str] = None, path: Optional[str] = None
) -> Optional[str]:
if domain and domain in self._cookies:
return self._cookies[domain].get(name, default)
return self._flat.get(name, default)
def set(self, name: str, value: str, domain: str = "localhost") -> None:
self._flat[name] = value
self._cookies.setdefault(domain, {})[name] = value
url = f"https://{domain.lstrip('.')}"
self._set_cookie_on_client(url, name, value)
def __getitem__(self, name: str) -> str:
return self._flat[name]
def __setitem__(self, name: str, value: str) -> None:
self.set(name, value)
def __delitem__(self, name: str) -> None:
self._flat.pop(name, None)
for domain_cookies in self._cookies.values():
domain_cookies.pop(name, None)
def __contains__(self, name: object) -> bool:
return name in self._flat
def __iter__(self) -> Iterator:
return iter(self._flat)
def __len__(self) -> int:
return len(self._flat)
def __bool__(self) -> bool:
return bool(self._flat)
def get_dict(self, domain: Optional[str] = None, path: Optional[str] = None) -> dict[str, str]:
"""Return cookies as a plain dict (requests RequestsCookieJar compat).
If *domain* is given, only cookies for that domain are returned.
*path* is accepted for API compatibility but ignored (flat storage).
"""
if domain is not None:
return dict(self._cookies.get(domain, {}))
return dict(self._flat)
def clear(self, domain: Optional[str] = None, path: Optional[str] = None, name: Optional[str] = None) -> None:
"""Remove cookies (requests RequestsCookieJar compat).
- ``clear()`` removes all cookies.
- ``clear(domain=..., path=..., name=...)`` removes a specific cookie.
"""
if name is not None:
self._flat.pop(name, None)
if domain is not None and domain in self._cookies:
self._cookies[domain].pop(name, None)
else:
for domain_cookies in self._cookies.values():
domain_cookies.pop(name, None)
elif domain is not None:
removed = self._cookies.pop(domain, {})
for k in removed:
# Only remove from flat if no other domain has same key
still_exists = any(k in dc for dc in self._cookies.values())
if not still_exists:
self._flat.pop(k, None)
else:
self._flat.clear()
self._cookies.clear()
def items(self) -> list[tuple[str, str]]:
return list(self._flat.items())
def keys(self) -> list[str]:
return list(self._flat.keys())
def values(self) -> list[str]:
return list(self._flat.values())
# ---------------------------------------------------------------------------
# Session proxy adapter
# ---------------------------------------------------------------------------
class RnetProxyDict(dict):
"""Dict-like proxy config that syncs to the rnet client.
Accepts ``{"all": url}``, ``{"https": url}``, or ``{"http": url}``
and applies via rnet's native ``proxies`` parameter (``List[rnet.Proxy]``).
Supports both lazy (pre-client) and live (post-client) proxy updates.
"""
def __init__(self, session: "RnetSession") -> None:
super().__init__()
self._session = session
def _sync(self) -> None:
proxy = self.get("all") or self.get("https") or self.get("http")
proxies = [rnet.Proxy.all(proxy)] if proxy else []
self._session._client_kwargs["proxies"] = proxies or None
if self._session._client is not None:
self._session._client.update(proxies=proxies or None)
def update(self, __m: Any = None, **kwargs: Any) -> None:
super().update(__m or {}, **kwargs)
self._sync()
def __setitem__(self, key: str, value: str) -> None:
super().__setitem__(key, value)
self._sync()
# ---------------------------------------------------------------------------
# Exceptions
# ---------------------------------------------------------------------------
class MaxRetriesError(Exception):
def __init__(self, message: str, cause: Optional[Exception] = None) -> None:
super().__init__(message)
self.__cause__ = cause
class CurlSession(Session):
# ---------------------------------------------------------------------------
# RnetSession — main session class
# ---------------------------------------------------------------------------
class RnetSession:
"""TLS-fingerprinted HTTP session powered by rnet (Rust/BoringSSL).
Drop-in replacement for CurlSession with requests-compatible API.
Supports browser impersonation (Chrome, Firefox, Edge, Safari, OkHttp),
retry with exponential backoff, cookie persistence, and proxy support.
The client is created lazily on the first request so that headers,
cookies, and proxies can be configured freely before any connection
is established.
"""
def __init__(
self,
max_retries: int = 5,
backoff_factor: float = 0.2,
max_backoff: float = 60.0,
status_forcelist: list[int] | None = None,
allowed_methods: set[str] | None = None,
catch_exceptions: tuple[type[Exception], ...] | None = None,
status_forcelist: Optional[list[int]] = None,
allowed_methods: Optional[set[str]] = None,
catch_exceptions: Optional[tuple[type[Exception], ...]] = None,
**session_kwargs: Any,
):
super().__init__(**session_kwargs)
) -> None:
self.max_retries = max_retries
self.backoff_factor = backoff_factor
self.max_backoff = max_backoff
self.status_forcelist = status_forcelist or [429, 500, 502, 503, 504]
self.allowed_methods = allowed_methods or {"GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE"}
self.catch_exceptions = catch_exceptions or (
exceptions.ConnectionError,
exceptions.ProxyError,
exceptions.SSLError,
exceptions.Timeout,
rnet.ConnectionError,
rnet.TimeoutError,
rnet.RequestError,
)
self.log = logging.getLogger(self.__class__.__name__)
def get_sleep_time(self, response: Response | None, attempt: int) -> float | None:
client_kwargs: dict[str, Any] = {}
for key in ("impersonate", "timeout", "proxies", "verify", "redirect"):
if key in session_kwargs:
client_kwargs[key] = session_kwargs.pop(key)
if "proxy" in session_kwargs:
proxy_url = session_kwargs.pop("proxy")
if proxy_url:
client_kwargs["proxies"] = [rnet.Proxy.all(proxy_url)]
client_kwargs["cookie_store"] = True
self.verify: bool = client_kwargs.pop("verify", True)
if not self.verify:
client_kwargs["danger_accept_invalid_certs"] = True
self._client_kwargs = dict(client_kwargs)
self._client: Optional[rnet.BlockingClient] = None
self.headers = RnetSessionHeaders(None)
self.cookies = RnetCookieAdapter(None)
self.proxies = RnetProxyDict(self)
if "headers" in session_kwargs:
self.headers.update(session_kwargs.pop("headers"))
if "cookies" in session_kwargs:
self.cookies.update(session_kwargs.pop("cookies"))
if "proxies" in session_kwargs:
self.proxies.update(session_kwargs.pop("proxies"))
def _ensure_client(self) -> rnet.BlockingClient:
"""Lazily create the rnet client on first use, flushing any buffered state."""
if self._client is None:
self._client = rnet.BlockingClient(**self._client_kwargs)
self.headers._client = self._client
self.headers._sync()
self.cookies._client = self._client
self.cookies._flush_to_client()
return self._client
def _build_url(self, url: str, params: Optional[Any] = None) -> str:
"""Encode params into the URL (rnet ignores the params kwarg).
Accepts the same shapes as requests: a mapping, a sequence of pairs, or a
pre-built query string/bytes. A string is appended verbatim (already encoded);
urlencode() would raise TypeError on it.
"""
if not params:
return url
if isinstance(params, bytes):
extra = params.decode("utf-8")
elif isinstance(params, str):
extra = params
else:
extra = urlencode(params, doseq=True)
parsed = urlparse(url)
separator = "&" if parsed.query else ""
query = parsed.query + separator + extra if parsed.query else extra
return urlunparse(parsed._replace(query=query))
def get_sleep_time(self, response: Optional[RnetResponse], attempt: int) -> Optional[float]:
if response:
retry_after = response.headers.get("Retry-After")
if retry_after:
@@ -108,19 +561,57 @@ class CurlSession(Session):
sleep_time = backoff_value + random.uniform(-jitter, jitter)
return min(sleep_time, self.max_backoff)
def request(self, method: str, url: str, **kwargs: Any) -> Response:
if method.upper() not in self.allowed_methods:
return super().request(method, url, **kwargs)
def request(self, method: str, url: str, **kwargs: Any) -> RnetResponse:
client = self._ensure_client()
method_upper = method.upper() if isinstance(method, str) else str(method).upper()
last_exception = None
response = None
# Build URL with params
url = self._build_url(url, kwargs.pop("params", None))
# Default allow_redirects=True
kwargs.setdefault("allow_redirects", True)
# Pass verify setting
if not self.verify:
kwargs.setdefault("verify", False)
# Remove kwargs rnet doesn't understand
kwargs.pop("stream", None) # rnet responses are always lazy
# Translate requests-compatible 'data' kwarg to rnet equivalents
data = kwargs.pop("data", None)
if data is not None:
if isinstance(data, dict):
kwargs["form"] = list(data.items())
elif isinstance(data, (str, bytes)):
kwargs["body"] = data
else:
kwargs["body"] = data
# Resolve method enum
rnet_method = _METHOD_MAP.get(method_upper)
if rnet_method is None:
raise ValueError(f"Unsupported HTTP method: {method}")
# Convert headers to standard dict once to resolve PyO3 CaseInsensitiveDict rejection.
if kwargs.get("headers") is not None:
kwargs["headers"] = dict(kwargs["headers"])
# Skip retry for non-allowed methods
if method_upper not in self.allowed_methods:
raw_resp = client.request(rnet_method, url, **kwargs)
return RnetResponse(raw_resp)
last_exception: Optional[Exception] = None
response: Optional[RnetResponse] = None
for attempt in range(self.max_retries + 1):
try:
response = super().request(method, url, **kwargs)
raw_resp = client.request(rnet_method, url, **kwargs)
response = RnetResponse(raw_resp)
if response.status_code not in self.status_forcelist:
return response
last_exception = exceptions.HTTPError(f"Received status code: {response.status_code}")
last_exception = HTTPError(f"Received status code: {response.status_code}")
self.log.warning(
f"{response.status_code} {response.reason}({urlparse(url).path}). Retrying... "
f"({attempt + 1}/{self.max_retries})"
@@ -142,120 +633,100 @@ class CurlSession(Session):
raise MaxRetriesError(f"Max retries exceeded for {method} {url}", cause=last_exception)
def get(self, url: str, **kwargs: Any) -> RnetResponse:
return self.request("GET", url, **kwargs)
def post(self, url: str, **kwargs: Any) -> RnetResponse:
return self.request("POST", url, **kwargs)
def put(self, url: str, **kwargs: Any) -> RnetResponse:
return self.request("PUT", url, **kwargs)
def delete(self, url: str, **kwargs: Any) -> RnetResponse:
return self.request("DELETE", url, **kwargs)
def head(self, url: str, **kwargs: Any) -> RnetResponse:
return self.request("HEAD", url, **kwargs)
def options(self, url: str, **kwargs: Any) -> RnetResponse:
return self.request("OPTIONS", url, **kwargs)
def patch(self, url: str, **kwargs: Any) -> RnetResponse:
return self.request("PATCH", url, **kwargs)
def prepare_request(self, req: Request) -> Request:
"""Compatibility shim for services using prepared requests."""
# Merge session headers into request headers
if req.headers:
merged = dict(self.headers)
merged.update(req.headers)
req.headers = merged
else:
req.headers = dict(self.headers)
return req
def send(self, req: Request, **kwargs: Any) -> RnetResponse:
"""Compatibility shim for services using prepared requests."""
method = req.method or "GET"
url = req.url or ""
send_kwargs: dict[str, Any] = {}
if req.headers:
send_kwargs["headers"] = dict(req.headers)
if req.body:
send_kwargs["data"] = req.body
if req.json:
send_kwargs["json"] = req.json
send_kwargs.update(kwargs)
return self.request(method, url, **send_kwargs)
def mount(self, prefix: str, adapter: Any) -> None:
"""No-op — rnet handles TLS and connection pooling natively."""
pass
def close(self) -> None:
"""No-op — rnet manages its own resources."""
pass
# ---------------------------------------------------------------------------
# session() factory
# ---------------------------------------------------------------------------
def session(
browser: str | None = None,
ja3: str | None = None,
akamai: str | None = None,
extra_fp: dict | None = None,
**kwargs,
) -> CurlSession:
browser: Optional[str] = None,
**kwargs: Any,
) -> RnetSession:
"""
Create a curl_cffi session that impersonates a browser or custom TLS/HTTP fingerprint.
This is a full replacement for requests.Session with browser impersonation
and anti-bot capabilities. The session uses curl-impersonate under the hood
to mimic real browser behavior.
Create an rnet session with TLS fingerprinting (browser/app impersonation).
Args:
browser: Browser to impersonate (e.g. "chrome124", "firefox", "safari") OR
fingerprint preset name (e.g. "okhttp4").
Uses the configured default from curl_impersonate.browser if not specified.
Available presets: okhttp4, okhttp5
See https://github.com/lexiforest/curl_cffi#sessions for browser options.
ja3: Custom JA3 TLS fingerprint string (format: "SSLVersion,Ciphers,Extensions,Curves,PointFormats").
When provided, curl_cffi will use this exact TLS fingerprint instead of the browser's default.
See https://curl-cffi.readthedocs.io/en/latest/impersonate/customize.html
akamai: Custom Akamai HTTP/2 fingerprint string (format: "SETTINGS|WINDOW_UPDATE|PRIORITY|PSEUDO_HEADERS").
When provided, curl_cffi will use this exact HTTP/2 fingerprint instead of the browser's default.
See https://curl-cffi.readthedocs.io/en/latest/impersonate/customize.html
extra_fp: Additional fingerprint parameters dict for advanced customization.
See https://curl-cffi.readthedocs.io/en/latest/impersonate/customize.html
**kwargs: Additional arguments passed to CurlSession constructor:
- headers: Additional headers (dict)
- cookies: Cookie jar or dict
- auth: HTTP basic auth tuple (username, password)
- proxies: Proxy configuration dict
- verify: SSL certificate verification (bool, default True)
- timeout: Request timeout in seconds (float or tuple)
- allow_redirects: Follow redirects (bool, default True)
- max_redirects: Maximum redirect count (int)
- cert: Client certificate (str or tuple)
Extra arguments for retry handler:
- max_retries: Maximum number of retries (int, default 5)
- backoff_factor: Backoff factor (float, default 0.2)
- max_backoff: Maximum backoff time (float, default 60.0)
- status_forcelist: List of status codes to force retry (list, default [429, 500, 502, 503, 504])
- allowed_methods: List of allowed HTTP methods (set, default {"GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE"})
- catch_exceptions: List of exceptions to catch (tuple, default (exceptions.ConnectionError, exceptions.ProxyError, exceptions.SSLError, exceptions.Timeout))
browser: Exact rnet.Impersonate preset name. Examples:
"Chrome131", "OkHttp4_12", "Edge101", "Firefox135",
"Safari18", "OkHttp5", "Opera118"
Uses the configured default from config if not specified.
See rnet.Impersonate for all available presets.
**kwargs: Additional arguments passed to RnetSession constructor.
Returns:
curl_cffi.requests.Session configured with browser impersonation or custom fingerprints,
common headers, and equivalent retry behavior to requests.Session.
RnetSession configured with browser impersonation and retry behavior.
Examples:
# Standard browser impersonation
from envied.core.session import session
class MyService(Service):
@staticmethod
def get_session():
return session() # Uses config default browser
# Use OkHttp 4.x preset for Android TV
class AndroidService(Service):
@staticmethod
def get_session():
return session("okhttp4")
# Custom fingerprint (manual)
class CustomService(Service):
@staticmethod
def get_session():
return session(
ja3="771,4865-4866-4867-49195...",
akamai="1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p",
)
# With retry configuration
class MyService(Service):
@staticmethod
def get_session():
return session(
"okhttp4",
max_retries=5,
status_forcelist=[429, 500],
allowed_methods={"GET", "HEAD", "OPTIONS"},
)
session() # Default browser from config
session("OkHttp4_12") # OkHttp 4.12 fingerprint
session("Chrome131") # Chrome 131
session("Edge101", max_retries=3) # Edge 101 with custom retry
"""
if browser is None:
browser = config.curl_impersonate.get("browser", "Chrome131")
if browser and browser in FINGERPRINT_PRESETS:
preset = FINGERPRINT_PRESETS[browser]
if ja3 is None:
ja3 = preset.get("ja3")
if akamai is None:
akamai = preset.get("akamai")
if extra_fp is None:
extra_fp = preset.get("extra_fp")
browser = None
impersonate = _resolve_impersonate(browser)
if browser is None and ja3 is None and akamai is None:
browser = config.curl_impersonate.get("browser", "chrome")
session_kwargs: dict[str, Any] = {"impersonate": impersonate}
session_kwargs.update(kwargs)
session_config = {}
if browser:
session_config["impersonate"] = browser
if ja3:
session_config["ja3"] = ja3
if akamai:
session_config["akamai"] = akamai
if extra_fp:
session_config["extra_fp"] = extra_fp
session_config.update(kwargs)
session_obj = CurlSession(**session_config)
session_obj = RnetSession(**session_kwargs)
session_obj.headers.update(config.headers)
return session_obj
@@ -8,4 +8,40 @@ Title_T = Union[Movie, Episode, Song]
Titles_T = Union[Movies, Series, Album]
__all__ = ("Episode", "Series", "Movie", "Movies", "Album", "Song", "Title_T", "Titles_T")
def remap_titles(titles: Titles_T, title_map: dict) -> Titles_T:
"""
Rewrite titles in-place using an exact-match ``title_map``.
Some services name a title differently from how the user wants it stored, which can
break library matching. ``title_map`` maps a source title string to the desired output
title. Episodes are matched on their ``title`` (the show name), Movies and Songs on
their ``name``. Returns the same collection for convenient chaining.
"""
if not title_map or not titles:
return titles
def remap_one(title: Title_T) -> None:
attr = "title" if isinstance(title, Episode) else "name"
current = getattr(title, attr, None)
if current and current in title_map:
setattr(title, attr, title_map[current])
if hasattr(titles, "__iter__"):
for title in titles:
remap_one(title)
else:
remap_one(titles)
return titles
__all__ = (
"Episode",
"Series",
"Movie",
"Movies",
"Album",
"Song",
"Title_T",
"Titles_T",
"remap_titles",
)
@@ -100,28 +100,39 @@ class Episode(Title):
def get_filename(self, media_info: MediaInfo, folder: bool = False, show_service: bool = True) -> str:
if folder:
series_template = config.output_template.get("series")
if series_template:
folder_template = series_template
folder_template = re.sub(r'\{episode\}', '', folder_template)
folder_template = re.sub(r'\{episode_name\?\}', '', folder_template)
folder_template = re.sub(r'\{episode_name\}', '', folder_template)
folder_template = re.sub(r'\{season_episode\}', '{season}', folder_template)
folder_template = re.sub(r'\.{2,}', '.', folder_template)
folder_template = re.sub(r'\s{2,}', ' ', folder_template)
folder_template = re.sub(r'^[\.\s]+|[\.\s]+$', '', folder_template)
formatter = TemplateFormatter(folder_template)
template = config.get_folder_template("series")
if template:
formatter = TemplateFormatter(template)
context = self._build_template_context(media_info, show_service)
context['season'] = f"S{self.season:02}"
context["season"] = f"S{self.season:02}"
folder_name = formatter.format(context)
if '.' in series_template and ' ' not in series_template:
return sanitize_filename(folder_name, ".")
else:
return sanitize_filename(folder_name, " ")
separators = re.sub(r"\{[^}]*\}", "", template)
spacer = "." if "." in separators and " " not in separators else " "
return sanitize_filename(folder_name, spacer)
series_template = config.output_template.get("series")
if series_template:
derived_template = series_template
derived_template = re.sub(r"\{episode\}", "", derived_template)
derived_template = re.sub(r"\{episode_name\?\}", "", derived_template)
derived_template = re.sub(r"\{episode_name\}", "", derived_template)
derived_template = re.sub(r"\{season_episode\}", "{season}", derived_template)
derived_template = re.sub(r"\.{2,}", ".", derived_template)
derived_template = re.sub(r"\s{2,}", " ", derived_template)
derived_template = re.sub(r"^[\.\s]+|[\.\s]+$", "", derived_template)
formatter = TemplateFormatter(derived_template)
context = self._build_template_context(media_info, show_service)
context["season"] = f"S{self.season:02}"
folder_name = formatter.format(context)
separators = re.sub(r"\{[^}]*\}", "", derived_template)
spacer = "." if "." in separators and " " not in separators else " "
return sanitize_filename(folder_name, spacer)
else:
name = f"{self.title}"
if self.year:
@@ -149,7 +160,7 @@ class Series(SortedKeyList, ABC):
sum(seasons.values())
season_breakdown = ", ".join(f"S{season}({count})" for season, count in sorted(seasons.items()))
tree = Tree(
f"{num_seasons} seasons, {season_breakdown}",
f"{num_seasons} season{'s'[:num_seasons^1]}, {season_breakdown}",
guide_style="bright_black",
)
if verbose:
@@ -1,3 +1,4 @@
import re
from abc import ABC
from typing import Any, Iterable, Optional, Union
@@ -59,6 +60,15 @@ class Movie(Title):
def get_filename(self, media_info: MediaInfo, folder: bool = False, show_service: bool = True) -> str:
if folder:
template = config.get_folder_template("movies")
if template:
formatter = TemplateFormatter(template)
context = self._build_template_context(media_info, show_service)
folder_name = formatter.format(context)
separators = re.sub(r"\{[^}]*\}", "", template)
spacer = "." if "." in separators and " " not in separators else " "
return sanitize_filename(folder_name, spacer)
name = f"{self.name}"
if self.year:
name += f" ({self.year})"
@@ -1,3 +1,4 @@
import re
from abc import ABC
from typing import Any, Iterable, Optional, Union
@@ -94,6 +95,15 @@ class Song(Title):
def get_filename(self, media_info: MediaInfo, folder: bool = False, show_service: bool = True) -> str:
if folder:
template = config.get_folder_template("songs")
if template:
formatter = TemplateFormatter(template)
context = self._build_template_context(media_info, show_service)
folder_name = formatter.format(context)
separators = re.sub(r"\{[^}]*\}", "", template)
spacer = "." if "." in separators and " " not in separators else " "
return sanitize_filename(folder_name, spacer)
name = f"{self.artist} - {self.album}"
if self.year:
name += f" ({self.year})"
@@ -61,7 +61,21 @@ class Title:
returned dict with their specific fields (e.g., season/episode).
"""
primary_video_track = next(iter(media_info.video_tracks), None)
primary_audio_track = next(iter(media_info.audio_tracks), None)
original_lang_tag = (
str(self.language).split("-")[0].lower() if self.language else ""
)
primary_audio_track = None
if original_lang_tag:
primary_audio_track = next(
(
t
for t in media_info.audio_tracks
if t.language and t.language.split("-")[0].lower() == original_lang_tag
),
None,
)
if primary_audio_track is None:
primary_audio_track = next(iter(media_info.audio_tracks), None)
unique_audio_languages = len({x.language.split("-")[0] for x in media_info.audio_tracks if x.language})
context: dict[str, Any] = {
@@ -99,7 +113,20 @@ class Title:
aspect_ratio.append(1)
ratio = aspect_ratio[0] / aspect_ratio[1]
if ratio not in (16 / 9, 4 / 3, 9 / 16, 3 / 4):
if abs(width - 3840) <= 50:
width = 3840
elif abs(width - 2560) <= 50:
width = 2560
elif abs(width - 1920) <= 50 or abs(width - 1620) <= 50:
width = 1920
elif abs(width - 1280) <= 50 or abs(width - 1080) <= 50:
width = 1280
resolution = int(max(width, primary_video_track.height) * (9 / 16))
track_height = primary_video_track.height
if abs(resolution - track_height) <= 10 or track_height in (2160, 1440, 1080, 720, 480):
resolution = track_height
except Exception:
pass
@@ -143,14 +170,16 @@ class Title:
channel_count = primary_audio_track.channel_s or primary_audio_track.channels or 0
channels = float(channel_count)
features = primary_audio_track.format_additionalfeatures or ""
has_atmos = any(
"JOC" in (t.format_additionalfeatures or "") or t.joc for t in media_info.audio_tracks
)
context.update(
{
"audio": AUDIO_CODEC_MAP.get(codec, codec),
"audio_channels": f"{channels:.1f}",
"audio_full": f"{AUDIO_CODEC_MAP.get(codec, codec)}{channels:.1f}",
"atmos": "Atmos" if ("JOC" in features or primary_audio_track.joc) else "",
"atmos": "Atmos" if has_atmos else "",
}
)
@@ -86,9 +86,7 @@ class Attachment:
raise ValueError(f"Failed to download attachment from URL: {e}")
if path is not None and not isinstance(path, (str, Path)):
raise ValueError(
f"Invalid attachment path type: expected str or Path, got {type(path).__name__}."
)
raise ValueError(f"Invalid attachment path type: expected str or Path, got {type(path).__name__}.")
if path is not None:
path = Path(path)
@@ -127,6 +125,10 @@ class Attachment:
def __str__(self) -> str:
return " | ".join(filter(bool, ["ATT", self.name, self.mime_type, self.description]))
def to_dict(self) -> dict[str, Optional[str]]:
"""Serialise a URL-backed attachment for export/import."""
return {"url": self.url, "name": self.name, "mime_type": self.mime_type, "description": self.description}
@property
def id(self) -> str:
"""Compute an ID from the attachment data."""
@@ -57,7 +57,7 @@ class Audio(Track):
@staticmethod
def from_netflix_profile(profile: str) -> Audio.Codec:
profile = profile.lower().strip()
if profile.startswith("heaac"):
if profile.startswith("heaac") or profile.startswith("xheaac"):
return Audio.Codec.AAC
if profile.startswith("dd-"):
return Audio.Codec.AC3
@@ -126,6 +126,31 @@ class Audio(Track):
self.joc = joc
self.descriptive = bool(descriptive)
def to_dict(self) -> dict[str, Any]:
data = super().to_dict()
data.update(
{
"codec": self.codec.name if self.codec else None,
"bitrate": self.bitrate,
"channels": self.channels,
"joc": self.joc,
"descriptive": self.descriptive,
}
)
return data
@classmethod
def from_dict(cls, data: dict[str, Any]) -> Audio:
kwargs = Track.base_kwargs_from_dict(data)
return cls(
**kwargs,
codec=Audio.Codec[data["codec"]] if data.get("codec") else None,
bitrate=data.get("bitrate"),
channels=data.get("channels"),
joc=data.get("joc"),
descriptive=data.get("descriptive", False),
)
@property
def atmos(self) -> bool:
"""Return True if the audio track contains Dolby Atmos."""
@@ -0,0 +1,117 @@
"""
DV fixup for HLS composite HEVC streams.
Some services deliver DV Profile 8.1 in a stream whose primary CODECS is plain
hvc1, with DV advertised only via SUPPLEMENTAL-CODECS. The fMP4 carries DV RPU NALs but
the container does not signal DV, so muxing produces an MKV that mediainfo and DV-capable
TVs see as plain HDR10/HDR10+.
A dovi_tool extract-rpu / inject-rpu round-trip rewrites the bitstream so it is recognised
as DV after muxing. HDR10+ SEI NALs and HDR10 base layer signaling survive untouched.
"""
from __future__ import annotations
import logging
from pathlib import Path
from typing import TYPE_CHECKING
from rich.padding import Padding
from rich.rule import Rule
from envied.core.binaries import FFMPEG, DoviTool
from envied.core.config import config
from envied.core.console import console
from envied.core.utilities import get_debug_logger
from envied.core.utils import dovi
from envied.core.utils.subprocess import run_step
if TYPE_CHECKING:
from envied.core.tracks import Video
class DVFixup:
"""Round-trip a DV-composite HEVC track through dovi_tool to restore DV signaling."""
def __init__(self, video: "Video") -> None:
self.log = logging.getLogger("dv-fixup")
self.debug_logger = get_debug_logger()
self.video = video
if not DoviTool:
raise EnvironmentError("dovi_tool is required for DV-composite fixup but was not found.")
if not FFMPEG:
raise EnvironmentError("ffmpeg is required for DV-composite fixup but was not found.")
if not video.path or not Path(video.path).exists():
raise ValueError(f"Video track {video.id} was not downloaded before DV fixup.")
def run(self) -> Path:
"""Execute the fixup. Returns the DV-signaled HEVC path, or the original
source path on any failure so muxing can proceed with the as-downloaded file."""
source = Path(self.video.path)
height = self.video.height or 0
console.print(Padding(Rule(f"[rule.text]DV Composite Fixup ({height}p)"), (1, 2)))
fixed_hevc = source.with_name(f"{self.video.id}.dv.hevc")
if fixed_hevc.exists() and fixed_hevc.stat().st_size > 0:
self.log.info("✓ DV signaling already restored (reusing existing fixup)")
return fixed_hevc
tmp = config.directories.temp
tmp.mkdir(parents=True, exist_ok=True)
suffix = f"{self.video.id}_{height or 'na'}"
raw_hevc = tmp / f"dvfix_{suffix}.hevc"
rpu = tmp / f"dvfix_{suffix}_rpu.bin"
try:
run_step(
[FFMPEG, "-nostdin", "-y", "-i", source, "-c:v", "copy", "-f", "hevc", raw_hevc],
status="Demuxing HEVC bitstream...",
output=raw_hevc,
label="ffmpeg demux",
)
dovi.extract_rpu_with_fallback(raw_hevc, rpu)
dovi.inject_rpu(raw_hevc, rpu, fixed_hevc, status="Re-injecting DV RPU with proper signaling...")
except Exception as e:
self.log.warning(f"DV fixup failed ({e}); muxing source as-is.")
if self.debug_logger:
self.debug_logger.log(
level="WARNING",
operation="dv_fixup",
message="DV fixup failed; falling back to source",
context={"error": str(e), "source": str(source)},
)
for leftover in (raw_hevc, rpu, fixed_hevc):
leftover.unlink(missing_ok=True)
return source
for leftover in (raw_hevc, rpu):
leftover.unlink(missing_ok=True)
self.log.info("✓ DV signaling restored")
if self.debug_logger:
self.debug_logger.log(
level="INFO",
operation="dv_fixup",
message="DV fixup complete",
context={"source": str(source), "output": str(fixed_hevc)},
success=True,
)
return fixed_hevc
def apply_dv_fixup(video: "Video") -> None:
"""Run DV fixup on `video` if flagged as DV-composite. Updates `video.path` in place
and deletes the original source file so the standard mux cleanup handles the new path."""
if not getattr(video, "dv_compatible_bitstream", False):
return
if not video.path or not Path(video.path).exists():
return
original = Path(video.path)
fixed = DVFixup(video).run()
if fixed != original:
video.path = fixed
original.unlink(missing_ok=True)
__all__ = ("DVFixup", "apply_dv_fixup")
+199 -197
View File
@@ -6,14 +6,17 @@ import re
import subprocess
import sys
from pathlib import Path
from typing import Optional
from rich.padding import Padding
from rich.rule import Rule
from envied.core.binaries import FFMPEG, DoviTool, FFProbe, HDR10PlusTool
from envied.core.binaries import FFMPEG, FFProbe, HDR10PlusTool
from envied.core.config import config
from envied.core.console import console
from envied.core.utilities import get_debug_logger
from envied.core.utils import dovi
from envied.core.utils.subprocess import run_step
class Hybrid:
@@ -113,9 +116,7 @@ class Hybrid:
# Edit L6 with actual luminance values from RPU, then L5 active area
self.level_6()
base_video = next(
(v for v in videos if v.range in (Video.Range.HDR10, Video.Range.HDR10P)), None
)
base_video = next((v for v in videos if v.range in (Video.Range.HDR10, Video.Range.HDR10P)), None)
if base_video and base_video.path:
self.level_5(base_video.path)
@@ -140,51 +141,27 @@ class Hybrid:
Path.unlink(config.directories.temp / "dv.mkv")
Path.unlink(config.directories.temp / "HDR10.hevc", missing_ok=True)
Path.unlink(config.directories.temp / "DV.hevc", missing_ok=True)
Path.unlink(config.directories.temp / f"{self.rpu_file}", missing_ok=True)
Path.unlink(config.directories.temp / "RPU_L6.bin", missing_ok=True)
Path.unlink(config.directories.temp / "RPU_L5.bin", missing_ok=True)
for rpu_name in ("RPU.bin", "RPU_UNT.bin", "RPU_L5.bin", "RPU_L6.bin"):
Path.unlink(config.directories.temp / rpu_name, missing_ok=True)
Path.unlink(config.directories.temp / "L5.json", missing_ok=True)
Path.unlink(config.directories.temp / "L6.json", missing_ok=True)
def ffmpeg_simple(self, save_path, output):
"""Simple ffmpeg execution without progress tracking"""
p = subprocess.run(
[
str(FFMPEG) if FFMPEG else "ffmpeg",
"-nostdin",
"-i",
str(save_path),
"-c:v",
"copy",
str(output),
"-y", # overwrite output
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
return p
def extract_stream(self, save_path, type_):
output = Path(config.directories.temp / f"{type_}.hevc")
with console.status(f"Extracting {type_} stream...", spinner="dots"):
result = self.ffmpeg_simple(save_path, output)
if result.returncode:
output.unlink(missing_ok=True)
try:
run_step(
[FFMPEG or "ffmpeg", "-nostdin", "-y", "-i", save_path, "-c:v", "copy", output],
status=f"Extracting {type_} stream...",
output=output,
label=f"ffmpeg extract {type_}",
)
except RuntimeError as e:
if self.debug_logger:
self.debug_logger.log(
level="ERROR",
operation="hybrid_extract_stream",
message=f"Failed extracting {type_} stream",
context={
"type": type_,
"input": str(save_path),
"output": str(output),
"returncode": result.returncode,
"stderr": (result.stderr or b"").decode(errors="replace"),
"stdout": (result.stdout or b"").decode(errors="replace"),
},
context={"type": type_, "input": str(save_path), "output": str(output), "error": str(e)},
)
self.log.error(f"x Failed extracting {type_} stream")
sys.exit(1)
@@ -204,48 +181,30 @@ class Hybrid:
):
return
with console.status(
f"Extracting{' untouched ' if untouched else ' '}RPU from Dolby Vision stream...", spinner="dots"
):
extraction_args = [str(DoviTool)]
if not untouched:
extraction_args += ["-m", "3"]
extraction_args += [
"extract-rpu",
config.directories.temp / "DV.hevc",
"-o",
config.directories.temp / f"{'RPU' if not untouched else 'RPU_UNT'}.bin",
]
rpu_name = "RPU_UNT" if untouched else "RPU"
rpu_path = config.directories.temp / f"{rpu_name}.bin"
dv_stream = config.directories.temp / "DV.hevc"
spinner = f"Extracting{' untouched ' if untouched else ' '}RPU from Dolby Vision stream..."
rpu_extraction = subprocess.run(
extraction_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
rpu_name = "RPU" if not untouched else "RPU_UNT"
if rpu_extraction.returncode:
Path.unlink(config.directories.temp / f"{rpu_name}.bin")
stderr_text = rpu_extraction.stderr.decode(errors="replace") if rpu_extraction.stderr else ""
try:
dovi.extract_rpu(dv_stream, rpu_path, mode=None if untouched else 3, status=spinner)
except RuntimeError as e:
stderr_text = str(e)
if self.debug_logger:
self.debug_logger.log(
level="ERROR",
operation="hybrid_extract_rpu",
message=f"Failed extracting{' untouched ' if untouched else ' '}RPU",
context={
"untouched": untouched,
"returncode": rpu_extraction.returncode,
"stderr": stderr_text,
"args": [str(a) for a in extraction_args],
},
context={"untouched": untouched, "error": stderr_text},
)
if b"MAX_PQ_LUMINANCE" in rpu_extraction.stderr:
if "MAX_PQ_LUMINANCE" in stderr_text:
self.extract_rpu(video, untouched=True)
elif b"Invalid PPS index" in rpu_extraction.stderr:
return
if "Invalid PPS index" in stderr_text:
raise ValueError("Dolby Vision VideoTrack seems to be corrupt")
else:
raise ValueError(f"Failed extracting{' untouched ' if untouched else ' '}RPU from Dolby Vision stream")
elif self.debug_logger:
raise ValueError(f"Failed extracting{' untouched ' if untouched else ' '}RPU from Dolby Vision stream")
if self.debug_logger:
self.debug_logger.log(
level="DEBUG",
operation="hybrid_extract_rpu",
@@ -383,11 +342,7 @@ class Hybrid:
for crop in crop_results:
crop_counts[crop] = crop_counts.get(crop, 0) + 1
most_common = max(crop_counts, key=crop_counts.get)
left, top, right, bottom = most_common
# If all borders are 0 there's nothing to correct
if left == 0 and top == 0 and right == 0 and bottom == 0:
return
left, top, right, bottom = most_common # frame instead of leaving phantom bars from the source.
l5_json = {
"active_area": {
@@ -401,31 +356,22 @@ class Hybrid:
with open(l5_path, "w") as f:
json.dump(l5_json, f, indent=4)
with console.status("Editing RPU Level 5 active area...", spinner="dots"):
result = subprocess.run(
[
str(DoviTool),
"editor",
"-i",
str(config.directories.temp / self.rpu_file),
"-j",
str(l5_path),
"-o",
str(config.directories.temp / "RPU_L5.bin"),
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
try:
dovi.editor(
config.directories.temp / self.rpu_file,
l5_path,
config.directories.temp / "RPU_L5.bin",
status="Editing RPU Level 5 active area...",
label="dovi_tool editor (L5)",
)
if result.returncode:
except RuntimeError as e:
if self.debug_logger:
self.debug_logger.log(
level="ERROR",
operation="hybrid_level5",
message="Failed editing RPU Level 5 values",
context={"returncode": result.returncode, "stderr": (result.stderr or b"").decode(errors="replace")},
context={"error": str(e)},
)
Path.unlink(config.directories.temp / "RPU_L5.bin", missing_ok=True)
raise ValueError("Failed editing RPU Level 5 values")
if self.debug_logger:
@@ -433,30 +379,45 @@ class Hybrid:
level="DEBUG",
operation="hybrid_level5",
message="Edited RPU Level 5 active area",
context={"crop": {"left": left, "right": right, "top": top, "bottom": bottom}, "samples": len(crop_results)},
context={
"crop": {"left": left, "right": right, "top": top, "bottom": bottom},
"samples": len(crop_results),
},
success=True,
)
self.rpu_file = "RPU_L5.bin"
@staticmethod
def sanitize_l6(
max_mdl: Optional[int], min_mdl: Optional[int], max_cll: Optional[int], max_fall: Optional[int]
) -> tuple[Optional[int], Optional[int], Optional[int], Optional[int]]:
"""Clamp static L6 values to a valid relationship.
MaxCLL must not exceed the mastering-display peak (some sources, e.g. ATV
HDR10+, ship MaxCLL 10000 on a 1000-nit master), and MaxFALL must not exceed
MaxCLL. A value of 0 means "unknown" and is preserved as-is.
"""
if max_mdl and max_cll and max_cll > max_mdl:
max_cll = max_mdl
if max_cll and max_fall and max_fall > max_cll:
max_fall = max_cll
return max_mdl, min_mdl, max_cll, max_fall
def level_6(self):
"""Edit RPU Level 6 values using actual luminance data from the RPU."""
"""Edit RPU Level 6 values using the static L6 luminance data from the RPU."""
if os.path.isfile(config.directories.temp / "RPU_L6.bin"):
return
with console.status("Reading RPU luminance metadata...", spinner="dots"):
result = subprocess.run(
[str(DoviTool), "info", "-i", str(config.directories.temp / self.rpu_file), "-s"],
capture_output=True,
text=True,
)
if result.returncode != 0:
try:
with console.status("Reading RPU luminance metadata...", spinner="dots"):
info_text = dovi.info_summary(config.directories.temp / self.rpu_file)
except RuntimeError as e:
if self.debug_logger:
self.debug_logger.log(
level="ERROR",
operation="hybrid_level6",
message="Failed reading RPU metadata for Level 6 values",
context={"returncode": result.returncode, "stderr": (result.stderr or "")},
context={"error": str(e)},
)
raise ValueError("Failed reading RPU metadata for Level 6 values")
@@ -465,17 +426,33 @@ class Hybrid:
max_mdl = None
min_mdl = None
for line in result.stdout.splitlines():
if "RPU content light level (L1):" in line:
parts = line.split("MaxCLL:")[1].split(",")
max_cll = int(float(parts[0].strip().split()[0]))
if len(parts) > 1 and "MaxFALL:" in parts[1]:
max_fall = int(float(parts[1].split("MaxFALL:")[1].strip().split()[0]))
elif "RPU mastering display:" in line:
mastering = line.split(":", 1)[1].strip()
in_l6 = False
for line in info_text.splitlines():
stripped = line.strip()
if "L6 metadata" in stripped:
in_l6 = True
if stripped.startswith("RPU mastering display:"):
mastering = stripped.split(":", 1)[1].strip()
min_lum, max_lum = mastering.split("/")[0], mastering.split("/")[1].split(" ")[0]
min_mdl = int(float(min_lum) * 10000)
max_mdl = int(float(max_lum))
elif in_l6 and "MaxCLL:" in stripped and max_cll is None:
max_cll = int(float(stripped.split("MaxCLL:")[1].split("nits")[0].strip().rstrip(",")))
if "MaxFALL:" in stripped:
max_fall = int(float(stripped.split("MaxFALL:")[1].split("nits")[0].strip().rstrip(",")))
if any(v is None for v in (max_cll, max_fall, max_mdl, min_mdl)):
base_max_mdl, base_min_mdl, base_cll, base_fall = self._probe_hdr_metadata()
if max_cll is None:
max_cll = base_cll
if max_fall is None:
max_fall = base_fall
if max_mdl is None:
max_mdl = base_max_mdl
if min_mdl is None:
min_mdl = base_min_mdl
max_mdl, min_mdl, max_cll, max_fall = self.sanitize_l6(max_mdl, min_mdl, max_cll, max_fall)
if any(v is None for v in (max_cll, max_fall, max_mdl, min_mdl)):
if self.debug_logger:
@@ -502,31 +479,22 @@ class Hybrid:
with open(l6_path, "w") as f:
json.dump(level6_data, f, indent=4)
with console.status("Editing RPU Level 6 values...", spinner="dots"):
result = subprocess.run(
[
str(DoviTool),
"editor",
"-i",
str(config.directories.temp / self.rpu_file),
"-j",
str(l6_path),
"-o",
str(config.directories.temp / "RPU_L6.bin"),
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
try:
dovi.editor(
config.directories.temp / self.rpu_file,
l6_path,
config.directories.temp / "RPU_L6.bin",
status="Editing RPU Level 6 values...",
label="dovi_tool editor (L6)",
)
if result.returncode:
except RuntimeError as e:
if self.debug_logger:
self.debug_logger.log(
level="ERROR",
operation="hybrid_level6",
message="Failed editing RPU Level 6 values",
context={"returncode": result.returncode, "stderr": (result.stderr or b"").decode(errors="replace")},
context={"error": str(e)},
)
Path.unlink(config.directories.temp / "RPU_L6.bin", missing_ok=True)
raise ValueError("Failed editing RPU Level 6 values")
if self.debug_logger:
@@ -548,38 +516,22 @@ class Hybrid:
if os.path.isfile(config.directories.temp / self.hevc_file):
return
with console.status(f"Injecting Dolby Vision metadata into {self.hdr_type} stream...", spinner="dots"):
inject_cmd = [
str(DoviTool),
"inject-rpu",
"-i",
try:
dovi.inject_rpu(
config.directories.temp / "HDR10.hevc",
"--rpu-in",
config.directories.temp / self.rpu_file,
]
inject_cmd.extend(["-o", config.directories.temp / self.hevc_file])
inject = subprocess.run(
inject_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
config.directories.temp / self.hevc_file,
status=f"Injecting Dolby Vision metadata into {self.hdr_type} stream...",
label="dovi_tool inject-rpu",
)
if inject.returncode:
except RuntimeError as e:
if self.debug_logger:
self.debug_logger.log(
level="ERROR",
operation="hybrid_inject_rpu",
message="Failed injecting Dolby Vision metadata into HDR10 stream",
context={
"returncode": inject.returncode,
"stderr": (inject.stderr or b"").decode(errors="replace"),
"stdout": (inject.stdout or b"").decode(errors="replace"),
"cmd": [str(a) for a in inject_cmd],
},
context={"error": str(e)},
)
Path.unlink(config.directories.temp / self.hevc_file)
raise ValueError("Failed injecting Dolby Vision metadata into HDR10 stream")
if self.debug_logger:
@@ -587,7 +539,12 @@ class Hybrid:
level="DEBUG",
operation="hybrid_inject_rpu",
message=f"Injected Dolby Vision metadata into {self.hdr_type} stream",
context={"hdr_type": self.hdr_type, "rpu_file": self.rpu_file, "output": self.hevc_file, "drop_hdr10plus": self.hdr10plus_to_dv},
context={
"hdr_type": self.hdr_type,
"rpu_file": self.rpu_file,
"output": self.hevc_file,
"drop_hdr10plus": self.hdr10plus_to_dv,
},
success=True,
)
@@ -599,35 +556,29 @@ class Hybrid:
if not HDR10PlusTool:
raise ValueError("HDR10Plus_tool not found. Please install it to use HDR10+ to DV conversion.")
with console.status("Extracting HDR10+ metadata...", spinner="dots"):
# HDR10Plus_tool needs raw HEVC stream
extraction = subprocess.run(
try:
run_step(
[
str(HDR10PlusTool),
HDR10PlusTool,
"extract",
str(config.directories.temp / "HDR10.hevc"),
config.directories.temp / "HDR10.hevc",
"-o",
str(config.directories.temp / self.hdr10plus_file),
config.directories.temp / self.hdr10plus_file,
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
status="Extracting HDR10+ metadata...",
output=config.directories.temp / self.hdr10plus_file,
label="hdr10plus_tool extract",
)
if extraction.returncode:
except RuntimeError as e:
if self.debug_logger:
self.debug_logger.log(
level="ERROR",
operation="hybrid_extract_hdr10plus",
message="Failed extracting HDR10+ metadata",
context={
"returncode": extraction.returncode,
"stderr": (extraction.stderr or b"").decode(errors="replace"),
"stdout": (extraction.stdout or b"").decode(errors="replace"),
},
context={"error": str(e)},
)
raise ValueError("Failed extracting HDR10+ metadata")
# Check if the extracted file has content
file_size = os.path.getsize(config.directories.temp / self.hdr10plus_file)
if file_size == 0:
if self.debug_logger:
@@ -648,54 +599,105 @@ class Hybrid:
success=True,
)
def _probe_hdr_metadata(self):
"""Extract mastering display and content light level metadata from the HDR10 stream via ffprobe.
Returns (max_mdl, min_mdl, max_cll, max_fall) in dovi_tool level6 units:
- max_mdl: nits (integer)
- min_mdl: 0.0001 nit units (integer)
- max_cll / max_fall: nits (integer)
"""
ffprobe_bin = str(FFProbe) if FFProbe else "ffprobe"
result = subprocess.run(
[
ffprobe_bin,
"-v",
"error",
"-select_streams",
"v:0",
"-show_entries",
"stream_side_data=max_luminance,min_luminance,max_content,max_average",
"-of",
"json",
str(config.directories.temp / "HDR10.hevc"),
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
max_mdl = 1000
min_mdl = 1
max_cll = 0
max_fall = 0
if result.returncode == 0 and result.stdout:
try:
probe = json.loads(result.stdout)
for stream in probe.get("streams", []):
for sd in stream.get("side_data_list", []):
if "max_luminance" in sd:
num, den = sd["max_luminance"].split("/")
max_mdl = int(int(num) / int(den))
if "min_luminance" in sd:
num, den = sd["min_luminance"].split("/")
min_mdl = int(int(num) / int(den) * 10000)
if "max_content" in sd:
max_cll = int(sd["max_content"])
if "max_average" in sd:
max_fall = int(sd["max_average"])
except (json.JSONDecodeError, KeyError, ValueError, ZeroDivisionError):
pass
if self.debug_logger:
self.debug_logger.log(
level="DEBUG",
operation="hybrid_probe_hdr_metadata",
message="Probed HDR metadata from source stream",
context={"max_mdl": max_mdl, "min_mdl": min_mdl, "max_cll": max_cll, "max_fall": max_fall},
)
return max_mdl, min_mdl, max_cll, max_fall
def convert_hdr10plus_to_dv(self):
"""Convert HDR10+ metadata to Dolby Vision RPU"""
if os.path.isfile(config.directories.temp / "RPU.bin"):
return
with console.status("Converting HDR10+ metadata to Dolby Vision...", spinner="dots"):
# Extract actual HDR metadata from the source stream
max_mdl, min_mdl, max_cll, max_fall = self._probe_hdr_metadata()
max_mdl, min_mdl, max_cll, max_fall = self.sanitize_l6(max_mdl, min_mdl, max_cll, max_fall)
# First create the extra metadata JSON for dovi_tool
extra_metadata = {
"cm_version": "V29",
"length": 0, # dovi_tool will figure this out
"level6": {
"max_display_mastering_luminance": 1000,
"min_display_mastering_luminance": 1,
"max_content_light_level": 0,
"max_frame_average_light_level": 0,
"max_display_mastering_luminance": max_mdl,
"min_display_mastering_luminance": min_mdl,
"max_content_light_level": max_cll,
"max_frame_average_light_level": max_fall,
},
}
with open(config.directories.temp / "extra.json", "w") as f:
json.dump(extra_metadata, f, indent=2)
# Generate DV RPU from HDR10+ metadata
conversion = subprocess.run(
[
str(DoviTool),
"generate",
"-j",
str(config.directories.temp / "extra.json"),
"--hdr10plus-json",
str(config.directories.temp / self.hdr10plus_file),
"-o",
str(config.directories.temp / "RPU.bin"),
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
try:
dovi.generate_from_hdr10plus(
config.directories.temp / "extra.json",
config.directories.temp / self.hdr10plus_file,
config.directories.temp / "RPU.bin",
label="dovi_tool generate",
)
if conversion.returncode:
except RuntimeError as e:
if self.debug_logger:
self.debug_logger.log(
level="ERROR",
operation="hybrid_convert_hdr10plus",
message="Failed converting HDR10+ to Dolby Vision",
context={
"returncode": conversion.returncode,
"stderr": (conversion.stderr or b"").decode(errors="replace"),
"stdout": (conversion.stdout or b"").decode(errors="replace"),
},
context={"error": str(e)},
)
raise ValueError("Failed converting HDR10+ to Dolby Vision")
@@ -195,6 +195,29 @@ class Subtitle(Track):
# Called after Track has been converted to another format
self.OnConverted: Optional[Callable[[Subtitle.Codec], None]] = None
def to_dict(self) -> dict[str, Any]:
data = super().to_dict()
data.update(
{
"codec": self.codec.name if self.codec else None,
"cc": self.cc,
"sdh": self.sdh,
"forced": self.forced,
}
)
return data
@classmethod
def from_dict(cls, data: dict[str, Any]) -> Subtitle:
kwargs = Track.base_kwargs_from_dict(data)
return cls(
**kwargs,
codec=Subtitle.Codec[data["codec"]] if data.get("codec") else None,
cc=data.get("cc", False),
sdh=data.get("sdh", False),
forced=data.get("forced", False),
)
def __str__(self) -> str:
return " | ".join(
filter(
@@ -221,8 +244,9 @@ class Subtitle(Track):
progress: Optional[partial] = None,
*,
cdm: Optional[object] = None,
no_proxy_download: bool = False,
):
super().download(session, prepare_drm, max_workers, progress, cdm=cdm)
super().download(session, prepare_drm, max_workers, progress, cdm=cdm, no_proxy_download=no_proxy_download)
if not self.path:
return
+98 -37
View File
@@ -13,7 +13,6 @@ from typing import Any, Callable, Iterable, Optional, Union
from uuid import UUID
from zlib import crc32
from curl_cffi.requests import Session as CurlSession
from langcodes import Language
from requests import Session
@@ -21,13 +20,33 @@ from envied.core import binaries
from envied.core.cdm.detect import is_playready_cdm, is_widevine_cdm
from envied.core.config import config
from envied.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY
from envied.core.downloaders import aria2c, curl_impersonate, n_m3u8dl_re, requests
from envied.core.downloaders import requests
from envied.core.drm import DRM_T, PlayReady, Widevine
from envied.core.events import events
from envied.core.session import RnetSession
from envied.core.utilities import get_boxes, try_ensure_utf8
from envied.core.utils.subprocess import ffprobe
def direct_session(session: Union[Session, "RnetSession"]) -> Session:
"""Vanilla requests.Session with copied headers/cookies, no proxy."""
new = Session()
headers = getattr(session, "headers", None)
if headers is not None:
try:
new.headers.update(dict(headers))
except Exception:
pass
cookies = getattr(session, "cookies", None)
if cookies is not None:
jar = getattr(cookies, "jar", None)
try:
new.cookies.update(jar if jar is not None else cookies)
except Exception:
pass
return new
class Track:
class Descriptor(Enum):
URL = 1 # Direct URL, nothing fancy
@@ -45,6 +64,7 @@ class Track:
name: Optional[str] = None,
drm: Optional[Iterable[DRM_T]] = None,
edition: Optional[str] = None,
session: Optional[Union[Session, "RnetSession"]] = None,
downloader: Optional[Callable] = None,
downloader_args: Optional[dict] = None,
from_file: Optional[Path] = None,
@@ -88,12 +108,7 @@ class Track:
raise TypeError(f"Expected drm to be an iterable, not {type(drm)}")
if downloader is None:
downloader = {
"aria2c": aria2c,
"curl_impersonate": curl_impersonate,
"requests": requests,
"n_m3u8dl_re": n_m3u8dl_re,
}[config.downloader]
downloader = requests
self.path: Optional[Path] = None
self.url = url
@@ -104,6 +119,7 @@ class Track:
self.name = name
self.drm = drm
self.edition: list[str] = [edition] if isinstance(edition, str) else (edition or [])
self.session = session
self.downloader = downloader
self.downloader_args = downloader_args
self.from_file = from_file
@@ -185,12 +201,13 @@ class Track:
def download(
self,
session: Session,
session: Union[Session, "RnetSession"],
prepare_drm: partial,
max_workers: Optional[int] = None,
progress: Optional[partial] = None,
*,
cdm: Optional[object] = None,
no_proxy_download: bool = False,
):
"""Download and optionally Decrypt this Track."""
from envied.core.manifests import DASH, HLS, ISM
@@ -206,28 +223,23 @@ class Track:
proxy = next(iter(session.proxies.values()), None)
dl_session = session
if no_proxy_download and proxy:
dl_session = direct_session(session)
proxy = None
track_type = self.__class__.__name__
save_path = config.directories.temp / f"{track_type}_{self.id}.mp4"
if track_type == "Subtitle":
save_path = save_path.with_suffix(f".{self.codec.extension}")
if self.downloader.__name__ == "n_m3u8dl_re" and (
self.descriptor == self.Descriptor.URL
or track_type in ("Subtitle", "Attachment")
):
self.downloader = requests
if self.descriptor != self.Descriptor.URL:
save_dir = save_path.with_name(save_path.name + "_segments")
else:
save_dir = save_path.parent
def cleanup():
# track file (e.g., "foo.mp4")
save_path.unlink(missing_ok=True)
# aria2c control file (e.g., "foo.mp4.aria2" or "foo.mp4.aria2__temp")
save_path.with_suffix(f"{save_path.suffix}.aria2").unlink(missing_ok=True)
save_path.with_suffix(f"{save_path.suffix}.aria2__temp").unlink(missing_ok=True)
if save_dir.exists() and save_dir.name.endswith("_segments"):
shutil.rmtree(save_dir)
@@ -250,7 +262,7 @@ class Track:
save_path=save_path,
save_dir=save_dir,
progress=progress,
session=session,
session=dl_session,
proxy=proxy,
max_workers=max_workers,
license_widevine=prepare_drm,
@@ -262,7 +274,7 @@ class Track:
save_path=save_path,
save_dir=save_dir,
progress=progress,
session=session,
session=dl_session,
proxy=proxy,
max_workers=max_workers,
license_widevine=prepare_drm,
@@ -274,7 +286,7 @@ class Track:
save_path=save_path,
save_dir=save_dir,
progress=progress,
session=session,
session=dl_session,
proxy=proxy,
max_workers=max_workers,
license_widevine=prepare_drm,
@@ -328,27 +340,24 @@ class Track:
if DOWNLOAD_LICENCE_ONLY.is_set():
progress(downloaded="[yellow]SKIPPED")
elif track_type != "Subtitle" and self.downloader.__name__ == "n_m3u8dl_re":
progress(downloaded="[red]FAILED")
error = f"[N_m3u8DL-RE]: {self.descriptor} is currently not supported"
raise ValueError(error)
else:
for status_update in self.downloader(
urls=self.url,
output_dir=save_path.parent,
filename=save_path.name,
headers=session.headers,
cookies=session.cookies,
headers=dl_session.headers,
cookies=dl_session.cookies,
proxy=proxy,
max_workers=max_workers,
session=dl_session,
):
file_downloaded = status_update.get("file_downloaded")
if not file_downloaded:
downloaded = status_update.get("downloaded")
if downloaded and downloaded.endswith("/s"):
status_update["downloaded"] = f"URL {downloaded}"
progress(**status_update)
# see https://github.com/devine-dl/devine/issues/71
save_path.with_suffix(f"{save_path.suffix}.aria2__temp").unlink(missing_ok=True)
self.path = save_path
events.emit(events.Types.TRACK_DOWNLOADED, track=self)
@@ -430,6 +439,57 @@ class Track:
self.path = target
return target
def to_dict(self) -> dict[str, Any]:
"""Serialise the track for export/import (identity/URL/descriptor/language).
DRM is not serialised here; the export writer attaches the licensed DRM + keys.
Subclasses add their own codec/quality fields.
"""
data: dict[str, Any] = {
"type": self.__class__.__name__,
"id": self.id,
"url": self.url,
"language": str(self.language),
"is_original_lang": self.is_original_lang,
"descriptor": self.descriptor.name,
"needs_repack": self.needs_repack,
"name": self.name,
"edition": self.edition,
}
return data
@staticmethod
def base_kwargs_from_dict(data: dict[str, Any]) -> dict[str, Any]:
"""Build the shared Track constructor kwargs from a ``to_dict()`` payload.
DRM is not reconstructed here ``to_dict`` does not serialise it, and the import
flow attaches the licensed DRM + content keys separately.
"""
return {
"url": data["url"],
"language": data.get("language") or "und",
"is_original_lang": data.get("is_original_lang", False),
"descriptor": Track.Descriptor[data.get("descriptor", "URL")],
"needs_repack": data.get("needs_repack", False),
"name": data.get("name"),
"edition": data.get("edition") or None,
"id_": data.get("id"),
}
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "Track":
"""Reconstruct the correct Track subclass from a ``to_dict()`` payload."""
from envied.core.tracks.audio import Audio
from envied.core.tracks.subtitle import Subtitle
from envied.core.tracks.video import Video
track_type = data.get("type")
builders = {"Video": Video, "Audio": Audio, "Subtitle": Subtitle}
builder = builders.get(track_type)
if builder is None:
raise ValueError(f"Cannot reconstruct unsupported track type: {track_type!r}")
return builder.from_dict(data)
def get_track_name(self) -> Optional[str]:
"""Get the Track Name."""
return self.name
@@ -602,8 +662,8 @@ class Track:
raise TypeError(f"Expected url to be a {str}, not {type(url)}")
if not isinstance(byte_range, (str, type(None))):
raise TypeError(f"Expected byte_range to be a {str}, not {type(byte_range)}")
if not isinstance(session, (Session, CurlSession, type(None))):
raise TypeError(f"Expected session to be a {Session} or {CurlSession}, not {type(session)}")
if not isinstance(session, (Session, RnetSession, type(None))):
raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {type(session)}")
if not url:
if self.descriptor != self.Descriptor.URL:
@@ -641,10 +701,11 @@ class Track:
init_data = res.content
else:
init_data = None
with session.get(url, stream=True) as s:
for chunk in s.iter_content(content_length):
init_data = chunk
break
s = session.get(url, stream=True)
for chunk in s.iter_content(content_length):
init_data = chunk
break
s.close()
if not init_data:
raise ValueError(f"Failed to read {content_length} bytes from the track URI.")
@@ -39,12 +39,14 @@ class Tracks:
*args: Union[
Tracks, Sequence[Union[AnyTrack, Chapter, Chapters, Attachment]], Track, Chapter, Chapters, Attachment
],
manifest_url: Optional[str] = None,
):
self.videos: list[Video] = []
self.audio: list[Audio] = []
self.subtitles: list[Subtitle] = []
self.chapters = Chapters()
self.attachments: list[Attachment] = []
self.manifest_url: Optional[str] = manifest_url
if args:
self.add(args)
@@ -195,6 +197,8 @@ class Tracks:
) -> None:
"""Add a provided track to its appropriate array and ensuring it's not a duplicate."""
if isinstance(tracks, Tracks):
if tracks.manifest_url and not self.manifest_url:
self.manifest_url = tracks.manifest_url
tracks = [*list(tracks), *tracks.chapters, *tracks.attachments]
duplicates = 0
@@ -245,12 +249,21 @@ class Tracks:
self.videos.sort(key=lambda x: str(x.language))
self.videos.sort(key=lambda x: not is_close_match(language, [x.language]))
def sort_audio(self, by_language: Optional[Sequence[Union[str, Language]]] = None) -> None:
"""Sort audio tracks by bitrate, Atmos, descriptive, and optionally language."""
def sort_audio(
self,
by_language: Optional[Sequence[Union[str, Language]]] = None,
codec_priority: Optional[Sequence[str]] = None,
) -> None:
"""Sort audio tracks by bitrate, codec priority, Atmos, descriptive, and optionally language."""
if not self.audio:
return
# bitrate (highest first)
self.audio.sort(key=lambda x: float(x.bitrate or 0.0), reverse=True)
# codec priority (listed codecs ranked in order; unlisted fall to end with bitrate order preserved)
if codec_priority:
rank = {str(c).upper(): i for i, c in enumerate(codec_priority)}
default_rank = len(rank)
self.audio.sort(key=lambda x: rank.get(x.codec.name if x.codec else "", default_rank))
# Atmos tracks first (prioritize over higher bitrate non-Atmos)
self.audio.sort(key=lambda x: not x.atmos)
# descriptive tracks last
@@ -302,25 +315,48 @@ class Tracks:
def select_subtitles(self, x: Callable[[Subtitle], bool]) -> None:
self.subtitles = list(filter(x, self.subtitles))
def select_hybrid(self, tracks, quality):
def filter(self, predicate: Callable[[AnyTrack], bool]) -> Tracks:
"""Return a new Tracks with tracks filtered by predicate, preserving metadata."""
new_tracks = Tracks(manifest_url=self.manifest_url)
new_tracks.videos = [t for t in self.videos if predicate(t)]
new_tracks.audio = [t for t in self.audio if predicate(t)]
new_tracks.subtitles = [t for t in self.subtitles if predicate(t)]
new_tracks.chapters = self.chapters
new_tracks.attachments = list(self.attachments)
return new_tracks
@staticmethod
def merge_video_selections(*groups: list[Video]) -> list[Video]:
"""Concatenate video selections, dropping duplicates (by track id, order-preserving).
A DV track can be chosen as both the hybrid ingredient (lowest) and an explicit
deliverable; without dedup the same track would be muxed/downloaded twice.
"""
merged: list[Video] = []
for group in groups:
for video in group:
if video not in merged:
merged.append(video)
return merged
def select_hybrid(self, tracks, quality, worst: bool = False):
# Prefer HDR10+ over HDR10 as the base layer (preserves dynamic metadata)
base_ranges = (Video.Range.HDR10P, Video.Range.HDR10)
base_tracks = []
for range_type in base_ranges:
base_tracks = [
v
for v in tracks
if v.range == range_type and (v.height in quality or int(v.width * 9 / 16) in quality)
v for v in tracks if v.range == range_type and (v.height in quality or int(v.width * 9 / 16) in quality)
]
if base_tracks:
break
pick = min if worst else max
base_selected = []
for res in quality:
candidates = [v for v in base_tracks if v.height == res or int(v.width * 9 / 16) == res]
if candidates:
best = max(candidates, key=lambda v: v.bitrate)
base_selected.append(best)
chosen = pick(candidates, key=lambda v: v.bitrate)
base_selected.append(chosen)
dv_tracks = [v for v in tracks if v.range == Video.Range.DV]
lowest_dv = min(dv_tracks, key=lambda v: v.height) if dv_tracks else None
@@ -415,13 +451,40 @@ class Tracks:
if config.muxing.get("set_title", True):
cl.extend(["--title", title])
default_language = config.muxing.get("default_language") or {}
preferred_video_lang = default_language.get("video")
preferred_audio_lang = default_language.get("audio")
preferred_subtitle_lang = default_language.get("subtitle")
preferred_video_idx: Optional[int] = None
if preferred_video_lang:
preferred_video_idx = next(
(idx for idx, v in enumerate(self.videos) if is_close_match(v.language, [preferred_video_lang])),
None,
)
preferred_audio_idx: Optional[int] = None
if preferred_audio_lang:
preferred_audio_idx = next(
(idx for idx, a in enumerate(self.audio) if is_close_match(a.language, [preferred_audio_lang])),
None,
)
preferred_subtitle_idx: Optional[int] = None
if preferred_subtitle_lang and not skip_subtitles:
preferred_subtitle_idx = next(
(idx for idx, s in enumerate(self.subtitles) if is_close_match(s.language, [preferred_subtitle_lang])),
None,
)
for i, vt in enumerate(self.videos):
if not vt.path or not vt.path.exists():
raise ValueError("Video Track must be downloaded before muxing...")
events.emit(events.Types.TRACK_MULTIPLEX, track=vt)
is_default = False
if title_language:
if preferred_video_idx is not None:
is_default = i == preferred_video_idx
elif title_language:
is_default = vt.language == title_language
if not any(v.language == title_language for v in self.videos):
is_default = vt.is_original_lang or i == 0
@@ -477,6 +540,10 @@ class Tracks:
if not at.path or not at.path.exists():
raise ValueError("Audio Track must be downloaded before muxing...")
events.emit(events.Types.TRACK_MULTIPLEX, track=at)
if preferred_audio_idx is not None:
audio_default = i == preferred_audio_idx
else:
audio_default = at.is_original_lang
cl.extend(
[
"--track-name",
@@ -484,7 +551,7 @@ class Tracks:
"--language",
f"0:{at.language}",
"--default-track",
f"0:{at.is_original_lang}",
f"0:{audio_default}",
"--visual-impaired-flag",
f"0:{at.descriptive}",
"--original-flag",
@@ -498,11 +565,14 @@ class Tracks:
)
if not skip_subtitles:
for st in self.subtitles:
for i, st in enumerate(self.subtitles):
if not st.path or not st.path.exists():
raise ValueError("Text Track must be downloaded before muxing...")
events.emit(events.Types.TRACK_MULTIPLEX, track=st)
default = bool(self.audio and is_close_match(st.language, [self.audio[0].language]) and st.forced)
if preferred_subtitle_idx is not None:
default = i == preferred_subtitle_idx
else:
default = bool(self.audio and is_close_match(st.language, [self.audio[0].language]) and st.forced)
cl.extend(
[
"--track-name",
+125 -3
View File
@@ -126,27 +126,48 @@ class Video(Track):
Reserved = 0
BT_709 = 1
Unspecified = 2
BT_470_M = 4
BT_601_625 = 5
BT_601_525 = 6
SMPTE_240M = 7
Generic_Film = 8
BT_2020_and_2100 = 9
SMPTE_ST_428_1 = 10
SMPTE_RP_431_2 = 11 # P3DCI
SMPTE_ST_2113_and_EG_4321 = 12 # P3D65
EBU_Tech_3213_E = 22
class Transfer(Enum):
Reserved = 0
BT_709 = 1
Unspecified = 2
BT_470_M = 4
BT_601 = 6
SMPTE_240M = 7
Linear = 8
Log_100 = 9
Log_316 = 10
IEC_61966_2_4 = 11
BT_1361 = 12
IEC_61966_2_1 = 13 # sRGB / sYCC
BT_2020 = 14
BT_2100 = 15
BT_2100_PQ = 16
SMPTE_ST_428_1 = 17
BT_2100_HLG = 18
class Matrix(Enum):
RGB = 0
YCbCr_BT_709 = 1
Unspecified = 2
YCbCr_FCC_73_682 = 4
YCbCr_BT_601_625 = 5
YCbCr_BT_601_525 = 6
SMPTE_240M = 7
YCgCo = 8
YCbCr_BT_2020_and_2100 = 9 # YCbCr BT.2100 shares the same CP
YCbCr_BT_2020_CL = 10
YCbCr_SMPTE_ST_2085 = 11
ICtCp_BT_2100 = 14
if transfer == 5:
@@ -155,9 +176,15 @@ class Video(Track):
# The codebase is currently agnostic to either, so a manual conversion to 6 is done.
transfer = 6
primaries = Primaries(primaries)
transfer = Transfer(transfer)
matrix = Matrix(matrix)
def _safe(enum_cls, value):
try:
return enum_cls(value)
except ValueError:
return enum_cls(2) # Unspecified for unknown/private-use codes
primaries = _safe(Primaries, primaries)
transfer = _safe(Transfer, transfer)
matrix = _safe(Matrix, matrix)
# primaries and matrix does not strictly correlate to a range
@@ -201,6 +228,7 @@ class Video(Track):
fps: Optional[Union[str, int, float]] = None,
scan_type: Optional[Video.ScanType] = None,
closed_captions: Optional[list[dict[str, Any]]] = None,
dv_compatible_bitstream: bool = False,
**kwargs: Any,
) -> None:
"""
@@ -267,6 +295,39 @@ class Video(Track):
self.scan_type = scan_type
self.closed_captions: list[dict[str, Any]] = closed_captions or []
self.needs_duration_fix = False
self.dv_compatible_bitstream = dv_compatible_bitstream
self.hybrid_base_only = False
def to_dict(self) -> dict[str, Any]:
data = super().to_dict()
data.update(
{
"codec": self.codec.name if self.codec else None,
"range": self.range.name if self.range else None,
"bitrate": self.bitrate,
"width": self.width,
"height": self.height,
"fps": str(self.fps) if self.fps else None,
"scan_type": self.scan_type.name if self.scan_type else None,
"dv_compatible_bitstream": self.dv_compatible_bitstream,
}
)
return data
@classmethod
def from_dict(cls, data: dict[str, Any]) -> Video:
kwargs = Track.base_kwargs_from_dict(data)
return cls(
**kwargs,
codec=Video.Codec[data["codec"]] if data.get("codec") else None,
range_=Video.Range[data["range"]] if data.get("range") else None,
bitrate=data.get("bitrate"),
width=data.get("width"),
height=data.get("height"),
fps=data.get("fps"),
scan_type=Video.ScanType[data["scan_type"]] if data.get("scan_type") else None,
dv_compatible_bitstream=data.get("dv_compatible_bitstream", False),
)
def __str__(self) -> str:
return " | ".join(
@@ -338,6 +399,67 @@ class Video(Track):
self.path = output_path
original_path.unlink()
def normalize_vui(self) -> bool:
"""Rewrite SPS VUI colour metadata to match ``self.range``.
Some services ship HDR10/HLG bitstreams with stale BT.709 VUI, which makes
downstream tools mis-classify the file. The manifest-derived range is the
source of truth. Skips SDR, DV, and HYBRID. Returns True if the bitstream
was rewritten.
"""
if not self.path or not self.path.exists():
return False
if self.codec not in (Video.Codec.AVC, Video.Codec.HEVC):
return False
if self.range in (Video.Range.SDR, Video.Range.DV, Video.Range.HYBRID):
return False
vui = {
Video.Range.HDR10: (9, 16, 9),
Video.Range.HDR10P: (9, 16, 9),
Video.Range.HLG: (9, 18, 9),
}.get(self.range)
if not vui:
return False
if not binaries.FFMPEG:
raise EnvironmentError('FFmpeg executable "ffmpeg" was not found but is required for this call.')
primaries, transfer, matrix = vui
filter_key = {Video.Codec.AVC: "h264_metadata", Video.Codec.HEVC: "hevc_metadata"}[self.codec]
bsf = (
f"{filter_key}=colour_primaries={primaries}"
f":transfer_characteristics={transfer}"
f":matrix_coefficients={matrix}"
)
original_path = self.path
output_path = original_path.with_stem(f"{original_path.stem}_vui")
try:
subprocess.run(
[
binaries.FFMPEG,
"-hide_banner",
"-loglevel",
"error",
"-i",
str(original_path),
"-codec",
"copy",
"-bsf:v",
bsf,
str(output_path),
],
check=True,
)
except subprocess.CalledProcessError:
output_path.unlink(missing_ok=True)
return False
self.path = output_path
original_path.unlink()
return True
def ccextractor(
self, track_id: Any, out_path: Union[Path, str], language: Language, original: bool = False
) -> Optional[Subtitle]:
+37 -111
View File
@@ -1,5 +1,6 @@
import ast
import contextlib
import gzip
import importlib.util
import json
import logging
@@ -10,6 +11,7 @@ import sys
import time
import traceback
import unicodedata
import zlib
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path
@@ -20,14 +22,12 @@ from uuid import uuid4
import chardet
import pycountry
import requests
from construct import ValidationError
from fontTools import ttLib
from langcodes import Language, closest_match
from pymp4.parser import Box
from unidecode import unidecode
from envied.core.cacher import Cacher
from envied.core.config import config
from envied.core.constants import LANGUAGE_EXACT_DISTANCE, LANGUAGE_MAX_DISTANCE
@@ -129,6 +129,8 @@ def sanitize_filename(filename: str, spacer: str = ".") -> str:
# optionally replace non-ASCII characters with ASCII equivalents
if not config.unicode_filenames:
filename = unidecode(filename)
filename = re.sub(r"\[\(+", "[", filename)
filename = re.sub(r"\)+\]", "]", filename)
# remove or replace further characters as needed
filename = "".join(c for c in filename if unicodedata.category(c) != "Mn") # hidden characters
@@ -158,6 +160,18 @@ def is_exact_match(language: Union[str, Language], languages: Sequence[Union[str
return closest_match(language, list(map(str, languages)))[1] <= LANGUAGE_EXACT_DISTANCE
def find_missing_langs(
requested: Sequence[str],
available: Sequence[Union[str, Language, None]],
*,
exact: bool = False,
) -> list[str]:
"""Return requested language tokens with no match in available languages."""
match_func = is_exact_match if exact else is_close_match
skip = {"all", "best", "orig"}
return [tok for tok in requested if tok not in skip and not match_func(tok, available)]
def get_boxes(data: bytes, box_type: bytes, as_bytes: bool = False) -> Box: # type: ignore
"""
Scan a byte array for a wanted MP4/ISOBMFF box, then parse and yield each find.
@@ -352,111 +366,6 @@ def get_country_code(name: str) -> Optional[str]:
return None
def get_ip_info(session: Optional[requests.Session] = None) -> dict:
"""
Use ipinfo.io to get IP location information.
If you provide a Requests Session with a Proxy, that proxies IP information
is what will be returned.
"""
return (session or requests.Session()).get("https://ipinfo.io/json").json()
def get_cached_ip_info(session: Optional[requests.Session] = None) -> Optional[dict]:
"""
Get IP location information with 24-hour caching and fallback providers.
This function uses a global cache to avoid repeated API calls when the IP
hasn't changed. Should only be used for local IP checks, not for proxy verification.
Implements smart provider rotation to handle rate limiting (429 errors).
Args:
session: Optional requests session (usually without proxy for local IP)
Returns:
Dict with IP info including 'country' key, or None if all providers fail
"""
log = logging.getLogger("get_cached_ip_info")
cache = Cacher("global").get("ip_info")
if cache and not cache.expired:
return cache.data
provider_state_cache = Cacher("global").get("ip_provider_state")
provider_state = provider_state_cache.data if provider_state_cache and not provider_state_cache.expired else {}
providers = {
"ipinfo": "https://ipinfo.io/json",
"ipapi": "https://ipapi.co/json",
}
session = session or requests.Session()
provider_order = ["ipinfo", "ipapi"]
current_time = time.time()
for provider_name in list(provider_order):
if provider_name in provider_state:
rate_limit_info = provider_state[provider_name]
if (current_time - rate_limit_info.get("rate_limited_at", 0)) < 300:
log.debug(f"Provider {provider_name} was rate limited recently, trying other provider first")
provider_order.remove(provider_name)
provider_order.append(provider_name)
break
for provider_name in provider_order:
provider_url = providers[provider_name]
try:
log.debug(f"Trying IP provider: {provider_name}")
response = session.get(provider_url, timeout=10)
if response.status_code == 429:
log.warning(f"Provider {provider_name} returned 429 (rate limited), trying next provider")
if provider_name not in provider_state:
provider_state[provider_name] = {}
provider_state[provider_name]["rate_limited_at"] = current_time
provider_state[provider_name]["rate_limit_count"] = (
provider_state[provider_name].get("rate_limit_count", 0) + 1
)
provider_state_cache.set(provider_state, expiration=300)
continue
elif response.status_code == 200:
data = response.json()
normalized_data = {}
if "country" in data:
normalized_data = data
elif "country_code" in data:
normalized_data = {
"country": data.get("country_code", "").lower(),
"region": data.get("region", ""),
"city": data.get("city", ""),
"ip": data.get("ip", ""),
}
if normalized_data and "country" in normalized_data:
log.debug(f"Successfully got IP info from provider: {provider_name}")
if provider_name in provider_state:
provider_state[provider_name].pop("rate_limited_at", None)
provider_state_cache.set(provider_state, expiration=300)
normalized_data["_provider"] = provider_name
cache.set(normalized_data, expiration=86400)
return normalized_data
else:
log.debug(f"Provider {provider_name} returned status {response.status_code}")
except Exception as e:
log.debug(f"Provider {provider_name} failed with exception: {e}")
continue
log.warning("All IP geolocation providers failed")
return None
def time_elapsed_since(start: float) -> str:
"""
Get time elapsed since a timestamp as a string.
@@ -478,12 +387,29 @@ def try_ensure_utf8(data: bytes) -> bytes:
"""
Try to ensure that the given data is encoded in UTF-8.
Automatically decompresses gzip/deflate/zlib data before encoding detection.
This handles cases where HTTP responses are saved with raw Content-Encoding
(e.g., when decode_content=False is used for performance).
Parameters:
data: Input data that may or may not yet be UTF-8 or another encoding.
Returns the input data encoded in UTF-8 if successful. If unable to detect the
encoding of the input data, then the original data is returned as-received.
"""
# Decompress gzip data (magic bytes: 1f 8b)
if data[:2] == b"\x1f\x8b":
try:
data = gzip.decompress(data)
except Exception:
pass
# Decompress raw deflate/zlib data (common zlib headers: 78 01, 78 5e, 78 9c, 78 da)
elif data[:1] == b"\x78" and len(data) > 1 and data[1:2] in (b"\x01", b"\x5e", b"\x9c", b"\xda"):
try:
data = zlib.decompress(data)
except Exception:
pass
try:
data.decode("utf8")
return data
@@ -497,10 +423,10 @@ def try_ensure_utf8(data: bytes) -> bytes:
# last ditch effort to detect encoding
detection_result = chardet.detect(data)
if not detection_result["encoding"]:
return data
return data.decode(detection_result["encoding"]).encode("utf8")
except UnicodeDecodeError:
return data
return data.decode("utf-8", errors="replace").encode("utf-8")
return data.decode(detection_result["encoding"], errors="replace").encode("utf8")
except (UnicodeDecodeError, LookupError):
return data.decode("utf-8", errors="replace").encode("utf-8")
def get_free_port() -> int:
@@ -0,0 +1,441 @@
from __future__ import annotations
import json
import logging
import subprocess
from collections import OrderedDict, defaultdict
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from typing import TYPE_CHECKING, Callable, Hashable, Optional, Union
from urllib.parse import urljoin
from requests import Session
from envied.core.binaries import FFProbe
from envied.core.session import RnetSession
if TYPE_CHECKING:
from envied.core.tracks import Track
# Default ISM timescale (ticks per second) per the Smooth Streaming spec.
ISM_DEFAULT_TIMESCALE = 10_000_000
# Bytes fetched to locate an mp4 moov box when probing duration via ffprobe.
MOOV_PROBE_BYTES = 4 * 1024 * 1024
# Network timeout (seconds) for probe requests.
PROBE_TIMEOUT = 15
@dataclass
class Segment:
"""One probe target: a media URL, optional byte range, its size, and duration."""
url: str
# The original byte-range string (e.g. "0-1023"), preserved as the segment's
# identity so distinct ranges of one file are never confused with each other.
byte_range: Optional[str]
# Size in bytes when derivable without a request (from a byte range); else None.
known_size: Optional[int]
duration: float
def measure_real_bitrate(
track: "Track",
session: Union[Session, RnetSession],
*,
samples: int = 40,
log: logging.Logger,
) -> Optional[int]:
"""
Probe a track's actual media size to compute its real average bitrate.
Manifests often declare an inaccurate bandwidth (DASH ``@bandwidth`` is a
leaky-bucket ceiling, not an average). This measures the true bitrate
(bits/sec) from real media byte sizes and durations using ``bytes * 8 / sec``.
Single-file tracks are measured exactly. Segmented tracks probe up to
``samples`` segments spread across the track and extrapolate; byte-range
segments need no request. Returns bits/sec, or ``None`` if it cannot be
measured. Never raises a probe failure must not abort a download.
"""
from envied.core.tracks.track import Track
try:
if track.descriptor == Track.Descriptor.DASH:
segments = extract_dash(track, session)
elif track.descriptor == Track.Descriptor.HLS:
segments = extract_hls(track, session)
elif track.descriptor == Track.Descriptor.ISM:
segments = extract_ism(track, session)
else:
# Descriptor.URL: a single file. Some services (e.g. AMZN) parse a DASH
# manifest then collapse each representation to its single BaseURL and
# flip the descriptor to URL, leaving the manifest (and its duration) in
# track.data — recover the duration from there, else probe the file.
segments = extract_url(track, session, log=log)
if not segments:
log.debug(f"{track.id}: cannot measure real bitrate (no known duration)")
return None
except Exception as e:
log.warning(f"{track.id}: failed to derive segments for real bitrate ({e})")
return None
if not segments:
return None
items = dedupe(segments)
chosen = pick_samples(items, samples)
total_bytes = 0
total_seconds = 0.0
for segment in chosen:
if segment.duration <= 0:
continue
size = segment.known_size if segment.known_size is not None else probe_size(segment, session)
if not size:
continue
total_bytes += size
total_seconds += segment.duration
log.debug(
f"{track.id}: real-bitrate probe desc={track.descriptor.name} "
f"n_seg={len(segments)} n_unique={len(items)} n_chosen={len(chosen)} "
f"sampled_bytes={total_bytes} sampled_seconds={round(total_seconds, 4)}"
)
if total_seconds <= 0 or total_bytes <= 0:
log.warning(f"{track.id}: real bitrate probe returned no usable data")
return None
return round(total_bytes * 8 / total_seconds)
def apply_real_bitrates(
tracks: list["Track"],
session: Union[Session, RnetSession],
*,
log: logging.Logger,
group_key: Callable[["Track"], Hashable],
per_group: int = 5,
workers: int = 8,
) -> None:
"""
Probe real bitrates and overwrite ``track.bitrate`` for the tracks worth probing.
Probing every rendition is slow when a service exposes dozens. Tracks are
grouped by ``group_key`` (a quality tier), and only the ``per_group`` highest
declared-bitrate tracks per group are probed, in parallel. Each group is then
extended downward: while the lowest probed bitrate in a group sits below the
next unprobed track's declared bitrate (so that track could outrank a probed
one), the next track is probed too until the probed set is safely above the
rest. Unprobed tracks keep their manifest-declared bitrate.
"""
groups: defaultdict[Hashable, list["Track"]] = defaultdict(list)
for track in tracks:
groups[group_key(track)].append(track)
for group in groups.values():
group.sort(key=lambda t: getattr(t, "bitrate", None) or 0, reverse=True)
# Initial pass: top per_group of every group, all probed concurrently.
initial = [track for group in groups.values() for track in group[:per_group]]
probe_batch(initial, session, log=log, workers=workers)
# Extend each group downward until unprobed tracks can't outrank probed ones.
for group in groups.values():
probed = min(per_group, len(group))
while probed < len(group):
lowest_probed = min((getattr(t, "bitrate", None) or 0) for t in group[:probed])
next_declared = getattr(group[probed], "bitrate", None) or 0
if next_declared <= lowest_probed:
break
probe_batch([group[probed]], session, log=log, workers=workers)
probed += 1
def probe_batch(
tracks: list["Track"],
session: Union[Session, RnetSession],
*,
log: logging.Logger,
workers: int,
) -> None:
"""Probe each track concurrently and overwrite its bitrate with the measured value."""
if not tracks:
return
def probe_one(track: "Track") -> tuple["Track", Optional[int]]:
return track, measure_real_bitrate(track, track.session or session, log=log)
with ThreadPoolExecutor(max_workers=min(workers, len(tracks))) as executor:
for track, measured in executor.map(probe_one, tracks):
if not measured:
continue
declared = getattr(track, "bitrate", None)
if declared and declared != measured:
log.debug(f"{track.id}: bitrate {declared // 1000}{measured // 1000} kb/s (real)")
setattr(track, "bitrate", measured)
def dedupe(segments: list[Segment]) -> list[Segment]:
"""
Collapse segments that address the same bytes so each object is measured once.
Manifests sometimes wrap a single file in several segment entries sharing one
URL with no byte range (a ``SegmentTemplate`` whose media pattern has no
``$Number$``) or with the same range. Each resolves to the whole file, so
counting them all would multiply the size by the segment count. Segments
sharing the same ``(url, byte_range)`` are merged into one entry whose duration
is the sum they cover. Distinct byte ranges of one file (different offsets) are
kept individual so their sizes still add up to the full track.
"""
merged: OrderedDict[tuple[str, Optional[str]], Segment] = OrderedDict()
for segment in segments:
key = (segment.url, segment.byte_range)
existing = merged.get(key)
if existing is None:
merged[key] = Segment(segment.url, segment.byte_range, segment.known_size, segment.duration)
else:
existing.duration += segment.duration
return list(merged.values())
def pick_samples(segments: list[Segment], samples: int) -> list[Segment]:
"""Pick up to ``samples`` segments spread evenly across the track."""
count = len(segments)
if count <= samples:
return segments
step = count / samples
indices = sorted({int(i * step) for i in range(samples)})
return [segments[i] for i in indices]
def probe_size(segment: Segment, session: Union[Session, RnetSession]) -> Optional[int]:
"""Return a segment's byte size via HEAD, falling back to a ranged GET. Validates status."""
try:
res = session.head(segment.url, allow_redirects=True, timeout=PROBE_TIMEOUT)
if getattr(res, "status_code", 0) in (200, 206):
content_length = res.headers.get("Content-Length")
if content_length:
return int(content_length)
except Exception:
pass
# Some hosts block or mishandle HEAD; ask for a single byte and read the total.
# Require a 206 so a server that ignores Range (returning the whole 200 body)
# is not mistaken for a valid size or downloaded wholesale.
try:
res = session.get(segment.url, headers={"Range": "bytes=0-0"}, timeout=PROBE_TIMEOUT)
if getattr(res, "status_code", 0) == 206:
content_range = res.headers.get("Content-Range")
if content_range and "/" in content_range:
total = content_range.rsplit("/", 1)[-1].strip()
if total.isdigit():
return int(total)
except Exception:
pass
return None
def range_size(byte_range: Optional[str]) -> Optional[int]:
"""Size in bytes of a ``start-end`` media range, inclusive."""
if not byte_range or "-" not in byte_range:
return None
start_s, _, end_s = byte_range.partition("-")
try:
start = int(start_s) if start_s else 0
if not end_s:
return None
return int(end_s) - start + 1
except ValueError:
return None
def uniform_segments(
raw_segments: list[tuple[str, Optional[str]]],
total_duration: Optional[float],
) -> list[Segment]:
"""
Build Segments giving each an equal share of the total duration.
Used for DASH: ``DASH._get_period_segments`` returns timeline *start times*
rather than per-segment durations, so they cannot be trusted. Segment lengths
are near-uniform in practice, so the track duration (from
``mediaPresentationDuration``) split evenly is both correct and timeline-safe.
"""
count = len(raw_segments)
if not count or not total_duration or total_duration <= 0:
return []
per_segment = total_duration / count
return [Segment(url, byte_range, range_size(byte_range), per_segment) for url, byte_range in raw_segments]
def extract_dash(track: "Track", session: Union[Session, RnetSession]) -> list[Segment]:
from envied.core.manifests import DASH
data = track.data["dash"]
manifest = data["manifest"]
rep_id = data.get("representation_id") or data["representation"].get("id")
filtered_period_ids = data.get("filtered_period_ids", [])
track_url = track.url if isinstance(track.url, str) else track.url[0]
content_periods = [p for p in manifest.findall("Period") if DASH._is_content_period(p, filtered_period_ids)]
raw_segments: list[tuple[str, Optional[str]]] = []
for period in content_periods:
matched_rep = matched_as = None
for as_ in period.findall("AdaptationSet"):
if DASH.is_trick_mode(as_):
continue
for rep in as_.findall("Representation"):
if rep.get("id") == rep_id:
matched_rep, matched_as = rep, as_
break
if matched_rep is not None:
break
if matched_rep is None or matched_as is None:
continue
_, period_segments, _, _, _ = DASH._get_period_segments(
period=period,
adaptation_set=matched_as,
representation=matched_rep,
manifest=manifest,
track=track,
track_url=track_url,
session=session,
)
raw_segments.extend(period_segments)
total_duration: Optional[float] = None
mpd_duration = manifest.get("mediaPresentationDuration")
if mpd_duration:
total_duration = DASH.pt_to_sec(mpd_duration)
return uniform_segments(raw_segments, total_duration)
def extract_hls(track: "Track", session: Union[Session, RnetSession]) -> list[Segment]:
import m3u8
playlist_url = track.url if isinstance(track.url, str) else track.url[0]
res = session.get(playlist_url, timeout=PROBE_TIMEOUT)
playlist = m3u8.loads(res.text, uri=playlist_url)
out: list[Segment] = []
for segment in playlist.segments:
url = urljoin(segment.base_uri or "", segment.uri)
byte_range = segment.byterange # "<length>[@<offset>]"
known_size: Optional[int] = None
if byte_range:
length = byte_range.split("@")[0].strip()
if length.isdigit():
known_size = int(length)
# EXTINF durations are reliable, so they are used directly (unlike DASH).
out.append(Segment(url, byte_range, known_size, float(segment.duration or 0)))
return out
def extract_ism(track: "Track", session: Union[Session, RnetSession]) -> list[Segment]:
data = track.data["ism"]
segments: list[str] = data.get("segments") or []
manifest = data["manifest"]
timescale = int(manifest.get("TimeScale") or ISM_DEFAULT_TIMESCALE)
duration_ticks = int(manifest.get("Duration") or 0)
total_duration = (duration_ticks / timescale) if timescale else 0.0
return uniform_segments([(url, None) for url in segments], total_duration)
def extract_url(track: "Track", session: Union[Session, RnetSession], *, log: logging.Logger) -> list[Segment]:
"""Single-file track: one whole-file URL with the duration from leftover manifest data."""
url = track.url if isinstance(track.url, str) else (track.url[0] if track.url else None)
if not url:
return []
duration: Optional[float] = None
dash_data = track.data.get("dash")
if dash_data and dash_data.get("manifest") is not None:
from envied.core.manifests import DASH
mpd_duration = dash_data["manifest"].get("mediaPresentationDuration")
if mpd_duration:
duration = DASH.pt_to_sec(mpd_duration)
else:
ism_data = track.data.get("ism")
if ism_data and ism_data.get("manifest") is not None:
manifest = ism_data["manifest"]
timescale = int(manifest.get("TimeScale") or ISM_DEFAULT_TIMESCALE)
duration_ticks = int(manifest.get("Duration") or 0)
if timescale and duration_ticks:
duration = duration_ticks / timescale
if not duration or duration <= 0:
# Services like AMZN clear the manifest data after collapsing to a single
# file; fall back to reading the duration straight from the remote file.
duration = ffprobe_duration(url, session, log=log)
if not duration or duration <= 0:
return []
return [Segment(url, None, None, duration)]
def ffprobe_duration(url: str, session: Union[Session, RnetSession], *, log: logging.Logger) -> Optional[float]:
"""
Read a single-file track's duration (seconds) without a manifest.
The bundled ffprobe segfaults on network input, so the file's ``moov`` box is
fetched over HTTP with the session (keeping the service's proxy/headers) and
piped to ffprobe as local bytes. The head of the file is tried first (VOD is
usually faststart), then the tail as a fallback for moov-at-end files.
"""
head = ranged_get(url, session, f"bytes=0-{MOOV_PROBE_BYTES - 1}")
duration = probe_bytes_duration(head, log)
if duration:
return duration
size = probe_size(Segment(url, None, None, 0.0), session)
if size and size > MOOV_PROBE_BYTES:
tail = ranged_get(url, session, f"bytes={size - MOOV_PROBE_BYTES}-{size - 1}")
duration = probe_bytes_duration(tail, log)
return duration
def ranged_get(url: str, session: Union[Session, RnetSession], byte_range: str) -> Optional[bytes]:
"""Fetch a byte range, only accepting a real 206 partial response (never a full 200 body)."""
try:
res = session.get(url, headers={"Range": byte_range}, timeout=PROBE_TIMEOUT)
if getattr(res, "status_code", 0) != 206:
return None
content = getattr(res, "content", None)
return content if content else None
except Exception:
return None
def probe_bytes_duration(data: Optional[bytes], log: logging.Logger) -> Optional[float]:
"""Pipe media bytes to ffprobe and return the format/stream duration in seconds."""
if not data:
return None
ffprobe_bin = str(FFProbe) if FFProbe else "ffprobe"
try:
result = subprocess.run(
[ffprobe_bin, "-v", "error", "-show_entries", "format=duration:stream=duration", "-of", "json", "pipe:"],
input=data,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=60,
)
info = json.loads(result.stdout or b"{}")
candidates = [info.get("format", {}).get("duration")]
candidates += [s.get("duration") for s in info.get("streams", [])]
for value in candidates:
if value:
return float(value)
log.debug(f"ffprobe found no duration (rc={result.returncode}): {result.stderr.decode(errors='replace')[:160]}")
return None
except (subprocess.SubprocessError, ValueError, json.JSONDecodeError) as e:
log.debug(f"ffprobe duration error: {e}")
return None
@@ -360,9 +360,32 @@ class MultipleChoice(click.Choice):
return super(self).shell_complete(ctx, param, incomplete)
class SlowDelayRange(click.ParamType):
"""Parses a delay range string like '20-40' into a tuple of (min, max) seconds."""
name = "delay_range"
def convert(self, value: Any, param: Optional[click.Parameter], ctx: Optional[click.Context]) -> tuple[int, int]:
if isinstance(value, tuple):
return value
match = re.match(r"^(\d+)-(\d+)$", str(value))
if not match:
self.fail(f"'{value}' is not a valid range. Use format: MIN-MAX (e.g., 20-40)", param, ctx)
low, high = int(match.group(1)), int(match.group(2))
if low < 20:
self.fail(f"Minimum delay must be at least 20 seconds, got {low}", param, ctx)
if low > high:
self.fail(f"Min ({low}) cannot be greater than max ({high})", param, ctx)
return (low, high)
SEASON_RANGE = SeasonRange()
LANGUAGE_RANGE = LanguageRange()
QUALITY_LIST = QualityList()
AUDIO_CODEC_LIST = AudioCodecList(Audio.Codec)
SLOW_DELAY_RANGE = SlowDelayRange()
# VIDEO_CODEC_CHOICE will be created dynamically when imported
@@ -0,0 +1,130 @@
"""Thin wrappers around dovi_tool subcommands used by DVFixup and Hybrid.
Centralises argv construction, status spinners, and error handling so callers do not
re-implement subprocess plumbing per call site. Each wrapper:
- Resolves `binaries.DoviTool` and raises EnvironmentError if missing.
- Delegates to `core.utils.subprocess.run_step` for execution, output validation, and
stderr-tail RuntimeError on failure.
- Returns captured stderr so callers can inspect specific failure modes (e.g. the
MAX_PQ_LUMINANCE retry path in extract_rpu).
"""
from __future__ import annotations
import subprocess
from pathlib import Path
from typing import Optional
from envied.core import binaries
from envied.core.utils.subprocess import run_step
def _require_dovi_tool() -> str:
if not binaries.DoviTool:
raise EnvironmentError("dovi_tool executable was not found but is required.")
return str(binaries.DoviTool)
def extract_rpu(
source: Path,
output: Path,
*,
mode: Optional[int] = 3,
status: Optional[str] = "Extracting DV RPU...",
label: str = "dovi_tool extract-rpu",
) -> bytes:
"""Extract DV RPU NALs from a raw HEVC stream. `mode=None` skips the -m flag (untouched)."""
tool = _require_dovi_tool()
args: list = [tool]
if mode is not None:
args += ["-m", str(mode)]
args += ["extract-rpu", source, "-o", output]
return run_step(args, status=status, output=output, label=label)
def inject_rpu(
source: Path,
rpu: Path,
output: Path,
*,
status: Optional[str] = "Re-injecting DV RPU...",
label: str = "dovi_tool inject-rpu",
) -> bytes:
"""Inject a DV RPU back into a raw HEVC stream, producing DV-signaled output."""
tool = _require_dovi_tool()
return run_step(
[tool, "inject-rpu", "-i", source, "--rpu-in", rpu, "-o", output],
status=status,
output=output,
label=label,
)
def editor(
source: Path,
json_spec: Path,
output: Path,
*,
status: Optional[str] = "Editing DV RPU...",
label: str = "dovi_tool editor",
) -> bytes:
"""Apply a JSON edit spec to an RPU file."""
tool = _require_dovi_tool()
return run_step(
[tool, "editor", "-i", source, "-j", json_spec, "-o", output],
status=status,
output=output,
label=label,
)
def info_summary(rpu: Path) -> str:
"""Return the textual summary (`dovi_tool info -i ... -s`) for an RPU file."""
tool = _require_dovi_tool()
p = subprocess.run([tool, "info", "-i", str(rpu), "-s"], capture_output=True, text=True)
if p.returncode != 0:
raise RuntimeError(f"dovi_tool info failed: {(p.stderr or '')[-400:]}")
return p.stdout
def generate_from_hdr10plus(
extra_json: Path,
hdr10plus_json: Path,
output: Path,
*,
status: Optional[str] = "Generating DV RPU from HDR10+ metadata...",
label: str = "dovi_tool generate",
) -> bytes:
"""Build a DV RPU from extracted HDR10+ metadata + an extra JSON descriptor."""
tool = _require_dovi_tool()
return run_step(
[tool, "generate", "-j", extra_json, "--hdr10plus-json", hdr10plus_json, "-o", output],
status=status,
output=output,
label=label,
)
def extract_rpu_with_fallback(source: Path, output: Path, *, label: str = "dovi_tool extract-rpu") -> bytes:
"""Try `-m 3` first; on MAX_PQ_LUMINANCE error, retry untouched (no -m). Returns stderr.
Used when the caller wants automatic normalization but cannot abort if the source
rejects mode-3 conversion.
"""
try:
return extract_rpu(source, output, mode=3, label=label)
except RuntimeError as e:
if "MAX_PQ_LUMINANCE" not in str(e):
raise
return extract_rpu(source, output, mode=None, status="Extracting DV RPU (untouched)...", label=label)
__all__ = (
"extract_rpu",
"extract_rpu_with_fallback",
"inject_rpu",
"editor",
"info_summary",
"generate_from_hdr10plus",
)
@@ -0,0 +1,252 @@
from __future__ import annotations
import logging
import time
from typing import Any, Callable, Optional
import requests
from envied.core.cacher import Cacher
CACHE_KEY = "ip_info_v3"
CACHE_TTL = 86400 # 24 hours
PROVIDER_STATE_KEY = "ip_provider_state"
RATE_LIMIT_COOLDOWN = 300 # 5 minutes
REQUEST_TIMEOUT = 10
# Only these keys are persisted to the global cache.
GEO_CACHE_KEYS = ("country", "country_code")
Fetcher = Callable[[requests.Session], Optional[dict]]
log = logging.getLogger("ip_info")
class RateLimited(Exception):
"""Raised by a provider fetcher when the upstream returns 429."""
def normalize(
*,
country_code: str,
ip: str = "",
region: str = "",
city: str = "",
org: str = "",
asn: str = "",
as_name: str = "",
continent_code: str = "",
) -> Optional[dict]:
"""Build the canonical IP-info dict, or None if no country code is present."""
code = country_code.strip()
if not code:
return None
return {
"ip": ip,
"country": code.lower(),
"country_code": code.upper(),
"region": region,
"city": city,
"org": org,
"asn": asn,
"as_name": as_name,
"continent_code": continent_code.upper(),
}
def parse_ipinfo_lite(data: dict) -> Optional[dict]:
asn = (data.get("asn") or "").strip()
as_name = (data.get("as_name") or "").strip()
return normalize(
country_code=data.get("country_code") or "",
ip=data.get("ip") or "",
org=f"{asn} {as_name}".strip(),
asn=asn,
as_name=as_name,
continent_code=data.get("continent_code") or "",
)
def parse_ipinfo(data: dict) -> Optional[dict]:
return normalize(
country_code=data.get("country") or "",
ip=data.get("ip") or "",
region=data.get("region") or "",
city=data.get("city") or "",
org=data.get("org") or "",
)
def parse_ip_api_in(data: dict) -> Optional[dict]:
asn = (data.get("asn") or "").strip()
org_name = (data.get("organization") or "").strip()
return normalize(
country_code=data.get("country_code") or "",
ip=data.get("ip") or "",
region=data.get("region") or "",
city=data.get("city") or "",
org=f"{asn} {org_name}".strip(),
asn=asn,
as_name=org_name,
continent_code=data.get("continent_code") or "",
)
def lookup_session(source: Optional[requests.Session]) -> requests.Session:
"""
Build a plain, retry-free requests session for IP geolocation.
Geolocation needs no TLS fingerprinting, so we skip the impersonated rnet
session and the base session's urllib3 retry loop — both retry 429 internally,
which hides the response and defeats fast provider handover. With a bare session
a 429 comes straight back so we can move to the next provider immediately. Only
the proxy is carried over so proxied lookups still report the proxy's exit IP.
"""
sess = requests.Session()
proxies = getattr(source, "proxies", None)
if proxies:
proxy = proxies.get("all") or proxies.get("https") or proxies.get("http")
if proxy:
sess.proxies.update({"http": proxy, "https": proxy})
return sess
def json_or_raise(response: requests.Response) -> Optional[dict]:
"""Raise RateLimited on 429, return parsed JSON on 200, else None."""
if response.status_code == 429:
raise RateLimited()
if response.status_code != 200:
return None
try:
return response.json()
except ValueError:
return None
def fetch_ipinfo_lite(token: str) -> Fetcher:
headers = {"Authorization": f"Bearer {token}"}
def fetch(session: requests.Session) -> Optional[dict]:
payload = json_or_raise(session.get("https://api.ipinfo.io/lite/me", headers=headers, timeout=REQUEST_TIMEOUT))
return parse_ipinfo_lite(payload) if payload else None
return fetch
def fetch_ipinfo(session: requests.Session) -> Optional[dict]:
payload = json_or_raise(session.get("https://ipinfo.io/json", timeout=REQUEST_TIMEOUT))
return parse_ipinfo(payload) if payload else None
def fetch_ip_api_in(session: requests.Session) -> Optional[dict]:
"""ip-api.in has no /me endpoint — resolve IP via ipify first, then look it up."""
ip_resp = session.get("https://api.ipify.org", timeout=REQUEST_TIMEOUT)
if ip_resp.status_code == 429:
raise RateLimited()
ip = (ip_resp.text or "").strip() if ip_resp.status_code == 200 else ""
if not ip:
return None
payload = json_or_raise(session.get(f"https://ip-api.in/api/v1/ip/{ip}", timeout=REQUEST_TIMEOUT))
if not payload or not payload.get("success"):
return None
return parse_ip_api_in(payload.get("data") or {})
def build_providers() -> list[tuple[str, Fetcher]]:
"""Return ordered (name, fetcher) pairs. Token is read at call time."""
from envied.core.config import config
providers: list[tuple[str, Fetcher]] = []
token = (getattr(config, "ipinfo_api_key", "") or "").strip()
if token:
providers.append(("ipinfo_lite", fetch_ipinfo_lite(token)))
providers.append(("ipinfo", fetch_ipinfo))
providers.append(("ip_api_in", fetch_ip_api_in))
return providers
def purge_stale_cache() -> None:
"""Delete superseded ip_info cache files (older CACHE_KEY versions)."""
from envied.core.config import config
global_dir = config.directories.cache / "global"
for stale in global_dir.glob("ip_info_v*.json"):
if stale.stem != CACHE_KEY:
stale.unlink(missing_ok=True)
def load_provider_state(cacher: Cacher) -> dict[str, Any]:
return cacher.data if cacher and not cacher.expired and isinstance(cacher.data, dict) else {}
def get_ip_info(
session: Optional[requests.Session] = None,
*,
cached: bool = False,
) -> Optional[dict]:
"""
Look up IP/geolocation info via ipinfo.io (Lite when `ipinfo_api_key` configured)
with fallback to ip-api.in.
Live lookups return a dict with `ip`, `country` (lowercase ISO2), `country_code`
(uppercase ISO2), `region`, `city`, `org`, `asn`, `as_name`, `continent_code` and
`_provider`. Cached lookups return only `country`/`country_code` (see GEO_CACHE_KEYS).
Returns None if every provider fails.
Args:
session: Optional requests session. If a proxied session is passed, the
returned info reflects the proxy's exit IP. Auth headers for ipinfo
are sent per-request; never mutated onto session.headers.
cached: When True, read/write a 24h Cacher-backed entry. Use only for
local IP lookups never with a proxied session.
"""
cache = None
if cached:
purge_stale_cache()
cache = Cacher("global").get(CACHE_KEY)
if cache and not cache.expired and cache.data:
return cache.data
state_cache = Cacher("global").get(PROVIDER_STATE_KEY)
state = load_provider_state(state_cache)
now = time.time()
def on_cooldown(item: tuple[str, Fetcher]) -> int:
rate_limited_at = (state.get(item[0]) or {}).get("rate_limited_at", 0)
return 1 if (now - rate_limited_at) < RATE_LIMIT_COOLDOWN else 0
providers = sorted(build_providers(), key=on_cooldown)
sess = lookup_session(session)
for name, fetcher in providers:
log.debug(f"Trying IP provider: {name}")
try:
normalized = fetcher(sess)
except RateLimited:
log.warning(f"Provider {name} returned 429 (rate limited), trying next provider")
entry = state.setdefault(name, {})
entry["rate_limited_at"] = now
entry["rate_limit_count"] = entry.get("rate_limit_count", 0) + 1
state_cache.set(state, expiration=RATE_LIMIT_COOLDOWN)
continue
except Exception as e:
log.debug(f"Provider {name} failed with exception: {e}")
continue
if not normalized:
log.debug(f"Provider {name} returned no usable data")
continue
normalized["_provider"] = name
log.debug(f"Successfully got IP info from provider: {name}")
if name in state and state[name].pop("rate_limited_at", None) is not None:
state_cache.set(state, expiration=RATE_LIMIT_COOLDOWN)
if cache is not None:
cache.set({k: normalized.get(k, "") for k in GEO_CACHE_KEYS}, expiration=CACHE_TTL)
return normalized
log.warning("All IP geolocation providers failed")
return None
@@ -1,9 +1,10 @@
import json
import subprocess
from pathlib import Path
from typing import Union
from typing import Optional, Sequence, Union
from envied.core import binaries
from envied.core.console import console
def ffprobe(uri: Union[bytes, Path]) -> dict:
@@ -23,3 +24,34 @@ def ffprobe(uri: Union[bytes, Path]) -> dict:
except subprocess.CalledProcessError:
return {}
return json.loads(ff.stdout.decode("utf8"))
def run_step(
args: Sequence[Union[str, Path]],
*,
status: Optional[str] = None,
output: Optional[Path] = None,
label: str = "subprocess step",
) -> bytes:
"""Run a CLI step that writes to `output` (when provided). Returns stderr bytes.
Raises RuntimeError with the stderr tail when the process exits non-zero, or when
`output` is given and does not exist / is empty after the run.
"""
if output is not None:
output.unlink(missing_ok=True)
str_args = [str(a) for a in args]
if status:
with console.status(status, spinner="dots"):
p = subprocess.run(str_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
else:
p = subprocess.run(str_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stderr = p.stderr or b""
bad_output = output is not None and (not output.exists() or output.stat().st_size == 0)
if p.returncode or bad_output:
if output is not None:
output.unlink(missing_ok=True)
raise RuntimeError(f"{label} failed: {stderr.decode(errors='replace')[-400:]}")
return stderr
+43 -37
View File
@@ -33,13 +33,16 @@ def apply_tags(path: Path, tags: dict[str, str]) -> None:
f.write("\n".join(xml_lines))
tmp_path = Path(f.name)
try:
subprocess.run(
result = subprocess.run(
[str(binaries.Mkvpropedit), str(path), "--tags", f"global:{tmp_path}"],
check=False,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
capture_output=True,
text=True,
)
log.debug("Tags applied via mkvpropedit")
if result.returncode != 0:
log.warning("mkvpropedit failed (exit %d): %s", result.returncode, result.stderr.strip())
else:
log.debug("Tags applied via mkvpropedit")
finally:
tmp_path.unlink(missing_ok=True)
@@ -92,43 +95,46 @@ def tag_file(
standard_tags: dict[str, str] = {}
if config.tag_imdb_tmdb:
providers = get_available_providers()
if not providers:
log.debug("No metadata providers available; skipping tag lookup")
apply_tags(path, custom_tags)
return
try:
providers = get_available_providers()
if not providers:
log.debug("No metadata providers available; skipping tag lookup")
apply_tags(path, custom_tags)
return
result: Optional[MetadataResult] = None
result: Optional[MetadataResult] = None
# Direct ID lookup path
if imdb_id:
imdbapi = get_provider("imdbapi")
if imdbapi:
result = imdbapi.get_by_id(imdb_id, kind)
if result:
result.external_ids.imdb_id = imdb_id
enrich_ids(result)
elif tmdb_id is not None:
tmdb = get_provider("tmdb")
if tmdb:
result = tmdb.get_by_id(tmdb_id, kind)
if result:
ext = tmdb.get_external_ids(tmdb_id, kind)
result.external_ids = ext
else:
# Search across providers in priority order
result = search_metadata(name, year, kind)
# Direct ID lookup path
if imdb_id:
imdbapi = get_provider("imdbapi")
if imdbapi:
result = imdbapi.get_by_id(imdb_id, kind)
if result:
result.external_ids.imdb_id = imdb_id
enrich_ids(result)
elif tmdb_id is not None:
tmdb = get_provider("tmdb")
if tmdb:
result = tmdb.get_by_id(tmdb_id, kind)
if result:
ext = tmdb.get_external_ids(tmdb_id, kind)
result.external_ids = ext
else:
# Search across providers in priority order
result = search_metadata(name, year, kind)
# If we got a TMDB ID from search but no full external IDs, fetch them
if result and result.external_ids.tmdb_id and not result.external_ids.imdb_id:
ext = fetch_external_ids(result.external_ids.tmdb_id, kind)
if ext.imdb_id:
result.external_ids.imdb_id = ext.imdb_id
if ext.tvdb_id:
result.external_ids.tvdb_id = ext.tvdb_id
# If we got a TMDB ID from search but no full external IDs, fetch them
if result and result.external_ids.tmdb_id and not result.external_ids.imdb_id:
ext = fetch_external_ids(result.external_ids.tmdb_id, kind)
if ext.imdb_id:
result.external_ids.imdb_id = ext.imdb_id
if ext.tvdb_id:
result.external_ids.tvdb_id = ext.tvdb_id
if result and result.external_ids:
standard_tags = _build_tags_from_ids(result.external_ids, kind)
if result and result.external_ids:
standard_tags = _build_tags_from_ids(result.external_ids, kind)
except Exception as e:
log.warning("Metadata lookup failed, applying custom tags only: %s", e)
apply_tags(path, {**custom_tags, **standard_tags})
@@ -73,7 +73,9 @@ class TemplateFormatter:
has_left = s[0] in ".- "
has_right = s[-1] in ".- "
if has_left and has_right:
return s[0] # keep left separator
if s[-1] == "-":
return s[-1]
return s[0]
return ""
result = re.sub(
+14 -1
View File
@@ -1,3 +1,4 @@
import logging
from typing import Any, Iterator, Optional, Union
from uuid import UUID
@@ -5,6 +6,8 @@ from envied.core.config import config
from envied.core.utilities import import_module_by_path
from envied.core.vault import Vault
log = logging.getLogger(__name__)
_VAULTS = sorted(
(path for path in config.directories.vaults.glob("*.py") if path.stem.lower() != "__init__"), key=lambda x: x.stem
)
@@ -48,7 +51,13 @@ class Vaults:
def get_key(self, kid: Union[UUID, str]) -> tuple[Optional[str], Optional[Vault]]:
"""Get Key from the first Vault it can by KID (Key ID) and Service."""
for vault in self.vaults:
key = vault.get_key(kid, self.service)
try:
key = vault.get_key(kid, self.service)
except (PermissionError, NotImplementedError):
continue
except Exception as e:
log.warning(f"Failed to get key from Vault '{vault.name}': {e}")
continue
if key and key.count("0") != len(key):
return key, vault
return None, None
@@ -62,6 +71,8 @@ class Vaults:
success += vault.add_key(self.service, kid, key)
except (PermissionError, NotImplementedError):
pass
except Exception as e:
log.warning(f"Failed to add key to Vault '{vault.name}': {e}")
return success
def add_keys(self, kid_keys: dict[Union[UUID, str], str]) -> int:
@@ -79,6 +90,8 @@ class Vaults:
success += 1
except (PermissionError, NotImplementedError):
pass
except Exception as e:
log.warning(f"Failed to add keys to Vault '{vault.name}': {e}")
return success
@@ -0,0 +1,758 @@
# API key for The Movie Database (TMDB)
tmdb_api_key: ""
# Client ID for SIMKL API (optional, improves metadata matching)
# Get your free client ID at: https://simkl.com/settings/developer/
simkl_client_id: ""
# Optional ipinfo.io API token. When set, unshackle uses the free Lite endpoint
# which has higher rate limits and richer IP info (ASN, org, continent).
# Get a free token at: https://ipinfo.io/signup
ipinfo_api_key: ""
# Group or Username to postfix to the end of all download filenames following a dash
tag: user_tag
# Enable/disable tagging with group name (default: true)
tag_group_name: true
# Enable/disable tagging with IMDB/TMDB/TVDB details (default: true)
tag_imdb_tmdb: true
# Set terminal background color (custom option not in CONFIG.md)
set_terminal_bg: false
# Custom output templates for filenames
# Configure output_template in your envied.yaml to control filename format.
# If not configured, default scene-style templates are used and a warning is shown.
# Available variables: {title}, {year}, {season}, {episode}, {season_episode}, {episode_name},
# {quality}, {resolution}, {source}, {audio}, {audio_channels}, {audio_full},
# {video}, {hdr}, {hfr}, {atmos}, {dual}, {multi}, {tag}, {edition}, {repack},
# {lang_tag}
# Conditional variables (included only if present): Add ? suffix like {year?}, {episode_name?}, {hdr?}
# Customize the templates below:
#
# Example outputs:
# Scene movies: 'The.Matrix.1999.1080p.SERVICE.WEB-DL.DDP5.1.H.264-EXAMPLE'
# Scene movies (HDR): 'Dune.2021.2160p.SERVICE.WEB-DL.DDP5.1.HDR10.H.265-EXAMPLE'
# Scene movies (REPACK): 'Dune.2021.REPACK.2160p.SERVICE.WEB-DL.DDP5.1.H.265-EXAMPLE'
# Scene series: 'Breaking.Bad.2008.S01E01.Pilot.1080p.SERVICE.WEB-DL.DDP5.1.H.264-EXAMPLE'
# Plex movies: 'The Matrix (1999) 1080p'
# Plex series: 'Breaking Bad S01E01 Pilot'
output_template:
# Scene-style naming (dot-separated)
movies: '{title}.{year}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}'
series: '{title}.{year?}.{season_episode}.{episode_name?}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}'
songs: '{track_number}.{title}.{repack?}.{edition?}.{source?}.WEB-DL.{audio_full}.{atmos?}-{tag}'
#
# Plex-friendly naming (space-separated, clean format)
# movies: '{title} ({year}) {quality}'
# series: '{title} {season_episode} {episode_name?}'
# songs: '{track_number}. {title}'
#
# Minimal naming (basic info only)
# movies: '{title}.{year}.{quality}'
# series: '{title}.{season_episode}.{episode_name?}'
#
# Custom scene-style with specific elements
# movies: '{title}.{year}.{quality}.{hdr?}.{source}.WEB-DL.{audio_full}.{video}-{tag}'
# series: '{title}.{year?}.{season_episode}.{episode_name?}.{quality}.{hdr?}.{source}.WEB-DL.{audio_full}.{atmos?}.{video}-{tag}'
#
# Folder naming (optional). Controls the folder name for downloaded content.
# If not configured, series folders are derived from the series template (minus episode info),
# movie folders use "{title} ({year})", and song folders use "{artist} - {album} ({year})".
# Uses the same template variables as the file templates above.
#
# Scene-style folder:
# folder: '{title}.{year?}.{repack?}.{edition?}.{lang_tag?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}'
#
# Plex-friendly folder:
# folder: '{title} ({year?})'
#
# Per-title-type folder templates (optional). Override folder naming separately for
# movies, series, and songs. Useful when music libraries need artist/album-style folders
# while movies/series follow a different scheme. Any kind omitted falls back to the
# default for that title type.
#
# folder:
# movies: '{title} ({year})'
# series: '{title} ({year?})'
# songs: '{artist}/{album} ({year?})'
# Language-based tagging for output filenames
# Automatically adds language identifiers (e.g., DANiSH, NORDiC, DKsubs) based on
# audio and subtitle track languages. Rules are evaluated in order; first match wins.
# Use {lang_tag?} in your output_template to place the tag in the filename.
#
# Conditions (all conditions in a rule must match):
# audio: <lang> - any audio track matches this language
# subs_contain: <lang> - any subtitle matches this language
# subs_contain_all: [lang, ...] - subtitles include ALL listed languages
#
# language_tags:
# rules:
# - audio: da
# tag: DANiSH
# - audio: sv
# tag: SWEDiSH
# - audio: nb
# tag: NORWEGiAN
# - audio: en
# subs_contain_all: [da, sv, nb]
# tag: NORDiC
# - audio: en
# subs_contain: da
# tag: DKsubs
# Check for updates from GitHub repository on startup (default: true)
update_checks: true
# How often to check for updates, in hours (default: 24)
update_check_interval: 24
# Title caching configuration
# Cache title metadata to reduce redundant API calls
title_cache_enabled: true # Enable/disable title caching globally (default: true)
title_cache_time: 1800 # Cache duration in seconds (default: 1800 = 30 minutes)
title_cache_max_retention: 86400 # Maximum cache retention for fallback when API fails (default: 86400 = 24 hours)
# Filename Configuration
unicode_filenames: false # optionally replace non-ASCII characters with ASCII equivalents
# Debug logging configuration
# Comprehensive JSON-based debug logging for troubleshooting and service development
debug:
false # Enable structured JSON debug logging (default: false)
# When enabled with --debug flag or set to true:
# - Creates JSON Lines (.jsonl) log files with complete debugging context
# - Logs: session info, CLI params, service config, CDM details, authentication,
# titles, tracks metadata, DRM operations, vault queries, errors with stack traces
# - File location: logs/unshackle_debug_{service}_{timestamp}.jsonl
# - Also creates text log: logs/unshackle_root_{timestamp}.log
debug_keys:
false # Log decryption keys in debug logs (default: false)
# Set to true to include actual decryption keys in logs
# Useful for debugging key retrieval and decryption issues
# SECURITY NOTE: Passwords, tokens, cookies, and session tokens
# are ALWAYS redacted regardless of this setting
# Only affects: content_key, key fields (the actual CEKs)
# Never affects: kid, keys_count, key_id (metadata is always logged)
# Muxing configuration
muxing:
set_title: false
# merge_audio: Merge all audio tracks into each output file
# true (default): All selected audio in one MKV per quality
# false: Separate MKV per (quality, audio_codec) combination
# Example: Title.1080p.AAC.mkv, Title.1080p.EC3.mkv
merge_audio: true
# default_language: Override which track is flagged as the default in the muxed MKV.
# audio: BCP-47 tag of the preferred default audio track (e.g. pl, en, pt-BR).
# Wins over the title's original_language. Falls back to is_original_lang
# if no matching track is present.
# video: BCP-47 tag of the preferred default video track. Falls back to the
# original-language / first-track rule if no match is found.
# subtitle: BCP-47 tag of the preferred default subtitle track. Falls back to
# the existing rule (forced sub matching the audio language) if no
# matching subtitle is present.
# default_language:
# audio: pl
# video: pl
# subtitle: pl
# Login credentials for each Service
credentials:
# Direct credentials (no profile support)
EXAMPLE: email@example.com:password
# Per-profile credentials with default fallback
SERVICE_NAME:
default: default@email.com:password # Used when no -p/--profile is specified
profile1: user1@email.com:password1
profile2: user2@email.com:password2
# Per-profile credentials without default (requires -p/--profile)
SERVICE_NAME2:
john: john@example.com:johnspassword
jane: jane@example.com:janespassword
# You can also use list format for passwords with special characters
SERVICE_NAME3:
default: ["user@email.com", ":PasswordWith:Colons"]
# Override default directories used across unshackle
directories:
cache: Cache
cookies: Cookies
dcsl: DCSL # Device Certificate Status List
downloads: Downloads
logs: Logs
temp: Temp
wvds: WVDs
prds: PRDs
exports: Exports # JSON export output from --export flag
# Additional directories that can be configured:
# commands: Commands
services:
- /path/to/services
- /other/path/to/services
# vaults: Vaults
# fonts: Fonts
# Pre-define which Widevine or PlayReady device to use for each Service
cdm:
# Global default CDM device (fallback for all services/profiles)
default: WVD_1
# Direct service-specific CDM
DIFFERENT_EXAMPLE: PRD_1
# Per-profile CDM configuration
EXAMPLE:
john_sd: chromecdm_903_l3 # Profile 'john_sd' uses Chrome CDM L3
jane_uhd: nexus_5_l1 # Profile 'jane_uhd' uses Nexus 5 L1
default: generic_android_l3 # Default CDM for this service
# NEW: Quality-based CDM selection
# Use different CDMs based on video resolution
# Supports operators: >=, >, <=, <, or exact match
EXAMPLE_QUALITY:
"<=1080": generic_android_l3 # Use L3 for 1080p and below
">1080": nexus_5_l1 # Use L1 for above 1080p (1440p, 2160p)
default: generic_android_l3 # Optional: fallback if no quality match
# You can mix profiles and quality thresholds in the same service
NETFLIX:
# Profile-based selection (existing functionality)
john: netflix_l3_profile
jane: netflix_l1_profile
# Quality-based selection (new functionality)
"<=720": netflix_mobile_l3
"1080": netflix_standard_l3
">=1440": netflix_premium_l1
# Fallback
default: netflix_standard_l3
# Use pywidevine Serve-compliant Remote CDMs
# Example: Custom CDM API Configuration
# This demonstrates the highly configurable custom_api type that can adapt to any CDM API format
# - name: "chrome"
# type: "custom_api"
# host: "http://remotecdm.test/"
# timeout: 30
# device:
# name: "ChromeCDM"
# type: "CHROME"
# system_id: 34312
# security_level: 3
# auth:
# type: "header"
# header_name: "x-api-key"
# key: "YOUR_API_KEY_HERE"
# custom_headers:
# User-Agent: "Unshackle/2.0.0"
# endpoints:
# get_request:
# path: "/get-challenge"
# method: "POST"
# timeout: 30
# decrypt_response:
# path: "/get-keys"
# method: "POST"
# timeout: 30
# request_mapping:
# get_request:
# param_names:
# scheme: "device"
# init_data: "init_data"
# static_params:
# scheme: "Widevine"
# decrypt_response:
# param_names:
# scheme: "device"
# license_request: "license_request"
# license_response: "license_response"
# static_params:
# scheme: "Widevine"
# response_mapping:
# get_request:
# fields:
# challenge: "challenge"
# session_id: "session_id"
# message: "message"
# message_type: "message_type"
# response_types:
# - condition: "message_type == 'license-request'"
# type: "license_request"
# success_conditions:
# - "message == 'success'"
# decrypt_response:
# fields:
# keys: "keys"
# message: "message"
# key_fields:
# kid: "kid"
# key: "key"
# type: "type"
# success_conditions:
# - "message == 'success'"
# caching:
# enabled: true
# use_vaults: true
# check_cached_first: true
remote_cdm:
- name: "chrome"
device_name: chrome
device_type: CHROME
system_id: 27175
security_level: 3
host: https://domain.com/api
secret: secret_key
- name: "chrome-2"
device_name: chrome
device_type: CHROME
system_id: 26830
security_level: 3
host: https://domain-2.com/api
secret: secret_key
- name: "decrypt_labs_chrome"
type: "decrypt_labs" # Required to identify as DecryptLabs CDM
device_name: "ChromeCDM" # Scheme identifier - must match exactly
device_type: CHROME
system_id: 4464 # Doesn't matter
security_level: 3
host: "https://keyxtractor.decryptlabs.com"
secret: "your_decrypt_labs_api_key_here" # Replace with your API key
- name: "decrypt_labs_l1"
type: "decrypt_labs"
device_name: "L1" # Scheme identifier - must match exactly
device_type: ANDROID
system_id: 4464
security_level: 1
host: "https://keyxtractor.decryptlabs.com"
secret: "your_decrypt_labs_api_key_here"
- name: "decrypt_labs_l2"
type: "decrypt_labs"
device_name: "L2" # Scheme identifier - must match exactly
device_type: ANDROID
system_id: 4464
security_level: 2
host: "https://keyxtractor.decryptlabs.com"
secret: "your_decrypt_labs_api_key_here"
- name: "decrypt_labs_playready_sl2"
type: "decrypt_labs"
device_name: "SL2" # Scheme identifier - must match exactly
device_type: PLAYREADY
system_id: 0
security_level: 2000
host: "https://keyxtractor.decryptlabs.com"
secret: "your_decrypt_labs_api_key_here"
- name: "decrypt_labs_playready_sl3"
type: "decrypt_labs"
device_name: "SL3" # Scheme identifier - must match exactly
device_type: PLAYREADY
system_id: 0
security_level: 3000
host: "https://keyxtractor.decryptlabs.com"
secret: "your_decrypt_labs_api_key_here"
# PyPlayReady RemoteCdm - connects to an unshackle serve instance
- name: "playready_remote"
device_name: "my_prd_device" # Device name on the serve instance
device_type: PLAYREADY
system_id: 0
security_level: 3000 # 2000 for SL2000, 3000 for SL3000
host: "http://127.0.0.1:8786/playready" # Include /playready path
secret: "your-api-secret-key"
# Key Vaults store your obtained Content Encryption Keys (CEKs)
# Use 'no_push: true' to prevent a vault from receiving pushed keys
# while still allowing it to provide keys when requested
key_vaults:
- type: SQLite
name: Local
path: key_store.db
# Additional vault types:
# - type: API
# name: "Remote Vault"
# uri: "https://key-vault.example.com"
# token: "secret_token"
# no_push: true # This vault will only provide keys, not receive them
# - type: MySQL
# name: "MySQL Vault"
# host: "127.0.0.1"
# port: 3306
# database: vault
# username: user
# password: pass
# no_push: false # Default behavior - vault both provides and receives keys
# Choose what software to use to download data
downloader: requests
# Options: requests
# Downloading now uses the unified in-process requests/rnet downloader; the legacy
# aria2c, curl_impersonate, and n_m3u8dl_re backends have been removed.
# rnet TLS impersonation preset (not a downloader). Selects the browser
# fingerprint the HTTP session impersonates.
curl_impersonate:
browser: chrome120
# Pre-define default options and switches of the dl command
# Audio track selection preferences
audio:
# Codec priority order used as a tiebreaker when multiple audio tracks share the same
# bitrate and language. Listed codecs are ranked in the order given; codecs not in the
# list keep their bitrate-based ordering and are placed after all listed codecs.
# Atmos still trumps codec priority. Valid names: AAC, AC3, EC3, AC4, OPUS, OGG, DTS, ALAC, FLAC.
# codec_priority: [FLAC, ALAC, AC4, EC3, DTS, AC3, OPUS, AAC, OGG]
dl:
sub_format: srt
downloads: 4
workers: 16
lang:
- en
- fr
EXAMPLE:
bitrate: CBR
# Chapter Name to use when exporting a Chapter without a Name
chapter_fallback_name: "Chapter {j:02}"
# Case-Insensitive dictionary of headers for all Services
headers:
Accept-Language: "en-US,en;q=0.8"
User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"
# Override default filenames used across unshackle
filenames:
debug_log: "unshackle_debug_{service}_{time}.jsonl" # JSON Lines debug log file
config: "config.yaml"
root_config: "envied.yaml"
chapters: "Chapters_{title}_{random}.txt"
subtitle: "Subtitle_{id}_{language}.srt"
# conversion_method:
# - auto (default): Smart routing - subby for WebVTT/SAMI, pycaption for others
# - subby: Always use subby with advanced processing
# - pycaption: Use only pycaption library (no SubtitleEdit, no subby)
# - subtitleedit: Prefer SubtitleEdit when available, fall back to pycaption
# - pysubs2: Use pysubs2 library (supports SRT/SSA/ASS/WebVTT/TTML/SAMI/MicroDVD/MPL2/TMP)
subtitle:
conversion_method: auto
# sdh_method: Method to use for SDH (hearing impaired) stripping
# - auto (default): Try subby (SRT only), then SubtitleEdit (if available), then subtitle-filter
# - subby: Use subby library (SRT only)
# - subtitleedit: Use SubtitleEdit tool (Windows only, falls back to subtitle-filter)
# - filter-subs: Use subtitle-filter library directly
sdh_method: auto
# strip_sdh: Automatically create stripped (non-SDH) versions of SDH subtitles
# Set to false to disable automatic SDH stripping entirely (default: true)
strip_sdh: true
# convert_before_strip: Auto-convert VTT/other formats to SRT before using subtitle-filter
# This ensures compatibility when subtitle-filter is used as fallback (default: true)
convert_before_strip: true
# preserve_formatting: Preserve original subtitle formatting (tags, positioning, styling)
# When true, skips pycaption processing for WebVTT files to keep tags like <i>, <b>, positioning intact
# Combined with no sub_format setting, ensures subtitles remain in their original format (default: true)
preserve_formatting: true
# output_mode: Output mode for subtitles
# - mux: Embed subtitles in MKV container only (default)
# - sidecar: Save subtitles as separate files only
# - both: Embed in MKV AND save as sidecar files
output_mode: mux
# sidecar_format: Format for sidecar subtitle files
# Options: srt, vtt, ass, original (keep current format)
sidecar_format: srt
# Configuration for pywidevine and pyplayready's serve functionality
# Also used for remote services (unshackle serve)
serve:
api_secret: "your-secret-key-here"
# Compression level for API payloads (manifests, cache, cookies)
# 0=off, 1=fast, 6=balanced, 9=max compression (default: 1)
compression_level: 1
# Session inactivity timeout in seconds (default: 300 = 5 minutes)
# Sessions are automatically deleted after this many seconds of inactivity
# Each API request resets the timer
session_ttl: 300
# Maximum concurrent sessions before oldest is evicted (default: 100)
max_sessions: 100
# Global service allowlist (optional)
# Only these services will be exposed on the API. If omitted, all services are available.
# services:
# - SERVICE_TAG_1
# - SERVICE_TAG_2
users:
secret_key_for_user:
devices: # Widevine devices (WVDs) this user can access
- generic_nexus_4464_l3
playready_devices: # PlayReady devices (PRDs) this user can access
- playready_device_sl3000
username: user
# Per-user service allowlist (optional)
# Restricts this user to only the listed services. If omitted, user can access
# all globally-allowed services. Effective access is the intersection of global
# and per-user allowlists.
# services:
# - SERVICE_TAG_1
# devices: # Widevine device paths (auto-populated from directories.wvds)
# - '/path/to/device.wvd'
# playready_devices: # PlayReady device paths (auto-populated from directories.prds)
# - '/path/to/device.prd'
# Optional: any /api/download flag can be set here as a server-side default.
# Per-request body values still win. Useful for raising concurrency without
# changing every client call. Full list of accepted keys: see docs/API.md.
# downloads: 4 # parallel tracks per download job
# workers: 16 # threads per track segment fetch
# best_available: true
# no_proxy_download: false
# Remote Services Configuration
# Connect to a remote unshackle server (unshackle serve) to use its services
# without needing the service code locally. Use with: unshackle dl --remote
# If multiple servers are configured, specify which with: --server <name>
remote_services:
# Server name (used with --server flag if multiple configured)
my-server:
url: "http://192.168.1.100:8786"
api_key: "your-secret-key-here"
# Server-CDM mode: server handles all DRM licensing using its own CDM devices
# When false (default), client uses its own CDM and proxies license requests through the server
server_cdm: false
# Per-service overrides for remote services
# Override downloader, decryption tool, or CDM settings per service on the client
services:
# Example: Override the decryption tool for specific services
# EXAMPLE_SERVICE:
# decryption: mp4decrypt # Override decryption tool (shaka, mp4decrypt)
# Example: Multiple servers
# us-server:
# url: "https://us.example.com:8786"
# api_key: "us-api-key"
# server_cdm: true
# services:
# EXAMPLE:
# decryption: mp4decrypt
# eu-server:
# url: "https://eu.example.com:8786"
# api_key: "eu-api-key"
# server_cdm: false
# Configuration data for each Service
services:
# Service-specific configuration goes here
# Profile-specific configurations can be nested under service names
# You can override ANY global configuration option on a per-service basis
# This allows fine-tuned control for services with special requirements
# Supported overrides: dl, curl_impersonate, subtitle, muxing, headers, etc.
# Example: Comprehensive service configuration showing all features
EXAMPLE:
# Standard service config
api_key: "service_api_key"
# Service certificate for Widevine L1/L2 (base64 encoded)
# This certificate is automatically used when L1/L2 schemes are selected
# Services obtain this from their DRM provider or license server
certificate: |
CAUSwwUKvQIIAxIQ5US6QAvBDzfTtjb4tU/7QxiH8c+TBSKOAjCCAQoCggEBAObzvlu2hZRsapAPx4Aa4GUZj4/GjxgXUtBH4THSkM40x63wQeyVxlEEo
# ... (full base64 certificate here)
# Profile-specific device configurations
profiles:
john_sd:
device:
app_name: "AIV"
device_model: "SHIELD Android TV"
jane_uhd:
device:
app_name: "AIV"
device_model: "Fire TV Stick 4K"
# Service-specific proxy mappings
# Override global proxy selection with specific servers for this service
# When --proxy matches a key in proxy_map, the mapped server will be used
# instead of the default/random server selection
proxy_map:
nordvpn:ca: ca1577 # Use ca1577 when --proxy nordvpn:ca is specified
nordvpn:us: us9842 # Use us9842 when --proxy nordvpn:us is specified
us: 123 # Use server 123 (from any provider) when --proxy us is specified
gb: 456 # Use server 456 (from any provider) when --proxy gb is specified
# Without this service, --proxy nordvpn:ca picks a random CA server
# With this config, --proxy nordvpn:ca EXAMPLE uses ca1577 specifically
# Other services or no service specified will still use random selection
# NEW: Configuration overrides (can be combined with profiles and certificates)
# Override dl command defaults for this service
dl:
downloads: 4 # Limit concurrent track downloads (global default: 6)
workers: 8 # Reduce workers per track (global default: 16)
lang: ["en", "es-419"] # Different language priority for this service
sub_format: srt # Force SRT subtitle format
# Override subtitle processing for this service
subtitle:
conversion_method: pycaption # Use specific subtitle converter
sdh_method: auto
# Service-specific headers
headers:
User-Agent: "Service-specific user agent string"
Accept-Language: "en-US,en;q=0.9"
# Override muxing options
muxing:
set_title: true
# Remap service-provided titles before naming/output
# Keyed by the exact title the service returns -> desired output title.
title_map:
Service Title: Desired Title
# Example: Service with different regions per profile
SERVICE_NAME:
profiles:
us_account:
region: "US"
api_endpoint: "https://api.us.service.com"
uk_account:
region: "GB"
api_endpoint: "https://api.uk.service.com"
# Example: Rate-limited service
RATE_LIMITED_SERVICE:
dl:
downloads: 2 # Limit concurrent downloads
workers: 4 # Reduce workers to avoid rate limits
# Notes on service-specific overrides:
# - Overrides are merged with global config, not replaced
# - Only specified keys are overridden, others use global defaults
# - Reserved keys (profiles, api_key, certificate, etc.) are NOT treated as overrides
# - Any dict-type config option can be overridden (dl, subtitle, muxing, headers, etc.)
# - CLI arguments always take priority over service-specific config
# External proxy provider services
proxy_providers:
nordvpn:
username: username_from_service_credentials
password: password_from_service_credentials
# server_map: global mapping that applies to ALL services
# Difference from service-specific proxy_map:
# - server_map: applies to ALL services when --proxy nordvpn:us is used
# - proxy_map: only applies to the specific service configured (see services: EXAMPLE: proxy_map above)
# - proxy_map takes precedence over server_map for that service
server_map:
us: 12 # force US server #12 for US proxies
ca:calgary: 2534 # force CA server #2534 for Calgary proxies
us:seattle: 7890 # force US server #7890 for Seattle proxies
surfsharkvpn:
username: your_surfshark_service_username # Service credentials from https://my.surfshark.com/vpn/manual-setup/main/openvpn
password: your_surfshark_service_password # Service credentials (not your login password)
server_map:
us: 3844 # force US server #3844 for US proxies
gb: 2697 # force GB server #2697 for GB proxies
au: 4621 # force AU server #4621 for AU proxies
us:seattle: 5678 # force US server #5678 for Seattle proxies
ca:toronto: 1234 # force CA server #1234 for Toronto proxies
windscribevpn:
username: your_windscribe_username # Service credentials from https://windscribe.com/getconfig/openvpn
password: your_windscribe_password # Service credentials (not your login password)
server_map:
us: "us-central-096.totallyacdn.com" # force US server
gb: "uk-london-055.totallyacdn.com" # force GB server
us:seattle: "us-west-011.totallyacdn.com" # force US Seattle server
ca:toronto: "ca-toronto-012.totallyacdn.com" # force CA Toronto server
# Gluetun: Dynamic Docker-based VPN proxy (supports 50+ VPN providers)
# Creates Docker containers running Gluetun to bridge VPN connections to HTTP proxies
# Requires Docker to be installed and running
# Usage: --proxy gluetun:windscribe:us or --proxy gluetun:nordvpn:de
gluetun:
# Global settings
base_port: 8888 # Starting port for HTTP proxies (increments for each container)
auto_cleanup: true # Automatically remove containers when done
container_prefix: "unshackle-gluetun" # Docker container name prefix
verify_ip: true # Verify VPN IP matches expected region
# Optional HTTP proxy authentication (for the proxy itself, not VPN)
# auth_user: proxy_user
# auth_password: proxy_password
# VPN provider configurations
providers:
# Windscribe (WireGuard) - Get credentials from https://windscribe.com/getconfig/wireguard
windscribe:
vpn_type: wireguard
credentials:
private_key: "YOUR_WIREGUARD_PRIVATE_KEY"
addresses: "YOUR_WIREGUARD_ADDRESS" # e.g., "10.x.x.x/32"
# Map friendly names to country codes
server_countries:
us: US
uk: GB
ca: CA
de: DE
# NordVPN (OpenVPN) - Get service credentials from https://my.nordaccount.com/dashboard/nordvpn/manual-configuration/
# Note: Service credentials are NOT your email+password - generate them from the link above
# nordvpn:
# vpn_type: openvpn
# credentials:
# username: "YOUR_NORDVPN_SERVICE_USERNAME"
# password: "YOUR_NORDVPN_SERVICE_PASSWORD"
# server_countries:
# us: US
# uk: GB
# ExpressVPN (OpenVPN) - Get credentials from ExpressVPN setup page
# expressvpn:
# vpn_type: openvpn
# credentials:
# username: "YOUR_EXPRESSVPN_USERNAME"
# password: "YOUR_EXPRESSVPN_PASSWORD"
# server_countries:
# us: US
# uk: GB
# Surfshark (WireGuard) - Get credentials from https://my.surfshark.com/vpn/manual-setup/main/wireguard
# surfshark:
# vpn_type: wireguard
# credentials:
# private_key: "YOUR_SURFSHARK_PRIVATE_KEY"
# addresses: "YOUR_SURFSHARK_ADDRESS"
# server_countries:
# us: US
# uk: GB
# Specific server selection: Use format like "us1239" to select specific servers
# Example: --proxy gluetun:nordvpn:us1239 connects to us1239.nordvpn.com
# Supported providers: nordvpn, surfshark, expressvpn, cyberghost
basic:
GB:
- "socks5://username:password@bhx.socks.ipvanish.com:1080" # 1 (Birmingham)
- "socks5://username:password@gla.socks.ipvanish.com:1080" # 2 (Glasgow)
AU:
- "socks5://username:password@syd.socks.ipvanish.com:1080" # 1 (Sydney)
- "https://username:password@au-syd.prod.surfshark.com" # 2 (Sydney)
- "https://username:password@au-bne.prod.surfshark.com" # 3 (Brisbane)
BG: "https://username:password@bg-sof.prod.surfshark.com"
@@ -0,0 +1,321 @@
from __future__ import annotations
import base64
import hashlib
import hmac
import json
import re
import time
from collections.abc import Generator
from typing import Optional, Union
from urllib.parse import urlencode
import click
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
from envied.core.constants import AnyTrack
from envied.core.manifests import DASH
from envied.core.search_result import SearchResult
from envied.core.service import Service
from envied.core.titles import Episode, Series, Title_T, Titles_T
from envied.core.tracks import Chapters, Tracks
# NBC ships the page metadata AND the obfuscated config (which contains
# drmProxySecret) inline in every nbc.com page's HTML, both inside a
# `PRELOAD={...}` JS global. Layout we depend on:
# PRELOAD.pages[<url-path>].base — same shape as the legacy GraphQL response
# (replaces friendship.nbc.com/v3/graphql)
# PRELOAD.client.oc — base64-encoded encrypted config blob
#
# Obfuscated-config payload format (after base64-decode):
# bytes 0..12 : AES-GCM IV (12 bytes)
# bytes 12..44 : AES-256 key (32 bytes - the key is shipped next to the
# ciphertext, this is obfuscation, not security)
# bytes 44..-4 : AES-GCM ciphertext (includes 16-byte auth tag)
# bytes -4.. : COMPATIBILITY_VERSION (uint32 big-endian) — sanity check
# Decrypted plaintext is UTF-8 JSON. Currently exposes `{coreVideo: {drmProxySecret}}`.
_OC_COMPATIBILITY_VERSION = 1
class NBC(Service):
"""
\b
Service code for NBC.com (https://www.nbc.com).
\b
Version: 0.1.0
Authorization: None (free-tier content only TV-provider auth not implemented)
Robustness:
Widevine: L3
\b
Tips:
- Input may be either:
SERIES: https://www.nbc.com/<show-slug> (current season only)
EPISODE: https://www.nbc.com/<show-slug>/video/<episode-slug>/<id>
- Series URLs enumerate only the most recent season older seasons are
typically behind Peacock auth and not surfaced by the free-tier API.
- Content with active TV-provider entitlement windows will fail wait until the
episode's free window opens (typically ~7 days after first air).
"""
GEOFENCE = ("us",)
TITLE_RE = re.compile(
r"^https?://www\.nbc\.com/(?P<show>[a-zA-Z0-9_-]+)"
r"(?:/video/[a-zA-Z0-9_-]+/(?P<id>\d+))?/?$"
)
@staticmethod
@click.command(name="NBC", short_help="https://www.nbc.com", help=__doc__)
@click.argument("title", type=str, required=True)
@click.pass_context
def cli(ctx, **kwargs) -> NBC:
return NBC(ctx, **kwargs)
def __init__(self, ctx, title):
self.title = title
super().__init__(ctx)
# URL parsing happens lazily in get_titles() — search() accepts a free-text
# query that won't match TITLE_RE.
# Service API
def search(self) -> Generator[SearchResult, None, None]:
algolia = self.config["algolia"]
entity_types = ["series", "episodes", "movies"]
facet_filters = json.dumps([[f"algoliaProperties.entityType:{t}" for t in entity_types]])
algolia_params = urlencode({
"query": self.title,
"facetFilters": facet_filters,
"page": 0,
"hitsPerPage": 20,
})
body = {"requests": [{"indexName": algolia["index"], "params": algolia_params}]}
r = self.session.post(
algolia["url"],
headers={
**self.config["headers"],
"content-type": "application/x-www-form-urlencoded",
"x-algolia-api-key": algolia["api_key"],
"x-algolia-application-id": algolia["app_id"],
},
json=body,
)
r.raise_for_status()
for hit in r.json().get("results", [{}])[0].get("hits") or []:
entity_type = (hit.get("algoliaProperties") or {}).get("entityType")
if entity_type == "series":
series_data = hit.get("series") or {}
slug = series_data.get("seriesName") or series_data.get("urlAlias")
if not slug:
continue
yield SearchResult(
id_=hit.get("objectID") or slug,
title=series_data.get("shortTitle") or slug,
description=series_data.get("shortDescription"),
label="Series",
url=f"https://www.nbc.com/{slug}",
)
elif entity_type == "episodes":
ep_data = hit.get("episegment") or {}
video = hit.get("video") or {}
season = hit.get("season") or {}
series_data = hit.get("series") or {}
permalink = (video.get("permalink") or "").replace("http://", "https://")
if not permalink:
continue
season_n = season.get("seasonNumber")
episode_n = ep_data.get("episodeNumber")
label_bits = [series_data.get("shortTitle")]
if season_n is not None and episode_n is not None:
label_bits.append(f"S{season_n:02d}E{episode_n:02d}")
yield SearchResult(
id_=video.get("mpxGuid") or hit.get("objectID"),
title=ep_data.get("title") or "(untitled)",
description=ep_data.get("shortDescription"),
label=" · ".join(b for b in label_bits if b),
url=permalink,
)
def get_titles(self) -> Titles_T:
match = self.TITLE_RE.match(self.title)
if not match:
raise ValueError(f"Could not parse NBC URL: {self.title!r}")
show_slug = match.group("show")
mpx_guid = match.group("id")
self.show_slug = show_slug # cached for _episode_from_meta fallback
if mpx_guid:
return Series([self._episode_from_url()])
return Series(self._show(show_slug))
def get_tracks(self, title: Title_T) -> Tracks:
manifest_url = self._fetch_manifest_url(title)
return DASH.from_url(url=manifest_url).to_tracks(language=title.language)
def get_chapters(self, title: Episode) -> Chapters:
return Chapters()
def certificate(self, **_):
return None # use common privacy cert
def get_widevine_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[Union[bytes, str]]:
# hash = HMAC-SHA256(drm_proxy_secret, str(time_ms) + "widevine")
# The secret is extracted per-title from the obfuscated `oc` blob on the
# episode/show page HTML — see _fetch_page / _decrypt_oc.
secret = title.data.get("drm_proxy_secret")
if not secret:
raise ValueError(
"NBC: title has no drm_proxy_secret on its data — was it created outside of get_titles()?"
)
time_ms = str(int(time.time() * 1000))
url_hash = hmac.new(
secret.encode(), (time_ms + "widevine").encode(), hashlib.sha256
).hexdigest()
r = self.session.post(
self.config["endpoints"]["license_url"],
params={"time": time_ms, "hash": url_hash, "device": "web"},
headers={**self.config["headers"], "content-type": "application/octet-stream"},
data=challenge,
)
if not r.ok:
raise ConnectionError(f"NBC license request failed (HTTP {r.status_code}): {r.text[:200]}")
return r.content
# Service-specific helpers ---------------------------------------------------
def _episode_from_url(self) -> Episode:
# The page URL path after nbc.com/, e.g.
# "law-and-order-special-victims-unit/video/monster/9000448060"
path = re.match(r"https?://www\.nbc\.com/(.+?)/?$", self.title).group(1)
page, secret = self._fetch_page(path)
return self._episode_from_meta(page["metadata"], secret)
def _show(self, slug: str) -> list[Episode]:
# Show landing page returns the current season's episodes (older seasons are
# typically Peacock-only and don't appear in itemLabelsConfig here).
page, secret = self._fetch_page(slug)
episodes: list[Episode] = []
for section in page.get("data", {}).get("sections") or []:
if section.get("component") != "LinksSelectableGroup":
continue
section_data = section.get("data") or {}
if section_data.get("optionalTitle") != "Episodes":
continue
for shelf in section_data.get("items") or []:
for tile in (shelf.get("data") or {}).get("items") or []:
tile_data = tile.get("data") or {}
if tile_data.get("programmingType") != "Full Episode":
continue
episodes.append(self._episode_from_meta(tile_data, secret))
if not episodes:
raise ValueError(f"NBC: no episodes found for show {slug!r}")
return episodes
def _episode_from_meta(self, meta: dict, drm_proxy_secret: str) -> Episode:
"""Build an Episode from a VIDEO-page `metadata` dict or a VideoTile `data` dict.
Both shapes expose the same field set (mpxGuid, mpxAccountId, season/episode
numbers, secondaryTitle, seriesShortTitle, programmingType, duration, permalink).
"""
return Episode(
id_=meta["mpxGuid"],
title=meta.get("seriesShortTitle") or self.show_slug,
season=int(meta["seasonNumber"]) if meta.get("seasonNumber") else 0,
number=int(meta["episodeNumber"]) if meta.get("episodeNumber") else 0,
name=meta.get("secondaryTitle"),
language="en-US",
service=self.__class__,
data={
"mpxAccountId": meta["mpxAccountId"],
"mpxGuid": meta["mpxGuid"],
"programmingType": meta.get("programmingType", "Full Episode"),
"duration": meta.get("duration"),
"permalink": meta.get("permalink"),
"drm_proxy_secret": drm_proxy_secret,
},
)
def _fetch_page(self, url_path: str) -> tuple[dict, str]:
"""Fetch an nbc.com page and return (page_base, drm_proxy_secret).
`page_base` is the same dict shape that friendship.nbc.com used to return
as `data.page` (keys: metadata, analytics, data, ...).
"""
url = f"https://www.nbc.com/{url_path.lstrip('/')}"
r = self.session.get(url, headers=self.config["headers"])
if r.status_code != 200:
raise ConnectionError(f"NBC page {r.status_code}: {url}")
preload = self._extract_preload(r.text)
pages = preload.get("pages") or {}
# The URL we requested is the key; sometimes the path is normalised with
# a leading slash, sometimes without — match whichever key is present.
page = next(iter(pages.values()), None)
if not page or "base" not in page:
raise ValueError(f"NBC page {url}: PRELOAD has no pages[*].base")
oc_b64 = (preload.get("client") or {}).get("oc")
if not oc_b64:
raise ValueError(f"NBC page {url}: PRELOAD has no client.oc")
secret = self._decrypt_oc(oc_b64)["coreVideo"]["drmProxySecret"]
return page["base"], secret
@staticmethod
def _extract_preload(html: str) -> dict:
"""Locate `PRELOAD={...}` in the inline <script> and parse the JSON object.
The PRELOAD assignment is the only statement in its <script> tag, so we just
slice from `PRELOAD=` to the closing `</script>` and strip JS punctuation.
"""
marker = "PRELOAD="
start = html.find(marker + "{")
if start < 0:
raise ValueError("NBC: PRELOAD global not found in page HTML")
start += len(marker)
end = html.find("</script>", start)
if end < 0:
raise ValueError("NBC: unterminated <script> after PRELOAD")
# JSON parser tolerates leading/trailing whitespace but not a trailing semicolon.
return json.loads(html[start:end].strip().rstrip(";"))
def _decrypt_oc(self, oc_b64: str) -> dict:
"""AES-GCM-decrypt the obfuscated-config blob from PRELOAD.client.oc."""
raw = base64.b64decode(oc_b64)
if len(raw) <= 12 + 32 + 4:
raise ValueError(f"NBC: oc payload too small ({len(raw)} bytes)")
iv, key, ct, ver_bytes = raw[:12], raw[12:44], raw[44:-4], raw[-4:]
ver = int.from_bytes(ver_bytes, "big")
if ver != _OC_COMPATIBILITY_VERSION:
# Not fatal - payload still decrypts. Worth knowing about because the
# underlying layout may have shifted; if downstream parsing then fails
# this warning will be the first breadcrumb.
self.log.warning(
f"NBC: oc COMPATIBILITY_VERSION mismatch (got {ver}, expected {_OC_COMPATIBILITY_VERSION}); "
f"obfuscated-config layout may have changed"
)
plaintext = AESGCM(key).decrypt(iv, ct, None)
return json.loads(plaintext.decode("utf-8"))
def _fetch_manifest_url(self, title: Episode) -> str:
url = self.config["endpoints"]["lemonade_url"].format(
account=title.data["mpxAccountId"],
guid=title.data["mpxGuid"],
)
self.session.headers.update(self.config["headers"])
r = self.session.get(
url,
params={
"platform": "web",
"browser": "other",
"programmingType": title.data.get("programmingType", "Full Episode"),
},
)
if r.status_code != 200:
raise ConnectionError(f"NBC lemonade {r.status_code}: {r.text[:200]}")
playback = r.json()
manifest_url = playback.get("playbackUrl")
if not manifest_url:
raise ValueError(f"NBC lemonade returned no playbackUrl: {playback}")
# Match the in-browser request which appends locale flags to unlock all
# audio / subtitle / forced-narrative tracks in the DASH manifest.
sep = "&" if "?" in manifest_url else "?"
return f"{manifest_url}{sep}audio=all&subtitle=all&forcedNarrative=true"
@@ -0,0 +1,15 @@
headers:
user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36
origin: https://www.nbc.com
referer: https://www.nbc.com/
endpoints:
lemonade_url: https://lemonade.nbc.com/v1/vod/{account}/{guid}
license_url: https://drmproxy.digitalsvc.apps.nbcuni.com/drm-proxy/license/widevine
# NBC uses Algolia for search. The app_id/api_key are
algolia:
url: https://3nkvntt7f3-dsn.algolia.net/1/indexes/*/queries
app_id: 3NKVNTT7F3
api_key: c2df90d0ff616a2726139c671d6e6e8e
index: prod_multi-brand-unified-web
@@ -5,6 +5,7 @@ import concurrent.futures
import json
import re
import time
import uuid
from collections.abc import Generator
from http.cookiejar import MozillaCookieJar
from typing import Any, Optional
@@ -15,6 +16,7 @@ import jwt
from click import Context
from langcodes import Language
from lxml import etree
from rich.prompt import Prompt
from envied.core import __version__
from envied.core.cdm.detect import is_playready_cdm
from envied.core.config import config
@@ -32,9 +34,9 @@ class TVNZ(Service):
Service code for TVNZ streaming service (https://www.tvnz.co.nz).
\b
Version: 2.0.2
Version: 2.0.3
Author: stabbedbybrick
Authorization: tokens
Authorization: Credentials (email + OTP)
Robustness:
Widevine:
L3: 1080p, DDP5.1
@@ -53,14 +55,11 @@ class TVNZ(Service):
\b
Notes:
TVNZ has moved to an OTP-only login system, with no username/password and no cookies.
Auth sessions are stored in the browser's local storage, so they need to be extracted once
before being cached for future use.
There are many ways to extract it, but the easiest is with a browser extension, such as
"Cookie & Storage Exporter" or similar. Name the exported JSON file 'local_storage.json' and place it
in the `TwinVine/Cache/TVNZ` directory and it'll be added to cache on the next run.
Do note that the session can't be shared between browser and script, and will invalidate the other session
when tokens are refreshed. It's recommended to use a separate account for ripping purposes.
- TVNZ has moved to an OTP-only login system, with no username/password and no cookies.
On first run with a new profile, the OTP code will be sent to the email address listed in the config
and you will be prompted to enter it. This is only needed once, subsequent logins will use the cached tokens.
- Since there are no passwords, simply set the password as 'none' in the config so Unshackle
doesn't trip on incorrect formats: 'username:password' -> 'username:none'.
"""
GEOFENCE = ("nz",)
@@ -82,6 +81,20 @@ class TVNZ(Service):
self.profile = ctx.parent.params.get("profile") or "default"
self.session.headers.update(self.config["headers"])
# handle cache and OTP input before calling authenticate() to avoid glitchy terminal
self.credential = self.get_credentials(self.__class__.__name__, self.profile)
if not self.credential:
self.log.error(f" - No credentials found for profile: {self.profile}")
exit(1)
self.cached_tokens = self.cache.get(f"tokens_{self.credential.sha1}")
if not self.cached_tokens:
self.log.info(" - No cached user tokens found, setting up new login...")
self._create_otp(self.credential.username)
self.log.info(" + OTP code was sent to your email address")
self.otp_input = Prompt.ask("\tEnter OTP code")
def search(self) -> Generator[SearchResult, None, None]:
params = {
"mode": "detail",
@@ -90,8 +103,8 @@ class TVNZ(Service):
"pageNumber": "1",
"pageSize": "50",
"reg": "nz",
"dt": "web",
"client": "tvnz-tvnz-web",
"dt": "androidtv",
"client": "tvnz-tvnz-androidtv",
"pf": "Regular",
"allowpg": "true",
}
@@ -119,19 +132,41 @@ class TVNZ(Service):
def authenticate(self, cookies: Optional[MozillaCookieJar] = None, credential: Optional[Credential] = None) -> None:
super().authenticate(cookies, credential)
user_tokens, session_tokens = self._get_cached_tokens(self.profile)
if not self.cached_tokens:
if not self.otp_input:
raise ValueError("OTP code not provided")
# If cache is missing or invalid, fallback to local storage
if not user_tokens or not session_tokens:
user_tokens, session_tokens = self._fetch_and_cache_local_storage(self.profile)
otp = self.otp_input.replace(" ", "")
device_id = str(uuid.uuid4())
confirmation = self._confirm_otp(self.credential.username, otp, device_id)
if not (params := confirmation.get("params", [])):
raise ValueError("OTP response is missing auth params")
self.access_token = user_tokens["access_token"]
self.device_ref = user_tokens["deviceref"]
self.contact_id = user_tokens["contact_id"]
self.xauthorization = session_tokens["xauthorization"]
tokens = {p.get("paramName"): p.get("paramValue") for p in params}
tokens["contactID"] = confirmation.get("contactID")
tokens["deviceID"] = device_id
self.cached_tokens.set(tokens, expiration=int(tokens["expiresIn"]) - 3600)
else:
if not self.cached_tokens.expired:
self.log.info(" + Using cached user tokens")
tokens = self.cached_tokens.data
else:
self.log.info(" + Refreshing cached user tokens")
tokens = self.cached_tokens.data.copy()
refreshed_data = self._refresh_user_tokens(self.cached_tokens.data)
tokens.update(refreshed_data)
self.cached_tokens.set(tokens, expiration=int(tokens["expiresIn"]) - 3600)
self.access_token = tokens["accessToken"]
self.device_id = tokens["deviceID"]
self.contact_id = tokens["contactID"]
self.session.headers.update({"Authorization": f"Bearer {self.access_token}"})
self.xauthorization, _ = self._get_entitlements(self.contact_id)
self.oauth_token = self._get_oauth_token()
self.secret = self._register_app(self.oauth_token, self.xauthorization, self.device_ref)
self.secret = self._register_app(self.oauth_token, self.xauthorization, self.device_id)
def get_titles(self) -> Movies | Series:
match = re.match(self.TITLE_RE, self.title)
@@ -153,49 +188,33 @@ class TVNZ(Service):
raise ValueError(f"Unsupported content type: {content_type}")
def get_tracks(self, title: Movie | Episode) -> Tracks:
device_token = self._get_device_token(self.secret, self.device_ref)
device_token = self._get_device_token(self.secret, self.device_id)
headers = {
"accept": "*/*",
"accept-language": "en-US,en;q=0.9",
"authorization": f"Bearer {self.oauth_token}",
"content-type": "application/json",
"origin": "https://tvnz.co.nz",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36",
"x-authorization": f"{self.xauthorization}",
"x-client-id": "tvnz-tvnz-web",
"x-device-id": f"{device_token}",
"x-device-type": "web",
}
json_data = {
"deviceName": "web",
"deviceId": f"{self.device_ref}",
"deviceName": "lgwebostv" if self.drm_system == "playready" else "androidtv",
"deviceId": self.device_id,
"deviceManufacturer": "Android TV",
"deviceModelName": "Android TV",
"deviceOs": "Android",
"deviceOsVersion": "10",
"contentId": title.id,
"mediaFormat": "dash",
"contentTypeId": "vod",
"catalogType": title.data.get("cty"),
"mediaFormat": "dash",
"drm": self.drm_system,
"delivery": "streaming",
"quality": "high",
"disableSsai": "true",
"deviceManufacturer": "web",
"deviceModelName": "Chrome browser on Windows",
"deviceModelNumber": "Chrome",
"deviceOs": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36",
"supportedResolution": "UHD",
"supportedAudioCodecs": "mp4a",
"supportedVideoCodecs": "avc,hevc,av01",
"supportedMaxWVSecurityLevel": "L3",
"deviceToken": f"{device_token}",
"urlParameters": {
"vpa": "click",
"rdid": f"{self.device_ref}",
"is_lat": "0",
"npa": "0",
"idtype": "dpid",
"endpoint": "web",
"endpoint-group": "desktop",
"endpoint_detail": "desktop",
},
"supportedMaxWVSecurityLevel": "L1",
}
response = self.session.post(
@@ -208,7 +227,7 @@ class TVNZ(Service):
data = response.json()
if data.get("header", {}).get("message", "").lower() != "success":
raise ConnectionError(f"Failed to authorize playback: {data}")
title.data["license_url"] = data.get("data", {}).get("licenseUrl")
title.data["markers"] = title.data.get("mar")
@@ -232,7 +251,7 @@ class TVNZ(Service):
def get_chapters(self, title: Movie | Episode) -> Chapters:
if not (markers := title.data.get("markers")):
return Chapters()
chapters = []
for marker in markers:
if marker.get("t", "").lower() == "postplay":
@@ -246,34 +265,26 @@ class TVNZ(Service):
return sorted(chapters, key=lambda x: x.timestamp)
def get_widevine_service_certificate(self, *, challenge: bytes, title: Episode | Movie, track: Any) -> bytes | str | None:
def get_widevine_service_certificate(
self, *, challenge: bytes, title: Episode | Movie, track: Any
) -> bytes | str | None:
return None
def get_widevine_license(self, *, challenge: bytes, title: Episode | Movie, track: Any) -> bytes | str | None:
if not (license_url := title.data.get("license_url")):
return None
headers = {
'accept': '*/*',
'authorization': 'Bearer {}'.format(self.oauth_token),
'origin': 'https://tvnz.co.nz',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36',
}
headers = {"authorization": "Bearer {}".format(self.oauth_token)}
r = self.session.post(url=license_url, headers=headers, data=challenge)
r.raise_for_status()
return r.content
def get_playready_license(self, *, challenge: bytes, title: Episode | Movie, track: Any) -> bytes | str | None:
if not (license_url := title.data.get("license_url")):
return None
headers = {
'accept': '*/*',
'authorization': 'Bearer {}'.format(self.oauth_token),
'origin': 'https://tvnz.co.nz',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36',
}
headers = {"authorization": "Bearer {}".format(self.oauth_token)}
r = self.session.post(url=license_url, headers=headers, data=challenge)
r.raise_for_status()
@@ -292,8 +303,8 @@ class TVNZ(Service):
"sortBy": "epnum",
"sortOrder": "asc",
"reg": "nz",
"dt": "web",
"client": "tvnz-tvnz-web",
"dt": "androidtv",
"client": "tvnz-tvnz-androidtv",
"pf": "Regular",
"allowpg": "true",
},
@@ -301,11 +312,11 @@ class TVNZ(Service):
)
response.raise_for_status()
data = response.json()
if not (episodes := data.get("data")):
self.log.error(f"Failed to get episodes for season {season_id}")
return []
return [
Episode(
id_=episode.get("nu"),
@@ -329,8 +340,8 @@ class TVNZ(Service):
url=self.config["endpoints"]["catalog"] + title_path,
params={
"reg": "nz",
"dt": "web",
"client": "tvnz-tvnz-web",
"dt": "androidtv",
"client": "tvnz-tvnz-androidtv",
"pf": "Regular",
"allowpg": "true",
},
@@ -350,8 +361,8 @@ class TVNZ(Service):
"sortBy": "asc",
"sortOrder": "desc",
"reg": "nz",
"dt": "web",
"client": "tvnz-tvnz-web",
"dt": "androidtv",
"client": "tvnz-tvnz-androidtv",
"pf": "Regular",
"allowpg": "true",
},
@@ -362,17 +373,14 @@ class TVNZ(Service):
seasons = [x.get("id") for x in data["data"]]
if not seasons:
raise ValueError(f"Failed to get seasons: {data}")
all_episodes = []
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
futures = [
executor.submit(self._fetch_season_episodes, series_id, season)
for season in seasons
]
futures = [executor.submit(self._fetch_season_episodes, series_id, season) for season in seasons]
for future in futures:
all_episodes.extend(future.result())
return Series(all_episodes)
def _get_movie(self, title_path: str) -> Movies:
@@ -380,8 +388,8 @@ class TVNZ(Service):
url=self.config["endpoints"]["catalog"] + title_path,
params={
"reg": "nz",
"dt": "web",
"client": "tvnz-tvnz-web",
"dt": "androidtv",
"client": "tvnz-tvnz-androidtv",
"pf": "Regular",
"allowpg": "true",
},
@@ -411,8 +419,8 @@ class TVNZ(Service):
url=self.config["endpoints"]["catalog"] + title_path,
params={
"reg": "nz",
"dt": "web",
"client": "tvnz-tvnz-web",
"dt": "androidtv",
"client": "tvnz-tvnz-androidtv",
"pf": "Regular",
"allowpg": "true",
},
@@ -439,14 +447,14 @@ class TVNZ(Service):
]
return Series(episodes)
def _get_single(self, title_path: str) -> Movies:
response = self.session.get(
url=self.config["endpoints"]["catalog"] + title_path,
params={
"reg": "nz",
"dt": "web",
"client": "tvnz-tvnz-web",
"dt": "androidtv",
"client": "tvnz-tvnz-androidtv",
"pf": "Regular",
"allowpg": "true",
},
@@ -457,7 +465,7 @@ class TVNZ(Service):
if not (video := data.get("data")):
raise ValueError(f"Failed to get episode: {data}")
events = [
Movie(
id_=video.get("nu"),
@@ -470,8 +478,8 @@ class TVNZ(Service):
]
return Movies(events)
@staticmethod
@staticmethod
def _get_device_token(secret_b64: str, device_id: str) -> str:
secret_bytes = base64.b64decode(secret_b64)
@@ -479,22 +487,13 @@ class TVNZ(Service):
"deviceId": device_id,
"aud": "playback-auth-service",
"iat": int(time.time()),
"exp": int(time.time()) + 30
"exp": int(time.time()) + 30,
}
device_token = jwt.encode(payload, secret_bytes, algorithm="HS256")
return device_token
def _get_entitlements(self, access_token: str, contact_id: str):
headers = {
"accept": "application/json, text/plain, */*",
"accept-language": "en-US,en;q=0.9",
"authorization": f"Bearer {access_token}",
"content-type": "application/json",
"origin": "https://tvnz.co.nz",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36",
}
def _get_entitlements(self, contact_id: str):
json_data = {
"GetEntitlementsRequestMessage": {
"contactID": contact_id,
@@ -506,7 +505,6 @@ class TVNZ(Service):
response = self.session.post(
url=self.config["endpoints"]["entitlements"],
headers=headers,
json=json_data,
timeout=30,
)
@@ -514,50 +512,17 @@ class TVNZ(Service):
data = response.json()
if data.get("GetEntitlementsResponseMessage").get("message", "").lower() != "success":
raise ConnectionError(f"Failed to get entitlements: {data}")
token = data.get("GetEntitlementsResponseMessage").get("ovatToken")
expiry = data.get("GetEntitlementsResponseMessage").get("ovatTokenExpiry")
if not token:
raise ValueError(f"Failed to get entitlements: {data}")
return token, expiry
def _get_contact_id(self, access_token: str):
headers = {
"accept": "application/json, text/plain, */*",
"accept-language": "en-US,en;q=0.9",
"authorization": f"Bearer {access_token}",
"content-type": "application/json",
"origin": "https://tvnz.co.nz",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36",
}
json_data = {"GetContactRequestMessage": {**self.config["contact"]}}
response = self.session.post(
url=self.config["endpoints"]["contact"],
headers=headers,
json=json_data,
timeout=30,
)
response.raise_for_status()
data = response.json()
if data.get("GetContactResponseMessage", {}).get("message", "").lower() != "success":
raise ConnectionError(f"Failed to get contact: {data}")
return data["GetContactResponseMessage"]["contactMessage"][0]["contactID"]
def _get_oauth_token(self) -> str:
headers = {
"accept": "*/*",
"accept-language": "en-US,en;q=0.9",
"content-type": "application/x-www-form-urlencoded;charset=UTF-8",
"origin": "https://tvnz.co.nz",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36",
}
data = {
**self.config["web_client"],
**self.config["androidtv_client"],
"grant_type": "client_credentials",
"audience": "edge-service",
"scope": "offline openid",
@@ -565,7 +530,6 @@ class TVNZ(Service):
response = self.session.post(
url=self.config["endpoints"]["oauth"],
headers=headers,
data=data,
timeout=30,
)
@@ -577,22 +541,16 @@ class TVNZ(Service):
return token
def _register_app(self, oauth_token: str, xauth: str, deviceref: str) -> str:
def _register_app(self, oauth_token: str, xauth: str, device_id: str) -> str:
headers = {
"accept": "*/*",
"accept-language": "en-US,en;q=0.9",
"authorization": f"Bearer {oauth_token}",
"content-type": "text/plain;charset=UTF-8",
"origin": "https://tvnz.co.nz",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36",
"x-authorization": f"{xauth}",
"x-client-id": "tvnz-tvnz-web",
}
response = self.session.post(
url=self.config["endpoints"]["register"],
headers=headers,
data=json.dumps({"uniqueId": deviceref}),
data=json.dumps({"uniqueId": device_id}),
timeout=30,
)
response.raise_for_status()
@@ -600,28 +558,19 @@ class TVNZ(Service):
if not (secret := registration.get("data", {}).get("secret")):
raise ValueError(f"Failed to register app: {registration}")
return secret
def _refresh_user_tokens(self, tokens: dict) -> tuple[dict, int]:
headers = {
"accept": "application/json, text/plain, */*",
"accept-language": "en-US,en;q=0.9",
"content-type": "application/json",
"origin": "https://tvnz.co.nz",
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36",
}
return secret
def _refresh_user_tokens(self, tokens: dict) -> tuple[dict, int]:
json_data = {
"RefreshTokenRequestMessage": {
**self.config["contact"],
"refreshToken": tokens.get("refresh_token"),
"refreshToken": tokens.get("refreshToken"),
},
}
response = self.session.post(
url=self.config["endpoints"]["refresh"],
headers=headers,
json=json_data,
timeout=30,
)
@@ -629,104 +578,77 @@ class TVNZ(Service):
data = response.json()
if data.get("RefreshTokenResponseMessage", {}).get("message", "").lower() != "success":
raise ConnectionError(f"Failed to refresh user tokens: {data}")
access_token = data["RefreshTokenResponseMessage"]["accessToken"]
refresh_token = data["RefreshTokenResponseMessage"]["refreshToken"]
expiry = data["RefreshTokenResponseMessage"]["expiresIn"]
return {"access_token": access_token, "refresh_token": refresh_token}, expiry
def _refresh_session_tokens(self, tokens: dict) -> tuple[dict, int]:
xauthorization, xexpiry = self._get_entitlements(tokens.get("access_token"), tokens.get("contact_id"))
return {"x-authorization": xauthorization}, xexpiry
return data.get("RefreshTokenResponseMessage")
def _get_cached_tokens(self, profile: str) -> tuple[dict | None, dict | None]:
user_tokens = self.cache.get(f"{profile}_user_tokens")
session_tokens = self.cache.get(f"{profile}_session_tokens")
if not (user_tokens and session_tokens):
return None, None
if not user_tokens.expired:
self.log.info(" + Using cached user tokens")
ptokens = user_tokens.data
else:
self.log.info(" + Refreshing cached user tokens..")
ptokens, pexpiry = self._refresh_user_tokens(user_tokens.data)
ptokens["deviceref"] = user_tokens.data["deviceref"]
ptokens["contact_id"] = user_tokens.data["contact_id"]
user_tokens.set(ptokens, expiration=int(pexpiry) - 3600)
if not session_tokens.expired:
self.log.info(" + Using cached session tokens")
xtokens = session_tokens.data
else:
self.log.info(" + Refreshing cached session tokens..")
xtokens, xexpiry = self._refresh_session_tokens(session_tokens.data)
session_tokens.set(xtokens, expiration=int(xexpiry) - 3600)
return ptokens, xtokens
def _fetch_and_cache_local_storage(self, profile: str) -> tuple[dict, dict]:
self.log.info(" + Fetching tokens from local storage JSON..")
cache_dir = config.directories.cache / "TVNZ"
storage = next((
f for f in cache_dir.rglob("*.json")
if f.is_file() and any(t in f.name for t in ("localStorage", "local_storage"))
),None,)
if not storage:
raise EnvironmentError("'localStorage' not found. \nRun 'envied. dl TVNZ --help' for more information.")
try:
user = json.loads(storage.read_text())
except json.JSONDecodeError:
raise ValueError(f"'{storage}' is corrupted. \nRun 'envied. dl TVNZ --help' for more information.")
access_token = user.get("accessToken")
refresh_token = user.get("refreshToken")
device_ref = user.get("deviceref")
for token in (access_token, refresh_token, device_ref):
if not token:
raise ValueError(
f"Required token '{token}' is missing from '{storage}'. \nRun 'envied. dl TVNZ --help' for more information."
)
pexpiry = jwt.decode(access_token, options={"verify_signature": False}).get("exp")
contact_id = self._get_contact_id(access_token)
xauthorization, xexpiry = self._get_entitlements(access_token, contact_id)
ptokens = {
"access_token": access_token,
"refresh_token": refresh_token,
"deviceref": device_ref,
"contact_id": contact_id,
def _create_otp(self, email: str) -> None:
json_data = {
"CreateOTPRequestMessage": {
**self.config["contact"],
"email": email,
},
}
xtokens = {
"xauthorization": xauthorization,
response = self.session.post("https://rest-prod-tvnz.evergentpd.com/tvnz/createOTP", json=json_data).json()
response_message = response.get("CreateOTPResponseMessage", {})
if not response_message.get("isUserExist", False):
raise Exception(f"User with email {email} not found. Please check your credentials.")
if response_message.get("status", "").lower() != "success":
raise Exception(f"Failed to create OTP: {response}")
return
def _confirm_otp(self, email: str, otp: str, device_id: str) -> dict:
json_data = {
"ConfirmOTPRequestMessage": {
**self.config["contact"],
"email": email,
"canCreateAccount": True,
"checkDeviceLimit": True,
"dmaId": "001",
"otp": otp,
"isGenerateJWT": True,
"isPrivacyPoliciesAccepted": True,
"isTAndCAccepted": True,
"deviceDetails": {
"deviceType": "Android TV",
"deviceName": "androidtv",
"modelNo": "Android TV",
"appType": "Android",
"serialNo": device_id,
},
},
}
user_tokens = self.cache.get(f"{profile}_user_tokens")
session_tokens = self.cache.get(f"{profile}_session_tokens")
response = self.session.post("https://rest-prod-tvnz.evergentpd.com/tvnz/confirmOTP", json=json_data).json()
response_message = response.get("ConfirmOTPResponseMessage", {})
if response_message.get("status", "").lower() != "success":
raise Exception(f"Failed to confirm OTP: {response}")
user_tokens.set(ptokens, expiration=int(pexpiry) - 3600)
session_tokens.set(xtokens, expiration=int(xexpiry) - 3600)
return response.get("ConfirmOTPResponseMessage", {})
@staticmethod
def get_credentials(service: str, profile: Optional[str]) -> Optional[Credential]:
"""We need this method here to avoid circular imports."""
credentials = config.credentials.get(service)
if credentials:
if isinstance(credentials, dict):
if profile:
credentials = credentials.get(profile) or credentials.get("default")
else:
credentials = credentials.get("default")
if credentials:
if isinstance(credentials, list):
return Credential(*credentials)
return Credential.loads(credentials) # type: ignore
return ptokens, xtokens
def _modify_transfer(self, source_manifest: str) -> str:
"""
Change transfer type to "2" until dev branch is merged
"""
"""Change transfer type to "2" until dev branch is merged."""
manifest = DASH.from_url(source_manifest, self.session).manifest
periods = manifest.findall("Period")
for period in periods:
for adaptation_set in period.findall("AdaptationSet"):
for prop in adaptation_set.findall("SupplementalProperty"):
if (
prop is not None
and prop.get("schemeIdUri") == "urn:mpeg:mpegB:cicp:TransferCharacteristics"
):
if prop is not None and prop.get("schemeIdUri") == "urn:mpeg:mpegB:cicp:TransferCharacteristics":
prop.set("value", "2")
return etree.tostring(manifest, encoding="unicode")
@@ -1,6 +1,8 @@
headers:
User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
x-device-type: "androidtv"
x-app-store-type: "androidtv"
x-client-id: "tvnz-tvnz-androidtv"
user-agent: "Dalvik/2.1.0 (Linux; U; Android 11; Android TV Build/RTMA.250416.082)"
endpoints:
authorize: "https://watch-cdn.edge-api.tvnz.co.nz/media/content/authorize"
+137
View File
@@ -0,0 +1,137 @@
# Clone from https://github.com/keis/base58
"""Base58 encoding
Implementations of Base58 and Base58Check encodings that are compatible
with the bitcoin network.
"""
# This module is based upon base58 snippets found scattered over many bitcoin
# tools written in python. From what I gather the original source is from a
# forum post by Gavin Andresen, so direct your praise to him.
# This module adds shiny packaging and support for python3.
from functools import lru_cache
from hashlib import sha256
from typing import Mapping, Union
__version__ = "2.1.1"
# 58 character alphabet used
BITCOIN_ALPHABET = b"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
RIPPLE_ALPHABET = b"rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz"
XRP_ALPHABET = RIPPLE_ALPHABET
# Retro compatibility
alphabet = BITCOIN_ALPHABET
def scrub_input(v: Union[str, bytes]) -> bytes:
if isinstance(v, str):
v = v.encode("ascii")
return v
def b58encode_int(i: int, default_one: bool = True, alphabet: bytes = BITCOIN_ALPHABET) -> bytes:
"""
Encode an integer using Base58
"""
if not i and default_one:
return alphabet[0:1]
string = b""
base = len(alphabet)
while i:
i, idx = divmod(i, base)
string = alphabet[idx : idx + 1] + string
return string
def b58encode(v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET) -> bytes:
"""
Encode a string using Base58
"""
v = scrub_input(v)
origlen = len(v)
v = v.lstrip(b"\0")
newlen = len(v)
acc = int.from_bytes(v, byteorder="big") # first byte is most significant
result = b58encode_int(acc, default_one=False, alphabet=alphabet)
return alphabet[0:1] * (origlen - newlen) + result
@lru_cache()
def _get_base58_decode_map(alphabet: bytes, autofix: bool) -> Mapping[int, int]:
invmap = {char: index for index, char in enumerate(alphabet)}
if autofix:
groups = [b"0Oo", b"Il1"]
for group in groups:
pivots = [c for c in group if c in invmap]
if len(pivots) == 1:
for alternative in group:
invmap[alternative] = invmap[pivots[0]]
return invmap
def b58decode_int(v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET, *, autofix: bool = False) -> int:
"""
Decode a Base58 encoded string as an integer
"""
if b" " not in alphabet:
v = v.rstrip()
v = scrub_input(v)
map = _get_base58_decode_map(alphabet, autofix=autofix)
decimal = 0
base = len(alphabet)
try:
for char in v:
decimal = decimal * base + map[char]
except KeyError as e:
raise ValueError("Invalid character {!r}".format(chr(e.args[0]))) from None
return decimal
def b58decode(v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET, *, autofix: bool = False) -> bytes:
"""
Decode a Base58 encoded string
"""
v = v.rstrip()
v = scrub_input(v)
origlen = len(v)
v = v.lstrip(alphabet[0:1])
newlen = len(v)
acc = b58decode_int(v, alphabet=alphabet, autofix=autofix)
return acc.to_bytes(origlen - newlen + (acc.bit_length() + 7) // 8, "big")
def b58encode_check(v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET) -> bytes:
"""
Encode a string using Base58 with a 4 character checksum
"""
v = scrub_input(v)
digest = sha256(sha256(v).digest()).digest()
return b58encode(v + digest[:4], alphabet=alphabet)
def b58decode_check(v: Union[str, bytes], alphabet: bytes = BITCOIN_ALPHABET, *, autofix: bool = False) -> bytes:
"""Decode and verify the checksum of a Base58 encoded string"""
result = b58decode(v, alphabet=alphabet, autofix=autofix)
result, check = result[:-4], result[-4:]
digest = sha256(sha256(result).digest()).digest()
if check != digest[:4]:
raise ValueError("Invalid checksum")
return result
+5 -1
View File
@@ -208,7 +208,11 @@ class ConnectionFactory:
self._store = threading.local()
def _create_connection(self) -> Connection:
return sqlite3.connect(self._path)
conn = sqlite3.connect(self._path, timeout=30.0)
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA synchronous=NORMAL")
conn.execute("PRAGMA busy_timeout=30000")
return conn
def get(self) -> Connection:
if not hasattr(self._store, "conn"):
@@ -3,7 +3,7 @@ from vinefeeder.base_loader import BaseLoader
from vinefeeder.parsing_utils import split_options, list_prettify
from rich.console import Console
from beaupy import select_multiple
import json
console = Console()
# TV Shows https://www.tvnz.co.nz/shows/boiling-point/episodes/s1-e1 or
@@ -120,7 +120,8 @@ class TvnzLoader(BaseLoader):
The function will prepare the series data, matching the search term for display.
"""
# returns json as type String
url = f"https://apis-public-prod.tech.tvnz.co.nz/api/v1/web/play/search?q={search_term}"
#url = f"https://apis-public-prod.tech.tvnz.co.nz/api/v1/web/play/search?q={search_term}"
url = f"https://search-cdn.cms-api.tvnz.co.nz/content/search?mode=detail&st=published&term={search_term}&pageNumber=1&pageSize=50&reg=nz&dt=web&client=tvnz-tvnz-web&pf=regular&allowpg=false"
try:
html = self.get_data(url)
if "No Matches" in html:
@@ -131,23 +132,25 @@ class TvnzLoader(BaseLoader):
except Exception:
print(f"No valid data returned for {url}")
return
# console.print_json(data=parsed_data)
"""f = open('tvnz.json', 'w')
'''# console.print_json(data=parsed_data)
f = open('tvnz.json', 'w')
f.write(json.dumps(parsed_data)) # parsed_data)
f.close()"""
if parsed_data and "results" in parsed_data:
for item in parsed_data["results"]:
series_name = item.get("title", "Unknown Series")
url = "https://apis-edge-prod.tech.tvnz.co.nz" + item.get(
"page", {}
).get("href", "")
f.close()'''
if parsed_data and "Success" in parsed_data["header"]["message"]:
for item in parsed_data["data"]:
title = item.get("lon", [{"n": ""}])[0].get("n")
content_type = item.get("cty")
content_id = item.get("nu")
synopsis = item.get("losd", [{"n": ""}])[0].get("n")
url=f"https://tvnz.co.nz/{content_type}/{content_id}"
episode = {
"type": item.get("type"), # 'type'
"title": item.get("title", "Unknown Title"),
"type": content_type,
"title": title,
"url": url,
"synopsis": item.get("synopsis", "No synopsis available."),
"synopsis": synopsis,
}
self.add_episode(series_name, episode)
self.add_episode(title, episode)
else:
print(f"No valid data returned for {url}")
return None
@@ -181,7 +184,7 @@ class TvnzLoader(BaseLoader):
beaupylist = []
# if sport video / news video - assume no episodes
# use data from first fetch directly
if type == "sportVideo" or type == "newsVideo":
if type == "tvseries":
for item in episodes[selected]: # existing data
url = "https://www.tvnz.co.nz/" + item.get("url").split("/page/")[1]
beaupylist.append(
@@ -210,7 +213,7 @@ class TvnzLoader(BaseLoader):
self.runsubprocess(command)
return None
elif type == "show" or type == "showVideo":
elif type == "movie" or type == "tvseries":
url = f"https://apis-public-prod.tech.tvnz.co.nz/api/v1/web/play/page/shows/{series_name}/episodes"
try:
html = self.get_data(url)
@@ -219,7 +222,7 @@ class TvnzLoader(BaseLoader):
try:
# direct download as seems only one episode
# https://www.tvnz.co.nz/shows/circle-of-friends/movie/s1-e1
url = f"https://www.tvnz.co.nz/shows/{series_name}/movie/s1-e1"
url = f"https://www.tvnz.co.nz/{type}/{series_name}/s1-e1"
if self.options_list[0] == "":
command = ['uv', 'run', 'envied', "dl", "TVNZ", url]
else:
@@ -1,6 +1,6 @@
service: TVNZ
options:
options: --select-titles
media_dict:
Drama: https://apis-edge-prod.tech.tvnz.co.nz/api/v1/web/play/page/categories/drama