From fe49a5f0337503123c062498b4935c389c17465d Mon Sep 17 00:00:00 2001 From: VineFeeder Date: Mon, 16 Feb 2026 12:19:21 +0000 Subject: [PATCH] envied to 3.0.0 --- UPDATES.md | 5 + packages/envied/pyproject.toml | 9 +- packages/envied/pyproject.toml.orig | 120 - packages/envied/src/envied/commands/dl.bak | 2774 +++++++++++++++++ packages/envied/src/envied/commands/dl.py | 696 ++++- packages/envied/src/envied/commands/env.py | 8 + packages/envied/src/envied/commands/search.py | 9 +- packages/envied/src/envied/commands/serve.py | 187 +- packages/envied/src/envied/core/__init__.py | 2 +- .../src/envied/core/api/download_manager.py | 1 + .../envied/src/envied/core/api/handlers.py | 130 +- packages/envied/src/envied/core/api/routes.py | 2 +- packages/envied/src/envied/core/binaries.py | 8 + .../envied/src/envied/core/cdm/__init__.py | 59 +- .../core/cdm/decrypt_labs_remote_cdm.py | 12 +- packages/envied/src/envied/core/cdm/detect.py | 187 ++ .../src/envied/core/cdm/monalisa/__init__.py | 3 + .../envied/core/cdm/monalisa/monalisa_cdm.py | 417 +++ packages/envied/src/envied/core/config.py | 1 + .../src/envied/core/downloaders/aria2c.py | 518 +-- .../envied/core/downloaders/n_m3u8dl_re.py | 81 +- .../src/envied/core/downloaders/requests.py | 13 +- .../envied/src/envied/core/drm/__init__.py | 5 +- .../envied/src/envied/core/drm/monalisa.py | 299 ++ .../envied/src/envied/core/drm/playready.py | 4 +- .../envied/src/envied/core/manifests/dash.py | 75 +- .../envied/src/envied/core/manifests/hls.py | 621 ++-- .../envied/src/envied/core/manifests/ism.py | 75 +- .../src/envied/core/proxies/__init__.py | 3 +- .../envied/src/envied/core/proxies/gluetun.py | 1408 +++++++++ .../envied/src/envied/core/proxies/nordvpn.py | 70 +- .../src/envied/core/proxies/surfsharkvpn.py | 71 +- .../src/envied/core/proxies/windscribevpn.py | 100 +- packages/envied/src/envied/core/service.py | 98 +- packages/envied/src/envied/core/services.py | 10 +- .../envied/src/envied/core/titles/episode.py | 210 +- .../envied/src/envied/core/titles/movie.py | 184 +- .../envied/src/envied/core/titles/song.py | 33 +- .../src/envied/core/tracks/attachment.py | 68 +- .../envied/src/envied/core/tracks/hybrid.py | 4 +- .../envied/src/envied/core/tracks/subtitle.py | 115 +- .../envied/src/envied/core/tracks/track.py | 32 +- .../envied/src/envied/core/tracks/tracks.py | 95 +- .../envied/src/envied/core/tracks/video.py | 8 + packages/envied/src/envied/core/utilities.py | 75 + .../src/envied/core/utils/click_types.py | 49 + .../envied/src/envied/core/utils/selector.py | 269 ++ .../envied/src/envied/core/utils/webvtt.py | 13 + .../src/envied/services/DSNP/__init__.py | 536 ++-- .../src/envied/services/DSNP/config.yaml | 30 +- .../src/envied/services/DSNP/queries.py | 24 +- packages/vinefeeder/pyproject.toml | 1 + uv.lock | 58 +- 53 files changed, 8567 insertions(+), 1318 deletions(-) delete mode 100644 packages/envied/pyproject.toml.orig create mode 100644 packages/envied/src/envied/commands/dl.bak create mode 100644 packages/envied/src/envied/core/cdm/detect.py create mode 100644 packages/envied/src/envied/core/cdm/monalisa/__init__.py create mode 100644 packages/envied/src/envied/core/cdm/monalisa/monalisa_cdm.py create mode 100644 packages/envied/src/envied/core/drm/monalisa.py create mode 100644 packages/envied/src/envied/core/proxies/gluetun.py create mode 100644 packages/envied/src/envied/core/utils/selector.py diff --git a/UPDATES.md b/UPDATES.md index 388bdaa..a2e893c 100644 --- a/UPDATES.md +++ b/UPDATES.md @@ -1,5 +1,10 @@ **Updates** +**16 Feb 2026** + +- upstream envied to 3.0.0 + envied's --select-titles option now follows dev branch of unshackle. + **19 Jan 2026** - upstream envied to 2.3.0 diff --git a/packages/envied/pyproject.toml b/packages/envied/pyproject.toml index 1598cfa..0c8ef60 100644 --- a/packages/envied/pyproject.toml +++ b/packages/envied/pyproject.toml @@ -1,10 +1,10 @@ [build-system] -requires = ["uv_build>=0.8.12,<0.9.0"] +requires = ["uv_build>=0.8.12,<0.9.21"] build-backend = "uv_build" [project] name = "envied" -version = "2.3.0" +version = "3.0.0" description = "Modular Movie, TV, and Music Archival Software." authors = [{ name = "rlaphoenix and others" }] requires-python = ">=3.10,<3.13" @@ -66,11 +66,14 @@ dependencies = [ "webvtt-py>=0.5.1,<0.6", "isodate>=0.7.2,<0.8", - "beaupy>=3.10.2,<4", + #"beaupy>=3.10.2,<4", "scrapy>=2.14.0,<3", "mypy>=1.19.1,<2", "pysubs2>=1.7.0,<2", "PyExecJS>=1.5.1,<2", + "pycountry>=24.6.1", + "language-data>=1.4.0", + "wasmtime>=41.0.0", # Keeping your existing cap style here; latest visible is 0.10.0. "aiohttp-swagger3>=0.10.0,<1", diff --git a/packages/envied/pyproject.toml.orig b/packages/envied/pyproject.toml.orig deleted file mode 100644 index cc4278f..0000000 --- a/packages/envied/pyproject.toml.orig +++ /dev/null @@ -1,120 +0,0 @@ -[build-system] -requires = ["uv_build>=0.8.12,<0.9.0"] -build-backend = "uv_build" - -[project] -name = "envied" -version = "1.4.8" -description = "Modular Movie, TV, and Music Archival Software." -authors = [{ name = "rlaphoenix and others" }] -requires-python = ">=3.10,<3.13" -readme = "README.md" -license = "GPL-3.0-only" -keywords = [ - "python", - "downloader", - "drm", - "widevine", -] -classifiers = [ - "Development Status :: 4 - Beta", - "Environment :: Console", - "Intended Audience :: End Users/Desktop", - "Natural Language :: English", - "Operating System :: OS Independent", - "Topic :: Multimedia :: Video", - "Topic :: Security :: Cryptography", -] -dependencies = [ - "appdirs>=1.4.4,<2", - "Brotli>=1.1.0,<2", - "click>=8.1.8,<9", - "construct>=2.8.8,<3", - "crccheck>=1.3.0,<2", - "fonttools>=4.60.1,<5", - "jsonpickle>=3.0.4,<4.5", - "langcodes>=3.4.0,<4", - "lxml>=6.0.2, <7", - "pproxy>=2.7.9,<3", - "protobuf>=4.25.3,<7", - "pycaption>=2.2.6,<3", - "pycryptodomex>=3.20.0,<4", - "pyjwt>=2.8.0,<3", - "pymediainfo>=6.1.0,<8", - "pymp4>=1.4.0,<2", - "pymysql>=1.1.0,<2", - "pywidevine[serve]>=1.8.0,<2", - "PyYAML>=6.0.1,<7", - "requests[socks]>=2.31.0,<3", - "rich>=13.7.1,<15", - "rlaphoenix.m3u8>=3.4.0,<4", - "ruamel.yaml>=0.18.6,<0.19", - "sortedcontainers>=2.4.0,<3", - "subtitle-filter>=1.4.9,<2", - "Unidecode>=1.3.8,<2", - "urllib3>=2.2.1,<3", - "chardet>=5.2.0,<6", - "curl-cffi>=0.8.0b6,<0.16", - "pyplayready>=0.6.3,<0.7", - "httpx>=0.28.1,<0.29", - "cryptography>=45.0.0,<47", - "subby", - "webvtt-py>=0.5.1", - "isodate>=0.7.2", - "beaupy>=3.10.2", - "scrapy>=2.13.3", - "mypy>=1.18.2,<2", - "pysubs2>=1.7.0,<2", - "PyExecJS>=1.5.1,<2", - "jwt>=1.4.0", - "langcodes>=3.5.0", - "aiohttp-swagger3>=0.10.0, <1", -] - -[project.urls] -Homepage = "https://github.com/envied-dl/envied" -Repository = "https://github.com/envied-dl/envied" -Issues = "https://github.com/envied-dl/envied/issues" -Discussions = "https://github.com/envied-dl/envied/discussions" -Changelog = "https://github.com/envied-dl/envied/blob/master/CHANGELOG.md" - -[project.scripts] -envied = "envied.core.__main__:main" - -[dependency-groups] -dev = [ - "pre-commit>=3.7.0,<4", - "mypy>=1.18.2,<2", - "mypy-protobuf>=3.6.0,<4", - "types-protobuf>=4.24.0.20240408,<5", - "types-PyMySQL>=1.1.0.1,<2", - "types-requests>=2.31.0.20240406,<3", - "isort>=5.13.2,<6", - "ruff~=0.3.7" -] - -#[tool.hatch.build.targets.wheel] -#packages = ["envied"] - - -[tool.ruff] -force-exclude = true -line-length = 120 - -[tool.ruff.lint] -select = ["E4", "E7", "E9", "F", "W"] - -[tool.isort] -line_length = 118 - -[tool.mypy] -check_untyped_defs = true -disallow_incomplete_defs = true -disallow_untyped_defs = true -follow_imports = "silent" -ignore_missing_imports = true -no_implicit_optional = true - -[tool.uv.sources] -#subby = { git = "https://github.com/vevv/subby.git" } -subby = { git = "https://github.com/vevv/subby.git", rev = "5a925c367ffb3f5e53fd114ae222d3be1fdff35d" } diff --git a/packages/envied/src/envied/commands/dl.bak b/packages/envied/src/envied/commands/dl.bak new file mode 100644 index 0000000..479f137 --- /dev/null +++ b/packages/envied/src/envied/commands/dl.bak @@ -0,0 +1,2774 @@ +from __future__ import annotations + +import html +import logging +import math +import random +import re +import shutil +import subprocess +import sys +import time +from concurrent import futures +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +from functools import partial +from http.cookiejar import CookieJar, MozillaCookieJar +from itertools import product +from pathlib import Path +from threading import Lock +from typing import Any, Callable, Optional +from uuid import UUID + +import click +import jsonpickle +import yaml +from construct import ConstError +from pymediainfo import MediaInfo +from pyplayready.cdm import Cdm as PlayReadyCdm +from pyplayready.device import Device as PlayReadyDevice +from pyplayready.remote.remotecdm import RemoteCdm as PlayReadyRemoteCdm +from pywidevine.cdm import Cdm as WidevineCdm +from pywidevine.device import Device +from pywidevine.remotecdm import RemoteCdm +from rich.console import Group +from rich.live import Live +from rich.padding import Padding +from rich.panel import Panel +from rich.progress import BarColumn, Progress, SpinnerColumn, TaskID, TextColumn, TimeRemainingColumn +from rich.rule import Rule +from rich.table import Table +from rich.text import Text +from rich.tree import Tree + +from envied.core import binaries +from envied.core.cdm import CustomRemoteCDM, DecryptLabsRemoteCDM +from envied.core.cdm.detect import is_playready_cdm, is_widevine_cdm +from envied.core.config import config +from envied.core.console import console +from envied.core.constants import DOWNLOAD_LICENCE_ONLY, AnyTrack, context_settings +from envied.core.credential import Credential +from envied.core.drm import DRM_T, MonaLisa, PlayReady, Widevine +from envied.core.events import events +from envied.core.proxies import Basic, Gluetun, Hola, NordVPN, SurfsharkVPN, WindscribeVPN +from envied.core.service import Service +from envied.core.services import Services +from envied.core.title_cacher import get_account_hash +from envied.core.titles import Movie, Movies, Series, Song, Title_T +from envied.core.titles.episode import Episode +from envied.core.tracks import Audio, Subtitle, Tracks, Video +from envied.core.tracks.attachment import Attachment +from envied.core.tracks.hybrid import Hybrid +from envied.core.utilities import (find_font_with_fallbacks, get_debug_logger, get_system_fonts, init_debug_logger, + is_close_match, suggest_font_packages, time_elapsed_since) +from envied.core.utils import tags +from envied.core.utils.click_types import (AUDIO_CODEC_LIST, LANGUAGE_RANGE, QUALITY_LIST, SEASON_RANGE, + ContextData, MultipleChoice, SubtitleCodecChoice, VideoCodecChoice) +from envied.core.utils.collections import merge_dict +from envied.core.utils.subprocess import ffprobe +from envied.core.vaults import Vaults + + +class dl: + @staticmethod + def truncate_pssh_for_display(pssh_string: str, drm_type: str) -> str: + """Truncate PSSH string for display when not in debug mode.""" + if logging.root.level == logging.DEBUG or not pssh_string: + return pssh_string + + max_width = console.width - len(drm_type) - 12 + if len(pssh_string) <= max_width: + return pssh_string + + return pssh_string[: max_width - 3] + "..." + + def find_custom_font(self, font_name: str) -> Optional[Path]: + """ + Find font in custom fonts directory. + + Args: + font_name: Font family name to find + + Returns: + Path to font file, or None if not found + """ + family_dir = Path(config.directories.fonts, font_name) + if family_dir.exists(): + fonts = list(family_dir.glob("*.*tf")) + return fonts[0] if fonts else None + return None + + def prepare_temp_font( + self, font_name: str, matched_font: Path, system_fonts: dict[str, Path], temp_font_files: list[Path] + ) -> Path: + """ + Copy system font to temp and log if using fallback. + + Args: + font_name: Requested font name + matched_font: Path to matched system font + system_fonts: Dictionary of available system fonts + temp_font_files: List to track temp files for cleanup + + Returns: + Path to temp font file + """ + # Find the matched name for logging + matched_name = next((name for name, path in system_fonts.items() if path == matched_font), None) + + if matched_name and matched_name.lower() != font_name.lower(): + self.log.info(f"Using '{matched_name}' as fallback for '{font_name}'") + + # Create unique temp file path + safe_name = font_name.replace(" ", "_").replace("/", "_") + temp_path = config.directories.temp / f"font_{safe_name}{matched_font.suffix}" + + # Copy if not already exists + if not temp_path.exists(): + shutil.copy2(matched_font, temp_path) + temp_font_files.append(temp_path) + + return temp_path + + def attach_subtitle_fonts( + self, font_names: list[str], title: Title_T, temp_font_files: list[Path] + ) -> tuple[int, list[str]]: + """ + Attach fonts for subtitle rendering. + + Args: + font_names: List of font names requested by subtitles + title: Title object to attach fonts to + temp_font_files: List to track temp files for cleanup + + Returns: + Tuple of (fonts_attached_count, missing_fonts_list) + """ + system_fonts = get_system_fonts() + + font_count = 0 + missing_fonts = [] + + for font_name in set(font_names): + # Try custom fonts first + if custom_font := self.find_custom_font(font_name): + title.tracks.add(Attachment(path=custom_font, name=f"{font_name} ({custom_font.stem})")) + font_count += 1 + continue + + # Try system fonts with fallback + if system_font := find_font_with_fallbacks(font_name, system_fonts): + temp_path = self.prepare_temp_font(font_name, system_font, system_fonts, temp_font_files) + title.tracks.add(Attachment(path=temp_path, name=f"{font_name} ({system_font.stem})")) + font_count += 1 + else: + self.log.warning(f"Subtitle uses font '{font_name}' but it could not be found") + missing_fonts.append(font_name) + + return font_count, missing_fonts + + def suggest_missing_fonts(self, missing_fonts: list[str]) -> None: + """ + Show package installation suggestions for missing fonts. + + Args: + missing_fonts: List of font names that couldn't be found + """ + if suggestions := suggest_font_packages(missing_fonts): + self.log.info("Install font packages to improve subtitle rendering:") + for package_cmd, fonts in suggestions.items(): + self.log.info(f" $ sudo apt install {package_cmd}") + self.log.info(f" → Provides: {', '.join(fonts)}") + + def generate_sidecar_subtitle_path( + self, + subtitle: Subtitle, + base_filename: str, + output_dir: Path, + target_codec: Optional[Subtitle.Codec] = None, + source_path: Optional[Path] = None, + ) -> Path: + """Generate sidecar path: {base}.{lang}[.forced][.sdh].{ext}""" + lang_suffix = str(subtitle.language) if subtitle.language else "und" + forced_suffix = ".forced" if subtitle.forced else "" + sdh_suffix = ".sdh" if (subtitle.sdh or subtitle.cc) else "" + + extension = (target_codec or subtitle.codec or Subtitle.Codec.SubRip).extension + if ( + not target_codec + and not subtitle.codec + and source_path + and source_path.suffix + ): + extension = source_path.suffix.lstrip(".") + + filename = f"{base_filename}.{lang_suffix}{forced_suffix}{sdh_suffix}.{extension}" + return output_dir / filename + + def output_subtitle_sidecars( + self, + subtitles: list[Subtitle], + base_filename: str, + output_dir: Path, + sidecar_format: str, + original_paths: Optional[dict[str, Path]] = None, + ) -> list[Path]: + """Output subtitles as sidecar files, converting if needed.""" + created_paths: list[Path] = [] + config.directories.temp.mkdir(parents=True, exist_ok=True) + + for subtitle in subtitles: + source_path = subtitle.path + if sidecar_format == "original" and original_paths and subtitle.id in original_paths: + source_path = original_paths[subtitle.id] + + if not source_path or not source_path.exists(): + continue + + # Determine target codec + if sidecar_format == "original": + target_codec = None + if source_path.suffix: + try: + target_codec = Subtitle.Codec.from_mime(source_path.suffix.lstrip(".")) + except ValueError: + target_codec = None + else: + target_codec = Subtitle.Codec.from_mime(sidecar_format) + + sidecar_path = self.generate_sidecar_subtitle_path( + subtitle, base_filename, output_dir, target_codec, source_path=source_path + ) + + # Copy or convert + if not target_codec or subtitle.codec == target_codec: + shutil.copy2(source_path, sidecar_path) + else: + # Create temp copy for conversion to preserve original + temp_path = config.directories.temp / f"sidecar_{subtitle.id}{source_path.suffix}" + shutil.copy2(source_path, temp_path) + + temp_sub = Subtitle( + subtitle.url, + subtitle.language, + is_original_lang=subtitle.is_original_lang, + descriptor=subtitle.descriptor, + codec=subtitle.codec, + forced=subtitle.forced, + sdh=subtitle.sdh, + cc=subtitle.cc, + id_=f"{subtitle.id}_sc", + ) + temp_sub.path = temp_path + try: + temp_sub.convert(target_codec) + if temp_sub.path and temp_sub.path.exists(): + shutil.copy2(temp_sub.path, sidecar_path) + finally: + if temp_sub.path and temp_sub.path.exists(): + temp_sub.path.unlink(missing_ok=True) + temp_path.unlink(missing_ok=True) + + created_paths.append(sidecar_path) + + return created_paths + + @click.command( + short_help="Download, Decrypt, and Mux tracks for titles from a Service.", + cls=Services, + context_settings=dict(**context_settings, default_map=config.dl, token_normalize_func=Services.get_tag), + ) + @click.option( + "-p", "--profile", type=str, default=None, help="Profile to use for Credentials and Cookies (if available)." + ) + @click.option( + "-q", + "--quality", + type=QUALITY_LIST, + default=[], + help="Download Resolution(s), defaults to the best available resolution.", + ) + @click.option( + "-v", + "--vcodec", + type=VideoCodecChoice(Video.Codec), + default=None, + help="Video Codec to download, defaults to any codec.", + ) + @click.option( + "-a", + "--acodec", + type=AUDIO_CODEC_LIST, + default=[], + help="Audio Codec(s) to download (comma-separated), e.g., 'AAC,EC3'. Defaults to any.", + ) + @click.option( + "-vb", + "--vbitrate", + type=int, + default=None, + help="Video Bitrate to download (in kbps), defaults to highest available.", + ) + @click.option( + "-ab", + "--abitrate", + type=int, + default=None, + help="Audio Bitrate to download (in kbps), defaults to highest available.", + ) + @click.option( + "-r", + "--range", + "range_", + type=MultipleChoice(Video.Range, case_sensitive=False), + default=[Video.Range.SDR], + help="Video Color Range(s) to download, defaults to SDR.", + ) + @click.option( + "-c", + "--channels", + type=float, + default=None, + help="Audio Channel(s) to download. Matches sub-channel layouts like 5.1 with 6.0 implicitly.", + ) + @click.option( + "-naa", + "--noatmos", + "no_atmos", + is_flag=True, + default=False, + help="Exclude Dolby Atmos audio tracks when selecting audio.", + ) + @click.option( + "--split-audio", + "split_audio", + is_flag=True, + default=None, + help="Create separate output files per audio codec instead of merging all audio.", + ) + @click.option( + "-w", + "--wanted", + type=SEASON_RANGE, + default=None, + help="Wanted episodes, e.g. `S01-S05,S07`, `S01E01-S02E03`, `S02-S02E03`, e.t.c, defaults to all.", + ) + @click.option( + "-l", + "--lang", + type=LANGUAGE_RANGE, + default="orig", + help="Language wanted for Video and Audio. Use 'orig' to select the original language, e.g. 'orig,en' for both original and English.", + ) + @click.option( + "--latest-episode", + is_flag=True, + default=False, + help="Download only the single most recent episode available.", + ) + @click.option( + "-vl", + "--v-lang", + type=LANGUAGE_RANGE, + default=[], + help="Language wanted for Video, you would use this if the video language doesn't match the audio.", + ) + @click.option( + "-al", + "--a-lang", + type=LANGUAGE_RANGE, + default=[], + help="Language wanted for Audio, overrides -l/--lang for audio tracks.", + ) + @click.option("-sl", "--s-lang", type=LANGUAGE_RANGE, default=["all"], help="Language wanted for Subtitles.") + @click.option( + "--require-subs", + type=LANGUAGE_RANGE, + default=[], + help="Required subtitle languages. Downloads all subtitles only if these languages exist. Cannot be used with --s-lang.", + ) + @click.option("-fs", "--forced-subs", is_flag=True, default=False, help="Include forced subtitle tracks.") + @click.option( + "--exact-lang", + is_flag=True, + default=False, + help="Use exact language matching (no variants). With this flag, -l es-419 matches ONLY es-419, not es-ES or other variants.", + ) + @click.option( + "--proxy", + type=str, + default=None, + help="Proxy URI to use. If a 2-letter country is provided, it will try get a proxy from the config.", + ) + @click.option( + "--tag", type=str, default=None, help="Set the Group Tag to be used, overriding the one in config if any." + ) + @click.option( + "--tmdb", + "tmdb_id", + type=int, + default=None, + help="Use this TMDB ID for tagging instead of automatic lookup.", + ) + @click.option( + "--tmdb-name", + "tmdb_name", + is_flag=True, + default=False, + help="Rename titles using the name returned from TMDB lookup.", + ) + @click.option( + "--tmdb-year", + "tmdb_year", + is_flag=True, + default=False, + help="Use the release year from TMDB for naming and tagging.", + ) + @click.option( + "--sub-format", + type=SubtitleCodecChoice(Subtitle.Codec), + default=None, + help="Set Output Subtitle Format, only converting if necessary.", + ) + @click.option("-V", "--video-only", is_flag=True, default=False, help="Only download video tracks.") + @click.option("-A", "--audio-only", is_flag=True, default=False, help="Only download audio tracks.") + @click.option("-S", "--subs-only", is_flag=True, default=False, help="Only download subtitle tracks.") + @click.option("-C", "--chapters-only", is_flag=True, default=False, help="Only download chapters.") + @click.option("-ns", "--no-subs", is_flag=True, default=False, help="Do not download subtitle tracks.") + @click.option("-na", "--no-audio", is_flag=True, default=False, help="Do not download audio tracks.") + @click.option("-nc", "--no-chapters", is_flag=True, default=False, help="Do not download chapters tracks.") + @click.option("-nv", "--no-video", is_flag=True, default=False, help="Do not download video tracks.") + @click.option("-ad", "--audio-description", is_flag=True, default=False, help="Download audio description tracks.") + @click.option( + "--slow", + is_flag=True, + default=False, + help="Add a 60-120 second delay between each Title download to act more like a real device. " + "This is recommended if you are downloading high-risk titles or streams.", + ) + @click.option( + "--list", + "list_", + is_flag=True, + default=False, + help="Skip downloading and list available tracks and what tracks would have been downloaded.", + ) + @click.option( + "--list-titles", + is_flag=True, + default=False, + help="Skip downloading, only list available titles that would have been downloaded.", + ) + @click.option( + "--skip-dl", is_flag=True, default=False, help="Skip downloading while still retrieving the decryption keys." + ) + @click.option("--export", type=Path, help="Export Decryption Keys as you obtain them to a JSON file.") + @click.option( + "--cdm-only/--vaults-only", + is_flag=True, + default=None, + help="Only use CDM, or only use Key Vaults for retrieval of Decryption Keys.", + ) + @click.option("--no-proxy", is_flag=True, default=False, help="Force disable all proxy use.") + @click.option("--no-folder", is_flag=True, default=False, help="Disable folder creation for TV Shows.") + @click.option( + "--no-source", is_flag=True, default=False, help="Disable the source tag from the output file name and path." + ) + @click.option("--no-mux", is_flag=True, default=False, help="Do not mux tracks into a container file.") + @click.option( + "--workers", + type=int, + default=None, + help="Max workers/threads to download with per-track. Default depends on the downloader.", + ) + @click.option("--downloads", type=int, default=1, help="Amount of tracks to download concurrently.") + @click.option("--no-cache", "no_cache", is_flag=True, default=False, help="Bypass title cache for this download.") + @click.option( + "--reset-cache", "reset_cache", is_flag=True, default=False, help="Clear title cache before fetching." + ) + @click.option( + "--best-available", + "best_available", + is_flag=True, + default=False, + help="Continue with best available quality if requested resolutions are not available.", + ) + @click.pass_context + def cli(ctx: click.Context, **kwargs: Any) -> dl: + return dl(ctx, **kwargs) + + DRM_TABLE_LOCK = Lock() + + def __init__( + self, + ctx: click.Context, + no_proxy: bool, + profile: Optional[str] = None, + proxy: Optional[str] = None, + tag: Optional[str] = None, + tmdb_id: Optional[int] = None, + tmdb_name: bool = False, + tmdb_year: bool = False, + *_: Any, + **__: Any, + ): + if not ctx.invoked_subcommand: + raise ValueError("A subcommand to invoke was not specified, the main code cannot continue.") + + self.log = logging.getLogger("download") + + self.service = Services.get_tag(ctx.invoked_subcommand) + service_dl_config = config.services.get(self.service, {}).get("dl", {}) + if service_dl_config: + param_types = {param.name: param.type for param in ctx.command.params if param.name} + + for param_name, service_value in service_dl_config.items(): + if param_name not in ctx.params: + continue + + current_value = ctx.params[param_name] + global_default = config.dl.get(param_name) + param_type = param_types.get(param_name) + + try: + if param_type and global_default is not None: + global_default = param_type.convert(global_default, None, ctx) + except Exception as e: + self.log.debug(f"Failed to convert global default for '{param_name}': {e}") + + if current_value == global_default or (current_value is None and global_default is None): + try: + converted_value = service_value + if param_type and service_value is not None: + converted_value = param_type.convert(service_value, None, ctx) + + ctx.params[param_name] = converted_value + self.log.debug(f"Applied service-specific '{param_name}' override: {converted_value}") + except Exception as e: + self.log.warning( + f"Failed to apply service-specific '{param_name}' override: {e}. " + f"Check that the value '{service_value}' is valid for this parameter." + ) + + self.profile = profile + self.tmdb_id = tmdb_id + self.tmdb_name = tmdb_name + self.tmdb_year = tmdb_year + + # Initialize debug logger with service name if debug logging is enabled + if config.debug or logging.root.level == logging.DEBUG: + from collections import defaultdict + from datetime import datetime + + debug_log_path = config.directories.logs / config.filenames.debug_log.format_map( + defaultdict(str, service=self.service, time=datetime.now().strftime("%Y%m%d-%H%M%S")) + ) + init_debug_logger(log_path=debug_log_path, enabled=True, log_keys=config.debug_keys) + self.debug_logger = get_debug_logger() + + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="download_init", + message=f"Download command initialized for service {self.service}", + service=self.service, + context={ + "profile": profile, + "proxy": proxy, + "tag": tag, + "tmdb_id": tmdb_id, + "tmdb_name": tmdb_name, + "tmdb_year": tmdb_year, + "cli_params": { + k: v + for k, v in ctx.params.items() + if k not in ["profile", "proxy", "tag", "tmdb_id", "tmdb_name", "tmdb_year"] + }, + }, + ) + else: + self.debug_logger = None + + if self.profile: + self.log.info(f"Using profile: '{self.profile}'") + + with console.status("Loading Service Config...", spinner="dots"): + service_config_path = Services.get_path(self.service) / config.filenames.config + if service_config_path.exists(): + self.service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) + self.log.info("Service Config loaded") + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="load_service_config", + service=self.service, + context={"config_path": str(service_config_path), "config": self.service_config}, + ) + else: + self.service_config = {} + merge_dict(config.services.get(self.service), self.service_config) + + if getattr(config, "downloader_map", None): + config.downloader = config.downloader_map.get(self.service, config.downloader) + + if getattr(config, "decryption_map", None): + config.decryption = config.decryption_map.get(self.service, config.decryption) + + service_config = config.services.get(self.service, {}) + if service_config: + reserved_keys = { + "profiles", + "api_key", + "certificate", + "api_endpoint", + "region", + "device", + "endpoints", + "client", + "dl", + } + + for config_key, override_value in service_config.items(): + if config_key in reserved_keys or not isinstance(override_value, dict): + continue + + if hasattr(config, config_key): + current_config = getattr(config, config_key, {}) + + if isinstance(current_config, dict): + merged_config = deepcopy(current_config) + merge_dict(override_value, merged_config) + setattr(config, config_key, merged_config) + + self.log.debug( + f"Applied service-specific '{config_key}' overrides for {self.service}: {override_value}" + ) + + cdm_only = ctx.params.get("cdm_only") + + if cdm_only: + self.vaults = Vaults(self.service) + self.log.info("CDM-only mode: Skipping vault loading") + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="vault_loading_skipped", + service=self.service, + context={"reason": "cdm_only flag set"}, + ) + else: + with console.status("Loading Key Vaults...", spinner="dots"): + self.vaults = Vaults(self.service) + total_vaults = len(config.key_vaults) + failed_vaults = [] + + for vault in config.key_vaults: + vault_type = vault["type"] + vault_name = vault.get("name", vault_type) + vault_copy = vault.copy() + del vault_copy["type"] + + if vault_type.lower() == "api" and "decrypt_labs" in vault_name.lower(): + if "token" not in vault_copy or not vault_copy["token"]: + if config.decrypt_labs_api_key: + vault_copy["token"] = config.decrypt_labs_api_key + else: + self.log.warning( + f"No token provided for DecryptLabs vault '{vault_name}' and no global " + "decrypt_labs_api_key configured" + ) + + if vault_type.lower() == "sqlite": + try: + self.vaults.load_critical(vault_type, **vault_copy) + self.log.debug(f"Successfully loaded vault: {vault_name} ({vault_type})") + except Exception as e: + self.log.error(f"vault failure: {vault_name} ({vault_type}) - {e}") + raise + else: + # Other vaults (MySQL, HTTP, API) - soft fail + if not self.vaults.load(vault_type, **vault_copy): + failed_vaults.append(vault_name) + self.log.debug(f"Failed to load vault: {vault_name} ({vault_type})") + else: + self.log.debug(f"Successfully loaded vault: {vault_name} ({vault_type})") + + loaded_count = len(self.vaults) + if failed_vaults: + self.log.warning(f"Failed to load {len(failed_vaults)} vault(s): {', '.join(failed_vaults)}") + self.log.info(f"Loaded {loaded_count}/{total_vaults} Vaults") + + # Debug: Show detailed vault status + if loaded_count > 0: + vault_names = [vault.name for vault in self.vaults] + self.log.debug(f"Active vaults: {', '.join(vault_names)}") + else: + self.log.debug("No vaults are currently active") + + with console.status("Loading DRM CDM...", spinner="dots"): + try: + self.cdm = self.get_cdm(self.service, self.profile) + except ValueError as e: + self.log.error(f"Failed to load CDM, {e}") + if self.debug_logger: + self.debug_logger.log_error("load_cdm", e, service=self.service) + sys.exit(1) + + if self.cdm: + cdm_info = {} + if isinstance(self.cdm, DecryptLabsRemoteCDM): + drm_type = "PlayReady" if self.cdm.is_playready else "Widevine" + self.log.info(f"Loaded {drm_type} Remote CDM: DecryptLabs (L{self.cdm.security_level})") + cdm_info = {"type": "DecryptLabs", "drm_type": drm_type, "security_level": self.cdm.security_level} + elif hasattr(self.cdm, "device_type") and self.cdm.device_type.name in ["ANDROID", "CHROME"]: + self.log.info(f"Loaded Widevine CDM: {self.cdm.system_id} (L{self.cdm.security_level})") + cdm_info = { + "type": "Widevine", + "system_id": self.cdm.system_id, + "security_level": self.cdm.security_level, + "device_type": self.cdm.device_type.name, + } + else: + # Handle both local PlayReady CDM and RemoteCdm (which has certificate_chain=None) + is_remote = self.cdm.certificate_chain is None and hasattr(self.cdm, "device_name") + if is_remote: + cdm_name = self.cdm.device_name + self.log.info(f"Loaded PlayReady Remote CDM: {cdm_name} (L{self.cdm.security_level})") + else: + cdm_name = self.cdm.certificate_chain.get_name() if self.cdm.certificate_chain else "Unknown" + self.log.info(f"Loaded PlayReady CDM: {cdm_name} (L{self.cdm.security_level})") + cdm_info = { + "type": "PlayReady", + "certificate": cdm_name, + "security_level": self.cdm.security_level, + } + + if self.debug_logger and cdm_info: + self.debug_logger.log( + level="INFO", operation="load_cdm", service=self.service, context={"cdm": cdm_info} + ) + + self.proxy_providers = [] + if no_proxy: + ctx.params["proxy"] = None + else: + with console.status("Loading Proxy Providers...", spinner="dots"): + if config.proxy_providers.get("basic"): + self.proxy_providers.append(Basic(**config.proxy_providers["basic"])) + if config.proxy_providers.get("nordvpn"): + self.proxy_providers.append(NordVPN(**config.proxy_providers["nordvpn"])) + if config.proxy_providers.get("surfsharkvpn"): + self.proxy_providers.append(SurfsharkVPN(**config.proxy_providers["surfsharkvpn"])) + if config.proxy_providers.get("windscribevpn"): + self.proxy_providers.append(WindscribeVPN(**config.proxy_providers["windscribevpn"])) + if config.proxy_providers.get("gluetun"): + self.proxy_providers.append(Gluetun(**config.proxy_providers["gluetun"])) + if binaries.HolaProxy: + self.proxy_providers.append(Hola()) + for proxy_provider in self.proxy_providers: + self.log.info(f"Loaded {proxy_provider.__class__.__name__}: {proxy_provider}") + + if proxy: + requested_provider = None + if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE): + # requesting proxy from a specific proxy provider + requested_provider, proxy = proxy.split(":", maxsplit=1) + # Match simple region codes (us, ca, uk1) or provider:region format (nordvpn:ca, windscribe:us) + if re.match(r"^[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE) or re.match( + r"^[a-z]+:[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE + ): + proxy = proxy.lower() + # Preserve the original user query (region code) for service-specific proxy_map overrides. + # NOTE: `proxy` may be overwritten with the resolved proxy URI later. + proxy_query = proxy + status_msg = ( + f"Connecting to VPN ({proxy})..." + if requested_provider == "gluetun" + else f"Getting a Proxy to {proxy}..." + ) + with console.status(status_msg, spinner="dots"): + if requested_provider: + proxy_provider = next( + (x for x in self.proxy_providers if x.__class__.__name__.lower() == requested_provider), + None, + ) + if not proxy_provider: + self.log.error(f"The proxy provider '{requested_provider}' was not recognised.") + sys.exit(1) + proxy_uri = proxy_provider.get_proxy(proxy) + if not proxy_uri: + self.log.error(f"The proxy provider {requested_provider} had no proxy for {proxy}") + sys.exit(1) + proxy = ctx.params["proxy"] = proxy_uri + # Show connection info for Gluetun (IP, location) instead of proxy URL + if hasattr(proxy_provider, "get_connection_info"): + conn_info = proxy_provider.get_connection_info(proxy_query) + if conn_info and conn_info.get("public_ip"): + location_parts = [conn_info.get("city"), conn_info.get("country")] + location = ", ".join(p for p in location_parts if p) + self.log.info(f"VPN Connected: {conn_info['public_ip']} ({location})") + else: + self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") + else: + self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") + else: + for proxy_provider in self.proxy_providers: + proxy_uri = proxy_provider.get_proxy(proxy) + if proxy_uri: + proxy = ctx.params["proxy"] = proxy_uri + # Show connection info for Gluetun (IP, location) instead of proxy URL + if hasattr(proxy_provider, "get_connection_info"): + conn_info = proxy_provider.get_connection_info(proxy_query) + if conn_info and conn_info.get("public_ip"): + location_parts = [conn_info.get("city"), conn_info.get("country")] + location = ", ".join(p for p in location_parts if p) + self.log.info(f"VPN Connected: {conn_info['public_ip']} ({location})") + else: + self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") + else: + self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") + break + # Store proxy query info for service-specific overrides + ctx.params["proxy_query"] = proxy_query + ctx.params["proxy_provider"] = requested_provider + else: + self.log.info(f"Using explicit Proxy: {proxy}") + # For explicit proxies, store None for query/provider + ctx.params["proxy_query"] = None + ctx.params["proxy_provider"] = None + + ctx.obj = ContextData( + config=self.service_config, cdm=self.cdm, proxy_providers=self.proxy_providers, profile=self.profile + ) + + if tag: + config.tag = tag + + # needs to be added this way instead of @cli.result_callback to be + # able to keep `self` as the first positional + self.cli._result_callback = self.result + + def result( + self, + service: Service, + quality: list[int], + vcodec: Optional[Video.Codec], + acodec: list[Audio.Codec], + vbitrate: int, + abitrate: int, + range_: list[Video.Range], + channels: float, + no_atmos: bool, + wanted: list[str], + latest_episode: bool, + lang: list[str], + v_lang: list[str], + a_lang: list[str], + s_lang: list[str], + require_subs: list[str], + forced_subs: bool, + exact_lang: bool, + sub_format: Optional[Subtitle.Codec], + video_only: bool, + audio_only: bool, + subs_only: bool, + chapters_only: bool, + no_subs: bool, + no_audio: bool, + no_chapters: bool, + no_video: bool, + audio_description: bool, + slow: bool, + list_: bool, + list_titles: bool, + skip_dl: bool, + export: Optional[Path], + cdm_only: Optional[bool], + no_proxy: bool, + no_folder: bool, + no_source: bool, + no_mux: bool, + workers: Optional[int], + downloads: int, + best_available: bool, + split_audio: Optional[bool] = None, + *_: Any, + **__: Any, + ) -> None: + self.tmdb_searched = False + self.search_source = None + start_time = time.time() + + if skip_dl: + DOWNLOAD_LICENCE_ONLY.set() + + if not acodec: + acodec = [] + elif isinstance(acodec, Audio.Codec): + acodec = [acodec] + elif isinstance(acodec, str) or ( + isinstance(acodec, list) and not all(isinstance(v, Audio.Codec) for v in acodec) + ): + acodec = AUDIO_CODEC_LIST.convert(acodec) + + if require_subs and s_lang != ["all"]: + self.log.error("--require-subs and --s-lang cannot be used together") + sys.exit(1) + + # Check if dovi_tool is available when hybrid mode is requested + if any(r == Video.Range.HYBRID for r in range_): + from envied.core.binaries import DoviTool + + if not DoviTool: + self.log.error("Unable to run hybrid mode: dovi_tool not detected") + self.log.error("Please install dovi_tool from https://github.com/quietvoid/dovi_tool") + sys.exit(1) + + if cdm_only is None: + vaults_only = None + else: + vaults_only = not cdm_only + + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="drm_mode_config", + service=self.service, + context={ + "cdm_only": cdm_only, + "vaults_only": vaults_only, + "mode": "CDM only" if cdm_only else ("Vaults only" if vaults_only else "Both CDM and Vaults"), + }, + ) + + with console.status("Authenticating with Service...", spinner="dots"): + try: + cookies = self.get_cookie_jar(self.service, self.profile) + credential = self.get_credentials(self.service, self.profile) + service.authenticate(cookies, credential) + if cookies or credential: + self.log.info("Authenticated with Service") + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="authenticate", + service=self.service, + context={ + "has_cookies": bool(cookies), + "has_credentials": bool(credential), + "profile": self.profile, + }, + ) + except Exception as e: + if self.debug_logger: + self.debug_logger.log_error( + "authenticate", e, service=self.service, context={"profile": self.profile} + ) + raise + + with console.status("Fetching Title Metadata...", spinner="dots"): + try: + titles = service.get_titles_cached() + if not titles: + self.log.error("No titles returned, nothing to download...") + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="get_titles", + service=self.service, + message="No titles returned from service", + success=False, + ) + sys.exit(1) + except Exception as e: + if self.debug_logger: + self.debug_logger.log_error("get_titles", e, service=self.service) + raise + + if self.debug_logger: + titles_info = { + "type": titles.__class__.__name__, + "count": len(titles) if hasattr(titles, "__len__") else 1, + "title": str(titles), + } + if hasattr(titles, "seasons"): + titles_info["seasons"] = len(titles.seasons) if hasattr(titles, "seasons") else 0 + self.debug_logger.log( + level="INFO", operation="get_titles", service=self.service, context={"titles": titles_info} + ) + + title_cacher = service.title_cache if hasattr(service, "title_cache") else None + cache_title_id = None + if hasattr(service, "title"): + cache_title_id = service.title + elif hasattr(service, "title_id"): + cache_title_id = service.title_id + cache_region = service.current_region if hasattr(service, "current_region") else None + cache_account_hash = get_account_hash(service.credential) if hasattr(service, "credential") else None + + if (self.tmdb_year or self.tmdb_name) and self.tmdb_id: + sample_title = titles[0] if hasattr(titles, "__getitem__") else titles + kind = "tv" if isinstance(sample_title, Episode) else "movie" + + tmdb_year_val = None + tmdb_name_val = None + + if self.tmdb_year: + tmdb_year_val = tags.get_year( + self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash + ) + + if self.tmdb_name: + tmdb_name_val = tags.get_title( + self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash + ) + + if isinstance(titles, (Series, Movies)): + for t in titles: + if tmdb_year_val: + t.year = tmdb_year_val + if tmdb_name_val: + if isinstance(t, Episode): + t.title = tmdb_name_val + else: + t.name = tmdb_name_val + else: + if tmdb_year_val: + titles.year = tmdb_year_val + if tmdb_name_val: + if isinstance(titles, Episode): + titles.title = tmdb_name_val + else: + titles.name = tmdb_name_val + + console.print(Padding(Rule(f"[rule.text]{titles.__class__.__name__}: {titles}"), (1, 2))) + + console.print(Padding(titles.tree(verbose=list_titles), (0, 5))) + if list_titles: + return + + # Determine the latest episode if --latest-episode is set + latest_episode_id = None + if latest_episode and isinstance(titles, Series) and len(titles) > 0: + # Series is already sorted by (season, number, year) + # The last episode in the sorted list is the latest + latest_ep = titles[-1] + latest_episode_id = f"{latest_ep.season}x{latest_ep.number}" + self.log.info(f"Latest episode mode: Selecting S{latest_ep.season:02}E{latest_ep.number:02}") + + for i, title in enumerate(titles): + if isinstance(title, Episode) and latest_episode and latest_episode_id: + # If --latest-episode is set, only process the latest episode + if f"{title.season}x{title.number}" != latest_episode_id: + continue + elif isinstance(title, Episode) and wanted and f"{title.season}x{title.number}" not in wanted: + continue + + console.print(Padding(Rule(f"[rule.text]{title}"), (1, 2))) + temp_font_files = [] + + if isinstance(title, Episode) and not self.tmdb_searched: + kind = "tv" + if self.tmdb_id: + tmdb_title = tags.get_title( + self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash + ) + else: + self.tmdb_id, tmdb_title, self.search_source = tags.search_show_info( + title.title, title.year, kind, title_cacher, cache_title_id, cache_region, cache_account_hash + ) + if not (self.tmdb_id and tmdb_title and tags.fuzzy_match(tmdb_title, title.title)): + self.tmdb_id = None + if list_ or list_titles: + if self.tmdb_id: + console.print( + Padding( + f"Search -> {tmdb_title or '?'} [bright_black](ID {self.tmdb_id})", + (0, 5), + ) + ) + else: + console.print(Padding("Search -> [bright_black]No match found[/]", (0, 5))) + self.tmdb_searched = True + + if isinstance(title, Movie) and (list_ or list_titles) and not self.tmdb_id: + movie_id, movie_title, _ = tags.search_show_info( + title.name, title.year, "movie", title_cacher, cache_title_id, cache_region, cache_account_hash + ) + if movie_id: + console.print( + Padding( + f"Search -> {movie_title or '?'} [bright_black](ID {movie_id})", + (0, 5), + ) + ) + else: + console.print(Padding("Search -> [bright_black]No match found[/]", (0, 5))) + + if self.tmdb_id and getattr(self, "search_source", None) != "simkl": + kind = "tv" if isinstance(title, Episode) else "movie" + tags.external_ids(self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash) + + if slow and i != 0: + delay = random.randint(60, 120) + with console.status(f"Delaying by {delay} seconds..."): + time.sleep(delay) + + with console.status("Subscribing to events...", spinner="dots"): + events.reset() + events.subscribe(events.Types.SEGMENT_DOWNLOADED, service.on_segment_downloaded) + events.subscribe(events.Types.TRACK_DOWNLOADED, service.on_track_downloaded) + events.subscribe(events.Types.TRACK_DECRYPTED, service.on_track_decrypted) + events.subscribe(events.Types.TRACK_REPACKED, service.on_track_repacked) + events.subscribe(events.Types.TRACK_MULTIPLEX, service.on_track_multiplex) + + if hasattr(service, "NO_SUBTITLES") and service.NO_SUBTITLES: + console.log("Skipping subtitles - service does not support subtitle downloads") + no_subs = True + s_lang = None + title.tracks.subtitles = [] + elif no_subs: + console.log("Skipped subtitles as --no-subs was used...") + s_lang = None + title.tracks.subtitles = [] + + if no_video: + console.log("Skipped video as --no-video was used...") + v_lang = None + title.tracks.videos = [] + + with console.status("Getting tracks...", spinner="dots"): + try: + title.tracks.add(service.get_tracks(title), warn_only=True) + title.tracks.chapters = service.get_chapters(title) + except Exception as e: + if self.debug_logger: + self.debug_logger.log_error( + "get_tracks", e, service=self.service, context={"title": str(title)} + ) + raise + + if self.debug_logger: + tracks_info = { + "title": str(title), + "video_tracks": len(title.tracks.videos), + "audio_tracks": len(title.tracks.audio), + "subtitle_tracks": len(title.tracks.subtitles), + "has_chapters": bool(title.tracks.chapters), + "videos": [ + { + "codec": str(v.codec), + "resolution": f"{v.width}x{v.height}" if v.width and v.height else "unknown", + "bitrate": v.bitrate, + "range": str(v.range), + "language": str(v.language) if v.language else None, + "drm": [str(type(d).__name__) for d in v.drm] if v.drm else [], + } + for v in title.tracks.videos + ], + "audio": [ + { + "codec": str(a.codec), + "bitrate": a.bitrate, + "channels": a.channels, + "language": str(a.language) if a.language else None, + "descriptive": a.descriptive, + "drm": [str(type(d).__name__) for d in a.drm] if a.drm else [], + } + for a in title.tracks.audio + ], + "subtitles": [ + { + "codec": str(s.codec), + "language": str(s.language) if s.language else None, + "forced": s.forced, + "sdh": s.sdh, + } + for s in title.tracks.subtitles + ], + } + self.debug_logger.log( + level="INFO", operation="get_tracks", service=self.service, context=tracks_info + ) + + # strip SDH subs to non-SDH if no equivalent same-lang non-SDH is available + # uses a loose check, e.g, wont strip en-US SDH sub if a non-SDH en-GB is available + # Check if automatic SDH stripping is enabled in config + if config.subtitle.get("strip_sdh", True): + for subtitle in title.tracks.subtitles: + if subtitle.sdh and not any( + is_close_match(subtitle.language, [x.language]) + for x in title.tracks.subtitles + if not x.sdh and not x.forced + ): + non_sdh_sub = deepcopy(subtitle) + non_sdh_sub.id += "_stripped" + non_sdh_sub.sdh = False + title.tracks.add(non_sdh_sub) + events.subscribe( + events.Types.TRACK_MULTIPLEX, + lambda track, sub_id=non_sdh_sub.id: (track.strip_hearing_impaired()) + if track.id == sub_id + else None, + ) + + with console.status("Sorting tracks by language and bitrate...", spinner="dots"): + video_sort_lang = v_lang or lang + processed_video_sort_lang = [] + for language in video_sort_lang: + if language == "orig": + if title.language: + orig_lang = str(title.language) if hasattr(title.language, "__str__") else title.language + if orig_lang not in processed_video_sort_lang: + processed_video_sort_lang.append(orig_lang) + else: + if language not in processed_video_sort_lang: + processed_video_sort_lang.append(language) + + audio_sort_lang = a_lang or lang + processed_audio_sort_lang = [] + for language in audio_sort_lang: + if language == "orig": + if title.language: + orig_lang = str(title.language) if hasattr(title.language, "__str__") else title.language + if orig_lang not in processed_audio_sort_lang: + processed_audio_sort_lang.append(orig_lang) + else: + if language not in processed_audio_sort_lang: + processed_audio_sort_lang.append(language) + + title.tracks.sort_videos(by_language=processed_video_sort_lang) + title.tracks.sort_audio(by_language=processed_audio_sort_lang) + title.tracks.sort_subtitles(by_language=s_lang) + + if list_: + available_tracks, _ = title.tracks.tree() + console.print(Padding(Panel(available_tracks, title="Available Tracks"), (0, 5))) + continue + + with console.status("Selecting tracks...", spinner="dots"): + if isinstance(title, (Movie, Episode)): + # filter video tracks + if vcodec: + title.tracks.select_video(lambda x: x.codec == vcodec) + if not title.tracks.videos: + self.log.error(f"There's no {vcodec.name} Video Track...") + sys.exit(1) + + if range_: + # Special handling for HYBRID - don't filter, keep all HDR10 and DV tracks + if Video.Range.HYBRID not in range_: + title.tracks.select_video(lambda x: x.range in range_) + missing_ranges = [r for r in range_ if not any(x.range == r for x in title.tracks.videos)] + for color_range in missing_ranges: + self.log.warning(f"Skipping {color_range.name} video tracks as none are available.") + + if vbitrate: + title.tracks.select_video(lambda x: x.bitrate and x.bitrate // 1000 == vbitrate) + if not title.tracks.videos: + self.log.error(f"There's no {vbitrate}kbps Video Track...") + sys.exit(1) + + video_languages = [lang for lang in (v_lang or lang) if lang != "best"] + if video_languages and "all" not in video_languages: + processed_video_lang = [] + for language in video_languages: + if language == "orig": + if title.language: + orig_lang = ( + str(title.language) if hasattr(title.language, "__str__") else title.language + ) + if orig_lang not in processed_video_lang: + processed_video_lang.append(orig_lang) + else: + self.log.warning( + "Original language not available for title, skipping 'orig' selection for video" + ) + else: + if language not in processed_video_lang: + processed_video_lang.append(language) + title.tracks.videos = title.tracks.by_language( + title.tracks.videos, processed_video_lang, exact_match=exact_lang + ) + if not title.tracks.videos: + self.log.error(f"There's no {processed_video_lang} Video Track...") + sys.exit(1) + + if quality: + missing_resolutions = [] + if any(r == Video.Range.HYBRID for r in range_): + title.tracks.select_video(title.tracks.select_hybrid(title.tracks.videos, quality)) + else: + title.tracks.by_resolutions(quality) + + for resolution in quality: + if any(v.height == resolution for v in title.tracks.videos): + continue + if any(int(v.width * 9 / 16) == resolution for v in title.tracks.videos): + continue + missing_resolutions.append(resolution) + + if missing_resolutions: + res_list = "" + if len(missing_resolutions) > 1: + res_list = ", ".join([f"{x}p" for x in missing_resolutions[:-1]]) + " or " + res_list = f"{res_list}{missing_resolutions[-1]}p" + plural = "s" if len(missing_resolutions) > 1 else "" + + if best_available: + self.log.warning( + f"There's no {res_list} Video Track{plural}, continuing with available qualities..." + ) + else: + self.log.error(f"There's no {res_list} Video Track{plural}...") + sys.exit(1) + + # choose best track by range and quality + if any(r == Video.Range.HYBRID for r in range_): + # For hybrid mode, always apply hybrid selection + # If no quality specified, use only the best (highest) resolution + if not quality: + # Get the highest resolution available + best_resolution = max((v.height for v in title.tracks.videos), default=None) + if best_resolution: + # Use the hybrid selection logic with only the best resolution + title.tracks.select_video( + title.tracks.select_hybrid(title.tracks.videos, [best_resolution]) + ) + # If quality was specified, hybrid selection was already applied above + else: + selected_videos: list[Video] = [] + for resolution, color_range in product(quality or [None], range_ or [None]): + match = next( + ( + t + for t in title.tracks.videos + if ( + not resolution + or t.height == resolution + or int(t.width * (9 / 16)) == resolution + ) + and (not color_range or t.range == color_range) + ), + None, + ) + if match and match not in selected_videos: + selected_videos.append(match) + title.tracks.videos = selected_videos + + # validate hybrid mode requirements + if any(r == Video.Range.HYBRID for r in range_): + hdr10_tracks = [v for v in title.tracks.videos if v.range == Video.Range.HDR10] + dv_tracks = [v for v in title.tracks.videos if v.range == Video.Range.DV] + + if not hdr10_tracks and not dv_tracks: + available_ranges = sorted(set(v.range.name for v in title.tracks.videos)) + self.log.error("HYBRID mode requires both HDR10 and DV tracks, but neither is available") + self.log.error( + f"Available ranges: {', '.join(available_ranges) if available_ranges else 'none'}" + ) + sys.exit(1) + elif not hdr10_tracks: + available_ranges = sorted(set(v.range.name for v in title.tracks.videos)) + self.log.error("HYBRID mode requires both HDR10 and DV tracks, but only DV is available") + self.log.error(f"Available ranges: {', '.join(available_ranges)}") + sys.exit(1) + elif not dv_tracks: + available_ranges = sorted(set(v.range.name for v in title.tracks.videos)) + self.log.error("HYBRID mode requires both HDR10 and DV tracks, but only HDR10 is available") + self.log.error(f"Available ranges: {', '.join(available_ranges)}") + sys.exit(1) + + # filter subtitle tracks + if require_subs: + missing_langs = [ + lang + for lang in require_subs + if not any(is_close_match(lang, [sub.language]) for sub in title.tracks.subtitles) + ] + + if missing_langs: + self.log.error(f"Required subtitle language(s) not found: {', '.join(missing_langs)}") + sys.exit(1) + + self.log.info( + f"Required languages found ({', '.join(require_subs)}), downloading all available subtitles" + ) + elif s_lang and "all" not in s_lang: + from envied.core.utilities import is_exact_match + + match_func = is_exact_match if exact_lang else is_close_match + + missing_langs = [ + lang_ + for lang_ in s_lang + if not any(match_func(lang_, [sub.language]) for sub in title.tracks.subtitles) + ] + if missing_langs: + self.log.error(", ".join(missing_langs) + " not found in tracks") + sys.exit(1) + + title.tracks.select_subtitles(lambda x: match_func(x.language, s_lang)) + if not title.tracks.subtitles: + self.log.error(f"There's no {s_lang} Subtitle Track...") + sys.exit(1) + + if not forced_subs: + title.tracks.select_subtitles(lambda x: not x.forced) + + # filter audio tracks + # might have no audio tracks if part of the video, e.g. transport stream hls + if len(title.tracks.audio) > 0: + if not audio_description: + title.tracks.select_audio(lambda x: not x.descriptive) # exclude descriptive audio + if acodec: + title.tracks.select_audio(lambda x: x.codec in acodec) + if not title.tracks.audio: + codec_names = ", ".join(c.name for c in acodec) + self.log.error(f"No audio tracks matching codecs: {codec_names}") + sys.exit(1) + if channels: + title.tracks.select_audio(lambda x: math.ceil(x.channels) == math.ceil(channels)) + if not title.tracks.audio: + self.log.error(f"There's no {channels} Audio Track...") + sys.exit(1) + if no_atmos: + title.tracks.audio = [x for x in title.tracks.audio if not x.atmos] + if not title.tracks.audio: + self.log.error("No non-Atmos audio tracks available...") + sys.exit(1) + if abitrate: + title.tracks.select_audio(lambda x: x.bitrate and x.bitrate // 1000 == abitrate) + if not title.tracks.audio: + self.log.error(f"There's no {abitrate}kbps Audio Track...") + sys.exit(1) + audio_languages = a_lang or lang + if audio_languages: + processed_lang = [] + for language in audio_languages: + if language == "orig": + if title.language: + orig_lang = ( + str(title.language) if hasattr(title.language, "__str__") else title.language + ) + if orig_lang not in processed_lang: + processed_lang.append(orig_lang) + else: + self.log.warning( + "Original language not available for title, skipping 'orig' selection" + ) + else: + if language not in processed_lang: + processed_lang.append(language) + + if "best" in processed_lang: + unique_languages = {track.language for track in title.tracks.audio} + selected_audio = [] + for language in unique_languages: + codecs_to_check = acodec if (acodec and len(acodec) > 1) else [None] + for codec in codecs_to_check: + base_candidates = [ + t + for t in title.tracks.audio + if t.language == language and (codec is None or t.codec == codec) + ] + if not base_candidates: + continue + if audio_description: + standards = [t for t in base_candidates if not t.descriptive] + if standards: + selected_audio.append(max(standards, key=lambda x: x.bitrate or 0)) + descs = [t for t in base_candidates if t.descriptive] + if descs: + selected_audio.append(max(descs, key=lambda x: x.bitrate or 0)) + else: + selected_audio.append(max(base_candidates, key=lambda x: x.bitrate or 0)) + title.tracks.audio = selected_audio + elif "all" not in processed_lang: + # If multiple codecs were explicitly requested, pick the best track per codec per + # requested language instead of selecting *all* bitrate variants of a codec. + if acodec and len(acodec) > 1: + selected_audio: list[Audio] = [] + + for language in processed_lang: + for codec in acodec: + codec_tracks = [a for a in title.tracks.audio if a.codec == codec] + if not codec_tracks: + continue + + candidates = title.tracks.by_language( + codec_tracks, [language], per_language=0, exact_match=exact_lang + ) + if not candidates: + continue + + if audio_description: + standards = [t for t in candidates if not t.descriptive] + if standards: + selected_audio.append(max(standards, key=lambda x: x.bitrate or 0)) + descs = [t for t in candidates if t.descriptive] + if descs: + selected_audio.append(max(descs, key=lambda x: x.bitrate or 0)) + else: + selected_audio.append(max(candidates, key=lambda x: x.bitrate or 0)) + + title.tracks.audio = selected_audio + else: + per_language = 1 + if audio_description: + standard_audio = [a for a in title.tracks.audio if not a.descriptive] + selected_standards = title.tracks.by_language( + standard_audio, processed_lang, per_language=per_language, exact_match=exact_lang + ) + desc_audio = [a for a in title.tracks.audio if a.descriptive] + # Include all descriptive tracks for the requested languages. + selected_descs = title.tracks.by_language( + desc_audio, processed_lang, per_language=0, exact_match=exact_lang + ) + title.tracks.audio = selected_standards + selected_descs + else: + title.tracks.audio = title.tracks.by_language( + title.tracks.audio, + processed_lang, + per_language=per_language, + exact_match=exact_lang, + ) + if not title.tracks.audio: + self.log.error(f"There's no {processed_lang} Audio Track, cannot continue...") + sys.exit(1) + + if ( + video_only + or audio_only + or subs_only + or chapters_only + or no_subs + or no_audio + or no_chapters + or no_video + ): + keep_videos = False + keep_audio = False + keep_subtitles = False + keep_chapters = False + + if video_only or audio_only or subs_only or chapters_only: + if video_only: + keep_videos = True + if audio_only: + keep_audio = True + if subs_only: + keep_subtitles = True + if chapters_only: + keep_chapters = True + else: + keep_videos = True + keep_audio = True + keep_subtitles = True + keep_chapters = True + + if no_subs: + keep_subtitles = False + if no_audio: + keep_audio = False + if no_chapters: + keep_chapters = False + if no_video: + keep_videos = False + + kept_tracks = [] + if keep_videos: + kept_tracks.extend(title.tracks.videos) + if keep_audio: + kept_tracks.extend(title.tracks.audio) + if keep_subtitles: + kept_tracks.extend(title.tracks.subtitles) + if keep_chapters: + kept_tracks.extend(title.tracks.chapters) + kept_tracks.extend(title.tracks.attachments) + + title.tracks = Tracks(kept_tracks) + + selected_tracks, tracks_progress_callables = title.tracks.tree(add_progress=True) + + for track in title.tracks: + if hasattr(track, "needs_drm_loading") and track.needs_drm_loading: + track.load_drm_if_needed(service) + + download_table = Table.grid() + download_table.add_row(selected_tracks) + + video_tracks = title.tracks.videos + if video_tracks: + highest_quality = max((track.height for track in video_tracks if track.height), default=0) + if highest_quality > 0: + if is_widevine_cdm(self.cdm): + quality_based_cdm = self.get_cdm( + self.service, self.profile, drm="widevine", quality=highest_quality + ) + if quality_based_cdm and quality_based_cdm != self.cdm: + self.log.debug( + f"Pre-selecting Widevine CDM based on highest quality {highest_quality}p across all video tracks" + ) + self.cdm = quality_based_cdm + elif is_playready_cdm(self.cdm): + quality_based_cdm = self.get_cdm( + self.service, self.profile, drm="playready", quality=highest_quality + ) + if quality_based_cdm and quality_based_cdm != self.cdm: + self.log.debug( + f"Pre-selecting PlayReady CDM based on highest quality {highest_quality}p across all video tracks" + ) + self.cdm = quality_based_cdm + + dl_start_time = time.time() + + try: + with Live(Padding(download_table, (1, 5)), console=console, refresh_per_second=5): + with ThreadPoolExecutor(downloads) as pool: + for download in futures.as_completed( + ( + pool.submit( + track.download, + session=service.session, + prepare_drm=partial( + partial(self.prepare_drm, table=download_table), + track=track, + title=title, + certificate=partial( + service.get_widevine_service_certificate, + title=title, + track=track, + ), + licence=partial( + service.get_playready_license + if ( + is_playready_cdm(self.cdm) + ) + and hasattr(service, "get_playready_license") + else service.get_widevine_license, + title=title, + track=track, + ), + cdm_only=cdm_only, + vaults_only=vaults_only, + export=export, + ), + cdm=self.cdm, + max_workers=workers, + progress=tracks_progress_callables[i], + ) + for i, track in enumerate(title.tracks) + ) + ): + download.result() + + except KeyboardInterrupt: + console.print(Padding(":x: Download Cancelled...", (0, 5, 1, 5))) + if self.debug_logger: + self.debug_logger.log( + level="WARNING", + operation="download_tracks", + service=self.service, + message="Download cancelled by user", + context={"title": str(title)}, + ) + return + except Exception as e: # noqa + error_messages = [ + ":x: Download Failed...", + ] + if isinstance(e, EnvironmentError): + error_messages.append(f" {e}") + if isinstance(e, ValueError): + error_messages.append(f" {e}") + if isinstance(e, (AttributeError, TypeError)): + console.print_exception() + else: + error_messages.append( + " An unexpected error occurred in one of the download workers.", + ) + if hasattr(e, "returncode"): + error_messages.append(f" Binary call failed, Process exit code: {e.returncode}") + error_messages.append(" See the error trace above for more information.") + if isinstance(e, subprocess.CalledProcessError): + # CalledProcessError already lists the exception trace + console.print_exception() + console.print(Padding(Group(*error_messages), (1, 5))) + + if self.debug_logger: + self.debug_logger.log_error( + "download_tracks", + e, + service=self.service, + context={ + "title": str(title), + "error_type": type(e).__name__, + "tracks_count": len(title.tracks), + "returncode": getattr(e, "returncode", None), + }, + ) + return + + if skip_dl: + console.log("Skipped downloads as --skip-dl was used...") + else: + dl_time = time_elapsed_since(dl_start_time) + console.print(Padding(f"Track downloads finished in [progress.elapsed]{dl_time}[/]", (0, 5))) + + video_track_n = 0 + + while ( + not title.tracks.subtitles + and not no_subs + and not (hasattr(service, "NO_SUBTITLES") and service.NO_SUBTITLES) + and not video_only + and not no_video + and len(title.tracks.videos) > video_track_n + and any( + x.get("codec_name", "").startswith("eia_") + for x in ffprobe(title.tracks.videos[video_track_n].path).get("streams", []) + ) + ): + with console.status(f"Checking Video track {video_track_n + 1} for Closed Captions..."): + try: + # TODO: Figure out the real language, it might be different + # EIA-CC tracks sadly don't carry language information :( + # TODO: Figure out if the CC language is original lang or not. + # Will need to figure out above first to do so. + video_track = title.tracks.videos[video_track_n] + track_id = f"ccextractor-{video_track.id}" + cc_lang = title.language or video_track.language + cc = video_track.ccextractor( + track_id=track_id, + out_path=config.directories.temp + / config.filenames.subtitle.format(id=track_id, language=cc_lang), + language=cc_lang, + original=False, + ) + if cc: + # will not appear in track listings as it's added after all times it lists + title.tracks.add(cc) + self.log.info(f"Extracted a Closed Caption from Video track {video_track_n + 1}") + else: + self.log.info(f"No Closed Captions were found in Video track {video_track_n + 1}") + except EnvironmentError: + self.log.error( + "Cannot extract Closed Captions as the ccextractor executable was not found..." + ) + break + video_track_n += 1 + + # Subtitle output mode configuration (for sidecar originals) + subtitle_output_mode = config.subtitle.get("output_mode", "mux") + sidecar_format = config.subtitle.get("sidecar_format", "srt") + skip_subtitle_mux = ( + subtitle_output_mode == "sidecar" and (title.tracks.videos or title.tracks.audio) + ) + sidecar_subtitles: list[Subtitle] = [] + sidecar_original_paths: dict[str, Path] = {} + if subtitle_output_mode in ("sidecar", "both") and not no_mux: + sidecar_subtitles = [s for s in title.tracks.subtitles if s.path and s.path.exists()] + if sidecar_format == "original": + config.directories.temp.mkdir(parents=True, exist_ok=True) + for subtitle in sidecar_subtitles: + original_path = ( + config.directories.temp / f"sidecar_original_{subtitle.id}{subtitle.path.suffix}" + ) + shutil.copy2(subtitle.path, original_path) + sidecar_original_paths[subtitle.id] = original_path + + with console.status("Converting Subtitles..."): + for subtitle in title.tracks.subtitles: + if sub_format: + if subtitle.codec != sub_format: + subtitle.convert(sub_format) + elif subtitle.codec == Subtitle.Codec.TimedTextMarkupLang: + # MKV does not support TTML, VTT is the next best option + subtitle.convert(Subtitle.Codec.WebVTT) + + with console.status("Checking Subtitles for Fonts..."): + font_names = [] + for subtitle in title.tracks.subtitles: + if subtitle.codec == Subtitle.Codec.SubStationAlphav4: + for line in subtitle.path.read_text("utf8").splitlines(): + if line.startswith("Style: "): + font_names.append(line.removeprefix("Style: ").split(",")[1].strip()) + + font_count, missing_fonts = self.attach_subtitle_fonts(font_names, title, temp_font_files) + + if font_count: + self.log.info(f"Attached {font_count} fonts for the Subtitles") + + if missing_fonts and sys.platform != "win32": + self.suggest_missing_fonts(missing_fonts) + + # Handle DRM decryption BEFORE repacking (must decrypt first!) + service_name = service.__class__.__name__.upper() + decryption_method = config.decryption_map.get(service_name, config.decryption) + decrypt_tool = "mp4decrypt" if decryption_method.lower() == "mp4decrypt" else "Shaka Packager" + + drm_tracks = [track for track in title.tracks if track.drm] + if drm_tracks: + with console.status(f"Decrypting tracks with {decrypt_tool}..."): + has_decrypted = False + for track in drm_tracks: + drm = track.get_drm_for_cdm(self.cdm) + if drm and hasattr(drm, "decrypt"): + drm.decrypt(track.path) + if not isinstance(drm, MonaLisa): + has_decrypted = True + events.emit(events.Types.TRACK_REPACKED, track=track) + else: + self.log.warning( + f"No matching DRM found for track {track} with CDM type {type(self.cdm).__name__}" + ) + if has_decrypted: + self.log.info(f"Decrypted tracks with {decrypt_tool}") + + # Now repack the decrypted tracks + with console.status("Repackaging tracks with FFMPEG..."): + has_repacked = False + for track in title.tracks: + if track.needs_repack: + track.repackage() + has_repacked = True + events.emit(events.Types.TRACK_REPACKED, track=track) + if has_repacked: + # we don't want to fill up the log with "Repacked x track" + self.log.info("Repacked one or more tracks with FFMPEG") + + muxed_paths = [] + muxed_audio_codecs: dict[Path, Optional[Audio.Codec]] = {} + append_audio_codec_suffix = True + + if no_mux: + # Skip muxing, handle individual track files + for track in title.tracks: + if track.path and track.path.exists(): + muxed_paths.append(track.path) + elif isinstance(title, (Movie, Episode)): + progress = Progress( + TextColumn("[progress.description]{task.description}"), + SpinnerColumn(finished_text=""), + BarColumn(), + "•", + TimeRemainingColumn(compact=True, elapsed_when_finished=True), + console=console, + ) + + merge_audio = ( + (not split_audio) if split_audio is not None else config.muxing.get("merge_audio", True) + ) + # When we split audio (merge_audio=False), multiple outputs may exist per title, so suffix codec. + append_audio_codec_suffix = not merge_audio + + multiplex_tasks: list[tuple[TaskID, Tracks, Optional[Audio.Codec]]] = [] + # Track hybrid-processing outputs explicitly so we can always clean them up, + # even if muxing fails early (e.g. SystemExit) before the normal delete loop. + hybrid_temp_paths: list[Path] = [] + + def clone_tracks_for_audio(base_tracks: Tracks, audio_tracks: list[Audio]) -> Tracks: + task_tracks = Tracks() + task_tracks.videos = list(base_tracks.videos) + task_tracks.audio = audio_tracks + task_tracks.subtitles = list(base_tracks.subtitles) + task_tracks.chapters = base_tracks.chapters + task_tracks.attachments = list(base_tracks.attachments) + return task_tracks + + def enqueue_mux_tasks(task_description: str, base_tracks: Tracks) -> None: + if merge_audio or not base_tracks.audio: + task_id = progress.add_task(f"{task_description}...", total=None, start=False) + multiplex_tasks.append((task_id, base_tracks, None)) + return + + audio_by_codec: dict[Optional[Audio.Codec], list[Audio]] = {} + for audio_track in base_tracks.audio: + audio_by_codec.setdefault(audio_track.codec, []).append(audio_track) + + for audio_codec, codec_audio_tracks in audio_by_codec.items(): + description = task_description + if audio_codec: + description = f"{task_description} {audio_codec.name}" + + task_id = progress.add_task(f"{description}...", total=None, start=False) + task_tracks = clone_tracks_for_audio(base_tracks, codec_audio_tracks) + multiplex_tasks.append((task_id, task_tracks, audio_codec)) + + # Check if we're in hybrid mode + if any(r == Video.Range.HYBRID for r in range_) and title.tracks.videos: + # Hybrid mode: process DV and HDR10 tracks separately for each resolution + self.log.info("Processing Hybrid HDR10+DV tracks...") + + # Group video tracks by resolution + resolutions_processed = set() + hdr10_tracks = [v for v in title.tracks.videos if v.range == Video.Range.HDR10] + dv_tracks = [v for v in title.tracks.videos if v.range == Video.Range.DV] + + for hdr10_track in hdr10_tracks: + resolution = hdr10_track.height + if resolution in resolutions_processed: + continue + resolutions_processed.add(resolution) + + # Find matching DV track for this resolution (use the lowest DV resolution) + matching_dv = min(dv_tracks, key=lambda v: v.height) if dv_tracks else None + + if matching_dv: + # Create track pair for this resolution + resolution_tracks = [hdr10_track, matching_dv] + + for track in resolution_tracks: + track.needs_duration_fix = True + + # Run the hybrid processing for this resolution + Hybrid(resolution_tracks, self.service) + + # Create unique output filename for this resolution + hybrid_filename = f"HDR10-DV-{resolution}p.hevc" + hybrid_output_path = config.directories.temp / hybrid_filename + hybrid_temp_paths.append(hybrid_output_path) + + # The Hybrid class creates HDR10-DV.hevc, rename it for this resolution + default_output = config.directories.temp / "HDR10-DV.hevc" + if default_output.exists(): + # If a previous run left this behind, replace it to avoid move() failures. + hybrid_output_path.unlink(missing_ok=True) + shutil.move(str(default_output), str(hybrid_output_path)) + + # Create tracks with the hybrid video output for this resolution + task_description = f"Multiplexing Hybrid HDR10+DV {resolution}p" + task_tracks = Tracks(title.tracks) + title.tracks.chapters + title.tracks.attachments + + # Create a new video track for the hybrid output + hybrid_track = deepcopy(hdr10_track) + hybrid_track.id = f"hybrid_{hdr10_track.id}_{resolution}" + hybrid_track.path = hybrid_output_path + hybrid_track.range = Video.Range.DV # It's now a DV track + hybrid_track.needs_duration_fix = True + title.tracks.add(hybrid_track) + task_tracks.videos = [hybrid_track] + + enqueue_mux_tasks(task_description, task_tracks) + + console.print() + else: + # Normal mode: process each video track separately + for video_track in title.tracks.videos or [None]: + task_description = "Multiplexing" + if video_track: + if len(quality) > 1: + task_description += f" {video_track.height}p" + if len(range_) > 1: + task_description += f" {video_track.range.name}" + + task_tracks = Tracks(title.tracks) + title.tracks.chapters + title.tracks.attachments + if video_track: + task_tracks.videos = [video_track] + + enqueue_mux_tasks(task_description, task_tracks) + + try: + with Live(Padding(progress, (0, 5, 1, 5)), console=console): + mux_index = 0 + for task_id, task_tracks, audio_codec in multiplex_tasks: + progress.start_task(task_id) # TODO: Needed? + audio_expected = not video_only and not no_audio + muxed_path, return_code, errors = task_tracks.mux( + str(title), + progress=partial(progress.update, task_id=task_id), + delete=False, + audio_expected=audio_expected, + title_language=title.language, + skip_subtitles=skip_subtitle_mux, + ) + if muxed_path.exists(): + mux_index += 1 + unique_path = muxed_path.with_name( + f"{muxed_path.stem}.{mux_index}{muxed_path.suffix}" + ) + if unique_path != muxed_path: + shutil.move(muxed_path, unique_path) + muxed_path = unique_path + muxed_paths.append(muxed_path) + muxed_audio_codecs[muxed_path] = audio_codec + if return_code >= 2: + self.log.error(f"Failed to Mux video to Matroska file ({return_code}):") + elif return_code == 1 or errors: + self.log.warning("mkvmerge had at least one warning or error, continuing anyway...") + for line in errors: + if line.startswith("#GUI#error"): + self.log.error(line) + else: + self.log.warning(line) + if return_code >= 2: + sys.exit(1) + + # Output sidecar subtitles before deleting track files + if sidecar_subtitles and not no_mux: + media_info = MediaInfo.parse(muxed_paths[0]) if muxed_paths else None + if media_info: + base_filename = title.get_filename(media_info, show_service=not no_source) + else: + base_filename = str(title) + + sidecar_dir = config.directories.downloads + if not no_folder and isinstance(title, (Episode, Song)) and media_info: + sidecar_dir /= title.get_filename(media_info, show_service=not no_source, folder=True) + sidecar_dir.mkdir(parents=True, exist_ok=True) + + with console.status("Saving subtitle sidecar files..."): + created = self.output_subtitle_sidecars( + sidecar_subtitles, + base_filename, + sidecar_dir, + sidecar_format, + original_paths=sidecar_original_paths or None, + ) + if created: + self.log.info(f"Saved {len(created)} sidecar subtitle files") + + for track in title.tracks: + track.delete() + + # Clear temp font attachment paths and delete other attachments + for attachment in title.tracks.attachments: + if attachment.path and attachment.path in temp_font_files: + attachment.path = None + else: + attachment.delete() + + # Clean up temp fonts + for temp_path in temp_font_files: + temp_path.unlink(missing_ok=True) + for temp_path in sidecar_original_paths.values(): + temp_path.unlink(missing_ok=True) + finally: + # Hybrid() produces a temp HEVC output we rename; make sure it's never left behind. + # Also attempt to remove the default hybrid output name if it still exists. + for temp_path in hybrid_temp_paths: + try: + temp_path.unlink(missing_ok=True) + except PermissionError: + self.log.warning(f"Failed to delete temp file (in use?): {temp_path}") + try: + (config.directories.temp / "HDR10-DV.hevc").unlink(missing_ok=True) + except PermissionError: + self.log.warning( + f"Failed to delete temp file (in use?): {config.directories.temp / 'HDR10-DV.hevc'}" + ) + + else: + # dont mux + muxed_paths.append(title.tracks.audio[0].path) + + if no_mux: + # Handle individual track files without muxing + final_dir = config.directories.downloads + if not no_folder and isinstance(title, (Episode, Song)): + # Create folder based on title + # Use first available track for filename generation + sample_track = ( + title.tracks.videos[0] + if title.tracks.videos + else ( + title.tracks.audio[0] + if title.tracks.audio + else (title.tracks.subtitles[0] if title.tracks.subtitles else None) + ) + ) + if sample_track and sample_track.path: + media_info = MediaInfo.parse(sample_track.path) + final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True) + + final_dir.mkdir(parents=True, exist_ok=True) + + for track_path in muxed_paths: + # Generate appropriate filename for each track + media_info = MediaInfo.parse(track_path) + base_filename = title.get_filename(media_info, show_service=not no_source) + + # Add track type suffix to filename + track = next((t for t in title.tracks if t.path == track_path), None) + if track: + if isinstance(track, Video): + track_suffix = f".{track.codec.name if hasattr(track.codec, 'name') else 'video'}" + elif isinstance(track, Audio): + lang_suffix = f".{track.language}" if track.language else "" + track_suffix = ( + f"{lang_suffix}.{track.codec.name if hasattr(track.codec, 'name') else 'audio'}" + ) + elif isinstance(track, Subtitle): + lang_suffix = f".{track.language}" if track.language else "" + forced_suffix = ".forced" if track.forced else "" + sdh_suffix = ".sdh" if track.sdh else "" + track_suffix = f"{lang_suffix}{forced_suffix}{sdh_suffix}" + else: + track_suffix = "" + + final_path = final_dir / f"{base_filename}{track_suffix}{track_path.suffix}" + else: + final_path = final_dir / f"{base_filename}{track_path.suffix}" + + shutil.move(track_path, final_path) + self.log.debug(f"Saved: {final_path.name}") + else: + # Handle muxed files + for muxed_path in muxed_paths: + media_info = MediaInfo.parse(muxed_path) + final_dir = config.directories.downloads + final_filename = title.get_filename(media_info, show_service=not no_source) + audio_codec_suffix = muxed_audio_codecs.get(muxed_path) + + if not no_folder and isinstance(title, (Episode, Song)): + final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True) + + final_dir.mkdir(parents=True, exist_ok=True) + final_path = final_dir / f"{final_filename}{muxed_path.suffix}" + if final_path.exists() and audio_codec_suffix and append_audio_codec_suffix: + sep = "." if config.scene_naming else " " + final_filename = f"{final_filename.rstrip()}{sep}{audio_codec_suffix.name}" + final_path = final_dir / f"{final_filename}{muxed_path.suffix}" + + if final_path.exists(): + sep = "." if config.scene_naming else " " + i = 2 + while final_path.exists(): + final_path = final_dir / f"{final_filename.rstrip()}{sep}{i}{muxed_path.suffix}" + i += 1 + + shutil.move(muxed_path, final_path) + tags.tag_file(final_path, title, self.tmdb_id) + + title_dl_time = time_elapsed_since(dl_start_time) + console.print( + Padding(f":tada: Title downloaded in [progress.elapsed]{title_dl_time}[/]!", (0, 5, 1, 5)) + ) + + # update cookies + cookie_file = self.get_cookie_path(self.service, self.profile) + if cookie_file: + self.save_cookies(cookie_file, service.session.cookies) + + dl_time = time_elapsed_since(start_time) + + console.print(Padding(f"Processed all titles in [progress.elapsed]{dl_time}", (0, 5, 1, 5))) + + def prepare_drm( + self, + drm: DRM_T, + track: AnyTrack, + title: Title_T, + certificate: Callable, + licence: Callable, + track_kid: Optional[UUID] = None, + table: Table = None, + cdm_only: bool = False, + vaults_only: bool = False, + export: Optional[Path] = None, + ) -> None: + """ + Prepare the DRM by getting decryption data like KIDs, Keys, and such. + The DRM object should be ready for decryption once this function ends. + """ + if not drm: + return + + track_quality = None + if isinstance(track, Video) and track.height: + track_quality = track.height + + if isinstance(drm, Widevine): + if not is_widevine_cdm(self.cdm): + widevine_cdm = self.get_cdm(self.service, self.profile, drm="widevine", quality=track_quality) + if widevine_cdm: + if track_quality: + self.log.info(f"Switching to Widevine CDM for Widevine {track_quality}p content") + else: + self.log.info("Switching to Widevine CDM for Widevine content") + self.cdm = widevine_cdm + + elif isinstance(drm, PlayReady): + if not is_playready_cdm(self.cdm): + playready_cdm = self.get_cdm(self.service, self.profile, drm="playready", quality=track_quality) + if playready_cdm: + if track_quality: + self.log.info(f"Switching to PlayReady CDM for PlayReady {track_quality}p content") + else: + self.log.info("Switching to PlayReady CDM for PlayReady content") + self.cdm = playready_cdm + + if isinstance(drm, Widevine): + if self.debug_logger: + self.debug_logger.log_drm_operation( + drm_type="Widevine", + operation="prepare_drm", + service=self.service, + context={ + "track": str(track), + "title": str(title), + "pssh": drm.pssh.dumps() if drm.pssh else None, + "kids": [k.hex for k in drm.kids], + "track_kid": track_kid.hex if track_kid else None, + }, + ) + + with self.DRM_TABLE_LOCK: + pssh_display = self.truncate_pssh_for_display(drm.pssh.dumps(), "Widevine") + cek_tree = Tree(Text.assemble(("Widevine", "cyan"), (f"({pssh_display})", "text"), overflow="fold")) + pre_existing_tree = next( + (x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None + ) + if pre_existing_tree: + cek_tree = pre_existing_tree + + need_license = False + all_kids = list(drm.kids) + if track_kid and track_kid not in all_kids: + all_kids.append(track_kid) + + for kid in all_kids: + if kid in drm.content_keys: + continue + + is_track_kid = ["", "*"][kid == track_kid] + + if not cdm_only: + content_key, vault_used = self.vaults.get_key(kid) + if content_key: + drm.content_keys[kid] = content_key + label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}" + if not any(f"{kid.hex}:{content_key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + self.vaults.add_key(kid, content_key, excluding=vault_used) + + if self.debug_logger: + self.debug_logger.log_vault_query( + vault_name=vault_used, + operation="get_key_success", + service=self.service, + context={ + "kid": kid.hex, + "content_key": content_key, + "track": str(track), + "from_cache": True, + }, + ) + elif vaults_only: + msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="vault_key_not_found", + service=self.service, + message=msg, + context={"kid": kid.hex, "track": str(track)}, + ) + raise Widevine.Exceptions.CEKNotFound(msg) + else: + need_license = True + + if kid not in drm.content_keys and cdm_only: + need_license = True + + if need_license and not vaults_only: + from_vaults = drm.content_keys.copy() + + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="get_license", + service=self.service, + message="Requesting Widevine license from service", + context={ + "track": str(track), + "kids_needed": [k.hex for k in all_kids if k not in drm.content_keys], + }, + ) + + try: + if self.service == "NF": + drm.get_NF_content_keys(cdm=self.cdm, licence=licence, certificate=certificate) + else: + drm.get_content_keys(cdm=self.cdm, licence=licence, certificate=certificate) + except Exception as e: + if isinstance(e, (Widevine.Exceptions.EmptyLicense, Widevine.Exceptions.CEKNotFound)): + msg = str(e) + else: + msg = f"An exception occurred in the Service's license function: {e}" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + if self.debug_logger: + self.debug_logger.log_error( + "get_license", + e, + service=self.service, + context={"track": str(track), "exception_type": type(e).__name__}, + ) + raise e + + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="license_keys_retrieved", + service=self.service, + context={ + "track": str(track), + "keys_count": len(drm.content_keys), + "kids": [k.hex for k in drm.content_keys.keys()], + }, + ) + + for kid_, key in drm.content_keys.items(): + if key == "0" * 32: + key = f"[red]{key}[/]" + is_track_kid_marker = ["", "*"][kid_ == track_kid] + label = f"[text2]{kid_.hex}:{key}{is_track_kid_marker}" + if not any(f"{kid_.hex}:{key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + + drm.content_keys = { + kid_: key for kid_, key in drm.content_keys.items() if key and key.count("0") != len(key) + } + + # The CDM keys may have returned blank content keys for KIDs we got from vaults. + # So we re-add the keys from vaults earlier overwriting blanks or removed KIDs data. + drm.content_keys.update(from_vaults) + + successful_caches = self.vaults.add_keys(drm.content_keys) + self.log.info( + f"Cached {len(drm.content_keys)} Key{'' if len(drm.content_keys) == 1 else 's'} to " + f"{successful_caches}/{len(self.vaults)} Vaults" + ) + + if track_kid and track_kid not in drm.content_keys: + msg = f"No Content Key for KID {track_kid.hex} was returned in the License" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + raise Widevine.Exceptions.CEKNotFound(msg) + + if cek_tree.children and not pre_existing_tree: + table.add_row() + table.add_row(cek_tree) + + if export: + keys = {} + if export.is_file(): + keys = jsonpickle.loads(export.read_text(encoding="utf8")) or {} + if str(title) not in keys: + keys[str(title)] = {} + if str(track) not in keys[str(title)]: + keys[str(title)][str(track)] = {} + + track_data = keys[str(title)][str(track)] + track_data["url"] = track.url + track_data["descriptor"] = track.descriptor.name + + if "keys" not in track_data: + track_data["keys"] = {} + for kid, key in drm.content_keys.items(): + track_data["keys"][kid.hex] = key + + export.write_text(jsonpickle.dumps(keys, indent=4), encoding="utf8") + + elif isinstance(drm, PlayReady): + if self.debug_logger: + self.debug_logger.log_drm_operation( + drm_type="PlayReady", + operation="prepare_drm", + service=self.service, + context={ + "track": str(track), + "title": str(title), + "pssh": drm.pssh_b64 or "", + "kids": [k.hex for k in drm.kids], + "track_kid": track_kid.hex if track_kid else None, + }, + ) + + with self.DRM_TABLE_LOCK: + pssh_display = self.truncate_pssh_for_display(drm.pssh_b64 or "", "PlayReady") + cek_tree = Tree( + Text.assemble( + ("PlayReady", "cyan"), + (f"({pssh_display})", "text"), + overflow="fold", + ) + ) + pre_existing_tree = next( + (x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None + ) + if pre_existing_tree: + cek_tree = pre_existing_tree + + need_license = False + all_kids = list(drm.kids) + if track_kid and track_kid not in all_kids: + all_kids.append(track_kid) + + for kid in all_kids: + if kid in drm.content_keys: + continue + + is_track_kid = ["", "*"][kid == track_kid] + + if not cdm_only: + content_key, vault_used = self.vaults.get_key(kid) + if content_key: + drm.content_keys[kid] = content_key + label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}" + if not any(f"{kid.hex}:{content_key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + self.vaults.add_key(kid, content_key, excluding=vault_used) + + if self.debug_logger: + self.debug_logger.log_vault_query( + vault_name=vault_used, + operation="get_key_success", + service=self.service, + context={ + "kid": kid.hex, + "content_key": content_key, + "track": str(track), + "from_cache": True, + "drm_type": "PlayReady", + }, + ) + elif vaults_only: + msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="vault_key_not_found", + service=self.service, + message=msg, + context={"kid": kid.hex, "track": str(track), "drm_type": "PlayReady"}, + ) + raise PlayReady.Exceptions.CEKNotFound(msg) + else: + need_license = True + + if kid not in drm.content_keys and cdm_only: + need_license = True + + if need_license and not vaults_only: + from_vaults = drm.content_keys.copy() + + try: + drm.get_content_keys(cdm=self.cdm, licence=licence, certificate=certificate) + except Exception as e: + if isinstance(e, (PlayReady.Exceptions.EmptyLicense, PlayReady.Exceptions.CEKNotFound)): + msg = str(e) + else: + msg = f"An exception occurred in the Service's license function: {e}" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + if self.debug_logger: + self.debug_logger.log_error( + "get_license_playready", + e, + service=self.service, + context={ + "track": str(track), + "exception_type": type(e).__name__, + "drm_type": "PlayReady", + }, + ) + raise e + + for kid_, key in drm.content_keys.items(): + is_track_kid_marker = ["", "*"][kid_ == track_kid] + label = f"[text2]{kid_.hex}:{key}{is_track_kid_marker}" + if not any(f"{kid_.hex}:{key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + + drm.content_keys.update(from_vaults) + + successful_caches = self.vaults.add_keys(drm.content_keys) + self.log.info( + f"Cached {len(drm.content_keys)} Key{'' if len(drm.content_keys) == 1 else 's'} to " + f"{successful_caches}/{len(self.vaults)} Vaults" + ) + + if track_kid and track_kid not in drm.content_keys: + msg = f"No Content Key for KID {track_kid.hex} was returned in the License" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + raise PlayReady.Exceptions.CEKNotFound(msg) + + if cek_tree.children and not pre_existing_tree: + table.add_row() + table.add_row(cek_tree) + + if export: + keys = {} + if export.is_file(): + keys = jsonpickle.loads(export.read_text(encoding="utf8")) or {} + if str(title) not in keys: + keys[str(title)] = {} + if str(track) not in keys[str(title)]: + keys[str(title)][str(track)] = {} + + track_data = keys[str(title)][str(track)] + track_data["url"] = track.url + track_data["descriptor"] = track.descriptor.name + + if "keys" not in track_data: + track_data["keys"] = {} + for kid, key in drm.content_keys.items(): + track_data["keys"][kid.hex] = key + + export.write_text(jsonpickle.dumps(keys, indent=4), encoding="utf8") + + elif isinstance(drm, MonaLisa): + with self.DRM_TABLE_LOCK: + display_id = drm.content_id or drm.pssh + pssh_display = self.truncate_pssh_for_display(display_id, "MonaLisa") + cek_tree = Tree(Text.assemble(("MonaLisa", "cyan"), (f"({pssh_display})", "text"), overflow="fold")) + pre_existing_tree = next( + (x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None + ) + if pre_existing_tree: + cek_tree = pre_existing_tree + + for kid_, key in drm.content_keys.items(): + label = f"[text2]{kid_.hex}:{key}" + if not any(f"{kid_.hex}:{key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + + if cek_tree.children and not pre_existing_tree: + table.add_row() + table.add_row(cek_tree) + + @staticmethod + def get_cookie_path(service: str, profile: Optional[str]) -> Optional[Path]: + """Get Service Cookie File Path for Profile.""" + direct_cookie_file = config.directories.cookies / f"{service}.txt" + profile_cookie_file = config.directories.cookies / service / f"{profile}.txt" + default_cookie_file = config.directories.cookies / service / "default.txt" + + if direct_cookie_file.exists(): + return direct_cookie_file + elif profile_cookie_file.exists(): + return profile_cookie_file + elif default_cookie_file.exists(): + return default_cookie_file + + @staticmethod + def get_cookie_jar(service: str, profile: Optional[str]) -> Optional[MozillaCookieJar]: + """Get Service Cookies for Profile.""" + cookie_file = dl.get_cookie_path(service, profile) + if cookie_file: + cookie_jar = MozillaCookieJar(cookie_file) + cookie_data = html.unescape(cookie_file.read_text("utf8")).splitlines(keepends=False) + for i, line in enumerate(cookie_data): + if line and not line.startswith("#"): + line_data = line.lstrip().split("\t") + # Disable client-side expiry checks completely across everywhere + # Even though the cookies are loaded under ignore_expires=True, stuff + # like python-requests may not use them if they are expired + line_data[4] = "" + cookie_data[i] = "\t".join(line_data) + cookie_data = "\n".join(cookie_data) + cookie_file.write_text(cookie_data, "utf8") + cookie_jar.load(ignore_discard=True, ignore_expires=True) + return cookie_jar + + @staticmethod + def save_cookies(path: Path, cookies: CookieJar): + if hasattr(cookies, "jar"): + cookies = cookies.jar + + cookie_jar = MozillaCookieJar(path) + cookie_jar.load() + for cookie in cookies: + cookie_jar.set_cookie(cookie) + cookie_jar.save(ignore_discard=True) + + @staticmethod + def get_credentials(service: str, profile: Optional[str]) -> Optional[Credential]: + """Get Service Credentials for Profile.""" + credentials = config.credentials.get(service) + if credentials: + if isinstance(credentials, dict): + if profile: + credentials = credentials.get(profile) or credentials.get("default") + else: + credentials = credentials.get("default") + if credentials: + if isinstance(credentials, list): + return Credential(*credentials) + return Credential.loads(credentials) # type: ignore + + def get_cdm( + self, + service: str, + profile: Optional[str] = None, + drm: Optional[str] = None, + quality: Optional[int] = None, + ) -> Optional[object]: + """ + Get CDM for a specified service (either Local or Remote CDM). + Now supports quality-based selection when quality is provided. + Raises a ValueError if there's a problem getting a CDM. + """ + cdm_name = config.cdm.get(service) or config.cdm.get("default") + if not cdm_name: + return None + + if isinstance(cdm_name, dict): + if quality: + quality_match = None + quality_keys = [] + + for key in cdm_name.keys(): + if ( + isinstance(key, str) + and any(op in key for op in [">=", ">", "<=", "<"]) + or (isinstance(key, str) and key.isdigit()) + ): + quality_keys.append(key) + + def sort_quality_key(key): + if key.isdigit(): + return (0, int(key)) # Exact matches first + elif key.startswith(">="): + return (1, -int(key[2:])) # >= descending + elif key.startswith(">"): + return (1, -int(key[1:])) # > descending + elif key.startswith("<="): + return (2, int(key[2:])) # <= ascending + elif key.startswith("<"): + return (2, int(key[1:])) # < ascending + return (3, 0) # Other keys last + + quality_keys.sort(key=sort_quality_key) + + for key in quality_keys: + if key.isdigit() and quality == int(key): + quality_match = cdm_name[key] + self.log.debug(f"Selected CDM based on exact quality match {quality}p: {quality_match}") + break + elif key.startswith(">="): + threshold = int(key[2:]) + if quality >= threshold: + quality_match = cdm_name[key] + self.log.debug(f"Selected CDM based on quality {quality}p >= {threshold}p: {quality_match}") + break + elif key.startswith(">"): + threshold = int(key[1:]) + if quality > threshold: + quality_match = cdm_name[key] + self.log.debug(f"Selected CDM based on quality {quality}p > {threshold}p: {quality_match}") + break + elif key.startswith("<="): + threshold = int(key[2:]) + if quality <= threshold: + quality_match = cdm_name[key] + self.log.debug(f"Selected CDM based on quality {quality}p <= {threshold}p: {quality_match}") + break + elif key.startswith("<"): + threshold = int(key[1:]) + if quality < threshold: + quality_match = cdm_name[key] + self.log.debug(f"Selected CDM based on quality {quality}p < {threshold}p: {quality_match}") + break + + if quality_match: + cdm_name = quality_match + + if isinstance(cdm_name, dict): + lower_keys = {k.lower(): v for k, v in cdm_name.items()} + if {"widevine", "playready"} & lower_keys.keys(): + drm_key = None + if drm: + drm_key = { + "wv": "widevine", + "widevine": "widevine", + "pr": "playready", + "playready": "playready", + }.get(drm.lower()) + cdm_name = lower_keys.get(drm_key or "widevine") or lower_keys.get("playready") + else: + cdm_name = cdm_name.get(profile) or cdm_name.get("default") or config.cdm.get("default") + if not cdm_name: + return None + + cdm_api = next(iter(x.copy() for x in config.remote_cdm if x["name"] == cdm_name), None) + if cdm_api: + cdm_type = cdm_api.get("type") + + if cdm_type == "decrypt_labs": + del cdm_api["name"] + del cdm_api["type"] + + if "secret" not in cdm_api or not cdm_api["secret"]: + if config.decrypt_labs_api_key: + cdm_api["secret"] = config.decrypt_labs_api_key + else: + raise ValueError( + f"No secret provided for DecryptLabs CDM '{cdm_name}' and no global " + "decrypt_labs_api_key configured" + ) + + # All DecryptLabs CDMs use DecryptLabsRemoteCDM + return DecryptLabsRemoteCDM(service_name=service, vaults=self.vaults, **cdm_api) + + elif cdm_type == "custom_api": + del cdm_api["name"] + del cdm_api["type"] + + # All Custom API CDMs use CustomRemoteCDM + return CustomRemoteCDM(service_name=service, vaults=self.vaults, **cdm_api) + + else: + device_type = cdm_api.get("Device Type", cdm_api.get("device_type", "")) + if str(device_type).upper() == "PLAYREADY": + return PlayReadyRemoteCdm( + security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)), + host=cdm_api.get("Host", cdm_api.get("host")), + secret=cdm_api.get("Secret", cdm_api.get("secret")), + device_name=cdm_api.get("Device Name", cdm_api.get("device_name")), + ) + else: + return RemoteCdm( + device_type=cdm_api.get("Device Type", cdm_api.get("device_type", "")), + system_id=cdm_api.get("System ID", cdm_api.get("system_id", "")), + security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)), + host=cdm_api.get("Host", cdm_api.get("host")), + secret=cdm_api.get("Secret", cdm_api.get("secret")), + device_name=cdm_api.get("Device Name", cdm_api.get("device_name")), + ) + + prd_path = config.directories.prds / f"{cdm_name}.prd" + if not prd_path.is_file(): + prd_path = config.directories.wvds / f"{cdm_name}.prd" + if prd_path.is_file(): + device = PlayReadyDevice.load(prd_path) + return PlayReadyCdm.from_device(device) + + cdm_path = config.directories.wvds / f"{cdm_name}.wvd" + if not cdm_path.is_file(): + raise ValueError(f"{cdm_name} does not exist or is not a file") + + try: + device = Device.load(cdm_path) + except ConstError as e: + if "expected 2 but parsed 1" in str(e): + raise ValueError( + f"{cdm_name}.wvd seems to be a v1 WVD file, use `pywidevine migrate --help` to migrate it to v2." + ) + raise ValueError(f"{cdm_name}.wvd is an invalid or corrupt Widevine Device file, {e}") + + return WidevineCdm.from_device(device) diff --git a/packages/envied/src/envied/commands/dl.py b/packages/envied/src/envied/commands/dl.py index 4eb1ff8..cfa3139 100644 --- a/packages/envied/src/envied/commands/dl.py +++ b/packages/envied/src/envied/commands/dl.py @@ -27,6 +27,7 @@ from construct import ConstError from pymediainfo import MediaInfo from pyplayready.cdm import Cdm as PlayReadyCdm from pyplayready.device import Device as PlayReadyDevice +from pyplayready.remote.remotecdm import RemoteCdm as PlayReadyRemoteCdm from pywidevine.cdm import Cdm as WidevineCdm from pywidevine.device import Device from pywidevine.remotecdm import RemoteCdm @@ -42,13 +43,14 @@ from rich.tree import Tree from envied.core import binaries from envied.core.cdm import CustomRemoteCDM, DecryptLabsRemoteCDM +from envied.core.cdm.detect import is_playready_cdm, is_widevine_cdm from envied.core.config import config from envied.core.console import console from envied.core.constants import DOWNLOAD_LICENCE_ONLY, AnyTrack, context_settings from envied.core.credential import Credential -from envied.core.drm import DRM_T, PlayReady, Widevine +from envied.core.drm import DRM_T, MonaLisa, PlayReady, Widevine from envied.core.events import events -from envied.core.proxies import Basic, Hola, NordVPN, SurfsharkVPN, WindscribeVPN +from envied.core.proxies import Basic, Gluetun, Hola, NordVPN, SurfsharkVPN, WindscribeVPN from envied.core.service import Service from envied.core.services import Services from envied.core.title_cacher import get_account_hash @@ -60,12 +62,13 @@ from envied.core.tracks.hybrid import Hybrid from envied.core.utilities import (find_font_with_fallbacks, get_debug_logger, get_system_fonts, init_debug_logger, is_close_match, suggest_font_packages, time_elapsed_since) from envied.core.utils import tags -from envied.core.utils.click_types import (LANGUAGE_RANGE, QUALITY_LIST, SEASON_RANGE, ContextData, MultipleChoice, - SubtitleCodecChoice, VideoCodecChoice) +from envied.core.utils.click_types import (AUDIO_CODEC_LIST, LANGUAGE_RANGE, QUALITY_LIST, SEASON_RANGE, + ContextData, MultipleChoice, SubtitleCodecChoice, VideoCodecChoice) from envied.core.utils.collections import merge_dict +from envied.core.utils.selector import select_multiple from envied.core.utils.subprocess import ffprobe from envied.core.vaults import Vaults -from beaupy import select_multiple, Config + class dl: @staticmethod @@ -97,11 +100,7 @@ class dl: return None def prepare_temp_font( - self, - font_name: str, - matched_font: Path, - system_fonts: dict[str, Path], - temp_font_files: list[Path] + self, font_name: str, matched_font: Path, system_fonts: dict[str, Path], temp_font_files: list[Path] ) -> Path: """ Copy system font to temp and log if using fallback. @@ -116,10 +115,7 @@ class dl: Path to temp font file """ # Find the matched name for logging - matched_name = next( - (name for name, path in system_fonts.items() if path == matched_font), - None - ) + matched_name = next((name for name, path in system_fonts.items() if path == matched_font), None) if matched_name and matched_name.lower() != font_name.lower(): self.log.info(f"Using '{matched_name}' as fallback for '{font_name}'") @@ -136,10 +132,7 @@ class dl: return temp_path def attach_subtitle_fonts( - self, - font_names: list[str], - title: Title_T, - temp_font_files: list[Path] + self, font_names: list[str], title: Title_T, temp_font_files: list[Path] ) -> tuple[int, list[str]]: """ Attach fonts for subtitle rendering. @@ -188,6 +181,99 @@ class dl: self.log.info(f" $ sudo apt install {package_cmd}") self.log.info(f" → Provides: {', '.join(fonts)}") + def generate_sidecar_subtitle_path( + self, + subtitle: Subtitle, + base_filename: str, + output_dir: Path, + target_codec: Optional[Subtitle.Codec] = None, + source_path: Optional[Path] = None, + ) -> Path: + """Generate sidecar path: {base}.{lang}[.forced][.sdh].{ext}""" + lang_suffix = str(subtitle.language) if subtitle.language else "und" + forced_suffix = ".forced" if subtitle.forced else "" + sdh_suffix = ".sdh" if (subtitle.sdh or subtitle.cc) else "" + + extension = (target_codec or subtitle.codec or Subtitle.Codec.SubRip).extension + if ( + not target_codec + and not subtitle.codec + and source_path + and source_path.suffix + ): + extension = source_path.suffix.lstrip(".") + + filename = f"{base_filename}.{lang_suffix}{forced_suffix}{sdh_suffix}.{extension}" + return output_dir / filename + + def output_subtitle_sidecars( + self, + subtitles: list[Subtitle], + base_filename: str, + output_dir: Path, + sidecar_format: str, + original_paths: Optional[dict[str, Path]] = None, + ) -> list[Path]: + """Output subtitles as sidecar files, converting if needed.""" + created_paths: list[Path] = [] + config.directories.temp.mkdir(parents=True, exist_ok=True) + + for subtitle in subtitles: + source_path = subtitle.path + if sidecar_format == "original" and original_paths and subtitle.id in original_paths: + source_path = original_paths[subtitle.id] + + if not source_path or not source_path.exists(): + continue + + # Determine target codec + if sidecar_format == "original": + target_codec = None + if source_path.suffix: + try: + target_codec = Subtitle.Codec.from_mime(source_path.suffix.lstrip(".")) + except ValueError: + target_codec = None + else: + target_codec = Subtitle.Codec.from_mime(sidecar_format) + + sidecar_path = self.generate_sidecar_subtitle_path( + subtitle, base_filename, output_dir, target_codec, source_path=source_path + ) + + # Copy or convert + if not target_codec or subtitle.codec == target_codec: + shutil.copy2(source_path, sidecar_path) + else: + # Create temp copy for conversion to preserve original + temp_path = config.directories.temp / f"sidecar_{subtitle.id}{source_path.suffix}" + shutil.copy2(source_path, temp_path) + + temp_sub = Subtitle( + subtitle.url, + subtitle.language, + is_original_lang=subtitle.is_original_lang, + descriptor=subtitle.descriptor, + codec=subtitle.codec, + forced=subtitle.forced, + sdh=subtitle.sdh, + cc=subtitle.cc, + id_=f"{subtitle.id}_sc", + ) + temp_sub.path = temp_path + try: + temp_sub.convert(target_codec) + if temp_sub.path and temp_sub.path.exists(): + shutil.copy2(temp_sub.path, sidecar_path) + finally: + if temp_sub.path and temp_sub.path.exists(): + temp_sub.path.unlink(missing_ok=True) + temp_path.unlink(missing_ok=True) + + created_paths.append(sidecar_path) + + return created_paths + @click.command( short_help="Download, Decrypt, and Mux tracks for titles from a Service.", cls=Services, @@ -213,9 +299,9 @@ class dl: @click.option( "-a", "--acodec", - type=click.Choice(Audio.Codec, case_sensitive=False), - default=None, - help="Audio Codec to download, defaults to any codec.", + type=AUDIO_CODEC_LIST, + default=[], + help="Audio Codec(s) to download (comma-separated), e.g., 'AAC,EC3'. Defaults to any.", ) @click.option( "-vb", @@ -254,13 +340,19 @@ class dl: default=False, help="Exclude Dolby Atmos audio tracks when selecting audio.", ) + @click.option( + "--split-audio", + "split_audio", + is_flag=True, + default=None, + help="Create separate output files per audio codec instead of merging all audio.", + ) @click.option( "--select-titles", is_flag=True, default=False, help="Interactively select downloads from a list. Only use with Series to select Episodes", ) - @click.option( "-w", "--wanted", @@ -268,13 +360,6 @@ class dl: default=None, help="Wanted episodes, e.g. `S01-S05,S07`, `S01E01-S02E03`, `S02-S02E03`, e.t.c, defaults to all.", ) - @click.option( - "-le", - "--latest-episode", - is_flag=True, - default=False, - help="Download only the single most recent episode available.", - ) @click.option( "-l", "--lang", @@ -282,6 +367,12 @@ class dl: default="orig", help="Language wanted for Video and Audio. Use 'orig' to select the original language, e.g. 'orig,en' for both original and English.", ) + @click.option( + "--latest-episode", + is_flag=True, + default=False, + help="Download only the single most recent episode available.", + ) @click.option( "-vl", "--v-lang", @@ -645,12 +736,17 @@ class dl: "device_type": self.cdm.device_type.name, } else: - self.log.info( - f"Loaded PlayReady CDM: {self.cdm.certificate_chain.get_name()} (L{self.cdm.security_level})" - ) + # Handle both local PlayReady CDM and RemoteCdm (which has certificate_chain=None) + is_remote = self.cdm.certificate_chain is None and hasattr(self.cdm, "device_name") + if is_remote: + cdm_name = self.cdm.device_name + self.log.info(f"Loaded PlayReady Remote CDM: {cdm_name} (L{self.cdm.security_level})") + else: + cdm_name = self.cdm.certificate_chain.get_name() if self.cdm.certificate_chain else "Unknown" + self.log.info(f"Loaded PlayReady CDM: {cdm_name} (L{self.cdm.security_level})") cdm_info = { "type": "PlayReady", - "certificate": self.cdm.certificate_chain.get_name(), + "certificate": cdm_name, "security_level": self.cdm.security_level, } @@ -672,6 +768,8 @@ class dl: self.proxy_providers.append(SurfsharkVPN(**config.proxy_providers["surfsharkvpn"])) if config.proxy_providers.get("windscribevpn"): self.proxy_providers.append(WindscribeVPN(**config.proxy_providers["windscribevpn"])) + if config.proxy_providers.get("gluetun"): + self.proxy_providers.append(Gluetun(**config.proxy_providers["gluetun"])) if binaries.HolaProxy: self.proxy_providers.append(Hola()) for proxy_provider in self.proxy_providers: @@ -682,9 +780,20 @@ class dl: if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE): # requesting proxy from a specific proxy provider requested_provider, proxy = proxy.split(":", maxsplit=1) - if re.match(r"^[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE): + # Match simple region codes (us, ca, uk1) or provider:region format (nordvpn:ca, windscribe:us) + if re.match(r"^[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE) or re.match( + r"^[a-z]+:[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE + ): proxy = proxy.lower() - with console.status(f"Getting a Proxy to {proxy}...", spinner="dots"): + # Preserve the original user query (region code) for service-specific proxy_map overrides. + # NOTE: `proxy` may be overwritten with the resolved proxy URI later. + proxy_query = proxy + status_msg = ( + f"Connecting to VPN ({proxy})..." + if requested_provider == "gluetun" + else f"Getting a Proxy to {proxy}..." + ) + with console.status(status_msg, spinner="dots"): if requested_provider: proxy_provider = next( (x for x in self.proxy_providers if x.__class__.__name__.lower() == requested_provider), @@ -698,16 +807,42 @@ class dl: self.log.error(f"The proxy provider {requested_provider} had no proxy for {proxy}") sys.exit(1) proxy = ctx.params["proxy"] = proxy_uri - self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") + # Show connection info for Gluetun (IP, location) instead of proxy URL + if hasattr(proxy_provider, "get_connection_info"): + conn_info = proxy_provider.get_connection_info(proxy_query) + if conn_info and conn_info.get("public_ip"): + location_parts = [conn_info.get("city"), conn_info.get("country")] + location = ", ".join(p for p in location_parts if p) + self.log.info(f"VPN Connected: {conn_info['public_ip']} ({location})") + else: + self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") + else: + self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") else: for proxy_provider in self.proxy_providers: proxy_uri = proxy_provider.get_proxy(proxy) if proxy_uri: proxy = ctx.params["proxy"] = proxy_uri - self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") + # Show connection info for Gluetun (IP, location) instead of proxy URL + if hasattr(proxy_provider, "get_connection_info"): + conn_info = proxy_provider.get_connection_info(proxy_query) + if conn_info and conn_info.get("public_ip"): + location_parts = [conn_info.get("city"), conn_info.get("country")] + location = ", ".join(p for p in location_parts if p) + self.log.info(f"VPN Connected: {conn_info['public_ip']} ({location})") + else: + self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") + else: + self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") break + # Store proxy query info for service-specific overrides + ctx.params["proxy_query"] = proxy_query + ctx.params["proxy_provider"] = requested_provider else: self.log.info(f"Using explicit Proxy: {proxy}") + # For explicit proxies, store None for query/provider + ctx.params["proxy_query"] = None + ctx.params["proxy_provider"] = None ctx.obj = ContextData( config=self.service_config, cdm=self.cdm, proxy_providers=self.proxy_providers, profile=self.profile @@ -725,7 +860,7 @@ class dl: service: Service, quality: list[int], vcodec: Optional[Video.Codec], - acodec: Optional[Audio.Codec], + acodec: list[Audio.Codec], vbitrate: int, abitrate: int, range_: list[Video.Range], @@ -764,6 +899,7 @@ class dl: workers: Optional[int], downloads: int, best_available: bool, + split_audio: Optional[bool] = None, *_: Any, **__: Any, ) -> None: @@ -771,6 +907,18 @@ class dl: self.search_source = None start_time = time.time() + if skip_dl: + DOWNLOAD_LICENCE_ONLY.set() + + if not acodec: + acodec = [] + elif isinstance(acodec, Audio.Codec): + acodec = [acodec] + elif isinstance(acodec, str) or ( + isinstance(acodec, list) and not all(isinstance(v, Audio.Codec) for v in acodec) + ): + acodec = AUDIO_CODEC_LIST.convert(acodec) + if require_subs and s_lang != ["all"]: self.log.error("--require-subs and --s-lang cannot be used together") sys.exit(1) @@ -906,41 +1054,85 @@ class dl: console.print(Padding(titles.tree(verbose=list_titles), (0, 5))) if list_titles: return - # modification to enable beaupy module to list titles for download for manual selection - # use --select-titles after dl in unshackle command - - Config.transient = True - # thanks CodeName393 for suggestion to use episode name and full SXXExx format - if select_titles and type(titles)==Series: - console.print(Padding(Rule(f"[rule.text]\nSelect Titles Option\n"), (1, 2))) + + # Enables manual selection for Series when --select-titles is set + if select_titles and type(titles) == Series: + console.print(Padding(Rule(f"[rule.text]Select Titles"), (1, 2))) - beaupy_titles = [ - f"{i+1}. {t.title} - S{t.season:02}E{t.number:02}" - + (f" - {t.name}" if t.name else "") - for i, t in enumerate(titles) - ] - - selected_idx = select_multiple( - beaupy_titles, - preprocessor=lambda val: f"[rgb(205,214,244)]{val}[/rgb(205,214,244)]", + selection_titles = [] + dependencies = {} + original_indices = {} + + current_season = None + current_season_header_idx = -1 + + unique_seasons = {t.season for t in titles} + multiple_seasons = len(unique_seasons) > 1 + + # Build selection options + for i, t in enumerate(titles): + # Insert season header only if multiple seasons exist + if multiple_seasons and t.season != current_season: + current_season = t.season + header_text = f"Season {t.season}" + selection_titles.append(header_text) + current_season_header_idx = len(selection_titles) - 1 + dependencies[current_season_header_idx] = [] + # Note: Headers are not mapped to actual title indices + + # Format display name + display_name = ((t.name[:35].rstrip() + "…") if len(t.name) > 35 else t.name) if t.name else None + + # Apply indentation only for multiple seasons + prefix = " " if multiple_seasons else "" + option_text = ( + f"{prefix}{t.number}" + (f". {display_name}" if t.name else "") + ) + + selection_titles.append(option_text) + current_ui_idx = len(selection_titles) - 1 + + # Map UI index to actual title index + original_indices[current_ui_idx] = i + + # Link episode to season header for group selection + if current_season_header_idx != -1: + dependencies[current_season_header_idx].append(current_ui_idx) + + selection_start = time.time() + + # Execute selector with dependencies (headers select all children) + selected_ui_idx = select_multiple( + selection_titles, minimal_count=1, page_size=8, - pagination=True, return_indices=True, - + dependencies=dependencies ) - # Keep indices unique & ordered + selection_end = time.time() + start_time += (selection_end - selection_start) + + # Map UI indices back to title indices (excluding headers) + selected_idx = [] + for idx in selected_ui_idx: + if idx in original_indices: + selected_idx.append(original_indices[idx]) + + # Ensure indices are unique and ordered selected_idx = sorted(set(selected_idx)) keep = set(selected_idx) - # In-place filter: delete everything not selected (walk backwards!) - # need to delete from original list to keep super classes meta data + # In-place filter: remove unselected items (iterate backwards) for i in range(len(titles) - 1, -1, -1): if i not in keep: del titles[i] - console.print(Padding(f"[rgb(205,214,244)]{len(titles)} {'titles' if len(titles) > 1 else 'title'} selected for download[/rgb(205,214,244)]", (0, 5))) - # end modification + + # Show selected count + if titles: + count = len(titles) + console.print(Padding(f"[text]Total selected: {count}[/]", (0, 5))) + # Determine the latest episode if --latest-episode is set latest_episode_id = None if latest_episode and isinstance(titles, Series) and len(titles) > 0: @@ -1101,7 +1293,9 @@ class dl: title.tracks.add(non_sdh_sub) events.subscribe( events.Types.TRACK_MULTIPLEX, - lambda track, sub_id=non_sdh_sub.id: (track.strip_hearing_impaired()) if track.id == sub_id else None, + lambda track, sub_id=non_sdh_sub.id: (track.strip_hearing_impaired()) + if track.id == sub_id + else None, ) with console.status("Sorting tracks by language and bitrate...", spinner="dots"): @@ -1314,9 +1508,10 @@ class dl: if not audio_description: title.tracks.select_audio(lambda x: not x.descriptive) # exclude descriptive audio if acodec: - title.tracks.select_audio(lambda x: x.codec == acodec) + title.tracks.select_audio(lambda x: x.codec in acodec) if not title.tracks.audio: - self.log.error(f"There's no {acodec.name} Audio Tracks...") + codec_names = ", ".join(c.name for c in acodec) + self.log.error(f"No audio tracks matching codecs: {codec_names}") sys.exit(1) if channels: title.tracks.select_audio(lambda x: math.ceil(x.channels) == math.ceil(channels)) @@ -1356,22 +1551,88 @@ class dl: unique_languages = {track.language for track in title.tracks.audio} selected_audio = [] for language in unique_languages: - highest_quality = max( - (track for track in title.tracks.audio if track.language == language), - key=lambda x: x.bitrate or 0, - ) - selected_audio.append(highest_quality) + codecs_to_check = acodec if (acodec and len(acodec) > 1) else [None] + for codec in codecs_to_check: + base_candidates = [ + t + for t in title.tracks.audio + if t.language == language and (codec is None or t.codec == codec) + ] + if not base_candidates: + continue + if audio_description: + standards = [t for t in base_candidates if not t.descriptive] + if standards: + selected_audio.append(max(standards, key=lambda x: x.bitrate or 0)) + descs = [t for t in base_candidates if t.descriptive] + if descs: + selected_audio.append(max(descs, key=lambda x: x.bitrate or 0)) + else: + selected_audio.append(max(base_candidates, key=lambda x: x.bitrate or 0)) title.tracks.audio = selected_audio elif "all" not in processed_lang: - per_language = 1 - title.tracks.audio = title.tracks.by_language( - title.tracks.audio, processed_lang, per_language=per_language, exact_match=exact_lang - ) + # If multiple codecs were explicitly requested, pick the best track per codec per + # requested language instead of selecting *all* bitrate variants of a codec. + if acodec and len(acodec) > 1: + selected_audio: list[Audio] = [] + + for language in processed_lang: + for codec in acodec: + codec_tracks = [a for a in title.tracks.audio if a.codec == codec] + if not codec_tracks: + continue + + candidates = title.tracks.by_language( + codec_tracks, [language], per_language=0, exact_match=exact_lang + ) + if not candidates: + continue + + if audio_description: + standards = [t for t in candidates if not t.descriptive] + if standards: + selected_audio.append(max(standards, key=lambda x: x.bitrate or 0)) + descs = [t for t in candidates if t.descriptive] + if descs: + selected_audio.append(max(descs, key=lambda x: x.bitrate or 0)) + else: + selected_audio.append(max(candidates, key=lambda x: x.bitrate or 0)) + + title.tracks.audio = selected_audio + else: + per_language = 1 + if audio_description: + standard_audio = [a for a in title.tracks.audio if not a.descriptive] + selected_standards = title.tracks.by_language( + standard_audio, processed_lang, per_language=per_language, exact_match=exact_lang + ) + desc_audio = [a for a in title.tracks.audio if a.descriptive] + # Include all descriptive tracks for the requested languages. + selected_descs = title.tracks.by_language( + desc_audio, processed_lang, per_language=0, exact_match=exact_lang + ) + title.tracks.audio = selected_standards + selected_descs + else: + title.tracks.audio = title.tracks.by_language( + title.tracks.audio, + processed_lang, + per_language=per_language, + exact_match=exact_lang, + ) if not title.tracks.audio: self.log.error(f"There's no {processed_lang} Audio Track, cannot continue...") sys.exit(1) - if video_only or audio_only or subs_only or chapters_only or no_subs or no_audio or no_chapters or no_video: + if ( + video_only + or audio_only + or subs_only + or chapters_only + or no_subs + or no_audio + or no_chapters + or no_video + ): keep_videos = False keep_audio = False keep_subtitles = False @@ -1427,9 +1688,7 @@ class dl: if video_tracks: highest_quality = max((track.height for track in video_tracks if track.height), default=0) if highest_quality > 0: - if isinstance(self.cdm, (WidevineCdm, DecryptLabsRemoteCDM)) and not ( - isinstance(self.cdm, DecryptLabsRemoteCDM) and self.cdm.is_playready - ): + if is_widevine_cdm(self.cdm): quality_based_cdm = self.get_cdm( self.service, self.profile, drm="widevine", quality=highest_quality ) @@ -1438,9 +1697,7 @@ class dl: f"Pre-selecting Widevine CDM based on highest quality {highest_quality}p across all video tracks" ) self.cdm = quality_based_cdm - elif isinstance(self.cdm, (PlayReadyCdm, DecryptLabsRemoteCDM)) and ( - isinstance(self.cdm, DecryptLabsRemoteCDM) and self.cdm.is_playready - ): + elif is_playready_cdm(self.cdm): quality_based_cdm = self.get_cdm( self.service, self.profile, drm="playready", quality=highest_quality ) @@ -1452,9 +1709,6 @@ class dl: dl_start_time = time.time() - if skip_dl: - DOWNLOAD_LICENCE_ONLY.set() - try: with Live(Padding(download_table, (1, 5)), console=console, refresh_per_second=5): with ThreadPoolExecutor(downloads) as pool: @@ -1475,10 +1729,7 @@ class dl: licence=partial( service.get_playready_license if ( - isinstance(self.cdm, PlayReadyCdm) - or ( - isinstance(self.cdm, DecryptLabsRemoteCDM) and self.cdm.is_playready - ) + is_playready_cdm(self.cdm) ) and hasattr(service, "get_playready_license") else service.get_widevine_license, @@ -1594,6 +1845,25 @@ class dl: break video_track_n += 1 + # Subtitle output mode configuration (for sidecar originals) + subtitle_output_mode = config.subtitle.get("output_mode", "mux") + sidecar_format = config.subtitle.get("sidecar_format", "srt") + skip_subtitle_mux = ( + subtitle_output_mode == "sidecar" and (title.tracks.videos or title.tracks.audio) + ) + sidecar_subtitles: list[Subtitle] = [] + sidecar_original_paths: dict[str, Path] = {} + if subtitle_output_mode in ("sidecar", "both") and not no_mux: + sidecar_subtitles = [s for s in title.tracks.subtitles if s.path and s.path.exists()] + if sidecar_format == "original": + config.directories.temp.mkdir(parents=True, exist_ok=True) + for subtitle in sidecar_subtitles: + original_path = ( + config.directories.temp / f"sidecar_original_{subtitle.id}{subtitle.path.suffix}" + ) + shutil.copy2(subtitle.path, original_path) + sidecar_original_paths[subtitle.id] = original_path + with console.status("Converting Subtitles..."): for subtitle in title.tracks.subtitles: if sub_format: @@ -1609,11 +1879,9 @@ class dl: if subtitle.codec == Subtitle.Codec.SubStationAlphav4: for line in subtitle.path.read_text("utf8").splitlines(): if line.startswith("Style: "): - font_names.append(line.removesuffix("Style: ").split(",")[1]).strip() + font_names.append(line.removeprefix("Style: ").split(",")[1].strip()) - font_count, missing_fonts = self.attach_subtitle_fonts( - font_names, title, temp_font_files - ) + font_count, missing_fonts = self.attach_subtitle_fonts(font_names, title, temp_font_files) if font_count: self.log.info(f"Attached {font_count} fonts for the Subtitles") @@ -1634,7 +1902,8 @@ class dl: drm = track.get_drm_for_cdm(self.cdm) if drm and hasattr(drm, "decrypt"): drm.decrypt(track.path) - has_decrypted = True + if not isinstance(drm, MonaLisa): + has_decrypted = True events.emit(events.Types.TRACK_REPACKED, track=track) else: self.log.warning( @@ -1656,6 +1925,8 @@ class dl: self.log.info("Repacked one or more tracks with FFMPEG") muxed_paths = [] + muxed_audio_codecs: dict[Path, Optional[Audio.Codec]] = {} + append_audio_codec_suffix = True if no_mux: # Skip muxing, handle individual track files @@ -1672,7 +1943,44 @@ class dl: console=console, ) - multiplex_tasks: list[tuple[TaskID, Tracks]] = [] + merge_audio = ( + (not split_audio) if split_audio is not None else config.muxing.get("merge_audio", True) + ) + # When we split audio (merge_audio=False), multiple outputs may exist per title, so suffix codec. + append_audio_codec_suffix = not merge_audio + + multiplex_tasks: list[tuple[TaskID, Tracks, Optional[Audio.Codec]]] = [] + # Track hybrid-processing outputs explicitly so we can always clean them up, + # even if muxing fails early (e.g. SystemExit) before the normal delete loop. + hybrid_temp_paths: list[Path] = [] + + def clone_tracks_for_audio(base_tracks: Tracks, audio_tracks: list[Audio]) -> Tracks: + task_tracks = Tracks() + task_tracks.videos = list(base_tracks.videos) + task_tracks.audio = audio_tracks + task_tracks.subtitles = list(base_tracks.subtitles) + task_tracks.chapters = base_tracks.chapters + task_tracks.attachments = list(base_tracks.attachments) + return task_tracks + + def enqueue_mux_tasks(task_description: str, base_tracks: Tracks) -> None: + if merge_audio or not base_tracks.audio: + task_id = progress.add_task(f"{task_description}...", total=None, start=False) + multiplex_tasks.append((task_id, base_tracks, None)) + return + + audio_by_codec: dict[Optional[Audio.Codec], list[Audio]] = {} + for audio_track in base_tracks.audio: + audio_by_codec.setdefault(audio_track.codec, []).append(audio_track) + + for audio_codec, codec_audio_tracks in audio_by_codec.items(): + description = task_description + if audio_codec: + description = f"{task_description} {audio_codec.name}" + + task_id = progress.add_task(f"{description}...", total=None, start=False) + task_tracks = clone_tracks_for_audio(base_tracks, codec_audio_tracks) + multiplex_tasks.append((task_id, task_tracks, audio_codec)) # Check if we're in hybrid mode if any(r == Video.Range.HYBRID for r in range_) and title.tracks.videos: @@ -1706,27 +2014,29 @@ class dl: # Create unique output filename for this resolution hybrid_filename = f"HDR10-DV-{resolution}p.hevc" hybrid_output_path = config.directories.temp / hybrid_filename + hybrid_temp_paths.append(hybrid_output_path) # The Hybrid class creates HDR10-DV.hevc, rename it for this resolution default_output = config.directories.temp / "HDR10-DV.hevc" if default_output.exists(): + # If a previous run left this behind, replace it to avoid move() failures. + hybrid_output_path.unlink(missing_ok=True) shutil.move(str(default_output), str(hybrid_output_path)) - # Create a mux task for this resolution - task_description = f"Multiplexing Hybrid HDR10+DV {resolution}p" - task_id = progress.add_task(f"{task_description}...", total=None, start=False) - # Create tracks with the hybrid video output for this resolution + task_description = f"Multiplexing Hybrid HDR10+DV {resolution}p" task_tracks = Tracks(title.tracks) + title.tracks.chapters + title.tracks.attachments # Create a new video track for the hybrid output hybrid_track = deepcopy(hdr10_track) + hybrid_track.id = f"hybrid_{hdr10_track.id}_{resolution}" hybrid_track.path = hybrid_output_path hybrid_track.range = Video.Range.DV # It's now a DV track hybrid_track.needs_duration_fix = True + title.tracks.add(hybrid_track) task_tracks.videos = [hybrid_track] - multiplex_tasks.append((task_id, task_tracks)) + enqueue_mux_tasks(task_description, task_tracks) console.print() else: @@ -1739,52 +2049,101 @@ class dl: if len(range_) > 1: task_description += f" {video_track.range.name}" - task_id = progress.add_task(f"{task_description}...", total=None, start=False) - task_tracks = Tracks(title.tracks) + title.tracks.chapters + title.tracks.attachments if video_track: task_tracks.videos = [video_track] - multiplex_tasks.append((task_id, task_tracks)) + enqueue_mux_tasks(task_description, task_tracks) - with Live(Padding(progress, (0, 5, 1, 5)), console=console): - for task_id, task_tracks in multiplex_tasks: - progress.start_task(task_id) # TODO: Needed? - audio_expected = not video_only and not no_audio - muxed_path, return_code, errors = task_tracks.mux( - str(title), - progress=partial(progress.update, task_id=task_id), - delete=False, - audio_expected=audio_expected, - title_language=title.language, - ) - muxed_paths.append(muxed_path) - if return_code >= 2: - self.log.error(f"Failed to Mux video to Matroska file ({return_code}):") - elif return_code == 1 or errors: - self.log.warning("mkvmerge had at least one warning or error, continuing anyway...") - for line in errors: - if line.startswith("#GUI#error"): - self.log.error(line) + try: + with Live(Padding(progress, (0, 5, 1, 5)), console=console): + mux_index = 0 + for task_id, task_tracks, audio_codec in multiplex_tasks: + progress.start_task(task_id) # TODO: Needed? + audio_expected = not video_only and not no_audio + muxed_path, return_code, errors = task_tracks.mux( + str(title), + progress=partial(progress.update, task_id=task_id), + delete=False, + audio_expected=audio_expected, + title_language=title.language, + skip_subtitles=skip_subtitle_mux, + ) + if muxed_path.exists(): + mux_index += 1 + unique_path = muxed_path.with_name( + f"{muxed_path.stem}.{mux_index}{muxed_path.suffix}" + ) + if unique_path != muxed_path: + shutil.move(muxed_path, unique_path) + muxed_path = unique_path + muxed_paths.append(muxed_path) + muxed_audio_codecs[muxed_path] = audio_codec + if return_code >= 2: + self.log.error(f"Failed to Mux video to Matroska file ({return_code}):") + elif return_code == 1 or errors: + self.log.warning("mkvmerge had at least one warning or error, continuing anyway...") + for line in errors: + if line.startswith("#GUI#error"): + self.log.error(line) + else: + self.log.warning(line) + if return_code >= 2: + sys.exit(1) + + # Output sidecar subtitles before deleting track files + if sidecar_subtitles and not no_mux: + media_info = MediaInfo.parse(muxed_paths[0]) if muxed_paths else None + if media_info: + base_filename = title.get_filename(media_info, show_service=not no_source) else: - self.log.warning(line) - if return_code >= 2: - sys.exit(1) - for video_track in task_tracks.videos: - video_track.delete() - for track in title.tracks: - track.delete() + base_filename = str(title) - # Clear temp font attachment paths and delete other attachments - for attachment in title.tracks.attachments: - if attachment.path and attachment.path in temp_font_files: - attachment.path = None - else: - attachment.delete() + sidecar_dir = config.directories.downloads + if not no_folder and isinstance(title, (Episode, Song)) and media_info: + sidecar_dir /= title.get_filename(media_info, show_service=not no_source, folder=True) + sidecar_dir.mkdir(parents=True, exist_ok=True) - # Clean up temp fonts - for temp_path in temp_font_files: - temp_path.unlink(missing_ok=True) + with console.status("Saving subtitle sidecar files..."): + created = self.output_subtitle_sidecars( + sidecar_subtitles, + base_filename, + sidecar_dir, + sidecar_format, + original_paths=sidecar_original_paths or None, + ) + if created: + self.log.info(f"Saved {len(created)} sidecar subtitle files") + + for track in title.tracks: + track.delete() + + # Clear temp font attachment paths and delete other attachments + for attachment in title.tracks.attachments: + if attachment.path and attachment.path in temp_font_files: + attachment.path = None + else: + attachment.delete() + + # Clean up temp fonts + for temp_path in temp_font_files: + temp_path.unlink(missing_ok=True) + for temp_path in sidecar_original_paths.values(): + temp_path.unlink(missing_ok=True) + finally: + # Hybrid() produces a temp HEVC output we rename; make sure it's never left behind. + # Also attempt to remove the default hybrid output name if it still exists. + for temp_path in hybrid_temp_paths: + try: + temp_path.unlink(missing_ok=True) + except PermissionError: + self.log.warning(f"Failed to delete temp file (in use?): {temp_path}") + try: + (config.directories.temp / "HDR10-DV.hevc").unlink(missing_ok=True) + except PermissionError: + self.log.warning( + f"Failed to delete temp file (in use?): {config.directories.temp / 'HDR10-DV.hevc'}" + ) else: # dont mux @@ -1846,12 +2205,24 @@ class dl: media_info = MediaInfo.parse(muxed_path) final_dir = config.directories.downloads final_filename = title.get_filename(media_info, show_service=not no_source) + audio_codec_suffix = muxed_audio_codecs.get(muxed_path) if not no_folder and isinstance(title, (Episode, Song)): final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True) final_dir.mkdir(parents=True, exist_ok=True) final_path = final_dir / f"{final_filename}{muxed_path.suffix}" + if final_path.exists() and audio_codec_suffix and append_audio_codec_suffix: + sep = "." if config.scene_naming else " " + final_filename = f"{final_filename.rstrip()}{sep}{audio_codec_suffix.name}" + final_path = final_dir / f"{final_filename}{muxed_path.suffix}" + + if final_path.exists(): + sep = "." if config.scene_naming else " " + i = 2 + while final_path.exists(): + final_path = final_dir / f"{final_filename.rstrip()}{sep}{i}{muxed_path.suffix}" + i += 1 shutil.move(muxed_path, final_path) tags.tag_file(final_path, title, self.tmdb_id) @@ -1895,9 +2266,7 @@ class dl: track_quality = track.height if isinstance(drm, Widevine): - if not isinstance(self.cdm, (WidevineCdm, DecryptLabsRemoteCDM)) or ( - isinstance(self.cdm, DecryptLabsRemoteCDM) and self.cdm.is_playready - ): + if not is_widevine_cdm(self.cdm): widevine_cdm = self.get_cdm(self.service, self.profile, drm="widevine", quality=track_quality) if widevine_cdm: if track_quality: @@ -1907,9 +2276,7 @@ class dl: self.cdm = widevine_cdm elif isinstance(drm, PlayReady): - if not isinstance(self.cdm, (PlayReadyCdm, DecryptLabsRemoteCDM)) or ( - isinstance(self.cdm, DecryptLabsRemoteCDM) and not self.cdm.is_playready - ): + if not is_playready_cdm(self.cdm): playready_cdm = self.get_cdm(self.service, self.profile, drm="playready", quality=track_quality) if playready_cdm: if track_quality: @@ -2250,6 +2617,26 @@ class dl: export.write_text(jsonpickle.dumps(keys, indent=4), encoding="utf8") + elif isinstance(drm, MonaLisa): + with self.DRM_TABLE_LOCK: + display_id = drm.content_id or drm.pssh + pssh_display = self.truncate_pssh_for_display(display_id, "MonaLisa") + cek_tree = Tree(Text.assemble(("MonaLisa", "cyan"), (f"({pssh_display})", "text"), overflow="fold")) + pre_existing_tree = next( + (x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None + ) + if pre_existing_tree: + cek_tree = pre_existing_tree + + for kid_, key in drm.content_keys.items(): + label = f"[text2]{kid_.hex}:{key}" + if not any(f"{kid_.hex}:{key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + + if cek_tree.children and not pre_existing_tree: + table.add_row() + table.add_row(cek_tree) + @staticmethod def get_cookie_path(service: str, profile: Optional[str]) -> Optional[Path]: """Get Service Cookie File Path for Profile.""" @@ -2432,14 +2819,23 @@ class dl: return CustomRemoteCDM(service_name=service, vaults=self.vaults, **cdm_api) else: - return RemoteCdm( - device_type=cdm_api["Device Type"], - system_id=cdm_api["System ID"], - security_level=cdm_api["Security Level"], - host=cdm_api["Host"], - secret=cdm_api["Secret"], - device_name=cdm_api["Device Name"], - ) + device_type = cdm_api.get("Device Type", cdm_api.get("device_type", "")) + if str(device_type).upper() == "PLAYREADY": + return PlayReadyRemoteCdm( + security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)), + host=cdm_api.get("Host", cdm_api.get("host")), + secret=cdm_api.get("Secret", cdm_api.get("secret")), + device_name=cdm_api.get("Device Name", cdm_api.get("device_name")), + ) + else: + return RemoteCdm( + device_type=cdm_api.get("Device Type", cdm_api.get("device_type", "")), + system_id=cdm_api.get("System ID", cdm_api.get("system_id", "")), + security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)), + host=cdm_api.get("Host", cdm_api.get("host")), + secret=cdm_api.get("Secret", cdm_api.get("secret")), + device_name=cdm_api.get("Device Name", cdm_api.get("device_name")), + ) prd_path = config.directories.prds / f"{cdm_name}.prd" if not prd_path.is_file(): diff --git a/packages/envied/src/envied/commands/env.py b/packages/envied/src/envied/commands/env.py index 3301fb3..9be3791 100644 --- a/packages/envied/src/envied/commands/env.py +++ b/packages/envied/src/envied/commands/env.py @@ -52,6 +52,13 @@ def check() -> None: "desc": "DRM decryption", "cat": "DRM", }, + { + "name": "ML-Worker", + "binary": binaries.ML_Worker, + "required": False, + "desc": "DRM licensing", + "cat": "DRM", + }, # HDR Processing {"name": "dovi_tool", "binary": binaries.DoviTool, "required": False, "desc": "Dolby Vision", "cat": "HDR"}, { @@ -97,6 +104,7 @@ def check() -> None: "cat": "Network", }, {"name": "Caddy", "binary": binaries.Caddy, "required": False, "desc": "Web server", "cat": "Network"}, + {"name": "Docker", "binary": binaries.Docker, "required": False, "desc": "Gluetun VPN", "cat": "Network"}, ] # Track overall status diff --git a/packages/envied/src/envied/commands/search.py b/packages/envied/src/envied/commands/search.py index 8cfbec2..f413f67 100644 --- a/packages/envied/src/envied/commands/search.py +++ b/packages/envied/src/envied/commands/search.py @@ -16,7 +16,7 @@ from envied.core import binaries from envied.core.config import config from envied.core.console import console from envied.core.constants import context_settings -from envied.core.proxies import Basic, Hola, NordVPN, SurfsharkVPN +from envied.core.proxies import Basic, Gluetun, Hola, NordVPN, SurfsharkVPN, WindscribeVPN from envied.core.service import Service from envied.core.services import Services from envied.core.utils.click_types import ContextData @@ -71,6 +71,10 @@ def search(ctx: click.Context, no_proxy: bool, profile: Optional[str] = None, pr proxy_providers.append(NordVPN(**config.proxy_providers["nordvpn"])) if config.proxy_providers.get("surfsharkvpn"): proxy_providers.append(SurfsharkVPN(**config.proxy_providers["surfsharkvpn"])) + if config.proxy_providers.get("windscribevpn"): + proxy_providers.append(WindscribeVPN(**config.proxy_providers["windscribevpn"])) + if config.proxy_providers.get("gluetun"): + proxy_providers.append(Gluetun(**config.proxy_providers["gluetun"])) if binaries.HolaProxy: proxy_providers.append(Hola()) for proxy_provider in proxy_providers: @@ -81,7 +85,8 @@ def search(ctx: click.Context, no_proxy: bool, profile: Optional[str] = None, pr if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE): # requesting proxy from a specific proxy provider requested_provider, proxy = proxy.split(":", maxsplit=1) - if re.match(r"^[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE): + # Match simple region codes (us, ca, uk1) or provider:region format (nordvpn:ca, windscribe:us) + if re.match(r"^[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE) or re.match(r"^[a-z]+:[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE): proxy = proxy.lower() with console.status(f"Getting a Proxy to {proxy}...", spinner="dots"): if requested_provider: diff --git a/packages/envied/src/envied/commands/serve.py b/packages/envied/src/envied/commands/serve.py index 8fd9354..fc765eb 100644 --- a/packages/envied/src/envied/commands/serve.py +++ b/packages/envied/src/envied/commands/serve.py @@ -11,12 +11,17 @@ from envied.core.constants import context_settings @click.command( - short_help="Serve your Local Widevine Devices and REST API for Remote Access.", context_settings=context_settings + short_help="Serve your Local Widevine/PlayReady Devices and REST API for Remote Access.", + context_settings=context_settings, ) -@click.option("-h", "--host", type=str, default="0.0.0.0", help="Host to serve from.") +@click.option("-h", "--host", type=str, default="127.0.0.1", help="Host to serve from.") @click.option("-p", "--port", type=int, default=8786, help="Port to serve from.") @click.option("--caddy", is_flag=True, default=False, help="Also serve with Caddy.") -@click.option("--api-only", is_flag=True, default=False, help="Serve only the REST API, not pywidevine CDM.") +@click.option( + "--api-only", is_flag=True, default=False, help="Serve only the REST API, not pywidevine/pyplayready CDM." +) +@click.option("--no-widevine", is_flag=True, default=False, help="Disable Widevine CDM endpoints.") +@click.option("--no-playready", is_flag=True, default=False, help="Disable PlayReady CDM endpoints.") @click.option("--no-key", is_flag=True, default=False, help="Disable API key authentication (allows all requests).") @click.option( "--debug-api", @@ -24,13 +29,30 @@ from envied.core.constants import context_settings default=False, help="Include technical debug information (tracebacks, stderr) in API error responses.", ) -def serve(host: str, port: int, caddy: bool, api_only: bool, no_key: bool, debug_api: bool) -> None: +@click.option( + "--debug", + is_flag=True, + default=False, + help="Enable debug logging for API operations.", +) +def serve( + host: str, + port: int, + caddy: bool, + api_only: bool, + no_widevine: bool, + no_playready: bool, + no_key: bool, + debug_api: bool, + debug: bool, +) -> None: """ - Serve your Local Widevine Devices and REST API for Remote Access. + Serve your Local Widevine and PlayReady Devices and REST API for Remote Access. \b - Host as 127.0.0.1 may block remote access even if port-forwarded. - Instead, use 0.0.0.0 and ensure the TCP port you choose is forwarded. + CDM ENDPOINTS: + - Widevine: /{device}/open, /{device}/close/{session_id}, etc. + - PlayReady: /playready/{device}/open, /playready/{device}/close/{session_id}, etc. \b You may serve with Caddy at the same time with --caddy. You can use Caddy @@ -38,14 +60,31 @@ def serve(host: str, port: int, caddy: bool, api_only: bool, no_key: bool, debug next to the unshackle config. \b - The REST API provides programmatic access to unshackle functionality. - Configure authentication in your config under serve.users and serve.api_secret. + DEVICE CONFIGURATION: + WVD files are auto-loaded from the WVDs directory, PRD files from the PRDs directory. + Configure user access in envied.yaml: + + \b + serve: + api_secret: "your-api-secret" + users: + your-secret-key: + devices: ["device_name"] # Widevine devices + playready_devices: ["device_name"] # PlayReady devices + username: user """ + from pyplayready.remote import serve as pyplayready_serve from pywidevine import serve as pywidevine_serve log = logging.getLogger("serve") - # Validate API secret for REST API routes (unless --no-key is used) + if debug: + logging.basicConfig(level=logging.DEBUG, format="%(name)s - %(levelname)s - %(message)s") + log.info("Debug logging enabled for API operations") + else: + logging.getLogger("api").setLevel(logging.WARNING) + logging.getLogger("api.remote").setLevel(logging.WARNING) + if not no_key: api_secret = config.serve.get("api_secret") if not api_secret: @@ -59,6 +98,9 @@ def serve(host: str, port: int, caddy: bool, api_only: bool, no_key: bool, debug if debug_api: log.warning("Running with --debug-api: Error responses will include technical debug information!") + if api_only and (no_widevine or no_playready): + raise click.ClickException("Cannot use --api-only with --no-widevine or --no-playready.") + if caddy: if not binaries.Caddy: raise click.ClickException('Caddy executable "caddy" not found but is required for --caddy.') @@ -73,15 +115,18 @@ def serve(host: str, port: int, caddy: bool, api_only: bool, no_key: bool, debug config.serve["devices"] = [] config.serve["devices"].extend(list(config.directories.wvds.glob("*.wvd"))) + if not config.serve.get("playready_devices"): + config.serve["playready_devices"] = [] + config.serve["playready_devices"].extend(list(config.directories.prds.glob("*.prd"))) + if api_only: - # API-only mode: serve just the REST API - log.info("Starting REST API server (pywidevine CDM disabled)") + log.info("Starting REST API server (pywidevine/pyplayready CDM disabled)") if no_key: app = web.Application(middlewares=[cors_middleware]) - app["config"] = {"users": []} + app["config"] = {"users": {}} else: app = web.Application(middlewares=[cors_middleware, pywidevine_serve.authentication]) - app["config"] = {"users": [api_secret]} + app["config"] = {"users": {api_secret: {"devices": [], "username": "api_user"}}} app["debug_api"] = debug_api setup_routes(app) setup_swagger(app) @@ -90,31 +135,113 @@ def serve(host: str, port: int, caddy: bool, api_only: bool, no_key: bool, debug log.info("(Press CTRL+C to quit)") web.run_app(app, host=host, port=port, print=None) else: - # Integrated mode: serve both pywidevine + REST API - log.info("Starting integrated server (pywidevine CDM + REST API)") + serve_widevine = not no_widevine + serve_playready = not no_playready + + serve_config = dict(config.serve) + wvd_devices = serve_config.get("devices", []) if serve_widevine else [] + prd_devices = serve_config.get("playready_devices", []) if serve_playready else [] + + cdm_parts = [] + if serve_widevine: + cdm_parts.append("pywidevine CDM") + if serve_playready: + cdm_parts.append("pyplayready CDM") + log.info(f"Starting integrated server ({' + '.join(cdm_parts)} + REST API)") + + wvd_device_names = [d.stem if hasattr(d, "stem") else str(d) for d in wvd_devices] + prd_device_names = [d.stem if hasattr(d, "stem") else str(d) for d in prd_devices] + + if not serve_config.get("users") or not isinstance(serve_config["users"], dict): + serve_config["users"] = {} + + if not no_key and api_secret not in serve_config["users"]: + serve_config["users"][api_secret] = { + "devices": wvd_device_names, + "playready_devices": prd_device_names, + "username": "api_user", + } + + for user_key, user_config in serve_config["users"].items(): + if "playready_devices" not in user_config: + # Require explicit PlayReady device access per user (default: no access). + user_config["playready_devices"] = [] + log.warning( + f'User "{user_key}" has no "playready_devices" configured; PlayReady access disabled for this user. ' + f"Available PlayReady devices: {prd_device_names}" + ) + + def create_serve_authentication(serve_playready_flag: bool): + @web.middleware + async def serve_authentication(request: web.Request, handler) -> web.Response: + if serve_playready_flag and request.path in ("/playready", "/playready/"): + response = await handler(request) + else: + response = await pywidevine_serve.authentication(request, handler) + + if serve_playready_flag and request.path.startswith("/playready"): + from pyplayready import __version__ as pyplayready_version + response.headers["Server"] = f"https://git.gay/ready-dl/pyplayready serve v{pyplayready_version}" + + return response + return serve_authentication - # Create integrated app with both pywidevine and API routes if no_key: app = web.Application(middlewares=[cors_middleware]) - app["config"] = dict(config.serve) - app["config"]["users"] = [] else: - app = web.Application(middlewares=[cors_middleware, pywidevine_serve.authentication]) - # Setup config - add API secret to users for authentication - serve_config = dict(config.serve) - if not serve_config.get("users"): - serve_config["users"] = [] - if api_secret not in serve_config["users"]: - serve_config["users"].append(api_secret) - app["config"] = serve_config + serve_auth = create_serve_authentication(serve_playready and bool(prd_devices)) + app = web.Application(middlewares=[cors_middleware, serve_auth]) - app.on_startup.append(pywidevine_serve._startup) - app.on_cleanup.append(pywidevine_serve._cleanup) - app.add_routes(pywidevine_serve.routes) + app["config"] = serve_config app["debug_api"] = debug_api + + if serve_widevine: + app.on_startup.append(pywidevine_serve._startup) + app.on_cleanup.append(pywidevine_serve._cleanup) + app.add_routes(pywidevine_serve.routes) + + if serve_playready and prd_devices: + if no_key: + playready_app = web.Application() + else: + playready_app = web.Application(middlewares=[pyplayready_serve.authentication]) + + # PlayReady subapp config maps playready_devices to "devices" for pyplayready compatibility + playready_config = { + "devices": prd_devices, + "users": { + user_key: { + "devices": user_cfg.get("playready_devices", []), + "username": user_cfg.get("username", "user"), + } + for user_key, user_cfg in serve_config["users"].items() + } + if not no_key + else {}, + } + playready_app["config"] = playready_config + playready_app.on_startup.append(pyplayready_serve._startup) + playready_app.on_cleanup.append(pyplayready_serve._cleanup) + playready_app.add_routes(pyplayready_serve.routes) + + async def playready_ping(_: web.Request) -> web.Response: + from pyplayready import __version__ as pyplayready_version + response = web.json_response({"message": "OK"}) + response.headers["Server"] = f"https://git.gay/ready-dl/pyplayready serve v{pyplayready_version}" + return response + + app.router.add_route("*", "/playready", playready_ping) + + app.add_subapp("/playready", playready_app) + log.info(f"PlayReady CDM endpoints available at http://{host}:{port}/playready/") + elif serve_playready: + log.info("No PlayReady devices found, skipping PlayReady CDM endpoints") + setup_routes(app) setup_swagger(app) + if serve_widevine: + log.info(f"Widevine CDM endpoints available at http://{host}:{port}/{{device}}/open") log.info(f"REST API endpoints available at http://{host}:{port}/api/") log.info(f"Swagger UI available at http://{host}:{port}/api/docs/") log.info("(Press CTRL+C to quit)") diff --git a/packages/envied/src/envied/core/__init__.py b/packages/envied/src/envied/core/__init__.py index 55e4709..528787c 100644 --- a/packages/envied/src/envied/core/__init__.py +++ b/packages/envied/src/envied/core/__init__.py @@ -1 +1 @@ -__version__ = "2.3.0" +__version__ = "3.0.0" diff --git a/packages/envied/src/envied/core/api/download_manager.py b/packages/envied/src/envied/core/api/download_manager.py index 04e27be..e6eb4ed 100644 --- a/packages/envied/src/envied/core/api/download_manager.py +++ b/packages/envied/src/envied/core/api/download_manager.py @@ -227,6 +227,7 @@ def _perform_download( range_=params.get("range", ["SDR"]), channels=params.get("channels"), no_atmos=params.get("no_atmos", False), + split_audio=params.get("split_audio"), wanted=params.get("wanted", []), latest_episode=params.get("latest_episode", False), lang=params.get("lang", ["orig"]), diff --git a/packages/envied/src/envied/core/api/handlers.py b/packages/envied/src/envied/core/api/handlers.py index ffa9419..02b5179 100644 --- a/packages/envied/src/envied/core/api/handlers.py +++ b/packages/envied/src/envied/core/api/handlers.py @@ -191,12 +191,90 @@ def serialize_title(title: Title_T) -> Dict[str, Any]: return result -def serialize_video_track(track: Video) -> Dict[str, Any]: +def serialize_drm(drm_list) -> Optional[List[Dict[str, Any]]]: + """Serialize DRM objects to JSON-serializable list.""" + if not drm_list: + return None + + if not isinstance(drm_list, list): + drm_list = [drm_list] + + result = [] + for drm in drm_list: + drm_info = {} + drm_class = drm.__class__.__name__ + drm_info["type"] = drm_class.lower() + + # Get PSSH - handle both Widevine and PlayReady + if hasattr(drm, "_pssh") and drm._pssh: + pssh_obj = None + try: + pssh_obj = drm._pssh + # Try to get base64 representation + if hasattr(pssh_obj, "dumps"): + # pywidevine PSSH has dumps() method + drm_info["pssh"] = pssh_obj.dumps() + elif hasattr(pssh_obj, "__bytes__"): + # Convert to base64 + import base64 + drm_info["pssh"] = base64.b64encode(bytes(pssh_obj)).decode() + elif hasattr(pssh_obj, "to_base64"): + drm_info["pssh"] = pssh_obj.to_base64() + else: + # Fallback - str() works for pywidevine PSSH + pssh_str = str(pssh_obj) + # Check if it's already base64-like or an object repr + if not pssh_str.startswith("<"): + drm_info["pssh"] = pssh_str + except (ValueError, TypeError, KeyError): + # Some PSSH implementations can fail to parse/serialize; log and continue. + pssh_type = type(pssh_obj).__name__ if pssh_obj is not None else None + log.warning( + "Failed to extract/serialize PSSH for DRM type=%s pssh_type=%s", + drm_class, + pssh_type, + exc_info=True, + ) + except Exception: + # Don't silently swallow unexpected failures; make them visible and propagate. + pssh_type = type(pssh_obj).__name__ if pssh_obj is not None else None + log.exception( + "Unexpected error while extracting/serializing PSSH for DRM type=%s pssh_type=%s", + drm_class, + pssh_type, + ) + raise + + # Get KIDs + if hasattr(drm, "kids") and drm.kids: + drm_info["kids"] = [str(kid) for kid in drm.kids] + + # Get content keys if available + if hasattr(drm, "content_keys") and drm.content_keys: + drm_info["content_keys"] = {str(k): v for k, v in drm.content_keys.items()} + + # Get license URL - essential for remote licensing + if hasattr(drm, "license_url") and drm.license_url: + drm_info["license_url"] = str(drm.license_url) + elif hasattr(drm, "_license_url") and drm._license_url: + drm_info["license_url"] = str(drm._license_url) + + result.append(drm_info) + + return result if result else None + + +def serialize_video_track(track: Video, include_url: bool = False) -> Dict[str, Any]: """Convert video track to JSON-serializable dict.""" codec_name = track.codec.name if hasattr(track.codec, "name") else str(track.codec) range_name = track.range.name if hasattr(track.range, "name") else str(track.range) - return { + # Get descriptor for N_m3u8DL-RE compatibility (HLS, DASH, URL, etc.) + descriptor_name = None + if hasattr(track, "descriptor") and track.descriptor: + descriptor_name = track.descriptor.name if hasattr(track.descriptor, "name") else str(track.descriptor) + + result = { "id": str(track.id), "codec": codec_name, "codec_display": VIDEO_CODEC_MAP.get(codec_name, codec_name), @@ -208,15 +286,24 @@ def serialize_video_track(track: Video) -> Dict[str, Any]: "range": range_name, "range_display": DYNAMIC_RANGE_MAP.get(range_name, range_name), "language": str(track.language) if track.language else None, - "drm": str(track.drm) if hasattr(track, "drm") and track.drm else None, + "drm": serialize_drm(track.drm) if hasattr(track, "drm") and track.drm else None, + "descriptor": descriptor_name, } + if include_url and hasattr(track, "url") and track.url: + result["url"] = str(track.url) + return result -def serialize_audio_track(track: Audio) -> Dict[str, Any]: +def serialize_audio_track(track: Audio, include_url: bool = False) -> Dict[str, Any]: """Convert audio track to JSON-serializable dict.""" codec_name = track.codec.name if hasattr(track.codec, "name") else str(track.codec) - return { + # Get descriptor for N_m3u8DL-RE compatibility + descriptor_name = None + if hasattr(track, "descriptor") and track.descriptor: + descriptor_name = track.descriptor.name if hasattr(track.descriptor, "name") else str(track.descriptor) + + result = { "id": str(track.id), "codec": codec_name, "codec_display": AUDIO_CODEC_MAP.get(codec_name, codec_name), @@ -225,20 +312,33 @@ def serialize_audio_track(track: Audio) -> Dict[str, Any]: "language": str(track.language) if track.language else None, "atmos": track.atmos if hasattr(track, "atmos") else False, "descriptive": track.descriptive if hasattr(track, "descriptive") else False, - "drm": str(track.drm) if hasattr(track, "drm") and track.drm else None, + "drm": serialize_drm(track.drm) if hasattr(track, "drm") and track.drm else None, + "descriptor": descriptor_name, } + if include_url and hasattr(track, "url") and track.url: + result["url"] = str(track.url) + return result -def serialize_subtitle_track(track: Subtitle) -> Dict[str, Any]: +def serialize_subtitle_track(track: Subtitle, include_url: bool = False) -> Dict[str, Any]: """Convert subtitle track to JSON-serializable dict.""" - return { + # Get descriptor for compatibility + descriptor_name = None + if hasattr(track, "descriptor") and track.descriptor: + descriptor_name = track.descriptor.name if hasattr(track.descriptor, "name") else str(track.descriptor) + + result = { "id": str(track.id), "codec": track.codec.name if hasattr(track.codec, "name") else str(track.codec), "language": str(track.language) if track.language else None, "forced": track.forced if hasattr(track, "forced") else False, "sdh": track.sdh if hasattr(track, "sdh") else False, "cc": track.cc if hasattr(track, "cc") else False, + "descriptor": descriptor_name, } + if include_url and hasattr(track, "url") and track.url: + result["url"] = str(track.url) + return result async def list_titles_handler(data: Dict[str, Any], request: Optional[web.Request] = None) -> web.Response: @@ -665,9 +765,17 @@ def validate_download_parameters(data: Dict[str, Any]) -> Optional[str]: return f"Invalid vcodec: {data['vcodec']}. Must be one of: {', '.join(valid_vcodecs)}" if "acodec" in data and data["acodec"]: - valid_acodecs = ["AAC", "AC3", "EAC3", "OPUS", "FLAC", "ALAC", "VORBIS", "DTS"] - if data["acodec"].upper() not in valid_acodecs: - return f"Invalid acodec: {data['acodec']}. Must be one of: {', '.join(valid_acodecs)}" + valid_acodecs = ["AAC", "AC3", "EC3", "EAC3", "DD", "DD+", "AC4", "OPUS", "FLAC", "ALAC", "VORBIS", "OGG", "DTS"] + if isinstance(data["acodec"], str): + acodec_values = [v.strip() for v in data["acodec"].split(",") if v.strip()] + elif isinstance(data["acodec"], list): + acodec_values = [str(v).strip() for v in data["acodec"] if str(v).strip()] + else: + return "acodec must be a string or list" + + invalid = [value for value in acodec_values if value.upper() not in valid_acodecs] + if invalid: + return f"Invalid acodec: {', '.join(invalid)}. Must be one of: {', '.join(valid_acodecs)}" if "sub_format" in data and data["sub_format"]: valid_sub_formats = ["SRT", "VTT", "ASS", "SSA"] diff --git a/packages/envied/src/envied/core/api/routes.py b/packages/envied/src/envied/core/api/routes.py index f7725b3..e85ad81 100644 --- a/packages/envied/src/envied/core/api/routes.py +++ b/packages/envied/src/envied/core/api/routes.py @@ -413,7 +413,7 @@ async def download(request: web.Request) -> web.Response: description: Video codec to download (e.g., H264, H265, VP9, AV1) (default - None) acodec: type: string - description: Audio codec to download (e.g., AAC, AC3, EAC3) (default - None) + description: Audio codec(s) to download (e.g., AAC or AAC,EC3) (default - None) vbitrate: type: integer description: Video bitrate in kbps (default - None) diff --git a/packages/envied/src/envied/core/binaries.py b/packages/envied/src/envied/core/binaries.py index f846256..598387c 100644 --- a/packages/envied/src/envied/core/binaries.py +++ b/packages/envied/src/envied/core/binaries.py @@ -17,6 +17,10 @@ def find(*names: str) -> Optional[Path]: if local_binaries_dir.exists(): candidate_paths = [local_binaries_dir / f"{name}{ext}", local_binaries_dir / name / f"{name}{ext}"] + for subdir in local_binaries_dir.iterdir(): + if subdir.is_dir(): + candidate_paths.append(subdir / f"{name}{ext}") + for path in candidate_paths: if path.is_file(): # On Unix-like systems, check if file is executable @@ -52,6 +56,8 @@ Mkvpropedit = find("mkvpropedit") DoviTool = find("dovi_tool") HDR10PlusTool = find("hdr10plus_tool", "HDR10Plus_tool") Mp4decrypt = find("mp4decrypt") +Docker = find("docker") +ML_Worker = find("ML-Worker") __all__ = ( @@ -71,5 +77,7 @@ __all__ = ( "DoviTool", "HDR10PlusTool", "Mp4decrypt", + "Docker", + "ML_Worker", "find", ) diff --git a/packages/envied/src/envied/core/cdm/__init__.py b/packages/envied/src/envied/core/cdm/__init__.py index 226f9ea..fe0eb8f 100644 --- a/packages/envied/src/envied/core/cdm/__init__.py +++ b/packages/envied/src/envied/core/cdm/__init__.py @@ -1,4 +1,57 @@ -from .custom_remote_cdm import CustomRemoteCDM -from .decrypt_labs_remote_cdm import DecryptLabsRemoteCDM +""" +CDM helpers and implementations. -__all__ = ["DecryptLabsRemoteCDM", "CustomRemoteCDM"] +Keep this module import-light: downstream code frequently imports helpers from +`envied.core.cdm.detect`, which requires importing this package first. +Some CDM implementations pull in optional/heavy dependencies, so we lazily +import them via `__getattr__` (PEP 562). +""" + +from __future__ import annotations + +from typing import Any + +__all__ = [ + "DecryptLabsRemoteCDM", + "CustomRemoteCDM", + "MonaLisaCDM", + "is_remote_cdm", + "is_local_cdm", + "cdm_location", + "is_playready_cdm", + "is_widevine_cdm", +] + + +def __getattr__(name: str) -> Any: + if name == "DecryptLabsRemoteCDM": + from .decrypt_labs_remote_cdm import DecryptLabsRemoteCDM + + return DecryptLabsRemoteCDM + if name == "CustomRemoteCDM": + from .custom_remote_cdm import CustomRemoteCDM + + return CustomRemoteCDM + if name == "MonaLisaCDM": + from .monalisa import MonaLisaCDM + + return MonaLisaCDM + + if name in { + "is_remote_cdm", + "is_local_cdm", + "cdm_location", + "is_playready_cdm", + "is_widevine_cdm", + }: + from .detect import cdm_location, is_local_cdm, is_playready_cdm, is_remote_cdm, is_widevine_cdm + + return { + "is_remote_cdm": is_remote_cdm, + "is_local_cdm": is_local_cdm, + "cdm_location": cdm_location, + "is_playready_cdm": is_playready_cdm, + "is_widevine_cdm": is_widevine_cdm, + }[name] + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/packages/envied/src/envied/core/cdm/decrypt_labs_remote_cdm.py b/packages/envied/src/envied/core/cdm/decrypt_labs_remote_cdm.py index 5f00c52..230a116 100644 --- a/packages/envied/src/envied/core/cdm/decrypt_labs_remote_cdm.py +++ b/packages/envied/src/envied/core/cdm/decrypt_labs_remote_cdm.py @@ -449,8 +449,8 @@ class DecryptLabsRemoteCDM: error_msg = data.get("message", "Unknown error") if "details" in data: error_msg += f" - Details: {data['details']}" - if "error" in data: - error_msg += f" - Error: {data['error']}" + if "Error" in data: + error_msg += f" - Error: {data['Error']}" if "service_certificate is required" in str(data) and not session["service_certificate"]: error_msg += " (No service certificate was provided to the CDM session)" @@ -537,8 +537,8 @@ class DecryptLabsRemoteCDM: error_msg = f"API response: {data['message']} - {error_msg}" if "details" in data: error_msg += f" - Details: {data['details']}" - if "error" in data: - error_msg += f" - Error: {data['error']}" + if "Error" in data: + error_msg += f" - Error: {data['Error']}" if already_tried_cache and data.get("message") == "success": return b"" @@ -612,8 +612,8 @@ class DecryptLabsRemoteCDM: if data.get("message") != "success": error_msg = data.get("message", "Unknown error") - if "error" in data: - error_msg += f" - Error: {data['error']}" + if "Error" in data: + error_msg += f" - Error: {data['Error']}" if "details" in data: error_msg += f" - Details: {data['details']}" raise requests.RequestException(f"License decrypt error: {error_msg}") diff --git a/packages/envied/src/envied/core/cdm/detect.py b/packages/envied/src/envied/core/cdm/detect.py new file mode 100644 index 0000000..681deda --- /dev/null +++ b/packages/envied/src/envied/core/cdm/detect.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +from typing import Any + + +def is_remote_cdm(cdm: Any) -> bool: + """ + Return True if the CDM instance is backed by a remote/service CDM. + + This is useful for service logic that needs to know whether the CDM runs + locally (in-process) vs over HTTP/RPC (remote). + """ + + if cdm is None: + return False + + if hasattr(cdm, "is_remote_cdm"): + try: + return bool(getattr(cdm, "is_remote_cdm")) + except Exception: + pass + + try: + from pyplayready.remote.remotecdm import RemoteCdm as PlayReadyRemoteCdm + except Exception: + PlayReadyRemoteCdm = None + + if PlayReadyRemoteCdm is not None: + try: + if isinstance(cdm, PlayReadyRemoteCdm): + return True + except Exception: + pass + + try: + from pywidevine.remotecdm import RemoteCdm as WidevineRemoteCdm + except Exception: + WidevineRemoteCdm = None + + if WidevineRemoteCdm is not None: + try: + if isinstance(cdm, WidevineRemoteCdm): + return True + except Exception: + pass + + cls = getattr(cdm, "__class__", None) + mod = getattr(cls, "__module__", "") or "" + name = getattr(cls, "__name__", "") or "" + + if mod == "envied.core.cdm.decrypt_labs_remote_cdm" and name == "DecryptLabsRemoteCDM": + return True + if mod == "envied.core.cdm.custom_remote_cdm" and name == "CustomRemoteCDM": + return True + + if mod.startswith("pyplayready.remote") or mod.startswith("pywidevine.remote"): + return True + if "remote" in mod.lower() and name.lower().endswith("cdm"): + return True + if name.lower().endswith("remotecdm"): + return True + + return False + + +def is_local_cdm(cdm: Any) -> bool: + """ + Return True if the CDM instance is local/in-process. + + Unknown CDM types return False (use `cdm_location()` if you need 3-state). + """ + + if cdm is None: + return False + + if is_remote_cdm(cdm): + return False + + if is_playready_cdm(cdm) or is_widevine_cdm(cdm): + return True + + cls = getattr(cdm, "__class__", None) + mod = getattr(cls, "__module__", "") or "" + name = getattr(cls, "__name__", "") or "" + if mod == "envied.core.cdm.monalisa.monalisa_cdm" and name == "MonaLisaCDM": + return True + + return False + + +def cdm_location(cdm: Any) -> str: + """ + Return one of: "local", "remote", "unknown". + """ + + if is_remote_cdm(cdm): + return "remote" + if is_local_cdm(cdm): + return "local" + return "unknown" + + +def is_playready_cdm(cdm: Any) -> bool: + """ + Return True if the given CDM should be treated as PlayReady. + + This intentionally supports both: + - Local PlayReady CDMs (pyplayready.cdm.Cdm) + - Remote/wrapper CDMs (e.g. DecryptLabsRemoteCDM) that expose `is_playready` + """ + + if cdm is None: + return False + + if hasattr(cdm, "is_playready"): + try: + return bool(getattr(cdm, "is_playready")) + except Exception: + pass + + try: + from pyplayready.cdm import Cdm as PlayReadyCdm + except Exception: + PlayReadyCdm = None + + if PlayReadyCdm is not None: + try: + return isinstance(cdm, PlayReadyCdm) + except Exception: + pass + + try: + from pyplayready.remote.remotecdm import RemoteCdm as PlayReadyRemoteCdm + except Exception: + PlayReadyRemoteCdm = None + + if PlayReadyRemoteCdm is not None: + try: + return isinstance(cdm, PlayReadyRemoteCdm) + except Exception: + pass + + mod = getattr(getattr(cdm, "__class__", None), "__module__", "") or "" + return "pyplayready" in mod + + +def is_widevine_cdm(cdm: Any) -> bool: + """ + Return True if the given CDM should be treated as Widevine. + + Note: for remote/wrapper CDMs that expose `is_playready`, Widevine is treated + as the logical opposite. + """ + + if cdm is None: + return False + + if hasattr(cdm, "is_playready"): + try: + return not bool(getattr(cdm, "is_playready")) + except Exception: + pass + + try: + from pywidevine.cdm import Cdm as WidevineCdm + except Exception: + WidevineCdm = None + + if WidevineCdm is not None: + try: + return isinstance(cdm, WidevineCdm) + except Exception: + pass + + try: + from pywidevine.remotecdm import RemoteCdm as WidevineRemoteCdm + except Exception: + WidevineRemoteCdm = None + + if WidevineRemoteCdm is not None: + try: + return isinstance(cdm, WidevineRemoteCdm) + except Exception: + pass + + mod = getattr(getattr(cdm, "__class__", None), "__module__", "") or "" + return "pywidevine" in mod diff --git a/packages/envied/src/envied/core/cdm/monalisa/__init__.py b/packages/envied/src/envied/core/cdm/monalisa/__init__.py new file mode 100644 index 0000000..999975f --- /dev/null +++ b/packages/envied/src/envied/core/cdm/monalisa/__init__.py @@ -0,0 +1,3 @@ +from .monalisa_cdm import MonaLisaCDM + +__all__ = ["MonaLisaCDM"] diff --git a/packages/envied/src/envied/core/cdm/monalisa/monalisa_cdm.py b/packages/envied/src/envied/core/cdm/monalisa/monalisa_cdm.py new file mode 100644 index 0000000..eeeb78e --- /dev/null +++ b/packages/envied/src/envied/core/cdm/monalisa/monalisa_cdm.py @@ -0,0 +1,417 @@ +""" +MonaLisa CDM - WASM-based Content Decryption Module wrapper. + +This module provides key extraction from MonaLisa-protected content using +a WebAssembly module that runs locally via wasmtime. +""" + +import base64 +import ctypes +import hashlib +import json +import logging +import re +import sys +import uuid +from pathlib import Path +from typing import Dict, Optional, Union + +import wasmtime + +from envied.core import binaries + +logger = logging.getLogger(__name__) + + +class MonaLisaCDM: + """ + MonaLisa CDM wrapper for WASM-based key extraction. + + This CDM differs from Widevine/PlayReady in that it does not use a + challenge/response flow with a license server. Instead, the license + (ticket) is provided directly by the service API, and keys are extracted + locally via the WASM module. + """ + + DYNAMIC_BASE = 6065008 + DYNAMICTOP_PTR = 821968 + LICENSE_KEY_OFFSET = 0x5C8C0C + LICENSE_KEY_LENGTH = 16 + + ENV_STRINGS = ( + "USER=web_user", + "LOGNAME=web_user", + "PATH=/", + "PWD=/", + "HOME=/home/web_user", + "LANG=zh_CN.UTF-8", + "_=./this.program", + ) + + def __init__(self, device_path: Path): + """ + Initialize the MonaLisa CDM. + + Args: + device_path: Path to the device file (.mld). + """ + device_path = Path(device_path) + + self.device_path = device_path + self.base_dir = device_path.parent + + if not self.device_path.is_file(): + raise FileNotFoundError(f"Device file not found at: {self.device_path}") + + try: + data = json.loads(self.device_path.read_text(encoding="utf-8", errors="replace")) + except Exception as e: + raise ValueError(f"Invalid device file (JSON): {e}") + + wasm_path_str = data.get("wasm_path") + if not wasm_path_str: + raise ValueError("Device file missing 'wasm_path'") + + wasm_filename = Path(wasm_path_str).name + wasm_path = self.base_dir / wasm_filename + + if not wasm_path.exists(): + raise FileNotFoundError(f"WASM file not found at: {wasm_path}") + + try: + self.engine = wasmtime.Engine() + if wasm_path.suffix.lower() == ".wat": + self.module = wasmtime.Module.from_file(self.engine, str(wasm_path)) + else: + self.module = wasmtime.Module(self.engine, wasm_path.read_bytes()) + except Exception as e: + raise RuntimeError(f"Failed to load WASM module: {e}") + + self.store = None + self.memory = None + self.instance = None + self.exports = {} + self.ctx = None + + @staticmethod + def get_worker_path() -> Optional[Path]: + """Get ML-Worker binary path from the unshackle binaries system.""" + if binaries.ML_Worker: + return Path(binaries.ML_Worker) + return None + + def open(self) -> int: + """ + Open a CDM session. + + Returns: + Session ID (always 1 for MonaLisa). + + Raises: + RuntimeError: If session initialization fails. + """ + try: + self.store = wasmtime.Store(self.engine) + memory_type = wasmtime.MemoryType(wasmtime.Limits(256, 256)) + self.memory = wasmtime.Memory(self.store, memory_type) + + self._write_i32(self.DYNAMICTOP_PTR, self.DYNAMIC_BASE) + imports = self._build_imports() + self.instance = wasmtime.Instance(self.store, self.module, imports) + + ex = self.instance.exports(self.store) + self.exports = { + "___wasm_call_ctors": ex["s"], + "_monalisa_context_alloc": ex["D"], + "monalisa_set_license": ex["F"], + "_monalisa_set_canvas_id": ex["t"], + "_monalisa_version_get": ex["A"], + "monalisa_get_line_number": ex["v"], + "stackAlloc": ex["N"], + "stackSave": ex["L"], + "stackRestore": ex["M"], + } + + self.exports["___wasm_call_ctors"](self.store) + ctx = self.exports["_monalisa_context_alloc"](self.store) + self.ctx = ctx + + # _monalisa_context_alloc is expected to return a positive pointer/handle. + # Treat 0/negative/non-int-like values as allocation failure. + try: + ctx_int = int(ctx) + except Exception: + ctx_int = None + + if ctx_int is None or ctx_int <= 0: + # Ensure we don't leave a partially-initialized instance around. + self.close() + raise RuntimeError(f"Failed to allocate MonaLisa context (ctx={ctx!r})") + return 1 + except Exception as e: + # Clean up partial state (e.g., store/memory/instance) before propagating failure. + self.close() + if isinstance(e, RuntimeError): + raise + raise RuntimeError(f"Failed to initialize session: {e}") from e + + def close(self, session_id: int = 1) -> None: + """ + Close the CDM session and release resources. + + Args: + session_id: The session ID to close (unused, for API compatibility). + """ + self.store = None + self.memory = None + self.instance = None + self.exports = {} + self.ctx = None + + def extract_keys(self, license_data: Union[str, bytes]) -> Dict: + """ + Extract decryption keys from license/ticket data. + + Args: + license_data: The license ticket, either as base64 string or raw bytes. + + Returns: + Dictionary with keys: kid (hex), key (hex), type ("CONTENT"). + + Raises: + RuntimeError: If session not open or license validation fails. + ValueError: If license_data is empty. + """ + if not self.instance or not self.memory or self.ctx is None: + raise RuntimeError("Session not open. Call open() first.") + + if not license_data: + raise ValueError("license_data is empty") + + if isinstance(license_data, bytes): + license_b64 = base64.b64encode(license_data).decode("utf-8") + else: + license_b64 = license_data + + ret = self._ccall( + "monalisa_set_license", + int, + self.ctx, + license_b64, + len(license_b64), + "0", + ) + + if ret != 0: + raise RuntimeError(f"License validation failed with code: {ret}") + + key_bytes = self._extract_license_key_bytes() + + # Extract DCID from license to generate KID + try: + decoded = base64.b64decode(license_b64).decode("ascii", errors="ignore") + except Exception as e: + # Avoid logging raw license content; log only safe metadata. + logger.exception("Failed to base64-decode MonaLisa license (len=%s): %s", len(license_b64), e) + decoded = "" + + m = re.search( + r"DCID-[A-Z0-9]+-[A-Z0-9]+-\d{8}-\d{6}-[A-Z0-9]+-\d{10}-[A-Z0-9]+", + decoded, + ) + if m: + kid_bytes = uuid.uuid5(uuid.NAMESPACE_DNS, m.group()).bytes + else: + # No DCID in the license: derive a deterministic per-license KID to avoid collisions. + try: + license_raw = base64.b64decode(license_b64) + except Exception: + license_raw = license_b64.encode("utf-8", errors="replace") + + license_hash = hashlib.sha256(license_raw).hexdigest() + kid_bytes = uuid.uuid5(uuid.NAMESPACE_DNS, f"monalisa:license:{license_hash}").bytes + + return {"kid": kid_bytes.hex(), "key": key_bytes.hex(), "type": "CONTENT"} + + def _extract_license_key_bytes(self) -> bytes: + """Extract the 16-byte decryption key from WASM memory.""" + data_ptr = self.memory.data_ptr(self.store) + data_len = self.memory.data_len(self.store) + + if self.LICENSE_KEY_OFFSET + self.LICENSE_KEY_LENGTH > data_len: + raise RuntimeError("License key offset beyond memory bounds") + + mem_ptr = ctypes.cast(data_ptr, ctypes.POINTER(ctypes.c_ubyte * data_len)) + start = self.LICENSE_KEY_OFFSET + end = self.LICENSE_KEY_OFFSET + self.LICENSE_KEY_LENGTH + + return bytes(mem_ptr.contents[start:end]) + + def _ccall(self, func_name: str, return_type: type, *args): + """Call a WASM function with automatic string conversion.""" + stack = 0 + converted_args = [] + + try: + for arg in args: + if isinstance(arg, str): + if stack == 0: + stack = self.exports["stackSave"](self.store) + max_length = (len(arg) << 2) + 1 + ptr = self.exports["stackAlloc"](self.store, max_length) + self._string_to_utf8(arg, ptr, max_length) + converted_args.append(ptr) + else: + converted_args.append(arg) + + result = self.exports[func_name](self.store, *converted_args) + finally: + # stackAlloc pointers live on the WASM stack; always restore even if the call throws. + if stack != 0: + exc = sys.exc_info()[1] + try: + self.exports["stackRestore"](self.store, stack) + except Exception: + # If we're already failing, don't mask the original exception. + if exc is None: + raise + + if return_type is bool: + return bool(result) + return result + + def _write_i32(self, addr: int, value: int) -> None: + """Write a 32-bit integer to WASM memory.""" + if addr % 4 != 0: + raise ValueError(f"Unaligned i32 write: addr={addr} (must be 4-byte aligned)") + + data_len = self.memory.data_len(self.store) + if addr < 0 or addr + 4 > data_len: + raise IndexError(f"i32 write out of bounds: addr={addr}, mem_len={data_len}") + + data = self.memory.data_ptr(self.store) + mem_ptr = ctypes.cast(data, ctypes.POINTER(ctypes.c_int32)) + mem_ptr[addr >> 2] = value + + def _string_to_utf8(self, data: str, ptr: int, max_length: int) -> int: + """Convert string to UTF-8 and write to WASM memory.""" + encoded = data.encode("utf-8") + write_length = min(len(encoded), max_length - 1) + + mem_data = self.memory.data_ptr(self.store) + mem_ptr = ctypes.cast(mem_data, ctypes.POINTER(ctypes.c_ubyte)) + + for i in range(write_length): + mem_ptr[ptr + i] = encoded[i] + mem_ptr[ptr + write_length] = 0 + return write_length + + def _write_ascii_to_memory(self, string: str, buffer: int, dont_add_null: int = 0) -> None: + """Write ASCII string to WASM memory.""" + mem_data = self.memory.data_ptr(self.store) + mem_ptr = ctypes.cast(mem_data, ctypes.POINTER(ctypes.c_ubyte)) + + encoded = string.encode("utf-8") + for i, byte_val in enumerate(encoded): + mem_ptr[buffer + i] = byte_val + + if dont_add_null == 0: + mem_ptr[buffer + len(encoded)] = 0 + + def _build_imports(self): + """Build the WASM import stubs required by the MonaLisa module.""" + + def sys_fcntl64(a, b, c): + return 0 + + def fd_write(a, b, c, d): + return 0 + + def fd_close(a): + return 0 + + def sys_ioctl(a, b, c): + return 0 + + def sys_open(a, b, c): + return 0 + + def sys_rmdir(a): + return 0 + + def sys_unlink(a): + return 0 + + def clock(): + return 0 + + def time(a): + return 0 + + def emscripten_run_script(a): + return None + + def fd_seek(a, b, c, d, e): + return 0 + + def emscripten_resize_heap(a): + return 0 + + def fd_read(a, b, c, d): + return 0 + + def emscripten_run_script_string(a): + return 0 + + def emscripten_run_script_int(a): + return 1 + + def emscripten_memcpy_big(dest, src, num): + mem_data = self.memory.data_ptr(self.store) + data_len = self.memory.data_len(self.store) + if num is None: + num = data_len - 1 + mem_ptr = ctypes.cast(mem_data, ctypes.POINTER(ctypes.c_ubyte)) + for i in range(num): + if dest + i < data_len and src + i < data_len: + mem_ptr[dest + i] = mem_ptr[src + i] + return dest + + def environ_get(environ_ptr, environ_buf): + buf_size = 0 + for index, string in enumerate(self.ENV_STRINGS): + ptr = environ_buf + buf_size + self._write_i32(environ_ptr + index * 4, ptr) + self._write_ascii_to_memory(string, ptr) + buf_size += len(string) + 1 + return 0 + + def environ_sizes_get(penviron_count, penviron_buf_size): + self._write_i32(penviron_count, len(self.ENV_STRINGS)) + buf_size = sum(len(s) + 1 for s in self.ENV_STRINGS) + self._write_i32(penviron_buf_size, buf_size) + return 0 + + i32 = wasmtime.ValType.i32() + + return [ + wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32], [i32]), sys_fcntl64), + wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32, i32], [i32]), fd_write), + wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), fd_close), + wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32], [i32]), sys_ioctl), + wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32], [i32]), sys_open), + wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), sys_rmdir), + wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), sys_unlink), + wasmtime.Func(self.store, wasmtime.FuncType([], [i32]), clock), + wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), time), + wasmtime.Func(self.store, wasmtime.FuncType([i32], []), emscripten_run_script), + wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32, i32, i32], [i32]), fd_seek), + wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32], [i32]), emscripten_memcpy_big), + wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), emscripten_resize_heap), + wasmtime.Func(self.store, wasmtime.FuncType([i32, i32], [i32]), environ_get), + wasmtime.Func(self.store, wasmtime.FuncType([i32, i32], [i32]), environ_sizes_get), + wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32, i32], [i32]), fd_read), + wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), emscripten_run_script_string), + wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), emscripten_run_script_int), + self.memory, + ] diff --git a/packages/envied/src/envied/core/config.py b/packages/envied/src/envied/core/config.py index be08269..e59fe63 100644 --- a/packages/envied/src/envied/core/config.py +++ b/packages/envied/src/envied/core/config.py @@ -94,6 +94,7 @@ class Config: self.update_checks: bool = kwargs.get("update_checks", True) self.update_check_interval: int = kwargs.get("update_check_interval", 24) self.scene_naming: bool = kwargs.get("scene_naming", True) + self.dash_naming: bool = kwargs.get("dash_naming", False) self.series_year: bool = kwargs.get("series_year", True) self.unicode_filenames: bool = kwargs.get("unicode_filenames", False) diff --git a/packages/envied/src/envied/core/downloaders/aria2c.py b/packages/envied/src/envied/core/downloaders/aria2c.py index e8b361f..be94b69 100644 --- a/packages/envied/src/envied/core/downloaders/aria2c.py +++ b/packages/envied/src/envied/core/downloaders/aria2c.py @@ -1,6 +1,8 @@ +import logging import os import subprocess import textwrap +import threading import time from functools import partial from http.cookiejar import CookieJar @@ -49,6 +51,196 @@ def rpc(caller: Callable, secret: str, method: str, params: Optional[list[Any]] return +class _Aria2Manager: + """Singleton manager to run one aria2c process and enqueue downloads via RPC.""" + + def __init__(self) -> None: + self._logger = logging.getLogger(__name__) + self._proc: Optional[subprocess.Popen] = None + self._rpc_port: Optional[int] = None + self._rpc_secret: Optional[str] = None + self._rpc_uri: Optional[str] = None + self._session: Session = Session() + self._max_workers: Optional[int] = None + self._max_concurrent_downloads: int = 0 + self._max_connection_per_server: int = 1 + self._split_default: int = 5 + self._file_allocation: str = "prealloc" + self._proxy: Optional[str] = None + self._lock: threading.Lock = threading.Lock() + + def _wait_for_rpc_ready(self, timeout_s: float = 8.0, interval_s: float = 0.1) -> None: + assert self._proc is not None + assert self._rpc_uri is not None + assert self._rpc_secret is not None + + deadline = time.monotonic() + timeout_s + + payload = { + "jsonrpc": "2.0", + "id": get_random_bytes(16).hex(), + "method": "aria2.getVersion", + "params": [f"token:{self._rpc_secret}"], + } + + while time.monotonic() < deadline: + if self._proc.poll() is not None: + raise RuntimeError( + f"aria2c exited before RPC became ready (exit code {self._proc.returncode})" + ) + try: + res = self._session.post(self._rpc_uri, json=payload, timeout=0.25) + data = res.json() + if isinstance(data, dict) and data.get("result") is not None: + return + except (requests.exceptions.RequestException, ValueError): + # Not ready yet (connection refused / bad response / etc.) + pass + time.sleep(interval_s) + + # Timed out: ensure we don't leave a zombie/stray aria2c process behind. + try: + self._proc.terminate() + self._proc.wait(timeout=2) + except Exception: + try: + self._proc.kill() + self._proc.wait(timeout=2) + except Exception: + pass + raise TimeoutError(f"aria2c RPC did not become ready within {timeout_s:.1f}s") + + def _build_args(self) -> list[str]: + args = [ + "--continue=true", + f"--max-concurrent-downloads={self._max_concurrent_downloads}", + f"--max-connection-per-server={self._max_connection_per_server}", + f"--split={self._split_default}", + "--max-file-not-found=5", + "--max-tries=5", + "--retry-wait=2", + "--allow-overwrite=true", + "--auto-file-renaming=false", + "--console-log-level=warn", + "--download-result=default", + f"--file-allocation={self._file_allocation}", + "--summary-interval=0", + "--enable-rpc=true", + f"--rpc-listen-port={self._rpc_port}", + f"--rpc-secret={self._rpc_secret}", + ] + if self._proxy: + args.extend(["--all-proxy", self._proxy]) + return args + + def ensure_started( + self, + proxy: Optional[str], + max_workers: Optional[int], + ) -> None: + with self._lock: + if not binaries.Aria2: + debug_logger = get_debug_logger() + if debug_logger: + debug_logger.log( + level="ERROR", + operation="downloader_aria2c_binary_missing", + message="Aria2c executable not found in PATH or local binaries directory", + context={"searched_names": ["aria2c", "aria2"]}, + ) + raise EnvironmentError("Aria2c executable not found...") + + effective_proxy = proxy or None + + if not max_workers: + effective_max_workers = min(32, (os.cpu_count() or 1) + 4) + elif not isinstance(max_workers, int): + raise TypeError(f"Expected max_workers to be {int}, not {type(max_workers)}") + else: + effective_max_workers = max_workers + + if self._proc and self._proc.poll() is None: + if effective_proxy != self._proxy or effective_max_workers != self._max_workers: + self._logger.warning( + "aria2c process is already running; requested proxy=%r, max_workers=%r, " + "but running process will continue with proxy=%r, max_workers=%r", + effective_proxy, + effective_max_workers, + self._proxy, + self._max_workers, + ) + return + + self._rpc_port = get_free_port() + self._rpc_secret = get_random_bytes(16).hex() + self._rpc_uri = f"http://127.0.0.1:{self._rpc_port}/jsonrpc" + + self._max_workers = effective_max_workers + self._max_concurrent_downloads = int( + config.aria2c.get("max_concurrent_downloads", effective_max_workers) + ) + self._max_connection_per_server = int(config.aria2c.get("max_connection_per_server", 1)) + self._split_default = int(config.aria2c.get("split", 5)) + self._file_allocation = config.aria2c.get("file_allocation", "prealloc") + self._proxy = effective_proxy + + args = self._build_args() + self._proc = subprocess.Popen( + [binaries.Aria2, *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + self._wait_for_rpc_ready() + + @property + def rpc_uri(self) -> str: + assert self._rpc_uri + return self._rpc_uri + + @property + def rpc_secret(self) -> str: + assert self._rpc_secret + return self._rpc_secret + + @property + def session(self) -> Session: + return self._session + + def add_uris(self, uris: list[str], options: dict[str, Any]) -> str: + """Add a single download with multiple URIs via RPC.""" + gid = rpc( + caller=partial(self._session.post, url=self.rpc_uri), + secret=self.rpc_secret, + method="aria2.addUri", + params=[uris, options], + ) + return gid or "" + + def get_global_stat(self) -> dict[str, Any]: + return rpc( + caller=partial(self.session.post, url=self.rpc_uri), + secret=self.rpc_secret, + method="aria2.getGlobalStat", + ) or {} + + def tell_status(self, gid: str) -> Optional[dict[str, Any]]: + return rpc( + caller=partial(self.session.post, url=self.rpc_uri), + secret=self.rpc_secret, + method="aria2.tellStatus", + params=[gid, ["status", "errorCode", "errorMessage", "files", "completedLength", "totalLength"]], + ) + + def remove(self, gid: str) -> None: + rpc( + caller=partial(self.session.post, url=self.rpc_uri), + secret=self.rpc_secret, + method="aria2.forceRemove", + params=[gid], + ) + + +_manager = _Aria2Manager() + + def download( urls: Union[str, list[str], dict[str, Any], list[dict[str, Any]]], output_dir: Path, @@ -58,6 +250,7 @@ def download( proxy: Optional[str] = None, max_workers: Optional[int] = None, ) -> Generator[dict[str, Any], None, None]: + """Enqueue downloads to the singleton aria2c instance via stdin and track per-call progress via RPC.""" debug_logger = get_debug_logger() if not urls: @@ -92,102 +285,10 @@ def download( if not isinstance(urls, list): urls = [urls] - if not binaries.Aria2: - if debug_logger: - debug_logger.log( - level="ERROR", - operation="downloader_aria2c_binary_missing", - message="Aria2c executable not found in PATH or local binaries directory", - context={"searched_names": ["aria2c", "aria2"]}, - ) - raise EnvironmentError("Aria2c executable not found...") - - if proxy and not proxy.lower().startswith("http://"): - raise ValueError("Only HTTP proxies are supported by aria2(c)") - if cookies and not isinstance(cookies, CookieJar): cookies = cookiejar_from_dict(cookies) - url_files = [] - for i, url in enumerate(urls): - if isinstance(url, str): - url_data = {"url": url} - else: - url_data: dict[str, Any] = url - url_filename = filename.format(i=i, ext=get_extension(url_data["url"])) - url_text = url_data["url"] - url_text += f"\n\tdir={output_dir}" - url_text += f"\n\tout={url_filename}" - if cookies: - mock_request = requests.Request(url=url_data["url"]) - cookie_header = get_cookie_header(cookies, mock_request) - if cookie_header: - url_text += f"\n\theader=Cookie: {cookie_header}" - for key, value in url_data.items(): - if key == "url": - continue - if key == "headers": - for header_name, header_value in value.items(): - url_text += f"\n\theader={header_name}: {header_value}" - else: - url_text += f"\n\t{key}={value}" - url_files.append(url_text) - url_file = "\n".join(url_files) - - rpc_port = get_free_port() - rpc_secret = get_random_bytes(16).hex() - rpc_uri = f"http://127.0.0.1:{rpc_port}/jsonrpc" - rpc_session = Session() - - max_concurrent_downloads = int(config.aria2c.get("max_concurrent_downloads", max_workers)) - max_connection_per_server = int(config.aria2c.get("max_connection_per_server", 1)) - split = int(config.aria2c.get("split", 5)) - file_allocation = config.aria2c.get("file_allocation", "prealloc") - if len(urls) > 1: - split = 1 - file_allocation = "none" - - arguments = [ - # [Basic Options] - "--input-file", - "-", - "--all-proxy", - proxy or "", - "--continue=true", - # [Connection Options] - f"--max-concurrent-downloads={max_concurrent_downloads}", - f"--max-connection-per-server={max_connection_per_server}", - f"--split={split}", # each split uses their own connection - "--max-file-not-found=5", # counted towards --max-tries - "--max-tries=5", - "--retry-wait=2", - # [Advanced Options] - "--allow-overwrite=true", - "--auto-file-renaming=false", - "--console-log-level=warn", - "--download-result=default", - f"--file-allocation={file_allocation}", - "--summary-interval=0", - # [RPC Options] - "--enable-rpc=true", - f"--rpc-listen-port={rpc_port}", - f"--rpc-secret={rpc_secret}", - ] - - for header, value in (headers or {}).items(): - if header.lower() == "cookie": - raise ValueError("You cannot set Cookies as a header manually, please use the `cookies` param.") - if header.lower() == "accept-encoding": - # we cannot set an allowed encoding, or it will return compressed - # and the code is not set up to uncompress the data - continue - if header.lower() == "referer": - arguments.extend(["--referer", value]) - continue - if header.lower() == "user-agent": - arguments.extend(["--user-agent", value]) - continue - arguments.extend(["--header", f"{header}: {value}"]) + _manager.ensure_started(proxy=proxy, max_workers=max_workers) if debug_logger: first_url = urls[0] if isinstance(urls[0], str) else urls[0].get("url", "") @@ -202,128 +303,151 @@ def download( "first_url": url_display, "output_dir": str(output_dir), "filename": filename, - "max_concurrent_downloads": max_concurrent_downloads, - "max_connection_per_server": max_connection_per_server, - "split": split, - "file_allocation": file_allocation, "has_proxy": bool(proxy), - "rpc_port": rpc_port, }, ) - yield dict(total=len(urls)) + # Build options for each URI and add via RPC + gids: list[str] = [] + + for i, url in enumerate(urls): + if isinstance(url, str): + url_data = {"url": url} + else: + url_data: dict[str, Any] = url + + url_filename = filename.format(i=i, ext=get_extension(url_data["url"])) + + opts: dict[str, Any] = { + "dir": str(output_dir), + "out": url_filename, + "split": str(1 if len(urls) > 1 else int(config.aria2c.get("split", 5))), + } + + # Cookies as header + if cookies: + mock_request = requests.Request(url=url_data["url"]) + cookie_header = get_cookie_header(cookies, mock_request) + if cookie_header: + opts.setdefault("header", []).append(f"Cookie: {cookie_header}") + + # Global headers + for header, value in (headers or {}).items(): + if header.lower() == "cookie": + raise ValueError("You cannot set Cookies as a header manually, please use the `cookies` param.") + if header.lower() == "accept-encoding": + continue + if header.lower() == "referer": + opts["referer"] = str(value) + continue + if header.lower() == "user-agent": + opts["user-agent"] = str(value) + continue + opts.setdefault("header", []).append(f"{header}: {value}") + + # Per-url extra args + for key, value in url_data.items(): + if key == "url": + continue + if key == "headers": + for header_name, header_value in value.items(): + opts.setdefault("header", []).append(f"{header_name}: {header_value}") + else: + opts[key] = str(value) + + # Add via RPC + gid = _manager.add_uris([url_data["url"]], opts) + if gid: + gids.append(gid) + + yield dict(total=len(gids)) + + completed: set[str] = set() try: - p = subprocess.Popen([binaries.Aria2, *arguments], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL) + while len(completed) < len(gids): + if DOWNLOAD_CANCELLED.is_set(): + # Remove tracked downloads on cancel + for gid in gids: + if gid not in completed: + _manager.remove(gid) + yield dict(downloaded="[yellow]CANCELLED") + raise KeyboardInterrupt() - p.stdin.write(url_file.encode()) - p.stdin.close() + stats = _manager.get_global_stat() + dl_speed = int(stats.get("downloadSpeed", -1)) - while p.poll() is None: - global_stats: dict[str, Any] = ( - rpc(caller=partial(rpc_session.post, url=rpc_uri), secret=rpc_secret, method="aria2.getGlobalStat") - or {} - ) + # Aggregate progress across all GIDs for this call + total_completed = 0 + total_size = 0 - number_stopped = int(global_stats.get("numStoppedTotal", 0)) - download_speed = int(global_stats.get("downloadSpeed", -1)) + # Check each tracked GID + for gid in gids: + if gid in completed: + continue - if number_stopped: - yield dict(completed=number_stopped) - if download_speed != -1: - yield dict(downloaded=f"{filesize.decimal(download_speed)}/s") + status = _manager.tell_status(gid) + if not status: + continue - stopped_downloads: list[dict[str, Any]] = ( - rpc( - caller=partial(rpc_session.post, url=rpc_uri), - secret=rpc_secret, - method="aria2.tellStopped", - params=[0, 999999], - ) - or [] - ) + completed_length = int(status.get("completedLength", 0)) + total_length = int(status.get("totalLength", 0)) + total_completed += completed_length + total_size += total_length - for dl in stopped_downloads: - if dl["status"] == "error": - used_uri = next( - uri["uri"] - for file in dl["files"] - if file["selected"] == "true" - for uri in file["uris"] - if uri["status"] == "used" - ) - error = f"Download Error (#{dl['gid']}): {dl['errorMessage']} ({dl['errorCode']}), {used_uri}" - error_pretty = "\n ".join( - textwrap.wrap(error, width=console.width - 20, initial_indent="") - ) - console.log(Text.from_ansi("\n[Aria2c]: " + error_pretty)) - if debug_logger: - debug_logger.log( - level="ERROR", - operation="downloader_aria2c_download_error", - message=f"Aria2c download failed: {dl['errorMessage']}", - context={ - "gid": dl["gid"], - "error_code": dl["errorCode"], - "error_message": dl["errorMessage"], - "used_uri": used_uri[:200] + "..." if len(used_uri) > 200 else used_uri, - "completed_length": dl.get("completedLength"), - "total_length": dl.get("totalLength"), - }, - ) - raise ValueError(error) + state = status.get("status") + if state in ("complete", "error"): + completed.add(gid) + yield dict(completed=len(completed)) - if number_stopped == len(urls): - rpc(caller=partial(rpc_session.post, url=rpc_uri), secret=rpc_secret, method="aria2.shutdown") - break + if state == "error": + used_uri = None + try: + used_uri = next( + uri["uri"] + for file in status.get("files", []) + for uri in file.get("uris", []) + if uri.get("status") == "used" + ) + except Exception: + used_uri = "unknown" + error = f"Download Error (#{gid}): {status.get('errorMessage')} ({status.get('errorCode')}), {used_uri}" + error_pretty = "\n ".join(textwrap.wrap(error, width=console.width - 20, initial_indent="")) + console.log(Text.from_ansi("\n[Aria2c]: " + error_pretty)) + if debug_logger: + debug_logger.log( + level="ERROR", + operation="downloader_aria2c_download_error", + message=f"Aria2c download failed: {status.get('errorMessage')}", + context={ + "gid": gid, + "error_code": status.get("errorCode"), + "error_message": status.get("errorMessage"), + "used_uri": used_uri[:200] + "..." if used_uri and len(used_uri) > 200 else used_uri, + "completed_length": status.get("completedLength"), + "total_length": status.get("totalLength"), + }, + ) + raise ValueError(error) + + # Yield aggregate progress for this call's downloads + if total_size > 0: + # Yield both advance (bytes downloaded this iteration) and total for rich progress + if dl_speed != -1: + yield dict(downloaded=f"{filesize.decimal(dl_speed)}/s", advance=0, completed=total_completed, total=total_size) + else: + yield dict(advance=0, completed=total_completed, total=total_size) + elif dl_speed != -1: + yield dict(downloaded=f"{filesize.decimal(dl_speed)}/s") time.sleep(1) - - p.wait() - - if p.returncode != 0: - if debug_logger: - debug_logger.log( - level="ERROR", - operation="downloader_aria2c_failed", - message=f"Aria2c exited with code {p.returncode}", - context={ - "returncode": p.returncode, - "url_count": len(urls), - "output_dir": str(output_dir), - }, - ) - raise subprocess.CalledProcessError(p.returncode, arguments) - - if debug_logger: - debug_logger.log( - level="DEBUG", - operation="downloader_aria2c_complete", - message="Aria2c download completed successfully", - context={ - "url_count": len(urls), - "output_dir": str(output_dir), - "filename": filename, - }, - ) - - except ConnectionResetError: - # interrupted while passing URI to download - raise KeyboardInterrupt() - except subprocess.CalledProcessError as e: - if e.returncode in (7, 0xC000013A): - # 7 is when Aria2(c) handled the CTRL+C - # 0xC000013A is when it never got the chance to - raise KeyboardInterrupt() - raise except KeyboardInterrupt: - DOWNLOAD_CANCELLED.set() # skip pending track downloads - yield dict(downloaded="[yellow]CANCELLED") + DOWNLOAD_CANCELLED.set() raise except Exception as e: - DOWNLOAD_CANCELLED.set() # skip pending track downloads + DOWNLOAD_CANCELLED.set() yield dict(downloaded="[red]FAILED") - if debug_logger and not isinstance(e, (subprocess.CalledProcessError, ValueError)): + if debug_logger and not isinstance(e, ValueError): debug_logger.log( level="ERROR", operation="downloader_aria2c_exception", @@ -335,8 +459,6 @@ def download( }, ) raise - finally: - rpc(caller=partial(rpc_session.post, url=rpc_uri), secret=rpc_secret, method="aria2.shutdown") def aria2c( diff --git a/packages/envied/src/envied/core/downloaders/n_m3u8dl_re.py b/packages/envied/src/envied/core/downloaders/n_m3u8dl_re.py index 311ca52..4b95384 100644 --- a/packages/envied/src/envied/core/downloaders/n_m3u8dl_re.py +++ b/packages/envied/src/envied/core/downloaders/n_m3u8dl_re.py @@ -10,6 +10,7 @@ import requests from requests.cookies import cookiejar_from_dict, get_cookie_header from envied.core import binaries +from envied.core.binaries import FFMPEG, Mp4decrypt, ShakaPackager from envied.core.config import config from envied.core.console import console from envied.core.constants import DOWNLOAD_CANCELLED @@ -19,7 +20,7 @@ PERCENT_RE = re.compile(r"(\d+\.\d+%)") SPEED_RE = re.compile(r"(\d+\.\d+(?:MB|KB)ps)") SIZE_RE = re.compile(r"(\d+\.\d+(?:MB|GB|KB)/\d+\.\d+(?:MB|GB|KB))") WARN_RE = re.compile(r"(WARN : Response.*|WARN : One or more errors occurred.*)") -ERROR_RE = re.compile(r"(ERROR.*)") +ERROR_RE = re.compile(r"(\bERROR\b.*|\bFAILED\b.*|\bException\b.*)") DECRYPTION_ENGINE = { "shaka": "SHAKA_PACKAGER", @@ -67,12 +68,17 @@ def get_track_selection_args(track: Any) -> list[str]: parts = [] if track_type == "Audio": - if track_id := representation.get("id") or adaptation_set.get("audioTrackId"): + track_id = representation.get("id") or adaptation_set.get("audioTrackId") + lang = representation.get("lang") or adaptation_set.get("lang") + + if track_id: parts.append(rf'"id=\b{track_id}\b"') + if lang: + parts.append(f"lang={lang}") else: if codecs := representation.get("codecs"): parts.append(f"codecs={codecs}") - if lang := representation.get("lang") or adaptation_set.get("lang"): + if lang: parts.append(f"lang={lang}") if bw := representation.get("bandwidth"): bitrate = int(bw) // 1000 @@ -176,17 +182,35 @@ def build_download_args( "--tmp-dir": output_dir, "--thread-count": thread_count, "--download-retry-count": retry_count, - "--write-meta-json": False, } + if FFMPEG: + args["--ffmpeg-binary-path"] = str(FFMPEG) if proxy: args["--custom-proxy"] = proxy if skip_merge: args["--skip-merge"] = skip_merge if ad_keyword: args["--ad-keyword"] = ad_keyword + + key_args = [] if content_keys: - args["--key"] = next((f"{kid.hex}:{key.lower()}" for kid, key in content_keys.items()), None) - args["--decryption-engine"] = DECRYPTION_ENGINE.get(config.decryption.lower()) or "SHAKA_PACKAGER" + for kid, key in content_keys.items(): + key_args.extend(["--key", f"{kid.hex}:{key.lower()}"]) + + decryption_config = config.decryption.lower() + engine_name = DECRYPTION_ENGINE.get(decryption_config) or "SHAKA_PACKAGER" + args["--decryption-engine"] = engine_name + + binary_path = None + if engine_name == "SHAKA_PACKAGER": + if ShakaPackager: + binary_path = str(ShakaPackager) + elif engine_name == "MP4DECRYPT": + if Mp4decrypt: + binary_path = str(Mp4decrypt) + if binary_path: + args["--decryption-binary-path"] = binary_path + if custom_args: args.update(custom_args) @@ -199,6 +223,9 @@ def build_download_args( elif value is not False and value is not None: command.extend([flag, str(value)]) + # Append all content keys (multiple --key flags supported by N_m3u8DL-RE) + command.extend(key_args) + if headers: for key, value in headers.items(): if key.lower() not in ("accept-encoding", "cookie"): @@ -283,7 +310,10 @@ def download( log_file_path: Path | None = None if debug_logger: log_file_path = output_dir / f".n_m3u8dl_re_{filename}.log" - arguments.extend(["--log-file-path", str(log_file_path)]) + arguments.extend([ + "--log-file-path", str(log_file_path), + "--log-level", "DEBUG", + ]) track_url_display = track.url[:200] + "..." if len(track.url) > 200 else track.url debug_logger.log( @@ -371,6 +401,14 @@ def download( raise subprocess.CalledProcessError(process.returncode, arguments) if debug_logger: + output_dir_exists = output_dir.exists() + output_files = [] + if output_dir_exists: + try: + output_files = [f.name for f in output_dir.iterdir() if f.is_file()][:20] + except Exception: + output_files = [""] + debug_logger.log( level="DEBUG", operation="downloader_n_m3u8dl_re_complete", @@ -379,10 +417,38 @@ def download( "track_id": getattr(track, "id", None), "track_type": track.__class__.__name__, "output_dir": str(output_dir), + "output_dir_exists": output_dir_exists, + "output_files_count": len(output_files), + "output_files": output_files, "filename": filename, }, ) + # Warn if no output was produced - include N_m3u8DL-RE's logs for diagnosis + if not output_dir_exists or not output_files: + # Read N_m3u8DL-RE's log file for debugging + n_m3u8dl_log = "" + if log_file_path and log_file_path.exists(): + try: + n_m3u8dl_log = log_file_path.read_text(encoding="utf-8", errors="replace") + except Exception: + n_m3u8dl_log = "" + + debug_logger.log( + level="WARNING", + operation="downloader_n_m3u8dl_re_no_output", + message="N_m3u8DL-RE exited successfully but produced no output files", + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "output_dir": str(output_dir), + "output_dir_exists": output_dir_exists, + "selection_args": selection_args, + "track_url": track.url[:200] + "..." if len(track.url) > 200 else track.url, + "n_m3u8dl_re_log": n_m3u8dl_log, + }, + ) + except ConnectionResetError: # interrupted while passing URI to download raise KeyboardInterrupt() @@ -414,6 +480,7 @@ def download( ) raise finally: + # Clean up temporary debug files if log_file_path and log_file_path.exists(): try: log_file_path.unlink() diff --git a/packages/envied/src/envied/core/downloaders/requests.py b/packages/envied/src/envied/core/downloaders/requests.py index d60ab49..0866009 100644 --- a/packages/envied/src/envied/core/downloaders/requests.py +++ b/packages/envied/src/envied/core/downloaders/requests.py @@ -122,7 +122,7 @@ def download( last_speed_refresh = now download_sizes.clear() - if content_length and written < content_length: + if not segmented and content_length and written < content_length: raise IOError(f"Failed to read {content_length} bytes from the track URI.") yield dict(file_downloaded=save_path, written=written) @@ -260,11 +260,18 @@ def requests( }, ) - yield dict(total=len(urls)) + # If we're downloading more than one URL, treat them as "segments" for progress purposes. + # For single-URL downloads we want per-chunk progress (and the inner `download()` will yield + # a chunk-based `total`), so don't set a segment total of 1 here. + segmented_batch = len(urls) > 1 + if segmented_batch: + yield dict(total=len(urls)) try: with ThreadPoolExecutor(max_workers=max_workers) as pool: - for future in as_completed(pool.submit(download, session=session, segmented=False, **url) for url in urls): + for future in as_completed( + pool.submit(download, session=session, segmented=segmented_batch, **url) for url in urls + ): try: yield from future.result() except KeyboardInterrupt: diff --git a/packages/envied/src/envied/core/drm/__init__.py b/packages/envied/src/envied/core/drm/__init__.py index 2b3c0ef..76db577 100644 --- a/packages/envied/src/envied/core/drm/__init__.py +++ b/packages/envied/src/envied/core/drm/__init__.py @@ -1,10 +1,11 @@ from typing import Union from envied.core.drm.clearkey import ClearKey +from envied.core.drm.monalisa import MonaLisa from envied.core.drm.playready import PlayReady from envied.core.drm.widevine import Widevine -DRM_T = Union[ClearKey, Widevine, PlayReady] +DRM_T = Union[ClearKey, Widevine, PlayReady, MonaLisa] -__all__ = ("ClearKey", "Widevine", "PlayReady", "DRM_T") +__all__ = ("ClearKey", "Widevine", "PlayReady", "MonaLisa", "DRM_T") diff --git a/packages/envied/src/envied/core/drm/monalisa.py b/packages/envied/src/envied/core/drm/monalisa.py new file mode 100644 index 0000000..44c1466 --- /dev/null +++ b/packages/envied/src/envied/core/drm/monalisa.py @@ -0,0 +1,299 @@ +""" +MonaLisa DRM System. + +A WASM-based DRM system that uses local key extraction and two-stage +segment decryption (ML-Worker binary + AES-ECB). +""" + +from __future__ import annotations + +import logging +import os +import subprocess +import sys +from pathlib import Path +from typing import Any, Optional, Union +from uuid import UUID + +from Cryptodome.Cipher import AES +from Cryptodome.Util.Padding import unpad + +log = logging.getLogger(__name__) + + +class MonaLisa: + """ + MonaLisa DRM System. + + Unlike Widevine/PlayReady, MonaLisa does not use a challenge/response flow + with a license server. Instead, the PSSH value (ticket) is provided directly + by the service API, and keys are extracted locally via a WASM module. + + Decryption is performed in two stages: + 1. ML-Worker binary: Removes MonaLisa encryption layer (bbts -> ents) + 2. AES-ECB decryption: Final decryption with service-provided key + """ + + class Exceptions: + class TicketNotFound(Exception): + """Raised when no PSSH/ticket data is provided.""" + + class KeyExtractionFailed(Exception): + """Raised when key extraction from the ticket fails.""" + + class WorkerNotFound(Exception): + """Raised when the ML-Worker binary is not found.""" + + class DecryptionFailed(Exception): + """Raised when segment decryption fails.""" + + def __init__( + self, + ticket: Union[str, bytes], + aes_key: Union[str, bytes], + device_path: Path, + **kwargs: Any, + ): + """ + Initialize MonaLisa DRM. + + Args: + ticket: PSSH value from service API (base64 string or raw bytes). + aes_key: AES-ECB key for second-stage decryption (hex string or bytes). + device_path: Path to the CDM device file (.mld). + **kwargs: Additional metadata stored in self.data. + + Raises: + TicketNotFound: If ticket/PSSH is empty. + KeyExtractionFailed: If key extraction fails. + """ + if not ticket: + raise MonaLisa.Exceptions.TicketNotFound("No PSSH/ticket data provided.") + + self._ticket = ticket + + # Store AES key for second-stage decryption + if isinstance(aes_key, str): + self._aes_key = bytes.fromhex(aes_key) + else: + self._aes_key = aes_key + + self._device_path = device_path + self._kid: Optional[UUID] = None + self._key: Optional[str] = None + self.data: dict = kwargs or {} + + # Extract keys immediately + self._extract_keys() + + def _extract_keys(self) -> None: + """Extract keys from the ticket using the MonaLisa CDM.""" + # Import here to avoid circular import + from envied.core.cdm.monalisa import MonaLisaCDM + + try: + cdm = MonaLisaCDM(device_path=self._device_path) + session_id = cdm.open() + try: + keys = cdm.extract_keys(self._ticket) + if keys: + kid_hex = keys.get("kid") + if kid_hex: + self._kid = UUID(hex=kid_hex) + self._key = keys.get("key") + finally: + cdm.close(session_id) + except Exception as e: + raise MonaLisa.Exceptions.KeyExtractionFailed(f"Failed to extract keys: {e}") + + @classmethod + def from_ticket( + cls, + ticket: Union[str, bytes], + aes_key: Union[str, bytes], + device_path: Path, + ) -> MonaLisa: + """ + Create a MonaLisa DRM instance from a PSSH/ticket. + + Args: + ticket: PSSH value from service API. + aes_key: AES-ECB key for second-stage decryption. + device_path: Path to the CDM device file (.mld). + + Returns: + MonaLisa DRM instance with extracted keys. + """ + return cls(ticket=ticket, aes_key=aes_key, device_path=device_path) + + @property + def kid(self) -> Optional[UUID]: + """Get the Key ID.""" + return self._kid + + @property + def key(self) -> Optional[str]: + """Get the content key as hex string.""" + return self._key + + @property + def pssh(self) -> str: + """ + Get the raw PSSH/ticket value as a string. + + Returns: + The raw PSSH value as a base64 string. + """ + if isinstance(self._ticket, bytes): + try: + return self._ticket.decode("utf-8") + except UnicodeDecodeError: + # Tickets are typically base64, so ASCII is a reasonable fallback. + try: + return self._ticket.decode("ascii") + except UnicodeDecodeError as e: + raise ValueError( + f"Ticket bytes must be UTF-8 text or ASCII base64; got undecodable bytes (len={len(self._ticket)})" + ) from e + return self._ticket + + @property + def content_id(self) -> Optional[str]: + """ + Extract the Content ID from the PSSH for display. + + The PSSH contains an embedded Content ID at bytes 21-75 with format: + H5DCID-V3-P1-YYYYMMDD-HHMMSS-MEDIAID-TIMESTAMP-SUFFIX + + Returns: + The Content ID string if extractable, None otherwise. + """ + import base64 + + try: + # Decode base64 PSSH to get raw bytes + if isinstance(self._ticket, bytes): + data = self._ticket + else: + data = base64.b64decode(self._ticket) + + # Content ID is at bytes 21-75 (55 bytes) + if len(data) >= 76: + content_id = data[21:76].decode("ascii") + # Validate it looks like a content ID + if content_id.startswith("H5DCID-"): + return content_id + except Exception: + pass + + return None + + @property + def content_keys(self) -> dict[UUID, str]: + """ + Get content keys in the same format as Widevine/PlayReady. + + Returns: + Dictionary mapping KID to key hex string. + """ + if self._kid and self._key: + return {self._kid: self._key} + return {} + + def decrypt_segment(self, segment_path: Path) -> None: + """ + Decrypt a single segment using two-stage decryption. + + Stage 1: ML-Worker binary (bbts -> ents) + Stage 2: AES-ECB decryption (ents -> ts) + + Args: + segment_path: Path to the encrypted segment file. + + Raises: + WorkerNotFound: If ML-Worker binary is not available. + DecryptionFailed: If decryption fails at any stage. + """ + if not self._key: + return + + # Import here to avoid circular import + from envied.core.cdm.monalisa import MonaLisaCDM + + worker_path = MonaLisaCDM.get_worker_path() + if not worker_path or not worker_path.exists(): + raise MonaLisa.Exceptions.WorkerNotFound("ML-Worker not found.") + + bbts_path = segment_path.with_suffix(".bbts") + ents_path = segment_path.with_suffix(".ents") + + try: + if segment_path.exists(): + segment_path.replace(bbts_path) + else: + raise MonaLisa.Exceptions.DecryptionFailed(f"Segment file does not exist: {segment_path}") + + # Stage 1: ML-Worker decryption + cmd = [str(worker_path), str(self._key), str(bbts_path), str(ents_path)] + + startupinfo = None + if sys.platform == "win32": + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + + worker_timeout_s = 60 + process = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + startupinfo=startupinfo, + timeout=worker_timeout_s, + ) + + if process.returncode != 0: + raise MonaLisa.Exceptions.DecryptionFailed( + f"ML-Worker failed for {segment_path.name}: {process.stderr}" + ) + + if not ents_path.exists(): + raise MonaLisa.Exceptions.DecryptionFailed( + f"Decrypted .ents file was not created for {segment_path.name}" + ) + + # Stage 2: AES-ECB decryption + with open(ents_path, "rb") as f: + ents_data = f.read() + + crypto = AES.new(self._aes_key, AES.MODE_ECB) + decrypted_data = unpad(crypto.decrypt(ents_data), AES.block_size) + + # Write decrypted segment back to original path + with open(segment_path, "wb") as f: + f.write(decrypted_data) + + except MonaLisa.Exceptions.DecryptionFailed: + raise + except subprocess.TimeoutExpired as e: + log.error("ML-Worker timed out after %ss for %s", worker_timeout_s, segment_path.name) + raise MonaLisa.Exceptions.DecryptionFailed( + f"ML-Worker timed out after {worker_timeout_s}s for {segment_path.name}" + ) from e + except Exception as e: + raise MonaLisa.Exceptions.DecryptionFailed(f"Failed to decrypt segment {segment_path.name}: {e}") + finally: + if ents_path.exists(): + os.remove(ents_path) + if bbts_path != segment_path and bbts_path.exists(): + os.remove(bbts_path) + + def decrypt(self, _path: Path) -> None: + """ + MonaLisa uses per-segment decryption during download via the + on_segment_downloaded callback. By the time this method is called, + the content has already been decrypted and muxed into a container. + + Args: + path: Path to the file (ignored). + """ + pass diff --git a/packages/envied/src/envied/core/drm/playready.py b/packages/envied/src/envied/core/drm/playready.py index 4acdd8a..969f373 100644 --- a/packages/envied/src/envied/core/drm/playready.py +++ b/packages/envied/src/envied/core/drm/playready.py @@ -154,7 +154,9 @@ class PlayReady: pssh_boxes.extend( Box.parse(base64.b64decode(x.uri.split(",")[-1])) for x in (master.session_keys or master.keys) - if x and x.keyformat and "playready" in x.keyformat.lower() + if x and x.keyformat and x.keyformat.lower() in { + f"urn:uuid:{PSSH.SYSTEM_ID}", "com.microsoft.playready" + } ) init_data = track.get_init_segment(session=session) diff --git a/packages/envied/src/envied/core/manifests/dash.py b/packages/envied/src/envied/core/manifests/dash.py index a426f0f..df2b3b7 100644 --- a/packages/envied/src/envied/core/manifests/dash.py +++ b/packages/envied/src/envied/core/manifests/dash.py @@ -19,12 +19,12 @@ import requests from curl_cffi.requests import Session as CurlSession from langcodes import Language, tag_is_valid from lxml.etree import Element, ElementTree -from pyplayready.cdm import Cdm as PlayReadyCdm from pyplayready.system.pssh import PSSH as PR_PSSH from pywidevine.cdm import Cdm as WidevineCdm from pywidevine.pssh import PSSH from requests import Session +from envied.core.cdm.detect import is_playready_cdm from envied.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack from envied.core.downloaders import requests as requests_downloader from envied.core.drm import DRM_T, PlayReady, Widevine @@ -151,6 +151,11 @@ class DASH: if not track_fps and segment_base is not None: track_fps = segment_base.get("timescale") + scan_type = None + scan_type_str = get("scanType") + if scan_type_str and scan_type_str.lower() == "interlaced": + scan_type = Video.ScanType.INTERLACED + track_args = dict( range_=self.get_video_range( codecs, findall("SupplementalProperty"), findall("EssentialProperty") @@ -159,6 +164,7 @@ class DASH: width=get("width") or 0, height=get("height") or 0, fps=track_fps or None, + scan_type=scan_type, ) elif content_type == "audio": track_type = Audio @@ -366,6 +372,9 @@ class DASH: if not end_number: end_number = len(segment_durations) + # Handle high startNumber in DVR/catch-up manifests where startNumber > segment count + if start_number > end_number: + end_number = start_number + len(segment_durations) - 1 for t, n in zip(segment_durations, range(start_number, end_number + 1)): segments.append( @@ -467,8 +476,9 @@ class DASH: track.data["dash"]["timescale"] = int(segment_timescale) track.data["dash"]["segment_durations"] = segment_durations - if init_data and isinstance(track, (Video, Audio)): - if isinstance(cdm, PlayReadyCdm): + if not track.drm and init_data and isinstance(track, (Video, Audio)): + prefers_playready = is_playready_cdm(cdm) + if prefers_playready: try: track.drm = [PlayReady.from_init_data(init_data)] except PlayReady.Exceptions.PSSHNotFound: @@ -572,8 +582,64 @@ class DASH: for control_file in save_dir.glob("*.aria2__temp"): control_file.unlink() + # Verify output directory exists and contains files + if not save_dir.exists(): + error_msg = f"Output directory does not exist: {save_dir}" + if debug_logger: + debug_logger.log( + level="ERROR", + operation="manifest_dash_download_output_missing", + message=error_msg, + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "save_dir": str(save_dir), + "save_path": str(save_path), + "downloader": downloader.__name__, + "skip_merge": skip_merge, + }, + ) + raise FileNotFoundError(error_msg) + segments_to_merge = [x for x in sorted(save_dir.iterdir()) if x.is_file()] + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="manifest_dash_download_complete", + message="DASH download complete, preparing to merge", + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "save_dir": str(save_dir), + "save_dir_exists": save_dir.exists(), + "segments_found": len(segments_to_merge), + "segment_files": [f.name for f in segments_to_merge[:10]], # Limit to first 10 + "downloader": downloader.__name__, + "skip_merge": skip_merge, + }, + ) + + if not segments_to_merge: + error_msg = f"No segment files found in output directory: {save_dir}" + if debug_logger: + # List all contents of the directory for debugging + all_contents = list(save_dir.iterdir()) if save_dir.exists() else [] + debug_logger.log( + level="ERROR", + operation="manifest_dash_download_no_segments", + message=error_msg, + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "save_dir": str(save_dir), + "directory_contents": [str(p) for p in all_contents], + "downloader": downloader.__name__, + "skip_merge": skip_merge, + }, + ) + raise FileNotFoundError(error_msg) + if skip_merge: # N_m3u8DL-RE handles merging and decryption internally shutil.move(segments_to_merge[0], save_path) @@ -800,7 +866,7 @@ class DASH: urn = (protection.get("schemeIdUri") or "").lower() if urn == WidevineCdm.urn: - pssh_text = protection.findtext("pssh") + pssh_text = protection.findtext("pssh") or protection.findtext("{urn:mpeg:cenc:2013}pssh") if not pssh_text: continue pssh = PSSH(pssh_text) @@ -831,6 +897,7 @@ class DASH: elif urn in ("urn:uuid:9a04f079-9840-4286-ab92-e65be0885f95", "urn:microsoft:playready"): pr_pssh_b64 = ( protection.findtext("pssh") + or protection.findtext("{urn:mpeg:cenc:2013}pssh") or protection.findtext("pro") or protection.findtext("{urn:microsoft:playready}pro") ) diff --git a/packages/envied/src/envied/core/manifests/hls.py b/packages/envied/src/envied/core/manifests/hls.py index dc8304d..5ac0504 100644 --- a/packages/envied/src/envied/core/manifests/hls.py +++ b/packages/envied/src/envied/core/manifests/hls.py @@ -12,6 +12,7 @@ from functools import partial from pathlib import Path from typing import Any, Callable, Optional, Union from urllib.parse import urljoin +from uuid import UUID from zlib import crc32 import m3u8 @@ -27,12 +28,13 @@ from pywidevine.pssh import PSSH as WV_PSSH from requests import Session from envied.core import binaries +from envied.core.cdm.detect import is_playready_cdm, is_widevine_cdm from envied.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack from envied.core.downloaders import requests as requests_downloader -from envied.core.drm import DRM_T, ClearKey, PlayReady, Widevine +from envied.core.drm import DRM_T, ClearKey, MonaLisa, PlayReady, Widevine from envied.core.events import events from envied.core.tracks import Audio, Subtitle, Tracks, Video -from envied.core.utilities import get_extension, is_close_match, try_ensure_utf8 +from envied.core.utilities import get_debug_logger, get_extension, is_close_match, try_ensure_utf8 class HLS: @@ -114,9 +116,14 @@ class HLS: for playlist in self.manifest.playlists: audio_group = playlist.stream_info.audio - if audio_group: - audio_codec = Audio.Codec.from_codecs(playlist.stream_info.codecs) - audio_codecs_by_group_id[audio_group] = audio_codec + audio_codec: Optional[Audio.Codec] = None + if audio_group and playlist.stream_info.codecs: + try: + audio_codec = Audio.Codec.from_codecs(playlist.stream_info.codecs) + except ValueError: + audio_codec = None + if audio_codec: + audio_codecs_by_group_id[audio_group] = audio_codec try: # TODO: Any better way to figure out the primary track type? @@ -224,6 +231,39 @@ class HLS: return tracks + @staticmethod + def _finalize_n_m3u8dl_re_output(*, track: AnyTrack, save_dir: Path, save_path: Path) -> Path: + """ + Finalize output from N_m3u8DL-RE. + + We call N_m3u8DL-RE with `--save-name track.id`, so the final file should be `{track.id}.*` under `save_dir`. + This moves that output to `save_path` (preserving the real suffix) and, for subtitles, updates `track.codec` + to match the produced file extension. + """ + matches = [p for p in save_dir.rglob(f"{track.id}.*") if p.is_file()] + if not matches: + raise FileNotFoundError(f"No output files produced by N_m3u8DL-RE for save-name={track.id} in: {save_dir}") + + primary = max(matches, key=lambda p: p.stat().st_size) + + final_save_path = save_path.with_suffix(primary.suffix) if primary.suffix else save_path + + final_save_path.parent.mkdir(parents=True, exist_ok=True) + if primary.absolute() != final_save_path.absolute(): + final_save_path.unlink(missing_ok=True) + shutil.move(str(primary), str(final_save_path)) + + if isinstance(track, Subtitle): + ext = final_save_path.suffix.lower().lstrip(".") + try: + track.codec = Subtitle.Codec.from_mime(ext) + except ValueError: + pass + + shutil.rmtree(save_dir, ignore_errors=True) + + return final_save_path + @staticmethod def download_track( track: AnyTrack, @@ -254,13 +294,15 @@ class HLS: else: # Get the playlist text and handle both session types response = session.get(track.url) - if isinstance(response, requests.Response): + if isinstance(response, requests.Response) or isinstance(response, CurlResponse): if not response.ok: log.error(f"Failed to request the invariant M3U8 playlist: {response.status_code}") sys.exit(1) playlist_text = response.text else: - raise TypeError(f"Expected response to be a requests.Response or curl_cffi.Response, not {type(response)}") + raise TypeError( + f"Expected response to be a requests.Response or curl_cffi.Response, not {type(response)}" + ) master = m3u8.loads(playlist_text, uri=track.url) @@ -268,23 +310,63 @@ class HLS: log.error("Track's HLS playlist has no segments, expecting an invariant M3U8 playlist.") sys.exit(1) + # Get session DRM as fallback but prefer media playlist keys for accurate KID matching if track.drm: session_drm = track.get_drm_for_cdm(cdm) - if isinstance(session_drm, (Widevine, PlayReady)): - # license and grab content keys - try: - if not license_widevine: - raise ValueError("license_widevine func must be supplied to use DRM") - progress(downloaded="LICENSING") - license_widevine(session_drm) - progress(downloaded="[yellow]LICENSED") - except Exception: # noqa - DOWNLOAD_CANCELLED.set() # skip pending track downloads - progress(downloaded="[red]FAILED") - raise else: session_drm = None + initial_drm_licensed = False + initial_drm_key = None # Track the EXT-X-KEY used for initial licensing + media_keys = [k for k in (master.keys or []) if k is not None] + if media_keys: + cdm_media_keys = HLS.filter_keys_for_cdm(media_keys, cdm) + media_playlist_key = HLS.get_supported_key(cdm_media_keys) if cdm_media_keys else None + + if media_playlist_key: + media_drm = HLS.get_drm(media_playlist_key, session) + if isinstance(media_drm, (Widevine, PlayReady)): + track_kid = HLS.get_track_kid_from_init(master, track, session) or media_drm.kid + try: + if not license_widevine: + raise ValueError("license_widevine func must be supplied to use DRM") + progress(downloaded="LICENSING") + license_widevine(media_drm, track_kid=track_kid) + progress(downloaded="[yellow]LICENSED") + initial_drm_licensed = True + initial_drm_key = media_playlist_key + track.drm = [media_drm] + session_drm = media_drm + except Exception: # noqa + DOWNLOAD_CANCELLED.set() # skip pending track downloads + progress(downloaded="[red]FAILED") + raise + + # Fall back to session DRM if media playlist has no matching keys + if not initial_drm_licensed and session_drm and isinstance(session_drm, (Widevine, PlayReady)): + try: + if not license_widevine: + raise ValueError("license_widevine func must be supplied to use DRM") + progress(downloaded="LICENSING") + license_widevine(session_drm) + progress(downloaded="[yellow]LICENSED") + except Exception: # noqa + DOWNLOAD_CANCELLED.set() # skip pending track downloads + progress(downloaded="[red]FAILED") + raise + + if not initial_drm_licensed and session_drm and isinstance(session_drm, MonaLisa): + try: + if not license_widevine: + raise ValueError("license_widevine func must be supplied to use DRM") + progress(downloaded="LICENSING") + license_widevine(session_drm) + progress(downloaded="[yellow]LICENSED") + except Exception: # noqa + DOWNLOAD_CANCELLED.set() # skip pending track downloads + progress(downloaded="[red]FAILED") + raise + if DOWNLOAD_LICENCE_ONLY.is_set(): progress(downloaded="[yellow]SKIPPED") return @@ -341,15 +423,36 @@ class HLS: if downloader.__name__ == "n_m3u8dl_re": skip_merge = True + # session_drm already has correct content_keys from initial licensing above + n_m3u8dl_content_keys = session_drm.content_keys if session_drm else None + downloader_args.update( { "output_dir": save_dir, "filename": track.id, "track": track, - "content_keys": session_drm.content_keys if session_drm else None, + "content_keys": n_m3u8dl_content_keys, } ) + debug_logger = get_debug_logger() + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="manifest_hls_download_start", + message="Starting HLS manifest download", + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "total_segments": total_segments, + "downloader": downloader.__name__, + "has_drm": bool(session_drm), + "drm_type": session_drm.__class__.__name__ if session_drm else None, + "skip_merge": skip_merge, + "save_path": str(save_path), + }, + ) + for status_update in downloader(**downloader_args): file_downloaded = status_update.get("file_downloaded") if file_downloaded: @@ -364,216 +467,228 @@ class HLS: for control_file in segment_save_dir.glob("*.aria2__temp"): control_file.unlink() - if not skip_merge: - progress(total=total_segments, completed=0, downloaded="Merging") + if skip_merge: + final_save_path = HLS._finalize_n_m3u8dl_re_output(track=track, save_dir=save_dir, save_path=save_path) + progress(downloaded="Downloaded") + track.path = final_save_path + events.emit(events.Types.TRACK_DOWNLOADED, track=track) + return - name_len = len(str(total_segments)) - discon_i = 0 - range_offset = 0 - map_data: Optional[tuple[m3u8.model.InitializationSection, bytes]] = None - if session_drm: - encryption_data: Optional[tuple[Optional[m3u8.Key], DRM_T]] = (None, session_drm) - else: - encryption_data: Optional[tuple[Optional[m3u8.Key], DRM_T]] = None + progress(total=total_segments, completed=0, downloaded="Merging") - i = -1 - for real_i, segment in enumerate(master.segments): - if segment not in unwanted_segments: - i += 1 + name_len = len(str(total_segments)) + discon_i = 0 + range_offset = 0 + map_data: Optional[tuple[m3u8.model.InitializationSection, bytes]] = None + if session_drm: + encryption_data: Optional[tuple[Optional[m3u8.Key], DRM_T]] = (initial_drm_key, session_drm) + else: + encryption_data: Optional[tuple[Optional[m3u8.Key], DRM_T]] = None - is_last_segment = (real_i + 1) == len(master.segments) + i = -1 + for real_i, segment in enumerate(master.segments): + if segment not in unwanted_segments: + i += 1 - def merge(to: Path, via: list[Path], delete: bool = False, include_map_data: bool = False): - """ - Merge all files to a given path, optionally including map data. + is_last_segment = (real_i + 1) == len(master.segments) - Parameters: - to: The output file with all merged data. - via: List of files to merge, in sequence. - delete: Delete the file once it's been merged. - include_map_data: Whether to include the init map data. - """ - with open(to, "wb") as x: - if include_map_data and map_data and map_data[1]: - x.write(map_data[1]) - for file in via: - x.write(file.read_bytes()) - x.flush() - if delete: - file.unlink() + def merge(to: Path, via: list[Path], delete: bool = False, include_map_data: bool = False): + """ + Merge all files to a given path, optionally including map data. - def decrypt(include_this_segment: bool) -> Path: - """ - Decrypt all segments that uses the currently set DRM. + Parameters: + to: The output file with all merged data. + via: List of files to merge, in sequence. + delete: Delete the file once it's been merged. + include_map_data: Whether to include the init map data. + """ + with open(to, "wb") as x: + if include_map_data and map_data and map_data[1]: + x.write(map_data[1]) + for file in via: + x.write(file.read_bytes()) + x.flush() + if delete: + file.unlink() - All segments that will be decrypted with this DRM will be merged together - in sequence, prefixed with the init data (if any), and then deleted. Once - merged they will be decrypted. The merged and decrypted file names state - the range of segments that were used. + def decrypt(include_this_segment: bool) -> Path: + """ + Decrypt all segments that uses the currently set DRM. - Parameters: - include_this_segment: Whether to include the current segment in the - list of segments to merge and decrypt. This should be False if - decrypting on EXT-X-KEY changes, or True when decrypting on the - last segment. + All segments that will be decrypted with this DRM will be merged together + in sequence, prefixed with the init data (if any), and then deleted. Once + merged they will be decrypted. The merged and decrypted file names state + the range of segments that were used. - Returns the decrypted path. - """ - drm = encryption_data[1] - first_segment_i = next( - int(file.stem) for file in sorted(segment_save_dir.iterdir()) if file.stem.isdigit() - ) - last_segment_i = max(0, i - int(not include_this_segment)) - range_len = (last_segment_i - first_segment_i) + 1 + Parameters: + include_this_segment: Whether to include the current segment in the + list of segments to merge and decrypt. This should be False if + decrypting on EXT-X-KEY changes, or True when decrypting on the + last segment. - segment_range = f"{str(first_segment_i).zfill(name_len)}-{str(last_segment_i).zfill(name_len)}" - merged_path = ( - segment_save_dir / f"{segment_range}{get_extension(master.segments[last_segment_i].uri)}" - ) - decrypted_path = segment_save_dir / f"{merged_path.stem}_decrypted{merged_path.suffix}" + Returns the decrypted path. + """ + drm = encryption_data[1] + first_segment_i = next( + int(file.stem) for file in sorted(segment_save_dir.iterdir()) if file.stem.isdigit() + ) + last_segment_i = max(0, i - int(not include_this_segment)) + range_len = (last_segment_i - first_segment_i) + 1 - files = [ - file - for file in sorted(segment_save_dir.iterdir()) - if file.stem.isdigit() and first_segment_i <= int(file.stem) <= last_segment_i - ] - if not files: - raise ValueError(f"None of the segment files for {segment_range} exist...") - elif len(files) != range_len: - raise ValueError(f"Missing {range_len - len(files)} segment files for {segment_range}...") + segment_range = f"{str(first_segment_i).zfill(name_len)}-{str(last_segment_i).zfill(name_len)}" + merged_path = segment_save_dir / f"{segment_range}{get_extension(master.segments[last_segment_i].uri)}" + decrypted_path = segment_save_dir / f"{merged_path.stem}_decrypted{merged_path.suffix}" - if isinstance(drm, (Widevine, PlayReady)): - # with widevine we can merge all segments and decrypt once - merge(to=merged_path, via=files, delete=True, include_map_data=True) - drm.decrypt(merged_path) - merged_path.rename(decrypted_path) - else: - # with other drm we must decrypt separately and then merge them - # for aes this is because each segment likely has 16-byte padding - for file in files: - drm.decrypt(file) - merge(to=merged_path, via=files, delete=True, include_map_data=True) + files = [ + file + for file in sorted(segment_save_dir.iterdir()) + if file.stem.isdigit() and first_segment_i <= int(file.stem) <= last_segment_i + ] + if not files: + raise ValueError(f"None of the segment files for {segment_range} exist...") + elif len(files) != range_len: + raise ValueError(f"Missing {range_len - len(files)} segment files for {segment_range}...") - events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=drm, segment=decrypted_path) + if isinstance(drm, (Widevine, PlayReady)): + # with widevine we can merge all segments and decrypt once + merge(to=merged_path, via=files, delete=True, include_map_data=True) + drm.decrypt(merged_path) + merged_path.rename(decrypted_path) + else: + # with other drm we must decrypt separately and then merge them + # for aes this is because each segment likely has 16-byte padding + for file in files: + drm.decrypt(file) + merge(to=merged_path, via=files, delete=True, include_map_data=True) - return decrypted_path + events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=drm, segment=decrypted_path) - def merge_discontinuity(include_this_segment: bool, include_map_data: bool = True): - """ - Merge all segments of the discontinuity. + return decrypted_path - All segment files for this discontinuity must already be downloaded and - already decrypted (if it needs to be decrypted). + def merge_discontinuity(include_this_segment: bool, include_map_data: bool = True): + """ + Merge all segments of the discontinuity. - Parameters: - include_this_segment: Whether to include the current segment in the - list of segments to merge and decrypt. This should be False if - decrypting on EXT-X-KEY changes, or True when decrypting on the - last segment. - include_map_data: Whether to prepend the init map data before the - segment files when merging. - """ - last_segment_i = max(0, i - int(not include_this_segment)) + All segment files for this discontinuity must already be downloaded and + already decrypted (if it needs to be decrypted). - files = [ - file - for file in sorted(segment_save_dir.iterdir()) - if int(file.stem.replace("_decrypted", "").split("-")[-1]) <= last_segment_i - ] - if files: - to_dir = segment_save_dir.parent - to_path = to_dir / f"{str(discon_i).zfill(name_len)}{files[-1].suffix}" - merge(to=to_path, via=files, delete=True, include_map_data=include_map_data) + Parameters: + include_this_segment: Whether to include the current segment in the + list of segments to merge and decrypt. This should be False if + decrypting on EXT-X-KEY changes, or True when decrypting on the + last segment. + include_map_data: Whether to prepend the init map data before the + segment files when merging. + """ + last_segment_i = max(0, i - int(not include_this_segment)) - if segment not in unwanted_segments: - if isinstance(track, Subtitle): - segment_file_ext = get_extension(segment.uri) - segment_file_path = segment_save_dir / f"{str(i).zfill(name_len)}{segment_file_ext}" - segment_data = try_ensure_utf8(segment_file_path.read_bytes()) - if track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML): - segment_data = ( - segment_data.decode("utf8") - .replace("‎", html.unescape("‎")) - .replace("‏", html.unescape("‏")) - .encode("utf8") - ) - segment_file_path.write_bytes(segment_data) + files = [ + file + for file in sorted(segment_save_dir.iterdir()) + if int(file.stem.replace("_decrypted", "").split("-")[-1]) <= last_segment_i + ] + if files: + to_dir = segment_save_dir.parent + to_path = to_dir / f"{str(discon_i).zfill(name_len)}{files[-1].suffix}" + merge(to=to_path, via=files, delete=True, include_map_data=include_map_data) - if segment.discontinuity and i != 0: - if encryption_data: - decrypt(include_this_segment=False) - merge_discontinuity( - include_this_segment=False, include_map_data=not encryption_data or not encryption_data[1] + if segment not in unwanted_segments: + if isinstance(track, Subtitle): + segment_file_ext = get_extension(segment.uri) + segment_file_path = segment_save_dir / f"{str(i).zfill(name_len)}{segment_file_ext}" + segment_data = try_ensure_utf8(segment_file_path.read_bytes()) + if track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML): + segment_data = ( + segment_data.decode("utf8") + .replace("‎", html.unescape("‎")) + .replace("‏", html.unescape("‏")) + .encode("utf8") ) + segment_file_path.write_bytes(segment_data) - discon_i += 1 - range_offset = 0 # TODO: Should this be reset or not? - map_data = None - if encryption_data: - encryption_data = (encryption_data[0], encryption_data[1]) - - if segment.init_section and (not map_data or segment.init_section != map_data[0]): - if segment.init_section.byterange: - init_byte_range = HLS.calculate_byte_range(segment.init_section.byterange, range_offset) - range_offset = init_byte_range.split("-")[0] - init_range_header = {"Range": f"bytes={init_byte_range}"} - else: - init_range_header = {} - - # Handle both session types for init section request - res = session.get( - url=urljoin(segment.init_section.base_uri, segment.init_section.uri), - headers=init_range_header, - ) - - # Check response based on session type - if isinstance(res, requests.Response): - res.raise_for_status() - init_content = res.content - else: - raise TypeError( - f"Expected response to be requests.Response or curl_cffi.Response, not {type(res)}" - ) - - map_data = (segment.init_section, init_content) - - segment_keys = getattr(segment, "keys", None) - if segment_keys: - key = HLS.get_supported_key(segment_keys) - if encryption_data and encryption_data[0] != key and i != 0 and segment not in unwanted_segments: - decrypt(include_this_segment=False) - - if key is None: - encryption_data = None - elif not encryption_data or encryption_data[0] != key: - drm = HLS.get_drm(key, session) - if isinstance(drm, (Widevine, PlayReady)): - try: - if map_data: - track_kid = track.get_key_id(map_data[1]) - else: - track_kid = None - progress(downloaded="LICENSING") - license_widevine(drm, track_kid=track_kid) - progress(downloaded="[yellow]LICENSED") - except Exception: # noqa - DOWNLOAD_CANCELLED.set() # skip pending track downloads - progress(downloaded="[red]FAILED") - raise - encryption_data = (key, drm) - - if DOWNLOAD_LICENCE_ONLY.is_set(): - continue - - if is_last_segment: - # required as it won't end with EXT-X-DISCONTINUITY nor a new key + if segment.discontinuity and i != 0: if encryption_data: - decrypt(include_this_segment=True) + decrypt(include_this_segment=False) merge_discontinuity( - include_this_segment=True, include_map_data=not encryption_data or not encryption_data[1] + include_this_segment=False, include_map_data=not encryption_data or not encryption_data[1] ) - progress(advance=1) + discon_i += 1 + range_offset = 0 # TODO: Should this be reset or not? + map_data = None + + if segment.init_section and (not map_data or segment.init_section != map_data[0]): + if segment.init_section.byterange: + init_byte_range = HLS.calculate_byte_range(segment.init_section.byterange, range_offset) + range_offset = int(init_byte_range.split("-")[0]) + init_range_header = {"Range": f"bytes={init_byte_range}"} + else: + init_range_header = {} + + # Handle both session types for init section request + res = session.get( + url=urljoin(segment.init_section.base_uri, segment.init_section.uri), + headers=init_range_header, + ) + + # Check response based on session type + if isinstance(res, requests.Response) or isinstance(res, CurlResponse): + res.raise_for_status() + init_content = res.content + else: + raise TypeError( + f"Expected response to be requests.Response or curl_cffi.Response, not {type(res)}" + ) + + map_data = (segment.init_section, init_content) + + segment_keys = getattr(segment, "keys", None) + if segment_keys: + if cdm: + cdm_segment_keys = HLS.filter_keys_for_cdm(segment_keys, cdm) + key = ( + HLS.get_supported_key(cdm_segment_keys) + if cdm_segment_keys + else HLS.get_supported_key(segment_keys) + ) + else: + key = HLS.get_supported_key(segment_keys) + if encryption_data and encryption_data[0] != key and i != 0 and segment not in unwanted_segments: + decrypt(include_this_segment=False) + + if key is None: + encryption_data = None + elif not encryption_data or encryption_data[0] != key: + drm = HLS.get_drm(key, session) + if isinstance(drm, (Widevine, PlayReady)): + try: + if map_data: + track_kid = track.get_key_id(map_data[1]) + else: + track_kid = None + if not track_kid: + track_kid = drm.kid + progress(downloaded="LICENSING") + license_widevine(drm, track_kid=track_kid) + progress(downloaded="[yellow]LICENSED") + except Exception: # noqa + DOWNLOAD_CANCELLED.set() # skip pending track downloads + progress(downloaded="[red]FAILED") + raise + encryption_data = (key, drm) + + if DOWNLOAD_LICENCE_ONLY.is_set(): + continue + + if is_last_segment: + # required as it won't end with EXT-X-DISCONTINUITY nor a new key + if encryption_data: + decrypt(include_this_segment=True) + merge_discontinuity( + include_this_segment=True, include_map_data=not encryption_data or not encryption_data[1] + ) + + progress(advance=1) if DOWNLOAD_LICENCE_ONLY.is_set(): return @@ -596,6 +711,44 @@ class HLS: # finally merge all the discontinuity save files together to the final path segments_to_merge = find_segments_recursively(save_dir) + + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="manifest_hls_download_complete", + message="HLS download complete, preparing to merge", + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "save_dir": str(save_dir), + "save_dir_exists": save_dir.exists(), + "segments_found": len(segments_to_merge), + "segment_files": [f.name for f in segments_to_merge[:10]], # Limit to first 10 + "downloader": downloader.__name__, + "skip_merge": skip_merge, + }, + ) + + if not segments_to_merge: + error_msg = f"No segment files found in output directory: {save_dir}" + if debug_logger: + all_contents = list(save_dir.iterdir()) if save_dir.exists() else [] + debug_logger.log( + level="ERROR", + operation="manifest_hls_download_no_segments", + message=error_msg, + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "save_dir": str(save_dir), + "save_dir_exists": save_dir.exists(), + "directory_contents": [str(p) for p in all_contents], + "downloader": downloader.__name__, + "skip_merge": skip_merge, + }, + ) + raise FileNotFoundError(error_msg) + if len(segments_to_merge) == 1: shutil.move(segments_to_merge[0], save_path) else: @@ -752,6 +905,55 @@ class HLS: return keys + @staticmethod + def filter_keys_for_cdm( + keys: list[Union[m3u8.model.SessionKey, m3u8.model.Key]], + cdm: object, + ) -> list[Union[m3u8.model.SessionKey, m3u8.model.Key]]: + """ + Filter EXT-X-KEY entries to only include those matching the CDM type. + + This ensures we select the correct DRM system (Widevine vs PlayReady) + based on what CDM is configured, avoiding license request failures. + """ + playready_urn = f"urn:uuid:{PR_PSSH.SYSTEM_ID}" + playready_keyformats = {playready_urn, "com.microsoft.playready"} + if is_widevine_cdm(cdm): + return [k for k in keys if k.keyformat and k.keyformat.lower() == WidevineCdm.urn] + elif is_playready_cdm(cdm): + return [k for k in keys if k.keyformat and k.keyformat.lower() in playready_keyformats] + return keys + + @staticmethod + def get_track_kid_from_init( + master: M3U8, + track: AnyTrack, + session: Union[Session, CurlSession], + ) -> Optional[UUID]: + """ + Extract the track's Key ID from its init segment (EXT-X-MAP). + + Returns None if no init segment exists or KID extraction fails. + The caller should fall back to drm.kid from the PSSH if this returns None. + """ + map_section = next((seg.init_section for seg in master.segments if seg.init_section), None) + if not map_section: + return None + + map_uri = urljoin(map_section.base_uri or master.base_uri or "", map_section.uri) + try: + if map_section.byterange: + byte_range = HLS.calculate_byte_range(map_section.byterange, 0) + headers = {"Range": f"bytes={byte_range}"} + else: + headers = {} + map_res = session.get(url=map_uri, headers=headers) + if map_res.ok: + return track.get_key_id(map_res.content) + except Exception: + pass + return None + @staticmethod def get_supported_key(keys: list[Union[m3u8.model.SessionKey, m3u8.model.Key]]) -> Optional[m3u8.Key]: """ @@ -780,9 +982,10 @@ class HLS: return key elif key.keyformat and key.keyformat.lower() == WidevineCdm.urn: return key - elif key.keyformat and ( - key.keyformat.lower() == PlayReadyCdm or "com.microsoft.playready" in key.keyformat.lower() - ): + elif key.keyformat and key.keyformat.lower() in { + f"urn:uuid:{PR_PSSH.SYSTEM_ID}", + "com.microsoft.playready", + }: return key else: unsupported_systems.append(key.method + (f" ({key.keyformat})" if key.keyformat else "")) @@ -819,9 +1022,7 @@ class HLS: pssh=WV_PSSH(key.uri.split(",")[-1]), **key._extra_params, # noqa ) - elif key.keyformat and ( - key.keyformat.lower() == PlayReadyCdm or "com.microsoft.playready" in key.keyformat.lower() - ): + elif key.keyformat and key.keyformat.lower() in {f"urn:uuid:{PR_PSSH.SYSTEM_ID}", "com.microsoft.playready"}: drm = PlayReady( pssh=PR_PSSH(key.uri.split(",")[-1]), pssh_b64=key.uri.split(",")[-1], diff --git a/packages/envied/src/envied/core/manifests/ism.py b/packages/envied/src/envied/core/manifests/ism.py index 8653db7..596421f 100644 --- a/packages/envied/src/envied/core/manifests/ism.py +++ b/packages/envied/src/envied/core/manifests/ism.py @@ -21,7 +21,7 @@ from envied.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, Any from envied.core.drm import DRM_T, PlayReady, Widevine from envied.core.events import events from envied.core.tracks import Audio, Subtitle, Track, Tracks, Video -from envied.core.utilities import try_ensure_utf8 +from envied.core.utilities import get_debug_logger, try_ensure_utf8 from envied.core.utils.xml import load_xml @@ -283,6 +283,24 @@ class ISM: } ) + debug_logger = get_debug_logger() + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="manifest_ism_download_start", + message="Starting ISM manifest download", + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "total_segments": len(segments), + "downloader": downloader.__name__, + "has_drm": bool(session_drm), + "drm_type": session_drm.__class__.__name__ if session_drm else None, + "skip_merge": skip_merge, + "save_path": str(save_path), + }, + ) + for status_update in downloader(**downloader_args): file_downloaded = status_update.get("file_downloaded") if file_downloaded: @@ -296,8 +314,63 @@ class ISM: for control_file in save_dir.glob("*.aria2__temp"): control_file.unlink() + # Verify output directory exists and contains files + if not save_dir.exists(): + error_msg = f"Output directory does not exist: {save_dir}" + if debug_logger: + debug_logger.log( + level="ERROR", + operation="manifest_ism_download_output_missing", + message=error_msg, + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "save_dir": str(save_dir), + "save_path": str(save_path), + "downloader": downloader.__name__, + "skip_merge": skip_merge, + }, + ) + raise FileNotFoundError(error_msg) + segments_to_merge = [x for x in sorted(save_dir.iterdir()) if x.is_file()] + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="manifest_ism_download_complete", + message="ISM download complete, preparing to merge", + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "save_dir": str(save_dir), + "save_dir_exists": save_dir.exists(), + "segments_found": len(segments_to_merge), + "segment_files": [f.name for f in segments_to_merge[:10]], # Limit to first 10 + "downloader": downloader.__name__, + "skip_merge": skip_merge, + }, + ) + + if not segments_to_merge: + error_msg = f"No segment files found in output directory: {save_dir}" + if debug_logger: + all_contents = list(save_dir.iterdir()) if save_dir.exists() else [] + debug_logger.log( + level="ERROR", + operation="manifest_ism_download_no_segments", + message=error_msg, + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "save_dir": str(save_dir), + "directory_contents": [str(p) for p in all_contents], + "downloader": downloader.__name__, + "skip_merge": skip_merge, + }, + ) + raise FileNotFoundError(error_msg) + if skip_merge: shutil.move(segments_to_merge[0], save_path) else: diff --git a/packages/envied/src/envied/core/proxies/__init__.py b/packages/envied/src/envied/core/proxies/__init__.py index ecb97de..4a53298 100644 --- a/packages/envied/src/envied/core/proxies/__init__.py +++ b/packages/envied/src/envied/core/proxies/__init__.py @@ -1,7 +1,8 @@ from .basic import Basic +from .gluetun import Gluetun from .hola import Hola from .nordvpn import NordVPN from .surfsharkvpn import SurfsharkVPN from .windscribevpn import WindscribeVPN -__all__ = ("Basic", "Hola", "NordVPN", "SurfsharkVPN", "WindscribeVPN") +__all__ = ("Basic", "Gluetun", "Hola", "NordVPN", "SurfsharkVPN", "WindscribeVPN") diff --git a/packages/envied/src/envied/core/proxies/gluetun.py b/packages/envied/src/envied/core/proxies/gluetun.py new file mode 100644 index 0000000..12cb10e --- /dev/null +++ b/packages/envied/src/envied/core/proxies/gluetun.py @@ -0,0 +1,1408 @@ +import atexit +import logging +import os +import re +import stat +import subprocess +import tempfile +import threading +import time +from typing import Optional + +import requests + +from envied.core import binaries +from envied.core.proxies.proxy import Proxy +from envied.core.utilities import get_country_code, get_country_name, get_debug_logger, get_ip_info + +# Global registry for cleanup on exit +_gluetun_instances: list["Gluetun"] = [] +_cleanup_lock = threading.Lock() +_cleanup_registered = False + + +def _cleanup_all_gluetun_containers(): + """Cleanup all Gluetun containers on exit.""" + # Get instances without holding the lock during cleanup + with _cleanup_lock: + instances = list(_gluetun_instances) + _gluetun_instances.clear() + + # Cleanup each instance (no lock held, so no deadlock possible) + for instance in instances: + try: + instance.cleanup() + except Exception: + pass + + +def _register_cleanup(): + """Register cleanup handlers (only once).""" + global _cleanup_registered + with _cleanup_lock: + if not _cleanup_registered: + # Only use atexit for cleanup - don't override signal handlers + # This allows Ctrl+C to work normally while still cleaning up on exit + atexit.register(_cleanup_all_gluetun_containers) + _cleanup_registered = True + + +class Gluetun(Proxy): + """ + Dynamic Gluetun VPN-to-HTTP Proxy Provider with multi-provider support. + + Automatically manages Docker containers running Gluetun for WireGuard/OpenVPN VPN connections. + Supports multiple VPN providers in a single configuration using query format: provider:region + + Supported VPN providers: windscribe, expressvpn, nordvpn, surfshark, protonvpn, mullvad, + privateinternetaccess, cyberghost, vyprvpn, torguard, and 50+ more. + + Configuration example in envied.yaml: + proxy_providers: + gluetun: + providers: + windscribe: + vpn_type: wireguard + credentials: + private_key: YOUR_KEY + addresses: YOUR_ADDRESS + server_countries: + us: US + uk: GB + nordvpn: + vpn_type: wireguard + credentials: + private_key: YOUR_KEY + addresses: YOUR_ADDRESS + server_countries: + us: US + de: DE + # Global settings (optional) + base_port: 8888 + auto_cleanup: true + container_prefix: "unshackle-gluetun" + + Usage: + --proxy gluetun:windscribe:us + --proxy gluetun:nordvpn:de + """ + + # Mapping of common VPN provider names to Gluetun identifiers + PROVIDER_MAPPING = { + "windscribe": "windscribe", + "expressvpn": "expressvpn", + "nordvpn": "nordvpn", + "surfshark": "surfshark", + "protonvpn": "protonvpn", + "mullvad": "mullvad", + "pia": "private internet access", + "privateinternetaccess": "private internet access", + "cyberghost": "cyberghost", + "vyprvpn": "vyprvpn", + "torguard": "torguard", + "ipvanish": "ipvanish", + "purevpn": "purevpn", + } + + # Windscribe uses specific region names instead of country codes + # See: https://github.com/qdm12/gluetun-wiki/blob/main/setup/providers/windscribe.md + WINDSCRIBE_REGION_MAP = { + # Country codes to Windscribe region names + "us": "US East", + "us-east": "US East", + "us-west": "US West", + "us-central": "US Central", + "ca": "Canada East", + "ca-east": "Canada East", + "ca-west": "Canada West", + "uk": "United Kingdom", + "gb": "United Kingdom", + "de": "Germany", + "fr": "France", + "nl": "Netherlands", + "au": "Australia", + "jp": "Japan", + "sg": "Singapore", + "hk": "Hong Kong", + "kr": "South Korea", + "in": "India", + "it": "Italy", + "es": "Spain", + "ch": "Switzerland", + "se": "Sweden", + "no": "Norway", + "dk": "Denmark", + "fi": "Finland", + "at": "Austria", + "be": "Belgium", + "ie": "Ireland", + "pl": "Poland", + "pt": "Portugal", + "cz": "Czech Republic", + "ro": "Romania", + "hu": "Hungary", + "gr": "Greece", + "tr": "Turkey", + "ru": "Russia", + "ua": "Ukraine", + "br": "Brazil", + "mx": "Mexico", + "ar": "Argentina", + "za": "South Africa", + "nz": "New Zealand", + "th": "Thailand", + "ph": "Philippines", + "id": "Indonesia", + "my": "Malaysia", + "vn": "Vietnam", + "tw": "Taiwan", + "ae": "United Arab Emirates", + "il": "Israel", + } + + def __init__( + self, + providers: Optional[dict] = None, + base_port: int = 8888, + auto_cleanup: bool = True, + container_prefix: str = "unshackle-gluetun", + auth_user: Optional[str] = None, + auth_password: Optional[str] = None, + verify_ip: bool = True, + **kwargs, + ): + """ + Initialize Gluetun proxy provider with multi-provider support. + + Args: + providers: Dict of VPN provider configurations + Format: { + "windscribe": { + "vpn_type": "wireguard", + "credentials": {"private_key": "...", "addresses": "..."}, + "server_countries": {"us": "US", "uk": "GB"} + }, + "nordvpn": {...} + } + base_port: Starting port for HTTP proxies (default: 8888) + auto_cleanup: Automatically remove stopped containers (default: True) + container_prefix: Docker container name prefix (default: "unshackle-gluetun") + auth_user: Optional HTTP proxy authentication username + auth_password: Optional HTTP proxy authentication password + verify_ip: Automatically verify IP and region after connection (default: True) + """ + # Check Docker availability using binaries module + if not binaries.Docker: + raise RuntimeError( + "Docker is not available. Please install Docker to use Gluetun proxy.\n" + "Visit: https://docs.docker.com/engine/install/" + ) + + self.providers = providers or {} + self.base_port = base_port + self.auto_cleanup = auto_cleanup + self.container_prefix = container_prefix + self.auth_user = auth_user + self.auth_password = auth_password + self.verify_ip = verify_ip + + # Track active containers: {query_key: {"container_name": ..., "port": ..., ...}} + self.active_containers = {} + + # Lock for thread-safe port allocation + self._port_lock = threading.Lock() + + # Validate provider configurations + for provider_name, config in self.providers.items(): + self._validate_provider_config(provider_name, config) + + # Register this instance for cleanup on exit + _register_cleanup() + with _cleanup_lock: + _gluetun_instances.append(self) + + # Log initialization + debug_logger = get_debug_logger() + if debug_logger: + debug_logger.log( + level="INFO", + operation="gluetun_init", + message=f"Gluetun proxy provider initialized with {len(self.providers)} provider(s)", + context={ + "providers": list(self.providers.keys()), + "base_port": base_port, + "auto_cleanup": auto_cleanup, + "verify_ip": verify_ip, + "container_prefix": container_prefix, + }, + ) + + def __repr__(self) -> str: + provider_count = len(self.providers) + return f"Gluetun ({provider_count} provider{['s', ''][provider_count == 1]})" + + def get_proxy(self, query: str) -> Optional[str]: + """ + Get an HTTP proxy URI for a Gluetun VPN connection. + + Args: + query: Query format: "provider:region" (e.g., "windscribe:us", "nordvpn:uk") + + Returns: + HTTP proxy URI or None if unavailable + """ + # Parse query + parts = query.split(":") + if len(parts) != 2: + raise ValueError(f"Invalid query format: '{query}'. Expected 'provider:region' (e.g., 'windscribe:us')") + + provider_name = parts[0].lower() + region = parts[1].lower() + + # Check if provider is configured + if provider_name not in self.providers: + available = ", ".join(self.providers.keys()) + raise ValueError(f"VPN provider '{provider_name}' not configured. Available providers: {available}") + + # Create query key for tracking + query_key = f"{provider_name}:{region}" + container_name = f"{self.container_prefix}-{provider_name}-{region}" + + debug_logger = get_debug_logger() + + # Check if container already exists (in memory OR in Docker) + # This handles multiple concurrent Unshackle sessions + if query_key in self.active_containers: + container = self.active_containers[query_key] + if self._is_container_running(container["container_name"]): + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="gluetun_container_reuse", + message=f"Reusing existing container (in-memory): {query_key}", + context={ + "query_key": query_key, + "container_name": container["container_name"], + "port": container["port"], + }, + ) + # Re-verify if needed + if self.verify_ip: + self._verify_container(query_key) + return self._build_proxy_uri(container["port"]) + else: + # Not in memory, but might exist in Docker (from another session) + existing_info = self._get_existing_container_info(container_name) + if existing_info: + # Container exists in Docker, reuse it + self.active_containers[query_key] = existing_info + if debug_logger: + debug_logger.log( + level="INFO", + operation="gluetun_container_reuse_docker", + message=f"Reusing existing Docker container: {query_key}", + context={ + "query_key": query_key, + "container_name": container_name, + "port": existing_info["port"], + }, + ) + # Re-verify if needed + if self.verify_ip: + self._verify_container(query_key) + return self._build_proxy_uri(existing_info["port"]) + + # Get provider configuration + provider_config = self.providers[provider_name] + + # Determine server location + server_countries = provider_config.get("server_countries", {}) + server_cities = provider_config.get("server_cities", {}) + server_hostnames = provider_config.get("server_hostnames", {}) + + country = server_countries.get(region) + city = server_cities.get(region) + hostname = server_hostnames.get(region) + + # Check if region is a specific server pattern (e.g., us1239, uk5678) + # Format: 2-letter country code + number + specific_server_match = re.match(r"^([a-z]{2})(\d+)$", region, re.IGNORECASE) + + if specific_server_match and not country and not city and not hostname: + # Specific server requested (e.g., us1239) + country_code = specific_server_match.group(1).upper() + server_num = specific_server_match.group(2) + + # Build hostname based on provider + hostname = self._build_server_hostname(provider_name, country_code, server_num) + country = country_code # Set country for verification + + # If not explicitly mapped and not a specific server, try to use query as country code + elif not country and not city and not hostname: + if re.match(r"^[a-z]{2}$", region): + # Convert country code to full name for Gluetun + country = get_country_name(region) + if not country: + raise ValueError( + f"Country code '{region}' not recognized. " + f"Configure it in server_countries or use a valid ISO 3166-1 alpha-2 code." + ) + else: + raise ValueError( + f"Region '{region}' not recognized for provider '{provider_name}'. " + f"Configure it in server_countries or server_cities, or use a 2-letter country code." + ) + + # Remove any stopped container with the same name + self._remove_stopped_container(container_name) + + # Find available port + port = self._get_available_port() + + # Create container (name already set above) + try: + self._create_container( + container_name=container_name, + port=port, + provider_name=provider_name, + provider_config=provider_config, + country=country, + city=city, + hostname=hostname, + ) + + # Store container info + self.active_containers[query_key] = { + "container_name": container_name, + "port": port, + "provider": provider_name, + "region": region, + "country": country, + "city": city, + "hostname": hostname, + } + + # Wait for container to be ready (60s timeout for VPN connection) + if not self._wait_for_container(container_name, timeout=60): + # Get container logs for better error message + logs = self._get_container_logs(container_name, tail=30) + error_msg = f"Gluetun container '{container_name}' failed to start" + if hasattr(self, "_last_wait_error") and self._last_wait_error: + error_msg += f": {self._last_wait_error}" + if logs: + # Extract last few relevant lines + log_lines = [line for line in logs.strip().split("\n") if line.strip()][-5:] + error_msg += "\nRecent logs:\n" + "\n".join(log_lines) + raise RuntimeError(error_msg) + + # Verify IP and region if enabled + if self.verify_ip: + self._verify_container(query_key) + + return self._build_proxy_uri(port) + + except Exception as e: + # Cleanup on failure + self._remove_container(container_name) + if query_key in self.active_containers: + del self.active_containers[query_key] + raise RuntimeError(f"Failed to create Gluetun container: {e}") + + def cleanup(self): + """Stop and remove all managed Gluetun containers.""" + debug_logger = get_debug_logger() + container_count = len(self.active_containers) + + if container_count > 0 and debug_logger: + debug_logger.log( + level="DEBUG", + operation="gluetun_cleanup_start", + message=f"Cleaning up {container_count} Gluetun container(s)", + context={ + "container_count": container_count, + "containers": list(self.active_containers.keys()), + }, + ) + + for query_key, container_info in list(self.active_containers.items()): + container_name = container_info["container_name"] + self._remove_container(container_name) + + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="gluetun_container_removed", + message=f"Removed Gluetun container: {container_name}", + context={ + "query_key": query_key, + "container_name": container_name, + }, + ) + + self.active_containers.clear() + + if container_count > 0 and debug_logger: + debug_logger.log( + level="INFO", + operation="gluetun_cleanup_complete", + message=f"Cleanup complete: removed {container_count} container(s)", + context={"container_count": container_count}, + success=True, + ) + + def get_connection_info(self, query: str) -> Optional[dict]: + """ + Get connection info for a proxy query. + + Args: + query: Query format "provider:region" (e.g., "windscribe:us") + + Returns: + Dict with connection info including public_ip, country, city, or None if not found. + """ + parts = query.split(":") + if len(parts) != 2: + return None + + provider_name = parts[0].lower() + region = parts[1].lower() + query_key = f"{provider_name}:{region}" + + container = self.active_containers.get(query_key) + if not container: + return None + + return { + "provider": container.get("provider"), + "region": container.get("region"), + "public_ip": container.get("public_ip"), + "country": container.get("ip_country"), + "city": container.get("ip_city"), + "org": container.get("ip_org"), + } + + def _validate_provider_config(self, provider_name: str, config: dict): + """Validate a provider's configuration.""" + vpn_type = config.get("vpn_type", "wireguard").lower() + credentials = config.get("credentials", {}) + + if vpn_type not in ["wireguard", "openvpn"]: + raise ValueError(f"Provider '{provider_name}': Invalid vpn_type '{vpn_type}'. Use 'wireguard' or 'openvpn'") + + if vpn_type == "wireguard": + # private_key is always required for WireGuard + if "private_key" not in credentials: + raise ValueError(f"Provider '{provider_name}': WireGuard requires 'private_key' in credentials") + + # Provider-specific WireGuard requirements based on Gluetun wiki: + # - NordVPN, ProtonVPN: only private_key required + # - Windscribe: private_key, addresses, AND preshared_key required (preshared_key MUST be set) + # - Surfshark, Mullvad, IVPN: private_key AND addresses required + provider_lower = provider_name.lower() + + # Windscribe requires preshared_key (can be empty string, but must be set) + if provider_lower == "windscribe": + if "preshared_key" not in credentials: + raise ValueError( + f"Provider '{provider_name}': Windscribe WireGuard requires 'preshared_key' in credentials " + "(can be empty string, but must be set). Get it from windscribe.com/getconfig/wireguard" + ) + if "addresses" not in credentials: + raise ValueError( + f"Provider '{provider_name}': Windscribe WireGuard requires 'addresses' in credentials. " + "Get it from windscribe.com/getconfig/wireguard" + ) + + # Providers that require addresses (but not preshared_key) + elif provider_lower in ["surfshark", "mullvad", "ivpn"]: + if "addresses" not in credentials: + raise ValueError(f"Provider '{provider_name}': WireGuard requires 'addresses' in credentials") + + elif vpn_type == "openvpn": + if "username" not in credentials or "password" not in credentials: + raise ValueError( + f"Provider '{provider_name}': OpenVPN requires 'username' and 'password' in credentials" + ) + + def _get_available_port(self) -> int: + """Find an available port starting from base_port (thread-safe).""" + with self._port_lock: + used_ports = {info["port"] for info in self.active_containers.values()} + port = self.base_port + while port in used_ports or self._is_port_in_use(port): + port += 1 + return port + + def _is_port_in_use(self, port: int) -> bool: + """Check if a port is in use on the system or by any Docker container.""" + import socket + + # First check if the port is available on the system + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", port)) + except OSError: + # Port is in use by something on the system + return True + + # Also check Docker containers (in case of port forwarding) + try: + result = subprocess.run( + ["docker", "ps", "--format", "{{.Ports}}"], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode == 0: + return f":{port}->" in result.stdout or f"0.0.0.0:{port}" in result.stdout + return False + except (subprocess.TimeoutExpired, FileNotFoundError): + return False + + def _build_server_hostname(self, provider_name: str, country_code: str, server_num: str) -> str: + """ + Build a server hostname for specific server selection. + + Args: + provider_name: VPN provider name (e.g., "nordvpn") + country_code: 2-letter country code (e.g., "US") + server_num: Server number (e.g., "1239") + + Returns: + Server hostname (e.g., "us1239.nordvpn.com") + """ + # Convert to lowercase for hostname + country_lower = country_code.lower() + + # Provider-specific hostname formats + hostname_formats = { + "nordvpn": f"{country_lower}{server_num}.nordvpn.com", + "surfshark": f"{country_lower}-{server_num}.prod.surfshark.com", + "expressvpn": f"{country_lower}-{server_num}.expressvpn.com", + "cyberghost": f"{country_lower}-s{server_num}.cg-dialup.net", + # Generic fallback for other providers + } + + # Get provider-specific format or use generic + if provider_name in hostname_formats: + return hostname_formats[provider_name] + else: + # Generic format: country_code + server_num + return f"{country_lower}{server_num}" + + def _ensure_image_available(self, image: str = "qmcgaw/gluetun:latest") -> bool: + """ + Ensure the Gluetun Docker image is available locally. + + If the image is not present, it will be pulled. This prevents + the container creation from timing out during the first run. + + Args: + image: Docker image name with tag + + Returns: + True if image is available, False otherwise + """ + log = logging.getLogger("Gluetun") + + # Check if image exists locally + try: + result = subprocess.run( + ["docker", "image", "inspect", image], + capture_output=True, + text=True, + timeout=10, + encoding="utf-8", + errors="replace", + ) + if result.returncode == 0: + return True + log.debug(f"Image inspect failed: {result.stderr}") + except subprocess.TimeoutExpired: + log.warning("Docker image inspect timed out") + except FileNotFoundError: + log.error("Docker command not found - is Docker installed and in PATH?") + return False + + # Image not found, pull it + log.info(f"Pulling Docker image {image}...") + try: + result = subprocess.run( + ["docker", "pull", image], + capture_output=True, + text=True, + timeout=300, # 5 minutes for pull + encoding="utf-8", + errors="replace", + ) + if result.returncode == 0: + return True + log.error(f"Docker pull failed: {result.stderr}") + return False + except subprocess.TimeoutExpired: + raise RuntimeError(f"Timed out pulling Docker image '{image}'") + + def _create_container( + self, + container_name: str, + port: int, + provider_name: str, + provider_config: dict, + country: Optional[str] = None, + city: Optional[str] = None, + hostname: Optional[str] = None, + ): + """Create and start a Gluetun Docker container.""" + debug_logger = get_debug_logger() + start_time = time.time() + + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="gluetun_container_create_start", + message=f"Creating Gluetun container: {container_name}", + context={ + "container_name": container_name, + "port": port, + "provider": provider_name, + "country": country, + "city": city, + "hostname": hostname, + }, + ) + + # Ensure the Gluetun image is available (pulls if needed) + gluetun_image = "qmcgaw/gluetun:latest" + if not self._ensure_image_available(gluetun_image): + if debug_logger: + debug_logger.log( + level="ERROR", + operation="gluetun_image_pull_failed", + message=f"Failed to pull Docker image: {gluetun_image}", + success=False, + ) + raise RuntimeError(f"Failed to ensure Gluetun Docker image '{gluetun_image}' is available") + + vpn_type = provider_config.get("vpn_type", "wireguard").lower() + credentials = provider_config.get("credentials", {}) + extra_env = provider_config.get("extra_env", {}) + + # Normalize provider name + gluetun_provider = self.PROVIDER_MAPPING.get(provider_name.lower(), provider_name.lower()) + + # Build environment variables + env_vars = { + "VPN_SERVICE_PROVIDER": gluetun_provider, + "VPN_TYPE": vpn_type, + "HTTPPROXY": "on", + "HTTPPROXY_LISTENING_ADDRESS": ":8888", + "HTTPPROXY_LOG": "on", + "TZ": os.environ.get("TZ", "UTC"), + "LOG_LEVEL": "info", + } + + # Add credentials + if vpn_type == "wireguard": + env_vars["WIREGUARD_PRIVATE_KEY"] = credentials["private_key"] + # addresses is optional - not needed for some providers like NordVPN + if "addresses" in credentials: + env_vars["WIREGUARD_ADDRESSES"] = credentials["addresses"] + # preshared_key is required for Windscribe, optional for others + if "preshared_key" in credentials: + env_vars["WIREGUARD_PRESHARED_KEY"] = credentials["preshared_key"] + elif vpn_type == "openvpn": + env_vars["OPENVPN_USER"] = credentials.get("username", "") + env_vars["OPENVPN_PASSWORD"] = credentials.get("password", "") + + # Add server location + # Priority: hostname > country + city > country only + # Note: Different providers support different server selection variables + # - Most providers: SERVER_COUNTRIES, SERVER_CITIES + # - Windscribe, VyprVPN, VPN Secure: SERVER_REGIONS, SERVER_CITIES (no SERVER_COUNTRIES) + if hostname: + # Specific server hostname requested (e.g., us1239.nordvpn.com) + env_vars["SERVER_HOSTNAMES"] = hostname + else: + # Providers that use SERVER_REGIONS instead of SERVER_COUNTRIES + region_only_providers = {"windscribe", "vyprvpn", "vpn secure"} + uses_regions = gluetun_provider in region_only_providers + + # Use country/city selection + if country: + if uses_regions: + # Convert country code to provider-specific region name + if gluetun_provider == "windscribe": + region_name = self.WINDSCRIBE_REGION_MAP.get(country.lower(), country) + env_vars["SERVER_REGIONS"] = region_name + else: + env_vars["SERVER_REGIONS"] = country + else: + env_vars["SERVER_COUNTRIES"] = country + if city: + env_vars["SERVER_CITIES"] = city + + # Add authentication if configured + if self.auth_user: + env_vars["HTTPPROXY_USER"] = self.auth_user + if self.auth_password: + env_vars["HTTPPROXY_PASSWORD"] = self.auth_password + + # Merge extra environment variables + env_vars.update(extra_env) + + # Debug log environment variables (redact sensitive values) + if debug_logger: + redact_markers = ("KEY", "PASSWORD", "PASS", "TOKEN", "SECRET", "USER") + safe_env = {k: ("***" if any(m in k for m in redact_markers) else v) for k, v in env_vars.items()} + debug_logger.log( + level="DEBUG", + operation="gluetun_env_vars", + message=f"Environment variables for {container_name}", + context={"env_vars": safe_env, "gluetun_provider": gluetun_provider}, + ) + + # Build docker run command + cmd = [ + "docker", + "run", + "-d", + "--name", + container_name, + "--cap-add=NET_ADMIN", + "--device=/dev/net/tun", + "-p", + f"127.0.0.1:{port}:8888/tcp", + ] + + # Avoid exposing credentials in process listings by using --env-file instead of many "-e KEY=VALUE". + env_file_path: str | None = None + try: + fd, env_file_path = tempfile.mkstemp(prefix=f"unshackle-{container_name}-", suffix=".env") + try: + # Best-effort restrictive permissions. + if os.name != "nt": + if hasattr(os, "fchmod"): + os.fchmod(fd, 0o600) + else: + os.chmod(env_file_path, 0o600) + else: + os.chmod(env_file_path, stat.S_IREAD | stat.S_IWRITE) + + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as f: + for key, value in env_vars.items(): + if "=" in key: + raise ValueError(f"Invalid env var name for docker env-file: {key!r}") + v = "" if value is None else str(value) + if "\n" in v or "\r" in v: + raise ValueError(f"Invalid env var value (contains newline) for {key!r}") + f.write(f"{key}={v}\n") + except Exception: + # If we fail before fdopen closes the descriptor, make sure it's not leaked. + try: + os.close(fd) + except Exception: + pass + raise + + cmd.extend(["--env-file", env_file_path]) + + # Add Gluetun image + cmd.append(gluetun_image) + + # Execute docker run + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=30, + encoding="utf-8", + errors="replace", + ) + except subprocess.TimeoutExpired: + if debug_logger: + debug_logger.log( + level="ERROR", + operation="gluetun_container_create_timeout", + message=f"Docker run timed out for {container_name}", + context={"container_name": container_name}, + success=False, + duration_ms=(time.time() - start_time) * 1000, + ) + raise RuntimeError("Docker run command timed out") + + if result.returncode != 0: + error_msg = result.stderr or "unknown error" + if debug_logger: + debug_logger.log( + level="ERROR", + operation="gluetun_container_create_failed", + message=f"Docker run failed for {container_name}", + context={ + "container_name": container_name, + "return_code": result.returncode, + "stderr": error_msg, + }, + success=False, + duration_ms=(time.time() - start_time) * 1000, + ) + raise RuntimeError(f"Docker run failed: {error_msg}") + + # Log successful container creation + if debug_logger: + duration_ms = (time.time() - start_time) * 1000 + debug_logger.log( + level="INFO", + operation="gluetun_container_created", + message=f"Gluetun container created: {container_name}", + context={ + "container_name": container_name, + "port": port, + "provider": provider_name, + "vpn_type": vpn_type, + "country": country, + "city": city, + "hostname": hostname, + "container_id": result.stdout.strip()[:12] if result.stdout else None, + }, + success=True, + duration_ms=duration_ms, + ) + finally: + if env_file_path: + # Best-effort "secure delete": overwrite then unlink (not guaranteed on all filesystems). + try: + with open(env_file_path, "r+b") as f: + try: + f.seek(0, os.SEEK_END) + length = f.tell() + f.seek(0) + if length > 0: + f.write(b"\x00" * length) + f.flush() + os.fsync(f.fileno()) + except Exception: + pass + except Exception: + pass + try: + os.remove(env_file_path) + except FileNotFoundError: + pass + except Exception: + pass + + def _is_container_running(self, container_name: str) -> bool: + """Check if a Docker container is running.""" + try: + result = subprocess.run( + [ + "docker", + "ps", + "--filter", + f"name=^{re.escape(container_name)}$", + "--format", + "{{.Names}}", + ], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode != 0: + return False + + names = [line.strip() for line in (result.stdout or "").splitlines() if line.strip()] + return any(name == container_name for name in names) + except (subprocess.TimeoutExpired, FileNotFoundError): + return False + + def _get_existing_container_info(self, container_name: str) -> Optional[dict]: + """ + Check if a container exists in Docker and get its info. + + This handles multiple Unshackle sessions - if another session already + created the container, we'll reuse it instead of trying to create a duplicate. + + Args: + container_name: Name of the container to check + + Returns: + Dict with container info if exists and running, None otherwise + """ + try: + # Check if container is running + if not self._is_container_running(container_name): + return None + + # Get container port mapping + # Format: "127.0.0.1:8888->8888/tcp" + result = subprocess.run( + ["docker", "inspect", container_name, "--format", "{{.NetworkSettings.Ports}}"], + capture_output=True, + text=True, + timeout=5, + ) + + if result.returncode != 0: + return None + + # Parse port from output like "map[8888/tcp:[{127.0.0.1 8888}]]" + port_match = re.search(r"127\.0\.0\.1\s+(\d+)", result.stdout) + if not port_match: + return None + + port = int(port_match.group(1)) + + # Extract provider and region from container name + # Format: unshackle-gluetun-provider-region + name_pattern = f"{self.container_prefix}-(.+)-([^-]+)$" + name_match = re.match(name_pattern, container_name) + if not name_match: + return None + + provider_name = name_match.group(1) + region = name_match.group(2) + + # Get expected country and hostname from config (if available) + country = None + hostname = None + + # Check if region is a specific server (e.g., us1239) + specific_server_match = re.match(r"^([a-z]{2})(\d+)$", region, re.IGNORECASE) + if specific_server_match: + country_code = specific_server_match.group(1).upper() + server_num = specific_server_match.group(2) + hostname = self._build_server_hostname(provider_name, country_code, server_num) + country = country_code + + # Otherwise check config + elif provider_name in self.providers: + provider_config = self.providers[provider_name] + server_countries = provider_config.get("server_countries", {}) + country = server_countries.get(region) + + if not country and re.match(r"^[a-z]{2}$", region): + country = region.upper() + + return { + "container_name": container_name, + "port": port, + "provider": provider_name, + "region": region, + "country": country, + "city": None, + "hostname": hostname, + } + + except (subprocess.TimeoutExpired, FileNotFoundError, ValueError): + return None + + def _wait_for_container(self, container_name: str, timeout: int = 60) -> bool: + """ + Wait for Gluetun container to be ready by checking logs for proxy readiness. + + Gluetun logs "http proxy listening" when the HTTP proxy is ready to accept connections. + + Args: + container_name: Name of the container to wait for + timeout: Maximum time to wait in seconds (default: 60) + + Returns: + True if container is ready, False if it failed or timed out + """ + debug_logger = get_debug_logger() + start_time = time.time() + last_error = None + + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="gluetun_container_wait_start", + message=f"Waiting for container to be ready: {container_name}", + context={"container_name": container_name, "timeout": timeout}, + ) + + while time.time() - start_time < timeout: + try: + # First check if container is still running + if not self._is_container_running(container_name): + # Container may have exited - check if it crashed + exit_info = self._get_container_exit_info(container_name) + if exit_info: + last_error = f"Container exited with code {exit_info.get('exit_code', 'unknown')}" + time.sleep(1) + continue + + # Check logs for readiness indicators + result = subprocess.run( + ["docker", "logs", container_name, "--tail", "100"], + capture_output=True, + text=True, + timeout=5, + encoding="utf-8", + errors="replace", + ) + + if result.returncode == 0: + # Combine stdout and stderr for checking (handle None values) + stdout = result.stdout or "" + stderr = result.stderr or "" + all_logs = (stdout + stderr).lower() + + # Gluetun needs both proxy listening AND VPN connected + # The proxy starts before VPN is ready, so we need to wait for VPN + proxy_ready = "[http proxy] listening" in all_logs + vpn_ready = "initialization sequence completed" in all_logs + + if proxy_ready and vpn_ready: + # Give a brief moment for the proxy to fully initialize + time.sleep(1) + duration_ms = (time.time() - start_time) * 1000 + if debug_logger: + debug_logger.log( + level="INFO", + operation="gluetun_container_ready", + message=f"Gluetun container is ready: {container_name}", + context={ + "container_name": container_name, + "proxy_ready": proxy_ready, + "vpn_ready": vpn_ready, + }, + success=True, + duration_ms=duration_ms, + ) + return True + + # Check for fatal errors that indicate VPN connection failure + error_indicators = [ + "fatal", + "cannot connect", + "authentication failed", + "invalid credentials", + "connection refused", + "no valid servers", + ] + + for error in error_indicators: + if error in all_logs: + # Extract the error line for better messaging + for line in (stdout + stderr).split("\n"): + if error in line.lower(): + last_error = line.strip() + break + # Fatal errors mean we should stop waiting + if "fatal" in all_logs or "invalid credentials" in all_logs: + return False + + except subprocess.TimeoutExpired: + pass + + time.sleep(2) + + # Store the last error for potential logging + if last_error: + self._last_wait_error = last_error + + # Log timeout/failure + duration_ms = (time.time() - start_time) * 1000 + if debug_logger: + debug_logger.log( + level="ERROR", + operation="gluetun_container_wait_timeout", + message=f"Gluetun container failed to become ready: {container_name}", + context={ + "container_name": container_name, + "timeout": timeout, + "last_error": last_error, + }, + success=False, + duration_ms=duration_ms, + ) + return False + + def _get_container_exit_info(self, container_name: str) -> Optional[dict]: + """Get exit information for a stopped container.""" + try: + result = subprocess.run( + ["docker", "inspect", container_name, "--format", "{{.State.ExitCode}}:{{.State.Error}}"], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode == 0: + parts = result.stdout.strip().split(":", 1) + return { + "exit_code": int(parts[0]) if parts[0].isdigit() else -1, + "error": parts[1] if len(parts) > 1 else "", + } + return None + except (subprocess.TimeoutExpired, FileNotFoundError, ValueError): + return None + + def _get_container_logs(self, container_name: str, tail: int = 50) -> str: + """Get recent logs from a container for error reporting.""" + try: + result = subprocess.run( + ["docker", "logs", container_name, "--tail", str(tail)], + capture_output=True, + text=True, + timeout=10, + encoding="utf-8", + errors="replace", + ) + return (result.stdout or "") + (result.stderr or "") + except (subprocess.TimeoutExpired, FileNotFoundError): + return "" + + def _verify_container(self, query_key: str, max_retries: int = 3): + """ + Verify container's VPN IP and region using ipinfo.io lookup. + + Uses the shared get_ip_info function with a session configured to use + the Gluetun proxy. Retries with exponential backoff if the network + isn't ready immediately after the VPN connects. + + Args: + query_key: The container query key (provider:region) + max_retries: Maximum number of retry attempts (default: 3) + + Raises: + RuntimeError: If verification fails after all retries + """ + debug_logger = get_debug_logger() + start_time = time.time() + + if query_key not in self.active_containers: + return + + container = self.active_containers[query_key] + proxy_url = self._build_proxy_uri(container["port"]) + expected_country = container.get("country", "").upper() + + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="gluetun_verify_start", + message=f"Verifying VPN IP for: {query_key}", + context={ + "query_key": query_key, + "container_name": container.get("container_name"), + "expected_country": expected_country, + "max_retries": max_retries, + }, + ) + + last_error = None + + # Create a session with the proxy configured + session = requests.Session() + try: + session.proxies = {"http": proxy_url, "https": proxy_url} + + # Retry with exponential backoff + for attempt in range(max_retries): + try: + # Get external IP through the proxy using shared utility + ip_info = get_ip_info(session) + + if ip_info: + actual_country = ip_info.get("country", "").upper() + + # Check if country matches (if we have an expected country) + # ipinfo.io returns country codes (CA), but we may have full names (Canada) + # Normalize both to country codes for comparison using shared utility + if expected_country: + # Convert expected country name to code if it's a full name + expected_code = get_country_code(expected_country) or expected_country + expected_code = expected_code.upper() + + if actual_country != expected_code: + duration_ms = (time.time() - start_time) * 1000 + if debug_logger: + debug_logger.log( + level="ERROR", + operation="gluetun_verify_mismatch", + message=f"Region mismatch for {query_key}", + context={ + "query_key": query_key, + "expected_country": expected_code, + "actual_country": actual_country, + "ip": ip_info.get("ip"), + "city": ip_info.get("city"), + "org": ip_info.get("org"), + }, + success=False, + duration_ms=duration_ms, + ) + raise RuntimeError( + f"Region mismatch for {container['provider']}:{container['region']}: " + f"Expected '{expected_code}' but got '{actual_country}' " + f"(IP: {ip_info.get('ip')}, City: {ip_info.get('city')})" + ) + + # Verification successful - store IP info in container record + if query_key in self.active_containers: + self.active_containers[query_key]["public_ip"] = ip_info.get("ip") + self.active_containers[query_key]["ip_country"] = actual_country + self.active_containers[query_key]["ip_city"] = ip_info.get("city") + self.active_containers[query_key]["ip_org"] = ip_info.get("org") + + duration_ms = (time.time() - start_time) * 1000 + if debug_logger: + debug_logger.log( + level="INFO", + operation="gluetun_verify_success", + message=f"VPN IP verified for: {query_key}", + context={ + "query_key": query_key, + "ip": ip_info.get("ip"), + "country": actual_country, + "city": ip_info.get("city"), + "org": ip_info.get("org"), + "attempts": attempt + 1, + }, + success=True, + duration_ms=duration_ms, + ) + return + + # ip_info was None, retry + last_error = "Failed to get IP info from ipinfo.io" + + except RuntimeError: + raise # Re-raise region mismatch errors immediately + except Exception as e: + last_error = str(e) + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="gluetun_verify_retry", + message=f"Verification attempt {attempt + 1} failed, retrying", + context={ + "query_key": query_key, + "attempt": attempt + 1, + "error": last_error, + }, + ) + + # Wait before retry (exponential backoff) + if attempt < max_retries - 1: + wait_time = 2**attempt # 1, 2, 4 seconds + time.sleep(wait_time) + finally: + try: + session.close() + except Exception: + pass + + # All retries exhausted + duration_ms = (time.time() - start_time) * 1000 + if debug_logger: + debug_logger.log( + level="ERROR", + operation="gluetun_verify_failed", + message=f"VPN verification failed after {max_retries} attempts", + context={ + "query_key": query_key, + "max_retries": max_retries, + "last_error": last_error, + }, + success=False, + duration_ms=duration_ms, + ) + raise RuntimeError( + f"Failed to verify VPN IP for {container['provider']}:{container['region']} " + f"after {max_retries} attempts. Last error: {last_error}" + ) + + def _remove_stopped_container(self, container_name: str) -> bool: + """ + Remove a stopped container with the given name if it exists. + + This prevents "container name already in use" errors when a previous + container wasn't properly cleaned up. + + Args: + container_name: Name of the container to check and remove + + Returns: + True if a container was removed, False otherwise + """ + try: + # Check if container exists (running or stopped) + result = subprocess.run( + ["docker", "ps", "-a", "--filter", f"name=^{container_name}$", "--format", "{{.Names}}:{{.Status}}"], + capture_output=True, + text=True, + timeout=5, + ) + + if result.returncode != 0 or not result.stdout.strip(): + return False + + # Parse status - format is "name:Up 2 hours" or "name:Exited (0) 2 hours ago" + output = result.stdout.strip() + if container_name not in output: + return False + + # Check if container is stopped (not running) + if "Exited" in output or "Created" in output or "Dead" in output: + # Container exists but is stopped - remove it + subprocess.run( + ["docker", "rm", "-f", container_name], + capture_output=True, + text=True, + timeout=10, + ) + return True + + return False + + except (subprocess.TimeoutExpired, FileNotFoundError): + return False + + def _remove_container(self, container_name: str): + """Stop and remove a Docker container.""" + try: + if self.auto_cleanup: + # Use docker rm -f to force remove (stops and removes in one command) + subprocess.run( + ["docker", "rm", "-f", container_name], + capture_output=True, + text=True, + timeout=10, + ) + else: + # Just stop the container + subprocess.run( + ["docker", "stop", container_name], + capture_output=True, + text=True, + timeout=10, + ) + except subprocess.TimeoutExpired: + # Force kill if timeout + try: + subprocess.run( + ["docker", "rm", "-f", container_name], + capture_output=True, + text=True, + timeout=5, + ) + except subprocess.TimeoutExpired: + pass + + def _build_proxy_uri(self, port: int) -> str: + """Build HTTP proxy URI.""" + if self.auth_user and self.auth_password: + return f"http://{self.auth_user}:{self.auth_password}@localhost:{port}" + return f"http://localhost:{port}" + + def __del__(self): + """Cleanup containers on object destruction.""" + if hasattr(self, "auto_cleanup") and self.auto_cleanup: + try: + if hasattr(self, "active_containers") and self.active_containers: + self.cleanup() + except Exception: + pass diff --git a/packages/envied/src/envied/core/proxies/nordvpn.py b/packages/envied/src/envied/core/proxies/nordvpn.py index ea0f0f1..1799039 100644 --- a/packages/envied/src/envied/core/proxies/nordvpn.py +++ b/packages/envied/src/envied/core/proxies/nordvpn.py @@ -1,4 +1,5 @@ import json +import random import re from typing import Optional @@ -46,8 +47,21 @@ class NordVPN(Proxy): HTTP proxies under port 80 were disabled on the 15th of Feb, 2021: https://nordvpn.com/blog/removing-http-proxies + + Supports: + - Country code: "us", "ca", "gb" + - Country ID: "228" + - Specific server: "us1234" + - City selection: "us:seattle", "ca:calgary" """ query = query.lower() + city = None + + # Check if query includes city specification (e.g., "ca:calgary") + if ":" in query: + query, city = query.split(":", maxsplit=1) + city = city.strip() + if re.match(r"^[a-z]{2}\d+$", query): # country and nordvpn server id, e.g., us1, fr1234 hostname = f"{query}.nordvpn.com" @@ -64,7 +78,12 @@ class NordVPN(Proxy): # NordVPN doesnt have servers in this region return - server_mapping = self.server_map.get(country["code"].lower()) + # Check server_map for pinned servers (can include city) + server_map_key = f"{country['code'].lower()}:{city}" if city else country["code"].lower() + server_mapping = self.server_map.get(server_map_key) or ( + self.server_map.get(country["code"].lower()) if not city else None + ) + if server_mapping: # country was set to a specific server ID in config hostname = f"{country['code'].lower()}{server_mapping}.nordvpn.com" @@ -76,7 +95,19 @@ class NordVPN(Proxy): f"The NordVPN Country {query} currently has no recommended servers. " "Try again later. If the issue persists, double-check the query." ) - hostname = recommended_servers[0]["hostname"] + + # Filter by city if specified + if city: + city_servers = self.filter_servers_by_city(recommended_servers, city) + if not city_servers: + raise ValueError( + f"No servers found in city '{city}' for country '{country['name']}'. " + "Try a different city or check the city name spelling." + ) + recommended_servers = city_servers + + # Pick a random server from the filtered list + hostname = random.choice(recommended_servers)["hostname"] if hostname.startswith("gb"): # NordVPN uses the alpha2 of 'GB' in API responses, but 'UK' in the hostname @@ -95,6 +126,41 @@ class NordVPN(Proxy): ): return country + @staticmethod + def filter_servers_by_city(servers: list[dict], city: str) -> list[dict]: + """ + Filter servers by city name. + + The API returns servers with location data that includes city information. + This method filters servers to only those in the specified city. + + Args: + servers: List of server dictionaries from the NordVPN API + city: City name to filter by (case-insensitive) + + Returns: + List of servers in the specified city + """ + city_lower = city.lower() + filtered = [] + + for server in servers: + # Each server has a 'locations' list with location data + locations = server.get("locations", []) + for location in locations: + # City data can be in different formats: + # - {"city": {"name": "Seattle", ...}} + # - {"city": "Seattle"} + city_data = location.get("city") + if city_data: + # Handle both dict and string formats + city_name = city_data.get("name") if isinstance(city_data, dict) else city_data + if city_name and city_name.lower() == city_lower: + filtered.append(server) + break # Found a match, no need to check other locations for this server + + return filtered + @staticmethod def get_recommended_servers(country_id: int) -> list[dict]: """ diff --git a/packages/envied/src/envied/core/proxies/surfsharkvpn.py b/packages/envied/src/envied/core/proxies/surfsharkvpn.py index 458dbc7..e811360 100644 --- a/packages/envied/src/envied/core/proxies/surfsharkvpn.py +++ b/packages/envied/src/envied/core/proxies/surfsharkvpn.py @@ -44,8 +44,21 @@ class SurfsharkVPN(Proxy): def get_proxy(self, query: str) -> Optional[str]: """ Get an HTTP(SSL) proxy URI for a SurfsharkVPN server. + + Supports: + - Country code: "us", "ca", "gb" + - Country ID: "228" + - Specific server: "us-bos" (Boston) + - City selection: "us:seattle", "ca:toronto" """ query = query.lower() + city = None + + # Check if query includes city specification (e.g., "us:seattle") + if ":" in query: + query, city = query.split(":", maxsplit=1) + city = city.strip() + if re.match(r"^[a-z]{2}\d+$", query): # country and surfsharkvpn server id, e.g., au-per, be-anr, us-bos hostname = f"{query}.prod.surfshark.com" @@ -62,13 +75,18 @@ class SurfsharkVPN(Proxy): # SurfsharkVPN doesnt have servers in this region return - server_mapping = self.server_map.get(country["countryCode"].lower()) + # Check server_map for pinned servers (can include city) + server_map_key = f"{country['countryCode'].lower()}:{city}" if city else country["countryCode"].lower() + server_mapping = self.server_map.get(server_map_key) or ( + self.server_map.get(country["countryCode"].lower()) if not city else None + ) + if server_mapping: # country was set to a specific server ID in config hostname = f"{country['code'].lower()}{server_mapping}.prod.surfshark.com" else: # get the random server ID - random_server = self.get_random_server(country["countryCode"]) + random_server = self.get_random_server(country["countryCode"], city) if not random_server: raise ValueError( f"The SurfsharkVPN Country {query} currently has no random servers. " @@ -92,18 +110,49 @@ class SurfsharkVPN(Proxy): ): return country - def get_random_server(self, country_id: str): + def get_random_server(self, country_id: str, city: Optional[str] = None): """ - Get the list of random Server for a Country. + Get a random server for a Country, optionally filtered by city. - Note: There may not always be more than one recommended server. + Args: + country_id: The country code (e.g., "US", "CA") + city: Optional city name to filter by (case-insensitive) + + Note: The API may include a 'location' field with city information. + If not available, this will return any server from the country. """ - country = [x["connectionName"] for x in self.countries if x["countryCode"].lower() == country_id.lower()] - try: - country = random.choice(country) - return country - except Exception: - raise ValueError("Could not get random countrycode from the countries list.") + servers = [x for x in self.countries if x["countryCode"].lower() == country_id.lower()] + + # Filter by city if specified + if city: + city_lower = city.lower() + # Check if servers have a 'location' field for city filtering + city_servers = [ + x + for x in servers + if x.get("location", "").lower() == city_lower or x.get("city", "").lower() == city_lower + ] + + if city_servers: + servers = city_servers + else: + raise ValueError( + f"No servers found in city '{city}' for country '{country_id}'. " + "Try a different city or check the city name spelling." + ) + + # Get connection names from filtered servers + if not servers: + raise ValueError(f"Could not get random server for country '{country_id}': no servers found.") + + # Only include servers that actually have a connection name to avoid KeyError. + connection_names = [x["connectionName"] for x in servers if "connectionName" in x] + if not connection_names: + raise ValueError( + f"Could not get random server for country '{country_id}': no servers with connectionName found." + ) + + return random.choice(connection_names) @staticmethod def get_countries() -> list[dict]: diff --git a/packages/envied/src/envied/core/proxies/windscribevpn.py b/packages/envied/src/envied/core/proxies/windscribevpn.py index f31947d..bffde71 100644 --- a/packages/envied/src/envied/core/proxies/windscribevpn.py +++ b/packages/envied/src/envied/core/proxies/windscribevpn.py @@ -45,22 +45,38 @@ class WindscribeVPN(Proxy): """ Get an HTTPS proxy URI for a WindscribeVPN server. - Note: Windscribe's static OpenVPN credentials work reliably on US, AU, and NZ servers. + Supports: + - Country code: "us", "ca", "gb" + - Specific server: "sg007", "us150" + - City selection: "us:seattle", "ca:toronto" """ query = query.lower() - supported_regions = {"us", "au", "nz"} + city = None - if query not in supported_regions and query not in self.server_map: - raise ValueError( - f"Windscribe proxy does not currently support the '{query.upper()}' region. " - f"Supported regions with reliable credentials: {', '.join(sorted(supported_regions))}. " - ) + # Check if query includes city specification (e.g., "ca:toronto") + if ":" in query: + query, city = query.split(":", maxsplit=1) + city = city.strip() - if query in self.server_map: + # Check server_map for pinned servers (can include city) + server_map_key = f"{query}:{city}" if city else query + if server_map_key in self.server_map: + hostname = self.server_map[server_map_key] + elif query in self.server_map: hostname = self.server_map[query] else: - if re.match(r"^[a-z]+$", query): - hostname = self.get_random_server(query) + server_match = re.match(r"^([a-z]{2})(\d+)$", query) + if server_match: + # Specific server selection, e.g., sg007, us150 + country_code, server_num = server_match.groups() + hostname = self.get_specific_server(country_code, server_num) + if not hostname: + raise ValueError( + f"No WindscribeVPN server found matching '{query}'. " + f"Check the server number or use just '{country_code}' for a random server." + ) + elif re.match(r"^[a-z]+$", query): + hostname = self.get_random_server(query, city) else: raise ValueError(f"The query provided is unsupported and unrecognized: {query}") @@ -70,22 +86,74 @@ class WindscribeVPN(Proxy): hostname = hostname.split(':')[0] return f"https://{self.username}:{self.password}@{hostname}:443" - def get_random_server(self, country_code: str) -> Optional[str]: + def get_specific_server(self, country_code: str, server_num: str) -> Optional[str]: """ - Get a random server hostname for a country. + Find a specific server by country code and server number. - Returns None if no servers are available for the country. + Matches against hostnames like "sg-007.totallyacdn.com" for query "sg007". + Tries both the raw number and zero-padded variants. + + Args: + country_code: Two-letter country code (e.g., "sg", "us") + server_num: Server number as string (e.g., "007", "7", "150") + + Returns: + The matching hostname, or None if not found. """ + num_stripped = server_num.lstrip("0") or "0" + candidates = { + f"{country_code}-{server_num}.", + f"{country_code}-{num_stripped}.", + f"{country_code}-{server_num.zfill(3)}.", + } + + for location in self.countries: + if location.get("country_code", "").lower() != country_code: + continue + for group in location.get("groups", []): + for host in group.get("hosts", []): + hostname = host.get("hostname", "") + if any(hostname.startswith(prefix) for prefix in candidates): + return hostname + + return None + + def get_random_server(self, country_code: str, city: Optional[str] = None) -> Optional[str]: + """ + Get a random server hostname for a country, optionally filtered by city. + + Args: + country_code: The country code (e.g., "us", "ca") + city: Optional city name to filter by (case-insensitive) + + Returns: + A random hostname from matching servers, or None if none available. + """ + hostnames = [] + + # Collect hostnames from ALL locations matching the country code for location in self.countries: if location.get("country_code", "").lower() == country_code.lower(): - hostnames = [] for group in location.get("groups", []): + # Filter by city if specified + if city: + group_city = group.get("city", "") + if group_city.lower() != city.lower(): + continue + + # Collect hostnames from this group for host in group.get("hosts", []): if hostname := host.get("hostname"): hostnames.append(hostname) - if hostnames: - return random.choice(hostnames) + if hostnames: + return random.choice(hostnames) + elif city: + # No servers found for the specified city + raise ValueError( + f"No servers found in city '{city}' for country code '{country_code}'. " + "Try a different city or check the city name spelling." + ) return None diff --git a/packages/envied/src/envied/core/service.py b/packages/envied/src/envied/core/service.py index 62f1c2f..1d7248e 100644 --- a/packages/envied/src/envied/core/service.py +++ b/packages/envied/src/envied/core/service.py @@ -5,7 +5,7 @@ from collections.abc import Generator from http.cookiejar import CookieJar from pathlib import Path from typing import Optional, Union -from urllib.parse import urlparse +from urllib.parse import urlparse, urlunparse import click import m3u8 @@ -27,6 +27,45 @@ from envied.core.tracks import Chapters, Tracks from envied.core.utilities import get_cached_ip_info, get_ip_info +def sanitize_proxy_for_log(uri: Optional[str]) -> Optional[str]: + """ + Sanitize a proxy URI for logs by redacting any embedded userinfo (username/password). + + Examples: + - http://user:pass@host:8080 -> http://REDACTED@host:8080 + - socks5h://user@host:1080 -> socks5h://REDACTED@host:1080 + """ + if uri is None: + return None + if not isinstance(uri, str): + return str(uri) + if not uri: + return uri + + try: + parsed = urlparse(uri) + + # Handle schemeless proxies like "user:pass@host:port" + if not parsed.scheme and not parsed.netloc and "@" in uri and "://" not in uri: + # Parse as netloc using a dummy scheme, then strip scheme back out. + dummy = urlparse(f"http://{uri}") + netloc = dummy.netloc + if "@" in netloc: + netloc = f"REDACTED@{netloc.split('@', 1)[1]}" + # urlparse("http://...") sets path to "" for typical netloc-only strings; keep it just in case. + return f"{netloc}{dummy.path}" + + netloc = parsed.netloc + if "@" in netloc: + netloc = f"REDACTED@{netloc.split('@', 1)[1]}" + + return urlunparse(parsed._replace(netloc=netloc)) + except Exception: + if "@" in uri: + return f"REDACTED@{uri.split('@', 1)[1]}" + return uri + + class Service(metaclass=ABCMeta): """The Service Base Class.""" @@ -53,8 +92,65 @@ class Service(metaclass=ABCMeta): if not ctx.parent or not ctx.parent.params.get("no_proxy"): if ctx.parent: proxy = ctx.parent.params["proxy"] + proxy_query = ctx.parent.params.get("proxy_query") + proxy_provider_name = ctx.parent.params.get("proxy_provider") else: proxy = None + proxy_query = None + proxy_provider_name = None + + # Check for service-specific proxy mapping + service_name = self.__class__.__name__ + service_config_dict = config.services.get(service_name, {}) + proxy_map = service_config_dict.get("proxy_map", {}) + + if proxy_map and proxy_query: + # Build the full proxy query key (e.g., "nordvpn:ca" or "us") + if proxy_provider_name: + full_proxy_key = f"{proxy_provider_name}:{proxy_query}" + else: + full_proxy_key = proxy_query + + # Check if there's a mapping for this query + mapped_value = proxy_map.get(full_proxy_key) + if mapped_value: + self.log.info( + f"Found service-specific proxy mapping: {full_proxy_key} -> {sanitize_proxy_for_log(mapped_value)}" + ) + # Query the proxy provider with the mapped value + if proxy_provider_name: + # Specific provider requested + proxy_provider = next( + (x for x in ctx.obj.proxy_providers if x.__class__.__name__.lower() == proxy_provider_name), + None, + ) + if proxy_provider: + mapped_proxy_uri = proxy_provider.get_proxy(mapped_value) + if mapped_proxy_uri: + proxy = mapped_proxy_uri + self.log.info( + f"Using mapped proxy from {proxy_provider.__class__.__name__}: {sanitize_proxy_for_log(proxy)}" + ) + else: + self.log.warning( + f"Failed to get proxy for mapped value '{sanitize_proxy_for_log(mapped_value)}', using default" + ) + else: + self.log.warning(f"Proxy provider '{proxy_provider_name}' not found, using default proxy") + else: + # No specific provider, try all providers + for proxy_provider in ctx.obj.proxy_providers: + mapped_proxy_uri = proxy_provider.get_proxy(mapped_value) + if mapped_proxy_uri: + proxy = mapped_proxy_uri + self.log.info( + f"Using mapped proxy from {proxy_provider.__class__.__name__}: {sanitize_proxy_for_log(proxy)}" + ) + break + else: + self.log.warning( + f"No provider could resolve mapped value '{sanitize_proxy_for_log(mapped_value)}', using default" + ) if not proxy: # don't override the explicit proxy set by the user, even if they may be geoblocked diff --git a/packages/envied/src/envied/core/services.py b/packages/envied/src/envied/core/services.py index bd7ddd5..776bc15 100644 --- a/packages/envied/src/envied/core/services.py +++ b/packages/envied/src/envied/core/services.py @@ -58,6 +58,7 @@ class Services(click.MultiCommand): def get_path(name: str) -> Path: """Get the directory path of a command.""" tag = Services.get_tag(name) + for service in _SERVICES: if service.parent.stem == tag: return service.parent @@ -72,19 +73,22 @@ class Services(click.MultiCommand): """ original_value = value value = value.lower() + for path in _SERVICES: tag = path.parent.stem if value in (tag.lower(), *_ALIASES.get(tag, [])): return tag + return original_value @staticmethod def load(tag: str) -> Service: """Load a Service module by Service tag.""" module = _MODULES.get(tag) - if not module: - raise KeyError(f"There is no Service added by the Tag '{tag}'") - return module + if module: + return module + + raise KeyError(f"There is no Service added by the Tag '{tag}'") __all__ = ("Services",) diff --git a/packages/envied/src/envied/core/titles/episode.py b/packages/envied/src/envied/core/titles/episode.py index b1be4b3..4197b7d 100644 --- a/packages/envied/src/envied/core/titles/episode.py +++ b/packages/envied/src/envied/core/titles/episode.py @@ -102,6 +102,27 @@ class Episode(Title): primary_audio_track = sorted_audio[0] unique_audio_languages = len({x.language.split("-")[0] for x in media_info.audio_tracks if x.language}) + def _get_resolution_token(track: Any) -> str: + if not track or not getattr(track, "height", None): + return "" + resolution = track.height + try: + dar = getattr(track, "other_display_aspect_ratio", None) or [] + if dar and dar[0]: + aspect_ratio = [int(float(plane)) for plane in str(dar[0]).split(":")] + if len(aspect_ratio) == 1: + aspect_ratio.append(1) + if aspect_ratio[0] / aspect_ratio[1] not in (16 / 9, 4 / 3): + resolution = int(track.width * (9 / 16)) + except Exception: + pass + + scan_suffix = "p" + scan_type = getattr(track, "scan_type", None) + if scan_type and str(scan_type).lower() == "interlaced": + scan_suffix = "i" + return f"{resolution}{scan_suffix}" + # Title [Year] SXXEXX Name (or Title [Year] SXX if folder) if folder: name = f"{self.title}" @@ -109,104 +130,121 @@ class Episode(Title): name += f" {self.year}" name += f" S{self.season:02}" else: - name = "{title}{year} S{season:02}E{number:02} {name}".format( - title=self.title.replace("$", "S"), # e.g., Arli$$ - year=f" {self.year}" if self.year and config.series_year else "", - season=self.season, - number=self.number, - name=self.name or "", - ).strip() + if config.dash_naming: + # Format: Title - SXXEXX - Episode Name + name = self.title.replace("$", "S") # e.g., Arli$$ - if config.scene_naming: - # Resolution - if primary_video_track: - resolution = primary_video_track.height - aspect_ratio = [ - int(float(plane)) for plane in primary_video_track.other_display_aspect_ratio[0].split(":") - ] - if len(aspect_ratio) == 1: - # e.g., aspect ratio of 2 (2.00:1) would end up as `(2.0,)`, add 1 - aspect_ratio.append(1) - if aspect_ratio[0] / aspect_ratio[1] not in (16 / 9, 4 / 3): - # We want the resolution represented in a 4:3 or 16:9 canvas. - # If it's not 4:3 or 16:9, calculate as if it's inside a 16:9 canvas, - # otherwise the track's height value is fine. - # We are assuming this title is some weird aspect ratio so most - # likely a movie or HD source, so it's most likely widescreen so - # 16:9 canvas makes the most sense. - resolution = int(primary_video_track.width * (9 / 16)) - name += f" {resolution}p" + # Add year if configured + if self.year and config.series_year: + name += f" {self.year}" - # Service - if show_service: - name += f" {self.service.__name__}" + # Add season and episode + name += f" - S{self.season:02}E{self.number:02}" - # 'WEB-DL' - name += " WEB-DL" + # Add episode name with dash separator + if self.name: + name += f" - {self.name}" - # DUAL - if unique_audio_languages == 2: - name += " DUAL" + name = name.strip() + else: + # Standard format without extra dashes + name = "{title}{year} S{season:02}E{number:02} {name}".format( + title=self.title.replace("$", "S"), # e.g., Arli$$ + year=f" {self.year}" if self.year and config.series_year else "", + season=self.season, + number=self.number, + name=self.name or "", + ).strip() - # MULTi - if unique_audio_languages > 2: - name += " MULTi" + if primary_video_track: + resolution_token = _get_resolution_token(primary_video_track) + if resolution_token: + name += f" {resolution_token}" - # Audio Codec + Channels (+ feature) - if primary_audio_track: - codec = primary_audio_track.format - channel_layout = primary_audio_track.channel_layout or primary_audio_track.channellayout_original - if channel_layout: - channels = float( - sum({"LFE": 0.1}.get(position.upper(), 1) for position in channel_layout.split(" ")) - ) - else: - channel_count = primary_audio_track.channel_s or primary_audio_track.channels or 0 - channels = float(channel_count) + # Service (use track source if available) + if show_service: + source_name = None + if self.tracks: + first_track = next(iter(self.tracks), None) + if first_track and hasattr(first_track, "source") and first_track.source: + source_name = first_track.source + name += f" {source_name or self.service.__name__}" - features = primary_audio_track.format_additionalfeatures or "" - name += f" {AUDIO_CODEC_MAP.get(codec, codec)}{channels:.1f}" - if "JOC" in features or primary_audio_track.joc: - name += " Atmos" + # 'WEB-DL' + name += " WEB-DL" - # Video (dynamic range + hfr +) Codec - if primary_video_track: - codec = primary_video_track.format - hdr_format = primary_video_track.hdr_format_commercial - hdr_format_full = primary_video_track.hdr_format or "" - trc = ( - primary_video_track.transfer_characteristics - or primary_video_track.transfer_characteristics_original - or "" + # DUAL + if unique_audio_languages == 2: + name += " DUAL" + + # MULTi + if unique_audio_languages > 2: + name += " MULTi" + + # Audio Codec + Channels (+ feature) + if primary_audio_track: + codec = primary_audio_track.format + channel_layout = primary_audio_track.channel_layout or primary_audio_track.channellayout_original + if channel_layout: + channels = float( + sum({"LFE": 0.1}.get(position.upper(), 1) for position in channel_layout.split(" ")) ) - frame_rate = float(primary_video_track.frame_rate) + else: + channel_count = primary_audio_track.channel_s or primary_audio_track.channels or 0 + channels = float(channel_count) - # Primary HDR format detection - if hdr_format: - if hdr_format_full.startswith("Dolby Vision"): - name += " DV" - if any( - indicator in (hdr_format_full + " " + hdr_format) - for indicator in ["HDR10", "SMPTE ST 2086"] - ): - name += " HDR" - else: - name += f" {DYNAMIC_RANGE_MAP.get(hdr_format)} " - elif "HLG" in trc or "Hybrid Log-Gamma" in trc or "ARIB STD-B67" in trc or "arib-std-b67" in trc.lower(): - name += " HLG" - elif any(indicator in trc for indicator in ["PQ", "SMPTE ST 2084", "BT.2100"]) or "smpte2084" in trc.lower() or "bt.2020-10" in trc.lower(): - name += " HDR" - if frame_rate > 30: - name += " HFR" - name += f" {VIDEO_CODEC_MAP.get(codec, codec)}" + features = primary_audio_track.format_additionalfeatures or "" + name += f" {AUDIO_CODEC_MAP.get(codec, codec)}{channels:.1f}" + if "JOC" in features or primary_audio_track.joc: + name += " Atmos" - if config.tag: - name += f"-{config.tag}" + # Video (dynamic range + hfr +) Codec + if primary_video_track: + codec = primary_video_track.format + hdr_format = primary_video_track.hdr_format_commercial + hdr_format_full = primary_video_track.hdr_format or "" + trc = ( + primary_video_track.transfer_characteristics + or primary_video_track.transfer_characteristics_original + or "" + ) + frame_rate = float(primary_video_track.frame_rate) - return sanitize_filename(name) - else: - # Simple naming style without technical details - use spaces instead of dots - return sanitize_filename(name, " ") + def _append_token(current: str, token: Optional[str]) -> str: + token = (token or "").strip() + current = current.rstrip() + if not token: + return current + if current.endswith(f" {token}"): + return current + return f"{current} {token}" + + # Primary HDR format detection + if hdr_format: + if hdr_format_full.startswith("Dolby Vision"): + name = _append_token(name, "DV") + if any( + indicator in (hdr_format_full + " " + hdr_format) + for indicator in ["HDR10", "SMPTE ST 2086"] + ): + name = _append_token(name, "HDR") + elif "HDR Vivid" in hdr_format: + name = _append_token(name, "HDR") + else: + dynamic_range = DYNAMIC_RANGE_MAP.get(hdr_format) or hdr_format or "" + name = _append_token(name, dynamic_range) + elif "HLG" in trc or "Hybrid Log-Gamma" in trc or "ARIB STD-B67" in trc or "arib-std-b67" in trc.lower(): + name += " HLG" + elif any(indicator in trc for indicator in ["PQ", "SMPTE ST 2084", "BT.2100"]) or "smpte2084" in trc.lower() or "bt.2020-10" in trc.lower(): + name += " HDR" + if frame_rate > 30: + name += " HFR" + name += f" {VIDEO_CODEC_MAP.get(codec, codec)}" + + if config.tag: + name += f"-{config.tag}" + + return sanitize_filename(name, "." if config.scene_naming else " ") class Series(SortedKeyList, ABC): diff --git a/packages/envied/src/envied/core/titles/movie.py b/packages/envied/src/envied/core/titles/movie.py index 756907c..8ce9ce0 100644 --- a/packages/envied/src/envied/core/titles/movie.py +++ b/packages/envied/src/envied/core/titles/movie.py @@ -47,6 +47,8 @@ class Movie(Title): def __str__(self) -> str: if self.year: + if config.dash_naming: + return f"{self.name} - {self.year}" return f"{self.name} ({self.year})" return self.name @@ -65,99 +67,119 @@ class Movie(Title): primary_audio_track = sorted_audio[0] unique_audio_languages = len({x.language.split("-")[0] for x in media_info.audio_tracks if x.language}) + def _get_resolution_token(track: Any) -> str: + if not track or not getattr(track, "height", None): + return "" + resolution = track.height + try: + dar = getattr(track, "other_display_aspect_ratio", None) or [] + if dar and dar[0]: + aspect_ratio = [int(float(plane)) for plane in str(dar[0]).split(":")] + if len(aspect_ratio) == 1: + aspect_ratio.append(1) + if aspect_ratio[0] / aspect_ratio[1] not in (16 / 9, 4 / 3): + resolution = int(track.width * (9 / 16)) + except Exception: + pass + + scan_suffix = "p" + scan_type = getattr(track, "scan_type", None) + if scan_type and str(scan_type).lower() == "interlaced": + scan_suffix = "i" + return f"{resolution}{scan_suffix}" + # Name (Year) name = str(self).replace("$", "S") # e.g., Arli$$ - if config.scene_naming: - # Resolution - if primary_video_track: - resolution = primary_video_track.height - aspect_ratio = [ - int(float(plane)) for plane in primary_video_track.other_display_aspect_ratio[0].split(":") - ] - if len(aspect_ratio) == 1: - # e.g., aspect ratio of 2 (2.00:1) would end up as `(2.0,)`, add 1 - aspect_ratio.append(1) - if aspect_ratio[0] / aspect_ratio[1] not in (16 / 9, 4 / 3): - # We want the resolution represented in a 4:3 or 16:9 canvas. - # If it's not 4:3 or 16:9, calculate as if it's inside a 16:9 canvas, - # otherwise the track's height value is fine. - # We are assuming this title is some weird aspect ratio so most - # likely a movie or HD source, so it's most likely widescreen so - # 16:9 canvas makes the most sense. - resolution = int(primary_video_track.width * (9 / 16)) - name += f" {resolution}p" + if primary_video_track: + resolution_token = _get_resolution_token(primary_video_track) + if resolution_token: + name += f" {resolution_token}" - # Service - if show_service: - name += f" {self.service.__name__}" + # Service (use track source if available) + if show_service: + source_name = None + if self.tracks: + first_track = next(iter(self.tracks), None) + if first_track and hasattr(first_track, "source") and first_track.source: + source_name = first_track.source + name += f" {source_name or self.service.__name__}" - # 'WEB-DL' - name += " WEB-DL" + # 'WEB-DL' + name += " WEB-DL" - # DUAL - if unique_audio_languages == 2: - name += " DUAL" + # DUAL + if unique_audio_languages == 2: + name += " DUAL" - # MULTi - if unique_audio_languages > 2: - name += " MULTi" + # MULTi + if unique_audio_languages > 2: + name += " MULTi" - # Audio Codec + Channels (+ feature) - if primary_audio_track: - codec = primary_audio_track.format - channel_layout = primary_audio_track.channel_layout or primary_audio_track.channellayout_original - if channel_layout: - channels = float( - sum({"LFE": 0.1}.get(position.upper(), 1) for position in channel_layout.split(" ")) - ) - else: - channel_count = primary_audio_track.channel_s or primary_audio_track.channels or 0 - channels = float(channel_count) - - features = primary_audio_track.format_additionalfeatures or "" - name += f" {AUDIO_CODEC_MAP.get(codec, codec)}{channels:.1f}" - if "JOC" in features or primary_audio_track.joc: - name += " Atmos" - - # Video (dynamic range + hfr +) Codec - if primary_video_track: - codec = primary_video_track.format - hdr_format = primary_video_track.hdr_format_commercial - hdr_format_full = primary_video_track.hdr_format or "" - trc = ( - primary_video_track.transfer_characteristics - or primary_video_track.transfer_characteristics_original - or "" + # Audio Codec + Channels (+ feature) + if primary_audio_track: + codec = primary_audio_track.format + channel_layout = primary_audio_track.channel_layout or primary_audio_track.channellayout_original + if channel_layout: + channels = float( + sum({"LFE": 0.1}.get(position.upper(), 1) for position in channel_layout.split(" ")) ) - frame_rate = float(primary_video_track.frame_rate) + else: + channel_count = primary_audio_track.channel_s or primary_audio_track.channels or 0 + channels = float(channel_count) - # Primary HDR format detection - if hdr_format: - if hdr_format_full.startswith("Dolby Vision"): - name += " DV" - if any( - indicator in (hdr_format_full + " " + hdr_format) - for indicator in ["HDR10", "SMPTE ST 2086"] - ): - name += " HDR" - else: - name += f" {DYNAMIC_RANGE_MAP.get(hdr_format)} " - elif "HLG" in trc or "Hybrid Log-Gamma" in trc or "ARIB STD-B67" in trc or "arib-std-b67" in trc.lower(): - name += " HLG" - elif any(indicator in trc for indicator in ["PQ", "SMPTE ST 2084", "BT.2100"]) or "smpte2084" in trc.lower() or "bt.2020-10" in trc.lower(): - name += " HDR" - if frame_rate > 30: - name += " HFR" - name += f" {VIDEO_CODEC_MAP.get(codec, codec)}" + features = primary_audio_track.format_additionalfeatures or "" + name += f" {AUDIO_CODEC_MAP.get(codec, codec)}{channels:.1f}" + if "JOC" in features or primary_audio_track.joc: + name += " Atmos" - if config.tag: - name += f"-{config.tag}" + # Video (dynamic range + hfr +) Codec + if primary_video_track: + codec = primary_video_track.format + hdr_format = primary_video_track.hdr_format_commercial + hdr_format_full = primary_video_track.hdr_format or "" + trc = ( + primary_video_track.transfer_characteristics + or primary_video_track.transfer_characteristics_original + or "" + ) + frame_rate = float(primary_video_track.frame_rate) - return sanitize_filename(name) - else: - # Simple naming style without technical details - use spaces instead of dots - return sanitize_filename(name, " ") + def _append_token(current: str, token: Optional[str]) -> str: + token = (token or "").strip() + current = current.rstrip() + if not token: + return current + if current.endswith(f" {token}"): + return current + return f"{current} {token}" + + # Primary HDR format detection + if hdr_format: + if hdr_format_full.startswith("Dolby Vision"): + name = _append_token(name, "DV") + if any( + indicator in (hdr_format_full + " " + hdr_format) + for indicator in ["HDR10", "SMPTE ST 2086"] + ): + name = _append_token(name, "HDR") + elif "HDR Vivid" in hdr_format: + name = _append_token(name, "HDR") + else: + dynamic_range = DYNAMIC_RANGE_MAP.get(hdr_format) or hdr_format or "" + name = _append_token(name, dynamic_range) + elif "HLG" in trc or "Hybrid Log-Gamma" in trc or "ARIB STD-B67" in trc or "arib-std-b67" in trc.lower(): + name += " HLG" + elif any(indicator in trc for indicator in ["PQ", "SMPTE ST 2084", "BT.2100"]) or "smpte2084" in trc.lower() or "bt.2020-10" in trc.lower(): + name += " HDR" + if frame_rate > 30: + name += " HFR" + name += f" {VIDEO_CODEC_MAP.get(codec, codec)}" + + if config.tag: + name += f"-{config.tag}" + + return sanitize_filename(name, "." if config.scene_naming else " ") class Movies(SortedKeyList, ABC): diff --git a/packages/envied/src/envied/core/titles/song.py b/packages/envied/src/envied/core/titles/song.py index 81c047c..6beece8 100644 --- a/packages/envied/src/envied/core/titles/song.py +++ b/packages/envied/src/envied/core/titles/song.py @@ -100,26 +100,27 @@ class Song(Title): # NN. Song Name name = str(self).split(" / ")[1] - if config.scene_naming: - # Service - if show_service: - name += f" {self.service.__name__}" + # Service (use track source if available) + if show_service: + source_name = None + if self.tracks: + first_track = next(iter(self.tracks), None) + if first_track and hasattr(first_track, "source") and first_track.source: + source_name = first_track.source + name += f" {source_name or self.service.__name__}" - # 'WEB-DL' - name += " WEB-DL" + # 'WEB-DL' + name += " WEB-DL" - # Audio Codec + Channels (+ feature) - name += f" {AUDIO_CODEC_MAP.get(codec, codec)}{channels:.1f}" - if "JOC" in features or audio_track.joc: - name += " Atmos" + # Audio Codec + Channels (+ feature) + name += f" {AUDIO_CODEC_MAP.get(codec, codec)}{channels:.1f}" + if "JOC" in features or audio_track.joc: + name += " Atmos" - if config.tag: - name += f"-{config.tag}" + if config.tag: + name += f"-{config.tag}" - return sanitize_filename(name, " ") - else: - # Simple naming style without technical details - return sanitize_filename(name, " ") + return sanitize_filename(name, " ") class Album(SortedKeyList, ABC): diff --git a/packages/envied/src/envied/core/tracks/attachment.py b/packages/envied/src/envied/core/tracks/attachment.py index ec62f09..b82d6be 100644 --- a/packages/envied/src/envied/core/tracks/attachment.py +++ b/packages/envied/src/envied/core/tracks/attachment.py @@ -10,6 +10,7 @@ from zlib import crc32 import requests from envied.core.config import config +from envied.core.constants import DOWNLOAD_LICENCE_ONLY class Attachment: @@ -43,6 +44,8 @@ class Attachment: if path is None and url is None: raise ValueError("Either path or url must be provided.") + self.url = url + if url: if not isinstance(url, str): raise ValueError("The attachment URL must be a string.") @@ -57,41 +60,55 @@ class Attachment: download_path = config.directories.temp / file_name - # Download the file - try: - session = session or requests.Session() - response = session.get(url, stream=True) - response.raise_for_status() - config.directories.temp.mkdir(parents=True, exist_ok=True) - download_path.parent.mkdir(parents=True, exist_ok=True) + # Download the file unless we're in license-only mode + if DOWNLOAD_LICENCE_ONLY.is_set(): + path = None + else: + try: + if session is None: + with requests.Session() as session: + response = session.get(url, stream=True) + response.raise_for_status() + else: + response = session.get(url, stream=True) + response.raise_for_status() + config.directories.temp.mkdir(parents=True, exist_ok=True) + download_path.parent.mkdir(parents=True, exist_ok=True) - with open(download_path, "wb") as f: - for chunk in response.iter_content(chunk_size=8192): - f.write(chunk) + with open(download_path, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) - path = download_path - except Exception as e: - raise ValueError(f"Failed to download attachment from URL: {e}") + path = download_path + except Exception as e: + raise ValueError(f"Failed to download attachment from URL: {e}") - if not isinstance(path, (str, Path)): - raise ValueError("The attachment path must be provided.") + if path is not None and not isinstance(path, (str, Path)): + raise ValueError( + f"Invalid attachment path type: expected str or Path, got {type(path).__name__}." + ) - path = Path(path) - if not path.exists(): - raise ValueError("The attachment file does not exist.") + if path is not None: + path = Path(path) + if not path.exists(): + raise ValueError("The attachment file does not exist.") - name = (name or path.stem).strip() + if path is not None: + name = (name or path.stem).strip() + else: + name = (name or Path(file_name).stem).strip() mime_type = (mime_type or "").strip() or None description = (description or "").strip() or None if not mime_type: + suffix = path.suffix.lower() if path is not None else Path(file_name).suffix.lower() mime_type = { ".ttf": "application/x-truetype-font", ".otf": "application/vnd.ms-opentype", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".png": "image/png", - }.get(path.suffix.lower(), mimetypes.guess_type(path)[0]) + }.get(suffix, mimetypes.guess_type(file_name if path is None else path)[0]) if not mime_type: raise ValueError("The attachment mime-type could not be automatically detected.") @@ -111,13 +128,18 @@ class Attachment: @property def id(self) -> str: """Compute an ID from the attachment data.""" - checksum = crc32(self.path.read_bytes()) + if self.path and self.path.exists(): + checksum = crc32(self.path.read_bytes()) + elif self.url: + checksum = crc32(self.url.encode("utf8")) + else: + checksum = crc32(self.name.encode("utf8")) return hex(checksum) def delete(self) -> None: - if self.path: + if self.path and self.path.exists(): self.path.unlink() - self.path = None + self.path = None @classmethod def from_url( diff --git a/packages/envied/src/envied/core/tracks/hybrid.py b/packages/envied/src/envied/core/tracks/hybrid.py index ec43c10..526a6c0 100644 --- a/packages/envied/src/envied/core/tracks/hybrid.py +++ b/packages/envied/src/envied/core/tracks/hybrid.py @@ -8,7 +8,7 @@ from pathlib import Path from rich.padding import Padding from rich.rule import Rule -from envied.core.binaries import DoviTool, HDR10PlusTool +from envied.core.binaries import FFMPEG, DoviTool, HDR10PlusTool from envied.core.config import config from envied.core.console import console @@ -109,7 +109,7 @@ class Hybrid: """Simple ffmpeg execution without progress tracking""" p = subprocess.run( [ - "ffmpeg", + str(FFMPEG) if FFMPEG else "ffmpeg", "-nostdin", "-i", str(save_path), diff --git a/packages/envied/src/envied/core/tracks/subtitle.py b/packages/envied/src/envied/core/tracks/subtitle.py index e87f507..58c59d7 100644 --- a/packages/envied/src/envied/core/tracks/subtitle.py +++ b/packages/envied/src/envied/core/tracks/subtitle.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import re import subprocess from collections import defaultdict @@ -16,7 +17,7 @@ from construct import Container from pycaption import Caption, CaptionList, CaptionNode, WebVTTReader from pycaption.geometry import Layout from pymp4.parser import MP4 -from subby import CommonIssuesFixer, SAMIConverter, SDHStripper, WebVTTConverter +from subby import CommonIssuesFixer, SAMIConverter, SDHStripper, WebVTTConverter, WVTTConverter from subtitle_filter import Subtitles from envied.core import binaries @@ -25,6 +26,9 @@ from envied.core.tracks.track import Track from envied.core.utilities import try_ensure_utf8 from envied.core.utils.webvtt import merge_segmented_webvtt +# silence srt library INFO logging +logging.getLogger("srt").setLevel(logging.ERROR) + class Subtitle(Track): class Codec(str, Enum): @@ -595,10 +599,13 @@ class Subtitle(Track): if self.codec == Subtitle.Codec.WebVTT: converter = WebVTTConverter() - srt_subtitles = converter.from_file(str(self.path)) + srt_subtitles = converter.from_file(self.path) + if self.codec == Subtitle.Codec.fVTT: + converter = WVTTConverter() + srt_subtitles = converter.from_file(self.path) elif self.codec == Subtitle.Codec.SAMI: converter = SAMIConverter() - srt_subtitles = converter.from_file(str(self.path)) + srt_subtitles = converter.from_file(self.path) if srt_subtitles is not None: # Apply common fixes @@ -607,11 +614,11 @@ class Subtitle(Track): # If target is SRT, we're done if codec == Subtitle.Codec.SubRip: - output_path.write_text(str(fixed_srt), encoding="utf8") + fixed_srt.save(output_path, encoding="utf8") else: # Convert from SRT to target format using existing pycaption logic temp_srt_path = self.path.with_suffix(".temp.srt") - temp_srt_path.write_text(str(fixed_srt), encoding="utf8") + fixed_srt.save(temp_srt_path, encoding="utf8") # Parse the SRT and convert to target format caption_set = self.parse(temp_srt_path.read_bytes(), Subtitle.Codec.SubRip) @@ -706,7 +713,8 @@ class Subtitle(Track): Convert this Subtitle to another Format. The conversion method is determined by the 'conversion_method' setting in config: - - 'auto' (default): Uses subby for WebVTT/SAMI, standard for others + - 'auto' (default): Uses subby for WebVTT/fVTT/SAMI; for SSA/ASS/MicroDVD/MPL2/TMP + uses SubtitleEdit if available, otherwise pysubs2; standard for others - 'subby': Always uses subby with CommonIssuesFixer - 'subtitleedit': Uses SubtitleEdit when available, falls back to pycaption - 'pycaption': Uses only pycaption library @@ -724,8 +732,19 @@ class Subtitle(Track): elif conversion_method == "pysubs2": return self.convert_with_pysubs2(codec) elif conversion_method == "auto": - if self.codec in (Subtitle.Codec.WebVTT, Subtitle.Codec.SAMI): + if self.codec in (Subtitle.Codec.WebVTT, Subtitle.Codec.fVTT, Subtitle.Codec.SAMI): return self.convert_with_subby(codec) + elif self.codec in ( + Subtitle.Codec.SubStationAlpha, + Subtitle.Codec.SubStationAlphav4, + Subtitle.Codec.MicroDVD, + Subtitle.Codec.MPL2, + Subtitle.Codec.TMP, + ): + if binaries.SubtitleEdit: + return self._convert_standard(codec) + else: + return self.convert_with_pysubs2(codec) else: return self._convert_standard(codec) else: @@ -810,13 +829,18 @@ class Subtitle(Track): if binaries.SubtitleEdit and self.codec not in (Subtitle.Codec.fTTML, Subtitle.Codec.fVTT): sub_edit_format = { - Subtitle.Codec.SubStationAlphav4: "AdvancedSubStationAlpha", - Subtitle.Codec.TimedTextMarkupLang: "TimedText1.0", - }.get(codec, codec.name) + Subtitle.Codec.SubRip: "subrip", + Subtitle.Codec.SubStationAlpha: "substationalpha", + Subtitle.Codec.SubStationAlphav4: "advancedsubstationalpha", + Subtitle.Codec.TimedTextMarkupLang: "timedtext1.0", + Subtitle.Codec.WebVTT: "webvtt", + Subtitle.Codec.SAMI: "sami", + Subtitle.Codec.MicroDVD: "microdvd", + }.get(codec, codec.name.lower()) sub_edit_args = [ - binaries.SubtitleEdit, - "/Convert", - self.path, + str(binaries.SubtitleEdit), + "/convert", + str(self.path), sub_edit_format, f"/outputfilename:{output_path.name}", "/encoding:utf8", @@ -1172,9 +1196,12 @@ class Subtitle(Track): if sdh_method == "subby" and self.codec == Subtitle.Codec.SubRip: # Use subby's SDHStripper directly on the file + fixer = CommonIssuesFixer() stripper = SDHStripper() - stripped_srt, _ = stripper.from_file(str(self.path)) - self.path.write_text(str(stripped_srt), encoding="utf8") + srt, _ = fixer.from_file(self.path) + stripped, status = stripper.from_srt(srt) + if status is True: + stripped.save(self.path) return elif sdh_method == "subtitleedit" and binaries.SubtitleEdit: # Force use of SubtitleEdit @@ -1200,25 +1227,36 @@ class Subtitle(Track): # Try subby first for SRT files, then fall back if self.codec == Subtitle.Codec.SubRip: try: + fixer = CommonIssuesFixer() stripper = SDHStripper() - stripped_srt, _ = stripper.from_file(str(self.path)) - self.path.write_text(str(stripped_srt), encoding="utf8") + srt, _ = fixer.from_file(self.path) + stripped, status = stripper.from_srt(srt) + if status is True: + stripped.save(self.path) return except Exception: pass # Fall through to other methods - if binaries.SubtitleEdit: - if self.codec == Subtitle.Codec.SubStationAlphav4: - output_format = "AdvancedSubStationAlpha" - elif self.codec == Subtitle.Codec.TimedTextMarkupLang: - output_format = "TimedText1.0" - else: - output_format = self.codec.name + conversion_method = config.subtitle.get("conversion_method", "auto") + use_subtitleedit = sdh_method == "subtitleedit" or ( + sdh_method == "auto" and conversion_method in ("auto", "subtitleedit") + ) + + if binaries.SubtitleEdit and use_subtitleedit: + output_format = { + Subtitle.Codec.SubRip: "subrip", + Subtitle.Codec.SubStationAlpha: "substationalpha", + Subtitle.Codec.SubStationAlphav4: "advancedsubstationalpha", + Subtitle.Codec.TimedTextMarkupLang: "timedtext1.0", + Subtitle.Codec.WebVTT: "webvtt", + Subtitle.Codec.SAMI: "sami", + Subtitle.Codec.MicroDVD: "microdvd", + }.get(self.codec, self.codec.name.lower()) subprocess.run( [ - binaries.SubtitleEdit, - "/Convert", - self.path, + str(binaries.SubtitleEdit), + "/convert", + str(self.path), output_format, "/encoding:utf8", "/overwrite", @@ -1226,6 +1264,7 @@ class Subtitle(Track): ], check=True, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, ) else: if config.subtitle.get("convert_before_strip", True) and self.codec != Subtitle.Codec.SubRip: @@ -1267,18 +1306,21 @@ class Subtitle(Track): if not binaries.SubtitleEdit: raise EnvironmentError("SubtitleEdit executable not found...") - if self.codec == Subtitle.Codec.SubStationAlphav4: - output_format = "AdvancedSubStationAlpha" - elif self.codec == Subtitle.Codec.TimedTextMarkupLang: - output_format = "TimedText1.0" - else: - output_format = self.codec.name + output_format = { + Subtitle.Codec.SubRip: "subrip", + Subtitle.Codec.SubStationAlpha: "substationalpha", + Subtitle.Codec.SubStationAlphav4: "advancedsubstationalpha", + Subtitle.Codec.TimedTextMarkupLang: "timedtext1.0", + Subtitle.Codec.WebVTT: "webvtt", + Subtitle.Codec.SAMI: "sami", + Subtitle.Codec.MicroDVD: "microdvd", + }.get(self.codec, self.codec.name.lower()) subprocess.run( [ - binaries.SubtitleEdit, - "/Convert", - self.path, + str(binaries.SubtitleEdit), + "/convert", + str(self.path), output_format, "/ReverseRtlStartEnd", "/encoding:utf8", @@ -1286,6 +1328,7 @@ class Subtitle(Track): ], check=True, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, ) diff --git a/packages/envied/src/envied/core/tracks/track.py b/packages/envied/src/envied/core/tracks/track.py index 4ded54d..f93917d 100644 --- a/packages/envied/src/envied/core/tracks/track.py +++ b/packages/envied/src/envied/core/tracks/track.py @@ -15,11 +15,10 @@ from zlib import crc32 from curl_cffi.requests import Session as CurlSession from langcodes import Language -from pyplayready.cdm import Cdm as PlayReadyCdm -from pywidevine.cdm import Cdm as WidevineCdm from requests import Session from envied.core import binaries +from envied.core.cdm.detect import is_playready_cdm, is_widevine_cdm from envied.core.config import config from envied.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY from envied.core.downloaders import aria2c, curl_impersonate, n_m3u8dl_re, requests @@ -215,7 +214,8 @@ class Track: # or when the subtitle has a direct file extension if self.downloader.__name__ == "n_m3u8dl_re" and ( self.descriptor == self.Descriptor.URL - or get_extension(self.url) in { + or get_extension(self.url) + in { ".srt", ".vtt", ".ttml", @@ -296,14 +296,16 @@ class Track: if not self.drm and track_type in ("Video", "Audio"): # the service might not have explicitly defined the `drm` property # try find DRM information from the init data of URL based on CDM type - if isinstance(cdm, PlayReadyCdm): + if is_playready_cdm(cdm): try: self.drm = [PlayReady.from_track(self, session)] except PlayReady.Exceptions.PSSHNotFound: try: self.drm = [Widevine.from_track(self, session)] except Widevine.Exceptions.PSSHNotFound: - log.warning("No PlayReady or Widevine PSSH was found for this track, is it DRM free?") + log.warning( + "No PlayReady or Widevine PSSH was found for this track, is it DRM free?" + ) else: try: self.drm = [Widevine.from_track(self, session)] @@ -311,7 +313,9 @@ class Track: try: self.drm = [PlayReady.from_track(self, session)] except PlayReady.Exceptions.PSSHNotFound: - log.warning("No Widevine or PlayReady PSSH was found for this track, is it DRM free?") + log.warning( + "No Widevine or PlayReady PSSH was found for this track, is it DRM free?" + ) if self.drm: track_kid = self.get_key_id(session=session) @@ -446,23 +450,14 @@ class Track: if not self.drm: return None - if isinstance(cdm, WidevineCdm): + if is_widevine_cdm(cdm): for drm in self.drm: if isinstance(drm, Widevine): return drm - elif isinstance(cdm, PlayReadyCdm): + elif is_playready_cdm(cdm): for drm in self.drm: if isinstance(drm, PlayReady): return drm - elif hasattr(cdm, "is_playready"): - if cdm.is_playready: - for drm in self.drm: - if isinstance(drm, PlayReady): - return drm - else: - for drm in self.drm: - if isinstance(drm, Widevine): - return drm return self.drm[0] @@ -548,7 +543,6 @@ class Track: try: import m3u8 - from pyplayready.cdm import Cdm as PlayReadyCdm from pyplayready.system.pssh import PSSH as PR_PSSH from pywidevine.cdm import Cdm as WidevineCdm from pywidevine.pssh import PSSH as WV_PSSH @@ -569,7 +563,7 @@ class Track: pssh_b64 = key.uri.split(",")[-1] drm = Widevine(pssh=WV_PSSH(pssh_b64)) drm_list.append(drm) - elif fmt == PlayReadyCdm or "com.microsoft.playready" in fmt: + elif fmt in {f"urn:uuid:{PR_PSSH.SYSTEM_ID}", "com.microsoft.playready"}: pssh_b64 = key.uri.split(",")[-1] drm = PlayReady(pssh=PR_PSSH(pssh_b64), pssh_b64=pssh_b64) drm_list.append(drm) diff --git a/packages/envied/src/envied/core/tracks/tracks.py b/packages/envied/src/envied/core/tracks/tracks.py index 901243b..d42b0c3 100644 --- a/packages/envied/src/envied/core/tracks/tracks.py +++ b/packages/envied/src/envied/core/tracks/tracks.py @@ -95,7 +95,7 @@ class Tracks: return rep - def tree(self, add_progress: bool = False) -> tuple[Tree, list[partial]]: + def tree(self, add_progress: bool = False) -> tuple[Tree, list[Callable[..., None]]]: all_tracks = [*list(self), *self.chapters, *self.attachments] progress_callables = [] @@ -121,7 +121,29 @@ class Tracks: speed_estimate_period=10, ) task = progress.add_task("", downloaded="-") - progress_callables.append(partial(progress.update, task_id=task)) + state = {"total": 100.0} + + def update_track_progress( + task_id: int = task, + _state: dict[str, float] = state, + _progress: Progress = progress, + **kwargs, + ) -> None: + """ + Ensure terminal status states render as a fully completed bar. + + Some downloaders can report completed slightly below total + before emitting the final "Downloaded" state. + """ + if "total" in kwargs and kwargs["total"] is not None: + _state["total"] = kwargs["total"] + + downloaded_state = kwargs.get("downloaded") + if downloaded_state in {"Downloaded", "Decrypted", "[yellow]SKIPPED"}: + kwargs["completed"] = _state["total"] + _progress.update(task_id=task_id, **kwargs) + + progress_callables.append(update_track_progress) track_table = Table.grid() track_table.add_row(str(track)[6:], style="text2") track_table.add_row(progress) @@ -199,13 +221,15 @@ class Tracks: self.videos.sort(key=lambda x: not is_close_match(language, [x.language])) def sort_audio(self, by_language: Optional[Sequence[Union[str, Language]]] = None) -> None: - """Sort audio tracks by bitrate, descriptive, and optionally language.""" + """Sort audio tracks by bitrate, Atmos, descriptive, and optionally language.""" if not self.audio: return - # descriptive - self.audio.sort(key=lambda x: x.descriptive) - # bitrate (within each descriptive group) + # bitrate (highest first) self.audio.sort(key=lambda x: float(x.bitrate or 0.0), reverse=True) + # Atmos tracks first (prioritize over higher bitrate non-Atmos) + self.audio.sort(key=lambda x: not x.atmos) + # descriptive tracks last + self.audio.sort(key=lambda x: x.descriptive) # language for language in reversed(by_language or []): if str(language) in ("all", "best"): @@ -314,6 +338,7 @@ class Tracks: progress: Optional[partial] = None, audio_expected: bool = True, title_language: Optional[Language] = None, + skip_subtitles: bool = False, ) -> tuple[Path, int, list[str]]: """ Multiplex all the Tracks into a Matroska Container file. @@ -328,6 +353,7 @@ class Tracks: if embedded audio metadata should be added. title_language: The title's intended language. Used to select the best video track for audio metadata when multiple video tracks exist. + skip_subtitles: Skip muxing subtitle tracks into the container. """ if self.videos and not self.audio and audio_expected: video_track = None @@ -439,34 +465,35 @@ class Tracks: ] ) - for st in self.subtitles: - if not st.path or not st.path.exists(): - raise ValueError("Text Track must be downloaded before muxing...") - events.emit(events.Types.TRACK_MULTIPLEX, track=st) - default = bool(self.audio and is_close_match(st.language, [self.audio[0].language]) and st.forced) - cl.extend( - [ - "--track-name", - f"0:{st.get_track_name() or ''}", - "--language", - f"0:{st.language}", - "--sub-charset", - "0:UTF-8", - "--forced-track", - f"0:{st.forced}", - "--default-track", - f"0:{default}", - "--hearing-impaired-flag", - f"0:{st.sdh}", - "--original-flag", - f"0:{st.is_original_lang}", - "--compression", - "0:none", # disable extra compression (probably zlib) - "(", - str(st.path), - ")", - ] - ) + if not skip_subtitles: + for st in self.subtitles: + if not st.path or not st.path.exists(): + raise ValueError("Text Track must be downloaded before muxing...") + events.emit(events.Types.TRACK_MULTIPLEX, track=st) + default = bool(self.audio and is_close_match(st.language, [self.audio[0].language]) and st.forced) + cl.extend( + [ + "--track-name", + f"0:{st.get_track_name() or ''}", + "--language", + f"0:{st.language}", + "--sub-charset", + "0:UTF-8", + "--forced-track", + f"0:{st.forced}", + "--default-track", + f"0:{default}", + "--hearing-impaired-flag", + f"0:{st.sdh}", + "--original-flag", + f"0:{st.is_original_lang}", + "--compression", + "0:none", # disable extra compression (probably zlib) + "(", + str(st.path), + ")", + ] + ) if self.chapters: chapters_path = config.directories.temp / config.filenames.chapters.format( diff --git a/packages/envied/src/envied/core/tracks/video.py b/packages/envied/src/envied/core/tracks/video.py index 0fdaa2d..0e3bc6a 100644 --- a/packages/envied/src/envied/core/tracks/video.py +++ b/packages/envied/src/envied/core/tracks/video.py @@ -186,6 +186,10 @@ class Video(Track): # for some reason there's no Dolby Vision info tag raise ValueError(f"The M3U Range Tag '{tag}' is not a supported Video Range") + class ScanType(str, Enum): + PROGRESSIVE = "progressive" + INTERLACED = "interlaced" + def __init__( self, *args: Any, @@ -195,6 +199,7 @@ class Video(Track): width: Optional[int] = None, height: Optional[int] = None, fps: Optional[Union[str, int, float]] = None, + scan_type: Optional[Video.ScanType] = None, **kwargs: Any, ) -> None: """ @@ -232,6 +237,8 @@ class Video(Track): raise TypeError(f"Expected height to be a {int}, not {height!r}") if not isinstance(fps, (str, int, float, type(None))): raise TypeError(f"Expected fps to be a {str}, {int}, or {float}, not {fps!r}") + if not isinstance(scan_type, (Video.ScanType, type(None))): + raise TypeError(f"Expected scan_type to be a {Video.ScanType}, not {scan_type!r}") self.codec = codec self.range = range_ or Video.Range.SDR @@ -256,6 +263,7 @@ class Video(Track): except Exception as e: raise ValueError("Expected fps to be a number, float, or a string as numerator/denominator form, " + str(e)) + self.scan_type = scan_type self.needs_duration_fix = False def __str__(self) -> str: diff --git a/packages/envied/src/envied/core/utilities.py b/packages/envied/src/envied/core/utilities.py index dfd2f53..e727c4e 100644 --- a/packages/envied/src/envied/core/utilities.py +++ b/packages/envied/src/envied/core/utilities.py @@ -19,6 +19,7 @@ from urllib.parse import ParseResult, urlparse from uuid import uuid4 import chardet +import pycountry import requests from construct import ValidationError from fontTools import ttLib @@ -277,6 +278,80 @@ def ap_case(text: str, keep_spaces: bool = False, stop_words: tuple[str] = None) ) +# Common country code aliases that differ from ISO 3166-1 alpha-2 +COUNTRY_CODE_ALIASES = { + "uk": "gb", # United Kingdom -> Great Britain +} + + +def get_country_name(code: str) -> Optional[str]: + """ + Convert a 2-letter country code to full country name. + + Args: + code: ISO 3166-1 alpha-2 country code (e.g., 'ca', 'us', 'gb', 'uk') + + Returns: + Full country name (e.g., 'Canada', 'United States', 'United Kingdom') or None if not found + + Examples: + >>> get_country_name('ca') + 'Canada' + >>> get_country_name('US') + 'United States' + >>> get_country_name('uk') + 'United Kingdom' + """ + # Handle common aliases + code = COUNTRY_CODE_ALIASES.get(code.lower(), code.lower()) + + try: + country = pycountry.countries.get(alpha_2=code.upper()) + if country: + return country.name + except (KeyError, LookupError): + pass + return None + + +def get_country_code(name: str) -> Optional[str]: + """ + Convert a country name to its 2-letter ISO 3166-1 alpha-2 code. + + Args: + name: Full country name (e.g., 'Canada', 'United States', 'United Kingdom') + + Returns: + 2-letter country code in uppercase (e.g., 'CA', 'US', 'GB') or None if not found + + Examples: + >>> get_country_code('Canada') + 'CA' + >>> get_country_code('united states') + 'US' + >>> get_country_code('United Kingdom') + 'GB' + """ + try: + # Try exact name match first + country = pycountry.countries.get(name=name.title()) + if country: + return country.alpha_2.upper() + + # Try common name (e.g., "Bolivia" vs "Bolivia, Plurinational State of") + country = pycountry.countries.get(common_name=name.title()) + if country: + return country.alpha_2.upper() + + # Try fuzzy search as fallback + results = pycountry.countries.search_fuzzy(name) + if results: + return results[0].alpha_2.upper() + except (KeyError, LookupError): + pass + return None + + def get_ip_info(session: Optional[requests.Session] = None) -> dict: """ Use ipinfo.io to get IP location information. diff --git a/packages/envied/src/envied/core/utils/click_types.py b/packages/envied/src/envied/core/utils/click_types.py index 8707732..d53c50e 100644 --- a/packages/envied/src/envied/core/utils/click_types.py +++ b/packages/envied/src/envied/core/utils/click_types.py @@ -5,6 +5,8 @@ import click from click.shell_completion import CompletionItem from pywidevine.cdm import Cdm as WidevineCdm +from envied.core.tracks.audio import Audio + class VideoCodecChoice(click.Choice): """ @@ -241,6 +243,52 @@ class QualityList(click.ParamType): return sorted(resolutions, reverse=True) +class AudioCodecList(click.ParamType): + """Parses comma-separated audio codecs like 'AAC,EC3'.""" + + name = "audio_codec_list" + + def __init__(self, codec_enum): + self.codec_enum = codec_enum + self._name_to_codec: dict[str, Audio.Codec] = {} + for codec in codec_enum: + self._name_to_codec[codec.name.lower()] = codec + self._name_to_codec[codec.value.lower()] = codec + + aliases = { + "eac3": "EC3", + "ddp": "EC3", + "vorbis": "OGG", + } + for alias, target in aliases.items(): + if target in codec_enum.__members__: + self._name_to_codec[alias] = codec_enum[target] + + def convert(self, value: Any, param: Optional[click.Parameter] = None, ctx: Optional[click.Context] = None) -> list: + if not value: + return [] + if isinstance(value, self.codec_enum): + return [value] + if isinstance(value, list): + if all(isinstance(v, self.codec_enum) for v in value): + return value + values = [str(v).strip() for v in value] + else: + values = [v.strip() for v in str(value).split(",")] + + codecs = [] + for val in values: + if not val: + continue + key = val.lower() + if key in self._name_to_codec: + codecs.append(self._name_to_codec[key]) + else: + valid = sorted(set(self._name_to_codec.keys())) + self.fail(f"'{val}' is not valid. Choices: {', '.join(valid)}", param, ctx) + return list(dict.fromkeys(codecs)) # Remove duplicates, preserve order + + class MultipleChoice(click.Choice): """ The multiple choice type allows multiple values to be checked against @@ -288,5 +336,6 @@ class MultipleChoice(click.Choice): SEASON_RANGE = SeasonRange() LANGUAGE_RANGE = LanguageRange() QUALITY_LIST = QualityList() +AUDIO_CODEC_LIST = AudioCodecList(Audio.Codec) # VIDEO_CODEC_CHOICE will be created dynamically when imported diff --git a/packages/envied/src/envied/core/utils/selector.py b/packages/envied/src/envied/core/utils/selector.py new file mode 100644 index 0000000..44548f4 --- /dev/null +++ b/packages/envied/src/envied/core/utils/selector.py @@ -0,0 +1,269 @@ +import click +import sys +from rich.console import Group +from rich.live import Live +from rich.padding import Padding +from rich.table import Table +from rich.text import Text +from envied.core.console import console + +IS_WINDOWS = sys.platform == "win32" +if IS_WINDOWS: + import msvcrt + +class Selector: + """ + A custom interactive selector class using the Rich library. + Allows for multi-selection of items with pagination. + """ + def __init__( + self, + options: list[str], + cursor_style: str = "pink", + text_style: str = "text", + page_size: int = 8, + minimal_count: int = 0, + dependencies: dict[int, list[int]] = None, + prefixes: list[str] = None + ): + """ + Initialize the Selector. + + Args: + options: List of strings to select from. + cursor_style: Rich style for the highlighted cursor item. + text_style: Rich style for normal items. + page_size: Number of items to show per page. + minimal_count: Minimum number of items that must be selected. + dependencies: Dictionary mapping parent index to list of child indices. + """ + self.options = options + self.cursor_style = cursor_style + self.text_style = text_style + self.page_size = page_size + self.minimal_count = minimal_count + self.dependencies = dependencies or {} + + self.cursor_index = 0 + self.selected_indices = set() + self.scroll_offset = 0 + + def get_renderable(self): + """ + Constructs and returns the renderable object (Table + Info) for the current state. + """ + table = Table(show_header=False, show_edge=False, box=None, pad_edge=False, padding=(0, 1, 0, 0)) + table.add_column("Indicator", justify="right", no_wrap=True) + table.add_column("Option", overflow="ellipsis", no_wrap=True) + + for i in range(self.page_size): + idx = self.scroll_offset + i + + if idx < len(self.options): + option = self.options[idx] + is_cursor = (idx == self.cursor_index) + is_selected = (idx in self.selected_indices) + + symbol = "[X]" if is_selected else "[ ]" + style = self.cursor_style if is_cursor else self.text_style + indicator_text = Text(f"{symbol}", style=style) + + content_text = Text.from_markup(option) + content_text.style = style + + table.add_row(indicator_text, content_text) + else: + table.add_row(Text(" "), Text(" ")) + + total_pages = (len(self.options) + self.page_size - 1) // self.page_size + current_page = (self.scroll_offset // self.page_size) + 1 + + info_text = Text( + f"\n[Space]: Toggle [a]: All [←/→]: Page [Enter]: Confirm (Page {current_page}/{total_pages})", + style="gray" + ) + + return Padding(Group(table, info_text), (0, 5)) + + def move_cursor(self, delta: int): + """ + Moves the cursor up or down by the specified delta. + Updates the scroll offset if the cursor moves out of the current view. + """ + self.cursor_index = (self.cursor_index + delta) % len(self.options) + new_page_idx = self.cursor_index // self.page_size + self.scroll_offset = new_page_idx * self.page_size + + def change_page(self, delta: int): + """ + Changes the current page view by the specified delta (previous/next page). + Also moves the cursor to the first item of the new page. + """ + current_page = self.scroll_offset // self.page_size + total_pages = (len(self.options) + self.page_size - 1) // self.page_size + new_page = current_page + delta + + if 0 <= new_page < total_pages: + self.scroll_offset = new_page * self.page_size + first_idx_of_page = self.scroll_offset + if first_idx_of_page < len(self.options): + self.cursor_index = first_idx_of_page + else: + self.cursor_index = len(self.options) - 1 + + def toggle_selection(self): + """ + Toggles the selection state of the item currently under the cursor. + Propagates selection to children if defined in dependencies. + """ + target_indices = {self.cursor_index} + + if self.cursor_index in self.dependencies: + target_indices.update(self.dependencies[self.cursor_index]) + + should_select = self.cursor_index not in self.selected_indices + + if should_select: + self.selected_indices.update(target_indices) + else: + self.selected_indices.difference_update(target_indices) + + def toggle_all(self): + """ + Toggles the selection of all items. + If all are selected, clears selection. Otherwise, selects all. + """ + if len(self.selected_indices) == len(self.options): + self.selected_indices.clear() + else: + self.selected_indices = set(range(len(self.options))) + + def get_input_windows(self): + """ + Captures and parses keyboard input on Windows systems using msvcrt. + Returns command strings like 'UP', 'DOWN', 'ENTER', etc. + """ + key = msvcrt.getch() + if key == b'\x03' or key == b'\x1b': + return 'CANCEL' + if key == b'\xe0' or key == b'\x00': + try: + key = msvcrt.getch() + if key == b'H': return 'UP' + if key == b'P': return 'DOWN' + if key == b'K': return 'LEFT' + if key == b'M': return 'RIGHT' + except: pass + + try: char = key.decode('utf-8', errors='ignore') + except: return None + + if char in ('\r', '\n'): return 'ENTER' + if char == ' ': return 'SPACE' + if char in ('q', 'Q'): return 'QUIT' + if char in ('a', 'A'): return 'ALL' + if char in ('w', 'W', 'k', 'K'): return 'UP' + if char in ('s', 'S', 'j', 'J'): return 'DOWN' + if char in ('h', 'H'): return 'LEFT' + if char in ('d', 'D', 'l', 'L'): return 'RIGHT' + return None + + def get_input_unix(self): + """ + Captures and parses keyboard input on Unix/Linux systems using click.getchar(). + Returns command strings like 'UP', 'DOWN', 'ENTER', etc. + """ + char = click.getchar() + if char == '\x03': + return 'CANCEL' + mapping = { + '\x1b[A': 'UP', + '\x1b[B': 'DOWN', + '\x1b[C': 'RIGHT', + '\x1b[D': 'LEFT', + } + if char in mapping: + return mapping[char] + if char == '\x1b': + try: + next1 = click.getchar() + if next1 in ('[', 'O'): + next2 = click.getchar() + if next2 == 'A': return 'UP' + if next2 == 'B': return 'DOWN' + if next2 == 'C': return 'RIGHT' + if next2 == 'D': return 'LEFT' + return 'CANCEL' + except: + return 'CANCEL' + + if char in ('\r', '\n'): return 'ENTER' + if char == ' ': return 'SPACE' + if char in ('q', 'Q'): return 'QUIT' + if char in ('a', 'A'): return 'ALL' + if char in ('w', 'W', 'k', 'K'): return 'UP' + if char in ('s', 'S', 'j', 'J'): return 'DOWN' + if char in ('h', 'H'): return 'LEFT' + if char in ('d', 'D', 'l', 'L'): return 'RIGHT' + return None + + def run(self) -> list[int]: + """ + Starts the main event loop for the selector. + Renders the UI and processes input until confirmed or cancelled. + + Returns: + list[int]: A sorted list of selected indices. + """ + try: + with Live(self.get_renderable(), console=console, auto_refresh=False, transient=True) as live: + while True: + live.update(self.get_renderable(), refresh=True) + if IS_WINDOWS: action = self.get_input_windows() + else: action = self.get_input_unix() + + if action == 'UP': self.move_cursor(-1) + elif action == 'DOWN': self.move_cursor(1) + elif action == 'LEFT': self.change_page(-1) + elif action == 'RIGHT': self.change_page(1) + elif action == 'SPACE': self.toggle_selection() + elif action == 'ALL': self.toggle_all() + elif action in ('ENTER', 'QUIT'): + if len(self.selected_indices) >= self.minimal_count: + return sorted(list(self.selected_indices)) + elif action == 'CANCEL': raise KeyboardInterrupt + except KeyboardInterrupt: + return [] + +def select_multiple( + options: list[str], + minimal_count: int = 1, + page_size: int = 8, + return_indices: bool = True, + cursor_style: str = "pink", + **kwargs +) -> list[int]: + """ + Drop-in replacement using custom Selector with global console. + + Args: + options: List of options to display. + minimal_count: Minimum number of selections required. + page_size: Number of items per page. + return_indices: If True, returns indices; otherwise returns the option strings. + cursor_style: Style color for the cursor. + """ + selector = Selector( + options=options, + cursor_style=cursor_style, + text_style="text", + page_size=page_size, + minimal_count=minimal_count, + **kwargs + ) + + selected_indices = selector.run() + + if return_indices: + return selected_indices + return [options[i] for i in selected_indices] diff --git a/packages/envied/src/envied/core/utils/webvtt.py b/packages/envied/src/envied/core/utils/webvtt.py index c6a5ecb..562dd25 100644 --- a/packages/envied/src/envied/core/utils/webvtt.py +++ b/packages/envied/src/envied/core/utils/webvtt.py @@ -168,6 +168,16 @@ def merge_segmented_webvtt(vtt_raw: str, segment_durations: Optional[list[int]] duplicate_index: list[int] = [] captions = vtt.get_captions(lang) + # Some providers can produce "segment_index" values that are + # outside the provided segment_durations list after normalization/merge. + # This used to crash with IndexError and abort the entire download. + if segment_durations and captions: + max_idx = max(getattr(c, "segment_index", 0) for c in captions) + if max_idx >= len(segment_durations): + # Pad with the last known duration (or 0 if empty) so indexing is safe. + pad_val = segment_durations[-1] if segment_durations else 0 + segment_durations = segment_durations + [pad_val] * (max_idx - len(segment_durations) + 1) + if captions[0].segment_index == 0: first_segment_mpegts = captions[0].mpegts else: @@ -179,6 +189,9 @@ def merge_segmented_webvtt(vtt_raw: str, segment_durations: Optional[list[int]] # calculate the timestamp from SegmentTemplate/SegmentList duration. likely_dash = first_segment_mpegts == 0 and caption.mpegts == 0 if likely_dash and segment_durations: + # Defensive: segment_index can still be out of range if captions are malformed. + if caption.segment_index < 0 or caption.segment_index >= len(segment_durations): + continue duration = segment_durations[caption.segment_index] caption.mpegts = MPEG_TIMESCALE * (duration / timescale) diff --git a/packages/envied/src/envied/services/DSNP/__init__.py b/packages/envied/src/envied/services/DSNP/__init__.py index b4d0c8a..a0aa19e 100644 --- a/packages/envied/src/envied/services/DSNP/__init__.py +++ b/packages/envied/src/envied/services/DSNP/__init__.py @@ -1,28 +1,30 @@ from __future__ import annotations +import base64 +import click import re +import secrets import sys import uuid -from datetime import datetime -from collections.abc import Generator -from http.cookiejar import CookieJar -from typing import Any, Optional, Union, List -from langcodes import Language -import click from click import Context +from collections.abc import Generator +from datetime import datetime +from http.cookiejar import CookieJar +from langcodes import Language from pyplayready.cdm import Cdm as PlayReadyCdm from requests import Request +from typing import Any, Optional, Union, List from envied.core.constants import AnyTrack from envied.core.credential import Credential +from envied.core.manifests import HLS from envied.core.search_result import SearchResult from envied.core.service import Service -from envied.core.manifests import HLS from envied.core.titles import Title_T, Titles_T, Episode, Movie, Movies, Series from envied.core.tracks import Chapter, Chapters, Tracks, Attachment, Video, Audio, Subtitle -from envied.core.utils.collections import as_list from envied.core.utilities import get_ip_info +from envied.core.utils.collections import as_list from . import queries @@ -46,11 +48,12 @@ class DSNP(Service): @click.argument("title", type=str) @click.option("--imax", is_flag=True, default=False, help="Prefer IMAX Enhanced version if available.") @click.option("--remastered-ar", is_flag=True, default=False, help="Prefer Remastered Aspect Ratio if available.") + @click.option("-ext", "--extras", is_flag=True, default=False, help="Select a extras video if available.") @click.pass_context def cli(ctx: Context, **kwargs: Any) -> DSNP: return DSNP(ctx, **kwargs) - def __init__(self, ctx: Context, title: str, imax: bool, remastered_ar: bool): + def __init__(self, ctx: Context, title: str, imax: bool, remastered_ar: bool, extras: bool): self.title = title super().__init__(ctx) @@ -61,8 +64,9 @@ class DSNP(Service): self.title_id = match.group("id") break - self.prefer_imax = imax or False - self.prefer_remastered_ar = remastered_ar or False + self.prefer_imax = imax + self.prefer_remastered_ar = remastered_ar + self.extras = extras self.vcodec = ctx.parent.params.get("vcodec") or Video.Codec.AVC self.acodec : Audio.Codec = ctx.parent.params.get("acodec") @@ -74,8 +78,8 @@ class DSNP(Service): self.chapters_only = ctx.parent.params.get("chapters_only") self.cdm = ctx.obj.cdm - self.playready = isinstance(self.cdm, PlayReadyCdm) - self.is_l3 = (self.cdm.security_level < 3000) if self.playready else (self.cdm.security_level == 3) + self.playready = isinstance(self.cdm, PlayReadyCdm) if self.cdm else False + self.is_l3 = (self.cdm.security_level < 3000) if self.playready else (self.cdm.security_level == 3) if self.cdm else False self.region = None self.prod_config = {} @@ -91,13 +95,13 @@ class DSNP(Service): self.quality = [720] self.log.warning(" + Switched video to HD. This CDM only support HD.") else: - if self.quality > [1080] and self.range == [Video.Range.SDR]: + if self.quality > [1080] and self.range[0] == [Video.Range.SDR]: self.range = [Video.Range.HDR10] self.log.info(" + Switched range to HDR10. 4K resolution requires HDR.") if (self.range != [Video.Range.SDR] or self.quality > [1080]) and self.vcodec != Video.Codec.HEVC: self.vcodec = Video.Codec.HEVC - self.log.info(f" + Switched video codec to H265 to be able to get {self.range} dynamic range.") + self.log.info(f" + Switched video codec to H265 to be able to get {self.range[0]} dynamic range.") if self.acodec == Audio.Codec.DTS and not self.prefer_imax: self.prefer_imax = True @@ -164,7 +168,7 @@ class DSNP(Service): self._perform_switch_profile(target_profile, self.session.headers) self.log.info(" + Refreshing session data after profile switch...") - full_account_info = self._get_account_info_raw() + full_account_info = self._get_account_info() self.active_session = full_account_info["activeSession"] self.active_session['account'] = full_account_info['account'] self.log.info("Session data updated successfully.") @@ -193,12 +197,14 @@ class DSNP(Service): current_imax_setting = full_profile_object["attributes"]["playbackSettings"]["preferImaxEnhancedVersion"] self.log.info(f" + IMAX Enhanced: {current_imax_setting}") if current_imax_setting is not self.prefer_imax: - self._set_imax_preference(self.prefer_imax) + update_tokens = self._set_imax_preference(self.prefer_imax) + self._apply_new_tokens(update_tokens["token"]) current_133_setting = full_profile_object["attributes"]["playbackSettings"]["prefer133"] # Original Aspect Ratio self.log.info(f" + Remastered Aspect Ratio: {not current_133_setting}") if not current_133_setting is not self.prefer_remastered_ar: - self._set_remastered_ar_preference(self.prefer_remastered_ar) + update_tokens = self._set_remastered_ar_preference(self.prefer_remastered_ar) + self._apply_new_tokens(update_tokens["token"]) def _login(self) -> None: cache = self.cache.get(f"tokens_{self.region}_{self.credentials.sha1}") @@ -208,7 +214,7 @@ class DSNP(Service): self.log.info(" + Using cached tokens...") self.account_tokens = cache.data - bearer = self.account_tokens["token"]["accessToken"] + bearer = self.account_tokens["accessToken"] if not bearer: raise ValueError("accessToken not found in cache") self.session.headers.update({'Authorization': f'Bearer {bearer}'}) @@ -231,19 +237,45 @@ class DSNP(Service): self._perform_full_login() self.log.info(" + Fetching session data...") - full_account_info = self._get_account_info_raw() + full_account_info = self._get_account_info() self.active_session = full_account_info["activeSession"] self.active_session['account'] = full_account_info['account'] self.log.info("Session data setup successfully.") def _perform_full_login(self) -> None: - device_token = self._register_device() + android_id = secrets.token_bytes(8).hex() + drm_id = f"{base64.urlsafe_b64encode(secrets.token_bytes(32)).decode('utf-8')}\n" + device_token = self._register_device(android_id, drm_id) email_status = self._check_email(self.credentials.username, device_token) + if email_status.lower() != "login": - if email_status.lower() == "OTP": - self.log.error(" - Account requires 2FA passcode.", exc_info=False) - sys.exit(1) + if email_status.lower() == "otp": + self.log.warning(" - Account requires OTP code login.") + self._request_otp(self.credentials.username, device_token) + + otp_code = None + try: + otp_code = input("Enter a OTP code (Check email): ") + if not otp_code: + self.log.error(" - OTP code is required, but no value was entered.", exc_info=False) + sys.exit(1) + if not otp_code.isdigit(): + self.log.error(" - Invalid OTP code. Please enter only numbers.", exc_info=False) + sys.exit(1) + if len(otp_code) < 6: + self.log.error(" - OTP code is too short. Please enter at least 6 digits.", exc_info=False) + sys.exit(1) + if len(otp_code) > 6: + self.log.warning(" - OTP code is longer than 6 digits. Using the first 6 digits.") + otp_code = otp_code[:6] + except KeyboardInterrupt: + self.log.error("\n - OTP code input cancelled by user.", exc_info=False) + sys.exit(1) + + auth_action = self._auth_action_with_otp(self.credentials.username, otp_code, device_token) + login_tokens = self._login_with_auth_action(auth_action, device_token) + elif email_status.lower() == "register": self.log.error(" - Account is not registered. Please register first.", exc_info=False) sys.exit(1) @@ -251,10 +283,11 @@ class DSNP(Service): self.log.error(f" - Email status is '{email_status}'. Account status verification required.", exc_info=False) sys.exit(1) - login_tokens = self._login_with_password(self.credentials.username, self.credentials.password, device_token) + else: + login_tokens = self._login_with_password(self.credentials.username, self.credentials.password, device_token) - temp_auth_header = {"Authorization": f'Bearer {login_tokens["accessToken"]}'} - account_info = self._get_account_info_raw(temp_auth_header) + temp_auth_header = {"Authorization": f'Bearer {login_tokens["token"]["accessToken"]}'} + account_info = self._get_account_info(temp_auth_header) profiles = account_info["account"]["profiles"] selected_profile = None @@ -308,10 +341,8 @@ class DSNP(Service): self.log.error("\n - PIN input cancelled by user.", exc_info=False) sys.exit(1) - switch_profile_data = self._switch_profile(target_profile['id'], auth_headers, profile_pin) - final_token_data = self._refresh_token(switch_profile_data["token"]["refreshToken"]) - self._apply_new_tokens(final_token_data) + self._apply_new_tokens(switch_profile_data["token"]) def _refresh(self) -> str: cache = self.cache.get(f"tokens_{self.region}_{self.credentials.sha1}") @@ -321,23 +352,21 @@ class DSNP(Service): self.log.warning(" + Token expired. Refreshing...") try: - refreshed_data = self._refresh_token(self.account_tokens["token"]["refreshToken"]) - bearer = self._apply_new_tokens(refreshed_data) - return bearer + refreshed_data = self._refresh_token(self.account_tokens["refreshToken"]) + self._apply_new_tokens(refreshed_data["token"]) except Exception as _: - self.log.error("Refresh Token Expired", exc_info=False) - sys.exit(1) + raise Exception("Refresh Token Expired") def _apply_new_tokens(self, token_data: dict) -> str: self.account_tokens = token_data - bearer = self.account_tokens["token"]["accessToken"] + bearer = self.account_tokens["accessToken"] if not bearer: self.log.error("Invalid token data: accessToken not found.", exc_info=False) sys.exit(1) self.session.headers.update({'Authorization': f'Bearer {bearer}'}) - expires_in = self.account_tokens["token"]["expiresIn"] or 3600 + expires_in = self.account_tokens["expiresIn"] or 3600 cache = self.cache.get(f"tokens_{self.region}_{self.credentials.sha1}") cache.set(self.account_tokens, expires_in - 60) self.log.debug(f" + New Token is valid until: {datetime.fromtimestamp(cache.expiration.timestamp()).strftime('%Y-%m-%d %H:%M:%S')}") @@ -363,64 +392,71 @@ class DSNP(Service): ) def get_titles(self) -> Titles_T: - try: - content_info = self._get_deeplink(self.title_id) - content_type = content_info["data"]["deeplink"]["actions"][0]["contentType"] - except Exception as e: + if not self.extras: try: - actions_info = self._get_deeplink_last(self.title_id) - if actions_info["data"]["deeplink"]["actions"][0]["type"] == "browse": - content_type = "other" - self.log.warning(" - The content is not standard. however, it tries to look up the data.") + content_info = self._get_deeplink(self.title_id) + content_type = content_info["data"]["deeplink"]["actions"][0]["contentType"] except Exception as e: - self.log.error(f" - Failed to determine content type via deeplink ({e}).", exc_info=False) - sys.exit(1) + try: + actions_info = self._get_deeplink_last(self.title_id) + if actions_info["data"]["deeplink"]["actions"][0]["type"] == "browse": + info_block = base64.b64decode(actions_info["data"]["deeplink"]["actions"][0]["infoBlock"]) + if b"movie" in info_block: + content_type = "movie" + elif b"series" in info_block: + content_type = "series" + else: + content_type = "other" + self.log.warning(" - The content is not standard. however, it tries to look up the data.") + except Exception as e: + self.log.error(f" - Failed to determine content type via deeplink ({e}).", exc_info=False) + sys.exit(1) + else: + content_type = "extras" self.log.debug(f" + Content Type: {content_type.upper()}") page = self._get_page(self.title_id) - orig_lang = "en" - if not content_type == "other": - playback_action = next(x for x in page["actions"] if x["type"] == "playback") - avail_id = playback_action["availId"] - self.log.debug(f" + Avail ID: {avail_id}") - lang_data = self._get_original_lang(avail_id) - orig_lang = lang_data["data"]["playerExperience"]["originalLanguage"] + year = None + if year_data := page["visuals"]["metastringParts"].get("releaseYearRange"): + year = year_data.get("startYear") + + if content_type != "extras": + playback_action = next( + (x for x in page["actions"] if x["type"] == "playback"), + None + ) + if not playback_action: + self.log.error(f" - No content is available. (Playback action not found)", exc_info=False) + sys.exit(1) + lang_data = self._get_original_lang(playback_action["availId"]) + player_exp = lang_data["data"]["playerExperience"] + orig_lang = player_exp.get("originalLanguage") or player_exp.get("targetLanguage") or "en" self.log.debug(f' + Original Language: {orig_lang}') - if content_type == "movie": - return Movies( - [ - Movie( - id_=page["id"], - service=self.__class__, - name=page["visuals"]["title"], - year=page["visuals"]["metastringParts"]["releaseYearRange"]["startYear"], - language=Language.get(orig_lang), - data=page - ) - ] - ) + if content_type in ("movie", "other"): + return Movies([ + Movie( + id_=page["id"], + service=self.__class__, + name=page["visuals"]["title"], + year=year, + language=Language.get(orig_lang), + data=page + ) + ]) elif content_type == "series": - return Series(self._get_series(page, orig_lang)) + return Series(self._get_series(page, year, orig_lang)) + + elif content_type == "extras": + return Series(self._get_extras(page, year)) - elif content_type == "other": - return Movies( - [ - Movie( - id_=page["id"], - service=self.__class__, - name=page["visuals"]["title"], - data=page - ) - ] - ) else: self.log.error(f" - Unsupported content type: {content_type}", exc_info=False) sys.exit(1) - def _get_series(self, page: dict, orig_lang: str) -> Series: + def _get_series(self, page: dict, year: int, orig_lang: str) -> Series: container = next(x for x in page["containers"] if x["type"] == "episodes") season_ids = [s["id"] for s in container["seasons"]] @@ -440,13 +476,62 @@ class DSNP(Service): season=int(ep["visuals"]["seasonNumber"]), number=int(ep["visuals"]["episodeNumber"]), name=ep["visuals"]["episodeTitle"], - year=page["visuals"]["metastringParts"]["releaseYearRange"]["startYear"], + year=year, language=Language.get(orig_lang), data=ep ) ) return episodes + + def _get_extras(self, page: dict, year: int) -> Series: + extras_containers = [ + x for x in page["containers"] + if x["type"] == "set" and x["style"]["name"] == "standard_compact_list" + ] + + if not extras_containers: + self.log.error(" - No extras found.", exc_info=False) + sys.exit(1) + + extras_episodes : List[Episode] = [] + ep_count = 1 + + first_item = extras_containers[0]["items"][0] + first_action = next((x for x in first_item["actions"] if x["type"] in ("playback", "trailer")), None) + if first_action: + lang_data = self._get_original_lang(first_action["availId"]) + player_exp = lang_data["data"]["playerExperience"] + orig_lang = player_exp.get("originalLanguage") or player_exp.get("targetLanguage") or "en" + self.log.debug(f' + Original Language: {orig_lang}') + + for container in extras_containers: + items = container["items"] + for item in items: + if item["type"] == "view": + action = next((x for x in item["actions"] if x["type"] in ("playback", "trailer")), None) + + if action: + extras_episodes.append( + Episode( + id_=item["id"], + service=self.__class__, + title=page["visuals"]["title"], + season=0, # Special + number=ep_count, + name=item["visuals"]["title"], + year=year, + language=Language.get(orig_lang), + data=item + ) + ) + ep_count += 1 + + if not extras_episodes: + self.log.error(" - No playable extras found.", exc_info=False) + sys.exit(1) + + return extras_episodes def get_tracks(self, title: Title_T) -> Tracks: playback = next(x for x in title.data["actions"] if x.get("type") == "playback") @@ -462,7 +547,7 @@ class DSNP(Service): self._refresh() # Safe Access - if Video.Range.HYBRID in self.range and not self.is_l3: + if Video.Range.HYBRID in self.range[0] and not self.is_l3: self.log.warning("DV+HDR Multi-range requested.") self.log.info(" + Fetching Dolby Vision tracks...") @@ -475,15 +560,14 @@ class DSNP(Service): else: video_ranges = [] if not self.is_l3: - if Video.Range.DV in self.range: + if Video.Range.DV in self.range[0]: video_ranges = ["DOLBY_VISION"] - elif Video.Range.HDR10 in self.range or Video.Range.HDR10P in self.range: + elif Video.Range.HDR10 in self.range[0] or Video.Range.HDR10P in self.range[0]: video_ranges = ["HDR10"] # HDR10PLUS tracks = self._fetch_manifest_tracks(title, media_id, scenario, video_ranges or None) tracks.add(self._get_thumbnail(title)) - return self._post_process_tracks(tracks) def _fetch_manifest_tracks(self, title: Title_T, media_id: str, scenario: str, video_ranges: List[str] = None) -> Tracks: @@ -494,8 +578,13 @@ class DSNP(Service): }, "protocol": "HTTPS", "frameRates": [60], - "assetInsertionStrategy": "SGAI", # Server-Guided Ad Insertion - "playbackInitiationContext": "ONLINE" + "assetInsertionStrategies": { + "point": "SGAI", # Server-Guided Ad Insertion + "range": "SGAI" # Server-Guided Ad Insertion + }, + "playbackInitiationContext": "ONLINE", + "slugDuration": "SLUG_500_MS", # SLUG_1000_MS, SLUG_750_MS ? + "maxSlideDuration": "4_HOUR" # 15_MIN ? } if self.is_l3: @@ -517,9 +606,9 @@ class DSNP(Service): "attributes": attributes } } - self.playback_data[title.id] = self._get_video(scenario, payload) - + self.playback_data[title.id] = self._get_playback(scenario, payload) manifest_url = self.playback_data[title.id]["sources"][0]['complete']['url'] + self.log.debug(f" + Manifest URL: {manifest_url}") return HLS.from_url(url=manifest_url, session=self.session).to_tracks(title.language) def _get_thumbnail(self, title: Title_T) -> Attachment: @@ -548,67 +637,39 @@ class DSNP(Service): audio.bitrate = 768_000 # DSNP lies about the Atmos bitrate if audio.channels == 6.0: audio.channels = 5.1 + if audio.channels == 10.0: # DTS-UHD + audio.channels = "5.1.4" # Unshackle does not recommend + audio.codec = Audio.Codec.DTS + audio.drm = None for subtitle in tracks.subtitles: subtitle.codec = Subtitle.Codec.WebVTT return tracks - + def get_chapters(self, title: Title_T) -> Chapters: try: editorial = self.playback_data[title.id]["editorial"] - if not editorial: - return [] + return Chapters() - label_to_group = { + LABEL_MAP = { "intro_start": "intro_start", - "FFEI": "intro_start", # First Frame Episode Intro "intro_end": "intro_end", - "LFEI": "intro_end", # Last Frame Episode Intro "recap_start": "recap_start", - "FFER": "recap_start", # First Frame Episode Recap "recap_end": "recap_end", - "LFER": "recap_end", # Last Frame Episode Recap + "FFER": "recap_start", # First Frame Episode Recap + "LFER": "recap_end", # Last Frame Episode Recap + "FFEI": "intro_start", # First Frame Episode Intro + "LFEI": "intro_end", # Last Frame Episode Intro "FFEC": "credits_start", # First Frame End Credits "LFEC": "lfec_marker", # Last Frame End Credits - "FFCB": None, # First Frame Credits Bumper - "LFCB": None, # Last Frame Credits Bumper "up_next": None, "tag_start": None, "tag_end": None, } - # Collision Correction - grouped_timestamps = {} - for marker in editorial: - label = marker.get("label") - group = label_to_group.get(label) - if group: - timestamp = marker.get("offsetMillis") - if timestamp is not None: - if group not in grouped_timestamps: - grouped_timestamps[group] = [] - grouped_timestamps[group].append(timestamp) - - resolved_markers = [] - for group, timestamps in grouped_timestamps.items(): - if not timestamps: - continue - - final_timestamp = 0 - if "start" in group: - final_timestamp = min(timestamps) - elif "end" in group: - final_timestamp = max(timestamps) - else: - final_timestamp = timestamps[0] - - resolved_markers.append({"group": group, "ms": final_timestamp}) - - # Create Chapter Data - raw_chapter_data = [] - group_to_name = { + NAME_MAP = { "recap_start": "Recap", "recap_end": "Scene", "intro_start": "Intro", @@ -616,66 +677,63 @@ class DSNP(Service): "credits_start": "Credits", } - total_runtime_ms = 0 - if "visuals" in title.data and "metastringParts" in title.data["visuals"]: - total_runtime_ms = title.data["visuals"]["metastringParts"]["runtime"]["runtimeMs"] + grouped = {} + for marker in editorial: + group = LABEL_MAP.get(marker["label"]) + offset = marker["offsetMillis"] + if group and offset is not None: + grouped.setdefault(group, []).append(offset) - for marker in resolved_markers: - group = marker["group"] - timestamp_ms = marker["ms"] - name = None + raw_chapters = [] + total_runtime = title.data["visuals"]["metastringParts"]["runtime"]["runtimeMs"] - if group == "lfec_marker": - if total_runtime_ms and (total_runtime_ms - timestamp_ms) > 5000: # 5 sec - name = "Scene" - else: - name = group_to_name.get(group) + for group, times in grouped.items(): + if not times: continue + + timestamp = min(times) if "start" in group else max(times) if "end" in group else times[0] + name = NAME_MAP.get(group) + if group == "lfec_marker" and (total_runtime - timestamp) > 5000: + name = "Scene" + if name: - raw_chapter_data.append({"ms": timestamp_ms, "name": name}) - - # Sorting and deduplication in chronological order - if not raw_chapter_data: - return [] + raw_chapters.append((timestamp, name)) - unique_chapters_data = [] + raw_chapters.sort(key=lambda x: x[0]) + unique_chapters = [] seen_ms = set() - for chap in sorted(raw_chapter_data, key=lambda x: x["ms"]): - if chap["ms"] not in seen_ms: - unique_chapters_data.append(chap) - seen_ms.add(chap["ms"]) - # Processe the First Chapter - if not unique_chapters_data: - unique_chapters_data.append({"ms": 0, "name": "Scene"}) + for ms, name in raw_chapters: + if ms not in seen_ms: + unique_chapters.append({"ms": ms, "name": name}) + seen_ms.add(ms) + + if not unique_chapters: + unique_chapters.append({"ms": 0, "name": "Scene"}) else: - first_chapter = unique_chapters_data[0] - if first_chapter["ms"] > 0: - if not (first_chapter["ms"] < 5000 and first_chapter["name"] in ["Intro", "Recap"]): - unique_chapters_data.insert(0, {"ms": 0, "name": "Scene"}) - - if unique_chapters_data: - first_chapter = unique_chapters_data[0] - if first_chapter["name"] in ["Intro", "Recap"] and first_chapter["ms"] > 0: - first_chapter["ms"] = 0 + first = unique_chapters[0] + if first["ms"] > 0: + if first["ms"] < 5000 and first["name"] in ("Intro", "Recap"): + first["ms"] = 0 + else: + unique_chapters.insert(0, {"ms": 0, "name": "Scene"}) - # Create Final Chapter List - final_chapters = [] - for i, chap_info in enumerate(unique_chapters_data): - name = chap_info["name"] - - final_chapters.append( + chapters: List[Chapter] = [] + for i, chap_data in enumerate(unique_chapters): + time_sec = chap_data["ms"] / 1000.000 + chapter_title = chap_data["name"] + chapters.append( Chapter( - timestamp=chap_info["ms"] / 1000.000, - name=name if name != "Scene" else None + timestamp=float(time_sec), + name=chapter_title if chapter_title != "Scene" else None ) ) - return final_chapters + return chapters except Exception as e: self.log.warning(f"Failed to extract chapters: {e}") - return [] + return Chapters() def get_widevine_service_certificate(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Union[bytes, str]: # endpoint = self.prod_config["services"]["drm"]["client"]["endpoints"]["widevineCertificate"]["href"] @@ -691,7 +749,7 @@ class DSNP(Service): res = self.session.post(endpoint, headers=headers, data=challenge) res.raise_for_status() except Exception as e: - self.log.error(f" - License request failed: {e}", exc_info=False) + self.log.error(f"License request failed: {e}", exc_info=False) sys.exit(1) return res.content @@ -707,7 +765,7 @@ class DSNP(Service): res = self.session.post(endpoint, headers=headers, data=challenge) res.raise_for_status() except Exception as e: - self.log.error(f" - License request failed: {e}", exc_info=False) + self.log.error(f"License request failed: {e}", exc_info=False) sys.exit(1) return res.content @@ -753,7 +811,7 @@ class DSNP(Service): data = self._request("GET", endpoint, params={'limit': 999})["data"]["season"]["items"] return data - def _get_video(self, scenario: str, payload: dict) -> dict: + def _get_playback(self, scenario: str, payload: dict) -> dict: endpoint = self._href( self.prod_config["services"]["media"]["client"]["endpoints"]["mediaPayload"]["href"], scenario=scenario @@ -765,7 +823,7 @@ class DSNP(Service): data = self._request("POST", endpoint, headers=headers, payload=payload) return data["stream"] - def _register_device(self) -> str: + def _register_device(self, android_id: str, drm_id: str) -> str: endpoint = self.prod_config["services"]["orchestration"]["client"]["endpoints"]["registerDevice"]["href"] headers = { "Authorization": self.config["bamsdk"]["api_key"], @@ -776,6 +834,16 @@ class DSNP(Service): "registerDevice": { "applicationRuntime": self.config["device"]["applicationRuntime"], "attributes": { + "osDeviceIds": [ + { + "identifier": android_id, + "type": "android.vendor.id" + }, + { + "identifier": drm_id, + "type": "android.drm.id" + } + ], "operatingSystem": self.config["device"]["operatingSystem"], "operatingSystemVersion": self.config["device"]["operatingSystemVersion"] }, @@ -797,7 +865,7 @@ class DSNP(Service): "X-BAMSDK-Platform-Id": self.config["device"]["platform_id"] } payload = { - "operationName": "Check", + "operationName": "check", "variables": { "email": email }, @@ -805,7 +873,7 @@ class DSNP(Service): } data = self._request("POST", endpoint, payload=payload, headers=headers) return data["data"]["check"]["operations"][0] - + def _login_with_password(self, email: str, password: str, token: str) -> str: endpoint = self.prod_config["services"]["orchestration"]["client"]["endpoints"]["query"]["href"] headers = { @@ -813,25 +881,88 @@ class DSNP(Service): "X-BAMSDK-Platform-Id": self.config["device"]["platform_id"] } payload = { - "operationName": "loginTv", + "operationName": "login", "variables": { "input": { "email": email, "password": password - } + }, + "includeIdentity": True, + "includeAccountConsentToken": True }, "query": queries.LOGIN } data = self._request("POST", endpoint, payload=payload, headers=headers) - return data["extensions"]["sdk"]["token"] + return data["extensions"]["sdk"] + + def _request_otp(self, email: str, token: str) -> dict: + endpoint = self.prod_config["services"]["orchestration"]["client"]["endpoints"]["query"]["href"] + headers = { + "Authorization": token, + "X-BAMSDK-Platform-Id": self.config["device"]["platform_id"] + } + payload = { + "operationName": "requestOtp", + "variables": { + "input": { + "email": email, + "reason": "Login" + } + }, + "query": queries.REQUESET_OTP + } + data = self._request("POST", endpoint, payload=payload, headers=headers) + if not data["data"]["requestOtp"]["accepted"]: + self.log.error(" - OTP code request failed.", exc_info=False) + sys.exit(1) - def _get_account_info_raw(self, headers: dict = {}) -> dict: + def _auth_action_with_otp(self, email: str, otp: str, token: str) -> dict: + endpoint = self.prod_config["services"]["orchestration"]["client"]["endpoints"]["query"]["href"] + headers = { + "Authorization": token, + "X-BAMSDK-Platform-Id": self.config["device"]["platform_id"] + } + payload = { + "operationName": "authenticateWithOtp", + "variables": { + "input": { + "email": email, + "passcode": otp + } + }, + "query": queries.LOGIN_OTP + } + data = self._request("POST", endpoint, payload=payload, headers=headers) + return data["data"]["authenticateWithOtp"]["actionGrant"] + + def _login_with_auth_action(self, auth_action: str, token: str) -> dict: + endpoint = self.prod_config["services"]["orchestration"]["client"]["endpoints"]["query"]["href"] + headers = { + "Authorization": token, + "X-BAMSDK-Platform-Id": self.config["device"]["platform_id"] + } + payload = { + "operationName": "loginWithActionGrant", + "variables": { + "input": { + "actionGrant": auth_action + }, + "includeAccountConsentToken": True + }, + "query": queries.LOGIN_ACTION_GRANT + } + data = self._request("POST", endpoint, payload=payload, headers=headers) + return data["extensions"]["sdk"] + + def _get_account_info(self, headers: dict = {}) -> dict: endpoint = self.prod_config["services"]["orchestration"]["client"]["endpoints"]["query"]["href"] headers.update({"X-BAMSDK-Platform-Id": self.config["device"]["platform_id"]}) payload = { - "operationName": "EntitledGraphMeQuery", - "variables": {}, - "query": queries.ENTITLEMENTS + "operationName": "me", + "variables": { + "includeAccountConsentToken": True + }, + "query": queries.ME } data = self._request("POST", endpoint, payload=payload, headers=headers) return data["data"]["me"] @@ -845,7 +976,9 @@ class DSNP(Service): payload = { "operationName": "switchProfile", "variables": { - "input": profile_input + "input": profile_input, + "includeIdentity": True, + "includeAccountConsentToken": True }, "query": queries.SWITCH_PROFILE } @@ -861,7 +994,7 @@ class DSNP(Service): payload = { "operationName": "refreshToken", "variables": { - "input": { + "refreshToken": { "refreshToken": refresh_token } }, @@ -870,7 +1003,7 @@ class DSNP(Service): data = self._request("POST", endpoint, payload=payload, headers=headers) return data["extensions"]["sdk"] - def _update_device(self) -> str: + def _update_device(self, android_id: str, drm_id: str) -> str: endpoint = self.prod_config["services"]["orchestration"]["client"]["endpoints"]["query"]["href"] headers = {"X-BAMSDK-Platform-Id": self.config["device"]["platform_id"]} payload = { @@ -878,7 +1011,17 @@ class DSNP(Service): "variables": { "updateDeviceOperatingSystem": { "operatingSystem": self.config["device"]["operatingSystem"], - "operatingSystemVersion": self.config["device"]["operatingSystemVersion"] + "operatingSystemVersion": self.config["device"]["operatingSystemVersion"], + "osDeviceIds": [ + { + "identifier": android_id, + "type": "android.vendor.id" + }, + { + "identifier": drm_id, + "type": "android.drm.id" + } + ] } }, "query": queries.UPDATE_DEVICE @@ -906,10 +1049,10 @@ class DSNP(Service): data = self._request("POST", endpoint, payload=payload, headers=headers) if data["data"]["updateProfileImaxEnhancedVersion"]["accepted"]: - self.log.info(f" + Updated IMAX Enhanced preference: {enabled}") + self.log.info(f" + Updated IMAX Enhanced preference: {enabled}") return data["extensions"]["sdk"] else: - self.log.warning(" - Failed to set IMAX preference.") + self.log.warning(" - Failed to set IMAX preference.") def _set_remastered_ar_preference(self, enabled: bool) -> str: endpoint = self.prod_config["services"]["orchestration"]["client"]["endpoints"]["query"]["href"] @@ -927,10 +1070,10 @@ class DSNP(Service): data = self._request("POST", endpoint, payload=payload, headers=headers) if data["data"]["updateProfileRemasteredAspectRatio"]["accepted"]: - self.log.info(f" + Updated Remastered Aspect Ratio preference: {enabled}") + self.log.info(f" + Updated Remastered Aspect Ratio preference: {enabled}") return data["extensions"]["sdk"] else: - self.log.warning(" - Failed to set Remastered Aspect Ratio preference.") + self.log.warning(" - Failed to set Remastered Aspect Ratio preference.") def _href(self, href: str, **kwargs: Any) -> str: _args = {"version": self.config["bamsdk"]["explore_version"]} @@ -947,7 +1090,6 @@ class DSNP(Service): req = Request(method, endpoint, headers=_headers, params=params, json=payload) prepped = self.session.prepare_request(req) - try: res = self.session.send(prepped) res.raise_for_status() @@ -956,13 +1098,15 @@ class DSNP(Service): error_code = data["errors"][0]["extensions"]["code"] if "token.service.invalid.grant" in error_code: raise ConnectionError(f"Refresh Token Expired: {error_code}") - if "token.service.unauthorized.client" in error_code: + elif "token.service.unauthorized.client" in error_code: raise ConnectionError(f"Unauthorized Client/IP: {error_code}") elif "idp.error.identity.bad-credentials" in error_code: raise ConnectionError(f"Bad Credentials: {error_code}") elif "account.profile.pin.invalid" in error_code: raise ConnectionError(f"Invalid PIN: {error_code}") raise ConnectionError(data["errors"]) + if data.get("data") and data["data"].get("errors"): + raise ConnectionError(data["data"]["errors"]) return data except Exception as e: if "Refresh Token Expired" in str(e) or "/deeplink" in endpoint: diff --git a/packages/envied/src/envied/services/DSNP/config.yaml b/packages/envied/src/envied/services/DSNP/config.yaml index 5def0ba..9540ddc 100644 --- a/packages/envied/src/envied/services/DSNP/config.yaml +++ b/packages/envied/src/envied/services/DSNP/config.yaml @@ -10,16 +10,16 @@ certificate: | ## config ( {configVersion}/{clientId}/{deviceFamily}/{sdkVersion}/{applicationRuntime}/{deviceProfile}/{environment} ) ## # Browser (windows, chrome) : /browser/v34.2/windows/chrome/prod.json -# Android Phone : /android/v12.0.0/google/handset/prod.json -# Android TV : /android/v12.0.0/google/tv/prod.json -# Amazon Fire TV : /android/v12.0.0/amazon/tv/prod.json +# Android Phone : /android/v15.0.0/google/handset/prod.json +# Android TV : /android/v15.0.0/google/tv/prod.json +# Amazon Fire TV : /android/v15.0.0/amazon/tv/prod.json endpoints: - config: "https://client-sdk-configs.bamgrid.com/bam-sdk/v7.0/disney-svod-3d9324fc/android/v13.0.0/google/tv/prod.json" + config: "https://client-sdk-configs.bamgrid.com/bam-sdk/v7.0/disney-svod-3d9324fc/android/v15.0.0/google/tv/prod.json" -## user_agent ## -# android-phone : BAMSDK/v13.3.0 (disney-svod-3d9324fc 4.18.1+rc6-2025.11.05.0; v7.0/v13.0.0; android; phone) -# android-tv : BAMSDK/v13.3.0 (disney-svod-3d9324fc 4.18.1+rc6-2025.11.05.0; v7.0/v13.0.0; android; tv) +## user_agent (okhttp/5.0.0-alpha.14) ## +# android-phone : BAMSDK/v15.0.1 (disney-svod-3d9324fc 4.21.1+rc3-2026.01.06.0; v7.0/v15.0.0; android; phone) +# android-tv : BAMSDK/v15.0.1 (disney-svod-3d9324fc 4.21.1+rc3-2026.01.06.0; v7.0/v15.0.0; android; tv) ## api_key ## # browser : ZGlzbmV5JmJyb3dzZXImMS4wLjA.Cu56AgSfBTDag5NiRA81oLHkDZfu5L3CKadnefEAY84 @@ -30,11 +30,11 @@ endpoints: # android : 624b805dafc5c73635b1a216 bamsdk: - sdk_version: "13.3.0" - application_version: "4.18.1+rc6-2025.11.05.0" - explore_version: "v1.11" + sdk_version: "15.0.1" + application_version: "4.21.1+rc3-2026.01.06.0" + explore_version: "v1.13" client: "disney-svod-3d9324fc" - user_agent: "BAMSDK/v13.3.0 (disney-svod-3d9324fc 4.18.1+rc6-2025.11.05.0; v7.0/v13.0.0; android; tv)" + user_agent: "BAMSDK/v15.0.1 (disney-svod-3d9324fc 4.21.1+rc3-2026.01.06.0; v7.0/v15.0.0; android; tv)" api_key: "ZGlzbmV5JmFuZHJvaWQmMS4wLjA.bkeb0m230uUhv8qrAXuNu39tbE_mD5EEhM_NAcohjyA" yp_service_id: "624b805dafc5c73635b1a216" @@ -46,7 +46,9 @@ device: applicationRuntime: "android" operatingSystem: "Android" operatingSystemVersion: "16" - deviceLanguage: "ko" # en + deviceLanguage: "ko" # Device language data independent of account language data -profile: - index: 0 +# Specifies the index of the profile to use. (0 = first profile, 1 = second profile, etc.) +# Automatically select a profile when commenting. +# profile: +# index: 0 diff --git a/packages/envied/src/envied/services/DSNP/queries.py b/packages/envied/src/envied/services/DSNP/queries.py index 975395a..f8ec5ba 100644 --- a/packages/envied/src/envied/services/DSNP/queries.py +++ b/packages/envied/src/envied/services/DSNP/queries.py @@ -1,13 +1,13 @@ -SWITCH_PROFILE = """mutation switchProfile($input: SwitchProfileInput!) { switchProfile(switchProfile: $input) { __typename account { __typename ...accountGraphFragment } activeSession { __typename ...sessionGraphFragment } } } fragment accountGraphFragment on Account { __typename id activeProfile { __typename id } profiles { __typename ...profileGraphFragment } parentalControls { __typename isProfileCreationProtected } flows { __typename star { __typename isOnboarded } } attributes { __typename email emailVerified userVerified locations { __typename manual { __typename country } purchase { __typename country } registration { __typename geoIp { __typename country } } } } } fragment profileGraphFragment on Profile { __typename id name maturityRating { __typename ratingSystem ratingSystemValues contentMaturityRating maxRatingSystemValue isMaxContentMaturityRating } isAge21Verified flows { __typename star { __typename eligibleForOnboarding isOnboarded } } attributes { __typename isDefault kidsModeEnabled groupWatch { __typename enabled } languagePreferences { __typename appLanguage playbackLanguage preferAudioDescription preferSDH subtitleLanguage subtitlesEnabled } parentalControls { __typename isPinProtected kidProofExitEnabled liveAndUnratedContent { __typename enabled } } playbackSettings { __typename autoplay backgroundVideo prefer133 } avatar { __typename id userSelected } } } fragment sessionGraphFragment on Session { __typename sessionId device { __typename id } entitlements experiments { __typename featureId variantId version } homeLocation { __typename countryCode } inSupportedLocation isSubscriber location { __typename countryCode } portabilityLocation { __typename countryCode } preferredMaturityRating { __typename impliedMaturityRating ratingSystem } }""" -ENTITLEMENTS = """query EntitledGraphMeQuery { me { __typename account { __typename ...accountGraphFragment } activeSession { __typename ...sessionGraphFragment } } } fragment accountGraphFragment on Account { __typename id activeProfile { __typename id } profiles { __typename ...profileGraphFragment } parentalControls { __typename isProfileCreationProtected } flows { __typename star { __typename isOnboarded } } attributes { __typename email emailVerified userVerified locations { __typename manual { __typename country } purchase { __typename country } registration { __typename geoIp { __typename country } } } } } fragment profileGraphFragment on Profile { __typename id name maturityRating { __typename ratingSystem ratingSystemValues contentMaturityRating maxRatingSystemValue isMaxContentMaturityRating } isAge21Verified flows { __typename star { __typename eligibleForOnboarding isOnboarded } } attributes { __typename isDefault kidsModeEnabled groupWatch { __typename enabled } languagePreferences { __typename appLanguage playbackLanguage preferAudioDescription preferSDH subtitleLanguage subtitlesEnabled } parentalControls { __typename isPinProtected kidProofExitEnabled liveAndUnratedContent { __typename enabled } } playbackSettings { __typename autoplay backgroundVideo prefer133 preferImaxEnhancedVersion} avatar { __typename id userSelected } } } fragment sessionGraphFragment on Session { __typename sessionId device { __typename id } entitlements experiments { __typename featureId variantId version } homeLocation { __typename countryCode } inSupportedLocation isSubscriber location { __typename countryCode } portabilityLocation { __typename countryCode } preferredMaturityRating { __typename impliedMaturityRating ratingSystem } }""" -REGISTER_DEVICE = """mutation ($registerDevice: RegisterDeviceInput!) {registerDevice(registerDevice: $registerDevice) {__typename}}""" -SET_IMAX = """mutation updateProfileImaxEnhancedVersion($input: UpdateProfileImaxEnhancedVersionInput!, $includeProfile: Boolean!) { updateProfileImaxEnhancedVersion(updateProfileImaxEnhancedVersion: $input) { accepted profile @include(if: $includeProfile) { __typename ...profileGraphFragment } } } fragment profileGraphFragment on Profile { id name personalInfo { dateOfBirth gender } maturityRating { ratingSystem ratingSystemValues contentMaturityRating maxRatingSystemValue isMaxContentMaturityRating suggestedMaturityRatings { minimumAge maximumAge ratingSystemValue } } isAge21Verified flows { star { eligibleForOnboarding isOnboarded } personalInfo { eligibleForCollection requiresCollection } } attributes { isDefault kidsModeEnabled languagePreferences { appLanguage playbackLanguage preferAudioDescription preferSDH subtitleLanguage subtitlesEnabled } parentalControls { isPinProtected kidProofExitEnabled liveAndUnratedContent { enabled available } } playbackSettings { autoplay backgroundVideo prefer133 preferImaxEnhancedVersion } avatar { id userSelected } privacySettings { consents { consentType value } } } }""" -SET_REMASTERED_AR = """mutation updateProfileRemasteredAspectRatio($input: UpdateProfileRemasteredAspectRatioInput!, $includeProfile: Boolean!) { updateProfileRemasteredAspectRatio(updateProfileRemasteredAspectRatio: $input) { accepted profile @include(if: $includeProfile) { __typename ...profileGraphFragment } } } fragment profileGraphFragment on Profile { id name personalInfo { dateOfBirth gender } maturityRating { ratingSystem ratingSystemValues contentMaturityRating maxRatingSystemValue isMaxContentMaturityRating suggestedMaturityRatings { minimumAge maximumAge ratingSystemValue } } isAge21Verified flows { star { eligibleForOnboarding isOnboarded } personalInfo { eligibleForCollection requiresCollection } } attributes { isDefault kidsModeEnabled languagePreferences { appLanguage playbackLanguage preferAudioDescription preferSDH subtitleLanguage subtitlesEnabled } parentalControls { isPinProtected kidProofExitEnabled liveAndUnratedContent { enabled available } } playbackSettings { autoplay backgroundVideo prefer133 preferImaxEnhancedVersion } avatar { id userSelected } privacySettings { consents { consentType value } } } }""" -# REQUEST_DEVICE_CODE = ("""mutation requestLicensePlate {requestLicensePlate {__typename licensePlate expirationTime expiresInSeconds}}""") -CHECK_EMAIL = """query Check($email: String!) { check(email: $email) { operations nextOperation } }""" -# REQUESET_OTP = """mutation requestOtp($input: RequestOtpInput!) { requestOtp(requestOtp: $input) { accepted } }""" -LOGIN = """mutation loginTv($input: LoginInput!) { login(login: $input) { __typename account { __typename ...accountGraphFragment } actionGrant activeSession { __typename ...sessionGraphFragment } }} fragment accountGraphFragment on Account { __typename id activeProfile { __typename id } profiles { __typename ...profileGraphFragment } parentalControls { __typename isProfileCreationProtected } flows { __typename star { __typename isOnboarded } } attributes { __typename email emailVerified userVerified locations { __typename manual { __typename country } purchase { __typename country } registration { __typename geoIp { __typename country } } } }}\nfragment profileGraphFragment on Profile { __typename id name maturityRating { __typename ratingSystem ratingSystemValues contentMaturityRating maxRatingSystemValue isMaxContentMaturityRating } isAge21Verified flows { __typename star { __typename eligibleForOnboarding isOnboarded } } attributes { __typename isDefault kidsModeEnabled groupWatch { __typename enabled } languagePreferences { __typename appLanguage playbackLanguage preferAudioDescription preferSDH subtitleLanguage subtitlesEnabled } parentalControls { __typename isPinProtected kidProofExitEnabled liveAndUnratedContent { __typename enabled } } playbackSettings { __typename autoplay backgroundVideo prefer133 } avatar { __typename id userSelected } }}\nfragment sessionGraphFragment on Session { __typename sessionId device { __typename id } entitlements experiments { __typename featureId variantId version } homeLocation { __typename countryCode } inSupportedLocation isSubscriber location { __typename countryCode } portabilityLocation { __typename countryCode } preferredMaturityRating { __typename impliedMaturityRating ratingSystem }}""" -# LOGIN_OTP = """mutation authenticateWithOtp($input: AuthenticateWithOtpInput!) { authenticateWithOtp(authenticateWithOtp: $input) { actionGrant securityAction } }""" -# LOGIN_ACTION_GRANT = """\n mutation loginWithActionGrant($input: LoginWithActionGrantInput!) {\n loginWithActionGrant(login: $input) {\n account {\n ...account\n\n profiles {\n ...profile\n }\n }\n activeSession {\n ...session\n }\n identity {\n ...identity\n }\n }\n }\n\n \nfragment identity on Identity {\n attributes {\n securityFlagged\n createdAt\n passwordResetRequired\n }\n flows {\n marketingPreferences {\n eligibleForOnboarding\n isOnboarded\n }\n personalInfo {\n eligibleForCollection\n requiresCollection\n }\n }\n personalInfo {\n dateOfBirth\n gender\n }\n subscriber {\n subscriberStatus\n subscriptionAtRisk\n overlappingSubscription\n doubleBilled\n doubleBilledProviders\n subscriptions {\n id\n groupId\n state\n partner\n isEntitled\n source {\n sourceType\n sourceProvider\n sourceRef\n subType\n }\n paymentProvider\n product {\n id\n sku\n offerId\n promotionId\n name\n nextPhase {\n sku\n offerId\n campaignCode\n voucherCode\n }\n entitlements {\n id\n name\n desc\n partner\n }\n categoryCodes\n redeemed {\n campaignCode\n redemptionCode\n voucherCode\n }\n bundle\n bundleType\n subscriptionPeriod\n earlyAccess\n trial {\n duration\n }\n }\n term {\n purchaseDate\n startDate\n expiryDate\n nextRenewalDate\n pausedDate\n churnedDate\n isFreeTrial\n }\n externalSubscriptionId,\n cancellation {\n type\n restartEligible\n }\n stacking {\n status\n overlappingSubscriptionProviders\n previouslyStacked\n previouslyStackedByProvider\n }\n }\n }\n}\n\n \nfragment account on Account {\n id\n attributes {\n blocks {\n expiry\n reason\n }\n consentPreferences {\n dataElements {\n name\n value\n }\n purposes {\n consentDate\n firstTransactionDate\n id\n lastTransactionCollectionPointId\n lastTransactionCollectionPointVersion\n lastTransactionDate\n name\n status\n totalTransactionCount\n version\n }\n }\n dssIdentityCreatedAt\n email\n emailVerified\n lastSecurityFlaggedAt\n locations {\n manual {\n country\n }\n purchase {\n country\n source\n }\n registration {\n geoIp {\n country\n }\n }\n }\n securityFlagged\n tags\n taxId\n userVerified\n }\n parentalControls {\n isProfileCreationProtected\n }\n flows {\n star {\n isOnboarded\n }\n }\n}\n\n \nfragment profile on Profile {\n id\n name\n isAge21Verified\n attributes {\n avatar {\n id\n userSelected\n }\n isDefault\n kidsModeEnabled\n languagePreferences {\n appLanguage\n playbackLanguage\n preferAudioDescription\n preferSDH\n subtitleAppearance {\n backgroundColor\n backgroundOpacity\n description\n font\n size\n textColor\n }\n subtitleLanguage\n subtitlesEnabled\n }\n groupWatch {\n enabled\n }\n parentalControls {\n kidProofExitEnabled\n isPinProtected\n }\n playbackSettings {\n autoplay\n backgroundVideo\n prefer133\n preferImaxEnhancedVersion\n previewAudioOnHome\n previewVideoOnHome\n }\n }\n personalInfo {\n dateOfBirth\n gender\n age\n }\n maturityRating {\n ...maturityRating\n }\n personalInfo {\n dateOfBirth\n age\n gender\n }\n flows {\n personalInfo {\n eligibleForCollection\n requiresCollection\n }\n star {\n eligibleForOnboarding\n isOnboarded\n }\n }\n}\n\n\nfragment maturityRating on MaturityRating {\n ratingSystem\n ratingSystemValues\n contentMaturityRating\n maxRatingSystemValue\n isMaxContentMaturityRating\n}\n\n\n \nfragment session on Session {\n device {\n id\n platform\n }\n entitlements\n features {\n coPlay\n }\n inSupportedLocation\n isSubscriber\n location {\n type\n countryCode\n dma\n asn\n regionName\n connectionType\n zipCode\n }\n sessionId\n experiments {\n featureId\n variantId\n version\n }\n identity {\n id\n }\n account {\n id\n }\n profile {\n id\n parentalControls {\n liveAndUnratedContent {\n enabled\n }\n }\n }\n partnerName\n preferredMaturityRating {\n impliedMaturityRating\n ratingSystem\n }\n homeLocation {\n countryCode\n }\n portabilityLocation {\n countryCode\n type\n }\n}\n\n""" -REFRESH_TOKEN = """mutation refreshToken($input:RefreshTokenInput!) { refreshToken(refreshToken:$input) { activeSession{sessionId} } }""" +# REQUEST_DEVICE_CODE = """mutation requestLicensePlate($input: RequestLicensePlateInput!) { requestLicensePlate(requestLicensePlate: $input) { licensePlate expirationTime expiresInSeconds } }""" +CHECK_EMAIL = """query check($email: String!) { check(email: $email) { operations nextOperation } }""" +LOGIN = """mutation login($input: LoginInput!, $includeIdentity: Boolean!, $includeAccountConsentToken: Boolean!) { login(login: $input) { account { __typename ...accountGraphFragment } actionGrant activeSession { __typename ...sessionGraphFragment } identity @include(if: $includeIdentity) { __typename ...identityGraphFragment } } } fragment profileGraphFragment on Profile { id name personalInfo { dateOfBirth gender } maturityRating { ratingSystem ratingSystemValues contentMaturityRating maxRatingSystemValue isMaxContentMaturityRating suggestedMaturityRatings { minimumAge maximumAge ratingSystemValue } } isAge21Verified flows { star { eligibleForOnboarding isOnboarded } personalInfo { eligibleForCollection requiresCollection } } attributes { isDefault kidsModeEnabled languagePreferences { appLanguage playbackLanguage preferAudioDescription preferSDH subtitleLanguage subtitlesEnabled } parentalControls { isPinProtected kidProofExitEnabled liveAndUnratedContent { enabled available } } playbackSettings { autoplay backgroundVideo prefer133 preferImaxEnhancedVersion } avatar { id userSelected } privacySettings { consents { consentType value } } } } fragment accountGraphFragment on Account { id umpMessages { data { messages { messageId messageSource displayLocations content } } } accountConsentToken @include(if: $includeAccountConsentToken) activeProfile { id umpMessages { data { messages { messageId content } } } } profiles { __typename ...profileGraphFragment } profileRequirements { primaryProfiles { personalInfo { requiresCollection } } secondaryProfiles { personalInfo { requiresCollection } personalInfoJrMode { requiresCollection } } } parentalControls { isProfileCreationProtected } flows { star { isOnboarded } } attributes { email emailVerified userVerified maxNumberOfProfilesAllowed locations { manual { country } purchase { country } registration { geoIp { country } } } } } fragment sessionGraphFragment on Session { sessionId device { id } entitlements experiments { featureId variantId version } features { coPlay download noAds } homeLocation { countryCode adsSupported } inSupportedLocation isSubscriber location { countryCode adsSupported } portabilityLocation { countryCode } preferredMaturityRating { impliedMaturityRating ratingSystem } } fragment identityGraphFragment on Identity { id email repromptSubscriberAgreement attributes { passwordResetRequired } commerce { notifications { subscriptionId type showNotification offerData { productType expectedTransition { date price { amount currency } } cypherKeys { key value type } } currentOffer { offerId price { amount currency frequency } } } } flows { marketingPreferences { isOnboarded eligibleForOnboarding } personalInfo { eligibleForCollection requiresCollection } } personalInfo { dateOfBirth gender } locations { purchase { country } } subscriber { subscriberStatus subscriptionAtRisk overlappingSubscription doubleBilled doubleBilledProviders subscriptions { id groupId state partner isEntitled source { sourceProvider sourceType subType sourceRef } product { id sku name entitlements { id name partner } bundle subscriptionPeriod earlyAccess trial { duration } categoryCodes } stacking { status overlappingSubscriptionProviders previouslyStacked previouslyStackedByProvider } term { purchaseDate startDate expiryDate nextRenewalDate pausedDate churnedDate isFreeTrial } } } consent { id idType token } }""" +LOGIN_ACTION_GRANT = """mutation loginWithActionGrant($input: LoginWithActionGrantInput!, $includeAccountConsentToken: Boolean!) { loginWithActionGrant(login: $input) { account { __typename ...accountGraphFragment } activeSession { __typename ...sessionGraphFragment } identity { __typename ...identityGraphFragment } actionGrant } } fragment profileGraphFragment on Profile { id name personalInfo { dateOfBirth gender } maturityRating { ratingSystem ratingSystemValues contentMaturityRating maxRatingSystemValue isMaxContentMaturityRating suggestedMaturityRatings { minimumAge maximumAge ratingSystemValue } } isAge21Verified flows { star { eligibleForOnboarding isOnboarded } personalInfo { eligibleForCollection requiresCollection } } attributes { isDefault kidsModeEnabled languagePreferences { appLanguage playbackLanguage preferAudioDescription preferSDH subtitleLanguage subtitlesEnabled } parentalControls { isPinProtected kidProofExitEnabled liveAndUnratedContent { enabled available } } playbackSettings { autoplay backgroundVideo backgroundAudio prefer133 preferImaxEnhancedVersion } avatar { id userSelected } privacySettings { consents { consentType value } } } } fragment accountGraphFragment on Account { id umpMessages { data { messages { messageId messageSource displayLocations content } } } accountConsentToken @include(if: $includeAccountConsentToken) activeProfile { id umpMessages { data { messages { messageId content } } } } profiles { __typename ...profileGraphFragment } profileRequirements { primaryProfiles { personalInfo { requiresCollection } } secondaryProfiles { personalInfo { requiresCollection } personalInfoJrMode { requiresCollection } } } parentalControls { isProfileCreationProtected } flows { star { isOnboarded } } attributes { email emailVerified userVerified maxNumberOfProfilesAllowed locations { manual { country } purchase { country } registration { geoIp { country } } } } } fragment sessionGraphFragment on Session { sessionId device { id } entitlements experiments { featureId variantId version } features { coPlay download noAds } homeLocation { countryCode adsSupported } inSupportedLocation isSubscriber location { countryCode adsSupported } portabilityLocation { countryCode } preferredMaturityRating { impliedMaturityRating ratingSystem } } fragment identityGraphFragment on Identity { id email repromptSubscriberAgreement attributes { passwordResetRequired } commerce { notifications { subscriptionId type showNotification offerData { productType expectedTransition { date price { amount currency } } cypherKeys { key value type } } currentOffer { offerId price { amount currency frequency } } } } flows { marketingPreferences { isOnboarded eligibleForOnboarding } personalInfo { eligibleForCollection requiresCollection } } personalInfo { dateOfBirth gender } locations { purchase { country } } subscriber { subscriberStatus subscriptionAtRisk overlappingSubscription doubleBilled doubleBilledProviders subscriptions { id groupId state partner isEntitled source { sourceProvider sourceType subType sourceRef } product { id sku name entitlements { id name partner } bundle subscriptionPeriod earlyAccess trial { duration } categoryCodes } stacking { status overlappingSubscriptionProviders previouslyStacked previouslyStackedByProvider } term { purchaseDate startDate expiryDate nextRenewalDate pausedDate churnedDate isFreeTrial } } } consent { id idType token } }""" +LOGIN_OTP = """mutation authenticateWithOtp($input: AuthenticateWithOtpInput!) { authenticateWithOtp(authenticateWithOtp: $input) { actionGrant securityAction passwordRules { __typename ...passwordRulesFragment } } } fragment passwordRulesFragment on PasswordRules { minLength charTypes }""" +ME = """query me($includeAccountConsentToken: Boolean!) { me { account { __typename ...accountGraphFragment } activeSession { __typename ...sessionGraphFragment } identity { __typename ...identityGraphFragment } } } fragment profileGraphFragment on Profile { id name personalInfo { dateOfBirth gender } maturityRating { ratingSystem ratingSystemValues contentMaturityRating maxRatingSystemValue isMaxContentMaturityRating suggestedMaturityRatings { minimumAge maximumAge ratingSystemValue } } isAge21Verified flows { star { eligibleForOnboarding isOnboarded } personalInfo { eligibleForCollection requiresCollection } } attributes { isDefault kidsModeEnabled languagePreferences { appLanguage playbackLanguage preferAudioDescription preferSDH subtitleLanguage subtitlesEnabled } parentalControls { isPinProtected kidProofExitEnabled liveAndUnratedContent { enabled available } } playbackSettings { autoplay backgroundVideo backgroundAudio prefer133 preferImaxEnhancedVersion } avatar { id userSelected } privacySettings { consents { consentType value } } } } fragment accountGraphFragment on Account { id umpMessages { data { messages { messageId messageSource displayLocations content } } } accountConsentToken @include(if: $includeAccountConsentToken) activeProfile { id umpMessages { data { messages { messageId content } } } } profiles { __typename ...profileGraphFragment } profileRequirements { primaryProfiles { personalInfo { requiresCollection } } secondaryProfiles { personalInfo { requiresCollection } personalInfoJrMode { requiresCollection } } } parentalControls { isProfileCreationProtected } flows { star { isOnboarded } } attributes { email emailVerified userVerified maxNumberOfProfilesAllowed locations { manual { country } purchase { country } registration { geoIp { country } } } } } fragment sessionGraphFragment on Session { sessionId device { id } entitlements experiments { featureId variantId version } features { coPlay download noAds } homeLocation { countryCode adsSupported } inSupportedLocation isSubscriber location { countryCode adsSupported } portabilityLocation { countryCode } preferredMaturityRating { impliedMaturityRating ratingSystem } } fragment identityGraphFragment on Identity { id email repromptSubscriberAgreement attributes { passwordResetRequired } commerce { notifications { subscriptionId type showNotification offerData { productType expectedTransition { date price { amount currency } } cypherKeys { key value type } } currentOffer { offerId price { amount currency frequency } } } } flows { marketingPreferences { isOnboarded eligibleForOnboarding } personalInfo { eligibleForCollection requiresCollection } } personalInfo { dateOfBirth gender } locations { purchase { country } } subscriber { subscriberStatus subscriptionAtRisk overlappingSubscription doubleBilled doubleBilledProviders subscriptions { id groupId state partner isEntitled source { sourceProvider sourceType subType sourceRef } product { id sku name entitlements { id name partner } bundle subscriptionPeriod earlyAccess trial { duration } categoryCodes } stacking { status overlappingSubscriptionProviders previouslyStacked previouslyStackedByProvider } term { purchaseDate startDate expiryDate nextRenewalDate pausedDate churnedDate isFreeTrial } } } consent { id idType token } }""" +REFRESH_TOKEN = """mutation refreshToken($refreshToken: RefreshTokenInput!) { refreshToken(refreshToken: $refreshToken) { activeSession { sessionId } } }""" +REGISTER_DEVICE = """mutation ($registerDevice: RegisterDeviceInput!) { registerDevice(registerDevice: $registerDevice) { __typename } }""" +REQUESET_OTP = """mutation requestOtp($input: RequestOtpInput!) { requestOtp(requestOtp: $input) { accepted } }""" +SET_IMAX = """mutation updateProfileImaxEnhancedVersion($input: UpdateProfileImaxEnhancedVersionInput!, $includeProfile: Boolean!) { updateProfileImaxEnhancedVersion(updateProfileImaxEnhancedVersion: $input) { accepted profile @include(if: $includeProfile) { __typename ...profileGraphFragment } } } fragment profileGraphFragment on Profile { id name personalInfo { dateOfBirth gender } maturityRating { ratingSystem ratingSystemValues contentMaturityRating maxRatingSystemValue isMaxContentMaturityRating suggestedMaturityRatings { minimumAge maximumAge ratingSystemValue } } isAge21Verified flows { star { eligibleForOnboarding isOnboarded } personalInfo { eligibleForCollection requiresCollection } } attributes { isDefault kidsModeEnabled languagePreferences { appLanguage playbackLanguage preferAudioDescription preferSDH subtitleLanguage subtitlesEnabled } parentalControls { isPinProtected kidProofExitEnabled liveAndUnratedContent { enabled available } } playbackSettings { autoplay backgroundVideo backgroundAudio prefer133 preferImaxEnhancedVersion } avatar { id userSelected } privacySettings { consents { consentType value } } } }""" +SET_REMASTERED_AR = """mutation updateProfileRemasteredAspectRatio($input: UpdateProfileRemasteredAspectRatioInput!, $includeProfile: Boolean!) { updateProfileRemasteredAspectRatio(updateProfileRemasteredAspectRatio: $input) { accepted profile @include(if: $includeProfile) { __typename ...profileGraphFragment } } } fragment profileGraphFragment on Profile { id name personalInfo { dateOfBirth gender } maturityRating { ratingSystem ratingSystemValues contentMaturityRating maxRatingSystemValue isMaxContentMaturityRating suggestedMaturityRatings { minimumAge maximumAge ratingSystemValue } } isAge21Verified flows { star { eligibleForOnboarding isOnboarded } personalInfo { eligibleForCollection requiresCollection } } attributes { isDefault kidsModeEnabled languagePreferences { appLanguage playbackLanguage preferAudioDescription preferSDH subtitleLanguage subtitlesEnabled } parentalControls { isPinProtected kidProofExitEnabled liveAndUnratedContent { enabled available } } playbackSettings { autoplay backgroundVideo backgroundAudio prefer133 preferImaxEnhancedVersion } avatar { id userSelected } privacySettings { consents { consentType value } } } }""" +SWITCH_PROFILE = """mutation switchProfile($input: SwitchProfileInput!, $includeIdentity: Boolean!, $includeAccountConsentToken: Boolean!) { switchProfile(switchProfile: $input) { account { __typename ...accountGraphFragment } activeSession { __typename ...sessionGraphFragment } identity @include(if: $includeIdentity) { __typename ...identityGraphFragment } } } fragment profileGraphFragment on Profile { id name personalInfo { dateOfBirth gender } maturityRating { ratingSystem ratingSystemValues contentMaturityRating maxRatingSystemValue isMaxContentMaturityRating suggestedMaturityRatings { minimumAge maximumAge ratingSystemValue } } isAge21Verified flows { star { eligibleForOnboarding isOnboarded } personalInfo { eligibleForCollection requiresCollection } } attributes { isDefault kidsModeEnabled languagePreferences { appLanguage playbackLanguage preferAudioDescription preferSDH subtitleLanguage subtitlesEnabled } parentalControls { isPinProtected kidProofExitEnabled liveAndUnratedContent { enabled available } } playbackSettings { autoplay backgroundVideo backgroundAudio prefer133 preferImaxEnhancedVersion } avatar { id userSelected } privacySettings { consents { consentType value } } } } fragment accountGraphFragment on Account { id umpMessages { data { messages { messageId messageSource displayLocations content } } } accountConsentToken @include(if: $includeAccountConsentToken) activeProfile { id umpMessages { data { messages { messageId content } } } } profiles { __typename ...profileGraphFragment } profileRequirements { primaryProfiles { personalInfo { requiresCollection } } secondaryProfiles { personalInfo { requiresCollection } personalInfoJrMode { requiresCollection } } } parentalControls { isProfileCreationProtected } flows { star { isOnboarded } } attributes { email emailVerified userVerified maxNumberOfProfilesAllowed locations { manual { country } purchase { country } registration { geoIp { country } } } } } fragment sessionGraphFragment on Session { sessionId device { id } entitlements experiments { featureId variantId version } features { coPlay download noAds } homeLocation { countryCode adsSupported } inSupportedLocation isSubscriber location { countryCode adsSupported } portabilityLocation { countryCode } preferredMaturityRating { impliedMaturityRating ratingSystem } } fragment identityGraphFragment on Identity { id email repromptSubscriberAgreement attributes { passwordResetRequired } commerce { notifications { subscriptionId type showNotification offerData { productType expectedTransition { date price { amount currency } } cypherKeys { key value type } } currentOffer { offerId price { amount currency frequency } } } } flows { marketingPreferences { isOnboarded eligibleForOnboarding } personalInfo { eligibleForCollection requiresCollection } } personalInfo { dateOfBirth gender } locations { purchase { country } } subscriber { subscriberStatus subscriptionAtRisk overlappingSubscription doubleBilled doubleBilledProviders subscriptions { id groupId state partner isEntitled source { sourceProvider sourceType subType sourceRef } product { id sku name entitlements { id name partner } bundle subscriptionPeriod earlyAccess trial { duration } categoryCodes } stacking { status overlappingSubscriptionProviders previouslyStacked previouslyStackedByProvider } term { purchaseDate startDate expiryDate nextRenewalDate pausedDate churnedDate isFreeTrial } } } consent { id idType token } }""" UPDATE_DEVICE = """mutation updateDeviceOperatingSystem($updateDeviceOperatingSystem: UpdateDeviceOperatingSystemInput!) {updateDeviceOperatingSystem(updateDeviceOperatingSystem: $updateDeviceOperatingSystem) {accepted}}""" \ No newline at end of file diff --git a/packages/vinefeeder/pyproject.toml b/packages/vinefeeder/pyproject.toml index f2b6601..e32eb51 100644 --- a/packages/vinefeeder/pyproject.toml +++ b/packages/vinefeeder/pyproject.toml @@ -10,6 +10,7 @@ dependencies = [ "PyQt6", # or GUI toolkit(s) "rich", "xmltodict>=1.0.2", + "beaupy>=3.10.2,<4", ] [project.scripts] diff --git a/uv.lock b/uv.lock index 2598187..cec7fe2 100644 --- a/uv.lock +++ b/uv.lock @@ -177,7 +177,7 @@ wheels = [ [[package]] name = "beaupy" -version = "3.10.2" +version = "3.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "emoji" }, @@ -185,9 +185,9 @@ dependencies = [ { name = "questo" }, { name = "rich" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d0/ce/81f6aa40d2c779a3f0a0ac11d130a9c14c6b4f53700cd279980f3b9ecdf0/beaupy-3.10.2.tar.gz", hash = "sha256:da1a644f71e60fddc599084a5ef8bd41118b9c779527ce501289729e60e7db90", size = 16535, upload-time = "2025-10-06T18:47:50.063Z" } +sdist = { url = "https://files.pythonhosted.org/packages/78/b2/1478a56b9669eda5fc6f9fed9e9aa47fbe73b564c5fb5cd55d719cf5b32e/beaupy-3.11.0.tar.gz", hash = "sha256:c264d13061c5741ceef5462b3752e572a0a7c3b8bd8dac870d10aec8a7d6b2ee", size = 16550, upload-time = "2026-01-08T14:03:33.794Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d6/bf/42536b458a7108a3b880bbfc99783e73b1c3b4c24d4ede3057bb22c44e07/beaupy-3.10.2-py3-none-any.whl", hash = "sha256:c5398416b6a9dfed25d56be4caf4b995c41c9d18aa132203ef69fa3d8953bcc5", size = 14842, upload-time = "2025-10-06T18:47:48.8Z" }, + { url = "https://files.pythonhosted.org/packages/56/0e/58181784f84f5944e2539e9605fdb1941bb2156c6382d5d11937a86128ec/beaupy-3.11.0-py3-none-any.whl", hash = "sha256:43dfa75f8a0ebb11c031426b481b27266f8623fbb28d695521ac6dc2c27479f0", size = 14860, upload-time = "2026-01-08T14:03:31.669Z" }, ] [[package]] @@ -552,12 +552,11 @@ wheels = [ [[package]] name = "envied" -version = "2.3.0" +version = "3.0.0" source = { editable = "packages/envied" } dependencies = [ { name = "aiohttp-swagger3" }, { name = "appdirs" }, - { name = "beaupy" }, { name = "brotli" }, { name = "chardet" }, { name = "click" }, @@ -570,11 +569,13 @@ dependencies = [ { name = "isodate" }, { name = "jsonpickle" }, { name = "langcodes" }, + { name = "language-data" }, { name = "lxml" }, { name = "mypy" }, { name = "pproxy" }, { name = "protobuf" }, { name = "pycaption" }, + { name = "pycountry" }, { name = "pycryptodomex" }, { name = "pyexecjs" }, { name = "pyjwt" }, @@ -595,6 +596,7 @@ dependencies = [ { name = "subtitle-filter" }, { name = "unidecode" }, { name = "urllib3" }, + { name = "wasmtime" }, { name = "webvtt-py" }, ] @@ -614,7 +616,6 @@ dev = [ requires-dist = [ { name = "aiohttp-swagger3", specifier = ">=0.10.0,<1" }, { name = "appdirs", specifier = ">=1.4.4,<2" }, - { name = "beaupy", specifier = ">=3.10.2,<4" }, { name = "brotli", specifier = ">=1.1.0,<2" }, { name = "chardet", specifier = ">=5.2.0,<6" }, { name = "click", specifier = ">=8.1.8,<9" }, @@ -627,11 +628,13 @@ requires-dist = [ { name = "isodate", specifier = ">=0.7.2,<0.8" }, { name = "jsonpickle", specifier = ">=4.1.1,<5" }, { name = "langcodes", specifier = ">=3.5.0,<4" }, + { name = "language-data", specifier = ">=1.4.0" }, { name = "lxml", specifier = ">=6.0.2,<7" }, { name = "mypy", specifier = ">=1.19.1,<2" }, { name = "pproxy", specifier = ">=2.7.9,<3" }, { name = "protobuf", specifier = ">=6.33.3,<7" }, { name = "pycaption", specifier = ">=2.2.6,<3" }, + { name = "pycountry", specifier = ">=24.6.1" }, { name = "pycryptodomex", specifier = ">=3.23.0,<4" }, { name = "pyexecjs", specifier = ">=1.5.1,<2" }, { name = "pyjwt", specifier = ">=2.10.1,<3" }, @@ -652,6 +655,7 @@ requires-dist = [ { name = "subtitle-filter", specifier = ">=1.4.9,<2" }, { name = "unidecode", specifier = ">=1.3.8,<2" }, { name = "urllib3", specifier = ">=2.6.2,<3" }, + { name = "wasmtime", specifier = ">=41.0.0" }, { name = "webvtt-py", specifier = ">=0.5.1,<0.6" }, ] @@ -961,14 +965,14 @@ wheels = [ [[package]] name = "language-data" -version = "1.3.0" +version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "marisa-trie" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/dd/ce/3f144716a9f2cbf42aa86ebc8b085a184be25c80aa453eea17c294d239c1/language_data-1.3.0.tar.gz", hash = "sha256:7600ef8aa39555145d06c89f0c324bf7dab834ea0b0a439d8243762e3ebad7ec", size = 5129310, upload-time = "2024-11-19T10:21:37.912Z" } +sdist = { url = "https://files.pythonhosted.org/packages/61/50/2518b4d0805f4d1f10166837829ad0bd71dcee3ec33fa84aa8c0db23c13c/language_data-1.4.0.tar.gz", hash = "sha256:800e6457e7beda781c156e02d7707e38db2ded026472e07e2c055dc8446ee574", size = 5309660, upload-time = "2025-11-28T13:45:25.217Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/e9/5a5ffd9b286db82be70d677d0a91e4d58f7912bb8dd026ddeeb4abe70679/language_data-1.3.0-py3-none-any.whl", hash = "sha256:e2ee943551b5ae5f89cd0e801d1fc3835bb0ef5b7e9c3a4e8e17b2b214548fbf", size = 5385760, upload-time = "2024-11-19T10:21:36.005Z" }, + { url = "https://files.pythonhosted.org/packages/ad/d1/68e2bcca94c9bbdc122a71e504e0b6a6c3e31541b1bad33fee0205996006/language_data-1.4.0-py3-none-any.whl", hash = "sha256:f741927c24ab14cbed2a57bc2bfe82b00cff266c427179597e8b14123364f084", size = 5572678, upload-time = "2025-11-28T13:45:23.381Z" }, ] [[package]] @@ -1466,6 +1470,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/c7/d13c57e5a3408df2e5d910853e957ec8e253b41ba531e0f32036c8321240/pycaption-2.2.19-py3-none-any.whl", hash = "sha256:7eb84a05d40bb80400689f9431d05d8b77dec6535938b419ebed2c9d67283a4f", size = 124970, upload-time = "2025-09-30T07:15:21.945Z" }, ] +[[package]] +name = "pycountry" +version = "24.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/57/c389fa68c50590881a75b7883eeb3dc15e9e73a0fdc001cdd45c13290c92/pycountry-24.6.1.tar.gz", hash = "sha256:b61b3faccea67f87d10c1f2b0fc0be714409e8fcdcc1315613174f6466c10221", size = 6043910, upload-time = "2024-06-01T04:12:15.05Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/ec/1fb891d8a2660716aadb2143235481d15ed1cbfe3ad669194690b0604492/pycountry-24.6.1-py3-none-any.whl", hash = "sha256:f1a4fb391cd7214f8eefd39556d740adcc233c778a27f8942c8dca351d6ce06f", size = 6335189, upload-time = "2024-06-01T04:11:49.711Z" }, +] + [[package]] name = "pycparser" version = "2.23" @@ -1788,15 +1801,15 @@ wheels = [ [[package]] name = "questo" -version = "0.4.1" +version = "0.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "python-yakh" }, { name = "rich" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1b/2a/ace6f441af4bf71a926183255ecf82dc2574c0203408ff466354f3c2256c/questo-0.4.1.tar.gz", hash = "sha256:efe4fb0ca68606555a5357d6424d4116081e9af7f10d8a6208039aa9b130d30f", size = 7418, upload-time = "2025-01-14T18:13:45.002Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a8/e6/4a86ccd5710c2fddae92786f15f2c69637b417905ff239d7424b55e342cb/questo-0.4.2.tar.gz", hash = "sha256:bcc917c2f1a65872e2bd02e79e226a716fd2d68a1ebe0d23c1828aa116aa6da6", size = 10474, upload-time = "2025-11-30T13:08:59.403Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/53/d5/71ad81de9c569b1b2e2baa5a60e85102c35fe898007de7ea2136c291b4ce/questo-0.4.1-py3-none-any.whl", hash = "sha256:57515abdec03e9b2ddcdfe7031e8b1742408fe61c9914b1395c0caa4f9abad7b", size = 10108, upload-time = "2025-01-14T18:13:43.926Z" }, + { url = "https://files.pythonhosted.org/packages/04/9e/2669ac7d718539a3783c3a6c440b3440b68193b59d763d7f89d33c0b1e8e/questo-0.4.2-py3-none-any.whl", hash = "sha256:1afce34a638601b54fbd697e99e7551813fafb513fe523165394eb12f9607b43", size = 12442, upload-time = "2025-11-30T13:08:57.602Z" }, ] [[package]] @@ -2154,6 +2167,7 @@ name = "vinefeeder" version = "0.2.0" source = { editable = "packages/vinefeeder" } dependencies = [ + { name = "beaupy" }, { name = "envied" }, { name = "pyqt6" }, { name = "rich" }, @@ -2162,6 +2176,7 @@ dependencies = [ [package.metadata] requires-dist = [ + { name = "beaupy", specifier = ">=3.10.2,<4" }, { name = "envied", editable = "packages/envied" }, { name = "pyqt6" }, { name = "rich" }, @@ -2192,6 +2207,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/58/dd/56f0d8af71e475ed194d702f8b4cf9cea812c95e82ad823d239023c6558c/w3lib-2.3.1-py3-none-any.whl", hash = "sha256:9ccd2ae10c8c41c7279cd8ad4fe65f834be894fe7bfdd7304b991fd69325847b", size = 21751, upload-time = "2025-01-27T14:22:09.421Z" }, ] +[[package]] +name = "wasmtime" +version = "41.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/68/6dc0e7156f883afe0129dd89e4031c8d1163131794ba6ce9e454a09168ad/wasmtime-41.0.0.tar.gz", hash = "sha256:fc2aaacf3ba794eac8baeb739939b2f7903e12d6b78edddc0b7f3ac3a9af6dfc", size = 117354, upload-time = "2026-01-20T18:18:00.565Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/f9/f6aef5de536d12652d97cf162f124cbdd642150c7da61ffa7863272cdab7/wasmtime-41.0.0-py3-none-android_26_arm64_v8a.whl", hash = "sha256:f5a6e237b5b94188ef9867926b447f779f540c729c92e4d91cc946f2bee7c282", size = 6837018, upload-time = "2026-01-20T18:17:41.489Z" }, + { url = "https://files.pythonhosted.org/packages/04/b9/42ec977972b2dcc8c61e3a40644d24d229b41fba151410644e44e35e6eb1/wasmtime-41.0.0-py3-none-android_26_x86_64.whl", hash = "sha256:4a3e33d0d3cf49062eaa231f748f54af991e89e9a795c5ab9d4f0eee85736e4c", size = 7654957, upload-time = "2026-01-20T18:17:43.285Z" }, + { url = "https://files.pythonhosted.org/packages/18/ca/6cce49b03c35c7fecb4437fd98990c64694a5e0024f9279bef0ddef000f7/wasmtime-41.0.0-py3-none-any.whl", hash = "sha256:5f6721406a6cd186d11f34e6d4991c4d536387b0c577d09a56bd93b8a3cf10c2", size = 6325757, upload-time = "2026-01-20T18:17:44.789Z" }, + { url = "https://files.pythonhosted.org/packages/a0/16/d91cb80322cc7ae10bfa5db8cea4e0b9bb112f0c100b4486783ab16c1c22/wasmtime-41.0.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:2107360212fce33ed2adcfc33b7e75ed7136380a17d3ed598a5bab376dcf9e1b", size = 7471888, upload-time = "2026-01-20T18:17:46.185Z" }, + { url = "https://files.pythonhosted.org/packages/bc/f0/dcc80973d2ec58a1978b838887ccbd84d56900cf66dec5fb730bec3bd081/wasmtime-41.0.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:f475df32ce9bfec4f6d0e124a49ca4a89e2ee71ccca460677f5237b1c8ee92ae", size = 6507285, upload-time = "2026-01-20T18:17:48.138Z" }, + { url = "https://files.pythonhosted.org/packages/bd/df/0867edd9ec26eb2e5eee7674a55f82c23ec27dd1d38d2d401f0e308eb920/wasmtime-41.0.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:ad7e866430313eb2ee07c85811e524344884489d00896f3b2246b65553fe322c", size = 7732024, upload-time = "2026-01-20T18:17:50.207Z" }, + { url = "https://files.pythonhosted.org/packages/bb/48/b748a2e70478feabc5c876d90e90a39f4aba35378f5ee822f607e8f29c69/wasmtime-41.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:e0ea44584f60dcfa620af82d4fc2589248bcf64a93905b54ac3144242113b48a", size = 6800017, upload-time = "2026-01-20T18:17:51.7Z" }, + { url = "https://files.pythonhosted.org/packages/14/29/43656c3a464d437d62421de16f2de2db645647bab0a0153deea30bfdade4/wasmtime-41.0.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:1dabb20a2751f01b835095013426a76091bd0bdb36ca9bcfc49c910b78347438", size = 6840763, upload-time = "2026-01-20T18:17:53.125Z" }, + { url = "https://files.pythonhosted.org/packages/9f/09/4608b65fa35ce5fc1479e138293a1166b4ea817cfa9a79f019ab6d7013d8/wasmtime-41.0.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:d9627dfc5625b4947ea35c819561da358838fe76f65bda8ffe01ce34df8b32b1", size = 7754016, upload-time = "2026-01-20T18:17:55.346Z" }, + { url = "https://files.pythonhosted.org/packages/f4/9d/236bb367270579e4f628fb7b04fe93541151df7953006f3766607fc667c9/wasmtime-41.0.0-py3-none-win_amd64.whl", hash = "sha256:4f29171d73b71f232b6fe86cba77526fee84139f1590071af5facba401b0c9eb", size = 6325764, upload-time = "2026-01-20T18:17:57.034Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4a/bba9c0368c377250ab24fd005a7a1e9076121778c1e83b1bcc092ab84f86/wasmtime-41.0.0-py3-none-win_arm64.whl", hash = "sha256:0c4bcaba055e78fc161f497b85f39f1d35d475f0341b1e0259fa0a4b49e223e8", size = 5392238, upload-time = "2026-01-20T18:17:59.052Z" }, +] + [[package]] name = "webvtt-py" version = "0.5.1"