From e473c818f40af9ae36f53d71863c3eee5b3792dd Mon Sep 17 00:00:00 2001 From: VineFeeder Date: Sat, 29 Nov 2025 12:02:04 +0000 Subject: [PATCH] V2.1.0 upstream --- README.md | 2 + UPDATES.md | 4 + packages/envied/cliff.toml | 71 -- packages/envied/pyproject.toml | 13 +- packages/envied/src/envied/commands/dl.py | 68 +- packages/envied/src/envied/core/__init__.py | 2 +- packages/envied/src/envied/core/__main__.py | 2 +- .../envied/src/envied/core/cacher.py.orig | 156 +++++ .../envied/src/envied/core/manifests/dash.py | 5 +- .../envied/src/envied/core/manifests/hls.py | 2 +- .../src/envied/core/proxies/windscribevpn.py | 10 + .../envied/src/envied/core/tracks/video.py | 30 +- packages/envied/src/envied/core/utilities.py | 2 + .../src/envied/services/DSNP/__init__.py | 632 ++++++++++-------- .../src/envied/services/DSNP/config.yaml | 20 +- .../src/envied/services/DSNP/queries.py | 2 + .../envied/src/envied/unshackle-example.yaml | 504 ++++++++++++++ packages/envied/src/envied/unshackle.yaml | 155 +++++ uv.lock | 23 +- vaults/API.py | 2 +- vaults/HTTP.py | 95 ++- vaults/MySQL.py | 52 +- vaults/SQLite.py | 47 +- 23 files changed, 1450 insertions(+), 449 deletions(-) delete mode 100644 packages/envied/cliff.toml create mode 100644 packages/envied/src/envied/core/cacher.py.orig create mode 100644 packages/envied/src/envied/unshackle-example.yaml create mode 100644 packages/envied/src/envied/unshackle.yaml diff --git a/README.md b/README.md index d23fd62..686affa 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,8 @@ pip install uv or python3 -m pip install uv + +or use your system's package manager to install python-uv ``` Then install TwinVine; the following installs the latest version directly from the GitHub repository: diff --git a/UPDATES.md b/UPDATES.md index 65afa5b..2a7b8ef 100644 --- a/UPDATES.md +++ b/UPDATES.md @@ -1,4 +1,8 @@ **Updates** +**29 Nov 2025** +- Followed upstream changes to envied V2.1.0 +- Updated some services +- removed incorrectly duplicated vaults folder within src/envied/ **13 Sept 2025:** - Added selected service name to screen display with GUI use. Featured in the original but got lost! diff --git a/packages/envied/cliff.toml b/packages/envied/cliff.toml deleted file mode 100644 index f8552cb..0000000 --- a/packages/envied/cliff.toml +++ /dev/null @@ -1,71 +0,0 @@ -# git-cliff ~ default configuration file -# https://git-cliff.org/docs/configuration - -[changelog] -header = """ -# Changelog\n -All notable changes to this project will be documented in this file. - -This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -Versions [3.0.0] and older use a format based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -but versions thereafter use a custom changelog format using [git-cliff](https://git-cliff.org).\n -""" -body = """ -{% if version -%} - ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }} -{% else -%} - ## [Unreleased] -{% endif -%} -{% for group, commits in commits | group_by(attribute="group") %} - ### {{ group | striptags | trim | upper_first }} - {% for commit in commits %} - - {% if commit.scope %}*{{ commit.scope }}*: {% endif %}\ - {% if commit.breaking %}[**breaking**] {% endif %}\ - {{ commit.message | upper_first }}\ - {% endfor %} -{% endfor %}\n -""" -footer = """ -{% for release in releases -%} - {% if release.version -%} - {% if release.previous.version -%} - [{{ release.version | trim_start_matches(pat="v") }}]: \ - https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}\ - /compare/{{ release.previous.version }}..{{ release.version }} - {% endif -%} - {% else -%} - [unreleased]: https://github.com/{{ remote.github.owner }}/{{ remote.github.repo }}\ - /compare/{{ release.previous.version }}..HEAD - {% endif -%} -{% endfor %} -""" -trim = true -postprocessors = [ - # { pattern = '', replace = "https://github.com/orhun/git-cliff" }, # replace repository URL -] - -[git] -conventional_commits = true -filter_unconventional = true -split_commits = false -commit_preprocessors = [] -commit_parsers = [ - { message = "^feat", group = "Features" }, - { message = "^fix|revert", group = "Bug Fixes" }, - { message = "^docs", group = "Documentation" }, - { message = "^style", skip = true }, - { message = "^refactor", group = "Changes" }, - { message = "^perf", group = "Performance Improvements" }, - { message = "^test", skip = true }, - { message = "^build", group = "Builds" }, - { message = "^ci", skip = true }, - { message = "^chore", skip = true }, -] -protect_breaking_commits = false -filter_commits = false -# tag_pattern = "v[0-9].*" -# skip_tags = "" -# ignore_tags = "" -topo_order = false -sort_commits = "oldest" diff --git a/packages/envied/pyproject.toml b/packages/envied/pyproject.toml index 7d4237d..be7566f 100644 --- a/packages/envied/pyproject.toml +++ b/packages/envied/pyproject.toml @@ -31,15 +31,16 @@ dependencies = [ "click>=8.1.8,<9", "construct>=2.8.8,<3", "crccheck>=1.3.0,<2", + "fonttools>=4.60.1,<5", "jsonpickle>=3.0.4,<4.5", "langcodes>=3.4.0,<4", - "lxml>=6.0.2", + "lxml>=6.0.2, <7", "pproxy>=2.7.9,<3", "protobuf>=4.25.3,<7", "pycaption>=2.2.6,<3", "pycryptodomex>=3.20.0,<4", "pyjwt>=2.8.0,<3", - "pymediainfo>=6.1.0,<7.5", + "pymediainfo>=6.1.0,<8", "pymp4>=1.4.0,<2", "pymysql>=1.1.0,<2", "pywidevine[serve]>=1.8.0,<2", @@ -53,10 +54,10 @@ dependencies = [ "Unidecode>=1.3.8,<2", "urllib3>=2.2.1,<3", "chardet>=5.2.0,<6", - "curl-cffi>=0.7.0b4,<0.8", + "curl-cffi>=0.7.0b4,<0.14", "pyplayready>=0.6.3,<0.7", "httpx>=0.28.1,<0.29", - "cryptography>=45.0.0", + "cryptography>=45.0.0,<47", "subby", "webvtt-py>=0.5.1", "isodate>=0.7.2", @@ -64,10 +65,10 @@ dependencies = [ "scrapy>=2.13.3", "mypy>=1.18.2,<2", "pysubs2>=1.7.0,<2", + "PyExecJS>=1.5.1,<2", "jwt>=1.4.0", "langcodes>=3.5.0", - "fonttools>=4.60.1", - "aiohttp-swagger3>=0.10.0", + "aiohttp-swagger3>=0.10.0, <1", ] [project.urls] diff --git a/packages/envied/src/envied/commands/dl.py b/packages/envied/src/envied/commands/dl.py index 27e3042..4070b24 100644 --- a/packages/envied/src/envied/commands/dl.py +++ b/packages/envied/src/envied/commands/dl.py @@ -65,10 +65,8 @@ from envied.core.utils.click_types import (LANGUAGE_RANGE, QUALITY_LIST, SEASON_ from envied.core.utils.collections import merge_dict from envied.core.utils.subprocess import ffprobe from envied.core.vaults import Vaults - from beaupy import select_multiple, Config - class dl: @staticmethod def truncate_pssh_for_display(pssh_string: str, drm_type: str) -> str: @@ -256,6 +254,13 @@ class dl: default=False, help="Exclude Dolby Atmos audio tracks when selecting audio.", ) + @click.option( + "--select-titles", + is_flag=True, + default=False, + help="Interactively select downloads from a list. Only use with Series to select Episodes", + ) + @click.option( "-w", "--wanted", @@ -270,12 +275,6 @@ class dl: default=False, help="Download only the single most recent episode available.", ) - @click.option( - "--select-titles", - is_flag=True, - default=False, - help="Interactively select downloads from a list. Only use with Series to select Episodes", - ) @click.option( "-l", "--lang", @@ -311,7 +310,6 @@ class dl: default=False, help="Use exact language matching (no variants). With this flag, -l es-419 matches ONLY es-419, not es-ES or other variants.", ) - @click.option( "--proxy", type=str, @@ -733,9 +731,9 @@ class dl: range_: list[Video.Range], channels: float, no_atmos: bool, + select_titles: bool, wanted: list[str], latest_episode: bool, - select_titles: bool, lang: list[str], v_lang: list[str], a_lang: list[str], @@ -908,7 +906,6 @@ class dl: console.print(Padding(titles.tree(verbose=list_titles), (0, 5))) if list_titles: return - # modification to enable beaupy module to list titles for download for manual selection # use --select-titles after dl in unshackle command @@ -942,8 +939,6 @@ class dl: if i not in keep: del titles[i] # end modification - - # Determine the latest episode if --latest-episode is set latest_episode_id = None if latest_episode and isinstance(titles, Series) and len(titles) > 0: @@ -1413,6 +1408,7 @@ class dl: kept_tracks.extend(title.tracks.subtitles) if keep_chapters: kept_tracks.extend(title.tracks.chapters) + kept_tracks.extend(title.tracks.attachments) title.tracks = Tracks(kept_tracks) @@ -1499,6 +1495,7 @@ class dl: ) ): download.result() + except KeyboardInterrupt: console.print(Padding(":x: Download Cancelled...", (0, 5, 1, 5))) if self.debug_logger: @@ -1891,25 +1888,32 @@ class dl: if not drm: return + track_quality = None if isinstance(track, Video) and track.height: - pass + track_quality = track.height if isinstance(drm, Widevine): if not isinstance(self.cdm, (WidevineCdm, DecryptLabsRemoteCDM)) or ( isinstance(self.cdm, DecryptLabsRemoteCDM) and self.cdm.is_playready ): - widevine_cdm = self.get_cdm(self.service, self.profile, drm="widevine") + widevine_cdm = self.get_cdm(self.service, self.profile, drm="widevine", quality=track_quality) if widevine_cdm: - self.log.info("Switching to Widevine CDM for Widevine content") + if track_quality: + self.log.info(f"Switching to Widevine CDM for Widevine {track_quality}p content") + else: + self.log.info("Switching to Widevine CDM for Widevine content") self.cdm = widevine_cdm elif isinstance(drm, PlayReady): if not isinstance(self.cdm, (PlayReadyCdm, DecryptLabsRemoteCDM)) or ( isinstance(self.cdm, DecryptLabsRemoteCDM) and not self.cdm.is_playready ): - playready_cdm = self.get_cdm(self.service, self.profile, drm="playready") + playready_cdm = self.get_cdm(self.service, self.profile, drm="playready", quality=track_quality) if playready_cdm: - self.log.info("Switching to PlayReady CDM for PlayReady content") + if track_quality: + self.log.info(f"Switching to PlayReady CDM for PlayReady {track_quality}p content") + else: + self.log.info("Switching to PlayReady CDM for PlayReady content") self.cdm = playready_cdm if isinstance(drm, Widevine): @@ -1929,7 +1933,7 @@ class dl: with self.DRM_TABLE_LOCK: pssh_display = self.truncate_pssh_for_display(drm.pssh.dumps(), "Widevine") - cek_tree = Tree(Text.assemble(("Widevine", "cyan"), (f"({pssh_display})", "text"), overflow="ellipses")) + cek_tree = Tree(Text.assemble(("Widevine", "cyan"), (f"({pssh_display})", "text"), overflow="fold")) pre_existing_tree = next( (x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None ) @@ -2073,12 +2077,21 @@ class dl: if export: keys = {} if export.is_file(): - keys = jsonpickle.loads(export.read_text(encoding="utf8")) + keys = jsonpickle.loads(export.read_text(encoding="utf8")) or {} if str(title) not in keys: keys[str(title)] = {} if str(track) not in keys[str(title)]: keys[str(title)][str(track)] = {} - keys[str(title)][str(track)].update(drm.content_keys) + + track_data = keys[str(title)][str(track)] + track_data["url"] = track.url + track_data["descriptor"] = track.descriptor.name + + if "keys" not in track_data: + track_data["keys"] = {} + for kid, key in drm.content_keys.items(): + track_data["keys"][kid.hex] = key + export.write_text(jsonpickle.dumps(keys, indent=4), encoding="utf8") elif isinstance(drm, PlayReady): @@ -2218,12 +2231,21 @@ class dl: if export: keys = {} if export.is_file(): - keys = jsonpickle.loads(export.read_text(encoding="utf8")) + keys = jsonpickle.loads(export.read_text(encoding="utf8")) or {} if str(title) not in keys: keys[str(title)] = {} if str(track) not in keys[str(title)]: keys[str(title)][str(track)] = {} - keys[str(title)][str(track)].update(drm.content_keys) + + track_data = keys[str(title)][str(track)] + track_data["url"] = track.url + track_data["descriptor"] = track.descriptor.name + + if "keys" not in track_data: + track_data["keys"] = {} + for kid, key in drm.content_keys.items(): + track_data["keys"][kid.hex] = key + export.write_text(jsonpickle.dumps(keys, indent=4), encoding="utf8") @staticmethod diff --git a/packages/envied/src/envied/core/__init__.py b/packages/envied/src/envied/core/__init__.py index 8c0d5d5..9aa3f90 100644 --- a/packages/envied/src/envied/core/__init__.py +++ b/packages/envied/src/envied/core/__init__.py @@ -1 +1 @@ -__version__ = "2.0.0" +__version__ = "2.1.0" diff --git a/packages/envied/src/envied/core/__main__.py b/packages/envied/src/envied/core/__main__.py index 26691cb..22037d8 100644 --- a/packages/envied/src/envied/core/__main__.py +++ b/packages/envied/src/envied/core/__main__.py @@ -63,7 +63,7 @@ def main(version: bool, debug: bool) -> None: expand=True, ), justify="center", -) + ) @atexit.register diff --git a/packages/envied/src/envied/core/cacher.py.orig b/packages/envied/src/envied/core/cacher.py.orig new file mode 100644 index 0000000..c88b9ed --- /dev/null +++ b/packages/envied/src/envied/core/cacher.py.orig @@ -0,0 +1,156 @@ +from __future__ import annotations + +import zlib +from datetime import datetime, timedelta +from os import stat_result +from pathlib import Path +from typing import Any, Optional, Union + +import jsonpickle +import jwt + +from envied.core.config import config + +EXP_T = Union[datetime, str, int, float] + + +class Cacher: + """Cacher for Services to get and set arbitrary data with expiration dates.""" + + def __init__( + self, + service_tag: str, + key: Optional[str] = None, + version: Optional[int] = 1, + data: Optional[Any] = None, + expiration: Optional[datetime] = None, + ) -> None: + self.service_tag = service_tag + self.key = key + self.version = version + self.data = data or {} + self.expiration = expiration + + if self.expiration and self.expired: + # if its expired, remove the data for safety and delete cache file + self.data = None + self.path.unlink() + + def __bool__(self) -> bool: + return bool(self.data) + + @property + def path(self) -> Path: + """Get the path at which the cache will be read and written.""" + return (config.directories.cache / self.service_tag / self.key).with_suffix(".json") + + @property + def expired(self) -> bool: + return self.expiration and self.expiration < datetime.now() + + def get(self, key: str, version: int = 1) -> Cacher: + """ + Get Cached data for the Service by Key. + :param key: the filename to save the data to, should be url-safe. + :param version: the config data version you expect to use. + :returns: Cache object containing the cached data or None if the file does not exist. + """ + cache = Cacher(self.service_tag, key, version) + if cache.path.is_file(): + data = jsonpickle.loads(cache.path.read_text(encoding="utf8")) + payload = data.copy() + del payload["crc32"] + checksum = data["crc32"] + calculated = zlib.crc32(jsonpickle.dumps(payload).encode("utf8")) + if calculated != checksum: + raise ValueError( + f"The checksum of the Cache payload mismatched. Checksum: {checksum} !== Calculated: {calculated}" + ) + cache.data = data["data"] + cache.expiration = data["expiration"] + cache.version = data["version"] + if cache.version != version: + raise ValueError( + f"The version of your {self.service_tag} {key} cache is outdated. Please delete: {cache.path}" + ) + return cache + + def set(self, data: Any, expiration: Optional[EXP_T] = None) -> Any: + """ + Set Cached data for the Service by Key. + :param data: absolutely anything including None. + :param expiration: when the data expires, optional. Can be ISO 8601, seconds + til expiration, unix timestamp, or a datetime object. + :returns: the data provided for quick wrapping of functions or vars. + """ + self.data = data + + if not expiration: + try: + expiration = jwt.decode(self.data, options={"verify_signature": False})["exp"] + except jwt.DecodeError: + pass + + self.expiration = self._resolve_datetime(expiration) if expiration else None + + payload = {"data": self.data, "expiration": self.expiration, "version": self.version} + payload["crc32"] = zlib.crc32(jsonpickle.dumps(payload).encode("utf8")) + + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text(jsonpickle.dumps(payload)) + + return self.data + + def stat(self) -> stat_result: + """ + Get Cache file OS Stat data like Creation Time, Modified Time, and such. + :returns: an os.stat_result tuple + """ + return self.path.stat() + + @staticmethod + def _resolve_datetime(timestamp: EXP_T) -> datetime: + """ + Resolve multiple formats of a Datetime or Timestamp to an absolute Datetime. + + Examples: + >>> now = datetime.now() + datetime.datetime(2022, 6, 27, 9, 49, 13, 657208) + >>> iso8601 = now.isoformat() + '2022-06-27T09:49:13.657208' + >>> Cacher._resolve_datetime(iso8601) + datetime.datetime(2022, 6, 27, 9, 49, 13, 657208) + >>> Cacher._resolve_datetime(iso8601 + "Z") + datetime.datetime(2022, 6, 27, 9, 49, 13, 657208) + >>> Cacher._resolve_datetime(3600) + datetime.datetime(2022, 6, 27, 10, 52, 50, 657208) + >>> Cacher._resolve_datetime('3600') + datetime.datetime(2022, 6, 27, 10, 52, 51, 657208) + >>> Cacher._resolve_datetime(7800.113) + datetime.datetime(2022, 6, 27, 11, 59, 13, 770208) + + In the int/float examples you may notice that it did not return now + 3600 seconds + but rather something a bit more than that. This is because it did not resolve 3600 + seconds from the `now` variable but from right now as the function was called. + """ + if isinstance(timestamp, datetime): + return timestamp + if isinstance(timestamp, str): + if timestamp.endswith("Z"): + # fromisoformat doesn't accept the final Z + timestamp = timestamp.split("Z")[0] + try: + return datetime.fromisoformat(timestamp) + except ValueError: + timestamp = float(timestamp) + try: + if len(str(int(timestamp))) == 13: # JS-style timestamp + timestamp /= 1000 + timestamp = datetime.fromtimestamp(timestamp) + except ValueError: + raise ValueError(f"Unrecognized Timestamp value {timestamp!r}") + if timestamp < datetime.now(): + # timestamp is likely an amount of seconds til expiration + # or, it's an already expired timestamp which is unlikely + timestamp = timestamp + timedelta(seconds=datetime.now().timestamp()) + return timestamp diff --git a/packages/envied/src/envied/core/manifests/dash.py b/packages/envied/src/envied/core/manifests/dash.py index c935feb..64e48ff 100644 --- a/packages/envied/src/envied/core/manifests/dash.py +++ b/packages/envied/src/envied/core/manifests/dash.py @@ -297,8 +297,9 @@ class DASH: manifest_base_url = track.url elif not re.match("^https?://", manifest_base_url, re.IGNORECASE): manifest_base_url = urljoin(track.url, f"./{manifest_base_url}") - period_base_url = urljoin(manifest_base_url, period.findtext("BaseURL")) - rep_base_url = urljoin(period_base_url, representation.findtext("BaseURL")) + period_base_url = urljoin(manifest_base_url, period.findtext("BaseURL") or "") + adaptation_set_base_url = urljoin(period_base_url, adaptation_set.findtext("BaseURL") or "") + rep_base_url = urljoin(adaptation_set_base_url, representation.findtext("BaseURL") or "") period_duration = period.get("duration") or manifest.get("mediaPresentationDuration") init_data: Optional[bytes] = None diff --git a/packages/envied/src/envied/core/manifests/hls.py b/packages/envied/src/envied/core/manifests/hls.py index 56aa6a1..dc8304d 100644 --- a/packages/envied/src/envied/core/manifests/hls.py +++ b/packages/envied/src/envied/core/manifests/hls.py @@ -313,7 +313,7 @@ class HLS: if segment.byterange: byte_range = HLS.calculate_byte_range(segment.byterange, range_offset) - range_offset = byte_range.split("-")[0] + range_offset = int(byte_range.split("-")[0]) else: byte_range = None diff --git a/packages/envied/src/envied/core/proxies/windscribevpn.py b/packages/envied/src/envied/core/proxies/windscribevpn.py index a44d6e0..f31947d 100644 --- a/packages/envied/src/envied/core/proxies/windscribevpn.py +++ b/packages/envied/src/envied/core/proxies/windscribevpn.py @@ -44,8 +44,17 @@ class WindscribeVPN(Proxy): def get_proxy(self, query: str) -> Optional[str]: """ Get an HTTPS proxy URI for a WindscribeVPN server. + + Note: Windscribe's static OpenVPN credentials work reliably on US, AU, and NZ servers. """ query = query.lower() + supported_regions = {"us", "au", "nz"} + + if query not in supported_regions and query not in self.server_map: + raise ValueError( + f"Windscribe proxy does not currently support the '{query.upper()}' region. " + f"Supported regions with reliable credentials: {', '.join(sorted(supported_regions))}. " + ) if query in self.server_map: hostname = self.server_map[query] @@ -58,6 +67,7 @@ class WindscribeVPN(Proxy): if not hostname: return None + hostname = hostname.split(':')[0] return f"https://{self.username}:{self.password}@{hostname}:443" def get_random_server(self, country_code: str) -> Optional[str]: diff --git a/packages/envied/src/envied/core/tracks/video.py b/packages/envied/src/envied/core/tracks/video.py index 00981f9..0fdaa2d 100644 --- a/packages/envied/src/envied/core/tracks/video.py +++ b/packages/envied/src/envied/core/tracks/video.py @@ -99,24 +99,42 @@ class Video(Track): @staticmethod def from_cicp(primaries: int, transfer: int, matrix: int) -> Video.Range: """ - ISO/IEC 23001-8 Coding-independent code points to Video Range. + Convert CICP (Coding-Independent Code Points) values to Video Range. + + CICP is defined in ITU-T H.273 and ISO/IEC 23091-2 for signaling video + color properties independently of the compression codec. These values are + used across AVC (H.264), HEVC (H.265), VVC, AV1, and other modern codecs. + + The enum values (Primaries, Transfer, Matrix) match the official specifications: + - ITU-T H.273: Coding-independent code points for video signal type identification + - ISO/IEC 23091-2: Information technology — Coding-independent code points — Part 2: Video + - H.264 Table E-3 (Colour Primaries) and Table E-4 (Transfer Characteristics) + - H.265 Table E.3 and E.4 (identical to H.264) + + Note: Value 0 = "Reserved" and Value 2 = "Unspecified" per specification. + While both effectively mean "unknown" in practice, the distinction matters for + spec compliance. Value 2 was added based on user feedback (GitHub issue) and + verified against FFmpeg's AVColorPrimaries/AVColorTransferCharacteristic enums. Sources: - https://www.itu.int/rec/T-REC-H.Sup19-202104-I + - https://www.itu.int/rec/T-REC-H.273 + - https://www.itu.int/rec/T-REC-H.Sup19-202104-I + - https://github.com/FFmpeg/FFmpeg/blob/master/libavutil/pixfmt.h """ class Primaries(Enum): - Unspecified = 0 + Reserved = 0 BT_709 = 1 + Unspecified = 2 BT_601_625 = 5 BT_601_525 = 6 BT_2020_and_2100 = 9 SMPTE_ST_2113_and_EG_4321 = 12 # P3D65 class Transfer(Enum): - Unspecified = 0 + Reserved = 0 BT_709 = 1 - Unspecified_Image = 2 + Unspecified = 2 BT_601 = 6 BT_2020 = 14 BT_2100 = 15 @@ -143,7 +161,7 @@ class Video(Track): # primaries and matrix does not strictly correlate to a range - if (primaries, transfer, matrix) == (0, 0, 0): + if (primaries, transfer, matrix) == (Primaries.Reserved, Transfer.Reserved, Matrix.RGB): return Video.Range.SDR elif primaries in (Primaries.BT_601_625, Primaries.BT_601_525): return Video.Range.SDR diff --git a/packages/envied/src/envied/core/utilities.py b/packages/envied/src/envied/core/utilities.py index aaf4f7e..566167f 100644 --- a/packages/envied/src/envied/core/utilities.py +++ b/packages/envied/src/envied/core/utilities.py @@ -127,6 +127,8 @@ def sanitize_filename(filename: str, spacer: str = ".") -> str: # remove or replace further characters as needed filename = "".join(c for c in filename if unicodedata.category(c) != "Mn") # hidden characters filename = filename.replace("/", " & ").replace(";", " & ") # e.g. multi-episode filenames + if spacer == ".": + filename = re.sub(r" - ", spacer, filename) # title separators to spacer (avoids .-. pattern) filename = re.sub(r"[:; ]", spacer, filename) # structural chars to (spacer) filename = re.sub(r"[\\*!?¿,'\"" "()<>|$#~]", "", filename) # not filename safe chars filename = re.sub(rf"[{spacer}]{{2,}}", spacer, filename) # remove extra neighbouring (spacer)s diff --git a/packages/envied/src/envied/services/DSNP/__init__.py b/packages/envied/src/envied/services/DSNP/__init__.py index 0283b28..8dc8550 100644 --- a/packages/envied/src/envied/services/DSNP/__init__.py +++ b/packages/envied/src/envied/services/DSNP/__init__.py @@ -1,4 +1,3 @@ - from __future__ import annotations import re @@ -12,14 +11,14 @@ import click from click import Context from requests import Request -#from envied.core.downloaders import n_m3u8dl_re -from envied.core.credential import Credential -from envied.core.manifests import HLS -from envied.core.search_result import SearchResult -from envied.core.service import Service -from envied.core.titles import Episode, Movie, Movies, Series -from envied.core.tracks import Chapter, Chapters, Tracks, Video -from envied.core.utils.collections import as_list +from unshackle.core.downloaders import n_m3u8dl_re +from unshackle.core.credential import Credential +from unshackle.core.manifests import HLS +from unshackle.core.search_result import SearchResult +from unshackle.core.service import Service +from unshackle.core.titles import Episode, Movie, Movies, Series +from unshackle.core.tracks import Chapters, Tracks, Video, Hybrid +from unshackle.core.utils.collections import as_list from . import queries @@ -57,30 +56,42 @@ class DSNP(Service): @staticmethod @click.command(name="DSNP", short_help="https://www.disneyplus.com", help=__doc__) @click.argument("title", type=str) + @click.option( + "-m", "--movie", is_flag=True, default=False, help="Title is a Movie." + ) @click.pass_context def cli(ctx: Context, **kwargs: Any) -> DSNP: return DSNP(ctx, **kwargs) - def __init__(self, ctx: Context, title: str): + def __init__(self, ctx: Context, title: str, movie): self.title = title + self.movie = movie super().__init__(ctx) self.cdm = ctx.obj.cdm - self.playback_data = {} vcodec = ctx.parent.params.get("vcodec") range = ctx.parent.params.get("range_") self.range = range[0].name if range else "SDR" - self.vcodec = "H265" if vcodec and vcodec == Video.Codec.HEVC else "H264" - if self.range != "SDR" and self.vcodec != "H265": - self.vcodec = "H265" + self.vcodec = "H.265" if vcodec and vcodec == Video.Codec.HEVC else "H.264" + if self.range != "SDR" and self.vcodec != "H.265": + self.vcodec = "H.265" def authenticate(self, cookies: Optional[CookieJar] = None, credential: Optional[Credential] = None) -> None: super().authenticate(cookies, credential) if not credential: raise EnvironmentError("Service requires Credentials for Authentication.") - self.session.headers.update(self.config["HEADERS"]) + # Use exact headers from working Vinetrimmer implementation to avoid geoblocking + self.session.headers.update({ + "Accept-Language": "en-US,en;q=0.5", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + "Origin": "https://www.disneyplus.com", + "x-bamsdk-platform": "javascript/windows/chrome", + "x-bamsdk-version": "28.0", + "x-bamsdk-client-id": "disney-svod", + "Accept-Encoding": "gzip", + }) self.session.headers.update({"x-bamsdk-transaction-id": str(uuid.uuid4())}) self.prd_config = self.session.get(self.config["CONFIG_URL"]).json() @@ -155,256 +166,229 @@ class DSNP(Service): ) def get_titles(self) -> Union[Movies, Series]: - if not self.title.startswith("entity"): - raise ValueError("Invalid input - Use only entity IDs.") + # Use Vinetrimmer logic - handle both entity IDs and other formats + if not "entity" in self.title: + # Convert to entity ID like Vinetrimmer does + try: + deeplinkId_response = self.session.get( + url='https://disney.api.edge.bamgrid.com/explore/v1.3/deeplink', + params={ + 'refId': self.title, + 'refIdType': 'encodedFamilyId' # Try movie first + } + ) + if deeplinkId_response.status_code == 200: + deeplinkId = deeplinkId_response.json() + self.title = deeplinkId["data"]["deeplink"]["actions"][0]["deeplinkId"] + else: + # Try with encodedSeriesId + deeplinkId_response = self.session.get( + url='https://disney.api.edge.bamgrid.com/explore/v1.3/deeplink', + params={ + 'refId': self.title, + 'refIdType': 'encodedSeriesId' + } + ) + if deeplinkId_response.status_code == 200: + deeplinkId = deeplinkId_response.json() + self.title = deeplinkId["data"]["deeplink"]["actions"][0]["deeplinkId"] + except Exception: + # If all fails, assume it's already an entity ID + pass content = self.get_deeplink(self.title) - _type = content["data"]["deeplink"]["actions"][0]["contentType"] + + # Handle deeplink response structure - determine if series or movie from infoBlock + if "data" in content and "deeplink" in content["data"]: + actions = content["data"]["deeplink"]["actions"] + if actions and len(actions) > 0: + action = actions[0] + # Check the infoBlock to determine content type like Vinetrimmer does + info_block = action.get("infoBlock", "") + if "urn:ds:cmp:eva:series" in info_block: + _type = "series" + elif "urn:ds:cmp:eva:movie" in info_block or "urn:ds:cmp:eva:film" in info_block: + _type = "movie" + else: + # Fallback: assume series for browse type + _type = "series" if action.get("type") == "browse" else "movie" + else: + raise ValueError("No actions found in deeplink response") + else: + raise ValueError("Invalid deeplink response structure") - if _type == "movie": + if _type == "movie" or self.movie: movie = self._movie(self.title) return Movies(movie) elif _type == "series": episodes = self._show(self.title) return Series(episodes) + + else: + raise ValueError(f"Unknown content type: {_type}") def get_tracks(self, title: Union[Movie, Episode]) -> Tracks: resource_id = title.data.get("resourceId") - content_id = title.data["partnerFeed"].get("dmcContentId") - content = self.get_video(content_id) - playback = content["video"]["mediaMetadata"]["playbackUrls"][0]["href"] - - token = self._refresh() - - headers = { - "accept": "application/vnd.media-service+json; version=5", - "authorization": token, - "x-dss-feature-filtering": "true", - } - - payload = { - "playbackId": resource_id, - "playback": { - "attributes": { - "codecs": { - "supportsMultiCodecMaster": False, + + # =============================== + # TOKEN REFRESH + # =============================== + if self._cache: + refresh_token = self._cache.data["token"]["refreshToken"] + fresh_token = self.refresh_token(refresh_token) + self._cache.set(fresh_token, expiration=fresh_token["token"]["expiresIn"] - 30) + + # Update session header + token = fresh_token["token"]["accessToken"] + self.session.headers.update({"Authorization": f"Bearer {token}"}) + + # =============================== + # INTERNAL MANIFEST FETCHER + # =============================== + def get_manifest_for_scenario(scenario_name, quality=None): + original_headers = dict(self.session.headers) + + # Vinetrimmer-style headers + self.session.headers.update({ + 'x-dss-feature-filtering': 'true', + 'x-application-version': '1.1.2', + 'x-bamsdk-client-id': 'disney-svod', + 'x-bamsdk-platform': 'javascript/windows/chrome', + 'x-bamsdk-version': '28.0' + }) + + # Default resolution (Vinetrimmer-style) + if quality is None: + quality = '1280' if hasattr(self, 'cdm') and self.cdm.security_level == 3 else '1920' + + json_data = { + 'playback': { + 'attributes': { + 'resolution': { + 'max': [quality], + }, + 'protocol': 'HTTPS', + 'assetInsertionStrategy': 'SGAI', + 'playbackInitiationContext': 'ONLINE', + 'frameRates': [60], }, - "protocol": "HTTPS", - # "ads": "", - "frameRates": [60], - "assetInsertionStrategy": "SGAI", - "playbackInitializationContext": "ONLINE", }, - }, - } - - video_ranges = [] - audio_types = [] - - audio_types.append("ATMOS") - audio_types.append("DTS_X") - - if not self.cdm.security_level == 3 and self.range == "DV": - video_ranges.append("DOLBY_VISION") - - if not self.cdm.security_level == 3 and self.range == "HDR10": - video_ranges.append("HDR10") - - if self.vcodec == "H265": - payload["playback"]["attributes"]["codecs"] = {"video": ["h264", "h265"]} - - if audio_types: - payload["playback"]["attributes"]["audioTypes"] = audio_types - - if video_ranges: - payload["playback"]["attributes"]["videoRanges"] = video_ranges - - if self.cdm.security_level == 3: - payload["playback"]["attributes"]["resolution"] = {"max": ["1280x720"]} - - scenario = "ctr-regular" if self.cdm.security_level == 3 else "ctr-high" - endpoint = playback.format(scenario=scenario) - - res = self._request("POST", endpoint, payload=payload, headers=headers) - self.playback_data[title.id] = self._request( - "POST", f"https://disney.playback.edge.bamgrid.com/v7/playback/{scenario}", payload=payload, headers=headers - ) - - manifest = res["stream"]["complete"][0]["url"] - - tracks = HLS.from_url(url=manifest, session=self.session).to_tracks(language=title.language) + 'playbackId': resource_id, + } + + try: + res = self.session.post( + f'https://disney.playback.edge.bamgrid.com/v7/playback/{scenario_name}', + json=json_data + ) + if res.status_code == 200: + data = res.json() + manifest_url = data["stream"]["sources"][0]['complete']['url'] + return HLS.from_url(url=manifest_url, session=self.session).to_tracks(language="en-US") + return None + finally: + self.session.headers.clear() + self.session.headers.update(original_headers) + + # ========================================================== + # =============== HYBRID RANGE (DV + HDR10) ================= + # ========================================================== + if self.range == "HYBRID": + self.log.info("HYBRID mode — fetching HDR10 + DV manifests") + + all_tracks = Tracks() + + # -------- Fetch HDR10 ------------ + self.log.info("Fetching HDR10 tracks") + self.range = Video.Range.HDR10 + hdr_scenario = 'tv-drm-cbcs-h265-hdr10' + hdr_tracks = get_manifest_for_scenario(hdr_scenario, '2160') + if hdr_tracks: + all_tracks.add(hdr_tracks, warn_only=True) + + # -------- Fetch DV --------------- + self.log.info("Fetching DV tracks") + self.range = Video.Range.DV + dv_scenario = 'tv-drm-cbcs-h265-dovi' + dv_tracks = get_manifest_for_scenario(dv_scenario, '2160') + if dv_tracks: + all_tracks.add(dv_tracks, warn_only=True) + + # Restore range + self.range = "HYBRID" + + tracks = all_tracks + self.log.info("HYBRID fetch complete — merge will occur after download") + + # ========================================================== + # =============== NON-HYBRID MODES ========================= + # ========================================================== + else: + if self.vcodec == "H.265" and self.range == 'HDR10': + scenario = 'tv-drm-cbcs-h265-hdr10' + quality = '2160' + elif self.vcodec == "H.265" and self.range == 'DV': + scenario = 'tv-drm-cbcs-h265-dovi' + quality = '2160' + else: + scenario = 'tv-drm-cbcs' + quality = '1920' + + tracks = get_manifest_for_scenario(scenario, quality) + if not tracks: + raise ValueError("Failed to fetch DSNP manifest") + + # ===================================================== + # Fetch ATMOS/H265 secondary manifest (like Vinetrimmer) + # ===================================================== + atmos_tracks = get_manifest_for_scenario('tv-drm-ctr-h265-atmos') + if atmos_tracks: + tracks.videos.extend(atmos_tracks.videos) + tracks.audio.extend(atmos_tracks.audio) + tracks.subtitles.extend(atmos_tracks.subtitles) + + # ===================================================== + # AUDIO BITRATE FIX + # ===================================================== for audio in tracks.audio: bitrate = re.search( r"(?<=r/composite_)\d+|\d+(?=_complete.m3u8)", as_list(audio.url)[0], ) audio.bitrate = int(bitrate.group()) * 1000 - if audio.bitrate == 1000_000: - # DSNP lies about the Atmos bitrate + if audio.bitrate == 1000_000: # DSNP lies about Atmos audio.bitrate = 768_000 - + + # ===================================================== + # FINAL CONFIG + # ===================================================== for track in tracks: if track not in tracks.attachments: - track.downloader = "N_m3u8DL-RE" + track.downloader = n_m3u8dl_re track.needs_repack = True - + return tracks def get_chapters(self, title: Union[Movie, Episode]) -> Chapters: - """ - Extract chapter information from the title data if available. - Returns chapter markers for intro, credits, and scenes. - """ - chapters = Chapters() + return Chapters() - try: - # First try to get chapters from the new API via playback data - if title.id in self.playback_data and "stream" in self.playback_data[title.id]: - playback_res = self.playback_data[title.id] + def get_widevine_service_certificate(self, **_: Any) -> str: + return None - # Check for editorial markers in playback data - if "editorial" in playback_res.get("stream", {}): - editorial = playback_res["stream"]["editorial"] - - # Add "Start" chapter if not already present - if not any(item.get("offsetMillis") == 0 for item in editorial): - chapters.add(Chapter(timestamp=0, name="Start")) - - # Map editorial labels to chapter names - mapping = { - "recap_start": "Recap", - "FFER": "Recap", # First Frame Episode Recap - "recap_end": "Scene", - "LFER": "Scene", # Last Frame Episode Recap - "intro_start": "Title Sequence", - "intro_end": "Scene", - "FFEI": "Title Sequence", # First Frame Episode Intro - "LFEI": "Scene", # Last Frame Episode Intro - "FFCB": None, # First Frame Credits Bumper - "LFCB": "Scene", # Last Frame Credits Bumper - "FFEC": "End Credits", # First Frame End Credits - "LFEC": None, # Last Frame End Credits - "up_next": None, - } - - # Sort by timestamp to ensure proper scene numbering - editorial.sort(key=lambda x: x.get("offsetMillis", 0)) - - # Track chapters we've already added by timestamp to avoid duplicates - seen_timestamps = set() - scene_count = 0 - - for marker in editorial: - if "label" in marker and "offsetMillis" in marker: - timestamp = marker["offsetMillis"] - name = mapping.get(marker["label"]) - - # Skip if no mapping or already processed timestamp - if not name or timestamp in seen_timestamps: - continue - - # Mark this timestamp as seen - seen_timestamps.add(timestamp) - - if name == "Scene": - scene_count += 1 - name = f"Scene {scene_count}" - - chapters.add(Chapter(timestamp=timestamp, name=name)) - - # If we found chapters in the playback data, return them - if chapters: - return chapters - - # If no chapters found in playback data, try the original method - content_id = title.data["partnerFeed"].get("dmcContentId") - content = self.get_video(content_id) - - # Check for chapter/milestone data - video_info = content.get("video", {}).get("milestone", {}) - - if not video_info: - return chapters - - # Mapping of milestone types to chapter names - mapping = { - "recap_start": "Recap", - "recap_end": "Scene", - "intro_start": "Title Sequence", - "intro_end": "Scene", - "FFEI": "Title Sequence", # First Frame Episode Intro - "LFEI": "Scene", # Last Frame Episode Intro - "FFCB": None, # First Frame Credits Bumper - "LFCB": "Scene", # Last Frame Credits Bumper - "FFEC": "End Credits", # First Frame End Credits - "LFEC": None, # Last Frame End Credits - "up_next": None, - } - - # Flatten the milestone data and sort by start time - flattened = [] - for chapter_type, items in video_info.items(): - for entry in items: - if "milestoneTime" in entry and entry["milestoneTime"]: - start = entry["milestoneTime"][0]["startMillis"] - flattened.append({"type": chapter_type, "start": start}) - - flattened.sort(key=lambda x: x["start"]) - - # Create chapters - chapter_list = [] - scene_count = 0 - for f in flattened: - name = mapping.get(f["type"]) - if not name: - continue - - if name == "Scene": - scene_count += 1 - name = f"Scene {scene_count}" - - chapter_list.append(Chapter(timestamp=f["start"], name=name)) - - # Add a "Start" chapter at 0 if we have end credits - if "FFEC" in video_info and not any(ch.timestamp == 0 for ch in chapter_list): - chapter_list.insert(0, Chapter(timestamp=0, name="Start")) - - # Remove duplicates (same time and name) - prev_time, prev_name = None, None - - for ch in chapter_list: - # Convert timestamp to milliseconds for comparison - if isinstance(ch.timestamp, str): - ts_parts = ch.timestamp.split(":") - hour, minute, second = int(ts_parts[0]), int(ts_parts[1]), float(ts_parts[2]) - ts_ms = (hour * 3600 + minute * 60 + second) * 1000 - else: - ts_ms = ch.timestamp - - if prev_time is None or (ts_ms != prev_time and ch.name != prev_name): - chapters.add(ch) - prev_time, prev_name = ts_ms, ch.name - - return chapters - - except Exception as e: - self.log.warning(f"Failed to extract chapters: {e}") - return chapters - - def get_playready_license(self, *, challenge: bytes, title, track) -> bytes: + def get_widevine_license(self, *, challenge: bytes, title, track) -> bytes: + """Get Widevine license for Disney+ content - Adapted from working Vinetrimmer implementation""" + # Force token refresh like Vinetrimmer does for license calls + if self._cache: + refresh_token = self._cache.data["token"]["refreshToken"] + fresh_token = self.refresh_token(refresh_token) + self._cache.set(fresh_token, expiration=fresh_token["token"]["expiresIn"] - 30) + token = fresh_token["token"]["accessToken"] + else: + return None + headers = { - "Authorization": f"Bearer {self._cache.data['token']['accessToken']}", - "Content-Type": "application/octet-stream", - } - r = self.session.post(url=self.config["PLAYREADY_LICENSE"], headers=headers, data=challenge) - if r.status_code != 200: - raise ConnectionError(r.text) - return r.content - - def get_widevine_license(self, *, challenge: bytes, title, track) -> None: - headers = { - "Authorization": f"Bearer {self._cache.data['token']['accessToken']}", + "Authorization": f'Bearer {token}', "Content-Type": "application/octet-stream", } r = self.session.post(url=self.config["LICENSE"], headers=headers, data=challenge) @@ -412,6 +396,23 @@ class DSNP(Service): raise ConnectionError(r.text) return r.content + def get_playready_license(self, *, challenge: bytes, title, track) -> Optional[bytes]: + """Get PlayReady license for Disney+ content - Adapted from working Vinetrimmer implementation""" + # Refresh token if needed + token = self._cache.data["token"]["accessToken"] if self._cache else None + if not token: + return None + + r = self.session.post( + url='https://disney.playback.edge.bamgrid.com/playready/v1/obtain-license.asmx', + headers={'authorization': f'Bearer {token}'}, + data=challenge + ) + + if r.status_code != 200: + raise ConnectionError(r.text) + return r.text.encode() + # Service specific functions def _show(self, title: str) -> Episode: @@ -421,13 +422,16 @@ class DSNP(Service): episodes = [] for season in season_ids: - endpoint = self.href( - self.prd_config["services"]["explore"]["client"]["endpoints"]["getSeason"]["href"], - version=self.config["EXPLORE_VERSION"], - seasonId=season, - ) - data = self.session.get(endpoint, params={'limit': 999}).json()["data"]["season"]["items"] - episodes.extend(data) + # Use direct Disney+ API URL like Vinetrimmer does + endpoint = f'https://disney.api.edge.bamgrid.com/explore/v1.3/season/{season}' + params = {'limit': 80} + + response = self.session.get(endpoint, params=params) + if response.status_code == 200: + data = response.json()["data"]["season"]["items"] + episodes.extend(data) + else: + self.log.warning(f"Failed to get season {season}: {response.status_code}") return [ Episode( @@ -438,7 +442,6 @@ class DSNP(Service): season=int(episode["visuals"].get("seasonNumber", 0)), number=int(episode["visuals"].get("episodeNumber", 0)), name=episode["visuals"].get("episodeTitle"), - language=self.get_original_lang(next(x for x in episode["actions"] if x.get("type") == "playback").get("availId")), data=next(x for x in episode["actions"] if x.get("type") == "playback"), ) for episode in episodes @@ -448,28 +451,16 @@ class DSNP(Service): def _movie(self, title: str) -> Movie: movie = self.get_page(title) - playback_action = next(x for x in movie["actions"] if x.get("type") == "playback") - original_lang = self.get_original_lang(playback_action.get("availId")) - return [ Movie( id_=movie.get("id"), service=self.__class__, name=movie["visuals"].get("title"), year=movie["visuals"]["metastringParts"].get("releaseYearRange", {}).get("startYear"), - language=original_lang, - data=playback_action, + data=next(x for x in movie["actions"] if x.get("type") == "playback"), ) ] - def get_original_lang(self, availId): - try: - title_lang = self.session.get(f'https://disney.api.edge.bamgrid.com/explore/v1.6/playerExperience/{availId}').json() - original_lang = title_lang["data"]["playerExperience"]["targetLanguage"] - except Exception: - original_lang = "en" - return original_lang - def _request( self, method: str, @@ -478,14 +469,18 @@ class DSNP(Service): headers: dict = None, payload: dict = None, ) -> Any[dict | str]: - _headers = {**self.session.headers, **(headers or {})} - + _headers = headers if headers else self.session.headers prep = self.session.prepare_request(Request(method, endpoint, headers=_headers, params=params, json=payload)) response = self.session.send(prep) - + + # Check for geoblocking + if response.status_code == 404 and 'x-dss-edge' in response.headers: + edge_error = response.headers.get('x-dss-edge', '') + if 'location.invalid' in edge_error: + raise ConnectionError("Disney+ content is geoblocked in your region. Please use a VPN to US/supported region.") + try: data = response.json() - if data.get("errors"): code = data["errors"][0]["extensions"].get("code") @@ -495,38 +490,111 @@ class DSNP(Service): raise ConnectionError("Bad Credentials: " + code) else: raise ConnectionError(data["errors"]) - return data - except Exception: - raise ConnectionError("Request failed: {}".format(response.content)) + except Exception as e: + if response.status_code == 404: + raise ConnectionError(f"Disney+ content not found or geoblocked. Status: {response.status_code}") + raise ConnectionError(f"Request failed. Status: {response.status_code}, Content: {response.content}") def get_page(self, title): + # Use direct Disney+ API URL like Vinetrimmer does - no need for SDK endpoints params = { - "disableSmartFocus": "true", - "limit": 999, - "enhancedContainersLimit": 0, + "disableSmartFocus": True, + "enhancedContainersLimit": 12, + "limit": 24, } - endpoint = self.href( - self.prd_config["services"]["explore"]["client"]["endpoints"]["getPage"]["href"], - version=self.config["EXPLORE_VERSION"], - pageId=title, - ) - return self._request("GET", endpoint, params=params)["data"]["page"] + + # Use exact Vinetrimmer URL pattern + endpoint = f'https://disney.api.edge.bamgrid.com/explore/v1.4/page/{title}' + + response = self.session.get(endpoint, params=params) + if response.status_code == 200: + return response.json()["data"]["page"] + else: + raise ConnectionError(f"Failed to get page data. Status: {response.status_code}") def get_video(self, content_id: str) -> dict: - endpoint = self.href( - self.prd_config["services"]["content"]["client"]["endpoints"]["getDmcVideo"]["href"], contentId=content_id - ) - return self._request("GET", endpoint)["data"]["DmcVideo"] + # Use Vinetrimmer's approach - get manifest directly using playback API + # Add special headers like Vinetrimmer does to avoid geoblocking + original_headers = dict(self.session.headers) + self.session.headers.update({ + 'x-dss-feature-filtering': 'true', + 'x-application-version': '1.1.2', + 'x-bamsdk-client-id': 'disney-svod', + 'x-bamsdk-platform': 'javascript/windows/chrome', + 'x-bamsdk-version': '28.0' + }) + + try: + # Use playback API like Vinetrimmer with exact configuration + # Use exact Vinetrimmer configuration - L3 CAN get 1080p if done correctly + json_data = { + 'playback': { + 'attributes': { + 'resolution': { + 'max': ['1920'], # Same as Vinetrimmer - no artificial L3 limitation + }, + 'protocol': 'HTTPS', + 'assetInsertionStrategy': 'SGAI', + 'playbackInitiationContext': 'ONLINE', + 'frameRates': [60], + }, + }, + 'playbackId': content_id, + } + + # Use exact Vinetrimmer playback URL and scenario + manifest_response = self.session.post( + 'https://disney.playback.edge.bamgrid.com/v7/playback/tv-drm-ctr', + json=json_data + ) + + if manifest_response.status_code == 200: + manifest_data = manifest_response.json() + # Return in expected format + return { + "video": { + "mediaMetadata": { + "playbackUrls": [{"url": manifest_data["stream"]["sources"][0]['complete']['url']}] + } + } + } + else: + # Fallback: try original method but with new headers + endpoint = self.href( + self.prd_config["services"]["content"]["client"]["endpoints"]["getDmcVideo"]["href"], + contentId=content_id + ) + data = self._request("GET", endpoint)["data"]["DmcVideo"] + return data + except Exception as e: + # Final fallback + endpoint = self.href( + self.prd_config["services"]["content"]["client"]["endpoints"]["getDmcVideo"]["href"], + contentId=content_id + ) + data = self._request("GET", endpoint)["data"]["DmcVideo"] + return data + + finally: + # Restore original headers + self.session.headers.clear() + self.session.headers.update(original_headers) def get_deeplink(self, ref_id: str) -> str: + # Use direct Disney+ API URL like Vinetrimmer does params = { "refId": ref_id, "refIdType": "deeplinkId", } - endpoint = "https://disney.content.edge.bamgrid.com/explore/v1.0/deeplink" - return self._request("GET", endpoint, params=params) + endpoint = "https://disney.api.edge.bamgrid.com/explore/v1.3/deeplink" + + response = self.session.get(endpoint, params=params) + if response.status_code == 200: + return response.json() + else: + raise ConnectionError(f"Failed to get deeplink. Status: {response.status_code}") def series_bundle(self, series_id: str) -> dict: endpoint = self.href( @@ -563,14 +631,14 @@ class DSNP(Service): payload = { "variables": { "registerDevice": { - "applicationRuntime": self.config["APPLICATION_RUNTIME"], + "applicationRuntime": self.config.get("BAM_APPLICATION_RUNTIME", "android"), "attributes": { "operatingSystem": "Android", "operatingSystemVersion": "8.1.0", }, - "deviceFamily": self.config["DEVICE_FAMILY"], + "deviceFamily": self.config.get("BAM_FAMILY", "browser"), # Use Vinetrimmer family "deviceLanguage": "en", - "deviceProfile": self.config["DEVICE_PROFILE"], + "deviceProfile": self.config.get("BAM_PROFILE", "tv"), } }, "query": queries.REGISTER_DEVICE, @@ -655,3 +723,5 @@ class DSNP(Service): endpoint = self.prd_config["services"]["orchestration"]["client"]["endpoints"]["query"]["href"] data = self._request("POST", endpoint, payload=payload) return data["extensions"]["sdk"] + + diff --git a/packages/envied/src/envied/services/DSNP/config.yaml b/packages/envied/src/envied/services/DSNP/config.yaml index c0f67d7..c0611d0 100644 --- a/packages/envied/src/envied/services/DSNP/config.yaml +++ b/packages/envied/src/envied/services/DSNP/config.yaml @@ -1,27 +1,29 @@ +# Use exact configuration from working Vinetrimmer implementation CLIENT_ID: disney-svod-3d9324fc -CLIENT_VERSION: "9.10.0" +CLIENT_VERSION: "6.0.4" API_KEY: "ZGlzbmV5JmFuZHJvaWQmMS4wLjA.bkeb0m230uUhv8qrAXuNu39tbE_mD5EEhM_NAcohjyA" -CONFIG_URL: https://bam-sdk-configs.bamgrid.com/bam-sdk/v5.0/disney-svod-3d9324fc/android/v9.10.0/google/tv/prod.json +CONFIG_URL: https://bam-sdk-configs.bamgrid.com/bam-sdk/v3.0/disney-svod-3d9324fc/android/v6.0.0/google/tv/prod.json PAGE_SIZE_SETS: 15 PAGE_SIZE_CONTENT: 30 SEARCH_QUERY_TYPE: ge BAM_PARTNER: disney EXPLORE_VERSION: v1.3 -DEVICE_FAMILY: browser -DEVICE_PROFILE: tv -APPLICATION_RUNTIME: android + +# Use Vinetrimmer family/platform combination that avoids geoblocking +BAM_FAMILY: browser +BAM_PLATFORM: android-tv +BAM_PROFILE: tv +BAM_APPLICATION_RUNTIME: android HEADERS: - User-Agent: BAMSDK/v9.10.0 (disney-svod-3d9324fc 2.26.2-rc1.0; v5.0/v9.10.0; android; tv) + User-Agent: BAMSDK/v6.0.4 (disney-svod-3d9324fc 1.15.0.0; v3.0/v6.0.0; android; tv) google Nexus Player (OPR2.170623.027; Linux; 8.0.0; API 26) x-application-version: google x-bamsdk-platform-id: android-tv x-bamsdk-client-id: disney-svod-3d9324fc x-bamsdk-platform: android-tv - x-bamsdk-version: "9.10.0" + x-bamsdk-version: "6.0.4" Accept-Encoding: gzip - Origin: https://www.disneyplus.com LICENSE: https://disney.playback.edge.bamgrid.com/widevine/v1/obtain-license -PLAYREADY_LICENSE: https://disney.playback.edge.bamgrid.com/playready/v1/obtain-license.asmx diff --git a/packages/envied/src/envied/services/DSNP/queries.py b/packages/envied/src/envied/services/DSNP/queries.py index e1bee92..a3d3246 100644 --- a/packages/envied/src/envied/services/DSNP/queries.py +++ b/packages/envied/src/envied/services/DSNP/queries.py @@ -11,3 +11,5 @@ LOGIN = """mutation loginTv($input: LoginInput!) { login(login: $input) { __type LOGIN_OTP = """mutation authenticateWithOtp($input: AuthenticateWithOtpInput!) { authenticateWithOtp(authenticateWithOtp: $input) { actionGrant securityAction } }""" LOGIN_ACTION_GRANT = """\n mutation loginWithActionGrant($input: LoginWithActionGrantInput!) {\n loginWithActionGrant(login: $input) {\n account {\n ...account\n\n profiles {\n ...profile\n }\n }\n activeSession {\n ...session\n }\n identity {\n ...identity\n }\n }\n }\n\n \nfragment identity on Identity {\n attributes {\n securityFlagged\n createdAt\n passwordResetRequired\n }\n flows {\n marketingPreferences {\n eligibleForOnboarding\n isOnboarded\n }\n personalInfo {\n eligibleForCollection\n requiresCollection\n }\n }\n personalInfo {\n dateOfBirth\n gender\n }\n subscriber {\n subscriberStatus\n subscriptionAtRisk\n overlappingSubscription\n doubleBilled\n doubleBilledProviders\n subscriptions {\n id\n groupId\n state\n partner\n isEntitled\n source {\n sourceType\n sourceProvider\n sourceRef\n subType\n }\n paymentProvider\n product {\n id\n sku\n offerId\n promotionId\n name\n nextPhase {\n sku\n offerId\n campaignCode\n voucherCode\n }\n entitlements {\n id\n name\n desc\n partner\n }\n categoryCodes\n redeemed {\n campaignCode\n redemptionCode\n voucherCode\n }\n bundle\n bundleType\n subscriptionPeriod\n earlyAccess\n trial {\n duration\n }\n }\n term {\n purchaseDate\n startDate\n expiryDate\n nextRenewalDate\n pausedDate\n churnedDate\n isFreeTrial\n }\n externalSubscriptionId,\n cancellation {\n type\n restartEligible\n }\n stacking {\n status\n overlappingSubscriptionProviders\n previouslyStacked\n previouslyStackedByProvider\n }\n }\n }\n}\n\n \nfragment account on Account {\n id\n attributes {\n blocks {\n expiry\n reason\n }\n consentPreferences {\n dataElements {\n name\n value\n }\n purposes {\n consentDate\n firstTransactionDate\n id\n lastTransactionCollectionPointId\n lastTransactionCollectionPointVersion\n lastTransactionDate\n name\n status\n totalTransactionCount\n version\n }\n }\n dssIdentityCreatedAt\n email\n emailVerified\n lastSecurityFlaggedAt\n locations {\n manual {\n country\n }\n purchase {\n country\n source\n }\n registration {\n geoIp {\n country\n }\n }\n }\n securityFlagged\n tags\n taxId\n userVerified\n }\n parentalControls {\n isProfileCreationProtected\n }\n flows {\n star {\n isOnboarded\n }\n }\n}\n\n \nfragment profile on Profile {\n id\n name\n isAge21Verified\n attributes {\n avatar {\n id\n userSelected\n }\n isDefault\n kidsModeEnabled\n languagePreferences {\n appLanguage\n playbackLanguage\n preferAudioDescription\n preferSDH\n subtitleAppearance {\n backgroundColor\n backgroundOpacity\n description\n font\n size\n textColor\n }\n subtitleLanguage\n subtitlesEnabled\n }\n groupWatch {\n enabled\n }\n parentalControls {\n kidProofExitEnabled\n isPinProtected\n }\n playbackSettings {\n autoplay\n backgroundVideo\n prefer133\n preferImaxEnhancedVersion\n previewAudioOnHome\n previewVideoOnHome\n }\n }\n personalInfo {\n dateOfBirth\n gender\n age\n }\n maturityRating {\n ...maturityRating\n }\n personalInfo {\n dateOfBirth\n age\n gender\n }\n flows {\n personalInfo {\n eligibleForCollection\n requiresCollection\n }\n star {\n eligibleForOnboarding\n isOnboarded\n }\n }\n}\n\n\nfragment maturityRating on MaturityRating {\n ratingSystem\n ratingSystemValues\n contentMaturityRating\n maxRatingSystemValue\n isMaxContentMaturityRating\n}\n\n\n \nfragment session on Session {\n device {\n id\n platform\n }\n entitlements\n features {\n coPlay\n }\n inSupportedLocation\n isSubscriber\n location {\n type\n countryCode\n dma\n asn\n regionName\n connectionType\n zipCode\n }\n sessionId\n experiments {\n featureId\n variantId\n version\n }\n identity {\n id\n }\n account {\n id\n }\n profile {\n id\n parentalControls {\n liveAndUnratedContent {\n enabled\n }\n }\n }\n partnerName\n preferredMaturityRating {\n impliedMaturityRating\n ratingSystem\n }\n homeLocation {\n countryCode\n }\n portabilityLocation {\n countryCode\n type\n }\n}\n\n""" REFRESH_TOKEN = """mutation refreshToken($input:RefreshTokenInput!) { refreshToken(refreshToken:$input) { activeSession{sessionId} } }""" + + diff --git a/packages/envied/src/envied/unshackle-example.yaml b/packages/envied/src/envied/unshackle-example.yaml new file mode 100644 index 0000000..d6e08c2 --- /dev/null +++ b/packages/envied/src/envied/unshackle-example.yaml @@ -0,0 +1,504 @@ +# API key for The Movie Database (TMDB) +tmdb_api_key: "" + +# Client ID for SIMKL API (optional, improves metadata matching) +# Get your free client ID at: https://simkl.com/settings/developer/ +simkl_client_id: "" + +# Group or Username to postfix to the end of all download filenames following a dash +tag: user_tag + +# Enable/disable tagging with group name (default: true) +tag_group_name: true + +# Enable/disable tagging with IMDB/TMDB/TVDB details (default: true) +tag_imdb_tmdb: true + +# Set terminal background color (custom option not in CONFIG.md) +set_terminal_bg: false + +# Set file naming convention +# true for style - Prime.Suspect.S07E01.The.Final.Act.Part.One.1080p.ITV.WEB-DL.AAC2.0.H.264 +# false for style - Prime Suspect S07E01 The Final Act - Part One +scene_naming: true + +# Whether to include the year in series names for episodes and folders (default: true) +# true for style - Show Name (2023) S01E01 Episode Name +# false for style - Show Name S01E01 Episode Name +series_year: true + +# Check for updates from GitHub repository on startup (default: true) +update_checks: true + +# How often to check for updates, in hours (default: 24) +update_check_interval: 24 + +# Title caching configuration +# Cache title metadata to reduce redundant API calls +title_cache_enabled: true # Enable/disable title caching globally (default: true) +title_cache_time: 1800 # Cache duration in seconds (default: 1800 = 30 minutes) +title_cache_max_retention: 86400 # Maximum cache retention for fallback when API fails (default: 86400 = 24 hours) + +# Debug logging configuration +# Comprehensive JSON-based debug logging for troubleshooting and service development +debug: + false # Enable structured JSON debug logging (default: false) + # When enabled with --debug flag or set to true: + # - Creates JSON Lines (.jsonl) log files with complete debugging context + # - Logs: session info, CLI params, service config, CDM details, authentication, + # titles, tracks metadata, DRM operations, vault queries, errors with stack traces + # - File location: logs/unshackle_debug_{service}_{timestamp}.jsonl + # - Also creates text log: logs/unshackle_root_{timestamp}.log + +debug_keys: + false # Log decryption keys in debug logs (default: false) + # Set to true to include actual decryption keys in logs + # Useful for debugging key retrieval and decryption issues + # SECURITY NOTE: Passwords, tokens, cookies, and session tokens + # are ALWAYS redacted regardless of this setting + # Only affects: content_key, key fields (the actual CEKs) + # Never affects: kid, keys_count, key_id (metadata is always logged) + +# Muxing configuration +muxing: + set_title: false + +# Login credentials for each Service +credentials: + # Direct credentials (no profile support) + EXAMPLE: email@example.com:password + + # Per-profile credentials with default fallback + SERVICE_NAME: + default: default@email.com:password # Used when no -p/--profile is specified + profile1: user1@email.com:password1 + profile2: user2@email.com:password2 + + # Per-profile credentials without default (requires -p/--profile) + SERVICE_NAME2: + john: john@example.com:johnspassword + jane: jane@example.com:janespassword + + # You can also use list format for passwords with special characters + SERVICE_NAME3: + default: ["user@email.com", ":PasswordWith:Colons"] + +# Override default directories used across unshackle +directories: + cache: Cache + cookies: Cookies + dcsl: DCSL # Device Certificate Status List + downloads: Downloads + logs: Logs + temp: Temp + wvds: WVDs + prds: PRDs + # Additional directories that can be configured: + # commands: Commands + services: + - /path/to/services + - /other/path/to/services + # vaults: Vaults + # fonts: Fonts + +# Pre-define which Widevine or PlayReady device to use for each Service +cdm: + # Global default CDM device (fallback for all services/profiles) + default: WVD_1 + + # Direct service-specific CDM + DIFFERENT_EXAMPLE: PRD_1 + + # Per-profile CDM configuration + EXAMPLE: + john_sd: chromecdm_903_l3 # Profile 'john_sd' uses Chrome CDM L3 + jane_uhd: nexus_5_l1 # Profile 'jane_uhd' uses Nexus 5 L1 + default: generic_android_l3 # Default CDM for this service + + # NEW: Quality-based CDM selection + # Use different CDMs based on video resolution + # Supports operators: >=, >, <=, <, or exact match + EXAMPLE_QUALITY: + "<=1080": generic_android_l3 # Use L3 for 1080p and below + ">1080": nexus_5_l1 # Use L1 for above 1080p (1440p, 2160p) + default: generic_android_l3 # Optional: fallback if no quality match + + # You can mix profiles and quality thresholds in the same service + NETFLIX: + # Profile-based selection (existing functionality) + john: netflix_l3_profile + jane: netflix_l1_profile + # Quality-based selection (new functionality) + "<=720": netflix_mobile_l3 + "1080": netflix_standard_l3 + ">=1440": netflix_premium_l1 + # Fallback + default: netflix_standard_l3 + +# Use pywidevine Serve-compliant Remote CDMs + +# Example: Custom CDM API Configuration +# This demonstrates the highly configurable custom_api type that can adapt to any CDM API format +# - name: "chrome" +# type: "custom_api" +# host: "http://remotecdm.test/" +# timeout: 30 +# device: +# name: "ChromeCDM" +# type: "CHROME" +# system_id: 34312 +# security_level: 3 +# auth: +# type: "header" +# header_name: "x-api-key" +# key: "YOUR_API_KEY_HERE" +# custom_headers: +# User-Agent: "Unshackle/2.0.0" +# endpoints: +# get_request: +# path: "/get-challenge" +# method: "POST" +# timeout: 30 +# decrypt_response: +# path: "/get-keys" +# method: "POST" +# timeout: 30 +# request_mapping: +# get_request: +# param_names: +# scheme: "device" +# init_data: "init_data" +# static_params: +# scheme: "Widevine" +# decrypt_response: +# param_names: +# scheme: "device" +# license_request: "license_request" +# license_response: "license_response" +# static_params: +# scheme: "Widevine" +# response_mapping: +# get_request: +# fields: +# challenge: "challenge" +# session_id: "session_id" +# message: "message" +# message_type: "message_type" +# response_types: +# - condition: "message_type == 'license-request'" +# type: "license_request" +# success_conditions: +# - "message == 'success'" +# decrypt_response: +# fields: +# keys: "keys" +# message: "message" +# key_fields: +# kid: "kid" +# key: "key" +# type: "type" +# success_conditions: +# - "message == 'success'" +# caching: +# enabled: true +# use_vaults: true +# check_cached_first: true + +remote_cdm: + - name: "chrome" + device_name: chrome + device_type: CHROME + system_id: 27175 + security_level: 3 + host: https://domain.com/api + secret: secret_key + - name: "chrome-2" + device_name: chrome + device_type: CHROME + system_id: 26830 + security_level: 3 + host: https://domain-2.com/api + secret: secret_key + + - name: "decrypt_labs_chrome" + type: "decrypt_labs" # Required to identify as DecryptLabs CDM + device_name: "ChromeCDM" # Scheme identifier - must match exactly + device_type: CHROME + system_id: 4464 # Doesn't matter + security_level: 3 + host: "https://keyxtractor.decryptlabs.com" + secret: "your_decrypt_labs_api_key_here" # Replace with your API key + - name: "decrypt_labs_l1" + type: "decrypt_labs" + device_name: "L1" # Scheme identifier - must match exactly + device_type: ANDROID + system_id: 4464 + security_level: 1 + host: "https://keyxtractor.decryptlabs.com" + secret: "your_decrypt_labs_api_key_here" + + - name: "decrypt_labs_l2" + type: "decrypt_labs" + device_name: "L2" # Scheme identifier - must match exactly + device_type: ANDROID + system_id: 4464 + security_level: 2 + host: "https://keyxtractor.decryptlabs.com" + secret: "your_decrypt_labs_api_key_here" + + - name: "decrypt_labs_playready_sl2" + type: "decrypt_labs" + device_name: "SL2" # Scheme identifier - must match exactly + device_type: PLAYREADY + system_id: 0 + security_level: 2000 + host: "https://keyxtractor.decryptlabs.com" + secret: "your_decrypt_labs_api_key_here" + + - name: "decrypt_labs_playready_sl3" + type: "decrypt_labs" + device_name: "SL3" # Scheme identifier - must match exactly + device_type: PLAYREADY + system_id: 0 + security_level: 3000 + host: "https://keyxtractor.decryptlabs.com" + secret: "your_decrypt_labs_api_key_here" + +# Key Vaults store your obtained Content Encryption Keys (CEKs) +# Use 'no_push: true' to prevent a vault from receiving pushed keys +# while still allowing it to provide keys when requested +key_vaults: + - type: SQLite + name: Local + path: key_store.db + # Additional vault types: + # - type: API + # name: "Remote Vault" + # uri: "https://key-vault.example.com" + # token: "secret_token" + # no_push: true # This vault will only provide keys, not receive them + # - type: MySQL + # name: "MySQL Vault" + # host: "127.0.0.1" + # port: 3306 + # database: vault + # username: user + # password: pass + # no_push: false # Default behavior - vault both provides and receives keys + +# Choose what software to use to download data +downloader: aria2c +# Options: requests | aria2c | curl_impersonate | n_m3u8dl_re +# Can also be a mapping: +# downloader: +# NF: requests +# AMZN: n_m3u8dl_re +# DSNP: n_m3u8dl_re +# default: requests + +# aria2c downloader configuration +aria2c: + max_concurrent_downloads: 4 + max_connection_per_server: 3 + split: 5 + file_allocation: falloc # none | prealloc | falloc | trunc + +# N_m3u8DL-RE downloader configuration +n_m3u8dl_re: + thread_count: 16 + ad_keyword: "advertisement" + use_proxy: true + +# curl_impersonate downloader configuration +curl_impersonate: + browser: chrome120 + +# Pre-define default options and switches of the dl command +dl: + sub_format: srt + downloads: 4 + workers: 16 + lang: + - en + - fr + EXAMPLE: + bitrate: CBR + +# Chapter Name to use when exporting a Chapter without a Name +chapter_fallback_name: "Chapter {j:02}" + +# Case-Insensitive dictionary of headers for all Services +headers: + Accept-Language: "en-US,en;q=0.8" + User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36" + +# Override default filenames used across unshackle +filenames: + debug_log: "unshackle_debug_{service}_{time}.jsonl" # JSON Lines debug log file + config: "config.yaml" + root_config: "envied.yaml" + chapters: "Chapters_{title}_{random}.txt" + subtitle: "Subtitle_{id}_{language}.srt" + +# conversion_method: +# - auto (default): Smart routing - subby for WebVTT/SAMI, pycaption for others +# - subby: Always use subby with advanced processing +# - pycaption: Use only pycaption library (no SubtitleEdit, no subby) +# - subtitleedit: Prefer SubtitleEdit when available, fall back to pycaption +# - pysubs2: Use pysubs2 library (supports SRT/SSA/ASS/WebVTT/TTML/SAMI/MicroDVD/MPL2/TMP) +subtitle: + conversion_method: auto + # sdh_method: Method to use for SDH (hearing impaired) stripping + # - auto (default): Try subby (SRT only), then SubtitleEdit (if available), then subtitle-filter + # - subby: Use subby library (SRT only) + # - subtitleedit: Use SubtitleEdit tool (Windows only, falls back to subtitle-filter) + # - filter-subs: Use subtitle-filter library directly + sdh_method: auto + # strip_sdh: Automatically create stripped (non-SDH) versions of SDH subtitles + # Set to false to disable automatic SDH stripping entirely (default: true) + strip_sdh: true + # convert_before_strip: Auto-convert VTT/other formats to SRT before using subtitle-filter + # This ensures compatibility when subtitle-filter is used as fallback (default: true) + convert_before_strip: true + # preserve_formatting: Preserve original subtitle formatting (tags, positioning, styling) + # When true, skips pycaption processing for WebVTT files to keep tags like , , positioning intact + # Combined with no sub_format setting, ensures subtitles remain in their original format (default: true) + preserve_formatting: true + +# Configuration for pywidevine's serve functionality +serve: + api_secret: "your-secret-key-here" + users: + secret_key_for_user: + devices: + - generic_nexus_4464_l3 + username: user + # devices: + # - '/path/to/device.wvd' + +# Configuration data for each Service +services: + # Service-specific configuration goes here + # Profile-specific configurations can be nested under service names + + # You can override ANY global configuration option on a per-service basis + # This allows fine-tuned control for services with special requirements + # Supported overrides: dl, aria2c, n_m3u8dl_re, curl_impersonate, subtitle, muxing, headers, etc. + + # Example: Comprehensive service configuration showing all features + EXAMPLE: + # Standard service config + api_key: "service_api_key" + + # Service certificate for Widevine L1/L2 (base64 encoded) + # This certificate is automatically used when L1/L2 schemes are selected + # Services obtain this from their DRM provider or license server + certificate: | + CAUSwwUKvQIIAxIQ5US6QAvBDzfTtjb4tU/7QxiH8c+TBSKOAjCCAQoCggEBAObzvlu2hZRsapAPx4Aa4GUZj4/GjxgXUtBH4THSkM40x63wQeyVxlEEo + # ... (full base64 certificate here) + + # Profile-specific device configurations + profiles: + john_sd: + device: + app_name: "AIV" + device_model: "SHIELD Android TV" + jane_uhd: + device: + app_name: "AIV" + device_model: "Fire TV Stick 4K" + + # NEW: Configuration overrides (can be combined with profiles and certificates) + # Override dl command defaults for this service + dl: + downloads: 4 # Limit concurrent track downloads (global default: 6) + workers: 8 # Reduce workers per track (global default: 16) + lang: ["en", "es-419"] # Different language priority for this service + sub_format: srt # Force SRT subtitle format + + # Override n_m3u8dl_re downloader settings + n_m3u8dl_re: + thread_count: 8 # Lower thread count for rate-limited service (global default: 16) + use_proxy: true # Force proxy usage for this service + retry_count: 10 # More retries for unstable connections + ad_keyword: "advertisement" # Service-specific ad filtering + + # Override aria2c downloader settings + aria2c: + max_concurrent_downloads: 2 # Limit concurrent downloads (global default: 4) + max_connection_per_server: 1 # Single connection per server + split: 3 # Fewer splits (global default: 5) + file_allocation: none # Faster allocation for this service + + # Override subtitle processing for this service + subtitle: + conversion_method: pycaption # Use specific subtitle converter + sdh_method: auto + + # Service-specific headers + headers: + User-Agent: "Service-specific user agent string" + Accept-Language: "en-US,en;q=0.9" + + # Override muxing options + muxing: + set_title: true + + # Example: Service with different regions per profile + SERVICE_NAME: + profiles: + us_account: + region: "US" + api_endpoint: "https://api.us.service.com" + uk_account: + region: "GB" + api_endpoint: "https://api.uk.service.com" + + # Example: Rate-limited service + RATE_LIMITED_SERVICE: + dl: + downloads: 2 # Limit concurrent downloads + workers: 4 # Reduce workers to avoid rate limits + n_m3u8dl_re: + thread_count: 4 # Very low thread count + retry_count: 20 # More retries for flaky service + aria2c: + max_concurrent_downloads: 1 # Download tracks one at a time + max_connection_per_server: 1 # Single connection only + + # Notes on service-specific overrides: + # - Overrides are merged with global config, not replaced + # - Only specified keys are overridden, others use global defaults + # - Reserved keys (profiles, api_key, certificate, etc.) are NOT treated as overrides + # - Any dict-type config option can be overridden (dl, aria2c, n_m3u8dl_re, subtitle, etc.) + # - CLI arguments always take priority over service-specific config + +# External proxy provider services +proxy_providers: + nordvpn: + username: username_from_service_credentials + password: password_from_service_credentials + server_map: + us: 12 # force US server #12 for US proxies + surfsharkvpn: + username: your_surfshark_service_username # Service credentials from https://my.surfshark.com/vpn/manual-setup/main/openvpn + password: your_surfshark_service_password # Service credentials (not your login password) + server_map: + us: 3844 # force US server #3844 for US proxies + gb: 2697 # force GB server #2697 for GB proxies + au: 4621 # force AU server #4621 for AU proxies + windscribevpn: + username: your_windscribe_username # Service credentials from https://windscribe.com/getconfig/openvpn + password: your_windscribe_password # Service credentials (not your login password) + server_map: + us: "us-central-096.totallyacdn.com" # force US server + gb: "uk-london-055.totallyacdn.com" # force GB server + basic: + GB: + - "socks5://username:password@bhx.socks.ipvanish.com:1080" # 1 (Birmingham) + - "socks5://username:password@gla.socks.ipvanish.com:1080" # 2 (Glasgow) + AU: + - "socks5://username:password@syd.socks.ipvanish.com:1080" # 1 (Sydney) + - "https://username:password@au-syd.prod.surfshark.com" # 2 (Sydney) + - "https://username:password@au-bne.prod.surfshark.com" # 3 (Brisbane) + BG: "https://username:password@bg-sof.prod.surfshark.com" diff --git a/packages/envied/src/envied/unshackle.yaml b/packages/envied/src/envied/unshackle.yaml new file mode 100644 index 0000000..c7e5def --- /dev/null +++ b/packages/envied/src/envied/unshackle.yaml @@ -0,0 +1,155 @@ +# Group or Username to postfix to the end of all download fienames following a dash +#tag: '' + +# Set terminal background color (custom option not in CONFIG.md) +set_terminal_bg: false + +# Muxing configuration +muxing: + set_title: false + +# +decryption: mp4decrypt + +scene_naming: false +series_year: false + +# Widevine pssh display; fold, crop or ellipsis +pssh_display: ellipsis + +# Login credentials for each Service +credentials: + ALL4: angela.slaney@gmail.com:Nood13sh + ROKU: moretonplumber@yandex.com:$WPJ~Mdh2:X@EtB + TVNZ: moretonplumber@yandex.com:penumbra + TPTV: moretonplumber@yandex.com:penumbra1234 + CBC: moretonplumber@yandex.com:penumbra +# Override default directories used across unshackle +directories: + cache: /home/angela/.local/tmp/devine + cookies: /home/angela/.local/share/devine/cookies + dcsl: DCSL # Device Certificate Status List + downloads: /home/angela/Downloads/devine + logs: /home/angela/.local/tmp/devine/ + temp: /home/angela/.local/tmp/devine/ + wvds: /home/angela/.local/share/devine/WVDs/ + prds: PRDs + User_Configs: /home/angela/.local/share/devine/ + # Additional directories that can be configured: + # commands: Commands + services: + - /home/angela/Programming/gits/production/envied/unshackle/services/ + vaults: /home/angela/.local/share/devine/vaults/ + # fonts: Fonts + +# Pre-define which Widevine or PlayReady device to use for each Service +cdm: + default: device + +# Use pywidevine Serve-compliant Remote CDMs +#remote_cdm: +# - name: "CDRM_Project_API" +# device_type: ANDROID +# system_id: 22590 +# security_level: 3 +# host: "https://cdrm-project.com/remotecdm/widevine" +# secret: "CDRM" +# device_name: "public" + + +# Key Vaults store your obtained Content Encryption Keys (CEKs) +key_vaults: +- type: SQLite + name: Local vault + path: /home/angela/Programming/gits/devine-services/key_store.db + +- type: HTTPAPI + name: drmlab + host: http://api.drmlab.io/vault/ + password: gEX75q7I5YVkvgF5SUkcNd41IbGrDtTT +#- type: API +# name: "Online Vault" +# uri: "https://cdrm-project.com/api/cache" +# token: "CDRM" + + +# Choose what software to use to download data +downloader: n_m3u8dl_re +# Options: requests | aria2c | curl_impersonate | n_m3u8dl_re +# Can also be a mapping: +# downloader: +# NF: requests +# AMZN: n_m3u8dl_re +# DSNP: n_m3u8dl_re +# default: requests + +# aria2c downloader configuration +aria2c: + max_concurrent_downloads: 4 + max_connection_per_server: 3 + split: 5 + file_allocation: falloc # none | prealloc | falloc | trunc + +# N_m3u8DL-RE downloader configuration +n_m3u8dl_re: + thread_count: 16 + ad_keyword: "advertisement" + use_proxy: true + +# curl_impersonate downloader configuration +curl_impersonate: + browser: chrome120 + +# Pre-define default options and switches of the dl command +dl: + best: true + sub_format: srt + downloads: 4 + workers: 16 + +# Chapter Name to use when exporting a Chapter without a Name +chapter_fallback_name: "Chapter {j:02}" + +# Case-Insensitive dictionary of headers for all Services +headers: + Accept-Language: "en-US,en;q=0.8" + User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36" + +# Override default filenames used across unshackle +filenames: + log: "unshackle_{name}_{time}.log" + config: "config.yaml" + root_config: "envied.yaml" + chapters: "Chapters_{title}_{random}.txt" + subtitle: "Subtitle_{id}_{language}.srt" + +# API key for The Movie Database (TMDB) +tmdb_api_key: "" + + +# conversion_method: +# - auto (default): Smart routing - subby for WebVTT/SAMI, standard for others +# - subby: Always use subby with advanced processing +# - pycaption: Use only pycaption library (no SubtitleEdit, no subby) +# - subtitleedit: Prefer SubtitleEdit when available, fall back to pycaption +subtitle: + conversion_method: auto + sdh_method: auto + + +# Configuration data for each Service +services: + # Service-specific configuration goes here + # EXAMPLE: + # api_key: "service_specific_key" + +# Legacy NordVPN configuration (use proxy_providers instead) +proxy_providers: + nordvpn: + username: gBM9ejjmY1J15jccH12VT7Qy + password: 1kmuCrRXnT55uAjiUTd8twv9 + server_map: + us: 6918 + uk: 2613 + nz: 100 + diff --git a/uv.lock b/uv.lock index ec7cffb..2b9ec23 100644 --- a/uv.lock +++ b/uv.lock @@ -576,6 +576,7 @@ dependencies = [ { name = "protobuf" }, { name = "pycaption" }, { name = "pycryptodomex" }, + { name = "pyexecjs" }, { name = "pyjwt" }, { name = "pymediainfo" }, { name = "pymp4" }, @@ -611,7 +612,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "aiohttp-swagger3", specifier = ">=0.10.0" }, + { name = "aiohttp-swagger3", specifier = ">=0.10.0,<1" }, { name = "appdirs", specifier = ">=1.4.4,<2" }, { name = "beaupy", specifier = ">=3.10.2" }, { name = "brotli", specifier = ">=1.1.0,<2" }, @@ -619,23 +620,24 @@ requires-dist = [ { name = "click", specifier = ">=8.1.8,<9" }, { name = "construct", specifier = ">=2.8.8,<3" }, { name = "crccheck", specifier = ">=1.3.0,<2" }, - { name = "cryptography", specifier = ">=45.0.0" }, - { name = "curl-cffi", specifier = ">=0.7.0b4,<0.8" }, - { name = "fonttools", specifier = ">=4.60.1" }, + { name = "cryptography", specifier = ">=45.0.0,<47" }, + { name = "curl-cffi", specifier = ">=0.7.0b4,<0.14" }, + { name = "fonttools", specifier = ">=4.60.1,<5" }, { name = "httpx", specifier = ">=0.28.1,<0.29" }, { name = "isodate", specifier = ">=0.7.2" }, { name = "jsonpickle", specifier = ">=3.0.4,<4.5" }, { name = "jwt", specifier = ">=1.4.0" }, { name = "langcodes", specifier = ">=3.4.0,<4" }, { name = "langcodes", specifier = ">=3.5.0" }, - { name = "lxml", specifier = ">=6.0.2" }, + { name = "lxml", specifier = ">=6.0.2,<7" }, { name = "mypy", specifier = ">=1.18.2,<2" }, { name = "pproxy", specifier = ">=2.7.9,<3" }, { name = "protobuf", specifier = ">=4.25.3,<7" }, { name = "pycaption", specifier = ">=2.2.6,<3" }, { name = "pycryptodomex", specifier = ">=3.20.0,<4" }, + { name = "pyexecjs", specifier = ">=1.5.1,<2" }, { name = "pyjwt", specifier = ">=2.8.0,<3" }, - { name = "pymediainfo", specifier = ">=6.1.0,<7.5" }, + { name = "pymediainfo", specifier = ">=6.1.0,<8" }, { name = "pymp4", specifier = ">=1.4.0,<2" }, { name = "pymysql", specifier = ">=1.1.0,<2" }, { name = "pyplayready", specifier = ">=0.6.3,<0.7" }, @@ -1502,6 +1504,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/66/0e/9ee7bc0b48ec45d93b302fa2d787830dca4dc454d31a237faa5815995988/PyDispatcher-2.0.7-py3-none-any.whl", hash = "sha256:96543bea04115ffde08f851e1d45cacbfd1ee866ac42127d9b476dc5aefa7de0", size = 12040, upload-time = "2023-02-17T20:11:11.991Z" }, ] +[[package]] +name = "pyexecjs" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ba/8e/aedef81641c8dca6fd0fb7294de5bed9c45f3397d67fddf755c1042c2642/PyExecJS-1.5.1.tar.gz", hash = "sha256:34cc1d070976918183ff7bdc0ad71f8157a891c92708c00c5fbbff7a769f505c", size = 13344, upload-time = "2018-01-18T04:33:55.126Z" } + [[package]] name = "pygments" version = "2.19.2" diff --git a/vaults/API.py b/vaults/API.py index 2cc5b4a..5049c1d 100644 --- a/vaults/API.py +++ b/vaults/API.py @@ -14,7 +14,7 @@ class API(Vault): super().__init__(name, no_push) self.uri = uri.rstrip("/") self.session = Session() - self.session.headers.update({"User-Agent": f"envied v{__version__}"}) + self.session.headers.update({"User-Agent": f"unshackle v{__version__}"}) self.session.headers.update({"Authorization": f"Bearer {token}"}) def get_key(self, kid: Union[UUID, str], service: str) -> Optional[str]: diff --git a/vaults/HTTP.py b/vaults/HTTP.py index 2fd171b..6c38079 100644 --- a/vaults/HTTP.py +++ b/vaults/HTTP.py @@ -16,13 +16,21 @@ class InsertResult(Enum): class HTTP(Vault): - """Key Vault using HTTP API with support for both query parameters and JSON payloads.""" + """ + Key Vault using HTTP API with support for multiple API modes. + + Supported modes: + - query: Uses GET requests with query parameters + - json: Uses POST requests with JSON payloads + - decrypt_labs: Uses DecryptLabs API format (read-only) + """ def __init__( self, name: str, host: str, - password: str, + password: Optional[str] = None, + api_key: Optional[str] = None, username: Optional[str] = None, api_mode: str = "query", no_push: bool = False, @@ -34,25 +42,33 @@ class HTTP(Vault): name: Vault name host: Host URL password: Password for query mode or API token for json mode - username: Username (required for query mode, ignored for json mode) - api_mode: "query" for query parameters or "json" for JSON API + api_key: API key (alternative to password, used for decrypt_labs mode) + username: Username (required for query mode, ignored for json/decrypt_labs mode) + api_mode: "query" for query parameters, "json" for JSON API, or "decrypt_labs" for DecryptLabs API no_push: If True, this vault will not receive pushed keys """ super().__init__(name, no_push) self.url = host - self.password = password + self.password = api_key or password + if not self.password: + raise ValueError("Either password or api_key is required") + self.username = username self.api_mode = api_mode.lower() self.current_title = None self.session = Session() - self.session.headers.update({"User-Agent": f"envied v{__version__}"}) + self.session.headers.update({"User-Agent": f"unshackle v{__version__}"}) self.api_session_id = None + if self.api_mode == "decrypt_labs": + self.session.headers.update({"decrypt-labs-api-key": self.password}) + self.no_push = True + # Validate configuration based on mode if self.api_mode == "query" and not self.username: raise ValueError("Username is required for query mode") - elif self.api_mode not in ["query", "json"]: - raise ValueError("api_mode must be either 'query' or 'json'") + elif self.api_mode not in ["query", "json", "decrypt_labs"]: + raise ValueError("api_mode must be either 'query', 'json', or 'decrypt_labs'") def request(self, method: str, params: dict = None) -> dict: """Make a request to the JSON API vault.""" @@ -95,7 +111,51 @@ class HTTP(Vault): if isinstance(kid, UUID): kid = kid.hex - if self.api_mode == "json": + if self.api_mode == "decrypt_labs": + try: + request_payload = {"service": service.lower(), "kid": kid} + + response = self.session.post(self.url, json=request_payload) + + if not response.ok: + return None + + data = response.json() + + if data.get("message") != "success": + return None + + cached_keys = data.get("cached_keys") + if not cached_keys: + return None + + if isinstance(cached_keys, str): + try: + cached_keys = json.loads(cached_keys) + except json.JSONDecodeError: + return cached_keys + + if isinstance(cached_keys, dict): + if cached_keys.get("kid") == kid: + return cached_keys.get("key") + if kid in cached_keys: + return cached_keys[kid] + elif isinstance(cached_keys, list): + for entry in cached_keys: + if isinstance(entry, dict): + if entry.get("kid") == kid: + return entry.get("key") + elif isinstance(entry, str) and ":" in entry: + entry_kid, entry_key = entry.split(":", 1) + if entry_kid == kid: + return entry_key + + except Exception as e: + print(f"Failed to get key from DecryptLabs ({e.__class__.__name__}: {e})") + return None + return None + + elif self.api_mode == "json": try: params = { "kid": kid, @@ -132,7 +192,9 @@ class HTTP(Vault): return data["keys"][0]["key"] def get_keys(self, service: str) -> Iterator[tuple[str, str]]: - if self.api_mode == "json": + if self.api_mode == "decrypt_labs": + return iter([]) + elif self.api_mode == "json": # JSON API doesn't support getting all keys, so return empty iterator # This will cause the copy command to rely on the API's internal duplicate handling return iter([]) @@ -153,6 +215,9 @@ class HTTP(Vault): if not key or key.count("0") == len(key): raise ValueError("You cannot add a NULL Content Key to a Vault.") + if self.api_mode == "decrypt_labs": + return False + if isinstance(kid, UUID): kid = kid.hex @@ -192,6 +257,9 @@ class HTTP(Vault): return data.get("status_code") == 200 def add_keys(self, service: str, kid_keys: dict[Union[UUID, str], str]) -> int: + if self.api_mode == "decrypt_labs": + return 0 + for kid, key in kid_keys.items(): if not key or key.count("0") == len(key): raise ValueError("You cannot add a NULL Content Key to a Vault.") @@ -243,7 +311,9 @@ class HTTP(Vault): return inserted_count def get_services(self) -> Iterator[str]: - if self.api_mode == "json": + if self.api_mode == "decrypt_labs": + return iter([]) + elif self.api_mode == "json": try: response = self.request("GetServices") services = response.get("services", []) @@ -283,6 +353,9 @@ class HTTP(Vault): if not key or key.count("0") == len(key): raise ValueError("You cannot add a NULL Content Key to a Vault.") + if self.api_mode == "decrypt_labs": + return InsertResult.FAILURE + if isinstance(kid, UUID): kid = kid.hex diff --git a/vaults/MySQL.py b/vaults/MySQL.py index e9c33c8..ab5bde3 100644 --- a/vaults/MySQL.py +++ b/vaults/MySQL.py @@ -28,26 +28,33 @@ class MySQL(Vault): raise PermissionError(f"MySQL vault {self.slug} has no SELECT permission.") def get_key(self, kid: Union[UUID, str], service: str) -> Optional[str]: - if not self.has_table(service): - # no table, no key, simple - return None - if isinstance(kid, UUID): kid = kid.hex + service_variants = [service] + if service != service.lower(): + service_variants.append(service.lower()) + if service != service.upper(): + service_variants.append(service.upper()) + conn = self.conn_factory.get() cursor = conn.cursor() try: - cursor.execute( - # TODO: SQL injection risk - f"SELECT `id`, `key_` FROM `{service}` WHERE `kid`=%s AND `key_`!=%s", - (kid, "0" * 32), - ) - cek = cursor.fetchone() - if not cek: - return None - return cek["key_"] + for service_name in service_variants: + if not self.has_table(service_name): + continue + + cursor.execute( + # TODO: SQL injection risk + f"SELECT `id`, `key_` FROM `{service_name}` WHERE `kid`=%s AND `key_`!=%s", + (kid, "0" * 32), + ) + cek = cursor.fetchone() + if cek: + return cek["key_"] + + return None finally: cursor.close() @@ -131,16 +138,27 @@ class MySQL(Vault): if any(isinstance(kid, UUID) for kid, key_ in kid_keys.items()): kid_keys = {kid.hex if isinstance(kid, UUID) else kid: key_ for kid, key_ in kid_keys.items()} + if not kid_keys: + return 0 + conn = self.conn_factory.get() cursor = conn.cursor() try: + placeholders = ",".join(["%s"] * len(kid_keys)) + cursor.execute(f"SELECT kid FROM `{service}` WHERE kid IN ({placeholders})", list(kid_keys.keys())) + existing_kids = {row["kid"] for row in cursor.fetchall()} + + new_keys = {kid: key for kid, key in kid_keys.items() if kid not in existing_kids} + + if not new_keys: + return 0 + cursor.executemany( - # TODO: SQL injection risk - f"INSERT IGNORE INTO `{service}` (kid, key_) VALUES (%s, %s)", - kid_keys.items(), + f"INSERT INTO `{service}` (kid, key_) VALUES (%s, %s)", + new_keys.items(), ) - return cursor.rowcount + return len(new_keys) finally: conn.commit() cursor.close() diff --git a/vaults/SQLite.py b/vaults/SQLite.py index 2ab50f7..1f1b733 100644 --- a/vaults/SQLite.py +++ b/vaults/SQLite.py @@ -19,22 +19,32 @@ class SQLite(Vault): self.conn_factory = ConnectionFactory(self.path) def get_key(self, kid: Union[UUID, str], service: str) -> Optional[str]: - if not self.has_table(service): - # no table, no key, simple - return None - if isinstance(kid, UUID): kid = kid.hex conn = self.conn_factory.get() cursor = conn.cursor() + # Try both the original service name and lowercase version to handle case sensitivity issues + service_variants = [service] + if service != service.lower(): + service_variants.append(service.lower()) + if service != service.upper(): + service_variants.append(service.upper()) + try: - cursor.execute(f"SELECT `id`, `key_` FROM `{service}` WHERE `kid`=? AND `key_`!=?", (kid, "0" * 32)) - cek = cursor.fetchone() - if not cek: - return None - return cek[1] + for service_name in service_variants: + if not self.has_table(service_name): + continue + + cursor.execute( + f"SELECT `id`, `key_` FROM `{service_name}` WHERE `kid`=? AND `key_`!=?", (kid, "0" * 32) + ) + cek = cursor.fetchone() + if cek: + return cek[1] + + return None finally: cursor.close() @@ -102,16 +112,27 @@ class SQLite(Vault): if any(isinstance(kid, UUID) for kid, key_ in kid_keys.items()): kid_keys = {kid.hex if isinstance(kid, UUID) else kid: key_ for kid, key_ in kid_keys.items()} + if not kid_keys: + return 0 + conn = self.conn_factory.get() cursor = conn.cursor() try: + placeholders = ",".join(["?"] * len(kid_keys)) + cursor.execute(f"SELECT kid FROM `{service}` WHERE kid IN ({placeholders})", list(kid_keys.keys())) + existing_kids = {row[0] for row in cursor.fetchall()} + + new_keys = {kid: key for kid, key in kid_keys.items() if kid not in existing_kids} + + if not new_keys: + return 0 + cursor.executemany( - # TODO: SQL injection risk - f"INSERT OR IGNORE INTO `{service}` (kid, key_) VALUES (?, ?)", - kid_keys.items(), + f"INSERT INTO `{service}` (kid, key_) VALUES (?, ?)", + new_keys.items(), ) - return cursor.rowcount + return len(new_keys) finally: conn.commit() cursor.close()