diff --git a/packages/envied/src/envied/envied-working-example.yaml b/packages/envied/src/envied/envied-working-example.yaml index 8230fd6..9d5cc7a 100644 --- a/packages/envied/src/envied/envied-working-example.yaml +++ b/packages/envied/src/envied/envied-working-example.yaml @@ -1,5 +1,6 @@ -# Group or Username to postfix to the end of all download fienames following a dash -#tag: '' +# Envied configuration file +# see unshackle, envied's parent for further configuration details: +# https://github.com/unshackle-dl/unshackle/blob/dev/docs/reference/configuration.md # Set terminal background color (custom option not in CONFIG.md) set_terminal_bg: false @@ -8,8 +9,11 @@ set_terminal_bg: false muxing: set_title: false -# shakapackager or mp4decrypt -decryption: shakapackager +# shaka or mp4decrypt +decryption: + default: shaka + ALL4: mp4decrypt + TPTV: mp4decrypt # Custom output templates for filenames # Configure output_template in your envied.yaml to control filename format. @@ -46,6 +50,13 @@ output_template: # Custom scene-style with specific elements # movies: '{title}.{year}.{quality}.{hdr?}.{source}.WEB-DL.{audio_full}.{video}-{tag}' # series: '{title}.{year?}.{season_episode}.{episode_name?}.{quality}.{hdr?}.{source}.WEB-DL.{audio_full}.{atmos?}.{video}-{tag}' + # + # Folder naming (optional). One template for all, or a per-kind map. + # Valid kinds: movies, series, songs, albums (album vars: see docs). + # folder: '{title} ({year?})' + folder: + movies: '{title} ({year})' + series: '{title} ({year?})' # Language-based tagging for output filenames # Automatically adds language identifiers (e.g., DANiSH, NORDiC, DKsubs) based on @@ -73,12 +84,16 @@ output_template: # tag: DKsubs # Login credentials for each Service +# email:password pairs should be provided for each service you want to use. +# eg All4: nosmo.king@gmail.com:12345678 credentials: - ALL4: email:password + ALL4: email:password ROKU: email:password TVNZ: email:password TPTV: email:password CBC: email:password + + # Override default directories used across envied directories: cache: Cache @@ -89,28 +104,12 @@ directories: temp: Temp wvds: WVDs prds: PRDs - # Additional directories that can be configured: - # commands: Commands - #services: - # - ./envied/services/ - vaults: vaults/ - # fonts: Fonts + # Pre-define which Widevine or PlayReady device to use for each Serviceuv run envied cdm: default: device -# Use pywidevine Serve-compliant Remote CDMs -# Currently not working! -#remote_cdm: -# - name: "CDRM_Project_API" -# device_type: ANDROID -# system_id: 22590 -# security_level: 3 -# host: "https://cdrm-project.com/remotecdm/widevine" -# secret: "CDRM" -# device_name: "public" - # Key Vaults store your obtained Content Encryption Keys (CEKs) key_vaults: @@ -129,32 +128,7 @@ key_vaults: password: N7kQp4XzL2mV8cHrY9tFw5BjUa6DeGs3 - -# Choose what software to use to download data -downloader: n_m3u8dl_re -# Options: requests | aria2c | curl_impersonate | n_m3u8dl_re -# Can also be a mapping: -# downloader: -# NF: requests -# AMZN: n_m3u8dl_re -# DSNP: n_m3u8dl_re -# default: requests - -# aria2c downloader configuration -aria2c: - max_concurrent_downloads: 4 - max_connection_per_server: 3 - split: 5 - file_allocation: falloc # none | prealloc | falloc | trunc - -# N_m3u8DL-RE downloader configuration -n_m3u8dl_re: - thread_count: 16 - ad_keyword: "advertisement" - use_proxy: true - -# curl_impersonate downloader configuration -curl_impersonate: +network: browser: chrome120 # Pre-define default options and switches of the dl command @@ -190,7 +164,7 @@ tmdb_api_key: "" # - subtitleedit: Prefer SubtitleEdit when available, fall back to pycaption # - pysubs2: Use pysubs2 library (supports SRT/SSA/ASS/WebVTT/TTML/SAMI/MicroDVD/MPL2/TMP) subtitle: - conversion_method: auto + conversion_method: pysubs2 # sdh_method: Method to use for SDH (hearing impaired) stripping # - auto (default): Try subby (SRT only), then SubtitleEdit (if available), then subtitle-filter # - subby: Use subby library (SRT only) @@ -220,7 +194,7 @@ subtitle: # Configuration data for each Service services: # Service-specific configuration goes here - # EXAMPLE: + # EXAMPLE_SERVICE: # api_key: "service_specific_key" # Legacy NordVPN configuration (use proxy_providers instead) diff --git a/vaults/API.py b/voolts/API.py similarity index 68% rename from vaults/API.py rename to voolts/API.py index 8145ffa..75bc699 100644 --- a/vaults/API.py +++ b/voolts/API.py @@ -114,32 +114,71 @@ class API(Vault): return added or updated def add_keys(self, service: str, kid_keys: dict[Union[UUID, str], str]) -> int: - data = self.session.post( - url=f"{self.uri}/{service.lower()}", - json={"content_keys": {str(kid).replace("-", ""): key for kid, key in kid_keys.items()}}, - headers={"Accept": "application/json"}, - ).json() + # Normalize keys + normalized_keys = {str(kid).replace("-", ""): key for kid, key in kid_keys.items()} + kid_list = list(normalized_keys.keys()) - code = int(data.get("code", 0)) - message = data.get("message") - error = { - 0: None, - 1: Exceptions.AuthRejected, - 2: Exceptions.TooManyRequests, - 3: Exceptions.ServiceTagInvalid, - 4: Exceptions.KeyIdInvalid, - 5: Exceptions.ContentKeyInvalid, - }.get(code, ValueError) + if not kid_list: + return 0 - if error: - raise error(f"{message} ({code})") + # Try batches starting at 500, stepping down by 100 on failure, fallback to 1 + batch_size = 500 + total_added = 0 + i = 0 - # each kid:key that was new to the vault (optional) - added = int(data.get("added")) - # each key for a kid that was changed/updated (optional) - updated = int(data.get("updated")) + while i < len(kid_list): + batch_kids = kid_list[i : i + batch_size] + batch_keys = {kid: normalized_keys[kid] for kid in batch_kids} - return added + updated + try: + response = self.session.post( + url=f"{self.uri}/{service.lower()}", + json={"content_keys": batch_keys}, + headers={"Accept": "application/json"}, + ) + + # Check for HTTP errors that suggest batch is too large + if response.status_code in (413, 414, 400) and batch_size > 1: + if batch_size > 100: + batch_size -= 100 + else: + batch_size = 1 + continue + + data = response.json() + except Exception: + # JSON decode error or connection issue - try smaller batch + if batch_size > 1: + if batch_size > 100: + batch_size -= 100 + else: + batch_size = 1 + continue + raise + + code = int(data.get("code", 0)) + message = data.get("message") + error = { + 0: None, + 1: Exceptions.AuthRejected, + 2: Exceptions.TooManyRequests, + 3: Exceptions.ServiceTagInvalid, + 4: Exceptions.KeyIdInvalid, + 5: Exceptions.ContentKeyInvalid, + }.get(code, ValueError) + + if error: + raise error(f"{message} ({code})") + + # each kid:key that was new to the vault (optional) + added = int(data.get("added", 0)) + # each key for a kid that was changed/updated (optional) + updated = int(data.get("updated", 0)) + + total_added += added + updated + i += batch_size + + return total_added def get_services(self) -> Iterator[str]: data = self.session.post(url=self.uri, headers={"Accept": "application/json"}).json() diff --git a/vaults/HTTP.py b/voolts/HTTP.py similarity index 100% rename from vaults/HTTP.py rename to voolts/HTTP.py diff --git a/vaults/HTTPAPI.py b/voolts/HTTPAPI.py similarity index 97% rename from vaults/HTTPAPI.py rename to voolts/HTTPAPI.py index 5836954..380a10c 100644 --- a/vaults/HTTPAPI.py +++ b/voolts/HTTPAPI.py @@ -1,215 +1,215 @@ -import json -from typing import Iterator, Optional, Union -from uuid import UUID -from enum import Enum - -from requests import Session - -from envied.core import __version__ -from envied.core.vault import Vault - - -class InsertResult(Enum): - FAILURE = 0 - SUCCESS = 1 - ALREADY_EXISTS = 2 - - -class HTTPAPI(Vault): - """Key Vault using a structured HTTP API with JSON payloads and token authentication.""" - - def __init__(self, name: str, host: str, password: str): - super().__init__(name) - self.url = host - self.password = password # This is the API token - self.current_title = None - self.session = Session() - self.session.headers.update({"User-Agent": f"Devine v{__version__}"}) - self.api_session_id = None - - def request(self, method: str, params: dict = None) -> dict: - """Make a request to the HTTPAPI vault.""" - request_payload = { - "method": method, - "params": { - **(params or {}), - "session_id": self.api_session_id, - }, - "token": self.password, - } - - r = self.session.post(self.url, json=request_payload) - - if r.status_code == 404: - return {"status": "not_found"} - - if not r.ok: - raise ValueError(f"API returned HTTP Error {r.status_code}: {r.reason.title()}") - - try: - res = r.json() - except json.JSONDecodeError: - if r.status_code == 404: - return {"status": "not_found"} - raise ValueError(f"API returned an invalid response: {r.text}") - - if res.get("status_code") != 200: - raise ValueError(f"API returned an error: {res['status_code']} - {res['message']}") - - if session_id := res.get("message", {}).get("session_id"): - self.api_session_id = session_id - - return res.get("message", res) - - def get_key(self, kid: Union[UUID, str], service: str) -> Optional[str]: - if isinstance(kid, UUID): - kid = kid.hex - - try: - title = getattr(self, "current_title", None) - response = self.request( - "GetKey", - { - "kid": kid, - "service": service.lower(), - "title": title, - }, - ) - if response.get("status") == "not_found": - return None - keys = response.get("keys", []) - for key_entry in keys: - if key_entry["kid"] == kid: - return key_entry["key"] - except Exception as e: - print(f"Failed to get key ({e.__class__.__name__}: {e})") - return None - - return None - - def get_keys(self, service: str) -> Iterator[tuple[str, str]]: - """Get all keys for a service - this may need to be implemented based on your API.""" - try: - response = self.request( - "GetKeys", - { - "service": service.lower(), - }, - ) - keys = response.get("keys", []) - for key_entry in keys: - yield key_entry["kid"], key_entry["key"] - except Exception: - return iter([]) - - def add_key(self, service: str, kid: Union[UUID, str], key: str) -> bool: - if not key or key.count("0") == len(key): - raise ValueError("You cannot add a NULL Content Key to a Vault.") - - if isinstance(kid, UUID): - kid = kid.hex - - title = getattr(self, "current_title", None) - - try: - response = self.request( - "InsertKey", - { - "kid": kid, - "key": key, - "service": service.lower(), - "title": title, - }, - ) - if response.get("status") == "not_found": - return False - return response.get("inserted", False) - except Exception: - return False - - def add_keys(self, service: str, kid_keys: dict[Union[UUID, str], str]) -> int: - for kid, key in kid_keys.items(): - if not key or key.count("0") == len(key): - raise ValueError("You cannot add a NULL Content Key to a Vault.") - - processed_kid_keys = { - str(kid).replace("-", "") if isinstance(kid, UUID) else kid: key for kid, key in kid_keys.items() - } - - inserted_count = 0 - title = getattr(self, "current_title", None) - - for kid, key in processed_kid_keys.items(): - try: - response = self.request( - "InsertKey", - { - "kid": kid, - "key": key, - "service": service.lower(), - "title": title, - }, - ) - if response.get("status") == "not_found": - continue - if response.get("inserted", False): - inserted_count += 1 - except Exception: - continue - - return inserted_count - - def get_services(self) -> Iterator[str]: - """Get available services - this may need to be implemented based on your API.""" - try: - response = self.request("GetServices") - services = response.get("services", []) - for service in services: - yield service - except Exception: - return iter([]) - - def set_title(self, title: str): - """ - Set a title to be used for the next key insertions. - This is optional and will be sent with add_key requests if available. - """ - self.current_title = title - - def insert_key_with_result( - self, service: str, kid: Union[UUID, str], key: str, title: Optional[str] = None - ) -> InsertResult: - """ - Insert a key and return detailed result information. - This method provides more granular feedback than the standard add_key method. - """ - if not key or key.count("0") == len(key): - raise ValueError("You cannot add a NULL Content Key to a Vault.") - - if isinstance(kid, UUID): - kid = kid.hex - - if title is None: - title = getattr(self, "current_title", None) - - try: - response = self.request( - "InsertKey", - { - "kid": kid, - "key": key, - "service": service.lower(), - "title": title, - }, - ) - - if response.get("status") == "not_found": - return InsertResult.FAILURE - - if response.get("inserted", False): - return InsertResult.SUCCESS - else: - return InsertResult.ALREADY_EXISTS - - except Exception: - return InsertResult.FAILURE +import json +from typing import Iterator, Optional, Union +from uuid import UUID +from enum import Enum + +from requests import Session + +from envied.core import __version__ +from envied.core.vault import Vault + + +class InsertResult(Enum): + FAILURE = 0 + SUCCESS = 1 + ALREADY_EXISTS = 2 + + +class HTTPAPI(Vault): + """Key Vault using a structured HTTP API with JSON payloads and token authentication.""" + + def __init__(self, name: str, host: str, password: str): + super().__init__(name) + self.url = host + self.password = password # This is the API token + self.current_title = None + self.session = Session() + self.session.headers.update({"User-Agent": f"Devine v{__version__}"}) + self.api_session_id = None + + def request(self, method: str, params: dict = None) -> dict: + """Make a request to the HTTPAPI vault.""" + request_payload = { + "method": method, + "params": { + **(params or {}), + "session_id": self.api_session_id, + }, + "token": self.password, + } + + r = self.session.post(self.url, json=request_payload) + + if r.status_code == 404: + return {"status": "not_found"} + + if not r.ok: + raise ValueError(f"API returned HTTP Error {r.status_code}: {r.reason.title()}") + + try: + res = r.json() + except json.JSONDecodeError: + if r.status_code == 404: + return {"status": "not_found"} + raise ValueError(f"API returned an invalid response: {r.text}") + + if res.get("status_code") != 200: + raise ValueError(f"API returned an error: {res['status_code']} - {res['message']}") + + if session_id := res.get("message", {}).get("session_id"): + self.api_session_id = session_id + + return res.get("message", res) + + def get_key(self, kid: Union[UUID, str], service: str) -> Optional[str]: + if isinstance(kid, UUID): + kid = kid.hex + + try: + title = getattr(self, "current_title", None) + response = self.request( + "GetKey", + { + "kid": kid, + "service": service.lower(), + "title": title, + }, + ) + if response.get("status") == "not_found": + return None + keys = response.get("keys", []) + for key_entry in keys: + if key_entry["kid"] == kid: + return key_entry["key"] + except Exception as e: + print(f"Failed to get key ({e.__class__.__name__}: {e})") + return None + + return None + + def get_keys(self, service: str) -> Iterator[tuple[str, str]]: + """Get all keys for a service - this may need to be implemented based on your API.""" + try: + response = self.request( + "GetKeys", + { + "service": service.lower(), + }, + ) + keys = response.get("keys", []) + for key_entry in keys: + yield key_entry["kid"], key_entry["key"] + except Exception: + return iter([]) + + def add_key(self, service: str, kid: Union[UUID, str], key: str) -> bool: + if not key or key.count("0") == len(key): + raise ValueError("You cannot add a NULL Content Key to a Vault.") + + if isinstance(kid, UUID): + kid = kid.hex + + title = getattr(self, "current_title", None) + + try: + response = self.request( + "InsertKey", + { + "kid": kid, + "key": key, + "service": service.lower(), + "title": title, + }, + ) + if response.get("status") == "not_found": + return False + return response.get("inserted", False) + except Exception: + return False + + def add_keys(self, service: str, kid_keys: dict[Union[UUID, str], str]) -> int: + for kid, key in kid_keys.items(): + if not key or key.count("0") == len(key): + raise ValueError("You cannot add a NULL Content Key to a Vault.") + + processed_kid_keys = { + str(kid).replace("-", "") if isinstance(kid, UUID) else kid: key for kid, key in kid_keys.items() + } + + inserted_count = 0 + title = getattr(self, "current_title", None) + + for kid, key in processed_kid_keys.items(): + try: + response = self.request( + "InsertKey", + { + "kid": kid, + "key": key, + "service": service.lower(), + "title": title, + }, + ) + if response.get("status") == "not_found": + continue + if response.get("inserted", False): + inserted_count += 1 + except Exception: + continue + + return inserted_count + + def get_services(self) -> Iterator[str]: + """Get available services - this may need to be implemented based on your API.""" + try: + response = self.request("GetServices") + services = response.get("services", []) + for service in services: + yield service + except Exception: + return iter([]) + + def set_title(self, title: str): + """ + Set a title to be used for the next key insertions. + This is optional and will be sent with add_key requests if available. + """ + self.current_title = title + + def insert_key_with_result( + self, service: str, kid: Union[UUID, str], key: str, title: Optional[str] = None + ) -> InsertResult: + """ + Insert a key and return detailed result information. + This method provides more granular feedback than the standard add_key method. + """ + if not key or key.count("0") == len(key): + raise ValueError("You cannot add a NULL Content Key to a Vault.") + + if isinstance(kid, UUID): + kid = kid.hex + + if title is None: + title = getattr(self, "current_title", None) + + try: + response = self.request( + "InsertKey", + { + "kid": kid, + "key": key, + "service": service.lower(), + "title": title, + }, + ) + + if response.get("status") == "not_found": + return InsertResult.FAILURE + + if response.get("inserted", False): + return InsertResult.SUCCESS + else: + return InsertResult.ALREADY_EXISTS + + except Exception: + return InsertResult.FAILURE diff --git a/vaults/MySQL.py b/voolts/MySQL.py similarity index 100% rename from vaults/MySQL.py rename to voolts/MySQL.py diff --git a/vaults/SQLite.py b/voolts/SQLite.py similarity index 84% rename from vaults/SQLite.py rename to voolts/SQLite.py index 1f1b733..e3a1805 100644 --- a/vaults/SQLite.py +++ b/voolts/SQLite.py @@ -119,9 +119,25 @@ class SQLite(Vault): cursor = conn.cursor() try: - placeholders = ",".join(["?"] * len(kid_keys)) - cursor.execute(f"SELECT kid FROM `{service}` WHERE kid IN ({placeholders})", list(kid_keys.keys())) - existing_kids = {row[0] for row in cursor.fetchall()} + # Query existing KIDs in batches to avoid SQLite variable limit + # Try larger batch first (newer SQLite supports 32766), fall back to 500 if needed + existing_kids: set[str] = set() + kid_list = list(kid_keys.keys()) + batch_size = 32000 + + i = 0 + while i < len(kid_list): + batch = kid_list[i : i + batch_size] + placeholders = ",".join(["?"] * len(batch)) + try: + cursor.execute(f"SELECT kid FROM `{service}` WHERE kid IN ({placeholders})", batch) + existing_kids.update(row[0] for row in cursor.fetchall()) + i += batch_size + except sqlite3.OperationalError as e: + if "too many SQL variables" in str(e) and batch_size > 500: + batch_size = 500 + continue + raise new_keys = {kid: key for kid, key in kid_keys.items() if kid not in existing_kids} @@ -192,7 +208,11 @@ class ConnectionFactory: self._store = threading.local() def _create_connection(self) -> Connection: - return sqlite3.connect(self._path) + conn = sqlite3.connect(self._path, timeout=30.0) + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=NORMAL") + conn.execute("PRAGMA busy_timeout=30000") + return conn def get(self) -> Connection: if not hasattr(self._store, "conn"): diff --git a/vaults/__init__.py b/voolts/__init__.py similarity index 100% rename from vaults/__init__.py rename to voolts/__init__.py