envied-working-example update

packages/envied/src/envied/envied-working-example.yaml

@@ -1,5 +1,6 @@
-# Group or Username to postfix to the end of all download fienames following a dash
+# Envied configuration file
-#tag: ''
+# see unshackle, envied's parent for further configuration details:
+# https://github.com/unshackle-dl/unshackle/blob/dev/docs/reference/configuration.md
 
 # Set terminal background color (custom option not in CONFIG.md)
 set_terminal_bg: false
@@ -8,8 +9,11 @@ set_terminal_bg: false
 muxing:
   set_title: false
 
-# shakapackager or mp4decrypt
+# shaka or mp4decrypt
-decryption: shakapackager
+decryption:
+  default: shaka
+  ALL4: mp4decrypt
+  TPTV: mp4decrypt
 
 # Custom output templates for filenames
 # Configure output_template in your envied.yaml to control filename format.
@@ -46,6 +50,13 @@ output_template:
   # Custom scene-style with specific elements
   # movies: '{title}.{year}.{quality}.{hdr?}.{source}.WEB-DL.{audio_full}.{video}-{tag}'
   # series: '{title}.{year?}.{season_episode}.{episode_name?}.{quality}.{hdr?}.{source}.WEB-DL.{audio_full}.{atmos?}.{video}-{tag}'
+  #
+  # Folder naming (optional). One template for all, or a per-kind map.
+  # Valid kinds: movies, series, songs, albums (album vars: see docs).
+  # folder: '{title} ({year?})'
+  folder:
+    movies: '{title} ({year})'
+    series: '{title} ({year?})'
 
 # Language-based tagging for output filenames
 # Automatically adds language identifiers (e.g., DANiSH, NORDiC, DKsubs) based on
@@ -73,12 +84,16 @@ output_template:
 #       tag: DKsubs
 
 # Login credentials for each Service
+# email:password pairs should be provided for each service you want to use.
+# eg All4: nosmo.king@gmail.com:12345678
 credentials:
-  ALL4: email:password
+  ALL4: email:password 
   ROKU: email:password
   TVNZ: email:password
   TPTV: email:password
   CBC: email:password
+
+
 # Override default directories used across envied
 directories:
   cache: Cache
@@ -89,28 +104,12 @@ directories:
   temp: Temp
   wvds: WVDs
   prds: PRDs
-  # Additional directories that can be configured:
+
-  # commands: Commands
-  #services:
-  #  - ./envied/services/
-  vaults: vaults/
-  # fonts: Fonts
 
 # Pre-define which Widevine or PlayReady device to use for each Serviceuv run envied
 cdm:
   default: device
 
-# Use pywidevine Serve-compliant Remote CDMs
-# Currently not working!
-#remote_cdm:
-#    - name: "CDRM_Project_API"
-#      device_type: ANDROID
-#      system_id: 22590
-#      security_level: 3
-#      host: "https://cdrm-project.com/remotecdm/widevine"
-#      secret: "CDRM"
-#      device_name: "public"
-
 
 # Key Vaults store your obtained Content Encryption Keys (CEKs)
 key_vaults:
@@ -129,32 +128,7 @@ key_vaults:
   password: N7kQp4XzL2mV8cHrY9tFw5BjUa6DeGs3
 
 
-
+network:
-# Choose what software to use to download data
-downloader: n_m3u8dl_re
-# Options: requests | aria2c | curl_impersonate | n_m3u8dl_re
-# Can also be a mapping:
-# downloader:
-#   NF: requests
-#   AMZN: n_m3u8dl_re
-#   DSNP: n_m3u8dl_re
-#   default: requests
-
-# aria2c downloader configuration
-aria2c:
-  max_concurrent_downloads: 4
-  max_connection_per_server: 3
-  split: 5
-  file_allocation: falloc # none | prealloc | falloc | trunc
-
-# N_m3u8DL-RE downloader configuration
-n_m3u8dl_re:
-  thread_count: 16
-  ad_keyword: "advertisement"
-  use_proxy: true
-
-# curl_impersonate downloader configuration
-curl_impersonate:
   browser: chrome120
 
 # Pre-define default options and switches of the dl command
@@ -190,7 +164,7 @@ tmdb_api_key: ""
 # - subtitleedit: Prefer SubtitleEdit when available, fall back to pycaption
 # - pysubs2: Use pysubs2 library (supports SRT/SSA/ASS/WebVTT/TTML/SAMI/MicroDVD/MPL2/TMP)
 subtitle:
-  conversion_method: auto
+  conversion_method: pysubs2
   # sdh_method: Method to use for SDH (hearing impaired) stripping
   # - auto (default): Try subby (SRT only), then SubtitleEdit (if available), then subtitle-filter
   # - subby: Use subby library (SRT only)
@@ -220,7 +194,7 @@ subtitle:
 # Configuration data for each Service
 services:
   # Service-specific configuration goes here
-  # EXAMPLE:
+  # EXAMPLE_SERVICE:
   #   api_key: "service_specific_key"
 
 # Legacy NordVPN configuration (use proxy_providers instead)

voolts/API.py

@@ -114,32 +114,71 @@ class API(Vault):
         return added or updated
 
     def add_keys(self, service: str, kid_keys: dict[Union[UUID, str], str]) -> int:
-        data = self.session.post(
+        # Normalize keys
-            url=f"{self.uri}/{service.lower()}",
+        normalized_keys = {str(kid).replace("-", ""): key for kid, key in kid_keys.items()}
-            json={"content_keys": {str(kid).replace("-", ""): key for kid, key in kid_keys.items()}},
+        kid_list = list(normalized_keys.keys())
-            headers={"Accept": "application/json"},
-        ).json()
 
-        code = int(data.get("code", 0))
+        if not kid_list:
-        message = data.get("message")
+            return 0
-        error = {
-            0: None,
-            1: Exceptions.AuthRejected,
-            2: Exceptions.TooManyRequests,
-            3: Exceptions.ServiceTagInvalid,
-            4: Exceptions.KeyIdInvalid,
-            5: Exceptions.ContentKeyInvalid,
-        }.get(code, ValueError)
 
-        if error:
+        # Try batches starting at 500, stepping down by 100 on failure, fallback to 1
-            raise error(f"{message} ({code})")
+        batch_size = 500
+        total_added = 0
+        i = 0
 
-        # each kid:key that was new to the vault (optional)
+        while i < len(kid_list):
-        added = int(data.get("added"))
+            batch_kids = kid_list[i : i + batch_size]
-        # each key for a kid that was changed/updated (optional)
+            batch_keys = {kid: normalized_keys[kid] for kid in batch_kids}
-        updated = int(data.get("updated"))
 
-        return added + updated
+            try:
+                response = self.session.post(
+                    url=f"{self.uri}/{service.lower()}",
+                    json={"content_keys": batch_keys},
+                    headers={"Accept": "application/json"},
+                )
+
+                # Check for HTTP errors that suggest batch is too large
+                if response.status_code in (413, 414, 400) and batch_size > 1:
+                    if batch_size > 100:
+                        batch_size -= 100
+                    else:
+                        batch_size = 1
+                    continue
+
+                data = response.json()
+            except Exception:
+                # JSON decode error or connection issue - try smaller batch
+                if batch_size > 1:
+                    if batch_size > 100:
+                        batch_size -= 100
+                    else:
+                        batch_size = 1
+                    continue
+                raise
+
+            code = int(data.get("code", 0))
+            message = data.get("message")
+            error = {
+                0: None,
+                1: Exceptions.AuthRejected,
+                2: Exceptions.TooManyRequests,
+                3: Exceptions.ServiceTagInvalid,
+                4: Exceptions.KeyIdInvalid,
+                5: Exceptions.ContentKeyInvalid,
+            }.get(code, ValueError)
+
+            if error:
+                raise error(f"{message} ({code})")
+
+            # each kid:key that was new to the vault (optional)
+            added = int(data.get("added", 0))
+            # each key for a kid that was changed/updated (optional)
+            updated = int(data.get("updated", 0))
+
+            total_added += added + updated
+            i += batch_size
+
+        return total_added
 
     def get_services(self) -> Iterator[str]:
         data = self.session.post(url=self.uri, headers={"Accept": "application/json"}).json()

voolts/HTTPAPI.py

@@ -1,215 +1,215 @@
-import json
+import json
-from typing import Iterator, Optional, Union
+from typing import Iterator, Optional, Union
-from uuid import UUID
+from uuid import UUID
-from enum import Enum
+from enum import Enum
-
+
-from requests import Session
+from requests import Session
-
+
-from envied.core import __version__
+from envied.core import __version__
-from envied.core.vault import Vault
+from envied.core.vault import Vault
-
+
-
+
-class InsertResult(Enum):
+class InsertResult(Enum):
-    FAILURE = 0
+    FAILURE = 0
-    SUCCESS = 1
+    SUCCESS = 1
-    ALREADY_EXISTS = 2
+    ALREADY_EXISTS = 2
-
+
-
+
-class HTTPAPI(Vault):
+class HTTPAPI(Vault):
-    """Key Vault using a structured HTTP API with JSON payloads and token authentication."""
+    """Key Vault using a structured HTTP API with JSON payloads and token authentication."""
-
+
-    def __init__(self, name: str, host: str, password: str):
+    def __init__(self, name: str, host: str, password: str):
-        super().__init__(name)
+        super().__init__(name)
-        self.url = host
+        self.url = host
-        self.password = password  # This is the API token
+        self.password = password  # This is the API token
-        self.current_title = None
+        self.current_title = None
-        self.session = Session()
+        self.session = Session()
-        self.session.headers.update({"User-Agent": f"Devine v{__version__}"})
+        self.session.headers.update({"User-Agent": f"Devine v{__version__}"})
-        self.api_session_id = None
+        self.api_session_id = None
-
+
-    def request(self, method: str, params: dict = None) -> dict:
+    def request(self, method: str, params: dict = None) -> dict:
-        """Make a request to the HTTPAPI vault."""
+        """Make a request to the HTTPAPI vault."""
-        request_payload = {
+        request_payload = {
-            "method": method,
+            "method": method,
-            "params": {
+            "params": {
-                **(params or {}),
+                **(params or {}),
-                "session_id": self.api_session_id,
+                "session_id": self.api_session_id,
-            },
+            },
-            "token": self.password,
+            "token": self.password,
-        }
+        }
-
+
-        r = self.session.post(self.url, json=request_payload)
+        r = self.session.post(self.url, json=request_payload)
-
+
-        if r.status_code == 404:
+        if r.status_code == 404:
-            return {"status": "not_found"}
+            return {"status": "not_found"}
-
+
-        if not r.ok:
+        if not r.ok:
-            raise ValueError(f"API returned HTTP Error {r.status_code}: {r.reason.title()}")
+            raise ValueError(f"API returned HTTP Error {r.status_code}: {r.reason.title()}")
-
+
-        try:
+        try:
-            res = r.json()
+            res = r.json()
-        except json.JSONDecodeError:
+        except json.JSONDecodeError:
-            if r.status_code == 404:
+            if r.status_code == 404:
-                return {"status": "not_found"}
+                return {"status": "not_found"}
-            raise ValueError(f"API returned an invalid response: {r.text}")
+            raise ValueError(f"API returned an invalid response: {r.text}")
-
+
-        if res.get("status_code") != 200:
+        if res.get("status_code") != 200:
-            raise ValueError(f"API returned an error: {res['status_code']} - {res['message']}")
+            raise ValueError(f"API returned an error: {res['status_code']} - {res['message']}")
-
+
-        if session_id := res.get("message", {}).get("session_id"):
+        if session_id := res.get("message", {}).get("session_id"):
-            self.api_session_id = session_id
+            self.api_session_id = session_id
-
+
-        return res.get("message", res)
+        return res.get("message", res)
-
+
-    def get_key(self, kid: Union[UUID, str], service: str) -> Optional[str]:
+    def get_key(self, kid: Union[UUID, str], service: str) -> Optional[str]:
-        if isinstance(kid, UUID):
+        if isinstance(kid, UUID):
-            kid = kid.hex
+            kid = kid.hex
-
+
-        try:
+        try:
-            title = getattr(self, "current_title", None)
+            title = getattr(self, "current_title", None)
-            response = self.request(
+            response = self.request(
-                "GetKey",
+                "GetKey",
-                {
+                {
-                    "kid": kid,
+                    "kid": kid,
-                    "service": service.lower(),
+                    "service": service.lower(),
-                    "title": title,
+                    "title": title,
-                },
+                },
-            )
+            )
-            if response.get("status") == "not_found":
+            if response.get("status") == "not_found":
-                return None
+                return None
-            keys = response.get("keys", [])
+            keys = response.get("keys", [])
-            for key_entry in keys:
+            for key_entry in keys:
-                if key_entry["kid"] == kid:
+                if key_entry["kid"] == kid:
-                    return key_entry["key"]
+                    return key_entry["key"]
-        except Exception as e:
+        except Exception as e:
-            print(f"Failed to get key ({e.__class__.__name__}: {e})")
+            print(f"Failed to get key ({e.__class__.__name__}: {e})")
-            return None
+            return None
-
+
-        return None
+        return None
-
+
-    def get_keys(self, service: str) -> Iterator[tuple[str, str]]:
+    def get_keys(self, service: str) -> Iterator[tuple[str, str]]:
-        """Get all keys for a service - this may need to be implemented based on your API."""
+        """Get all keys for a service - this may need to be implemented based on your API."""
-        try:
+        try:
-            response = self.request(
+            response = self.request(
-                "GetKeys",
+                "GetKeys",
-                {
+                {
-                    "service": service.lower(),
+                    "service": service.lower(),
-                },
+                },
-            )
+            )
-            keys = response.get("keys", [])
+            keys = response.get("keys", [])
-            for key_entry in keys:
+            for key_entry in keys:
-                yield key_entry["kid"], key_entry["key"]
+                yield key_entry["kid"], key_entry["key"]
-        except Exception:
+        except Exception:
-            return iter([])
+            return iter([])
-
+
-    def add_key(self, service: str, kid: Union[UUID, str], key: str) -> bool:
+    def add_key(self, service: str, kid: Union[UUID, str], key: str) -> bool:
-        if not key or key.count("0") == len(key):
+        if not key or key.count("0") == len(key):
-            raise ValueError("You cannot add a NULL Content Key to a Vault.")
+            raise ValueError("You cannot add a NULL Content Key to a Vault.")
-
+
-        if isinstance(kid, UUID):
+        if isinstance(kid, UUID):
-            kid = kid.hex
+            kid = kid.hex
-
+
-        title = getattr(self, "current_title", None)
+        title = getattr(self, "current_title", None)
-
+
-        try:
+        try:
-            response = self.request(
+            response = self.request(
-                "InsertKey",
+                "InsertKey",
-                {
+                {
-                    "kid": kid,
+                    "kid": kid,
-                    "key": key,
+                    "key": key,
-                    "service": service.lower(),
+                    "service": service.lower(),
-                    "title": title,
+                    "title": title,
-                },
+                },
-            )
+            )
-            if response.get("status") == "not_found":
+            if response.get("status") == "not_found":
-                return False
+                return False
-            return response.get("inserted", False)
+            return response.get("inserted", False)
-        except Exception:
+        except Exception:
-            return False
+            return False
-
+
-    def add_keys(self, service: str, kid_keys: dict[Union[UUID, str], str]) -> int:
+    def add_keys(self, service: str, kid_keys: dict[Union[UUID, str], str]) -> int:
-        for kid, key in kid_keys.items():
+        for kid, key in kid_keys.items():
-            if not key or key.count("0") == len(key):
+            if not key or key.count("0") == len(key):
-                raise ValueError("You cannot add a NULL Content Key to a Vault.")
+                raise ValueError("You cannot add a NULL Content Key to a Vault.")
-
+
-        processed_kid_keys = {
+        processed_kid_keys = {
-            str(kid).replace("-", "") if isinstance(kid, UUID) else kid: key for kid, key in kid_keys.items()
+            str(kid).replace("-", "") if isinstance(kid, UUID) else kid: key for kid, key in kid_keys.items()
-        }
+        }
-
+
-        inserted_count = 0
+        inserted_count = 0
-        title = getattr(self, "current_title", None)
+        title = getattr(self, "current_title", None)
-
+
-        for kid, key in processed_kid_keys.items():
+        for kid, key in processed_kid_keys.items():
-            try:
+            try:
-                response = self.request(
+                response = self.request(
-                    "InsertKey",
+                    "InsertKey",
-                    {
+                    {
-                        "kid": kid,
+                        "kid": kid,
-                        "key": key,
+                        "key": key,
-                        "service": service.lower(),
+                        "service": service.lower(),
-                        "title": title,
+                        "title": title,
-                    },
+                    },
-                )
+                )
-                if response.get("status") == "not_found":
+                if response.get("status") == "not_found":
-                    continue
+                    continue
-                if response.get("inserted", False):
+                if response.get("inserted", False):
-                    inserted_count += 1
+                    inserted_count += 1
-            except Exception:
+            except Exception:
-                continue
+                continue
-
+
-        return inserted_count
+        return inserted_count
-
+
-    def get_services(self) -> Iterator[str]:
+    def get_services(self) -> Iterator[str]:
-        """Get available services - this may need to be implemented based on your API."""
+        """Get available services - this may need to be implemented based on your API."""
-        try:
+        try:
-            response = self.request("GetServices")
+            response = self.request("GetServices")
-            services = response.get("services", [])
+            services = response.get("services", [])
-            for service in services:
+            for service in services:
-                yield service
+                yield service
-        except Exception:
+        except Exception:
-            return iter([])
+            return iter([])
-
+
-    def set_title(self, title: str):
+    def set_title(self, title: str):
-        """
+        """
-        Set a title to be used for the next key insertions.
+        Set a title to be used for the next key insertions.
-        This is optional and will be sent with add_key requests if available.
+        This is optional and will be sent with add_key requests if available.
-        """
+        """
-        self.current_title = title
+        self.current_title = title
-
+
-    def insert_key_with_result(
+    def insert_key_with_result(
-        self, service: str, kid: Union[UUID, str], key: str, title: Optional[str] = None
+        self, service: str, kid: Union[UUID, str], key: str, title: Optional[str] = None
-    ) -> InsertResult:
+    ) -> InsertResult:
-        """
+        """
-        Insert a key and return detailed result information.
+        Insert a key and return detailed result information.
-        This method provides more granular feedback than the standard add_key method.
+        This method provides more granular feedback than the standard add_key method.
-        """
+        """
-        if not key or key.count("0") == len(key):
+        if not key or key.count("0") == len(key):
-            raise ValueError("You cannot add a NULL Content Key to a Vault.")
+            raise ValueError("You cannot add a NULL Content Key to a Vault.")
-
+
-        if isinstance(kid, UUID):
+        if isinstance(kid, UUID):
-            kid = kid.hex
+            kid = kid.hex
-
+
-        if title is None:
+        if title is None:
-            title = getattr(self, "current_title", None)
+            title = getattr(self, "current_title", None)
-
+
-        try:
+        try:
-            response = self.request(
+            response = self.request(
-                "InsertKey",
+                "InsertKey",
-                {
+                {
-                    "kid": kid,
+                    "kid": kid,
-                    "key": key,
+                    "key": key,
-                    "service": service.lower(),
+                    "service": service.lower(),
-                    "title": title,
+                    "title": title,
-                },
+                },
-            )
+            )
-
+
-            if response.get("status") == "not_found":
+            if response.get("status") == "not_found":
-                return InsertResult.FAILURE
+                return InsertResult.FAILURE
-
+
-            if response.get("inserted", False):
+            if response.get("inserted", False):
-                return InsertResult.SUCCESS
+                return InsertResult.SUCCESS
-            else:
+            else:
-                return InsertResult.ALREADY_EXISTS
+                return InsertResult.ALREADY_EXISTS
-
+
-        except Exception:
+        except Exception:
-            return InsertResult.FAILURE
+            return InsertResult.FAILURE

voolts/SQLite.py

@@ -119,9 +119,25 @@ class SQLite(Vault):
         cursor = conn.cursor()
 
         try:
-            placeholders = ",".join(["?"] * len(kid_keys))
+            # Query existing KIDs in batches to avoid SQLite variable limit
-            cursor.execute(f"SELECT kid FROM `{service}` WHERE kid IN ({placeholders})", list(kid_keys.keys()))
+            # Try larger batch first (newer SQLite supports 32766), fall back to 500 if needed
-            existing_kids = {row[0] for row in cursor.fetchall()}
+            existing_kids: set[str] = set()
+            kid_list = list(kid_keys.keys())
+            batch_size = 32000
+
+            i = 0
+            while i < len(kid_list):
+                batch = kid_list[i : i + batch_size]
+                placeholders = ",".join(["?"] * len(batch))
+                try:
+                    cursor.execute(f"SELECT kid FROM `{service}` WHERE kid IN ({placeholders})", batch)
+                    existing_kids.update(row[0] for row in cursor.fetchall())
+                    i += batch_size
+                except sqlite3.OperationalError as e:
+                    if "too many SQL variables" in str(e) and batch_size > 500:
+                        batch_size = 500
+                        continue
+                    raise
 
             new_keys = {kid: key for kid, key in kid_keys.items() if kid not in existing_kids}
 
@@ -192,7 +208,11 @@ class ConnectionFactory:
         self._store = threading.local()
 
     def _create_connection(self) -> Connection:
-        return sqlite3.connect(self._path)
+        conn = sqlite3.connect(self._path, timeout=30.0)
+        conn.execute("PRAGMA journal_mode=WAL")
+        conn.execute("PRAGMA synchronous=NORMAL")
+        conn.execute("PRAGMA busy_timeout=30000")
+        return conn
 
     def get(self) -> Connection:
         if not hasattr(self._store, "conn"):