envied-working-example update

packages/envied/src/envied/envied-working-example.yaml

@@ -1,5 +1,6 @@
-# Group or Username to postfix to the end of all download fienames following a dash
-#tag: ''
+# Envied configuration file
+# see unshackle, envied's parent for further configuration details:
+# https://github.com/unshackle-dl/unshackle/blob/dev/docs/reference/configuration.md
 
 # Set terminal background color (custom option not in CONFIG.md)
 set_terminal_bg: false
@@ -8,8 +9,11 @@ set_terminal_bg: false
 muxing:
   set_title: false
 
-# shakapackager or mp4decrypt
-decryption: shakapackager
+# shaka or mp4decrypt
+decryption:
+  default: shaka
+  ALL4: mp4decrypt
+  TPTV: mp4decrypt
 
 # Custom output templates for filenames
 # Configure output_template in your envied.yaml to control filename format.
@@ -46,6 +50,13 @@ output_template:
   # Custom scene-style with specific elements
   # movies: '{title}.{year}.{quality}.{hdr?}.{source}.WEB-DL.{audio_full}.{video}-{tag}'
   # series: '{title}.{year?}.{season_episode}.{episode_name?}.{quality}.{hdr?}.{source}.WEB-DL.{audio_full}.{atmos?}.{video}-{tag}'
+  #
+  # Folder naming (optional). One template for all, or a per-kind map.
+  # Valid kinds: movies, series, songs, albums (album vars: see docs).
+  # folder: '{title} ({year?})'
+  folder:
+    movies: '{title} ({year})'
+    series: '{title} ({year?})'
 
 # Language-based tagging for output filenames
 # Automatically adds language identifiers (e.g., DANiSH, NORDiC, DKsubs) based on
@@ -73,12 +84,16 @@ output_template:
 #       tag: DKsubs
 
 # Login credentials for each Service
+# email:password pairs should be provided for each service you want to use.
+# eg All4: nosmo.king@gmail.com:12345678
 credentials:
   ALL4: email:password 
   ROKU: email:password
   TVNZ: email:password
   TPTV: email:password
   CBC: email:password
+
+
 # Override default directories used across envied
 directories:
   cache: Cache
@@ -89,28 +104,12 @@ directories:
   temp: Temp
   wvds: WVDs
   prds: PRDs
-  # Additional directories that can be configured:
-  # commands: Commands
-  #services:
-  #  - ./envied/services/
-  vaults: vaults/
-  # fonts: Fonts
+
 
 # Pre-define which Widevine or PlayReady device to use for each Serviceuv run envied
 cdm:
   default: device
 
-# Use pywidevine Serve-compliant Remote CDMs
-# Currently not working!
-#remote_cdm:
-#    - name: "CDRM_Project_API"
-#      device_type: ANDROID
-#      system_id: 22590
-#      security_level: 3
-#      host: "https://cdrm-project.com/remotecdm/widevine"
-#      secret: "CDRM"
-#      device_name: "public"
-
 
 # Key Vaults store your obtained Content Encryption Keys (CEKs)
 key_vaults:
@@ -129,32 +128,7 @@ key_vaults:
   password: N7kQp4XzL2mV8cHrY9tFw5BjUa6DeGs3
 
 
-
-# Choose what software to use to download data
-downloader: n_m3u8dl_re
-# Options: requests | aria2c | curl_impersonate | n_m3u8dl_re
-# Can also be a mapping:
-# downloader:
-#   NF: requests
-#   AMZN: n_m3u8dl_re
-#   DSNP: n_m3u8dl_re
-#   default: requests
-
-# aria2c downloader configuration
-aria2c:
-  max_concurrent_downloads: 4
-  max_connection_per_server: 3
-  split: 5
-  file_allocation: falloc # none | prealloc | falloc | trunc
-
-# N_m3u8DL-RE downloader configuration
-n_m3u8dl_re:
-  thread_count: 16
-  ad_keyword: "advertisement"
-  use_proxy: true
-
-# curl_impersonate downloader configuration
-curl_impersonate:
+network:
   browser: chrome120
 
 # Pre-define default options and switches of the dl command
@@ -190,7 +164,7 @@ tmdb_api_key: ""
 # - subtitleedit: Prefer SubtitleEdit when available, fall back to pycaption
 # - pysubs2: Use pysubs2 library (supports SRT/SSA/ASS/WebVTT/TTML/SAMI/MicroDVD/MPL2/TMP)
 subtitle:
-  conversion_method: auto
+  conversion_method: pysubs2
   # sdh_method: Method to use for SDH (hearing impaired) stripping
   # - auto (default): Try subby (SRT only), then SubtitleEdit (if available), then subtitle-filter
   # - subby: Use subby library (SRT only)
@@ -220,7 +194,7 @@ subtitle:
 # Configuration data for each Service
 services:
   # Service-specific configuration goes here
-  # EXAMPLE:
+  # EXAMPLE_SERVICE:
   #   api_key: "service_specific_key"
 
 # Legacy NordVPN configuration (use proxy_providers instead)

voolts/API.py

@@ -114,32 +114,71 @@ class API(Vault):
         return added or updated
 
     def add_keys(self, service: str, kid_keys: dict[Union[UUID, str], str]) -> int:
-        data = self.session.post(
-            url=f"{self.uri}/{service.lower()}",
-            json={"content_keys": {str(kid).replace("-", ""): key for kid, key in kid_keys.items()}},
-            headers={"Accept": "application/json"},
-        ).json()
+        # Normalize keys
+        normalized_keys = {str(kid).replace("-", ""): key for kid, key in kid_keys.items()}
+        kid_list = list(normalized_keys.keys())
 
-        code = int(data.get("code", 0))
-        message = data.get("message")
-        error = {
-            0: None,
-            1: Exceptions.AuthRejected,
-            2: Exceptions.TooManyRequests,
-            3: Exceptions.ServiceTagInvalid,
-            4: Exceptions.KeyIdInvalid,
-            5: Exceptions.ContentKeyInvalid,
-        }.get(code, ValueError)
+        if not kid_list:
+            return 0
 
-        if error:
-            raise error(f"{message} ({code})")
+        # Try batches starting at 500, stepping down by 100 on failure, fallback to 1
+        batch_size = 500
+        total_added = 0
+        i = 0
 
-        # each kid:key that was new to the vault (optional)
-        added = int(data.get("added"))
-        # each key for a kid that was changed/updated (optional)
-        updated = int(data.get("updated"))
+        while i < len(kid_list):
+            batch_kids = kid_list[i : i + batch_size]
+            batch_keys = {kid: normalized_keys[kid] for kid in batch_kids}
 
-        return added + updated
+            try:
+                response = self.session.post(
+                    url=f"{self.uri}/{service.lower()}",
+                    json={"content_keys": batch_keys},
+                    headers={"Accept": "application/json"},
+                )
+
+                # Check for HTTP errors that suggest batch is too large
+                if response.status_code in (413, 414, 400) and batch_size > 1:
+                    if batch_size > 100:
+                        batch_size -= 100
+                    else:
+                        batch_size = 1
+                    continue
+
+                data = response.json()
+            except Exception:
+                # JSON decode error or connection issue - try smaller batch
+                if batch_size > 1:
+                    if batch_size > 100:
+                        batch_size -= 100
+                    else:
+                        batch_size = 1
+                    continue
+                raise
+
+            code = int(data.get("code", 0))
+            message = data.get("message")
+            error = {
+                0: None,
+                1: Exceptions.AuthRejected,
+                2: Exceptions.TooManyRequests,
+                3: Exceptions.ServiceTagInvalid,
+                4: Exceptions.KeyIdInvalid,
+                5: Exceptions.ContentKeyInvalid,
+            }.get(code, ValueError)
+
+            if error:
+                raise error(f"{message} ({code})")
+
+            # each kid:key that was new to the vault (optional)
+            added = int(data.get("added", 0))
+            # each key for a kid that was changed/updated (optional)
+            updated = int(data.get("updated", 0))
+
+            total_added += added + updated
+            i += batch_size
+
+        return total_added
 
     def get_services(self) -> Iterator[str]:
         data = self.session.post(url=self.uri, headers={"Accept": "application/json"}).json()

voolts/SQLite.py

@@ -119,9 +119,25 @@ class SQLite(Vault):
         cursor = conn.cursor()
 
         try:
-            placeholders = ",".join(["?"] * len(kid_keys))
-            cursor.execute(f"SELECT kid FROM `{service}` WHERE kid IN ({placeholders})", list(kid_keys.keys()))
-            existing_kids = {row[0] for row in cursor.fetchall()}
+            # Query existing KIDs in batches to avoid SQLite variable limit
+            # Try larger batch first (newer SQLite supports 32766), fall back to 500 if needed
+            existing_kids: set[str] = set()
+            kid_list = list(kid_keys.keys())
+            batch_size = 32000
+
+            i = 0
+            while i < len(kid_list):
+                batch = kid_list[i : i + batch_size]
+                placeholders = ",".join(["?"] * len(batch))
+                try:
+                    cursor.execute(f"SELECT kid FROM `{service}` WHERE kid IN ({placeholders})", batch)
+                    existing_kids.update(row[0] for row in cursor.fetchall())
+                    i += batch_size
+                except sqlite3.OperationalError as e:
+                    if "too many SQL variables" in str(e) and batch_size > 500:
+                        batch_size = 500
+                        continue
+                    raise
 
             new_keys = {kid: key for kid, key in kid_keys.items() if kid not in existing_kids}
 
@@ -192,7 +208,11 @@ class ConnectionFactory:
         self._store = threading.local()
 
     def _create_connection(self) -> Connection:
-        return sqlite3.connect(self._path)
+        conn = sqlite3.connect(self._path, timeout=30.0)
+        conn.execute("PRAGMA journal_mode=WAL")
+        conn.execute("PRAGMA synchronous=NORMAL")
+        conn.execute("PRAGMA busy_timeout=30000")
+        return conn
 
     def get(self) -> Connection:
         if not hasattr(self._store, "conn"):