From 7e1049360930dff5b99c32dc9479396bb0c88d16 Mon Sep 17 00:00:00 2001 From: VineFeeder Date: Thu, 4 Jun 2026 13:12:55 +0100 Subject: [PATCH] updates 5.1.0 --- README.md | 2 +- docs/ADVANCED_CONFIG.md | 232 +++ docs/API.md | 781 +++++++ docs/DOWNLOAD_CONFIG.md | 322 +++ docs/DRM_CONFIG.md | 481 +++++ docs/GLUETUN.md | 243 +++ docs/NETWORK_CONFIG.md | 178 ++ docs/OUTPUT_CONFIG.md | 287 +++ docs/SERVICE_CONFIG.md | 246 +++ docs/SUBTITLE_CONFIG.md | 73 + packages/envied/pyproject.toml | 29 +- packages/envied/src/envied/commands/dl.py | 1026 ++++++--- packages/envied/src/envied/commands/env.py | 9 - packages/envied/src/envied/commands/import.py | 54 + packages/envied/src/envied/commands/kv.py | 131 +- packages/envied/src/envied/commands/search.py | 4 +- packages/envied/src/envied/commands/serve.py | 102 +- packages/envied/src/envied/core/__init__.py | 2 +- packages/envied/src/envied/core/__main__.py | 29 +- .../envied/src/envied/core/api/compression.py | 40 + .../src/envied/core/api/download_manager.py | 73 +- packages/envied/src/envied/core/api/errors.py | 4 + .../envied/src/envied/core/api/handlers.py | 1832 ++++++++++++++--- .../src/envied/core/api/input_bridge.py | 139 ++ packages/envied/src/envied/core/api/routes.py | 505 ++++- .../src/envied/core/api/session_store.py | 209 ++ packages/envied/src/envied/core/binaries.py | 4 - .../envied/src/envied/core/cdm/__init__.py | 5 + packages/envied/src/envied/core/cdm/loader.py | 128 ++ packages/envied/src/envied/core/commands.py | 55 +- packages/envied/src/envied/core/config.py | 60 +- .../src/envied/core/downloaders/__init__.py | 5 +- .../src/envied/core/downloaders/requests.py | 623 ++++-- .../envied/src/envied/core/drm/__init__.py | 37 +- .../envied/src/envied/core/drm/clearkey.py | 7 +- .../envied/src/envied/core/drm/playready.py | 37 +- .../envied/src/envied/core/drm/widevine.py | 50 +- .../envied/src/envied/core/import_service.py | 304 +++ .../envied/src/envied/core/manifests/dash.py | 625 +++--- .../envied/src/envied/core/manifests/hls.py | 399 +++- .../envied/src/envied/core/manifests/ism.py | 83 +- .../envied/src/envied/core/manifests/m3u8.py | 7 +- .../envied/src/envied/core/proxies/gluetun.py | 13 +- .../envied/src/envied/core/proxies/nordvpn.py | 7 +- .../envied/src/envied/core/proxies/resolve.py | 88 + .../src/envied/core/proxies/windscribevpn.py | 2 +- .../envied/src/envied/core/remote_service.py | 853 ++++++++ packages/envied/src/envied/core/service.py | 69 +- packages/envied/src/envied/core/services.py | 190 +- packages/envied/src/envied/core/session.py | 805 ++++++-- .../envied/src/envied/core/titles/__init__.py | 38 +- .../envied/src/envied/core/titles/episode.py | 49 +- .../envied/src/envied/core/titles/movie.py | 10 + .../envied/src/envied/core/titles/song.py | 10 + .../envied/src/envied/core/titles/title.py | 35 +- .../src/envied/core/tracks/attachment.py | 8 +- .../envied/src/envied/core/tracks/audio.py | 27 +- .../envied/src/envied/core/tracks/dv_fixup.py | 117 ++ .../envied/src/envied/core/tracks/hybrid.py | 396 ++-- .../envied/src/envied/core/tracks/subtitle.py | 26 +- .../envied/src/envied/core/tracks/track.py | 135 +- .../envied/src/envied/core/tracks/tracks.py | 96 +- .../envied/src/envied/core/tracks/video.py | 128 +- packages/envied/src/envied/core/utilities.py | 148 +- .../envied/src/envied/core/utils/bitrate.py | 441 ++++ .../src/envied/core/utils/click_types.py | 23 + packages/envied/src/envied/core/utils/dovi.py | 130 ++ .../envied/src/envied/core/utils/ip_info.py | 252 +++ .../src/envied/core/utils/subprocess.py | 34 +- packages/envied/src/envied/core/utils/tags.py | 80 +- .../envied/core/utils/template_formatter.py | 4 +- packages/envied/src/envied/core/vaults.py | 15 +- .../envied/src/envied/envied-example.yaml | 758 +++++++ .../src/envied/services/NBC/__init__.py | 321 +++ .../src/envied/services/NBC/config.yaml | 15 + .../src/envied/services/TVNZ/__init__.py | 408 ++-- .../src/envied/services/TVNZ/config.yaml | 6 +- packages/envied/src/envied/utils/base58.py | 137 ++ packages/envied/src/envied/vaults/SQLite.py | 6 +- .../src/vinefeeder/services/TVNZ/__init__.py | 39 +- .../src/vinefeeder/services/TVNZ/config.yaml | 2 +- uv.lock | 327 +-- 82 files changed, 13340 insertions(+), 2370 deletions(-) create mode 100644 docs/ADVANCED_CONFIG.md create mode 100644 docs/API.md create mode 100644 docs/DOWNLOAD_CONFIG.md create mode 100644 docs/DRM_CONFIG.md create mode 100644 docs/GLUETUN.md create mode 100644 docs/NETWORK_CONFIG.md create mode 100644 docs/OUTPUT_CONFIG.md create mode 100644 docs/SERVICE_CONFIG.md create mode 100644 docs/SUBTITLE_CONFIG.md create mode 100644 packages/envied/src/envied/commands/import.py create mode 100644 packages/envied/src/envied/core/api/compression.py create mode 100644 packages/envied/src/envied/core/api/input_bridge.py create mode 100644 packages/envied/src/envied/core/api/session_store.py create mode 100644 packages/envied/src/envied/core/cdm/loader.py create mode 100644 packages/envied/src/envied/core/import_service.py create mode 100644 packages/envied/src/envied/core/proxies/resolve.py create mode 100644 packages/envied/src/envied/core/remote_service.py create mode 100644 packages/envied/src/envied/core/tracks/dv_fixup.py create mode 100644 packages/envied/src/envied/core/utils/bitrate.py create mode 100644 packages/envied/src/envied/core/utils/dovi.py create mode 100644 packages/envied/src/envied/core/utils/ip_info.py create mode 100644 packages/envied/src/envied/envied-example.yaml create mode 100644 packages/envied/src/envied/services/NBC/__init__.py create mode 100644 packages/envied/src/envied/services/NBC/config.yaml create mode 100644 packages/envied/src/envied/utils/base58.py diff --git a/README.md b/README.md index 7edadf8..6047133 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ These services have web-origins and not all have been tested by me. **Other README's"" TwinVine/packages/vinefeeder/src/vinefeeder/README.md for details for confuring Envied download options on a service by service basis. - TwinVine/packages/envied/README.md links to wiki (unshackle - envied's parent) + TwinVine/packages/envied/README.md links to wiki (envied - envied's parent) diff --git a/docs/ADVANCED_CONFIG.md b/docs/ADVANCED_CONFIG.md new file mode 100644 index 0000000..6b69bd0 --- /dev/null +++ b/docs/ADVANCED_CONFIG.md @@ -0,0 +1,232 @@ +# Advanced & System Configuration + +This document covers advanced features, debugging, and system-level configuration options. + +## serve (dict) + +Configuration for the integrated server that provides CDM endpoints (Widevine/PlayReady) and a REST API for remote downloading. + +Start the server with: + +```bash +envied serve # Default: localhost:8786 +envied serve -h 0.0.0.0 -p 8888 # Listen on all interfaces +envied serve --no-key # Disable authentication +envied serve --api-only # REST API only, no CDM endpoints +envied serve --remote-only # Only expose remote service session endpoints +``` + +### CLI Options + +| Option | Default | Description | +| --- | --- | --- | +| `-h, --host` | `127.0.0.1` | Host to serve from | +| `-p, --port` | `8786` | Port to serve from | +| `--caddy` | `false` | Also serve with Caddy reverse-proxy for HTTPS | +| `--api-only` | `false` | Serve only the REST API, disable CDM endpoints | +| `--no-widevine` | `false` | Disable Widevine CDM endpoints | +| `--no-playready` | `false` | Disable PlayReady CDM endpoints | +| `--no-key` | `false` | Disable API key authentication (allows all requests) | +| `--debug-api` | `false` | Include tracebacks and stderr in API error responses | +| `--debug` | `false` | Enable debug logging for API operations | +| `--remote-only` | `false` | Only expose remote service session endpoints (health, services, search, session) | + +### Configuration + +- `api_secret` - Secret key for REST API authentication. Required unless `--no-key` is used. All API requests must include this key via the `X-Secret-Key` header. +- `compression_level` - Compression level for API payloads (manifests, cache, cookies). `0`=off, `1`=fast, `6`=balanced, `9`=max. Default: `1`. +- `session_ttl` - Session inactivity timeout in seconds. Each request resets the timer. Default: `300`. +- `max_sessions` - Maximum concurrent sessions before the oldest is evicted. Default: `100`. +- `services` - Optional global service allowlist. Only these service tags are exposed. If omitted, all services are available. +- `devices` - List of Widevine device files (.wvd). If not specified, auto-populated from the WVDs directory. +- `playready_devices` - List of PlayReady device files (.prd). If not specified, auto-populated from the PRDs directory. +- `users` - Dictionary mapping user secret keys to their access configuration: + - `devices` - List of Widevine devices this user can access + - `playready_devices` - List of PlayReady devices this user can access + - `username` - Internal logging name for the user (not visible to users) + - `services` - Optional per-user service allowlist. Effective access is the intersection of global and per-user allowlists. + +#### Server-side `dl` defaults + +Any key accepted by `/api/download` (see `docs/API.md`) can also be declared directly under `serve:` and the REST API will treat it as a default. Per-request bodies still win. Use this to raise concurrency, force `best_available`, etc. without each client repeating the values: + +```yaml +serve: + api_secret: "..." + users: { ... } + downloads: 4 # parallel tracks per job + workers: 16 # threads per track + best_available: true + no_proxy_download: false +``` + +Layering: built-in defaults < `serve.*` overrides < service-specific defaults < request body. + +For example, + +```yaml +serve: + api_secret: "your-secret-key-here" + compression_level: 1 + session_ttl: 300 + max_sessions: 100 + # services: # global allowlist (optional) + # - EXAMPLE1 + # - EXAMPLE2 + users: + secret_key_for_jane: # 32bit hex recommended, case-sensitive + devices: # list of allowed Widevine devices for this user + - generic_nexus_4464_l3 + playready_devices: # list of allowed PlayReady devices for this user + - my_playready_device + username: jane # only for internal logging, users will not see this name + # services: # per-user allowlist (optional) + # - EXAMPLE1 + secret_key_for_james: + devices: + - generic_nexus_4464_l3 + username: james + # devices can be manually specified by path if you don't want to add it to + # envied's WVDs directory for whatever reason + # devices: + # - 'C:\Users\john\Devices\test_devices_001.wvd' + # playready_devices: + # - '/path/to/device.prd' +``` + +### REST API + +When the server is running, interactive API documentation is available at: + +- **Swagger UI**: `http://localhost:8786/api/docs/` + +See [API.md](API.md) for full REST API documentation with endpoints, parameters, and examples. + +--- + +## max_concurrent_downloads (int) + +Maximum number of `/api/download` jobs the serve queue manager will execute in parallel. Each job runs the full `dl` pipeline (authenticate, fetch tracks, decrypt, mux) in its own worker subprocess. This is independent of `serve.downloads`, which controls parallel tracks **inside** a single job. Default: `2`. + +```yaml +max_concurrent_downloads: 4 +``` + +--- + +## download_job_retention_hours (int) + +How long completed, failed, or cancelled download jobs remain queryable via `/api/download/jobs/{job_id}` before the periodic cleanup loop drops them. Default: `24`. + +```yaml +download_job_retention_hours: 48 +``` + +--- + +## debug (bool) + +Enables comprehensive debug logging. Default: `false` + +When enabled (either via config or the `-d`/`--debug` CLI flag): +- Sets console log level to DEBUG for verbose output +- Creates JSON Lines (`.jsonl`) debug log files with structured logging +- Logs detailed information about sessions, service configuration, DRM operations, and errors with full stack traces + +For example, + +```yaml +debug: true +``` + +--- + +## debug_keys (bool) + +Controls whether actual decryption keys (CEKs) are included in debug logs. Default: `false` + +When enabled: +- Content encryption keys are logged in debug output +- Only affects `content_key` and `key` fields (the actual CEKs) +- Key metadata (`kid`, `keys_count`, `key_id`) is always logged regardless of this setting +- Passwords, tokens, cookies, and session tokens remain redacted even when enabled + +For example, + +```yaml +debug_keys: true +``` + +--- + +## set_terminal_bg (bool) + +Controls whether envied should set the terminal background color. Default: `false` + +For example, + +```yaml +set_terminal_bg: true +``` + +--- + +## update_checks (bool) + +Check for updates from the GitHub repository on startup. Default: `true`. + +--- + +## update_check_interval (int) + +How often to check for updates, in hours. Default: `24`. + +--- + +## title_cache_enabled (bool) + +Enable or disable title metadata caching globally. Default: `true`. + +--- + +## title_cache_time (int) + +Title cache duration in seconds. Default: `1800` (30 minutes). + +--- + +## title_cache_max_retention (int) + +Maximum cache retention in seconds, used as fallback when the upstream API fails. Default: `86400` (24 hours). + +--- + +## unicode_filenames (bool) + +When `false`, replaces non-ASCII characters in output filenames with ASCII equivalents. Default: `false`. + +--- + +## ipinfo_api_key (str) + +Optional ipinfo.io token. When set, envied uses the ipinfo.io Lite endpoint for IP/geolocation lookups instead of the unauthenticated fallback. + +--- + +## tmdb_api_key (str) + +Optional TMDB API key, used for metadata enrichment and IMDb/TMDb tagging. + +--- + +## simkl_client_id (str) + +Optional Simkl client ID for metadata lookups. + +--- + +## decrypt_labs_api_key (str) + +Optional Decrypt Labs API key, used by services that integrate with the service. + +--- diff --git a/docs/API.md b/docs/API.md new file mode 100644 index 0000000..73f9091 --- /dev/null +++ b/docs/API.md @@ -0,0 +1,781 @@ +# REST API Documentation + +The envied REST API allows you to control downloads, search services, drive remote downloads from a thin client, and (optionally) co-host the pywidevine/pyplayready CDM. Start the server with `envied serve` and access the interactive Swagger UI at `http://localhost:8786/api/docs/`. + +The server is built on **aiohttp** (not FastAPI). Implementation lives in `envied/commands/serve.py` and `envied/core/api/` (`routes.py`, `handlers.py`, `session_store.py`, `input_bridge.py`, `download_manager.py`, `download_worker.py`). + +## Quick Start + +```bash +# Start the server (no authentication) +envied serve --no-key + +# Start with authentication (api_secret in envied.yaml) +envied serve + +# Serve only the REST API (no pywidevine/pyplayready CDM) +envied serve --api-only + +# Serve only the remote-dl session endpoints (CORS/Cloudflare friendly) +envied serve --remote-only + +# Disable just one CDM +envied serve --no-widevine +envied serve --no-playready + +# Verbose error responses (tracebacks/stderr in JSON) +envied serve --debug-api +``` + +`serve` flags: + +| Flag | Description | +| --- | --- | +| `-h, --host` | Bind host (default `127.0.0.1`) | +| `-p, --port` | Bind port (default `8786`) | +| `--caddy` | Also launch Caddy using `Caddyfile` next to the envied config | +| `--api-only` | REST API only; skip the bundled pywidevine/pyplayready CDM endpoints | +| `--no-widevine` | Disable Widevine CDM endpoints | +| `--no-playready` | Disable PlayReady CDM endpoints | +| `--no-key` | Disable API key authentication entirely | +| `--debug-api` | Include tracebacks/stderr in error responses | +| `--debug` | Enable DEBUG-level logging for API operations | +| `--remote-only` | Expose only `/api/health`, `/api/services`, `/api/search`, and `/api/session/*` (implies `--api-only`) | + +## Authentication + +When `api_secret` is set in `envied.yaml`, all API requests require the **`X-Secret-Key`** header. There is no query-parameter fallback. `/api/health` is always reachable without authentication. `--no-key` disables auth entirely (not recommended for public-facing servers). + +```yaml +# envied.yaml +serve: + api_secret: "your-master-secret" # falls back to global users map below + remote_only: false # also toggleable via --remote-only + services: ["EXAMPLE1", "EXAMPLE2"] # optional global service allowlist + users: + user-secret-1: + username: alice + devices: ["my_widevine_l3"] # Widevine WVD names this user may use + playready_devices: ["my_pr_sl2000"] # PlayReady PRD names; defaults to [] (no access) + services: ["EXAMPLE1"] # optional per-user allowlist (intersected with global) + user-secret-2: + username: bob + devices: [] + playready_devices: [] +``` + +### Service allowlists + +`config.serve.services` is the global allowlist; `users..services` further narrows it per key. The effective set is the intersection. Endpoints affected: `/api/services`, `/api/search`, `/api/list-titles`, `/api/list-tracks`, `/api/download`, and all `/api/session/*` routes. + +### CDM access (server-side decryption) + +There is no separate "tier" flag. Whether the server can return KID:KEY for a session-mode download depends solely on the device lists configured for the calling user key: + +- Empty `devices` and `playready_devices` -> server can only proxy CDM challenges; the client must run its own CDM and parse the license. +- Populated lists -> the client may set `mode: "server_cdm"` on `/api/session/{id}/license` and receive `{ "keys": { "": { "": "" } } }` instead of raw license bytes. + +Per-service CDM type can be pinned via `config.cdm` (`widevine`/`playready`) or per-service `cdm_type`; otherwise the server picks the type the user has devices for. + +### Server-side `dl` defaults + +Any flag accepted by `/api/download` (see the table below) can be declared under `serve:` in `envied.yaml` and the API will apply it as a default. Request-body values still win. Useful for raising concurrency without changing every client call: + +```yaml +serve: + api_secret: "..." + users: { ... } + downloads: 4 # parallel tracks per download job + workers: 16 # threads per track segment fetch + best_available: true + no_proxy_download: false +``` + +Layering order: built-in defaults < `serve.*` overrides < service-specific click defaults < request body. + +--- + +## Endpoint Map + +Standard endpoints (suppressed in `--remote-only` mode are marked R): + +| Method | Path | R | +| --- | --- | :-: | +| GET | `/api/health` | ok | +| GET | `/api/services` | ok | +| POST | `/api/search` | ok | +| POST | `/api/list-titles` | hidden | +| POST | `/api/list-tracks` | hidden | +| POST | `/api/download` | hidden | +| GET | `/api/download/jobs` | hidden | +| GET | `/api/download/jobs/{job_id}` | hidden | +| DELETE | `/api/download/jobs/{job_id}` | hidden | +| POST | `/api/session/create` | ok | +| GET | `/api/session/{session_id}` | ok | +| DELETE | `/api/session/{session_id}` | ok | +| GET | `/api/session/{session_id}/titles` | ok | +| POST | `/api/session/{session_id}/tracks` | ok | +| POST | `/api/session/{session_id}/segments` | ok | +| POST | `/api/session/{session_id}/license` | ok | +| GET | `/api/session/{session_id}/prompt` | ok | +| POST | `/api/session/{session_id}/prompt` | ok | + +CDM endpoints (`/{wvd}/...`, `/playready/{prd}/...`) are exposed unless `--api-only` / `--remote-only` / `--no-widevine` / `--no-playready` is set, and use pywidevine / pyplayready's own auth scheme. + +--- + +## Endpoints + +### GET /api/health + +Health check with version and update information. Always reachable without auth. + +```bash +curl http://localhost:8786/api/health +``` + +```json +{ + "status": "ok", + "version": "4.0.0", + "update_check": { + "update_available": false, + "current_version": "4.0.0", + "latest_version": null + } +} +``` + +--- + +### GET /api/services + +List all available streaming services (filtered by the effective allowlist for the caller). + +```bash +curl -H "X-Secret-Key: $KEY" http://localhost:8786/api/services +``` + +Returns `{"services": [...]}`. Each entry has `tag`, `aliases`, `geofence`, `title_regex`, `url` (from `cli.short_help`), `help` (full docstring), and `cli_params` describing the service-level Click parameters. + +--- + +### POST /api/search + +Search for titles from a streaming service. + +**Required parameters:** +| Parameter | Type | Description | +| --- | --- | --- | +| `service` | string | Service tag | +| `query` | string | Search query | + +**Optional parameters:** +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `profile` | string | `null` | Profile for credentials/cookies | +| `proxy` | string | `null` | Proxy URI or country code | +| `no_proxy` | boolean | `false` | Disable all proxy use | + +```bash +curl -X POST http://localhost:8786/api/search \ + -H "X-Secret-Key: $KEY" \ + -H "Content-Type: application/json" \ + -d '{"service": "EXAMPLE1", "query": "example show"}' +``` + +```json +{ + "results": [ + { + "id": "abc123def456", + "title": "Example Show", + "description": null, + "label": "TV Show", + "url": "https://example.com/show/abc123def456" + } + ], + "count": 1 +} +``` + +--- + +### POST /api/list-titles + +Get available titles (seasons/episodes/movies) for a service and title ID. Disabled in `--remote-only` mode. + +**Required parameters:** +| Parameter | Type | Description | +| --- | --- | --- | +| `service` | string | Service tag | +| `title_id` | string | Title ID or URL | + +```bash +curl -X POST http://localhost:8786/api/list-titles \ + -H "X-Secret-Key: $KEY" \ + -H "Content-Type: application/json" \ + -d '{"service": "EXAMPLE1", "title_id": "abc123def456"}' +``` + +--- + +### POST /api/list-tracks + +Get video, audio, and subtitle tracks for a title. Disabled in `--remote-only` mode. + +**Required parameters:** +| Parameter | Type | Description | +| --- | --- | --- | +| `service` | string | Service tag | +| `title_id` | string | Title ID or URL | + +**Optional parameters:** +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `wanted` | array | all | Episode filter (e.g., `["S01E01"]`) | +| `profile` | string | `null` | Profile for credentials/cookies | +| `proxy` | string | `null` | Proxy URI or country code | +| `no_proxy` | boolean | `false` | Disable all proxy use | + +Returns video, audio, and subtitle tracks with codec, bitrate, resolution, language, and DRM information. + +--- + +### POST /api/download + +Start a download job. Returns immediately with a job ID (HTTP 202). Disabled in `--remote-only` mode. + +**Required parameters:** +| Parameter | Type | Description | +| --- | --- | --- | +| `service` | string | Service tag | +| `title_id` | string | Title ID or URL | + +**Quality and codec parameters:** +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `quality` | array[int] | best | Resolution(s) (e.g., `[1080, 2160]`) | +| `vcodec` | string or array | any | Video codec(s): `H264`, `H265`/`HEVC`, `VP9`, `AV1`, `VC1`, `VP8` | +| `acodec` | string or array | any | Audio codec(s): `AAC`, `AC3`, `EC3`, `AC4`, `OPUS`, `FLAC`, `ALAC`, `DTS`, `OGG` | +| `vbitrate` | int | highest | Video bitrate in kbps | +| `abitrate` | int | highest | Audio bitrate in kbps | +| `range` | array[string] | `["SDR"]` | Color range(s): `SDR`, `HDR10`, `HDR10+`, `HLG`, `DV`, `HYBRID` | +| `channels` | float | any | Audio channels (e.g., `5.1`, `7.1`) | +| `no_atmos` | boolean | `false` | Exclude Dolby Atmos tracks | +| `split_audio` | boolean | `null` | Create separate output per audio codec | +| `sub_format` | string | `null` | Output subtitle format: `SRT`, `VTT`, `ASS`, `SSA`, `TTML` | + +**Episode selection:** +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `wanted` | array[string] | all | Episodes (e.g., `["S01E01", "S01E02-S01E05"]`) | +| `latest_episode` | boolean | `false` | Download only the most recent episode | + +**Language parameters:** +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `lang` | array[string] | `["orig"]` | Language for video and audio (`orig` = original) | +| `v_lang` | array[string] | `[]` | Language override for video tracks only | +| `a_lang` | array[string] | `[]` | Language override for audio tracks only | +| `s_lang` | array[string] | `["all"]` | Language for subtitles | +| `require_subs` | array[string] | `[]` | Required subtitle languages (skip if missing) | +| `forced_subs` | boolean | `false` | Include forced subtitle tracks | +| `exact_lang` | boolean | `false` | Exact language matching (no variants) | + +**Track selection:** +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `video_only` | boolean | `false` | Only download video tracks | +| `audio_only` | boolean | `false` | Only download audio tracks | +| `subs_only` | boolean | `false` | Only download subtitle tracks | +| `chapters_only` | boolean | `false` | Only download chapters | +| `no_video` | boolean | `false` | Skip video tracks | +| `no_audio` | boolean | `false` | Skip audio tracks | +| `no_subs` | boolean | `false` | Skip subtitle tracks | +| `no_chapters` | boolean | `false` | Skip chapters | +| `audio_description` | boolean | `false` | Include audio description tracks | + +**Output and tagging:** +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `tag` | string | `null` | Override group tag | +| `repack` | boolean | `false` | Add REPACK tag to filename | +| `tmdb_id` | int | `null` | Use specific TMDB ID for tagging | +| `imdb_id` | string | `null` | Use specific IMDB ID (e.g., `tt1375666`) | +| `animeapi_id` | string | `null` | Anime database ID via AnimeAPI (e.g., `mal:12345`) | +| `enrich` | boolean | `false` | Override show title and year from external source | +| `no_folder` | boolean | `false` | Disable folder creation for TV shows | +| `no_source` | boolean | `false` | Remove source tag from filename | +| `no_mux` | boolean | `false` | Do not mux tracks into container | +| `output_dir` | string | `null` | Override output directory | + +**Download behavior:** +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `profile` | string | `null` | Profile for credentials/cookies | +| `proxy` | string | `null` | Proxy URI or country code | +| `no_proxy` | boolean | `false` | Disable all proxy use | +| `no_proxy_download` | boolean | `false` | Bypass proxy for segment downloads only. Manifest, license, and auth still use proxy | +| `workers` | int | `null` | Max threads per track download | +| `downloads` | int | `1` | Concurrent track downloads | +| `slow` | boolean or string | `null` | Add randomized delay between titles. `true` = 60-120s, or `"MIN-MAX"` string (e.g., `"20-40"`). Min must be >= 20 | +| `best_available` | boolean | `false` | Continue if requested quality unavailable | +| `worst` | boolean | `false` | Select the lowest bitrate track within the specified quality. Requires `quality` | +| `skip_dl` | boolean | `false` | Skip download, only get decryption keys | +| `export` | boolean | `false` | Export manifest, track URLs, keys, and subtitles to JSON in the exports directory | +| `cdm_only` | boolean | `null` | Only use CDM (`true`) or only vaults (`false`) | +| `no_cache` | boolean | `false` | Bypass title cache | +| `reset_cache` | boolean | `false` | Clear title cache before fetching | + +**Example:** + +```bash +curl -X POST http://localhost:8786/api/download \ + -H "X-Secret-Key: $KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "service": "EXAMPLE1", + "title_id": "abc123def456", + "wanted": ["S01E01"], + "quality": [1080, 2160], + "vcodec": ["H265"], + "acodec": ["AAC", "EC3"], + "range": ["HDR10", "SDR"], + "split_audio": true, + "lang": ["en"] + }' +``` + +```json +{ + "job_id": "504db959-80b0-446c-a764-7924b761d613", + "status": "queued", + "created_time": "2026-02-27T18:00:00.000000" +} +``` + +--- + +### GET /api/download/jobs + +List all download jobs with optional filtering and sorting. Disabled in `--remote-only` mode. + +**Query parameters:** +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `status` | string | all | Filter by status: `queued`, `downloading`, `completed`, `failed`, `cancelled` | +| `service` | string | all | Filter by service tag | +| `sort_by` | string | `created_time` | Sort field: `created_time`, `started_time`, `completed_time`, `progress`, `status`, `service` | +| `sort_order` | string | `desc` | Sort order: `asc`, `desc` | + +```bash +curl -H "X-Secret-Key: $KEY" "http://localhost:8786/api/download/jobs?status=completed" +``` + +--- + +### GET /api/download/jobs/{job_id} + +Get detailed information about a specific download job including progress, parameters, and error details. + +```json +{ + "job_id": "504db959-80b0-446c-a764-7924b761d613", + "status": "completed", + "created_time": "2026-02-27T18:00:00.000000", + "service": "EXAMPLE1", + "title_id": "abc123def456", + "progress": 100.0, + "parameters": { }, + "started_time": "2026-02-27T18:00:01.000000", + "completed_time": "2026-02-27T18:00:15.000000", + "output_files": [], + "error_message": null, + "error_details": null +} +``` + +--- + +### DELETE /api/download/jobs/{job_id} + +Cancel a queued or running download job. Returns 400 if the job has already terminated. + +--- + +## Remote Service Sessions + +These endpoints back the `RemoteService` adapter in `envied/core/remote_service.py`. They let a thin `dl` client (or any consumer) authenticate against a service on the server, fetch titles/tracks/manifests, and either proxy CDM challenges or have the server resolve KID:KEY directly. The `dl` command's `RemoteService` adapter replaces the old `remote_dl` command. These endpoints are the only `/api/*` routes available in `--remote-only` mode (in addition to `health`, `services`, and `search`). + +### POST /api/session/create + +Authenticate against a service and open a session. Body fields: + +| Field | Type | Description | +| --- | --- | --- | +| `service` | string | Service tag (required) | +| `title_id` | string | Title ID/URL (required) | +| `credentials` | object | Auth credentials forwarded to `Service.authenticate` | +| `cookies` | string | Cookie blob (Netscape or JSON) | +| `proxy` | string | Proxy URI or country code | +| `no_proxy` | bool | Force-disable proxies | +| `profile` | string | Profile name | +| `cache` | object | Optional pre-warmed title cache payload | + +If the service requires interactive input during authentication, poll `GET /api/session/{id}/prompt` and submit responses via `POST /api/session/{id}/prompt` until status is `authenticated`. + +**Request:** + +```json +{ + "service": "EXAMPLE1", + "title_id": "abc123def456", + "credentials": {"username": "alice", "password": "hunter2"}, + "cookies": "# Netscape HTTP Cookie File\n...", + "proxy": "us", + "no_proxy": false, + "profile": "default", + "cache": {} +} +``` + +**Response (202-style; auth runs asynchronously):** + +```json +{ + "session_id": "f1c4a8b2-9c7e-4d2a-bf91-2d3e4f5a6b7c", + "service": "EXAMPLE1", + "status": "authenticating" +} +``` + +### GET /api/session/{session_id} + +Returns session metadata. 404 if expired or unknown. + +```json +{ + "session_id": "f1c4a8b2-9c7e-4d2a-bf91-2d3e4f5a6b7c", + "service": "EXAMPLE1", + "valid": true, + "expires_in": 3600, + "track_count": 0, + "title_count": 0 +} +``` + +### DELETE /api/session/{session_id} + +Tears down the session, cancels any pending prompts, and returns any updated per-session cache files (base64-encoded, zlib-compressed) so the client can re-warm next time. + +```json +{ + "status": "ok", + "cache": { + "tokens": "eJzLSM3JyVcozy/KSVGo5AIAGgQEvQ==" + } +} +``` + +### GET /api/session/{session_id}/titles + +Returns the resolved titles list. + +```json +{ + "session_id": "f1c4a8b2-9c7e-4d2a-bf91-2d3e4f5a6b7c", + "titles": [ + { + "type": "episode", + "name": "Pilot", + "series_title": "Example Show", + "season": 1, + "number": 1, + "year": 2024, + "id": "ep-0001", + "language": "en" + }, + { + "type": "movie", + "name": "Example Movie", + "year": 2024, + "id": "mov-0001", + "language": "en" + } + ] +} +``` + +### POST /api/session/{session_id}/tracks + +**Request:** + +```json +{"title_id": "ep-0001"} +``` + +**Response:** + +```json +{ + "title": { + "type": "episode", + "name": "Pilot", + "series_title": "Example Show", + "season": 1, + "number": 1, + "year": 2024, + "id": "ep-0001", + "language": "en" + }, + "video": [ + { + "id": "v-1080p-h264", + "codec": "H264", + "codec_display": "H.264", + "bitrate": 6000, + "width": 1920, + "height": 1080, + "resolution": "1920x1080", + "fps": "23.976", + "range": "SDR", + "range_display": "SDR", + "language": "en", + "drm": [ + { + "type": "widevine", + "pssh": "AAAAW3Bzc2gAAAAA7e+...", + "kids": ["abcdef0123456789abcdef0123456789"], + "license_url": "https://license.example.com/widevine" + } + ], + "descriptor": "DASH", + "url": "https://cdn.example.com/manifest.mpd" + } + ], + "audio": [ + { + "id": "a-en-eac3", + "codec": "EC3", + "codec_display": "Dolby Digital Plus", + "bitrate": 640, + "channels": "5.1", + "language": "en", + "atmos": false, + "descriptive": false, + "drm": null, + "descriptor": "DASH", + "url": "https://cdn.example.com/manifest.mpd" + } + ], + "subtitles": [ + { + "id": "s-en-vtt", + "codec": "WebVTT", + "language": "en", + "forced": false, + "sdh": false, + "cc": false, + "descriptor": "DASH", + "url": "https://cdn.example.com/subs/en.vtt" + } + ], + "chapters": [ + {"timestamp": "00:00:00.000", "name": "Chapter 1"} + ], + "attachments": [], + "manifests": [ + { + "type": "dash", + "url": "https://cdn.example.com/manifest.mpd", + "data": "eJzNVk1v2zAM/Ss..." + } + ], + "session_headers": { + "User-Agent": "Mozilla/5.0 ..." + }, + "session_cookies": { + "session": "abc123" + }, + "server_cdm_type": "widevine" +} +``` + +### POST /api/session/{session_id}/segments + +**Request:** + +```json +{"track_ids": ["v-1080p-h264", "a-en-eac3"]} +``` + +**Response:** + +```json +{ + "tracks": { + "v-1080p-h264": { + "descriptor": "DASH", + "url": "https://cdn.example.com/manifest.mpd", + "drm": [ + { + "type": "widevine", + "pssh": "AAAAW3Bzc2gAAAAA7e+...", + "kids": ["abcdef0123456789abcdef0123456789"], + "license_url": "https://license.example.com/widevine" + } + ], + "headers": {"User-Agent": "Mozilla/5.0 ..."}, + "cookies": {"session": "abc123"}, + "data": {} + }, + "a-en-eac3": { + "descriptor": "DASH", + "url": "https://cdn.example.com/manifest.mpd", + "drm": null, + "headers": {"User-Agent": "Mozilla/5.0 ..."}, + "cookies": {"session": "abc123"}, + "data": {} + } + } +} +``` + +### POST /api/session/{session_id}/license + +Two modes, selected by the `mode` field. + +**`mode: "proxy"` (default)** -- forward a client-built CDM challenge to the service's license endpoint. + +Request: + +```json +{ + "mode": "proxy", + "track_id": "v-1080p-h264", + "challenge": "CAESxQEK...", + "drm_type": "widevine", + "pssh": "AAAAW3Bzc2gAAAAA7e+..." +} +``` + +Response: + +```json +{"license": "CAIS3wIK..."} +``` + +**`mode: "server_cdm"`** -- the server uses its own CDM to license the track and extract keys. Single-track form takes `track_id`; batch form takes `track_ids`. Requires the calling user key to have a matching device (`devices` for Widevine, `playready_devices` for PlayReady) in `envied.yaml`. + +Request (batch): + +```json +{ + "mode": "server_cdm", + "track_ids": ["v-1080p-h264", "a-en-eac3"], + "drm_type": "widevine" +} +``` + +Response: + +```json +{ + "keys": { + "v-1080p-h264": { + "abcdef0123456789abcdef0123456789": "00112233445566778899aabbccddeeff" + }, + "a-en-eac3": { + "abcdef0123456789abcdef0123456789": "00112233445566778899aabbccddeeff" + } + }, + "drm_type": "widevine" +} +``` + +### GET /api/session/{session_id}/prompt + +Polled by the client during interactive authentication (OTP, PIN, device codes). Backed by the `InputBridge` in `envied/core/api/input_bridge.py`; `Service.request_input()` blocks server-side until the client posts a response. + +Pending input: + +```json +{"status": "pending_input", "prompt": "Enter OTP code: "} +``` + +Other states: + +```json +{"status": "authenticating"} +``` + +```json +{"status": "authenticated"} +``` + +```json +{"status": "failed", "error": "Invalid credentials"} +``` + +### POST /api/session/{session_id}/prompt + +Unblocks the server-side `request_input()` call. + +Request: + +```json +{"response": "123456"} +``` + +Response: + +```json +{"status": "accepted"} +``` + +--- + +## Error Responses + +All endpoints return consistent error responses: + +```json +{ + "status": "error", + "error_code": "INVALID_PARAMETERS", + "message": "Invalid vcodec: XYZ. Must be one of: H264, H265, VP9, AV1, VC1, VP8", + "timestamp": "2026-02-27T18:00:00.000000+00:00", + "details": { } +} +``` + +Common error codes: + +- `INVALID_INPUT` -- malformed request body +- `INVALID_PARAMETERS` -- invalid parameter values +- `MISSING_SERVICE` -- service tag not provided +- `INVALID_SERVICE` -- service not found or not in the caller's allowlist +- `SERVICE_ERROR` -- service initialization or runtime error +- `AUTH_FAILED` -- authentication failure +- `NOT_FOUND` / `TRACK_NOT_FOUND` / session not found -- job/session/track/title missing +- `INTERNAL_ERROR` -- unexpected server error + +When `--debug-api` is enabled, error responses include additional `debug_info` with tracebacks and stderr output. + +Authentication errors from the auth middleware are returned as `{"status": 401, "message": "..."}` (not the standard error envelope). + +--- + +## Download Job Lifecycle + +``` +queued -> downloading -> completed + \-> failed +queued -> cancelled +downloading -> cancelled +``` + +Jobs are retained for 24 hours after completion (override via top-level `download_job_retention_hours` in `envied.yaml`). The server runs up to 2 concurrent download jobs by default; override via top-level `max_concurrent_downloads`. This is independent of `serve.downloads`, which controls parallel tracks **within** a single job. + +Remote sessions are managed by `SessionStore` (`envied/core/api/session_store.py`); idle sessions and their `InputBridge` instances are cleaned up by a background loop started/stopped with the app lifecycle. diff --git a/docs/DOWNLOAD_CONFIG.md b/docs/DOWNLOAD_CONFIG.md new file mode 100644 index 0000000..24a842f --- /dev/null +++ b/docs/DOWNLOAD_CONFIG.md @@ -0,0 +1,322 @@ +# Download & Processing Configuration + +This document covers configuration options related to downloading and processing media content. + +## downloader + +envied ships a single unified downloader at `envied/core/downloaders/requests.py`. The legacy +`aria2c`, `curl_impersonate`, and `n_m3u8dl_re` backends have been removed; their config blocks no +longer have any effect. + +The unified downloader: + +- Works with both a standard `requests.Session` and `RnetSession` (rnet/BoringSSL TLS impersonation, + which replaces the previous `curl_cffi` backend). When a service exposes its own session via + `self.session`, TLS fingerprinting is preserved on every segment. +- Uses adaptive chunk sizing between **512 KB and 4 MB**, picked from the response `Content-Length`. +- Spawns **up to `min(16, cpu_count + 4)` worker threads** by default for segmented downloads + (override via `--workers` / `dl.workers`). +- Resumes interrupted downloads via HTTP `Range` requests (a sibling `.!dev` control file + marks an in-progress download). +- Has a single-URL fast path: if the server supports byte ranges and the file is at least 64 MB, + the file is split into 16 MB parts and downloaded in parallel into a pre-allocated file. +- Is selected per-track via `track.downloader`, which defaults to this unified `requests` downloader. + +There is no `downloader:` config key to set anymore. Setting one to a legacy value will emit a +`DeprecationWarning` and otherwise be ignored. + +--- + +## dl (dict) + +Pre-define default options and switches of the `dl` command. +The values will be ignored if explicitly set in the CLI call. + +The Key must be the same value Python click would resolve it to as an argument. +E.g., `@click.option("-r", "--range", "range_", type=...` actually resolves as `range_` variable. + +For example to set the default primary language to download to German, + +```yaml +lang: de +``` + +You can also set multiple preferred languages using a list, e.g., + +```yaml +lang: + - en + - fr +``` + +to set how many tracks to download concurrently to 4 and download threads to 16, + +```yaml +downloads: 4 +workers: 16 +``` + +to set `--bitrate=CVBR` for a specific service, + +```yaml +lang: de +EXAMPLE: + bitrate: CVBR +``` + +or to change the output subtitle format from the default (original format) to WebVTT, + +```yaml +sub_format: vtt +``` + +### All Available `dl` Keys + +Below is a comprehensive list of keys that can be pre-defined in the `dl` section. Each corresponds +to a CLI option on the `dl` command. CLI arguments always take priority over config values. + +**Quality and codec:** + +| Key | Type | Default | Description | +| --- | --- | --- | --- | +| `quality` | int or list | best | Resolution(s) to download (e.g., `1080`, `[1080, 2160]`) | +| `vcodec` | str or list | any | Video codec(s): `H264`, `H265`, `VP9`, `AV1`, `VC1` | +| `acodec` | str or list | any | Audio codec(s): `AAC`, `AC3`, `EC3`, `AC4`, `OPUS`, `FLAC`, `ALAC`, `DTS` | +| `vbitrate` | int | highest | Video bitrate in kbps | +| `abitrate` | int | highest | Audio bitrate in kbps | +| `vbitrate_range` | str | none | Video bitrate window in kbps, format `MIN-MAX` (e.g., `6000-7000`) | +| `abitrate_range` | str | none | Audio bitrate window in kbps, format `MIN-MAX` | +| `real_video_bitrate` | bool | `false` | Probe actual media size to compute true video bitrates, overriding the manifest's declared value (`-rvb`). See [Real bitrate probing](#real-bitrate-probing) | +| `real_audio_bitrate` | bool | `false` | Same as above for audio tracks (`-rab`). Slower than video (more renditions) | +| `range_` | str or list | `SDR` | Color range(s): `SDR`, `HDR10`, `HDR10+`, `HLG`, `DV`, `HYBRID` | +| `channels` | float | any | Audio channels (e.g., `5.1`, `7.1`) | +| `worst` | bool | `false` | Select the lowest bitrate track within the specified quality. Requires `quality` | +| `best_available` | bool | `false` | Continue if requested quality is unavailable | + +**Language:** + +| Key | Type | Default | Description | +| --- | --- | --- | --- | +| `lang` | str or list | `orig` | Language for video and audio (`orig` = original language) | +| `v_lang` | list | `[]` | Language override for video tracks only | +| `a_lang` | list | `[]` | Language override for audio tracks only | +| `s_lang` | list | `["all"]` | Language for subtitles | +| `require_subs` | list | `[]` | Required subtitle languages (skip title if missing) | +| `forced_subs` | bool | `false` | Include forced subtitle tracks | +| `exact_lang` | bool | `false` | Exact language matching (no regional variants) | +| `latest_episode` | bool | `false` | Download only the single most recent episode of a series | + +**Track selection:** + +| Key | Type | Default | Description | +| --- | --- | --- | --- | +| `video_only` | bool | `false` | Only download video tracks | +| `audio_only` | bool | `false` | Only download audio tracks | +| `subs_only` | bool | `false` | Only download subtitle tracks | +| `chapters_only` | bool | `false` | Only download chapters | +| `no_video` | bool | `false` | Skip video tracks | +| `no_audio` | bool | `false` | Skip audio tracks | +| `no_subs` | bool | `false` | Skip subtitle tracks | +| `no_chapters` | bool | `false` | Skip chapters | +| `no_atmos` | bool | `false` | Exclude Dolby Atmos audio tracks | +| `audio_description` | bool | `false` | Include audio description tracks | + +**Output and tagging:** + +| Key | Type | Default | Description | +| --- | --- | --- | --- | +| `tag` | str | config default | Override group tag | +| `repack` | bool | `false` | Add REPACK tag to output filename | +| `sub_format` | str | original | Output subtitle format: `srt`, `vtt`, `ass`, `ssa`, `ttml` | +| `no_folder` | bool | `false` | Disable folder creation for TV shows | +| `no_source` | bool | `false` | Remove source tag from filename | +| `no_mux` | bool | `false` | Do not mux tracks into a container file | +| `split_audio` | bool | `false` | Create separate output files per audio codec | +| `export` | bool | `false` | Write a JSON sidecar with manifest URLs, subtitles, per-track KID:KEY, codec/track info | + +**Metadata enrichment:** + +| Key | Type | Default | Description | +| --- | --- | --- | --- | +| `tmdb_id` | int | `null` | Use specific TMDB ID for tagging | +| `imdb_id` | str | `null` | Use specific IMDB ID (e.g., `tt1375666`) | +| `animeapi_id` | str | `null` | Anime database ID via AnimeAPI (e.g., `mal:12345`, `anilist:98765`) | +| `enrich` | bool | `false` | Override show title and year from external source. Requires `tmdb_id`, `imdb_id`, or `animeapi_id` | + +**Download behavior:** + +| Key | Type | Default | Description | +| --- | --- | --- | --- | +| `downloads` | int | `1` | Concurrent track downloads | +| `workers` | int | `min(16, cpu_count + 4)` | Max threads per track download (segments / ranged parts) | +| `slow` | bool or `MIN-MAX` | `false` | Randomized delay between titles. `true` uses 60-120s; pass `MIN-MAX` (e.g., `20-40`) for a custom range | +| `no_proxy_download` | bool | `false` | Bypass proxy for segment downloads only. Manifest, license, and auth still use proxy | +| `skip_dl` | bool | `false` | Skip download, only get decryption keys | +| `cdm_only` | bool | `null` | Only use CDM (`true`) or only vaults (`false`) | + +### Real bitrate probing + +Some services declare inaccurate `bandwidth`/`BANDWIDTH` in their manifests — often +a peak or nominal figure that is far from the real average. Because `track.bitrate` +drives the track listing, sorting, and `--vbitrate` / `--vbitrate-range` selection, +a wrong value picks the wrong track. + +`-rvb` / `--real-video-bitrate` (and `-rab` / `--real-audio-bitrate` for audio) +probe the actual media size and overwrite `track.bitrate` with the measured value +(`bytes * 8 / duration`) before listing and selection. So `-rvb --list` shows the +true numbers, and `-rvb --vbitrate-range 6000-7000` selects against them. Without +the flag, behaviour is unchanged (the manifest value is used). + +How it works: + +- **Single-file tracks** (one whole file per rendition — e.g. DASH `SegmentBase` + or services that collapse to a `BaseURL`) are measured **exactly**: the whole + file size over the track duration. +- **Multi-segment tracks** (most HLS) are a **sampled estimate** — a spread of + segments is probed and extrapolated, typically within a few percent. Segment + bytes include container overhead, so MPEG-TS HLS reads a few percent above the + demuxed stream (this is the real *delivered* size). +- Only the top renditions per quality tier are probed (video grouped by + codec + range, audio by codec + channels + language), in parallel, then extended + downward only as far as needed to keep ranking correct. This keeps the pass fast + even when a service exposes dozens of renditions. +- Tracks whose duration cannot be determined fall back to `ffprobe`; probe failures + are non-fatal and leave the manifest bitrate in place. + +Per-track before→after values are logged at debug level (run with `-d`); the +corrected values always appear in the Available Tracks panel. + +You can also set per-service `dl` overrides (see [Service Integration & Authentication Configuration](SERVICE_CONFIG.md)): + +```yaml +dl: + lang: en + downloads: 4 + workers: 16 + EXAMPLE: + bitrate: CVBR + EXAMPLE2: + worst: true + quality: 1080 +``` + +--- + +## audio (dict) + +Configuration for audio track selection. + +- `codec_priority` + Optional list of audio codec names defining the preferred order when multiple audio + tracks share the same bitrate and language. Listed codecs are ranked in the order given. + Codecs not in the list retain their bitrate-based ordering and are placed after all + listed codecs (i.e. soft priority — nothing is dropped). + + Atmos tracks still take precedence over codec priority, and audio description tracks + are still moved to the end. + + Valid codec names: `AAC`, `AC3`, `EC3`, `AC4`, `OPUS`, `OGG`, `DTS`, `ALAC`, `FLAC`. + +For example, + +```yaml +audio: + codec_priority: [FLAC, ALAC, AC4, EC3, DTS, AC3, OPUS, AAC, OGG] +``` + +Or to only prefer a subset (e.g. surround codecs first, everything else falls back to +bitrate order): + +```yaml +audio: + codec_priority: [EC3, DTS, AC3, AAC] +``` + +When unset, audio tracks are sorted by bitrate alone (with Atmos/descriptive rules still +applied). + +--- + +## subtitle (dict) + +Configuration for subtitle processing and conversion. + +- `conversion_method` + Method to use for converting subtitles between formats. Default: `"auto"` + - `"auto"` — Smart routing: uses subby for WebVTT/SAMI, pycaption for others. + - `"subby"` — Always use subby with advanced processing. + - `"pycaption"` — Use only pycaption library (no SubtitleEdit, no subby). + - `"subtitleedit"` — Prefer SubtitleEdit when available, fall back to pycaption. + - `"pysubs2"` — Use pysubs2 library (supports SRT/SSA/ASS/WebVTT/TTML/SAMI/MicroDVD/MPL2/TMP). +- `sdh_method` + Method to use for SDH (hearing impaired) stripping. Default: `"auto"` + - `"auto"` — Try subby (SRT only), then SubtitleEdit (if available), then subtitle-filter. + - `"subby"` — Use subby library (SRT only). + - `"subtitleedit"` — Use SubtitleEdit tool (Windows only, falls back to subtitle-filter). + - `"filter-subs"` — Use subtitle-filter library directly. +- `strip_sdh` + Automatically create stripped (non-SDH) versions of SDH subtitles. Default: `true` +- `convert_before_strip` + Auto-convert VTT/other formats to SRT before using subtitle-filter for SDH stripping. + Ensures compatibility when subtitle-filter is used as fallback. Default: `true` +- `preserve_formatting` + Preserve original subtitle formatting (tags, positioning, styling). + When `true`, skips pycaption processing for WebVTT files to keep tags like ``, ``, + positioning intact. Combined with no `sub_format` setting, ensures subtitles remain in + their original format. Default: `true` +- `output_mode` + Output mode for subtitles. Default: `"mux"` + - `"mux"` — Embed subtitles in MKV container only. + - `"sidecar"` — Save subtitles as separate files only. + - `"both"` — Embed in MKV and save as sidecar files. +- `sidecar_format` + Format for sidecar subtitle files when `output_mode` is `"sidecar"` or `"both"`. Default: `"srt"` + Options: `srt`, `vtt`, `ass`, `original` (keep current format). + +For example, + +```yaml +subtitle: + conversion_method: auto + sdh_method: auto + strip_sdh: true + convert_before_strip: true + preserve_formatting: true + output_mode: mux + sidecar_format: srt +``` + +--- + +## decryption (str | dict) + +Choose what software to use to decrypt DRM-protected content throughout envied where needed. +You may provide a single decryption method globally or a mapping of service tags to +decryption methods. + +Options: + +- `shaka` (default) - Shaka Packager - +- `mp4decrypt` - mp4decrypt from Bento4 - + +Note that Shaka Packager is the traditional method and works with most services. mp4decrypt +is an alternative that may work better with certain services that have specific encryption formats. + +Example mapping: + +```yaml +decryption: + EXAMPLE: mp4decrypt + EXAMPLE2: shaka + default: shaka +``` + +The `default` entry is optional. If omitted, `shaka` will be used for services not listed. + +Simple configuration (single method for all services): + +```yaml +decryption: mp4decrypt +``` + +--- diff --git a/docs/DRM_CONFIG.md b/docs/DRM_CONFIG.md new file mode 100644 index 0000000..acfb3cd --- /dev/null +++ b/docs/DRM_CONFIG.md @@ -0,0 +1,481 @@ +# DRM & CDM Configuration + +This document covers Digital Rights Management (DRM) and Content Decryption Module (CDM) configuration options. + +## cdm (dict) + +Pre-define which Widevine or PlayReady device to use for each Service by Service Tag as Key (case-sensitive). +The value should be a WVD or PRD filename without the file extension. When +loading the device, envied will look in both the `WVDs` and `PRDs` directories +for a matching file. + +For example, + +```yaml +EXAMPLE: chromecdm_903_l3 +EXAMPLE2: nexus_6_l1 +``` + +You may also specify this device based on the profile used. + +For example, + +```yaml +EXAMPLE: chromecdm_903_l3 +EXAMPLE2: nexus_6_l1 +EXAMPLE3: + john_sd: chromecdm_903_l3 + jane_uhd: nexus_5_l1 +``` + +You can also specify a fallback value to predefine if a match was not made. +This can be done using `default` key. This can help reduce redundancy in your specifications. + +For example, the following has the same result as the previous example, as well as all other +services and profiles being pre-defined to use `chromecdm_903_l3`. + +```yaml +EXAMPLE2: nexus_6_l1 +EXAMPLE3: + jane_uhd: nexus_5_l1 +default: chromecdm_903_l3 +``` + +You can also select CDMs based on video resolution using comparison operators (`>=`, `>`, `<=`, `<`) +or exact match on the resolution height. + +For example, + +```yaml +EXAMPLE: + "<=1080": generic_android_l3 # Use L3 for 1080p and below + ">1080": nexus_5_l1 # Use L1 for above 1080p (1440p, 2160p) + default: generic_android_l3 # Fallback if no quality match +``` + +You can mix profiles and quality thresholds in the same service: + +```yaml +EXAMPLE: + john: example_l3_profile # Profile-based selection + "<=720": example_mobile_l3 # Quality-based selection + "1080": example_standard_l3 # Exact match for 1080p + ">=1440": example_premium_l1 # Quality-based selection + default: example_standard_l3 # Fallback +``` + +--- + +## remote_cdm (list\[dict]) + +Configure remote CDM (Content Decryption Module) APIs to use for decrypting DRM-protected content. +Remote CDMs allow you to use high-security CDMs (L1/L2 for Widevine, SL2000/SL3000 for PlayReady) without +having the physical device files locally. + +envied supports multiple types of remote CDM providers: + +1. **DecryptLabs CDM** - Official DecryptLabs KeyXtractor API with intelligent caching +2. **Custom API CDM** - Highly configurable adapter for any third-party CDM API +3. **Legacy PyWidevine Serve** - Standard pywidevine serve-compliant APIs + +The name of each defined remote CDM can be referenced in the `cdm` configuration as if it was a local device file. + +### DecryptLabs Remote CDM + +DecryptLabs provides a professional CDM API service with support for multiple device types and intelligent key caching. + +**Supported Devices:** +- **Widevine**: `ChromeCDM` (L3), `L1` (Security Level 1), `L2` (Security Level 2) +- **PlayReady**: `SL2` (SL2000), `SL3` (SL3000) + +**Configuration:** + +```yaml +remote_cdm: + # Widevine L1 Device + - name: decrypt_labs_l1 + type: decrypt_labs # Required: identifies as DecryptLabs CDM + device_name: L1 # Required: must match exactly (L1, L2, ChromeCDM, SL2, SL3) + host: https://keyxtractor.decryptlabs.com + secret: YOUR_API_KEY # Your DecryptLabs API key + + # Widevine L2 Device + - name: decrypt_labs_l2 + type: decrypt_labs + device_name: L2 + host: https://keyxtractor.decryptlabs.com + secret: YOUR_API_KEY + + # Chrome CDM (L3) + - name: decrypt_labs_chrome + type: decrypt_labs + device_name: ChromeCDM + host: https://keyxtractor.decryptlabs.com + secret: YOUR_API_KEY + + # PlayReady SL2000 + - name: decrypt_labs_playready_sl2 + type: decrypt_labs + device_name: SL2 + device_type: PLAYREADY # Required for PlayReady + host: https://keyxtractor.decryptlabs.com + secret: YOUR_API_KEY + + # PlayReady SL3000 + - name: decrypt_labs_playready_sl3 + type: decrypt_labs + device_name: SL3 + device_type: PLAYREADY + host: https://keyxtractor.decryptlabs.com + secret: YOUR_API_KEY +``` + +**Features:** +- Intelligent key caching system (reduces API calls) +- Automatic integration with envied's vault system +- Support for both Widevine and PlayReady +- Multiple security levels (L1, L2, L3, SL2000, SL3000) + +**Note:** The `device_type` field determines whether the CDM operates in PlayReady or Widevine mode. +Setting `device_type: PLAYREADY` (or using `device_name: SL2` / `SL3`) activates PlayReady mode. +The `security_level` field is auto-computed from `device_name` when not specified (e.g., SL2 defaults +to 2000, SL3 to 3000, and Widevine devices default to 3). You can override these if needed. + +### Custom API Remote CDM + +A highly configurable CDM adapter that can work with virtually any third-party CDM API through YAML configuration. +This allows you to integrate custom CDM services without writing code. + +**Basic Example:** + +```yaml +remote_cdm: + - name: custom_chrome_cdm + type: custom_api # Required: identifies as Custom API CDM + host: https://your-cdm-api.com + timeout: 30 # Optional: request timeout in seconds + + device: + name: ChromeCDM + type: CHROME # CHROME, ANDROID, PLAYREADY + system_id: 27175 + security_level: 3 + + auth: + type: bearer # bearer, header, basic, body + key: YOUR_API_TOKEN + + endpoints: + get_request: + path: /get-challenge + method: POST + decrypt_response: + path: /get-keys + method: POST + + caching: + enabled: true # Enable key caching + use_vaults: true # Integrate with vault system +``` + +**Advanced Example with Field Mapping:** + +```yaml +remote_cdm: + - name: advanced_custom_api + type: custom_api + host: https://api.example.com + device: + name: L1 + type: ANDROID + security_level: 1 + + # Authentication configuration + auth: + type: header + header_name: X-API-Key + key: YOUR_SECRET_KEY + custom_headers: + User-Agent: envied/3.1.0 + X-Client-Version: "1.0" + + # Endpoint configuration + endpoints: + get_request: + path: /v2/challenge + method: POST + timeout: 30 + decrypt_response: + path: /v2/decrypt + method: POST + timeout: 30 + + # Request parameter mapping + request_mapping: + get_request: + param_names: + init_data: pssh # Rename 'init_data' to 'pssh' + scheme: device_type # Rename 'scheme' to 'device_type' + static_params: + api_version: "2.0" # Add static parameter + decrypt_response: + param_names: + license_request: challenge + license_response: license + + # Response field mapping + response_mapping: + get_request: + fields: + challenge: data.challenge # Deep field access + session_id: session.id + success_conditions: + - status == 'ok' # Validate response + decrypt_response: + fields: + keys: data.keys + key_fields: + kid: key_id # Map 'kid' field + key: content_key # Map 'key' field + + caching: + enabled: true + use_vaults: true + check_cached_first: true # Check cache before API calls +``` + +**Supported Authentication Types:** +- `bearer` - Bearer token authentication +- `header` - Custom header authentication +- `basic` - HTTP Basic authentication +- `body` - Credentials in request body +- `query` - Authentication added to query string parameters + +### Legacy PyWidevine Serve Format + +Standard [pywidevine] serve-compliant remote CDM configuration (backwards compatibility). + +```yaml +remote_cdm: + - name: legacy_chrome_cdm + device_name: chrome + device_type: CHROME + system_id: 27175 + security_level: 3 + host: https://domain.com/api + secret: secret_key +``` + +**Note:** If the `type` field is not specified, the entry is treated as a legacy pywidevine serve CDM. + +[pywidevine]: https://github.com/rlaphoenix/pywidevine + +--- + +## decrypt_labs_api_key (str) + +API key for DecryptLabs CDM service integration. + +When set, enables the use of DecryptLabs remote CDM services in your `remote_cdm` configuration. +This is used specifically for `type: "decrypt_labs"` entries in the remote CDM list. + +For example, + +```yaml +decrypt_labs_api_key: "your_api_key_here" +``` + +**Note**: This is different from the per-CDM `secret` field in `remote_cdm` entries. This provides a global +API key that can be referenced across multiple DecryptLabs CDM configurations. If a `remote_cdm` entry with +`type: "decrypt_labs"` does not have a `secret` field specified, the global `decrypt_labs_api_key` will be +used as a fallback. + +--- + +## decryption (str|dict) + +Configure which decryption tool to use for DRM-protected content. Default: `shaka`. + +Supported values: +- `shaka` - Shaka Packager (default) +- `mp4decrypt` - Bento4 mp4decrypt + +You can specify a single decrypter for all services: + +```yaml +decryption: shaka +``` + +Or configure per-service with a `DEFAULT` fallback: + +```yaml +decryption: + DEFAULT: shaka + EXAMPLE: mp4decrypt + EXAMPLE2: shaka +``` + +Service keys are case-insensitive (normalized to uppercase internally). + +--- + +## MonaLisa DRM + +MonaLisa is a WASM-based DRM system that uses local key extraction and two-stage segment decryption. +Unlike Widevine and PlayReady, MonaLisa does not use a challenge/response flow with a license server. +Instead, the PSSH value (ticket) is provided directly by the service API, and keys are extracted +locally via a WASM module. + +### Requirements + +- **ML-Worker binary**: Must be available on your system `PATH` (discovered via `binaries.ML_Worker`). + This is the binary that performs stage-1 decryption. + +### Decryption stages + +1. **ML-Worker binary**: Removes MonaLisa encryption layer (bbts -> ents). The key is passed via command-line argument. +2. **AES-ECB decryption**: Final decryption with service-provided key. + +MonaLisa uses per-segment decryption during download (not post-download like Widevine/PlayReady), +so segments are decrypted as they are downloaded. + +**Note:** MonaLisa is configured per-service rather than through global config options. Services +that use MonaLisa handle ticket/key retrieval and CDM initialization internally. + +--- + +## key_vaults (list\[dict]) + +Key Vaults store your obtained Content Encryption Keys (CEKs) and Key IDs per-service. + +This can help reduce unnecessary License calls even during the first download. This is because a Service may +provide the same Key ID and CEK for both Video and Audio, as well as for multiple resolutions or bitrates. + +You can have as many Key Vaults as you would like. It's nice to share Key Vaults or use a unified Vault on +Teams as sharing CEKs immediately can help reduce License calls drastically. + +Four types of Vaults are in the Core codebase: API, SQLite, MySQL, and HTTP. API and HTTP make HTTP requests to a RESTful API, +whereas SQLite and MySQL directly connect to an SQLite or MySQL Database. + +Note: SQLite and MySQL vaults have to connect directly to the Host/IP. It cannot be in front of a PHP API or such. +Beware that some Hosting Providers do not let you access the MySQL server outside their intranet and may not be +accessible outside their hosting platform. + +Additional behavior: + +- `no_push` (bool): Optional per-vault flag. When `true`, the vault will not receive pushed keys (writes) but + will still be queried and can provide keys for lookups. Useful for read-only/backup vaults. + +### Using an API Vault + +API vaults use a specific HTTP request format, therefore API or HTTP Key Vault APIs from other projects or services may +not work in envied. The API format can be seen in the [API Vault Code](envied/vaults/API.py). + +```yaml +- type: API + name: "John#0001's Vault" # arbitrary vault name + uri: "https://key-vault.example.com" # api base uri (can also be an IP or IP:Port) + # uri: "127.0.0.1:80/key-vault" + # uri: "https://api.example.com/key-vault" + token: "random secret key" # authorization token + # no_push: true # optional; make this API vault read-only (lookups only) +``` + +### Using a MySQL Vault + +MySQL vaults can be either MySQL or MariaDB servers. I recommend MariaDB. +A MySQL Vault can be on a local or remote network, but I recommend SQLite for local Vaults. + +```yaml +- type: MySQL + name: "John#0001's Vault" # arbitrary vault name + host: "127.0.0.1" # host/ip + # port: 3306 # port (defaults to 3306) + database: vault # database used for envied + username: jane11 + password: Doe123 + # no_push: false # optional; defaults to false +``` + +I recommend giving only a trustable user (or yourself) CREATE permission and then use envied to cache at least one CEK +per Service to have it create the tables. If you don't give any user permissions to create tables, you will need to +make tables yourself. + +- Use a password on all user accounts. +- Never use the root account with envied (even if it's you). +- Do not give multiple users the same username and/or password. +- Only give users access to the database used for envied. +- You may give trusted users CREATE permission so envied can create tables if needed. +- Other uses should only be given SELECT and INSERT permissions. + +### Using an SQLite Vault + +SQLite Vaults are usually only used for locally stored vaults. This vault may be stored on a mounted Cloud storage +drive, but I recommend using SQLite exclusively as an offline-only vault. Effectively this is your backup vault in +case something happens to your MySQL Vault. + +```yaml +- type: SQLite + name: "My Local Vault" # arbitrary vault name + path: "C:/Users/Jane11/Documents/envied/data/key_vault.db" + # no_push: true # optional; commonly true for local backup vaults +``` + +**Note**: You do not need to create the file at the specified path. +SQLite will create a new SQLite database at that path if one does not exist. +Try not to accidentally move the `db` file once created without reflecting the change in the config, or you will end +up with multiple databases. + +If you work on a Team I recommend every team member having their own SQLite Vault even if you all use a MySQL vault +together. + +### Using an HTTP Vault + +HTTP Vaults provide flexible HTTP-based key storage with support for multiple API modes. This vault type +is useful for integrating with various third-party key vault APIs. + +```yaml +- type: HTTP + name: "My HTTP Vault" + host: "https://vault-api.example.com" + api_key: "your_api_key" # or use 'password' field + api_mode: "json" # query, json, or decrypt_labs + # username: "user" # required for query mode only + # no_push: false # optional; defaults to false +``` + +**Supported API Modes:** + +- `query` - Uses GET requests with query parameters. Requires `username` field. +- `json` - Uses POST requests with JSON payloads. Token-based authentication. +- `decrypt_labs` - DecryptLabs API format. Read-only mode (`no_push` is forced to `true`). + +**Example configurations:** + +```yaml +# Query mode (requires username) +- type: HTTP + name: "Query Vault" + host: "https://api.example.com/keys" + username: "myuser" + password: "mypassword" + api_mode: "query" + +# JSON mode +- type: HTTP + name: "JSON Vault" + host: "https://api.example.com/vault" + api_key: "secret_token" + api_mode: "json" + +# DecryptLabs mode (read-only) +- type: HTTP + name: "DecryptLabs Cache" + host: "https://keyxtractor.decryptlabs.com/cache" + api_key: "your_decrypt_labs_api_key" + api_mode: "decrypt_labs" +``` + +**Note**: The `decrypt_labs` mode is always read-only and cannot receive pushed keys. + +--- diff --git a/docs/GLUETUN.md b/docs/GLUETUN.md new file mode 100644 index 0000000..0a9599e --- /dev/null +++ b/docs/GLUETUN.md @@ -0,0 +1,243 @@ +# Gluetun VPN Proxy + +Gluetun provides Docker-managed VPN proxies supporting 50+ VPN providers. + +## Prerequisites + +**Docker must be installed and running.** + +```bash +# Linux +curl -fsSL https://get.docker.com | sh +sudo usermod -aG docker $USER # Then log out/in + +# Windows/Mac +# Install Docker Desktop: https://www.docker.com/products/docker-desktop/ +``` + +## Quick Start + +### 1. Configuration + +Add to `~/.config/envied/envied.yaml`: + +```yaml +proxy_providers: + gluetun: + providers: + windscribe: + vpn_type: openvpn + credentials: + username: "YOUR_OPENVPN_USERNAME" + password: "YOUR_OPENVPN_PASSWORD" +``` + +### 2. Usage + +Use 2-letter country codes directly: + +```bash +envied dl SERVICE CONTENT --proxy gluetun:windscribe:us +envied dl SERVICE CONTENT --proxy gluetun:windscribe:uk +``` + +Format: `gluetun:provider:region` + +## Provider Credential Requirements + +**OpenVPN (Recommended)**: Most providers support OpenVPN with just `username` and `password` - the simplest setup. + +**WireGuard**: Requires private keys and varies by provider. See the [Gluetun Wiki](https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers) for provider-specific requirements. Note that `vpn_type` defaults to `wireguard` if not specified. + +## Getting Your Credentials + +### Windscribe (OpenVPN) + +1. Go to [windscribe.com/getconfig/openvpn](https://windscribe.com/getconfig/openvpn) +2. Log in with your Windscribe account +3. Select any location and click "Get Config" +4. Copy the username and password shown + +### NordVPN (OpenVPN) + +1. Go to [NordVPN Service Credentials](https://my.nordaccount.com/dashboard/nordvpn/manual-configuration/service-credentials/) +2. Log in with your NordVPN account +3. Generate or view your service credentials +4. Copy the username and password + +> **Note**: Use service credentials, NOT your account email/password. + +### WireGuard Credentials (Advanced) + +WireGuard requires private keys instead of username/password. See the [Gluetun Wiki](https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers) for provider-specific WireGuard setup. + +## Configuration Examples + +**OpenVPN (Recommended)** + +Most providers support OpenVPN with just username and password: + +```yaml +providers: + windscribe: + vpn_type: openvpn + credentials: + username: YOUR_OPENVPN_USERNAME + password: YOUR_OPENVPN_PASSWORD + + nordvpn: + vpn_type: openvpn + credentials: + username: YOUR_SERVICE_USERNAME + password: YOUR_SERVICE_PASSWORD +``` + +**WireGuard (Advanced)** + +WireGuard can be faster but requires more complex credential setup: + +```yaml +# NordVPN/ProtonVPN (only private_key needed) +providers: + nordvpn: + vpn_type: wireguard + credentials: + private_key: YOUR_PRIVATE_KEY + +# Surfshark/Mullvad/IVPN (private_key AND addresses required) + surfshark: + vpn_type: wireguard + credentials: + private_key: YOUR_PRIVATE_KEY + addresses: 10.x.x.x/32 + +# Windscribe (all three credentials required) + windscribe: + vpn_type: wireguard + credentials: + private_key: YOUR_PRIVATE_KEY + addresses: 10.x.x.x/32 + preshared_key: YOUR_PRESHARED_KEY +``` + +## Server Selection + +Most providers use `SERVER_COUNTRIES`, but some use `SERVER_REGIONS`: + +| Variable | Providers | +|----------|-----------| +| `SERVER_COUNTRIES` | NordVPN, ProtonVPN, Surfshark, Mullvad, ExpressVPN, and most others | +| `SERVER_REGIONS` | Windscribe, VyprVPN, VPN Secure | + +envied handles this automatically - just use 2-letter country codes. + +### Per-Provider Server Mapping + +You can explicitly map region codes to country names, cities, or hostnames per provider: + +```yaml +providers: + nordvpn: + vpn_type: openvpn + credentials: + username: YOUR_USERNAME + password: YOUR_PASSWORD + server_countries: + us: "United States" + uk: "United Kingdom" + server_cities: + us: "New York" + server_hostnames: + us: "us1239.nordvpn.com" +``` + +### Specific Server Selection + +Use a `` region (e.g. `us1239`) to target a specific server. envied builds the +hostname automatically per provider: + +| Provider | Hostname format | +|----------|-----------------| +| NordVPN | `us1239.nordvpn.com` | +| Surfshark | `us-1239.prod.surfshark.com` | +| ExpressVPN | `us-1239.expressvpn.com` | +| CyberGhost | `us-s1239.cg-dialup.net` | +| Other | `us1239` (passed as-is to `SERVER_HOSTNAMES`) | + +### Extra Environment Variables + +You can pass additional Gluetun environment variables per provider using `extra_env`: + +```yaml +providers: + nordvpn: + vpn_type: openvpn + credentials: + username: YOUR_USERNAME + password: YOUR_PASSWORD + extra_env: + LOG_LEVEL: debug +``` + +## Global Settings + +```yaml +proxy_providers: + gluetun: + providers: {...} + base_port: 8888 # Starting port (default: 8888) + auto_cleanup: true # Remove containers on exit (default: true) + verify_ip: true # Verify IP matches region (default: true) + container_prefix: "envied-gluetun" # Docker container name prefix (default: "envied-gluetun") + auth_user: username # Proxy auth (optional) + auth_password: password # Proxy auth (optional) +``` + +## Features + +- **Container Reuse**: First request takes 10-30s; subsequent requests are instant. Containers created by other envied processes are auto-detected via `docker inspect` and reused. +- **Ready Detection**: Waits up to 60s for both the HTTP proxy to listen (`[http proxy] listening`) and the VPN tunnel to come up (`initialization sequence completed` or `public ip address is`) before returning the proxy URI. Bails early on `fatal` or `invalid credentials` log lines. +- **IP Verification**: When `verify_ip: true` (default), looks up the exit IP via `ipinfo.io` through the proxy and compares country code to the requested region. Retries 3 times with exponential backoff (1s, 2s, 4s). +- **Concurrent Sessions**: Multiple downloads share the same container; ports are allocated thread-safely starting at `base_port`. +- **Specific Servers**: Use `--proxy gluetun:nordvpn:us1239` for specific server selection (see table above). +- **Automatic Image Pull**: The Gluetun Docker image (`qmcgaw/gluetun:latest`) is pulled automatically on first use (5 min timeout). +- **Secure Credentials**: Credentials are passed via temporary env files (mode 0600), then zero-overwritten and unlinked after `docker run`. They never appear in process listings. +- **Auto Cleanup**: Containers are removed via `atexit` (Ctrl+C still works normally). Disable with `auto_cleanup: false` to leave them stopped instead. + +## Container Management + +```bash +# View containers +docker ps | grep envied-gluetun + +# Check logs +docker logs envied-gluetun-nordvpn-us + +# Remove all containers +docker ps -a | grep envied-gluetun | awk '{print $1}' | xargs docker rm -f +``` + +## Troubleshooting + +### Docker Permission Denied (Linux) +```bash +sudo usermod -aG docker $USER +# Then log out and log back in +``` + +### VPN Connection Failed +Check container logs for specific errors: +```bash +docker logs envied-gluetun-nordvpn-us +``` + +Common issues: +- Invalid/missing credentials +- Windscribe WireGuard requires `preshared_key` (can be empty string, but must be set in credentials) +- VPN provider server issues +- Container startup timeout (default 60 seconds) + +## Resources + +- [Gluetun Wiki](https://github.com/qdm12/gluetun-wiki) - Official provider documentation +- [Gluetun GitHub](https://github.com/qdm12/gluetun) diff --git a/docs/NETWORK_CONFIG.md b/docs/NETWORK_CONFIG.md new file mode 100644 index 0000000..10368b9 --- /dev/null +++ b/docs/NETWORK_CONFIG.md @@ -0,0 +1,178 @@ +# Network & Proxy Configuration + +This document covers network and proxy configuration options for bypassing geofencing and managing connections. + +## proxy_providers (dict) + +Enable external proxy provider services. These proxies will be used automatically where needed as defined by the +Service's GEOFENCE class property, but can also be explicitly used with `--proxy`. You can specify which provider +to use by prefixing it with the provider key name, e.g., `--proxy basic:de` or `--proxy nordvpn:de`. Some providers +support specific query formats for selecting a country/server. + +### basic (dict[str, str|list]) + +Define a mapping of country to proxy to use where required. +The keys are region Alpha 2 Country Codes. Alpha 2 Country Codes are `[a-z]{2}` codes, e.g., `us`, `gb`, and `jp`. +Don't get this mixed up with language codes like `en` vs. `gb`, or `ja` vs. `jp`. + +Do note that each key's value can be a list of strings, or a string. For example, + +```yaml +us: + - "http://john%40email.tld:password123@proxy-us.domain.tld:8080" + - "http://jane%40email.tld:password456@proxy-us.domain2.tld:8080" +de: "https://127.0.0.1:8080" +``` + +Note that if multiple proxies are defined for a region, then by default one will be randomly chosen. +You can choose a specific one by specifying it's number, e.g., `--proxy basic:us2` will choose the +second proxy of the US list. + +### nordvpn (dict) + +Set your NordVPN Service credentials with `username` and `password` keys to automate the use of NordVPN as a Proxy +system where required. + +You can also specify specific servers to use per-region with the `server_map` key. +Sometimes a specific server works best for a service than others, so hard-coding one for a day or two helps. + +You can also select servers by city using the format `--proxy nordvpn:us:seattle` or `--proxy nordvpn:ca:calgary`. + +For example, + +```yaml +username: zxqsR7C5CyGwmGb6KSvk8qsZ # example of the login format +password: wXVHmht22hhRKUEQ32PQVjCZ +server_map: + us: 12 # force US server #12 for US proxies +``` + +The username and password should NOT be your normal NordVPN Account Credentials. +They should be the `Service credentials` which can be found on your Nord Account Dashboard. + +Once set, you can also specifically opt in to use a NordVPN proxy by specifying `--proxy nordvpn:gb` or such. +You can even set a specific server number this way, e.g., `--proxy nordvpn:gb2366`. + +Note that `gb` is used instead of `uk` to be more consistent across regional systems. + +### surfsharkvpn (dict) + +Enable Surfshark VPN proxy service using Surfshark Service credentials (not your login password). +You may pin specific server IDs per region using `server_map`. + +You can also select servers by city using the format `--proxy surfsharkvpn:us:seattle`. + +```yaml +username: your_surfshark_service_username # https://my.surfshark.com/vpn/manual-setup/main/openvpn +password: your_surfshark_service_password # service credentials, not account password +server_map: + us: 3844 # force US server #3844 + gb: 2697 # force GB server #2697 + au: 4621 # force AU server #4621 +``` + +### hola + +Enable Hola VPN proxy service. Requires the `hola-proxy` binary to be installed and available in your PATH. +No configuration is needed under `proxy_providers`. Hola is loaded automatically when the `hola-proxy` binary +is detected. + +Once available, use `--proxy hola:us` or similar to connect through Hola. + +### windscribevpn (dict) + +Enable Windscribe VPN proxy service using static OpenVPN service credentials. + +Use the service credentials from https://windscribe.com/getconfig/openvpn (not your account login credentials). + +```yaml +proxy_providers: + windscribevpn: + username: openvpn_username # From https://windscribe.com/getconfig/openvpn + password: openvpn_password # Service credentials, NOT your account password +``` + +#### Server Mapping + +You can optionally pin specific servers using `server_map`: + +```yaml +proxy_providers: + windscribevpn: + username: openvpn_username + password: openvpn_password + server_map: + us: us-central-096.totallyacdn.com # Force specific US server + gb: uk-london-001.totallyacdn.com # Force specific UK server +``` + +Once configured, use `--proxy windscribevpn:us` or `--proxy windscribevpn:gb` etc. to connect through Windscribe. + +You can also select specific servers by number (e.g., `--proxy windscribevpn:sg007`) or filter by city +(e.g., `--proxy windscribevpn:ca:toronto`). + +### gluetun (dict) + +Docker-managed VPN proxy supporting 50+ VPN providers via Gluetun. See [GLUETUN.md](GLUETUN.md) for full +configuration and usage details. + +```yaml +proxy_providers: + gluetun: + providers: + windscribe: + vpn_type: openvpn + credentials: + username: "YOUR_OPENVPN_USERNAME" + password: "YOUR_OPENVPN_PASSWORD" +``` + +Usage: `--proxy gluetun:windscribe:us` + +--- + +## headers (dict) + +Case-Insensitive dictionary of headers that all Services begin their Request Session state with. +All requests will use these unless changed explicitly or implicitly via a Server response. +These should be sane defaults and anything that would only be useful for some Services should not +be put here. + +Avoid headers like 'Accept-Encoding' as that would be a compatibility header that the underlying +HTTP backend (rnet) will set for you as part of its browser impersonation profile. + +I recommend using, + +```yaml +Accept-Language: "en-US,en;q=0.8" +User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" +``` + +--- + +## HTTP Session Backend + +envied uses [`rnet`](https://github.com/0x676e67/rnet) (Rust + BoringSSL) for HTTP with TLS +fingerprinting. `RnetSession` is a drop-in `requests.Session` replacement and is what +`self.session` exposes to services. It supports: + +- Browser/app impersonation via named `rnet.Impersonate` presets (Chrome, Edge, Firefox, Safari, + OkHttp, etc.) — picks JA3, ALPN, HTTP/2 SETTINGS and header order to match the chosen client. +- Native rnet proxy support (HTTP, HTTPS, SOCKS5) — used by all proxy providers below. +- Cookie-jar and `requests`-style `data=` / `json=` / `headers=` kwargs for compatibility. + +The legacy `curl_cffi` backend has been removed. The config key is still spelled +`curl_impersonate` for backward compatibility, but its value now selects an rnet preset. + +### curl_impersonate (dict) + +```yaml +curl_impersonate: + browser: Chrome131 # exact rnet.Impersonate preset name +``` + +`browser` must be an exact `rnet.Impersonate` preset name (e.g. `Chrome131`, `Chrome124`, +`Edge101`, `Firefox133`, `Safari18`, `OkHttp4_12`). See the rnet README for the full list. +Default when unset: `Chrome131`. + +--- diff --git a/docs/OUTPUT_CONFIG.md b/docs/OUTPUT_CONFIG.md new file mode 100644 index 0000000..2e3275e --- /dev/null +++ b/docs/OUTPUT_CONFIG.md @@ -0,0 +1,287 @@ +# Output & Naming Configuration + +This document covers output file organization and naming configuration options. + +## filenames (dict) + +Override the default filenames used across envied. +The filenames use various variables that are replaced during runtime. + +The following filenames are available and may be overridden: + +- `log` - Log filenames. Uses `{name}` and `{time}` variables. +- `debug_log` - Debug log filenames. Uses `{service}` and `{time}` variables. +- `config` - Service configuration filenames. +- `root_config` - Root configuration filename. +- `chapters` - Chapter export filenames. Uses `{title}` and `{random}` variables. +- `subtitle` - Subtitle export filenames. Uses `{id}` and `{language}` variables. + +For example, + +```yaml +filenames: + log: "envied_{name}_{time}.log" + debug_log: "envied_debug_{service}_{time}.jsonl" + config: "config.yaml" + root_config: "envied.yaml" + chapters: "Chapters_{title}_{random}.txt" + subtitle: "Subtitle_{id}_{language}.srt" +``` + +--- + +## output_template (dict) + +Configure custom output filename templates for movies, series, and songs. +This is **required** in your `envied.yaml` — a warning is shown if not configured. + +Available variables: `{title}`, `{year}`, `{season}`, `{episode}`, `{season_episode}`, `{episode_name}`, +`{quality}`, `{resolution}`, `{source}`, `{audio}`, `{audio_channels}`, `{audio_full}`, +`{video}`, `{hdr}`, `{hfr}`, `{atmos}`, `{dual}`, `{multi}`, `{tag}`, `{edition}`, `{repack}`, +`{lang_tag}`, `{track_number}`, `{artist}`, `{album}`, `{disc}` + +Add `?` suffix to make a variable conditional (omitted when empty): `{year?}`, `{hdr?}`, `{repack?}` + +```yaml +output_template: + # Scene-style (dot-separated) + movies: '{title}.{year}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + series: '{title}.{year?}.{season_episode}.{episode_name?}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + songs: '{track_number}.{title}.{repack?}.{edition?}.{source?}.WEB-DL.{audio_full}.{atmos?}-{tag}' + + # Plex-friendly (space-separated) + # movies: '{title} ({year}) {quality}' + # series: '{title} {season_episode} {episode_name?}' + # songs: '{track_number}. {title}' +``` + +Example outputs: +- Scene movies: `Example.Movie.2024.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG` +- Scene movies (REPACK): `Example.Movie.2024.REPACK.2160p.EXAMPLE.WEB-DL.DDP5.1.H.265-TAG` +- Scene series: `Example.Show.2024.S01E01.Pilot.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG` +- Plex movies: `Example Movie (2024) 1080p` + +### folder (optional) + +Controls the folder name for downloaded content. Uses the same template variables as the file templates above. + +If not configured, the default folder naming is used: +- Movies: `Title (Year)` +- Series: Derived from the `series` template with episode-specific variables removed +- Songs: `Artist - Album (Year)` + +`folder` accepts either a single string (applies to all title kinds) or a mapping with per-kind +templates keyed by `movies`, `series`, and/or `songs`. Unknown keys are warned about and ignored. + +```yaml +output_template: + movies: '{title}.{year}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + series: '{title}.{year?}.{season_episode}.{episode_name?}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + songs: '{track_number}.{title}.{repack?}.{edition?}.{source?}.WEB-DL.{audio_full}.{atmos?}-{tag}' + + # Scene-style folder (single template, applies to all kinds) + folder: '{title}.{year?}.{repack?}.{edition?}.{lang_tag?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + + # Plex-friendly folder + # folder: '{title} ({year?})' + + # Per-kind folder templates + # folder: + # movies: '{title} ({year})' + # series: '{title} ({year?})' + # songs: '{artist} - {album} ({year?})' +``` + +Example outputs: +- Scene folder: `Example.Show.2024.S01.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG/` +- Plex folder: `Example Show (2024)/` + +--- + +--- + +## language_tags (dict) + +Automatically adds language-based identifiers (e.g., `DANiSH`, `NORDiC`, `DKsubs`) to output filenames +based on audio and subtitle track languages. Use `{lang_tag?}` in your `output_template` to place the tag. + +Rules are evaluated in order; the first matching rule wins. All conditions within a single rule +must match (AND logic). If no rules match, `{lang_tag?}` is cleanly removed from the filename. + +### Conditions + +| Condition | Type | Description | +|-----------|------|-------------| +| `audio` | string | Matches if any selected audio track has this language | +| `subs_contain` | string | Matches if any selected subtitle has this language | +| `subs_contain_all` | list | Matches if subtitles include ALL listed languages | + +Language matching uses fuzzy matching (e.g., `en` matches `en-US`, `en-GB`). + +### Example: Nordic tagging + +```yaml +language_tags: + rules: + - audio: da + tag: DANiSH + - audio: sv + tag: SWEDiSH + - audio: nb + tag: NORWEGiAN + - audio: en + subs_contain_all: [da, sv, nb] + tag: NORDiC + - audio: en + subs_contain: da + tag: DKsubs + +output_template: + movies: '{title}.{year?}.{lang_tag?}.{quality}.{source}.WEB-DL.{audio_full}.{video}-{tag}' +``` + +Example outputs: +- Danish audio: `Example.Show.S01E01.DANiSH.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG` +- English audio + multiple Nordic subs: `Example.Show.S01E01.NORDiC.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG` +- English audio + Danish subs only: `Example.Show.S01E01.DKsubs.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG` +- No matching languages: `Example.Show.S01E01.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG` + +### Example: Other regional tags + +```yaml +language_tags: + rules: + - audio: nl + tag: DUTCH + - audio: de + tag: GERMAN + - audio: fr + subs_contain: en + tag: ENGFR + - audio: fr + tag: FRENCH +``` + +--- + +## unicode_filenames (bool) + +Allow Unicode characters in output filenames. When `false`, Unicode characters are transliterated +to ASCII equivalents. Default: `false`. + +--- + +## tag (str) + +Group or Username to postfix to the end of download filenames following a dash. +Use `{tag}` in your output template to include it. +For example, `tag: "J0HN"` will have `-J0HN` at the end of all download filenames. + +--- + +## tag_group_name (bool) + +Enable/disable tagging downloads with your group name when `tag` is set. Default: `true`. + +--- + +## tag_imdb_tmdb (bool) + +Enable/disable tagging downloaded files with IMDB/TMDB/TVDB identifiers (when available). Default: `true`. + +--- + +## muxing (dict) + +- `set_title` + Set the container title to `Show SXXEXX Episode Name` or `Movie (Year)`. Default: `true` +- `merge_audio` + Merge all audio tracks into each output file. Default: `true` + - `true`: All selected audio tracks are muxed into one MKV per quality. + - `false`: Separate MKV per (quality, audio_codec) combination. + For example: `Title.1080p.AAC.mkv`, `Title.1080p.EC3.mkv`. + + Note: The `--split-audio` CLI flag overrides this setting. When `--split-audio` is passed, + `merge_audio` is effectively set to `false` for that run. + +- `default_language` (dict) + Override which track is flagged as the default in the muxed MKV, regardless + of the title's original language. Useful when you always want your player to + open on a specific language (e.g. always default to Polish audio even on + English originals). Only affects the MKV `--default-track` flag — track + selection (`-l`, `--alang`, etc.) is unchanged. All keys are optional; each + track type falls back to its previous default rule when the configured + language isn't present in the manifest. + + - `audio`: BCP-47 tag (e.g. `pl`, `en`, `pt-BR`). Wins over `is_original_lang`. + The `--original-flag` continues to mark the true original-audio track. + - `video`: BCP-47 tag. Wins over the title-language / first-track rule. + - `subtitle`: BCP-47 tag. Wins over the "forced sub matching audio" rule. + + Languages are matched with the same close-match logic used elsewhere + (`pt` matches `pt-BR`, etc.). Supports per-service overrides like the rest + of `muxing`. + + ```yaml + muxing: + default_language: + audio: pl + video: pl + subtitle: pl + ``` + +--- + +## chapter_fallback_name (str) + +The Chapter Name to use when exporting a Chapter without a Name. +The default is no fallback name at all and no Chapter name will be set. + +The fallback name can use the following variables in f-string style: + +- `{i}`: The Chapter number starting at 1. + E.g., `"Chapter {i}"`: "Chapter 1", "Intro", "Chapter 3". +- `{j}`: A number starting at 1 that increments any time a Chapter has no title. + E.g., `"Chapter {j}"`: "Chapter 1", "Intro", "Chapter 2". + +These are formatted with f-strings, directives are supported. +For example, `"Chapter {i:02}"` will result in `"Chapter 01"`. + +--- + +## directories (dict) + +Override the default directories used across envied. +The directories are set to common values by default. + +The following directories are available and may be overridden, + +- `commands` - CLI Command Classes. +- `services` - Service Classes. +- `vaults` - Vault Classes. +- `fonts` - Font files (ttf or otf). +- `downloads` - Downloads. +- `temp` - Temporary files or conversions during download. +- `cache` - Expiring data like Authorization tokens, or other misc data. +- `cookies` - Expiring Cookie data. +- `logs` - Logs. +- `exports` - JSON sidecar exports written when `--export` is used on `dl`. +- `wvds` - Widevine Devices. +- `prds` - PlayReady Devices. +- `dcsl` - Device Certificate Status List. + +Notes: + +- `services` accepts either a single directory or a list of directories to search for service modules. + +For example, + +```yaml +directories: + downloads: "D:/Downloads/envied" + temp: "D:/Temp/envied" +``` + +There are directories not listed that cannot be modified as they are crucial to the operation of envied. + +--- diff --git a/docs/SERVICE_CONFIG.md b/docs/SERVICE_CONFIG.md new file mode 100644 index 0000000..19ca2b8 --- /dev/null +++ b/docs/SERVICE_CONFIG.md @@ -0,0 +1,246 @@ +# Service Integration & Authentication Configuration + +This document covers service-specific configuration, authentication, and metadata integration options. + +## services (dict) + +Configuration data for each Service. The Service will have the data within this section merged into the per-service +`config.yaml` (located in the service's directory) before being provided to the Service class. + +Think of this config to be used for more sensitive configuration data, like user or device-specific API keys, IDs, +device attributes, and so on. A per-service `config.yaml` file is typically shared and not meant to be modified, +so use this for any sensitive configuration data. + +The Key is the Service Tag, but can take any arbitrary form for its value. It's expected to begin as either a list or +a dictionary. + +For example, + +```yaml +EXAMPLE: + client: + auth_scheme: MESSO + # ... more sensitive data +``` + +### Per-Service Configuration Overrides + +You can override many global configuration options on a per-service basis by nesting them under the +service tag in the `services` section. Supported override keys include: `dl`, `subtitle`, `muxing`, +`headers`, `proxy_map`, `title_map`, and more. + +Overrides are merged with global config (not replaced) -- only specified keys are overridden, others +use global defaults. CLI arguments always take priority over service-specific config. + +For example, + +```yaml +services: + RATE_LIMITED_SERVICE: + dl: + downloads: 2 # Limit concurrent track downloads + workers: 4 # Reduce workers to avoid rate limits + headers: + User-Agent: "..." # Service-specific UA override +``` + +Note: envied uses a single unified `requests`-based downloader. The legacy `aria2c`, +`n_m3u8dl_re`, and `curl_impersonate` override sections have been removed. + +### title_map (dict) + +Rewrites service-provided titles before naming and output. Some services name a title differently +from how you want it stored, which can break library matching (e.g. a regional variant reusing the +international name). Keys are the exact title string the service returns; values are the desired +output title. + +```yaml +services: + EXAMPLE: + title_map: + Service Title: Desired Title +``` + +Episodes are matched on their show title, Movies and Songs on their name. The remap is applied +after the title cache (so edits take effect without a cache reset) and before any `--enrich` +override (so an explicit enrich still wins). + +It applies on the local `dl` path, the `import` command, and the remote client (`dl --remote`). +For remote services the **client's** `title_map` is applied to the titles returned by the server, +so you can rename titles for services you don't have installed locally. The server sends raw +titles and does not remap, leaving the final name fully under the client's control. + +### Service Class Conventions + +Each service directory under `envied/services/` exports a class extending +`envied.core.service.Service`. The class name must match the directory name (the service tag). + +Key class variables (defined on `Service` or by service-level idiom): + +- `ALIASES: tuple[str, ...]` — alternative tags accepted on the CLI. Empty by default. +- `GEOFENCE: tuple[str, ...]` — ISO country codes the service is available in. Empty == no geofence. +- `TITLE_RE: str` — regex (with named groups, e.g. `(?P...)`, `(?P...)`) used by the + service to parse the CLI title argument. Service-level idiom, not declared on the base class. +- `NO_SUBTITLES: bool` — service-level idiom indicating the service has no subtitle tracks. + +`self.*` helpers available after `super().__init__(ctx)`: + +- `self.session` — pre-configured HTTP session (`requests.Session`, or `RnetSession` when TLS + impersonation is active). Cookies, headers, proxies pre-applied. +- `self.config` — merged service config (per-service `config.yaml` plus the `services.` block + from `envied.yaml`). +- `self.log` — `logging.Logger` named for the service class. +- `self.cache` — generic `Cacher` for arbitrary key/value persistence. +- `self.title_cache` — specialized `TitleCacher` for title metadata. +- `self.track_request` — `TrackRequest` built from CLI flags. Fields: `codecs: list[Video.Codec]`, + `ranges: list[Video.Range]` (defaults to `[SDR]`), `best_available: bool`. Services may + read or rewrite these (e.g. force HEVC for HDR ranges). +- `self.credential` — set during `authenticate()`; `None` if cookies-only. +- `self.current_region` — lowercase ISO country code from proxy/geolocation, or `None`. +- `self.request_input(prompt: str) -> str` — interactive prompt. Falls through to `input()` + locally; under `serve`, the attached `InputBridge` relays the prompt to the remote client. + +Driving CLI flags (parsed into `self.track_request`): + +- `-v` / `--vcodec` — comma-separated `Video.Codec` list (e.g. `H264,H265`). +- `-a` / `--acodec` — comma-separated audio codec list. +- `-r` / `--range` — comma-separated `Video.Range` list (`SDR`, `HDR10`, `HDR10+`, `DV`, + `HYBRID`). Defaults to `[SDR]`. +- `-q` / `--quality` — resolution list. +- `--vbitrate-range` / `--abitrate-range` — `MIN-MAX` kbps windows. + +--- + +## credentials (dict[str, str|list|dict]) + +Specify login credentials to use for each Service, and optionally per-profile. + +For example, + +```yaml +EXAMPLE: jane@example.tld:LoremIpsum100 # directly +EXAMPLE2: # or per-profile, optionally with a default + default: jane@example.tld:LoremIpsum99 # <-- used by default if -p/--profile is not used + james: james@example.tld:TheFriend97 + john: john@example.tld:LoremIpsum98 +EXAMPLE3: # the `default` key is not necessary, but no credential will be used by default + john: john@example.tld:SecretPassword123 +``` + +The value should be in string form, i.e. `john@example.tld:password123` or `john:password123`. +Any arbitrary values can be used on the left (username/password/phone) and right (password/secret). +You can also specify these in list form, i.e., `["john@example.tld", ":PasswordWithAColon"]`. + +If you specify multiple credentials with keys like the `EXAMPLE2` and `EXAMPLE3` example above, then you should +use a `default` key or no credential will be loaded automatically unless you use `-p/--profile`. You +do not have to use a `default` key at all. + +Please be aware that this information is sensitive and to keep it safe. Do not share your config. + +--- + +## tmdb_api_key (str) + +API key for The Movie Database (TMDB). This is used for tagging downloaded files with TMDB, +IMDB and TVDB identifiers. Leave empty to disable automatic lookups. + +To obtain a TMDB API key: + +1. Create an account at +2. Go to to register for API access +3. Fill out the API application form with your project details +4. Once approved, you'll receive your API key + +For example, + +```yaml +tmdb_api_key: cf66bf18956kca5311ada3bebb84eb9a # Not a real key +``` + +**Note**: Keep your API key secure and do not share it publicly. This key is used by the `core/providers/tmdb.py` metadata provider to fetch metadata from TMDB for proper file tagging and ID enrichment. + +--- + +## simkl_client_id (str) + +Client ID for SIMKL API integration. SIMKL is used as a metadata source for improved title matching and tagging, +especially when a TMDB API key is not configured. + +To obtain a SIMKL Client ID: + +1. Create an account at +2. Go to +3. Register a new application to receive your Client ID + +For example, + +```yaml +simkl_client_id: "your_client_id_here" +``` + +**Note**: While optional, having a SIMKL Client ID improves metadata lookup reliability. SIMKL serves as an alternative or fallback metadata source to TMDB. This is used by the `core/providers/simkl.py` metadata provider. + +--- + +## ipinfo_api_key (str) + +Optional API token for [ipinfo.io](https://ipinfo.io). When set, envied uses the free authenticated **Lite** endpoint (`https://api.ipinfo.io/lite/me`), which has substantially higher rate limits than the anonymous endpoint and returns richer fields (ASN, organization name, continent). Leave empty to use the anonymous ipinfo.io endpoint, with [ip-api.in](https://ip-api.in) as a final fallback. + +To obtain an ipinfo.io token: + +1. Sign up for a free account at +2. Copy the token from your dashboard + +For example, + +```yaml +ipinfo_api_key: "12a3b45cd678ef" # Not a real key +``` + +**Note**: The token is only ever sent to `api.ipinfo.io` as a per-request `Authorization` header — it is never attached to your session for service requests. Used by `core/utils/ip_info.py` for region detection and proxy verification. + +--- + +## title_cache_enabled (bool) + +Enable/disable caching of title metadata to reduce redundant API calls. Default: `true`. + +--- + +## title_cache_time (int) + +Cache duration in seconds for title metadata. Default: `1800` (30 minutes). + +--- + +## title_cache_max_retention (int) + +Maximum retention time in seconds for serving slightly stale cached title metadata when API calls fail. +Default: `86400` (24 hours). Effective retention is `min(title_cache_time + grace, title_cache_max_retention)`. + +--- + +## debug (bool) + +Enable structured JSON debug logging for troubleshooting and service development. Default: `false`. + +When enabled (via config or the `--debug` CLI flag): +- Creates JSON Lines (`.jsonl`) log files with complete debugging context +- Logs: session info, CLI params, service config, CDM details, authentication, titles, tracks metadata, + DRM operations, vault queries, errors with stack traces +- File location: `logs/envied_debug_{service}_{timestamp}.jsonl` + +--- + +## debug_keys (bool) + +Log decryption keys in debug logs. Default: `false`. + +When `true`, actual content encryption keys (CEKs) are included in debug log output. Useful for +debugging key retrieval and decryption issues. + +**Security note:** Passwords, tokens, cookies, and session tokens are always redacted regardless +of this setting. Only content keys (`content_key`, `key` fields) are affected. Key IDs (`kid`), +key counts, and other metadata are always logged. + +--- diff --git a/docs/SUBTITLE_CONFIG.md b/docs/SUBTITLE_CONFIG.md new file mode 100644 index 0000000..17a3c3e --- /dev/null +++ b/docs/SUBTITLE_CONFIG.md @@ -0,0 +1,73 @@ +# Subtitle Processing Configuration + +This document covers subtitle processing and formatting options under the top-level `subtitle:` key in `envied.yaml`. + +For the canonical example, see `envied/envied-example.yaml`. + +## subtitle (dict) + +Control subtitle conversion, SDH (hearing-impaired) stripping, formatting preservation, and output behavior. + +- `conversion_method`: How to convert subtitles between formats. Default: `auto`. + - `auto`: Smart routing - subby for WebVTT/fVTT/SAMI; for SSA/ASS/MicroDVD/MPL2/TMP use SubtitleEdit when available, otherwise pysubs2; standard pycaption/SubtitleEdit pipeline for everything else. + - `subby`: Always use subby with `CommonIssuesFixer` (falls back to standard if the source codec isn't supported by subby). + - `subtitleedit`: Prefer SubtitleEdit when available; otherwise fall back to the standard pycaption pipeline. + - `pycaption`: Use only the pycaption library (no SubtitleEdit, no subby). Limited to SRT, TTML, and WebVTT outputs. + - `pysubs2`: Use pysubs2 (supports SRT, SSA, ASS, WebVTT, TTML, SAMI, MicroDVD, MPL2, TMP). + +- `sdh_method`: How to strip SDH cues. Default: `auto`. + - `auto`: Try subby for SRT first, then SubtitleEdit (when `conversion_method` is `auto`/`subtitleedit` and the binary is available), then subtitle-filter as the final fallback. + - `subby`: Use subby's `SDHStripper`. **Only operates on SRT**; for other codecs the call returns without stripping. + - `subtitleedit`: Use SubtitleEdit's `/RemoveTextForHI` when the binary is available; otherwise falls through to subtitle-filter. + - `filter-subs`: Use the `subtitle-filter` library directly (`rm_fonts`, `rm_ast`, `rm_music`, `rm_effects`, `rm_names`, `rm_author`). + +- `strip_sdh`: Enable/disable automatic SDH stripping for tracks flagged as SDH. Default: `true`. + +- `convert_before_strip`: When falling through to the subtitle-filter path, auto-convert non-SRT subtitles to SRT first for better compatibility. Default: `true`. Has no effect when SubtitleEdit handles stripping directly. + +- `preserve_formatting`: Keep original subtitle tags and positioning during WebVTT processing. When `true`, sanitized WebVTT is written back without round-tripping through pycaption, preserving tags like ``, ``, and `line:` positioning. Default: `true`. + +- `output_mode`: Controls how subtitles are included in the output. Default: `mux`. + - `mux`: Embed subtitles in the MKV container only. + - `sidecar`: Save subtitles as separate files only (not muxed). + - `both`: Embed in the MKV container and save as sidecar files. + +- `sidecar_format`: Format for sidecar subtitle files (used when `output_mode` is `sidecar` or `both`). Default: `srt`. + - `srt`: SubRip. + - `vtt`: WebVTT. + - `ass`: Advanced SubStation Alpha. + - `original`: Keep the subtitle in its current format without conversion. + +Example: + +```yaml +subtitle: + conversion_method: auto + sdh_method: auto + strip_sdh: true + convert_before_strip: true + preserve_formatting: true + output_mode: mux + sidecar_format: srt +``` + +## WebVTT Sanitization (automatic, not configurable) + +After download, WebVTT and segmented WebVTT (`fVTT`/`WVTT`) tracks pass through a fixed sanitization pipeline before any conversion or muxing: + +1. **Segment merge** — segmented DASH/HLS WebVTT is stitched via `merge_segmented_webvtt` (uses pysubs2 for lenient parsing when `conversion_method` is `auto` or `pysubs2`, otherwise pycaption directly). +2. **Negative timestamps** — `sanitize_webvtt_timestamps` rewrites `-HH:MM:SS.mmm` cues to `00:00:00.000`. +3. **Cue identifiers** — `sanitize_webvtt_cue_identifiers` strips letter+digit IDs (e.g. `Q0`, `S12`) on their own line before a timing line, which otherwise confuse parsers like pysubs2. +4. **Overlapping cues** — `merge_overlapping_webvtt_cues` collapses cues with start times within 50 ms and matching end times into a single multi-line cue, ordered by `line:` percentage (lower % = higher on screen = first line). +5. **Fallback hardening** — when `preserve_formatting` is `false` and the first pycaption parse fails, `sanitize_webvtt` retries with a `WEBVTT` header guard, hour-padded timings, and another negative-timestamp pass; if that still fails, the sanitized text is written as-is. + +`sanitize_broken_webvtt` and `space_webvtt_headers` additionally run inside `Subtitle.parse()` to drop malformed `-->` lines and reflow merged-segment headers. `merge_same_cues` and `filter_unwanted_cues` (drops ` `/whitespace-only cues) run only on the pycaption path. + +These behaviors are intentional and have no config knobs — they apply to every WebVTT track regardless of `conversion_method`. + +## Related + +- Filename sanitization (e.g. parenthesis handling, unidecode bracket artifacts from PR #105) lives in `envied/core/utilities.py::sanitize_filename` and is governed by `output_template`, not the `subtitle:` config block. +- Subtitle codec support and the conversion matrix are defined in `envied/core/tracks/subtitle.py`. + +--- diff --git a/packages/envied/pyproject.toml b/packages/envied/pyproject.toml index 037d153..c64e262 100644 --- a/packages/envied/pyproject.toml +++ b/packages/envied/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "uv_build" [project] name = "envied" -version = "4.0.0" +version = "5.1.0" description = "Modular Movie, TV, and Music Archival Software." authors = [{ name = "rlaphoenix and others" }] requires-python = ">=3.10,<=3.14" @@ -26,7 +26,6 @@ classifiers = [ "Topic :: Security :: Cryptography", ] dependencies = [ - "appdirs>=1.4.4,<2", "Brotli>=1.1.0,<2", "click>=8.1.8,<9", @@ -36,19 +35,18 @@ dependencies = [ "fonttools>=4.60.2,<5", "jsonpickle>=3.0.4,<5", "langcodes>=3.4.0,<4", - "lxml>=6.1.0,<7", - "pproxy>=2.7.9,<3", + "lxml>=5.2.1,<7", "protobuf>=4.25.3,<7", "pycaption>=2.2.6,<3", "pycryptodomex>=3.20.0,<4", - "pyjwt>=2.8.0,<3", + "pyjwt>=2.12.0,<3", "pymediainfo>=6.1.0,<8", "pymp4>=1.4.0,<2", "pymysql>=1.1.0,<2", "pywidevine[serve]>=1.8.0,<2", "PyYAML>=6.0.1,<7", "requests[socks]>=2.32.5,<3", - "rich>=13.7.1,<15", + "rich>=13.7.1,<15.1", "rlaphoenix.m3u8>=3.4.0,<4", "ruamel.yaml>=0.18.6,<0.19", "sortedcontainers>=2.4.0,<3", @@ -56,12 +54,10 @@ dependencies = [ "Unidecode>=1.3.8,<2", "urllib3>=2.6.3,<3", "chardet>=5.2.0,<6", - "curl-cffi>=0.7.0b4,<0.14", "pyplayready>=0.8.3,<0.9", - "httpx>=0.28.1,<0.29", "cryptography>=45.0.0,<47", "subby", - "aiohttp>=3.13.3,<4", + "aiohttp>=3.13.4,<4", "aiohttp-swagger3>=0.9.0,<1", "pysubs2>=1.7.0,<2", "PyExecJS>=1.5.1,<2", @@ -69,13 +65,14 @@ dependencies = [ "language-data>=1.4.0", "wasmtime>=41.0.0", "animeapi-py>=0.6.0", - - # envied specifix - "webvtt-py>=0.5.1,<0.6", - "isodate>=0.7.2,<0.8", - "scrapy>=2.14.0,<3", - "mypy>=1.19.1,<2", - + "rnet>=2.4.2", + "bandit>=1.9.4", + # envied specific dependencies + "webvtt-py>=0.5.1,<0.6", + "isodate>=0.7.2,<0.8", + "scrapy>=2.14.0,<3", + "mypy>=1.19.1,<2", + "httpx>=0.28.0,<0.35", # Keeping your existing cap style here; latest visible is 0.10.0. diff --git a/packages/envied/src/envied/commands/dl.py b/packages/envied/src/envied/commands/dl.py index 25ec99a..dae48b7 100644 --- a/packages/envied/src/envied/commands/dl.py +++ b/packages/envied/src/envied/commands/dl.py @@ -1,6 +1,7 @@ from __future__ import annotations import html +import json import logging import math import os @@ -22,29 +23,22 @@ from typing import Any, Callable, Optional from uuid import UUID import click -import jsonpickle import yaml -from construct import ConstError from langcodes import Language from pymediainfo import MediaInfo -from pyplayready.cdm import Cdm as PlayReadyCdm -from pyplayready.device import Device as PlayReadyDevice -from pyplayready.remote.remotecdm import RemoteCdm as PlayReadyRemoteCdm -from pywidevine.cdm import Cdm as WidevineCdm -from pywidevine.device import Device -from pywidevine.remotecdm import RemoteCdm from rich.console import Group from rich.live import Live from rich.padding import Padding from rich.panel import Panel from rich.progress import BarColumn, Progress, SpinnerColumn, TaskID, TextColumn, TimeRemainingColumn from rich.rule import Rule +from rich.spinner import Spinner from rich.table import Table from rich.text import Text from rich.tree import Tree from envied.core import binaries, providers -from envied.core.cdm import CustomRemoteCDM, DecryptLabsRemoteCDM +from envied.core.cdm import DecryptLabsRemoteCDM from envied.core.cdm.detect import is_playready_cdm, is_widevine_cdm from envied.core.config import config from envied.core.console import console @@ -60,12 +54,15 @@ from envied.core.titles import Movie, Movies, Series, Song, Title_T from envied.core.titles.episode import Episode from envied.core.tracks import Audio, Subtitle, Tracks, Video from envied.core.tracks.attachment import Attachment +from envied.core.tracks.dv_fixup import apply_dv_fixup from envied.core.tracks.hybrid import Hybrid -from envied.core.utilities import (find_font_with_fallbacks, get_debug_logger, get_system_fonts, init_debug_logger, - is_close_match, is_exact_match, suggest_font_packages, time_elapsed_since) +from envied.core.utilities import (find_font_with_fallbacks, find_missing_langs, get_debug_logger, + get_system_fonts, init_debug_logger, is_close_match, suggest_font_packages, + time_elapsed_since) from envied.core.utils import tags +from envied.core.utils.bitrate import apply_real_bitrates from envied.core.utils.click_types import (AUDIO_CODEC_LIST, LANGUAGE_RANGE, QUALITY_LIST, SEASON_RANGE, - ContextData, MultipleChoice, MultipleVideoCodecChoice, + SLOW_DELAY_RANGE, ContextData, MultipleChoice, MultipleVideoCodecChoice, SubtitleCodecChoice) from envied.core.utils.collections import merge_dict from envied.core.utils.selector import select_multiple @@ -315,6 +312,20 @@ class dl: default=None, help="Audio Bitrate to download (in kbps), defaults to highest available.", ) + @click.option( + "-vb-range", + "--vbitrate-range", + type=str, + default=None, + help="Video Bitrate range in kbps (e.g., '6000-7000'). Selects the highest bitrate within the range.", + ) + @click.option( + "-ab-range", + "--abitrate-range", + type=str, + default=None, + help="Audio Bitrate range in kbps (e.g., '128-256'). Selects the highest bitrate within the range.", + ) @click.option( "-r", "--range", @@ -349,21 +360,21 @@ class dl: "--select-titles", is_flag=True, default=False, - help="Interactively select downloads from a list. Only use with Series to select Episodes", + help="Interactively select downloads from a list. Only use with Series to select Episodes.", ) @click.option( "-w", "--wanted", type=SEASON_RANGE, default=None, - help="Wanted episodes, e.g. `S01-S05,S07`, `S01E01-S02E03`, `S02-S02E03`, e.t.c, defaults to all.", + help="Wanted episodes, e.g. `S01-S05,S07`, `S01E01-S02E03`, `S02-S02E03`, etc., defaults to all.", ) @click.option( "-l", "--lang", type=LANGUAGE_RANGE, default="orig", - help="Language wanted for Video and Audio. Use 'orig' to select the original language, e.g. 'orig,en' for both original and English.", + help="Language(s) wanted for Video and Audio (comma-separated). Use 'orig' to select the original language, e.g. 'orig,en' for both original and English.", ) @click.option( "--latest-episode", @@ -376,7 +387,7 @@ class dl: "--v-lang", type=LANGUAGE_RANGE, default=[], - help="Language wanted for Video, you would use this if the video language doesn't match the audio.", + help="Language wanted for Video. You would use this if the video language doesn't match the audio.", ) @click.option( "-al", @@ -403,12 +414,28 @@ class dl: "--proxy", type=str, default=None, - help="Proxy URI to use. If a 2-letter country is provided, it will try get a proxy from the config.", + help="Proxy URI to use. If a 2-letter country is provided, it will try to get a proxy from the config.", ) @click.option( "--tag", type=str, default=None, help="Set the Group Tag to be used, overriding the one in config if any." ) @click.option("--repack", is_flag=True, default=False, help="Add REPACK tag to the output filename.") + @click.option( + "-rvb", + "--real-video-bitrate", + is_flag=True, + default=False, + help="Probe actual media size to compute true video bitrates (top renditions per codec/range), " + "overriding the manifest's declared bitrate.", + ) + @click.option( + "-rab", + "--real-audio-bitrate", + is_flag=True, + default=False, + help="Probe actual media size to compute true audio bitrates (top renditions per codec/channels/language), " + "overriding the manifest's declared bitrate. Slower than --real-video-bitrate (more renditions).", + ) @click.option( "--tmdb", "tmdb_id", @@ -445,18 +472,19 @@ class dl: @click.option("-V", "--video-only", is_flag=True, default=False, help="Only download video tracks.") @click.option("-A", "--audio-only", is_flag=True, default=False, help="Only download audio tracks.") @click.option("-S", "--subs-only", is_flag=True, default=False, help="Only download subtitle tracks.") - @click.option("-C", "--chapters-only", is_flag=True, default=False, help="Only download chapters.") + @click.option("-C", "--chapters-only", is_flag=True, default=False, help="Only download chapter markers.") @click.option("-ns", "--no-subs", is_flag=True, default=False, help="Do not download subtitle tracks.") @click.option("-na", "--no-audio", is_flag=True, default=False, help="Do not download audio tracks.") - @click.option("-nc", "--no-chapters", is_flag=True, default=False, help="Do not download chapters tracks.") + @click.option("-nc", "--no-chapters", is_flag=True, default=False, help="Do not download chapter markers.") @click.option("-nv", "--no-video", is_flag=True, default=False, help="Do not download video tracks.") @click.option("-ad", "--audio-description", is_flag=True, default=False, help="Download audio description tracks.") @click.option( "--slow", - is_flag=True, - default=False, - help="Add a 60-120 second delay between each Title download to act more like a real device. " - "This is recommended if you are downloading high-risk titles or streams.", + type=SLOW_DELAY_RANGE, + default=None, + help="Add a delay between each Title download to act more like a real device. " + "Use --slow for a 60-120s delay, or --slow MIN-MAX (e.g., --slow 20-40) for a custom range. " + "Minimum delay is 20 seconds.", ) @click.option( "--list", @@ -474,7 +502,20 @@ class dl: @click.option( "--skip-dl", is_flag=True, default=False, help="Skip downloading while still retrieving the decryption keys." ) - @click.option("--export", type=Path, help="Export Decryption Keys as you obtain them to a JSON file.") + @click.option( + "--export", + is_flag=True, + default=False, + help="Export track info and decryption keys to a JSON file in the exports directory.", + ) + @click.option( + "--import", + "import_file", + type=str, + default=None, + hidden=True, + help="Internal: path to an export JSON to reconstruct a download from (used by 'unshackle import').", + ) @click.option( "--cdm-only/--vaults-only", is_flag=True, @@ -482,6 +523,12 @@ class dl: help="Only use CDM, or only use Key Vaults for retrieval of Decryption Keys.", ) @click.option("--no-proxy", is_flag=True, default=False, help="Force disable all proxy use.") + @click.option( + "--no-proxy-download", + is_flag=True, + default=False, + help="Bypass proxy for segment downloads only. Manifest, license, and auth still use proxy.", + ) @click.option("--no-folder", is_flag=True, default=False, help="Disable folder creation for TV Shows.") @click.option( "--no-source", is_flag=True, default=False, help="Disable the source tag from the output file name and path." @@ -519,11 +566,27 @@ class dl: default=False, help="Continue with best available quality if requested resolutions are not available.", ) + @click.option( + "--remote", + is_flag=True, + default=False, + is_eager=True, + help="Use a remote unshackle server instead of local service code.", + ) + @click.option( + "--server", + type=str, + default=None, + is_eager=True, + help="Name of the remote server from remote_services config (if multiple configured).", + ) @click.pass_context def cli(ctx: click.Context, **kwargs: Any) -> dl: return dl(ctx, **kwargs) DRM_TABLE_LOCK = Lock() + EXPORT_LOCK = Lock() + LICENSE_KEY_CACHE: dict[UUID, str] = {} def __init__( self, @@ -545,6 +608,14 @@ class dl: raise ValueError("A subcommand to invoke was not specified, the main code cannot continue.") self.log = logging.getLogger("download") + self.completed_files: list[Path] = [] + + if not config.output_template: + raise click.ClickException( + "No 'output_template' configured in your envied.yaml.\n" + "Please add an 'output_template' section with movies, series, and songs templates.\n" + "See unshackle-example.yaml for examples." + ) self.service = Services.get_tag(ctx.invoked_subcommand) service_dl_config = config.services.get(self.service, {}).get("dl", {}) @@ -580,6 +651,7 @@ class dl: ) self.profile = profile + self.proxy_requested = bool(proxy) self.tmdb_id = tmdb_id self.imdb_id = imdb_id self.enrich = enrich @@ -646,7 +718,6 @@ class dl: for name, binary in [ ("shaka_packager", binaries.ShakaPackager), ("mp4decrypt", binaries.Mp4decrypt), - ("n_m3u8dl_re", binaries.N_m3u8DL_RE), ("mkvmerge", binaries.MKVToolNix), ("ffmpeg", binaries.FFMPEG), ("ffprobe", binaries.FFProbe), @@ -672,11 +743,6 @@ class dl: output = (r.stdout or "") + (r.stderr or "") lines = [line.strip() for line in output.split("\n") if line.strip()] version = " | ".join(lines[:2]) if lines else None - elif name == "n_m3u8dl_re": - r = subprocess.run( - [str(binary), "--version"], capture_output=True, text=True, timeout=5 - ) - version = (r.stdout or r.stderr or "").strip().split("\n")[0] except Exception: version = "" binary_versions[name] = {"path": str(binary), "version": version} @@ -695,24 +761,26 @@ class dl: if self.profile: self.log.info(f"Using profile: '{self.profile}'") - with console.status("Loading Service Config...", spinner="dots"): - service_config_path = Services.get_path(self.service) / config.filenames.config - if service_config_path.exists(): - self.service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) - self.log.info("Service Config loaded") - if self.debug_logger: - self.debug_logger.log( - level="DEBUG", - operation="load_service_config", - service=self.service, - context={"config_path": str(service_config_path), "config": self.service_config}, - ) - else: - self.service_config = {} - merge_dict(config.services.get(self.service), self.service_config) + self.is_remote = bool(ctx.params.get("remote")) - if getattr(config, "downloader_map", None): - config.downloader = config.downloader_map.get(self.service, config.downloader) + with console.status("Loading Service Config...", spinner="dots"): + self.service_config = {} + if not self.is_remote: + try: + service_config_path = Services.get_path(self.service) / config.filenames.config + if service_config_path.exists(): + self.service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) + self.log.info("Service Config loaded") + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="load_service_config", + service=self.service, + context={"config_path": str(service_config_path), "config": self.service_config}, + ) + except KeyError: + pass + merge_dict(config.services.get(self.service), self.service_config) if getattr(config, "decryption_map", None): config.decryption = config.decryption_map.get(self.service, config.decryption) @@ -962,6 +1030,8 @@ class dl: acodec: list[Audio.Codec], vbitrate: int, abitrate: int, + vbitrate_range: Optional[str], + abitrate_range: Optional[str], range_: list[Video.Range], channels: float, no_atmos: bool, @@ -985,13 +1055,14 @@ class dl: no_chapters: bool, no_video: bool, audio_description: bool, - slow: bool, + slow: Optional[tuple[int, int]], list_: bool, list_titles: bool, skip_dl: bool, - export: Optional[Path], + export: bool, cdm_only: Optional[bool], no_proxy: bool, + no_proxy_download: bool, no_folder: bool, no_source: bool, no_mux: bool, @@ -1000,16 +1071,60 @@ class dl: worst: bool, best_available: bool, split_audio: Optional[bool] = None, + real_video_bitrate: bool = False, + real_audio_bitrate: bool = False, *_: Any, **__: Any, ) -> None: self.tmdb_searched = False self.search_source = None + self.server_cdm = getattr(service, "_server_cdm", False) + self._remote_service = service if self.server_cdm else None start_time = time.time() if skip_dl: DOWNLOAD_LICENCE_ONLY.set() + if export: + config.directories.exports.mkdir(parents=True, exist_ok=True) + export_path = config.directories.exports / f"export_{self.service}_{int(time.time())}.json" + self.export_service = service + else: + export_path = None + + # Parse bitrate range options + vbitrate_min, vbitrate_max = None, None + if vbitrate_range: + if vbitrate and vbitrate_range: + self.log.error("Cannot use both --vbitrate and --vbitrate-range at the same time.") + sys.exit(1) + try: + parts = vbitrate_range.split("-") + if len(parts) != 2: + raise ValueError + vbitrate_min, vbitrate_max = int(parts[0]), int(parts[1]) + if vbitrate_min > vbitrate_max: + vbitrate_min, vbitrate_max = vbitrate_max, vbitrate_min + except (ValueError, IndexError): + self.log.error("Invalid --vbitrate-range format. Use 'MIN-MAX' (e.g., '6000-7000').") + sys.exit(1) + + abitrate_min, abitrate_max = None, None + if abitrate_range: + if abitrate and abitrate_range: + self.log.error("Cannot use both --abitrate and --abitrate-range at the same time.") + sys.exit(1) + try: + parts = abitrate_range.split("-") + if len(parts) != 2: + raise ValueError + abitrate_min, abitrate_max = int(parts[0]), int(parts[1]) + if abitrate_min > abitrate_max: + abitrate_min, abitrate_max = abitrate_max, abitrate_min + except (ValueError, IndexError): + self.log.error("Invalid --abitrate-range format. Use 'MIN-MAX' (e.g., '128-256').") + sys.exit(1) + if not acodec: acodec = [] elif isinstance(acodec, Audio.Codec): @@ -1057,7 +1172,10 @@ class dl: }, ) - with console.status("Authenticating with Service...", spinner="dots"): + with console.status( + "Authenticating with Remote Service..." if self.is_remote else "Authenticating with Service...", + spinner="dots", + ): try: cookies = self.get_cookie_jar(self.service, self.profile) credential = self.get_credentials(self.service, self.profile) @@ -1082,7 +1200,9 @@ class dl: ) raise - with console.status("Fetching Title Metadata...", spinner="dots"): + with console.status( + "Fetching Remote Title Metadata..." if self.is_remote else "Fetching Title Metadata...", spinner="dots" + ): try: titles = service.get_titles_cached() if not titles: @@ -1326,9 +1446,12 @@ class dl: ) if slow and i != 0: - delay = random.randint(60, 120) - with console.status(f"Delaying by {delay} seconds..."): - time.sleep(delay) + delay = random.randint(slow[0], slow[1]) + spinner = Spinner("dots", text=f"Delaying by {delay} seconds...") + with Live(Padding(spinner, (0, 5)), console=console, refresh_per_second=12.5, transient=True): + for remaining in range(delay, 0, -1): + spinner.update(text=f"Delaying by {remaining} seconds...") + time.sleep(1) with console.status("Subscribing to events...", spinner="dots"): events.reset() @@ -1362,7 +1485,11 @@ class dl: console.log("Skipped chapters as --no-chapters was used...") title.tracks.chapters = [] - with console.status("Getting tracks...", spinner="dots"): + if no_proxy_download and any(service.session.proxies.values()): + console.log("Bypassing proxy for downloads as --no-proxy-download was used...") + + tracks_label = "Getting Remote Tracks..." if self.is_remote else "Getting Tracks..." + with console.status(tracks_label, spinner="dots"): try: title.tracks.add(service.get_tracks(title), warn_only=True) title.tracks.chapters = service.get_chapters(title) @@ -1432,11 +1559,29 @@ class dl: title.tracks.add(non_sdh_sub) events.subscribe( events.Types.TRACK_MULTIPLEX, - lambda track, sub_id=non_sdh_sub.id: (track.strip_hearing_impaired()) - if track.id == sub_id - else None, + lambda track, sub_id=non_sdh_sub.id: ( + (track.strip_hearing_impaired()) if track.id == sub_id else None + ), ) + if real_video_bitrate: + with console.status("Probing real video bitrates...", spinner="dots"): + apply_real_bitrates( + title.tracks.videos, + service.session, + log=self.log, + group_key=lambda t: (t.codec, t.range), + ) + + if real_audio_bitrate: + with console.status("Probing real audio bitrates...", spinner="dots"): + apply_real_bitrates( + title.tracks.audio, + service.session, + log=self.log, + group_key=lambda t: (t.codec, t.channels, str(t.language), t.descriptive), + ) + with console.status("Sorting tracks by language and bitrate...", spinner="dots"): video_sort_lang = v_lang or lang processed_video_sort_lang = [] @@ -1463,7 +1608,10 @@ class dl: processed_audio_sort_lang.append(language) title.tracks.sort_videos(by_language=processed_video_sort_lang) - title.tracks.sort_audio(by_language=processed_audio_sort_lang) + title.tracks.sort_audio( + by_language=processed_audio_sort_lang, + codec_priority=config.audio.get("codec_priority"), + ) title.tracks.sort_subtitles(by_language=s_lang) if list_: @@ -1490,6 +1638,9 @@ class dl: missing_ranges = [r for r in range_ if not any(x.range == r for x in title.tracks.videos)] for color_range in missing_ranges: self.log.warning(f"Skipping {color_range.name} video tracks as none are available.") + if not title.tracks.videos: + self.log.error(f"There's no {', '.join(r.name for r in range_)} Video Track...") + sys.exit(1) if vbitrate: if any(r == Video.Range.HYBRID for r in range_): @@ -1507,7 +1658,19 @@ class dl: self.log.error(f"There's no {vbitrate}kbps Video Track...") sys.exit(1) - video_languages = [lang for lang in (v_lang or lang) if lang != "best"] + if vbitrate_min is not None and vbitrate_max is not None: + title.tracks.select_video( + lambda x: x.bitrate and vbitrate_min <= x.bitrate // 1000 <= vbitrate_max + ) + if not title.tracks.videos: + self.log.error(f"No Video Track in {vbitrate_min}-{vbitrate_max}kbps range...") + sys.exit(1) + + effective_video_lang = v_lang or lang + video_languages = [lang for lang in effective_video_lang if lang != "best"] + video_multi_lang = ( + "best" in effective_video_lang or "all" in effective_video_lang or len(video_languages) > 1 + ) if video_languages and "all" not in video_languages: processed_video_lang = [] for language in video_languages: @@ -1534,11 +1697,13 @@ class dl: has_hybrid = any(r == Video.Range.HYBRID for r in range_) non_hybrid_ranges = [r for r in range_ if r != Video.Range.HYBRID] + # DV is both a hybrid ingredient (lowest track) and, when explicitly + # requested, a standalone deliverable (best track per resolution). + dv_is_deliverable = Video.Range.DV in non_hybrid_ranges if quality: missing_resolutions = [] if has_hybrid: - # Split tracks: hybrid candidates vs non-hybrid hybrid_candidate_tracks = [ v for v in title.tracks.videos @@ -1548,20 +1713,19 @@ class dl: v for v in title.tracks.videos if v.range not in (Video.Range.HDR10, Video.Range.HDR10P, Video.Range.DV) + or (dv_is_deliverable and v.range == Video.Range.DV) ] - # Apply hybrid selection to HDR10+DV tracks - hybrid_filter = title.tracks.select_hybrid(hybrid_candidate_tracks, quality) + hybrid_filter = title.tracks.select_hybrid(hybrid_candidate_tracks, quality, worst=worst) hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks)) if non_hybrid_ranges and non_hybrid_tracks: - # Also filter non-hybrid tracks by resolution non_hybrid_selected = [ v for v in non_hybrid_tracks if any(v.height == res or int(v.width * (9 / 16)) == res for res in quality) ] - title.tracks.videos = hybrid_selected + non_hybrid_selected + title.tracks.videos = Tracks.merge_video_selections(hybrid_selected, non_hybrid_selected) else: title.tracks.videos = hybrid_selected else: @@ -1590,6 +1754,7 @@ class dl: sys.exit(1) # choose best track by range and quality + pre_hybrid_videos: list[Video] = list(title.tracks.videos) if has_hybrid else [] if has_hybrid: # Apply hybrid selection for HYBRID tracks hybrid_candidate_tracks = [ @@ -1601,24 +1766,32 @@ class dl: v for v in title.tracks.videos if v.range not in (Video.Range.HDR10, Video.Range.HDR10P, Video.Range.DV) + or (dv_is_deliverable and v.range == Video.Range.DV) ] if not quality: best_resolution = max((v.height for v in hybrid_candidate_tracks), default=None) if best_resolution: - hybrid_filter = title.tracks.select_hybrid(hybrid_candidate_tracks, [best_resolution]) + hybrid_filter = title.tracks.select_hybrid( + hybrid_candidate_tracks, [best_resolution], worst=worst + ) hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks)) else: hybrid_selected = [] else: - hybrid_filter = title.tracks.select_hybrid(hybrid_candidate_tracks, quality) + hybrid_filter = title.tracks.select_hybrid(hybrid_candidate_tracks, quality, worst=worst) hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks)) # For non-hybrid ranges, apply Cartesian product selection non_hybrid_selected: list[Video] = [] if non_hybrid_ranges and non_hybrid_tracks: - for resolution, color_range, codec in product( - quality or [None], non_hybrid_ranges, vcodec or [None] + # Include language dimension when multiple video languages were requested + if video_multi_lang: + non_hybrid_langs = list(dict.fromkeys(str(v.language) for v in non_hybrid_tracks)) + else: + non_hybrid_langs = [None] + for resolution, color_range, codec, vlang in product( + quality or [None], non_hybrid_ranges, vcodec or [None], non_hybrid_langs ): candidates = [ t @@ -1630,27 +1803,38 @@ class dl: ) and (not color_range or t.range == color_range) and (not codec or t.codec == codec) + and (vlang is None or str(t.language) == vlang) ] match = candidates[-1] if worst and candidates else next(iter(candidates), None) if match and match not in non_hybrid_selected: non_hybrid_selected.append(match) - title.tracks.videos = hybrid_selected + non_hybrid_selected + title.tracks.videos = Tracks.merge_video_selections(hybrid_selected, non_hybrid_selected) + + # Flag the lowest DV track as ingredient-only so mux skips it standalone, + # unless it is itself the chosen DV deliverable (single DV rendition). + selected_dv = [v for v in title.tracks.videos if v.range == Video.Range.DV] + if selected_dv: + ingredient_dv = min(selected_dv, key=lambda v: v.height) + deliverable_dv = [v for v in non_hybrid_selected if v.range == Video.Range.DV] + if not (dv_is_deliverable and ingredient_dv in deliverable_dv): + ingredient_dv.hybrid_base_only = True else: selected_videos: list[Video] = [] - for resolution, color_range, codec in product( - quality or [None], range_ or [None], vcodec or [None] + if video_multi_lang: + unique_video_langs = list(dict.fromkeys(str(v.language) for v in title.tracks.videos)) + else: + unique_video_langs = [None] + for resolution, color_range, codec, vlang in product( + quality or [None], range_ or [None], vcodec or [None], unique_video_langs ): candidates = [ t for t in title.tracks.videos - if ( - not resolution - or t.height == resolution - or int(t.width * (9 / 16)) == resolution - ) + if (not resolution or t.height == resolution or int(t.width * (9 / 16)) == resolution) and (not color_range or t.range == color_range) and (not codec or t.codec == codec) + and (vlang is None or str(t.language) == vlang) ] match = candidates[-1] if worst and candidates else next(iter(candidates), None) if match and match not in selected_videos: @@ -1691,6 +1875,31 @@ class dl: f"Continuing with remaining range(s): {', '.join(r.name for r in other_ranges)}" ) range_ = other_ranges + fallback_pool = pre_hybrid_videos + if video_multi_lang: + fallback_langs = list(dict.fromkeys(str(v.language) for v in fallback_pool)) + else: + fallback_langs = [None] + fallback_selected: list[Video] = [] + for resolution, color_range, codec, vlang in product( + quality or [None], other_ranges, vcodec or [None], fallback_langs + ): + candidates = [ + t + for t in fallback_pool + if ( + not resolution + or t.height == resolution + or int(t.width * (9 / 16)) == resolution + ) + and (not color_range or t.range == color_range) + and (not codec or t.codec == codec) + and (vlang is None or str(t.language) == vlang) + ] + match = candidates[-1] if worst and candidates else next(iter(candidates), None) + if match and match not in fallback_selected: + fallback_selected.append(match) + title.tracks.videos = fallback_selected else: self.log.error(msg) self.log.error(msg_detail) @@ -1712,21 +1921,38 @@ class dl: f"Required languages found ({', '.join(require_subs)}), downloading all available subtitles" ) elif s_lang and "all" not in s_lang: + from envied.core.utilities import is_exact_match + match_func = is_exact_match if exact_lang else is_close_match - missing_langs = [ - lang_ - for lang_ in s_lang - if not any(match_func(lang_, [sub.language]) for sub in title.tracks.subtitles) - ] + missing_langs = find_missing_langs( + s_lang, + [sub.language for sub in title.tracks.subtitles], + exact=exact_lang, + ) if missing_langs: - self.log.error(", ".join(missing_langs) + " not found in tracks") - sys.exit(1) + missing_str = ", ".join(missing_langs) + if best_available: + remaining = [tok for tok in s_lang if tok not in missing_langs] + if remaining: + self.log.warning( + f"{missing_str} not found in subtitle tracks, continuing with: {', '.join(remaining)}" + ) + s_lang = remaining + else: + self.log.warning( + f"{missing_str} not found in subtitle tracks, continuing without subtitles" + ) + title.tracks.subtitles = [] + else: + self.log.error(missing_str + " not found in tracks") + sys.exit(1) - title.tracks.select_subtitles(lambda x: match_func(x.language, s_lang)) - if not title.tracks.subtitles: - self.log.error(f"There's no {s_lang} Subtitle Track...") - sys.exit(1) + if s_lang and title.tracks.subtitles: + title.tracks.select_subtitles(lambda x: match_func(x.language, s_lang)) + if not title.tracks.subtitles and not best_available: + self.log.error(f"There's no {s_lang} Subtitle Track...") + sys.exit(1) if not forced_subs: title.tracks.select_subtitles(lambda x: not x.forced) @@ -1757,6 +1983,13 @@ class dl: if not title.tracks.audio: self.log.error(f"There's no {abitrate}kbps Audio Track...") sys.exit(1) + if abitrate_min is not None and abitrate_max is not None: + title.tracks.select_audio( + lambda x: x.bitrate and abitrate_min <= x.bitrate // 1000 <= abitrate_max + ) + if not title.tracks.audio: + self.log.error(f"No Audio Track in {abitrate_min}-{abitrate_max}kbps range...") + sys.exit(1) audio_languages = a_lang or lang if audio_languages: processed_lang = [] @@ -1776,9 +2009,34 @@ class dl: if language not in processed_lang: processed_lang.append(language) + if not any(tok in processed_lang for tok in ("best", "all")): + missing_a_langs = find_missing_langs( + processed_lang, + [a.language for a in title.tracks.audio], + exact=exact_lang, + ) + if missing_a_langs: + missing_str = ", ".join(missing_a_langs) + if best_available: + remaining = [tok for tok in processed_lang if tok not in missing_a_langs] + if remaining: + self.log.warning( + f"{missing_str} not found in audio tracks, continuing with: {', '.join(remaining)}" + ) + processed_lang = remaining + else: + self.log.error( + f"{missing_str} not found in audio tracks and no fallback available" + ) + sys.exit(1) + else: + self.log.error(missing_str + " not found in audio tracks") + sys.exit(1) + if "best" in processed_lang or "all" in processed_lang: unique_languages = {track.language for track in title.tracks.audio} selected_audio = [] + best_key = lambda x: (bool(x.atmos), x.bitrate or 0) # noqa: E731 for language in unique_languages: codecs_to_check = acodec if (acodec and len(acodec) > 1) else [None] for codec in codecs_to_check: @@ -1792,18 +2050,19 @@ class dl: if audio_description: standards = [t for t in base_candidates if not t.descriptive] if standards: - selected_audio.append(max(standards, key=lambda x: x.bitrate or 0)) + selected_audio.append(max(standards, key=best_key)) descs = [t for t in base_candidates if t.descriptive] if descs: - selected_audio.append(max(descs, key=lambda x: x.bitrate or 0)) + selected_audio.append(max(descs, key=best_key)) else: - selected_audio.append(max(base_candidates, key=lambda x: x.bitrate or 0)) + selected_audio.append(max(base_candidates, key=best_key)) title.tracks.audio = selected_audio else: # If multiple codecs were explicitly requested, pick the best track per codec per # requested language instead of selecting *all* bitrate variants of a codec. if acodec and len(acodec) > 1: selected_audio: list[Audio] = [] + best_key = lambda x: (bool(x.atmos), x.bitrate or 0) # noqa: E731 for language in processed_lang: for codec in acodec: @@ -1820,12 +2079,12 @@ class dl: if audio_description: standards = [t for t in candidates if not t.descriptive] if standards: - selected_audio.append(max(standards, key=lambda x: x.bitrate or 0)) + selected_audio.append(max(standards, key=best_key)) descs = [t for t in candidates if t.descriptive] if descs: - selected_audio.append(max(descs, key=lambda x: x.bitrate or 0)) + selected_audio.append(max(descs, key=best_key)) else: - selected_audio.append(max(candidates, key=lambda x: x.bitrate or 0)) + selected_audio.append(max(candidates, key=best_key)) title.tracks.audio = selected_audio else: @@ -1905,7 +2164,7 @@ class dl: kept_tracks.extend(title.tracks.chapters) kept_tracks.extend(title.tracks.attachments) - title.tracks = Tracks(kept_tracks) + title.tracks = Tracks(kept_tracks, manifest_url=title.tracks.manifest_url) selected_tracks, tracks_progress_callables = title.tracks.tree(add_progress=True) @@ -1939,6 +2198,9 @@ class dl: ) self.cdm = quality_based_cdm + if hasattr(service, "resolve_server_keys"): + service.resolve_server_keys(title) + dl_start_time = time.time() try: @@ -1948,7 +2210,8 @@ class dl: ( pool.submit( track.download, - session=service.session, + session=track.session or service.session, + no_proxy_download=no_proxy_download, prepare_drm=partial( partial(self.prepare_drm, table=download_table), track=track, @@ -1967,7 +2230,7 @@ class dl: ), cdm_only=cdm_only, vaults_only=vaults_only, - export=export, + export=export_path, ), cdm=self.cdm, max_workers=workers, @@ -1992,24 +2255,15 @@ class dl: except Exception as e: # noqa error_messages = [ ":x: Download Failed...", + f" {type(e).__name__}: {e}", ] - if isinstance(e, EnvironmentError): - error_messages.append(f" {e}") - if isinstance(e, ValueError): - error_messages.append(f" {e}") - if isinstance(e, (AttributeError, TypeError)): - console.print_exception() - else: - error_messages.append( - " An unexpected error occurred in one of the download workers.", - ) - if hasattr(e, "returncode"): - error_messages.append(f" Binary call failed, Process exit code: {e.returncode}") - error_messages.append(" See the error trace above for more information.") - if isinstance(e, subprocess.CalledProcessError): - # CalledProcessError already lists the exception trace - console.print_exception() + if hasattr(e, "returncode"): + error_messages.append(f" Binary call failed, Process exit code: {e.returncode}") + error_messages.append( + " An unexpected error occurred in one of the download workers.", + ) console.print(Padding(Group(*error_messages), (1, 5))) + console.print_exception() if self.debug_logger: self.debug_logger.log_error( @@ -2103,7 +2357,6 @@ class dl: and not video_only and not no_video ): - match_func = is_exact_match if exact_lang else is_close_match for video_track_n, video_track in enumerate(title.tracks.videos): has_manifest_cc = bool(getattr(video_track, "closed_captions", None)) has_eia_cc = ( @@ -2117,48 +2370,27 @@ class dl: if not has_manifest_cc and not has_eia_cc: continue - # Build list of CC entries to extract - if has_manifest_cc: - cc_entries = video_track.closed_captions - # Filter CC languages against --s-lang if specified - if s_lang and "all" not in s_lang: - cc_entries = [ - entry for entry in cc_entries - if entry.get("language") - and match_func(Language.get(entry["language"]), s_lang) - ] - if not cc_entries: - continue - else: - # EIA fallback: single entry with unknown language - cc_entries = [{}] - with console.status(f"Checking Video track {video_track_n + 1} for Closed Captions..."): try: - for cc_idx, cc_entry in enumerate(cc_entries): - cc_lang = ( - Language.get(cc_entry["language"]) - if cc_entry.get("language") - else title.language or video_track.language - ) - track_id = f"ccextractor-{video_track.id}-{cc_idx}" - cc = video_track.ccextractor( - track_id=track_id, - out_path=config.directories.temp - / config.filenames.subtitle.format(id=track_id, language=cc_lang), - language=cc_lang, - original=False, - ) - if cc: - cc.cc = True - title.tracks.add(cc) - self.log.info( - f"Extracted a Closed Caption ({cc_lang}) from Video track {video_track_n + 1}" - ) - else: - self.log.info( - f"No Closed Captions were found in Video track {video_track_n + 1}" - ) + cc_lang = ( + Language.get(video_track.closed_captions[0]["language"]) + if has_manifest_cc and video_track.closed_captions[0].get("language") + else title.language or video_track.language + ) + track_id = f"ccextractor-{video_track.id}" + cc = video_track.ccextractor( + track_id=track_id, + out_path=config.directories.temp + / config.filenames.subtitle.format(id=track_id, language=cc_lang), + language=cc_lang, + original=False, + ) + if cc: + cc.cc = True + title.tracks.add(cc) + self.log.info(f"Extracted a Closed Caption from Video track {video_track_n + 1}") + else: + self.log.info(f"No Closed Captions were found in Video track {video_track_n + 1}") except EnvironmentError: self.log.error( "Cannot extract Closed Captions as the ccextractor executable was not found..." @@ -2177,6 +2409,13 @@ class dl: # we don't want to fill up the log with "Repacked x track" self.log.info("Repacked one or more tracks with FFMPEG") + with console.status("Normalizing video VUI..."): + for track in title.tracks.videos: + try: + track.normalize_vui() + except Exception as e: # noqa: BLE001 + self.log.warning(f"VUI normalization skipped for {track.id}: {e}") + muxed_paths = [] muxed_audio_codecs: dict[Path, Optional[Audio.Codec]] = {} append_audio_codec_suffix = True @@ -2193,6 +2432,8 @@ class dl: BarColumn(), "•", TimeRemainingColumn(compact=True, elapsed_when_finished=True), + "•", + TextColumn("{task.fields[downloaded]}"), console=console, ) @@ -2218,7 +2459,9 @@ class dl: def enqueue_mux_tasks(task_description: str, base_tracks: Tracks) -> None: if merge_audio or not base_tracks.audio: - task_id = progress.add_task(f"{task_description}...", total=None, start=False) + task_id = progress.add_task( + f"{task_description}...", total=None, start=False, downloaded="" + ) multiplex_tasks.append((task_id, base_tracks, None)) return @@ -2231,16 +2474,37 @@ class dl: if audio_codec: description = f"{task_description} {audio_codec.name}" - task_id = progress.add_task(f"{description}...", total=None, start=False) + task_id = progress.add_task(f"{description}...", total=None, start=False, downloaded="") task_tracks = clone_tracks_for_audio(base_tracks, codec_audio_tracks) multiplex_tasks.append((task_id, task_tracks, audio_codec)) - # Check if we're in hybrid mode + def mux_video_standalone(video_track: Optional[Video]) -> None: + if video_track and video_track.dv_compatible_bitstream: + apply_dv_fixup(video_track) + + task_description = "Multiplexing" + if video_track: + if len(quality) > 1: + task_description += f" {video_track.height}p" + if len(range_) > 1: + task_description += f" {video_track.range.name}" + if len(vcodec) > 1: + task_description += f" {video_track.codec.name}" + + task_tracks = Tracks(title.tracks) + title.tracks.chapters + title.tracks.attachments + if video_track: + task_tracks.videos = [video_track] + + enqueue_mux_tasks(task_description, task_tracks) + if any(r == Video.Range.HYBRID for r in range_) and title.tracks.videos: - # Hybrid mode: process DV and HDR10 tracks separately for each resolution self.log.info("Processing Hybrid HDR10+DV tracks...") - # Group video tracks by resolution (prefer HDR10+ over HDR10 as base) + # Snapshot videos before hybrid tracks are added so the originals + # can still be muxed standalone afterwards. + original_videos = list(title.tracks.videos) + + # Prefer HDR10+ over HDR10 as the hybrid base layer. resolutions_processed = set() base_tracks_list = [ v for v in title.tracks.videos if v.range in (Video.Range.HDR10P, Video.Range.HDR10) @@ -2251,38 +2515,33 @@ class dl: resolution = hdr10_track.height if resolution in resolutions_processed: continue - resolutions_processed.add(resolution) - # Find matching DV track for this resolution (use the lowest DV resolution) + # DV layer only supplies RPU metadata, so the lowest resolution suffices. matching_dv = min(dv_tracks, key=lambda v: v.height) if dv_tracks else None if matching_dv: - # Create track pair for this resolution - resolution_tracks = [hdr10_track, matching_dv] + resolutions_processed.add(resolution) + # Operate on copies so the originals stay muxable standalone. + resolution_tracks = [deepcopy(hdr10_track), deepcopy(matching_dv)] for track in resolution_tracks: track.needs_duration_fix = True - # Run the hybrid processing for this resolution Hybrid(resolution_tracks, self.service) - # Create unique output filename for this resolution hybrid_filename = f"HDR10-DV-{resolution}p.hevc" hybrid_output_path = config.directories.temp / hybrid_filename hybrid_temp_paths.append(hybrid_output_path) - # The Hybrid class creates HDR10-DV.hevc, rename it for this resolution + # Hybrid always writes HDR10-DV.hevc; rename it per resolution. default_output = config.directories.temp / "HDR10-DV.hevc" if default_output.exists(): - # If a previous run left this behind, replace it to avoid move() failures. hybrid_output_path.unlink(missing_ok=True) shutil.move(str(default_output), str(hybrid_output_path)) - # Create tracks with the hybrid video output for this resolution task_description = f"Multiplexing Hybrid HDR10+DV {resolution}p" task_tracks = Tracks(title.tracks) + title.tracks.chapters + title.tracks.attachments - # Create a new video track for the hybrid output hybrid_track = deepcopy(hdr10_track) hybrid_track.id = f"hybrid_{hdr10_track.id}_{resolution}" hybrid_track.path = hybrid_output_path @@ -2293,24 +2552,17 @@ class dl: enqueue_mux_tasks(task_description, task_tracks) + # Mux every requested range standalone, skipping the ingredient-only DV. + for video_track in original_videos: + if video_track.hybrid_base_only: + continue + mux_video_standalone(video_track) + console.print() else: # Normal mode: process each video track separately for video_track in title.tracks.videos or [None]: - task_description = "Multiplexing" - if video_track: - if len(quality) > 1: - task_description += f" {video_track.height}p" - if len(range_) > 1: - task_description += f" {video_track.range.name}" - if len(vcodec) > 1: - task_description += f" {video_track.codec.name}" - - task_tracks = Tracks(title.tracks) + title.tracks.chapters + title.tracks.attachments - if video_track: - task_tracks.videos = [video_track] - - enqueue_mux_tasks(task_description, task_tracks) + mux_video_standalone(video_track) try: with Live(Padding(progress, (0, 5, 1, 5)), console=console): @@ -2457,6 +2709,7 @@ class dl: final_path = final_dir / f"{base_filename}{track_path.suffix}" shutil.move(track_path, final_path) + self.completed_files.append(final_path) self.log.debug(f"Saved: {final_path.name}") else: # Handle muxed files @@ -2494,6 +2747,7 @@ class dl: final_path.unlink() shutil.move(muxed_path, final_path) used_final_paths.add(final_path) + self.completed_files.append(final_path) tags.tag_file(final_path, title, self.tmdb_id, self.imdb_id) title_dl_time = time_elapsed_since(dl_start_time) @@ -2501,15 +2755,96 @@ class dl: Padding(f":tada: Title downloaded in [progress.elapsed]{title_dl_time}[/]!", (0, 5, 1, 5)) ) - # update cookies - cookie_file = self.get_cookie_path(self.service, self.profile) - if cookie_file: - self.save_cookies(cookie_file, service.session.cookies) + if not hasattr(service, "close"): + cookie_file = self.get_cookie_path(self.service, self.profile) + if cookie_file: + self.save_cookies(cookie_file, service.session.cookies) + + if hasattr(service, "close"): + service.close() dl_time = time_elapsed_since(start_time) console.print(Padding(f"Processed all titles in [progress.elapsed]{dl_time}", (0, 5, 1, 5))) + @staticmethod + def title_to_meta(title: Title_T) -> dict[str, Any]: + """Capture the title fields needed to rebuild a Title for output naming on import.""" + from envied.core.titles import Episode, Movie + + meta: dict[str, Any] = { + "id": str(title.id), + "language": str(title.language) if getattr(title, "language", None) else None, + } + if isinstance(title, Episode): + meta.update( + type="episode", + series_title=title.title, + season=title.season, + number=title.number, + name=title.name, + year=title.year, + ) + elif isinstance(title, Movie): + meta.update(type="movie", name=title.name, year=title.year) + else: + meta.update(type="movie", name=str(title)) + return meta + + def write_export(self, export: Path, title: Title_T, track: AnyTrack, drm: Any) -> None: + """Write a shareable v2 export usable by ``unshackle import``. + + Carries no session/cookies/dl-flags. Region (country code) is stored only when the + export used ``--proxy``, as an import geofence. Each track records only the licensed + DRM system; content keys live once under the track's ``keys``. + """ + with self.EXPORT_LOCK: + doc: dict[str, Any] = {} + if export.is_file(): + doc = json.loads(export.read_text(encoding="utf8")) or {} + + doc.setdefault("version", 2) + doc.setdefault("service", self.service) + if "region" not in doc and getattr(self, "proxy_requested", False): + region = getattr(getattr(self, "export_service", None), "current_region", None) + if region: + doc["region"] = region + + titles = doc.setdefault("titles", {}) + tinfo = titles.setdefault(str(title.id), {}) + tinfo.setdefault("meta", self.title_to_meta(title)) + + if title.tracks.manifest_url: + tinfo.setdefault("manifest_url", title.tracks.manifest_url) + + all_tracks = [*title.tracks.videos, *title.tracks.audio, *title.tracks.subtitles] + if "manifest_type" not in tinfo: + tinfo["manifest_type"] = next( + (t.descriptor.name for t in all_tracks if t.descriptor != Video.Descriptor.URL), None + ) + + tracks_map = tinfo.setdefault("tracks", {}) + if not tracks_map: + for t in all_tracks: + tracks_map[str(t.id)] = t.to_dict() + + track_data = tracks_map.setdefault(str(track.id), track.to_dict()) + if hasattr(drm, "to_dict"): + track_data["drm"] = [drm.to_dict()] + keys = track_data.setdefault("keys", {}) + for kid, key in drm.content_keys.items(): + keys[kid.hex] = key + + if "chapters" not in tinfo: + tinfo["chapters"] = [ + {"timestamp": chapter.timestamp, "name": chapter.name} for chapter in (title.tracks.chapters or []) + ] + + if "attachments" not in tinfo: + tinfo["attachments"] = [a.to_dict() for a in (title.tracks.attachments or []) if a.url] + + export.write_text(json.dumps(doc, indent=4, ensure_ascii=False), encoding="utf8") + def prepare_drm( self, drm: DRM_T, @@ -2530,29 +2865,72 @@ class dl: if not drm: return + server_cdm = getattr(self, "server_cdm", False) + + if server_cdm: + if not drm.content_keys: + self.log.warning("Server CDM did not resolve any keys for this track") + return + svc = getattr(self, "_remote_service", None) + server_drm_type = getattr(svc, "_server_cdm_type", None) if svc else None + drm_name = {"widevine": "Widevine", "playready": "PlayReady"}.get( + server_drm_type or "", drm.__class__.__name__ + ) + with self.DRM_TABLE_LOCK: + pssh_str = "" + expected_class = "PlayReady" if server_drm_type == "playready" else "Widevine" + matching_drm = next( + (d for d in (track.drm or []) if d.__class__.__name__ == expected_class), + drm, + ) + if hasattr(matching_drm, "pssh") and matching_drm.pssh: + if hasattr(matching_drm.pssh, "dumps"): + pssh_str = self.truncate_pssh_for_display(matching_drm.pssh.dumps(), drm_name) + elif hasattr(matching_drm, "data") and matching_drm.data.get("pssh_b64"): + pssh_str = self.truncate_pssh_for_display(matching_drm.data["pssh_b64"], drm_name) + if pssh_str: + cek_tree = Tree(Text.assemble((drm_name, "cyan"), (f"({pssh_str})", "text"), overflow="fold")) + else: + cek_tree = Tree(Text.assemble((drm_name, "cyan"), overflow="fold")) + all_kids = list(getattr(drm, "kids", [])) + if track_kid and track_kid not in all_kids: + all_kids.append(track_kid) + for kid in all_kids: + if kid in drm.content_keys: + is_track_kid = ["", "*"][kid == track_kid] + key = drm.content_keys[kid] + cek_tree.add(f"[text2]{kid.hex}:{key}{is_track_kid}") + for kid, key in drm.content_keys.items(): + if kid not in all_kids: + cek_tree.add(f"[text2]{kid.hex}:{key}") + if not any(isinstance(x, Tree) and x.label == cek_tree.label for x in table.columns[0].cells): + table.add_row(cek_tree) + return + track_quality = None if isinstance(track, Video) and track.height: track_quality = track.height - if isinstance(drm, Widevine): - if not is_widevine_cdm(self.cdm): - widevine_cdm = self.get_cdm(self.service, self.profile, drm="widevine", quality=track_quality) - if widevine_cdm: - if track_quality: - self.log.info(f"Switching to Widevine CDM for Widevine {track_quality}p content") - else: - self.log.info("Switching to Widevine CDM for Widevine content") - self.cdm = widevine_cdm + if not server_cdm: + if isinstance(drm, Widevine): + if not is_widevine_cdm(self.cdm): + widevine_cdm = self.get_cdm(self.service, self.profile, drm="widevine", quality=track_quality) + if widevine_cdm: + if track_quality: + self.log.info(f"Switching to Widevine CDM for Widevine {track_quality}p content") + else: + self.log.info("Switching to Widevine CDM for Widevine content") + self.cdm = widevine_cdm - elif isinstance(drm, PlayReady): - if not is_playready_cdm(self.cdm): - playready_cdm = self.get_cdm(self.service, self.profile, drm="playready", quality=track_quality) - if playready_cdm: - if track_quality: - self.log.info(f"Switching to PlayReady CDM for PlayReady {track_quality}p content") - else: - self.log.info("Switching to PlayReady CDM for PlayReady content") - self.cdm = playready_cdm + elif isinstance(drm, PlayReady): + if not is_playready_cdm(self.cdm): + playready_cdm = self.get_cdm(self.service, self.profile, drm="playready", quality=track_quality) + if playready_cdm: + if track_quality: + self.log.info(f"Switching to PlayReady CDM for PlayReady {track_quality}p content") + else: + self.log.info("Switching to PlayReady CDM for PlayReady content") + self.cdm = playready_cdm if isinstance(drm, Widevine): if self.debug_logger: @@ -2585,10 +2963,35 @@ class dl: for kid in all_kids: if kid in drm.content_keys: + is_track_kid = ["", "*"][kid == track_kid] + key = drm.content_keys[kid] + label = f"[text2]{kid.hex}:{key}{is_track_kid}" + if not any(f"{kid.hex}:{key}" in x.label for x in cek_tree.children): + cek_tree.add(label) continue is_track_kid = ["", "*"][kid == track_kid] + cached_key = self.LICENSE_KEY_CACHE.get(kid) + if cached_key: + drm.content_keys[kid] = cached_key + label = f"[text2]{kid.hex}:{cached_key}{is_track_kid} from cache" + if not any(f"{kid.hex}:{cached_key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="license_cache_hit", + service=self.service, + context={ + "kid": kid.hex, + "content_key": cached_key, + "track": str(track), + "drm_type": "Widevine", + }, + ) + continue + if not cdm_only: content_key, vault_used = self.vaults.get_key(kid) if content_key: @@ -2597,6 +3000,7 @@ class dl: if not any(f"{kid.hex}:{content_key}" in x.label for x in cek_tree.children): cek_tree.add(label) self.vaults.add_key(kid, content_key, excluding=vault_used) + self.LICENSE_KEY_CACHE[kid] = content_key if self.debug_logger: self.debug_logger.log_vault_query( @@ -2630,6 +3034,9 @@ class dl: if kid not in drm.content_keys and cdm_only: need_license = True + if need_license and all(kid in drm.content_keys for kid in all_kids): + need_license = False + if need_license and not vaults_only: from_vaults = drm.content_keys.copy() @@ -2651,21 +3058,24 @@ class dl: else: drm.get_content_keys(cdm=self.cdm, licence=licence, certificate=certificate) except Exception as e: - if isinstance(e, (Widevine.Exceptions.EmptyLicense, Widevine.Exceptions.CEKNotFound)): - msg = str(e) + if drm.content_keys: + self.log.debug(f"License call failed but keys already in content_keys: {e}") else: - msg = f"An exception occurred in the Service's license function: {e}" - cek_tree.add(f"[logging.level.error]{msg}") - if not pre_existing_tree: - table.add_row(cek_tree) - if self.debug_logger: - self.debug_logger.log_error( - "get_license", - e, - service=self.service, - context={"track": str(track), "exception_type": type(e).__name__}, - ) - raise e + if isinstance(e, (Widevine.Exceptions.EmptyLicense, Widevine.Exceptions.CEKNotFound)): + msg = str(e) + else: + msg = f"An exception occurred in the Service's license function: {e}" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + if self.debug_logger: + self.debug_logger.log_error( + "get_license", + e, + service=self.service, + context={"track": str(track), "exception_type": type(e).__name__}, + ) + raise e if self.debug_logger: self.debug_logger.log( @@ -2695,6 +3105,8 @@ class dl: # So we re-add the keys from vaults earlier overwriting blanks or removed KIDs data. drm.content_keys.update(from_vaults) + self.LICENSE_KEY_CACHE.update(drm.content_keys) + successful_caches = self.vaults.add_keys(drm.content_keys) self.log.info( f"Cached {len(drm.content_keys)} Key{'' if len(drm.content_keys) == 1 else 's'} to " @@ -2713,24 +3125,7 @@ class dl: table.add_row(cek_tree) if export: - keys = {} - if export.is_file(): - keys = jsonpickle.loads(export.read_text(encoding="utf8")) or {} - if str(title) not in keys: - keys[str(title)] = {} - if str(track) not in keys[str(title)]: - keys[str(title)][str(track)] = {} - - track_data = keys[str(title)][str(track)] - track_data["url"] = track.url - track_data["descriptor"] = track.descriptor.name - - if "keys" not in track_data: - track_data["keys"] = {} - for kid, key in drm.content_keys.items(): - track_data["keys"][kid.hex] = key - - export.write_text(jsonpickle.dumps(keys, indent=4), encoding="utf8") + self.write_export(export, title, track, drm) elif isinstance(drm, PlayReady): if self.debug_logger: @@ -2769,10 +3164,35 @@ class dl: for kid in all_kids: if kid in drm.content_keys: + is_track_kid = ["", "*"][kid == track_kid] + key = drm.content_keys[kid] + label = f"[text2]{kid.hex}:{key}{is_track_kid}" + if not any(f"{kid.hex}:{key}" in x.label for x in cek_tree.children): + cek_tree.add(label) continue is_track_kid = ["", "*"][kid == track_kid] + cached_key = self.LICENSE_KEY_CACHE.get(kid) + if cached_key: + drm.content_keys[kid] = cached_key + label = f"[text2]{kid.hex}:{cached_key}{is_track_kid} from cache" + if not any(f"{kid.hex}:{cached_key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="license_cache_hit", + service=self.service, + context={ + "kid": kid.hex, + "content_key": cached_key, + "track": str(track), + "drm_type": "PlayReady", + }, + ) + continue + if not cdm_only: content_key, vault_used = self.vaults.get_key(kid) if content_key: @@ -2781,6 +3201,7 @@ class dl: if not any(f"{kid.hex}:{content_key}" in x.label for x in cek_tree.children): cek_tree.add(label) self.vaults.add_key(kid, content_key, excluding=vault_used) + self.LICENSE_KEY_CACHE[kid] = content_key if self.debug_logger: self.debug_logger.log_vault_query( @@ -2815,31 +3236,37 @@ class dl: if kid not in drm.content_keys and cdm_only: need_license = True + if need_license and all(kid in drm.content_keys for kid in all_kids): + need_license = False + if need_license and not vaults_only: from_vaults = drm.content_keys.copy() try: drm.get_content_keys(cdm=self.cdm, licence=licence, certificate=certificate) except Exception as e: - if isinstance(e, (PlayReady.Exceptions.EmptyLicense, PlayReady.Exceptions.CEKNotFound)): - msg = str(e) + if drm.content_keys: + self.log.debug(f"License call failed but keys already in content_keys: {e}") else: - msg = f"An exception occurred in the Service's license function: {e}" - cek_tree.add(f"[logging.level.error]{msg}") - if not pre_existing_tree: - table.add_row(cek_tree) - if self.debug_logger: - self.debug_logger.log_error( - "get_license_playready", - e, - service=self.service, - context={ - "track": str(track), - "exception_type": type(e).__name__, - "drm_type": "PlayReady", - }, - ) - raise e + if isinstance(e, (PlayReady.Exceptions.EmptyLicense, PlayReady.Exceptions.CEKNotFound)): + msg = str(e) + else: + msg = f"An exception occurred in the Service's license function: {e}" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + if self.debug_logger: + self.debug_logger.log_error( + "get_license_playready", + e, + service=self.service, + context={ + "track": str(track), + "exception_type": type(e).__name__, + "drm_type": "PlayReady", + }, + ) + raise e for kid_, key in drm.content_keys.items(): is_track_kid_marker = ["", "*"][kid_ == track_kid] @@ -2849,6 +3276,8 @@ class dl: drm.content_keys.update(from_vaults) + self.LICENSE_KEY_CACHE.update(drm.content_keys) + successful_caches = self.vaults.add_keys(drm.content_keys) self.log.info( f"Cached {len(drm.content_keys)} Key{'' if len(drm.content_keys) == 1 else 's'} to " @@ -2867,24 +3296,7 @@ class dl: table.add_row(cek_tree) if export: - keys = {} - if export.is_file(): - keys = jsonpickle.loads(export.read_text(encoding="utf8")) or {} - if str(title) not in keys: - keys[str(title)] = {} - if str(track) not in keys[str(title)]: - keys[str(title)][str(track)] = {} - - track_data = keys[str(title)][str(track)] - track_data["url"] = track.url - track_data["descriptor"] = track.descriptor.name - - if "keys" not in track_data: - track_data["keys"] = {} - for kid, key in drm.content_keys.items(): - track_data["keys"][kid.hex] = key - - export.write_text(jsonpickle.dumps(keys, indent=4), encoding="utf8") + self.write_export(export, title, track, drm) elif isinstance(drm, MonaLisa): with self.DRM_TABLE_LOCK: @@ -3060,70 +3472,6 @@ class dl: if not cdm_name: return None - cdm_api = next(iter(x.copy() for x in config.remote_cdm if x["name"] == cdm_name), None) - if cdm_api: - cdm_type = cdm_api.get("type") + from envied.core.cdm import load_cdm - if cdm_type == "decrypt_labs": - del cdm_api["name"] - del cdm_api["type"] - - if "secret" not in cdm_api or not cdm_api["secret"]: - if config.decrypt_labs_api_key: - cdm_api["secret"] = config.decrypt_labs_api_key - else: - raise ValueError( - f"No secret provided for DecryptLabs CDM '{cdm_name}' and no global " - "decrypt_labs_api_key configured" - ) - - # All DecryptLabs CDMs use DecryptLabsRemoteCDM - return DecryptLabsRemoteCDM(service_name=service, vaults=self.vaults, **cdm_api) - - elif cdm_type == "custom_api": - del cdm_api["name"] - del cdm_api["type"] - - # All Custom API CDMs use CustomRemoteCDM - return CustomRemoteCDM(service_name=service, vaults=self.vaults, **cdm_api) - - else: - device_type = cdm_api.get("Device Type", cdm_api.get("device_type", "")) - if str(device_type).upper() == "PLAYREADY": - return PlayReadyRemoteCdm( - security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)), - host=cdm_api.get("Host", cdm_api.get("host")), - secret=cdm_api.get("Secret", cdm_api.get("secret")), - device_name=cdm_api.get("Device Name", cdm_api.get("device_name")), - ) - else: - return RemoteCdm( - device_type=cdm_api.get("Device Type", cdm_api.get("device_type", "")), - system_id=cdm_api.get("System ID", cdm_api.get("system_id", "")), - security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)), - host=cdm_api.get("Host", cdm_api.get("host")), - secret=cdm_api.get("Secret", cdm_api.get("secret")), - device_name=cdm_api.get("Device Name", cdm_api.get("device_name")), - ) - - prd_path = config.directories.prds / f"{cdm_name}.prd" - if not prd_path.is_file(): - prd_path = config.directories.wvds / f"{cdm_name}.prd" - if prd_path.is_file(): - device = PlayReadyDevice.load(prd_path) - return PlayReadyCdm.from_device(device) - - cdm_path = config.directories.wvds / f"{cdm_name}.wvd" - if not cdm_path.is_file(): - raise ValueError(f"{cdm_name} does not exist or is not a file") - - try: - device = Device.load(cdm_path) - except ConstError as e: - if "expected 2 but parsed 1" in str(e): - raise ValueError( - f"{cdm_name}.wvd seems to be a v1 WVD file, use `pywidevine migrate --help` to migrate it to v2." - ) - raise ValueError(f"{cdm_name}.wvd is an invalid or corrupt Widevine Device file, {e}") - - return WidevineCdm.from_device(device) + return load_cdm(cdm_name, service_name=service, vaults=self.vaults) diff --git a/packages/envied/src/envied/commands/env.py b/packages/envied/src/envied/commands/env.py index 9be3791..ae58920 100644 --- a/packages/envied/src/envied/commands/env.py +++ b/packages/envied/src/envied/commands/env.py @@ -68,15 +68,6 @@ def check() -> None: "desc": "HDR10+ metadata", "cat": "HDR", }, - # Downloaders - {"name": "aria2c", "binary": binaries.Aria2, "required": False, "desc": "Multi-thread DL", "cat": "Download"}, - { - "name": "N_m3u8DL-RE", - "binary": binaries.N_m3u8DL_RE, - "required": False, - "desc": "HLS/DASH/ISM", - "cat": "Download", - }, # Subtitle Tools { "name": "SubtitleEdit", diff --git a/packages/envied/src/envied/commands/import.py b/packages/envied/src/envied/commands/import.py new file mode 100644 index 0000000..b6a58d3 --- /dev/null +++ b/packages/envied/src/envied/commands/import.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import click + +from envied.commands.dl import dl +from envied.core.constants import context_settings + + +class ImportCommand: + @staticmethod + @click.command( + name="import", + short_help="Reconstruct a download (download, decrypt, mux) from an --export JSON file.", + context_settings={**context_settings, "ignore_unknown_options": True}, + ) + @click.argument("export_file", type=Path) + @click.argument("dl_args", nargs=-1, type=click.UNPROCESSED) + @click.pass_context + def cli(ctx: click.Context, export_file: Path, dl_args: tuple[str, ...]) -> None: + """ + Reconstruct an exported download without re-contacting the service. + + Re-fetches the manifest, injects the stored keys, then downloads/decrypts/muxes as a + normal `dl` run. Any `dl` options after the file are forwarded verbatim: + + unshackle import export.json -r HDR10 --proxy US + """ + if not export_file.is_file(): + raise click.ClickException(f"Export file not found: {export_file}") + + try: + data: dict[str, Any] = json.loads(export_file.read_text(encoding="utf8")) + except json.JSONDecodeError as e: + raise click.ClickException(f"Export file is not valid JSON: {e}") + + if data.get("version") != 2: + raise click.ClickException( + f"Unsupported export version {data.get('version')!r}. " + "Re-create the export with a current build of envied." + ) + + service_tag = data.get("service") + if not service_tag: + raise click.ClickException("Export file is missing the 'service' tag.") + + args = [*dl_args, "--import", str(export_file), service_tag] + dl.cli.main(args=args, prog_name="unshackle dl", standalone_mode=False) + + +globals()["import"] = ImportCommand diff --git a/packages/envied/src/envied/commands/kv.py b/packages/envied/src/envied/commands/kv.py index 9ad8071..13bf5a1 100644 --- a/packages/envied/src/envied/commands/kv.py +++ b/packages/envied/src/envied/commands/kv.py @@ -4,8 +4,12 @@ from pathlib import Path from typing import Optional import click +from rich.padding import Padding +from rich.text import Text +from rich.tree import Tree from envied.core.config import config +from envied.core.console import console from envied.core.constants import context_settings from envied.core.services import Services from envied.core.vault import Vault @@ -77,7 +81,19 @@ def kv() -> None: @click.argument("to_vault_name", type=str) @click.argument("from_vault_names", nargs=-1, type=click.UNPROCESSED) @click.option("-s", "--service", type=str, default=None, help="Only copy data to and from a specific service.") -def copy(to_vault_name: str, from_vault_names: list[str], service: Optional[str] = None) -> None: +@click.option( + "-l", + "--local-only", + is_flag=True, + default=False, + help="Only copy data for services installed locally (skip vault tables for services not present in the configured services path).", +) +def copy( + to_vault_name: str, + from_vault_names: list[str], + service: Optional[str] = None, + local_only: bool = False, +) -> None: """ Copy data from multiple Key Vaults into a single Key Vault. Rows with matching KIDs are skipped unless there's no KEY set. @@ -103,13 +119,28 @@ def copy(to_vault_name: str, from_vault_names: list[str], service: Optional[str] vault_names = ", ".join([v.name for v in from_vaults]) log.info(f"Copying data from {vault_names} → {to_vault.name}") + if service and local_only: + raise click.UsageError("--service and --local-only are mutually exclusive.") + if service: service = Services.get_tag(service) log.info(f"Filtering by service: {service}") + installed: Optional[set[str]] = None + if local_only: + installed = {t.upper() for t in Services.get_tags()} + log.info(f"Filtering by locally installed services ({len(installed)} found)") + total_added = 0 for from_vault in from_vaults: - services_to_copy = [service] if service else from_vault.get_services() + services_to_copy = [service] if service else list(from_vault.get_services()) + + if installed is not None: + before = len(services_to_copy) + services_to_copy = [s for s in services_to_copy if s and s.upper() in installed] + skipped = before - len(services_to_copy) + if skipped: + log.info(f"{from_vault.name}: skipping {skipped} service(s) not installed locally") for service_tag in services_to_copy: added = copy_service_data(to_vault, from_vault, service_tag, log) @@ -124,8 +155,20 @@ def copy(to_vault_name: str, from_vault_names: list[str], service: Optional[str] @kv.command() @click.argument("vaults", nargs=-1, type=click.UNPROCESSED) @click.option("-s", "--service", type=str, default=None, help="Only sync data to and from a specific service.") +@click.option( + "-l", + "--local-only", + is_flag=True, + default=False, + help="Only sync data for services installed locally (skip vault tables for services not present in the configured services path).", +) @click.pass_context -def sync(ctx: click.Context, vaults: list[str], service: Optional[str] = None) -> None: +def sync( + ctx: click.Context, + vaults: list[str], + service: Optional[str] = None, + local_only: bool = False, +) -> None: """ Ensure multiple Key Vaults copies of all keys as each other. It's essentially just a bi-way copy between each vault. @@ -135,9 +178,21 @@ def sync(ctx: click.Context, vaults: list[str], service: Optional[str] = None) - if not len(vaults) > 1: raise click.ClickException("You must provide more than one Vault to sync.") - ctx.invoke(copy, to_vault_name=vaults[0], from_vault_names=vaults[1:], service=service) + ctx.invoke( + copy, + to_vault_name=vaults[0], + from_vault_names=vaults[1:], + service=service, + local_only=local_only, + ) for i in range(1, len(vaults)): - ctx.invoke(copy, to_vault_name=vaults[i], from_vault_names=[vaults[i - 1]], service=service) + ctx.invoke( + copy, + to_vault_name=vaults[i], + from_vault_names=[vaults[i - 1]], + service=service, + local_only=local_only, + ) @kv.command() @@ -188,6 +243,72 @@ def add(file: Path, service: str, vaults: list[str]) -> None: log.info("Done!") +@kv.command() +@click.argument("kid", type=str) +@click.option("-s", "--service", type=str, default=None, help="Limit search to a specific service tag.") +@click.option( + "-v", "--vault", "vault_name", type=str, default=None, help="Limit search to a specific configured vault by name." +) +def search(kid: str, service: Optional[str], vault_name: Optional[str]) -> None: + """ + Search configured Key Vault(s) for a KID and report any matching KEY. + + KID must be 32 hex characters (no dashes). If --service is omitted, every + service table in each vault is scanned. If --vault is omitted, every + vault in the config is searched. + """ + log = logging.getLogger("kv") + + kid_norm = kid.replace("-", "").lower() + if not re.fullmatch(r"[0-9a-f]{32}", kid_norm): + raise click.ClickException(f"KID '{kid}' is not 32 hex characters.") + + if vault_name: + vault_names = [vault_name] + else: + vault_names = [v["name"] for v in config.key_vaults] + if not vault_names: + raise click.ClickException("No Key Vaults are configured.") + + vaults_ = load_vaults(vault_names) + + service_tag = Services.get_tag(service) if service else None + + hit: Optional[tuple[str, str, str]] = None + for vault in vaults_: + if service_tag: + services_to_check: list[str] = [service_tag] + else: + try: + services_to_check = list(vault.get_services()) + except Exception as e: + log.debug(f"{vault}: get_services() failed ({e})") + services_to_check = [] + if not services_to_check: + log.warning(f"{vault}: cannot search without a service (remote vault requires --service). Skipping.") + continue + + for svc in services_to_check: + try: + key = vault.get_key(kid_norm, svc) + except Exception as e: + log.debug(f"{vault} [{svc}]: lookup error ({e})") + continue + if key and key.count("0") != len(key): + hit = (vault.name, svc, key) + break + if hit: + break + + if hit: + vname, svc, key = hit + tree = Tree(Text.assemble((svc, "cyan"), (f"({vname})", "text"), overflow="fold")) + tree.add(f"[text2]{kid_norm}:{key}") + console.print(Padding(tree, (1, 5))) + else: + log.info(f"KID {kid_norm} not found in {len(vaults_)} vault(s).") + + @kv.command() @click.argument("vaults", nargs=-1, type=click.UNPROCESSED) def prepare(vaults: list[str]) -> None: diff --git a/packages/envied/src/envied/commands/search.py b/packages/envied/src/envied/commands/search.py index f413f67..4eaae82 100644 --- a/packages/envied/src/envied/commands/search.py +++ b/packages/envied/src/envied/commands/search.py @@ -86,7 +86,9 @@ def search(ctx: click.Context, no_proxy: bool, profile: Optional[str] = None, pr # requesting proxy from a specific proxy provider requested_provider, proxy = proxy.split(":", maxsplit=1) # Match simple region codes (us, ca, uk1) or provider:region format (nordvpn:ca, windscribe:us) - if re.match(r"^[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE) or re.match(r"^[a-z]+:[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE): + if re.match(r"^[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE) or re.match( + r"^[a-z]+:[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE + ): proxy = proxy.lower() with console.status(f"Getting a Proxy to {proxy}...", spinner="dots"): if requested_provider: diff --git a/packages/envied/src/envied/commands/serve.py b/packages/envied/src/envied/commands/serve.py index fc765eb..b41ee49 100644 --- a/packages/envied/src/envied/commands/serve.py +++ b/packages/envied/src/envied/commands/serve.py @@ -6,6 +6,7 @@ from aiohttp import web from envied.core import binaries from envied.core.api import cors_middleware, setup_routes, setup_swagger +from envied.core.api.compression import compression_middleware from envied.core.config import config from envied.core.constants import context_settings @@ -35,6 +36,12 @@ from envied.core.constants import context_settings default=False, help="Enable debug logging for API operations.", ) +@click.option( + "--remote-only", + is_flag=True, + default=False, + help="Only expose remote service session endpoints (health, services, search, session).", +) def serve( host: str, port: int, @@ -45,6 +52,7 @@ def serve( no_key: bool, debug_api: bool, debug: bool, + remote_only: bool, ) -> None: """ Serve your Local Widevine and PlayReady Devices and REST API for Remote Access. @@ -119,19 +127,64 @@ def serve( config.serve["playready_devices"] = [] config.serve["playready_devices"].extend(list(config.directories.prds.glob("*.prd"))) + @web.middleware + async def api_key_authentication(request: web.Request, handler) -> web.Response: + """Authenticate API requests using X-Secret-Key header.""" + if request.path == "/api/health": + return await handler(request) + secret_key = request.headers.get("X-Secret-Key") + if not secret_key: + return web.json_response({"status": 401, "message": "Secret Key is Empty."}, status=401) + if secret_key not in request.app["config"]["users"]: + return web.json_response({"status": 401, "message": "Secret Key is Invalid."}, status=401) + return await handler(request) + + remote_only = remote_only or config.serve.get("remote_only", False) + if remote_only: + api_only = True + + global_services = config.serve.get("services") + if global_services: + log.info(f"Global service allowlist: {', '.join(global_services)}") + users = config.serve.get("users", {}) + for user_key, user_cfg in users.items() if isinstance(users, dict) else []: + user_services = user_cfg.get("services") if isinstance(user_cfg, dict) else None + if user_services: + username = user_cfg.get("username", user_key[:8] + "...") + log.info(f"User '{username}' restricted to services: {', '.join(user_services)}") + if api_only: log.info("Starting REST API server (pywidevine/pyplayready CDM disabled)") if no_key: - app = web.Application(middlewares=[cors_middleware]) + app = web.Application(middlewares=[cors_middleware, compression_middleware]) app["config"] = {"users": {}} else: - app = web.Application(middlewares=[cors_middleware, pywidevine_serve.authentication]) + app = web.Application(middlewares=[cors_middleware, api_key_authentication, compression_middleware]) app["config"] = {"users": {api_secret: {"devices": [], "username": "api_user"}}} app["debug_api"] = debug_api - setup_routes(app) - setup_swagger(app) - log.info(f"REST API endpoints available at http://{host}:{port}/api/") - log.info(f"Swagger UI available at http://{host}:{port}/api/docs/") + + # Start session cleanup loop for remote-dl sessions + from envied.core.api.session_store import get_session_store + + session_store = get_session_store() + + async def start_session_cleanup(_app: web.Application) -> None: + await session_store.start_cleanup_loop() + + async def stop_session_cleanup(_app: web.Application) -> None: + await session_store.cancel_all_bridges() + await session_store.stop_cleanup_loop() + + app.on_startup.append(start_session_cleanup) + app.on_cleanup.append(stop_session_cleanup) + + setup_routes(app, remote_only=remote_only) + if not remote_only: + setup_swagger(app) + log.info(f"REST API endpoints available at http://{host}:{port}/api/") + log.info(f"Swagger UI available at http://{host}:{port}/api/docs/") + else: + log.info(f"Remote service endpoints available at http://{host}:{port}/api/session/") log.info("(Press CTRL+C to quit)") web.run_app(app, host=host, port=port, print=None) else: @@ -181,20 +234,39 @@ def serve( if serve_playready_flag and request.path.startswith("/playready"): from pyplayready import __version__ as pyplayready_version - response.headers["Server"] = f"https://git.gay/ready-dl/pyplayready serve v{pyplayready_version}" + + response.headers["Server"] = ( + f"https://git.gay/ready-dl/pyplayready serve v{pyplayready_version}" + ) return response + return serve_authentication if no_key: - app = web.Application(middlewares=[cors_middleware]) + app = web.Application(middlewares=[cors_middleware, compression_middleware]) else: serve_auth = create_serve_authentication(serve_playready and bool(prd_devices)) - app = web.Application(middlewares=[cors_middleware, serve_auth]) + app = web.Application(middlewares=[cors_middleware, serve_auth, compression_middleware]) app["config"] = serve_config app["debug_api"] = debug_api + # Start session cleanup loop for remote-dl sessions + from envied.core.api.session_store import get_session_store + + session_store = get_session_store() + + async def start_session_cleanup(_app: web.Application) -> None: + await session_store.start_cleanup_loop() + + async def stop_session_cleanup(_app: web.Application) -> None: + await session_store.cancel_all_bridges() + await session_store.stop_cleanup_loop() + + app.on_startup.append(start_session_cleanup) + app.on_cleanup.append(stop_session_cleanup) + if serve_widevine: app.on_startup.append(pywidevine_serve._startup) app.on_cleanup.append(pywidevine_serve._cleanup) @@ -226,6 +298,7 @@ def serve( async def playready_ping(_: web.Request) -> web.Response: from pyplayready import __version__ as pyplayready_version + response = web.json_response({"message": "OK"}) response.headers["Server"] = f"https://git.gay/ready-dl/pyplayready serve v{pyplayready_version}" return response @@ -237,13 +310,16 @@ def serve( elif serve_playready: log.info("No PlayReady devices found, skipping PlayReady CDM endpoints") - setup_routes(app) - setup_swagger(app) + setup_routes(app, remote_only=remote_only) if serve_widevine: log.info(f"Widevine CDM endpoints available at http://{host}:{port}/{{device}}/open") - log.info(f"REST API endpoints available at http://{host}:{port}/api/") - log.info(f"Swagger UI available at http://{host}:{port}/api/docs/") + if remote_only: + log.info(f"Remote service endpoints available at http://{host}:{port}/api/session/") + else: + setup_swagger(app) + log.info(f"REST API endpoints available at http://{host}:{port}/api/") + log.info(f"Swagger UI available at http://{host}:{port}/api/docs/") log.info("(Press CTRL+C to quit)") web.run_app(app, host=host, port=port, print=None) finally: diff --git a/packages/envied/src/envied/core/__init__.py b/packages/envied/src/envied/core/__init__.py index ce1305b..0d72820 100644 --- a/packages/envied/src/envied/core/__init__.py +++ b/packages/envied/src/envied/core/__init__.py @@ -1 +1 @@ -__version__ = "4.0.0" +__version__ = "5.1.0" diff --git a/packages/envied/src/envied/core/__main__.py b/packages/envied/src/envied/core/__main__.py index d25c718..cef083d 100644 --- a/packages/envied/src/envied/core/__main__.py +++ b/packages/envied/src/envied/core/__main__.py @@ -49,22 +49,23 @@ def main(version: bool, debug: bool) -> None: traceback.install(console=console, width=80, suppress=[click]) console.print( - Padding( - Group( - Text( - r"░█▀▀░█▀█░█░█░▀█▀░█▀▀░█▀▄" + "\n" - r"░█▀▀░█░█░▀▄▀░░█░░█▀▀░█░█" + "\n" - r"░▀▀▀░▀░▀░░▀░░▀▀▀░▀▀▀░▀▀░" + "\n" , - style="ascii.art", + Padding( + Group( + Text( + r"░█▀▀░█▀█░█░█░▀█▀░█▀▀░█▀▄" + "\n" + r"░█▀▀░█░█░▀▄▀░░█░░█▀▀░█░█" + "\n" + r"░▀▀▀░▀░▀░░▀░░▀▀▀░▀▀▀░▀▀░" + "\n" , + style="ascii.art", + ), + Text(" and more than unshackled...", style = "ascii.art"), + f"\nv [repr.number]{__version__}[/] - https://github.com/vinefeeder/envied", ), - Text(" and more than unshackled...", style = "ascii.art"), - f"\nv [repr.number]{__version__}[/] - https://github.com/vinefeeder/envied", + (1, 11, 1, 10), + expand=True, ), - (1, 11, 1, 10), - expand=True, - ), - justify="center", - ) + justify="center", + ) + @atexit.register diff --git a/packages/envied/src/envied/core/api/compression.py b/packages/envied/src/envied/core/api/compression.py new file mode 100644 index 0000000..df568ac --- /dev/null +++ b/packages/envied/src/envied/core/api/compression.py @@ -0,0 +1,40 @@ +"""aiohttp middleware for gzip transport compression.""" + +from __future__ import annotations + +import gzip + +from aiohttp import web + + +@web.middleware +async def compression_middleware(request: web.Request, handler) -> web.Response: + """Compress JSON responses with gzip when the client supports it.""" + response = await handler(request) + + accept_encoding = request.headers.get("Accept-Encoding", "") + if "gzip" not in accept_encoding: + return response + + if response.content_type and "json" not in response.content_type: + return response + + body = response.body + if body is None or len(body) < 256: + return response + + from envied.core.config import config + + level = config.serve.get("compression_level", 1) + if not level: + return response + + compressed = gzip.compress(body, compresslevel=level) + headers = dict(response.headers) + headers["Content-Encoding"] = "gzip" + headers["Content-Length"] = str(len(compressed)) + return web.Response( + body=compressed, + status=response.status, + headers=headers, + ) diff --git a/packages/envied/src/envied/core/api/download_manager.py b/packages/envied/src/envied/core/api/download_manager.py index b8a15db..95673a0 100644 --- a/packages/envied/src/envied/core/api/download_manager.py +++ b/packages/envied/src/envied/core/api/download_manager.py @@ -2,6 +2,7 @@ import asyncio import json import logging import os +import re import sys import tempfile import threading @@ -16,6 +17,11 @@ from typing import Any, Callable, Dict, List, Optional log = logging.getLogger("download_manager") +def _sanitize_log(value: object) -> str: + """Sanitize a value for safe logging by removing newlines and control characters.""" + return str(value).replace("\n", "").replace("\r", "").replace("\x00", "") + + class JobStatus(Enum): QUEUED = "queued" DOWNLOADING = "downloading" @@ -141,8 +147,37 @@ def _perform_download( sub_map.update({c.value.upper(): c for c in Subtitle.Codec}) params["sub_format"] = sub_map.get(sub_format_raw.upper()) - if params.get("export") and isinstance(params["export"], str): - params["export"] = Path(params["export"]) + if params.get("export"): + params["export"] = bool(params["export"]) + + # Normalize slow: accept string "MIN-MAX", list/tuple, or True (default 60-120) + slow_raw = params.get("slow") + if slow_raw is not None and not isinstance(slow_raw, tuple): + if isinstance(slow_raw, bool): + params["slow"] = (60, 120) if slow_raw else None + elif isinstance(slow_raw, list) and len(slow_raw) == 2: + params["slow"] = (int(slow_raw[0]), int(slow_raw[1])) + elif isinstance(slow_raw, str): + from envied.core.utils.click_types import SLOW_DELAY_RANGE + + try: + params["slow"] = SLOW_DELAY_RANGE.convert(slow_raw, None, None) + except click.BadParameter as exc: + raise Exception(f"Invalid slow parameter: {exc}") + + # Convert wanted episode strings to internal "SxE" format + # Accepts: "S01E01", "S01-S03", "s1e1", "1x1", or already-parsed format + wanted_raw = params.get("wanted") + if wanted_raw: + from envied.core.utils.click_types import SeasonRange + + if isinstance(wanted_raw, str): + wanted_raw = [wanted_raw] + # Only convert if not already in internal "SxE" format + needs_conversion = any(not re.match(r"^\d+x\d+$", w) for w in wanted_raw) + if needs_conversion: + season_range = SeasonRange() + params["wanted"] = season_range.parse_tokens(*wanted_raw) # Load service configuration service_config_path = Services.get_path(service) / config.filenames.config @@ -156,10 +191,14 @@ def _perform_download( ctx = click.Context(dl_command.cli) ctx.invoked_subcommand = service - ctx.obj = ContextData(config=service_config, cdm=None, proxy_providers=[], profile=params.get("profile")) + from envied.core.api.handlers import load_full_cdm + + cdm = load_full_cdm(service, params.get("profile"), params.get("cdm_type")) + ctx.obj = ContextData(config=service_config, cdm=cdm, proxy_providers=[], profile=params.get("profile")) ctx.params = { "proxy": params.get("proxy"), "no_proxy": params.get("no_proxy", False), + "no_proxy_download": params.get("no_proxy_download", False), "profile": params.get("profile"), "repack": params.get("repack", False), "tag": params.get("tag"), @@ -266,6 +305,8 @@ def _perform_download( acodec=params.get("acodec"), vbitrate=params.get("vbitrate"), abitrate=params.get("abitrate"), + vbitrate_range=params.get("vbitrate_range"), + abitrate_range=params.get("abitrate_range"), range_=params.get("range", ["SDR"]), channels=params.get("channels"), no_atmos=params.get("no_atmos", False), @@ -289,18 +330,20 @@ def _perform_download( no_chapters=params.get("no_chapters", False), no_video=params.get("no_video", False), audio_description=params.get("audio_description", False), - slow=params.get("slow", False), + slow=params.get("slow", None), list_=False, list_titles=False, skip_dl=params.get("skip_dl", False), export=params.get("export"), cdm_only=params.get("cdm_only"), no_proxy=params.get("no_proxy", False), + no_proxy_download=params.get("no_proxy_download", False), no_folder=params.get("no_folder", False), no_source=params.get("no_source", False), no_mux=params.get("no_mux", False), workers=params.get("workers"), downloads=params.get("downloads", 1), + worst=params.get("worst", False), best_available=params.get("best_available", False), split_audio=params.get("split_audio"), ) @@ -322,9 +365,10 @@ def _perform_download( log.error(f"Stderr: {stderr_str}") raise - log.info(f"Download completed for job {job_id}, files in {original_download_dir}") + output_files = [str(p) for p in dl_instance.completed_files] + log.info(f"Download completed for job {job_id}, {len(output_files)} file(s) in {original_download_dir}") - return [] + return output_files class DownloadQueueManager: @@ -361,7 +405,7 @@ class DownloadQueueManager: self._jobs[job_id] = job self._job_queue.put_nowait(job) - log.info(f"Created download job {job_id} for {service}:{title_id}") + log.info(f"Created download job {job_id} for {_sanitize_log(service)}:{_sanitize_log(title_id)}") return job def get_job(self, job_id: str) -> Optional[DownloadJob]: @@ -381,27 +425,27 @@ class DownloadQueueManager: if job.status == JobStatus.QUEUED: job.status = JobStatus.CANCELLED job.cancel_event.set() # Signal cancellation - log.info(f"Cancelled queued job {job_id}") + log.info(f"Cancelled queued job {_sanitize_log(job_id)}") return True elif job.status == JobStatus.DOWNLOADING: # Set the cancellation event first - this will be checked by the download thread job.cancel_event.set() job.status = JobStatus.CANCELLED - log.info(f"Signaled cancellation for downloading job {job_id}") + log.info(f"Signaled cancellation for downloading job {_sanitize_log(job_id)}") # Cancel the active download task task = self._active_downloads.get(job_id) if task: task.cancel() - log.info(f"Cancelled download task for job {job_id}") + log.info(f"Cancelled download task for job {_sanitize_log(job_id)}") process = self._download_processes.get(job_id) if process: try: process.terminate() - log.info(f"Terminated worker process for job {job_id}") + log.info(f"Terminated worker process for job {_sanitize_log(job_id)}") except ProcessLookupError: - log.debug(f"Worker process for job {job_id} already exited") + log.debug(f"Worker process for job {_sanitize_log(job_id)} already exited") return True @@ -629,8 +673,11 @@ class DownloadQueueManager: if stdout.strip(): log.debug(f"Worker stdout for job {job.job_id}: {stdout.strip()}") if stderr.strip(): - log.warning(f"Worker stderr for job {job.job_id}: {stderr.strip()}") job.worker_stderr = stderr.strip() + if returncode != 0: + log.warning(f"Worker stderr for job {job.job_id}: {stderr.strip()}") + else: + log.debug(f"Worker stderr for job {job.job_id}: {stderr.strip()}") result_data: Optional[Dict[str, Any]] = None try: diff --git a/packages/envied/src/envied/core/api/errors.py b/packages/envied/src/envied/core/api/errors.py index 312ee12..2d5a106 100644 --- a/packages/envied/src/envied/core/api/errors.py +++ b/packages/envied/src/envied/core/api/errors.py @@ -35,6 +35,8 @@ class APIErrorCode(str, Enum): NOT_FOUND = "NOT_FOUND" # Resource not found (title, job, etc.) NO_CONTENT = "NO_CONTENT" # No titles/tracks/episodes found JOB_NOT_FOUND = "JOB_NOT_FOUND" # Download job doesn't exist + SESSION_NOT_FOUND = "SESSION_NOT_FOUND" # Remote-dl session doesn't exist or expired + TRACK_NOT_FOUND = "TRACK_NOT_FOUND" # Track ID not found in session RATE_LIMITED = "RATE_LIMITED" # Service rate limiting @@ -97,6 +99,8 @@ class APIError(Exception): APIErrorCode.NOT_FOUND: 404, APIErrorCode.NO_CONTENT: 404, APIErrorCode.JOB_NOT_FOUND: 404, + APIErrorCode.SESSION_NOT_FOUND: 404, + APIErrorCode.TRACK_NOT_FOUND: 404, # 429 Too Many Requests APIErrorCode.RATE_LIMITED: 429, # 500 Internal Server Error diff --git a/packages/envied/src/envied/core/api/handlers.py b/packages/envied/src/envied/core/api/handlers.py index f01e73e..a4cdaa4 100644 --- a/packages/envied/src/envied/core/api/handlers.py +++ b/packages/envied/src/envied/core/api/handlers.py @@ -1,3 +1,4 @@ +import asyncio import enum import logging from typing import Any, Dict, List, Optional @@ -5,17 +6,22 @@ from typing import Any, Dict, List, Optional from aiohttp import web from envied.core.api.errors import APIError, APIErrorCode, handle_api_exception +from envied.core.api.input_bridge import AuthStatus, InputBridge +from envied.core.config import config from envied.core.constants import AUDIO_CODEC_MAP, DYNAMIC_RANGE_MAP, VIDEO_CODEC_MAP -from envied.core.proxies.basic import Basic -from envied.core.proxies.hola import Hola -from envied.core.proxies.nordvpn import NordVPN -from envied.core.proxies.surfsharkvpn import SurfsharkVPN +from envied.core.proxies.resolve import initialize_proxy_providers, resolve_proxy from envied.core.services import Services from envied.core.titles import Episode, Movie, Title_T from envied.core.tracks import Audio, Subtitle, Video log = logging.getLogger("api") + +def sanitize_log(value: object) -> str: + """Sanitize a value for safe logging by removing newlines and control characters.""" + return str(value).replace("\n", "").replace("\r", "").replace("\x00", "") + + DEFAULT_DOWNLOAD_PARAMS = { "profile": None, "quality": [], @@ -23,6 +29,8 @@ DEFAULT_DOWNLOAD_PARAMS = { "acodec": None, "vbitrate": None, "abitrate": None, + "vbitrate_range": None, + "abitrate_range": None, "range": ["SDR"], "channels": None, "no_atmos": False, @@ -45,125 +53,197 @@ DEFAULT_DOWNLOAD_PARAMS = { "no_chapters": False, "no_video": False, "audio_description": False, - "slow": False, + "slow": None, "split_audio": None, "skip_dl": False, - "export": None, + "export": False, "cdm_only": None, + "proxy": None, "no_proxy": False, + "no_proxy_download": False, "no_folder": False, "no_source": False, "no_mux": False, "workers": None, "downloads": 1, + "worst": False, "best_available": False, "repack": False, + "tag": None, + "tmdb_id": None, "imdb_id": None, + "animeapi_id": None, + "enrich": False, "output_dir": None, "no_cache": False, "reset_cache": False, } -def initialize_proxy_providers() -> List[Any]: - """Initialize and return available proxy providers.""" - proxy_providers = [] - try: - from envied.core import binaries - # Load the main unshackle config to get proxy provider settings - from envied.core.config import config as main_config - - log.debug(f"Main config proxy providers: {getattr(main_config, 'proxy_providers', {})}") - log.debug(f"Available proxy provider configs: {list(getattr(main_config, 'proxy_providers', {}).keys())}") - - # Use main_config instead of the service-specific config for proxy providers - proxy_config = getattr(main_config, "proxy_providers", {}) - - if proxy_config.get("basic"): - log.debug("Loading Basic proxy provider") - proxy_providers.append(Basic(**proxy_config["basic"])) - if proxy_config.get("nordvpn"): - log.debug("Loading NordVPN proxy provider") - proxy_providers.append(NordVPN(**proxy_config["nordvpn"])) - if proxy_config.get("surfsharkvpn"): - log.debug("Loading SurfsharkVPN proxy provider") - proxy_providers.append(SurfsharkVPN(**proxy_config["surfsharkvpn"])) - if hasattr(binaries, "HolaProxy") and binaries.HolaProxy: - log.debug("Loading Hola proxy provider") - proxy_providers.append(Hola()) - - for proxy_provider in proxy_providers: - log.info(f"Loaded {proxy_provider.__class__.__name__}: {proxy_provider}") - - if not proxy_providers: - log.warning("No proxy providers were loaded. Check your proxy provider configuration in envied.yaml") - - except Exception as e: - log.warning(f"Failed to initialize some proxy providers: {e}") - - return proxy_providers +# Keys that are part of the API transport envelope, not service.cli options. +# Used by instantiate_service to avoid passing them as kwargs to a service. +LIST_HANDLER_TRANSPORT_KEYS = { + "service", + "title_id", + "profile", + "season", + "episode", + "wanted", + "proxy", + "no_proxy", + "query", +} -def resolve_proxy(proxy: str, proxy_providers: List[Any]) -> str: - """Resolve proxy parameter to actual proxy URI.""" - import re +def load_full_cdm(service: str, profile: Optional[str], cdm_type: Optional[str] = None) -> Optional[Any]: + """Load a real CDM object for the given service. - if not proxy: - return proxy + Services often touch ``ctx.obj.cdm.security_level`` / ``.device_type`` / ``.system_id`` + inside ``__init__``, so the lightweight ``_resolve_server_cdm`` stub is not enough + for list_titles / list_tracks / search. Mirrors ``dl.get_cdm`` selection logic but + skips the quality-tier shortcuts (no track context yet) and falls back to the stub + if no device is configured or loading fails. + """ + from envied.core.cdm import load_cdm + from envied.core.config import config as app_config - # Check if explicit proxy URI - if re.match(r"^https?://", proxy): - return proxy - - # Handle provider:country format (e.g., "nordvpn:us") - requested_provider = None - if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE): - requested_provider, proxy = proxy.split(":", maxsplit=1) - - # Handle country code format (e.g., "us", "uk") - if re.match(r"^[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE): - proxy = proxy.lower() - - if requested_provider: - # Find specific provider (case-insensitive matching) - proxy_provider = next( - (x for x in proxy_providers if x.__class__.__name__.lower() == requested_provider.lower()), - None, - ) - if not proxy_provider: - available_providers = [x.__class__.__name__ for x in proxy_providers] - raise ValueError( - f"The proxy provider '{requested_provider}' was not recognized. Available providers: {available_providers}" + cdm_name = app_config.cdm.get(service) or app_config.cdm.get("default") + if isinstance(cdm_name, dict): + lower_keys = {k.lower(): v for k, v in cdm_name.items()} + if {"widevine", "playready"} & lower_keys.keys(): + drm_key = None + if cdm_type: + drm_key = {"wv": "widevine", "widevine": "widevine", "pr": "playready", "playready": "playready"}.get( + cdm_type.lower() ) - - proxy_uri = proxy_provider.get_proxy(proxy) - if not proxy_uri: - raise ValueError(f"The proxy provider {requested_provider} had no proxy for {proxy}") - - log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy_uri}") - return proxy_uri + cdm_name = lower_keys.get(drm_key or "widevine") or lower_keys.get("playready") else: - # Try all providers - for proxy_provider in proxy_providers: - proxy_uri = proxy_provider.get_proxy(proxy) - if proxy_uri: - log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy_uri}") - return proxy_uri + cdm_name = cdm_name.get(profile) or cdm_name.get("default") or app_config.cdm.get("default") - raise ValueError(f"No proxy provider had a proxy for {proxy}") + if not cdm_name or not isinstance(cdm_name, str): + return _resolve_server_cdm(service, profile, cdm_type) - # Return as-is if not recognized format - log.info(f"Using explicit Proxy: {proxy}") - return proxy + try: + return load_cdm(cdm_name, service_name=service) + except Exception as exc: # noqa: BLE001 - fall back to stub on load failure + log.warning(f"load_cdm({cdm_name!r}) failed for {service}: {exc}; using lightweight stub") + return _resolve_server_cdm(service, profile, cdm_type) -def validate_service(service_tag: str) -> Optional[str]: - """Validate and normalize service tag.""" +def load_service_yaml(normalized_service: str) -> dict: + """Load a service's config.yaml and merge it with the global override block.""" + import yaml + + from envied.core.utils.collections import merge_dict + + service_config_path = Services.get_path(normalized_service) / config.filenames.config + if service_config_path.exists(): + service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) or {} + else: + service_config = {} + merge_dict(config.services.get(normalized_service), service_config) + return service_config + + +def build_parent_ctx( + profile: Optional[str], + cdm: Any, + proxy_param: Optional[str], + no_proxy: bool, + proxy_providers: list, + service_config: dict, + extra_params: Optional[Dict[str, Any]] = None, +) -> Any: + """Build a parent click Context for invoking a service.cli via ctx.invoke(). + + The service's CLI callback uses ``ctx.parent.params`` (proxy, range_, vcodec, etc.) + and ``ctx.obj`` (ContextData). Both flow through Click's parent chain. + """ + import click + + from envied.core.utils.click_types import ContextData + + @click.command() + @click.pass_context + def dummy(ctx: click.Context) -> None: + pass + + parent = click.Context(dummy) + parent.obj = ContextData(config=service_config, cdm=cdm, proxy_providers=proxy_providers, profile=profile) + params = {"proxy": proxy_param, "no_proxy": no_proxy} + if extra_params: + params.update(extra_params) + parent.params = params + return parent + + +def instantiate_service( + parent_ctx: Any, + service_module: Any, + title: str, + data: Optional[Dict[str, Any]] = None, + transport_keys: Optional[set] = None, +) -> Any: + """Instantiate a service by invoking its click cli through Click. + + Click fills option defaults via ``param.get_default()`` and runs type coercion, + so we no longer have to inspect ``__init__`` or stitch defaults by hand. Extra + kwargs are pulled from ``data`` when the key matches a cli option name and is + not in the transport-key blocklist. + """ + cli_params = getattr(getattr(service_module, "cli", None), "params", []) or [] + cli_param_names = {p.name for p in cli_params if hasattr(p, "name") and p.name} + transport_keys = transport_keys or set() + extras: Dict[str, Any] = {} + if data: + for k, v in data.items(): + if k in cli_param_names and k not in transport_keys and k != "title": + extras[k] = v + return parent_ctx.invoke(service_module.cli, title=title, **extras) + + +def get_allowed_services(request: Optional[web.Request] = None) -> Optional[List[str]]: + """Get effective service allowlist considering global + per-key config. + + Returns None if all services are allowed. + """ + global_allowed = config.serve.get("services") + global_set: Optional[set[str]] = None + if global_allowed: + global_set = {Services.get_tag(s) for s in global_allowed} + + key_set: Optional[set[str]] = None + if request: + secret_key = request.headers.get("X-Secret-Key") + if secret_key: + users = config.serve.get("users", {}) + user_config = users.get(secret_key, {}) + user_services = user_config.get("services") + if user_services: + key_set = {Services.get_tag(s) for s in user_services} + + if global_set and key_set: + result = global_set & key_set + elif global_set: + result = global_set + elif key_set: + result = key_set + else: + return None + + return list(result) + + +def validate_service(service_tag: str, request: Optional[web.Request] = None) -> Optional[str]: + """Validate, normalize, and check allowlist for service tag.""" try: normalized = Services.get_tag(service_tag) service_path = Services.get_path(normalized) if not service_path.exists(): return None + allowed = get_allowed_services(request) + if allowed is not None and normalized not in allowed: + return None return normalized except Exception: return None @@ -171,6 +251,8 @@ def validate_service(service_tag: str) -> Optional[str]: def serialize_title(title: Title_T) -> Dict[str, Any]: """Convert a title object to JSON-serializable dict.""" + title_language = str(title.language) if hasattr(title, "language") and title.language else None + if isinstance(title, Episode): episode_name = title.name if title.name else f"Episode {title.number:02d}" result = { @@ -181,6 +263,7 @@ def serialize_title(title: Title_T) -> Dict[str, Any]: "number": title.number, "year": title.year, "id": str(title.id) if hasattr(title, "id") else None, + "language": title_language, } elif isinstance(title, Movie): result = { @@ -188,17 +271,69 @@ def serialize_title(title: Title_T) -> Dict[str, Any]: "name": str(title.name) if hasattr(title, "name") else str(title), "year": title.year, "id": str(title.id) if hasattr(title, "id") else None, + "language": title_language, } else: result = { "type": "other", "name": str(title.name) if hasattr(title, "name") else str(title), "id": str(title.id) if hasattr(title, "id") else None, + "language": title_language, } return result +def _extract_manifests(tracks) -> List[Dict[str, Any]]: + """Extract manifest data from tracks for client-side re-parsing. + + Serializes DASH and ISM manifest XML as zlib-compressed base64 strings + so the client can reconstruct track.data locally. HLS tracks download + directly from their URL so no manifest serialization is needed. + """ + import base64 + import zlib + + from lxml import etree + + from envied.core.config import config as app_config + + compression_level = app_config.serve.get("compression_level", 1) + + seen: set[str] = set() + manifests: List[Dict[str, Any]] = [] + + for track in list(tracks.videos) + list(tracks.audio) + list(tracks.subtitles): + manifest_url = str(track.url) if track.url else None + if not manifest_url or manifest_url in seen: + continue + + if track.data.get("dash") and track.data["dash"].get("manifest"): + seen.add(manifest_url) + xml_bytes = etree.tostring(track.data["dash"]["manifest"], xml_declaration=True, encoding="UTF-8") + compressed = zlib.compress(xml_bytes, compression_level) if compression_level else xml_bytes + manifests.append( + { + "type": "dash", + "url": manifest_url, + "data": base64.b64encode(compressed).decode("ascii"), + } + ) + elif track.data.get("ism") and track.data["ism"].get("manifest"): + seen.add(manifest_url) + xml_bytes = etree.tostring(track.data["ism"]["manifest"], xml_declaration=True, encoding="UTF-8") + compressed = zlib.compress(xml_bytes, compression_level) if compression_level else xml_bytes + manifests.append( + { + "type": "ism", + "url": manifest_url, + "data": base64.b64encode(compressed).decode("ascii"), + } + ) + + return manifests + + def serialize_drm(drm_list) -> Optional[List[Dict[str, Any]]]: """Serialize DRM objects to JSON-serializable list.""" if not drm_list: @@ -225,6 +360,7 @@ def serialize_drm(drm_list) -> Optional[List[Dict[str, Any]]]: elif hasattr(pssh_obj, "__bytes__"): # Convert to base64 import base64 + drm_info["pssh"] = base64.b64encode(bytes(pssh_obj)).decode() elif hasattr(pssh_obj, "to_base64"): drm_info["pssh"] = pssh_obj.to_base64() @@ -277,7 +413,7 @@ def serialize_video_track(track: Video, include_url: bool = False) -> Dict[str, codec_name = track.codec.name if hasattr(track.codec, "name") else str(track.codec) range_name = track.range.name if hasattr(track.range, "name") else str(track.range) - # Get descriptor for N_m3u8DL-RE compatibility (HLS, DASH, URL, etc.) + # Serialize the manifest descriptor (HLS, DASH, URL, etc.) descriptor_name = None if hasattr(track, "descriptor") and track.descriptor: descriptor_name = track.descriptor.name if hasattr(track.descriptor, "name") else str(track.descriptor) @@ -306,7 +442,7 @@ def serialize_audio_track(track: Audio, include_url: bool = False) -> Dict[str, """Convert audio track to JSON-serializable dict.""" codec_name = track.codec.name if hasattr(track.codec, "name") else str(track.codec) - # Get descriptor for N_m3u8DL-RE compatibility + # Serialize the manifest descriptor (HLS, DASH, URL, etc.) descriptor_name = None if hasattr(track, "descriptor") and track.descriptor: descriptor_name = track.descriptor.name if hasattr(track.descriptor, "name") else str(track.descriptor) @@ -351,16 +487,7 @@ def serialize_subtitle_track(track: Subtitle, include_url: bool = False) -> Dict async def search_handler(data: Dict[str, Any], request: Optional[web.Request] = None) -> web.Response: """Handle search request.""" - import inspect - - import click - import yaml - from envied.commands.dl import dl - from envied.core.config import config - from envied.core.services import Services - from envied.core.utils.click_types import ContextData - from envied.core.utils.collections import merge_dict service_tag = data.get("service") query = data.get("query") @@ -378,16 +505,19 @@ async def search_handler(data: Dict[str, Any], request: Optional[web.Request] = details={"service": service_tag}, ) + allowed = get_allowed_services(request) + if allowed is not None and normalized_service not in allowed: + raise APIError( + APIErrorCode.INVALID_SERVICE, + f"Service '{service_tag}' not found", + details={"service": service_tag}, + ) + profile = data.get("profile") proxy_param = data.get("proxy") no_proxy = data.get("no_proxy", False) - service_config_path = Services.get_path(normalized_service) / config.filenames.config - if service_config_path.exists(): - service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) - else: - service_config = {} - merge_dict(config.services.get(normalized_service), service_config) + service_config = load_service_yaml(normalized_service) proxy_providers = [] if not no_proxy: @@ -404,49 +534,12 @@ async def search_handler(data: Dict[str, Any], request: Optional[web.Request] = details={"proxy": proxy_param, "service": normalized_service}, ) - @click.command() - @click.pass_context - def dummy_service(ctx: click.Context) -> None: - pass - - ctx = click.Context(dummy_service) - ctx.obj = ContextData(config=service_config, cdm=None, proxy_providers=proxy_providers, profile=profile) - ctx.params = {"proxy": proxy_param, "no_proxy": no_proxy} - + cdm = load_full_cdm(normalized_service, profile, data.get("cdm_type")) + parent_ctx = build_parent_ctx(profile, cdm, proxy_param, no_proxy, proxy_providers, service_config) service_module = Services.load(normalized_service) - dummy_service.name = normalized_service - ctx.invoked_subcommand = normalized_service - - service_ctx = click.Context(dummy_service, parent=ctx) - service_ctx.obj = ctx.obj - - service_init_params = inspect.signature(service_module.__init__).parameters - service_kwargs = {"title": query} - - # Extract default values from the click command - if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): - for param in service_module.cli.params: - if hasattr(param, "name") and param.name not in service_kwargs: - if hasattr(param, "default") and param.default is not None and not isinstance(param.default, enum.Enum): - service_kwargs[param.name] = param.default - - for param_name, param_info in service_init_params.items(): - if param_name not in service_kwargs and param_name not in ["self", "ctx"]: - if param_info.default is inspect.Parameter.empty: - if param_name == "meta_lang": - service_kwargs[param_name] = None - elif param_name == "movie": - service_kwargs[param_name] = False - else: - service_kwargs[param_name] = None - - # Filter to only accepted params - accepted_params = set(service_init_params.keys()) - {"self", "ctx"} - service_kwargs = {k: v for k, v in service_kwargs.items() if k in accepted_params} - try: - service_instance = service_module(service_ctx, **service_kwargs) + service_instance = instantiate_service(parent_ctx, service_module, query) except Exception as exc: raise APIError( APIErrorCode.SERVICE_ERROR, @@ -463,13 +556,15 @@ async def search_handler(data: Dict[str, Any], request: Optional[web.Request] = results = [] try: for result in service_instance.search(): - results.append({ - "id": result.id, - "title": result.title, - "description": result.description, - "label": result.label, - "url": result.url, - }) + results.append( + { + "id": result.id, + "title": result.title, + "description": result.description, + "label": result.label, + "url": result.url, + } + ) except NotImplementedError: raise APIError( APIErrorCode.SERVICE_ERROR, @@ -500,7 +595,7 @@ async def list_titles_handler(data: Dict[str, Any], request: Optional[web.Reques details={"missing_parameter": "title_id"}, ) - normalized_service = validate_service(service_tag) + normalized_service = validate_service(service_tag, request) if not normalized_service: raise APIError( APIErrorCode.INVALID_SERVICE, @@ -509,29 +604,10 @@ async def list_titles_handler(data: Dict[str, Any], request: Optional[web.Reques ) try: - import inspect - - import click - import yaml - from envied.commands.dl import dl - from envied.core.config import config - from envied.core.utils.click_types import ContextData - from envied.core.utils.collections import merge_dict - service_config_path = Services.get_path(normalized_service) / config.filenames.config - if service_config_path.exists(): - service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) - else: - service_config = {} - merge_dict(config.services.get(normalized_service), service_config) + service_config = load_service_yaml(normalized_service) - @click.command() - @click.pass_context - def dummy_service(ctx: click.Context) -> None: - pass - - # Handle proxy configuration proxy_param = data.get("proxy") no_proxy = data.get("no_proxy", False) proxy_providers = [] @@ -550,58 +626,10 @@ async def list_titles_handler(data: Dict[str, Any], request: Optional[web.Reques details={"proxy": proxy_param, "service": normalized_service}, ) - ctx = click.Context(dummy_service) - ctx.obj = ContextData(config=service_config, cdm=None, proxy_providers=proxy_providers, profile=profile) - ctx.params = {"proxy": proxy_param, "no_proxy": no_proxy} - + cdm = load_full_cdm(normalized_service, profile, data.get("cdm_type")) + parent_ctx = build_parent_ctx(profile, cdm, proxy_param, no_proxy, proxy_providers, service_config) service_module = Services.load(normalized_service) - - dummy_service.name = normalized_service - dummy_service.params = [click.Argument([title_id], type=str)] - ctx.invoked_subcommand = normalized_service - - service_ctx = click.Context(dummy_service, parent=ctx) - service_ctx.obj = ctx.obj - - service_kwargs = {"title": title_id} - - # Add additional parameters from request data - for key, value in data.items(): - if key not in ["service", "title_id", "profile", "season", "episode", "wanted", "proxy", "no_proxy"]: - service_kwargs[key] = value - - # Get service parameter info and click command defaults - service_init_params = inspect.signature(service_module.__init__).parameters - - # Extract default values from the click command - if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): - for param in service_module.cli.params: - if hasattr(param, "name") and param.name not in service_kwargs: - # Add default value if parameter is not already provided - if hasattr(param, "default") and param.default is not None and not isinstance(param.default, enum.Enum): - service_kwargs[param.name] = param.default - - # Handle required parameters that don't have click defaults - for param_name, param_info in service_init_params.items(): - if param_name not in service_kwargs and param_name not in ["self", "ctx"]: - # Check if parameter is required (no default value in signature) - if param_info.default is inspect.Parameter.empty: - # Provide sensible defaults for common required parameters - if param_name == "meta_lang": - service_kwargs[param_name] = None - elif param_name == "movie": - service_kwargs[param_name] = False - else: - # Log warning for unknown required parameters - log.warning(f"Unknown required parameter '{param_name}' for service {normalized_service}") - - # Filter out any parameters that the service doesn't accept - filtered_kwargs = {} - for key, value in service_kwargs.items(): - if key in service_init_params: - filtered_kwargs[key] = value - - service_instance = service_module(service_ctx, **filtered_kwargs) + service_instance = instantiate_service(parent_ctx, service_module, title_id, data, LIST_HANDLER_TRANSPORT_KEYS) cookies = dl.get_cookie_jar(normalized_service, profile) credential = dl.get_credentials(normalized_service, profile) @@ -618,7 +646,7 @@ async def list_titles_handler(data: Dict[str, Any], request: Optional[web.Reques except APIError: raise - except Exception as e: + except (Exception, SystemExit) as e: log.exception("Error listing titles") debug_mode = request.app.get("debug_api", False) if request else False return handle_api_exception( @@ -648,7 +676,7 @@ async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Reques details={"missing_parameter": "title_id"}, ) - normalized_service = validate_service(service_tag) + normalized_service = validate_service(service_tag, request) if not normalized_service: raise APIError( APIErrorCode.INVALID_SERVICE, @@ -657,29 +685,10 @@ async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Reques ) try: - import inspect - - import click - import yaml - from envied.commands.dl import dl - from envied.core.config import config - from envied.core.utils.click_types import ContextData - from envied.core.utils.collections import merge_dict - service_config_path = Services.get_path(normalized_service) / config.filenames.config - if service_config_path.exists(): - service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) - else: - service_config = {} - merge_dict(config.services.get(normalized_service), service_config) + service_config = load_service_yaml(normalized_service) - @click.command() - @click.pass_context - def dummy_service(ctx: click.Context) -> None: - pass - - # Handle proxy configuration proxy_param = data.get("proxy") no_proxy = data.get("no_proxy", False) proxy_providers = [] @@ -698,58 +707,10 @@ async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Reques details={"proxy": proxy_param, "service": normalized_service}, ) - ctx = click.Context(dummy_service) - ctx.obj = ContextData(config=service_config, cdm=None, proxy_providers=proxy_providers, profile=profile) - ctx.params = {"proxy": proxy_param, "no_proxy": no_proxy} - + cdm = load_full_cdm(normalized_service, profile, data.get("cdm_type")) + parent_ctx = build_parent_ctx(profile, cdm, proxy_param, no_proxy, proxy_providers, service_config) service_module = Services.load(normalized_service) - - dummy_service.name = normalized_service - dummy_service.params = [click.Argument([title_id], type=str)] - ctx.invoked_subcommand = normalized_service - - service_ctx = click.Context(dummy_service, parent=ctx) - service_ctx.obj = ctx.obj - - service_kwargs = {"title": title_id} - - # Add additional parameters from request data - for key, value in data.items(): - if key not in ["service", "title_id", "profile", "season", "episode", "wanted", "proxy", "no_proxy"]: - service_kwargs[key] = value - - # Get service parameter info and click command defaults - service_init_params = inspect.signature(service_module.__init__).parameters - - # Extract default values from the click command - if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): - for param in service_module.cli.params: - if hasattr(param, "name") and param.name not in service_kwargs: - # Add default value if parameter is not already provided - if hasattr(param, "default") and param.default is not None and not isinstance(param.default, enum.Enum): - service_kwargs[param.name] = param.default - - # Handle required parameters that don't have click defaults - for param_name, param_info in service_init_params.items(): - if param_name not in service_kwargs and param_name not in ["self", "ctx"]: - # Check if parameter is required (no default value in signature) - if param_info.default is inspect.Parameter.empty: - # Provide sensible defaults for common required parameters - if param_name == "meta_lang": - service_kwargs[param_name] = None - elif param_name == "movie": - service_kwargs[param_name] = False - else: - # Log warning for unknown required parameters - log.warning(f"Unknown required parameter '{param_name}' for service {normalized_service}") - - # Filter out any parameters that the service doesn't accept - filtered_kwargs = {} - for key, value in service_kwargs.items(): - if key in service_init_params: - filtered_kwargs[key] = value - - service_instance = service_module(service_ctx, **filtered_kwargs) + service_instance = instantiate_service(parent_ctx, service_module, title_id, data, LIST_HANDLER_TRANSPORT_KEYS) cookies = dl.get_cookie_jar(normalized_service, profile) credential = dl.get_credentials(normalized_service, profile) @@ -775,7 +736,7 @@ async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Reques else: wanted = season_range.parse_tokens(wanted_param) log.debug(f"Parsed wanted '{wanted_param}' into {len(wanted)} episodes: {wanted[:10]}...") - except Exception as e: + except (Exception, SystemExit) as e: raise APIError( APIErrorCode.INVALID_PARAMETERS, f"Invalid wanted parameter: {e}", @@ -839,7 +800,7 @@ async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Reques failed_episodes.append(f"S{title.season}E{title.number:02d}") log.debug(f"Episode {title.season}x{title.number} not available, skipping") continue - except Exception as e: + except (Exception, SystemExit) as e: # Handle other errors gracefully failed_episodes.append(f"S{title.season}E{title.number:02d}") log.debug(f"Error getting tracks for {title.season}x{title.number}: {e}") @@ -884,7 +845,7 @@ async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Reques except APIError: raise - except Exception as e: + except (Exception, SystemExit) as e: log.exception("Error listing tracks") debug_mode = request.app.get("debug_api", False) if request else False return handle_api_exception( @@ -915,7 +876,21 @@ def validate_download_parameters(data: Dict[str, Any]) -> Optional[str]: return f"Invalid vcodec: {', '.join(invalid)}. Must be one of: {', '.join(valid_vcodecs)}" if "acodec" in data and data["acodec"]: - valid_acodecs = ["AAC", "AC3", "EC3", "EAC3", "DD", "DD+", "AC4", "OPUS", "FLAC", "ALAC", "VORBIS", "OGG", "DTS"] + valid_acodecs = [ + "AAC", + "AC3", + "EC3", + "EAC3", + "DD", + "DD+", + "AC4", + "OPUS", + "FLAC", + "ALAC", + "VORBIS", + "OGG", + "DTS", + ] if isinstance(data["acodec"], str): acodec_values = [v.strip() for v in data["acodec"].split(",") if v.strip()] elif isinstance(data["acodec"], list): @@ -940,6 +915,14 @@ def validate_download_parameters(data: Dict[str, Any]) -> Optional[str]: if not isinstance(data["abitrate"], int) or data["abitrate"] <= 0: return "abitrate must be a positive integer" + if "vbitrate_range" in data and data["vbitrate_range"] is not None: + if not isinstance(data["vbitrate_range"], str) or "-" not in data["vbitrate_range"]: + return "vbitrate_range must be a string in 'MIN-MAX' format (e.g., '6000-7000')" + + if "abitrate_range" in data and data["abitrate_range"] is not None: + if not isinstance(data["abitrate_range"], str) or "-" not in data["abitrate_range"]: + return "abitrate_range must be a string in 'MIN-MAX' format (e.g., '128-256')" + if "channels" in data and data["channels"] is not None: if not isinstance(data["channels"], (int, float)) or data["channels"] <= 0: return "channels must be a positive number" @@ -974,7 +957,7 @@ def validate_download_parameters(data: Dict[str, Any]) -> Optional[str]: return "Cannot use both s_lang and require_subs" if "range" in data and data["range"]: - valid_ranges = ["SDR", "HDR10", "HDR10+", "DV", "HLG"] + valid_ranges = ["SDR", "HDR10", "HDR10+", "DV", "HLG", "HYBRID"] if isinstance(data["range"], list): for r in data["range"]: if r.upper() not in valid_ranges: @@ -1006,7 +989,7 @@ async def download_handler(data: Dict[str, Any], request: Optional[web.Request] details={"missing_parameter": "title_id"}, ) - normalized_service = validate_service(service_tag) + normalized_service = validate_service(service_tag, request) if not normalized_service: raise APIError( APIErrorCode.INVALID_SERVICE, @@ -1027,10 +1010,13 @@ async def download_handler(data: Dict[str, Any], request: Optional[web.Request] service_module = Services.load(normalized_service) service_specific_defaults = {} - # Extract default values from the service's click command + # Extract default values from the service's click command. + # Skip None defaults here: this dict overlays into job params; injecting + # None for keys like `profile` would clobber serve-config overrides. + # Missing required __init__ params are handled in download_manager._perform_download. if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): for param in service_module.cli.params: - if hasattr(param, "name") and hasattr(param, "default") and param.default is not None and not isinstance(param.default, enum.Enum): + if hasattr(param, "name") and param.default is not None and not isinstance(param.default, enum.Enum): # Store service-specific defaults (e.g., drm_system, hydrate_track, profile for NF) service_specific_defaults[param.name] = param.default @@ -1040,8 +1026,17 @@ async def download_handler(data: Dict[str, Any], request: Optional[web.Request] # Create download job with filtered parameters (exclude service and title_id as they're already passed) filtered_params = {k: v for k, v in data.items() if k not in ["service", "title_id"]} - # Merge defaults with provided parameters (user params override service defaults, which override global defaults) - params_with_defaults = {**DEFAULT_DOWNLOAD_PARAMS, **service_specific_defaults, **filtered_params} + # Overlay any dl-relevant keys from `serve:` config (e.g. downloads, workers) so the API + # respects server-side defaults without each client having to send them. + serve_overrides = { + k: v for k, v in (config.serve or {}).items() if k in DEFAULT_DOWNLOAD_PARAMS and v is not None + } + params_with_defaults = { + **DEFAULT_DOWNLOAD_PARAMS, + **serve_overrides, + **service_specific_defaults, + **filtered_params, + } job = manager.create_job(normalized_service, title_id, **params_with_defaults) return web.json_response( @@ -1050,7 +1045,7 @@ async def download_handler(data: Dict[str, Any], request: Optional[web.Request] except APIError: raise - except Exception as e: + except (Exception, SystemExit) as e: log.exception("Error creating download job") debug_mode = request.app.get("debug_api", False) if request else False return handle_api_exception( @@ -1118,7 +1113,7 @@ async def list_download_jobs_handler(data: Dict[str, Any], request: Optional[web except APIError: raise - except Exception as e: + except (Exception, SystemExit) as e: log.exception("Error listing download jobs") debug_mode = request.app.get("debug_api", False) if request else False return handle_api_exception( @@ -1147,8 +1142,8 @@ async def get_download_job_handler(job_id: str, request: Optional[web.Request] = except APIError: raise - except Exception as e: - log.exception(f"Error getting download job {job_id}") + except (Exception, SystemExit) as e: + log.exception(f"Error getting download job {sanitize_log(job_id)}") debug_mode = request.app.get("debug_api", False) if request else False return handle_api_exception( e, @@ -1184,11 +1179,1218 @@ async def cancel_download_job_handler(job_id: str, request: Optional[web.Request except APIError: raise - except Exception as e: - log.exception(f"Error cancelling download job {job_id}") + except (Exception, SystemExit) as e: + log.exception(f"Error cancelling download job {sanitize_log(job_id)}") debug_mode = request.app.get("debug_api", False) if request else False return handle_api_exception( e, context={"operation": "cancel_download_job", "job_id": job_id}, debug_mode=debug_mode, ) + + +# --------------------------------------------------------------------------- +# Remote-DL Session Handlers +# --------------------------------------------------------------------------- + + +SESSION_TRANSPORT_KEYS = { + "service", + "title_id", + "season", + "episode", + "wanted", + "proxy", + "no_proxy", + "credentials", + "cookies", + "cache", + "client_region", + "cdm_type", + "range_", + "vcodec", + "quality", + "best_available", +} + + +def _create_service_instance( + normalized_service: str, + title_id: str, + data: Dict[str, Any], + proxy_param: Optional[str], + proxy_providers: list, + profile: Optional[str], +) -> Any: + """Create and authenticate a service instance. + + Supports client-sent credentials/cookies (for remote-dl) with fallback + to server-local config (for backward compatibility). + """ + from envied.commands.dl import dl + from envied.core.credential import Credential + from envied.core.tracks import Video + + service_config = load_service_yaml(normalized_service) + cdm = load_full_cdm(normalized_service, profile, data.get("cdm_type")) + + # Reconstruct enum track-selection params from client data so service code that reads + # ctx.parent.params (Service.__init__ proxy/range/vcodec/best_available block) sees enums. + range_names = data.get("range_") + range_values: Optional[list] = None + if range_names: + range_values = [] + for name in range_names: + try: + range_values.append(Video.Range[name]) + except KeyError: + pass + range_values = range_values or None + + vcodec_names = data.get("vcodec") + vcodec_values: Optional[list] = None + if vcodec_names: + vcodec_values = [] + for name in vcodec_names: + try: + vcodec_values.append(Video.Codec[name]) + except KeyError: + pass + vcodec_values = vcodec_values or None + + extra_params = { + "range_": range_values, + "vcodec": vcodec_values, + "quality": data.get("quality"), + "best_available": data.get("best_available", False), + } + + parent_ctx = build_parent_ctx( + profile, + cdm, + proxy_param, + data.get("no_proxy", False), + proxy_providers, + service_config, + extra_params=extra_params, + ) + + service_module = Services.load(normalized_service) + service_instance = instantiate_service(parent_ctx, service_module, title_id, data, SESSION_TRANSPORT_KEYS) + + # Resolve credentials: client-sent > server-local + cred_data = data.get("credentials") + if cred_data and isinstance(cred_data, dict): + credential = Credential( + username=cred_data["username"], + password=cred_data["password"], + extra=cred_data.get("extra"), + ) + else: + credential = dl.get_credentials(normalized_service, profile) + + # Resolve cookies: client-sent > server-local + cookie_text = data.get("cookies") + if cookie_text and isinstance(cookie_text, str): + import base64 + import tempfile + import zlib + from http.cookiejar import MozillaCookieJar + + cookie_str = zlib.decompress(base64.b64decode(cookie_text)).decode("utf-8") + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False, encoding="utf-8") as f: + f.write(cookie_str) + tmp_path = f.name + try: + cookies = MozillaCookieJar(tmp_path) + cookies.load(ignore_discard=True, ignore_expires=True) + finally: + import os + + os.unlink(tmp_path) + else: + cookies = dl.get_cookie_jar(normalized_service, profile) + + return service_instance, cookies, credential + + +async def session_create_handler(data: Dict[str, Any], request: Optional[web.Request] = None) -> web.Response: + """Handle session creation: authenticate + get titles + get tracks + get chapters. + + This is the main entry point for remote-dl clients. It creates a persistent + session on the server with the authenticated service instance, fetches all + titles and tracks, and returns everything the client needs for track selection. + """ + from envied.core.api.session_store import get_session_store + + service_tag = data.get("service") + title_id = data.get("title_id") + profile = data.get("profile") + + if not service_tag: + raise APIError(APIErrorCode.INVALID_INPUT, "Missing required parameter: service") + if not title_id: + raise APIError(APIErrorCode.INVALID_INPUT, "Missing required parameter: title_id") + + normalized_service = validate_service(service_tag, request) + if not normalized_service: + raise APIError( + APIErrorCode.INVALID_SERVICE, + f"Invalid or unavailable service: {service_tag}", + details={"service": service_tag}, + ) + + try: + proxy_param, proxy_providers = _resolve_handler_proxy(data, normalized_service) + + import hashlib + import uuid as uuid_mod + + from envied.core.cacher import Cacher + from envied.core.config import config as app_config + + session_id = str(uuid_mod.uuid4()) + api_key = request.headers.get("X-Secret-Key", "anonymous") if request else "anonymous" + api_key_hash = hashlib.sha256(api_key.encode()).hexdigest()[:12] + session_cache_tag = f"_sessions/{api_key_hash}/{session_id}/{normalized_service}" + + service_instance, cookies, credential = _create_service_instance( + normalized_service, + title_id, + data, + proxy_param, + proxy_providers, + profile, + ) + + service_instance.cache = Cacher(session_cache_tag) + + cache_data = data.get("cache", {}) + if cache_data: + import base64 + import zlib + + cache_dir = app_config.directories.cache / session_cache_tag + cache_dir.mkdir(parents=True, exist_ok=True) + for key, content in cache_data.items(): + decompressed = zlib.decompress(base64.b64decode(content)).decode("utf-8") + (cache_dir / key).with_suffix(".json").write_text(decompressed, encoding="utf-8") + + bridge = InputBridge() + service_instance._input_bridge = bridge + + store = get_session_store() + session = await store.create( + normalized_service, + service_instance, + session_id=session_id, + ) + session.creator_ip = request.remote if request else None + session.cache_tag = session_cache_tag + session.input_bridge = bridge + session.auth_status = AuthStatus.AUTHENTICATING + + async def _run_auth() -> None: + try: + await asyncio.to_thread(service_instance.authenticate, cookies, credential) + session.auth_status = AuthStatus.AUTHENTICATED + bridge.status = AuthStatus.AUTHENTICATED + except (Exception, SystemExit) as e: + log.exception("Auth failed for session %s", session_id) + session.auth_status = AuthStatus.FAILED + session.auth_error = str(e) + bridge.status = AuthStatus.FAILED + bridge.error = str(e) + + asyncio.create_task(_run_auth()) + + return web.json_response( + { + "session_id": session.session_id, + "service": normalized_service, + "status": "authenticating", + } + ) + + except APIError: + raise + except (Exception, SystemExit) as e: + log.exception("Error creating session") + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "session_create", "service": service_tag, "title_id": title_id}, + debug_mode=debug_mode, + ) + + +async def session_titles_handler(session_id: str, request: Optional[web.Request] = None) -> web.Response: + """Get titles for the authenticated session. + + Called after session/create. This is separate from auth so that + interactive auth flows (OTP, captcha) can complete before titles + are fetched. + """ + session = await _get_validated_session(session_id, request) + _require_authenticated(session) + + try: + service_instance = session.service_instance + titles = service_instance.get_titles() + session.titles = titles + + # Serialize titles and build title map + if hasattr(titles, "__iter__") and not isinstance(titles, str): + titles_list = list(titles) + else: + titles_list = [titles] + + serialized_titles = [] + for t in titles_list: + tid = str(t.id) if hasattr(t, "id") else str(id(t)) + session.title_map[tid] = t + serialized_titles.append(serialize_title(t)) + + return web.json_response( + { + "session_id": session_id, + "titles": serialized_titles, + } + ) + + except (Exception, SystemExit) as e: + log.exception("Error getting titles") + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "session_titles", "session_id": session_id}, + debug_mode=debug_mode, + ) + + +async def session_tracks_handler( + data: Dict[str, Any], session_id: str, request: Optional[web.Request] = None +) -> web.Response: + """Get tracks and chapters for a specific title in the session. + + Called per-title by the client after session/create returns titles. + This keeps auth separate from track fetching, allowing interactive + auth flows (OTP, captcha) before any tracks are requested. + """ + session = await _get_validated_session(session_id, request) + _require_authenticated(session) + + title_id = data.get("title_id") + if not title_id: + raise APIError(APIErrorCode.INVALID_INPUT, "Missing required parameter: title_id") + + title = session.title_map.get(str(title_id)) + if not title: + raise APIError( + APIErrorCode.INVALID_INPUT, + f"Title not found in session: {title_id}", + details={"available_titles": list(session.title_map.keys())}, + ) + + try: + service_instance = session.service_instance + tracks = service_instance.get_tracks(title) + + title_tracks: Dict[str, Any] = {} + for track in tracks.videos: + title_tracks[str(track.id)] = track + session.tracks[str(track.id)] = track + for track in tracks.audio: + title_tracks[str(track.id)] = track + session.tracks[str(track.id)] = track + for track in tracks.subtitles: + title_tracks[str(track.id)] = track + session.tracks[str(track.id)] = track + session.tracks_by_title[str(title_id)] = title_tracks + + try: + chapters = service_instance.get_chapters(title) + session.chapters_by_title[str(title_id)] = chapters if chapters else [] + except (NotImplementedError, Exception): + session.chapters_by_title[str(title_id)] = [] + + video_tracks = sorted(tracks.videos, key=lambda t: t.bitrate or 0, reverse=True) + audio_tracks = sorted(tracks.audio, key=lambda t: t.bitrate or 0, reverse=True) + + manifests = _extract_manifests(tracks) + + svc_session = session.service_instance.session + session_headers = dict(svc_session.headers) if hasattr(svc_session, "headers") else {} + session_cookies = {} + if hasattr(svc_session, "cookies"): + for cookie in svc_session.cookies: + if hasattr(cookie, "name") and hasattr(cookie, "value"): + session_cookies[cookie.name] = cookie.value + + from envied.core.config import config as app_config + + api_key = request.headers.get("X-Secret-Key", "anonymous") if request else "anonymous" + user_cfg = app_config.serve.get("users", {}).get(api_key, {}) + has_wv = bool(user_cfg.get("devices")) + has_pr = bool(user_cfg.get("playready_devices")) + + service_tag = session.service_tag + config_cdm_type = _detect_cdm_type_for_service(service_tag, app_config) + + track_has_wv = any( + d.__class__.__name__ == "Widevine" for t in list(tracks.videos) + list(tracks.audio) if t.drm for d in t.drm + ) + track_has_pr = any( + d.__class__.__name__ == "PlayReady" + for t in list(tracks.videos) + list(tracks.audio) + if t.drm + for d in t.drm + ) + + if config_cdm_type: + server_cdm_type = config_cdm_type + elif track_has_pr and has_pr: + server_cdm_type = "playready" + elif track_has_wv and has_wv: + server_cdm_type = "widevine" + elif has_wv: + server_cdm_type = "widevine" + else: + server_cdm_type = "playready" + + return web.json_response( + { + "title": serialize_title(title), + "video": [serialize_video_track(t, include_url=True) for t in video_tracks], + "audio": [serialize_audio_track(t, include_url=True) for t in audio_tracks], + "subtitles": [serialize_subtitle_track(t, include_url=True) for t in tracks.subtitles], + "chapters": [ + {"timestamp": ch.timestamp, "name": ch.name} + for ch in session.chapters_by_title.get(str(title_id), []) + ], + "attachments": [ + {"url": a.url, "name": a.name, "mime_type": a.mime_type, "description": a.description} + for a in tracks.attachments + if hasattr(a, "url") and a.url + ], + "manifests": manifests, + "session_headers": session_headers, + "session_cookies": session_cookies, + "server_cdm_type": server_cdm_type, + } + ) + + except (Exception, SystemExit) as e: + log.exception(f"Error getting tracks for title {sanitize_log(title_id)}") + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "session_tracks", "session_id": session_id, "title_id": title_id}, + debug_mode=debug_mode, + ) + + +async def session_segments_handler( + data: Dict[str, Any], session_id: str, request: Optional[web.Request] = None +) -> web.Response: + """Resolve segment URLs for selected tracks. + + The client calls this after selecting which tracks to download. + Returns segment URLs, init data, DRM info, and any headers/cookies + needed for CDN download. + """ + session = await _get_validated_session(session_id, request) + + track_ids = data.get("track_ids", []) + if not track_ids: + raise APIError(APIErrorCode.INVALID_INPUT, "Missing required parameter: track_ids") + + try: + result: Dict[str, Any] = {} + + for track_id in track_ids: + track = session.tracks.get(track_id) + if not track: + raise APIError( + APIErrorCode.TRACK_NOT_FOUND, + f"Track not found in session: {track_id}", + details={"track_id": track_id, "session_id": session_id}, + ) + + descriptor_name = track.descriptor.name if hasattr(track.descriptor, "name") else str(track.descriptor) + + track_info: Dict[str, Any] = { + "descriptor": descriptor_name, + "url": str(track.url) if track.url else None, + "drm": serialize_drm(track.drm) if hasattr(track, "drm") and track.drm else None, + } + + # Extract session headers/cookies for CDN access + service_session = session.service_instance.session + if hasattr(service_session, "headers"): + # Only include relevant headers, not all session headers + headers = dict(service_session.headers) if service_session.headers else {} + track_info["headers"] = headers + else: + track_info["headers"] = {} + + if hasattr(service_session, "cookies"): + cookie_dict = {} + for cookie in service_session.cookies: + if hasattr(cookie, "name") and hasattr(cookie, "value"): + cookie_dict[cookie.name] = cookie.value + elif isinstance(cookie, str): + pass # Skip non-standard cookie objects + track_info["cookies"] = cookie_dict + else: + track_info["cookies"] = {} + + # Include manifest-specific data for segment resolution + if hasattr(track, "data") and track.data: + track_data = {} + for key, val in track.data.items(): + if isinstance(val, dict): + # Convert non-serializable values + serializable = {} + for k, v in val.items(): + try: + import json + + json.dumps(v) + serializable[k] = v + except (TypeError, ValueError): + serializable[k] = str(v) + track_data[key] = serializable + else: + try: + import json + + json.dumps(val) + track_data[key] = val + except (TypeError, ValueError): + track_data[key] = str(val) + track_info["data"] = track_data + else: + track_info["data"] = {} + + result[track_id] = track_info + + return web.json_response({"tracks": result}) + + except APIError: + raise + except (Exception, SystemExit) as e: + log.exception("Error resolving segments") + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "session_segments", "session_id": session_id}, + debug_mode=debug_mode, + ) + + +class _CdmTypeStub: + """Lightweight CDM stub so ``is_playready_cdm()`` can detect CDM type. + + Used on the server when the client sends ``cdm_type`` but the server + does not need a full CDM (e.g. for cache key / device selection only). + """ + + def __init__(self, cdm_type: str) -> None: + self.is_playready = cdm_type == "playready" + + +def _resolve_server_cdm(service: str, profile: Optional[str], cdm_type: Optional[str]) -> Optional[Any]: + """Resolve CDM for the server context. + + Checks the server's own CDM config (``config.cdm[service]``) to + determine the CDM type without loading the full CDM object. This + ensures that when ``server_cdm: true`` is used, the server's CDM + determines device selection (e.g. PlayReady vs Widevine for AMZN). + + Falls back to a lightweight stub from *cdm_type* only if no server + CDM is configured for the service. + """ + from envied.core.config import config as app_config + + cdm_name = app_config.cdm.get(service) + if cdm_name: + if isinstance(cdm_name, dict): + lower_keys = {k.lower(): v for k, v in cdm_name.items()} + if {"widevine", "playready"} & lower_keys.keys(): + cdm_name = lower_keys.get("playready") or lower_keys.get("widevine") + else: + cdm_name = cdm_name.get("default") or next(iter(cdm_name.values()), None) + + if cdm_name and isinstance(cdm_name, str): + detected_type = _detect_cdm_type(cdm_name, app_config) + if detected_type: + return _CdmTypeStub(detected_type) + + if cdm_type: + return _CdmTypeStub(cdm_type) + return None + + +def _detect_cdm_type_for_service(service: str, app_config: Any) -> Optional[str]: + """Detect the CDM type configured for a service in config.cdm.""" + cdm_name = app_config.cdm.get(service) + if not cdm_name: + return None + if isinstance(cdm_name, dict): + lower_keys = {k.lower(): v for k, v in cdm_name.items()} + if {"widevine", "playready"} & lower_keys.keys(): + return "playready" if "playready" in lower_keys else "widevine" + cdm_name = cdm_name.get("default") or next(iter(cdm_name.values()), None) + if cdm_name and isinstance(cdm_name, str): + return _detect_cdm_type(cdm_name, app_config) + return None + + +def _detect_cdm_type(cdm_name: str, app_config: Any) -> Optional[str]: + """Detect CDM type (playready/widevine) from config without loading it. + + Checks remote_cdm entries and local file extensions to determine the type. + """ + for entry in getattr(app_config, "remote_cdm", []) or []: + if entry.get("name") == cdm_name: + device_type = str(entry.get("device_type", entry.get("Device Type", ""))).upper() + return "playready" if device_type == "PLAYREADY" else "widevine" + + prd_path = app_config.directories.prds / f"{cdm_name}.prd" + if not prd_path.is_file(): + prd_path = app_config.directories.wvds / f"{cdm_name}.prd" + if prd_path.is_file(): + return "playready" + + wvd_path = app_config.directories.wvds / f"{cdm_name}.wvd" + if wvd_path.is_file(): + return "widevine" + + return None + + +def _require_authenticated(session: Any) -> None: + """Raise if the session has not finished authenticating.""" + if session.auth_status == AuthStatus.FAILED: + raise APIError( + APIErrorCode.AUTH_FAILED, + f"Authentication failed: {session.auth_error or 'unknown error'}", + ) + if session.auth_status in (AuthStatus.AUTHENTICATING, AuthStatus.PENDING_INPUT): + raise APIError( + APIErrorCode.INVALID_INPUT, + f"Session authentication not complete (status: {session.auth_status.value})", + details={"auth_status": session.auth_status.value}, + ) + + +async def session_prompt_get_handler(session_id: str, request: Optional[web.Request] = None) -> web.Response: + """Poll for pending interactive prompts during authentication. + + Returns the current auth status and any pending prompt that the + remote client should display to the user. + """ + session = await _get_validated_session(session_id, request) + + if session.auth_status == AuthStatus.AUTHENTICATED: + return web.json_response({"status": "authenticated"}) + + if session.auth_status == AuthStatus.FAILED: + return web.json_response({"status": "failed", "error": session.auth_error or "unknown error"}) + + bridge = session.input_bridge + if bridge: + prompt = bridge.get_pending_prompt() + if prompt: + return web.json_response({"status": "pending_input", "prompt": prompt}) + + return web.json_response({"status": "authenticating"}) + + +async def session_prompt_post_handler( + data: Dict[str, Any], session_id: str, request: Optional[web.Request] = None +) -> web.Response: + """Submit a response to a pending interactive prompt. + + The remote client calls this after collecting user input (OTP code, + PIN, or device-code confirmation) to unblock the server auth thread. + """ + session = await _get_validated_session(session_id, request) + + response_text = data.get("response") + if response_text is None: + raise APIError(APIErrorCode.INVALID_INPUT, "Missing required field: response") + + bridge = session.input_bridge + if bridge is None or bridge.status != AuthStatus.PENDING_INPUT: + raise APIError(APIErrorCode.INVALID_INPUT, "No prompt pending for this session") + + bridge.submit_response(str(response_text)) + return web.json_response({"status": "accepted"}) + + +async def _get_validated_session(session_id: str, request: Optional[web.Request]) -> Any: + """Fetch a session and verify the requesting IP matches the creator.""" + from envied.core.api.session_store import get_session_store + + store = get_session_store() + session = await store.get(session_id) + if not session: + raise APIError( + APIErrorCode.SESSION_NOT_FOUND, + f"Session not found or expired: {session_id}", + details={"session_id": session_id}, + ) + if session.creator_ip and request and request.remote != session.creator_ip: + raise APIError( + APIErrorCode.FORBIDDEN, + "Session access denied", + ) + return session + + +def _resolve_handler_proxy(data: Dict[str, Any], normalized_service: str) -> tuple[Optional[str], list]: + """Resolve proxy and initialize providers from API request data. + + Handles explicit proxy param, provider:country format, and + client_region-based auto-proxy when server region differs. + """ + proxy_param = data.get("proxy") + no_proxy = data.get("no_proxy", False) + proxy_providers: list = [] + + if not no_proxy: + proxy_providers = initialize_proxy_providers() + + if proxy_param and not no_proxy: + try: + proxy_param = resolve_proxy(proxy_param, proxy_providers) + except ValueError as e: + raise APIError( + APIErrorCode.INVALID_PROXY, + f"Proxy error: {e}", + details={"proxy": data.get("proxy"), "service": normalized_service}, + ) + + client_region = data.get("client_region") + if not proxy_param and not no_proxy and client_region and proxy_providers: + try: + from envied.core.utils.ip_info import get_ip_info + + server_ip_info = get_ip_info(None, cached=True) + server_region = server_ip_info.get("country", "").lower() if server_ip_info else None + except Exception: + server_region = None + + if server_region and server_region == client_region.lower(): + log.info(f"Server already in client region '{client_region}', no proxy needed") + else: + try: + proxy_param = resolve_proxy(client_region, proxy_providers) + log.info(f"Using server proxy for client region '{client_region}'") + except ValueError: + log.debug(f"No server proxy available for client region '{client_region}'") + + return proxy_param, proxy_providers + + +def _find_title_for_track(track_id: str, session: Any) -> Any: + """Find the title object that owns a given track.""" + for t_id, tracks_dict in session.tracks_by_title.items(): + if track_id in tracks_dict: + return session.title_map.get(t_id) + if session.title_map: + return next(iter(session.title_map.values())) + return None + + +def _extract_pssh_from_track(track: Any, drm_type: str) -> Optional[str]: + """Extract PSSH base64 string from a track's DRM objects.""" + if not track.drm: + return None + pssh_b64 = None + for drm_obj in track.drm: + drm_class = drm_obj.__class__.__name__ + if drm_class == "Widevine" and hasattr(drm_obj, "_pssh") and drm_obj._pssh: + if hasattr(drm_obj._pssh, "dumps"): + pssh_b64 = drm_obj._pssh.dumps() + if drm_type == "widevine": + break + elif drm_class == "PlayReady": + if hasattr(drm_obj, "data") and drm_obj.data.get("pssh_b64"): + pssh_b64 = drm_obj.data["pssh_b64"] + if drm_type == "playready": + break + return pssh_b64 + + +def _ensure_track_drm(track: Any) -> None: + """Extract DRM from manifest data if track has none. + + Supports DASH (ContentProtection elements), HLS (EXT-X-KEY from + playlist fetch), and ISM (ProtectionHeader elements). + """ + if track.drm: + return + + # DASH: extract from ContentProtection elements + if track.data.get("dash"): + from envied.core.manifests import DASH as DASHManifest + + rep = track.data["dash"].get("representation") + ada = track.data["dash"].get("adaptation_set") + if rep is not None and ada is not None: + track.drm = DASHManifest.get_drm(rep.findall("ContentProtection") + ada.findall("ContentProtection")) + if track.drm: + return + + # HLS: fetch playlist and extract DRM from EXT-X-KEY + if track.data.get("hls") and track.url: + try: + import m3u8 + + from envied.core.drm import PlayReady, Widevine + from envied.core.manifests import HLS + + playlist = m3u8.load(track.url) + keys = [k for k in (playlist.keys or []) + (playlist.session_keys or []) if k is not None] + for key in keys: + try: + drm = HLS.get_drm(key) + if isinstance(drm, (Widevine, PlayReady)): + track.drm = [drm] + return + except Exception: + continue + except Exception: + pass + + # ISM: extract from ProtectionHeader elements + if track.data.get("ism"): + try: + from envied.core.manifests import ISM as ISMManifest + + stream_index = track.data["ism"].get("stream_index") + if stream_index is not None: + track.drm = ISMManifest.get_drm(stream_index) + except Exception: + pass + + +def _resolve_device_name(user_config: dict, drm_type: str, service_tag: str = "") -> str: + """Get the CDM device name, checking service-specific config.cdm first. + + Resolution order: + 1. config.cdm[service_tag] (service-specific CDM mapping) + 2. serve.users.{key}.devices / playready_devices (user device list) + """ + from envied.core.config import config as app_config + + cdm_name = app_config.cdm.get(service_tag) if service_tag else None + if isinstance(cdm_name, dict): + drm_key = {"widevine": "widevine", "playready": "playready"}.get(drm_type) + lower_keys = {k.lower(): v for k, v in cdm_name.items()} + cdm_name = lower_keys.get(drm_key) or lower_keys.get("default") or app_config.cdm.get("default") + if cdm_name and isinstance(cdm_name, str): + return cdm_name + + if drm_type == "playready": + device_name = (user_config.get("playready_devices") or [None])[0] + if not device_name: + raise APIError(APIErrorCode.INVALID_INPUT, "No PlayReady device configured for this API key") + else: + device_name = (user_config.get("devices") or [None])[0] + if not device_name: + raise APIError(APIErrorCode.INVALID_INPUT, "No Widevine device configured for this API key") + return device_name + + +def _load_server_vaults(service_name: str) -> Any: + """Load server vaults from config.key_vaults.""" + from envied.core.config import config as app_config + from envied.core.vaults import Vaults + + vaults = Vaults(service_name) + for vault_config in app_config.key_vaults: + cfg = vault_config.copy() + vault_type = cfg.pop("type", None) + if vault_type: + try: + vaults.load(vault_type, **cfg) + except (Exception, SystemExit) as e: + log.warning(f"Could not load vault '{vault_type}': {e}") + return vaults + + +def _check_vaults(kids: list, service_name: str) -> Optional[Dict[str, str]]: + """Check server vaults for existing keys matching all KIDs. + + Returns a KID:KEY dict if ALL KIDs are found, None otherwise. + """ + from uuid import UUID + + try: + vaults = _load_server_vaults(service_name) + if not vaults.vaults: + return None + keys: Dict[str, str] = {} + for kid in kids: + kid_uuid = kid if isinstance(kid, UUID) else UUID(hex=str(kid)) + content_key, vault_used = vaults.get_key(kid_uuid) + if content_key: + keys[kid_uuid.hex] = content_key + else: + return None + if keys: + log.info(f"Vault hit: {len(keys)} key(s) from server vaults, skipping CDM") + return keys + except Exception: + pass + return None + + +def _cache_to_vaults(keys: Dict[str, str], service_name: str) -> None: + """Cache newly obtained keys to server vaults.""" + from uuid import UUID + + try: + vaults = _load_server_vaults(service_name) + if not vaults.vaults: + return + + key_map = {UUID(hex=kid): key for kid, key in keys.items()} + cached = vaults.add_keys(key_map) + if cached: + log.info(f"Cached {cached} key(s) to {cached} server vault(s)") + except (Exception, SystemExit) as e: + log.warning(f"Failed to cache keys to vaults: {e}") + + +def _handle_single_server_cdm( + service: Any, + title: Any, + track: Any, + pssh_b64: Optional[str], + drm_type: str, + request: Optional[web.Request], +) -> Dict[str, str]: + """Handle single-track server_cdm licensing using the DRM class get_content_keys() flow.""" + import base64 + + from envied.core.cdm import load_cdm + from envied.core.config import config as app_config + + _ensure_track_drm(track) + + if not pssh_b64: + pssh_b64 = _extract_pssh_from_track(track, drm_type) + if not pssh_b64: + raise APIError(APIErrorCode.INVALID_INPUT, "No PSSH available for server_cdm licensing") + + api_key = request.headers.get("X-Secret-Key", "anonymous") if request else "anonymous" + user_config = app_config.serve.get("users", {}).get(api_key, {}) + + if drm_type == "playready": + from pyplayready.system.pssh import PSSH as PlayReadyPSSH + + from envied.core.drm import PlayReady + + pr_pssh = PlayReadyPSSH(base64.b64decode(pssh_b64)) + pr_drm = PlayReady(pssh=pr_pssh, pssh_b64=pssh_b64) + + vault_keys = _check_vaults(pr_drm.kids, service.__class__.__name__) + if vault_keys: + return vault_keys + + device_name = _resolve_device_name(user_config, drm_type, service.__class__.__name__) + cdm = load_cdm(device_name, service_name=service.__class__.__name__) + pr_drm.get_content_keys( + cdm=cdm, + certificate=lambda challenge, **_: None, + licence=lambda challenge, **_: service.get_playready_license(challenge=challenge, title=title, track=track), + ) + keys = {kid.hex: key for kid, key in pr_drm.content_keys.items()} + elif drm_type == "widevine": + from pywidevine.pssh import PSSH as WvPSSH + + from envied.core.drm import Widevine + + wv_pssh = WvPSSH(pssh_b64) + wv_drm = Widevine(pssh=wv_pssh) + + vault_keys = _check_vaults(wv_drm.kids, service.__class__.__name__) + if vault_keys: + return vault_keys + + device_name = _resolve_device_name(user_config, drm_type, service.__class__.__name__) + cdm = load_cdm(device_name, service_name=service.__class__.__name__) + wv_drm.get_content_keys( + cdm=cdm, + certificate=lambda challenge, **_: service.get_widevine_service_certificate( + challenge=challenge, title=title, track=track + ), + licence=lambda challenge, **_: service.get_widevine_license(challenge=challenge, title=title, track=track), + ) + keys = {kid.hex: key for kid, key in wv_drm.content_keys.items()} + else: + raise APIError( + APIErrorCode.INVALID_PARAMETERS, + f"Unsupported DRM type for server_cdm: {drm_type}", + ) + + if not keys: + raise APIError(APIErrorCode.NO_CONTENT, "Server CDM returned no content keys") + + _cache_to_vaults(keys, service.__class__.__name__) + return keys + + +def _handle_proxy_license( + service: Any, + title: Any, + track: Any, + challenge_b64: Optional[str], + drm_type: str, +) -> web.Response: + """Forward a client CDM challenge to the service license endpoint.""" + import base64 + + if not challenge_b64: + raise APIError(APIErrorCode.INVALID_INPUT, "Missing required parameter: challenge") + challenge_bytes = base64.b64decode(challenge_b64) + + if drm_type == "widevine": + license_response = service.get_widevine_license(challenge=challenge_bytes, title=title, track=track) + elif drm_type == "playready": + license_response = service.get_playready_license(challenge=challenge_bytes, title=title, track=track) + else: + raise APIError( + APIErrorCode.INVALID_PARAMETERS, + f"Unsupported DRM type: {drm_type}", + details={"drm_type": drm_type, "supported": ["widevine", "playready"]}, + ) + + if isinstance(license_response, str): + license_response = license_response.encode("utf-8") + + return web.json_response({"license": base64.b64encode(license_response).decode("ascii")}) + + +async def session_license_handler( + data: Dict[str, Any], session_id: str, request: Optional[web.Request] = None +) -> web.Response: + """Handle DRM licensing in proxy or server_cdm mode. + + Proxy mode (default): forwards client CDM challenge to the service's + license endpoint, returns raw license bytes for client-side parsing. + + Server-CDM mode (mode="server_cdm"): server uses its own CDM to generate + the challenge, obtain the license, and extract KID:KEY pairs. Supports + batch (track_ids list) and single-track requests. + """ + import base64 + + session = await _get_validated_session(session_id, request) + + track_id = data.get("track_id") + track_ids = data.get("track_ids") + challenge_b64 = data.get("challenge") + drm_type = data.get("drm_type", "widevine") + mode = data.get("mode", "proxy") + + if mode == "server_cdm" and track_ids: + from envied.core.config import config as app_config + + api_key = request.headers.get("X-Secret-Key", "anonymous") if request else "anonymous" + user_config = app_config.serve.get("users", {}).get(api_key, {}) + service = session.service_instance + has_wv_device = bool(user_config.get("devices")) + has_pr_device = bool(user_config.get("playready_devices")) + + service_tag = session.service_tag + config_cdm_type = _detect_cdm_type_for_service(service_tag, app_config) + + all_keys: Dict[str, Dict[str, str]] = {} + seen_pssh: set[str] = set() + actual_drm_type: Optional[str] = None + + for tid in track_ids: + track = session.tracks.get(tid) + if not track: + continue + + _ensure_track_drm(track) + if not track.drm: + continue + + title = _find_title_for_track(tid, session) + + track_drm_type = None + pssh_str = None + if config_cdm_type == "playready": + pssh_str = _extract_pssh_from_track(track, "playready") + if pssh_str: + track_drm_type = "playready" + if not pssh_str: + pssh_str = _extract_pssh_from_track(track, "widevine") + if pssh_str: + track_drm_type = "widevine" + elif config_cdm_type == "widevine": + pssh_str = _extract_pssh_from_track(track, "widevine") + if pssh_str: + track_drm_type = "widevine" + if not pssh_str: + pssh_str = _extract_pssh_from_track(track, "playready") + if pssh_str: + track_drm_type = "playready" + else: + if has_wv_device: + pssh_str = _extract_pssh_from_track(track, "widevine") + if pssh_str: + track_drm_type = "widevine" + if not pssh_str and has_pr_device: + pssh_str = _extract_pssh_from_track(track, "playready") + if pssh_str: + track_drm_type = "playready" + + if not pssh_str or not track_drm_type: + continue + + if pssh_str in seen_pssh: + for prev_keys in all_keys.values(): + if prev_keys: + all_keys[tid] = prev_keys + break + continue + seen_pssh.add(pssh_str) + + try: + keys = _handle_single_server_cdm(service, title, track, pssh_str, track_drm_type, request) + if keys: + all_keys[tid] = keys + if track_drm_type: + actual_drm_type = track_drm_type + except SystemExit: + log.warning(f"Service exited while resolving keys for track {tid[:12]}, skipping") + except (Exception, SystemExit) as e: + log.warning(f"Failed to resolve keys for track {tid[:12]}: {e}") + + response: Dict[str, Any] = {"keys": all_keys} + if actual_drm_type: + response["drm_type"] = actual_drm_type + return web.json_response(response) + + if not track_id: + raise APIError(APIErrorCode.INVALID_INPUT, "Missing required parameter: track_id") + + track = session.tracks.get(track_id) + if not track: + raise APIError( + APIErrorCode.TRACK_NOT_FOUND, + f"Track not found in session: {track_id}", + details={"track_id": track_id, "session_id": session_id}, + ) + + try: + title = _find_title_for_track(track_id, session) + service = session.service_instance + + pssh_b64 = data.get("pssh") + if pssh_b64: + if not track.drm: + track.drm = [] + if drm_type == "playready": + track.pr_pssh = pssh_b64 + from pyplayready.system.pssh import PSSH as PlayReadyPSSH + + from envied.core.drm import PlayReady + + pr_pssh = PlayReadyPSSH(base64.b64decode(pssh_b64)) + pr_drm = PlayReady(pssh=pr_pssh, pssh_b64=pssh_b64) + track.drm.append(pr_drm) + elif drm_type == "widevine": + from pywidevine.pssh import PSSH as WidevinePSSH + + from envied.core.drm import Widevine + + wv_pssh = WidevinePSSH(pssh_b64) + wv_drm = Widevine(pssh=wv_pssh) + track.drm.append(wv_drm) + + if mode == "server_cdm": + keys = _handle_single_server_cdm(service, title, track, pssh_b64, drm_type, request) + log.info(f"Server CDM resolved {len(keys)} key(s) for track {track_id[:12]}") + return web.json_response({"keys": keys}) + + return _handle_proxy_license(service, title, track, challenge_b64, drm_type) + + except APIError: + raise + except SystemExit: + raise APIError(APIErrorCode.SERVICE_ERROR, "Service exited during license request") + except (Exception, SystemExit) as e: + log.exception(f"Error proxying license for track {track_id}") + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={ + "operation": "session_license", + "session_id": session_id, + "track_id": track_id, + "drm_type": drm_type, + }, + debug_mode=debug_mode, + ) + + +async def session_info_handler(session_id: str, request: Optional[web.Request] = None) -> web.Response: + """Check session validity and get session info.""" + session = await _get_validated_session(session_id, request) + + from envied.core.api.session_store import get_session_store + + return web.json_response( + { + "session_id": session.session_id, + "service": session.service_tag, + "valid": True, + "expires_in": get_session_store()._ttl, + "track_count": len(session.tracks), + "title_count": len(session.title_map), + } + ) + + +async def session_delete_handler(session_id: str, request: Optional[web.Request] = None) -> web.Response: + """Delete a session, return updated cache files, and clean up server-side data.""" + import base64 + import zlib + + from envied.core.api.session_store import get_session_store + from envied.core.config import config as app_config + + session = await _get_validated_session(session_id, request) + store = get_session_store() + + if session.input_bridge: + session.input_bridge.cancel() + + cache_tag = session.cache_tag + cache_data: Dict[str, str] = {} + if cache_tag: + cache_dir = app_config.directories.cache / cache_tag + if cache_dir.is_dir(): + for f in cache_dir.glob("*.json"): + if not f.stem.startswith("titles_"): + try: + cache_data[f.stem] = base64.b64encode(zlib.compress(f.read_bytes())).decode("ascii") + except Exception: + pass + + await store.delete(session_id) + + response: Dict[str, Any] = {"status": "ok"} + if cache_data: + response["cache"] = cache_data + return web.json_response(response) diff --git a/packages/envied/src/envied/core/api/input_bridge.py b/packages/envied/src/envied/core/api/input_bridge.py new file mode 100644 index 0000000..fd3e3e9 --- /dev/null +++ b/packages/envied/src/envied/core/api/input_bridge.py @@ -0,0 +1,139 @@ +"""Thread-safe bridge for interactive input during remote authentication. + +When a service calls ``request_input()`` during ``authenticate()`` on the +server, the InputBridge pauses the auth thread and exposes the prompt to +the HTTP layer so a remote client can poll for it, collect the user's +response, and submit it back. The auth thread then resumes with the +response value. +""" + +from __future__ import annotations + +import threading +import time +from dataclasses import dataclass, field +from enum import Enum +from typing import Optional + + +class AuthStatus(Enum): + """Authentication lifecycle states for a remote session.""" + + AUTHENTICATING = "authenticating" + PENDING_INPUT = "pending_input" + AUTHENTICATED = "authenticated" + FAILED = "failed" + + +class BridgeCancelledError(Exception): + """Raised when the bridge is cancelled (client disconnected, session deleted).""" + + +@dataclass +class InputBridge: + """Thread-safe bridge between a sync auth thread and the async HTTP layer. + + The auth thread calls :meth:`request_input` which blocks until the + remote client submits a response via the HTTP prompt endpoints. + """ + + _prompt: Optional[str] = field(default=None, init=False, repr=False) + _response: Optional[str] = field(default=None, init=False, repr=False) + _status: AuthStatus = field(default=AuthStatus.AUTHENTICATING, init=False) + _error: Optional[str] = field(default=None, init=False, repr=False) + _cancelled: bool = field(default=False, init=False, repr=False) + _response_ready: threading.Event = field(default_factory=threading.Event, init=False, repr=False) + _lock: threading.Lock = field(default_factory=threading.Lock, init=False, repr=False) + + def request_input(self, prompt: str, timeout: float = 600.0) -> str: + """Block until the remote client submits a response for *prompt*. + + Args: + prompt: The message to display to the remote user. + timeout: Maximum seconds to wait for a response. + + Returns: + The string response from the remote client. + + Raises: + TimeoutError: If no response is received within *timeout*. + BridgeCancelledError: If the bridge was cancelled. + """ + with self._lock: + if self._cancelled: + raise BridgeCancelledError("Session was cancelled") + self._prompt = prompt + self._response = None + self._status = AuthStatus.PENDING_INPUT + self._response_ready.clear() + + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if self._response_ready.wait(timeout=0.5): + break + with self._lock: + if self._cancelled: + raise BridgeCancelledError("Session was cancelled") + else: + with self._lock: + self._status = AuthStatus.FAILED + self._error = "Input request timed out waiting for client response" + raise TimeoutError(f"No client response for prompt within {timeout}s") + + with self._lock: + if self._cancelled: + raise BridgeCancelledError("Session was cancelled") + response = self._response or "" + self._prompt = None + self._response = None + self._status = AuthStatus.AUTHENTICATING + return response + + def get_pending_prompt(self) -> Optional[str]: + """Return the current prompt if the auth thread is waiting for input.""" + with self._lock: + if self._status == AuthStatus.PENDING_INPUT: + return self._prompt + return None + + def submit_response(self, response: str) -> bool: + """Deliver the client's response and unblock the auth thread. + + Returns: + ``True`` if a prompt was pending and the response was accepted, + ``False`` otherwise. + """ + with self._lock: + if self._status != AuthStatus.PENDING_INPUT: + return False + self._response = response + self._response_ready.set() + return True + + def cancel(self) -> None: + """Cancel the bridge, unblocking any waiting auth thread.""" + with self._lock: + self._cancelled = True + self._status = AuthStatus.FAILED + self._error = "Session cancelled" + self._response_ready.set() + + @property + def status(self) -> AuthStatus: + with self._lock: + return self._status + + @status.setter + def status(self, value: AuthStatus) -> None: + with self._lock: + self._status = value + + @property + def error(self) -> Optional[str]: + with self._lock: + return self._error + + @error.setter + def error(self, value: Optional[str]) -> None: + with self._lock: + self._error = value diff --git a/packages/envied/src/envied/core/api/routes.py b/packages/envied/src/envied/core/api/routes.py index 704847b..a20c22e 100644 --- a/packages/envied/src/envied/core/api/routes.py +++ b/packages/envied/src/envied/core/api/routes.py @@ -1,14 +1,18 @@ import logging import re +import click from aiohttp import web from aiohttp_swagger3 import SwaggerDocs, SwaggerInfo, SwaggerUiSettings from envied.core import __version__ from envied.core.api.errors import APIError, APIErrorCode, build_error_response, handle_api_exception -from envied.core.api.handlers import (cancel_download_job_handler, download_handler, get_download_job_handler, - list_download_jobs_handler, list_titles_handler, list_tracks_handler, - search_handler) +from envied.core.api.handlers import (cancel_download_job_handler, download_handler, get_allowed_services, + get_download_job_handler, list_download_jobs_handler, list_titles_handler, + list_tracks_handler, search_handler, session_create_handler, + session_delete_handler, session_info_handler, session_license_handler, + session_prompt_get_handler, session_prompt_post_handler, + session_segments_handler, session_titles_handler, session_tracks_handler) from envied.core.services import Services from envied.core.update_checker import UpdateChecker @@ -25,7 +29,7 @@ async def cors_middleware(request: web.Request, handler): # Add CORS headers response.headers["Access-Control-Allow-Origin"] = "*" response.headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, DELETE, OPTIONS" - response.headers["Access-Control-Allow-Headers"] = "Content-Type, X-API-Key, Authorization" + response.headers["Access-Control-Allow-Headers"] = "Content-Type, X-Secret-Key, Authorization" response.headers["Access-Control-Max-Age"] = "3600" return response @@ -151,6 +155,9 @@ async def services(request: web.Request) -> web.Response: """ try: service_tags = Services.get_tags() + allowed = get_allowed_services(request) + if allowed is not None: + service_tags = [t for t in service_tags if t in allowed] services_info = [] for tag in service_tags: @@ -185,6 +192,32 @@ async def services(request: web.Request) -> web.Response: if hasattr(service_module, "cli") and hasattr(service_module.cli, "short_help"): service_data["url"] = service_module.cli.short_help + if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): + cli_params = [] + for param in service_module.cli.params: + param_info: dict = {"name": getattr(param, "name", None)} + if isinstance(param, click.Argument): + param_info["kind"] = "argument" + param_info["required"] = param.required + else: + param_info["kind"] = "option" + param_info["opts"] = list(param.opts) if hasattr(param, "opts") else [] + param_info["is_flag"] = getattr(param, "is_flag", False) + default = param.default + if default is None: + pass + elif callable(default) or type(default).__name__ == "Sentinel": + default = None + elif hasattr(default, "name"): + default = default.name + elif not isinstance(default, (str, int, float, bool, list)): + default = str(default) + param_info["default"] = default + param_info["help"] = getattr(param, "help", None) + param_info["type"] = param.type.name if hasattr(param.type, "name") else str(param.type) + cli_params.append(param_info) + service_data["cli_params"] = cli_params + if service_module.__doc__: service_data["help"] = service_module.__doc__.strip() @@ -218,7 +251,7 @@ async def search(request: web.Request) -> web.Response: properties: service: type: string - description: Service tag (e.g., NF, AMZN, ATV) + description: Service tag query: type: string description: Search query string @@ -597,8 +630,10 @@ async def download(request: web.Request) -> web.Response: type: boolean description: Download audio description tracks (default - false) slow: - type: boolean - description: Add 60-120s delay between downloads (default - false) + oneOf: + - type: boolean + - type: string + description: Add randomized delay between downloads. `true` for default 60-120s, or `"MIN-MAX"` string (e.g., `"20-40"`). Min must be >= 20 (default - null) split_audio: type: boolean description: Create separate output files per audio codec instead of merging all audio (default - null) @@ -606,8 +641,8 @@ async def download(request: web.Request) -> web.Response: type: boolean description: Skip downloading, only retrieve decryption keys (default - false) export: - type: string - description: Path to export decryption keys as JSON (default - None) + type: boolean + description: Export manifest, track URLs, keys, and subtitles to JSON in the exports directory (default - false) cdm_only: type: boolean description: Only use CDM for key retrieval (true) or only vaults (false) (default - None) @@ -617,6 +652,9 @@ async def download(request: web.Request) -> web.Response: no_proxy: type: boolean description: Force disable all proxy use (default - false) + no_proxy_download: + type: boolean + description: Bypass proxy for segment downloads only. Manifest, license, and auth still use proxy (default - false) tag: type: string description: Set the group tag to be used (default - None) @@ -647,6 +685,9 @@ async def download(request: web.Request) -> web.Response: best_available: type: boolean description: Continue with best available if requested quality unavailable (default - false) + worst: + type: boolean + description: Select the lowest bitrate track within the specified quality. Requires `quality` (default - false) repack: type: boolean description: Add REPACK tag to the output filename (default - false) @@ -836,8 +877,429 @@ async def cancel_download_job(request: web.Request) -> web.Response: return build_error_response(e, debug_mode) -def setup_routes(app: web.Application) -> None: - """Setup all API routes.""" +async def session_create(request: web.Request) -> web.Response: + """ + Create a remote-dl session. + --- + summary: Create session + description: Authenticate with a service, get titles, tracks, and chapters in one call + requestBody: + required: true + content: + application/json: + schema: + type: object + additionalProperties: true + required: + - service + - title_id + properties: + service: + type: string + title_id: + type: string + credentials: + type: object + additionalProperties: true + cookies: + type: string + proxy: + type: string + no_proxy: + type: boolean + profile: + type: string + cache: + type: object + additionalProperties: true + responses: + '200': + description: Session created with titles, tracks, and chapters + '400': + description: Invalid request + '401': + description: Authentication failed + """ + try: + data = await request.json() + except Exception as e: + return build_error_response( + APIError(APIErrorCode.INVALID_INPUT, "Invalid JSON request body", details={"error": str(e)}), + request.app.get("debug_api", False), + ) + try: + return await session_create_handler(data, request) + except APIError as e: + return build_error_response(e, request.app.get("debug_api", False)) + except Exception as e: + log.exception("Error in session create") + return handle_api_exception( + e, context={"operation": "session_create"}, debug_mode=request.app.get("debug_api", False) + ) + + +async def session_titles(request: web.Request) -> web.Response: + """ + Get titles for an authenticated session. + --- + summary: Get titles + description: Fetch titles from the authenticated service session + parameters: + - name: session_id + in: path + required: true + schema: + type: string + responses: + '200': + description: List of titles + '404': + description: Session not found + """ + session_id = request.match_info["session_id"] + try: + return await session_titles_handler(session_id, request) + except APIError as e: + return build_error_response(e, request.app.get("debug_api", False)) + except Exception as e: + log.exception("Error in session titles") + return handle_api_exception( + e, context={"operation": "session_titles"}, debug_mode=request.app.get("debug_api", False) + ) + + +async def session_tracks(request: web.Request) -> web.Response: + """ + Get tracks and chapters for a specific title. + --- + summary: Get tracks + description: Fetch tracks and chapters for a title in the session + parameters: + - name: session_id + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - title_id + properties: + title_id: + type: string + description: ID of the title to get tracks for + responses: + '200': + description: Tracks and chapters for the title + '404': + description: Session or title not found + """ + session_id = request.match_info["session_id"] + try: + data = await request.json() + except Exception as e: + return build_error_response( + APIError(APIErrorCode.INVALID_INPUT, "Invalid JSON request body", details={"error": str(e)}), + request.app.get("debug_api", False), + ) + try: + return await session_tracks_handler(data, session_id, request) + except APIError as e: + return build_error_response(e, request.app.get("debug_api", False)) + except Exception as e: + log.exception("Error in session tracks") + return handle_api_exception( + e, context={"operation": "session_tracks"}, debug_mode=request.app.get("debug_api", False) + ) + + +async def session_segments(request: web.Request) -> web.Response: + """ + Resolve segment URLs for selected tracks. + --- + summary: Resolve segments + description: Get download URLs, DRM info, and headers for selected tracks + parameters: + - name: session_id + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - track_ids + properties: + track_ids: + type: array + items: + type: string + description: List of track IDs to resolve + responses: + '200': + description: Segment URLs and DRM info for each track + '404': + description: Session or track not found + """ + session_id = request.match_info["session_id"] + try: + data = await request.json() + except Exception as e: + return build_error_response( + APIError(APIErrorCode.INVALID_INPUT, "Invalid JSON request body", details={"error": str(e)}), + request.app.get("debug_api", False), + ) + try: + return await session_segments_handler(data, session_id, request) + except APIError as e: + return build_error_response(e, request.app.get("debug_api", False)) + except Exception as e: + log.exception("Error in session segments") + return handle_api_exception( + e, context={"operation": "session_segments"}, debug_mode=request.app.get("debug_api", False) + ) + + +async def session_license(request: web.Request) -> web.Response: + """ + Proxy DRM license through authenticated service. + --- + summary: Proxy license + description: Forward a CDM challenge to the service's license endpoint + parameters: + - name: session_id + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - track_id + - challenge + properties: + track_id: + type: string + description: Track ID this license is for + challenge: + type: string + description: Base64-encoded CDM challenge + drm_type: + type: string + enum: [widevine, playready] + description: DRM type (default widevine) + responses: + '200': + description: License response + '404': + description: Session or track not found + """ + session_id = request.match_info["session_id"] + try: + data = await request.json() + except Exception as e: + return build_error_response( + APIError(APIErrorCode.INVALID_INPUT, "Invalid JSON request body", details={"error": str(e)}), + request.app.get("debug_api", False), + ) + try: + return await session_license_handler(data, session_id, request) + except APIError as e: + return build_error_response(e, request.app.get("debug_api", False)) + except Exception as e: + log.exception("Error in session license") + return handle_api_exception( + e, context={"operation": "session_license"}, debug_mode=request.app.get("debug_api", False) + ) + + +async def session_info(request: web.Request) -> web.Response: + """ + Get session info. + --- + summary: Session info + description: Check session validity and get metadata + parameters: + - name: session_id + in: path + required: true + schema: + type: string + responses: + '200': + description: Session info + '404': + description: Session not found + """ + session_id = request.match_info["session_id"] + try: + return await session_info_handler(session_id, request) + except APIError as e: + return build_error_response(e, request.app.get("debug_api", False)) + + +async def session_delete(request: web.Request) -> web.Response: + """ + Delete a session. + --- + summary: Delete session + description: Clean up a remote-dl session + parameters: + - name: session_id + in: path + required: true + schema: + type: string + responses: + '200': + description: Session deleted + '404': + description: Session not found + """ + session_id = request.match_info["session_id"] + try: + return await session_delete_handler(session_id, request) + except APIError as e: + return build_error_response(e, request.app.get("debug_api", False)) + + +async def session_prompt_get(request: web.Request) -> web.Response: + """ + Poll for pending interactive prompts during authentication. + --- + summary: Get auth prompt + description: Poll for pending interactive prompts (OTP, device code, PIN) during session authentication + parameters: + - name: session_id + in: path + required: true + schema: + type: string + responses: + '200': + description: Auth status and optional prompt + content: + application/json: + schema: + type: object + properties: + status: + type: string + enum: [authenticating, pending_input, authenticated, failed] + prompt: + type: string + description: Prompt to display to the user (only when status is pending_input) + error: + type: string + description: Error message (only when status is failed) + '404': + description: Session not found + """ + session_id = request.match_info["session_id"] + try: + return await session_prompt_get_handler(session_id, request) + except APIError as e: + return build_error_response(e, request.app.get("debug_api", False)) + except Exception as e: + log.exception("Error in session prompt get") + return handle_api_exception( + e, context={"operation": "session_prompt_get"}, debug_mode=request.app.get("debug_api", False) + ) + + +async def session_prompt_submit(request: web.Request) -> web.Response: + """ + Submit a response to a pending interactive prompt. + --- + summary: Submit prompt response + description: Submit user input (OTP code, PIN, device code confirmation) to unblock server authentication + parameters: + - name: session_id + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - response + properties: + response: + type: string + description: User's response to the prompt + responses: + '200': + description: Response accepted + content: + application/json: + schema: + type: object + properties: + status: + type: string + example: accepted + '400': + description: No prompt pending or invalid request + '404': + description: Session not found + """ + session_id = request.match_info["session_id"] + try: + data = await request.json() + except Exception as e: + return build_error_response( + APIError(APIErrorCode.INVALID_INPUT, "Invalid JSON request body", details={"error": str(e)}), + request.app.get("debug_api", False), + ) + try: + return await session_prompt_post_handler(data, session_id, request) + except APIError as e: + return build_error_response(e, request.app.get("debug_api", False)) + except Exception as e: + log.exception("Error in session prompt submit") + return handle_api_exception( + e, context={"operation": "session_prompt_submit"}, debug_mode=request.app.get("debug_api", False) + ) + + +def _setup_remote_session_routes(app: web.Application) -> None: + """Setup remote-DL session endpoints only.""" + app.router.add_get("/api/health", health) + app.router.add_get("/api/services", services) + app.router.add_post("/api/search", search) + app.router.add_post("/api/session/create", session_create) + app.router.add_get("/api/session/{session_id}/titles", session_titles) + app.router.add_post("/api/session/{session_id}/tracks", session_tracks) + app.router.add_post("/api/session/{session_id}/segments", session_segments) + app.router.add_post("/api/session/{session_id}/license", session_license) + app.router.add_get("/api/session/{session_id}/prompt", session_prompt_get) + app.router.add_post("/api/session/{session_id}/prompt", session_prompt_submit) + app.router.add_get("/api/session/{session_id}", session_info) + app.router.add_delete("/api/session/{session_id}", session_delete) + + +def setup_routes(app: web.Application, remote_only: bool = False) -> None: + """Setup API routes. When remote_only=True, only expose remote session endpoints.""" + if remote_only: + _setup_remote_session_routes(app) + return + app.router.add_get("/api/health", health) app.router.add_get("/api/services", services) app.router.add_post("/api/search", search) @@ -848,6 +1310,17 @@ def setup_routes(app: web.Application) -> None: app.router.add_get("/api/download/jobs/{job_id}", download_job_detail) app.router.add_delete("/api/download/jobs/{job_id}", cancel_download_job) + # Remote-DL session endpoints + app.router.add_post("/api/session/create", session_create) + app.router.add_get("/api/session/{session_id}/titles", session_titles) + app.router.add_post("/api/session/{session_id}/tracks", session_tracks) + app.router.add_post("/api/session/{session_id}/segments", session_segments) + app.router.add_post("/api/session/{session_id}/license", session_license) + app.router.add_get("/api/session/{session_id}/prompt", session_prompt_get) + app.router.add_post("/api/session/{session_id}/prompt", session_prompt_submit) + app.router.add_get("/api/session/{session_id}", session_info) + app.router.add_delete("/api/session/{session_id}", session_delete) + def setup_swagger(app: web.Application) -> None: """Setup Swagger UI documentation.""" @@ -873,5 +1346,15 @@ def setup_swagger(app: web.Application) -> None: web.get("/api/download/jobs", download_jobs), web.get("/api/download/jobs/{job_id}", download_job_detail), web.delete("/api/download/jobs/{job_id}", cancel_download_job), + # Remote-DL session endpoints + web.post("/api/session/create", session_create), + web.get("/api/session/{session_id}/titles", session_titles), + web.post("/api/session/{session_id}/tracks", session_tracks), + web.post("/api/session/{session_id}/segments", session_segments), + web.post("/api/session/{session_id}/license", session_license), + web.get("/api/session/{session_id}/prompt", session_prompt_get), + web.post("/api/session/{session_id}/prompt", session_prompt_submit), + web.get("/api/session/{session_id}", session_info), + web.delete("/api/session/{session_id}", session_delete), ] ) diff --git a/packages/envied/src/envied/core/api/session_store.py b/packages/envied/src/envied/core/api/session_store.py new file mode 100644 index 0000000..e57dc8d --- /dev/null +++ b/packages/envied/src/envied/core/api/session_store.py @@ -0,0 +1,209 @@ +"""Server-side session store for remote-dl client-server architecture. + +Maintains authenticated service instances between API calls so that +a client can authenticate once and then make multiple requests (list tracks, +resolve segments, proxy license) using the same session. +""" + +from __future__ import annotations + +import asyncio +import logging +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +from envied.core.api.input_bridge import AuthStatus, InputBridge +from envied.core.config import config +from envied.core.tracks import Track + +log = logging.getLogger("api.session") + + +@dataclass +class SessionEntry: + """A single authenticated session with a service.""" + + session_id: str + service_tag: str + service_instance: Any # Service instance (authenticated) + titles: Any = None # Titles_T from get_titles() + title_map: Dict[str, Any] = field(default_factory=dict) # title_id -> Title object + tracks: Dict[str, Track] = field(default_factory=dict) # track_id -> Track object + tracks_by_title: Dict[str, Dict[str, Track]] = field(default_factory=dict) # title_key -> {track_id -> Track} + chapters_by_title: Dict[str, List[Any]] = field(default_factory=dict) # title_key -> [Chapter] + creator_ip: Optional[str] = None + cache_tag: Optional[str] = None # per-session cache directory tag + input_bridge: Optional[InputBridge] = None + auth_status: AuthStatus = AuthStatus.AUTHENTICATED + auth_error: Optional[str] = None + created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + last_accessed: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + + def touch(self) -> None: + """Update last_accessed timestamp.""" + self.last_accessed = datetime.now(timezone.utc) + + +class SessionStore: + """Thread-safe session store with TTL-based expiration.""" + + def __init__(self) -> None: + self._sessions: Dict[str, SessionEntry] = {} + self._lock = asyncio.Lock() + self._cleanup_task: Optional[asyncio.Task] = None + + @property + def _ttl(self) -> int: + """Session TTL in seconds from config.""" + return config.serve.get("session_ttl", 300) # 5 min default + + @property + def _max_sessions(self) -> int: + """Max concurrent sessions from config.""" + return config.serve.get("max_sessions", 100) + + async def create( + self, + service_tag: str, + service_instance: Any, + session_id: Optional[str] = None, + ) -> SessionEntry: + """Create a new session with an authenticated service instance.""" + async with self._lock: + if len(self._sessions) >= self._max_sessions: + oldest_id = min(self._sessions, key=lambda k: self._sessions[k].last_accessed) + log.warning(f"Max sessions reached ({self._max_sessions}), evicting oldest: {oldest_id}") + del self._sessions[oldest_id] + + session_id = session_id or str(uuid.uuid4()) + entry = SessionEntry( + session_id=session_id, + service_tag=service_tag, + service_instance=service_instance, + ) + self._sessions[session_id] = entry + log.info(f"Created session {session_id} for service {service_tag}") + return entry + + async def get(self, session_id: str) -> Optional[SessionEntry]: + """Get a session by ID, returns None if not found or expired.""" + async with self._lock: + entry = self._sessions.get(session_id) + if entry is None: + return None + + if entry.auth_status not in (AuthStatus.AUTHENTICATING, AuthStatus.PENDING_INPUT): + elapsed = (datetime.now(timezone.utc) - entry.last_accessed).total_seconds() + if elapsed > self._ttl: + log.info(f"Session {session_id} expired (elapsed={elapsed:.0f}s, ttl={self._ttl}s)") + del self._sessions[session_id] + return None + + entry.touch() + return entry + + async def delete(self, session_id: str) -> bool: + """Delete a session. Returns True if it existed.""" + async with self._lock: + entry = self._sessions.pop(session_id, None) + if entry: + if entry.input_bridge: + entry.input_bridge.cancel() + self._cleanup_cache_dir(entry.cache_tag) + log.info(f"Deleted session {session_id}") + return True + return False + + async def cleanup_expired(self) -> int: + """Remove all expired sessions. Returns count of removed sessions.""" + async with self._lock: + now = datetime.now(timezone.utc) + expired = [] + for sid, entry in self._sessions.items(): + elapsed = (now - entry.last_accessed).total_seconds() + if entry.auth_status in (AuthStatus.AUTHENTICATING, AuthStatus.PENDING_INPUT): + if elapsed > 600: + expired.append(sid) + elif elapsed > self._ttl: + expired.append(sid) + for sid in expired: + entry = self._sessions.pop(sid) + if entry.input_bridge: + entry.input_bridge.cancel() + self._cleanup_cache_dir(entry.cache_tag) + if expired: + log.info(f"Cleaned up {len(expired)} expired sessions") + return len(expired) + + async def start_cleanup_loop(self) -> None: + """Start periodic cleanup of expired sessions.""" + if self._cleanup_task is not None: + return + + async def _loop() -> None: + while True: + await asyncio.sleep(60) # Check every minute + try: + await self.cleanup_expired() + except Exception: + log.exception("Error during session cleanup") + + self._cleanup_task = asyncio.create_task(_loop()) + log.info("Session cleanup loop started") + + async def stop_cleanup_loop(self) -> None: + """Stop the periodic cleanup task.""" + if self._cleanup_task is not None: + self._cleanup_task.cancel() + self._cleanup_task = None + + async def cancel_all_bridges(self) -> None: + """Cancel all active input bridges (called on server shutdown).""" + async with self._lock: + for entry in self._sessions.values(): + if entry.input_bridge: + entry.input_bridge.cancel() + count = len(self._sessions) + if count: + log.info(f"Cancelled bridges for {count} active session(s)") + + @staticmethod + def _cleanup_cache_dir(cache_tag: Optional[str]) -> None: + """Remove session cache directory and empty parents.""" + if not cache_tag: + return + import shutil + + cache_dir = config.directories.cache / cache_tag + if cache_dir.is_dir(): + try: + shutil.rmtree(cache_dir) + except Exception as e: + log.warning(f"Failed to remove session cache {cache_dir}: {e}") + for parent in cache_dir.parents: + if parent == config.directories.cache: + break + try: + if parent.is_dir() and not any(parent.iterdir()): + parent.rmdir() + except Exception: + break + + @property + def session_count(self) -> int: + """Number of active sessions.""" + return len(self._sessions) + + +# Singleton instance +_session_store: Optional[SessionStore] = None + + +def get_session_store() -> SessionStore: + """Get or create the global session store singleton.""" + global _session_store + if _session_store is None: + _session_store = SessionStore() + return _session_store diff --git a/packages/envied/src/envied/core/binaries.py b/packages/envied/src/envied/core/binaries.py index 598387c..dd0a114 100644 --- a/packages/envied/src/envied/core/binaries.py +++ b/packages/envied/src/envied/core/binaries.py @@ -45,12 +45,10 @@ ShakaPackager = find( f"packager-{__shaka_platform}-arm64", f"packager-{__shaka_platform}-x64", ) -Aria2 = find("aria2c", "aria2") CCExtractor = find("ccextractor", "ccextractorwin", "ccextractorwinfull") HolaProxy = find("hola-proxy") MPV = find("mpv") Caddy = find("caddy") -N_m3u8DL_RE = find("N_m3u8DL-RE", "n-m3u8dl-re") MKVToolNix = find("mkvmerge") Mkvpropedit = find("mkvpropedit") DoviTool = find("dovi_tool") @@ -66,12 +64,10 @@ __all__ = ( "FFPlay", "SubtitleEdit", "ShakaPackager", - "Aria2", "CCExtractor", "HolaProxy", "MPV", "Caddy", - "N_m3u8DL_RE", "MKVToolNix", "Mkvpropedit", "DoviTool", diff --git a/packages/envied/src/envied/core/cdm/__init__.py b/packages/envied/src/envied/core/cdm/__init__.py index fe0eb8f..f705992 100644 --- a/packages/envied/src/envied/core/cdm/__init__.py +++ b/packages/envied/src/envied/core/cdm/__init__.py @@ -15,6 +15,7 @@ __all__ = [ "DecryptLabsRemoteCDM", "CustomRemoteCDM", "MonaLisaCDM", + "load_cdm", "is_remote_cdm", "is_local_cdm", "cdm_location", @@ -36,6 +37,10 @@ def __getattr__(name: str) -> Any: from .monalisa import MonaLisaCDM return MonaLisaCDM + if name == "load_cdm": + from .loader import load_cdm + + return load_cdm if name in { "is_remote_cdm", diff --git a/packages/envied/src/envied/core/cdm/loader.py b/packages/envied/src/envied/core/cdm/loader.py new file mode 100644 index 0000000..7ee5281 --- /dev/null +++ b/packages/envied/src/envied/core/cdm/loader.py @@ -0,0 +1,128 @@ +"""Shared CDM loading utility. + +Instantiates a CDM object (local or remote) given a resolved device name. +Name resolution (quality-based, profile-based, DRM-type) is the caller's +responsibility — this module only handles the instantiation step. +""" + +from __future__ import annotations + +import logging +from typing import Any, Optional + +log = logging.getLogger("cdm") + + +def load_cdm( + cdm_name: str, + *, + service_name: str = "", + vaults: Optional[Any] = None, +) -> Any: + """Instantiate a CDM by device name. + + Looks up the name in config.remote_cdm first (for remote/API CDMs), + then falls back to local .prd / .wvd files. + + Returns a CDM object (WidevineCdm, PlayReadyCdm, RemoteCdm, + DecryptLabsRemoteCDM, CustomRemoteCDM, or PlayReadyRemoteCdm). + + Raises ValueError if the device cannot be found or loaded. + """ + from envied.core.config import config + + cdm_api = next(iter(x.copy() for x in config.remote_cdm if x["name"] == cdm_name), None) + if cdm_api: + return _load_remote_cdm(cdm_api, cdm_name, service_name, vaults) + + return _load_local_cdm(cdm_name) + + +def _load_remote_cdm( + cdm_api: dict, + cdm_name: str, + service_name: str, + vaults: Optional[Any], +) -> Any: + """Instantiate a remote CDM from a config.remote_cdm entry.""" + from envied.core.config import config + + cdm_type = cdm_api.get("type") + + if cdm_type == "decrypt_labs": + from envied.core.cdm.decrypt_labs_remote_cdm import DecryptLabsRemoteCDM + + del cdm_api["name"] + del cdm_api["type"] + + if "secret" not in cdm_api or not cdm_api["secret"]: + if config.decrypt_labs_api_key: + cdm_api["secret"] = config.decrypt_labs_api_key + else: + raise ValueError( + f"No secret provided for DecryptLabs CDM '{cdm_name}' and no global decrypt_labs_api_key configured" + ) + + return DecryptLabsRemoteCDM(service_name=service_name, vaults=vaults, **cdm_api) + + if cdm_type == "custom_api": + from envied.core.cdm.custom_remote_cdm import CustomRemoteCDM + + del cdm_api["name"] + del cdm_api["type"] + return CustomRemoteCDM(service_name=service_name, vaults=vaults, **cdm_api) + + device_type = cdm_api.get("Device Type", cdm_api.get("device_type", "")) + if str(device_type).upper() == "PLAYREADY": + from pyplayready.remote.remotecdm import RemoteCdm as PlayReadyRemoteCdm + + return PlayReadyRemoteCdm( + security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)), + host=cdm_api.get("Host", cdm_api.get("host")), + secret=cdm_api.get("Secret", cdm_api.get("secret")), + device_name=cdm_api.get("Device Name", cdm_api.get("device_name")), + ) + + from pywidevine.remotecdm import RemoteCdm + + return RemoteCdm( + device_type=cdm_api.get("Device Type", cdm_api.get("device_type", "")), + system_id=cdm_api.get("System ID", cdm_api.get("system_id", "")), + security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)), + host=cdm_api.get("Host", cdm_api.get("host")), + secret=cdm_api.get("Secret", cdm_api.get("secret")), + device_name=cdm_api.get("Device Name", cdm_api.get("device_name")), + ) + + +def _load_local_cdm(cdm_name: str) -> Any: + """Instantiate a local CDM from a .prd or .wvd file.""" + from envied.core.config import config + + prd_path = config.directories.prds / f"{cdm_name}.prd" + if not prd_path.is_file(): + prd_path = config.directories.wvds / f"{cdm_name}.prd" + if prd_path.is_file(): + from pyplayready.cdm import Cdm as PlayReadyCdm + from pyplayready.device import Device as PlayReadyDevice + + return PlayReadyCdm.from_device(PlayReadyDevice.load(prd_path)) + + cdm_path = config.directories.wvds / f"{cdm_name}.wvd" + if not cdm_path.is_file(): + raise ValueError(f"{cdm_name} does not exist or is not a file") + + from construct import ConstError + from pywidevine.cdm import Cdm as WidevineCdm + from pywidevine.device import Device + + try: + device = Device.load(cdm_path) + except ConstError as e: + if "expected 2 but parsed 1" in str(e): + raise ValueError( + f"{cdm_name}.wvd seems to be a v1 WVD file, use `pywidevine migrate --help` to migrate it to v2." + ) + raise ValueError(f"{cdm_name}.wvd is an invalid or corrupt Widevine Device file, {e}") + + return WidevineCdm.from_device(device) diff --git a/packages/envied/src/envied/core/commands.py b/packages/envied/src/envied/core/commands.py index 7f0fb52..fa03f35 100644 --- a/packages/envied/src/envied/core/commands.py +++ b/packages/envied/src/envied/core/commands.py @@ -1,3 +1,5 @@ +import logging +from pathlib import Path from typing import Optional import click @@ -5,11 +7,56 @@ import click from envied.core.config import config from envied.core.utilities import import_module_by_path +log = logging.getLogger("commands") + _COMMANDS = sorted( (path for path in config.directories.commands.glob("*.py") if path.stem.lower() != "__init__"), key=lambda x: x.stem ) -_MODULES = {path.stem: getattr(import_module_by_path(path), path.stem) for path in _COMMANDS} + +def load_command(path: Path) -> object: + """Load one command module, returning its stem-named attribute. + + Raises a concise, single-line error naming the command and the real cause so + a broken command never surfaces as a raw traceback pointing at the loader. + """ + try: + module = import_module_by_path(path) + except Exception as e: + raise RuntimeError(f"{path.stem}: failed to import — {type(e).__name__}: {e} ({path})") from e + try: + return getattr(module, path.stem) + except AttributeError as e: + raise RuntimeError( + f"{path.stem}: no object named '{path.stem}' found in {path} — it must match the filename" + ) from e + + +def load_commands(paths: list[Path]) -> tuple[dict[str, object], list[str]]: + """Load every command, returning the good ones plus a list of load errors. + + Importing this module must never raise (it runs at CLI startup, before Rich + is installed, so a raise here prints an ugly pre-setup traceback). Instead we + collect failures and surface them once, cleanly, when the CLI is used. + """ + modules: dict[str, object] = {} + errors: list[str] = [] + for path in paths: + try: + modules[path.stem] = load_command(path) + except Exception as e: + errors.append(str(e)) + return modules, errors + + +_MODULES, LOAD_ERRORS = load_commands(_COMMANDS) + + +def check_load_errors() -> None: + """Raise a single clean error if any command failed to load.""" + if LOAD_ERRORS: + joined = "\n".join(f" - {err}" for err in LOAD_ERRORS) + raise click.ClickException(f"Failed to load {len(LOAD_ERRORS)} command(s):\n{joined}") class Commands(click.MultiCommand): @@ -17,11 +64,13 @@ class Commands(click.MultiCommand): def list_commands(self, ctx: click.Context) -> list[str]: """Returns a list of command names from the command filenames.""" - return [x.stem for x in _COMMANDS] + check_load_errors() + return [x.stem.replace("_", "-") for x in _COMMANDS] def get_command(self, ctx: click.Context, name: str) -> Optional[click.Command]: """Load the command code and return the main click command function.""" - module = _MODULES.get(name) + check_load_errors() + module = _MODULES.get(name) or _MODULES.get(name.replace("-", "_")) if not module: raise click.ClickException(f"Unable to find command by the name '{name}'") diff --git a/packages/envied/src/envied/core/config.py b/packages/envied/src/envied/core/config.py index 36b0796..a02814a 100644 --- a/packages/envied/src/envied/core/config.py +++ b/packages/envied/src/envied/core/config.py @@ -26,6 +26,7 @@ class Config: cache = data / "cache" cookies = data / "cookies" logs = data / "logs" + exports = data / "exports" wvds = data / "WVDs" prds = data / "PRDs" dcsl = data / "DCSL" @@ -41,8 +42,6 @@ class Config: def __init__(self, **kwargs: Any): self.dl: dict = kwargs.get("dl") or {} - self.aria2c: dict = kwargs.get("aria2c") or {} - self.n_m3u8dl_re: dict = kwargs.get("n_m3u8dl_re") or {} self.cdm: dict = kwargs.get("cdm") or {} self.chapter_fallback_name: str = kwargs.get("chapter_fallback_name") or "" self.curl_impersonate: dict = kwargs.get("curl_impersonate") or {} @@ -60,22 +59,24 @@ class Config: else: setattr(self.directories, name, Path(path).expanduser()) - downloader_cfg = kwargs.get("downloader") or "requests" - if isinstance(downloader_cfg, dict): - self.downloader_map = {k.upper(): v for k, v in downloader_cfg.items()} - self.downloader = self.downloader_map.get("DEFAULT", "requests") - else: - self.downloader_map = {} - self.downloader = downloader_cfg + downloader_cfg = kwargs.get("downloader") + if downloader_cfg and downloader_cfg != "requests": + warnings.warn( + f"downloader '{downloader_cfg}' is deprecated. The unified requests downloader is now used.", + DeprecationWarning, + stacklevel=2, + ) self.filenames = self._Filenames() for name, filename in (kwargs.get("filenames") or {}).items(): setattr(self.filenames, name, filename) + self.audio: dict = kwargs.get("audio") or {} self.headers: dict = kwargs.get("headers") or {} self.key_vaults: list[dict[str, Any]] = kwargs.get("key_vaults", []) self.muxing: dict = kwargs.get("muxing") or {} self.proxy_providers: dict = kwargs.get("proxy_providers") or {} + self.remote_services: dict = kwargs.get("remote_services") or {} self.serve: dict = kwargs.get("serve") or {} self.services: dict = kwargs.get("services") or {} decryption_cfg = kwargs.get("decryption") or {} @@ -93,11 +94,19 @@ class Config: self.tmdb_api_key: str = kwargs.get("tmdb_api_key") or "" self.simkl_client_id: str = kwargs.get("simkl_client_id") or "" self.decrypt_labs_api_key: str = kwargs.get("decrypt_labs_api_key") or "" + self.ipinfo_api_key: str = kwargs.get("ipinfo_api_key") or "" self.update_checks: bool = kwargs.get("update_checks", True) self.update_check_interval: int = kwargs.get("update_check_interval", 24) self.language_tags: dict = kwargs.get("language_tags") or {} self.output_template: dict = kwargs.get("output_template") or {} + folder_cfg = self.output_template.pop("folder", "") + self.folder_template: str = "" + self.folder_templates: dict = {} + if isinstance(folder_cfg, dict): + self.folder_templates = {k: v for k, v in folder_cfg.items() if isinstance(v, str) and v} + elif isinstance(folder_cfg, str): + self.folder_template = folder_cfg or "" if kwargs.get("scene_naming") is not None: raise SystemExit( @@ -106,14 +115,8 @@ class Config: "See unshackle-example.yaml for examples." ) - if not self.output_template: - raise SystemExit( - "ERROR: No 'output_template' configured in your envied.yaml.\n" - "Please add an 'output_template' section with movies, series, and songs templates.\n" - "See unshackle-example.yaml for examples." - ) - - self._validate_output_templates() + if self.output_template: + self._validate_output_templates() self.unicode_filenames: bool = kwargs.get("unicode_filenames", False) @@ -160,7 +163,16 @@ class Config: unsafe_chars = r'[<>:"/\\|?*]' - for template_type, template_str in self.output_template.items(): + all_templates = dict(self.output_template) + if self.folder_template: + all_templates["folder"] = self.folder_template + for kind, tmpl in self.folder_templates.items(): + if kind not in {"movies", "series", "songs"}: + warnings.warn(f"Unknown folder template kind '{kind}' (expected movies/series/songs)") + continue + all_templates[f"folder.{kind}"] = tmpl + + for template_type, template_str in all_templates.items(): if not isinstance(template_str, str): warnings.warn(f"Template '{template_type}' must be a string, got {type(template_str).__name__}") continue @@ -179,6 +191,18 @@ class Config: if not template_str.strip(): warnings.warn(f"Template '{template_type}' is empty") + def get_folder_template(self, kind: str) -> str: + """Resolve the folder template for the given title kind. + + kind: one of "movies", "series", "songs". + Falls back to the legacy single-string folder template, then "". + """ + if self.folder_templates: + tmpl = self.folder_templates.get(kind) + if tmpl: + return tmpl + return self.folder_template or "" + def get_template_separator(self, template_type: str = "movies") -> str: """Get the filename separator for the given template type. diff --git a/packages/envied/src/envied/core/downloaders/__init__.py b/packages/envied/src/envied/core/downloaders/__init__.py index aa0aecb..66d3b42 100644 --- a/packages/envied/src/envied/core/downloaders/__init__.py +++ b/packages/envied/src/envied/core/downloaders/__init__.py @@ -1,6 +1,3 @@ -from .aria2c import aria2c -from .curl_impersonate import curl_impersonate -from .n_m3u8dl_re import n_m3u8dl_re from .requests import requests -__all__ = ("aria2c", "curl_impersonate", "requests", "n_m3u8dl_re") +__all__ = ("requests",) diff --git a/packages/envied/src/envied/core/downloaders/requests.py b/packages/envied/src/envied/core/downloaders/requests.py index 0866009..3da2815 100644 --- a/packages/envied/src/envied/core/downloaders/requests.py +++ b/packages/envied/src/envied/core/downloaders/requests.py @@ -1,10 +1,11 @@ import math import os import time -from concurrent.futures import as_completed +from concurrent.futures import FIRST_COMPLETED, wait from concurrent.futures.thread import ThreadPoolExecutor from http.cookiejar import CookieJar from pathlib import Path +from queue import Empty, Queue from typing import Any, Generator, MutableMapping, Optional, Union from requests import Session @@ -16,19 +17,175 @@ from envied.core.utilities import get_debug_logger, get_extension MAX_ATTEMPTS = 5 RETRY_WAIT = 2 -CHUNK_SIZE = 1024 -PROGRESS_WINDOW = 5 +PROGRESS_WINDOW = 2 -DOWNLOAD_SIZES = [] -LAST_SPEED_REFRESH = time.time() +# Adaptive chunk sizing — benchmarked optimal range +MIN_CHUNK = 524_288 # 512KB +MAX_CHUNK = 4_194_304 # 4MB +DEFAULT_CHUNK = 524_288 # 512KB +SPEED_ROLLING_WINDOW = 10 # seconds of history to keep for speed calculation + +RANGE_PARALLEL_MIN_SIZE = 64 * 1024 * 1024 +RANGE_PARALLEL_PART_SIZE = 16 * 1024 * 1024 + + +def _adaptive_chunk_size(content_length: int) -> int: + """Pick chunk size based on content length. Benchmarked sweet spot: 512KB-4MB.""" + if content_length <= 0: + return DEFAULT_CHUNK + return min(MAX_CHUNK, max(MIN_CHUNK, content_length // 4)) + + +def _is_requests_session(session: Any) -> bool: + """Check if the session is a standard requests.Session (supports resp.raw).""" + return isinstance(session, Session) + + +def _is_rnet_session(session: Any) -> bool: + """Check if the session is an RnetSession (uses resp.stream()).""" + from envied.core.session import RnetSession + + return isinstance(session, RnetSession) + + +def _probe_ranged(url: str, session: Any, **kwargs: Any) -> tuple[int, bool]: + headers = {**(kwargs.get("headers") or {}), "Range": "bytes=0-0"} + rest = {k: v for k, v in kwargs.items() if k != "headers"} + try: + resp = session.get(url, stream=True, headers=headers, **rest) + except Exception: + return 0, False + try: + if resp.status_code != 206: + return 0, False + ce = (resp.headers.get("Content-Encoding") or resp.headers.get("content-encoding") or "").lower() + if ce in ("gzip", "deflate", "br"): + return 0, False + content_range = resp.headers.get("Content-Range") or resp.headers.get("content-range") or "" + total = content_range.rsplit("/", 1)[-1].strip() + return (int(total), True) if total.isdigit() else (0, False) + finally: + try: + resp.close() + except Exception: + pass + + +def _dispatch_parts( + url: str, + save_path: Path, + session: Any, + total_size: int, + max_workers: int, + **kwargs: Any, +) -> Generator[dict[str, Any], None, None]: + save_path.parent.mkdir(parents=True, exist_ok=True) + control_file = save_path.with_name(f"{save_path.name}.!dev") + + n_parts = max(1, min(max_workers, math.ceil(total_size / RANGE_PARALLEL_PART_SIZE))) + part_size = math.ceil(total_size / n_parts) + parts = [ + (s, e) + for i in range(n_parts) + for s, e in [(i * part_size, min(total_size - 1, (i + 1) * part_size - 1))] + if s <= e + ] + + control_file.write_bytes(b"") + with open(save_path, "wb") as f: + f.truncate(total_size) + + events: Queue[dict[str, Any]] = Queue() + + def _worker(start: int, end: int) -> None: + for ev in download( + url=url, + save_path=save_path, + session=session, + part_offset=start, + part_end=end, + **kwargs, + ): + events.put(ev) + + pool = ThreadPoolExecutor(max_workers=len(parts)) + futures = [pool.submit(_worker, s, e) for s, e in parts] + pending = set(futures) + + yield {"total": total_size} + + total_bytes = 0 + start_time = last_report = time.time() + completed = False + worker_error = False + + try: + while pending: + advance = 0 + while not events.empty(): + try: + ev = events.get_nowait() + except Empty: + break + a = ev.get("advance") + if a: + advance += a + if advance: + total_bytes += advance + yield {"advance": advance} + + now = time.time() + if now - last_report > 0.5 and total_bytes > 0: + yield {"downloaded": f"{filesize.decimal(math.ceil(total_bytes / (now - start_time)))}/s"} + last_report = now + + done, pending = wait(pending, timeout=0.1, return_when=FIRST_COMPLETED) + for fut in done: + exc = fut.exception() + if exc: + worker_error = True + DOWNLOAD_CANCELLED.set() + raise exc + + advance = 0 + while not events.empty(): + try: + ev = events.get_nowait() + except Empty: + break + a = ev.get("advance") + if a: + advance += a + if advance: + total_bytes += advance + yield {"advance": advance} + + yield {"file_downloaded": save_path, "written": total_size} + completed = True + except KeyboardInterrupt: + DOWNLOAD_CANCELLED.set() + pool.shutdown(wait=False, cancel_futures=True) + raise + finally: + pool.shutdown(wait=worker_error, cancel_futures=True) + if completed: + control_file.unlink(missing_ok=True) def download( - url: str, save_path: Path, session: Optional[Session] = None, segmented: bool = False, **kwargs: Any + url: str, + save_path: Path, + session: Optional[Any] = None, + segmented: bool = False, + part_offset: Optional[int] = None, + part_end: Optional[int] = None, + **kwargs: Any, ) -> Generator[dict[str, Any], None, None]: """ - Download a file using Python Requests. - https://requests.readthedocs.io + Download a file with optimized I/O. + + Supports both requests.Session and RnetSession for TLS fingerprinting. + Uses raw socket reads for requests.Session and native rnet streaming for RnetSession. Yields the following download status updates while chunks are downloading: @@ -38,116 +195,180 @@ def download( - {downloaded: "10.1 MB/s"} (currently downloading at a rate of 10.1 MB/s) - {file_downloaded: Path(...), written: 1024} (download finished, has the save path and size) - The data is in the same format accepted by rich's progress.update() function. The - `downloaded` key is custom and is not natively accepted by all rich progress bars. - Parameters: url: Web URL of a file to download. save_path: The path to save the file to. If the save path's directory does not exist then it will be made automatically. - session: The Requests Session to make HTTP requests with. Useful to set Header, - Cookie, and Proxy data. Connections are saved and re-used with the session - so long as the server keeps the connection alive. + session: A requests.Session or RnetSession to make HTTP requests with. + RnetSession preserves TLS fingerprinting for services that need it. segmented: If downloads are segments or parts of one bigger file. + part_offset: Byte offset to write at within a pre-allocated file. When set + (with `part_end`), enables part mode for parallel ranged downloads — + no truncate, no skip-if-exists, no control file; emits only `advance` + events; retries resume mid-part via Range. + part_end: Inclusive end byte of the part. Required when `part_offset` is set. kwargs: Any extra keyword arguments to pass to the session.get() call. Use this for one-time request changes like a header, cookie, or proxy. For example, to request Byte-ranges use e.g., `headers={"Range": "bytes=0-128"}`. """ - global LAST_SPEED_REFRESH - session = session or Session() + part_mode = part_offset is not None and part_end is not None save_dir = save_path.parent control_file = save_path.with_name(f"{save_path.name}.!dev") - save_dir.mkdir(parents=True, exist_ok=True) - if control_file.exists(): - # consider the file corrupt if the control file exists - save_path.unlink(missing_ok=True) - control_file.unlink() - elif save_path.exists(): - # if it exists, and no control file, then it should be safe - yield dict(file_downloaded=save_path, written=save_path.stat().st_size) - # TODO: This should return, potential recovery bug + resume_offset = 0 + if not part_mode: + if control_file.exists() and save_path.exists(): + resume_offset = save_path.stat().st_size + elif control_file.exists(): + control_file.unlink() + elif save_path.exists(): + yield dict(file_downloaded=save_path, written=save_path.stat().st_size) + return + control_file.write_bytes(b"") - # TODO: Design a control file format so we know how much of the file is missing - control_file.write_bytes(b"") + _time = time.time + use_raw = _is_requests_session(session) attempts = 1 + completed = False + written = 0 try: while True: - written = 0 - - # these are for single-url speed calcs only - download_sizes = [] - last_speed_refresh = time.time() + if not part_mode: + written = 0 + last_speed_refresh = _time() try: - stream = session.get(url, stream=True, **kwargs) + use_rnet = _is_rnet_session(session) + + request_kwargs = dict(kwargs) + if part_mode: + req_headers = dict(request_kwargs.get("headers", {}) or {}) + req_headers["Range"] = f"bytes={part_offset + written}-{part_end}" + request_kwargs["headers"] = req_headers + elif resume_offset > 0: + req_headers = dict(request_kwargs.get("headers", {}) or {}) + req_headers["Range"] = f"bytes={resume_offset}-" + request_kwargs["headers"] = req_headers + + stream = session.get(url, stream=True, **request_kwargs) stream.raise_for_status() - if not segmented: + resumed = (not part_mode) and resume_offset > 0 and stream.status_code == 206 + if (not part_mode) and resume_offset > 0 and not resumed: + resume_offset = 0 + if part_mode and stream.status_code != 206: + raise IOError(f"expected 206 for ranged part, got {stream.status_code}") + if use_rnet: + content_length = stream.content_length or 0 + else: try: content_length = int(stream.headers.get("Content-Length", "0")) - - # Skip Content-Length validation for compressed responses since - # requests automatically decompresses but Content-Length shows compressed size if stream.headers.get("Content-Encoding", "").lower() in ["gzip", "deflate", "br"]: content_length = 0 except ValueError: content_length = 0 - if content_length > 0: - yield dict(total=math.ceil(content_length / CHUNK_SIZE)) - else: - # we have no data to calculate total chunks - yield dict(total=None) # indeterminate mode + chunk_size = _adaptive_chunk_size(content_length) + total_size = (resume_offset + content_length) if resumed and content_length > 0 else content_length - with open(save_path, "wb") as f: - for chunk in stream.iter_content(chunk_size=CHUNK_SIZE): + if not segmented and not part_mode: + if total_size > 0: + yield dict(total=total_size) + else: + yield dict(total=None) + if resumed and resume_offset > 0: + yield dict(advance=resume_offset) + + if part_mode: + file_mode = "r+b" + file_buffering = 0 + else: + file_mode = "ab" if resumed else "wb" + file_buffering = 1_048_576 + with open(save_path, file_mode, buffering=file_buffering) as f: + if part_mode: + f.seek(part_offset + written) + elif not resumed and content_length > 0: + f.truncate(content_length) + f.seek(0) + + _write = f.write + + if use_rnet: + chunks = stream.stream() + elif use_raw: + chunks = iter(lambda: stream.raw.read(chunk_size), b"") + else: + chunks = stream.iter_content(chunk_size=chunk_size) + + _data_accumulated = 0 + _bytes_since_yield = 0 + emit_progress = (not segmented) or part_mode + for chunk in chunks: + if DOWNLOAD_CANCELLED.is_set(): + break + _write(chunk) download_size = len(chunk) - f.write(chunk) written += download_size - if not segmented: - yield dict(advance=1) - now = time.time() + if emit_progress: + _bytes_since_yield += download_size + _data_accumulated += download_size + now = _time() time_since = now - last_speed_refresh - download_sizes.append(download_size) - if time_since > PROGRESS_WINDOW or download_size < CHUNK_SIZE: - data_size = sum(download_sizes) - download_speed = math.ceil(data_size / (time_since or 1)) - yield dict(downloaded=f"{filesize.decimal(download_speed)}/s") + if time_since > PROGRESS_WINDOW: + yield dict(advance=_bytes_since_yield) + _bytes_since_yield = 0 + if not part_mode: + download_speed = math.ceil(_data_accumulated / (time_since or 1)) + yield dict(downloaded=f"{filesize.decimal(download_speed)}/s") last_speed_refresh = now - download_sizes.clear() + _data_accumulated = 0 - if not segmented and content_length and written < content_length: + if emit_progress and _bytes_since_yield > 0: + yield dict(advance=_bytes_since_yield) + + try: + stream.close() + except Exception: + pass + + if not part_mode and not resumed and content_length > 0 and written != content_length: + f.truncate(written) + + if part_mode: + expected = part_end - part_offset + 1 + if written < expected: + raise IOError(f"Failed to read part {part_offset}-{part_end}: got {written}/{expected}") + elif not segmented and content_length and written < content_length: raise IOError(f"Failed to read {content_length} bytes from the track URI.") - yield dict(file_downloaded=save_path, written=written) - - if segmented: - yield dict(advance=1) - now = time.time() - time_since = now - LAST_SPEED_REFRESH - if written: # no size == skipped dl - DOWNLOAD_SIZES.append(written) - if DOWNLOAD_SIZES and time_since > PROGRESS_WINDOW: - data_size = sum(DOWNLOAD_SIZES) - download_speed = math.ceil(data_size / (time_since or 1)) - yield dict(downloaded=f"{filesize.decimal(download_speed)}/s") - LAST_SPEED_REFRESH = now - DOWNLOAD_SIZES.clear() + if not part_mode: + yield dict(file_downloaded=save_path, written=resume_offset + written) + if segmented: + yield dict(advance=1) + completed = True break - except Exception as e: - save_path.unlink(missing_ok=True) + except Exception: + try: + stream.close() + except Exception: + pass if DOWNLOAD_CANCELLED.is_set() or attempts == MAX_ATTEMPTS: - raise e + if part_mode and not DOWNLOAD_CANCELLED.is_set(): + raise + return + if not part_mode and save_path.exists(): + resume_offset = save_path.stat().st_size time.sleep(RETRY_WAIT) attempts += 1 finally: - control_file.unlink() + if completed and not part_mode: + control_file.unlink(missing_ok=True) def requests( @@ -158,10 +379,14 @@ def requests( cookies: Optional[Union[MutableMapping[str, str], CookieJar]] = None, proxy: Optional[str] = None, max_workers: Optional[int] = None, + session: Optional[Any] = None, ) -> Generator[dict[str, Any], None, None]: """ - Download a file using Python Requests. - https://requests.readthedocs.io + Download files with optimized I/O and adaptive chunk sizing. + + Supports both requests.Session and RnetSession. When a RnetSession is + provided (e.g. from a service's get_session()), TLS fingerprinting is preserved + on all segment downloads. Yields the following download status updates while chunks are downloading: @@ -186,7 +411,10 @@ def requests( cookies: A mapping of Cookie Key/Values or a Cookie Jar to use for all downloads. proxy: An optional proxy URI to route connections through for all downloads. max_workers: The maximum amount of threads to use for downloads. Defaults to - min(32,(cpu_count+4)). + min(12,(cpu_count+4)). + session: An optional requests.Session or RnetSession to use. If provided, + it will be used directly (preserving TLS fingerprinting). If None, a new + requests.Session with HTTPAdapter connection pooling will be created. """ if not urls: raise ValueError("urls must be provided and not empty") @@ -221,7 +449,7 @@ def requests( urls = [urls] if not max_workers: - max_workers = min(32, (os.cpu_count() or 1) + 4) + max_workers = min(16, (os.cpu_count() or 1) + 4) urls = [ dict(save_path=save_path, **url) if isinstance(url, dict) else dict(url=url, save_path=save_path) @@ -231,25 +459,33 @@ def requests( ] ] - session = Session() - session.mount("https://", HTTPAdapter(pool_connections=max_workers, pool_maxsize=max_workers, pool_block=True)) - session.mount("http://", session.adapters["https://"]) + # Use provided session or create a new optimized requests.Session + # When a session is provided (e.g., service's RnetSession), don't mutate headers/cookies/proxy — + # they're already set and the session may be shared across tracks. + if session is None: + session = Session() + if headers: + headers = {k: v for k, v in headers.items() if k.lower() != "accept-encoding"} + session.headers.update(headers) + if cookies: + session.cookies.update(cookies) + if proxy: + session.proxies.update({"all": proxy}) - if headers: - headers = {k: v for k, v in headers.items() if k.lower() != "accept-encoding"} - session.headers.update(headers) - if cookies: - session.cookies.update(cookies) - if proxy: - session.proxies.update({"all": proxy}) + # Mount HTTPAdapter with connection pooling sized to worker count. + # Safe to do on any requests.Session — improves connection reuse for parallel downloads. + if _is_requests_session(session): + adapter = HTTPAdapter(pool_connections=max_workers, pool_maxsize=max_workers, pool_block=True) + session.mount("https://", adapter) + session.mount("http://", adapter) if debug_logger: first_url = urls[0].get("url", "") if urls else "" url_display = first_url[:200] + "..." if len(first_url) > 200 else first_url debug_logger.log( level="DEBUG", - operation="downloader_requests_start", - message="Starting requests download", + operation="downloader_start", + message="Starting download", context={ "url_count": len(urls), "first_url": url_display, @@ -257,64 +493,171 @@ def requests( "filename": filename, "max_workers": max_workers, "has_proxy": bool(proxy), + "session_type": type(session).__name__, }, ) - # If we're downloading more than one URL, treat them as "segments" for progress purposes. - # For single-URL downloads we want per-chunk progress (and the inner `download()` will yield - # a chunk-based `total`), so don't set a segment total of 1 here. segmented_batch = len(urls) > 1 - if segmented_batch: - yield dict(total=len(urls)) - try: - with ThreadPoolExecutor(max_workers=max_workers) as pool: - for future in as_completed( - pool.submit(download, session=session, segmented=segmented_batch, **url) for url in urls - ): - try: - yield from future.result() - except KeyboardInterrupt: - DOWNLOAD_CANCELLED.set() # skip pending track downloads - yield dict(downloaded="[yellow]CANCELLING") - pool.shutdown(wait=True, cancel_futures=True) - yield dict(downloaded="[yellow]CANCELLED") - # tell dl that it was cancelled - # the pool is already shut down, so exiting loop is fine - raise - except Exception as e: - DOWNLOAD_CANCELLED.set() # skip pending track downloads - yield dict(downloaded="[red]FAILING") - pool.shutdown(wait=True, cancel_futures=True) - yield dict(downloaded="[red]FAILED") - if debug_logger: - debug_logger.log( - level="ERROR", - operation="downloader_requests_failed", - message=f"Requests download failed: {e}", - error=e, - context={ - "url_count": len(urls), - "output_dir": str(output_dir), - }, + if len(urls) == 1: + url_item = urls[0] + try: + ranged_used = False + if max_workers > 1: + total_size, supports_ranges = _probe_ranged(url_item["url"], session) + if supports_ranges and total_size >= RANGE_PARALLEL_MIN_SIZE: + try: + yield from _dispatch_parts( + session=session, + total_size=total_size, + max_workers=max_workers, + **url_item, ) - # tell dl that it failed - # the pool is already shut down, so exiting loop is fine - raise + ranged_used = True + except KeyboardInterrupt: + raise + except Exception: + save_path = url_item.get("save_path") + if save_path: + sp = Path(save_path) + for target in (sp, sp.with_name(f"{sp.name}.!dev")): + try: + target.unlink(missing_ok=True) + except OSError: + pass + if not ranged_used: + yield from download( + session=session, + segmented=segmented_batch, + **url_item, + ) + except KeyboardInterrupt: + DOWNLOAD_CANCELLED.set() + yield dict(downloaded="[yellow]CANCELLED") + raise + else: + # Segmented download with thread pool + # Speed is tracked here on the main thread, not in workers + total_bytes = 0 + start_time = time.time() + last_speed_report = start_time - if debug_logger: - debug_logger.log( - level="DEBUG", - operation="downloader_requests_complete", - message="Requests download completed successfully", - context={ - "url_count": len(urls), - "output_dir": str(output_dir), - "filename": filename, - }, - ) - finally: - DOWNLOAD_SIZES.clear() + pool = ThreadPoolExecutor(max_workers=max_workers) + event_queue: Queue[dict[str, Any]] = Queue() + + def _download_worker(url_item: dict[str, Any]) -> None: + for event in download( + session=session, + segmented=segmented_batch, + **url_item, + ): + event_queue.put(event) + + futures = [pool.submit(_download_worker, url) for url in urls] + pending = set(futures) + + pending_advance = 0 + + try: + while pending: + # Drain queued events — batch advances, track bytes for speed + while True: + try: + event = event_queue.get_nowait() + except Empty: + break + # Accumulate advance events for batched yield + advance = event.get("advance") + if advance: + pending_advance += advance + continue + # Track bytes from completed segments for speed calculation + written = event.get("written") + if written: + total_bytes += written + # Pass through other events (file_downloaded, total, etc.) + yield event + + # Yield batched advances every drain cycle for responsive progress bar + if pending_advance > 0: + yield dict(advance=pending_advance) + pending_advance = 0 + + # Yield speed every 0.5s (throttled to avoid spamming Rich) + now = time.time() + if now - last_speed_report > 0.5 and total_bytes > 0: + elapsed = now - start_time + if elapsed > 0: + download_speed = math.ceil(total_bytes / elapsed) + yield dict(downloaded=f"{filesize.decimal(download_speed)}/s") + last_speed_report = now + + # Wait efficiently for next future completion (OS condition variable) + completed, pending = wait(pending, timeout=0.1, return_when=FIRST_COMPLETED) + for future in completed: + exc = future.exception() + if isinstance(exc, KeyboardInterrupt): + raise KeyboardInterrupt() + elif exc: + DOWNLOAD_CANCELLED.set() + yield dict(downloaded="[red]FAILING") + pool.shutdown(wait=False, cancel_futures=True) + yield dict(downloaded="[red]FAILED") + if debug_logger: + debug_logger.log( + level="ERROR", + operation="downloader_failed", + message=f"Download failed: {exc}", + error=exc, + context={ + "url_count": len(urls), + "output_dir": str(output_dir), + }, + ) + raise exc + except KeyboardInterrupt: + DOWNLOAD_CANCELLED.set() + yield dict(downloaded="[yellow]CANCELLING") + pool.shutdown(wait=False, cancel_futures=True) + yield dict(downloaded="[yellow]CANCELLED") + raise + finally: + pool.shutdown(wait=False, cancel_futures=True) + + # Drain remaining events + while True: + try: + event = event_queue.get_nowait() + except Empty: + break + advance = event.get("advance") + if advance: + pending_advance += advance + continue + written = event.get("written") + if written: + total_bytes += written + yield event + + # Flush remaining advances and final speed + if pending_advance > 0: + yield dict(advance=pending_advance) + elapsed = time.time() - start_time + if elapsed > 0 and total_bytes > 0: + download_speed = math.ceil(total_bytes / elapsed) + yield dict(downloaded=f"{filesize.decimal(download_speed)}/s") + + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="downloader_complete", + message="Download completed successfully", + context={ + "url_count": len(urls), + "output_dir": str(output_dir), + "filename": filename, + }, + ) __all__ = ("requests",) diff --git a/packages/envied/src/envied/core/drm/__init__.py b/packages/envied/src/envied/core/drm/__init__.py index 76db577..aa64295 100644 --- a/packages/envied/src/envied/core/drm/__init__.py +++ b/packages/envied/src/envied/core/drm/__init__.py @@ -1,4 +1,6 @@ -from typing import Union +import base64 +from typing import Any, Union +from uuid import UUID from envied.core.drm.clearkey import ClearKey from envied.core.drm.monalisa import MonaLisa @@ -8,4 +10,35 @@ from envied.core.drm.widevine import Widevine DRM_T = Union[ClearKey, Widevine, PlayReady, MonaLisa] -__all__ = ("ClearKey", "Widevine", "PlayReady", "MonaLisa", "DRM_T") +def drm_from_dict(data: dict[str, Any]) -> Union[Widevine, PlayReady]: + """Reconstruct a Widevine/PlayReady DRM instance from its ``to_dict()`` form. + + Rebuilds the PSSH from the stored base64 and re-injects any saved content keys + so the resulting object can decrypt without contacting a license server. + """ + system = data.get("system") + pssh_b64 = data.get("pssh_b64") + kids = data.get("kids") or [] + content_keys = data.get("content_keys") or {} + + if not pssh_b64: + raise ValueError("Cannot reconstruct DRM without a stored PSSH.") + + if system == "PlayReady": + from pyplayready.system.pssh import PSSH as PlayReadyPSSH + + drm: Union[Widevine, PlayReady] = PlayReady(pssh=PlayReadyPSSH(base64.b64decode(pssh_b64)), pssh_b64=pssh_b64) + elif system == "Widevine": + from pywidevine.pssh import PSSH as WidevinePSSH + + drm = Widevine(pssh=WidevinePSSH(pssh_b64), kid=kids[0] if kids else None) + else: + raise ValueError(f"Unsupported DRM system for reconstruction: {system!r}") + + for kid_hex, key in content_keys.items(): + drm.content_keys[UUID(hex=kid_hex)] = key + + return drm + + +__all__ = ("ClearKey", "Widevine", "PlayReady", "MonaLisa", "DRM_T", "drm_from_dict") diff --git a/packages/envied/src/envied/core/drm/clearkey.py b/packages/envied/src/envied/core/drm/clearkey.py index 089fa71..4e7d0e9 100644 --- a/packages/envied/src/envied/core/drm/clearkey.py +++ b/packages/envied/src/envied/core/drm/clearkey.py @@ -8,10 +8,11 @@ from urllib.parse import urljoin from Cryptodome.Cipher import AES from Cryptodome.Util.Padding import unpad -from curl_cffi.requests import Session as CurlSession from m3u8.model import Key from requests import Session +from envied.core.session import RnetSession + class ClearKey: """AES Clear Key DRM System.""" @@ -70,8 +71,8 @@ class ClearKey: """ if not isinstance(m3u_key, Key): raise ValueError(f"Provided M3U Key is in an unexpected type {m3u_key!r}") - if not isinstance(session, (Session, CurlSession, type(None))): - raise TypeError(f"Expected session to be a {Session} or {CurlSession}, not a {type(session)}") + if not isinstance(session, (Session, RnetSession, type(None))): + raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not a {type(session)}") if not m3u_key.method.startswith("AES"): raise ValueError(f"Provided M3U Key is not an AES Clear Key, {m3u_key.method}") diff --git a/packages/envied/src/envied/core/drm/playready.py b/packages/envied/src/envied/core/drm/playready.py index 951c760..8b3f260 100644 --- a/packages/envied/src/envied/core/drm/playready.py +++ b/packages/envied/src/envied/core/drm/playready.py @@ -179,9 +179,9 @@ class PlayReady: pssh_boxes.extend( Box.parse(base64.b64decode(x.uri.split(",")[-1])) for x in (master.session_keys or master.keys) - if x and x.keyformat and x.keyformat.lower() in { - f"urn:uuid:{PSSH.SYSTEM_ID}", "com.microsoft.playready" - } + if x + and x.keyformat + and x.keyformat.lower() in {f"urn:uuid:{PSSH.SYSTEM_ID}", "com.microsoft.playready"} ) init_data = track.get_init_segment(session=session) @@ -247,6 +247,17 @@ class PlayReady: def kid(self) -> Optional[UUID]: return next(iter(self.kids), None) + def to_dict(self) -> dict[str, Any]: + """Serialise this DRM instance for export/import (PSSH + KIDs). + + Content keys are stored once at the export's track level, not duplicated here. + """ + return { + "system": "PlayReady", + "pssh_b64": self.pssh_b64, + "kids": [kid.hex for kid in self.kids], + } + @property def kids(self) -> list[UUID]: return self._kids @@ -295,7 +306,10 @@ class PlayReady: if challenge: try: - license_res = licence(challenge=challenge) + try: + license_res = licence(challenge=challenge, pssh_b64=self.pssh_b64) + except TypeError: + license_res = licence(challenge=challenge) if isinstance(license_res, bytes): license_str = license_res.decode(errors="ignore") else: @@ -356,6 +370,15 @@ class PlayReady: key_hex = key if isinstance(key, str) else key.hex() key_args.extend(["--key", f"{kid_hex}:{key_hex}"]) + # Fallback for tracks whose tenc default_KID is all-zero and whose real + # KID is signalled out-of-band: emit a zero-KID entry per content key. + zero_kid = "00" * 16 + existing_kids = {kid.hex if hasattr(kid, "hex") else str(kid).replace("-", "") for kid in self.content_keys} + if zero_kid not in existing_kids: + for key in self.content_keys.values(): + key_hex = key if isinstance(key, str) else key.hex() + key_args.extend(["--key", f"{zero_kid}:{key_hex}"]) + cmd = [ str(binaries.Mp4decrypt), "--show-progress", @@ -365,7 +388,7 @@ class PlayReady: ] try: - subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8') + subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8") except subprocess.CalledProcessError as e: error_msg = e.stderr if e.stderr else f"mp4decrypt failed with exit code {e.returncode}" raise subprocess.CalledProcessError(e.returncode, cmd, output=e.stdout, stderr=error_msg) @@ -411,7 +434,9 @@ class PlayReady: [binaries.ShakaPackager, *arguments], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, - universal_newlines=True, + text=True, + encoding="utf-8", + errors="replace", ) stream_skipped = False diff --git a/packages/envied/src/envied/core/drm/widevine.py b/packages/envied/src/envied/core/drm/widevine.py index 6a22a6c..f4d7a82 100644 --- a/packages/envied/src/envied/core/drm/widevine.py +++ b/packages/envied/src/envied/core/drm/widevine.py @@ -27,6 +27,12 @@ from envied.core.utils.subprocess import ffprobe class Widevine: """Widevine DRM System.""" + PLACEHOLDER_KIDS = { + UUID("00000000-0000-0000-0000-000000000000"), # All zeros (key rotation default) + UUID("00010203-0405-0607-0809-0a0b0c0d0e0f"), # Sequential 0x00-0x0f + UUID("00010203-0405-0607-0809-101112131415"), # Shaka Packager test pattern + } + def __init__(self, pssh: PSSH, kid: Union[UUID, str, bytes, None] = None, **kwargs: Any): if not pssh: raise ValueError("Provided PSSH is empty.") @@ -36,6 +42,7 @@ class Widevine: if pssh.system_id == PSSH.SystemId.PlayReady: pssh.to_widevine() + self._kid: Optional[UUID] = None if kid: if isinstance(kid, str): kid = UUID(hex=kid) @@ -43,7 +50,11 @@ class Widevine: kid = UUID(bytes=kid) if not isinstance(kid, UUID): raise ValueError(f"Expected kid to be a {UUID}, str, or bytes, not {kid!r}") - pssh.set_key_ids([kid]) + self._kid = kid + if pssh.key_ids and all(k in self.PLACEHOLDER_KIDS for k in pssh.key_ids): + pssh.set_key_ids([kid]) + elif kid not in (pssh.key_ids or []): + pssh.set_key_ids([*(pssh.key_ids or []), kid]) self._pssh = pssh @@ -161,8 +172,24 @@ class Widevine: @property def kids(self) -> list[UUID]: - """Get all Key IDs.""" - return self._pssh.key_ids + """Get all Key IDs from PSSH, falling back to the externally provided KID.""" + pssh_kids = self._pssh.key_ids + if pssh_kids: + return pssh_kids + if self._kid: + return [self._kid] + return [] + + def to_dict(self) -> dict[str, Any]: + """Serialise this DRM instance for export/import (PSSH + KIDs). + + Content keys are stored once at the export's track level, not duplicated here. + """ + return { + "system": "Widevine", + "pssh_b64": self.pssh.dumps(), + "kids": [kid.hex for kid in self.kids], + } def get_content_keys(self, cdm: WidevineCdm, certificate: Callable, licence: Callable) -> None: """ @@ -189,7 +216,11 @@ class Widevine: if hasattr(cdm, "has_cached_keys") and cdm.has_cached_keys(session_id): pass else: - cdm.parse_license(session_id, licence(challenge=challenge)) + try: + license_res = licence(challenge=challenge, pssh=self.pssh) + except TypeError: + license_res = licence(challenge=challenge) + cdm.parse_license(session_id, license_res) self.content_keys = {key.kid: key.key.hex() for key in cdm.get_keys(session_id, "CONTENT")} if not self.content_keys: @@ -276,6 +307,15 @@ class Widevine: key_hex = key if isinstance(key, str) else key.hex() key_args.extend(["--key", f"{kid_hex}:{key_hex}"]) + # Fallback for tracks whose tenc default_KID is all-zero and whose real + # KID is signalled out-of-band: emit a zero-KID entry per content key. + zero_kid = "00" * 16 + existing_kids = {kid.hex if hasattr(kid, "hex") else str(kid).replace("-", "") for kid in self.content_keys} + if zero_kid not in existing_kids: + for key in self.content_keys.values(): + key_hex = key if isinstance(key, str) else key.hex() + key_args.extend(["--key", f"{zero_kid}:{key_hex}"]) + cmd = [ str(binaries.Mp4decrypt), "--show-progress", @@ -285,7 +325,7 @@ class Widevine: ] try: - subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8') + subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8") except subprocess.CalledProcessError as e: error_msg = e.stderr if e.stderr else f"mp4decrypt failed with exit code {e.returncode}" raise subprocess.CalledProcessError(e.returncode, cmd, output=e.stdout, stderr=error_msg) diff --git a/packages/envied/src/envied/core/import_service.py b/packages/envied/src/envied/core/import_service.py new file mode 100644 index 0000000..5506269 --- /dev/null +++ b/packages/envied/src/envied/core/import_service.py @@ -0,0 +1,304 @@ +from __future__ import annotations + +import json +import logging +from http.cookiejar import CookieJar +from pathlib import Path +from typing import Any, Optional, Union +from uuid import UUID + +import click +import requests + +from envied.core.config import config +from envied.core.constants import AnyTrack +from envied.core.credential import Credential +from envied.core.drm import drm_from_dict +from envied.core.manifests import DASH, HLS, ISM +from envied.core.remote_service import RemoteService, _build_title, _resolve_proxy +from envied.core.titles import Episode, Movies, Series, Title_T, Titles_T, remap_titles +from envied.core.tracks import Audio, Chapter, Chapters, Tracks, Video +from envied.core.tracks.attachment import Attachment +from envied.core.tracks.track import Track + +log = logging.getLogger("import") + +PARSERS = {"DASH": DASH, "HLS": HLS, "ISM": ISM} + + +class ImportService: + """Reconstructs a download from an export JSON. + + Auth and licensing are skipped; tracks are rebuilt from the export and keys injected + directly. ``_server_cdm``/``_server_cdm_type`` keep their underscores: dl.py reads them + via getattr as the server-CDM contract that skips client licensing. + """ + + ALIASES: tuple[str, ...] = () + GEOFENCE: tuple[str, ...] = () + NO_SUBTITLES: bool = False + + def __init__(self, ctx: click.Context, service_tag: str, title: str, import_file: Optional[str]) -> None: + self.__class__.__name__ = service_tag + self.service_tag = service_tag + self.title_id = title + self.ctx = ctx + self.log = logging.getLogger(service_tag) + self.credential: Optional[Credential] = None + self.current_region: Optional[str] = None + self.title_cache = None + + if not import_file: + raise click.ClickException("No export file was provided to import from.") + export_path = Path(import_file) + if not export_path.is_file(): + raise click.ClickException(f"Export file not found: {export_path}") + + self.data: dict[str, Any] = json.loads(export_path.read_text(encoding="utf8")) + version = self.data.get("version") + if version != 2: + raise click.ClickException( + f"Unsupported export version {version!r}. Re-create the export with a current build." + ) + + self.titles_data: dict[str, Any] = self.data.get("titles", {}) + self.region: Optional[str] = self.data.get("region") + self.titles: Optional[Titles_T] = None + self.tracks_by_title: dict[str, Tracks] = {} + + self._server_cdm = True + self._server_cdm_type = "widevine" + + self.session = self.build_session(ctx, self.region) + + @staticmethod + def build_session(ctx: click.Context, region: Optional[str] = None) -> requests.Session: + """Session for re-fetching the manifest. + + Honours the importer's ``--proxy``; otherwise falls back to the export region as a + geofence. An explicit proxy that fails to resolve raises; a region fallback warns. + """ + session = requests.Session() + session.headers.update(config.headers) + + params = ctx.parent.params if ctx.parent else {} + if params.get("no_proxy"): + return session + + explicit = params.get("proxy") + proxy_query = explicit or region + if not proxy_query: + return session + + try: + proxy = _resolve_proxy(proxy_query) + except Exception as e: + if explicit: + raise click.ClickException(f"Failed to resolve proxy '{proxy_query}': {e}") + log.warning(f"Could not auto-select a proxy for export region '{region}': {e}. Continuing without proxy.") + proxy = None + + if proxy: + session.proxies.update({"all": proxy}) + if not explicit: + log.info(f"No --proxy given; using export region '{region}' via your proxy provider.") + return session + + @property + def title(self) -> str: + return self.title_id + + def authenticate(self, cookies: Optional[CookieJar] = None, credential: Optional[Credential] = None) -> None: + self.credential = credential + + def get_titles(self) -> Titles_T: + if self.titles is not None: + return self.titles + titles_list = [ + _build_title(entry.get("meta", {}), self.service_tag, fallback_id=title_id) + for title_id, entry in self.titles_data.items() + ] + self.titles = ( + Series(titles_list) if titles_list and isinstance(titles_list[0], Episode) else Movies(titles_list) + ) + return self.titles + + def get_titles_cached(self, title_id: Optional[str] = None) -> Titles_T: + """Apply the service's title_map to titles reconstructed from the export sidecar.""" + title_map = (config.services.get(self.service_tag) or {}).get("title_map") or {} + return remap_titles(self.get_titles(), title_map) + + def get_tracks(self, title: Title_T) -> Tracks: + """Reconstruct the title's tracks from the export. + + DASH/ISM: re-fetch and re-parse the manifest and return the full ladder (the importer + picks quality with normal dl flags; keys are injected by KID later). HLS/URL: rebuild + from the stored per-track dicts, since the variant is re-fetched from track.url at + download time and ATV-style master playlists carry unstable per-fetch tokens. + """ + title_id = str(title.id) + if title_id in self.tracks_by_title: + return self.tracks_by_title[title_id] + + entry = self.titles_data.get(title_id, {}) + tracks_map: dict[str, Any] = entry.get("tracks") or {} + manifest_url = entry.get("manifest_url") + manifest_type = entry.get("manifest_type") + + tracks = Tracks() + tracks.manifest_url = manifest_url + + parser = PARSERS.get(manifest_type or "") + if manifest_url and parser is not None and manifest_type in ("DASH", "ISM"): + try: + parsed = parser.from_url(url=manifest_url, session=self.session).to_tracks(language=title.language) + except Exception as e: + raise click.ClickException( + f"Failed to re-fetch/parse the {manifest_type} manifest for '{title}'. " + f"The manifest URL may have expired since export. ({e})" + ) + for track in parsed: + tracks.add(track) + else: + for track_dict in tracks_map.values(): + track = Track.from_dict(track_dict) + drm = self.rebuild_drm(track_dict) + if drm: + track.drm = drm + tracks.add(track) + + for attachment in entry.get("attachments") or []: + url = attachment.get("url") + if not url: + continue + try: + tracks.attachments.append( + Attachment.from_url( + url, + name=attachment.get("name"), + mime_type=attachment.get("mime_type"), + description=attachment.get("description"), + session=self.session, + ) + ) + except Exception as e: + self.log.warning(f"Skipping attachment '{attachment.get('name')}': {e}") + + self.tracks_by_title[title_id] = tracks + return tracks + + def key_pool(self) -> dict[UUID, str]: + """All exported KID:KEY pairs across every title, as {UUID: key_hex}.""" + pool: dict[UUID, str] = {} + for entry in self.titles_data.values(): + for track_dict in (entry.get("tracks") or {}).values(): + for kid_hex, key in (track_dict.get("keys") or {}).items(): + pool[UUID(hex=kid_hex)] = key + return pool + + def rebuild_drm(self, track_dict: dict[str, Any]) -> Optional[list[Any]]: + """Rebuild a DRM object (from stored PSSH, falling back to a stub) with the exported keys.""" + keys = track_dict.get("keys") or {} + drm_dicts = track_dict.get("drm") or [] + if not drm_dicts and not keys: + return None + + drm_obj = None + if drm_dicts: + try: + drm_obj = drm_from_dict(drm_dicts[0]) + except Exception as e: + self.log.debug(f"Falling back to DRM stub (PSSH rebuild failed: {e})") + + if drm_obj is None and keys: + drm_type = (drm_dicts[0].get("system", "Widevine").lower()) if drm_dicts else "widevine" + drm_obj = RemoteService._create_drm_stub(drm_type, list(keys.keys())) + + if drm_obj is None: + return None + + for kid_hex, key in keys.items(): + drm_obj.content_keys[UUID(hex=kid_hex)] = key + return [drm_obj] + + def resolve_server_keys(self, title: Title_T) -> None: + """Inject exported keys into the selected encrypted tracks by KID (no network). + + Called by dl.py after selection. Only encrypted video/audio are touched; encrypted + DASH tracks (no DRM at parse time) get a stub holding the keys, which + DASH.download_track preserves. decrypt() applies the key whose KID matches the media. + """ + pool = self.key_pool() + if not pool: + return + + system = self.exported_drm_system() + kid_hexes = [kid.hex for kid in pool] + + for track in title.tracks: + if not isinstance(track, (Video, Audio)) or not self.track_is_encrypted(track): + continue + drm_obj = track.drm[0] if track.drm else RemoteService._create_drm_stub(system, kid_hexes) + for kid, key in pool.items(): + drm_obj.content_keys[kid] = key + track.drm = [drm_obj] + self._server_cdm_type = drm_obj.__class__.__name__.lower() + + @staticmethod + def track_is_encrypted(track: Any) -> bool: + """True if the track carries DRM or its DASH manifest declares ContentProtection.""" + if track.drm: + return True + dash = track.data.get("dash") if getattr(track, "data", None) else None + if dash: + for element in (dash.get("representation"), dash.get("adaptation_set")): + if element is not None and element.findall("ContentProtection"): + return True + return False + + def exported_drm_system(self) -> str: + """The DRM system the exporter licensed (e.g. 'playready'), defaulting to widevine.""" + for entry in self.titles_data.values(): + for track_dict in (entry.get("tracks") or {}).values(): + for drm_dict in track_dict.get("drm") or []: + if drm_dict.get("system"): + return drm_dict["system"].lower() + return "widevine" + + def get_chapters(self, title: Title_T) -> Chapters: + entry = self.titles_data.get(str(title.id), {}) + return Chapters( + [Chapter(ch["timestamp"], ch.get("name")) for ch in (entry.get("chapters") or []) if ch.get("timestamp")] + ) + + def get_widevine_service_certificate(self, **_: Any) -> Optional[str]: + return None + + def get_widevine_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[Union[bytes, str]]: + raise RuntimeError("ImportService should not request a license; keys come from the export.") + + def get_playready_license( + self, *, challenge: bytes, title: Title_T, track: AnyTrack + ) -> Optional[Union[bytes, str]]: + raise RuntimeError("ImportService should not request a license; keys come from the export.") + + def on_segment_downloaded(self, track: AnyTrack, segment: Any) -> None: + pass + + def on_track_downloaded(self, track: AnyTrack) -> None: + pass + + def on_track_decrypted(self, track: AnyTrack, drm: Any, segment: Any = None) -> None: + pass + + def on_track_repacked(self, track: AnyTrack) -> None: + pass + + def on_track_multiplex(self, track: AnyTrack) -> None: + pass + + def close(self) -> None: + try: + self.session.close() + except Exception: + pass diff --git a/packages/envied/src/envied/core/manifests/dash.py b/packages/envied/src/envied/core/manifests/dash.py index 7e183b2..2fae7a8 100644 --- a/packages/envied/src/envied/core/manifests/dash.py +++ b/packages/envied/src/envied/core/manifests/dash.py @@ -7,7 +7,7 @@ import math import re import shutil import sys -from copy import copy, deepcopy +from copy import deepcopy from functools import partial from pathlib import Path from typing import Any, Callable, Optional, Union @@ -16,9 +16,7 @@ from uuid import UUID from zlib import crc32 import requests -from curl_cffi.requests import Session as CurlSession from langcodes import Language, tag_is_valid -from lxml import etree from lxml.etree import Element, ElementTree from pyplayready.system.pssh import PSSH as PR_PSSH from pywidevine.cdm import Cdm as WidevineCdm @@ -27,9 +25,9 @@ from requests import Session from envied.core.cdm.detect import is_playready_cdm from envied.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack -from envied.core.downloaders import requests as requests_downloader from envied.core.drm import DRM_T, PlayReady, Widevine from envied.core.events import events +from envied.core.session import RnetSession from envied.core.tracks import Audio, Subtitle, Tracks, Video from envied.core.utilities import get_debug_logger, is_close_match, try_ensure_utf8 from envied.core.utils.xml import load_xml @@ -51,7 +49,7 @@ class DASH: self.url = url @classmethod - def from_url(cls, url: str, session: Optional[Union[Session, CurlSession]] = None, **args: Any) -> DASH: + def from_url(cls, url: str, session: Optional[Union[Session, RnetSession]] = None, **args: Any) -> DASH: if not url: raise requests.URLRequired("DASH manifest URL must be provided for relative path computations.") if not isinstance(url, str): @@ -59,8 +57,8 @@ class DASH: if not session: session = Session() - elif not isinstance(session, (Session, CurlSession)): - raise TypeError(f"Expected session to be a {Session} or {CurlSession}, not {session!r}") + elif not isinstance(session, (Session, RnetSession)): + raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {session!r}") res = session.get(url, **args) if res.url != url: @@ -109,13 +107,7 @@ class DASH: if period_id := period.get("id"): filtered_period_ids.append(period_id) continue - if next(iter(period.xpath("SegmentType/@value")), "content") != "content": - if period_id := period.get("id"): - filtered_period_ids.append(period_id) - continue - if "urn:amazon:primevideo:cachingBreadth" in [ - x.get("schemeIdUri") for x in period.findall("SupplementalProperty") - ]: + if not DASH._is_content_period(period, []): if period_id := period.get("id"): filtered_period_ids.append(period_id) continue @@ -244,6 +236,7 @@ class DASH: "period": period, "adaptation_set": adaptation_set, "representation": rep, + "representation_id": rep.get("id"), "filtered_period_ids": filtered_period_ids, } }, @@ -254,6 +247,7 @@ class DASH: # only get tracks from the first main-content period break + tracks.manifest_url = self.url return tracks @staticmethod @@ -271,8 +265,8 @@ class DASH: ): if not session: session = Session() - elif not isinstance(session, (Session, CurlSession)): - raise TypeError(f"Expected session to be a {Session} or {CurlSession}, not {session!r}") + elif not isinstance(session, (Session, RnetSession)): + raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {session!r}") if proxy: session.proxies.update({"all": proxy}) @@ -280,9 +274,10 @@ class DASH: log = logging.getLogger("DASH") manifest: ElementTree = track.data["dash"]["manifest"] - period: Element = track.data["dash"]["period"] adaptation_set: Element = track.data["dash"]["adaptation_set"] representation: Element = track.data["dash"]["representation"] + rep_id: Optional[str] = track.data["dash"].get("representation_id") or representation.get("id") + filtered_period_ids: list[str] = track.data["dash"].get("filtered_period_ids", []) # Preserve existing DRM if it was set by the service, especially when service set Widevine # but manifest only contains PlayReady protection (common scenario for some services) @@ -305,16 +300,346 @@ class DASH: or (existing_drm and not any(isinstance(drm, Widevine) for drm in existing_drm)) ) + pre_existing_keys = {} + if existing_drm: + for drm_obj in existing_drm: + if hasattr(drm_obj, "content_keys") and drm_obj.content_keys: + pre_existing_keys.update(drm_obj.content_keys) + if should_override_drm: track.drm = manifest_drm else: track.drm = existing_drm + if pre_existing_keys and track.drm: + for drm_obj in track.drm: + if hasattr(drm_obj, "content_keys"): + for kid, key in pre_existing_keys.items(): + if kid not in drm_obj.content_keys: + drm_obj.content_keys[kid] = key + + # Collect segments from all content periods in the manifest + all_periods = manifest.findall("Period") + segments: list[tuple[str, Optional[str]]] = [] + segment_durations: list[int] = [] + segment_timescale: float = 0 + init_data: Optional[bytes] = None + track_kid: Optional[UUID] = None + + content_periods = [p for p in all_periods if DASH._is_content_period(p, filtered_period_ids)] + period_count = len(content_periods) + + if period_count > 1: + log.debug(f"Multi-period manifest detected with {period_count} content periods") + + for period_idx, content_period in enumerate(content_periods): + # Find the matching representation in this period + matched_rep = None + matched_as = None + for as_ in content_period.findall("AdaptationSet"): + if DASH.is_trick_mode(as_): + continue + for rep in as_.findall("Representation"): + if rep.get("id") == rep_id: + matched_rep = rep + matched_as = as_ + break + if matched_rep is not None: + break + + if matched_rep is None or matched_as is None: + period_id = content_period.get("id", period_idx) + log.warning(f"Representation '{rep_id}' not found in period '{period_id}', skipping") + continue + + p_init, p_segments, p_timescale, p_durations, p_kid = DASH._get_period_segments( + period=content_period, + adaptation_set=matched_as, + representation=matched_rep, + manifest=manifest, + track=track, + track_url=track.url, + session=session, + ) + + if period_idx == 0: + # First period: use its init data and KID for DRM licensing + init_data = p_init + track_kid = p_kid + segment_timescale = p_timescale + else: + if p_kid and track_kid and p_kid != track_kid: + log.debug(f"Period {content_period.get('id', period_idx)} has different KID: {p_kid}") + + for seg in p_segments: + if seg not in segments: + segments.append(seg) + segment_durations.extend(p_durations) + + if not segments: + log.error("Could not find a way to get segments from this MPD manifest.") + log.debug(track.url) + sys.exit(1) + + # TODO: Should we floor/ceil/round, or is int() ok? + track.data["dash"]["timescale"] = int(segment_timescale) + track.data["dash"]["segment_durations"] = segment_durations + + if not track.drm and init_data and isinstance(track, (Video, Audio)): + prefers_playready = is_playready_cdm(cdm) + if prefers_playready: + try: + track.drm = [PlayReady.from_init_data(init_data)] + except PlayReady.Exceptions.PSSHNotFound: + try: + track.drm = [Widevine.from_init_data(init_data)] + except Widevine.Exceptions.PSSHNotFound: + log.warning("No PlayReady or Widevine PSSH was found for this track, is it DRM free?") + else: + try: + track.drm = [Widevine.from_init_data(init_data)] + except Widevine.Exceptions.PSSHNotFound: + try: + track.drm = [PlayReady.from_init_data(init_data)] + except PlayReady.Exceptions.PSSHNotFound: + log.warning("No Widevine or PlayReady PSSH was found for this track, is it DRM free?") + + if track.drm: + track_kid = track_kid or track.get_key_id(url=segments[0][0], session=session) + drm = track.get_drm_for_cdm(cdm) + if isinstance(drm, (Widevine, PlayReady)): + # license and grab content keys + try: + if not license_widevine: + raise ValueError("license_widevine func must be supplied to use DRM") + progress(downloaded="LICENSING") + license_widevine(drm, track_kid=track_kid) + progress(downloaded="[yellow]LICENSED") + except Exception: # noqa + DOWNLOAD_CANCELLED.set() # skip pending track downloads + progress(downloaded="[red]FAILED") + raise + else: + drm = None + + if DOWNLOAD_LICENCE_ONLY.is_set(): + progress(downloaded="[yellow]SKIPPED") + return + + progress(total=len(segments)) + + downloader = track.downloader + + downloader_args = dict( + urls=[ + {"url": url, "headers": {"Range": f"bytes={bytes_range}"} if bytes_range else {}} + for url, bytes_range in segments + ], + output_dir=save_dir, + filename="{i:0%d}.mp4" % (len(str(len(segments)))), + headers=session.headers, + cookies=session.cookies, + proxy=proxy, + max_workers=max_workers, + session=session, + ) + + debug_logger = get_debug_logger() + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="manifest_dash_download_start", + message="Starting DASH manifest download", + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "total_segments": len(segments), + "has_drm": bool(track.drm), + "drm_types": [drm.__class__.__name__ for drm in (track.drm or [])], + "save_path": str(save_path), + "has_init_data": bool(init_data), + }, + ) + + for status_update in downloader(**downloader_args): + file_downloaded = status_update.get("file_downloaded") + if file_downloaded: + events.emit(events.Types.SEGMENT_DOWNLOADED, track=track, segment=file_downloaded) + else: + downloaded = status_update.get("downloaded") + if downloaded and downloaded.endswith("/s"): + status_update["downloaded"] = f"DASH {downloaded}" + progress(**status_update) + + # Verify output directory exists and contains files + if not save_dir.exists(): + error_msg = f"Output directory does not exist: {save_dir}" + if debug_logger: + debug_logger.log( + level="ERROR", + operation="manifest_dash_download_output_missing", + message=error_msg, + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "save_dir": str(save_dir), + "save_path": str(save_path), + "downloader": "requests", + }, + ) + raise FileNotFoundError(error_msg) + + for control_file in save_dir.glob("*.!dev"): + control_file.unlink(missing_ok=True) + + segments_to_merge = [x for x in sorted(save_dir.iterdir()) if x.is_file()] + + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="manifest_dash_download_complete", + message="DASH download complete, preparing to merge", + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "save_dir": str(save_dir), + "save_dir_exists": save_dir.exists(), + "segments_found": len(segments_to_merge), + "segment_files": [f.name for f in segments_to_merge[:10]], # Limit to first 10 + "downloader": "requests", + }, + ) + + if not segments_to_merge: + error_msg = f"No segment files found in output directory: {save_dir}" + if debug_logger: + # List all contents of the directory for debugging + all_contents = list(save_dir.iterdir()) if save_dir.exists() else [] + debug_logger.log( + level="ERROR", + operation="manifest_dash_download_no_segments", + message=error_msg, + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "save_dir": str(save_dir), + "directory_contents": [str(p) for p in all_contents], + "downloader": "requests", + }, + ) + raise FileNotFoundError(error_msg) + + with open(save_path, "wb") as f: + if init_data: + f.write(init_data) + if len(segments_to_merge) > 1: + progress(downloaded="Merging", completed=0, total=len(segments_to_merge)) + for segment_file in segments_to_merge: + segment_data = segment_file.read_bytes() + if ( + not drm + and isinstance(track, Subtitle) + and track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML) + ): + segment_data = try_ensure_utf8(segment_data) + segment_data = ( + segment_data.decode("utf8") + .replace("‎", html.unescape("‎")) + .replace("‏", html.unescape("‏")) + .encode("utf8") + ) + f.write(segment_data) + f.flush() + segment_file.unlink() + progress(advance=1) + + track.path = save_path + events.emit(events.Types.TRACK_DOWNLOADED, track=track) + + if drm: + progress(downloaded="Decrypting", completed=0, total=100) + drm.decrypt(save_path) + track.drm = None + events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=drm, segment=None) + progress(downloaded="Decrypting", advance=100) + + # Clean up empty segment directory + if save_dir.exists() and save_dir.name.endswith("_segments"): + try: + save_dir.rmdir() + except OSError: + # Directory might not be empty, try removing recursively + shutil.rmtree(save_dir, ignore_errors=True) + + progress(downloaded="Downloaded") + + @staticmethod + def _is_content_period(period: Element, filtered_period_ids: list[str]) -> bool: + """Check if a period is a valid content period (not an ad, not filtered, not trick mode).""" + period_id = period.get("id") + if period_id and period_id in filtered_period_ids: + return False + if next(iter(period.xpath("SegmentType/@value")), "content") != "content": + return False + if "urn:amazon:primevideo:cachingBreadth" in [ + x.get("schemeIdUri") for x in period.findall("SupplementalProperty") + ]: + return False + return True + + @staticmethod + def _merge_segment_templates(adaptation_set: Element, representation: Element) -> Optional[Element]: + """ + Build the effective SegmentTemplate for a Representation by cascading the + AdaptationSet > Representation levels (ISO/IEC 23009-1 5.3.9.1). + + The Representation-level node, when present, is the base; attributes and the + SegmentTimeline child it does not declare are inherited from the AdaptationSet-level + node. Returns None if no SegmentTemplate exists at either level. + """ + levels = [node.find("SegmentTemplate") for node in (adaptation_set, representation)] + present = [node for node in levels if node is not None] + if not present: + return None + + merged = deepcopy(present[-1]) + for ancestor in reversed(present[:-1]): + for attr, value in ancestor.attrib.items(): + if merged.get(attr) is None: + merged.set(attr, value) + if merged.find("SegmentTimeline") is None: + timeline = ancestor.find("SegmentTimeline") + if timeline is not None: + merged.append(deepcopy(timeline)) + return merged + + @staticmethod + def _get_period_segments( + period: Element, + adaptation_set: Element, + representation: Element, + manifest: ElementTree, + track: AnyTrack, + track_url: str, + session: Union[Session, RnetSession], + ) -> tuple[ + Optional[bytes], + list[tuple[str, Optional[str]]], + float, + list[int], + Optional[UUID], + ]: + """ + Extract segments from a single period's representation. + + Returns: + A tuple of (init_data, segments, segment_timescale, segment_durations, track_kid). + """ manifest_base_url = manifest.findtext("BaseURL") if not manifest_base_url: - manifest_base_url = track.url + manifest_base_url = track_url elif not re.match("^https?://", manifest_base_url, re.IGNORECASE): - manifest_base_url = urljoin(track.url, f"./{manifest_base_url}") + manifest_base_url = urljoin(track_url, f"./{manifest_base_url}") period_base_url = urljoin(manifest_base_url, period.findtext("BaseURL") or "") adaptation_set_base_url = urljoin(period_base_url, adaptation_set.findtext("BaseURL") or "") rep_base_url = urljoin(adaptation_set_base_url, representation.findtext("BaseURL") or "") @@ -322,9 +647,7 @@ class DASH: period_duration = period.get("duration") or manifest.get("mediaPresentationDuration") init_data: Optional[bytes] = None - segment_template = representation.find("SegmentTemplate") - if segment_template is None: - segment_template = adaptation_set.find("SegmentTemplate") + segment_template = DASH._merge_segment_templates(adaptation_set, representation) segment_list = representation.find("SegmentList") if segment_list is None: @@ -340,7 +663,6 @@ class DASH: track_kid: Optional[UUID] = None if segment_template is not None: - segment_template = copy(segment_template) start_number = int(segment_template.get("startNumber") or 1) end_number = int(segment_template.get("endNumber") or 0) or None segment_timeline = segment_template.find("SegmentTimeline") @@ -355,7 +677,7 @@ class DASH: raise ValueError("Resolved Segment URL is not absolute, and no Base URL is available.") value = urljoin(rep_base_url, value) if not urlparse(value).query: - manifest_url_query = urlparse(track.url).query + manifest_url_query = urlparse(track_url).query if manifest_url_query: value += f"?{manifest_url_query}" segment_template.set(item, value) @@ -477,255 +799,8 @@ class DASH: segments.append((rep_base_url, media_range)) elif rep_base_url: segments.append((rep_base_url, None)) - else: - log.error("Could not find a way to get segments from this MPD manifest.") - log.debug(track.url) - sys.exit(1) - # TODO: Should we floor/ceil/round, or is int() ok? - track.data["dash"]["timescale"] = int(segment_timescale) - track.data["dash"]["segment_durations"] = segment_durations - - if not track.drm and init_data and isinstance(track, (Video, Audio)): - prefers_playready = is_playready_cdm(cdm) - if prefers_playready: - try: - track.drm = [PlayReady.from_init_data(init_data)] - except PlayReady.Exceptions.PSSHNotFound: - try: - track.drm = [Widevine.from_init_data(init_data)] - except Widevine.Exceptions.PSSHNotFound: - log.warning("No PlayReady or Widevine PSSH was found for this track, is it DRM free?") - else: - try: - track.drm = [Widevine.from_init_data(init_data)] - except Widevine.Exceptions.PSSHNotFound: - try: - track.drm = [PlayReady.from_init_data(init_data)] - except PlayReady.Exceptions.PSSHNotFound: - log.warning("No Widevine or PlayReady PSSH was found for this track, is it DRM free?") - - if track.drm: - track_kid = track_kid or track.get_key_id(url=segments[0][0], session=session) - drm = track.get_drm_for_cdm(cdm) - if isinstance(drm, (Widevine, PlayReady)): - # license and grab content keys - try: - if not license_widevine: - raise ValueError("license_widevine func must be supplied to use DRM") - progress(downloaded="LICENSING") - license_widevine(drm, track_kid=track_kid) - progress(downloaded="[yellow]LICENSED") - except Exception: # noqa - DOWNLOAD_CANCELLED.set() # skip pending track downloads - progress(downloaded="[red]FAILED") - raise - else: - drm = None - - if DOWNLOAD_LICENCE_ONLY.is_set(): - progress(downloaded="[yellow]SKIPPED") - return - - progress(total=len(segments)) - - downloader = track.downloader - if downloader.__name__ == "aria2c" and any(bytes_range is not None for url, bytes_range in segments): - # aria2(c) is shit and doesn't support the Range header, fallback to the requests downloader - downloader = requests_downloader - log.warning("Falling back to the requests downloader as aria2(c) doesn't support the Range header") - - downloader_args = dict( - urls=[ - {"url": url, "headers": {"Range": f"bytes={bytes_range}"} if bytes_range else {}} - for url, bytes_range in segments - ], - output_dir=save_dir, - filename="{i:0%d}.mp4" % (len(str(len(segments)))), - headers=session.headers, - cookies=session.cookies, - proxy=proxy, - max_workers=max_workers, - ) - - skip_merge = False - if downloader.__name__ == "n_m3u8dl_re": - skip_merge = True - - # When periods were filtered out during to_tracks(), n_m3u8dl_re will re-parse - # the raw MPD and download ALL periods (including ads/pre-rolls). Write a filtered - # MPD with the rejected periods removed so n_m3u8dl_re downloads the correct content. - filtered_period_ids = track.data.get("dash", {}).get("filtered_period_ids", []) - if filtered_period_ids: - filtered_manifest = deepcopy(manifest) - for child in list(filtered_manifest): - if not hasattr(child.tag, "find"): - continue - if child.tag == "Period" and child.get("id") in filtered_period_ids: - filtered_manifest.remove(child) - - filtered_mpd_path = save_dir / f".{track.id}_filtered.mpd" - filtered_mpd_path.parent.mkdir(parents=True, exist_ok=True) - etree.ElementTree(filtered_manifest).write( - str(filtered_mpd_path), xml_declaration=True, encoding="utf-8" - ) - track.from_file = filtered_mpd_path - - downloader_args.update( - { - "filename": track.id, - "track": track, - "content_keys": drm.content_keys if drm else None, - } - ) - - debug_logger = get_debug_logger() - if debug_logger: - debug_logger.log( - level="DEBUG", - operation="manifest_dash_download_start", - message="Starting DASH manifest download", - context={ - "track_id": getattr(track, "id", None), - "track_type": track.__class__.__name__, - "total_segments": len(segments), - "downloader": downloader.__name__, - "has_drm": bool(track.drm), - "drm_types": [drm.__class__.__name__ for drm in (track.drm or [])], - "skip_merge": skip_merge, - "save_path": str(save_path), - "has_init_data": bool(init_data), - }, - ) - - for status_update in downloader(**downloader_args): - file_downloaded = status_update.get("file_downloaded") - if file_downloaded: - events.emit(events.Types.SEGMENT_DOWNLOADED, track=track, segment=file_downloaded) - else: - downloaded = status_update.get("downloaded") - if downloaded and downloaded.endswith("/s"): - status_update["downloaded"] = f"DASH {downloaded}" - progress(**status_update) - - # Clean up filtered MPD temp file before enumerating segments - filtered_mpd_path = save_dir / f".{track.id}_filtered.mpd" - if filtered_mpd_path.exists(): - filtered_mpd_path.unlink() - - # see https://github.com/devine-dl/devine/issues/71 - for control_file in save_dir.glob("*.aria2__temp"): - control_file.unlink() - - # Verify output directory exists and contains files - if not save_dir.exists(): - error_msg = f"Output directory does not exist: {save_dir}" - if debug_logger: - debug_logger.log( - level="ERROR", - operation="manifest_dash_download_output_missing", - message=error_msg, - context={ - "track_id": getattr(track, "id", None), - "track_type": track.__class__.__name__, - "save_dir": str(save_dir), - "save_path": str(save_path), - "downloader": downloader.__name__, - "skip_merge": skip_merge, - }, - ) - raise FileNotFoundError(error_msg) - - segments_to_merge = [x for x in sorted(save_dir.iterdir()) if x.is_file()] - - if debug_logger: - debug_logger.log( - level="DEBUG", - operation="manifest_dash_download_complete", - message="DASH download complete, preparing to merge", - context={ - "track_id": getattr(track, "id", None), - "track_type": track.__class__.__name__, - "save_dir": str(save_dir), - "save_dir_exists": save_dir.exists(), - "segments_found": len(segments_to_merge), - "segment_files": [f.name for f in segments_to_merge[:10]], # Limit to first 10 - "downloader": downloader.__name__, - "skip_merge": skip_merge, - }, - ) - - if not segments_to_merge: - error_msg = f"No segment files found in output directory: {save_dir}" - if debug_logger: - # List all contents of the directory for debugging - all_contents = list(save_dir.iterdir()) if save_dir.exists() else [] - debug_logger.log( - level="ERROR", - operation="manifest_dash_download_no_segments", - message=error_msg, - context={ - "track_id": getattr(track, "id", None), - "track_type": track.__class__.__name__, - "save_dir": str(save_dir), - "directory_contents": [str(p) for p in all_contents], - "downloader": downloader.__name__, - "skip_merge": skip_merge, - }, - ) - raise FileNotFoundError(error_msg) - - if skip_merge: - # N_m3u8DL-RE handles merging and decryption internally - shutil.move(segments_to_merge[0], save_path) - if drm: - track.drm = None - events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=drm, segment=None) - else: - with open(save_path, "wb") as f: - if init_data: - f.write(init_data) - if len(segments_to_merge) > 1: - progress(downloaded="Merging", completed=0, total=len(segments_to_merge)) - for segment_file in segments_to_merge: - segment_data = segment_file.read_bytes() - # TODO: fix encoding after decryption? - if ( - not drm - and isinstance(track, Subtitle) - and track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML) - ): - segment_data = try_ensure_utf8(segment_data) - segment_data = ( - segment_data.decode("utf8") - .replace("‎", html.unescape("‎")) - .replace("‏", html.unescape("‏")) - .encode("utf8") - ) - f.write(segment_data) - f.flush() - segment_file.unlink() - progress(advance=1) - - track.path = save_path - events.emit(events.Types.TRACK_DOWNLOADED, track=track) - - if not skip_merge and drm: - progress(downloaded="Decrypting", completed=0, total=100) - drm.decrypt(save_path) - track.drm = None - events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=drm, segment=None) - progress(downloaded="Decrypting", advance=100) - - # Clean up empty segment directory - if save_dir.exists() and save_dir.name.endswith("_segments"): - try: - save_dir.rmdir() - except OSError: - # Directory might not be empty, try removing recursively - shutil.rmtree(save_dir, ignore_errors=True) - - progress(downloaded="Downloaded") + return init_data, segments, segment_timescale, segment_durations, track_kid @staticmethod def _get(item: str, adaptation_set: Element, representation: Optional[Element] = None) -> Optional[Any]: @@ -801,6 +876,10 @@ class DASH: def get_video_range( codecs: str, all_supplemental_props: list[Element], all_essential_props: list[Element] ) -> Video.Range: + # TODO: Detect Dolby Vision composite streams in DASH manifests (DV RPU embedded but + # primary codec is plain hvc1, signaled via a separate AdaptationSet/Representation + # with DV codec strings or DolbyVisionConfigurationBox). When found, mark the track + # with dv_compatible_bitstream=True so DVFixup runs pre-mux. No DASH samples seen yet. if codecs.startswith(("dva1", "dvav", "dvhe", "dvh1")): return Video.Range.DV @@ -924,7 +1003,7 @@ class DASH: None, ) - if kid and (not pssh.key_ids or all(k.int == 0 or k in PLACEHOLDER_KIDS for k in pssh.key_ids)): + if kid and pssh.key_ids and all(k.int == 0 or k in PLACEHOLDER_KIDS for k in pssh.key_ids): pssh.set_key_ids([kid]) drm.append(Widevine(pssh=pssh, kid=kid)) diff --git a/packages/envied/src/envied/core/manifests/hls.py b/packages/envied/src/envied/core/manifests/hls.py index bc6fe6e..faa4c9e 100644 --- a/packages/envied/src/envied/core/manifests/hls.py +++ b/packages/envied/src/envied/core/manifests/hls.py @@ -5,6 +5,7 @@ import html import json import logging import os +import re import shutil import subprocess import sys @@ -17,8 +18,6 @@ from zlib import crc32 import m3u8 import requests -from curl_cffi.requests import Response as CurlResponse -from curl_cffi.requests import Session as CurlSession from langcodes import Language, tag_is_valid from m3u8 import M3U8 from pyplayready.cdm import Cdm as PlayReadyCdm @@ -30,15 +29,23 @@ from requests import Session from envied.core import binaries from envied.core.cdm.detect import is_playready_cdm, is_widevine_cdm from envied.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack -from envied.core.downloaders import requests as requests_downloader from envied.core.drm import DRM_T, ClearKey, MonaLisa, PlayReady, Widevine from envied.core.events import events +from envied.core.session import RnetResponse, RnetSession from envied.core.tracks import Audio, Subtitle, Tracks, Video from envied.core.utilities import get_debug_logger, get_extension, is_close_match, try_ensure_utf8 class HLS: - def __init__(self, manifest: M3U8, session: Optional[Union[Session, CurlSession]] = None): + SUPP_CODECS_RE = re.compile(r'SUPPLEMENTAL-CODECS="([^"]+)"', re.IGNORECASE) + + def __init__( + self, + manifest: M3U8, + session: Optional[Union[Session, RnetSession]] = None, + url: Optional[str] = None, + raw_text: Optional[str] = None, + ): if not manifest: raise ValueError("HLS manifest must be provided.") if not isinstance(manifest, M3U8): @@ -48,9 +55,11 @@ class HLS: self.manifest = manifest self.session = session or Session() + self.url = url + self.raw_text = raw_text @classmethod - def from_url(cls, url: str, session: Optional[Union[Session, CurlSession]] = None, **args: Any) -> HLS: + def from_url(cls, url: str, session: Optional[Union[Session, RnetSession]] = None, **args: Any) -> HLS: if not url: raise requests.URLRequired("HLS manifest URL must be provided.") if not isinstance(url, str): @@ -58,26 +67,26 @@ class HLS: if not session: session = Session() - elif not isinstance(session, (Session, CurlSession)): - raise TypeError(f"Expected session to be a {Session} or {CurlSession}, not {session!r}") + elif not isinstance(session, (Session, RnetSession)): + raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {session!r}") res = session.get(url, **args) - # Handle requests and curl_cffi response objects + # Handle requests and rnet response objects if isinstance(res, requests.Response): if not res.ok: raise requests.ConnectionError("Failed to request the M3U(8) document.", response=res) content = res.text - elif isinstance(res, CurlResponse): + elif isinstance(res, RnetResponse): if not res.ok: raise requests.ConnectionError("Failed to request the M3U(8) document.", response=res) content = res.text else: - raise TypeError(f"Expected response to be a requests.Response or curl_cffi.Response, not {type(res)}") + raise TypeError(f"Expected response to be a requests.Response or rnet.Response, not {type(res)}") master = m3u8.loads(content, uri=url) - return cls(master, session) + return cls(master, session, url=url, raw_text=content) @classmethod def from_text(cls, text: str, url: str) -> HLS: @@ -93,7 +102,31 @@ class HLS: master = m3u8.loads(text, uri=url) - return cls(master) + return cls(master, raw_text=text) + + def supplemental_codecs_by_uri(self) -> dict[str, str]: + """Map each variant URI to its SUPPLEMENTAL-CODECS value. + + python-m3u8 drops this attribute, so we re-parse the raw text to recover it for + Dolby Vision composite detection (dvh1.08.x advertised only in SUPPLEMENTAL-CODECS + while primary CODECS stays plain hvc1). + """ + if not self.raw_text: + return {} + out: dict[str, str] = {} + lines = self.raw_text.splitlines() + for i, line in enumerate(lines): + if not line.startswith("#EXT-X-STREAM-INF"): + continue + supp_match = self.SUPP_CODECS_RE.search(line) + if not supp_match: + continue + for j in range(i + 1, len(lines)): + uri = lines[j].strip() + if uri and not uri.startswith("#"): + out[uri] = supp_match.group(1) + break + return out def to_tracks(self, language: Union[str, Language]) -> Tracks: """ @@ -115,14 +148,19 @@ class HLS: cc_by_group_id: dict[str, list[dict[str, Any]]] = {} for media in self.manifest.media: if media.type == "CLOSED-CAPTIONS": - cc_by_group_id.setdefault(media.group_id, []).append({ - "language": media.language, - "name": media.name, - "instream_id": media.instream_id, - "characteristics": media.characteristics, - }) + cc_by_group_id.setdefault(media.group_id, []).append( + { + "language": media.language, + "name": media.name, + "instream_id": media.instream_id, + "characteristics": media.characteristics, + } + ) tracks = Tracks() + supplemental_codecs = self.supplemental_codecs_by_uri() + dv_supp_prefixes = ("dva1", "dvav", "dvhe", "dvh1") + for playlist in self.manifest.playlists: audio_group = playlist.stream_info.audio audio_codec: Optional[Audio.Codec] = None @@ -143,6 +181,26 @@ class HLS: else: primary_track_type = Video + primary_codecs = (playlist.stream_info.codecs or "").lower() + primary_has_dv = any(codec.split(".")[0] in dv_supp_prefixes for codec in primary_codecs.split(",")) + + supp_codecs_str = supplemental_codecs.get(playlist.uri, "") + supp_dv_codec: Optional[str] = None + for codec in supp_codecs_str.lower().split(","): + token = codec.strip().split("/")[0] + if token.split(".")[0] in dv_supp_prefixes: + supp_dv_codec = token + break + + video_range = ( + Video.Range.DV if primary_has_dv else Video.Range.from_m3u_range_tag(playlist.stream_info.video_range) + ) + # DV-composite track: primary codec is plain HEVC but SUPPLEMENTAL-CODECS advertises + # a DV codec. Range stays whatever VIDEO-RANGE signaled (HDR10/HLG/SDR); DVFixup will + # restore DV signaling post-download. Services that know their encoder embeds HDR10+ + # SEI must override `range` themselves (see services/ATV). + dv_compatible_bitstream = primary_track_type is Video and not primary_has_dv and supp_dv_codec is not None + tracks.add( primary_track_type( id_=hex(crc32(str(playlist).encode()))[2:], @@ -161,18 +219,14 @@ class HLS: # video track args **( dict( - range_=Video.Range.DV - if any( - codec.split(".")[0] in ("dva1", "dvav", "dvhe", "dvh1") - for codec in (playlist.stream_info.codecs or "").lower().split(",") - ) - else Video.Range.from_m3u_range_tag(playlist.stream_info.video_range), + range_=video_range, width=playlist.stream_info.resolution[0] if playlist.stream_info.resolution else None, height=playlist.stream_info.resolution[1] if playlist.stream_info.resolution else None, fps=playlist.stream_info.frame_rate, closed_captions=cc_by_group_id.get( (playlist.stream_info.closed_captions or "").strip('"'), [] ), + dv_compatible_bitstream=dv_compatible_bitstream, ) if primary_track_type is Video else {} @@ -241,40 +295,200 @@ class HLS: ) ) + for video in tracks.videos: + has_resolution = video.width and video.height + has_codec = video.codec is not None + if has_resolution and has_codec: + continue + try: + probe = HLS._probe_ts_info(video.url, self.session) + if probe: + width, height, codec = probe + if not has_resolution: + video.width, video.height = width, height + if not has_codec: + video.codec = codec + except Exception: + pass + + if self.url: + tracks.manifest_url = self.url return tracks @staticmethod - def _finalize_n_m3u8dl_re_output(*, track: AnyTrack, save_dir: Path, save_path: Path) -> Path: - """ - Finalize output from N_m3u8DL-RE. + def _probe_ts_info( + variant_url: str, session: Optional[Union[Session, RnetSession]] = None + ) -> Optional[tuple[int, int, Video.Codec]]: + """Probe the first TS segment of a variant playlist to extract resolution and codec.""" + if not session: + session = Session() - We call N_m3u8DL-RE with `--save-name track.id`, so the final file should be `{track.id}.*` under `save_dir`. - This moves that output to `save_path` (preserving the real suffix) and, for subtitles, updates `track.codec` - to match the produced file extension. - """ - matches = [p for p in save_dir.rglob(f"{track.id}.*") if p.is_file()] - if not matches: - raise FileNotFoundError(f"No output files produced by N_m3u8DL-RE for save-name={track.id} in: {save_dir}") + res = session.get(variant_url) + variant = m3u8.loads(res.text if hasattr(res, "text") else res.text, uri=variant_url) + if not variant.segments: + return None - primary = max(matches, key=lambda p: p.stat().st_size) + seg_uri = urljoin(variant_url, variant.segments[0].uri) - final_save_path = save_path.with_suffix(primary.suffix) if primary.suffix else save_path + # Download only the first 8KB — SPS is always near the start of the first TS packet + res = session.get(seg_uri, headers={"Range": "bytes=0-8191"}) + data = res.content - final_save_path.parent.mkdir(parents=True, exist_ok=True) - if primary.absolute() != final_save_path.absolute(): - final_save_path.unlink(missing_ok=True) - shutil.move(str(primary), str(final_save_path)) + return HLS._parse_ts_video_info(data) - if isinstance(track, Subtitle): - ext = final_save_path.suffix.lower().lstrip(".") - try: - track.codec = Subtitle.Codec.from_mime(ext) - except ValueError: - pass + @staticmethod + def _parse_ts_video_info(data: bytes) -> Optional[tuple[int, int, Video.Codec]]: + """Parse H.264/H.265 NAL units from TS segment data to extract resolution and codec.""" - shutil.rmtree(save_dir, ignore_errors=True) + class _BitReader: + def __init__(self, buf: bytes) -> None: + self.data = buf + self.pos = 0 - return final_save_path + def bits(self, n: int) -> int: + val = 0 + for _ in range(n): + val = (val << 1) | ((self.data[self.pos >> 3] >> (7 - (self.pos & 7))) & 1) + self.pos += 1 + return val + + def ue(self) -> int: + zeros = 0 + while self.bits(1) == 0: + zeros += 1 + return (1 << zeros) - 1 + self.bits(zeros) if zeros else 0 + + def se(self) -> int: + val = self.ue() + return (val + 1) // 2 if val & 1 else -(val // 2) + + # Find SPS NAL unit via start code + # H.264: NAL type 7 (SPS), identified by byte & 0x1F == 7 + # H.265: NAL type 33 (SPS), identified by (byte >> 1) & 0x3F == 33 + for i in range(len(data) - 4): + start3 = data[i : i + 3] == b"\x00\x00\x01" + start4 = data[i : i + 4] == b"\x00\x00\x00\x01" + if not start3 and not start4: + continue + offset = i + (4 if start4 else 3) + if offset >= len(data): + continue + + nal_byte = data[offset] + h264_type = nal_byte & 0x1F + h265_type = (nal_byte >> 1) & 0x3F + + # H.264 SPS (NAL type 7) + if h264_type == 7: + sps = data[offset : offset + 64] + if len(sps) < 5: + continue + try: + r = _BitReader(sps[1:]) # skip NAL header byte + profile = r.bits(8) + r.bits(8) # constraint flags + r.bits(8) # level + r.ue() # sps_id + + if profile in (100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134): + chroma = r.ue() + if chroma == 3: + r.bits(1) + r.ue() # bit_depth_luma + r.ue() # bit_depth_chroma + r.bits(1) # qpprime_y_zero_transform_bypass + if r.bits(1): # scaling_matrix_present + for j in range(6 if chroma != 3 else 12): + if r.bits(1): + last = 8 + for _ in range(16 if j < 6 else 64): + if last != 0: + last = (last + r.se()) & 0xFF + + r.ue() # log2_max_frame_num + poc_type = r.ue() + if poc_type == 0: + r.ue() + elif poc_type == 1: + r.bits(1) + r.se() + r.se() + for _ in range(r.ue()): + r.se() + + r.ue() # max_num_ref_frames + r.bits(1) # gaps_in_frame_num + + w_mbs = r.ue() + 1 + h_map = r.ue() + 1 + frame_mbs_only = r.bits(1) + if not frame_mbs_only: + r.bits(1) + r.bits(1) # direct_8x8_inference + + cl = cr = ct = cb = 0 + if r.bits(1): # crop + cl, cr, ct, cb = r.ue(), r.ue(), r.ue(), r.ue() + + width = w_mbs * 16 - (cl + cr) * 2 + height = (2 - frame_mbs_only) * h_map * 16 - (ct + cb) * 2 + return (width, height, Video.Codec.AVC) + except (IndexError, ValueError): + continue + + # H.265 SPS (NAL type 33) + elif h265_type == 33: + sps = data[offset : offset + 128] + if len(sps) < 10: + continue + try: + r = _BitReader(sps[2:]) # skip 2-byte NAL header + r.bits(4) # sps_video_parameter_set_id + max_sub_layers = r.bits(3) + r.bits(1) # sps_temporal_id_nesting + + # profile_tier_level + r.bits(2) # general_profile_space + r.bits(1) # general_tier + r.bits(5) # general_profile_idc + r.bits(32) # general_profile_compatibility_flags + r.bits(48) # general_constraint_indicator_flags + r.bits(8) # general_level_idc + sub_layer_flags = [] + for _ in range(max_sub_layers - 1): + sub_layer_flags.append((r.bits(1), r.bits(1))) + if max_sub_layers - 1 > 0: + for _ in range(8 - (max_sub_layers - 1)): + r.bits(2) + for profile_present, level_present in sub_layer_flags: + if profile_present: + r.bits(2 + 1 + 5 + 32 + 48 + 8) + if level_present: + r.bits(8) + + r.ue() # sps_seq_parameter_set_id + chroma = r.ue() + if chroma == 3: + r.bits(1) + + width = r.ue() + height = r.ue() + + if r.bits(1): # conformance_window + cl = r.ue() + cr = r.ue() + ct = r.ue() + cb = r.ue() + sub_w = 2 if chroma in (1, 2) else 1 + sub_h = 2 if chroma == 1 else 1 + width -= (cl + cr) * sub_w + height -= (ct + cb) * sub_h + + return (width, height, Video.Codec.HEVC) + except (IndexError, ValueError): + continue + + return None @staticmethod def download_track( @@ -282,7 +496,7 @@ class HLS: save_path: Path, save_dir: Path, progress: partial, - session: Optional[Union[Session, CurlSession]] = None, + session: Optional[Union[Session, RnetSession]] = None, proxy: Optional[str] = None, max_workers: Optional[int] = None, license_widevine: Optional[Callable] = None, @@ -291,8 +505,8 @@ class HLS: ) -> None: if not session: session = Session() - elif not isinstance(session, (Session, CurlSession)): - raise TypeError(f"Expected session to be a {Session} or {CurlSession}, not {session!r}") + elif not isinstance(session, (Session, RnetSession)): + raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {session!r}") if proxy: # Handle proxies differently based on session type @@ -306,15 +520,13 @@ class HLS: else: # Get the playlist text and handle both session types response = session.get(track.url) - if isinstance(response, requests.Response) or isinstance(response, CurlResponse): + if isinstance(response, requests.Response) or isinstance(response, RnetResponse): if not response.ok: log.error(f"Failed to request the invariant M3U8 playlist: {response.status_code}") sys.exit(1) playlist_text = response.text else: - raise TypeError( - f"Expected response to be a requests.Response or curl_cffi.Response, not {type(response)}" - ) + raise TypeError(f"Expected response to be a requests.Response or rnet.Response, not {type(response)}") master = m3u8.loads(playlist_text, uri=track.url) @@ -339,6 +551,12 @@ class HLS: media_drm = HLS.get_drm(media_playlist_key, session) if isinstance(media_drm, (Widevine, PlayReady)): track_kid = HLS.get_track_kid_from_init(master, track, session) or media_drm.kid + # Preserve pre-existing keys (e.g. from server_cdm) + if track.drm: + for existing_drm in track.drm: + if hasattr(existing_drm, "content_keys") and existing_drm.content_keys: + media_drm.content_keys.update(existing_drm.content_keys) + track.drm = [media_drm] try: if not license_widevine: raise ValueError("license_widevine func must be supplied to use DRM") @@ -347,7 +565,6 @@ class HLS: progress(downloaded="[yellow]LICENSED") initial_drm_licensed = True initial_drm_key = media_playlist_key - track.drm = [media_drm] session_drm = media_drm except Exception: # noqa DOWNLOAD_CANCELLED.set() # skip pending track downloads @@ -359,8 +576,9 @@ class HLS: try: if not license_widevine: raise ValueError("license_widevine func must be supplied to use DRM") + track_kid = HLS.get_track_kid_from_init(master, track, session) or session_drm.kid progress(downloaded="LICENSING") - license_widevine(session_drm) + license_widevine(session_drm, track_kid=track_kid) progress(downloaded="[yellow]LICENSED") except Exception: # noqa DOWNLOAD_CANCELLED.set() # skip pending track downloads @@ -391,9 +609,6 @@ class HLS: progress(total=total_segments) downloader = track.downloader - if downloader.__name__ == "aria2c" and any(x.byterange for x in master.segments if x not in unwanted_segments): - downloader = requests_downloader - log.warning("Falling back to the requests downloader as aria2(c) doesn't support the Range header") urls: list[dict[str, Any]] = [] segment_durations: list[int] = [] @@ -422,7 +637,6 @@ class HLS: segment_save_dir = save_dir / "segments" - skip_merge = False downloader_args = dict( urls=urls, output_dir=segment_save_dir, @@ -431,22 +645,9 @@ class HLS: cookies=session.cookies, proxy=proxy, max_workers=max_workers, + session=session, ) - if downloader.__name__ == "n_m3u8dl_re": - skip_merge = True - # session_drm already has correct content_keys from initial licensing above - n_m3u8dl_content_keys = session_drm.content_keys if session_drm else None - - downloader_args.update( - { - "output_dir": save_dir, - "filename": track.id, - "track": track, - "content_keys": n_m3u8dl_content_keys, - } - ) - debug_logger = get_debug_logger() if debug_logger: debug_logger.log( @@ -457,10 +658,8 @@ class HLS: "track_id": getattr(track, "id", None), "track_type": track.__class__.__name__, "total_segments": total_segments, - "downloader": downloader.__name__, "has_drm": bool(session_drm), "drm_type": session_drm.__class__.__name__ if session_drm else None, - "skip_merge": skip_merge, "save_path": str(save_path), }, ) @@ -475,16 +674,8 @@ class HLS: status_update["downloaded"] = f"HLS {downloaded}" progress(**status_update) - # see https://github.com/devine-dl/devine/issues/71 - for control_file in segment_save_dir.glob("*.aria2__temp"): - control_file.unlink() - - if skip_merge: - final_save_path = HLS._finalize_n_m3u8dl_re_output(track=track, save_dir=save_dir, save_path=save_path) - progress(downloaded="Downloaded") - track.path = final_save_path - events.emit(events.Types.TRACK_DOWNLOADED, track=track) - return + for control_file in segment_save_dir.glob("*.!dev"): + control_file.unlink(missing_ok=True) progress(total=total_segments, completed=0, downloaded="Merging") @@ -644,13 +835,11 @@ class HLS: ) # Check response based on session type - if isinstance(res, requests.Response) or isinstance(res, CurlResponse): + if isinstance(res, requests.Response) or isinstance(res, RnetResponse): res.raise_for_status() init_content = res.content else: - raise TypeError( - f"Expected response to be requests.Response or curl_cffi.Response, not {type(res)}" - ) + raise TypeError(f"Expected response to be requests.Response or rnet.Response, not {type(res)}") map_data = (segment.init_section, init_content) @@ -687,6 +876,14 @@ class HLS: DOWNLOAD_CANCELLED.set() # skip pending track downloads progress(downloaded="[red]FAILED") raise + if ( + encryption_data + and isinstance(drm, (Widevine, PlayReady)) + and isinstance(encryption_data[1], type(drm)) + and getattr(encryption_data[1], "content_keys", None) + ): + for prev_kid, prev_key in encryption_data[1].content_keys.items(): + drm.content_keys.setdefault(prev_kid, prev_key) encryption_data = (key, drm) if DOWNLOAD_LICENCE_ONLY.is_set(): @@ -736,8 +933,7 @@ class HLS: "save_dir_exists": save_dir.exists(), "segments_found": len(segments_to_merge), "segment_files": [f.name for f in segments_to_merge[:10]], # Limit to first 10 - "downloader": downloader.__name__, - "skip_merge": skip_merge, + "downloader": "requests", }, ) @@ -755,8 +951,7 @@ class HLS: "save_dir": str(save_dir), "save_dir_exists": save_dir.exists(), "directory_contents": [str(p) for p in all_contents], - "downloader": downloader.__name__, - "skip_merge": skip_merge, + "downloader": "requests", }, ) raise FileNotFoundError(error_msg) @@ -787,6 +982,10 @@ class HLS: progress(downloaded="Downloaded") track.path = save_path + + if session_drm: + track.drm = None + events.emit(events.Types.TRACK_DOWNLOADED, track=track) @staticmethod @@ -865,7 +1064,7 @@ class HLS: @staticmethod def parse_session_data_keys( - manifest: M3U8, session: Optional[Union[Session, CurlSession]] = None + manifest: M3U8, session: Optional[Union[Session, RnetSession]] = None ) -> list[m3u8.model.Key]: """Parse `com.apple.hls.keys` session data and return Key objects.""" keys: list[m3u8.model.Key] = [] @@ -940,7 +1139,7 @@ class HLS: def get_track_kid_from_init( master: M3U8, track: AnyTrack, - session: Union[Session, CurlSession], + session: Union[Session, RnetSession], ) -> Optional[UUID]: """ Extract the track's Key ID from its init segment (EXT-X-MAP). @@ -1007,7 +1206,7 @@ class HLS: @staticmethod def get_drm( key: Union[m3u8.model.SessionKey, m3u8.model.Key], - session: Optional[Union[Session, CurlSession]] = None, + session: Optional[Union[Session, RnetSession]] = None, ) -> DRM_T: """ Convert HLS EXT-X-KEY data to an initialized DRM object. @@ -1019,8 +1218,8 @@ class HLS: Raises a NotImplementedError if the key system is not supported. """ - if not isinstance(session, (Session, CurlSession, type(None))): - raise TypeError(f"Expected session to be a {Session} or {CurlSession}, not {type(session)}") + if not isinstance(session, (Session, RnetSession, type(None))): + raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {type(session)}") if not session: session = Session() diff --git a/packages/envied/src/envied/core/manifests/ism.py b/packages/envied/src/envied/core/manifests/ism.py index 38e2395..2ba9378 100644 --- a/packages/envied/src/envied/core/manifests/ism.py +++ b/packages/envied/src/envied/core/manifests/ism.py @@ -3,14 +3,12 @@ from __future__ import annotations import base64 import hashlib import html -import shutil import urllib.parse from functools import partial from pathlib import Path from typing import Any, Callable, Optional, Union import requests -from curl_cffi.requests import Session as CurlSession from langcodes import Language, tag_is_valid from lxml.etree import Element from pyplayready.system.pssh import PSSH as PR_PSSH @@ -20,6 +18,7 @@ from requests import Session from envied.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack from envied.core.drm import DRM_T, PlayReady, Widevine from envied.core.events import events +from envied.core.session import RnetSession from envied.core.tracks import Audio, Subtitle, Track, Tracks, Video from envied.core.utilities import get_debug_logger, try_ensure_utf8 from envied.core.utils.xml import load_xml @@ -35,13 +34,13 @@ class ISM: self.url = url @classmethod - def from_url(cls, url: str, session: Optional[Union[Session, CurlSession]] = None, **kwargs: Any) -> "ISM": + def from_url(cls, url: str, session: Optional[Union[Session, RnetSession]] = None, **kwargs: Any) -> "ISM": if not url: raise requests.URLRequired("ISM manifest URL must be provided") if not session: session = Session() - elif not isinstance(session, (Session, CurlSession)): - raise TypeError(f"Expected session to be a {Session} or {CurlSession}, not {session!r}") + elif not isinstance(session, (Session, RnetSession)): + raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {session!r}") res = session.get(url, **kwargs) if res.url != url: url = res.url @@ -220,6 +219,7 @@ class ISM: data=data, ) ) + tracks.manifest_url = self.url return tracks @staticmethod @@ -269,7 +269,6 @@ class ISM: progress(total=len(segments)) downloader = track.downloader - skip_merge = False downloader_args = dict( urls=[{"url": url} for url in segments], output_dir=save_dir, @@ -278,18 +277,9 @@ class ISM: cookies=session.cookies, proxy=proxy, max_workers=max_workers, + session=session, ) - if downloader.__name__ == "n_m3u8dl_re": - skip_merge = True - downloader_args.update( - { - "filename": track.id, - "track": track, - "content_keys": session_drm.content_keys if session_drm else None, - } - ) - debug_logger = get_debug_logger() if debug_logger: debug_logger.log( @@ -300,10 +290,9 @@ class ISM: "track_id": getattr(track, "id", None), "track_type": track.__class__.__name__, "total_segments": len(segments), - "downloader": downloader.__name__, + "downloader": "requests", "has_drm": bool(session_drm), "drm_type": session_drm.__class__.__name__ if session_drm else None, - "skip_merge": skip_merge, "save_path": str(save_path), }, ) @@ -318,9 +307,6 @@ class ISM: status_update["downloaded"] = f"ISM {downloaded}" progress(**status_update) - for control_file in save_dir.glob("*.aria2__temp"): - control_file.unlink() - # Verify output directory exists and contains files if not save_dir.exists(): error_msg = f"Output directory does not exist: {save_dir}" @@ -334,12 +320,14 @@ class ISM: "track_type": track.__class__.__name__, "save_dir": str(save_dir), "save_path": str(save_path), - "downloader": downloader.__name__, - "skip_merge": skip_merge, + "downloader": "requests", }, ) raise FileNotFoundError(error_msg) + for control_file in save_dir.glob("*.!dev"): + control_file.unlink(missing_ok=True) + segments_to_merge = [x for x in sorted(save_dir.iterdir()) if x.is_file()] if debug_logger: @@ -354,8 +342,7 @@ class ISM: "save_dir_exists": save_dir.exists(), "segments_found": len(segments_to_merge), "segment_files": [f.name for f in segments_to_merge[:10]], # Limit to first 10 - "downloader": downloader.__name__, - "skip_merge": skip_merge, + "downloader": "requests", }, ) @@ -372,39 +359,35 @@ class ISM: "track_type": track.__class__.__name__, "save_dir": str(save_dir), "directory_contents": [str(p) for p in all_contents], - "downloader": downloader.__name__, - "skip_merge": skip_merge, + "downloader": "requests", }, ) raise FileNotFoundError(error_msg) - if skip_merge: - shutil.move(segments_to_merge[0], save_path) - else: - with open(save_path, "wb") as f: - for segment_file in segments_to_merge: - segment_data = segment_file.read_bytes() - if ( - not session_drm - and isinstance(track, Subtitle) - and track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML) - ): - segment_data = try_ensure_utf8(segment_data) - segment_data = ( - segment_data.decode("utf8") - .replace("‎", html.unescape("‎")) - .replace("‏", html.unescape("‏")) - .encode("utf8") - ) - f.write(segment_data) - f.flush() - segment_file.unlink() - progress(advance=1) + with open(save_path, "wb") as f: + for segment_file in segments_to_merge: + segment_data = segment_file.read_bytes() + if ( + not session_drm + and isinstance(track, Subtitle) + and track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML) + ): + segment_data = try_ensure_utf8(segment_data) + segment_data = ( + segment_data.decode("utf8") + .replace("‎", html.unescape("‎")) + .replace("‏", html.unescape("‏")) + .encode("utf8") + ) + f.write(segment_data) + f.flush() + segment_file.unlink() + progress(advance=1) track.path = save_path events.emit(events.Types.TRACK_DOWNLOADED, track=track) - if not skip_merge and session_drm: + if session_drm: progress(downloaded="Decrypting", completed=0, total=100) session_drm.decrypt(save_path) track.drm = None diff --git a/packages/envied/src/envied/core/manifests/m3u8.py b/packages/envied/src/envied/core/manifests/m3u8.py index 8252c88..a9111e3 100644 --- a/packages/envied/src/envied/core/manifests/m3u8.py +++ b/packages/envied/src/envied/core/manifests/m3u8.py @@ -5,10 +5,10 @@ from __future__ import annotations from typing import Optional, Union import m3u8 -from curl_cffi.requests import Session as CurlSession from requests import Session from envied.core.manifests.hls import HLS +from envied.core.session import RnetSession from envied.core.tracks import Tracks @@ -16,10 +16,11 @@ def parse( master: m3u8.M3U8, language: str, *, - session: Optional[Union[Session, CurlSession]] = None, + session: Optional[Union[Session, RnetSession]] = None, + url: Optional[str] = None, ) -> Tracks: """Parse a variant playlist to ``Tracks`` with basic information, defer DRM loading.""" - tracks = HLS(master, session=session).to_tracks(language) + tracks = HLS(master, session=session, url=url).to_tracks(language) bool(master.session_keys or HLS.parse_session_data_keys(master, session or Session())) diff --git a/packages/envied/src/envied/core/proxies/gluetun.py b/packages/envied/src/envied/core/proxies/gluetun.py index 12cb10e..f121cc6 100644 --- a/packages/envied/src/envied/core/proxies/gluetun.py +++ b/packages/envied/src/envied/core/proxies/gluetun.py @@ -13,7 +13,8 @@ import requests from envied.core import binaries from envied.core.proxies.proxy import Proxy -from envied.core.utilities import get_country_code, get_country_name, get_debug_logger, get_ip_info +from envied.core.utilities import get_country_code, get_country_name, get_debug_logger +from envied.core.utils.ip_info import get_ip_info # Global registry for cleanup on exit _gluetun_instances: list["Gluetun"] = [] @@ -1052,7 +1053,7 @@ class Gluetun(Proxy): # Gluetun needs both proxy listening AND VPN connected # The proxy starts before VPN is ready, so we need to wait for VPN proxy_ready = "[http proxy] listening" in all_logs - vpn_ready = "initialization sequence completed" in all_logs + vpn_ready = "initialization sequence completed" in all_logs or "public ip address is" in all_logs if proxy_ready and vpn_ready: # Give a brief moment for the proxy to fully initialize @@ -1235,10 +1236,10 @@ class Gluetun(Proxy): duration_ms=duration_ms, ) raise RuntimeError( - f"Region mismatch for {container['provider']}:{container['region']}: " - f"Expected '{expected_code}' but got '{actual_country}' " - f"(IP: {ip_info.get('ip')}, City: {ip_info.get('city')})" - ) + f"Region mismatch for {container['provider']}:{container['region']}: " + f"Expected '{expected_code}' but got '{actual_country}' " + f"(IP: {ip_info.get('ip')}, City: {ip_info.get('city')})" + ) # Verification successful - store IP info in container record if query_key in self.active_containers: diff --git a/packages/envied/src/envied/core/proxies/nordvpn.py b/packages/envied/src/envied/core/proxies/nordvpn.py index 1799039..bc987ed 100644 --- a/packages/envied/src/envied/core/proxies/nordvpn.py +++ b/packages/envied/src/envied/core/proxies/nordvpn.py @@ -64,7 +64,7 @@ class NordVPN(Proxy): if re.match(r"^[a-z]{2}\d+$", query): # country and nordvpn server id, e.g., us1, fr1234 - hostname = f"{query}.nordvpn.com" + hostname = f"{query}.proxy.nordvpn.com" else: if query.isdigit(): # country id @@ -86,7 +86,7 @@ class NordVPN(Proxy): if server_mapping: # country was set to a specific server ID in config - hostname = f"{country['code'].lower()}{server_mapping}.nordvpn.com" + hostname = f"{country['code'].lower()}{server_mapping}.proxy.nordvpn.com" else: # get the recommended server ID recommended_servers = self.get_recommended_servers(country["id"]) @@ -113,6 +113,9 @@ class NordVPN(Proxy): # NordVPN uses the alpha2 of 'GB' in API responses, but 'UK' in the hostname hostname = f"gb{hostname[2:]}" + if hostname.endswith(".nordvpn.com") and not hostname.endswith(".proxy.nordvpn.com"): + hostname = hostname[: -len(".nordvpn.com")] + ".proxy.nordvpn.com" + return f"https://{self.username}:{self.password}@{hostname}:89" def get_country(self, by_id: Optional[int] = None, by_code: Optional[str] = None) -> Optional[dict]: diff --git a/packages/envied/src/envied/core/proxies/resolve.py b/packages/envied/src/envied/core/proxies/resolve.py new file mode 100644 index 0000000..faf5624 --- /dev/null +++ b/packages/envied/src/envied/core/proxies/resolve.py @@ -0,0 +1,88 @@ +"""Shared proxy provider initialization and resolution. + +Used by both the REST API handlers and the remote service client. +""" + +from __future__ import annotations + +import logging +import re +from typing import Any, List, Optional + +log = logging.getLogger("proxies") + + +def initialize_proxy_providers() -> List[Any]: + """Initialize and return available proxy providers from config.""" + proxy_providers: list = [] + try: + from envied.core import binaries + from envied.core.config import config as main_config + from envied.core.proxies.basic import Basic + from envied.core.proxies.hola import Hola + from envied.core.proxies.nordvpn import NordVPN + from envied.core.proxies.surfsharkvpn import SurfsharkVPN + + proxy_config = getattr(main_config, "proxy_providers", {}) + + if proxy_config.get("basic"): + proxy_providers.append(Basic(**proxy_config["basic"])) + if proxy_config.get("nordvpn"): + proxy_providers.append(NordVPN(**proxy_config["nordvpn"])) + if proxy_config.get("surfsharkvpn"): + proxy_providers.append(SurfsharkVPN(**proxy_config["surfsharkvpn"])) + if hasattr(binaries, "HolaProxy") and binaries.HolaProxy: + proxy_providers.append(Hola()) + + for provider in proxy_providers: + log.info(f"Loaded {provider.__class__.__name__}: {provider}") + + if not proxy_providers: + log.warning("No proxy providers were loaded. Check your proxy provider configuration in envied.yaml") + + except Exception as e: + log.warning(f"Failed to initialize some proxy providers: {e}") + + return proxy_providers + + +def resolve_proxy(proxy: str, proxy_providers: List[Any]) -> Optional[str]: + """Resolve a proxy parameter to an actual proxy URI. + + Accepts: + - Direct URI: "https://...", "socks5://..." + - Country code: "us", "uk" + - Provider:country: "nordvpn:us" + """ + if not proxy: + return None + + if re.match(r"^(https?://|socks)", proxy): + return proxy + + requested_provider = None + query = proxy + if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE): + requested_provider, query = proxy.split(":", maxsplit=1) + + if requested_provider: + provider = next( + (x for x in proxy_providers if x.__class__.__name__.lower() == requested_provider.lower()), + None, + ) + if not provider: + available = [x.__class__.__name__ for x in proxy_providers] + raise ValueError(f"Proxy provider '{requested_provider}' not found. Available: {available}") + proxy_uri = provider.get_proxy(query) + if not proxy_uri: + raise ValueError(f"Proxy provider {requested_provider} had no proxy for {query}") + log.info(f"Using {provider.__class__.__name__} Proxy: {proxy_uri}") + return proxy_uri + + for provider in proxy_providers: + proxy_uri = provider.get_proxy(query) + if proxy_uri: + log.info(f"Using {provider.__class__.__name__} Proxy: {proxy_uri}") + return proxy_uri + + raise ValueError(f"No proxy provider had a proxy for {proxy}") diff --git a/packages/envied/src/envied/core/proxies/windscribevpn.py b/packages/envied/src/envied/core/proxies/windscribevpn.py index bffde71..4a00192 100644 --- a/packages/envied/src/envied/core/proxies/windscribevpn.py +++ b/packages/envied/src/envied/core/proxies/windscribevpn.py @@ -83,7 +83,7 @@ class WindscribeVPN(Proxy): if not hostname: return None - hostname = hostname.split(':')[0] + hostname = hostname.split(":")[0] return f"https://{self.username}:{self.password}@{hostname}:443" def get_specific_server(self, country_code: str, server_num: str) -> Optional[str]: diff --git a/packages/envied/src/envied/core/remote_service.py b/packages/envied/src/envied/core/remote_service.py new file mode 100644 index 0000000..e754f1a --- /dev/null +++ b/packages/envied/src/envied/core/remote_service.py @@ -0,0 +1,853 @@ +"""Remote service adapter for envied. + +Implements the Service interface by proxying authenticate, get_titles, +get_tracks, get_chapters, and license methods to a remote unshackle server. +Everything else (track selection, download, decrypt, mux) runs locally. +""" + +from __future__ import annotations + +import base64 +import logging +import time +from enum import Enum +from http.cookiejar import CookieJar +from typing import Any, Dict, Optional, Union + +import click +import requests +from langcodes import Language +from requests.adapters import HTTPAdapter, Retry +from rich.padding import Padding +from rich.rule import Rule + +from envied.core.config import config +from envied.core.console import console +from envied.core.constants import AnyTrack +from envied.core.credential import Credential +from envied.core.titles import Title_T, Titles_T, remap_titles +from envied.core.titles.episode import Episode, Series +from envied.core.titles.movie import Movie, Movies +from envied.core.tracks import Audio, Chapter, Chapters, Subtitle, Tracks, Video +from envied.core.tracks.attachment import Attachment +from envied.core.tracks.track import Track + +log = logging.getLogger("remote_service") + + +class RemoteClient: + """HTTP client for the unshackle serve API.""" + + def __init__(self, server_url: str, api_key: str) -> None: + self.server_url = server_url.rstrip("/") + self.api_key = api_key + self._session: Optional[requests.Session] = None + + @property + def session(self) -> requests.Session: + if self._session is None: + from envied.core import __version__ + + self._session = requests.Session() + self._session.headers["User-Agent"] = f"unshackle/{__version__}" + if self.api_key: + self._session.headers["X-Secret-Key"] = self.api_key + return self._session + + def _request(self, method: str, endpoint: str, data: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + url = f"{self.server_url}{endpoint}" + try: + resp = getattr(self.session, method)(url, json=data, timeout=120 if method == "post" else 30) + except requests.ConnectionError: + log.error(f"Could not connect to remote server at {self.server_url}. Is it running? (unshackle serve)") + raise SystemExit(1) + except requests.Timeout: + log.error(f"Request to remote server timed out: {endpoint}") + raise SystemExit(1) + result = resp.json() + if resp.status_code >= 400: + error_msg = result.get("message", resp.text) + error_code = result.get("error_code", "UNKNOWN") + log.error(f"Server error [{error_code}]: {error_msg}") + raise SystemExit(1) + return result + + def post(self, endpoint: str, data: Dict[str, Any]) -> Dict[str, Any]: + return self._request("post", endpoint, data) + + def get(self, endpoint: str) -> Dict[str, Any]: + return self._request("get", endpoint) + + def delete(self, endpoint: str) -> Dict[str, Any]: + return self._request("delete", endpoint) + + +def _enum_get(enum_cls: type[Enum], name: Optional[str], default: Any = None) -> Any: + """Safely get an enum value by name.""" + if not name: + return default + try: + return enum_cls[name] + except KeyError: + return default + + +def _deserialize_video(data: Dict[str, Any]) -> Video: + v = Video( + url=data.get("url") or "https://placeholder", + language=Language.get(data.get("language") or "und"), + descriptor=_enum_get(Track.Descriptor, data.get("descriptor"), Track.Descriptor.URL), + codec=_enum_get(Video.Codec, data.get("codec")), + range_=_enum_get(Video.Range, data.get("range"), Video.Range.SDR), + bitrate=data["bitrate"] * 1000 if data.get("bitrate") else 0, + width=data.get("width") or 0, + height=data.get("height") or 0, + fps=data.get("fps"), + id_=data.get("id"), + ) + return v + + +def _deserialize_audio(data: Dict[str, Any]) -> Audio: + a = Audio( + url=data.get("url") or "https://placeholder", + language=Language.get(data.get("language") or "und"), + descriptor=_enum_get(Track.Descriptor, data.get("descriptor"), Track.Descriptor.URL), + codec=_enum_get(Audio.Codec, data.get("codec")), + bitrate=data["bitrate"] * 1000 if data.get("bitrate") else 0, + channels=data.get("channels"), + joc=1 if data.get("atmos") else 0, + descriptive=data.get("descriptive", False), + id_=data.get("id"), + ) + return a + + +def _deserialize_subtitle(data: Dict[str, Any]) -> Subtitle: + return Subtitle( + url=data.get("url") or "https://placeholder", + language=Language.get(data.get("language") or "und"), + descriptor=_enum_get(Track.Descriptor, data.get("descriptor"), Track.Descriptor.URL), + codec=_enum_get(Subtitle.Codec, data.get("codec")), + cc=data.get("cc", False), + sdh=data.get("sdh", False), + forced=data.get("forced", False), + id_=data.get("id"), + ) + + +def _reconstruct_drm(drm_list: Optional[list]) -> list: + """Reconstruct DRM objects from serialized API data.""" + if not drm_list: + return [] + result = [] + for drm_info in drm_list: + drm_type = drm_info.get("type", "") + pssh_str = drm_info.get("pssh") + if not pssh_str: + continue + try: + if drm_type == "widevine": + from pywidevine.pssh import PSSH as WidevinePSSH + + from envied.core.drm import Widevine + + wv_pssh = WidevinePSSH(pssh_str) + result.append(Widevine(pssh=wv_pssh)) + elif drm_type == "playready": + import base64 as b64 + + from pyplayready.system.pssh import PSSH as PlayReadyPSSH + + from envied.core.drm import PlayReady + + pr_pssh = PlayReadyPSSH(b64.b64decode(pssh_str)) + result.append(PlayReady(pssh=pr_pssh, pssh_b64=pssh_str)) + except Exception: + continue + return result + + +def _build_tracks(data: Dict[str, Any]) -> Tracks: + tracks = Tracks() + tracks.videos = [_deserialize_video(v) for v in data.get("video", [])] + tracks.audio = [_deserialize_audio(a) for a in data.get("audio", [])] + tracks.subtitles = [_deserialize_subtitle(s) for s in data.get("subtitles", [])] + + for track_data, track_obj in [ + *zip(data.get("video", []), tracks.videos), + *zip(data.get("audio", []), tracks.audio), + ]: + drm_objs = _reconstruct_drm(track_data.get("drm")) + if drm_objs: + track_obj.drm = drm_objs + tracks.attachments = [ + Attachment(url=a["url"], name=a.get("name"), mime_type=a.get("mime_type"), description=a.get("description")) + for a in data.get("attachments", []) + ] + return tracks + + +def _resolve_manifest_data(tracks: Tracks, manifests: list, session: Any) -> None: + """Re-parse serialized manifests and populate track.data for downloading. + + The server serializes DASH and ISM manifest XML as zlib-compressed base64. + We decode and decompress locally, re-parse with the appropriate manifest + parser, then match each remote track to the locally-parsed track by ID + to copy track.data. HLS is skipped as it re-fetches from track.url. + """ + import base64 as b64 + import zlib + + if not manifests: + return + + log_m = logging.getLogger("remote_service") + all_tracks = list(tracks.videos) + list(tracks.audio) + list(tracks.subtitles) + + for manifest_info in manifests: + m_type = manifest_info.get("type") + m_url = manifest_info.get("url") + m_data = manifest_info.get("data") + if not m_data or not m_url: + continue + + try: + raw = zlib.decompress(b64.b64decode(m_data)) + + if m_type == "dash": + from lxml import etree + + from envied.core.manifests import DASH + + xml_tree = etree.fromstring(raw) + fallback_lang = next( + (t.language for t in all_tracks if t.language and str(t.language) != "und"), + None, + ) + local_tracks = DASH(xml_tree, m_url).to_tracks(language=fallback_lang) + elif m_type == "ism": + from lxml import etree + + from envied.core.manifests import ISM + + local_tracks = ISM(etree.fromstring(raw), m_url).to_tracks() + else: + continue + + local_all = list(local_tracks.videos) + list(local_tracks.audio) + list(local_tracks.subtitles) + for remote_track in all_tracks: + if remote_track.data.get(m_type): + continue + matched = _match_track(remote_track, local_all) + if matched and matched.data.get(m_type): + remote_track.data.update(matched.data) + remote_track.descriptor = matched.descriptor + if matched.drm and not remote_track.drm: + remote_track.drm = matched.drm + + except Exception as e: + log_m.warning("Failed to re-parse %s manifest from %s: %s", m_type, m_url, e) + + +def _match_track(remote_track: Track, local_tracks: list) -> Optional[Track]: + """Match a remote track to a locally-parsed track by ID or attributes.""" + remote_id = str(remote_track.id) + for lt in local_tracks: + if str(lt.id) == remote_id: + return lt + + for lt in local_tracks: + if type(lt).__name__ != type(remote_track).__name__: + continue + if lt.codec != remote_track.codec or str(lt.language) != str(remote_track.language): + continue + if hasattr(lt, "width") and hasattr(remote_track, "width"): + if lt.width == remote_track.width and lt.height == remote_track.height: + return lt + elif hasattr(lt, "channels") and hasattr(remote_track, "channels"): + if lt.bitrate == remote_track.bitrate: + return lt + elif hasattr(lt, "forced"): + if lt.forced == remote_track.forced and lt.sdh == remote_track.sdh: + return lt + return None + + +def _build_title(info: Dict[str, Any], service_tag: str, fallback_id: str) -> Union[Episode, Movie]: + svc_class = type(service_tag, (), {}) + lang = Language.get(info["language"]) if info.get("language") else None + if info.get("type") == "episode": + return Episode( + id_=info.get("id", fallback_id), + service=svc_class, + title=info.get("series_title", "Unknown"), + season=info.get("season", 0), + number=info.get("number", 0), + name=info.get("name"), + year=info.get("year"), + language=lang, + ) + return Movie( + id_=info.get("id", fallback_id), + service=svc_class, + name=info.get("name", "Unknown"), + year=info.get("year"), + language=lang, + ) + + +def resolve_server(server_name: Optional[str]) -> tuple[str, str, dict]: + """Resolve server URL, API key, and per-service config from remote_services.""" + remote_services = config.remote_services + if not remote_services: + raise click.ClickException( + "No remote services configured. Add 'remote_services' to your envied.yaml:\n\n" + " remote_services:\n" + " my_server:\n" + ' url: "https://server:8080"\n' + ' api_key: "your-api-key"' + ) + + if server_name: + svc = remote_services.get(server_name) + if not svc: + available = ", ".join(remote_services.keys()) + raise click.ClickException(f"Remote service '{server_name}' not found. Available: {available}") + services = svc.get("services", {}) + services["_server_cdm"] = svc.get("server_cdm", False) + return svc["url"], svc.get("api_key", ""), services + + if len(remote_services) == 1: + name, svc = next(iter(remote_services.items())) + log.info(f"Using remote service: {name}") + services = svc.get("services", {}) + services["_server_cdm"] = svc.get("server_cdm", False) + return svc["url"], svc.get("api_key", ""), services + + available = ", ".join(remote_services.keys()) + raise click.ClickException(f"Multiple remote services configured. Use --server to select one: {available}") + + +def _load_credentials_for_transport(service_tag: str, profile: Optional[str]) -> Optional[Dict[str, str]]: + from envied.commands.dl import dl + + credential = dl.get_credentials(service_tag, profile) + if credential: + result: Dict[str, str] = {"username": credential.username, "password": credential.password} + if credential.extra: + result["extra"] = credential.extra + return result + return None + + +def _load_cookies_for_transport(service_tag: str, profile: Optional[str]) -> Optional[str]: + import zlib + + from envied.commands.dl import dl + + cookie_path = dl.get_cookie_path(service_tag, profile) + if cookie_path and cookie_path.exists(): + return base64.b64encode(zlib.compress(cookie_path.read_bytes())).decode("ascii") + return None + + +def _resolve_proxy(proxy_arg: Optional[str]) -> Optional[str]: + if not proxy_arg: + return None + + from envied.core.proxies.resolve import initialize_proxy_providers, resolve_proxy + + try: + providers = initialize_proxy_providers() + return resolve_proxy(proxy_arg, providers) + except ValueError as e: + raise click.ClickException(str(e)) + + +class RemoteService: + """Service adapter that proxies to a remote unshackle server. + + Implements the same interface dl.py's result() expects without + subclassing Service (avoids proxy/geofence setup in __init__). + """ + + ALIASES: tuple[str, ...] = () + GEOFENCE: tuple[str, ...] = () + NO_SUBTITLES: bool = False + + def __init__( + self, + ctx: click.Context, + service_tag: str, + title_id: str, + server_url: str, + api_key: str, + services_config: dict, + service_params: Optional[Dict[str, Any]] = None, + ) -> None: + self.__class__.__name__ = service_tag + console.print(Padding(Rule(f"[rule.text]Service: {service_tag} (Remote)"), (1, 2))) + + self.service_tag = service_tag + self.title_id = title_id + self.client = RemoteClient(server_url, api_key) + self.ctx = ctx + self._service_params = service_params or {} + self.log = logging.getLogger(service_tag) + self.credential: Optional[Credential] = None + self.current_region: Optional[str] = None + self.title_cache = None + self._titles: Optional[Titles_T] = None + self._tracks_by_title: Dict[str, Tracks] = {} + self._chapters_by_title: Dict[str, list] = {} + self._session_id: Optional[str] = None + self._server_cdm_type: str = "widevine" + + self._session = requests.Session() + self._session.headers.update(config.headers) + self._session.mount( + "https://", + HTTPAdapter( + max_retries=Retry(total=5, backoff_factor=0.2, status_forcelist=[429, 500, 502, 503, 504]), + pool_block=True, + ), + ) + self._session.mount("http://", self._session.adapters["https://"]) + + svc_config = services_config.get(service_tag, {}) + self._server_cdm = services_config.get("_server_cdm", False) + self._apply_service_config(svc_config) + + def _apply_service_config(self, svc_config: dict) -> None: + if not svc_config: + return + config_maps = { + "cdm": ("cdm", self.service_tag), + "decryption": ("decryption_map", self.service_tag), + "downloader": ("downloader_map", self.service_tag), + } + for key, (attr, tag) in config_maps.items(): + if svc_config.get(key): + target = getattr(config, attr, None) + if target is None: + setattr(config, attr, {}) + target = getattr(config, attr) + target[tag] = svc_config[key] + + if svc_config.get("downloader"): + config.downloader = svc_config["downloader"] + if svc_config.get("decryption"): + config.decryption = svc_config["decryption"] + + extra = {k: v for k, v in svc_config.items() if k not in config_maps} + if extra: + existing = config.services.get(self.service_tag, {}) + for key, value in extra.items(): + if key in existing and isinstance(existing[key], dict) and isinstance(value, dict): + existing[key].update(value) + else: + existing[key] = value + config.services[self.service_tag] = existing + + @property + def session(self) -> requests.Session: + return self._session + + @property + def title(self) -> str: + return self.title_id + + def authenticate(self, cookies: Optional[CookieJar] = None, credential: Optional[Credential] = None) -> None: + self.credential = credential + profile = self.ctx.parent.params.get("profile") if self.ctx.parent else None + proxy = self.ctx.parent.params.get("proxy") if self.ctx.parent else None + no_proxy = self.ctx.parent.params.get("no_proxy", False) if self.ctx.parent else False + + create_data: Dict[str, Any] = {"service": self.service_tag, "title_id": self.title_id} + + credentials = _load_credentials_for_transport(self.service_tag, profile) + if credentials: + create_data["credentials"] = credentials + + cookies_text = _load_cookies_for_transport(self.service_tag, profile) + if cookies_text: + create_data["cookies"] = cookies_text + + if not no_proxy and proxy: + resolved_proxy = _resolve_proxy(proxy) + if resolved_proxy: + create_data["proxy"] = resolved_proxy + + if not no_proxy and not proxy: + try: + from envied.core.utils.ip_info import get_ip_info + + ip_info = get_ip_info(self._session, cached=True) + if ip_info and ip_info.get("country"): + create_data["client_region"] = ip_info["country"].lower() + except Exception: + pass + + if profile: + create_data["profile"] = profile + if no_proxy: + create_data["no_proxy"] = True + # Forward track selection params so the server fetches the right manifests + if self.ctx.parent: + range_ = self.ctx.parent.params.get("range_") + if range_: + create_data["range_"] = [r.name for r in range_] + vcodec = self.ctx.parent.params.get("vcodec") + if vcodec: + create_data["vcodec"] = [c.name for c in vcodec] + quality = self.ctx.parent.params.get("quality") + if quality: + create_data["quality"] = list(quality) + if self.ctx.parent.params.get("best_available"): + create_data["best_available"] = True + + if self._service_params: + create_data.update(self._service_params) + + cdm = self.ctx.obj.cdm if self.ctx.obj else None + if cdm is not None: + from envied.core.cdm.detect import is_playready_cdm + + create_data["cdm_type"] = "playready" if is_playready_cdm(cdm) else "widevine" + + cache_data = self._load_cache_files() + if cache_data: + create_data["cache"] = cache_data + + result = self.client.post("/api/session/create", create_data) + self._session_id = result["session_id"] + + status = result.get("status", "authenticated") + if status == "authenticating": + self._poll_auth_completion() + + def _poll_auth_completion(self, poll_interval: float = 2.0, timeout: float = 600.0) -> None: + """Poll the server until authentication completes, handling interactive prompts. + + When the server needs user input (OTP, device code, PIN), it returns + ``pending_input`` with a prompt. We display it locally, collect the + response, and POST it back. The server resumes its auth flow. + """ + deadline = time.monotonic() + timeout + + while time.monotonic() < deadline: + resp = self.client.get(f"/api/session/{self._session_id}/prompt") + status = resp.get("status") + + if status == "authenticated": + return + + if status == "failed": + error = resp.get("error", "Authentication failed on server") + log.error(f"Remote auth failed: {error}") + raise SystemExit(1) + + if status == "pending_input": + prompt = resp.get("prompt", "Enter input: ") + user_response = click.prompt(prompt.rstrip("\n "), default="", show_default=False) + self.client.post( + f"/api/session/{self._session_id}/prompt", + {"response": user_response}, + ) + continue + + time.sleep(poll_interval) + + log.error("Remote authentication timed out") + raise SystemExit(1) + + def get_titles(self) -> Titles_T: + if self._titles is not None: + return self._titles + result = self.client.get(f"/api/session/{self._session_id}/titles") + titles_list = [_build_title(t, self.service_tag, self.title_id) for t in result.get("titles", [])] + self._titles = ( + Series(titles_list) if titles_list and isinstance(titles_list[0], Episode) else Movies(titles_list) + ) + return self._titles + + def get_titles_cached(self, title_id: str = None) -> Titles_T: + """Apply the client's local title_map to titles fetched from the remote server. + + Lets users rename titles for remote services they don't have installed locally. + The server sends raw titles; the client's own ``services..title_map`` wins. + """ + title_map = (config.services.get(self.service_tag) or {}).get("title_map") or {} + return remap_titles(self.get_titles(), title_map) + + def get_tracks(self, title: Title_T) -> Tracks: + title_id = str(title.id) + if title_id in self._tracks_by_title: + return self._tracks_by_title[title_id] + result = self.client.post(f"/api/session/{self._session_id}/tracks", {"title_id": title_id}) + tracks = _build_tracks(result) + + for k, v in result.get("session_headers", {}).items(): + if k.lower() not in ("host", "content-length", "content-type"): + self._session.headers[k] = v + for k, v in result.get("session_cookies", {}).items(): + self._session.cookies.set(k, v) + + _resolve_manifest_data(tracks, result.get("manifests", []), self._session) + + self._server_cdm_type = result.get("server_cdm_type", "widevine") + + self._tracks_by_title[title_id] = tracks + self._chapters_by_title[title_id] = result.get("chapters", []) + + return tracks + + def resolve_server_keys(self, title: Title_T) -> None: + """Resolve DRM keys via server CDM for all tracks on a title. + + Called by dl.py between track selection and download. The server + decides which CDM device to use and tells the client via + server_cdm_type. We send track IDs and the server does the full + CDM flow, returning KID:KEY pairs. + """ + if not self._server_cdm: + return + + from uuid import UUID + + track_ids = [str(t.id) for t in title.tracks.videos + title.tracks.audio] + if not track_ids: + return + + drm_type = getattr(self, "_server_cdm_type", "widevine") + self.log.debug(f"Requesting server CDM keys (server_cdm_type={drm_type})") + + try: + with console.status("Retrieving Remote License...", spinner="dots"): + resp = self.client.post( + f"/api/session/{self._session_id}/license", + { + "track_ids": track_ids, + "mode": "server_cdm", + "drm_type": drm_type, + }, + ) + keys_by_track = resp.get("keys", {}) + server_drm_type = resp.get("drm_type", drm_type) + self._server_cdm_type = server_drm_type + self.log.debug(f"Server responded with drm_type={server_drm_type}, keys for {len(keys_by_track)} track(s)") + + for track in title.tracks: + track_keys = keys_by_track.get(str(track.id), {}) + if not track_keys: + continue + + kid_list = list(track_keys.keys()) + drm_obj = self._create_drm_stub(server_drm_type, kid_list) + for kid_hex, key_hex in track_keys.items(): + drm_obj.content_keys[UUID(hex=kid_hex)] = key_hex + track.drm = [drm_obj] + self.log.debug( + f"Track {track.id}: set DRM to {drm_obj.__class__.__name__} with {len(track_keys)} key(s)" + ) + key_count = sum(len(v) for v in keys_by_track.values()) + if key_count: + self.log.debug(f"Server CDM resolved {key_count} key(s) using {server_drm_type.upper()}") + except Exception as e: + self.log.warning("Failed to resolve server CDM keys: %s", e) + + @staticmethod + def _create_drm_stub(drm_type: str, kid_hexes: list[str]) -> Any: + """Create a DRM object stub matching the type the server actually used. + + For server_cdm mode, this is only used for display — keys are already + resolved. We build a minimal DRM object that holds content_keys. + """ + from uuid import UUID + + if drm_type == "playready": + import base64 as b64 + import struct + + from pyplayready.system.pssh import PSSH as PlayReadyPSSH + + from envied.core.drm import PlayReady + + kid_uuids = [UUID(hex=k) for k in kid_hexes] + kid_b64 = b64.b64encode(kid_uuids[0].bytes_le).decode() + wrm_xml = ( + '' + f"16AESCTR" + f"{kid_b64}" + ) + wrm_bytes = wrm_xml.encode("utf-16-le") + record_length = len(wrm_bytes) + obj_length = 4 + 2 + 2 + 2 + record_length + pr_obj = struct.pack(" Chapters: + title_id = str(title.id) + if title_id not in self._chapters_by_title: + self.get_tracks(title) + raw = self._chapters_by_title.get(title_id, []) + return Chapters([Chapter(ch["timestamp"], ch.get("name")) for ch in raw]) + + def get_widevine_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[Union[bytes, str]]: + return self._proxy_license(challenge, track, "widevine") + + def get_playready_license( + self, *, challenge: bytes, title: Title_T, track: AnyTrack + ) -> Optional[Union[bytes, str]]: + return self._proxy_license(challenge, track, "playready") + + def get_widevine_service_certificate( + self, + *, + challenge: bytes, + title: Title_T, + track: AnyTrack, + ) -> Union[bytes, str]: + try: + resp = self.client.post( + f"/api/session/{self._session_id}/license", + { + "track_id": str(track.id), + "challenge": base64.b64encode(challenge).decode("ascii"), + "drm_type": "widevine", + "is_certificate": True, + }, + ) + return base64.b64decode(resp["license"]) + except Exception: + return None + + def _proxy_license(self, challenge: Union[bytes, str], track: AnyTrack, drm_type: str) -> bytes: + if isinstance(challenge, str): + challenge = challenge.encode("utf-8") + + pssh_b64 = None + if track.drm: + for drm_obj in track.drm: + drm_class = drm_obj.__class__.__name__ + if drm_type == "playready" and drm_class == "PlayReady": + pssh_b64 = drm_obj.data["pssh_b64"] + break + elif drm_type == "widevine" and drm_class == "Widevine": + pssh_b64 = drm_obj.pssh.dumps() + break + + if self._server_cdm: + from uuid import UUID + + if pssh_b64: + try: + resp = self.client.post( + f"/api/session/{self._session_id}/license", + { + "track_id": str(track.id), + "drm_type": drm_type, + "mode": "server_cdm", + "pssh": pssh_b64, + }, + ) + keys = resp.get("keys", {}) + if keys and track.drm: + for drm_obj in track.drm: + if hasattr(drm_obj, "content_keys"): + for kid_hex, key_hex in keys.items(): + drm_obj.content_keys[UUID(hex=kid_hex)] = key_hex + return challenge + except Exception as e: + self.log.warning("server_cdm license failed: %s", e) + return challenge + + payload = { + "track_id": str(track.id), + "challenge": base64.b64encode(challenge).decode("ascii"), + "drm_type": drm_type, + } + if pssh_b64: + payload["pssh"] = pssh_b64 + + resp = self.client.post(f"/api/session/{self._session_id}/license", payload) + return base64.b64decode(resp["license"]) + + def on_segment_downloaded(self, track: AnyTrack, segment: Any) -> None: + pass + + def on_track_downloaded(self, track: AnyTrack) -> None: + pass + + def on_track_decrypted(self, track: AnyTrack, drm: Any, segment: Any = None) -> None: + pass + + def on_track_repacked(self, track: AnyTrack) -> None: + pass + + def on_track_multiplex(self, track: AnyTrack) -> None: + pass + + def close(self) -> None: + if self._session_id: + try: + result = self.client.delete(f"/api/session/{self._session_id}") + self._save_returned_cache(result.get("cache", {})) + except Exception as e: + self.log.warning(f"Failed to clean up remote session: {e}") + self._session_id = None + + def _save_returned_cache(self, cache_data: Dict[str, str]) -> None: + """Save cache files returned by the server to the local cache directory. + + The server returns updated cache files (e.g. refreshed tokens) on + session close. Writing them locally means the next remote session + can forward them back, skipping interactive auth. + """ + if not cache_data: + return + + import zlib + + cache_dir = config.directories.cache / self.service_tag + cache_dir.mkdir(parents=True, exist_ok=True) + + for key, content in cache_data.items(): + try: + decompressed = zlib.decompress(base64.b64decode(content)) + (cache_dir / key).with_suffix(".json").write_bytes(decompressed) + except Exception as e: + self.log.warning(f"Failed to save returned cache file '{key}': {e}") + + self.log.info(f"Saved {len(cache_data)} cache file(s) from server") + + def _load_cache_files(self) -> Dict[str, str]: + import zlib + + cache_dir = config.directories.cache / self.service_tag + if not cache_dir.is_dir(): + return {} + return { + f.stem: base64.b64encode(zlib.compress(f.read_bytes())).decode("ascii") + for f in cache_dir.glob("*.json") + if not f.stem.startswith("titles_") + } + + +__all__ = ("RemoteClient", "RemoteService", "resolve_server") diff --git a/packages/envied/src/envied/core/service.py b/packages/envied/src/envied/core/service.py index da6cbf4..5758404 100644 --- a/packages/envied/src/envied/core/service.py +++ b/packages/envied/src/envied/core/service.py @@ -1,19 +1,22 @@ -import base64 import logging from abc import ABCMeta, abstractmethod from collections.abc import Callable, Generator from dataclasses import dataclass, field from http.cookiejar import CookieJar from pathlib import Path -from typing import Optional, Union +from typing import TYPE_CHECKING, Optional, Union from urllib.parse import urlparse, urlunparse +if TYPE_CHECKING: + from envied.core.api.input_bridge import InputBridge + import click import m3u8 import requests from requests.adapters import HTTPAdapter, Retry from rich.padding import Padding from rich.rule import Rule +from rich.text import Text from envied.core.cacher import Cacher from envied.core.config import config @@ -23,10 +26,10 @@ from envied.core.credential import Credential from envied.core.drm import DRM_T from envied.core.search_result import SearchResult from envied.core.title_cacher import TitleCacher, get_account_hash, get_region_from_proxy -from envied.core.titles import Title_T, Titles_T +from envied.core.titles import Title_T, Titles_T, remap_titles from envied.core.tracks import Chapters, Tracks from envied.core.tracks.video import Video -from envied.core.utilities import get_cached_ip_info, get_ip_info +from envied.core.utils.ip_info import get_ip_info @dataclass @@ -101,11 +104,13 @@ class Service(metaclass=ABCMeta): self.session = self.get_session() self.cache = Cacher(self.__class__.__name__) self.title_cache = TitleCacher(self.__class__.__name__) + self.cache_dir = config.directories.cache / self.__class__.__name__ # Store context for cache control flags and credential self.ctx = ctx self.credential = None # Will be set in authenticate() self.current_region = None # Will be set based on proxy/geolocation + self._input_bridge: Optional[InputBridge] = None # Set track request from CLI params - services can read/override in their __init__ vcodec = ctx.parent.params.get("vcodec") if ctx.parent else None @@ -207,15 +212,10 @@ class Service(metaclass=ABCMeta): if proxy: self.session.proxies.update({"all": proxy}) - proxy_parse = urlparse(proxy) - if proxy_parse.username and proxy_parse.password: - self.session.headers.update( - { - "Proxy-Authorization": base64.b64encode( - f"{proxy_parse.username}:{proxy_parse.password}".encode("utf8") - ).decode() - } - ) + # Don't set Proxy-Authorization manually: both rnet (Proxy.all) and + # requests authenticate from the credentials embedded in the proxy URL. + # A manual header here was malformed (no "Basic " scheme) and broke + # plaintext-http forward-proxy requests with HTTP 407. # Always verify proxy IP - proxies can change exit nodes try: proxy_ip_info = get_ip_info(self.session) @@ -227,7 +227,7 @@ class Service(metaclass=ABCMeta): else: # No proxy, use cached IP info for title caching (non-critical) try: - ip_info = get_cached_ip_info(self.session) + ip_info = get_ip_info(self.session, cached=True) self.current_region = ip_info.get("country", "").lower() if ip_info else None except Exception as e: self.log.debug(f"Failed to get cached IP info: {e}") @@ -271,6 +271,8 @@ class Service(metaclass=ABCMeta): raise if first: all_tracks.add(hdr_tracks, warn_only=True) + if hdr_tracks.manifest_url and not all_tracks.manifest_url: + all_tracks.manifest_url = hdr_tracks.manifest_url first = False else: for video in hdr_tracks.videos: @@ -289,13 +291,13 @@ class Service(metaclass=ABCMeta): except (ValueError, SystemExit) as e: if self.track_request.best_available: codec_name = codec_val.name if codec_val else "default" - self.log.warning( - f" - {range_val.name}/{codec_name} not available, skipping ({e})" - ) + self.log.warning(f" - {range_val.name}/{codec_name} not available, skipping ({e})") continue raise if first: all_tracks.add(tracks, warn_only=True) + if tracks.manifest_url and not all_tracks.manifest_url: + all_tracks.manifest_url = tracks.manifest_url first = False else: for video in tracks.videos: @@ -349,6 +351,20 @@ class Service(metaclass=ABCMeta): # Store credential for cache key generation self.credential = credential + def request_input(self, prompt: str) -> str: + """Request interactive input from the user. + + When running locally (CLI), prompts via the shared rich console so the + prompt renders correctly alongside Live progress / log handlers. + When running in serve mode with an :class:`InputBridge` attached, + delegates to the bridge which relays the prompt to the remote client. + """ + if self._input_bridge is not None: + return self._input_bridge.request_input(prompt) + indent = " " * 5 + padded = indent + prompt.replace("\n", "\n" + indent) + return console.input(Text(padded, style="text")) + def search(self) -> Generator[SearchResult, None, None]: """ Search by query for titles from the Service. @@ -394,7 +410,9 @@ class Service(metaclass=ABCMeta): Decode the data, return as is to reduce unnecessary computations. """ - def get_playready_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[Union[bytes, str]]: + def get_playready_license( + self, *, challenge: bytes, title: Title_T, track: AnyTrack + ) -> Optional[Union[bytes, str]]: """ Get a PlayReady License message by sending a License Request (challenge). @@ -458,7 +476,7 @@ class Service(metaclass=ABCMeta): else: # If we can't determine title_id, just call get_titles directly self.log.debug("Cannot determine title_id for caching, bypassing cache") - return self.get_titles() + return self.apply_title_map(self.get_titles()) # Get cache control flags from context no_cache = False @@ -471,7 +489,7 @@ class Service(metaclass=ABCMeta): account_hash = get_account_hash(self.credential) # Use title cache to get titles with fallback support - return self.title_cache.get_cached_titles( + titles = self.title_cache.get_cached_titles( title_id=str(title_id), fetch_function=self.get_titles, region=self.current_region, @@ -479,6 +497,17 @@ class Service(metaclass=ABCMeta): no_cache=no_cache, reset_cache=reset_cache, ) + return self.apply_title_map(titles) + + def apply_title_map(self, titles: Titles_T) -> Titles_T: + """ + Rewrite service-provided titles using the per-service ``title_map`` config. + + ``title_map`` lives under ``services.`` in envied.yaml. Applied after the + title cache so config edits take effect without a cache reset, and before any + ``--enrich`` override so enrich wins. See ``remap_titles`` for the match rules. + """ + return remap_titles(titles, (self.config or {}).get("title_map") or {}) @abstractmethod def get_tracks(self, title: Title_T) -> Tracks: diff --git a/packages/envied/src/envied/core/services.py b/packages/envied/src/envied/core/services.py index 776bc15..c91e2a4 100644 --- a/packages/envied/src/envied/core/services.py +++ b/packages/envied/src/envied/core/services.py @@ -1,3 +1,7 @@ +from __future__ import annotations + +import logging +import re from pathlib import Path import click @@ -6,6 +10,8 @@ from envied.core.config import config from envied.core.service import Service from envied.core.utilities import import_module_by_path +log = logging.getLogger("services") + _service_dirs = config.directories.services if not isinstance(_service_dirs, list): _service_dirs = [_service_dirs] @@ -15,23 +21,118 @@ _SERVICES = sorted( key=lambda x: x.parent.stem, ) -_MODULES = {path.parent.stem: getattr(import_module_by_path(path), path.parent.stem) for path in _SERVICES} -_ALIASES = {tag: getattr(module, "ALIASES") for tag, module in _MODULES.items()} +def load_service(path: Path) -> object: + """Load one Service module, returning its tag-named class. + + Raises a concise, single-line error naming the Service and the real cause so + a broken Service never surfaces as a raw traceback pointing at the loader. + """ + tag = path.parent.stem + try: + module = import_module_by_path(path) + except Exception as e: + raise RuntimeError(f"{tag}: failed to import — {type(e).__name__}: {e} ({path})") from e + try: + return getattr(module, tag) + except AttributeError as e: + raise RuntimeError( + f"{tag}: no class named '{tag}' found in {path} — the class name must match the directory name" + ) from e + + +def load_services(paths: list[Path]) -> tuple[dict[str, object], list[str]]: + """Load every Service, returning the good ones plus a list of load errors. + + Importing this module must never raise: it is imported by several commands, + and a failed import is not cached by Python, so raising here would re-run and + re-report for every command. Instead we collect failures and let the caller + surface them once, cleanly, at the point services are actually used. + """ + modules: dict[str, object] = {} + errors: list[str] = [] + for path in paths: + try: + modules[path.parent.stem] = load_service(path) + except Exception as e: + errors.append(str(e)) + return modules, errors + + +_MODULES, LOAD_ERRORS = load_services(_SERVICES) + +_ALIASES = {tag: getattr(module, "ALIASES", ()) for tag, module in _MODULES.items()} + + +def check_load_errors() -> None: + """Raise a single clean error if any Service failed to load. + + Called when services are actually needed (listing/resolving) so the message + is rendered once by Click, without a traceback and without cascading through + every command that imports this module. + """ + if LOAD_ERRORS: + joined = "\n".join(f" - {err}" for err in LOAD_ERRORS) + raise click.ClickException(f"Failed to load {len(LOAD_ERRORS)} service(s):\n{joined}") class Services(click.MultiCommand): """Lazy-loaded command group of project services.""" + _remote_services_cache: list[dict] | None = None + # Click-specific methods + def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]: + """Preprocess --slow to support optional range value before Click parses args.""" + processed = [] + i = 0 + while i < len(args): + if args[i] == "--slow": + if i + 1 < len(args) and re.match(r"^\d+-\d+$", args[i + 1]): + processed.append(f"--slow={args[i + 1]}") + i += 2 + else: + processed.append("--slow=60-120") + i += 1 + else: + processed.append(args[i]) + i += 1 + return super().parse_args(ctx, processed) + def list_commands(self, ctx: click.Context) -> list[str]: - """Returns a list of all available Services as command names for Click.""" + """Returns a list of all available Services as command names for Click. + + In remote mode, fetches the service list from the remote server + so the user sees exactly what's available remotely. + """ + remote = ctx.params.get("remote") or (ctx.parent and ctx.parent.params.get("remote")) + if remote: + remote_services = Services._fetch_remote_services(ctx) + if remote_services is not None: + return [s["tag"] for s in remote_services] + tags = Services.get_tags() + for svc_cfg in config.remote_services.values(): + for remote_tag in svc_cfg.get("services", {}).keys(): + if remote_tag not in tags: + tags.append(remote_tag) + return tags + check_load_errors() return Services.get_tags() def get_command(self, ctx: click.Context, name: str) -> click.Command: """Load the Service and return the Click CLI method.""" + check_load_errors() tag = Services.get_tag(name) + + import_file = ctx.params.get("import_file") or (ctx.parent and ctx.parent.params.get("import_file")) + if import_file: + return Services._make_import_command(tag, ctx) + + remote = ctx.params.get("remote") or (ctx.parent and ctx.parent.params.get("remote")) + if remote: + return Services._make_remote_command(tag, ctx) + try: service = Services.load(tag) except KeyError as e: @@ -47,6 +148,89 @@ class Services(click.MultiCommand): raise click.ClickException(f"Service '{tag}' has no 'cli' method configured.") + @staticmethod + def _fetch_remote_services(ctx: click.Context) -> list[dict] | None: + """Fetch the service list from the remote server (cached per process).""" + if Services._remote_services_cache is not None: + return Services._remote_services_cache + try: + from envied.core.remote_service import RemoteClient, resolve_server + + server_name = ctx.params.get("server") + server_url, api_key, _ = resolve_server(server_name) + client = RemoteClient(server_url, api_key) + result = client.get("/api/services") + Services._remote_services_cache = result.get("services", []) + return Services._remote_services_cache + except Exception: + return None + + @staticmethod + def _make_remote_command(tag: str, ctx: click.Context) -> click.Command: + """Create a Click command for a remote service with server-provided options.""" + svc_info = Services._fetch_remote_service_info(tag, ctx) + short_help = svc_info.get("url") if svc_info else None + cli_params = svc_info.get("cli_params") if svc_info else None + + @click.command(name=tag, short_help=short_help) + @click.argument("title", type=str) + @click.pass_context + def remote_cli(ctx: click.Context, title: str, **kwargs: object) -> object: + from envied.core.remote_service import RemoteService, resolve_server + + server_name = ctx.parent.params.get("server") if ctx.parent else None + server_url, api_key, services_config = resolve_server(server_name) + service_params = {k: v for k, v in kwargs.items() if v is not None and v is not False} + return RemoteService(ctx, tag, title, server_url, api_key, services_config, service_params=service_params) + + if cli_params: + for param in cli_params: + if param.get("kind") == "option": + opts = param.get("opts", [f"--{param['name']}"]) + kwargs: dict = {} + if param.get("is_flag"): + kwargs["is_flag"] = True + kwargs["default"] = param.get("default", False) + else: + kwargs["default"] = param.get("default") + kwargs["type"] = str + if param.get("help"): + kwargs["help"] = param["help"] + remote_cli = click.option(*opts, **kwargs)(remote_cli) + + return remote_cli + + @staticmethod + def _make_import_command(tag: str, ctx: click.Context) -> click.Command: + """Create a synthetic command that yields an ImportService from an export JSON. + + Mirrors how remote services are wired so dl.py's result() runs unchanged. + """ + + @click.command(name=tag, short_help="Reconstruct a download from an export JSON.") + @click.argument("title", type=str, required=False, default="") + @click.pass_context + def import_cli(ctx: click.Context, title: str, **kwargs: object) -> object: + from envied.core.import_service import ImportService + + import_file = ctx.params.get("import_file") or (ctx.parent and ctx.parent.params.get("import_file")) + return ImportService(ctx, tag, title, import_file) + + return import_cli + + @staticmethod + def _fetch_remote_service_info(tag: str, ctx: click.Context) -> dict | None: + """Fetch service info for a specific service from the remote server.""" + try: + services = Services._fetch_remote_services(ctx) + if services: + for svc in services: + if svc.get("tag") == tag: + return svc + except Exception: + pass + return None + # Methods intended to be used anywhere @staticmethod diff --git a/packages/envied/src/envied/core/session.py b/packages/envied/src/envied/core/session.py index 0d7d04b..8b45ebc 100644 --- a/packages/envied/src/envied/core/session.py +++ b/packages/envied/src/envied/core/session.py @@ -1,96 +1,549 @@ -"""Session utilities for creating HTTP sessions with different backends.""" +"""Session utilities for creating HTTP sessions with TLS fingerprinting via rnet (Rust/BoringSSL).""" from __future__ import annotations +import http import logging import random import time -import warnings +from collections.abc import Iterator, MutableMapping from datetime import datetime, timezone from email.utils import parsedate_to_datetime -from typing import Any -from urllib.parse import urlparse +from http.cookiejar import CookieJar +from typing import Any, Optional +from urllib.parse import urlencode, urlparse, urlunparse -from curl_cffi.requests import Response, Session, exceptions +import rnet +from requests import HTTPError, Request +from requests.structures import CaseInsensitiveDict from envied.core.config import config -# Globally suppress curl_cffi HTTPS proxy warnings since some proxy providers -# (like NordVPN) require HTTPS URLs but curl_cffi expects HTTP format -warnings.filterwarnings( - "ignore", message="Make sure you are using https over https proxy.*", category=RuntimeWarning, module="curl_cffi.*" -) +# --------------------------------------------------------------------------- +# Impersonate preset mapping — rnet uses named presets (no custom JA3/Akamai) +# --------------------------------------------------------------------------- -FINGERPRINT_PRESETS = { - "okhttp4": { - "ja3": ( - "771," # TLS 1.2 - "4865-4866-4867-49195-49196-52393-49199-49200-52392-49171-49172-156-157-47-53," # Ciphers - "0-23-65281-10-11-35-16-5-13-51-45-43," # Extensions - "29-23-24," # Named groups (x25519, secp256r1, secp384r1) - "0" # EC point formats - ), - "akamai": "4:16777216|16711681|0|m,p,a,s", - "description": "OkHttp 3.x/4.x (BoringSSL TLS stack)", - }, - "okhttp5": { - "ja3": ( - "771," # TLS 1.2 - "4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53," # Ciphers - "0-23-65281-10-11-35-16-5-13-51-45-43," # Extensions - "29-23-24," # Named groups (x25519, secp256r1, secp384r1) - "0" # EC point formats - ), - "akamai": "4:16777216|16711681|0|m,p,a,s", - "description": "OkHttp 5.x (BoringSSL TLS stack)", - }, - "shield_okhttp": { - "ja3": ( - "771," # TLS 1.2 - "4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53," # Ciphers (OkHttp 4.11) - "0-23-65281-10-11-35-16-5-13-51-45-43-21," # Extensions (incl padding ext 21) - "29-23-24," # Named groups (x25519, secp256r1, secp384r1) - "0" # EC point formats - ), - "akamai": "4:16777216|16711681|0|m,p,a,s", - "description": "NVIDIA SHIELD Android TV OkHttp 4.11 (captured JA3)", - }, +DEFAULT_IMPERSONATE = rnet.Impersonate.Chrome131 + + +def _resolve_impersonate(browser: str) -> rnet.Impersonate: + """Resolve a browser string to an rnet.Impersonate preset. + + Accepts exact rnet preset names (e.g. "Chrome131", "OkHttp4_12", "Edge101"). + See https://github.com/0x676e67/rnet for the full list of available presets. + """ + preset = getattr(rnet.Impersonate, browser, None) + if preset is not None: + return preset + raise ValueError( + f"Unknown impersonate preset: {browser!r}. " + f"Use exact rnet preset names like 'Chrome131', 'OkHttp4_12', 'Edge101'. " + f"See rnet.Impersonate for all available presets." + ) + + +# Map string method names to rnet.Method enum +_METHOD_MAP: dict[str, rnet.Method] = { + "GET": rnet.Method.GET, + "POST": rnet.Method.POST, + "PUT": rnet.Method.PUT, + "DELETE": rnet.Method.DELETE, + "HEAD": rnet.Method.HEAD, + "OPTIONS": rnet.Method.OPTIONS, + "PATCH": rnet.Method.PATCH, + "TRACE": rnet.Method.TRACE, } -class MaxRetriesError(exceptions.RequestException): - def __init__(self, message, cause=None): +# --------------------------------------------------------------------------- +# Response headers adapter — bytes → str +# --------------------------------------------------------------------------- + + +class RnetResponseHeaders(MutableMapping): + """Read-only str-based view over rnet's bytes-based HeaderMap.""" + + def __init__(self, header_map: Any) -> None: + self._map = header_map + + def _decode(self, val: Any) -> str: + return val.decode("utf-8", errors="replace") if isinstance(val, (bytes, bytearray)) else str(val) + + def __getitem__(self, key: str) -> str: + val = self._map[key] + return self._decode(val) + + def __setitem__(self, key: str, value: str) -> None: + raise TypeError("Response headers are read-only") + + def __delitem__(self, key: str) -> None: + raise TypeError("Response headers are read-only") + + def __contains__(self, key: object) -> bool: + if not isinstance(key, str): + return False + return self._map.contains_key(key) + + def __iter__(self) -> Iterator[str]: + seen: set[str] = set() + for k, _ in self._map.items(): + dk = self._decode(k) + if dk not in seen: + seen.add(dk) + yield dk + + def __len__(self) -> int: + return self._map.keys_len() + + def get(self, key: str, default: Optional[str] = None) -> Optional[str]: + val = self._map.get(key) + if val is None: + return default + return self._decode(val) + + def items(self) -> list[tuple[str, str]]: + return [(self._decode(k), self._decode(v)) for k, v in self._map.items()] + + +# --------------------------------------------------------------------------- +# Response wrapper — requests-compatible interface +# --------------------------------------------------------------------------- + + +class RnetResponse: + """Wraps rnet.BlockingResponse with a requests-compatible API.""" + + def __init__(self, resp: Any) -> None: + self._resp = resp + self._headers: Optional[RnetResponseHeaders] = None + self._content: Optional[bytes] = None + self._text: Optional[str] = None + self._streamed = False + + @property + def status_code(self) -> int: + return int(str(self._resp.status_code)) + + @property + def ok(self) -> bool: + return self._resp.ok + + @property + def headers(self) -> RnetResponseHeaders: + if self._headers is None: + self._headers = RnetResponseHeaders(self._resp.headers) + return self._headers + + @property + def url(self) -> str: + return str(self._resp.url) + + @property + def content_length(self) -> Optional[int]: + return self._resp.content_length + + @property + def content(self) -> bytes: + if self._content is None: + self._content = self._resp.bytes() + return self._content + + @property + def text(self) -> str: + if self._text is None: + encoding = self._resp.encoding or "utf-8" + self._text = self.content.decode(encoding, errors="replace") + return self._text + + @property + def reason(self) -> str: + try: + return http.HTTPStatus(self.status_code).phrase + except ValueError: + return "Unknown" + + @property + def cookies(self) -> Any: + return self._resp.cookies + + def json(self, **kwargs: Any) -> Any: + import json as _json + + return _json.loads(self.content) + + def raise_for_status(self) -> None: + if not self.ok: + raise HTTPError( + f"{self.status_code} {self.reason}: {self.url}", + response=self, + ) + + def iter_content(self, chunk_size: Optional[int] = None) -> Iterator[bytes]: + """Re-chunk rnet's variable-size stream into fixed-size pieces.""" + self._streamed = True + if chunk_size is None or chunk_size <= 0: + yield from self._resp.stream() + return + + buf = bytearray() + for chunk in self._resp.stream(): + buf.extend(chunk) + while len(buf) >= chunk_size: + yield bytes(buf[:chunk_size]) + buf = buf[chunk_size:] + if buf: + yield bytes(buf) + + def stream(self) -> Iterator[bytes]: + """Direct pass-through of rnet's native stream iterator.""" + self._streamed = True + yield from self._resp.stream() + + def close(self) -> None: + try: + self._resp.close() + except Exception: + pass + + +# --------------------------------------------------------------------------- +# Session headers adapter — persists via client.update() +# --------------------------------------------------------------------------- + + +class RnetSessionHeaders(CaseInsensitiveDict): + """Dict-like headers that persist to the rnet client via update().""" + + def __init__(self, client: Any) -> None: + self._client = client + super().__init__() + + def _sync(self) -> None: + """Push current headers to the rnet client.""" + if self._client is not None and hasattr(self, "_store") and self._store: + self._client.update(headers={k: v for k, v in self.items()}) + + def __setitem__(self, key: str, value: str) -> None: + super().__setitem__(key, value) + self._sync() + + def update(self, __m: Any = None, **kwargs: Any) -> None: + if __m: + if hasattr(__m, "items"): + for k, v in __m.items(): + super().__setitem__(k, v) + else: + for k, v in __m: + super().__setitem__(k, v) + for k, v in kwargs.items(): + super().__setitem__(k, v) + self._sync() + + def pop(self, key: str, *args: Any) -> Any: + result = super().pop(key, *args) + # rnet doesn't support removing individual headers, but we track locally + # and always send the full set on next update + return result + + def __delitem__(self, key: str) -> None: + super().__delitem__(key) + + +# --------------------------------------------------------------------------- +# Session cookies adapter +# --------------------------------------------------------------------------- + + +class RnetCookieAdapter(MutableMapping): + """Cookie adapter that bridges requests-style cookie access to rnet.""" + + def __init__(self, client: Any) -> None: + self._client = client + self._cookies: dict[str, dict[str, str]] = {} + self._flat: dict[str, str] = {} + self._original_cookies: list[Any] = [] + + def _set_cookie_on_client(self, url: str, name: str, value: str) -> None: + """Set a cookie on the rnet client, or buffer locally if the client is not yet created.""" + if self._client is not None: + try: + self._client.set_cookie(url, rnet.Cookie(name, value)) + except Exception: + pass + + def _flush_to_client(self) -> None: + """Push all buffered cookies to the rnet client once it is created.""" + if self._client is None: + return + for domain, cookies in self._cookies.items(): + url = f"https://{domain.lstrip('.')}" if domain else "https://localhost" + for name, value in cookies.items(): + try: + self._client.set_cookie(url, rnet.Cookie(name, value)) + except Exception: + pass + + @property + def jar(self) -> CookieJar: + """Return a CookieJar with original Cookie objects (requests compat). + + Used by ``save_cookies`` in dl.py to persist cookies back to disk. + """ + jar = CookieJar() + for cookie in self._original_cookies: + jar.set_cookie(cookie) + return jar + + def update(self, other: Any = None, **kwargs: Any) -> None: + if other is None: + other = {} + if isinstance(other, CookieJar): + for cookie in other: + domain = cookie.domain or "" + name = cookie.name + value = cookie.value or "" + self._flat[name] = value + self._cookies.setdefault(domain, {})[name] = value + self._original_cookies.append(cookie) + url = f"https://{domain.lstrip('.')}" if domain else "https://localhost" + self._set_cookie_on_client(url, name, value) + elif isinstance(other, dict): + for name, value in other.items(): + self._flat[name] = value + self._set_cookie_on_client("https://localhost", name, str(value)) + self._flat.update(other) + elif hasattr(other, "items"): + for name, value in other.items(): + self._flat[name] = str(value) + self._set_cookie_on_client("https://localhost", name, str(value)) + + for name, value in kwargs.items(): + self._flat[name] = value + self._set_cookie_on_client("https://localhost", name, value) + + def get( + self, name: str, default: Optional[str] = None, domain: Optional[str] = None, path: Optional[str] = None + ) -> Optional[str]: + if domain and domain in self._cookies: + return self._cookies[domain].get(name, default) + return self._flat.get(name, default) + + def set(self, name: str, value: str, domain: str = "localhost") -> None: + self._flat[name] = value + self._cookies.setdefault(domain, {})[name] = value + url = f"https://{domain.lstrip('.')}" + self._set_cookie_on_client(url, name, value) + + def __getitem__(self, name: str) -> str: + return self._flat[name] + + def __setitem__(self, name: str, value: str) -> None: + self.set(name, value) + + def __delitem__(self, name: str) -> None: + self._flat.pop(name, None) + for domain_cookies in self._cookies.values(): + domain_cookies.pop(name, None) + + def __contains__(self, name: object) -> bool: + return name in self._flat + + def __iter__(self) -> Iterator: + return iter(self._flat) + + def __len__(self) -> int: + return len(self._flat) + + def __bool__(self) -> bool: + return bool(self._flat) + + def get_dict(self, domain: Optional[str] = None, path: Optional[str] = None) -> dict[str, str]: + """Return cookies as a plain dict (requests RequestsCookieJar compat). + + If *domain* is given, only cookies for that domain are returned. + *path* is accepted for API compatibility but ignored (flat storage). + """ + if domain is not None: + return dict(self._cookies.get(domain, {})) + return dict(self._flat) + + def clear(self, domain: Optional[str] = None, path: Optional[str] = None, name: Optional[str] = None) -> None: + """Remove cookies (requests RequestsCookieJar compat). + + - ``clear()`` removes all cookies. + - ``clear(domain=..., path=..., name=...)`` removes a specific cookie. + """ + if name is not None: + self._flat.pop(name, None) + if domain is not None and domain in self._cookies: + self._cookies[domain].pop(name, None) + else: + for domain_cookies in self._cookies.values(): + domain_cookies.pop(name, None) + elif domain is not None: + removed = self._cookies.pop(domain, {}) + for k in removed: + # Only remove from flat if no other domain has same key + still_exists = any(k in dc for dc in self._cookies.values()) + if not still_exists: + self._flat.pop(k, None) + else: + self._flat.clear() + self._cookies.clear() + + def items(self) -> list[tuple[str, str]]: + return list(self._flat.items()) + + def keys(self) -> list[str]: + return list(self._flat.keys()) + + def values(self) -> list[str]: + return list(self._flat.values()) + + +# --------------------------------------------------------------------------- +# Session proxy adapter +# --------------------------------------------------------------------------- + + +class RnetProxyDict(dict): + """Dict-like proxy config that syncs to the rnet client. + + Accepts ``{"all": url}``, ``{"https": url}``, or ``{"http": url}`` + and applies via rnet's native ``proxies`` parameter (``List[rnet.Proxy]``). + Supports both lazy (pre-client) and live (post-client) proxy updates. + """ + + def __init__(self, session: "RnetSession") -> None: + super().__init__() + self._session = session + + def _sync(self) -> None: + proxy = self.get("all") or self.get("https") or self.get("http") + proxies = [rnet.Proxy.all(proxy)] if proxy else [] + self._session._client_kwargs["proxies"] = proxies or None + if self._session._client is not None: + self._session._client.update(proxies=proxies or None) + + def update(self, __m: Any = None, **kwargs: Any) -> None: + super().update(__m or {}, **kwargs) + self._sync() + + def __setitem__(self, key: str, value: str) -> None: + super().__setitem__(key, value) + self._sync() + + +# --------------------------------------------------------------------------- +# Exceptions +# --------------------------------------------------------------------------- + + +class MaxRetriesError(Exception): + def __init__(self, message: str, cause: Optional[Exception] = None) -> None: super().__init__(message) self.__cause__ = cause -class CurlSession(Session): +# --------------------------------------------------------------------------- +# RnetSession — main session class +# --------------------------------------------------------------------------- + + +class RnetSession: + """TLS-fingerprinted HTTP session powered by rnet (Rust/BoringSSL). + + Drop-in replacement for CurlSession with requests-compatible API. + Supports browser impersonation (Chrome, Firefox, Edge, Safari, OkHttp), + retry with exponential backoff, cookie persistence, and proxy support. + + The client is created lazily on the first request so that headers, + cookies, and proxies can be configured freely before any connection + is established. + """ + def __init__( self, max_retries: int = 5, backoff_factor: float = 0.2, max_backoff: float = 60.0, - status_forcelist: list[int] | None = None, - allowed_methods: set[str] | None = None, - catch_exceptions: tuple[type[Exception], ...] | None = None, + status_forcelist: Optional[list[int]] = None, + allowed_methods: Optional[set[str]] = None, + catch_exceptions: Optional[tuple[type[Exception], ...]] = None, **session_kwargs: Any, - ): - super().__init__(**session_kwargs) - + ) -> None: self.max_retries = max_retries self.backoff_factor = backoff_factor self.max_backoff = max_backoff self.status_forcelist = status_forcelist or [429, 500, 502, 503, 504] self.allowed_methods = allowed_methods or {"GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE"} self.catch_exceptions = catch_exceptions or ( - exceptions.ConnectionError, - exceptions.ProxyError, - exceptions.SSLError, - exceptions.Timeout, + rnet.ConnectionError, + rnet.TimeoutError, + rnet.RequestError, ) self.log = logging.getLogger(self.__class__.__name__) - def get_sleep_time(self, response: Response | None, attempt: int) -> float | None: + client_kwargs: dict[str, Any] = {} + for key in ("impersonate", "timeout", "proxies", "verify", "redirect"): + if key in session_kwargs: + client_kwargs[key] = session_kwargs.pop(key) + if "proxy" in session_kwargs: + proxy_url = session_kwargs.pop("proxy") + if proxy_url: + client_kwargs["proxies"] = [rnet.Proxy.all(proxy_url)] + + client_kwargs["cookie_store"] = True + + self.verify: bool = client_kwargs.pop("verify", True) + if not self.verify: + client_kwargs["danger_accept_invalid_certs"] = True + + self._client_kwargs = dict(client_kwargs) + self._client: Optional[rnet.BlockingClient] = None + + self.headers = RnetSessionHeaders(None) + self.cookies = RnetCookieAdapter(None) + self.proxies = RnetProxyDict(self) + + if "headers" in session_kwargs: + self.headers.update(session_kwargs.pop("headers")) + if "cookies" in session_kwargs: + self.cookies.update(session_kwargs.pop("cookies")) + if "proxies" in session_kwargs: + self.proxies.update(session_kwargs.pop("proxies")) + + def _ensure_client(self) -> rnet.BlockingClient: + """Lazily create the rnet client on first use, flushing any buffered state.""" + if self._client is None: + self._client = rnet.BlockingClient(**self._client_kwargs) + self.headers._client = self._client + self.headers._sync() + self.cookies._client = self._client + self.cookies._flush_to_client() + return self._client + + def _build_url(self, url: str, params: Optional[Any] = None) -> str: + """Encode params into the URL (rnet ignores the params kwarg). + + Accepts the same shapes as requests: a mapping, a sequence of pairs, or a + pre-built query string/bytes. A string is appended verbatim (already encoded); + urlencode() would raise TypeError on it. + """ + if not params: + return url + if isinstance(params, bytes): + extra = params.decode("utf-8") + elif isinstance(params, str): + extra = params + else: + extra = urlencode(params, doseq=True) + parsed = urlparse(url) + separator = "&" if parsed.query else "" + query = parsed.query + separator + extra if parsed.query else extra + return urlunparse(parsed._replace(query=query)) + + def get_sleep_time(self, response: Optional[RnetResponse], attempt: int) -> Optional[float]: if response: retry_after = response.headers.get("Retry-After") if retry_after: @@ -108,19 +561,57 @@ class CurlSession(Session): sleep_time = backoff_value + random.uniform(-jitter, jitter) return min(sleep_time, self.max_backoff) - def request(self, method: str, url: str, **kwargs: Any) -> Response: - if method.upper() not in self.allowed_methods: - return super().request(method, url, **kwargs) + def request(self, method: str, url: str, **kwargs: Any) -> RnetResponse: + client = self._ensure_client() + method_upper = method.upper() if isinstance(method, str) else str(method).upper() - last_exception = None - response = None + # Build URL with params + url = self._build_url(url, kwargs.pop("params", None)) + + # Default allow_redirects=True + kwargs.setdefault("allow_redirects", True) + + # Pass verify setting + if not self.verify: + kwargs.setdefault("verify", False) + + # Remove kwargs rnet doesn't understand + kwargs.pop("stream", None) # rnet responses are always lazy + + # Translate requests-compatible 'data' kwarg to rnet equivalents + data = kwargs.pop("data", None) + if data is not None: + if isinstance(data, dict): + kwargs["form"] = list(data.items()) + elif isinstance(data, (str, bytes)): + kwargs["body"] = data + else: + kwargs["body"] = data + + # Resolve method enum + rnet_method = _METHOD_MAP.get(method_upper) + if rnet_method is None: + raise ValueError(f"Unsupported HTTP method: {method}") + + # Convert headers to standard dict once to resolve PyO3 CaseInsensitiveDict rejection. + if kwargs.get("headers") is not None: + kwargs["headers"] = dict(kwargs["headers"]) + + # Skip retry for non-allowed methods + if method_upper not in self.allowed_methods: + raw_resp = client.request(rnet_method, url, **kwargs) + return RnetResponse(raw_resp) + + last_exception: Optional[Exception] = None + response: Optional[RnetResponse] = None for attempt in range(self.max_retries + 1): try: - response = super().request(method, url, **kwargs) + raw_resp = client.request(rnet_method, url, **kwargs) + response = RnetResponse(raw_resp) if response.status_code not in self.status_forcelist: return response - last_exception = exceptions.HTTPError(f"Received status code: {response.status_code}") + last_exception = HTTPError(f"Received status code: {response.status_code}") self.log.warning( f"{response.status_code} {response.reason}({urlparse(url).path}). Retrying... " f"({attempt + 1}/{self.max_retries})" @@ -142,120 +633,100 @@ class CurlSession(Session): raise MaxRetriesError(f"Max retries exceeded for {method} {url}", cause=last_exception) + def get(self, url: str, **kwargs: Any) -> RnetResponse: + return self.request("GET", url, **kwargs) + + def post(self, url: str, **kwargs: Any) -> RnetResponse: + return self.request("POST", url, **kwargs) + + def put(self, url: str, **kwargs: Any) -> RnetResponse: + return self.request("PUT", url, **kwargs) + + def delete(self, url: str, **kwargs: Any) -> RnetResponse: + return self.request("DELETE", url, **kwargs) + + def head(self, url: str, **kwargs: Any) -> RnetResponse: + return self.request("HEAD", url, **kwargs) + + def options(self, url: str, **kwargs: Any) -> RnetResponse: + return self.request("OPTIONS", url, **kwargs) + + def patch(self, url: str, **kwargs: Any) -> RnetResponse: + return self.request("PATCH", url, **kwargs) + + def prepare_request(self, req: Request) -> Request: + """Compatibility shim for services using prepared requests.""" + # Merge session headers into request headers + if req.headers: + merged = dict(self.headers) + merged.update(req.headers) + req.headers = merged + else: + req.headers = dict(self.headers) + return req + + def send(self, req: Request, **kwargs: Any) -> RnetResponse: + """Compatibility shim for services using prepared requests.""" + method = req.method or "GET" + url = req.url or "" + + send_kwargs: dict[str, Any] = {} + if req.headers: + send_kwargs["headers"] = dict(req.headers) + if req.body: + send_kwargs["data"] = req.body + if req.json: + send_kwargs["json"] = req.json + + send_kwargs.update(kwargs) + return self.request(method, url, **send_kwargs) + + def mount(self, prefix: str, adapter: Any) -> None: + """No-op — rnet handles TLS and connection pooling natively.""" + pass + + def close(self) -> None: + """No-op — rnet manages its own resources.""" + pass + + +# --------------------------------------------------------------------------- +# session() factory +# --------------------------------------------------------------------------- + def session( - browser: str | None = None, - ja3: str | None = None, - akamai: str | None = None, - extra_fp: dict | None = None, - **kwargs, -) -> CurlSession: + browser: Optional[str] = None, + **kwargs: Any, +) -> RnetSession: """ - Create a curl_cffi session that impersonates a browser or custom TLS/HTTP fingerprint. - - This is a full replacement for requests.Session with browser impersonation - and anti-bot capabilities. The session uses curl-impersonate under the hood - to mimic real browser behavior. + Create an rnet session with TLS fingerprinting (browser/app impersonation). Args: - browser: Browser to impersonate (e.g. "chrome124", "firefox", "safari") OR - fingerprint preset name (e.g. "okhttp4"). - Uses the configured default from curl_impersonate.browser if not specified. - Available presets: okhttp4, okhttp5 - See https://github.com/lexiforest/curl_cffi#sessions for browser options. - ja3: Custom JA3 TLS fingerprint string (format: "SSLVersion,Ciphers,Extensions,Curves,PointFormats"). - When provided, curl_cffi will use this exact TLS fingerprint instead of the browser's default. - See https://curl-cffi.readthedocs.io/en/latest/impersonate/customize.html - akamai: Custom Akamai HTTP/2 fingerprint string (format: "SETTINGS|WINDOW_UPDATE|PRIORITY|PSEUDO_HEADERS"). - When provided, curl_cffi will use this exact HTTP/2 fingerprint instead of the browser's default. - See https://curl-cffi.readthedocs.io/en/latest/impersonate/customize.html - extra_fp: Additional fingerprint parameters dict for advanced customization. - See https://curl-cffi.readthedocs.io/en/latest/impersonate/customize.html - **kwargs: Additional arguments passed to CurlSession constructor: - - headers: Additional headers (dict) - - cookies: Cookie jar or dict - - auth: HTTP basic auth tuple (username, password) - - proxies: Proxy configuration dict - - verify: SSL certificate verification (bool, default True) - - timeout: Request timeout in seconds (float or tuple) - - allow_redirects: Follow redirects (bool, default True) - - max_redirects: Maximum redirect count (int) - - cert: Client certificate (str or tuple) - - Extra arguments for retry handler: - - max_retries: Maximum number of retries (int, default 5) - - backoff_factor: Backoff factor (float, default 0.2) - - max_backoff: Maximum backoff time (float, default 60.0) - - status_forcelist: List of status codes to force retry (list, default [429, 500, 502, 503, 504]) - - allowed_methods: List of allowed HTTP methods (set, default {"GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE"}) - - catch_exceptions: List of exceptions to catch (tuple, default (exceptions.ConnectionError, exceptions.ProxyError, exceptions.SSLError, exceptions.Timeout)) + browser: Exact rnet.Impersonate preset name. Examples: + "Chrome131", "OkHttp4_12", "Edge101", "Firefox135", + "Safari18", "OkHttp5", "Opera118" + Uses the configured default from config if not specified. + See rnet.Impersonate for all available presets. + **kwargs: Additional arguments passed to RnetSession constructor. Returns: - curl_cffi.requests.Session configured with browser impersonation or custom fingerprints, - common headers, and equivalent retry behavior to requests.Session. + RnetSession configured with browser impersonation and retry behavior. Examples: - # Standard browser impersonation - from envied.core.session import session - - class MyService(Service): - @staticmethod - def get_session(): - return session() # Uses config default browser - - # Use OkHttp 4.x preset for Android TV - class AndroidService(Service): - @staticmethod - def get_session(): - return session("okhttp4") - - # Custom fingerprint (manual) - class CustomService(Service): - @staticmethod - def get_session(): - return session( - ja3="771,4865-4866-4867-49195...", - akamai="1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p", - ) - - # With retry configuration - class MyService(Service): - @staticmethod - def get_session(): - return session( - "okhttp4", - max_retries=5, - status_forcelist=[429, 500], - allowed_methods={"GET", "HEAD", "OPTIONS"}, - ) + session() # Default browser from config + session("OkHttp4_12") # OkHttp 4.12 fingerprint + session("Chrome131") # Chrome 131 + session("Edge101", max_retries=3) # Edge 101 with custom retry """ + if browser is None: + browser = config.curl_impersonate.get("browser", "Chrome131") - if browser and browser in FINGERPRINT_PRESETS: - preset = FINGERPRINT_PRESETS[browser] - if ja3 is None: - ja3 = preset.get("ja3") - if akamai is None: - akamai = preset.get("akamai") - if extra_fp is None: - extra_fp = preset.get("extra_fp") - browser = None + impersonate = _resolve_impersonate(browser) - if browser is None and ja3 is None and akamai is None: - browser = config.curl_impersonate.get("browser", "chrome") + session_kwargs: dict[str, Any] = {"impersonate": impersonate} + session_kwargs.update(kwargs) - session_config = {} - if browser: - session_config["impersonate"] = browser - - if ja3: - session_config["ja3"] = ja3 - if akamai: - session_config["akamai"] = akamai - if extra_fp: - session_config["extra_fp"] = extra_fp - - session_config.update(kwargs) - - session_obj = CurlSession(**session_config) + session_obj = RnetSession(**session_kwargs) session_obj.headers.update(config.headers) return session_obj diff --git a/packages/envied/src/envied/core/titles/__init__.py b/packages/envied/src/envied/core/titles/__init__.py index 304a8f1..9c95645 100644 --- a/packages/envied/src/envied/core/titles/__init__.py +++ b/packages/envied/src/envied/core/titles/__init__.py @@ -8,4 +8,40 @@ Title_T = Union[Movie, Episode, Song] Titles_T = Union[Movies, Series, Album] -__all__ = ("Episode", "Series", "Movie", "Movies", "Album", "Song", "Title_T", "Titles_T") +def remap_titles(titles: Titles_T, title_map: dict) -> Titles_T: + """ + Rewrite titles in-place using an exact-match ``title_map``. + + Some services name a title differently from how the user wants it stored, which can + break library matching. ``title_map`` maps a source title string to the desired output + title. Episodes are matched on their ``title`` (the show name), Movies and Songs on + their ``name``. Returns the same collection for convenient chaining. + """ + if not title_map or not titles: + return titles + + def remap_one(title: Title_T) -> None: + attr = "title" if isinstance(title, Episode) else "name" + current = getattr(title, attr, None) + if current and current in title_map: + setattr(title, attr, title_map[current]) + + if hasattr(titles, "__iter__"): + for title in titles: + remap_one(title) + else: + remap_one(titles) + return titles + + +__all__ = ( + "Episode", + "Series", + "Movie", + "Movies", + "Album", + "Song", + "Title_T", + "Titles_T", + "remap_titles", +) diff --git a/packages/envied/src/envied/core/titles/episode.py b/packages/envied/src/envied/core/titles/episode.py index b965833..a440ac3 100644 --- a/packages/envied/src/envied/core/titles/episode.py +++ b/packages/envied/src/envied/core/titles/episode.py @@ -100,28 +100,39 @@ class Episode(Title): def get_filename(self, media_info: MediaInfo, folder: bool = False, show_service: bool = True) -> str: if folder: - series_template = config.output_template.get("series") - if series_template: - folder_template = series_template - folder_template = re.sub(r'\{episode\}', '', folder_template) - folder_template = re.sub(r'\{episode_name\?\}', '', folder_template) - folder_template = re.sub(r'\{episode_name\}', '', folder_template) - folder_template = re.sub(r'\{season_episode\}', '{season}', folder_template) - - folder_template = re.sub(r'\.{2,}', '.', folder_template) - folder_template = re.sub(r'\s{2,}', ' ', folder_template) - folder_template = re.sub(r'^[\.\s]+|[\.\s]+$', '', folder_template) - - formatter = TemplateFormatter(folder_template) + template = config.get_folder_template("series") + if template: + formatter = TemplateFormatter(template) context = self._build_template_context(media_info, show_service) - context['season'] = f"S{self.season:02}" + context["season"] = f"S{self.season:02}" folder_name = formatter.format(context) - if '.' in series_template and ' ' not in series_template: - return sanitize_filename(folder_name, ".") - else: - return sanitize_filename(folder_name, " ") + separators = re.sub(r"\{[^}]*\}", "", template) + spacer = "." if "." in separators and " " not in separators else " " + return sanitize_filename(folder_name, spacer) + + series_template = config.output_template.get("series") + if series_template: + derived_template = series_template + derived_template = re.sub(r"\{episode\}", "", derived_template) + derived_template = re.sub(r"\{episode_name\?\}", "", derived_template) + derived_template = re.sub(r"\{episode_name\}", "", derived_template) + derived_template = re.sub(r"\{season_episode\}", "{season}", derived_template) + + derived_template = re.sub(r"\.{2,}", ".", derived_template) + derived_template = re.sub(r"\s{2,}", " ", derived_template) + derived_template = re.sub(r"^[\.\s]+|[\.\s]+$", "", derived_template) + + formatter = TemplateFormatter(derived_template) + context = self._build_template_context(media_info, show_service) + context["season"] = f"S{self.season:02}" + + folder_name = formatter.format(context) + + separators = re.sub(r"\{[^}]*\}", "", derived_template) + spacer = "." if "." in separators and " " not in separators else " " + return sanitize_filename(folder_name, spacer) else: name = f"{self.title}" if self.year: @@ -149,7 +160,7 @@ class Series(SortedKeyList, ABC): sum(seasons.values()) season_breakdown = ", ".join(f"S{season}({count})" for season, count in sorted(seasons.items())) tree = Tree( - f"{num_seasons} seasons, {season_breakdown}", + f"{num_seasons} season{'s'[:num_seasons^1]}, {season_breakdown}", guide_style="bright_black", ) if verbose: diff --git a/packages/envied/src/envied/core/titles/movie.py b/packages/envied/src/envied/core/titles/movie.py index 6d260ad..7a2e9f3 100644 --- a/packages/envied/src/envied/core/titles/movie.py +++ b/packages/envied/src/envied/core/titles/movie.py @@ -1,3 +1,4 @@ +import re from abc import ABC from typing import Any, Iterable, Optional, Union @@ -59,6 +60,15 @@ class Movie(Title): def get_filename(self, media_info: MediaInfo, folder: bool = False, show_service: bool = True) -> str: if folder: + template = config.get_folder_template("movies") + if template: + formatter = TemplateFormatter(template) + context = self._build_template_context(media_info, show_service) + folder_name = formatter.format(context) + + separators = re.sub(r"\{[^}]*\}", "", template) + spacer = "." if "." in separators and " " not in separators else " " + return sanitize_filename(folder_name, spacer) name = f"{self.name}" if self.year: name += f" ({self.year})" diff --git a/packages/envied/src/envied/core/titles/song.py b/packages/envied/src/envied/core/titles/song.py index aac9d0d..59f91b5 100644 --- a/packages/envied/src/envied/core/titles/song.py +++ b/packages/envied/src/envied/core/titles/song.py @@ -1,3 +1,4 @@ +import re from abc import ABC from typing import Any, Iterable, Optional, Union @@ -94,6 +95,15 @@ class Song(Title): def get_filename(self, media_info: MediaInfo, folder: bool = False, show_service: bool = True) -> str: if folder: + template = config.get_folder_template("songs") + if template: + formatter = TemplateFormatter(template) + context = self._build_template_context(media_info, show_service) + folder_name = formatter.format(context) + + separators = re.sub(r"\{[^}]*\}", "", template) + spacer = "." if "." in separators and " " not in separators else " " + return sanitize_filename(folder_name, spacer) name = f"{self.artist} - {self.album}" if self.year: name += f" ({self.year})" diff --git a/packages/envied/src/envied/core/titles/title.py b/packages/envied/src/envied/core/titles/title.py index 7e37a92..0551b6f 100644 --- a/packages/envied/src/envied/core/titles/title.py +++ b/packages/envied/src/envied/core/titles/title.py @@ -61,7 +61,21 @@ class Title: returned dict with their specific fields (e.g., season/episode). """ primary_video_track = next(iter(media_info.video_tracks), None) - primary_audio_track = next(iter(media_info.audio_tracks), None) + original_lang_tag = ( + str(self.language).split("-")[0].lower() if self.language else "" + ) + primary_audio_track = None + if original_lang_tag: + primary_audio_track = next( + ( + t + for t in media_info.audio_tracks + if t.language and t.language.split("-")[0].lower() == original_lang_tag + ), + None, + ) + if primary_audio_track is None: + primary_audio_track = next(iter(media_info.audio_tracks), None) unique_audio_languages = len({x.language.split("-")[0] for x in media_info.audio_tracks if x.language}) context: dict[str, Any] = { @@ -99,7 +113,20 @@ class Title: aspect_ratio.append(1) ratio = aspect_ratio[0] / aspect_ratio[1] if ratio not in (16 / 9, 4 / 3, 9 / 16, 3 / 4): + if abs(width - 3840) <= 50: + width = 3840 + elif abs(width - 2560) <= 50: + width = 2560 + elif abs(width - 1920) <= 50 or abs(width - 1620) <= 50: + width = 1920 + elif abs(width - 1280) <= 50 or abs(width - 1080) <= 50: + width = 1280 + resolution = int(max(width, primary_video_track.height) * (9 / 16)) + + track_height = primary_video_track.height + if abs(resolution - track_height) <= 10 or track_height in (2160, 1440, 1080, 720, 480): + resolution = track_height except Exception: pass @@ -143,14 +170,16 @@ class Title: channel_count = primary_audio_track.channel_s or primary_audio_track.channels or 0 channels = float(channel_count) - features = primary_audio_track.format_additionalfeatures or "" + has_atmos = any( + "JOC" in (t.format_additionalfeatures or "") or t.joc for t in media_info.audio_tracks + ) context.update( { "audio": AUDIO_CODEC_MAP.get(codec, codec), "audio_channels": f"{channels:.1f}", "audio_full": f"{AUDIO_CODEC_MAP.get(codec, codec)}{channels:.1f}", - "atmos": "Atmos" if ("JOC" in features or primary_audio_track.joc) else "", + "atmos": "Atmos" if has_atmos else "", } ) diff --git a/packages/envied/src/envied/core/tracks/attachment.py b/packages/envied/src/envied/core/tracks/attachment.py index 82da0ed..7e94225 100644 --- a/packages/envied/src/envied/core/tracks/attachment.py +++ b/packages/envied/src/envied/core/tracks/attachment.py @@ -86,9 +86,7 @@ class Attachment: raise ValueError(f"Failed to download attachment from URL: {e}") if path is not None and not isinstance(path, (str, Path)): - raise ValueError( - f"Invalid attachment path type: expected str or Path, got {type(path).__name__}." - ) + raise ValueError(f"Invalid attachment path type: expected str or Path, got {type(path).__name__}.") if path is not None: path = Path(path) @@ -127,6 +125,10 @@ class Attachment: def __str__(self) -> str: return " | ".join(filter(bool, ["ATT", self.name, self.mime_type, self.description])) + def to_dict(self) -> dict[str, Optional[str]]: + """Serialise a URL-backed attachment for export/import.""" + return {"url": self.url, "name": self.name, "mime_type": self.mime_type, "description": self.description} + @property def id(self) -> str: """Compute an ID from the attachment data.""" diff --git a/packages/envied/src/envied/core/tracks/audio.py b/packages/envied/src/envied/core/tracks/audio.py index a682729..03ee9cd 100644 --- a/packages/envied/src/envied/core/tracks/audio.py +++ b/packages/envied/src/envied/core/tracks/audio.py @@ -57,7 +57,7 @@ class Audio(Track): @staticmethod def from_netflix_profile(profile: str) -> Audio.Codec: profile = profile.lower().strip() - if profile.startswith("heaac"): + if profile.startswith("heaac") or profile.startswith("xheaac"): return Audio.Codec.AAC if profile.startswith("dd-"): return Audio.Codec.AC3 @@ -126,6 +126,31 @@ class Audio(Track): self.joc = joc self.descriptive = bool(descriptive) + def to_dict(self) -> dict[str, Any]: + data = super().to_dict() + data.update( + { + "codec": self.codec.name if self.codec else None, + "bitrate": self.bitrate, + "channels": self.channels, + "joc": self.joc, + "descriptive": self.descriptive, + } + ) + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> Audio: + kwargs = Track.base_kwargs_from_dict(data) + return cls( + **kwargs, + codec=Audio.Codec[data["codec"]] if data.get("codec") else None, + bitrate=data.get("bitrate"), + channels=data.get("channels"), + joc=data.get("joc"), + descriptive=data.get("descriptive", False), + ) + @property def atmos(self) -> bool: """Return True if the audio track contains Dolby Atmos.""" diff --git a/packages/envied/src/envied/core/tracks/dv_fixup.py b/packages/envied/src/envied/core/tracks/dv_fixup.py new file mode 100644 index 0000000..5810ea5 --- /dev/null +++ b/packages/envied/src/envied/core/tracks/dv_fixup.py @@ -0,0 +1,117 @@ +""" +DV fixup for HLS composite HEVC streams. + +Some services deliver DV Profile 8.1 in a stream whose primary CODECS is plain +hvc1, with DV advertised only via SUPPLEMENTAL-CODECS. The fMP4 carries DV RPU NALs but +the container does not signal DV, so muxing produces an MKV that mediainfo and DV-capable +TVs see as plain HDR10/HDR10+. + +A dovi_tool extract-rpu / inject-rpu round-trip rewrites the bitstream so it is recognised +as DV after muxing. HDR10+ SEI NALs and HDR10 base layer signaling survive untouched. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import TYPE_CHECKING + +from rich.padding import Padding +from rich.rule import Rule + +from envied.core.binaries import FFMPEG, DoviTool +from envied.core.config import config +from envied.core.console import console +from envied.core.utilities import get_debug_logger +from envied.core.utils import dovi +from envied.core.utils.subprocess import run_step + +if TYPE_CHECKING: + from envied.core.tracks import Video + + +class DVFixup: + """Round-trip a DV-composite HEVC track through dovi_tool to restore DV signaling.""" + + def __init__(self, video: "Video") -> None: + self.log = logging.getLogger("dv-fixup") + self.debug_logger = get_debug_logger() + self.video = video + + if not DoviTool: + raise EnvironmentError("dovi_tool is required for DV-composite fixup but was not found.") + if not FFMPEG: + raise EnvironmentError("ffmpeg is required for DV-composite fixup but was not found.") + if not video.path or not Path(video.path).exists(): + raise ValueError(f"Video track {video.id} was not downloaded before DV fixup.") + + def run(self) -> Path: + """Execute the fixup. Returns the DV-signaled HEVC path, or the original + source path on any failure so muxing can proceed with the as-downloaded file.""" + source = Path(self.video.path) + height = self.video.height or 0 + console.print(Padding(Rule(f"[rule.text]DV Composite Fixup ({height}p)"), (1, 2))) + + fixed_hevc = source.with_name(f"{self.video.id}.dv.hevc") + if fixed_hevc.exists() and fixed_hevc.stat().st_size > 0: + self.log.info("✓ DV signaling already restored (reusing existing fixup)") + return fixed_hevc + + tmp = config.directories.temp + tmp.mkdir(parents=True, exist_ok=True) + suffix = f"{self.video.id}_{height or 'na'}" + raw_hevc = tmp / f"dvfix_{suffix}.hevc" + rpu = tmp / f"dvfix_{suffix}_rpu.bin" + + try: + run_step( + [FFMPEG, "-nostdin", "-y", "-i", source, "-c:v", "copy", "-f", "hevc", raw_hevc], + status="Demuxing HEVC bitstream...", + output=raw_hevc, + label="ffmpeg demux", + ) + dovi.extract_rpu_with_fallback(raw_hevc, rpu) + dovi.inject_rpu(raw_hevc, rpu, fixed_hevc, status="Re-injecting DV RPU with proper signaling...") + except Exception as e: + self.log.warning(f"DV fixup failed ({e}); muxing source as-is.") + if self.debug_logger: + self.debug_logger.log( + level="WARNING", + operation="dv_fixup", + message="DV fixup failed; falling back to source", + context={"error": str(e), "source": str(source)}, + ) + for leftover in (raw_hevc, rpu, fixed_hevc): + leftover.unlink(missing_ok=True) + return source + + for leftover in (raw_hevc, rpu): + leftover.unlink(missing_ok=True) + + self.log.info("✓ DV signaling restored") + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="dv_fixup", + message="DV fixup complete", + context={"source": str(source), "output": str(fixed_hevc)}, + success=True, + ) + return fixed_hevc + + +def apply_dv_fixup(video: "Video") -> None: + """Run DV fixup on `video` if flagged as DV-composite. Updates `video.path` in place + and deletes the original source file so the standard mux cleanup handles the new path.""" + if not getattr(video, "dv_compatible_bitstream", False): + return + if not video.path or not Path(video.path).exists(): + return + original = Path(video.path) + fixed = DVFixup(video).run() + if fixed != original: + video.path = fixed + original.unlink(missing_ok=True) + + +__all__ = ("DVFixup", "apply_dv_fixup") diff --git a/packages/envied/src/envied/core/tracks/hybrid.py b/packages/envied/src/envied/core/tracks/hybrid.py index 553548b..ccc4444 100644 --- a/packages/envied/src/envied/core/tracks/hybrid.py +++ b/packages/envied/src/envied/core/tracks/hybrid.py @@ -6,14 +6,17 @@ import re import subprocess import sys from pathlib import Path +from typing import Optional from rich.padding import Padding from rich.rule import Rule -from envied.core.binaries import FFMPEG, DoviTool, FFProbe, HDR10PlusTool +from envied.core.binaries import FFMPEG, FFProbe, HDR10PlusTool from envied.core.config import config from envied.core.console import console from envied.core.utilities import get_debug_logger +from envied.core.utils import dovi +from envied.core.utils.subprocess import run_step class Hybrid: @@ -113,9 +116,7 @@ class Hybrid: # Edit L6 with actual luminance values from RPU, then L5 active area self.level_6() - base_video = next( - (v for v in videos if v.range in (Video.Range.HDR10, Video.Range.HDR10P)), None - ) + base_video = next((v for v in videos if v.range in (Video.Range.HDR10, Video.Range.HDR10P)), None) if base_video and base_video.path: self.level_5(base_video.path) @@ -140,51 +141,27 @@ class Hybrid: Path.unlink(config.directories.temp / "dv.mkv") Path.unlink(config.directories.temp / "HDR10.hevc", missing_ok=True) Path.unlink(config.directories.temp / "DV.hevc", missing_ok=True) - Path.unlink(config.directories.temp / f"{self.rpu_file}", missing_ok=True) - Path.unlink(config.directories.temp / "RPU_L6.bin", missing_ok=True) - Path.unlink(config.directories.temp / "RPU_L5.bin", missing_ok=True) + for rpu_name in ("RPU.bin", "RPU_UNT.bin", "RPU_L5.bin", "RPU_L6.bin"): + Path.unlink(config.directories.temp / rpu_name, missing_ok=True) Path.unlink(config.directories.temp / "L5.json", missing_ok=True) Path.unlink(config.directories.temp / "L6.json", missing_ok=True) - def ffmpeg_simple(self, save_path, output): - """Simple ffmpeg execution without progress tracking""" - p = subprocess.run( - [ - str(FFMPEG) if FFMPEG else "ffmpeg", - "-nostdin", - "-i", - str(save_path), - "-c:v", - "copy", - str(output), - "-y", # overwrite output - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - return p - def extract_stream(self, save_path, type_): output = Path(config.directories.temp / f"{type_}.hevc") - - with console.status(f"Extracting {type_} stream...", spinner="dots"): - result = self.ffmpeg_simple(save_path, output) - - if result.returncode: - output.unlink(missing_ok=True) + try: + run_step( + [FFMPEG or "ffmpeg", "-nostdin", "-y", "-i", save_path, "-c:v", "copy", output], + status=f"Extracting {type_} stream...", + output=output, + label=f"ffmpeg extract {type_}", + ) + except RuntimeError as e: if self.debug_logger: self.debug_logger.log( level="ERROR", operation="hybrid_extract_stream", message=f"Failed extracting {type_} stream", - context={ - "type": type_, - "input": str(save_path), - "output": str(output), - "returncode": result.returncode, - "stderr": (result.stderr or b"").decode(errors="replace"), - "stdout": (result.stdout or b"").decode(errors="replace"), - }, + context={"type": type_, "input": str(save_path), "output": str(output), "error": str(e)}, ) self.log.error(f"x Failed extracting {type_} stream") sys.exit(1) @@ -204,48 +181,30 @@ class Hybrid: ): return - with console.status( - f"Extracting{' untouched ' if untouched else ' '}RPU from Dolby Vision stream...", spinner="dots" - ): - extraction_args = [str(DoviTool)] - if not untouched: - extraction_args += ["-m", "3"] - extraction_args += [ - "extract-rpu", - config.directories.temp / "DV.hevc", - "-o", - config.directories.temp / f"{'RPU' if not untouched else 'RPU_UNT'}.bin", - ] + rpu_name = "RPU_UNT" if untouched else "RPU" + rpu_path = config.directories.temp / f"{rpu_name}.bin" + dv_stream = config.directories.temp / "DV.hevc" + spinner = f"Extracting{' untouched ' if untouched else ' '}RPU from Dolby Vision stream..." - rpu_extraction = subprocess.run( - extraction_args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - rpu_name = "RPU" if not untouched else "RPU_UNT" - if rpu_extraction.returncode: - Path.unlink(config.directories.temp / f"{rpu_name}.bin") - stderr_text = rpu_extraction.stderr.decode(errors="replace") if rpu_extraction.stderr else "" + try: + dovi.extract_rpu(dv_stream, rpu_path, mode=None if untouched else 3, status=spinner) + except RuntimeError as e: + stderr_text = str(e) if self.debug_logger: self.debug_logger.log( level="ERROR", operation="hybrid_extract_rpu", message=f"Failed extracting{' untouched ' if untouched else ' '}RPU", - context={ - "untouched": untouched, - "returncode": rpu_extraction.returncode, - "stderr": stderr_text, - "args": [str(a) for a in extraction_args], - }, + context={"untouched": untouched, "error": stderr_text}, ) - if b"MAX_PQ_LUMINANCE" in rpu_extraction.stderr: + if "MAX_PQ_LUMINANCE" in stderr_text: self.extract_rpu(video, untouched=True) - elif b"Invalid PPS index" in rpu_extraction.stderr: + return + if "Invalid PPS index" in stderr_text: raise ValueError("Dolby Vision VideoTrack seems to be corrupt") - else: - raise ValueError(f"Failed extracting{' untouched ' if untouched else ' '}RPU from Dolby Vision stream") - elif self.debug_logger: + raise ValueError(f"Failed extracting{' untouched ' if untouched else ' '}RPU from Dolby Vision stream") + + if self.debug_logger: self.debug_logger.log( level="DEBUG", operation="hybrid_extract_rpu", @@ -383,11 +342,7 @@ class Hybrid: for crop in crop_results: crop_counts[crop] = crop_counts.get(crop, 0) + 1 most_common = max(crop_counts, key=crop_counts.get) - left, top, right, bottom = most_common - - # If all borders are 0 there's nothing to correct - if left == 0 and top == 0 and right == 0 and bottom == 0: - return + left, top, right, bottom = most_common # frame instead of leaving phantom bars from the source. l5_json = { "active_area": { @@ -401,31 +356,22 @@ class Hybrid: with open(l5_path, "w") as f: json.dump(l5_json, f, indent=4) - with console.status("Editing RPU Level 5 active area...", spinner="dots"): - result = subprocess.run( - [ - str(DoviTool), - "editor", - "-i", - str(config.directories.temp / self.rpu_file), - "-j", - str(l5_path), - "-o", - str(config.directories.temp / "RPU_L5.bin"), - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + try: + dovi.editor( + config.directories.temp / self.rpu_file, + l5_path, + config.directories.temp / "RPU_L5.bin", + status="Editing RPU Level 5 active area...", + label="dovi_tool editor (L5)", ) - - if result.returncode: + except RuntimeError as e: if self.debug_logger: self.debug_logger.log( level="ERROR", operation="hybrid_level5", message="Failed editing RPU Level 5 values", - context={"returncode": result.returncode, "stderr": (result.stderr or b"").decode(errors="replace")}, + context={"error": str(e)}, ) - Path.unlink(config.directories.temp / "RPU_L5.bin", missing_ok=True) raise ValueError("Failed editing RPU Level 5 values") if self.debug_logger: @@ -433,30 +379,45 @@ class Hybrid: level="DEBUG", operation="hybrid_level5", message="Edited RPU Level 5 active area", - context={"crop": {"left": left, "right": right, "top": top, "bottom": bottom}, "samples": len(crop_results)}, + context={ + "crop": {"left": left, "right": right, "top": top, "bottom": bottom}, + "samples": len(crop_results), + }, success=True, ) self.rpu_file = "RPU_L5.bin" + @staticmethod + def sanitize_l6( + max_mdl: Optional[int], min_mdl: Optional[int], max_cll: Optional[int], max_fall: Optional[int] + ) -> tuple[Optional[int], Optional[int], Optional[int], Optional[int]]: + """Clamp static L6 values to a valid relationship. + + MaxCLL must not exceed the mastering-display peak (some sources, e.g. ATV + HDR10+, ship MaxCLL 10000 on a 1000-nit master), and MaxFALL must not exceed + MaxCLL. A value of 0 means "unknown" and is preserved as-is. + """ + if max_mdl and max_cll and max_cll > max_mdl: + max_cll = max_mdl + if max_cll and max_fall and max_fall > max_cll: + max_fall = max_cll + return max_mdl, min_mdl, max_cll, max_fall + def level_6(self): - """Edit RPU Level 6 values using actual luminance data from the RPU.""" + """Edit RPU Level 6 values using the static L6 luminance data from the RPU.""" if os.path.isfile(config.directories.temp / "RPU_L6.bin"): return - with console.status("Reading RPU luminance metadata...", spinner="dots"): - result = subprocess.run( - [str(DoviTool), "info", "-i", str(config.directories.temp / self.rpu_file), "-s"], - capture_output=True, - text=True, - ) - - if result.returncode != 0: + try: + with console.status("Reading RPU luminance metadata...", spinner="dots"): + info_text = dovi.info_summary(config.directories.temp / self.rpu_file) + except RuntimeError as e: if self.debug_logger: self.debug_logger.log( level="ERROR", operation="hybrid_level6", message="Failed reading RPU metadata for Level 6 values", - context={"returncode": result.returncode, "stderr": (result.stderr or "")}, + context={"error": str(e)}, ) raise ValueError("Failed reading RPU metadata for Level 6 values") @@ -465,17 +426,33 @@ class Hybrid: max_mdl = None min_mdl = None - for line in result.stdout.splitlines(): - if "RPU content light level (L1):" in line: - parts = line.split("MaxCLL:")[1].split(",") - max_cll = int(float(parts[0].strip().split()[0])) - if len(parts) > 1 and "MaxFALL:" in parts[1]: - max_fall = int(float(parts[1].split("MaxFALL:")[1].strip().split()[0])) - elif "RPU mastering display:" in line: - mastering = line.split(":", 1)[1].strip() + in_l6 = False + for line in info_text.splitlines(): + stripped = line.strip() + if "L6 metadata" in stripped: + in_l6 = True + if stripped.startswith("RPU mastering display:"): + mastering = stripped.split(":", 1)[1].strip() min_lum, max_lum = mastering.split("/")[0], mastering.split("/")[1].split(" ")[0] min_mdl = int(float(min_lum) * 10000) max_mdl = int(float(max_lum)) + elif in_l6 and "MaxCLL:" in stripped and max_cll is None: + max_cll = int(float(stripped.split("MaxCLL:")[1].split("nits")[0].strip().rstrip(","))) + if "MaxFALL:" in stripped: + max_fall = int(float(stripped.split("MaxFALL:")[1].split("nits")[0].strip().rstrip(","))) + + if any(v is None for v in (max_cll, max_fall, max_mdl, min_mdl)): + base_max_mdl, base_min_mdl, base_cll, base_fall = self._probe_hdr_metadata() + if max_cll is None: + max_cll = base_cll + if max_fall is None: + max_fall = base_fall + if max_mdl is None: + max_mdl = base_max_mdl + if min_mdl is None: + min_mdl = base_min_mdl + + max_mdl, min_mdl, max_cll, max_fall = self.sanitize_l6(max_mdl, min_mdl, max_cll, max_fall) if any(v is None for v in (max_cll, max_fall, max_mdl, min_mdl)): if self.debug_logger: @@ -502,31 +479,22 @@ class Hybrid: with open(l6_path, "w") as f: json.dump(level6_data, f, indent=4) - with console.status("Editing RPU Level 6 values...", spinner="dots"): - result = subprocess.run( - [ - str(DoviTool), - "editor", - "-i", - str(config.directories.temp / self.rpu_file), - "-j", - str(l6_path), - "-o", - str(config.directories.temp / "RPU_L6.bin"), - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + try: + dovi.editor( + config.directories.temp / self.rpu_file, + l6_path, + config.directories.temp / "RPU_L6.bin", + status="Editing RPU Level 6 values...", + label="dovi_tool editor (L6)", ) - - if result.returncode: + except RuntimeError as e: if self.debug_logger: self.debug_logger.log( level="ERROR", operation="hybrid_level6", message="Failed editing RPU Level 6 values", - context={"returncode": result.returncode, "stderr": (result.stderr or b"").decode(errors="replace")}, + context={"error": str(e)}, ) - Path.unlink(config.directories.temp / "RPU_L6.bin", missing_ok=True) raise ValueError("Failed editing RPU Level 6 values") if self.debug_logger: @@ -548,38 +516,22 @@ class Hybrid: if os.path.isfile(config.directories.temp / self.hevc_file): return - with console.status(f"Injecting Dolby Vision metadata into {self.hdr_type} stream...", spinner="dots"): - inject_cmd = [ - str(DoviTool), - "inject-rpu", - "-i", + try: + dovi.inject_rpu( config.directories.temp / "HDR10.hevc", - "--rpu-in", config.directories.temp / self.rpu_file, - ] - - inject_cmd.extend(["-o", config.directories.temp / self.hevc_file]) - - inject = subprocess.run( - inject_cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + config.directories.temp / self.hevc_file, + status=f"Injecting Dolby Vision metadata into {self.hdr_type} stream...", + label="dovi_tool inject-rpu", ) - - if inject.returncode: + except RuntimeError as e: if self.debug_logger: self.debug_logger.log( level="ERROR", operation="hybrid_inject_rpu", message="Failed injecting Dolby Vision metadata into HDR10 stream", - context={ - "returncode": inject.returncode, - "stderr": (inject.stderr or b"").decode(errors="replace"), - "stdout": (inject.stdout or b"").decode(errors="replace"), - "cmd": [str(a) for a in inject_cmd], - }, + context={"error": str(e)}, ) - Path.unlink(config.directories.temp / self.hevc_file) raise ValueError("Failed injecting Dolby Vision metadata into HDR10 stream") if self.debug_logger: @@ -587,7 +539,12 @@ class Hybrid: level="DEBUG", operation="hybrid_inject_rpu", message=f"Injected Dolby Vision metadata into {self.hdr_type} stream", - context={"hdr_type": self.hdr_type, "rpu_file": self.rpu_file, "output": self.hevc_file, "drop_hdr10plus": self.hdr10plus_to_dv}, + context={ + "hdr_type": self.hdr_type, + "rpu_file": self.rpu_file, + "output": self.hevc_file, + "drop_hdr10plus": self.hdr10plus_to_dv, + }, success=True, ) @@ -599,35 +556,29 @@ class Hybrid: if not HDR10PlusTool: raise ValueError("HDR10Plus_tool not found. Please install it to use HDR10+ to DV conversion.") - with console.status("Extracting HDR10+ metadata...", spinner="dots"): - # HDR10Plus_tool needs raw HEVC stream - extraction = subprocess.run( + try: + run_step( [ - str(HDR10PlusTool), + HDR10PlusTool, "extract", - str(config.directories.temp / "HDR10.hevc"), + config.directories.temp / "HDR10.hevc", "-o", - str(config.directories.temp / self.hdr10plus_file), + config.directories.temp / self.hdr10plus_file, ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + status="Extracting HDR10+ metadata...", + output=config.directories.temp / self.hdr10plus_file, + label="hdr10plus_tool extract", ) - - if extraction.returncode: + except RuntimeError as e: if self.debug_logger: self.debug_logger.log( level="ERROR", operation="hybrid_extract_hdr10plus", message="Failed extracting HDR10+ metadata", - context={ - "returncode": extraction.returncode, - "stderr": (extraction.stderr or b"").decode(errors="replace"), - "stdout": (extraction.stdout or b"").decode(errors="replace"), - }, + context={"error": str(e)}, ) raise ValueError("Failed extracting HDR10+ metadata") - # Check if the extracted file has content file_size = os.path.getsize(config.directories.temp / self.hdr10plus_file) if file_size == 0: if self.debug_logger: @@ -648,54 +599,105 @@ class Hybrid: success=True, ) + def _probe_hdr_metadata(self): + """Extract mastering display and content light level metadata from the HDR10 stream via ffprobe. + + Returns (max_mdl, min_mdl, max_cll, max_fall) in dovi_tool level6 units: + - max_mdl: nits (integer) + - min_mdl: 0.0001 nit units (integer) + - max_cll / max_fall: nits (integer) + """ + ffprobe_bin = str(FFProbe) if FFProbe else "ffprobe" + result = subprocess.run( + [ + ffprobe_bin, + "-v", + "error", + "-select_streams", + "v:0", + "-show_entries", + "stream_side_data=max_luminance,min_luminance,max_content,max_average", + "-of", + "json", + str(config.directories.temp / "HDR10.hevc"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + max_mdl = 1000 + min_mdl = 1 + max_cll = 0 + max_fall = 0 + + if result.returncode == 0 and result.stdout: + try: + probe = json.loads(result.stdout) + for stream in probe.get("streams", []): + for sd in stream.get("side_data_list", []): + if "max_luminance" in sd: + num, den = sd["max_luminance"].split("/") + max_mdl = int(int(num) / int(den)) + if "min_luminance" in sd: + num, den = sd["min_luminance"].split("/") + min_mdl = int(int(num) / int(den) * 10000) + if "max_content" in sd: + max_cll = int(sd["max_content"]) + if "max_average" in sd: + max_fall = int(sd["max_average"]) + except (json.JSONDecodeError, KeyError, ValueError, ZeroDivisionError): + pass + + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="hybrid_probe_hdr_metadata", + message="Probed HDR metadata from source stream", + context={"max_mdl": max_mdl, "min_mdl": min_mdl, "max_cll": max_cll, "max_fall": max_fall}, + ) + + return max_mdl, min_mdl, max_cll, max_fall + def convert_hdr10plus_to_dv(self): """Convert HDR10+ metadata to Dolby Vision RPU""" if os.path.isfile(config.directories.temp / "RPU.bin"): return with console.status("Converting HDR10+ metadata to Dolby Vision...", spinner="dots"): + # Extract actual HDR metadata from the source stream + max_mdl, min_mdl, max_cll, max_fall = self._probe_hdr_metadata() + max_mdl, min_mdl, max_cll, max_fall = self.sanitize_l6(max_mdl, min_mdl, max_cll, max_fall) + # First create the extra metadata JSON for dovi_tool extra_metadata = { "cm_version": "V29", "length": 0, # dovi_tool will figure this out "level6": { - "max_display_mastering_luminance": 1000, - "min_display_mastering_luminance": 1, - "max_content_light_level": 0, - "max_frame_average_light_level": 0, + "max_display_mastering_luminance": max_mdl, + "min_display_mastering_luminance": min_mdl, + "max_content_light_level": max_cll, + "max_frame_average_light_level": max_fall, }, } with open(config.directories.temp / "extra.json", "w") as f: json.dump(extra_metadata, f, indent=2) - # Generate DV RPU from HDR10+ metadata - conversion = subprocess.run( - [ - str(DoviTool), - "generate", - "-j", - str(config.directories.temp / "extra.json"), - "--hdr10plus-json", - str(config.directories.temp / self.hdr10plus_file), - "-o", - str(config.directories.temp / "RPU.bin"), - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + try: + dovi.generate_from_hdr10plus( + config.directories.temp / "extra.json", + config.directories.temp / self.hdr10plus_file, + config.directories.temp / "RPU.bin", + label="dovi_tool generate", ) - - if conversion.returncode: + except RuntimeError as e: if self.debug_logger: self.debug_logger.log( level="ERROR", operation="hybrid_convert_hdr10plus", message="Failed converting HDR10+ to Dolby Vision", - context={ - "returncode": conversion.returncode, - "stderr": (conversion.stderr or b"").decode(errors="replace"), - "stdout": (conversion.stdout or b"").decode(errors="replace"), - }, + context={"error": str(e)}, ) raise ValueError("Failed converting HDR10+ to Dolby Vision") diff --git a/packages/envied/src/envied/core/tracks/subtitle.py b/packages/envied/src/envied/core/tracks/subtitle.py index 58c59d7..7135692 100644 --- a/packages/envied/src/envied/core/tracks/subtitle.py +++ b/packages/envied/src/envied/core/tracks/subtitle.py @@ -195,6 +195,29 @@ class Subtitle(Track): # Called after Track has been converted to another format self.OnConverted: Optional[Callable[[Subtitle.Codec], None]] = None + def to_dict(self) -> dict[str, Any]: + data = super().to_dict() + data.update( + { + "codec": self.codec.name if self.codec else None, + "cc": self.cc, + "sdh": self.sdh, + "forced": self.forced, + } + ) + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> Subtitle: + kwargs = Track.base_kwargs_from_dict(data) + return cls( + **kwargs, + codec=Subtitle.Codec[data["codec"]] if data.get("codec") else None, + cc=data.get("cc", False), + sdh=data.get("sdh", False), + forced=data.get("forced", False), + ) + def __str__(self) -> str: return " | ".join( filter( @@ -221,8 +244,9 @@ class Subtitle(Track): progress: Optional[partial] = None, *, cdm: Optional[object] = None, + no_proxy_download: bool = False, ): - super().download(session, prepare_drm, max_workers, progress, cdm=cdm) + super().download(session, prepare_drm, max_workers, progress, cdm=cdm, no_proxy_download=no_proxy_download) if not self.path: return diff --git a/packages/envied/src/envied/core/tracks/track.py b/packages/envied/src/envied/core/tracks/track.py index b0bd5eb..bcc285a 100644 --- a/packages/envied/src/envied/core/tracks/track.py +++ b/packages/envied/src/envied/core/tracks/track.py @@ -13,7 +13,6 @@ from typing import Any, Callable, Iterable, Optional, Union from uuid import UUID from zlib import crc32 -from curl_cffi.requests import Session as CurlSession from langcodes import Language from requests import Session @@ -21,13 +20,33 @@ from envied.core import binaries from envied.core.cdm.detect import is_playready_cdm, is_widevine_cdm from envied.core.config import config from envied.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY -from envied.core.downloaders import aria2c, curl_impersonate, n_m3u8dl_re, requests +from envied.core.downloaders import requests from envied.core.drm import DRM_T, PlayReady, Widevine from envied.core.events import events +from envied.core.session import RnetSession from envied.core.utilities import get_boxes, try_ensure_utf8 from envied.core.utils.subprocess import ffprobe +def direct_session(session: Union[Session, "RnetSession"]) -> Session: + """Vanilla requests.Session with copied headers/cookies, no proxy.""" + new = Session() + headers = getattr(session, "headers", None) + if headers is not None: + try: + new.headers.update(dict(headers)) + except Exception: + pass + cookies = getattr(session, "cookies", None) + if cookies is not None: + jar = getattr(cookies, "jar", None) + try: + new.cookies.update(jar if jar is not None else cookies) + except Exception: + pass + return new + + class Track: class Descriptor(Enum): URL = 1 # Direct URL, nothing fancy @@ -45,6 +64,7 @@ class Track: name: Optional[str] = None, drm: Optional[Iterable[DRM_T]] = None, edition: Optional[str] = None, + session: Optional[Union[Session, "RnetSession"]] = None, downloader: Optional[Callable] = None, downloader_args: Optional[dict] = None, from_file: Optional[Path] = None, @@ -88,12 +108,7 @@ class Track: raise TypeError(f"Expected drm to be an iterable, not {type(drm)}") if downloader is None: - downloader = { - "aria2c": aria2c, - "curl_impersonate": curl_impersonate, - "requests": requests, - "n_m3u8dl_re": n_m3u8dl_re, - }[config.downloader] + downloader = requests self.path: Optional[Path] = None self.url = url @@ -104,6 +119,7 @@ class Track: self.name = name self.drm = drm self.edition: list[str] = [edition] if isinstance(edition, str) else (edition or []) + self.session = session self.downloader = downloader self.downloader_args = downloader_args self.from_file = from_file @@ -185,12 +201,13 @@ class Track: def download( self, - session: Session, + session: Union[Session, "RnetSession"], prepare_drm: partial, max_workers: Optional[int] = None, progress: Optional[partial] = None, *, cdm: Optional[object] = None, + no_proxy_download: bool = False, ): """Download and optionally Decrypt this Track.""" from envied.core.manifests import DASH, HLS, ISM @@ -206,28 +223,23 @@ class Track: proxy = next(iter(session.proxies.values()), None) + dl_session = session + if no_proxy_download and proxy: + dl_session = direct_session(session) + proxy = None + track_type = self.__class__.__name__ save_path = config.directories.temp / f"{track_type}_{self.id}.mp4" if track_type == "Subtitle": save_path = save_path.with_suffix(f".{self.codec.extension}") - if self.downloader.__name__ == "n_m3u8dl_re" and ( - self.descriptor == self.Descriptor.URL - or track_type in ("Subtitle", "Attachment") - ): - self.downloader = requests - if self.descriptor != self.Descriptor.URL: save_dir = save_path.with_name(save_path.name + "_segments") else: save_dir = save_path.parent def cleanup(): - # track file (e.g., "foo.mp4") save_path.unlink(missing_ok=True) - # aria2c control file (e.g., "foo.mp4.aria2" or "foo.mp4.aria2__temp") - save_path.with_suffix(f"{save_path.suffix}.aria2").unlink(missing_ok=True) - save_path.with_suffix(f"{save_path.suffix}.aria2__temp").unlink(missing_ok=True) if save_dir.exists() and save_dir.name.endswith("_segments"): shutil.rmtree(save_dir) @@ -250,7 +262,7 @@ class Track: save_path=save_path, save_dir=save_dir, progress=progress, - session=session, + session=dl_session, proxy=proxy, max_workers=max_workers, license_widevine=prepare_drm, @@ -262,7 +274,7 @@ class Track: save_path=save_path, save_dir=save_dir, progress=progress, - session=session, + session=dl_session, proxy=proxy, max_workers=max_workers, license_widevine=prepare_drm, @@ -274,7 +286,7 @@ class Track: save_path=save_path, save_dir=save_dir, progress=progress, - session=session, + session=dl_session, proxy=proxy, max_workers=max_workers, license_widevine=prepare_drm, @@ -328,27 +340,24 @@ class Track: if DOWNLOAD_LICENCE_ONLY.is_set(): progress(downloaded="[yellow]SKIPPED") - elif track_type != "Subtitle" and self.downloader.__name__ == "n_m3u8dl_re": - progress(downloaded="[red]FAILED") - error = f"[N_m3u8DL-RE]: {self.descriptor} is currently not supported" - raise ValueError(error) else: for status_update in self.downloader( urls=self.url, output_dir=save_path.parent, filename=save_path.name, - headers=session.headers, - cookies=session.cookies, + headers=dl_session.headers, + cookies=dl_session.cookies, proxy=proxy, max_workers=max_workers, + session=dl_session, ): file_downloaded = status_update.get("file_downloaded") if not file_downloaded: + downloaded = status_update.get("downloaded") + if downloaded and downloaded.endswith("/s"): + status_update["downloaded"] = f"URL {downloaded}" progress(**status_update) - # see https://github.com/devine-dl/devine/issues/71 - save_path.with_suffix(f"{save_path.suffix}.aria2__temp").unlink(missing_ok=True) - self.path = save_path events.emit(events.Types.TRACK_DOWNLOADED, track=self) @@ -430,6 +439,57 @@ class Track: self.path = target return target + def to_dict(self) -> dict[str, Any]: + """Serialise the track for export/import (identity/URL/descriptor/language). + + DRM is not serialised here; the export writer attaches the licensed DRM + keys. + Subclasses add their own codec/quality fields. + """ + data: dict[str, Any] = { + "type": self.__class__.__name__, + "id": self.id, + "url": self.url, + "language": str(self.language), + "is_original_lang": self.is_original_lang, + "descriptor": self.descriptor.name, + "needs_repack": self.needs_repack, + "name": self.name, + "edition": self.edition, + } + return data + + @staticmethod + def base_kwargs_from_dict(data: dict[str, Any]) -> dict[str, Any]: + """Build the shared Track constructor kwargs from a ``to_dict()`` payload. + + DRM is not reconstructed here — ``to_dict`` does not serialise it, and the import + flow attaches the licensed DRM + content keys separately. + """ + return { + "url": data["url"], + "language": data.get("language") or "und", + "is_original_lang": data.get("is_original_lang", False), + "descriptor": Track.Descriptor[data.get("descriptor", "URL")], + "needs_repack": data.get("needs_repack", False), + "name": data.get("name"), + "edition": data.get("edition") or None, + "id_": data.get("id"), + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "Track": + """Reconstruct the correct Track subclass from a ``to_dict()`` payload.""" + from envied.core.tracks.audio import Audio + from envied.core.tracks.subtitle import Subtitle + from envied.core.tracks.video import Video + + track_type = data.get("type") + builders = {"Video": Video, "Audio": Audio, "Subtitle": Subtitle} + builder = builders.get(track_type) + if builder is None: + raise ValueError(f"Cannot reconstruct unsupported track type: {track_type!r}") + return builder.from_dict(data) + def get_track_name(self) -> Optional[str]: """Get the Track Name.""" return self.name @@ -602,8 +662,8 @@ class Track: raise TypeError(f"Expected url to be a {str}, not {type(url)}") if not isinstance(byte_range, (str, type(None))): raise TypeError(f"Expected byte_range to be a {str}, not {type(byte_range)}") - if not isinstance(session, (Session, CurlSession, type(None))): - raise TypeError(f"Expected session to be a {Session} or {CurlSession}, not {type(session)}") + if not isinstance(session, (Session, RnetSession, type(None))): + raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {type(session)}") if not url: if self.descriptor != self.Descriptor.URL: @@ -641,10 +701,11 @@ class Track: init_data = res.content else: init_data = None - with session.get(url, stream=True) as s: - for chunk in s.iter_content(content_length): - init_data = chunk - break + s = session.get(url, stream=True) + for chunk in s.iter_content(content_length): + init_data = chunk + break + s.close() if not init_data: raise ValueError(f"Failed to read {content_length} bytes from the track URI.") diff --git a/packages/envied/src/envied/core/tracks/tracks.py b/packages/envied/src/envied/core/tracks/tracks.py index 5853a96..f14d47c 100644 --- a/packages/envied/src/envied/core/tracks/tracks.py +++ b/packages/envied/src/envied/core/tracks/tracks.py @@ -39,12 +39,14 @@ class Tracks: *args: Union[ Tracks, Sequence[Union[AnyTrack, Chapter, Chapters, Attachment]], Track, Chapter, Chapters, Attachment ], + manifest_url: Optional[str] = None, ): self.videos: list[Video] = [] self.audio: list[Audio] = [] self.subtitles: list[Subtitle] = [] self.chapters = Chapters() self.attachments: list[Attachment] = [] + self.manifest_url: Optional[str] = manifest_url if args: self.add(args) @@ -195,6 +197,8 @@ class Tracks: ) -> None: """Add a provided track to its appropriate array and ensuring it's not a duplicate.""" if isinstance(tracks, Tracks): + if tracks.manifest_url and not self.manifest_url: + self.manifest_url = tracks.manifest_url tracks = [*list(tracks), *tracks.chapters, *tracks.attachments] duplicates = 0 @@ -245,12 +249,21 @@ class Tracks: self.videos.sort(key=lambda x: str(x.language)) self.videos.sort(key=lambda x: not is_close_match(language, [x.language])) - def sort_audio(self, by_language: Optional[Sequence[Union[str, Language]]] = None) -> None: - """Sort audio tracks by bitrate, Atmos, descriptive, and optionally language.""" + def sort_audio( + self, + by_language: Optional[Sequence[Union[str, Language]]] = None, + codec_priority: Optional[Sequence[str]] = None, + ) -> None: + """Sort audio tracks by bitrate, codec priority, Atmos, descriptive, and optionally language.""" if not self.audio: return # bitrate (highest first) self.audio.sort(key=lambda x: float(x.bitrate or 0.0), reverse=True) + # codec priority (listed codecs ranked in order; unlisted fall to end with bitrate order preserved) + if codec_priority: + rank = {str(c).upper(): i for i, c in enumerate(codec_priority)} + default_rank = len(rank) + self.audio.sort(key=lambda x: rank.get(x.codec.name if x.codec else "", default_rank)) # Atmos tracks first (prioritize over higher bitrate non-Atmos) self.audio.sort(key=lambda x: not x.atmos) # descriptive tracks last @@ -302,25 +315,48 @@ class Tracks: def select_subtitles(self, x: Callable[[Subtitle], bool]) -> None: self.subtitles = list(filter(x, self.subtitles)) - def select_hybrid(self, tracks, quality): + def filter(self, predicate: Callable[[AnyTrack], bool]) -> Tracks: + """Return a new Tracks with tracks filtered by predicate, preserving metadata.""" + new_tracks = Tracks(manifest_url=self.manifest_url) + new_tracks.videos = [t for t in self.videos if predicate(t)] + new_tracks.audio = [t for t in self.audio if predicate(t)] + new_tracks.subtitles = [t for t in self.subtitles if predicate(t)] + new_tracks.chapters = self.chapters + new_tracks.attachments = list(self.attachments) + return new_tracks + + @staticmethod + def merge_video_selections(*groups: list[Video]) -> list[Video]: + """Concatenate video selections, dropping duplicates (by track id, order-preserving). + + A DV track can be chosen as both the hybrid ingredient (lowest) and an explicit + deliverable; without dedup the same track would be muxed/downloaded twice. + """ + merged: list[Video] = [] + for group in groups: + for video in group: + if video not in merged: + merged.append(video) + return merged + + def select_hybrid(self, tracks, quality, worst: bool = False): # Prefer HDR10+ over HDR10 as the base layer (preserves dynamic metadata) base_ranges = (Video.Range.HDR10P, Video.Range.HDR10) base_tracks = [] for range_type in base_ranges: base_tracks = [ - v - for v in tracks - if v.range == range_type and (v.height in quality or int(v.width * 9 / 16) in quality) + v for v in tracks if v.range == range_type and (v.height in quality or int(v.width * 9 / 16) in quality) ] if base_tracks: break + pick = min if worst else max base_selected = [] for res in quality: candidates = [v for v in base_tracks if v.height == res or int(v.width * 9 / 16) == res] if candidates: - best = max(candidates, key=lambda v: v.bitrate) - base_selected.append(best) + chosen = pick(candidates, key=lambda v: v.bitrate) + base_selected.append(chosen) dv_tracks = [v for v in tracks if v.range == Video.Range.DV] lowest_dv = min(dv_tracks, key=lambda v: v.height) if dv_tracks else None @@ -415,13 +451,40 @@ class Tracks: if config.muxing.get("set_title", True): cl.extend(["--title", title]) + default_language = config.muxing.get("default_language") or {} + preferred_video_lang = default_language.get("video") + preferred_audio_lang = default_language.get("audio") + preferred_subtitle_lang = default_language.get("subtitle") + + preferred_video_idx: Optional[int] = None + if preferred_video_lang: + preferred_video_idx = next( + (idx for idx, v in enumerate(self.videos) if is_close_match(v.language, [preferred_video_lang])), + None, + ) + + preferred_audio_idx: Optional[int] = None + if preferred_audio_lang: + preferred_audio_idx = next( + (idx for idx, a in enumerate(self.audio) if is_close_match(a.language, [preferred_audio_lang])), + None, + ) + + preferred_subtitle_idx: Optional[int] = None + if preferred_subtitle_lang and not skip_subtitles: + preferred_subtitle_idx = next( + (idx for idx, s in enumerate(self.subtitles) if is_close_match(s.language, [preferred_subtitle_lang])), + None, + ) + for i, vt in enumerate(self.videos): if not vt.path or not vt.path.exists(): raise ValueError("Video Track must be downloaded before muxing...") events.emit(events.Types.TRACK_MULTIPLEX, track=vt) - is_default = False - if title_language: + if preferred_video_idx is not None: + is_default = i == preferred_video_idx + elif title_language: is_default = vt.language == title_language if not any(v.language == title_language for v in self.videos): is_default = vt.is_original_lang or i == 0 @@ -477,6 +540,10 @@ class Tracks: if not at.path or not at.path.exists(): raise ValueError("Audio Track must be downloaded before muxing...") events.emit(events.Types.TRACK_MULTIPLEX, track=at) + if preferred_audio_idx is not None: + audio_default = i == preferred_audio_idx + else: + audio_default = at.is_original_lang cl.extend( [ "--track-name", @@ -484,7 +551,7 @@ class Tracks: "--language", f"0:{at.language}", "--default-track", - f"0:{at.is_original_lang}", + f"0:{audio_default}", "--visual-impaired-flag", f"0:{at.descriptive}", "--original-flag", @@ -498,11 +565,14 @@ class Tracks: ) if not skip_subtitles: - for st in self.subtitles: + for i, st in enumerate(self.subtitles): if not st.path or not st.path.exists(): raise ValueError("Text Track must be downloaded before muxing...") events.emit(events.Types.TRACK_MULTIPLEX, track=st) - default = bool(self.audio and is_close_match(st.language, [self.audio[0].language]) and st.forced) + if preferred_subtitle_idx is not None: + default = i == preferred_subtitle_idx + else: + default = bool(self.audio and is_close_match(st.language, [self.audio[0].language]) and st.forced) cl.extend( [ "--track-name", diff --git a/packages/envied/src/envied/core/tracks/video.py b/packages/envied/src/envied/core/tracks/video.py index 17356b9..fd8727d 100644 --- a/packages/envied/src/envied/core/tracks/video.py +++ b/packages/envied/src/envied/core/tracks/video.py @@ -126,27 +126,48 @@ class Video(Track): Reserved = 0 BT_709 = 1 Unspecified = 2 + BT_470_M = 4 BT_601_625 = 5 BT_601_525 = 6 + SMPTE_240M = 7 + Generic_Film = 8 BT_2020_and_2100 = 9 + SMPTE_ST_428_1 = 10 + SMPTE_RP_431_2 = 11 # P3DCI SMPTE_ST_2113_and_EG_4321 = 12 # P3D65 + EBU_Tech_3213_E = 22 class Transfer(Enum): Reserved = 0 BT_709 = 1 Unspecified = 2 + BT_470_M = 4 BT_601 = 6 + SMPTE_240M = 7 + Linear = 8 + Log_100 = 9 + Log_316 = 10 + IEC_61966_2_4 = 11 + BT_1361 = 12 + IEC_61966_2_1 = 13 # sRGB / sYCC BT_2020 = 14 BT_2100 = 15 BT_2100_PQ = 16 + SMPTE_ST_428_1 = 17 BT_2100_HLG = 18 class Matrix(Enum): RGB = 0 YCbCr_BT_709 = 1 + Unspecified = 2 + YCbCr_FCC_73_682 = 4 YCbCr_BT_601_625 = 5 YCbCr_BT_601_525 = 6 + SMPTE_240M = 7 + YCgCo = 8 YCbCr_BT_2020_and_2100 = 9 # YCbCr BT.2100 shares the same CP + YCbCr_BT_2020_CL = 10 + YCbCr_SMPTE_ST_2085 = 11 ICtCp_BT_2100 = 14 if transfer == 5: @@ -155,9 +176,15 @@ class Video(Track): # The codebase is currently agnostic to either, so a manual conversion to 6 is done. transfer = 6 - primaries = Primaries(primaries) - transfer = Transfer(transfer) - matrix = Matrix(matrix) + def _safe(enum_cls, value): + try: + return enum_cls(value) + except ValueError: + return enum_cls(2) # Unspecified for unknown/private-use codes + + primaries = _safe(Primaries, primaries) + transfer = _safe(Transfer, transfer) + matrix = _safe(Matrix, matrix) # primaries and matrix does not strictly correlate to a range @@ -201,6 +228,7 @@ class Video(Track): fps: Optional[Union[str, int, float]] = None, scan_type: Optional[Video.ScanType] = None, closed_captions: Optional[list[dict[str, Any]]] = None, + dv_compatible_bitstream: bool = False, **kwargs: Any, ) -> None: """ @@ -267,6 +295,39 @@ class Video(Track): self.scan_type = scan_type self.closed_captions: list[dict[str, Any]] = closed_captions or [] self.needs_duration_fix = False + self.dv_compatible_bitstream = dv_compatible_bitstream + self.hybrid_base_only = False + + def to_dict(self) -> dict[str, Any]: + data = super().to_dict() + data.update( + { + "codec": self.codec.name if self.codec else None, + "range": self.range.name if self.range else None, + "bitrate": self.bitrate, + "width": self.width, + "height": self.height, + "fps": str(self.fps) if self.fps else None, + "scan_type": self.scan_type.name if self.scan_type else None, + "dv_compatible_bitstream": self.dv_compatible_bitstream, + } + ) + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> Video: + kwargs = Track.base_kwargs_from_dict(data) + return cls( + **kwargs, + codec=Video.Codec[data["codec"]] if data.get("codec") else None, + range_=Video.Range[data["range"]] if data.get("range") else None, + bitrate=data.get("bitrate"), + width=data.get("width"), + height=data.get("height"), + fps=data.get("fps"), + scan_type=Video.ScanType[data["scan_type"]] if data.get("scan_type") else None, + dv_compatible_bitstream=data.get("dv_compatible_bitstream", False), + ) def __str__(self) -> str: return " | ".join( @@ -338,6 +399,67 @@ class Video(Track): self.path = output_path original_path.unlink() + def normalize_vui(self) -> bool: + """Rewrite SPS VUI colour metadata to match ``self.range``. + + Some services ship HDR10/HLG bitstreams with stale BT.709 VUI, which makes + downstream tools mis-classify the file. The manifest-derived range is the + source of truth. Skips SDR, DV, and HYBRID. Returns True if the bitstream + was rewritten. + """ + if not self.path or not self.path.exists(): + return False + if self.codec not in (Video.Codec.AVC, Video.Codec.HEVC): + return False + if self.range in (Video.Range.SDR, Video.Range.DV, Video.Range.HYBRID): + return False + + vui = { + Video.Range.HDR10: (9, 16, 9), + Video.Range.HDR10P: (9, 16, 9), + Video.Range.HLG: (9, 18, 9), + }.get(self.range) + if not vui: + return False + + if not binaries.FFMPEG: + raise EnvironmentError('FFmpeg executable "ffmpeg" was not found but is required for this call.') + + primaries, transfer, matrix = vui + filter_key = {Video.Codec.AVC: "h264_metadata", Video.Codec.HEVC: "hevc_metadata"}[self.codec] + bsf = ( + f"{filter_key}=colour_primaries={primaries}" + f":transfer_characteristics={transfer}" + f":matrix_coefficients={matrix}" + ) + + original_path = self.path + output_path = original_path.with_stem(f"{original_path.stem}_vui") + try: + subprocess.run( + [ + binaries.FFMPEG, + "-hide_banner", + "-loglevel", + "error", + "-i", + str(original_path), + "-codec", + "copy", + "-bsf:v", + bsf, + str(output_path), + ], + check=True, + ) + except subprocess.CalledProcessError: + output_path.unlink(missing_ok=True) + return False + + self.path = output_path + original_path.unlink() + return True + def ccextractor( self, track_id: Any, out_path: Union[Path, str], language: Language, original: bool = False ) -> Optional[Subtitle]: diff --git a/packages/envied/src/envied/core/utilities.py b/packages/envied/src/envied/core/utilities.py index 83c61c1..7c51441 100644 --- a/packages/envied/src/envied/core/utilities.py +++ b/packages/envied/src/envied/core/utilities.py @@ -1,5 +1,6 @@ import ast import contextlib +import gzip import importlib.util import json import logging @@ -10,6 +11,7 @@ import sys import time import traceback import unicodedata +import zlib from collections import defaultdict from datetime import datetime, timezone from pathlib import Path @@ -20,14 +22,12 @@ from uuid import uuid4 import chardet import pycountry -import requests from construct import ValidationError from fontTools import ttLib from langcodes import Language, closest_match from pymp4.parser import Box from unidecode import unidecode -from envied.core.cacher import Cacher from envied.core.config import config from envied.core.constants import LANGUAGE_EXACT_DISTANCE, LANGUAGE_MAX_DISTANCE @@ -129,6 +129,8 @@ def sanitize_filename(filename: str, spacer: str = ".") -> str: # optionally replace non-ASCII characters with ASCII equivalents if not config.unicode_filenames: filename = unidecode(filename) + filename = re.sub(r"\[\(+", "[", filename) + filename = re.sub(r"\)+\]", "]", filename) # remove or replace further characters as needed filename = "".join(c for c in filename if unicodedata.category(c) != "Mn") # hidden characters @@ -158,6 +160,18 @@ def is_exact_match(language: Union[str, Language], languages: Sequence[Union[str return closest_match(language, list(map(str, languages)))[1] <= LANGUAGE_EXACT_DISTANCE +def find_missing_langs( + requested: Sequence[str], + available: Sequence[Union[str, Language, None]], + *, + exact: bool = False, +) -> list[str]: + """Return requested language tokens with no match in available languages.""" + match_func = is_exact_match if exact else is_close_match + skip = {"all", "best", "orig"} + return [tok for tok in requested if tok not in skip and not match_func(tok, available)] + + def get_boxes(data: bytes, box_type: bytes, as_bytes: bool = False) -> Box: # type: ignore """ Scan a byte array for a wanted MP4/ISOBMFF box, then parse and yield each find. @@ -352,111 +366,6 @@ def get_country_code(name: str) -> Optional[str]: return None -def get_ip_info(session: Optional[requests.Session] = None) -> dict: - """ - Use ipinfo.io to get IP location information. - - If you provide a Requests Session with a Proxy, that proxies IP information - is what will be returned. - """ - return (session or requests.Session()).get("https://ipinfo.io/json").json() - - -def get_cached_ip_info(session: Optional[requests.Session] = None) -> Optional[dict]: - """ - Get IP location information with 24-hour caching and fallback providers. - - This function uses a global cache to avoid repeated API calls when the IP - hasn't changed. Should only be used for local IP checks, not for proxy verification. - Implements smart provider rotation to handle rate limiting (429 errors). - - Args: - session: Optional requests session (usually without proxy for local IP) - - Returns: - Dict with IP info including 'country' key, or None if all providers fail - """ - - log = logging.getLogger("get_cached_ip_info") - cache = Cacher("global").get("ip_info") - - if cache and not cache.expired: - return cache.data - - provider_state_cache = Cacher("global").get("ip_provider_state") - provider_state = provider_state_cache.data if provider_state_cache and not provider_state_cache.expired else {} - - providers = { - "ipinfo": "https://ipinfo.io/json", - "ipapi": "https://ipapi.co/json", - } - - session = session or requests.Session() - provider_order = ["ipinfo", "ipapi"] - - current_time = time.time() - for provider_name in list(provider_order): - if provider_name in provider_state: - rate_limit_info = provider_state[provider_name] - if (current_time - rate_limit_info.get("rate_limited_at", 0)) < 300: - log.debug(f"Provider {provider_name} was rate limited recently, trying other provider first") - provider_order.remove(provider_name) - provider_order.append(provider_name) - break - - for provider_name in provider_order: - provider_url = providers[provider_name] - try: - log.debug(f"Trying IP provider: {provider_name}") - response = session.get(provider_url, timeout=10) - - if response.status_code == 429: - log.warning(f"Provider {provider_name} returned 429 (rate limited), trying next provider") - if provider_name not in provider_state: - provider_state[provider_name] = {} - provider_state[provider_name]["rate_limited_at"] = current_time - provider_state[provider_name]["rate_limit_count"] = ( - provider_state[provider_name].get("rate_limit_count", 0) + 1 - ) - - provider_state_cache.set(provider_state, expiration=300) - continue - - elif response.status_code == 200: - data = response.json() - normalized_data = {} - - if "country" in data: - normalized_data = data - elif "country_code" in data: - normalized_data = { - "country": data.get("country_code", "").lower(), - "region": data.get("region", ""), - "city": data.get("city", ""), - "ip": data.get("ip", ""), - } - - if normalized_data and "country" in normalized_data: - log.debug(f"Successfully got IP info from provider: {provider_name}") - - if provider_name in provider_state: - provider_state[provider_name].pop("rate_limited_at", None) - provider_state_cache.set(provider_state, expiration=300) - - normalized_data["_provider"] = provider_name - cache.set(normalized_data, expiration=86400) - return normalized_data - else: - log.debug(f"Provider {provider_name} returned status {response.status_code}") - - except Exception as e: - log.debug(f"Provider {provider_name} failed with exception: {e}") - continue - - log.warning("All IP geolocation providers failed") - return None - - def time_elapsed_since(start: float) -> str: """ Get time elapsed since a timestamp as a string. @@ -478,12 +387,29 @@ def try_ensure_utf8(data: bytes) -> bytes: """ Try to ensure that the given data is encoded in UTF-8. + Automatically decompresses gzip/deflate/zlib data before encoding detection. + This handles cases where HTTP responses are saved with raw Content-Encoding + (e.g., when decode_content=False is used for performance). + Parameters: data: Input data that may or may not yet be UTF-8 or another encoding. Returns the input data encoded in UTF-8 if successful. If unable to detect the encoding of the input data, then the original data is returned as-received. """ + # Decompress gzip data (magic bytes: 1f 8b) + if data[:2] == b"\x1f\x8b": + try: + data = gzip.decompress(data) + except Exception: + pass + # Decompress raw deflate/zlib data (common zlib headers: 78 01, 78 5e, 78 9c, 78 da) + elif data[:1] == b"\x78" and len(data) > 1 and data[1:2] in (b"\x01", b"\x5e", b"\x9c", b"\xda"): + try: + data = zlib.decompress(data) + except Exception: + pass + try: data.decode("utf8") return data @@ -497,10 +423,10 @@ def try_ensure_utf8(data: bytes) -> bytes: # last ditch effort to detect encoding detection_result = chardet.detect(data) if not detection_result["encoding"]: - return data - return data.decode(detection_result["encoding"]).encode("utf8") - except UnicodeDecodeError: - return data + return data.decode("utf-8", errors="replace").encode("utf-8") + return data.decode(detection_result["encoding"], errors="replace").encode("utf8") + except (UnicodeDecodeError, LookupError): + return data.decode("utf-8", errors="replace").encode("utf-8") def get_free_port() -> int: diff --git a/packages/envied/src/envied/core/utils/bitrate.py b/packages/envied/src/envied/core/utils/bitrate.py new file mode 100644 index 0000000..046809f --- /dev/null +++ b/packages/envied/src/envied/core/utils/bitrate.py @@ -0,0 +1,441 @@ +from __future__ import annotations + +import json +import logging +import subprocess +from collections import OrderedDict, defaultdict +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from typing import TYPE_CHECKING, Callable, Hashable, Optional, Union +from urllib.parse import urljoin + +from requests import Session + +from envied.core.binaries import FFProbe +from envied.core.session import RnetSession + +if TYPE_CHECKING: + from envied.core.tracks import Track + +# Default ISM timescale (ticks per second) per the Smooth Streaming spec. +ISM_DEFAULT_TIMESCALE = 10_000_000 + +# Bytes fetched to locate an mp4 moov box when probing duration via ffprobe. +MOOV_PROBE_BYTES = 4 * 1024 * 1024 + +# Network timeout (seconds) for probe requests. +PROBE_TIMEOUT = 15 + + +@dataclass +class Segment: + """One probe target: a media URL, optional byte range, its size, and duration.""" + + url: str + # The original byte-range string (e.g. "0-1023"), preserved as the segment's + # identity so distinct ranges of one file are never confused with each other. + byte_range: Optional[str] + # Size in bytes when derivable without a request (from a byte range); else None. + known_size: Optional[int] + duration: float + + +def measure_real_bitrate( + track: "Track", + session: Union[Session, RnetSession], + *, + samples: int = 40, + log: logging.Logger, +) -> Optional[int]: + """ + Probe a track's actual media size to compute its real average bitrate. + + Manifests often declare an inaccurate bandwidth (DASH ``@bandwidth`` is a + leaky-bucket ceiling, not an average). This measures the true bitrate + (bits/sec) from real media byte sizes and durations using ``bytes * 8 / sec``. + + Single-file tracks are measured exactly. Segmented tracks probe up to + ``samples`` segments spread across the track and extrapolate; byte-range + segments need no request. Returns bits/sec, or ``None`` if it cannot be + measured. Never raises — a probe failure must not abort a download. + """ + from envied.core.tracks.track import Track + + try: + if track.descriptor == Track.Descriptor.DASH: + segments = extract_dash(track, session) + elif track.descriptor == Track.Descriptor.HLS: + segments = extract_hls(track, session) + elif track.descriptor == Track.Descriptor.ISM: + segments = extract_ism(track, session) + else: + # Descriptor.URL: a single file. Some services (e.g. AMZN) parse a DASH + # manifest then collapse each representation to its single BaseURL and + # flip the descriptor to URL, leaving the manifest (and its duration) in + # track.data — recover the duration from there, else probe the file. + segments = extract_url(track, session, log=log) + if not segments: + log.debug(f"{track.id}: cannot measure real bitrate (no known duration)") + return None + except Exception as e: + log.warning(f"{track.id}: failed to derive segments for real bitrate ({e})") + return None + + if not segments: + return None + + items = dedupe(segments) + chosen = pick_samples(items, samples) + + total_bytes = 0 + total_seconds = 0.0 + for segment in chosen: + if segment.duration <= 0: + continue + size = segment.known_size if segment.known_size is not None else probe_size(segment, session) + if not size: + continue + total_bytes += size + total_seconds += segment.duration + + log.debug( + f"{track.id}: real-bitrate probe desc={track.descriptor.name} " + f"n_seg={len(segments)} n_unique={len(items)} n_chosen={len(chosen)} " + f"sampled_bytes={total_bytes} sampled_seconds={round(total_seconds, 4)}" + ) + + if total_seconds <= 0 or total_bytes <= 0: + log.warning(f"{track.id}: real bitrate probe returned no usable data") + return None + + return round(total_bytes * 8 / total_seconds) + + +def apply_real_bitrates( + tracks: list["Track"], + session: Union[Session, RnetSession], + *, + log: logging.Logger, + group_key: Callable[["Track"], Hashable], + per_group: int = 5, + workers: int = 8, +) -> None: + """ + Probe real bitrates and overwrite ``track.bitrate`` for the tracks worth probing. + + Probing every rendition is slow when a service exposes dozens. Tracks are + grouped by ``group_key`` (a quality tier), and only the ``per_group`` highest + declared-bitrate tracks per group are probed, in parallel. Each group is then + extended downward: while the lowest probed bitrate in a group sits below the + next unprobed track's declared bitrate (so that track could outrank a probed + one), the next track is probed too — until the probed set is safely above the + rest. Unprobed tracks keep their manifest-declared bitrate. + """ + groups: defaultdict[Hashable, list["Track"]] = defaultdict(list) + for track in tracks: + groups[group_key(track)].append(track) + for group in groups.values(): + group.sort(key=lambda t: getattr(t, "bitrate", None) or 0, reverse=True) + + # Initial pass: top per_group of every group, all probed concurrently. + initial = [track for group in groups.values() for track in group[:per_group]] + probe_batch(initial, session, log=log, workers=workers) + + # Extend each group downward until unprobed tracks can't outrank probed ones. + for group in groups.values(): + probed = min(per_group, len(group)) + while probed < len(group): + lowest_probed = min((getattr(t, "bitrate", None) or 0) for t in group[:probed]) + next_declared = getattr(group[probed], "bitrate", None) or 0 + if next_declared <= lowest_probed: + break + probe_batch([group[probed]], session, log=log, workers=workers) + probed += 1 + + +def probe_batch( + tracks: list["Track"], + session: Union[Session, RnetSession], + *, + log: logging.Logger, + workers: int, +) -> None: + """Probe each track concurrently and overwrite its bitrate with the measured value.""" + if not tracks: + return + + def probe_one(track: "Track") -> tuple["Track", Optional[int]]: + return track, measure_real_bitrate(track, track.session or session, log=log) + + with ThreadPoolExecutor(max_workers=min(workers, len(tracks))) as executor: + for track, measured in executor.map(probe_one, tracks): + if not measured: + continue + declared = getattr(track, "bitrate", None) + if declared and declared != measured: + log.debug(f"{track.id}: bitrate {declared // 1000} → {measured // 1000} kb/s (real)") + setattr(track, "bitrate", measured) + + +def dedupe(segments: list[Segment]) -> list[Segment]: + """ + Collapse segments that address the same bytes so each object is measured once. + + Manifests sometimes wrap a single file in several segment entries sharing one + URL — with no byte range (a ``SegmentTemplate`` whose media pattern has no + ``$Number$``) or with the same range. Each resolves to the whole file, so + counting them all would multiply the size by the segment count. Segments + sharing the same ``(url, byte_range)`` are merged into one entry whose duration + is the sum they cover. Distinct byte ranges of one file (different offsets) are + kept individual so their sizes still add up to the full track. + """ + merged: OrderedDict[tuple[str, Optional[str]], Segment] = OrderedDict() + for segment in segments: + key = (segment.url, segment.byte_range) + existing = merged.get(key) + if existing is None: + merged[key] = Segment(segment.url, segment.byte_range, segment.known_size, segment.duration) + else: + existing.duration += segment.duration + return list(merged.values()) + + +def pick_samples(segments: list[Segment], samples: int) -> list[Segment]: + """Pick up to ``samples`` segments spread evenly across the track.""" + count = len(segments) + if count <= samples: + return segments + step = count / samples + indices = sorted({int(i * step) for i in range(samples)}) + return [segments[i] for i in indices] + + +def probe_size(segment: Segment, session: Union[Session, RnetSession]) -> Optional[int]: + """Return a segment's byte size via HEAD, falling back to a ranged GET. Validates status.""" + try: + res = session.head(segment.url, allow_redirects=True, timeout=PROBE_TIMEOUT) + if getattr(res, "status_code", 0) in (200, 206): + content_length = res.headers.get("Content-Length") + if content_length: + return int(content_length) + except Exception: + pass + + # Some hosts block or mishandle HEAD; ask for a single byte and read the total. + # Require a 206 so a server that ignores Range (returning the whole 200 body) + # is not mistaken for a valid size or downloaded wholesale. + try: + res = session.get(segment.url, headers={"Range": "bytes=0-0"}, timeout=PROBE_TIMEOUT) + if getattr(res, "status_code", 0) == 206: + content_range = res.headers.get("Content-Range") + if content_range and "/" in content_range: + total = content_range.rsplit("/", 1)[-1].strip() + if total.isdigit(): + return int(total) + except Exception: + pass + + return None + + +def range_size(byte_range: Optional[str]) -> Optional[int]: + """Size in bytes of a ``start-end`` media range, inclusive.""" + if not byte_range or "-" not in byte_range: + return None + start_s, _, end_s = byte_range.partition("-") + try: + start = int(start_s) if start_s else 0 + if not end_s: + return None + return int(end_s) - start + 1 + except ValueError: + return None + + +def uniform_segments( + raw_segments: list[tuple[str, Optional[str]]], + total_duration: Optional[float], +) -> list[Segment]: + """ + Build Segments giving each an equal share of the total duration. + + Used for DASH: ``DASH._get_period_segments`` returns timeline *start times* + rather than per-segment durations, so they cannot be trusted. Segment lengths + are near-uniform in practice, so the track duration (from + ``mediaPresentationDuration``) split evenly is both correct and timeline-safe. + """ + count = len(raw_segments) + if not count or not total_duration or total_duration <= 0: + return [] + per_segment = total_duration / count + return [Segment(url, byte_range, range_size(byte_range), per_segment) for url, byte_range in raw_segments] + + +def extract_dash(track: "Track", session: Union[Session, RnetSession]) -> list[Segment]: + from envied.core.manifests import DASH + + data = track.data["dash"] + manifest = data["manifest"] + rep_id = data.get("representation_id") or data["representation"].get("id") + filtered_period_ids = data.get("filtered_period_ids", []) + track_url = track.url if isinstance(track.url, str) else track.url[0] + + content_periods = [p for p in manifest.findall("Period") if DASH._is_content_period(p, filtered_period_ids)] + + raw_segments: list[tuple[str, Optional[str]]] = [] + for period in content_periods: + matched_rep = matched_as = None + for as_ in period.findall("AdaptationSet"): + if DASH.is_trick_mode(as_): + continue + for rep in as_.findall("Representation"): + if rep.get("id") == rep_id: + matched_rep, matched_as = rep, as_ + break + if matched_rep is not None: + break + if matched_rep is None or matched_as is None: + continue + + _, period_segments, _, _, _ = DASH._get_period_segments( + period=period, + adaptation_set=matched_as, + representation=matched_rep, + manifest=manifest, + track=track, + track_url=track_url, + session=session, + ) + raw_segments.extend(period_segments) + + total_duration: Optional[float] = None + mpd_duration = manifest.get("mediaPresentationDuration") + if mpd_duration: + total_duration = DASH.pt_to_sec(mpd_duration) + + return uniform_segments(raw_segments, total_duration) + + +def extract_hls(track: "Track", session: Union[Session, RnetSession]) -> list[Segment]: + import m3u8 + + playlist_url = track.url if isinstance(track.url, str) else track.url[0] + res = session.get(playlist_url, timeout=PROBE_TIMEOUT) + playlist = m3u8.loads(res.text, uri=playlist_url) + + out: list[Segment] = [] + for segment in playlist.segments: + url = urljoin(segment.base_uri or "", segment.uri) + byte_range = segment.byterange # "[@]" + known_size: Optional[int] = None + if byte_range: + length = byte_range.split("@")[0].strip() + if length.isdigit(): + known_size = int(length) + # EXTINF durations are reliable, so they are used directly (unlike DASH). + out.append(Segment(url, byte_range, known_size, float(segment.duration or 0))) + return out + + +def extract_ism(track: "Track", session: Union[Session, RnetSession]) -> list[Segment]: + data = track.data["ism"] + segments: list[str] = data.get("segments") or [] + manifest = data["manifest"] + + timescale = int(manifest.get("TimeScale") or ISM_DEFAULT_TIMESCALE) + duration_ticks = int(manifest.get("Duration") or 0) + total_duration = (duration_ticks / timescale) if timescale else 0.0 + + return uniform_segments([(url, None) for url in segments], total_duration) + + +def extract_url(track: "Track", session: Union[Session, RnetSession], *, log: logging.Logger) -> list[Segment]: + """Single-file track: one whole-file URL with the duration from leftover manifest data.""" + url = track.url if isinstance(track.url, str) else (track.url[0] if track.url else None) + if not url: + return [] + + duration: Optional[float] = None + dash_data = track.data.get("dash") + if dash_data and dash_data.get("manifest") is not None: + from envied.core.manifests import DASH + + mpd_duration = dash_data["manifest"].get("mediaPresentationDuration") + if mpd_duration: + duration = DASH.pt_to_sec(mpd_duration) + else: + ism_data = track.data.get("ism") + if ism_data and ism_data.get("manifest") is not None: + manifest = ism_data["manifest"] + timescale = int(manifest.get("TimeScale") or ISM_DEFAULT_TIMESCALE) + duration_ticks = int(manifest.get("Duration") or 0) + if timescale and duration_ticks: + duration = duration_ticks / timescale + + if not duration or duration <= 0: + # Services like AMZN clear the manifest data after collapsing to a single + # file; fall back to reading the duration straight from the remote file. + duration = ffprobe_duration(url, session, log=log) + + if not duration or duration <= 0: + return [] + return [Segment(url, None, None, duration)] + + +def ffprobe_duration(url: str, session: Union[Session, RnetSession], *, log: logging.Logger) -> Optional[float]: + """ + Read a single-file track's duration (seconds) without a manifest. + + The bundled ffprobe segfaults on network input, so the file's ``moov`` box is + fetched over HTTP with the session (keeping the service's proxy/headers) and + piped to ffprobe as local bytes. The head of the file is tried first (VOD is + usually faststart), then the tail as a fallback for moov-at-end files. + """ + head = ranged_get(url, session, f"bytes=0-{MOOV_PROBE_BYTES - 1}") + duration = probe_bytes_duration(head, log) + if duration: + return duration + + size = probe_size(Segment(url, None, None, 0.0), session) + if size and size > MOOV_PROBE_BYTES: + tail = ranged_get(url, session, f"bytes={size - MOOV_PROBE_BYTES}-{size - 1}") + duration = probe_bytes_duration(tail, log) + return duration + + +def ranged_get(url: str, session: Union[Session, RnetSession], byte_range: str) -> Optional[bytes]: + """Fetch a byte range, only accepting a real 206 partial response (never a full 200 body).""" + try: + res = session.get(url, headers={"Range": byte_range}, timeout=PROBE_TIMEOUT) + if getattr(res, "status_code", 0) != 206: + return None + content = getattr(res, "content", None) + return content if content else None + except Exception: + return None + + +def probe_bytes_duration(data: Optional[bytes], log: logging.Logger) -> Optional[float]: + """Pipe media bytes to ffprobe and return the format/stream duration in seconds.""" + if not data: + return None + ffprobe_bin = str(FFProbe) if FFProbe else "ffprobe" + try: + result = subprocess.run( + [ffprobe_bin, "-v", "error", "-show_entries", "format=duration:stream=duration", "-of", "json", "pipe:"], + input=data, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=60, + ) + info = json.loads(result.stdout or b"{}") + candidates = [info.get("format", {}).get("duration")] + candidates += [s.get("duration") for s in info.get("streams", [])] + for value in candidates: + if value: + return float(value) + log.debug(f"ffprobe found no duration (rc={result.returncode}): {result.stderr.decode(errors='replace')[:160]}") + return None + except (subprocess.SubprocessError, ValueError, json.JSONDecodeError) as e: + log.debug(f"ffprobe duration error: {e}") + return None diff --git a/packages/envied/src/envied/core/utils/click_types.py b/packages/envied/src/envied/core/utils/click_types.py index 4ac0dda..a01031b 100644 --- a/packages/envied/src/envied/core/utils/click_types.py +++ b/packages/envied/src/envied/core/utils/click_types.py @@ -360,9 +360,32 @@ class MultipleChoice(click.Choice): return super(self).shell_complete(ctx, param, incomplete) +class SlowDelayRange(click.ParamType): + """Parses a delay range string like '20-40' into a tuple of (min, max) seconds.""" + + name = "delay_range" + + def convert(self, value: Any, param: Optional[click.Parameter], ctx: Optional[click.Context]) -> tuple[int, int]: + if isinstance(value, tuple): + return value + + match = re.match(r"^(\d+)-(\d+)$", str(value)) + if not match: + self.fail(f"'{value}' is not a valid range. Use format: MIN-MAX (e.g., 20-40)", param, ctx) + + low, high = int(match.group(1)), int(match.group(2)) + if low < 20: + self.fail(f"Minimum delay must be at least 20 seconds, got {low}", param, ctx) + if low > high: + self.fail(f"Min ({low}) cannot be greater than max ({high})", param, ctx) + + return (low, high) + + SEASON_RANGE = SeasonRange() LANGUAGE_RANGE = LanguageRange() QUALITY_LIST = QualityList() AUDIO_CODEC_LIST = AudioCodecList(Audio.Codec) +SLOW_DELAY_RANGE = SlowDelayRange() # VIDEO_CODEC_CHOICE will be created dynamically when imported diff --git a/packages/envied/src/envied/core/utils/dovi.py b/packages/envied/src/envied/core/utils/dovi.py new file mode 100644 index 0000000..157b578 --- /dev/null +++ b/packages/envied/src/envied/core/utils/dovi.py @@ -0,0 +1,130 @@ +"""Thin wrappers around dovi_tool subcommands used by DVFixup and Hybrid. + +Centralises argv construction, status spinners, and error handling so callers do not +re-implement subprocess plumbing per call site. Each wrapper: + +- Resolves `binaries.DoviTool` and raises EnvironmentError if missing. +- Delegates to `core.utils.subprocess.run_step` for execution, output validation, and + stderr-tail RuntimeError on failure. +- Returns captured stderr so callers can inspect specific failure modes (e.g. the + MAX_PQ_LUMINANCE retry path in extract_rpu). +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from typing import Optional + +from envied.core import binaries +from envied.core.utils.subprocess import run_step + + +def _require_dovi_tool() -> str: + if not binaries.DoviTool: + raise EnvironmentError("dovi_tool executable was not found but is required.") + return str(binaries.DoviTool) + + +def extract_rpu( + source: Path, + output: Path, + *, + mode: Optional[int] = 3, + status: Optional[str] = "Extracting DV RPU...", + label: str = "dovi_tool extract-rpu", +) -> bytes: + """Extract DV RPU NALs from a raw HEVC stream. `mode=None` skips the -m flag (untouched).""" + tool = _require_dovi_tool() + args: list = [tool] + if mode is not None: + args += ["-m", str(mode)] + args += ["extract-rpu", source, "-o", output] + return run_step(args, status=status, output=output, label=label) + + +def inject_rpu( + source: Path, + rpu: Path, + output: Path, + *, + status: Optional[str] = "Re-injecting DV RPU...", + label: str = "dovi_tool inject-rpu", +) -> bytes: + """Inject a DV RPU back into a raw HEVC stream, producing DV-signaled output.""" + tool = _require_dovi_tool() + return run_step( + [tool, "inject-rpu", "-i", source, "--rpu-in", rpu, "-o", output], + status=status, + output=output, + label=label, + ) + + +def editor( + source: Path, + json_spec: Path, + output: Path, + *, + status: Optional[str] = "Editing DV RPU...", + label: str = "dovi_tool editor", +) -> bytes: + """Apply a JSON edit spec to an RPU file.""" + tool = _require_dovi_tool() + return run_step( + [tool, "editor", "-i", source, "-j", json_spec, "-o", output], + status=status, + output=output, + label=label, + ) + + +def info_summary(rpu: Path) -> str: + """Return the textual summary (`dovi_tool info -i ... -s`) for an RPU file.""" + tool = _require_dovi_tool() + p = subprocess.run([tool, "info", "-i", str(rpu), "-s"], capture_output=True, text=True) + if p.returncode != 0: + raise RuntimeError(f"dovi_tool info failed: {(p.stderr or '')[-400:]}") + return p.stdout + + +def generate_from_hdr10plus( + extra_json: Path, + hdr10plus_json: Path, + output: Path, + *, + status: Optional[str] = "Generating DV RPU from HDR10+ metadata...", + label: str = "dovi_tool generate", +) -> bytes: + """Build a DV RPU from extracted HDR10+ metadata + an extra JSON descriptor.""" + tool = _require_dovi_tool() + return run_step( + [tool, "generate", "-j", extra_json, "--hdr10plus-json", hdr10plus_json, "-o", output], + status=status, + output=output, + label=label, + ) + + +def extract_rpu_with_fallback(source: Path, output: Path, *, label: str = "dovi_tool extract-rpu") -> bytes: + """Try `-m 3` first; on MAX_PQ_LUMINANCE error, retry untouched (no -m). Returns stderr. + + Used when the caller wants automatic normalization but cannot abort if the source + rejects mode-3 conversion. + """ + try: + return extract_rpu(source, output, mode=3, label=label) + except RuntimeError as e: + if "MAX_PQ_LUMINANCE" not in str(e): + raise + return extract_rpu(source, output, mode=None, status="Extracting DV RPU (untouched)...", label=label) + + +__all__ = ( + "extract_rpu", + "extract_rpu_with_fallback", + "inject_rpu", + "editor", + "info_summary", + "generate_from_hdr10plus", +) diff --git a/packages/envied/src/envied/core/utils/ip_info.py b/packages/envied/src/envied/core/utils/ip_info.py new file mode 100644 index 0000000..2ad5f58 --- /dev/null +++ b/packages/envied/src/envied/core/utils/ip_info.py @@ -0,0 +1,252 @@ +from __future__ import annotations + +import logging +import time +from typing import Any, Callable, Optional + +import requests + +from envied.core.cacher import Cacher + +CACHE_KEY = "ip_info_v3" +CACHE_TTL = 86400 # 24 hours +PROVIDER_STATE_KEY = "ip_provider_state" +RATE_LIMIT_COOLDOWN = 300 # 5 minutes +REQUEST_TIMEOUT = 10 + +# Only these keys are persisted to the global cache. +GEO_CACHE_KEYS = ("country", "country_code") + +Fetcher = Callable[[requests.Session], Optional[dict]] + +log = logging.getLogger("ip_info") + + +class RateLimited(Exception): + """Raised by a provider fetcher when the upstream returns 429.""" + + +def normalize( + *, + country_code: str, + ip: str = "", + region: str = "", + city: str = "", + org: str = "", + asn: str = "", + as_name: str = "", + continent_code: str = "", +) -> Optional[dict]: + """Build the canonical IP-info dict, or None if no country code is present.""" + code = country_code.strip() + if not code: + return None + return { + "ip": ip, + "country": code.lower(), + "country_code": code.upper(), + "region": region, + "city": city, + "org": org, + "asn": asn, + "as_name": as_name, + "continent_code": continent_code.upper(), + } + + +def parse_ipinfo_lite(data: dict) -> Optional[dict]: + asn = (data.get("asn") or "").strip() + as_name = (data.get("as_name") or "").strip() + return normalize( + country_code=data.get("country_code") or "", + ip=data.get("ip") or "", + org=f"{asn} {as_name}".strip(), + asn=asn, + as_name=as_name, + continent_code=data.get("continent_code") or "", + ) + + +def parse_ipinfo(data: dict) -> Optional[dict]: + return normalize( + country_code=data.get("country") or "", + ip=data.get("ip") or "", + region=data.get("region") or "", + city=data.get("city") or "", + org=data.get("org") or "", + ) + + +def parse_ip_api_in(data: dict) -> Optional[dict]: + asn = (data.get("asn") or "").strip() + org_name = (data.get("organization") or "").strip() + return normalize( + country_code=data.get("country_code") or "", + ip=data.get("ip") or "", + region=data.get("region") or "", + city=data.get("city") or "", + org=f"{asn} {org_name}".strip(), + asn=asn, + as_name=org_name, + continent_code=data.get("continent_code") or "", + ) + + +def lookup_session(source: Optional[requests.Session]) -> requests.Session: + """ + Build a plain, retry-free requests session for IP geolocation. + + Geolocation needs no TLS fingerprinting, so we skip the impersonated rnet + session and the base session's urllib3 retry loop — both retry 429 internally, + which hides the response and defeats fast provider handover. With a bare session + a 429 comes straight back so we can move to the next provider immediately. Only + the proxy is carried over so proxied lookups still report the proxy's exit IP. + """ + sess = requests.Session() + proxies = getattr(source, "proxies", None) + if proxies: + proxy = proxies.get("all") or proxies.get("https") or proxies.get("http") + if proxy: + sess.proxies.update({"http": proxy, "https": proxy}) + return sess + + +def json_or_raise(response: requests.Response) -> Optional[dict]: + """Raise RateLimited on 429, return parsed JSON on 200, else None.""" + if response.status_code == 429: + raise RateLimited() + if response.status_code != 200: + return None + try: + return response.json() + except ValueError: + return None + + +def fetch_ipinfo_lite(token: str) -> Fetcher: + headers = {"Authorization": f"Bearer {token}"} + + def fetch(session: requests.Session) -> Optional[dict]: + payload = json_or_raise(session.get("https://api.ipinfo.io/lite/me", headers=headers, timeout=REQUEST_TIMEOUT)) + return parse_ipinfo_lite(payload) if payload else None + + return fetch + + +def fetch_ipinfo(session: requests.Session) -> Optional[dict]: + payload = json_or_raise(session.get("https://ipinfo.io/json", timeout=REQUEST_TIMEOUT)) + return parse_ipinfo(payload) if payload else None + + +def fetch_ip_api_in(session: requests.Session) -> Optional[dict]: + """ip-api.in has no /me endpoint — resolve IP via ipify first, then look it up.""" + ip_resp = session.get("https://api.ipify.org", timeout=REQUEST_TIMEOUT) + if ip_resp.status_code == 429: + raise RateLimited() + ip = (ip_resp.text or "").strip() if ip_resp.status_code == 200 else "" + if not ip: + return None + payload = json_or_raise(session.get(f"https://ip-api.in/api/v1/ip/{ip}", timeout=REQUEST_TIMEOUT)) + if not payload or not payload.get("success"): + return None + return parse_ip_api_in(payload.get("data") or {}) + + +def build_providers() -> list[tuple[str, Fetcher]]: + """Return ordered (name, fetcher) pairs. Token is read at call time.""" + from envied.core.config import config + + providers: list[tuple[str, Fetcher]] = [] + token = (getattr(config, "ipinfo_api_key", "") or "").strip() + if token: + providers.append(("ipinfo_lite", fetch_ipinfo_lite(token))) + providers.append(("ipinfo", fetch_ipinfo)) + providers.append(("ip_api_in", fetch_ip_api_in)) + return providers + + +def purge_stale_cache() -> None: + """Delete superseded ip_info cache files (older CACHE_KEY versions).""" + from envied.core.config import config + + global_dir = config.directories.cache / "global" + for stale in global_dir.glob("ip_info_v*.json"): + if stale.stem != CACHE_KEY: + stale.unlink(missing_ok=True) + + +def load_provider_state(cacher: Cacher) -> dict[str, Any]: + return cacher.data if cacher and not cacher.expired and isinstance(cacher.data, dict) else {} + + +def get_ip_info( + session: Optional[requests.Session] = None, + *, + cached: bool = False, +) -> Optional[dict]: + """ + Look up IP/geolocation info via ipinfo.io (Lite when `ipinfo_api_key` configured) + with fallback to ip-api.in. + + Live lookups return a dict with `ip`, `country` (lowercase ISO2), `country_code` + (uppercase ISO2), `region`, `city`, `org`, `asn`, `as_name`, `continent_code` and + `_provider`. Cached lookups return only `country`/`country_code` (see GEO_CACHE_KEYS). + Returns None if every provider fails. + + Args: + session: Optional requests session. If a proxied session is passed, the + returned info reflects the proxy's exit IP. Auth headers for ipinfo + are sent per-request; never mutated onto session.headers. + cached: When True, read/write a 24h Cacher-backed entry. Use only for + local IP lookups — never with a proxied session. + """ + cache = None + if cached: + purge_stale_cache() + cache = Cacher("global").get(CACHE_KEY) + if cache and not cache.expired and cache.data: + return cache.data + + state_cache = Cacher("global").get(PROVIDER_STATE_KEY) + state = load_provider_state(state_cache) + now = time.time() + + def on_cooldown(item: tuple[str, Fetcher]) -> int: + rate_limited_at = (state.get(item[0]) or {}).get("rate_limited_at", 0) + return 1 if (now - rate_limited_at) < RATE_LIMIT_COOLDOWN else 0 + + providers = sorted(build_providers(), key=on_cooldown) + sess = lookup_session(session) + + for name, fetcher in providers: + log.debug(f"Trying IP provider: {name}") + try: + normalized = fetcher(sess) + except RateLimited: + log.warning(f"Provider {name} returned 429 (rate limited), trying next provider") + entry = state.setdefault(name, {}) + entry["rate_limited_at"] = now + entry["rate_limit_count"] = entry.get("rate_limit_count", 0) + 1 + state_cache.set(state, expiration=RATE_LIMIT_COOLDOWN) + continue + except Exception as e: + log.debug(f"Provider {name} failed with exception: {e}") + continue + + if not normalized: + log.debug(f"Provider {name} returned no usable data") + continue + + normalized["_provider"] = name + log.debug(f"Successfully got IP info from provider: {name}") + + if name in state and state[name].pop("rate_limited_at", None) is not None: + state_cache.set(state, expiration=RATE_LIMIT_COOLDOWN) + + if cache is not None: + cache.set({k: normalized.get(k, "") for k in GEO_CACHE_KEYS}, expiration=CACHE_TTL) + + return normalized + + log.warning("All IP geolocation providers failed") + return None diff --git a/packages/envied/src/envied/core/utils/subprocess.py b/packages/envied/src/envied/core/utils/subprocess.py index 68e4354..d712402 100644 --- a/packages/envied/src/envied/core/utils/subprocess.py +++ b/packages/envied/src/envied/core/utils/subprocess.py @@ -1,9 +1,10 @@ import json import subprocess from pathlib import Path -from typing import Union +from typing import Optional, Sequence, Union from envied.core import binaries +from envied.core.console import console def ffprobe(uri: Union[bytes, Path]) -> dict: @@ -23,3 +24,34 @@ def ffprobe(uri: Union[bytes, Path]) -> dict: except subprocess.CalledProcessError: return {} return json.loads(ff.stdout.decode("utf8")) + + +def run_step( + args: Sequence[Union[str, Path]], + *, + status: Optional[str] = None, + output: Optional[Path] = None, + label: str = "subprocess step", +) -> bytes: + """Run a CLI step that writes to `output` (when provided). Returns stderr bytes. + + Raises RuntimeError with the stderr tail when the process exits non-zero, or when + `output` is given and does not exist / is empty after the run. + """ + if output is not None: + output.unlink(missing_ok=True) + + str_args = [str(a) for a in args] + if status: + with console.status(status, spinner="dots"): + p = subprocess.run(str_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + else: + p = subprocess.run(str_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + stderr = p.stderr or b"" + bad_output = output is not None and (not output.exists() or output.stat().st_size == 0) + if p.returncode or bad_output: + if output is not None: + output.unlink(missing_ok=True) + raise RuntimeError(f"{label} failed: {stderr.decode(errors='replace')[-400:]}") + return stderr diff --git a/packages/envied/src/envied/core/utils/tags.py b/packages/envied/src/envied/core/utils/tags.py index 4b79385..6836d06 100644 --- a/packages/envied/src/envied/core/utils/tags.py +++ b/packages/envied/src/envied/core/utils/tags.py @@ -33,13 +33,16 @@ def apply_tags(path: Path, tags: dict[str, str]) -> None: f.write("\n".join(xml_lines)) tmp_path = Path(f.name) try: - subprocess.run( + result = subprocess.run( [str(binaries.Mkvpropedit), str(path), "--tags", f"global:{tmp_path}"], check=False, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + capture_output=True, + text=True, ) - log.debug("Tags applied via mkvpropedit") + if result.returncode != 0: + log.warning("mkvpropedit failed (exit %d): %s", result.returncode, result.stderr.strip()) + else: + log.debug("Tags applied via mkvpropedit") finally: tmp_path.unlink(missing_ok=True) @@ -92,43 +95,46 @@ def tag_file( standard_tags: dict[str, str] = {} if config.tag_imdb_tmdb: - providers = get_available_providers() - if not providers: - log.debug("No metadata providers available; skipping tag lookup") - apply_tags(path, custom_tags) - return + try: + providers = get_available_providers() + if not providers: + log.debug("No metadata providers available; skipping tag lookup") + apply_tags(path, custom_tags) + return - result: Optional[MetadataResult] = None + result: Optional[MetadataResult] = None - # Direct ID lookup path - if imdb_id: - imdbapi = get_provider("imdbapi") - if imdbapi: - result = imdbapi.get_by_id(imdb_id, kind) - if result: - result.external_ids.imdb_id = imdb_id - enrich_ids(result) - elif tmdb_id is not None: - tmdb = get_provider("tmdb") - if tmdb: - result = tmdb.get_by_id(tmdb_id, kind) - if result: - ext = tmdb.get_external_ids(tmdb_id, kind) - result.external_ids = ext - else: - # Search across providers in priority order - result = search_metadata(name, year, kind) + # Direct ID lookup path + if imdb_id: + imdbapi = get_provider("imdbapi") + if imdbapi: + result = imdbapi.get_by_id(imdb_id, kind) + if result: + result.external_ids.imdb_id = imdb_id + enrich_ids(result) + elif tmdb_id is not None: + tmdb = get_provider("tmdb") + if tmdb: + result = tmdb.get_by_id(tmdb_id, kind) + if result: + ext = tmdb.get_external_ids(tmdb_id, kind) + result.external_ids = ext + else: + # Search across providers in priority order + result = search_metadata(name, year, kind) - # If we got a TMDB ID from search but no full external IDs, fetch them - if result and result.external_ids.tmdb_id and not result.external_ids.imdb_id: - ext = fetch_external_ids(result.external_ids.tmdb_id, kind) - if ext.imdb_id: - result.external_ids.imdb_id = ext.imdb_id - if ext.tvdb_id: - result.external_ids.tvdb_id = ext.tvdb_id + # If we got a TMDB ID from search but no full external IDs, fetch them + if result and result.external_ids.tmdb_id and not result.external_ids.imdb_id: + ext = fetch_external_ids(result.external_ids.tmdb_id, kind) + if ext.imdb_id: + result.external_ids.imdb_id = ext.imdb_id + if ext.tvdb_id: + result.external_ids.tvdb_id = ext.tvdb_id - if result and result.external_ids: - standard_tags = _build_tags_from_ids(result.external_ids, kind) + if result and result.external_ids: + standard_tags = _build_tags_from_ids(result.external_ids, kind) + except Exception as e: + log.warning("Metadata lookup failed, applying custom tags only: %s", e) apply_tags(path, {**custom_tags, **standard_tags}) diff --git a/packages/envied/src/envied/core/utils/template_formatter.py b/packages/envied/src/envied/core/utils/template_formatter.py index 1775dac..1c36684 100644 --- a/packages/envied/src/envied/core/utils/template_formatter.py +++ b/packages/envied/src/envied/core/utils/template_formatter.py @@ -73,7 +73,9 @@ class TemplateFormatter: has_left = s[0] in ".- " has_right = s[-1] in ".- " if has_left and has_right: - return s[0] # keep left separator + if s[-1] == "-": + return s[-1] + return s[0] return "" result = re.sub( diff --git a/packages/envied/src/envied/core/vaults.py b/packages/envied/src/envied/core/vaults.py index 5b655cf..ee672ce 100644 --- a/packages/envied/src/envied/core/vaults.py +++ b/packages/envied/src/envied/core/vaults.py @@ -1,3 +1,4 @@ +import logging from typing import Any, Iterator, Optional, Union from uuid import UUID @@ -5,6 +6,8 @@ from envied.core.config import config from envied.core.utilities import import_module_by_path from envied.core.vault import Vault +log = logging.getLogger(__name__) + _VAULTS = sorted( (path for path in config.directories.vaults.glob("*.py") if path.stem.lower() != "__init__"), key=lambda x: x.stem ) @@ -48,7 +51,13 @@ class Vaults: def get_key(self, kid: Union[UUID, str]) -> tuple[Optional[str], Optional[Vault]]: """Get Key from the first Vault it can by KID (Key ID) and Service.""" for vault in self.vaults: - key = vault.get_key(kid, self.service) + try: + key = vault.get_key(kid, self.service) + except (PermissionError, NotImplementedError): + continue + except Exception as e: + log.warning(f"Failed to get key from Vault '{vault.name}': {e}") + continue if key and key.count("0") != len(key): return key, vault return None, None @@ -62,6 +71,8 @@ class Vaults: success += vault.add_key(self.service, kid, key) except (PermissionError, NotImplementedError): pass + except Exception as e: + log.warning(f"Failed to add key to Vault '{vault.name}': {e}") return success def add_keys(self, kid_keys: dict[Union[UUID, str], str]) -> int: @@ -79,6 +90,8 @@ class Vaults: success += 1 except (PermissionError, NotImplementedError): pass + except Exception as e: + log.warning(f"Failed to add keys to Vault '{vault.name}': {e}") return success diff --git a/packages/envied/src/envied/envied-example.yaml b/packages/envied/src/envied/envied-example.yaml new file mode 100644 index 0000000..789a774 --- /dev/null +++ b/packages/envied/src/envied/envied-example.yaml @@ -0,0 +1,758 @@ +# API key for The Movie Database (TMDB) +tmdb_api_key: "" + +# Client ID for SIMKL API (optional, improves metadata matching) +# Get your free client ID at: https://simkl.com/settings/developer/ +simkl_client_id: "" + +# Optional ipinfo.io API token. When set, unshackle uses the free Lite endpoint +# which has higher rate limits and richer IP info (ASN, org, continent). +# Get a free token at: https://ipinfo.io/signup +ipinfo_api_key: "" + +# Group or Username to postfix to the end of all download filenames following a dash +tag: user_tag + +# Enable/disable tagging with group name (default: true) +tag_group_name: true + +# Enable/disable tagging with IMDB/TMDB/TVDB details (default: true) +tag_imdb_tmdb: true + +# Set terminal background color (custom option not in CONFIG.md) +set_terminal_bg: false + +# Custom output templates for filenames +# Configure output_template in your envied.yaml to control filename format. +# If not configured, default scene-style templates are used and a warning is shown. +# Available variables: {title}, {year}, {season}, {episode}, {season_episode}, {episode_name}, +# {quality}, {resolution}, {source}, {audio}, {audio_channels}, {audio_full}, +# {video}, {hdr}, {hfr}, {atmos}, {dual}, {multi}, {tag}, {edition}, {repack}, +# {lang_tag} +# Conditional variables (included only if present): Add ? suffix like {year?}, {episode_name?}, {hdr?} +# Customize the templates below: +# +# Example outputs: +# Scene movies: 'The.Matrix.1999.1080p.SERVICE.WEB-DL.DDP5.1.H.264-EXAMPLE' +# Scene movies (HDR): 'Dune.2021.2160p.SERVICE.WEB-DL.DDP5.1.HDR10.H.265-EXAMPLE' +# Scene movies (REPACK): 'Dune.2021.REPACK.2160p.SERVICE.WEB-DL.DDP5.1.H.265-EXAMPLE' +# Scene series: 'Breaking.Bad.2008.S01E01.Pilot.1080p.SERVICE.WEB-DL.DDP5.1.H.264-EXAMPLE' +# Plex movies: 'The Matrix (1999) 1080p' +# Plex series: 'Breaking Bad S01E01 Pilot' +output_template: + # Scene-style naming (dot-separated) + movies: '{title}.{year}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + series: '{title}.{year?}.{season_episode}.{episode_name?}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + songs: '{track_number}.{title}.{repack?}.{edition?}.{source?}.WEB-DL.{audio_full}.{atmos?}-{tag}' + # + # Plex-friendly naming (space-separated, clean format) + # movies: '{title} ({year}) {quality}' + # series: '{title} {season_episode} {episode_name?}' + # songs: '{track_number}. {title}' + # + # Minimal naming (basic info only) + # movies: '{title}.{year}.{quality}' + # series: '{title}.{season_episode}.{episode_name?}' + # + # Custom scene-style with specific elements + # movies: '{title}.{year}.{quality}.{hdr?}.{source}.WEB-DL.{audio_full}.{video}-{tag}' + # series: '{title}.{year?}.{season_episode}.{episode_name?}.{quality}.{hdr?}.{source}.WEB-DL.{audio_full}.{atmos?}.{video}-{tag}' + # + # Folder naming (optional). Controls the folder name for downloaded content. + # If not configured, series folders are derived from the series template (minus episode info), + # movie folders use "{title} ({year})", and song folders use "{artist} - {album} ({year})". + # Uses the same template variables as the file templates above. + # + # Scene-style folder: + # folder: '{title}.{year?}.{repack?}.{edition?}.{lang_tag?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + # + # Plex-friendly folder: + # folder: '{title} ({year?})' + # + # Per-title-type folder templates (optional). Override folder naming separately for + # movies, series, and songs. Useful when music libraries need artist/album-style folders + # while movies/series follow a different scheme. Any kind omitted falls back to the + # default for that title type. + # + # folder: + # movies: '{title} ({year})' + # series: '{title} ({year?})' + # songs: '{artist}/{album} ({year?})' + +# Language-based tagging for output filenames +# Automatically adds language identifiers (e.g., DANiSH, NORDiC, DKsubs) based on +# audio and subtitle track languages. Rules are evaluated in order; first match wins. +# Use {lang_tag?} in your output_template to place the tag in the filename. +# +# Conditions (all conditions in a rule must match): +# audio: - any audio track matches this language +# subs_contain: - any subtitle matches this language +# subs_contain_all: [lang, ...] - subtitles include ALL listed languages +# +# language_tags: +# rules: +# - audio: da +# tag: DANiSH +# - audio: sv +# tag: SWEDiSH +# - audio: nb +# tag: NORWEGiAN +# - audio: en +# subs_contain_all: [da, sv, nb] +# tag: NORDiC +# - audio: en +# subs_contain: da +# tag: DKsubs + +# Check for updates from GitHub repository on startup (default: true) +update_checks: true + +# How often to check for updates, in hours (default: 24) +update_check_interval: 24 + +# Title caching configuration +# Cache title metadata to reduce redundant API calls +title_cache_enabled: true # Enable/disable title caching globally (default: true) +title_cache_time: 1800 # Cache duration in seconds (default: 1800 = 30 minutes) +title_cache_max_retention: 86400 # Maximum cache retention for fallback when API fails (default: 86400 = 24 hours) + +# Filename Configuration +unicode_filenames: false # optionally replace non-ASCII characters with ASCII equivalents + +# Debug logging configuration +# Comprehensive JSON-based debug logging for troubleshooting and service development +debug: + false # Enable structured JSON debug logging (default: false) + # When enabled with --debug flag or set to true: + # - Creates JSON Lines (.jsonl) log files with complete debugging context + # - Logs: session info, CLI params, service config, CDM details, authentication, + # titles, tracks metadata, DRM operations, vault queries, errors with stack traces + # - File location: logs/unshackle_debug_{service}_{timestamp}.jsonl + # - Also creates text log: logs/unshackle_root_{timestamp}.log + +debug_keys: + false # Log decryption keys in debug logs (default: false) + # Set to true to include actual decryption keys in logs + # Useful for debugging key retrieval and decryption issues + # SECURITY NOTE: Passwords, tokens, cookies, and session tokens + # are ALWAYS redacted regardless of this setting + # Only affects: content_key, key fields (the actual CEKs) + # Never affects: kid, keys_count, key_id (metadata is always logged) + +# Muxing configuration +muxing: + set_title: false + # merge_audio: Merge all audio tracks into each output file + # true (default): All selected audio in one MKV per quality + # false: Separate MKV per (quality, audio_codec) combination + # Example: Title.1080p.AAC.mkv, Title.1080p.EC3.mkv + merge_audio: true + # default_language: Override which track is flagged as the default in the muxed MKV. + # audio: BCP-47 tag of the preferred default audio track (e.g. pl, en, pt-BR). + # Wins over the title's original_language. Falls back to is_original_lang + # if no matching track is present. + # video: BCP-47 tag of the preferred default video track. Falls back to the + # original-language / first-track rule if no match is found. + # subtitle: BCP-47 tag of the preferred default subtitle track. Falls back to + # the existing rule (forced sub matching the audio language) if no + # matching subtitle is present. + # default_language: + # audio: pl + # video: pl + # subtitle: pl + +# Login credentials for each Service +credentials: + # Direct credentials (no profile support) + EXAMPLE: email@example.com:password + + # Per-profile credentials with default fallback + SERVICE_NAME: + default: default@email.com:password # Used when no -p/--profile is specified + profile1: user1@email.com:password1 + profile2: user2@email.com:password2 + + # Per-profile credentials without default (requires -p/--profile) + SERVICE_NAME2: + john: john@example.com:johnspassword + jane: jane@example.com:janespassword + + # You can also use list format for passwords with special characters + SERVICE_NAME3: + default: ["user@email.com", ":PasswordWith:Colons"] + +# Override default directories used across unshackle +directories: + cache: Cache + cookies: Cookies + dcsl: DCSL # Device Certificate Status List + downloads: Downloads + logs: Logs + temp: Temp + wvds: WVDs + prds: PRDs + exports: Exports # JSON export output from --export flag + # Additional directories that can be configured: + # commands: Commands + services: + - /path/to/services + - /other/path/to/services + # vaults: Vaults + # fonts: Fonts + +# Pre-define which Widevine or PlayReady device to use for each Service +cdm: + # Global default CDM device (fallback for all services/profiles) + default: WVD_1 + + # Direct service-specific CDM + DIFFERENT_EXAMPLE: PRD_1 + + # Per-profile CDM configuration + EXAMPLE: + john_sd: chromecdm_903_l3 # Profile 'john_sd' uses Chrome CDM L3 + jane_uhd: nexus_5_l1 # Profile 'jane_uhd' uses Nexus 5 L1 + default: generic_android_l3 # Default CDM for this service + + # NEW: Quality-based CDM selection + # Use different CDMs based on video resolution + # Supports operators: >=, >, <=, <, or exact match + EXAMPLE_QUALITY: + "<=1080": generic_android_l3 # Use L3 for 1080p and below + ">1080": nexus_5_l1 # Use L1 for above 1080p (1440p, 2160p) + default: generic_android_l3 # Optional: fallback if no quality match + + # You can mix profiles and quality thresholds in the same service + NETFLIX: + # Profile-based selection (existing functionality) + john: netflix_l3_profile + jane: netflix_l1_profile + # Quality-based selection (new functionality) + "<=720": netflix_mobile_l3 + "1080": netflix_standard_l3 + ">=1440": netflix_premium_l1 + # Fallback + default: netflix_standard_l3 + +# Use pywidevine Serve-compliant Remote CDMs + +# Example: Custom CDM API Configuration +# This demonstrates the highly configurable custom_api type that can adapt to any CDM API format +# - name: "chrome" +# type: "custom_api" +# host: "http://remotecdm.test/" +# timeout: 30 +# device: +# name: "ChromeCDM" +# type: "CHROME" +# system_id: 34312 +# security_level: 3 +# auth: +# type: "header" +# header_name: "x-api-key" +# key: "YOUR_API_KEY_HERE" +# custom_headers: +# User-Agent: "Unshackle/2.0.0" +# endpoints: +# get_request: +# path: "/get-challenge" +# method: "POST" +# timeout: 30 +# decrypt_response: +# path: "/get-keys" +# method: "POST" +# timeout: 30 +# request_mapping: +# get_request: +# param_names: +# scheme: "device" +# init_data: "init_data" +# static_params: +# scheme: "Widevine" +# decrypt_response: +# param_names: +# scheme: "device" +# license_request: "license_request" +# license_response: "license_response" +# static_params: +# scheme: "Widevine" +# response_mapping: +# get_request: +# fields: +# challenge: "challenge" +# session_id: "session_id" +# message: "message" +# message_type: "message_type" +# response_types: +# - condition: "message_type == 'license-request'" +# type: "license_request" +# success_conditions: +# - "message == 'success'" +# decrypt_response: +# fields: +# keys: "keys" +# message: "message" +# key_fields: +# kid: "kid" +# key: "key" +# type: "type" +# success_conditions: +# - "message == 'success'" +# caching: +# enabled: true +# use_vaults: true +# check_cached_first: true + +remote_cdm: + - name: "chrome" + device_name: chrome + device_type: CHROME + system_id: 27175 + security_level: 3 + host: https://domain.com/api + secret: secret_key + - name: "chrome-2" + device_name: chrome + device_type: CHROME + system_id: 26830 + security_level: 3 + host: https://domain-2.com/api + secret: secret_key + + - name: "decrypt_labs_chrome" + type: "decrypt_labs" # Required to identify as DecryptLabs CDM + device_name: "ChromeCDM" # Scheme identifier - must match exactly + device_type: CHROME + system_id: 4464 # Doesn't matter + security_level: 3 + host: "https://keyxtractor.decryptlabs.com" + secret: "your_decrypt_labs_api_key_here" # Replace with your API key + - name: "decrypt_labs_l1" + type: "decrypt_labs" + device_name: "L1" # Scheme identifier - must match exactly + device_type: ANDROID + system_id: 4464 + security_level: 1 + host: "https://keyxtractor.decryptlabs.com" + secret: "your_decrypt_labs_api_key_here" + + - name: "decrypt_labs_l2" + type: "decrypt_labs" + device_name: "L2" # Scheme identifier - must match exactly + device_type: ANDROID + system_id: 4464 + security_level: 2 + host: "https://keyxtractor.decryptlabs.com" + secret: "your_decrypt_labs_api_key_here" + + - name: "decrypt_labs_playready_sl2" + type: "decrypt_labs" + device_name: "SL2" # Scheme identifier - must match exactly + device_type: PLAYREADY + system_id: 0 + security_level: 2000 + host: "https://keyxtractor.decryptlabs.com" + secret: "your_decrypt_labs_api_key_here" + + - name: "decrypt_labs_playready_sl3" + type: "decrypt_labs" + device_name: "SL3" # Scheme identifier - must match exactly + device_type: PLAYREADY + system_id: 0 + security_level: 3000 + host: "https://keyxtractor.decryptlabs.com" + secret: "your_decrypt_labs_api_key_here" + + # PyPlayReady RemoteCdm - connects to an unshackle serve instance + - name: "playready_remote" + device_name: "my_prd_device" # Device name on the serve instance + device_type: PLAYREADY + system_id: 0 + security_level: 3000 # 2000 for SL2000, 3000 for SL3000 + host: "http://127.0.0.1:8786/playready" # Include /playready path + secret: "your-api-secret-key" + +# Key Vaults store your obtained Content Encryption Keys (CEKs) +# Use 'no_push: true' to prevent a vault from receiving pushed keys +# while still allowing it to provide keys when requested +key_vaults: + - type: SQLite + name: Local + path: key_store.db + # Additional vault types: + # - type: API + # name: "Remote Vault" + # uri: "https://key-vault.example.com" + # token: "secret_token" + # no_push: true # This vault will only provide keys, not receive them + # - type: MySQL + # name: "MySQL Vault" + # host: "127.0.0.1" + # port: 3306 + # database: vault + # username: user + # password: pass + # no_push: false # Default behavior - vault both provides and receives keys + +# Choose what software to use to download data +downloader: requests +# Options: requests +# Downloading now uses the unified in-process requests/rnet downloader; the legacy +# aria2c, curl_impersonate, and n_m3u8dl_re backends have been removed. + +# rnet TLS impersonation preset (not a downloader). Selects the browser +# fingerprint the HTTP session impersonates. +curl_impersonate: + browser: chrome120 + +# Pre-define default options and switches of the dl command +# Audio track selection preferences +audio: + # Codec priority order used as a tiebreaker when multiple audio tracks share the same + # bitrate and language. Listed codecs are ranked in the order given; codecs not in the + # list keep their bitrate-based ordering and are placed after all listed codecs. + # Atmos still trumps codec priority. Valid names: AAC, AC3, EC3, AC4, OPUS, OGG, DTS, ALAC, FLAC. + # codec_priority: [FLAC, ALAC, AC4, EC3, DTS, AC3, OPUS, AAC, OGG] + +dl: + sub_format: srt + downloads: 4 + workers: 16 + lang: + - en + - fr + EXAMPLE: + bitrate: CBR + +# Chapter Name to use when exporting a Chapter without a Name +chapter_fallback_name: "Chapter {j:02}" + +# Case-Insensitive dictionary of headers for all Services +headers: + Accept-Language: "en-US,en;q=0.8" + User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36" + +# Override default filenames used across unshackle +filenames: + debug_log: "unshackle_debug_{service}_{time}.jsonl" # JSON Lines debug log file + config: "config.yaml" + root_config: "envied.yaml" + chapters: "Chapters_{title}_{random}.txt" + subtitle: "Subtitle_{id}_{language}.srt" + +# conversion_method: +# - auto (default): Smart routing - subby for WebVTT/SAMI, pycaption for others +# - subby: Always use subby with advanced processing +# - pycaption: Use only pycaption library (no SubtitleEdit, no subby) +# - subtitleedit: Prefer SubtitleEdit when available, fall back to pycaption +# - pysubs2: Use pysubs2 library (supports SRT/SSA/ASS/WebVTT/TTML/SAMI/MicroDVD/MPL2/TMP) +subtitle: + conversion_method: auto + # sdh_method: Method to use for SDH (hearing impaired) stripping + # - auto (default): Try subby (SRT only), then SubtitleEdit (if available), then subtitle-filter + # - subby: Use subby library (SRT only) + # - subtitleedit: Use SubtitleEdit tool (Windows only, falls back to subtitle-filter) + # - filter-subs: Use subtitle-filter library directly + sdh_method: auto + # strip_sdh: Automatically create stripped (non-SDH) versions of SDH subtitles + # Set to false to disable automatic SDH stripping entirely (default: true) + strip_sdh: true + # convert_before_strip: Auto-convert VTT/other formats to SRT before using subtitle-filter + # This ensures compatibility when subtitle-filter is used as fallback (default: true) + convert_before_strip: true + # preserve_formatting: Preserve original subtitle formatting (tags, positioning, styling) + # When true, skips pycaption processing for WebVTT files to keep tags like , , positioning intact + # Combined with no sub_format setting, ensures subtitles remain in their original format (default: true) + preserve_formatting: true + # output_mode: Output mode for subtitles + # - mux: Embed subtitles in MKV container only (default) + # - sidecar: Save subtitles as separate files only + # - both: Embed in MKV AND save as sidecar files + output_mode: mux + # sidecar_format: Format for sidecar subtitle files + # Options: srt, vtt, ass, original (keep current format) + sidecar_format: srt + +# Configuration for pywidevine and pyplayready's serve functionality +# Also used for remote services (unshackle serve) +serve: + api_secret: "your-secret-key-here" + + # Compression level for API payloads (manifests, cache, cookies) + # 0=off, 1=fast, 6=balanced, 9=max compression (default: 1) + compression_level: 1 + + # Session inactivity timeout in seconds (default: 300 = 5 minutes) + # Sessions are automatically deleted after this many seconds of inactivity + # Each API request resets the timer + session_ttl: 300 + + # Maximum concurrent sessions before oldest is evicted (default: 100) + max_sessions: 100 + + # Global service allowlist (optional) + # Only these services will be exposed on the API. If omitted, all services are available. + # services: + # - SERVICE_TAG_1 + # - SERVICE_TAG_2 + + users: + secret_key_for_user: + devices: # Widevine devices (WVDs) this user can access + - generic_nexus_4464_l3 + playready_devices: # PlayReady devices (PRDs) this user can access + - playready_device_sl3000 + username: user + # Per-user service allowlist (optional) + # Restricts this user to only the listed services. If omitted, user can access + # all globally-allowed services. Effective access is the intersection of global + # and per-user allowlists. + # services: + # - SERVICE_TAG_1 + # devices: # Widevine device paths (auto-populated from directories.wvds) + # - '/path/to/device.wvd' + # playready_devices: # PlayReady device paths (auto-populated from directories.prds) + # - '/path/to/device.prd' + + # Optional: any /api/download flag can be set here as a server-side default. + # Per-request body values still win. Useful for raising concurrency without + # changing every client call. Full list of accepted keys: see docs/API.md. + # downloads: 4 # parallel tracks per download job + # workers: 16 # threads per track segment fetch + # best_available: true + # no_proxy_download: false + +# Remote Services Configuration +# Connect to a remote unshackle server (unshackle serve) to use its services +# without needing the service code locally. Use with: unshackle dl --remote +# If multiple servers are configured, specify which with: --server +remote_services: + # Server name (used with --server flag if multiple configured) + my-server: + url: "http://192.168.1.100:8786" + api_key: "your-secret-key-here" + + # Server-CDM mode: server handles all DRM licensing using its own CDM devices + # When false (default), client uses its own CDM and proxies license requests through the server + server_cdm: false + + # Per-service overrides for remote services + # Override downloader, decryption tool, or CDM settings per service on the client + services: + # Example: Override the decryption tool for specific services + # EXAMPLE_SERVICE: + # decryption: mp4decrypt # Override decryption tool (shaka, mp4decrypt) + + # Example: Multiple servers + # us-server: + # url: "https://us.example.com:8786" + # api_key: "us-api-key" + # server_cdm: true + # services: + # EXAMPLE: + # decryption: mp4decrypt + # eu-server: + # url: "https://eu.example.com:8786" + # api_key: "eu-api-key" + # server_cdm: false + +# Configuration data for each Service +services: + # Service-specific configuration goes here + # Profile-specific configurations can be nested under service names + + # You can override ANY global configuration option on a per-service basis + # This allows fine-tuned control for services with special requirements + # Supported overrides: dl, curl_impersonate, subtitle, muxing, headers, etc. + + # Example: Comprehensive service configuration showing all features + EXAMPLE: + # Standard service config + api_key: "service_api_key" + + # Service certificate for Widevine L1/L2 (base64 encoded) + # This certificate is automatically used when L1/L2 schemes are selected + # Services obtain this from their DRM provider or license server + certificate: | + CAUSwwUKvQIIAxIQ5US6QAvBDzfTtjb4tU/7QxiH8c+TBSKOAjCCAQoCggEBAObzvlu2hZRsapAPx4Aa4GUZj4/GjxgXUtBH4THSkM40x63wQeyVxlEEo + # ... (full base64 certificate here) + + # Profile-specific device configurations + profiles: + john_sd: + device: + app_name: "AIV" + device_model: "SHIELD Android TV" + jane_uhd: + device: + app_name: "AIV" + device_model: "Fire TV Stick 4K" + + # Service-specific proxy mappings + # Override global proxy selection with specific servers for this service + # When --proxy matches a key in proxy_map, the mapped server will be used + # instead of the default/random server selection + proxy_map: + nordvpn:ca: ca1577 # Use ca1577 when --proxy nordvpn:ca is specified + nordvpn:us: us9842 # Use us9842 when --proxy nordvpn:us is specified + us: 123 # Use server 123 (from any provider) when --proxy us is specified + gb: 456 # Use server 456 (from any provider) when --proxy gb is specified + # Without this service, --proxy nordvpn:ca picks a random CA server + # With this config, --proxy nordvpn:ca EXAMPLE uses ca1577 specifically + # Other services or no service specified will still use random selection + + # NEW: Configuration overrides (can be combined with profiles and certificates) + # Override dl command defaults for this service + dl: + downloads: 4 # Limit concurrent track downloads (global default: 6) + workers: 8 # Reduce workers per track (global default: 16) + lang: ["en", "es-419"] # Different language priority for this service + sub_format: srt # Force SRT subtitle format + + # Override subtitle processing for this service + subtitle: + conversion_method: pycaption # Use specific subtitle converter + sdh_method: auto + + # Service-specific headers + headers: + User-Agent: "Service-specific user agent string" + Accept-Language: "en-US,en;q=0.9" + + # Override muxing options + muxing: + set_title: true + + # Remap service-provided titles before naming/output + # Keyed by the exact title the service returns -> desired output title. + title_map: + Service Title: Desired Title + + # Example: Service with different regions per profile + SERVICE_NAME: + profiles: + us_account: + region: "US" + api_endpoint: "https://api.us.service.com" + uk_account: + region: "GB" + api_endpoint: "https://api.uk.service.com" + + # Example: Rate-limited service + RATE_LIMITED_SERVICE: + dl: + downloads: 2 # Limit concurrent downloads + workers: 4 # Reduce workers to avoid rate limits + + # Notes on service-specific overrides: + # - Overrides are merged with global config, not replaced + # - Only specified keys are overridden, others use global defaults + # - Reserved keys (profiles, api_key, certificate, etc.) are NOT treated as overrides + # - Any dict-type config option can be overridden (dl, subtitle, muxing, headers, etc.) + # - CLI arguments always take priority over service-specific config + +# External proxy provider services +proxy_providers: + nordvpn: + username: username_from_service_credentials + password: password_from_service_credentials + # server_map: global mapping that applies to ALL services + # Difference from service-specific proxy_map: + # - server_map: applies to ALL services when --proxy nordvpn:us is used + # - proxy_map: only applies to the specific service configured (see services: EXAMPLE: proxy_map above) + # - proxy_map takes precedence over server_map for that service + server_map: + us: 12 # force US server #12 for US proxies + ca:calgary: 2534 # force CA server #2534 for Calgary proxies + us:seattle: 7890 # force US server #7890 for Seattle proxies + surfsharkvpn: + username: your_surfshark_service_username # Service credentials from https://my.surfshark.com/vpn/manual-setup/main/openvpn + password: your_surfshark_service_password # Service credentials (not your login password) + server_map: + us: 3844 # force US server #3844 for US proxies + gb: 2697 # force GB server #2697 for GB proxies + au: 4621 # force AU server #4621 for AU proxies + us:seattle: 5678 # force US server #5678 for Seattle proxies + ca:toronto: 1234 # force CA server #1234 for Toronto proxies + windscribevpn: + username: your_windscribe_username # Service credentials from https://windscribe.com/getconfig/openvpn + password: your_windscribe_password # Service credentials (not your login password) + server_map: + us: "us-central-096.totallyacdn.com" # force US server + gb: "uk-london-055.totallyacdn.com" # force GB server + us:seattle: "us-west-011.totallyacdn.com" # force US Seattle server + ca:toronto: "ca-toronto-012.totallyacdn.com" # force CA Toronto server + + # Gluetun: Dynamic Docker-based VPN proxy (supports 50+ VPN providers) + # Creates Docker containers running Gluetun to bridge VPN connections to HTTP proxies + # Requires Docker to be installed and running + # Usage: --proxy gluetun:windscribe:us or --proxy gluetun:nordvpn:de + gluetun: + # Global settings + base_port: 8888 # Starting port for HTTP proxies (increments for each container) + auto_cleanup: true # Automatically remove containers when done + container_prefix: "unshackle-gluetun" # Docker container name prefix + verify_ip: true # Verify VPN IP matches expected region + # Optional HTTP proxy authentication (for the proxy itself, not VPN) + # auth_user: proxy_user + # auth_password: proxy_password + + # VPN provider configurations + providers: + # Windscribe (WireGuard) - Get credentials from https://windscribe.com/getconfig/wireguard + windscribe: + vpn_type: wireguard + credentials: + private_key: "YOUR_WIREGUARD_PRIVATE_KEY" + addresses: "YOUR_WIREGUARD_ADDRESS" # e.g., "10.x.x.x/32" + # Map friendly names to country codes + server_countries: + us: US + uk: GB + ca: CA + de: DE + + # NordVPN (OpenVPN) - Get service credentials from https://my.nordaccount.com/dashboard/nordvpn/manual-configuration/ + # Note: Service credentials are NOT your email+password - generate them from the link above + # nordvpn: + # vpn_type: openvpn + # credentials: + # username: "YOUR_NORDVPN_SERVICE_USERNAME" + # password: "YOUR_NORDVPN_SERVICE_PASSWORD" + # server_countries: + # us: US + # uk: GB + + # ExpressVPN (OpenVPN) - Get credentials from ExpressVPN setup page + # expressvpn: + # vpn_type: openvpn + # credentials: + # username: "YOUR_EXPRESSVPN_USERNAME" + # password: "YOUR_EXPRESSVPN_PASSWORD" + # server_countries: + # us: US + # uk: GB + + # Surfshark (WireGuard) - Get credentials from https://my.surfshark.com/vpn/manual-setup/main/wireguard + # surfshark: + # vpn_type: wireguard + # credentials: + # private_key: "YOUR_SURFSHARK_PRIVATE_KEY" + # addresses: "YOUR_SURFSHARK_ADDRESS" + # server_countries: + # us: US + # uk: GB + + # Specific server selection: Use format like "us1239" to select specific servers + # Example: --proxy gluetun:nordvpn:us1239 connects to us1239.nordvpn.com + # Supported providers: nordvpn, surfshark, expressvpn, cyberghost + + basic: + GB: + - "socks5://username:password@bhx.socks.ipvanish.com:1080" # 1 (Birmingham) + - "socks5://username:password@gla.socks.ipvanish.com:1080" # 2 (Glasgow) + AU: + - "socks5://username:password@syd.socks.ipvanish.com:1080" # 1 (Sydney) + - "https://username:password@au-syd.prod.surfshark.com" # 2 (Sydney) + - "https://username:password@au-bne.prod.surfshark.com" # 3 (Brisbane) + BG: "https://username:password@bg-sof.prod.surfshark.com" diff --git a/packages/envied/src/envied/services/NBC/__init__.py b/packages/envied/src/envied/services/NBC/__init__.py new file mode 100644 index 0000000..44eaab8 --- /dev/null +++ b/packages/envied/src/envied/services/NBC/__init__.py @@ -0,0 +1,321 @@ +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +import re +import time +from collections.abc import Generator +from typing import Optional, Union +from urllib.parse import urlencode + +import click +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from envied.core.constants import AnyTrack +from envied.core.manifests import DASH +from envied.core.search_result import SearchResult +from envied.core.service import Service +from envied.core.titles import Episode, Series, Title_T, Titles_T +from envied.core.tracks import Chapters, Tracks + + +# NBC ships the page metadata AND the obfuscated config (which contains +# drmProxySecret) inline in every nbc.com page's HTML, both inside a +# `PRELOAD={...}` JS global. Layout we depend on: +# PRELOAD.pages[].base — same shape as the legacy GraphQL response +# (replaces friendship.nbc.com/v3/graphql) +# PRELOAD.client.oc — base64-encoded encrypted config blob +# +# Obfuscated-config payload format (after base64-decode): +# bytes 0..12 : AES-GCM IV (12 bytes) +# bytes 12..44 : AES-256 key (32 bytes - the key is shipped next to the +# ciphertext, this is obfuscation, not security) +# bytes 44..-4 : AES-GCM ciphertext (includes 16-byte auth tag) +# bytes -4.. : COMPATIBILITY_VERSION (uint32 big-endian) — sanity check +# Decrypted plaintext is UTF-8 JSON. Currently exposes `{coreVideo: {drmProxySecret}}`. +_OC_COMPATIBILITY_VERSION = 1 + + +class NBC(Service): + """ + \b + Service code for NBC.com (https://www.nbc.com). + + \b + Version: 0.1.0 + Authorization: None (free-tier content only — TV-provider auth not implemented) + Robustness: + Widevine: L3 + + \b + Tips: + - Input may be either: + SERIES: https://www.nbc.com/ (current season only) + EPISODE: https://www.nbc.com//video// + - Series URLs enumerate only the most recent season — older seasons are + typically behind Peacock auth and not surfaced by the free-tier API. + - Content with active TV-provider entitlement windows will fail — wait until the + episode's free window opens (typically ~7 days after first air). + """ + + GEOFENCE = ("us",) + + TITLE_RE = re.compile( + r"^https?://www\.nbc\.com/(?P[a-zA-Z0-9_-]+)" + r"(?:/video/[a-zA-Z0-9_-]+/(?P\d+))?/?$" + ) + + @staticmethod + @click.command(name="NBC", short_help="https://www.nbc.com", help=__doc__) + @click.argument("title", type=str, required=True) + @click.pass_context + def cli(ctx, **kwargs) -> NBC: + return NBC(ctx, **kwargs) + + def __init__(self, ctx, title): + self.title = title + super().__init__(ctx) + # URL parsing happens lazily in get_titles() — search() accepts a free-text + # query that won't match TITLE_RE. + + # Service API + + def search(self) -> Generator[SearchResult, None, None]: + algolia = self.config["algolia"] + entity_types = ["series", "episodes", "movies"] + facet_filters = json.dumps([[f"algoliaProperties.entityType:{t}" for t in entity_types]]) + algolia_params = urlencode({ + "query": self.title, + "facetFilters": facet_filters, + "page": 0, + "hitsPerPage": 20, + }) + body = {"requests": [{"indexName": algolia["index"], "params": algolia_params}]} + r = self.session.post( + algolia["url"], + headers={ + **self.config["headers"], + "content-type": "application/x-www-form-urlencoded", + "x-algolia-api-key": algolia["api_key"], + "x-algolia-application-id": algolia["app_id"], + }, + json=body, + ) + r.raise_for_status() + for hit in r.json().get("results", [{}])[0].get("hits") or []: + entity_type = (hit.get("algoliaProperties") or {}).get("entityType") + if entity_type == "series": + series_data = hit.get("series") or {} + slug = series_data.get("seriesName") or series_data.get("urlAlias") + if not slug: + continue + yield SearchResult( + id_=hit.get("objectID") or slug, + title=series_data.get("shortTitle") or slug, + description=series_data.get("shortDescription"), + label="Series", + url=f"https://www.nbc.com/{slug}", + ) + elif entity_type == "episodes": + ep_data = hit.get("episegment") or {} + video = hit.get("video") or {} + season = hit.get("season") or {} + series_data = hit.get("series") or {} + permalink = (video.get("permalink") or "").replace("http://", "https://") + if not permalink: + continue + season_n = season.get("seasonNumber") + episode_n = ep_data.get("episodeNumber") + label_bits = [series_data.get("shortTitle")] + if season_n is not None and episode_n is not None: + label_bits.append(f"S{season_n:02d}E{episode_n:02d}") + yield SearchResult( + id_=video.get("mpxGuid") or hit.get("objectID"), + title=ep_data.get("title") or "(untitled)", + description=ep_data.get("shortDescription"), + label=" · ".join(b for b in label_bits if b), + url=permalink, + ) + + def get_titles(self) -> Titles_T: + match = self.TITLE_RE.match(self.title) + if not match: + raise ValueError(f"Could not parse NBC URL: {self.title!r}") + show_slug = match.group("show") + mpx_guid = match.group("id") + self.show_slug = show_slug # cached for _episode_from_meta fallback + if mpx_guid: + return Series([self._episode_from_url()]) + return Series(self._show(show_slug)) + + def get_tracks(self, title: Title_T) -> Tracks: + manifest_url = self._fetch_manifest_url(title) + return DASH.from_url(url=manifest_url).to_tracks(language=title.language) + + def get_chapters(self, title: Episode) -> Chapters: + return Chapters() + + def certificate(self, **_): + return None # use common privacy cert + + def get_widevine_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[Union[bytes, str]]: + # hash = HMAC-SHA256(drm_proxy_secret, str(time_ms) + "widevine") + # The secret is extracted per-title from the obfuscated `oc` blob on the + # episode/show page HTML — see _fetch_page / _decrypt_oc. + secret = title.data.get("drm_proxy_secret") + if not secret: + raise ValueError( + "NBC: title has no drm_proxy_secret on its data — was it created outside of get_titles()?" + ) + time_ms = str(int(time.time() * 1000)) + url_hash = hmac.new( + secret.encode(), (time_ms + "widevine").encode(), hashlib.sha256 + ).hexdigest() + + r = self.session.post( + self.config["endpoints"]["license_url"], + params={"time": time_ms, "hash": url_hash, "device": "web"}, + headers={**self.config["headers"], "content-type": "application/octet-stream"}, + data=challenge, + ) + if not r.ok: + raise ConnectionError(f"NBC license request failed (HTTP {r.status_code}): {r.text[:200]}") + return r.content + + # Service-specific helpers --------------------------------------------------- + + def _episode_from_url(self) -> Episode: + # The page URL path after nbc.com/, e.g. + # "law-and-order-special-victims-unit/video/monster/9000448060" + path = re.match(r"https?://www\.nbc\.com/(.+?)/?$", self.title).group(1) + page, secret = self._fetch_page(path) + return self._episode_from_meta(page["metadata"], secret) + + def _show(self, slug: str) -> list[Episode]: + # Show landing page returns the current season's episodes (older seasons are + # typically Peacock-only and don't appear in itemLabelsConfig here). + page, secret = self._fetch_page(slug) + episodes: list[Episode] = [] + for section in page.get("data", {}).get("sections") or []: + if section.get("component") != "LinksSelectableGroup": + continue + section_data = section.get("data") or {} + if section_data.get("optionalTitle") != "Episodes": + continue + for shelf in section_data.get("items") or []: + for tile in (shelf.get("data") or {}).get("items") or []: + tile_data = tile.get("data") or {} + if tile_data.get("programmingType") != "Full Episode": + continue + episodes.append(self._episode_from_meta(tile_data, secret)) + if not episodes: + raise ValueError(f"NBC: no episodes found for show {slug!r}") + return episodes + + def _episode_from_meta(self, meta: dict, drm_proxy_secret: str) -> Episode: + """Build an Episode from a VIDEO-page `metadata` dict or a VideoTile `data` dict. + Both shapes expose the same field set (mpxGuid, mpxAccountId, season/episode + numbers, secondaryTitle, seriesShortTitle, programmingType, duration, permalink). + """ + return Episode( + id_=meta["mpxGuid"], + title=meta.get("seriesShortTitle") or self.show_slug, + season=int(meta["seasonNumber"]) if meta.get("seasonNumber") else 0, + number=int(meta["episodeNumber"]) if meta.get("episodeNumber") else 0, + name=meta.get("secondaryTitle"), + language="en-US", + service=self.__class__, + data={ + "mpxAccountId": meta["mpxAccountId"], + "mpxGuid": meta["mpxGuid"], + "programmingType": meta.get("programmingType", "Full Episode"), + "duration": meta.get("duration"), + "permalink": meta.get("permalink"), + "drm_proxy_secret": drm_proxy_secret, + }, + ) + + def _fetch_page(self, url_path: str) -> tuple[dict, str]: + """Fetch an nbc.com page and return (page_base, drm_proxy_secret). + + `page_base` is the same dict shape that friendship.nbc.com used to return + as `data.page` (keys: metadata, analytics, data, ...). + """ + url = f"https://www.nbc.com/{url_path.lstrip('/')}" + r = self.session.get(url, headers=self.config["headers"]) + if r.status_code != 200: + raise ConnectionError(f"NBC page {r.status_code}: {url}") + preload = self._extract_preload(r.text) + pages = preload.get("pages") or {} + # The URL we requested is the key; sometimes the path is normalised with + # a leading slash, sometimes without — match whichever key is present. + page = next(iter(pages.values()), None) + if not page or "base" not in page: + raise ValueError(f"NBC page {url}: PRELOAD has no pages[*].base") + oc_b64 = (preload.get("client") or {}).get("oc") + if not oc_b64: + raise ValueError(f"NBC page {url}: PRELOAD has no client.oc") + secret = self._decrypt_oc(oc_b64)["coreVideo"]["drmProxySecret"] + return page["base"], secret + + @staticmethod + def _extract_preload(html: str) -> dict: + """Locate `PRELOAD={...}` in the inline ` and strip JS punctuation. + """ + marker = "PRELOAD=" + start = html.find(marker + "{") + if start < 0: + raise ValueError("NBC: PRELOAD global not found in page HTML") + start += len(marker) + end = html.find("", start) + if end < 0: + raise ValueError("NBC: unterminated