diff --git a/README.md b/README.md index da1cbff..5d07133 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ wget -qO- https://astral.sh/uv/install.sh | sh 6. Then initialize TwinVine: ```bash +git clone https://github.com/vinefeeder/TwinVine.git cd TwinVine uv lock uv sync diff --git a/UPDATES.md b/UPDATES.md index 4972ba2..c893607 100644 --- a/UPDATES.md +++ b/UPDATES.md @@ -1,4 +1,7 @@ **Updates** +**11 July 2026** +vinefeeder update to BBC service to better select UHD quality availability. Granularity is now at the episode level rather than series. +meaning that any series with mixed UHD and HD titles will no longer fail at the download stage **11 May 2026** Vinefeeder update to BBC UHD detection to enable automatic HLG selection. diff --git a/packages/envied/src/envied/commands/search.py b/packages/envied/src/envied/commands/search.py index 4eaae82..c687125 100644 --- a/packages/envied/src/envied/commands/search.py +++ b/packages/envied/src/envied/commands/search.py @@ -136,6 +136,7 @@ def result(service: Service, profile: Optional[str] = None, **_: Any) -> None: result_text = f"[bold text]{result.title}[/]" if result.url: result_text = f"[link={result.url}]{result_text}[/link]" + if result.label: result_text += f" [pink]{result.label}[/]" if result.description: @@ -143,7 +144,7 @@ def result(service: Service, profile: Optional[str] = None, **_: Any) -> None: result_text += f"\n[bright_black]id: {result.id}[/]" search_results.add(result_text + "\n") - # update cookies + # update cookie cookie_file = dl.get_cookie_path(service_tag, profile) if cookie_file: dl.save_cookies(cookie_file, service.session.cookies) diff --git a/packages/envied/src/envied/core/search_result.py b/packages/envied/src/envied/core/search_result.py index c902995..3bfe46c 100644 --- a/packages/envied/src/envied/core/search_result.py +++ b/packages/envied/src/envied/core/search_result.py @@ -9,6 +9,7 @@ class SearchResult: description: Optional[str] = None, label: Optional[str] = None, url: Optional[str] = None, + image: Optional[str] = None ): """ A Search Result for any support Title Type. @@ -33,12 +34,15 @@ class SearchResult: raise TypeError(f"Expected label to be a {str}, not {type(label)}") if not isinstance(url, (str, type(None))): raise TypeError(f"Expected url to be a {str}, not {type(url)}") - + if not isinstance(image, (str, type(None))): + raise TypeError(f"Expected image to be a {str}, not {type(image)}") + self.id = id_ self.title = title self.description = description self.label = label self.url = url + self.image = image __all__ = ("SearchResult",) diff --git a/packages/envied/src/envied/services/STV/__init__.py b/packages/envied/src/envied/services/STV/__init__.py index d05c51d..5c24352 100644 --- a/packages/envied/src/envied/services/STV/__init__.py +++ b/packages/envied/src/envied/services/STV/__init__.py @@ -22,8 +22,8 @@ class STV(Service): Service code for STV Player streaming service (https://player.stv.tv/). \b - Version: 1.0.3 - Author: stabbedbybrick + Version: 1.0.4 + Author: stabbedbybrick; search corrected by Angela Authorization: None Robustness: L3: 1080p @@ -68,6 +68,7 @@ class STV(Service): synopsis = result['attributes']["long_description"] label = result["attributes"]["subGenre"] + #image = result["attributes"]["images"][0]["_filepath"].split("&w")[0] if result["attributes"]["images"] else None yield SearchResult( id_= self.config["endpoints"]["prefix"].strip("/") + result["attributes"]["permalink"], @@ -75,6 +76,7 @@ class STV(Service): description=synopsis, label=label, url=url, + #image=image ) def get_titles(self) -> Union[Movies, Series]: diff --git a/packages/envied/src/envied/services/TPTV/__init__.py b/packages/envied/src/envied/services/TPTV/__init__.py index 722650a..2a802cd 100644 --- a/packages/envied/src/envied/services/TPTV/__init__.py +++ b/packages/envied/src/envied/services/TPTV/__init__.py @@ -43,10 +43,10 @@ class TPTV(Service): Service code for TPTVencore streaming service (https://www.TPTVencore.co.uk/). \b - version 1.1.0 - Date: June 2026 + version 1.3.0 + Date: July 2026 Author: A_n_g_e_l_a - Authorization: email/password for service in envied.yaml + Authorization: email/password for service in envied.yaml Robustness: DRM free... with rare exceptions L3 @@ -190,8 +190,29 @@ class TPTV(Service): self.authorization = tokens def search(self) -> Generator[SearchResult, None, None]: - print(f"Searching not implemented in Envied for TPTVencore. Please use VineFeeder instead.") - return None + + search_term = self.title.replace(" ", "+") + response = self.session.get(self.config["endpoints"]["search"].format(query=search_term)) + response.raise_for_status() + results = json.loads(response.content.decode("utf-8")) + + for result in results["data"]: + url = result['video']['playback'].replace("api/core/play", "playback") + title = result['title'] + synopsis = result['description'].replace('\n', ' ') + label = result["subtype"] + id = result["video"]["playback"].replace("api/core/play", "playback") + + + yield SearchResult( + id_= id, + title=title, + description=synopsis, + label=label, + url=url, + ) + + def get_titles(self) -> Union[Movies, Series]: data = self.get_data(self.title) diff --git a/packages/vinefeeder/src/vinefeeder/base_loader.py b/packages/vinefeeder/src/vinefeeder/base_loader.py index 23a17f6..a2fef4d 100644 --- a/packages/vinefeeder/src/vinefeeder/base_loader.py +++ b/packages/vinefeeder/src/vinefeeder/base_loader.py @@ -178,7 +178,7 @@ class BaseLoader: return - def add_episode(self, series_name, episode): + def add_episode(self, series_name, episode, uhd=None): """Add an episode to the series in memory. Episode may be Any""" if series_name not in self.series_data: @@ -246,13 +246,19 @@ class BaseLoader: def get_final_episode_list(self): return self.final_episode_data - def display_final_episode_list(self, final_episode_data): + def display_final_episode_list(self, final_episode_data, check_uhd=False): """Use beaupy to display episodes for a selected series.""" # episodes = self.series_data.get(series_name, []) - episode_list = [ - f"{ep['series_no']}, {ep['title']}, {ep['url']}, \n\t {ep['synopsis']}" - for ep in final_episode_data - ] + if check_uhd: + episode_list = [ + f"{ep['series_no']}, {ep['title']}, {ep['url']}, UHD={ep['uhd']}, \n\t {ep['synopsis']}" + for ep in final_episode_data + ] + else: + episode_list = [ + f"{ep['series_no']}, {ep['title']}, {ep['url']}, \n\t {ep['synopsis']}" + for ep in final_episode_data + ] selected_episodes = select_multiple( episode_list, preprocessor=lambda val: prettify(val), diff --git a/packages/vinefeeder/src/vinefeeder/services/BBC/__init__.py b/packages/vinefeeder/src/vinefeeder/services/BBC/__init__.py index 3f6eae1..ca72bfc 100644 --- a/packages/vinefeeder/src/vinefeeder/services/BBC/__init__.py +++ b/packages/vinefeeder/src/vinefeeder/services/BBC/__init__.py @@ -9,6 +9,7 @@ from vinefeeder.parsing_utils import extract_params_json, parse_json, split, spl from rich.console import Console import jmespath from scrapy.selector import Selector +import json console = Console() @@ -151,7 +152,7 @@ class BbcLoader(BaseLoader): print( "Error downloading video:", e, - "Is self.DOWNLOAD_ORCHESTRATOR installed correctly via 'pip install self.DOWNLOAD_ORCHESTRATOR?", + f"Is {self.DOWNLOAD_ORCHESTRATOR} installed correctly via 'pip install {self.DOWNLOAD_ORCHESTRATOR}?", ) else: @@ -166,7 +167,7 @@ class BbcLoader(BaseLoader): print( "Error downloading video:", e, - "Is self.DOWNLOAD_ORCHESTRATOR installed correctly via 'pip install self.DOWNLOAD_ORCHESTRATOR?", + f"Is {self.DOWNLOAD_ORCHESTRATOR} installed correctly via 'pip install {self.DOWNLOAD_ORCHESTRATOR}?", ) return @@ -213,15 +214,11 @@ class BbcLoader(BaseLoader): def fetch_videos(self, search_term): """Fetch videos from BBC using a search term.""" - url = "https://ibl.api.bbc.co.uk/ibl/v1/new-search" - # url = f"https://search.api.bbci.co.uk/formula/iplayer-ibl-root?q={search_term}&apikey=D2FgtcTxGqqIgLsfBWTJdrQh2tVdeaAp&seqId=0582e0f0-b911-11ee-806c-11c6c885ab56" - params = { - "q": search_term, - "rights": "web", - #'mixin': 'live' - } + url = f"https://ibl.api.bbc.co.uk/ibl/v1/new-search?q={search_term}&rights=mobile&size=10&sort=relevance&start=0" + + try: - html = self.get_data(url, self.headers, params) + html = self.get_data(url, self.headers) if "new_search" not in html: print("Nothing found for that search; try again.") return @@ -290,10 +287,10 @@ class BbcLoader(BaseLoader): parsed_data = parse_json(myhtml) # testing - #file = open("init_data.json", "w") - #file.write(json.dumps(parsed_data)) - #file.close() - #console.print_json(data=parsed_data) + # file = open("init_data.json", "w") + # file.write(json.dumps(parsed_data)) + # file.close() + # console.print_json(data=parsed_data) self.clear_series_data() # Clear existing series data self.options_list = split_options(self.options) @@ -317,6 +314,8 @@ class BbcLoader(BaseLoader): episodes = parsed_data.get("programme_episodes").get("elements") for item in episodes: try: + # programme_episodes►elements►0►versions►0►uhd + uhd = item.get("versions", [{}])[0].get("uhd", False) series_no = item["subtitle"].split(":")[0].split(" ")[1] if int(series_no): pass @@ -330,21 +329,23 @@ class BbcLoader(BaseLoader): ], # .split(' ')[-1] or '01', "url": "https://www.bbc.co.uk/iplayer/episode/" + item["id"], "synopsis": item["synopses"]["small"] or None, + "uhd": uhd or False, } except Exception: try: episode = { - "series_no": 100, # special or one-off' + "series_no": int(100), # special or one-off' # 'title' is episode number here, some services use descriptive text "title": item["subtitle"], # could be date "url": "https://www.bbc.co.uk/iplayer/episode/" + item["id"], "synopsis": item["synopses"]["small"] or None, + "uhd": item.get("versions", [{}])[0].get("uhd", False) or False, } except KeyError as e: print(f"Error: {e}") continue # Skip any episode that doesn't have the required information - self.add_episode(series_name, episode) + self.add_episode(series_name, episode, uhd=uhd) # Single episode expedite download else: @@ -356,6 +357,9 @@ class BbcLoader(BaseLoader): self.AVAILABLE_HLG = True break # self.options_list = split_options(self.options) + + + try: if BbcLoader.HLG and self.AVAILABLE_HLG: command = ( @@ -378,7 +382,7 @@ class BbcLoader(BaseLoader): print( "Error downloading video:", e, - "Is self.DOWNLOAD_ORCHESTRATOR installed correctly via 'pip install self.DOWNLOAD_ORCHESTRATOR?", + f"Is {self.DOWNLOAD_ORCHESTRATOR} installed correctly via 'pip install {self.DOWNLOAD_ORCHESTRATOR}'?", ) return @@ -386,17 +390,22 @@ class BbcLoader(BaseLoader): series_name ) # creates list of series; allows user selection of wanted series prepares an episode list over chosen series selected_final_episodes = self.display_final_episode_list( - self.final_episode_data + self.final_episode_data, check_uhd=True ) # specific to BBC # self.options_list = split_options(self.options) for item in selected_final_episodes: - # check for UHD content - for hlg_item in self.uhd_list: - if series_name.lower() in hlg_item: - self.AVAILABLE_HLG = True - break + #print(type(item)) + #print(item) + for part in item.split(","): + if "UHD" in part: + uhd = part.split("UHD=")[1].strip() + # Sanity check for UHD content + if uhd=="True": + self.AVAILABLE_HLG = True + else: + self.AVAILABLE_HLG = False for part in item.split(","): if "https" in part: url = part.strip() @@ -419,7 +428,7 @@ class BbcLoader(BaseLoader): Returns: None """ - beaupylist = [] + #beaupylist = [] req = self.client.get(browse_url, headers=self.headers) init_data = extract_params_json(req.content.decode(), "__IPLAYER_REDUX_STATE__") diff --git a/reset/.gitignore b/reset/.gitignore new file mode 100644 index 0000000..5888731 --- /dev/null +++ b/reset/.gitignore @@ -0,0 +1,218 @@ +Cache/ +Logs/ +Downloads +Temp/ +packages/envied/src/envied/logs/ +packages/envied/src/envied/envied.yaml +packages/envied/src/envied/cache/TVNZ/local_storage.jsonv +images/* +tools/ +batch.txt +*.json +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock +#poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +#pdm.lock +#pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +#pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +.vscode/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Cursor +# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to +# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data +# refer to https://docs.cursor.com/context/ignore-files +.cursorignore +.cursorindexingignore + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ diff --git a/reset/Install-media-tools.ps1 b/reset/Install-media-tools.ps1 new file mode 100644 index 0000000..a18a9c0 --- /dev/null +++ b/reset/Install-media-tools.ps1 @@ -0,0 +1,231 @@ +# Install-media-tools.ps1 +# Run in elevated PowerShell: +# Inside the VM: + +# Open Start + +# Type PowerShell + +# Right-click Windows PowerShell → Run as administrator + +# chdir to the location of this script + +# powershell -ExecutionPolicy Bypass -File .\Install-media-tools.ps1 + +$ErrorActionPreference = "Stop" + +# Where to install (bin folder on PATH) +$BinDir = "C:\Tools\bin" +$ToolsRoot = "C:\Tools" +$SubtitleEditDir = Join-Path $ToolsRoot "SubtitleEdit" +$WorkDir = Join-Path $env:TEMP ("media-tools-" + [guid]::NewGuid().ToString("N")) + +New-Item -ItemType Directory -Force -Path $BinDir | Out-Null +New-Item -ItemType Directory -Force -Path $WorkDir | Out-Null +New-Item -ItemType Directory -Force -Path $SubtitleEditDir | Out-Null + +function Download-File([string]$Url, [string]$OutFile) { + Write-Host "Downloading: $Url" + Invoke-WebRequest -Uri $Url -OutFile $OutFile -UseBasicParsing +} + +function Add-ToMachinePath([string]$PathToAdd) { + $current = [Environment]::GetEnvironmentVariable("Path", "Machine") + if (-not $current) { $current = "" } + + # Compare case-insensitively on Windows + $parts = $current.Split(";", [System.StringSplitOptions]::RemoveEmptyEntries) + $exists = $parts | Where-Object { $_.TrimEnd("\") -ieq $PathToAdd.TrimEnd("\") } + + if (-not $exists) { + $newPath = ($current.TrimEnd(";") + ";" + $PathToAdd).TrimStart(";") + [Environment]::SetEnvironmentVariable("Path", $newPath, "Machine") + Write-Host "Added to MACHINE PATH: $PathToAdd" + Write-Host "Restart your terminal/session to pick it up." + } else { + Write-Host "MACHINE PATH already contains: $PathToAdd" + } +} + +try { + Push-Location $WorkDir + + # ----------------------------- + # Install FFmpeg (full build, Windows x64) from GitHub ZIP + # ----------------------------- + $ffmpegZip = Join-Path $WorkDir "ffmpeg-full_build.zip" + Download-File ` + "https://github.com/GyanD/codexffmpeg/releases/download/2025-12-14-git-3332b2db84/ffmpeg-2025-12-14-git-3332b2db84-full_build.zip" ` + $ffmpegZip + + $ffmpegExtract = Join-Path $WorkDir "ffmpeg" + New-Item -ItemType Directory -Force -Path $ffmpegExtract | Out-Null + Expand-Archive -Path $ffmpegZip -DestinationPath $ffmpegExtract -Force + + # Find the extracted "bin" folder (contains ffmpeg.exe etc.) + $ffmpegBin = Get-ChildItem $ffmpegExtract -Recurse -Directory | + Where-Object { $_.Name -ieq "bin" } | + Select-Object -First 1 + + if (-not $ffmpegBin) { + throw "FFmpeg bin directory not found after extraction." + } + + Get-ChildItem $ffmpegBin.FullName -File | + Where-Object { $_.Name -match '^ff(mpeg|probe|play)\.exe$' } | + ForEach-Object { + Copy-Item -Force $_.FullName (Join-Path $BinDir $_.Name) + } + + Write-Host "FFmpeg installed to $BinDir" + + + # ----------------------------- + # Install Bento4 (mp4decrypt.exe, etc.) + # ----------------------------- + $bentoZip = Join-Path $WorkDir "Bento4.zip" + Download-File ` + "https://www.bok.net/Bento4/binaries/Bento4-SDK-1-6-0-641.x86_64-microsoft-win32.zip" ` + $bentoZip + Expand-Archive -Path $bentoZip -DestinationPath (Join-Path $WorkDir "bento") -Force + + Get-ChildItem -Path (Join-Path $WorkDir "bento") -Recurse -File | + Where-Object { $_.FullName -match "\\bin\\.*\.exe$" } | + ForEach-Object { Copy-Item -Force $_.FullName $BinDir } + + # ----------------------------- + # Install MKVToolNix (Windows x64) + # ----------------------------- + $mkvInstaller = Join-Path $WorkDir "mkvtoolnix-64-bit-96.0-setup.exe" + Download-File ` + "https://mkvtoolnix.download/windows/releases/96.0/mkvtoolnix-64-bit-96.0-setup.exe" ` + $mkvInstaller + + # Silent install (NSIS): /S = silent + # Note: no UI will appear; wait for it to finish. + Start-Process -FilePath $mkvInstaller -ArgumentList "/S" -Wait + + # Optional: add MKVToolNix install dir to PATH if present + $mkvDefaultPath = "C:\Program Files\MKVToolNix" + if (Test-Path $mkvDefaultPath) { + # If you kept Add-ToMachinePath, use that: + Add-ToMachinePath $mkvDefaultPath + + # Or if you switched to Add-ToBestPath, use that instead: + # Add-ToBestPath $mkvDefaultPath + } else { + Write-Host "MKVToolNix installed, but default path not found: $mkvDefaultPath (maybe different install location)" + } + + # ----------------------------- + # Install N_m3u8DL-RE (Windows x64) + # ----------------------------- + $nreZip = Join-Path $WorkDir "N_m3u8DL-RE.zip" + Download-File ` + "https://github.com/nilaoda/N_m3u8DL-RE/releases/download/v0.3.0-beta/N_m3u8DL-RE_v0.3.0-beta_win-x64_20241203.zip" ` + $nreZip + Expand-Archive -Path $nreZip -DestinationPath (Join-Path $WorkDir "nre") -Force + + Get-ChildItem (Join-Path $WorkDir "nre") -Recurse -File | + Where-Object { $_.Name -match "^N_m3u8DL-RE(\.exe)?$" } | + Select-Object -First 1 | + ForEach-Object { Copy-Item -Force $_.FullName (Join-Path $BinDir "N_m3u8DL-RE.exe") } + + '''# ----------------------------- + # Install Git for Windows (64-bit) + # ----------------------------- + $gitInstaller = Join-Path $WorkDir "Git-2.52.0-64-bit.exe" + Download-File ` + "https://github.com/git-for-windows/git/releases/download/v2.52.0.windows.1/Git-2.52.0-64-bit.exe" ` + $gitInstaller + + # Silent install (NSIS): /S = silent + Start-Process -FilePath $gitInstaller -ArgumentList "/S" -Wait + + # Add the typical Git cmd folder to MACHINE PATH if present + $gitDefaultPath = "C:\Program Files\Git\cmd" + if (Test-Path $gitDefaultPath) { + Add-ToMachinePath $gitDefaultPath + } else { + Write-Host "Git installed, but default path not found: $gitDefaultPath (maybe different install location)" + }''' + + # ----------------------------- + # Install Shaka Packager (Windows x64) + # ----------------------------- + $shakaExe = Join-Path $BinDir "shaka-packager.exe" + Download-File ` + "https://github.com/shaka-project/shaka-packager/releases/download/v2.6.1/packager-win-x64.exe" ` + $shakaExe + + # ----------------------------- + # Install dovi_tool (Windows x64) + # ----------------------------- + $doviZip = Join-Path $WorkDir "dovi_tool.zip" + Download-File ` + "https://github.com/quietvoid/dovi_tool/releases/download/2.3.1/dovi_tool-2.3.1-x86_64-pc-windows-msvc.zip" ` + $doviZip + Expand-Archive -Path $doviZip -DestinationPath (Join-Path $WorkDir "dovi") -Force + + Get-ChildItem (Join-Path $WorkDir "dovi") -Recurse -File | + Where-Object { $_.Name -ieq "dovi_tool.exe" } | + Select-Object -First 1 | + ForEach-Object { Copy-Item -Force $_.FullName (Join-Path $BinDir "dovi_tool.exe") } + + # ----------------------------- + # Install hdr10plus_tool (Windows x64) + # ----------------------------- + $hdrZip = Join-Path $WorkDir "hdr10plus_tool.zip" + Download-File ` + "https://github.com/quietvoid/hdr10plus_tool/releases/download/1.7.1/hdr10plus_tool-1.7.1-x86_64-pc-windows-msvc.zip" ` + $hdrZip + Expand-Archive -Path $hdrZip -DestinationPath (Join-Path $WorkDir "hdr") -Force + + Get-ChildItem (Join-Path $WorkDir "hdr") -Recurse -File | + Where-Object { $_.Name -ieq "hdr10plus_tool.exe" } | + Select-Object -First 1 | + ForEach-Object { Copy-Item -Force $_.FullName (Join-Path $BinDir "hdr10plus_tool.exe") } + + # ----------------------------- + # Install SubtitleEdit permanently (Portable zip) + # ----------------------------- + $seZip = Join-Path $WorkDir "SubtitleEdit.zip" + Download-File ` + "https://github.com/SubtitleEdit/subtitleedit/releases/download/4.0.14/SE4014.zip" ` + $seZip + + # Replace any previous install cleanly + if (Test-Path $SubtitleEditDir) { Remove-Item -Recurse -Force $SubtitleEditDir } + New-Item -ItemType Directory -Force -Path $SubtitleEditDir | Out-Null + + Expand-Archive -Path $seZip -DestinationPath $SubtitleEditDir -Force + + # Launcher in bin (stable path) + $launcher = Join-Path $BinDir "SubtitleEdit.cmd" + @( + '@echo off' + '"C:\Tools\SubtitleEdit\SubtitleEdit.exe" %*' + ) | Set-Content -Encoding ASCII $launcher + + # ----------------------------- + # Install uv (Python required) + # ----------------------------- + python -m pip install --upgrade uv + + # PATH (Machine) + Add-ToMachinePath $BinDir + + Write-Host "`nDone. Open a NEW terminal and try:" + Write-Host " mp4decrypt.exe --help" + Write-Host " N_m3u8DL-RE.exe --help" + Write-Host " shaka-packager.exe --help" + Write-Host " dovi_tool.exe --help" + Write-Host " hdr10plus_tool.exe --help" + Write-Host " SubtitleEdit.cmd" + Write-Host " uv --version" + +} finally { + Pop-Location + # Clean up temp work dir + if (Test-Path $WorkDir) { Remove-Item -Recurse -Force $WorkDir } +} diff --git a/reset/Install-media-tools.sh b/reset/Install-media-tools.sh new file mode 100644 index 0000000..2824a65 --- /dev/null +++ b/reset/Install-media-tools.sh @@ -0,0 +1,53 @@ +# Thanks Akirainblack for providing this routine +# run script as superuser using command:- +# sudo bash Install-media-tools.sh + +# Install MKVToolNix and ffmpeg from Ubuntu repos +apt-get update && \ + apt-get install -y mkvtoolnix mkvtoolnix-gui ffmpeg && \ + rm -rf /var/lib/apt/lists/* + +# Install Bento4 (mp4decrypt) +wget https://www.bok.net/Bento4/binaries/Bento4-SDK-1-6-0-641.x86_64-unknown-linux.zip && \ + unzip -j Bento4-SDK-1-6-0-641.x86_64-unknown-linux.zip \ + 'Bento4-SDK-1-6-0-641.x86_64-unknown-linux/bin/*' -d /usr/local/bin/ && \ + rm Bento4-SDK-1-6-0-641.x86_64-unknown-linux.zip && \ + chmod +x /usr/local/bin/* + +# Install N_m3u8DL-RE +wget https://github.com/nilaoda/N_m3u8DL-RE/releases/download/v0.3.0-beta/N_m3u8DL-RE_v0.3.0-beta_linux-x64_20241203.tar.gz && \ + tar -xzf N_m3u8DL-RE_v0.3.0-beta_linux-x64_20241203.tar.gz && \ + find . -name "N_m3u8DL-RE" -type f -exec mv {} /usr/local/bin/ \; && \ + chmod +x /usr/local/bin/N_m3u8DL-RE && \ + rm -rf N_m3u8DL-RE_v0.3.0-beta_linux-x64_20241203.tar.gz + +# Install Shaka Packager +wget https://github.com/shaka-project/shaka-packager/releases/download/v3.2.0/packager-linux-x64 && \ + mv packager-linux-x64 /usr/local/bin/shaka-packager && \ + chmod +x /usr/local/bin/shaka-packager + +# Install dovi_tool +wget https://github.com/quietvoid/dovi_tool/releases/download/2.3.1/dovi_tool-2.3.1-x86_64-unknown-linux-musl.tar.gz && \ + tar -xzf dovi_tool-2.3.1-x86_64-unknown-linux-musl.tar.gz && \ + find . -name "dovi_tool" -type f -exec mv {} /usr/local/bin/ \; && \ + chmod +x /usr/local/bin/dovi_tool && \ + rm -rf dovi_tool-2.3.1-x86_64-unknown-linux-musl.tar.gz + +# Install HDR10Plus +wget https://github.com/quietvoid/hdr10plus_tool/releases/download/1.7.1/hdr10plus_tool-1.7.1-x86_64-unknown-linux-musl.tar.gz && \ + tar -xzf hdr10plus_tool-1.7.1-x86_64-unknown-linux-musl.tar.gz && \ + find . -name "hdr10plus_tool" -type f -exec mv {} /usr/local/bin/ \; && \ + chmod +x /usr/local/bin/hdr10plus_tool && \ + rm -rf hdr10plus_tool-1.7.1-x86_64-unknown-linux-musl.tar.gz + +# Install SubtitleEdit +wget https://github.com/SubtitleEdit/subtitleedit/releases/download/4.0.14/SE4014.zip && \ + unzip SE4014.zip -d /SE && \ + awk 'BEGIN{print "#!/bin/bash"}' >> /usr/local/bin/SubtitleEdit && \ + echo "exec mono /SE/SubtitleEdit.exe \"\$@\"" >> /usr/local/bin/SubtitleEdit && \ + chmod +x /usr/local/bin/SubtitleEdit && \ + rm -rf SE4014.zip + +# Install uv by copying from official image +# cp --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ +python -m pip install uv diff --git a/reset/LICENSE b/reset/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/reset/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/reset/README.md b/reset/README.md new file mode 100644 index 0000000..5d07133 --- /dev/null +++ b/reset/README.md @@ -0,0 +1,317 @@ +# TwinVine + +![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeederA.png) + +TwinVine combines two Python packages: + +- [Vinefeeder](https://github.com/vinefeeder/TwinVine/blob/main/packages/vinefeeder/src/vinefeeder/README.md) +- [Envied](https://github.com/vinefeeder/TwinVine/blob/main/packages/envied/README.md) + +TwinVine helps you find, select, and download media. Use the graphical front end when you want search and selection assistance, and use the command-line downloader for exact URLs. + +Envied is forked from unshackle github.com/unshackle-dl/unshackle and I thank the developers for their effort. + +## Key workflows + +- Use `envied` when you already have an exact program URL. +- Use `vinefeeder` search when you only know a program name. +- Use the browse feature when you want to explore categories like Film, Drama, or Sport. +- Use Batch Mode to select and download multiple items from several services. + +## Usage + +TwinVine runs through the Python package manager `uv`. + +### Run the main tools + +```bash +uv run vinefeeder +uv run envied dl --select-titles +``` + +### Access envied from the GUI + +- On Linux: choose **run envied** after clicking the GUI `envied` button. +- On Windows: close the GUI or press `Ctrl+C` to return to the terminal. + +## Installation + +### Prerequisites + +Install `uv` if needed: + +```bash +pip install uv +python3 -m pip install uv +``` + +Or install `python-uv` through your system package manager. + +### Install TwinVine + +```bash +git clone https://github.com/vinefeeder/TwinVine.git +cd TwinVine +uv clean +uv lock +uv sync +uv run vinefeeder --help +uv run envied --help +uv run envied dl -? +``` + +## Windows installation + +This section is aimed at novice Windows users. +A printer-friendly checklist is available here: + +[WINDOWS_INSTALL_CHECKLIST.txt](WINDOWS_INSTALL_CHECKLIST.txt) + +### Windows quick overview + +1. Install Git for Windows. +2. Clone the TwinVine repository. +3. Run the Windows installer script. +4. Verify that `uv` is installed. +5. Initialize TwinVine and verify the commands. + +## Linux installation + +On Linux, use the bundled shell script to install required binaries. + +1. Open `Install-media-tools.sh` in a text editor. +2. Change the package manager command on lines 6 and 7 from `apt-get` to your distro's package manager (`dnf`, `pacman`, `zypper`, etc.). +3. Save the file. +4. Run: + +```bash +sudo bash ./Install-media-tools.sh +``` + +5. If `uv` is still not installed, run: + +```bash +wget -qO- https://astral.sh/uv/install.sh | sh +``` + +6. Then initialize TwinVine: + +```bash +git clone https://github.com/vinefeeder/TwinVine.git +cd TwinVine +uv lock +uv sync +cp ./packages/envied/src/envied/envied-working-example.yaml ./packages/envied/src/envied/envied.yaml +uv run vinefeeder --help +uv run envied --help +uv run envied dl -? +``` + +## Locations + +By default, downloaded files are saved to: + +- `TwinVine/packages/envied/src/downloads/` + +Edit `packages/envied/src/envied/envied.yaml` to change the media-download location and other values like email:password for a service. + +- Use a full path. +- On Windows, use forward slashes: `C:/Users/Downloads`. +- On Linux, use `/home/user/Downloads`. + +Cookies live in `packages/envied/src/`. Each cookie file should be named exactly for the service, for example `DNSP.txt`, and contain the service login cookie. + +WVD files are stored in `TwinVine/WVDs/`, such as `device.wvd`. + +Vaults are not configured locally by default. TwinVine uses a remote vault for caching and license retrieval and may display `DRMLab` as the license source. + +## Linux note + +Linux terminals can sometimes freeze after `envied` completes a download. If that happens, ensure `TERMINAL_RESET: True` is set in: + +- `TwinVine/packages/vinefeeder/src/vinefeeder/config.yaml` + +## Services + +### Vinefeeder services + +Vinefeeder currently supports search, browse, and list-select for these services: + +- ALL4 +- BBC +- ITVX +- MY5 +- PLEX +- RTE +- STV +- TPTV +- TVNZ +- U + +### Envied services + +Envied supports a broader set of direct-download services: + +- ALL4 +- AUBC +- CBS +- CWTV +- DSCP +- iP +- MAX +- MY5 +- NF +- PCOK +- PLEX +- RTE +- ROKU +- SPOT +- TPTV +- TVNZ +- YTBE +- ARD +- CBC +- CTV +- DSNP +- ITV +- MTSP +- NBLA +- NRK +- PLUTO +- STV +- TUBI +- UKTV +- ZDF + +These services have web origins and not all have been tested. + +## You can use AI to create your own Vinefeeder service!! + +The prompt below was written by ChatGPT after refactoring a Vinefeeder service to reuse Envied service logic. + +``` +I am working on the TwinVine project: + +https://github.com/vinefeeder/TwinVine + +TwinVine contains two Python packages: + +* `envied`: a command-line video downloader. +* `vinefeeder`: a graphical / interactive front end that searches, lists, selects, and then calls `envied` by subprocess. + +I want to create or rewrite a Vinefeeder service by reusing the matching envied service instead of duplicating API logic. + +Target service: + +* Envied service path: + `packages/envied/src/envied/services//` + +* Vinefeeder service path: + `packages/vinefeeder/src/vinefeeder/services//` + +Please inspect the current code in both locations. + +Goal: + +Rewrite the Vinefeeder `` service so that it imports and reuses the existing envied `` service class wherever possible. + +The Vinefeeder service should: + +1. Preserve the standard Vinefeeder service interface: + + * `receive()` + * `fetch_videos()` + * `second_fetch()` + * `fetch_videos_by_category()` + +2. Use the envied service for: + + * authentication / token handling + * search, if the envied service provides `search()` + * programme/title expansion via `get_titles()` or `get_titles_cached()` + * service-specific API calls already implemented in envied + +3. Avoid reimplementing API endpoints that already exist in envied. + +4. Keep Vinefeeder responsible only for: + + * presenting search results + * letting the user choose a title + * expanding a multi-episode series into selectable individual episodes + * building one envied subprocess command per selected single title + +5. Use `beaupy.select_multiple()` when there are multiple episodes available. + +6. If the selected item is a movie, sport event, clip, highlight, or single episode, skip the beaupy episode list and call envied directly. + +7. Preserve Vinefeeder’s existing download behaviour by calling: + + `self.runsubprocess(command)` + + where command should normally be shaped like: + + `["uv", "run", "envied", "dl", *self.options_list, "", url]` + +8. Preserve support for Vinefeeder service options from config, using `split_options()` as existing services do. + +9. If an envied service object needs a Click context, create the smallest safe adapter context needed rather than copying envied command-line internals unnecessarily. + +10. If the envied service requires credentials, token cache, proxy state, or service config, reuse envied’s existing config-loading patterns as closely as possible. + +11. Be careful not to authenticate twice unnecessarily. If `fetch_videos()` and `second_fetch()` are sequential, prefer reusing an already-authenticated envied service instance where safe. + +12. Produce a complete replacement `__init__.py` for the Vinefeeder service. + +13. Also explain: + +* which envied methods/classes are being reused +* what Vinefeeder-specific logic remains +* any assumptions or caveats +* how I should test the rewritten service + +Please do not merely describe the approach. Provide the actual rewritten Vinefeeder service code. + +``` +## Linux users only: Adding a little automation + +In .bashrc in your home directory you can add useful aliases and scripts to speed the start-up of TwinVine. + +Add the following lines at the end of your .bashrc, (or your system's equivalent), to start Vinefeeder by typing 'v' into a terminal window; +and 'e' to start envied with a pre-filled command. BE SURE TO EDIT your/path/to/Twinvine to reflect your needs. +``` +alias v="cd /your/path/to/TwinVine/;uv run vinefeeder" + +unalias e 2>/dev/null + +e() { + cd /your/path/to/TwinVine || return + + local cmd + read -e -i "uv run envied dl --select-titles " -p "TwinVine> " cmd || return + + [[ -n "$cmd" ]] && history -s "$cmd" && eval "$cmd" +} +``` + +## Other README's + TwinVine/packages/vinefeeder/src/vinefeeder/README.md + for details for confuring Envied download options on a service by service basis. + TwinVine/packages/envied/README.md links to wiki (envied - envied's parent) + + + +Images + ![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeederA1.png) + ![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder1.png) + ![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder2.png) + ![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder4.png) + ![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder5.png) + ![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder6.png) + ![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder7.png) + ![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder8.png) + ![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder9.png) + ![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder10.png) + ![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder11.png) + ![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeederB.png) + ![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/hellyes.png) + \ No newline at end of file diff --git a/reset/UPDATES.md b/reset/UPDATES.md new file mode 100644 index 0000000..c893607 --- /dev/null +++ b/reset/UPDATES.md @@ -0,0 +1,48 @@ +**Updates** +**11 July 2026** +vinefeeder update to BBC service to better select UHD quality availability. Granularity is now at the episode level rather than series. +meaning that any series with mixed UHD and HD titles will no longer fail at the download stage + + **11 May 2026** + Vinefeeder update to BBC UHD detection to enable automatic HLG selection. + Some service updates. + + **Mar 2026** + Envied updated to match upstream to version 4.0.0 + service updates + +**19 Jan 2026** + +- upstream envied to 2.3.0 +- correction RTE vinefeeder service + +**31 Dec 2025** + +**Altered:** + - **Colour Render:** of --select-titles option title display + - **Display Ttiles:** for --select-titles option now include episode title and standard SxxExx designation + + +**29 Nov 2025** +- Followed upstream changes to envied V2.1.0 +- Updated some services +- removed incorrectly duplicated vaults folder within src/envied/ + +**13 Sept 2025:** +- Added selected service name to screen display with GUI use. Featured in the original but got lost! +- corrected uv.lock to uv lock for initializing project + +**18 Sept 2025:** + - Fully implemented Plex service - categories now selectable. + +**22 Sept 2025:** + - Added RTE service with search and direct url entry. + - updated README +**28 Sept 2025** + - added HellYes GUI + - added images to reflect GUI change + +**15 Oct 2025** + - corrected keyword search when using menu in terminal window. Previously that mode of search only took a url striing; now it takes keywords. + - code additions to reflect upstream envied changes. + diff --git a/reset/WINDOWS_INSTALL_CHECKLIST.txt b/reset/WINDOWS_INSTALL_CHECKLIST.txt new file mode 100644 index 0000000..2d56b44 --- /dev/null +++ b/reset/WINDOWS_INSTALL_CHECKLIST.txt @@ -0,0 +1,57 @@ +TwinVine Windows Installation Checklist + +1. Install Git for Windows: + - Download from https://github.com/git-for-windows/git/releases/download/v2.52.0.windows.1/Git-2.52.0-64-bit.exe + - Run the installer. + +2. Restart your computer. + +3. Open PowerShell. + +4. Change directory to your chosen install location. + +5. Clone TwinVine: + ```powershell +git clone https://github.com/vinefeeder/TwinVine.git + ``` + +6. Close PowerShell. + +7. Open PowerShell as Administrator: + - Right-click Windows PowerShell → Run as administrator. + +8. Change directory to TwinVine: + ```powershell +cd TwinVine + ``` + +9. Run the Windows installer script: + ```powershell +powershell -ExecutionPolicy Bypass -File .\Install-media-tools.ps1 + ``` + +10. After installation completes, close PowerShell and restart your computer. + +11. Open PowerShell and verify uv is installed: + ```powershell +uv + ``` + +12. If uv is missing, install it as Administrator: + ```powershell +powershell -ExecutionPolicy Bypass -c "irm https://github.com/astral-sh/uv/releases/download/0.9.18/uv-installer.ps1 | iex" + ``` + +13. Close PowerShell and restart your computer. + +14. Open PowerShell, go to TwinVine, and run: + ```powershell +uv lock +uv sync +cp .\packages\envied\src\envied\envied-working-example.yaml .\packages\envied\src\envied\envied.yaml +uv run vinefeeder --help +uv run envied --help +uv run envied dl -? + ``` + +15. You are ready to use TwinVine on Windows. diff --git a/reset/WVDs/device.wvd b/reset/WVDs/device.wvd new file mode 100644 index 0000000..e55d373 Binary files /dev/null and b/reset/WVDs/device.wvd differ diff --git a/reset/docs/ADVANCED_CONFIG.md b/reset/docs/ADVANCED_CONFIG.md new file mode 100644 index 0000000..6b69bd0 --- /dev/null +++ b/reset/docs/ADVANCED_CONFIG.md @@ -0,0 +1,232 @@ +# Advanced & System Configuration + +This document covers advanced features, debugging, and system-level configuration options. + +## serve (dict) + +Configuration for the integrated server that provides CDM endpoints (Widevine/PlayReady) and a REST API for remote downloading. + +Start the server with: + +```bash +envied serve # Default: localhost:8786 +envied serve -h 0.0.0.0 -p 8888 # Listen on all interfaces +envied serve --no-key # Disable authentication +envied serve --api-only # REST API only, no CDM endpoints +envied serve --remote-only # Only expose remote service session endpoints +``` + +### CLI Options + +| Option | Default | Description | +| --- | --- | --- | +| `-h, --host` | `127.0.0.1` | Host to serve from | +| `-p, --port` | `8786` | Port to serve from | +| `--caddy` | `false` | Also serve with Caddy reverse-proxy for HTTPS | +| `--api-only` | `false` | Serve only the REST API, disable CDM endpoints | +| `--no-widevine` | `false` | Disable Widevine CDM endpoints | +| `--no-playready` | `false` | Disable PlayReady CDM endpoints | +| `--no-key` | `false` | Disable API key authentication (allows all requests) | +| `--debug-api` | `false` | Include tracebacks and stderr in API error responses | +| `--debug` | `false` | Enable debug logging for API operations | +| `--remote-only` | `false` | Only expose remote service session endpoints (health, services, search, session) | + +### Configuration + +- `api_secret` - Secret key for REST API authentication. Required unless `--no-key` is used. All API requests must include this key via the `X-Secret-Key` header. +- `compression_level` - Compression level for API payloads (manifests, cache, cookies). `0`=off, `1`=fast, `6`=balanced, `9`=max. Default: `1`. +- `session_ttl` - Session inactivity timeout in seconds. Each request resets the timer. Default: `300`. +- `max_sessions` - Maximum concurrent sessions before the oldest is evicted. Default: `100`. +- `services` - Optional global service allowlist. Only these service tags are exposed. If omitted, all services are available. +- `devices` - List of Widevine device files (.wvd). If not specified, auto-populated from the WVDs directory. +- `playready_devices` - List of PlayReady device files (.prd). If not specified, auto-populated from the PRDs directory. +- `users` - Dictionary mapping user secret keys to their access configuration: + - `devices` - List of Widevine devices this user can access + - `playready_devices` - List of PlayReady devices this user can access + - `username` - Internal logging name for the user (not visible to users) + - `services` - Optional per-user service allowlist. Effective access is the intersection of global and per-user allowlists. + +#### Server-side `dl` defaults + +Any key accepted by `/api/download` (see `docs/API.md`) can also be declared directly under `serve:` and the REST API will treat it as a default. Per-request bodies still win. Use this to raise concurrency, force `best_available`, etc. without each client repeating the values: + +```yaml +serve: + api_secret: "..." + users: { ... } + downloads: 4 # parallel tracks per job + workers: 16 # threads per track + best_available: true + no_proxy_download: false +``` + +Layering: built-in defaults < `serve.*` overrides < service-specific defaults < request body. + +For example, + +```yaml +serve: + api_secret: "your-secret-key-here" + compression_level: 1 + session_ttl: 300 + max_sessions: 100 + # services: # global allowlist (optional) + # - EXAMPLE1 + # - EXAMPLE2 + users: + secret_key_for_jane: # 32bit hex recommended, case-sensitive + devices: # list of allowed Widevine devices for this user + - generic_nexus_4464_l3 + playready_devices: # list of allowed PlayReady devices for this user + - my_playready_device + username: jane # only for internal logging, users will not see this name + # services: # per-user allowlist (optional) + # - EXAMPLE1 + secret_key_for_james: + devices: + - generic_nexus_4464_l3 + username: james + # devices can be manually specified by path if you don't want to add it to + # envied's WVDs directory for whatever reason + # devices: + # - 'C:\Users\john\Devices\test_devices_001.wvd' + # playready_devices: + # - '/path/to/device.prd' +``` + +### REST API + +When the server is running, interactive API documentation is available at: + +- **Swagger UI**: `http://localhost:8786/api/docs/` + +See [API.md](API.md) for full REST API documentation with endpoints, parameters, and examples. + +--- + +## max_concurrent_downloads (int) + +Maximum number of `/api/download` jobs the serve queue manager will execute in parallel. Each job runs the full `dl` pipeline (authenticate, fetch tracks, decrypt, mux) in its own worker subprocess. This is independent of `serve.downloads`, which controls parallel tracks **inside** a single job. Default: `2`. + +```yaml +max_concurrent_downloads: 4 +``` + +--- + +## download_job_retention_hours (int) + +How long completed, failed, or cancelled download jobs remain queryable via `/api/download/jobs/{job_id}` before the periodic cleanup loop drops them. Default: `24`. + +```yaml +download_job_retention_hours: 48 +``` + +--- + +## debug (bool) + +Enables comprehensive debug logging. Default: `false` + +When enabled (either via config or the `-d`/`--debug` CLI flag): +- Sets console log level to DEBUG for verbose output +- Creates JSON Lines (`.jsonl`) debug log files with structured logging +- Logs detailed information about sessions, service configuration, DRM operations, and errors with full stack traces + +For example, + +```yaml +debug: true +``` + +--- + +## debug_keys (bool) + +Controls whether actual decryption keys (CEKs) are included in debug logs. Default: `false` + +When enabled: +- Content encryption keys are logged in debug output +- Only affects `content_key` and `key` fields (the actual CEKs) +- Key metadata (`kid`, `keys_count`, `key_id`) is always logged regardless of this setting +- Passwords, tokens, cookies, and session tokens remain redacted even when enabled + +For example, + +```yaml +debug_keys: true +``` + +--- + +## set_terminal_bg (bool) + +Controls whether envied should set the terminal background color. Default: `false` + +For example, + +```yaml +set_terminal_bg: true +``` + +--- + +## update_checks (bool) + +Check for updates from the GitHub repository on startup. Default: `true`. + +--- + +## update_check_interval (int) + +How often to check for updates, in hours. Default: `24`. + +--- + +## title_cache_enabled (bool) + +Enable or disable title metadata caching globally. Default: `true`. + +--- + +## title_cache_time (int) + +Title cache duration in seconds. Default: `1800` (30 minutes). + +--- + +## title_cache_max_retention (int) + +Maximum cache retention in seconds, used as fallback when the upstream API fails. Default: `86400` (24 hours). + +--- + +## unicode_filenames (bool) + +When `false`, replaces non-ASCII characters in output filenames with ASCII equivalents. Default: `false`. + +--- + +## ipinfo_api_key (str) + +Optional ipinfo.io token. When set, envied uses the ipinfo.io Lite endpoint for IP/geolocation lookups instead of the unauthenticated fallback. + +--- + +## tmdb_api_key (str) + +Optional TMDB API key, used for metadata enrichment and IMDb/TMDb tagging. + +--- + +## simkl_client_id (str) + +Optional Simkl client ID for metadata lookups. + +--- + +## decrypt_labs_api_key (str) + +Optional Decrypt Labs API key, used by services that integrate with the service. + +--- diff --git a/reset/docs/API.md b/reset/docs/API.md new file mode 100644 index 0000000..73f9091 --- /dev/null +++ b/reset/docs/API.md @@ -0,0 +1,781 @@ +# REST API Documentation + +The envied REST API allows you to control downloads, search services, drive remote downloads from a thin client, and (optionally) co-host the pywidevine/pyplayready CDM. Start the server with `envied serve` and access the interactive Swagger UI at `http://localhost:8786/api/docs/`. + +The server is built on **aiohttp** (not FastAPI). Implementation lives in `envied/commands/serve.py` and `envied/core/api/` (`routes.py`, `handlers.py`, `session_store.py`, `input_bridge.py`, `download_manager.py`, `download_worker.py`). + +## Quick Start + +```bash +# Start the server (no authentication) +envied serve --no-key + +# Start with authentication (api_secret in envied.yaml) +envied serve + +# Serve only the REST API (no pywidevine/pyplayready CDM) +envied serve --api-only + +# Serve only the remote-dl session endpoints (CORS/Cloudflare friendly) +envied serve --remote-only + +# Disable just one CDM +envied serve --no-widevine +envied serve --no-playready + +# Verbose error responses (tracebacks/stderr in JSON) +envied serve --debug-api +``` + +`serve` flags: + +| Flag | Description | +| --- | --- | +| `-h, --host` | Bind host (default `127.0.0.1`) | +| `-p, --port` | Bind port (default `8786`) | +| `--caddy` | Also launch Caddy using `Caddyfile` next to the envied config | +| `--api-only` | REST API only; skip the bundled pywidevine/pyplayready CDM endpoints | +| `--no-widevine` | Disable Widevine CDM endpoints | +| `--no-playready` | Disable PlayReady CDM endpoints | +| `--no-key` | Disable API key authentication entirely | +| `--debug-api` | Include tracebacks/stderr in error responses | +| `--debug` | Enable DEBUG-level logging for API operations | +| `--remote-only` | Expose only `/api/health`, `/api/services`, `/api/search`, and `/api/session/*` (implies `--api-only`) | + +## Authentication + +When `api_secret` is set in `envied.yaml`, all API requests require the **`X-Secret-Key`** header. There is no query-parameter fallback. `/api/health` is always reachable without authentication. `--no-key` disables auth entirely (not recommended for public-facing servers). + +```yaml +# envied.yaml +serve: + api_secret: "your-master-secret" # falls back to global users map below + remote_only: false # also toggleable via --remote-only + services: ["EXAMPLE1", "EXAMPLE2"] # optional global service allowlist + users: + user-secret-1: + username: alice + devices: ["my_widevine_l3"] # Widevine WVD names this user may use + playready_devices: ["my_pr_sl2000"] # PlayReady PRD names; defaults to [] (no access) + services: ["EXAMPLE1"] # optional per-user allowlist (intersected with global) + user-secret-2: + username: bob + devices: [] + playready_devices: [] +``` + +### Service allowlists + +`config.serve.services` is the global allowlist; `users..services` further narrows it per key. The effective set is the intersection. Endpoints affected: `/api/services`, `/api/search`, `/api/list-titles`, `/api/list-tracks`, `/api/download`, and all `/api/session/*` routes. + +### CDM access (server-side decryption) + +There is no separate "tier" flag. Whether the server can return KID:KEY for a session-mode download depends solely on the device lists configured for the calling user key: + +- Empty `devices` and `playready_devices` -> server can only proxy CDM challenges; the client must run its own CDM and parse the license. +- Populated lists -> the client may set `mode: "server_cdm"` on `/api/session/{id}/license` and receive `{ "keys": { "": { "": "" } } }` instead of raw license bytes. + +Per-service CDM type can be pinned via `config.cdm` (`widevine`/`playready`) or per-service `cdm_type`; otherwise the server picks the type the user has devices for. + +### Server-side `dl` defaults + +Any flag accepted by `/api/download` (see the table below) can be declared under `serve:` in `envied.yaml` and the API will apply it as a default. Request-body values still win. Useful for raising concurrency without changing every client call: + +```yaml +serve: + api_secret: "..." + users: { ... } + downloads: 4 # parallel tracks per download job + workers: 16 # threads per track segment fetch + best_available: true + no_proxy_download: false +``` + +Layering order: built-in defaults < `serve.*` overrides < service-specific click defaults < request body. + +--- + +## Endpoint Map + +Standard endpoints (suppressed in `--remote-only` mode are marked R): + +| Method | Path | R | +| --- | --- | :-: | +| GET | `/api/health` | ok | +| GET | `/api/services` | ok | +| POST | `/api/search` | ok | +| POST | `/api/list-titles` | hidden | +| POST | `/api/list-tracks` | hidden | +| POST | `/api/download` | hidden | +| GET | `/api/download/jobs` | hidden | +| GET | `/api/download/jobs/{job_id}` | hidden | +| DELETE | `/api/download/jobs/{job_id}` | hidden | +| POST | `/api/session/create` | ok | +| GET | `/api/session/{session_id}` | ok | +| DELETE | `/api/session/{session_id}` | ok | +| GET | `/api/session/{session_id}/titles` | ok | +| POST | `/api/session/{session_id}/tracks` | ok | +| POST | `/api/session/{session_id}/segments` | ok | +| POST | `/api/session/{session_id}/license` | ok | +| GET | `/api/session/{session_id}/prompt` | ok | +| POST | `/api/session/{session_id}/prompt` | ok | + +CDM endpoints (`/{wvd}/...`, `/playready/{prd}/...`) are exposed unless `--api-only` / `--remote-only` / `--no-widevine` / `--no-playready` is set, and use pywidevine / pyplayready's own auth scheme. + +--- + +## Endpoints + +### GET /api/health + +Health check with version and update information. Always reachable without auth. + +```bash +curl http://localhost:8786/api/health +``` + +```json +{ + "status": "ok", + "version": "4.0.0", + "update_check": { + "update_available": false, + "current_version": "4.0.0", + "latest_version": null + } +} +``` + +--- + +### GET /api/services + +List all available streaming services (filtered by the effective allowlist for the caller). + +```bash +curl -H "X-Secret-Key: $KEY" http://localhost:8786/api/services +``` + +Returns `{"services": [...]}`. Each entry has `tag`, `aliases`, `geofence`, `title_regex`, `url` (from `cli.short_help`), `help` (full docstring), and `cli_params` describing the service-level Click parameters. + +--- + +### POST /api/search + +Search for titles from a streaming service. + +**Required parameters:** +| Parameter | Type | Description | +| --- | --- | --- | +| `service` | string | Service tag | +| `query` | string | Search query | + +**Optional parameters:** +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `profile` | string | `null` | Profile for credentials/cookies | +| `proxy` | string | `null` | Proxy URI or country code | +| `no_proxy` | boolean | `false` | Disable all proxy use | + +```bash +curl -X POST http://localhost:8786/api/search \ + -H "X-Secret-Key: $KEY" \ + -H "Content-Type: application/json" \ + -d '{"service": "EXAMPLE1", "query": "example show"}' +``` + +```json +{ + "results": [ + { + "id": "abc123def456", + "title": "Example Show", + "description": null, + "label": "TV Show", + "url": "https://example.com/show/abc123def456" + } + ], + "count": 1 +} +``` + +--- + +### POST /api/list-titles + +Get available titles (seasons/episodes/movies) for a service and title ID. Disabled in `--remote-only` mode. + +**Required parameters:** +| Parameter | Type | Description | +| --- | --- | --- | +| `service` | string | Service tag | +| `title_id` | string | Title ID or URL | + +```bash +curl -X POST http://localhost:8786/api/list-titles \ + -H "X-Secret-Key: $KEY" \ + -H "Content-Type: application/json" \ + -d '{"service": "EXAMPLE1", "title_id": "abc123def456"}' +``` + +--- + +### POST /api/list-tracks + +Get video, audio, and subtitle tracks for a title. Disabled in `--remote-only` mode. + +**Required parameters:** +| Parameter | Type | Description | +| --- | --- | --- | +| `service` | string | Service tag | +| `title_id` | string | Title ID or URL | + +**Optional parameters:** +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `wanted` | array | all | Episode filter (e.g., `["S01E01"]`) | +| `profile` | string | `null` | Profile for credentials/cookies | +| `proxy` | string | `null` | Proxy URI or country code | +| `no_proxy` | boolean | `false` | Disable all proxy use | + +Returns video, audio, and subtitle tracks with codec, bitrate, resolution, language, and DRM information. + +--- + +### POST /api/download + +Start a download job. Returns immediately with a job ID (HTTP 202). Disabled in `--remote-only` mode. + +**Required parameters:** +| Parameter | Type | Description | +| --- | --- | --- | +| `service` | string | Service tag | +| `title_id` | string | Title ID or URL | + +**Quality and codec parameters:** +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `quality` | array[int] | best | Resolution(s) (e.g., `[1080, 2160]`) | +| `vcodec` | string or array | any | Video codec(s): `H264`, `H265`/`HEVC`, `VP9`, `AV1`, `VC1`, `VP8` | +| `acodec` | string or array | any | Audio codec(s): `AAC`, `AC3`, `EC3`, `AC4`, `OPUS`, `FLAC`, `ALAC`, `DTS`, `OGG` | +| `vbitrate` | int | highest | Video bitrate in kbps | +| `abitrate` | int | highest | Audio bitrate in kbps | +| `range` | array[string] | `["SDR"]` | Color range(s): `SDR`, `HDR10`, `HDR10+`, `HLG`, `DV`, `HYBRID` | +| `channels` | float | any | Audio channels (e.g., `5.1`, `7.1`) | +| `no_atmos` | boolean | `false` | Exclude Dolby Atmos tracks | +| `split_audio` | boolean | `null` | Create separate output per audio codec | +| `sub_format` | string | `null` | Output subtitle format: `SRT`, `VTT`, `ASS`, `SSA`, `TTML` | + +**Episode selection:** +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `wanted` | array[string] | all | Episodes (e.g., `["S01E01", "S01E02-S01E05"]`) | +| `latest_episode` | boolean | `false` | Download only the most recent episode | + +**Language parameters:** +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `lang` | array[string] | `["orig"]` | Language for video and audio (`orig` = original) | +| `v_lang` | array[string] | `[]` | Language override for video tracks only | +| `a_lang` | array[string] | `[]` | Language override for audio tracks only | +| `s_lang` | array[string] | `["all"]` | Language for subtitles | +| `require_subs` | array[string] | `[]` | Required subtitle languages (skip if missing) | +| `forced_subs` | boolean | `false` | Include forced subtitle tracks | +| `exact_lang` | boolean | `false` | Exact language matching (no variants) | + +**Track selection:** +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `video_only` | boolean | `false` | Only download video tracks | +| `audio_only` | boolean | `false` | Only download audio tracks | +| `subs_only` | boolean | `false` | Only download subtitle tracks | +| `chapters_only` | boolean | `false` | Only download chapters | +| `no_video` | boolean | `false` | Skip video tracks | +| `no_audio` | boolean | `false` | Skip audio tracks | +| `no_subs` | boolean | `false` | Skip subtitle tracks | +| `no_chapters` | boolean | `false` | Skip chapters | +| `audio_description` | boolean | `false` | Include audio description tracks | + +**Output and tagging:** +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `tag` | string | `null` | Override group tag | +| `repack` | boolean | `false` | Add REPACK tag to filename | +| `tmdb_id` | int | `null` | Use specific TMDB ID for tagging | +| `imdb_id` | string | `null` | Use specific IMDB ID (e.g., `tt1375666`) | +| `animeapi_id` | string | `null` | Anime database ID via AnimeAPI (e.g., `mal:12345`) | +| `enrich` | boolean | `false` | Override show title and year from external source | +| `no_folder` | boolean | `false` | Disable folder creation for TV shows | +| `no_source` | boolean | `false` | Remove source tag from filename | +| `no_mux` | boolean | `false` | Do not mux tracks into container | +| `output_dir` | string | `null` | Override output directory | + +**Download behavior:** +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `profile` | string | `null` | Profile for credentials/cookies | +| `proxy` | string | `null` | Proxy URI or country code | +| `no_proxy` | boolean | `false` | Disable all proxy use | +| `no_proxy_download` | boolean | `false` | Bypass proxy for segment downloads only. Manifest, license, and auth still use proxy | +| `workers` | int | `null` | Max threads per track download | +| `downloads` | int | `1` | Concurrent track downloads | +| `slow` | boolean or string | `null` | Add randomized delay between titles. `true` = 60-120s, or `"MIN-MAX"` string (e.g., `"20-40"`). Min must be >= 20 | +| `best_available` | boolean | `false` | Continue if requested quality unavailable | +| `worst` | boolean | `false` | Select the lowest bitrate track within the specified quality. Requires `quality` | +| `skip_dl` | boolean | `false` | Skip download, only get decryption keys | +| `export` | boolean | `false` | Export manifest, track URLs, keys, and subtitles to JSON in the exports directory | +| `cdm_only` | boolean | `null` | Only use CDM (`true`) or only vaults (`false`) | +| `no_cache` | boolean | `false` | Bypass title cache | +| `reset_cache` | boolean | `false` | Clear title cache before fetching | + +**Example:** + +```bash +curl -X POST http://localhost:8786/api/download \ + -H "X-Secret-Key: $KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "service": "EXAMPLE1", + "title_id": "abc123def456", + "wanted": ["S01E01"], + "quality": [1080, 2160], + "vcodec": ["H265"], + "acodec": ["AAC", "EC3"], + "range": ["HDR10", "SDR"], + "split_audio": true, + "lang": ["en"] + }' +``` + +```json +{ + "job_id": "504db959-80b0-446c-a764-7924b761d613", + "status": "queued", + "created_time": "2026-02-27T18:00:00.000000" +} +``` + +--- + +### GET /api/download/jobs + +List all download jobs with optional filtering and sorting. Disabled in `--remote-only` mode. + +**Query parameters:** +| Parameter | Type | Default | Description | +| --- | --- | --- | --- | +| `status` | string | all | Filter by status: `queued`, `downloading`, `completed`, `failed`, `cancelled` | +| `service` | string | all | Filter by service tag | +| `sort_by` | string | `created_time` | Sort field: `created_time`, `started_time`, `completed_time`, `progress`, `status`, `service` | +| `sort_order` | string | `desc` | Sort order: `asc`, `desc` | + +```bash +curl -H "X-Secret-Key: $KEY" "http://localhost:8786/api/download/jobs?status=completed" +``` + +--- + +### GET /api/download/jobs/{job_id} + +Get detailed information about a specific download job including progress, parameters, and error details. + +```json +{ + "job_id": "504db959-80b0-446c-a764-7924b761d613", + "status": "completed", + "created_time": "2026-02-27T18:00:00.000000", + "service": "EXAMPLE1", + "title_id": "abc123def456", + "progress": 100.0, + "parameters": { }, + "started_time": "2026-02-27T18:00:01.000000", + "completed_time": "2026-02-27T18:00:15.000000", + "output_files": [], + "error_message": null, + "error_details": null +} +``` + +--- + +### DELETE /api/download/jobs/{job_id} + +Cancel a queued or running download job. Returns 400 if the job has already terminated. + +--- + +## Remote Service Sessions + +These endpoints back the `RemoteService` adapter in `envied/core/remote_service.py`. They let a thin `dl` client (or any consumer) authenticate against a service on the server, fetch titles/tracks/manifests, and either proxy CDM challenges or have the server resolve KID:KEY directly. The `dl` command's `RemoteService` adapter replaces the old `remote_dl` command. These endpoints are the only `/api/*` routes available in `--remote-only` mode (in addition to `health`, `services`, and `search`). + +### POST /api/session/create + +Authenticate against a service and open a session. Body fields: + +| Field | Type | Description | +| --- | --- | --- | +| `service` | string | Service tag (required) | +| `title_id` | string | Title ID/URL (required) | +| `credentials` | object | Auth credentials forwarded to `Service.authenticate` | +| `cookies` | string | Cookie blob (Netscape or JSON) | +| `proxy` | string | Proxy URI or country code | +| `no_proxy` | bool | Force-disable proxies | +| `profile` | string | Profile name | +| `cache` | object | Optional pre-warmed title cache payload | + +If the service requires interactive input during authentication, poll `GET /api/session/{id}/prompt` and submit responses via `POST /api/session/{id}/prompt` until status is `authenticated`. + +**Request:** + +```json +{ + "service": "EXAMPLE1", + "title_id": "abc123def456", + "credentials": {"username": "alice", "password": "hunter2"}, + "cookies": "# Netscape HTTP Cookie File\n...", + "proxy": "us", + "no_proxy": false, + "profile": "default", + "cache": {} +} +``` + +**Response (202-style; auth runs asynchronously):** + +```json +{ + "session_id": "f1c4a8b2-9c7e-4d2a-bf91-2d3e4f5a6b7c", + "service": "EXAMPLE1", + "status": "authenticating" +} +``` + +### GET /api/session/{session_id} + +Returns session metadata. 404 if expired or unknown. + +```json +{ + "session_id": "f1c4a8b2-9c7e-4d2a-bf91-2d3e4f5a6b7c", + "service": "EXAMPLE1", + "valid": true, + "expires_in": 3600, + "track_count": 0, + "title_count": 0 +} +``` + +### DELETE /api/session/{session_id} + +Tears down the session, cancels any pending prompts, and returns any updated per-session cache files (base64-encoded, zlib-compressed) so the client can re-warm next time. + +```json +{ + "status": "ok", + "cache": { + "tokens": "eJzLSM3JyVcozy/KSVGo5AIAGgQEvQ==" + } +} +``` + +### GET /api/session/{session_id}/titles + +Returns the resolved titles list. + +```json +{ + "session_id": "f1c4a8b2-9c7e-4d2a-bf91-2d3e4f5a6b7c", + "titles": [ + { + "type": "episode", + "name": "Pilot", + "series_title": "Example Show", + "season": 1, + "number": 1, + "year": 2024, + "id": "ep-0001", + "language": "en" + }, + { + "type": "movie", + "name": "Example Movie", + "year": 2024, + "id": "mov-0001", + "language": "en" + } + ] +} +``` + +### POST /api/session/{session_id}/tracks + +**Request:** + +```json +{"title_id": "ep-0001"} +``` + +**Response:** + +```json +{ + "title": { + "type": "episode", + "name": "Pilot", + "series_title": "Example Show", + "season": 1, + "number": 1, + "year": 2024, + "id": "ep-0001", + "language": "en" + }, + "video": [ + { + "id": "v-1080p-h264", + "codec": "H264", + "codec_display": "H.264", + "bitrate": 6000, + "width": 1920, + "height": 1080, + "resolution": "1920x1080", + "fps": "23.976", + "range": "SDR", + "range_display": "SDR", + "language": "en", + "drm": [ + { + "type": "widevine", + "pssh": "AAAAW3Bzc2gAAAAA7e+...", + "kids": ["abcdef0123456789abcdef0123456789"], + "license_url": "https://license.example.com/widevine" + } + ], + "descriptor": "DASH", + "url": "https://cdn.example.com/manifest.mpd" + } + ], + "audio": [ + { + "id": "a-en-eac3", + "codec": "EC3", + "codec_display": "Dolby Digital Plus", + "bitrate": 640, + "channels": "5.1", + "language": "en", + "atmos": false, + "descriptive": false, + "drm": null, + "descriptor": "DASH", + "url": "https://cdn.example.com/manifest.mpd" + } + ], + "subtitles": [ + { + "id": "s-en-vtt", + "codec": "WebVTT", + "language": "en", + "forced": false, + "sdh": false, + "cc": false, + "descriptor": "DASH", + "url": "https://cdn.example.com/subs/en.vtt" + } + ], + "chapters": [ + {"timestamp": "00:00:00.000", "name": "Chapter 1"} + ], + "attachments": [], + "manifests": [ + { + "type": "dash", + "url": "https://cdn.example.com/manifest.mpd", + "data": "eJzNVk1v2zAM/Ss..." + } + ], + "session_headers": { + "User-Agent": "Mozilla/5.0 ..." + }, + "session_cookies": { + "session": "abc123" + }, + "server_cdm_type": "widevine" +} +``` + +### POST /api/session/{session_id}/segments + +**Request:** + +```json +{"track_ids": ["v-1080p-h264", "a-en-eac3"]} +``` + +**Response:** + +```json +{ + "tracks": { + "v-1080p-h264": { + "descriptor": "DASH", + "url": "https://cdn.example.com/manifest.mpd", + "drm": [ + { + "type": "widevine", + "pssh": "AAAAW3Bzc2gAAAAA7e+...", + "kids": ["abcdef0123456789abcdef0123456789"], + "license_url": "https://license.example.com/widevine" + } + ], + "headers": {"User-Agent": "Mozilla/5.0 ..."}, + "cookies": {"session": "abc123"}, + "data": {} + }, + "a-en-eac3": { + "descriptor": "DASH", + "url": "https://cdn.example.com/manifest.mpd", + "drm": null, + "headers": {"User-Agent": "Mozilla/5.0 ..."}, + "cookies": {"session": "abc123"}, + "data": {} + } + } +} +``` + +### POST /api/session/{session_id}/license + +Two modes, selected by the `mode` field. + +**`mode: "proxy"` (default)** -- forward a client-built CDM challenge to the service's license endpoint. + +Request: + +```json +{ + "mode": "proxy", + "track_id": "v-1080p-h264", + "challenge": "CAESxQEK...", + "drm_type": "widevine", + "pssh": "AAAAW3Bzc2gAAAAA7e+..." +} +``` + +Response: + +```json +{"license": "CAIS3wIK..."} +``` + +**`mode: "server_cdm"`** -- the server uses its own CDM to license the track and extract keys. Single-track form takes `track_id`; batch form takes `track_ids`. Requires the calling user key to have a matching device (`devices` for Widevine, `playready_devices` for PlayReady) in `envied.yaml`. + +Request (batch): + +```json +{ + "mode": "server_cdm", + "track_ids": ["v-1080p-h264", "a-en-eac3"], + "drm_type": "widevine" +} +``` + +Response: + +```json +{ + "keys": { + "v-1080p-h264": { + "abcdef0123456789abcdef0123456789": "00112233445566778899aabbccddeeff" + }, + "a-en-eac3": { + "abcdef0123456789abcdef0123456789": "00112233445566778899aabbccddeeff" + } + }, + "drm_type": "widevine" +} +``` + +### GET /api/session/{session_id}/prompt + +Polled by the client during interactive authentication (OTP, PIN, device codes). Backed by the `InputBridge` in `envied/core/api/input_bridge.py`; `Service.request_input()` blocks server-side until the client posts a response. + +Pending input: + +```json +{"status": "pending_input", "prompt": "Enter OTP code: "} +``` + +Other states: + +```json +{"status": "authenticating"} +``` + +```json +{"status": "authenticated"} +``` + +```json +{"status": "failed", "error": "Invalid credentials"} +``` + +### POST /api/session/{session_id}/prompt + +Unblocks the server-side `request_input()` call. + +Request: + +```json +{"response": "123456"} +``` + +Response: + +```json +{"status": "accepted"} +``` + +--- + +## Error Responses + +All endpoints return consistent error responses: + +```json +{ + "status": "error", + "error_code": "INVALID_PARAMETERS", + "message": "Invalid vcodec: XYZ. Must be one of: H264, H265, VP9, AV1, VC1, VP8", + "timestamp": "2026-02-27T18:00:00.000000+00:00", + "details": { } +} +``` + +Common error codes: + +- `INVALID_INPUT` -- malformed request body +- `INVALID_PARAMETERS` -- invalid parameter values +- `MISSING_SERVICE` -- service tag not provided +- `INVALID_SERVICE` -- service not found or not in the caller's allowlist +- `SERVICE_ERROR` -- service initialization or runtime error +- `AUTH_FAILED` -- authentication failure +- `NOT_FOUND` / `TRACK_NOT_FOUND` / session not found -- job/session/track/title missing +- `INTERNAL_ERROR` -- unexpected server error + +When `--debug-api` is enabled, error responses include additional `debug_info` with tracebacks and stderr output. + +Authentication errors from the auth middleware are returned as `{"status": 401, "message": "..."}` (not the standard error envelope). + +--- + +## Download Job Lifecycle + +``` +queued -> downloading -> completed + \-> failed +queued -> cancelled +downloading -> cancelled +``` + +Jobs are retained for 24 hours after completion (override via top-level `download_job_retention_hours` in `envied.yaml`). The server runs up to 2 concurrent download jobs by default; override via top-level `max_concurrent_downloads`. This is independent of `serve.downloads`, which controls parallel tracks **within** a single job. + +Remote sessions are managed by `SessionStore` (`envied/core/api/session_store.py`); idle sessions and their `InputBridge` instances are cleaned up by a background loop started/stopped with the app lifecycle. diff --git a/reset/docs/DOWNLOAD_CONFIG.md b/reset/docs/DOWNLOAD_CONFIG.md new file mode 100644 index 0000000..24a842f --- /dev/null +++ b/reset/docs/DOWNLOAD_CONFIG.md @@ -0,0 +1,322 @@ +# Download & Processing Configuration + +This document covers configuration options related to downloading and processing media content. + +## downloader + +envied ships a single unified downloader at `envied/core/downloaders/requests.py`. The legacy +`aria2c`, `curl_impersonate`, and `n_m3u8dl_re` backends have been removed; their config blocks no +longer have any effect. + +The unified downloader: + +- Works with both a standard `requests.Session` and `RnetSession` (rnet/BoringSSL TLS impersonation, + which replaces the previous `curl_cffi` backend). When a service exposes its own session via + `self.session`, TLS fingerprinting is preserved on every segment. +- Uses adaptive chunk sizing between **512 KB and 4 MB**, picked from the response `Content-Length`. +- Spawns **up to `min(16, cpu_count + 4)` worker threads** by default for segmented downloads + (override via `--workers` / `dl.workers`). +- Resumes interrupted downloads via HTTP `Range` requests (a sibling `.!dev` control file + marks an in-progress download). +- Has a single-URL fast path: if the server supports byte ranges and the file is at least 64 MB, + the file is split into 16 MB parts and downloaded in parallel into a pre-allocated file. +- Is selected per-track via `track.downloader`, which defaults to this unified `requests` downloader. + +There is no `downloader:` config key to set anymore. Setting one to a legacy value will emit a +`DeprecationWarning` and otherwise be ignored. + +--- + +## dl (dict) + +Pre-define default options and switches of the `dl` command. +The values will be ignored if explicitly set in the CLI call. + +The Key must be the same value Python click would resolve it to as an argument. +E.g., `@click.option("-r", "--range", "range_", type=...` actually resolves as `range_` variable. + +For example to set the default primary language to download to German, + +```yaml +lang: de +``` + +You can also set multiple preferred languages using a list, e.g., + +```yaml +lang: + - en + - fr +``` + +to set how many tracks to download concurrently to 4 and download threads to 16, + +```yaml +downloads: 4 +workers: 16 +``` + +to set `--bitrate=CVBR` for a specific service, + +```yaml +lang: de +EXAMPLE: + bitrate: CVBR +``` + +or to change the output subtitle format from the default (original format) to WebVTT, + +```yaml +sub_format: vtt +``` + +### All Available `dl` Keys + +Below is a comprehensive list of keys that can be pre-defined in the `dl` section. Each corresponds +to a CLI option on the `dl` command. CLI arguments always take priority over config values. + +**Quality and codec:** + +| Key | Type | Default | Description | +| --- | --- | --- | --- | +| `quality` | int or list | best | Resolution(s) to download (e.g., `1080`, `[1080, 2160]`) | +| `vcodec` | str or list | any | Video codec(s): `H264`, `H265`, `VP9`, `AV1`, `VC1` | +| `acodec` | str or list | any | Audio codec(s): `AAC`, `AC3`, `EC3`, `AC4`, `OPUS`, `FLAC`, `ALAC`, `DTS` | +| `vbitrate` | int | highest | Video bitrate in kbps | +| `abitrate` | int | highest | Audio bitrate in kbps | +| `vbitrate_range` | str | none | Video bitrate window in kbps, format `MIN-MAX` (e.g., `6000-7000`) | +| `abitrate_range` | str | none | Audio bitrate window in kbps, format `MIN-MAX` | +| `real_video_bitrate` | bool | `false` | Probe actual media size to compute true video bitrates, overriding the manifest's declared value (`-rvb`). See [Real bitrate probing](#real-bitrate-probing) | +| `real_audio_bitrate` | bool | `false` | Same as above for audio tracks (`-rab`). Slower than video (more renditions) | +| `range_` | str or list | `SDR` | Color range(s): `SDR`, `HDR10`, `HDR10+`, `HLG`, `DV`, `HYBRID` | +| `channels` | float | any | Audio channels (e.g., `5.1`, `7.1`) | +| `worst` | bool | `false` | Select the lowest bitrate track within the specified quality. Requires `quality` | +| `best_available` | bool | `false` | Continue if requested quality is unavailable | + +**Language:** + +| Key | Type | Default | Description | +| --- | --- | --- | --- | +| `lang` | str or list | `orig` | Language for video and audio (`orig` = original language) | +| `v_lang` | list | `[]` | Language override for video tracks only | +| `a_lang` | list | `[]` | Language override for audio tracks only | +| `s_lang` | list | `["all"]` | Language for subtitles | +| `require_subs` | list | `[]` | Required subtitle languages (skip title if missing) | +| `forced_subs` | bool | `false` | Include forced subtitle tracks | +| `exact_lang` | bool | `false` | Exact language matching (no regional variants) | +| `latest_episode` | bool | `false` | Download only the single most recent episode of a series | + +**Track selection:** + +| Key | Type | Default | Description | +| --- | --- | --- | --- | +| `video_only` | bool | `false` | Only download video tracks | +| `audio_only` | bool | `false` | Only download audio tracks | +| `subs_only` | bool | `false` | Only download subtitle tracks | +| `chapters_only` | bool | `false` | Only download chapters | +| `no_video` | bool | `false` | Skip video tracks | +| `no_audio` | bool | `false` | Skip audio tracks | +| `no_subs` | bool | `false` | Skip subtitle tracks | +| `no_chapters` | bool | `false` | Skip chapters | +| `no_atmos` | bool | `false` | Exclude Dolby Atmos audio tracks | +| `audio_description` | bool | `false` | Include audio description tracks | + +**Output and tagging:** + +| Key | Type | Default | Description | +| --- | --- | --- | --- | +| `tag` | str | config default | Override group tag | +| `repack` | bool | `false` | Add REPACK tag to output filename | +| `sub_format` | str | original | Output subtitle format: `srt`, `vtt`, `ass`, `ssa`, `ttml` | +| `no_folder` | bool | `false` | Disable folder creation for TV shows | +| `no_source` | bool | `false` | Remove source tag from filename | +| `no_mux` | bool | `false` | Do not mux tracks into a container file | +| `split_audio` | bool | `false` | Create separate output files per audio codec | +| `export` | bool | `false` | Write a JSON sidecar with manifest URLs, subtitles, per-track KID:KEY, codec/track info | + +**Metadata enrichment:** + +| Key | Type | Default | Description | +| --- | --- | --- | --- | +| `tmdb_id` | int | `null` | Use specific TMDB ID for tagging | +| `imdb_id` | str | `null` | Use specific IMDB ID (e.g., `tt1375666`) | +| `animeapi_id` | str | `null` | Anime database ID via AnimeAPI (e.g., `mal:12345`, `anilist:98765`) | +| `enrich` | bool | `false` | Override show title and year from external source. Requires `tmdb_id`, `imdb_id`, or `animeapi_id` | + +**Download behavior:** + +| Key | Type | Default | Description | +| --- | --- | --- | --- | +| `downloads` | int | `1` | Concurrent track downloads | +| `workers` | int | `min(16, cpu_count + 4)` | Max threads per track download (segments / ranged parts) | +| `slow` | bool or `MIN-MAX` | `false` | Randomized delay between titles. `true` uses 60-120s; pass `MIN-MAX` (e.g., `20-40`) for a custom range | +| `no_proxy_download` | bool | `false` | Bypass proxy for segment downloads only. Manifest, license, and auth still use proxy | +| `skip_dl` | bool | `false` | Skip download, only get decryption keys | +| `cdm_only` | bool | `null` | Only use CDM (`true`) or only vaults (`false`) | + +### Real bitrate probing + +Some services declare inaccurate `bandwidth`/`BANDWIDTH` in their manifests — often +a peak or nominal figure that is far from the real average. Because `track.bitrate` +drives the track listing, sorting, and `--vbitrate` / `--vbitrate-range` selection, +a wrong value picks the wrong track. + +`-rvb` / `--real-video-bitrate` (and `-rab` / `--real-audio-bitrate` for audio) +probe the actual media size and overwrite `track.bitrate` with the measured value +(`bytes * 8 / duration`) before listing and selection. So `-rvb --list` shows the +true numbers, and `-rvb --vbitrate-range 6000-7000` selects against them. Without +the flag, behaviour is unchanged (the manifest value is used). + +How it works: + +- **Single-file tracks** (one whole file per rendition — e.g. DASH `SegmentBase` + or services that collapse to a `BaseURL`) are measured **exactly**: the whole + file size over the track duration. +- **Multi-segment tracks** (most HLS) are a **sampled estimate** — a spread of + segments is probed and extrapolated, typically within a few percent. Segment + bytes include container overhead, so MPEG-TS HLS reads a few percent above the + demuxed stream (this is the real *delivered* size). +- Only the top renditions per quality tier are probed (video grouped by + codec + range, audio by codec + channels + language), in parallel, then extended + downward only as far as needed to keep ranking correct. This keeps the pass fast + even when a service exposes dozens of renditions. +- Tracks whose duration cannot be determined fall back to `ffprobe`; probe failures + are non-fatal and leave the manifest bitrate in place. + +Per-track before→after values are logged at debug level (run with `-d`); the +corrected values always appear in the Available Tracks panel. + +You can also set per-service `dl` overrides (see [Service Integration & Authentication Configuration](SERVICE_CONFIG.md)): + +```yaml +dl: + lang: en + downloads: 4 + workers: 16 + EXAMPLE: + bitrate: CVBR + EXAMPLE2: + worst: true + quality: 1080 +``` + +--- + +## audio (dict) + +Configuration for audio track selection. + +- `codec_priority` + Optional list of audio codec names defining the preferred order when multiple audio + tracks share the same bitrate and language. Listed codecs are ranked in the order given. + Codecs not in the list retain their bitrate-based ordering and are placed after all + listed codecs (i.e. soft priority — nothing is dropped). + + Atmos tracks still take precedence over codec priority, and audio description tracks + are still moved to the end. + + Valid codec names: `AAC`, `AC3`, `EC3`, `AC4`, `OPUS`, `OGG`, `DTS`, `ALAC`, `FLAC`. + +For example, + +```yaml +audio: + codec_priority: [FLAC, ALAC, AC4, EC3, DTS, AC3, OPUS, AAC, OGG] +``` + +Or to only prefer a subset (e.g. surround codecs first, everything else falls back to +bitrate order): + +```yaml +audio: + codec_priority: [EC3, DTS, AC3, AAC] +``` + +When unset, audio tracks are sorted by bitrate alone (with Atmos/descriptive rules still +applied). + +--- + +## subtitle (dict) + +Configuration for subtitle processing and conversion. + +- `conversion_method` + Method to use for converting subtitles between formats. Default: `"auto"` + - `"auto"` — Smart routing: uses subby for WebVTT/SAMI, pycaption for others. + - `"subby"` — Always use subby with advanced processing. + - `"pycaption"` — Use only pycaption library (no SubtitleEdit, no subby). + - `"subtitleedit"` — Prefer SubtitleEdit when available, fall back to pycaption. + - `"pysubs2"` — Use pysubs2 library (supports SRT/SSA/ASS/WebVTT/TTML/SAMI/MicroDVD/MPL2/TMP). +- `sdh_method` + Method to use for SDH (hearing impaired) stripping. Default: `"auto"` + - `"auto"` — Try subby (SRT only), then SubtitleEdit (if available), then subtitle-filter. + - `"subby"` — Use subby library (SRT only). + - `"subtitleedit"` — Use SubtitleEdit tool (Windows only, falls back to subtitle-filter). + - `"filter-subs"` — Use subtitle-filter library directly. +- `strip_sdh` + Automatically create stripped (non-SDH) versions of SDH subtitles. Default: `true` +- `convert_before_strip` + Auto-convert VTT/other formats to SRT before using subtitle-filter for SDH stripping. + Ensures compatibility when subtitle-filter is used as fallback. Default: `true` +- `preserve_formatting` + Preserve original subtitle formatting (tags, positioning, styling). + When `true`, skips pycaption processing for WebVTT files to keep tags like ``, ``, + positioning intact. Combined with no `sub_format` setting, ensures subtitles remain in + their original format. Default: `true` +- `output_mode` + Output mode for subtitles. Default: `"mux"` + - `"mux"` — Embed subtitles in MKV container only. + - `"sidecar"` — Save subtitles as separate files only. + - `"both"` — Embed in MKV and save as sidecar files. +- `sidecar_format` + Format for sidecar subtitle files when `output_mode` is `"sidecar"` or `"both"`. Default: `"srt"` + Options: `srt`, `vtt`, `ass`, `original` (keep current format). + +For example, + +```yaml +subtitle: + conversion_method: auto + sdh_method: auto + strip_sdh: true + convert_before_strip: true + preserve_formatting: true + output_mode: mux + sidecar_format: srt +``` + +--- + +## decryption (str | dict) + +Choose what software to use to decrypt DRM-protected content throughout envied where needed. +You may provide a single decryption method globally or a mapping of service tags to +decryption methods. + +Options: + +- `shaka` (default) - Shaka Packager - +- `mp4decrypt` - mp4decrypt from Bento4 - + +Note that Shaka Packager is the traditional method and works with most services. mp4decrypt +is an alternative that may work better with certain services that have specific encryption formats. + +Example mapping: + +```yaml +decryption: + EXAMPLE: mp4decrypt + EXAMPLE2: shaka + default: shaka +``` + +The `default` entry is optional. If omitted, `shaka` will be used for services not listed. + +Simple configuration (single method for all services): + +```yaml +decryption: mp4decrypt +``` + +--- diff --git a/reset/docs/DRM_CONFIG.md b/reset/docs/DRM_CONFIG.md new file mode 100644 index 0000000..acfb3cd --- /dev/null +++ b/reset/docs/DRM_CONFIG.md @@ -0,0 +1,481 @@ +# DRM & CDM Configuration + +This document covers Digital Rights Management (DRM) and Content Decryption Module (CDM) configuration options. + +## cdm (dict) + +Pre-define which Widevine or PlayReady device to use for each Service by Service Tag as Key (case-sensitive). +The value should be a WVD or PRD filename without the file extension. When +loading the device, envied will look in both the `WVDs` and `PRDs` directories +for a matching file. + +For example, + +```yaml +EXAMPLE: chromecdm_903_l3 +EXAMPLE2: nexus_6_l1 +``` + +You may also specify this device based on the profile used. + +For example, + +```yaml +EXAMPLE: chromecdm_903_l3 +EXAMPLE2: nexus_6_l1 +EXAMPLE3: + john_sd: chromecdm_903_l3 + jane_uhd: nexus_5_l1 +``` + +You can also specify a fallback value to predefine if a match was not made. +This can be done using `default` key. This can help reduce redundancy in your specifications. + +For example, the following has the same result as the previous example, as well as all other +services and profiles being pre-defined to use `chromecdm_903_l3`. + +```yaml +EXAMPLE2: nexus_6_l1 +EXAMPLE3: + jane_uhd: nexus_5_l1 +default: chromecdm_903_l3 +``` + +You can also select CDMs based on video resolution using comparison operators (`>=`, `>`, `<=`, `<`) +or exact match on the resolution height. + +For example, + +```yaml +EXAMPLE: + "<=1080": generic_android_l3 # Use L3 for 1080p and below + ">1080": nexus_5_l1 # Use L1 for above 1080p (1440p, 2160p) + default: generic_android_l3 # Fallback if no quality match +``` + +You can mix profiles and quality thresholds in the same service: + +```yaml +EXAMPLE: + john: example_l3_profile # Profile-based selection + "<=720": example_mobile_l3 # Quality-based selection + "1080": example_standard_l3 # Exact match for 1080p + ">=1440": example_premium_l1 # Quality-based selection + default: example_standard_l3 # Fallback +``` + +--- + +## remote_cdm (list\[dict]) + +Configure remote CDM (Content Decryption Module) APIs to use for decrypting DRM-protected content. +Remote CDMs allow you to use high-security CDMs (L1/L2 for Widevine, SL2000/SL3000 for PlayReady) without +having the physical device files locally. + +envied supports multiple types of remote CDM providers: + +1. **DecryptLabs CDM** - Official DecryptLabs KeyXtractor API with intelligent caching +2. **Custom API CDM** - Highly configurable adapter for any third-party CDM API +3. **Legacy PyWidevine Serve** - Standard pywidevine serve-compliant APIs + +The name of each defined remote CDM can be referenced in the `cdm` configuration as if it was a local device file. + +### DecryptLabs Remote CDM + +DecryptLabs provides a professional CDM API service with support for multiple device types and intelligent key caching. + +**Supported Devices:** +- **Widevine**: `ChromeCDM` (L3), `L1` (Security Level 1), `L2` (Security Level 2) +- **PlayReady**: `SL2` (SL2000), `SL3` (SL3000) + +**Configuration:** + +```yaml +remote_cdm: + # Widevine L1 Device + - name: decrypt_labs_l1 + type: decrypt_labs # Required: identifies as DecryptLabs CDM + device_name: L1 # Required: must match exactly (L1, L2, ChromeCDM, SL2, SL3) + host: https://keyxtractor.decryptlabs.com + secret: YOUR_API_KEY # Your DecryptLabs API key + + # Widevine L2 Device + - name: decrypt_labs_l2 + type: decrypt_labs + device_name: L2 + host: https://keyxtractor.decryptlabs.com + secret: YOUR_API_KEY + + # Chrome CDM (L3) + - name: decrypt_labs_chrome + type: decrypt_labs + device_name: ChromeCDM + host: https://keyxtractor.decryptlabs.com + secret: YOUR_API_KEY + + # PlayReady SL2000 + - name: decrypt_labs_playready_sl2 + type: decrypt_labs + device_name: SL2 + device_type: PLAYREADY # Required for PlayReady + host: https://keyxtractor.decryptlabs.com + secret: YOUR_API_KEY + + # PlayReady SL3000 + - name: decrypt_labs_playready_sl3 + type: decrypt_labs + device_name: SL3 + device_type: PLAYREADY + host: https://keyxtractor.decryptlabs.com + secret: YOUR_API_KEY +``` + +**Features:** +- Intelligent key caching system (reduces API calls) +- Automatic integration with envied's vault system +- Support for both Widevine and PlayReady +- Multiple security levels (L1, L2, L3, SL2000, SL3000) + +**Note:** The `device_type` field determines whether the CDM operates in PlayReady or Widevine mode. +Setting `device_type: PLAYREADY` (or using `device_name: SL2` / `SL3`) activates PlayReady mode. +The `security_level` field is auto-computed from `device_name` when not specified (e.g., SL2 defaults +to 2000, SL3 to 3000, and Widevine devices default to 3). You can override these if needed. + +### Custom API Remote CDM + +A highly configurable CDM adapter that can work with virtually any third-party CDM API through YAML configuration. +This allows you to integrate custom CDM services without writing code. + +**Basic Example:** + +```yaml +remote_cdm: + - name: custom_chrome_cdm + type: custom_api # Required: identifies as Custom API CDM + host: https://your-cdm-api.com + timeout: 30 # Optional: request timeout in seconds + + device: + name: ChromeCDM + type: CHROME # CHROME, ANDROID, PLAYREADY + system_id: 27175 + security_level: 3 + + auth: + type: bearer # bearer, header, basic, body + key: YOUR_API_TOKEN + + endpoints: + get_request: + path: /get-challenge + method: POST + decrypt_response: + path: /get-keys + method: POST + + caching: + enabled: true # Enable key caching + use_vaults: true # Integrate with vault system +``` + +**Advanced Example with Field Mapping:** + +```yaml +remote_cdm: + - name: advanced_custom_api + type: custom_api + host: https://api.example.com + device: + name: L1 + type: ANDROID + security_level: 1 + + # Authentication configuration + auth: + type: header + header_name: X-API-Key + key: YOUR_SECRET_KEY + custom_headers: + User-Agent: envied/3.1.0 + X-Client-Version: "1.0" + + # Endpoint configuration + endpoints: + get_request: + path: /v2/challenge + method: POST + timeout: 30 + decrypt_response: + path: /v2/decrypt + method: POST + timeout: 30 + + # Request parameter mapping + request_mapping: + get_request: + param_names: + init_data: pssh # Rename 'init_data' to 'pssh' + scheme: device_type # Rename 'scheme' to 'device_type' + static_params: + api_version: "2.0" # Add static parameter + decrypt_response: + param_names: + license_request: challenge + license_response: license + + # Response field mapping + response_mapping: + get_request: + fields: + challenge: data.challenge # Deep field access + session_id: session.id + success_conditions: + - status == 'ok' # Validate response + decrypt_response: + fields: + keys: data.keys + key_fields: + kid: key_id # Map 'kid' field + key: content_key # Map 'key' field + + caching: + enabled: true + use_vaults: true + check_cached_first: true # Check cache before API calls +``` + +**Supported Authentication Types:** +- `bearer` - Bearer token authentication +- `header` - Custom header authentication +- `basic` - HTTP Basic authentication +- `body` - Credentials in request body +- `query` - Authentication added to query string parameters + +### Legacy PyWidevine Serve Format + +Standard [pywidevine] serve-compliant remote CDM configuration (backwards compatibility). + +```yaml +remote_cdm: + - name: legacy_chrome_cdm + device_name: chrome + device_type: CHROME + system_id: 27175 + security_level: 3 + host: https://domain.com/api + secret: secret_key +``` + +**Note:** If the `type` field is not specified, the entry is treated as a legacy pywidevine serve CDM. + +[pywidevine]: https://github.com/rlaphoenix/pywidevine + +--- + +## decrypt_labs_api_key (str) + +API key for DecryptLabs CDM service integration. + +When set, enables the use of DecryptLabs remote CDM services in your `remote_cdm` configuration. +This is used specifically for `type: "decrypt_labs"` entries in the remote CDM list. + +For example, + +```yaml +decrypt_labs_api_key: "your_api_key_here" +``` + +**Note**: This is different from the per-CDM `secret` field in `remote_cdm` entries. This provides a global +API key that can be referenced across multiple DecryptLabs CDM configurations. If a `remote_cdm` entry with +`type: "decrypt_labs"` does not have a `secret` field specified, the global `decrypt_labs_api_key` will be +used as a fallback. + +--- + +## decryption (str|dict) + +Configure which decryption tool to use for DRM-protected content. Default: `shaka`. + +Supported values: +- `shaka` - Shaka Packager (default) +- `mp4decrypt` - Bento4 mp4decrypt + +You can specify a single decrypter for all services: + +```yaml +decryption: shaka +``` + +Or configure per-service with a `DEFAULT` fallback: + +```yaml +decryption: + DEFAULT: shaka + EXAMPLE: mp4decrypt + EXAMPLE2: shaka +``` + +Service keys are case-insensitive (normalized to uppercase internally). + +--- + +## MonaLisa DRM + +MonaLisa is a WASM-based DRM system that uses local key extraction and two-stage segment decryption. +Unlike Widevine and PlayReady, MonaLisa does not use a challenge/response flow with a license server. +Instead, the PSSH value (ticket) is provided directly by the service API, and keys are extracted +locally via a WASM module. + +### Requirements + +- **ML-Worker binary**: Must be available on your system `PATH` (discovered via `binaries.ML_Worker`). + This is the binary that performs stage-1 decryption. + +### Decryption stages + +1. **ML-Worker binary**: Removes MonaLisa encryption layer (bbts -> ents). The key is passed via command-line argument. +2. **AES-ECB decryption**: Final decryption with service-provided key. + +MonaLisa uses per-segment decryption during download (not post-download like Widevine/PlayReady), +so segments are decrypted as they are downloaded. + +**Note:** MonaLisa is configured per-service rather than through global config options. Services +that use MonaLisa handle ticket/key retrieval and CDM initialization internally. + +--- + +## key_vaults (list\[dict]) + +Key Vaults store your obtained Content Encryption Keys (CEKs) and Key IDs per-service. + +This can help reduce unnecessary License calls even during the first download. This is because a Service may +provide the same Key ID and CEK for both Video and Audio, as well as for multiple resolutions or bitrates. + +You can have as many Key Vaults as you would like. It's nice to share Key Vaults or use a unified Vault on +Teams as sharing CEKs immediately can help reduce License calls drastically. + +Four types of Vaults are in the Core codebase: API, SQLite, MySQL, and HTTP. API and HTTP make HTTP requests to a RESTful API, +whereas SQLite and MySQL directly connect to an SQLite or MySQL Database. + +Note: SQLite and MySQL vaults have to connect directly to the Host/IP. It cannot be in front of a PHP API or such. +Beware that some Hosting Providers do not let you access the MySQL server outside their intranet and may not be +accessible outside their hosting platform. + +Additional behavior: + +- `no_push` (bool): Optional per-vault flag. When `true`, the vault will not receive pushed keys (writes) but + will still be queried and can provide keys for lookups. Useful for read-only/backup vaults. + +### Using an API Vault + +API vaults use a specific HTTP request format, therefore API or HTTP Key Vault APIs from other projects or services may +not work in envied. The API format can be seen in the [API Vault Code](envied/vaults/API.py). + +```yaml +- type: API + name: "John#0001's Vault" # arbitrary vault name + uri: "https://key-vault.example.com" # api base uri (can also be an IP or IP:Port) + # uri: "127.0.0.1:80/key-vault" + # uri: "https://api.example.com/key-vault" + token: "random secret key" # authorization token + # no_push: true # optional; make this API vault read-only (lookups only) +``` + +### Using a MySQL Vault + +MySQL vaults can be either MySQL or MariaDB servers. I recommend MariaDB. +A MySQL Vault can be on a local or remote network, but I recommend SQLite for local Vaults. + +```yaml +- type: MySQL + name: "John#0001's Vault" # arbitrary vault name + host: "127.0.0.1" # host/ip + # port: 3306 # port (defaults to 3306) + database: vault # database used for envied + username: jane11 + password: Doe123 + # no_push: false # optional; defaults to false +``` + +I recommend giving only a trustable user (or yourself) CREATE permission and then use envied to cache at least one CEK +per Service to have it create the tables. If you don't give any user permissions to create tables, you will need to +make tables yourself. + +- Use a password on all user accounts. +- Never use the root account with envied (even if it's you). +- Do not give multiple users the same username and/or password. +- Only give users access to the database used for envied. +- You may give trusted users CREATE permission so envied can create tables if needed. +- Other uses should only be given SELECT and INSERT permissions. + +### Using an SQLite Vault + +SQLite Vaults are usually only used for locally stored vaults. This vault may be stored on a mounted Cloud storage +drive, but I recommend using SQLite exclusively as an offline-only vault. Effectively this is your backup vault in +case something happens to your MySQL Vault. + +```yaml +- type: SQLite + name: "My Local Vault" # arbitrary vault name + path: "C:/Users/Jane11/Documents/envied/data/key_vault.db" + # no_push: true # optional; commonly true for local backup vaults +``` + +**Note**: You do not need to create the file at the specified path. +SQLite will create a new SQLite database at that path if one does not exist. +Try not to accidentally move the `db` file once created without reflecting the change in the config, or you will end +up with multiple databases. + +If you work on a Team I recommend every team member having their own SQLite Vault even if you all use a MySQL vault +together. + +### Using an HTTP Vault + +HTTP Vaults provide flexible HTTP-based key storage with support for multiple API modes. This vault type +is useful for integrating with various third-party key vault APIs. + +```yaml +- type: HTTP + name: "My HTTP Vault" + host: "https://vault-api.example.com" + api_key: "your_api_key" # or use 'password' field + api_mode: "json" # query, json, or decrypt_labs + # username: "user" # required for query mode only + # no_push: false # optional; defaults to false +``` + +**Supported API Modes:** + +- `query` - Uses GET requests with query parameters. Requires `username` field. +- `json` - Uses POST requests with JSON payloads. Token-based authentication. +- `decrypt_labs` - DecryptLabs API format. Read-only mode (`no_push` is forced to `true`). + +**Example configurations:** + +```yaml +# Query mode (requires username) +- type: HTTP + name: "Query Vault" + host: "https://api.example.com/keys" + username: "myuser" + password: "mypassword" + api_mode: "query" + +# JSON mode +- type: HTTP + name: "JSON Vault" + host: "https://api.example.com/vault" + api_key: "secret_token" + api_mode: "json" + +# DecryptLabs mode (read-only) +- type: HTTP + name: "DecryptLabs Cache" + host: "https://keyxtractor.decryptlabs.com/cache" + api_key: "your_decrypt_labs_api_key" + api_mode: "decrypt_labs" +``` + +**Note**: The `decrypt_labs` mode is always read-only and cannot receive pushed keys. + +--- diff --git a/reset/docs/GLUETUN.md b/reset/docs/GLUETUN.md new file mode 100644 index 0000000..0a9599e --- /dev/null +++ b/reset/docs/GLUETUN.md @@ -0,0 +1,243 @@ +# Gluetun VPN Proxy + +Gluetun provides Docker-managed VPN proxies supporting 50+ VPN providers. + +## Prerequisites + +**Docker must be installed and running.** + +```bash +# Linux +curl -fsSL https://get.docker.com | sh +sudo usermod -aG docker $USER # Then log out/in + +# Windows/Mac +# Install Docker Desktop: https://www.docker.com/products/docker-desktop/ +``` + +## Quick Start + +### 1. Configuration + +Add to `~/.config/envied/envied.yaml`: + +```yaml +proxy_providers: + gluetun: + providers: + windscribe: + vpn_type: openvpn + credentials: + username: "YOUR_OPENVPN_USERNAME" + password: "YOUR_OPENVPN_PASSWORD" +``` + +### 2. Usage + +Use 2-letter country codes directly: + +```bash +envied dl SERVICE CONTENT --proxy gluetun:windscribe:us +envied dl SERVICE CONTENT --proxy gluetun:windscribe:uk +``` + +Format: `gluetun:provider:region` + +## Provider Credential Requirements + +**OpenVPN (Recommended)**: Most providers support OpenVPN with just `username` and `password` - the simplest setup. + +**WireGuard**: Requires private keys and varies by provider. See the [Gluetun Wiki](https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers) for provider-specific requirements. Note that `vpn_type` defaults to `wireguard` if not specified. + +## Getting Your Credentials + +### Windscribe (OpenVPN) + +1. Go to [windscribe.com/getconfig/openvpn](https://windscribe.com/getconfig/openvpn) +2. Log in with your Windscribe account +3. Select any location and click "Get Config" +4. Copy the username and password shown + +### NordVPN (OpenVPN) + +1. Go to [NordVPN Service Credentials](https://my.nordaccount.com/dashboard/nordvpn/manual-configuration/service-credentials/) +2. Log in with your NordVPN account +3. Generate or view your service credentials +4. Copy the username and password + +> **Note**: Use service credentials, NOT your account email/password. + +### WireGuard Credentials (Advanced) + +WireGuard requires private keys instead of username/password. See the [Gluetun Wiki](https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers) for provider-specific WireGuard setup. + +## Configuration Examples + +**OpenVPN (Recommended)** + +Most providers support OpenVPN with just username and password: + +```yaml +providers: + windscribe: + vpn_type: openvpn + credentials: + username: YOUR_OPENVPN_USERNAME + password: YOUR_OPENVPN_PASSWORD + + nordvpn: + vpn_type: openvpn + credentials: + username: YOUR_SERVICE_USERNAME + password: YOUR_SERVICE_PASSWORD +``` + +**WireGuard (Advanced)** + +WireGuard can be faster but requires more complex credential setup: + +```yaml +# NordVPN/ProtonVPN (only private_key needed) +providers: + nordvpn: + vpn_type: wireguard + credentials: + private_key: YOUR_PRIVATE_KEY + +# Surfshark/Mullvad/IVPN (private_key AND addresses required) + surfshark: + vpn_type: wireguard + credentials: + private_key: YOUR_PRIVATE_KEY + addresses: 10.x.x.x/32 + +# Windscribe (all three credentials required) + windscribe: + vpn_type: wireguard + credentials: + private_key: YOUR_PRIVATE_KEY + addresses: 10.x.x.x/32 + preshared_key: YOUR_PRESHARED_KEY +``` + +## Server Selection + +Most providers use `SERVER_COUNTRIES`, but some use `SERVER_REGIONS`: + +| Variable | Providers | +|----------|-----------| +| `SERVER_COUNTRIES` | NordVPN, ProtonVPN, Surfshark, Mullvad, ExpressVPN, and most others | +| `SERVER_REGIONS` | Windscribe, VyprVPN, VPN Secure | + +envied handles this automatically - just use 2-letter country codes. + +### Per-Provider Server Mapping + +You can explicitly map region codes to country names, cities, or hostnames per provider: + +```yaml +providers: + nordvpn: + vpn_type: openvpn + credentials: + username: YOUR_USERNAME + password: YOUR_PASSWORD + server_countries: + us: "United States" + uk: "United Kingdom" + server_cities: + us: "New York" + server_hostnames: + us: "us1239.nordvpn.com" +``` + +### Specific Server Selection + +Use a `` region (e.g. `us1239`) to target a specific server. envied builds the +hostname automatically per provider: + +| Provider | Hostname format | +|----------|-----------------| +| NordVPN | `us1239.nordvpn.com` | +| Surfshark | `us-1239.prod.surfshark.com` | +| ExpressVPN | `us-1239.expressvpn.com` | +| CyberGhost | `us-s1239.cg-dialup.net` | +| Other | `us1239` (passed as-is to `SERVER_HOSTNAMES`) | + +### Extra Environment Variables + +You can pass additional Gluetun environment variables per provider using `extra_env`: + +```yaml +providers: + nordvpn: + vpn_type: openvpn + credentials: + username: YOUR_USERNAME + password: YOUR_PASSWORD + extra_env: + LOG_LEVEL: debug +``` + +## Global Settings + +```yaml +proxy_providers: + gluetun: + providers: {...} + base_port: 8888 # Starting port (default: 8888) + auto_cleanup: true # Remove containers on exit (default: true) + verify_ip: true # Verify IP matches region (default: true) + container_prefix: "envied-gluetun" # Docker container name prefix (default: "envied-gluetun") + auth_user: username # Proxy auth (optional) + auth_password: password # Proxy auth (optional) +``` + +## Features + +- **Container Reuse**: First request takes 10-30s; subsequent requests are instant. Containers created by other envied processes are auto-detected via `docker inspect` and reused. +- **Ready Detection**: Waits up to 60s for both the HTTP proxy to listen (`[http proxy] listening`) and the VPN tunnel to come up (`initialization sequence completed` or `public ip address is`) before returning the proxy URI. Bails early on `fatal` or `invalid credentials` log lines. +- **IP Verification**: When `verify_ip: true` (default), looks up the exit IP via `ipinfo.io` through the proxy and compares country code to the requested region. Retries 3 times with exponential backoff (1s, 2s, 4s). +- **Concurrent Sessions**: Multiple downloads share the same container; ports are allocated thread-safely starting at `base_port`. +- **Specific Servers**: Use `--proxy gluetun:nordvpn:us1239` for specific server selection (see table above). +- **Automatic Image Pull**: The Gluetun Docker image (`qmcgaw/gluetun:latest`) is pulled automatically on first use (5 min timeout). +- **Secure Credentials**: Credentials are passed via temporary env files (mode 0600), then zero-overwritten and unlinked after `docker run`. They never appear in process listings. +- **Auto Cleanup**: Containers are removed via `atexit` (Ctrl+C still works normally). Disable with `auto_cleanup: false` to leave them stopped instead. + +## Container Management + +```bash +# View containers +docker ps | grep envied-gluetun + +# Check logs +docker logs envied-gluetun-nordvpn-us + +# Remove all containers +docker ps -a | grep envied-gluetun | awk '{print $1}' | xargs docker rm -f +``` + +## Troubleshooting + +### Docker Permission Denied (Linux) +```bash +sudo usermod -aG docker $USER +# Then log out and log back in +``` + +### VPN Connection Failed +Check container logs for specific errors: +```bash +docker logs envied-gluetun-nordvpn-us +``` + +Common issues: +- Invalid/missing credentials +- Windscribe WireGuard requires `preshared_key` (can be empty string, but must be set in credentials) +- VPN provider server issues +- Container startup timeout (default 60 seconds) + +## Resources + +- [Gluetun Wiki](https://github.com/qdm12/gluetun-wiki) - Official provider documentation +- [Gluetun GitHub](https://github.com/qdm12/gluetun) diff --git a/reset/docs/NETWORK_CONFIG.md b/reset/docs/NETWORK_CONFIG.md new file mode 100644 index 0000000..10368b9 --- /dev/null +++ b/reset/docs/NETWORK_CONFIG.md @@ -0,0 +1,178 @@ +# Network & Proxy Configuration + +This document covers network and proxy configuration options for bypassing geofencing and managing connections. + +## proxy_providers (dict) + +Enable external proxy provider services. These proxies will be used automatically where needed as defined by the +Service's GEOFENCE class property, but can also be explicitly used with `--proxy`. You can specify which provider +to use by prefixing it with the provider key name, e.g., `--proxy basic:de` or `--proxy nordvpn:de`. Some providers +support specific query formats for selecting a country/server. + +### basic (dict[str, str|list]) + +Define a mapping of country to proxy to use where required. +The keys are region Alpha 2 Country Codes. Alpha 2 Country Codes are `[a-z]{2}` codes, e.g., `us`, `gb`, and `jp`. +Don't get this mixed up with language codes like `en` vs. `gb`, or `ja` vs. `jp`. + +Do note that each key's value can be a list of strings, or a string. For example, + +```yaml +us: + - "http://john%40email.tld:password123@proxy-us.domain.tld:8080" + - "http://jane%40email.tld:password456@proxy-us.domain2.tld:8080" +de: "https://127.0.0.1:8080" +``` + +Note that if multiple proxies are defined for a region, then by default one will be randomly chosen. +You can choose a specific one by specifying it's number, e.g., `--proxy basic:us2` will choose the +second proxy of the US list. + +### nordvpn (dict) + +Set your NordVPN Service credentials with `username` and `password` keys to automate the use of NordVPN as a Proxy +system where required. + +You can also specify specific servers to use per-region with the `server_map` key. +Sometimes a specific server works best for a service than others, so hard-coding one for a day or two helps. + +You can also select servers by city using the format `--proxy nordvpn:us:seattle` or `--proxy nordvpn:ca:calgary`. + +For example, + +```yaml +username: zxqsR7C5CyGwmGb6KSvk8qsZ # example of the login format +password: wXVHmht22hhRKUEQ32PQVjCZ +server_map: + us: 12 # force US server #12 for US proxies +``` + +The username and password should NOT be your normal NordVPN Account Credentials. +They should be the `Service credentials` which can be found on your Nord Account Dashboard. + +Once set, you can also specifically opt in to use a NordVPN proxy by specifying `--proxy nordvpn:gb` or such. +You can even set a specific server number this way, e.g., `--proxy nordvpn:gb2366`. + +Note that `gb` is used instead of `uk` to be more consistent across regional systems. + +### surfsharkvpn (dict) + +Enable Surfshark VPN proxy service using Surfshark Service credentials (not your login password). +You may pin specific server IDs per region using `server_map`. + +You can also select servers by city using the format `--proxy surfsharkvpn:us:seattle`. + +```yaml +username: your_surfshark_service_username # https://my.surfshark.com/vpn/manual-setup/main/openvpn +password: your_surfshark_service_password # service credentials, not account password +server_map: + us: 3844 # force US server #3844 + gb: 2697 # force GB server #2697 + au: 4621 # force AU server #4621 +``` + +### hola + +Enable Hola VPN proxy service. Requires the `hola-proxy` binary to be installed and available in your PATH. +No configuration is needed under `proxy_providers`. Hola is loaded automatically when the `hola-proxy` binary +is detected. + +Once available, use `--proxy hola:us` or similar to connect through Hola. + +### windscribevpn (dict) + +Enable Windscribe VPN proxy service using static OpenVPN service credentials. + +Use the service credentials from https://windscribe.com/getconfig/openvpn (not your account login credentials). + +```yaml +proxy_providers: + windscribevpn: + username: openvpn_username # From https://windscribe.com/getconfig/openvpn + password: openvpn_password # Service credentials, NOT your account password +``` + +#### Server Mapping + +You can optionally pin specific servers using `server_map`: + +```yaml +proxy_providers: + windscribevpn: + username: openvpn_username + password: openvpn_password + server_map: + us: us-central-096.totallyacdn.com # Force specific US server + gb: uk-london-001.totallyacdn.com # Force specific UK server +``` + +Once configured, use `--proxy windscribevpn:us` or `--proxy windscribevpn:gb` etc. to connect through Windscribe. + +You can also select specific servers by number (e.g., `--proxy windscribevpn:sg007`) or filter by city +(e.g., `--proxy windscribevpn:ca:toronto`). + +### gluetun (dict) + +Docker-managed VPN proxy supporting 50+ VPN providers via Gluetun. See [GLUETUN.md](GLUETUN.md) for full +configuration and usage details. + +```yaml +proxy_providers: + gluetun: + providers: + windscribe: + vpn_type: openvpn + credentials: + username: "YOUR_OPENVPN_USERNAME" + password: "YOUR_OPENVPN_PASSWORD" +``` + +Usage: `--proxy gluetun:windscribe:us` + +--- + +## headers (dict) + +Case-Insensitive dictionary of headers that all Services begin their Request Session state with. +All requests will use these unless changed explicitly or implicitly via a Server response. +These should be sane defaults and anything that would only be useful for some Services should not +be put here. + +Avoid headers like 'Accept-Encoding' as that would be a compatibility header that the underlying +HTTP backend (rnet) will set for you as part of its browser impersonation profile. + +I recommend using, + +```yaml +Accept-Language: "en-US,en;q=0.8" +User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" +``` + +--- + +## HTTP Session Backend + +envied uses [`rnet`](https://github.com/0x676e67/rnet) (Rust + BoringSSL) for HTTP with TLS +fingerprinting. `RnetSession` is a drop-in `requests.Session` replacement and is what +`self.session` exposes to services. It supports: + +- Browser/app impersonation via named `rnet.Impersonate` presets (Chrome, Edge, Firefox, Safari, + OkHttp, etc.) — picks JA3, ALPN, HTTP/2 SETTINGS and header order to match the chosen client. +- Native rnet proxy support (HTTP, HTTPS, SOCKS5) — used by all proxy providers below. +- Cookie-jar and `requests`-style `data=` / `json=` / `headers=` kwargs for compatibility. + +The legacy `curl_cffi` backend has been removed. The config key is still spelled +`curl_impersonate` for backward compatibility, but its value now selects an rnet preset. + +### curl_impersonate (dict) + +```yaml +curl_impersonate: + browser: Chrome131 # exact rnet.Impersonate preset name +``` + +`browser` must be an exact `rnet.Impersonate` preset name (e.g. `Chrome131`, `Chrome124`, +`Edge101`, `Firefox133`, `Safari18`, `OkHttp4_12`). See the rnet README for the full list. +Default when unset: `Chrome131`. + +--- diff --git a/reset/docs/OUTPUT_CONFIG.md b/reset/docs/OUTPUT_CONFIG.md new file mode 100644 index 0000000..2e3275e --- /dev/null +++ b/reset/docs/OUTPUT_CONFIG.md @@ -0,0 +1,287 @@ +# Output & Naming Configuration + +This document covers output file organization and naming configuration options. + +## filenames (dict) + +Override the default filenames used across envied. +The filenames use various variables that are replaced during runtime. + +The following filenames are available and may be overridden: + +- `log` - Log filenames. Uses `{name}` and `{time}` variables. +- `debug_log` - Debug log filenames. Uses `{service}` and `{time}` variables. +- `config` - Service configuration filenames. +- `root_config` - Root configuration filename. +- `chapters` - Chapter export filenames. Uses `{title}` and `{random}` variables. +- `subtitle` - Subtitle export filenames. Uses `{id}` and `{language}` variables. + +For example, + +```yaml +filenames: + log: "envied_{name}_{time}.log" + debug_log: "envied_debug_{service}_{time}.jsonl" + config: "config.yaml" + root_config: "envied.yaml" + chapters: "Chapters_{title}_{random}.txt" + subtitle: "Subtitle_{id}_{language}.srt" +``` + +--- + +## output_template (dict) + +Configure custom output filename templates for movies, series, and songs. +This is **required** in your `envied.yaml` — a warning is shown if not configured. + +Available variables: `{title}`, `{year}`, `{season}`, `{episode}`, `{season_episode}`, `{episode_name}`, +`{quality}`, `{resolution}`, `{source}`, `{audio}`, `{audio_channels}`, `{audio_full}`, +`{video}`, `{hdr}`, `{hfr}`, `{atmos}`, `{dual}`, `{multi}`, `{tag}`, `{edition}`, `{repack}`, +`{lang_tag}`, `{track_number}`, `{artist}`, `{album}`, `{disc}` + +Add `?` suffix to make a variable conditional (omitted when empty): `{year?}`, `{hdr?}`, `{repack?}` + +```yaml +output_template: + # Scene-style (dot-separated) + movies: '{title}.{year}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + series: '{title}.{year?}.{season_episode}.{episode_name?}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + songs: '{track_number}.{title}.{repack?}.{edition?}.{source?}.WEB-DL.{audio_full}.{atmos?}-{tag}' + + # Plex-friendly (space-separated) + # movies: '{title} ({year}) {quality}' + # series: '{title} {season_episode} {episode_name?}' + # songs: '{track_number}. {title}' +``` + +Example outputs: +- Scene movies: `Example.Movie.2024.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG` +- Scene movies (REPACK): `Example.Movie.2024.REPACK.2160p.EXAMPLE.WEB-DL.DDP5.1.H.265-TAG` +- Scene series: `Example.Show.2024.S01E01.Pilot.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG` +- Plex movies: `Example Movie (2024) 1080p` + +### folder (optional) + +Controls the folder name for downloaded content. Uses the same template variables as the file templates above. + +If not configured, the default folder naming is used: +- Movies: `Title (Year)` +- Series: Derived from the `series` template with episode-specific variables removed +- Songs: `Artist - Album (Year)` + +`folder` accepts either a single string (applies to all title kinds) or a mapping with per-kind +templates keyed by `movies`, `series`, and/or `songs`. Unknown keys are warned about and ignored. + +```yaml +output_template: + movies: '{title}.{year}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + series: '{title}.{year?}.{season_episode}.{episode_name?}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + songs: '{track_number}.{title}.{repack?}.{edition?}.{source?}.WEB-DL.{audio_full}.{atmos?}-{tag}' + + # Scene-style folder (single template, applies to all kinds) + folder: '{title}.{year?}.{repack?}.{edition?}.{lang_tag?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + + # Plex-friendly folder + # folder: '{title} ({year?})' + + # Per-kind folder templates + # folder: + # movies: '{title} ({year})' + # series: '{title} ({year?})' + # songs: '{artist} - {album} ({year?})' +``` + +Example outputs: +- Scene folder: `Example.Show.2024.S01.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG/` +- Plex folder: `Example Show (2024)/` + +--- + +--- + +## language_tags (dict) + +Automatically adds language-based identifiers (e.g., `DANiSH`, `NORDiC`, `DKsubs`) to output filenames +based on audio and subtitle track languages. Use `{lang_tag?}` in your `output_template` to place the tag. + +Rules are evaluated in order; the first matching rule wins. All conditions within a single rule +must match (AND logic). If no rules match, `{lang_tag?}` is cleanly removed from the filename. + +### Conditions + +| Condition | Type | Description | +|-----------|------|-------------| +| `audio` | string | Matches if any selected audio track has this language | +| `subs_contain` | string | Matches if any selected subtitle has this language | +| `subs_contain_all` | list | Matches if subtitles include ALL listed languages | + +Language matching uses fuzzy matching (e.g., `en` matches `en-US`, `en-GB`). + +### Example: Nordic tagging + +```yaml +language_tags: + rules: + - audio: da + tag: DANiSH + - audio: sv + tag: SWEDiSH + - audio: nb + tag: NORWEGiAN + - audio: en + subs_contain_all: [da, sv, nb] + tag: NORDiC + - audio: en + subs_contain: da + tag: DKsubs + +output_template: + movies: '{title}.{year?}.{lang_tag?}.{quality}.{source}.WEB-DL.{audio_full}.{video}-{tag}' +``` + +Example outputs: +- Danish audio: `Example.Show.S01E01.DANiSH.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG` +- English audio + multiple Nordic subs: `Example.Show.S01E01.NORDiC.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG` +- English audio + Danish subs only: `Example.Show.S01E01.DKsubs.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG` +- No matching languages: `Example.Show.S01E01.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG` + +### Example: Other regional tags + +```yaml +language_tags: + rules: + - audio: nl + tag: DUTCH + - audio: de + tag: GERMAN + - audio: fr + subs_contain: en + tag: ENGFR + - audio: fr + tag: FRENCH +``` + +--- + +## unicode_filenames (bool) + +Allow Unicode characters in output filenames. When `false`, Unicode characters are transliterated +to ASCII equivalents. Default: `false`. + +--- + +## tag (str) + +Group or Username to postfix to the end of download filenames following a dash. +Use `{tag}` in your output template to include it. +For example, `tag: "J0HN"` will have `-J0HN` at the end of all download filenames. + +--- + +## tag_group_name (bool) + +Enable/disable tagging downloads with your group name when `tag` is set. Default: `true`. + +--- + +## tag_imdb_tmdb (bool) + +Enable/disable tagging downloaded files with IMDB/TMDB/TVDB identifiers (when available). Default: `true`. + +--- + +## muxing (dict) + +- `set_title` + Set the container title to `Show SXXEXX Episode Name` or `Movie (Year)`. Default: `true` +- `merge_audio` + Merge all audio tracks into each output file. Default: `true` + - `true`: All selected audio tracks are muxed into one MKV per quality. + - `false`: Separate MKV per (quality, audio_codec) combination. + For example: `Title.1080p.AAC.mkv`, `Title.1080p.EC3.mkv`. + + Note: The `--split-audio` CLI flag overrides this setting. When `--split-audio` is passed, + `merge_audio` is effectively set to `false` for that run. + +- `default_language` (dict) + Override which track is flagged as the default in the muxed MKV, regardless + of the title's original language. Useful when you always want your player to + open on a specific language (e.g. always default to Polish audio even on + English originals). Only affects the MKV `--default-track` flag — track + selection (`-l`, `--alang`, etc.) is unchanged. All keys are optional; each + track type falls back to its previous default rule when the configured + language isn't present in the manifest. + + - `audio`: BCP-47 tag (e.g. `pl`, `en`, `pt-BR`). Wins over `is_original_lang`. + The `--original-flag` continues to mark the true original-audio track. + - `video`: BCP-47 tag. Wins over the title-language / first-track rule. + - `subtitle`: BCP-47 tag. Wins over the "forced sub matching audio" rule. + + Languages are matched with the same close-match logic used elsewhere + (`pt` matches `pt-BR`, etc.). Supports per-service overrides like the rest + of `muxing`. + + ```yaml + muxing: + default_language: + audio: pl + video: pl + subtitle: pl + ``` + +--- + +## chapter_fallback_name (str) + +The Chapter Name to use when exporting a Chapter without a Name. +The default is no fallback name at all and no Chapter name will be set. + +The fallback name can use the following variables in f-string style: + +- `{i}`: The Chapter number starting at 1. + E.g., `"Chapter {i}"`: "Chapter 1", "Intro", "Chapter 3". +- `{j}`: A number starting at 1 that increments any time a Chapter has no title. + E.g., `"Chapter {j}"`: "Chapter 1", "Intro", "Chapter 2". + +These are formatted with f-strings, directives are supported. +For example, `"Chapter {i:02}"` will result in `"Chapter 01"`. + +--- + +## directories (dict) + +Override the default directories used across envied. +The directories are set to common values by default. + +The following directories are available and may be overridden, + +- `commands` - CLI Command Classes. +- `services` - Service Classes. +- `vaults` - Vault Classes. +- `fonts` - Font files (ttf or otf). +- `downloads` - Downloads. +- `temp` - Temporary files or conversions during download. +- `cache` - Expiring data like Authorization tokens, or other misc data. +- `cookies` - Expiring Cookie data. +- `logs` - Logs. +- `exports` - JSON sidecar exports written when `--export` is used on `dl`. +- `wvds` - Widevine Devices. +- `prds` - PlayReady Devices. +- `dcsl` - Device Certificate Status List. + +Notes: + +- `services` accepts either a single directory or a list of directories to search for service modules. + +For example, + +```yaml +directories: + downloads: "D:/Downloads/envied" + temp: "D:/Temp/envied" +``` + +There are directories not listed that cannot be modified as they are crucial to the operation of envied. + +--- diff --git a/reset/docs/SERVICE_CONFIG.md b/reset/docs/SERVICE_CONFIG.md new file mode 100644 index 0000000..19ca2b8 --- /dev/null +++ b/reset/docs/SERVICE_CONFIG.md @@ -0,0 +1,246 @@ +# Service Integration & Authentication Configuration + +This document covers service-specific configuration, authentication, and metadata integration options. + +## services (dict) + +Configuration data for each Service. The Service will have the data within this section merged into the per-service +`config.yaml` (located in the service's directory) before being provided to the Service class. + +Think of this config to be used for more sensitive configuration data, like user or device-specific API keys, IDs, +device attributes, and so on. A per-service `config.yaml` file is typically shared and not meant to be modified, +so use this for any sensitive configuration data. + +The Key is the Service Tag, but can take any arbitrary form for its value. It's expected to begin as either a list or +a dictionary. + +For example, + +```yaml +EXAMPLE: + client: + auth_scheme: MESSO + # ... more sensitive data +``` + +### Per-Service Configuration Overrides + +You can override many global configuration options on a per-service basis by nesting them under the +service tag in the `services` section. Supported override keys include: `dl`, `subtitle`, `muxing`, +`headers`, `proxy_map`, `title_map`, and more. + +Overrides are merged with global config (not replaced) -- only specified keys are overridden, others +use global defaults. CLI arguments always take priority over service-specific config. + +For example, + +```yaml +services: + RATE_LIMITED_SERVICE: + dl: + downloads: 2 # Limit concurrent track downloads + workers: 4 # Reduce workers to avoid rate limits + headers: + User-Agent: "..." # Service-specific UA override +``` + +Note: envied uses a single unified `requests`-based downloader. The legacy `aria2c`, +`n_m3u8dl_re`, and `curl_impersonate` override sections have been removed. + +### title_map (dict) + +Rewrites service-provided titles before naming and output. Some services name a title differently +from how you want it stored, which can break library matching (e.g. a regional variant reusing the +international name). Keys are the exact title string the service returns; values are the desired +output title. + +```yaml +services: + EXAMPLE: + title_map: + Service Title: Desired Title +``` + +Episodes are matched on their show title, Movies and Songs on their name. The remap is applied +after the title cache (so edits take effect without a cache reset) and before any `--enrich` +override (so an explicit enrich still wins). + +It applies on the local `dl` path, the `import` command, and the remote client (`dl --remote`). +For remote services the **client's** `title_map` is applied to the titles returned by the server, +so you can rename titles for services you don't have installed locally. The server sends raw +titles and does not remap, leaving the final name fully under the client's control. + +### Service Class Conventions + +Each service directory under `envied/services/` exports a class extending +`envied.core.service.Service`. The class name must match the directory name (the service tag). + +Key class variables (defined on `Service` or by service-level idiom): + +- `ALIASES: tuple[str, ...]` — alternative tags accepted on the CLI. Empty by default. +- `GEOFENCE: tuple[str, ...]` — ISO country codes the service is available in. Empty == no geofence. +- `TITLE_RE: str` — regex (with named groups, e.g. `(?P...)`, `(?P...)`) used by the + service to parse the CLI title argument. Service-level idiom, not declared on the base class. +- `NO_SUBTITLES: bool` — service-level idiom indicating the service has no subtitle tracks. + +`self.*` helpers available after `super().__init__(ctx)`: + +- `self.session` — pre-configured HTTP session (`requests.Session`, or `RnetSession` when TLS + impersonation is active). Cookies, headers, proxies pre-applied. +- `self.config` — merged service config (per-service `config.yaml` plus the `services.` block + from `envied.yaml`). +- `self.log` — `logging.Logger` named for the service class. +- `self.cache` — generic `Cacher` for arbitrary key/value persistence. +- `self.title_cache` — specialized `TitleCacher` for title metadata. +- `self.track_request` — `TrackRequest` built from CLI flags. Fields: `codecs: list[Video.Codec]`, + `ranges: list[Video.Range]` (defaults to `[SDR]`), `best_available: bool`. Services may + read or rewrite these (e.g. force HEVC for HDR ranges). +- `self.credential` — set during `authenticate()`; `None` if cookies-only. +- `self.current_region` — lowercase ISO country code from proxy/geolocation, or `None`. +- `self.request_input(prompt: str) -> str` — interactive prompt. Falls through to `input()` + locally; under `serve`, the attached `InputBridge` relays the prompt to the remote client. + +Driving CLI flags (parsed into `self.track_request`): + +- `-v` / `--vcodec` — comma-separated `Video.Codec` list (e.g. `H264,H265`). +- `-a` / `--acodec` — comma-separated audio codec list. +- `-r` / `--range` — comma-separated `Video.Range` list (`SDR`, `HDR10`, `HDR10+`, `DV`, + `HYBRID`). Defaults to `[SDR]`. +- `-q` / `--quality` — resolution list. +- `--vbitrate-range` / `--abitrate-range` — `MIN-MAX` kbps windows. + +--- + +## credentials (dict[str, str|list|dict]) + +Specify login credentials to use for each Service, and optionally per-profile. + +For example, + +```yaml +EXAMPLE: jane@example.tld:LoremIpsum100 # directly +EXAMPLE2: # or per-profile, optionally with a default + default: jane@example.tld:LoremIpsum99 # <-- used by default if -p/--profile is not used + james: james@example.tld:TheFriend97 + john: john@example.tld:LoremIpsum98 +EXAMPLE3: # the `default` key is not necessary, but no credential will be used by default + john: john@example.tld:SecretPassword123 +``` + +The value should be in string form, i.e. `john@example.tld:password123` or `john:password123`. +Any arbitrary values can be used on the left (username/password/phone) and right (password/secret). +You can also specify these in list form, i.e., `["john@example.tld", ":PasswordWithAColon"]`. + +If you specify multiple credentials with keys like the `EXAMPLE2` and `EXAMPLE3` example above, then you should +use a `default` key or no credential will be loaded automatically unless you use `-p/--profile`. You +do not have to use a `default` key at all. + +Please be aware that this information is sensitive and to keep it safe. Do not share your config. + +--- + +## tmdb_api_key (str) + +API key for The Movie Database (TMDB). This is used for tagging downloaded files with TMDB, +IMDB and TVDB identifiers. Leave empty to disable automatic lookups. + +To obtain a TMDB API key: + +1. Create an account at +2. Go to to register for API access +3. Fill out the API application form with your project details +4. Once approved, you'll receive your API key + +For example, + +```yaml +tmdb_api_key: cf66bf18956kca5311ada3bebb84eb9a # Not a real key +``` + +**Note**: Keep your API key secure and do not share it publicly. This key is used by the `core/providers/tmdb.py` metadata provider to fetch metadata from TMDB for proper file tagging and ID enrichment. + +--- + +## simkl_client_id (str) + +Client ID for SIMKL API integration. SIMKL is used as a metadata source for improved title matching and tagging, +especially when a TMDB API key is not configured. + +To obtain a SIMKL Client ID: + +1. Create an account at +2. Go to +3. Register a new application to receive your Client ID + +For example, + +```yaml +simkl_client_id: "your_client_id_here" +``` + +**Note**: While optional, having a SIMKL Client ID improves metadata lookup reliability. SIMKL serves as an alternative or fallback metadata source to TMDB. This is used by the `core/providers/simkl.py` metadata provider. + +--- + +## ipinfo_api_key (str) + +Optional API token for [ipinfo.io](https://ipinfo.io). When set, envied uses the free authenticated **Lite** endpoint (`https://api.ipinfo.io/lite/me`), which has substantially higher rate limits than the anonymous endpoint and returns richer fields (ASN, organization name, continent). Leave empty to use the anonymous ipinfo.io endpoint, with [ip-api.in](https://ip-api.in) as a final fallback. + +To obtain an ipinfo.io token: + +1. Sign up for a free account at +2. Copy the token from your dashboard + +For example, + +```yaml +ipinfo_api_key: "12a3b45cd678ef" # Not a real key +``` + +**Note**: The token is only ever sent to `api.ipinfo.io` as a per-request `Authorization` header — it is never attached to your session for service requests. Used by `core/utils/ip_info.py` for region detection and proxy verification. + +--- + +## title_cache_enabled (bool) + +Enable/disable caching of title metadata to reduce redundant API calls. Default: `true`. + +--- + +## title_cache_time (int) + +Cache duration in seconds for title metadata. Default: `1800` (30 minutes). + +--- + +## title_cache_max_retention (int) + +Maximum retention time in seconds for serving slightly stale cached title metadata when API calls fail. +Default: `86400` (24 hours). Effective retention is `min(title_cache_time + grace, title_cache_max_retention)`. + +--- + +## debug (bool) + +Enable structured JSON debug logging for troubleshooting and service development. Default: `false`. + +When enabled (via config or the `--debug` CLI flag): +- Creates JSON Lines (`.jsonl`) log files with complete debugging context +- Logs: session info, CLI params, service config, CDM details, authentication, titles, tracks metadata, + DRM operations, vault queries, errors with stack traces +- File location: `logs/envied_debug_{service}_{timestamp}.jsonl` + +--- + +## debug_keys (bool) + +Log decryption keys in debug logs. Default: `false`. + +When `true`, actual content encryption keys (CEKs) are included in debug log output. Useful for +debugging key retrieval and decryption issues. + +**Security note:** Passwords, tokens, cookies, and session tokens are always redacted regardless +of this setting. Only content keys (`content_key`, `key` fields) are affected. Key IDs (`kid`), +key counts, and other metadata are always logged. + +--- diff --git a/reset/docs/SUBTITLE_CONFIG.md b/reset/docs/SUBTITLE_CONFIG.md new file mode 100644 index 0000000..17a3c3e --- /dev/null +++ b/reset/docs/SUBTITLE_CONFIG.md @@ -0,0 +1,73 @@ +# Subtitle Processing Configuration + +This document covers subtitle processing and formatting options under the top-level `subtitle:` key in `envied.yaml`. + +For the canonical example, see `envied/envied-example.yaml`. + +## subtitle (dict) + +Control subtitle conversion, SDH (hearing-impaired) stripping, formatting preservation, and output behavior. + +- `conversion_method`: How to convert subtitles between formats. Default: `auto`. + - `auto`: Smart routing - subby for WebVTT/fVTT/SAMI; for SSA/ASS/MicroDVD/MPL2/TMP use SubtitleEdit when available, otherwise pysubs2; standard pycaption/SubtitleEdit pipeline for everything else. + - `subby`: Always use subby with `CommonIssuesFixer` (falls back to standard if the source codec isn't supported by subby). + - `subtitleedit`: Prefer SubtitleEdit when available; otherwise fall back to the standard pycaption pipeline. + - `pycaption`: Use only the pycaption library (no SubtitleEdit, no subby). Limited to SRT, TTML, and WebVTT outputs. + - `pysubs2`: Use pysubs2 (supports SRT, SSA, ASS, WebVTT, TTML, SAMI, MicroDVD, MPL2, TMP). + +- `sdh_method`: How to strip SDH cues. Default: `auto`. + - `auto`: Try subby for SRT first, then SubtitleEdit (when `conversion_method` is `auto`/`subtitleedit` and the binary is available), then subtitle-filter as the final fallback. + - `subby`: Use subby's `SDHStripper`. **Only operates on SRT**; for other codecs the call returns without stripping. + - `subtitleedit`: Use SubtitleEdit's `/RemoveTextForHI` when the binary is available; otherwise falls through to subtitle-filter. + - `filter-subs`: Use the `subtitle-filter` library directly (`rm_fonts`, `rm_ast`, `rm_music`, `rm_effects`, `rm_names`, `rm_author`). + +- `strip_sdh`: Enable/disable automatic SDH stripping for tracks flagged as SDH. Default: `true`. + +- `convert_before_strip`: When falling through to the subtitle-filter path, auto-convert non-SRT subtitles to SRT first for better compatibility. Default: `true`. Has no effect when SubtitleEdit handles stripping directly. + +- `preserve_formatting`: Keep original subtitle tags and positioning during WebVTT processing. When `true`, sanitized WebVTT is written back without round-tripping through pycaption, preserving tags like ``, ``, and `line:` positioning. Default: `true`. + +- `output_mode`: Controls how subtitles are included in the output. Default: `mux`. + - `mux`: Embed subtitles in the MKV container only. + - `sidecar`: Save subtitles as separate files only (not muxed). + - `both`: Embed in the MKV container and save as sidecar files. + +- `sidecar_format`: Format for sidecar subtitle files (used when `output_mode` is `sidecar` or `both`). Default: `srt`. + - `srt`: SubRip. + - `vtt`: WebVTT. + - `ass`: Advanced SubStation Alpha. + - `original`: Keep the subtitle in its current format without conversion. + +Example: + +```yaml +subtitle: + conversion_method: auto + sdh_method: auto + strip_sdh: true + convert_before_strip: true + preserve_formatting: true + output_mode: mux + sidecar_format: srt +``` + +## WebVTT Sanitization (automatic, not configurable) + +After download, WebVTT and segmented WebVTT (`fVTT`/`WVTT`) tracks pass through a fixed sanitization pipeline before any conversion or muxing: + +1. **Segment merge** — segmented DASH/HLS WebVTT is stitched via `merge_segmented_webvtt` (uses pysubs2 for lenient parsing when `conversion_method` is `auto` or `pysubs2`, otherwise pycaption directly). +2. **Negative timestamps** — `sanitize_webvtt_timestamps` rewrites `-HH:MM:SS.mmm` cues to `00:00:00.000`. +3. **Cue identifiers** — `sanitize_webvtt_cue_identifiers` strips letter+digit IDs (e.g. `Q0`, `S12`) on their own line before a timing line, which otherwise confuse parsers like pysubs2. +4. **Overlapping cues** — `merge_overlapping_webvtt_cues` collapses cues with start times within 50 ms and matching end times into a single multi-line cue, ordered by `line:` percentage (lower % = higher on screen = first line). +5. **Fallback hardening** — when `preserve_formatting` is `false` and the first pycaption parse fails, `sanitize_webvtt` retries with a `WEBVTT` header guard, hour-padded timings, and another negative-timestamp pass; if that still fails, the sanitized text is written as-is. + +`sanitize_broken_webvtt` and `space_webvtt_headers` additionally run inside `Subtitle.parse()` to drop malformed `-->` lines and reflow merged-segment headers. `merge_same_cues` and `filter_unwanted_cues` (drops ` `/whitespace-only cues) run only on the pycaption path. + +These behaviors are intentional and have no config knobs — they apply to every WebVTT track regardless of `conversion_method`. + +## Related + +- Filename sanitization (e.g. parenthesis handling, unidecode bracket artifacts from PR #105) lives in `envied/core/utilities.py::sanitize_filename` and is governed by `output_template`, not the `subtitle:` config block. +- Subtitle codec support and the conversion matrix are defined in `envied/core/tracks/subtitle.py`. + +--- diff --git a/reset/envied_gui.py b/reset/envied_gui.py new file mode 100644 index 0000000..becf92e --- /dev/null +++ b/reset/envied_gui.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- +from PyQt6.QtWidgets import QWidget, QVBoxLayout, QPushButton, QFrame, QLabel +from PyQt6.QtCore import QProcess +from PyQt6.QtCore import Qt +import sys +import os +import subprocess +import shutil + + +CFG = os.path.abspath("./packages/envied/src/envied/envied.yaml") +class EnviedPanel(QFrame): + def __init__(self, parent=None): + super().__init__(parent) + self.setWindowTitle("Envied Control ") + # Frame styling: 1px white border (no pink) + self.setObjectName("enviedFrame") + self.setStyleSheet("#enviedFrame { background-color: #2D2D2D; border: 1px solid pink; }") + # ("color: #f5c2e7; border: none; background-color:#1E1E2E;padding: 5px;") + + # Layout + layout = QVBoxLayout(self) + layout.setContentsMargins(5, 5, 5, 5) + layout.setSpacing(5) + + + # Helper to make consistently styled buttons + def make_btn(text): + b = QPushButton(text) + b.setStyleSheet( + "background-color: #1E1E2E; \ + color: #f5c2e7; \ + border-width: 2px;\ + border-color: pink;\ + font: 14px;\ + min-width: 10em;\ + padding: 5px;" + ) + b.setAutoFillBackground(True) + b.setAttribute(Qt.WidgetAttribute.WA_StyledBackground, True) + return b + + # Buttons + btn_run = make_btn("run envied") + btn_check = make_btn("env check") + btn_info = make_btn("env info") + btn_cfg = make_btn("config") + + layout.addWidget(btn_run) + layout.addWidget(btn_check) + layout.addWidget(btn_info) + layout.addWidget(btn_cfg) + + # --- Wire up clicks --- + # Option A: non-blocking (recommended): use QProcess so the GUI stays responsive + btn_run.clicked.connect(lambda: open_prefilled_terminal()) + btn_check.clicked.connect(lambda: self.run_cmd(["uv", "run", "envied", "env", "check"])) + btn_info.clicked.connect(lambda: self.run_cmd(["uv", "run", "envied", "env", "info"])) + '''btn_cfg.clicked.connect( + lambda: os.system('nano ./packages/envied/src/envied/envied.yaml') if os.name == 'posix' + else os.system('notepad.exe ./packages/envied/src/envied/envied.yaml') + )''' + btn_cfg.clicked.connect(open_config_detached) + + # ---- Helpers ---- + def run_cmd(self, argv): + """ + Non-blocking run using QProcess. + """ + proc = QProcess(self) + # Optional: forward output to your terminal + proc.setProgram(argv[0]) + proc.setArguments(argv[1:]) + # Show output in your launching terminal (detach these if not needed) + proc.readyReadStandardOutput.connect(lambda p=proc: print(p.readAllStandardOutput().data().decode(), end="")) + proc.readyReadStandardError.connect(lambda p=proc: print(p.readAllStandardError().data().decode(), end="")) + proc.start() + + def run_script(self, what): + """ + Placeholder hook for your 'config' action. + e.g., open a config dialog or run a script. + """ + print(f"[EnviedPanel] run_script({what}) not implemented yet.") + +def get_terminal(): + terminals = ["gnome-terminal", "xterm", "konsole", "lxterminal", "xfce4-terminal"] + for term in terminals: + if shutil.which(term): + return term + elif os.__name__ =="nt": + TERMINALS = ["WindowsTerminal.exe", "OpenConsole.exe","powershell.exe", "Terminal.exe", "cmd.exe"] + for TERMINAL in TERMINALS: + if shutil.which(TERMINAL): + return TERMINAL + raise EnvironmentError("No suitable terminal emulator found.") + +def open_prefilled_terminal(): + prefill = 'uv run envied dl --select-titles ' + term = get_terminal() + + if term == "gnome-terminal": + # -l starts login shell; -c runs the command, then we keep the shell open with 'exec bash' + cmd = [ + "gnome-terminal", + "--", + "bash", "-lc", + # read: -e enables readline, -i sets initial text + f'read -e -i "{prefill}" cmd; eval "$cmd"; exec bash' + ] + elif term == "konsole": + cmd = [ + "konsole", "-e", + "bash", "-lc", + f'read -e -i "{prefill}" cmd; eval "$cmd"; exec bash' + ] + elif term =="lxterminal": # xterm fallback + cmd = [ + "xterm", "-e", + "bash", "-lc", + f'read -e -i "{prefill}" cmd; eval "$cmd"; exec bash' + ] + + elif os.name == "nt": + # --- Windows: open PowerShell (or Windows Terminal -> PowerShell) + # and pre-fill the command line using PSReadLine. + # This requires PSReadLine (present by default on modern Windows). + ps_prefill = prefill.replace('"', r'`"') # escape quotes for PowerShell + + if shutil.which("wt.exe"): + # Windows Terminal present: open a new tab running PowerShell + cmd = [ + "wt.exe", + "powershell", + "-NoExit", + "-Command", + f"[Microsoft.PowerShell.PSConsoleReadLine]::Insert(\"{ps_prefill}\")" + ] + elif shutil.which("powershell.exe"): + # Fallback: plain PowerShell + cmd = [ + "powershell.exe", + "-NoExit", + "-Command", + f"[Microsoft.PowerShell.PSConsoleReadLine]::Insert(\"{ps_prefill}\")" + ] + else: + # Last resort: cmd.exe can't truly prefill. Keep the window open and hint user. + cmd = [ + "cmd.exe", "/K", + f'echo Please use PowerShell for prefill. Intended command: {prefill}' + ] + + + + subprocess.Popen(cmd) + + +def open_config_detached(): + if os.name == "nt": + QProcess.startDetached("notepad.exe", [CFG]) + return + + # POSIX + # Easiest: open in the default GUI editor + if shutil.which("xdg-open"): + QProcess.startDetached("xdg-open", [CFG]) + return + + # If you insist on nano, run it *inside* a terminal emulator + # (xterm is very common and works out of the box) + if shutil.which("xterm"): + QProcess.startDetached("xterm", ["-e", "nano", CFG]) + return + + # Last resort (may fail without a TTY) + QProcess.startDetached("nano", [CFG]) + +if __name__ == "__main__": + from PyQt6.QtWidgets import QApplication + app = QApplication(sys.argv) + envied_panel = EnviedPanel() + window = envied_panel + window.show() + sys.exit(app.exec()) + + + + + subprocess.Popen(cmd) diff --git a/reset/gui.py b/reset/gui.py new file mode 100644 index 0000000..30f3261 --- /dev/null +++ b/reset/gui.py @@ -0,0 +1,449 @@ +import sys +from PyQt6.QtWidgets import ( + QApplication, QWidget, QVBoxLayout, QLabel, QLineEdit, QTextEdit, QPushButton, QCheckBox, QFrame, QMessageBox, QHBoxLayout +) +from PyQt6.QtCore import Qt, QThread, pyqtSignal +from PyQt6.QtGui import QPalette, QColor +import subprocess +import httpx +import base64 +import re +import urllib.parse +import xml.etree.ElementTree as ET +from pywidevine.cdm import Cdm +from pywidevine.device import Device +from pywidevine.pssh import PSSH +from base64 import b64encode +import codecs +from pathlib import Path +import os +import shlex + +WVD_PATH = "./WVDs/device.wvd" +WIDEVINE_SYSTEM_ID = 'EDEF8BA9-79D6-4ACE-A3C8-27DCD51D21ED' + +class DownloadThread(QThread): + finished = pyqtSignal() + error = pyqtSignal(str) + + def __init__(self, command): + super().__init__() + self.command = command + + def run(self): + try: + subprocess.run(self.command, check=True) + self.finished.emit() + except subprocess.CalledProcessError as e: + self.error.emit(str(e)) + +class AllHell3App(QWidget): + def __init__(self): + super().__init__() + self.init_ui() + + def init_ui(self): + self.setWindowTitle("HellYes!") + layout = QVBoxLayout() + + self.mpd_url_label = QLabel("MPD URL") + layout.addWidget(self.mpd_url_label) + self.mpd_url_entry = QLineEdit() + layout.addWidget(self.mpd_url_entry) + + self.curl_label = QLabel("cURL of License Request") + layout.addWidget(self.curl_label) + self.curl_text = QTextEdit() + layout.addWidget(self.curl_text) + + highlighted_frame = QFrame() + highlighted_layout = QVBoxLayout() + highlighted_frame.setLayout(highlighted_layout) + highlighted_frame.setStyleSheet("border: 1px solid red;") + + self.video_name_label = QLabel("Video Name") + highlighted_layout.addWidget(self.video_name_label) + self.video_name_entry = QLineEdit() + highlighted_layout.addWidget(self.video_name_entry) + + self.fetch_keys_button = QPushButton("Get Keys") + self.fetch_keys_button.clicked.connect(self.fetch_keys) + highlighted_layout.addWidget(self.fetch_keys_button) + + self.download_button = QPushButton("Download Nm~RE") + self.download_button.clicked.connect(self.download_video) + highlighted_layout.addWidget(self.download_button) + + self.download_button2 = QPushButton("Download DASH") + self.download_button2.clicked.connect(self.download_dash) + highlighted_layout.addWidget(self.download_button2) + + layout.addWidget(highlighted_frame) + + self.keys_output = QTextEdit() + layout.addWidget(self.keys_output) + + self.n_m3u8dl_re_label = QLabel("N_m3u8DL-RE command") + layout.addWidget(self.n_m3u8dl_re_label) + self.n_m3u8dl_re_output = QTextEdit() + layout.addWidget(self.n_m3u8dl_re_output) + + self.dash_mpd_cli_label = QLabel("Dash-MPD-CLI command") + layout.addWidget(self.dash_mpd_cli_label) + self.dash_mpd_cli_output = QTextEdit() + layout.addWidget(self.dash_mpd_cli_output) + + checkbox_layout = QHBoxLayout() + self.dark_mode_checkbox = QCheckBox("Dark Mode") + self.dark_mode_checkbox.setChecked(True) + self.dark_mode_checkbox.stateChanged.connect(self.toggle_dark_mode) + checkbox_layout.addWidget(self.dark_mode_checkbox) + + self.reset_button = QPushButton("Reset") + self.reset_button.clicked.connect(self.reset_fields) + checkbox_layout.addWidget(self.reset_button) + + layout.addLayout(checkbox_layout) + + self.setLayout(layout) + self.toggle_dark_mode() + + def reset_fields(self): + self.mpd_url_entry.clear() + self.curl_text.clear() + self.video_name_entry.clear() + self.keys_output.clear() + self.n_m3u8dl_re_output.clear() + self.dash_mpd_cli_output.clear() + + + def toggle_dark_mode(self): + if self.dark_mode_checkbox.isChecked(): + palette = QPalette() + palette.setColor(QPalette.ColorRole.Window, QColor(53, 53, 53)) + palette.setColor(QPalette.ColorRole.WindowText, Qt.GlobalColor.white) + palette.setColor(QPalette.ColorRole.Base, QColor(35, 35, 35)) + palette.setColor(QPalette.ColorRole.AlternateBase, QColor(53, 53, 53)) + palette.setColor(QPalette.ColorRole.ToolTipBase, Qt.GlobalColor.black) + palette.setColor(QPalette.ColorRole.ToolTipText, Qt.GlobalColor.white) + palette.setColor(QPalette.ColorRole.Text, Qt.GlobalColor.white) + palette.setColor(QPalette.ColorRole.Button, QColor(53, 53, 53)) + palette.setColor(QPalette.ColorRole.ButtonText, Qt.GlobalColor.white) + palette.setColor(QPalette.ColorRole.BrightText, Qt.GlobalColor.red) + palette.setColor(QPalette.ColorRole.Link, QColor(42, 130, 218)) + palette.setColor(QPalette.ColorRole.Highlight, QColor(42, 130, 218)) + palette.setColor(QPalette.ColorRole.HighlightedText, Qt.GlobalColor.black) + self.setPalette(palette) + + self.mpd_url_label.setStyleSheet("color: white;") + self.curl_label.setStyleSheet("color: white;") + self.video_name_label.setStyleSheet("color: white;") + self.n_m3u8dl_re_label.setStyleSheet("color: white;") + self.dash_mpd_cli_label.setStyleSheet("color: white;") + self.dark_mode_checkbox.setStyleSheet("color: white;") + self.fetch_keys_button.setStyleSheet("""color: white; + background-color: #4e4e4e; + border: 1px solid #6e6e6e; + padding: 5px;""") + self.download_button.setStyleSheet("""color: white; + background-color: #4e4e4e; + border: 1px solid #6e6e6e; + padding: 5px;""") + + self.download_button2.setStyleSheet("""color: white; + background-color: #4e4e4e; + border: 1px solid #6e6e6e; + padding: 5px;""") + + self.curl_text.setStyleSheet("""color: white; border: 1px solid red; + background-color: #232323;""") + self.video_name_label.setStyleSheet(""" + color: white; border: None;""") + self.mpd_url_entry.setStyleSheet("""color: white; border: 1px solid red; + background-color: #232323;""") + + else: + self.setPalette(QApplication.palette()) + + self.mpd_url_label.setStyleSheet("color: black;") + self.curl_label.setStyleSheet("color: black;") + self.video_name_label.setStyleSheet("color: black;") + self.n_m3u8dl_re_label.setStyleSheet("color: black;") + self.dash_mpd_cli_label.setStyleSheet("color: black;") + self.dark_mode_checkbox.setStyleSheet("color: black;") + self.fetch_keys_button.setStyleSheet("""color: black; + background-color: #aeaeae; + border: 1px solid #6e6e6e; + padding: 5px;""") + self.download_button.setStyleSheet("""color: black; + background-color: #aeaeae; + border: 1px solid #6e6e6e; + padding: 5px;""") + self.download_button2.setStyleSheet("""color: black; + background-color: #aeaeae; + border: 1px solid #6e6e6e; + padding: 5px;""") + + self.curl_text.setStyleSheet("""color: black; border: 1px solid red; + background-color: white;""") + self.video_name_label.setStyleSheet(""" + color: black; border: None;""") + self.mpd_url_entry.setStyleSheet("""color: black; border: 1px solid red; + background-color: white;""") + + def fetch_mpd_content(self, mpd_url): + response = httpx.get(mpd_url) + response.raise_for_status() + return response.text + + def find_default_kid_with_regex(self, mpd_content): + match = re.search(r'cenc:default_KID="([A-F0-9-]+)"', mpd_content) + if match: + return match.group(1) + return None + + def extract_or_generate_pssh(self, mpd_content): + try: + tree = ET.ElementTree(ET.fromstring(mpd_content)) + root = tree.getroot() + + namespaces = { + 'cenc': 'urn:mpeg:cenc:2013', + '': 'urn:mpeg:dash:schema:mpd:2011' + } + + default_kid = None + for elem in root.findall('.//ContentProtection', namespaces): + scheme_id_uri = elem.attrib.get('schemeIdUri', '').upper() + if scheme_id_uri == 'URN:MPEG:DASH:MP4PROTECTION:2011': + default_kid = elem.attrib.get('cenc:default_KID') + if default_kid: + break + + if not default_kid: + default_kid = self.find_default_kid_with_regex(mpd_content) + + pssh = None + for elem in root.findall('.//ContentProtection', namespaces): + scheme_id_uri = elem.attrib.get('schemeIdUri', '').upper() + if scheme_id_uri == f'URN:UUID:{WIDEVINE_SYSTEM_ID}': + pssh_elem = elem.find('cenc:pssh', namespaces) + if pssh_elem is not None: + pssh = pssh_elem.text + break + + if pssh is not None: + return pssh + elif default_kid is not None: + default_kid = default_kid.replace('-', '') + s = f'000000387073736800000000edef8ba979d64acea3c827dcd51d21ed000000181210{default_kid}48e3dc959b06' + return b64encode(bytes.fromhex(s)).decode() + + else: + # No pssh or default_KID found + try: + pssh = self.get_pssh_from_mpd(self.mpd_url_entry.text()) # init.m4f method + if pssh: + return pssh + else: + return None + except: + return None + + except ET.ParseError as e: + try: + pssh = self.get_pssh_from_mpd(self.mpd_url_entry.text()) # init.m4f method + return pssh + except: + pass + print(f"Error parsing MPD content: {e}") + return None + + + def get_pssh_from_mpd(self, mpd: str): + print("Extracting PSSH from MPD...") + + yt_dl = 'yt-dlp' + init = 'init.m4f' + + files_to_delete = [init] + + for file_name in files_to_delete: + if os.path.exists(file_name): + os.remove(file_name) + print(f"{file_name} file successfully deleted.") + + try: + subprocess.run([yt_dl, '-q', '--no-warning', '--test', '--allow-u', '-f', 'bestvideo[ext=mp4]/bestaudio[ext=m4a]/best', '-o', init, mpd]) + except FileNotFoundError: + print("yt-dlp not found. Trying installing it...") + exit(0) + + pssh_list = self.extract_pssh_from_file('init.m4f') + pssh = None + for target_pssh in pssh_list: + if 20 < len(target_pssh) < 220: + pssh = target_pssh + + print(f'\n{pssh}\n') + + for file_name in files_to_delete: + if os.path.exists(file_name): + os.remove(file_name) + print(f"{file_name} file successfully deleted.") + return pssh + + + + def extract_pssh_from_file(self,file_path: str) -> list: + print('Extracting PSSHs from init file:', file_path) + return self.to_pssh(Path(file_path).read_bytes()) + + def find_wv_pssh_offsets(self, raw: bytes) -> list: + offsets = [] + offset = 0 + while True: + offset = raw.find(b'pssh', offset) + if offset == -1: + break + size = int.from_bytes(raw[offset-4:offset], byteorder='big') + pssh_offset = offset - 4 + offsets.append(raw[pssh_offset:pssh_offset+size]) + offset += size + return offsets + + def to_pssh(self, content: bytes) -> list: + wv_offsets = self.find_wv_pssh_offsets(content) + return [base64.b64encode(wv_offset).decode() for wv_offset in wv_offsets] + + + + def parse_curl(self, curl_command): + url_match = re.search(r"curl\s+'(.*?)'", curl_command) + url = url_match.group(1) if url_match else "" + + method_match = re.search(r"-X\s+(\w+)", curl_command) + method = method_match.group(1) if method_match else "UNDEFINED" + + headers = {} + headers_matches = re.findall(r"-H\s+'([^:]+):\s*(.*?)'", curl_command) + for header in headers_matches: + headers[header[0]] = header[1] + + data_match = re.search(r"--data(?:-raw)?\s+(?:(\$?')|(\$?{?))(.*?)'", curl_command, re.DOTALL) + if data_match: + raw_prefix = data_match.group(1) + data = data_match.group(3) + if raw_prefix and raw_prefix.startswith('$'): + data = None + else: + data = data.replace('\\\\', '\\').replace('\\x', '\\\\x') + try: + data = codecs.decode(data, 'unicode_escape') + except Exception as e: + data = "" + else: + data = "" + return url, method, headers, data + + def get_key(self, pssh, license_url, headers, data): + device = Device.load(WVD_PATH) + cdm = Cdm.from_device(device) + session_id = cdm.open() + + challenge = cdm.get_license_challenge(session_id, PSSH(pssh)) + + if data: + if match := re.search(r'"(CAQ=.*?)"', data): + challenge = data.replace(match.group(1), base64.b64encode(challenge).decode()) + elif match := re.search(r'"(CAES.*?)"', data): + challenge = data.replace(match.group(1), base64.b64encode(challenge).decode()) + elif match := re.search(r'=(CAES.*?)(&.*)?$', data): + b64challenge = base64.b64encode(challenge).decode() + quoted = urllib.parse.quote_plus(b64challenge) + challenge = data.replace(match.group(1), quoted) + + payload = challenge if data is None else challenge + + license_response = httpx.post(url=license_url, data=payload, headers=headers) + try: + license_response.raise_for_status() + except httpx.HTTPStatusError as e: + raise e + + license_content = license_response.content + try: + match = re.search(r'"(CAIS.*?)"', license_response.content.decode('utf-8')) + if match: + license_content = base64.b64decode(match.group(1)) + except: + pass + + if isinstance(license_content, str): + license_content = base64.b64decode(license_content) + + cdm.parse_license(session_id, license_content) + + keys = [] + for key in cdm.get_keys(session_id): + if key.type == 'CONTENT': + keys.append(f"--key {key.kid.hex}:{key.key.hex()}") + + cdm.close(session_id) + return keys + + def fetch_keys(self): + mpd_url = self.mpd_url_entry.text() + try: + mpd_content = self.fetch_mpd_content(mpd_url) + pssh = self.extract_or_generate_pssh(mpd_content) + if not pssh: + QMessageBox.critical(self, "Error", "Failed to extract or generate PSSH\nAre you sure the video is Widevine encrypted?.") + return + + curl_command = self.curl_text.toPlainText().strip() + license_url, method, headers, data = self.parse_curl(curl_command) + keys = self.get_key(pssh, license_url, headers, data) + self.keys_output.clear() + self.keys_output.append("\n".join( keys)) + key_str = " ".join(keys) + + self.n_m3u8dl_re_command = f"N_m3u8DL-RE '{mpd_url}' {key_str} --save-name {self.video_name_entry.text()} -mt -M:format=mkv:muxer=mkvmerge" + self.n_m3u8dl_re_output.setText(self.n_m3u8dl_re_command) + + self.dash_mpd_cli_command = f"dash-mpd-cli --quality best --muxer-preference mkv:mkvmerge {key_str} \"{mpd_url}\" --write-subs --output '{self.video_name_entry.text()}.mkv'" + + self.dash_mpd_cli_output.setText(str(self.dash_mpd_cli_command)) + + except Exception as e: + QMessageBox.critical(self, "Error", str(e)) + + def download_video(self): + try: + self.thread = DownloadThread(shlex.split(self.n_m3u8dl_re_command)) + self.thread.finished.connect(self.on_download_finished) + self.thread.error.connect(self.on_download_error) + self.thread.start() + except Exception as e: + QMessageBox.critical(self, "Error", str(e)) + + def download_dash(self): + try: + self.thread = DownloadThread(shlex.split(self.dash_mpd_cli_command)) + self.thread.finished.connect(self.on_download_finished) + self.thread.error.connect(self.on_download_error) + self.thread.start() + except Exception as e: + QMessageBox.critical(self, "Error", str(e)) + + def on_download_finished(self): + QMessageBox.information(self, "Success", "Download finished successfully.") + + def on_download_error(self, error_message): + QMessageBox.critical(self, "Error", f"Download failed: {error_message}") + +if __name__ == "__main__": + app = QApplication(sys.argv) + window = AllHell3App() + window.show() + sys.exit(app.exec()) diff --git a/reset/packages/envied/CONFIG.md b/reset/packages/envied/CONFIG.md new file mode 100644 index 0000000..499e234 --- /dev/null +++ b/reset/packages/envied/CONFIG.md @@ -0,0 +1,582 @@ +# Config Documentation + +This page documents configuration values and what they do. You begin with an empty configuration file. +You may alter your configuration with `envied cfg --help`, or find the direct location with `envied env info`. +Configuration values are listed in alphabetical order. + +Avoid putting comments in the config file as they may be removed. Comments are currently kept only thanks +to the usage of `ruamel.yaml` to parse and write YAML files. In the future `yaml` may be used instead, +which does not keep comments. + +## aria2c (dict) + +- `max_concurrent_downloads` + Maximum number of parallel downloads. Default: `min(32,(cpu_count+4))` + Note: Overrides the `max_workers` parameter of the aria2(c) downloader function. +- `max_connection_per_server` + Maximum number of connections to one server for each download. Default: `1` +- `split` + Split a file into N chunks and download each chunk on its own connection. Default: `5` +- `file_allocation` + Specify file allocation method. Default: `"prealloc"` + + - `"none"` doesn't pre-allocate file space. + - `"prealloc"` pre-allocates file space before download begins. This may take some time depending on the size of the + file. + - `"falloc"` is your best choice if you are using newer file systems such as ext4 (with extents support), btrfs, xfs + or NTFS (MinGW build only). It allocates large(few GiB) files almost instantly. Don't use falloc with legacy file + systems such as ext3 and FAT32 because it takes almost same time as prealloc, and it blocks aria2 entirely until + allocation finishes. falloc may not be available if your system doesn't have posix_fallocate(3) function. + - `"trunc"` uses ftruncate(2) system call or platform-specific counterpart to truncate a file to a specified length. + +## cdm (dict) + +Pre-define which Widevine or PlayReady device to use for each Service by Service Tag as Key (case-sensitive). +The value should be a WVD or PRD filename without the file extension. When +loading the device, envied will look in both the `WVDs` and `PRDs` directories +for a matching file. + +For example, + +```yaml +AMZN: chromecdm_903_l3 +NF: nexus_6_l1 +``` + +You may also specify this device based on the profile used. + +For example, + +```yaml +AMZN: chromecdm_903_l3 +NF: nexus_6_l1 +DSNP: + john_sd: chromecdm_903_l3 + jane_uhd: nexus_5_l1 +``` + +You can also specify a fallback value to predefine if a match was not made. +This can be done using `default` key. This can help reduce redundancy in your specifications. + +For example, the following has the same result as the previous example, as well as all other +services and profiles being pre-defined to use `chromecdm_903_l3`. + +```yaml +NF: nexus_6_l1 +DSNP: + jane_uhd: nexus_5_l1 +default: chromecdm_903_l3 +``` + +## chapter_fallback_name (str) + +The Chapter Name to use when exporting a Chapter without a Name. +The default is no fallback name at all and no Chapter name will be set. + +The fallback name can use the following variables in f-string style: + +- `{i}`: The Chapter number starting at 1. + E.g., `"Chapter {i}"`: "Chapter 1", "Intro", "Chapter 3". +- `{j}`: A number starting at 1 that increments any time a Chapter has no title. + E.g., `"Chapter {j}"`: "Chapter 1", "Intro", "Chapter 2". + +These are formatted with f-strings, directives are supported. +For example, `"Chapter {i:02}"` will result in `"Chapter 01"`. + +## credentials (dict[str, str|list|dict]) + +Specify login credentials to use for each Service, and optionally per-profile. + +For example, + +```yaml +ALL4: jane@gmail.com:LoremIpsum100 # directly +AMZN: # or per-profile, optionally with a default + default: jane@example.tld:LoremIpsum99 # <-- used by default if -p/--profile is not used + james: james@gmail.com:TheFriend97 + john: john@example.tld:LoremIpsum98 +NF: # the `default` key is not necessary, but no credential will be used by default + john: john@gmail.com:TheGuyWhoPaysForTheNetflix69420 +``` + +The value should be in string form, i.e. `john@gmail.com:password123` or `john:password123`. +Any arbitrary values can be used on the left (username/password/phone) and right (password/secret). +You can also specify these in list form, i.e., `["john@gmail.com", ":PasswordWithAColon"]`. + +If you specify multiple credentials with keys like the `AMZN` and `NF` example above, then you should +use a `default` key or no credential will be loaded automatically unless you use `-p/--profile`. You +do not have to use a `default` key at all. + +Please be aware that this information is sensitive and to keep it safe. Do not share your config. + +## curl_impersonate (dict) + +- `browser` - The Browser to impersonate as. A list of available Browsers and Versions are listed here: + + + Default: `"chrome124"` + +For example, + +```yaml +curl_impersonate: + browser: "chrome120" +``` + +## directories (dict) + +Override the default directories used across envied. +The directories are set to common values by default. + +The following directories are available and may be overridden, + +- `commands` - CLI Command Classes. +- `services` - Service Classes. +- `vaults` - Vault Classes. +- `fonts` - Font files (ttf or otf). +- `downloads` - Downloads. +- `temp` - Temporary files or conversions during download. +- `cache` - Expiring data like Authorization tokens, or other misc data. +- `cookies` - Expiring Cookie data. +- `logs` - Logs. +- `wvds` - Widevine Devices. +- `prds` - PlayReady Devices. + +For example, + +```yaml +downloads: "D:/Downloads/envied" +temp: "D:/Temp/envied" +``` + +There are directories not listed that cannot be modified as they are crucial to the operation of envied. + +## dl (dict) + +Pre-define default options and switches of the `dl` command. +The values will be ignored if explicitly set in the CLI call. + +The Key must be the same value Python click would resolve it to as an argument. +E.g., `@click.option("-r", "--range", "range_", type=...` actually resolves as `range_` variable. + +For example to set the default primary language to download to German, + +```yaml +lang: de +``` + +to set how many tracks to download concurrently to 4 and download threads to 16, + +```yaml +downloads: 4 +workers: 16 +``` + +to set `--bitrate=CVBR` for the AMZN service, + +```yaml +lang: de +AMZN: + bitrate: CVBR +``` + +or to change the output subtitle format from the default (original format) to WebVTT, + +```yaml +sub_format: vtt +``` + +## downloader (str | dict) + +Choose what software to use to download data throughout envied where needed. +You may provide a single downloader globally or a mapping of service tags to +downloaders. + +Options: + +- `requests` (default) - +- `aria2c` - +- `curl_impersonate` - (via ) +- `n_m3u8dl_re` - + +Note that aria2c can reach the highest speeds as it utilizes threading and more connections than the other downloaders. However, aria2c can also be one of the more unstable downloaders. It will work one day, then not another day. It also does not support HTTP(S) proxies while the other downloaders do. + +Example mapping: + +```yaml +downloader: + NF: requests + AMZN: n_m3u8dl_re + DSNP: n_m3u8dl_re + default: requests +``` + +The `default` entry is optional. If omitted, `requests` will be used for services not listed. + +## decryption (str | dict) + +Choose what software to use to decrypt DRM-protected content throughout envied where needed. +You may provide a single decryption method globally or a mapping of service tags to +decryption methods. + +Options: + +- `shaka` (default) - Shaka Packager - +- `mp4decrypt` - mp4decrypt from Bento4 - + +Note that Shaka Packager is the traditional method and works with most services. mp4decrypt +is an alternative that may work better with certain services that have specific encryption formats. + +Example mapping: + +```yaml +decryption: + ATVP: mp4decrypt + AMZN: shaka + default: shaka +``` + +The `default` entry is optional. If omitted, `shaka` will be used for services not listed. + +Simple configuration (single method for all services): + +```yaml +decryption: mp4decrypt +``` + +## filenames (dict) + +Override the default filenames used across envied. +The filenames use various variables that are replaced during runtime. + +The following filenames are available and may be overridden: + +- `log` - Log filenames. Uses `{name}` and `{time}` variables. +- `config` - Service configuration filenames. +- `root_config` - Root configuration filename. +- `chapters` - Chapter export filenames. Uses `{title}` and `{random}` variables. +- `subtitle` - Subtitle export filenames. Uses `{id}` and `{language}` variables. + +For example, + +```yaml +filenames: + log: "envied_{name}_{time}.log" + config: "config.yaml" + root_config: "envied.yaml" + chapters: "Chapters_{title}_{random}.txt" + subtitle: "Subtitle_{id}_{language}.srt" +``` + +## headers (dict) + +Case-Insensitive dictionary of headers that all Services begin their Request Session state with. +All requests will use these unless changed explicitly or implicitly via a Server response. +These should be sane defaults and anything that would only be useful for some Services should not +be put here. + +Avoid headers like 'Accept-Encoding' as that would be a compatibility header that Python-requests will +set for you. + +I recommend using, + +```yaml +Accept-Language: "en-US,en;q=0.8" +User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36" +``` + +## key_vaults (list\[dict]) + +Key Vaults store your obtained Content Encryption Keys (CEKs) and Key IDs per-service. + +This can help reduce unnecessary License calls even during the first download. This is because a Service may +provide the same Key ID and CEK for both Video and Audio, as well as for multiple resolutions or bitrates. + +You can have as many Key Vaults as you would like. It's nice to share Key Vaults or use a unified Vault on +Teams as sharing CEKs immediately can help reduce License calls drastically. + +Three types of Vaults are in the Core codebase, API, SQLite and MySQL. API makes HTTP requests to a RESTful API, +whereas SQLite and MySQL directly connect to an SQLite or MySQL Database. + +Note: SQLite and MySQL vaults have to connect directly to the Host/IP. It cannot be in front of a PHP API or such. +Beware that some Hosting Providers do not let you access the MySQL server outside their intranet and may not be +accessible outside their hosting platform. + +### Using an API Vault + +API vaults use a specific HTTP request format, therefore API or HTTP Key Vault APIs from other projects or services may +not work in envied.The API format can be seen in the [API Vault Code](envied/vaults/API.py). + +```yaml +- type: API + name: "John#0001's Vault" # arbitrary vault name + uri: "https://key-vault.example.com" # api base uri (can also be an IP or IP:Port) + # uri: "127.0.0.1:80/key-vault" + # uri: "https://api.example.com/key-vault" + token: "random secret key" # authorization token +``` + +### Using a MySQL Vault + +MySQL vaults can be either MySQL or MariaDB servers. I recommend MariaDB. +A MySQL Vault can be on a local or remote network, but I recommend SQLite for local Vaults. + +```yaml +- type: MySQL + name: "John#0001's Vault" # arbitrary vault name + host: "127.0.0.1" # host/ip + # port: 3306 # port (defaults to 3306) + database: vault # database used for envied + username: jane11 + password: Doe123 +``` + +I recommend giving only a trustable user (or yourself) CREATE permission and then use envied to cache at least one CEK +per Service to have it create the tables. If you don't give any user permissions to create tables, you will need to +make tables yourself. + +- Use a password on all user accounts. +- Never use the root account with envied (even if it's you). +- Do not give multiple users the same username and/or password. +- Only give users access to the database used for envied. +- You may give trusted users CREATE permission so envied can create tables if needed. +- Other uses should only be given SELECT and INSERT permissions. + +### Using an SQLite Vault + +SQLite Vaults are usually only used for locally stored vaults. This vault may be stored on a mounted Cloud storage +drive, but I recommend using SQLite exclusively as an offline-only vault. Effectively this is your backup vault in +case something happens to your MySQL Vault. + +```yaml +- type: SQLite + name: "My Local Vault" # arbitrary vault name + path: "C:/Users/Jane11/Documents/envied/data/key_vault.db" +``` + +**Note**: You do not need to create the file at the specified path. +SQLite will create a new SQLite database at that path if one does not exist. +Try not to accidentally move the `db` file once created without reflecting the change in the config, or you will end +up with multiple databases. + +If you work on a Team I recommend every team member having their own SQLite Vault even if you all use a MySQL vault +together. + +## muxing (dict) + +- `set_title` + Set the container title to `Show SXXEXX Episode Name` or `Movie (Year)`. Default: `true` + +## n_m3u8dl_re (dict) + +Configuration for N_m3u8DL-RE downloader. This downloader is particularly useful for HLS streams. + +- `thread_count` + Number of threads to use for downloading. Default: Uses the same value as max_workers from the command. +- `ad_keyword` + Keyword to identify and potentially skip advertisement segments. Default: `None` +- `use_proxy` + Whether to use proxy when downloading. Default: `true` + +For example, + +```yaml +n_m3u8dl_re: + thread_count: 16 + ad_keyword: "advertisement" + use_proxy: true +``` + +## nordvpn (dict) + +**Legacy configuration. Use `proxy_providers.nordvpn` instead.** + +Set your NordVPN Service credentials with `username` and `password` keys to automate the use of NordVPN as a Proxy +system where required. + +You can also specify specific servers to use per-region with the `servers` key. +Sometimes a specific server works best for a service than others, so hard-coding one for a day or two helps. + +For example, + +```yaml +nordvpn: + username: zxqsR7C5CyGwmGb6KSvk8qsZ # example of the login format + password: wXVHmht22hhRKUEQ32PQVjCZ + servers: + - us: 12 # force US server #12 for US proxies +``` + +The username and password should NOT be your normal NordVPN Account Credentials. +They should be the `Service credentials` which can be found on your Nord Account Dashboard. + +Note that `gb` is used instead of `uk` to be more consistent across regional systems. + +## proxy_providers (dict) + +Enable external proxy provider services. These proxies will be used automatically where needed as defined by the +Service's GEOFENCE class property, but can also be explicitly used with `--proxy`. You can specify which provider +to use by prefixing it with the provider key name, e.g., `--proxy basic:de` or `--proxy nordvpn:de`. Some providers +support specific query formats for selecting a country/server. + +### basic (dict[str, str|list]) + +Define a mapping of country to proxy to use where required. +The keys are region Alpha 2 Country Codes. Alpha 2 Country Codes are `[a-z]{2}` codes, e.g., `us`, `gb`, and `jp`. +Don't get this mixed up with language codes like `en` vs. `gb`, or `ja` vs. `jp`. + +Do note that each key's value can be a list of strings, or a string. For example, + +```yaml +us: + - "http://john%40email.tld:password123@proxy-us.domain.tld:8080" + - "http://jane%40email.tld:password456@proxy-us.domain2.tld:8080" +de: "https://127.0.0.1:8080" +``` + +Note that if multiple proxies are defined for a region, then by default one will be randomly chosen. +You can choose a specific one by specifying it's number, e.g., `--proxy basic:us2` will choose the +second proxy of the US list. + +### nordvpn (dict) + +Set your NordVPN Service credentials with `username` and `password` keys to automate the use of NordVPN as a Proxy +system where required. + +You can also specify specific servers to use per-region with the `servers` key. +Sometimes a specific server works best for a service than others, so hard-coding one for a day or two helps. + +For example, + +```yaml +username: zxqsR7C5CyGwmGb6KSvk8qsZ # example of the login format +password: wXVHmht22hhRKUEQ32PQVjCZ +servers: + - us: 12 # force US server #12 for US proxies +``` + +The username and password should NOT be your normal NordVPN Account Credentials. +They should be the `Service credentials` which can be found on your Nord Account Dashboard. + +Once set, you can also specifically opt in to use a NordVPN proxy by specifying `--proxy=gb` or such. +You can even set a specific server number this way, e.g., `--proxy=gb2366`. + +Note that `gb` is used instead of `uk` to be more consistent across regional systems. + +### hola (dict) + +Enable Hola VPN proxy service. This is a simple provider that doesn't require configuration. + +For example, + +```yaml +proxy_providers: + hola: {} +``` + +Note: Hola VPN is automatically enabled when proxy_providers is configured, no additional setup is required. + +## remote_cdm (list\[dict]) + +Use [pywidevine] Serve-compliant Remote CDMs in envied as if it was a local widevine device file. +The name of each defined device maps as if it was a local device and should be used like a local device. + +For example, + +```yaml +- name: chromecdm_903_l3 # name must be unique for each remote CDM + # the device type, system id and security level must match the values of the device on the API + # if any of the information is wrong, it will raise an error, if you do not know it ask the API owner + device_type: CHROME + system_id: 1234 + security_level: 3 + host: "http://xxxxxxxxxxxxxxxx/the_cdm_endpoint" + secret: "secret/api key" + device_name: "remote device to use" # the device name from the API, usually a wvd filename +``` + +[pywidevine]: https://github.com/rlaphoenix/pywidevine + +## serve (dict) + +Configuration data for pywidevine's serve functionality run through envied. +This effectively allows you to run `envied serve` to start serving pywidevine Serve-compliant CDMs right from your +local widevine device files. + +For example, + +```yaml +users: + secret_key_for_jane: # 32bit hex recommended, case-sensitive + devices: # list of allowed devices for this user + - generic_nexus_4464_l3 + username: jane # only for internal logging, users will not see this name + secret_key_for_james: + devices: + - generic_nexus_4464_l3 + username: james + secret_key_for_john: + devices: + - generic_nexus_4464_l3 + username: john +# devices can be manually specified by path if you don't want to add it to +# envied's WVDs directory for whatever reason +# devices: +# - 'C:\Users\john\Devices\test_devices_001.wvd' +``` + +## services (dict) + +Configuration data for each Service. The Service will have the data within this section merged into the `config.yaml` +before provided to the Service class. + +Think of this config to be used for more sensitive configuration data, like user or device-specific API keys, IDs, +device attributes, and so on. A `config.yaml` file is typically shared and not meant to be modified, so use this for +any sensitive configuration data. + +The Key is the Service Tag, but can take any arbitrary form for its value. It's expected to begin as either a list or +a dictionary. + +For example, + +```yaml +NOW: + client: + auth_scheme: MESSO + # ... more sensitive data +``` + +## set_terminal_bg (bool) + +Controls whether envied should set the terminal background color. Default: `false` + +For example, + +```yaml +set_terminal_bg: true +``` + +## tag (str) + +Group or Username to postfix to the end of all download filenames following a dash. +For example, `tag: "J0HN"` will have `-J0HN` at the end of all download filenames. + +## tmdb_api_key (str) + +API key for The Movie Database (TMDB). This is used for tagging downloaded files with TMDB, +IMDB and TVDB identifiers. Leave empty to disable automatic lookups. + +To obtain a TMDB API key: + +1. Create an account at +2. Go to to register for API access +3. Fill out the API application form with your project details +4. Once approved, you'll receive your API key + +For example, + +```yaml +tmdb_api_key: cf66bf18956kca5311ada3bebb84eb9a # Not a real key +``` + +**Note**: Keep your API key secure and do not share it publicly. This key is used by the core/utils/tags.py module to fetch metadata from TMDB for proper file tagging. diff --git a/reset/packages/envied/LICENSE b/reset/packages/envied/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/reset/packages/envied/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/reset/packages/envied/README.md b/reset/packages/envied/README.md new file mode 100644 index 0000000..c95ec64 --- /dev/null +++ b/reset/packages/envied/README.md @@ -0,0 +1,76 @@ +# Envied in twinvine + +**Note** + +*Envied has already been installed by TwinVine. These notes are retained for information only* + + +## What is envied? + +Envied is a fork of [Devine](https://github.com/devine-dl/devine/). The name 'envied' is an anagram of Devine, and as such, pays homage to the original author. +Is is based on v 1.4.7 of envied.It is a powerful archival tool for downloading movies, TV shows, and music from streaming services. Built with a focus on modularity and extensibility, it provides a robust framework for content acquisition with support for DRM-protected content. + +No commands have been changed 'uv run envied' still works as usual. + +The major difference is that envied comes complete and needs little configuration. +A basic CDM and services are taken care of. +The prime reason for the existence of envied is a --select-titles function. + +If you already use envied you'll probably just want to replace envied/envied/envied.yaml +with your own. But the exisiting yaml is close to working - just needs a few directory locations. + +Envied's existence helps keep download tools free. Unshackle is flirting with pay-for access, having a free, closley matched, aternative puts a break on any pay-me aspirations unshackle might have now or in the future. + +## Divergence from Envied's Parent +- **select-titles option** avoid the uncertainty of -w S26E03 gobbledegook to get your video +- **Singleton Design Pattern** Good coding practice: where possible, a single instance of a Class is created and re-used. Saving time and resources. +- **Multitron Design Pattern** For those times when a Singleton will not do. Re-use Classes with care for the calling parameters. +- **Clear Branding** Clear presentation of the program name! +- **No Free Ride for Spammers** Envied will not implement any methods used to connect to Decrypt Labs or other payfor-CDM-access sites + + + **Recommended:** Use `uv run envied` instead of direct command execution to ensure proper virtual environment activation. + + +### Basic Usage + +```shell + + +# Download content (requires configured services) +# from inside the TwinVine top level folder:- +uv run envied dl SERVICE_NAME CONTENT_ID +``` + +## Documentation + +For comprehensive setup guides, configuration options, and advanced usage: + +📖 **[Visit their WIKI](https://github.com/unshackle-dl/unshackle/wiki)** + +The WIKI contains detailed information on: + +- Service configuration +- DRM configuration +- Advanced features and troubleshooting + +For guidance on creating services, see their [WIKI documentation](https://github.com/unshackle-dl/unshackle/wiki). + +## End User License Agreement + +Envied, and it's community pages, should be treated with the same kindness as other projects. +Please refrain from spam or asking for questions that infringe upon a Service's End User License Agreement. + +1. Do not use envied for any purposes of which you do not have the rights to do so. +2. Do not share or request infringing content; this includes widevine Provision Keys, Content Encryption Keys, + or Service API Calls or Code. +3. The Core codebase is meant to stay Free and Open-Source while the Service code should be kept private. +4. Do not sell any part of this project, neither alone nor as part of a bundle. + If you paid for this software or received it as part of a bundle following payment, you should demand your money + back immediately. +5. Be kind to one another and do not single anyone out. + +## Licensing + +This software is licensed under the terms of [GNU General Public License, Version 3.0](LICENSE). +You can find a copy of the license in the LICENSE file in the root folder. diff --git a/reset/packages/envied/img/envied1.png b/reset/packages/envied/img/envied1.png new file mode 100644 index 0000000..7a62320 Binary files /dev/null and b/reset/packages/envied/img/envied1.png differ diff --git a/reset/packages/envied/pyproject.toml b/reset/packages/envied/pyproject.toml new file mode 100644 index 0000000..67eb40e --- /dev/null +++ b/reset/packages/envied/pyproject.toml @@ -0,0 +1,133 @@ +[build-system] +requires = ["uv_build>=0.8.12,<0.9.21"] +build-backend = "uv_build" + +[project] +name = "envied" +version = "5.3.0" +description = "Modular Movie, TV, and Music Archival Software." +authors = [{ name = "rlaphoenix and others" }] +requires-python = ">=3.10,<=3.14" +readme = "README.md" +license = "GPL-3.0-only" +keywords = [ + "python", + "downloader", + "drm", + "widevine", +] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: End Users/Desktop", + "Natural Language :: English", + "Operating System :: OS Independent", + "Topic :: Multimedia :: Video", + "Topic :: Security :: Cryptography", +] +dependencies = [ + "appdirs>=1.4.4,<2", + "Brotli>=1.2.0,<2", + "click>=8.1.8,<9", + "construct>=2.8.8,<3", + "crccheck>=1.3.0,<2", + "filelock>=3.20.3,<4", + "fonttools>=4.60.2,<5", + "jsonpickle>=3.0.4,<5", + "langcodes>=3.4.0,<4", + "lxml>=5.2.1,<7", + "protobuf>=4.25.3,<7", + "pycaption>=2.2.6,<3", + "pycryptodomex>=3.20.0,<4", + "pyjwt>=2.12.0,<3", + "pymediainfo>=6.1.0,<8", + "pymp4>=1.4.0,<2", + "pymysql>=1.1.0,<2", + "pywidevine[serve]>=1.8.0,<2", + "PyYAML>=6.0.1,<7", + "requests[socks]>=2.32.5,<3", + "rich>=13.7.1,<15.1", + "rlaphoenix.m3u8>=3.4.0,<4", + "ruamel.yaml>=0.18.6,<0.19", + "sortedcontainers>=2.4.0,<3", + "subtitle-filter>=1.4.9,<2", + "Unidecode>=1.3.8,<2", + "urllib3>=2.6.3,<3", + "chardet>=5.2.0,<6", + "pyplayready>=0.8.3,<0.9", + "cryptography>=45.0.0,<47", + "subby", + "aiohttp>=3.13.4,<4", + "aiohttp-swagger3>=0.9.0,<1", + "pysubs2>=1.7.0,<2", + "PyExecJS>=1.5.1,<2", + "pycountry>=24.6.1", + "language-data>=1.4.0", + "wasmtime>=41.0.0", + "animeapi-py>=0.6.0", + "rnet>=2.4.2", + "bandit>=1.9.4", + "defusedxml>=0.7.1", + "jsonpath-ng>=1.8.0", + "mutagen>=1.47.0", + # envied specific dependencies + "webvtt-py>=0.5.1,<0.6", + "isodate>=0.7.2,<0.8", + "scrapy>=2.14.0,<3", + "mypy>=1.19.1,<=2.1.0", + "httpx>=0.28.0,<0.35", + + + # Keeping your existing cap style here; latest visible is 0.10.0. + #"aiohttp-swagger3>=0.10.0,<1", +] + + +[project.urls] +Homepage = "https://github.com/envied-dl/envied" +Repository = "https://github.com/envied-dl/envied" +Issues = "https://github.com/envied-dl/envied/issues" +Discussions = "https://github.com/envied-dl/envied/discussions" +Changelog = "https://github.com/envied-dl/envied/blob/master/CHANGELOG.md" + +[project.scripts] +envied = "envied.core.__main__:main" + +[dependency-groups] +dev = [ + "pre-commit>=3.7.0,<4", + "mypy>=1.18.2,<2", + "mypy-protobuf>=3.6.0,<4", + "types-protobuf>=4.24.0.20240408,<5", + "types-PyMySQL>=1.1.0.1,<2", + "types-requests>=2.31.0.20240406,<3", + "isort>=5.13.2,<6", + "ruff~=0.3.7" +] + +#[tool.hatch.build.targets.wheel] +#packages = ["envied"] + + +[tool.ruff] +force-exclude = true +line-length = 120 + +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "W"] + +[tool.isort] +line_length = 118 + +[tool.mypy] +check_untyped_defs = true +disallow_incomplete_defs = true +disallow_untyped_defs = true +follow_imports = "silent" +ignore_missing_imports = true +no_implicit_optional = true + +[tool.uv.sources] +#subby = { git = "https://github.com/vevv/subby.git" } +subby = { git = "https://github.com/vevv/subby.git", rev = "a057280f31acd3a0ba5c6dc69eaefbb88aea974e" } + diff --git a/reset/packages/envied/src/envied/CHANGELOG.md b/reset/packages/envied/src/envied/CHANGELOG.md new file mode 100644 index 0000000..8a1bbce --- /dev/null +++ b/reset/packages/envied/src/envied/CHANGELOG.md @@ -0,0 +1,747 @@ +# Changelog + +All notable changes to this project will be documented in this file. + + +## 5.3.0 01-07-2026 + +Repository: unshackle-dl/unshackle · Tag: 5.3.0 · Commit: d4a5826 · Released by: imSp4rky +Features + + --merge-video: merge video language variants into one MKV per (resolution, range, codec) group + Native ExpressVPN HTTPS proxy provider (full OAuth PKCE flow, no external tools) + Proton VPN proxy provider with country/city/server-number selection, plus ProtonVPN TV-code login + Load service plugins from git repos (clone + TTL-pull cache, refresh-services command, dirty-clone guard) + Python 3.14 support (supported range widened to 3.11-3.14) + Automatic Firefox cookie and localStorage extraction + Nested directories in folder templates (/ treated as path separator) + air_date for date-based episode naming (daily/sports content named by date instead of SxxExx) + Per-vault network timeout for remote API/HTTP vaults + Richer API job progress: active-track segment counts, transfer speed, title description/date/cover_url + +Bug Fixes + + drm: don't switch to a mismatched CDM type during licensing; raise a clear error instead + config: match per-service cdm keys case-insensitively + dl: apply per-service decryption tool override before decrypt + hls: stream segment merges to avoid OOM on large (multi-GB) tracks + console: pause active live/spinner contexts during terminal input so prompts stay visible + output: treat both / and \ as folder separators on any OS + env: only resolve Path items in directory lists (git repo specs pass through) + +Performance & Changes + + perf(hls): rename single decrypted range instead of re-copying a full track + perf(tracks): skip redundant DV/VUI bitstream passes when VUI already matches range + api: collapse route registration, dedup handlers, remove dead serve/remote code paths + + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [2.3.0] - 2026-01-18 + +### Added + +- **Unicode Filenames Option**: New `unicode_filenames` config option to preserve native characters + - Allows disabling ASCII transliteration in filenames + - Preserves Korean, Japanese, Chinese, and other native language characters + - Closes #49 + +### Fixed + +- **WebVTT Cue Handling**: Handle WebVTT cue identifiers and overlapping multi-line cues + - Added detection and sanitization for cue identifiers (Q0, Q1, etc.) before timing lines + - Added merging of overlapping cues with different line positions into multi-line subtitles + - Fixes parsing issues with pysubs2/pycaption on certain WebVTT files +- **Widevine PSSH Filtering**: Filter Widevine PSSH by system ID instead of sorting + - Fixes KeyError crash when unsupported DRM systems are present in init segments +- **TTML Negative Values**: Handle negative values in multi-value TTML attributes + - Fixes pycaption parse errors for attributes like `tts:extent="-5% 7.5%"` + - Closes #47 +- **ASS Font Names**: Strip whitespace from ASS font names + - Handles ASS subtitle files with spaces after commas in Style definitions + - Fixes #57 +- **Shaka-Packager Error Messages**: Include shaka-packager binary path in error messages +- **N_m3u8DL-RE Merge and Decryption**: Handle merge and decryption properly + - Prevents audio corruption ("Box 'OG 2' size is too large") with DASH manifests + - Fixes duplicate init segment writing when using N_m3u8DL-RE +- **DASH Placeholder KIDs**: Handle placeholder KIDs and improve DRM init from segments + - Detects and replaces placeholder/test KIDs in Widevine PSSH + - Adds CENC namespace support for kid/default_KID attributes +- **PlayReady PSSH Comparison**: Correct PSSH system ID comparison in PlayReady + - Removes erroneous `.bytes` accessor from PSSH.SYSTEM_ID comparisons + +## [2.2.0] - 2026-01-15 + +### Added + +- **CDM-Aware PlayReady Fallback Detection**: Intelligent DRM fallback based on selected CDM + - Adds PlayReady PSSH/KID extraction from track and init data with CDM-aware ordering + - When PlayReady CDM is selected, tries PlayReady first then falls back to Widevine + - When Widevine CDM is selected (default), tries Widevine first then falls back to PlayReady +- **Comprehensive Debug Logging**: Enhanced debug logging for downloaders and muxing + - Added detailed debug logging to aria2c, curl_impersonate, n_m3u8dl_re, and requests downloaders + - Enhanced manifest parsers (DASH, HLS, ISM) with debug logging + - Added debug logging to track muxing operations + +### Fixed + +- **Hybrid DV+HDR10 Filename Detection**: Fixed HDR10 detection in hybrid Dolby Vision filenames + - Hybrid DV+HDR10 files were incorrectly named "DV.H.265" instead of "DV.HDR.H.265" + - Now checks both `hdr_format_full` and `hdr_format_commercial` fields for HDR10 indicators +- **Vault Adaptive Batch Sizing**: Improved bulk key operations with adaptive batch sizing + - Prevents query limit issues when retrieving large numbers of keys from vaults + - Dynamically adjusts batch sizes based on vault response characteristics +- **Test Command Improvements**: Enhanced test command error detection and sorting + - Improved error detection in test command output + - Added natural sorting for test results + +## [2.1.0] - 2025-11-27 + +### Added + +- **Per-Track Quality-Based CDM Selection**: Dynamic CDM switching during runtime DRM operations + - Enables quality-based CDM selection during runtime DRM switching + - Different CDMs can be used for different video quality levels within the same download session + - Example: Use Widevine L3 for SD/HD and PlayReady SL3 for 4K content +- **Enhanced Track Export**: Improved export functionality with additional metadata + - Added URL field to track export for easier identification + - Added descriptor information in export output + - Keys now exported in hex-formatted strings + +### Changed + +- **Dependencies**: Upgraded to latest compatible versions + - Updated various dependencies to their latest versions + +### Fixed + +- **Attachment Preservation**: Fixed attachments being dropped during track filtering + - Attachments (screenshots, fonts) were being lost when track list was rebuilt + - Fixes image files remaining in temp directory after muxing +- **DASH BaseURL Resolution**: Added AdaptationSet-level BaseURL support per DASH spec + - URL resolution chain now properly follows: MPD → Period → AdaptationSet → Representation +- **WindscribeVPN Region Support**: Restricted to supported regions with proper error handling + - Added error handling for unsupported regions in get_proxy method + - Prevents cryptic errors when using unsupported region codes +- **Filename Sanitization**: Fixed space-hyphen-space handling in filenames + - Pre-process space-hyphen-space patterns (e.g., "Title - Episode") before other replacements + - Made space-hyphen-space handling conditional on scene_naming setting + - Addresses PR #44 by fixing the root cause +- **CICP Enum Values**: Corrected values to match ITU-T H.273 specification + - Added Primaries.Unspecified (value 2) per H.273 spec + - Renamed Primaries/Transfer value 0 from Unspecified to Reserved for spec accuracy + - Simplified Transfer value 2 from Unspecified_Image to Unspecified + - Verified against ITU-T H.273, ISO/IEC 23091-2, H.264/H.265 specs, and FFmpeg enums +- **HLS Byte Range Parsing**: Fixed TypeError in range_offset conversion + - Converted range_offset to int to prevent TypeError in calculate_byte_range +- **pyplayready Compatibility**: Pinned to <0.7 to avoid KID extraction bug + +## [2.0.0] - 2025-11-10 + +### Breaking Changes + +- **REST API Integration**: Core architecture modified to support REST API functionality + - Changes to internal APIs for download management and tracking + - Title and track classes updated with API integration points + - Core component interfaces modified for queue management support +- **Configuration Changes**: New required configuration options for API and enhanced features + - Added `simkl_client_id` now required for Simkl functionality + - Service-specific configuration override structure introduced + - Debug logging configuration options added +- **Forced Subtitles**: Behavior change for forced subtitle inclusion + - Forced subs no longer auto-included, requires explicit `--forced-subs` flag + +### Added + +- **REST API Server**: Complete download management via REST API (early development) + - Implemented download queue management and worker system + - Added OpenAPI/Swagger documentation for easy API exploration + - Included download progress tracking and status endpoints + - API authentication and comprehensive error handling + - Updated core components to support API integration + - Early development work with more changes planned +- **CustomRemoteCDM**: Highly configurable custom CDM API support + - Support for third-party CDM API providers with maximum configurability + - Full configuration through YAML without code changes + - Addresses GitHub issue #26 for flexible CDM integration +- **WindscribeVPN Proxy Provider**: New VPN provider support + - Added WindscribeVPN following NordVPN and SurfsharkVPN patterns + - Fixes GitHub issue #29 +- **Latest Episode Download**: New `--latest-episode` CLI option + - `-le, --latest-episode` flag to download only the most recent episode + - Automatically selects the single most recent episode regardless of season + - Fixes GitHub issue #28 +- **Video Track Exclusion**: New `--no-video` CLI option + - `-nv, --no-video` flag to skip downloading video tracks + - Allows downloading only audio, subtitles, attachments, and chapters + - Useful for audio-only or subtitle extraction workflows + - Fixes GitHub issue #39 +- **Service-Specific Configuration Overrides**: Per-service fine-tuned control + - Support for per-service configuration overrides in YAML + - Fine-tuned control of downloader and command options per service + - Fixes GitHub issue #13 +- **Comprehensive JSON Debug Logging**: Structured logging for troubleshooting + - Binary toggle via `--debug` flag or `debug: true` in config + - JSON Lines (.jsonl) format for easy parsing and analysis + - Comprehensive logging of all operations (session info, CLI params, CDM details, auth status, title/track metadata, DRM operations, vault queries) + - Configurable key logging via `debug_keys` option with smart redaction + - Error logging for all critical operations + - Removed old text logging system +- **curl_cffi Retry Handler**: Enhanced session reliability + - Added automatic retry mechanism to curl_cffi Session + - Improved download reliability with configurable retries +- **Simkl API Configuration**: New API key support + - Added `simkl_client_id` configuration option + - Simkl now requires client_id from https://simkl.com/settings/developer/ +- **Custom Session Fingerprints**: Enhanced browser impersonation capabilities + - Added custom fingerprint and preset support for better service compatibility + - Configurable fingerprint presets for different device types + - Improved success rate with services using advanced bot detection +- **TMDB and Simkl Metadata Caching**: Enhanced title cache system + - Added metadata caching to title cache to reduce API calls + - Caches movie/show metadata alongside title information + - Improves performance for repeated title lookups and reduces API rate limiting +- **API Enhancements**: Improved REST API functionality + - Added default parameter handling for better request processing + - Added URL field to services endpoint response for easier service identification + - Complete API enhancements for production readiness + - Improved error responses with better detail and debugging information + +### Changed + +- **Binary Search Enhancement**: Improved binary discovery + - Refactored to search for binaries in root of binary folder or subfolder named after the binary + - Better organization of binary dependencies +- **Type Annotations**: Modernized to PEP 604 syntax + - Updated session.py type annotations to use modern Python syntax + - Improved code readability and type checking + +### Fixed + +- **Audio Description Track Support**: Added option to download audio description tracks + - Added `--audio-description/-ad` flag to optionally include descriptive audio tracks + - Previously, audio description tracks were always filtered out + - Users can now choose to download AD tracks when needed + - Fixes GitHub issue #33 +- **Config Directory Support**: Cross-platform user config directory support + - Fixed config loading to properly support user config directories across all platforms + - Fixes GitHub issue #23 +- **HYBRID Mode Validation**: Pre-download validation for hybrid processing + - Added validation to check both HDR10 and DV tracks are available before download + - Prevents wasted downloads when hybrid processing would fail +- **TMDB/Simkl API Keys**: Graceful handling of missing API keys + - Improved error handling when TMDB or Simkl API keys are not configured + - Better user messaging for API configuration requirements +- **Forced Subtitles Behavior**: Correct forced subtitle filtering + - Fixed forced subtitles being incorrectly included without `--forced-subs` flag + - Forced subs now only included when explicitly requested +- **Font Attachment Constructor**: Fixed ASS/SSA font attachment + - Use keyword arguments for Attachment constructor in font attachment + - Fixes "Invalid URL: No scheme supplied" error + - Fixes GitHub issue #24 +- **Binary Subdirectory Checking**: Enhanced binary location discovery (by @TPD94, PR #19) + - Updated binaries.py to check subdirectories in binaries folders named after the binary + - Improved binary detection and loading +- **HLS Manifest Processing**: Minor HLS parser fix (by @TPD94, PR #19) +- **lxml and pyplayready**: Updated dependencies (by @Sp5rky) + - Updated lxml constraint and pyplayready import path for compatibility +- **DASH Segment Calculation**: Corrected segment handling + - Fixed segment count calculation for DASH manifests with startNumber=0 + - Ensures accurate segment processing for all DASH manifest configurations + - Prevents off-by-one errors in segment downloads +- **HDR Detection and Naming**: Comprehensive HDR format support + - Improved HDR detection with comprehensive transfer characteristics checks + - Added hybrid DV+HDR10 support for accurate file naming + - Better identification of HDR formats across different streaming services + - More accurate HDR/DV detection in filename generation +- **Subtitle Processing**: VTT subtitle handling improvements + - Resolved SDH (Subtitles for Deaf and Hard of hearing) stripping crash when processing VTT files + - More robust subtitle processing pipeline with better error handling + - Fixes crashes when filtering specific VTT subtitle formats +- **DRM Processing**: Enhanced encoding handling + - Added explicit UTF-8 encoding to mp4decrypt subprocess calls + - Prevents encoding issues on systems with non-UTF-8 default encodings + - Improves cross-platform compatibility for Windows and some Linux configurations +- **Session Fingerprints**: Updated OkHttp presets + - Updated OkHttp fingerprint presets for better Android TV compatibility + - Improved success rate with services using fingerprint-based detection + +### Documentation + +- **GitHub Issue Templates**: Enhanced issue reporting + - Improved bug report template with better structure and required fields + - Enhanced feature request template for clearer specifications + - Added helpful guidance for contributors to provide complete information + +### Refactored + +- **Import Cleanup**: Removed unused imports + - Removed unused mypy import from binaries.py + - Fixed import ordering in API download_manager and handlers + +### Contributors + +This release includes contributions from: + +- @Sp5rky - REST API server implementation, dependency updates +- @stabbedbybrick - curl_cffi retry handler (PR #31) +- @stabbedbybrick - n_m3u8dl-re refactor (PR #38) +- @TPD94 - Binary search enhancements, manifest parser fixes (PR #19) +- @scene (Andy) - Core features, configuration system, bug fixes + +## [1.4.8] - 2025-10-08 + +### Added + +- **Exact Language Matching**: New `--exact-lang` flag for precise language matching + - Enables strict language code matching without fallbacks +- **No-Mux Flag**: New `--no-mux` flag to skip muxing tracks into container files + - Useful for keeping individual track files separate +- **DecryptLabs API Integration for HTTP Vault**: Enhanced vault support + - Added DecryptLabs API support to HTTP vault for improved key retrieval +- **AC4 Audio Codec Support**: Enhanced audio format handling + - Added AC4 codec support in Audio class with updated mime/profile handling +- **pysubs2 Subtitle Conversion**: Extended subtitle format support + - Added pysubs2 subtitle conversion with extended format support + - Configurable conversion method in configuration + +### Changed + +- **Audio Track Sorting**: Optimized audio track selection logic + - Improved audio track sorting by grouping descriptive tracks and sorting by bitrate + - Better identification of ATMOS and DD+ as highest quality for filenaming +- **pyplayready Update**: Upgraded to version 0.6.3 + - Updated import paths to resolve compatibility issues + - Fixed lxml constraints for better dependency management +- **pysubs2 Conversion Method**: Moved from auto to manual configuration + - pysubs2 no longer auto-selected during testing phase + +### Fixed + +- **Remote CDM**: Fixed curl_cffi compatibility + - Added curl_cffi to instance checks in RemoteCDM +- **Temporary File Handling**: Improved encoding handling + - Specified UTF-8 encoding when opening temporary files + +### Reverted + +- **tinycss SyntaxWarning Suppression**: Removed ineffective warning filter + - Reverted warnings filter that didn't work as expected for suppressing tinycss warnings + +## [1.4.7] - 2025-09-25 + +### Added + +- **curl_cffi Session Support**: Enhanced anti-bot protection with browser impersonation + - Added new session utility with curl_cffi support for bypassing anti-bot measures + - Browser impersonation support for Chrome, Firefox, and Safari user agents + - Full backward compatibility with requests.Session maintained + - Suppressed HTTPS proxy warnings for improved user experience +- **Download Retry Functionality**: Configurable retry mechanism for failed downloads + - Added retry count option to download function for improved reliability +- **Subtitle Requirements Options**: Enhanced subtitle download control + - Added options for required subtitles in download command + - Better control over subtitle track selection and requirements +- **Quality Selection Enhancement**: Improved quality selection options + - Added best available quality option in download command for optimal track selection +- **DecryptLabs API Integration**: Enhanced remote CDM configuration + - Added decrypt_labs_api_key to Config initialization for better API integration + +### Changed + +- **Manifest Parser Updates**: Enhanced compatibility across all parsers + - Updated DASH, HLS, ISM, and M3U8 parsers to accept curl_cffi sessions + - Improved cookie handling compatibility between requests and curl_cffi +- **Logging Improvements**: Reduced log verbosity for better user experience + - Changed duplicate track log level to debug to reduce console noise + - Dynamic CDM selection messages moved to debug-only output + +### Fixed + +- **Remote CDM Reuse**: Fixed KeyError in dynamic CDM selection + - Prevents KeyError when reusing remote CDMs in dynamic selection process + - Creates copy of CDM dictionary before modification to prevent configuration mutation + - Allows same CDM to be selected multiple times within session without errors + +## [1.4.6] - 2025-09-13 + +### Added + +- **Quality-Based CDM Selection**: Dynamic CDM selection based on video resolution + - Automatically selects appropriate CDM (L3/L1) based on video track quality + - Supports quality thresholds in configuration (>=, >, <=, <, exact match) + - Pre-selects optimal CDM based on highest quality across all video tracks + - Maintains backward compatibility with existing CDM configurations +- **Automatic Audio Language Metadata**: Intelligent embedded audio language detection + - Automatically sets audio language metadata when no separate audio tracks exist + - Smart video track selection based on title language with fallbacks + - Enhanced FFmpeg repackaging with audio stream metadata injection +- **Lazy DRM Loading**: Deferred DRM loading for multi-track key retrieval optimization + - Add deferred DRM loading to M3U8 parser to mark tracks for later processing + - Just-in-time DRM loading during download process for better performance + +### Changed + +- **Enhanced CDM Management**: Improved CDM switching logic for multi-quality downloads + - CDM selection now based on highest quality track to avoid inefficient switching + - Quality-based selection only within same DRM type (Widevine-to-Widevine, PlayReady-to-PlayReady) + - Single CDM used per session for better performance and reliability + +### Fixed + +- **Vault Caching Issues**: Fixed vault count display and NoneType iteration errors + - Fix 'NoneType' object is not iterable error in DecryptLabsRemoteCDM + - Fix vault count display showing 0/3 instead of actual successful vault count +- **Service Name Transmission**: Resolved DecryptLabsRemoteCDM service name issues + - Fixed DecryptLabsRemoteCDM sending 'generic' instead of proper service names + - Added case-insensitive vault lookups for SQLite/MySQL vaults + - Added local vault integration to DecryptLabsRemoteCDM +- **Import Organization**: Improved import ordering and code formatting + - Reorder imports in decrypt_labs_remote_cdm.py for better organization + - Clean up trailing whitespace in vault files + +### Configuration + +- **New CDM Configuration Format**: Extended `cdm:` section supports quality-based selection + ```yaml + cdm: + SERVICE_NAME: + "<=1080": l3_cdm_name + ">1080": l1_cdm_name + default: l3_cdm_name + ``` + +## [1.4.5] - 2025-09-09 + +### Added + +- **Enhanced CDM Key Caching**: Improved key caching and session management for L1/L2 devices + - Optimized `get_cached_keys_if_exists` functionality for better performance with L1/L2 devices + - Enhanced cached key retrieval logic with improved session handling +- **Widevine Common Certificate Fallback**: Added fallback to Widevine common certificate for L1 devices + - Improved compatibility for L1 devices when service certificates are unavailable +- **Enhanced Vault Loading**: Improved vault loading and key copying logic + - Better error handling and key management in vault operations +- **PSSH Display Optimization**: Truncated PSSH string display in non-debug mode for cleaner output +- **CDM Error Messaging**: Added error messages for missing service certificates in CDM sessions + +### Changed + +- **Dynamic Version Headers**: Updated User-Agent headers to use dynamic version strings + - DecryptLabsRemoteCDM now uses dynamic version import instead of hardcoded version +- **Intelligent CDM Caching**: Implemented intelligent caching system for CDM license requests + - Enhanced caching logic reduces redundant license requests and improves performance +- **Enhanced Tag Handling**: Improved tag handling for TV shows and movies from Simkl data + - Better metadata processing and formatting for improved media tagging + +### Fixed + +- **CDM Session Management**: Clean up session data when retrieving cached keys + - Remove decrypt_labs_session_id and challenge from session when cached keys exist but there are missing kids + - Ensures clean state for subsequent requests and prevents session conflicts +- **Tag Formatting**: Fixed formatting issues in tag processing +- **Import Order**: Fixed import order issues in tags module + +## [1.4.4] - 2025-09-02 + +### Added + +- **Enhanced DecryptLabs CDM Support**: Comprehensive remote CDM functionality + - Full support for Widevine, PlayReady, and ChromeCDM through DecryptLabsRemoteCDM + - Enhanced session management and caching support for remote WV/PR operations + - Support for cached keys and improved license handling + - New CDM configurations for Chrome and PlayReady devices with updated User-Agent and service certificate +- **Advanced Configuration Options**: New device and language preferences + - Added configuration options for device certificate status list + - Enhanced language preference settings + +### Changed + +- **DRM Decryption Enhancements**: Streamlined decryption process + - Simplified decrypt method by removing unused parameter and streamlined logic + - Improved DecryptLabs CDM configurations with better device support + +### Fixed + +- **Matroska Tag Compliance**: Enhanced media container compatibility + - Fixed Matroska tag compliance with official specification +- **Application Branding**: Cleaned up version display + - Removed old devine version reference from banner to avoid developer confusion + - Updated branding while maintaining original GNU license compliance +- **IP Information Handling**: Improved geolocation services + - Enhanced get_ip_info functionality with better failover handling + - Added support for 429 error handling and multiple API provider fallback + - Implemented cached IP info retrieval with fallback tester to avoid rate limiting +- **Dependencies**: Streamlined package requirements + - Removed unnecessary data extra requirement from langcodes + +### Removed + +- Deprecated version references in application banner for clarity + +## [1.4.3] - 2025-08-20 + +### Added + +- Cached IP info helper for region detection + - New `get_cached_ip_info()` with 24h cache and provider rotation (ipinfo/ipapi) with 429 handling. + - Reduces external calls and stabilizes non-proxy region lookups for caching/logging. + +### Changed + +- DRM decryption selection is fully configuration-driven + - Widevine and PlayReady now select the decrypter based solely on `decryption` in YAML (including per-service mapping). + - Shaka Packager remains the default decrypter when not specified. + - `dl.py` logs the chosen tool based on the resolved configuration. +- Geofencing and proxy verification improvements + - Safer geofence checks with error handling and clearer logs. + - Always verify proxy exit region via live IP lookup; fallback to proxy parsing on failure. +- Example config updated to default to Shaka + - `envied.yaml`/example now sets `decryption.default: shaka` (service overrides still supported). + +### Removed + +- Deprecated parameter `use_mp4decrypt` + - Removed from `Widevine.decrypt()` and `PlayReady.decrypt()` and all callsites. + - Internal naming switched from mp4decrypt-specific flags to generic `decrypter` selection. + +## [1.4.2] - 2025-08-14 + +### Added + +- **Session Management for API Requests**: Enhanced API reliability with retry logic + - Implemented session management for tags functionality with automatic retry mechanisms + - Improved API request stability and error handling +- **Series Year Configuration**: New `series_year` option for title naming control + - Added configurable `series_year` option to control year inclusion in series titles + - Enhanced YAML configuration with series year handling options +- **Audio Language Override**: New audio language selection option + - Added `audio_language` option to override default language selection for audio tracks + - Provides more granular control over audio track selection +- **Vault Key Reception Control**: Enhanced vault security options + - Added `no_push` option to Vault and its subclasses to control key reception + - Improved key management security and flexibility + +### Changed + +- **HLS Segment Processing**: Enhanced segment retrieval and merging capabilities + - Enhanced segment retrieval to allow all file types for better compatibility + - Improved segment merging with recursive file search and fallback to binary concatenation + - Fixed issues with VTT files from HLS not being found correctly due to format changes + - Added cleanup of empty segment directories after processing +- **Documentation**: Updated README.md with latest information + +### Fixed + +- **Audio Track Selection**: Improved per-language logic for audio tracks + - Adjusted `per_language` logic to ensure correct audio track selection + - Fixed issue where all tracks for selected language were being downloaded instead of just the intended ones + +## [1.4.1] - 2025-08-08 + +### Added + +- **Title Caching System**: Intelligent title caching to reduce redundant API calls + - Configurable title caching with 30-minute default cache duration + - 24-hour fallback cache on API failures for improved reliability + - Region-aware caching to handle geo-restricted content properly + - SHA256 hashing for cache keys to handle complex title IDs + - Added `--no-cache` CLI flag to bypass caching when needed + - Added `--reset-cache` CLI flag to clear existing cache data + - New cache configuration variables in config system + - Documented caching options in example configuration file + - Significantly improves performance when debugging or modifying CLI parameters +- **Enhanced Tagging Configuration**: New options for customizing tag behavior + - Added `tag_group_name` config option to control group name inclusion in tags + - Added `tag_imdb_tmdb` config option to control IMDB/TMDB details in tags + - Added Simkl API endpoint support as fallback when no TMDB API key is provided + - Enhanced tag_file function to prioritize provided TMDB ID when `--tmdb` flag is used + - Improved TMDB ID handling with better prioritization logic + +### Changed + +- **Language Selection Enhancement**: Improved default language handling + - Updated language option default to 'orig' when no `-l` flag is set + - Avoids hardcoded 'en' default and respects original content language +- **Tagging Logic Improvements**: Simplified and enhanced tagging functionality + - Simplified Simkl search logic with soft-fail when no results found + - Enhanced tag_file function with better TMDB ID prioritization + - Improved error handling in tagging operations + +### Fixed + +- **Subtitle Processing**: Enhanced subtitle filtering for edge cases + - Fixed ValueError in subtitle filtering for multiple colons in time references + - Improved handling of subtitles containing complex time formatting + - Better error handling for malformed subtitle timestamps + +### Removed + +- **Docker Support**: Removed Docker configuration from repository + - Removed Dockerfile and .dockerignore files + - Cleaned up README.md Docker-related documentation + - Focuses on direct installation methods + +## [1.4.0] - 2025-08-05 + +### Added + +- **HLG Transfer Characteristics Preservation**: Enhanced video muxing to preserve HLG color metadata + - Added automatic detection of HLG video tracks during muxing process + - Implemented `--color-transfer-characteristics 0:18` argument for mkvmerge when processing HLG content + - Prevents incorrect conversion from HLG (18) to BT.2020 (14) transfer characteristics + - Ensures proper HLG playback support on compatible hardware without manual editing +- **Original Language Support**: Enhanced language selection with 'orig' keyword support + - Added support for 'orig' language selector for both video and audio tracks + - Automatically detects and uses the title's original language when 'orig' is specified + - Improved language processing logic with better duplicate handling + - Enhanced help text to document original language selection usage +- **Forced Subtitle Support**: Added option to include forced subtitle tracks + - New functionality to download and include forced subtitle tracks alongside regular subtitles +- **WebVTT Subtitle Filtering**: Enhanced subtitle processing capabilities + - Added filtering for unwanted cues in WebVTT subtitles + - Improved subtitle quality by removing unnecessary metadata + +### Changed + +- **DRM Track Decryption**: Improved DRM decryption track selection logic + - Enhanced `get_drm_for_cdm()` method usage for better DRM-CDM matching + - Added warning messages when no matching DRM is found for tracks + - Improved error handling and logging for DRM decryption failures +- **Series Tree Representation**: Enhanced episode tree display formatting + - Updated series tree to show season breakdown with episode counts + - Improved visual representation with "S{season}({count})" format + - Better organization of series information in console output +- **Hybrid Processing UI**: Enhanced extraction and conversion processes + - Added dynamic spinning bars to follow the rest of the codebase design + - Improved visual feedback during hybrid HDR processing operations +- **Track Selection Logic**: Enhanced multi-track selection capabilities + - Fixed track selection to support combining -V, -A, -S flags properly + - Improved flexibility in selecting multiple track types simultaneously +- **Service Subtitle Support**: Added configuration for services without subtitle support + - Services can now indicate if they don't support subtitle downloads + - Prevents unnecessary subtitle download attempts for unsupported services +- **Update Checker**: Enhanced update checking logic and cache handling + - Improved rate limiting and caching mechanisms for update checks + - Better performance and reduced API calls to GitHub + +### Fixed + +- **PlayReady KID Extraction**: Enhanced KID extraction from PSSH data + - Added base64 support and XML parsing for better KID detection + - Fixed issue where only one KID was being extracted for certain services + - Improved multi-KID support for PlayReady protected content +- **Dolby Vision Detection**: Improved DV codec detection across all formats + - Fixed detection of dvhe.05.06 codec which was not being recognized correctly + - Enhanced detection logic in Episode and Movie title classes + - Better support for various Dolby Vision codec variants + +## [1.3.0] - 2025-08-03 + +### Added + +- **mp4decrypt Support**: Alternative DRM decryption method using mp4decrypt from Bento4 + - Added `mp4decrypt` binary detection and support in binaries module + - New `decryption` configuration option in envied.yaml for service-specific decryption methods + - Enhanced PlayReady and Widevine DRM classes with mp4decrypt decryption support + - Service-specific decryption mapping allows choosing between `shaka` and `mp4decrypt` per service + - Improved error handling and progress reporting for mp4decrypt operations +- **Scene Naming Configuration**: New `scene_naming` option for controlling file naming conventions + - Added scene naming logic to movie, episode, and song title classes + - Configurable through envied.yaml to enable/disable scene naming standards +- **Terminal Cleanup and Signal Handling**: Enhanced console management + - Implemented proper terminal cleanup on application exit + - Added signal handling for graceful shutdown in ComfyConsole +- **Configuration Template**: New `unshackle-example.yaml` template file + - Replaced main `envied.yaml` with example template to prevent git conflicts + - Users can now modify their local config without affecting repository updates +- **Enhanced Credential Management**: Improved CDM and vault configuration + - Expanded credential management documentation in configuration + - Enhanced CDM configuration examples and guidelines +- **Video Transfer Standards**: Added `Unspecified_Image` option to Transfer enum + - Implements ITU-T H.Sup19 standard value 2 for image characteristics + - Supports still image coding systems and unknown transfer characteristics +- **Update Check Rate Limiting**: Enhanced update checking system + - Added configurable update check intervals to prevent excessive API calls + - Improved rate limiting for GitHub API requests + +### Changed + +- **DRM Decryption Architecture**: Enhanced decryption system with dual method support + - Updated `dl.py` to handle service-specific decryption method selection + - Refactored `Config` class to manage decryption method mapping per service + - Enhanced DRM decrypt methods with `use_mp4decrypt` parameter for method selection +- **Error Handling**: Improved exception handling in Hybrid class + - Replaced log.exit calls with ValueError exceptions for better error propagation + - Enhanced error handling consistency across hybrid processing + +### Fixed + +- **Proxy Configuration**: Fixed proxy server mapping in configuration + - Renamed 'servers' to 'server_map' in proxy configuration to resolve Nord/Surfshark naming conflicts + - Updated configuration structure for better compatibility with proxy providers +- **HTTP Vault**: Improved URL handling and key retrieval logic + - Fixed URL processing issues in HTTP-based key vaults + - Enhanced key retrieval reliability and error handling + +## [1.2.0] - 2025-07-30 + +### Added + +- **Update Checker**: Automatic GitHub release version checking on startup + - Configurable update notifications via `update_checks` setting in envied.yaml + - Non-blocking HTTP requests with 5-second timeout for performance + - Smart semantic version comparison supporting all version formats (x.y.z, x.y, x) + - Graceful error handling for network issues and API failures + - User-friendly update notifications with current → latest version display + - Direct links to GitHub releases page for easy updates +- **HDR10+ Support**: Enhanced HDR10+ metadata processing for hybrid tracks + - HDR10+ tool binary support (`hdr10plus_tool`) added to binaries module + - HDR10+ to Dolby Vision conversion capabilities in hybrid processing + - Enhanced metadata extraction for HDR10+ content +- **Duration Fix Handling**: Added duration correction for video and hybrid tracks +- **Temporary Directory Management**: Automatic creation of temp directories for attachment downloads + +### Changed + +- Enhanced configuration system with new `update_checks` boolean option (defaults to true) +- Updated sample envied.yaml with update checker configuration documentation +- Improved console styling consistency using `bright_black` for dimmed text +- **Environment Dependency Check**: Complete overhaul with detailed categorization and status summary + - Organized dependencies by category (Core, HDR, Download, Subtitle, Player, Network) + - Enhanced status reporting with compact summary display + - Improved tool requirement tracking and missing dependency alerts +- **Hybrid Track Processing**: Significant improvements to HDR10+ and Dolby Vision handling + - Enhanced metadata extraction and processing workflows + - Better integration with HDR processing tools + +### Removed + +- **Docker Workflow**: Removed Docker build and publish GitHub Actions workflow for manual builds + +## [1.1.0] - 2025-07-29 + +### Added + +- **HDR10+DV Hybrid Processing**: New `-r HYBRID` command for processing HDR10 and Dolby Vision tracks + - Support for hybrid HDR processing and injection using dovi_tool + - New hybrid track processing module for seamless HDR10/DV conversion + - Automatic detection and handling of HDR10 and DV metadata +- Support for HDR10 and DV tracks in hybrid mode for EXAMPLE service +- Binary availability check for dovi_tool in hybrid mode operations +- Enhanced track processing capabilities for HDR content + +### Fixed + +- Import order issues and missing json import in hybrid processing +- UV installation process and error handling improvements +- Binary search functionality updated to use `binaries.find` + +### Changed + +- Updated package version from 1.0.2 to 1.1.0 +- Enhanced dl.py command processing for hybrid mode support +- Improved core titles (episode/movie) processing for HDR content +- Extended tracks module with hybrid processing capabilities diff --git a/reset/packages/envied/src/envied/__init__.py b/reset/packages/envied/src/envied/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/reset/packages/envied/src/envied/__main__.py b/reset/packages/envied/src/envied/__main__.py new file mode 100644 index 0000000..cf26b60 --- /dev/null +++ b/reset/packages/envied/src/envied/__main__.py @@ -0,0 +1,4 @@ +if __name__ == "__main__": + from envied.core.__main__ import main + + main() diff --git a/reset/packages/envied/src/envied/binaries/ML-Worker b/reset/packages/envied/src/envied/binaries/ML-Worker new file mode 100644 index 0000000..fbc9b2b Binary files /dev/null and b/reset/packages/envied/src/envied/binaries/ML-Worker differ diff --git a/reset/packages/envied/src/envied/binaries/placehere.txt b/reset/packages/envied/src/envied/binaries/placehere.txt new file mode 100644 index 0000000..e69de29 diff --git a/reset/packages/envied/src/envied/commands/__init__.py b/reset/packages/envied/src/envied/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/reset/packages/envied/src/envied/commands/cfg.py b/reset/packages/envied/src/envied/commands/cfg.py new file mode 100644 index 0000000..b21872f --- /dev/null +++ b/reset/packages/envied/src/envied/commands/cfg.py @@ -0,0 +1,90 @@ +import ast +import logging +import sys + +import click +from ruamel.yaml import YAML + +from envied.core.config import config, get_config_path +from envied.core.constants import context_settings + + +@click.command( + short_help="Manage configuration values for the program and its services.", context_settings=context_settings +) +@click.argument("key", type=str, required=False) +@click.argument("value", type=str, required=False) +@click.option("--unset", is_flag=True, default=False, help="Unset/remove the configuration value.") +@click.option("--list", "list_", is_flag=True, default=False, help="List all set configuration values.") +@click.pass_context +def cfg(ctx: click.Context, key: str, value: str, unset: bool, list_: bool) -> None: + """ + Manage configuration values for the program and its services. + + \b + Known Issues: + - Config changes remove all comments of the changed files, which may hold critical data. (#14) + """ + if not key and not value and not list_: + raise click.UsageError("Nothing to do.", ctx) + + if value: + try: + value = ast.literal_eval(value) + except (ValueError, SyntaxError): + pass # probably a str without quotes or similar, assume it's a string value + + log = logging.getLogger("cfg") + + yaml, data = YAML(), None + yaml.default_flow_style = False + + config_path = get_config_path() or config.directories.user_configs / config.filenames.root_config + if config_path.exists(): + data = yaml.load(config_path) + + if not data: + log.warning("No config file was found or it has no data, yet") + # yaml.load() returns `None` if the input data is blank instead of a usable object + # force a usable object by making one and removing the only item within it + data = yaml.load("""__TEMP__: null""") + del data["__TEMP__"] + + if list_: + yaml.dump(data, sys.stdout) + return + + key_items = key.split(".") + parent_key = key_items[:-1] + trailing_key = key_items[-1] + + is_write = value is not None + is_delete = unset + if is_write and is_delete: + raise click.ClickException("You cannot set a value and use --unset at the same time.") + + if not is_write and not is_delete: + data = data.mlget(key_items, default=KeyError) + if data is KeyError: + raise click.ClickException(f"Key '{key}' does not exist in the config.") + yaml.dump(data, sys.stdout) + else: + try: + parent_data = data + if parent_key: + parent_data = data.mlget(parent_key, default=data) + if parent_data == data: + for key in parent_key: + if not hasattr(parent_data, key): + parent_data[key] = {} + parent_data = parent_data[key] + if is_write: + parent_data[trailing_key] = value + log.info(f"Set {key} to {repr(value)}") + elif is_delete: + del parent_data[trailing_key] + log.info(f"Unset {key}") + except KeyError: + raise click.ClickException(f"Key '{key}' does not exist in the config.") + config_path.parent.mkdir(parents=True, exist_ok=True) + yaml.dump(data, config_path) diff --git a/reset/packages/envied/src/envied/commands/dl.bak b/reset/packages/envied/src/envied/commands/dl.bak new file mode 100644 index 0000000..479f137 --- /dev/null +++ b/reset/packages/envied/src/envied/commands/dl.bak @@ -0,0 +1,2774 @@ +from __future__ import annotations + +import html +import logging +import math +import random +import re +import shutil +import subprocess +import sys +import time +from concurrent import futures +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +from functools import partial +from http.cookiejar import CookieJar, MozillaCookieJar +from itertools import product +from pathlib import Path +from threading import Lock +from typing import Any, Callable, Optional +from uuid import UUID + +import click +import jsonpickle +import yaml +from construct import ConstError +from pymediainfo import MediaInfo +from pyplayready.cdm import Cdm as PlayReadyCdm +from pyplayready.device import Device as PlayReadyDevice +from pyplayready.remote.remotecdm import RemoteCdm as PlayReadyRemoteCdm +from pywidevine.cdm import Cdm as WidevineCdm +from pywidevine.device import Device +from pywidevine.remotecdm import RemoteCdm +from rich.console import Group +from rich.live import Live +from rich.padding import Padding +from rich.panel import Panel +from rich.progress import BarColumn, Progress, SpinnerColumn, TaskID, TextColumn, TimeRemainingColumn +from rich.rule import Rule +from rich.table import Table +from rich.text import Text +from rich.tree import Tree + +from envied.core import binaries +from envied.core.cdm import CustomRemoteCDM, DecryptLabsRemoteCDM +from envied.core.cdm.detect import is_playready_cdm, is_widevine_cdm +from envied.core.config import config +from envied.core.console import console +from envied.core.constants import DOWNLOAD_LICENCE_ONLY, AnyTrack, context_settings +from envied.core.credential import Credential +from envied.core.drm import DRM_T, MonaLisa, PlayReady, Widevine +from envied.core.events import events +from envied.core.proxies import Basic, Gluetun, Hola, NordVPN, SurfsharkVPN, WindscribeVPN +from envied.core.service import Service +from envied.core.services import Services +from envied.core.title_cacher import get_account_hash +from envied.core.titles import Movie, Movies, Series, Song, Title_T +from envied.core.titles.episode import Episode +from envied.core.tracks import Audio, Subtitle, Tracks, Video +from envied.core.tracks.attachment import Attachment +from envied.core.tracks.hybrid import Hybrid +from envied.core.utilities import (find_font_with_fallbacks, get_debug_logger, get_system_fonts, init_debug_logger, + is_close_match, suggest_font_packages, time_elapsed_since) +from envied.core.utils import tags +from envied.core.utils.click_types import (AUDIO_CODEC_LIST, LANGUAGE_RANGE, QUALITY_LIST, SEASON_RANGE, + ContextData, MultipleChoice, SubtitleCodecChoice, VideoCodecChoice) +from envied.core.utils.collections import merge_dict +from envied.core.utils.subprocess import ffprobe +from envied.core.vaults import Vaults + + +class dl: + @staticmethod + def truncate_pssh_for_display(pssh_string: str, drm_type: str) -> str: + """Truncate PSSH string for display when not in debug mode.""" + if logging.root.level == logging.DEBUG or not pssh_string: + return pssh_string + + max_width = console.width - len(drm_type) - 12 + if len(pssh_string) <= max_width: + return pssh_string + + return pssh_string[: max_width - 3] + "..." + + def find_custom_font(self, font_name: str) -> Optional[Path]: + """ + Find font in custom fonts directory. + + Args: + font_name: Font family name to find + + Returns: + Path to font file, or None if not found + """ + family_dir = Path(config.directories.fonts, font_name) + if family_dir.exists(): + fonts = list(family_dir.glob("*.*tf")) + return fonts[0] if fonts else None + return None + + def prepare_temp_font( + self, font_name: str, matched_font: Path, system_fonts: dict[str, Path], temp_font_files: list[Path] + ) -> Path: + """ + Copy system font to temp and log if using fallback. + + Args: + font_name: Requested font name + matched_font: Path to matched system font + system_fonts: Dictionary of available system fonts + temp_font_files: List to track temp files for cleanup + + Returns: + Path to temp font file + """ + # Find the matched name for logging + matched_name = next((name for name, path in system_fonts.items() if path == matched_font), None) + + if matched_name and matched_name.lower() != font_name.lower(): + self.log.info(f"Using '{matched_name}' as fallback for '{font_name}'") + + # Create unique temp file path + safe_name = font_name.replace(" ", "_").replace("/", "_") + temp_path = config.directories.temp / f"font_{safe_name}{matched_font.suffix}" + + # Copy if not already exists + if not temp_path.exists(): + shutil.copy2(matched_font, temp_path) + temp_font_files.append(temp_path) + + return temp_path + + def attach_subtitle_fonts( + self, font_names: list[str], title: Title_T, temp_font_files: list[Path] + ) -> tuple[int, list[str]]: + """ + Attach fonts for subtitle rendering. + + Args: + font_names: List of font names requested by subtitles + title: Title object to attach fonts to + temp_font_files: List to track temp files for cleanup + + Returns: + Tuple of (fonts_attached_count, missing_fonts_list) + """ + system_fonts = get_system_fonts() + + font_count = 0 + missing_fonts = [] + + for font_name in set(font_names): + # Try custom fonts first + if custom_font := self.find_custom_font(font_name): + title.tracks.add(Attachment(path=custom_font, name=f"{font_name} ({custom_font.stem})")) + font_count += 1 + continue + + # Try system fonts with fallback + if system_font := find_font_with_fallbacks(font_name, system_fonts): + temp_path = self.prepare_temp_font(font_name, system_font, system_fonts, temp_font_files) + title.tracks.add(Attachment(path=temp_path, name=f"{font_name} ({system_font.stem})")) + font_count += 1 + else: + self.log.warning(f"Subtitle uses font '{font_name}' but it could not be found") + missing_fonts.append(font_name) + + return font_count, missing_fonts + + def suggest_missing_fonts(self, missing_fonts: list[str]) -> None: + """ + Show package installation suggestions for missing fonts. + + Args: + missing_fonts: List of font names that couldn't be found + """ + if suggestions := suggest_font_packages(missing_fonts): + self.log.info("Install font packages to improve subtitle rendering:") + for package_cmd, fonts in suggestions.items(): + self.log.info(f" $ sudo apt install {package_cmd}") + self.log.info(f" → Provides: {', '.join(fonts)}") + + def generate_sidecar_subtitle_path( + self, + subtitle: Subtitle, + base_filename: str, + output_dir: Path, + target_codec: Optional[Subtitle.Codec] = None, + source_path: Optional[Path] = None, + ) -> Path: + """Generate sidecar path: {base}.{lang}[.forced][.sdh].{ext}""" + lang_suffix = str(subtitle.language) if subtitle.language else "und" + forced_suffix = ".forced" if subtitle.forced else "" + sdh_suffix = ".sdh" if (subtitle.sdh or subtitle.cc) else "" + + extension = (target_codec or subtitle.codec or Subtitle.Codec.SubRip).extension + if ( + not target_codec + and not subtitle.codec + and source_path + and source_path.suffix + ): + extension = source_path.suffix.lstrip(".") + + filename = f"{base_filename}.{lang_suffix}{forced_suffix}{sdh_suffix}.{extension}" + return output_dir / filename + + def output_subtitle_sidecars( + self, + subtitles: list[Subtitle], + base_filename: str, + output_dir: Path, + sidecar_format: str, + original_paths: Optional[dict[str, Path]] = None, + ) -> list[Path]: + """Output subtitles as sidecar files, converting if needed.""" + created_paths: list[Path] = [] + config.directories.temp.mkdir(parents=True, exist_ok=True) + + for subtitle in subtitles: + source_path = subtitle.path + if sidecar_format == "original" and original_paths and subtitle.id in original_paths: + source_path = original_paths[subtitle.id] + + if not source_path or not source_path.exists(): + continue + + # Determine target codec + if sidecar_format == "original": + target_codec = None + if source_path.suffix: + try: + target_codec = Subtitle.Codec.from_mime(source_path.suffix.lstrip(".")) + except ValueError: + target_codec = None + else: + target_codec = Subtitle.Codec.from_mime(sidecar_format) + + sidecar_path = self.generate_sidecar_subtitle_path( + subtitle, base_filename, output_dir, target_codec, source_path=source_path + ) + + # Copy or convert + if not target_codec or subtitle.codec == target_codec: + shutil.copy2(source_path, sidecar_path) + else: + # Create temp copy for conversion to preserve original + temp_path = config.directories.temp / f"sidecar_{subtitle.id}{source_path.suffix}" + shutil.copy2(source_path, temp_path) + + temp_sub = Subtitle( + subtitle.url, + subtitle.language, + is_original_lang=subtitle.is_original_lang, + descriptor=subtitle.descriptor, + codec=subtitle.codec, + forced=subtitle.forced, + sdh=subtitle.sdh, + cc=subtitle.cc, + id_=f"{subtitle.id}_sc", + ) + temp_sub.path = temp_path + try: + temp_sub.convert(target_codec) + if temp_sub.path and temp_sub.path.exists(): + shutil.copy2(temp_sub.path, sidecar_path) + finally: + if temp_sub.path and temp_sub.path.exists(): + temp_sub.path.unlink(missing_ok=True) + temp_path.unlink(missing_ok=True) + + created_paths.append(sidecar_path) + + return created_paths + + @click.command( + short_help="Download, Decrypt, and Mux tracks for titles from a Service.", + cls=Services, + context_settings=dict(**context_settings, default_map=config.dl, token_normalize_func=Services.get_tag), + ) + @click.option( + "-p", "--profile", type=str, default=None, help="Profile to use for Credentials and Cookies (if available)." + ) + @click.option( + "-q", + "--quality", + type=QUALITY_LIST, + default=[], + help="Download Resolution(s), defaults to the best available resolution.", + ) + @click.option( + "-v", + "--vcodec", + type=VideoCodecChoice(Video.Codec), + default=None, + help="Video Codec to download, defaults to any codec.", + ) + @click.option( + "-a", + "--acodec", + type=AUDIO_CODEC_LIST, + default=[], + help="Audio Codec(s) to download (comma-separated), e.g., 'AAC,EC3'. Defaults to any.", + ) + @click.option( + "-vb", + "--vbitrate", + type=int, + default=None, + help="Video Bitrate to download (in kbps), defaults to highest available.", + ) + @click.option( + "-ab", + "--abitrate", + type=int, + default=None, + help="Audio Bitrate to download (in kbps), defaults to highest available.", + ) + @click.option( + "-r", + "--range", + "range_", + type=MultipleChoice(Video.Range, case_sensitive=False), + default=[Video.Range.SDR], + help="Video Color Range(s) to download, defaults to SDR.", + ) + @click.option( + "-c", + "--channels", + type=float, + default=None, + help="Audio Channel(s) to download. Matches sub-channel layouts like 5.1 with 6.0 implicitly.", + ) + @click.option( + "-naa", + "--noatmos", + "no_atmos", + is_flag=True, + default=False, + help="Exclude Dolby Atmos audio tracks when selecting audio.", + ) + @click.option( + "--split-audio", + "split_audio", + is_flag=True, + default=None, + help="Create separate output files per audio codec instead of merging all audio.", + ) + @click.option( + "-w", + "--wanted", + type=SEASON_RANGE, + default=None, + help="Wanted episodes, e.g. `S01-S05,S07`, `S01E01-S02E03`, `S02-S02E03`, e.t.c, defaults to all.", + ) + @click.option( + "-l", + "--lang", + type=LANGUAGE_RANGE, + default="orig", + help="Language wanted for Video and Audio. Use 'orig' to select the original language, e.g. 'orig,en' for both original and English.", + ) + @click.option( + "--latest-episode", + is_flag=True, + default=False, + help="Download only the single most recent episode available.", + ) + @click.option( + "-vl", + "--v-lang", + type=LANGUAGE_RANGE, + default=[], + help="Language wanted for Video, you would use this if the video language doesn't match the audio.", + ) + @click.option( + "-al", + "--a-lang", + type=LANGUAGE_RANGE, + default=[], + help="Language wanted for Audio, overrides -l/--lang for audio tracks.", + ) + @click.option("-sl", "--s-lang", type=LANGUAGE_RANGE, default=["all"], help="Language wanted for Subtitles.") + @click.option( + "--require-subs", + type=LANGUAGE_RANGE, + default=[], + help="Required subtitle languages. Downloads all subtitles only if these languages exist. Cannot be used with --s-lang.", + ) + @click.option("-fs", "--forced-subs", is_flag=True, default=False, help="Include forced subtitle tracks.") + @click.option( + "--exact-lang", + is_flag=True, + default=False, + help="Use exact language matching (no variants). With this flag, -l es-419 matches ONLY es-419, not es-ES or other variants.", + ) + @click.option( + "--proxy", + type=str, + default=None, + help="Proxy URI to use. If a 2-letter country is provided, it will try get a proxy from the config.", + ) + @click.option( + "--tag", type=str, default=None, help="Set the Group Tag to be used, overriding the one in config if any." + ) + @click.option( + "--tmdb", + "tmdb_id", + type=int, + default=None, + help="Use this TMDB ID for tagging instead of automatic lookup.", + ) + @click.option( + "--tmdb-name", + "tmdb_name", + is_flag=True, + default=False, + help="Rename titles using the name returned from TMDB lookup.", + ) + @click.option( + "--tmdb-year", + "tmdb_year", + is_flag=True, + default=False, + help="Use the release year from TMDB for naming and tagging.", + ) + @click.option( + "--sub-format", + type=SubtitleCodecChoice(Subtitle.Codec), + default=None, + help="Set Output Subtitle Format, only converting if necessary.", + ) + @click.option("-V", "--video-only", is_flag=True, default=False, help="Only download video tracks.") + @click.option("-A", "--audio-only", is_flag=True, default=False, help="Only download audio tracks.") + @click.option("-S", "--subs-only", is_flag=True, default=False, help="Only download subtitle tracks.") + @click.option("-C", "--chapters-only", is_flag=True, default=False, help="Only download chapters.") + @click.option("-ns", "--no-subs", is_flag=True, default=False, help="Do not download subtitle tracks.") + @click.option("-na", "--no-audio", is_flag=True, default=False, help="Do not download audio tracks.") + @click.option("-nc", "--no-chapters", is_flag=True, default=False, help="Do not download chapters tracks.") + @click.option("-nv", "--no-video", is_flag=True, default=False, help="Do not download video tracks.") + @click.option("-ad", "--audio-description", is_flag=True, default=False, help="Download audio description tracks.") + @click.option( + "--slow", + is_flag=True, + default=False, + help="Add a 60-120 second delay between each Title download to act more like a real device. " + "This is recommended if you are downloading high-risk titles or streams.", + ) + @click.option( + "--list", + "list_", + is_flag=True, + default=False, + help="Skip downloading and list available tracks and what tracks would have been downloaded.", + ) + @click.option( + "--list-titles", + is_flag=True, + default=False, + help="Skip downloading, only list available titles that would have been downloaded.", + ) + @click.option( + "--skip-dl", is_flag=True, default=False, help="Skip downloading while still retrieving the decryption keys." + ) + @click.option("--export", type=Path, help="Export Decryption Keys as you obtain them to a JSON file.") + @click.option( + "--cdm-only/--vaults-only", + is_flag=True, + default=None, + help="Only use CDM, or only use Key Vaults for retrieval of Decryption Keys.", + ) + @click.option("--no-proxy", is_flag=True, default=False, help="Force disable all proxy use.") + @click.option("--no-folder", is_flag=True, default=False, help="Disable folder creation for TV Shows.") + @click.option( + "--no-source", is_flag=True, default=False, help="Disable the source tag from the output file name and path." + ) + @click.option("--no-mux", is_flag=True, default=False, help="Do not mux tracks into a container file.") + @click.option( + "--workers", + type=int, + default=None, + help="Max workers/threads to download with per-track. Default depends on the downloader.", + ) + @click.option("--downloads", type=int, default=1, help="Amount of tracks to download concurrently.") + @click.option("--no-cache", "no_cache", is_flag=True, default=False, help="Bypass title cache for this download.") + @click.option( + "--reset-cache", "reset_cache", is_flag=True, default=False, help="Clear title cache before fetching." + ) + @click.option( + "--best-available", + "best_available", + is_flag=True, + default=False, + help="Continue with best available quality if requested resolutions are not available.", + ) + @click.pass_context + def cli(ctx: click.Context, **kwargs: Any) -> dl: + return dl(ctx, **kwargs) + + DRM_TABLE_LOCK = Lock() + + def __init__( + self, + ctx: click.Context, + no_proxy: bool, + profile: Optional[str] = None, + proxy: Optional[str] = None, + tag: Optional[str] = None, + tmdb_id: Optional[int] = None, + tmdb_name: bool = False, + tmdb_year: bool = False, + *_: Any, + **__: Any, + ): + if not ctx.invoked_subcommand: + raise ValueError("A subcommand to invoke was not specified, the main code cannot continue.") + + self.log = logging.getLogger("download") + + self.service = Services.get_tag(ctx.invoked_subcommand) + service_dl_config = config.services.get(self.service, {}).get("dl", {}) + if service_dl_config: + param_types = {param.name: param.type for param in ctx.command.params if param.name} + + for param_name, service_value in service_dl_config.items(): + if param_name not in ctx.params: + continue + + current_value = ctx.params[param_name] + global_default = config.dl.get(param_name) + param_type = param_types.get(param_name) + + try: + if param_type and global_default is not None: + global_default = param_type.convert(global_default, None, ctx) + except Exception as e: + self.log.debug(f"Failed to convert global default for '{param_name}': {e}") + + if current_value == global_default or (current_value is None and global_default is None): + try: + converted_value = service_value + if param_type and service_value is not None: + converted_value = param_type.convert(service_value, None, ctx) + + ctx.params[param_name] = converted_value + self.log.debug(f"Applied service-specific '{param_name}' override: {converted_value}") + except Exception as e: + self.log.warning( + f"Failed to apply service-specific '{param_name}' override: {e}. " + f"Check that the value '{service_value}' is valid for this parameter." + ) + + self.profile = profile + self.tmdb_id = tmdb_id + self.tmdb_name = tmdb_name + self.tmdb_year = tmdb_year + + # Initialize debug logger with service name if debug logging is enabled + if config.debug or logging.root.level == logging.DEBUG: + from collections import defaultdict + from datetime import datetime + + debug_log_path = config.directories.logs / config.filenames.debug_log.format_map( + defaultdict(str, service=self.service, time=datetime.now().strftime("%Y%m%d-%H%M%S")) + ) + init_debug_logger(log_path=debug_log_path, enabled=True, log_keys=config.debug_keys) + self.debug_logger = get_debug_logger() + + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="download_init", + message=f"Download command initialized for service {self.service}", + service=self.service, + context={ + "profile": profile, + "proxy": proxy, + "tag": tag, + "tmdb_id": tmdb_id, + "tmdb_name": tmdb_name, + "tmdb_year": tmdb_year, + "cli_params": { + k: v + for k, v in ctx.params.items() + if k not in ["profile", "proxy", "tag", "tmdb_id", "tmdb_name", "tmdb_year"] + }, + }, + ) + else: + self.debug_logger = None + + if self.profile: + self.log.info(f"Using profile: '{self.profile}'") + + with console.status("Loading Service Config...", spinner="dots"): + service_config_path = Services.get_path(self.service) / config.filenames.config + if service_config_path.exists(): + self.service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) + self.log.info("Service Config loaded") + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="load_service_config", + service=self.service, + context={"config_path": str(service_config_path), "config": self.service_config}, + ) + else: + self.service_config = {} + merge_dict(config.services.get(self.service), self.service_config) + + if getattr(config, "downloader_map", None): + config.downloader = config.downloader_map.get(self.service, config.downloader) + + if getattr(config, "decryption_map", None): + config.decryption = config.decryption_map.get(self.service, config.decryption) + + service_config = config.services.get(self.service, {}) + if service_config: + reserved_keys = { + "profiles", + "api_key", + "certificate", + "api_endpoint", + "region", + "device", + "endpoints", + "client", + "dl", + } + + for config_key, override_value in service_config.items(): + if config_key in reserved_keys or not isinstance(override_value, dict): + continue + + if hasattr(config, config_key): + current_config = getattr(config, config_key, {}) + + if isinstance(current_config, dict): + merged_config = deepcopy(current_config) + merge_dict(override_value, merged_config) + setattr(config, config_key, merged_config) + + self.log.debug( + f"Applied service-specific '{config_key}' overrides for {self.service}: {override_value}" + ) + + cdm_only = ctx.params.get("cdm_only") + + if cdm_only: + self.vaults = Vaults(self.service) + self.log.info("CDM-only mode: Skipping vault loading") + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="vault_loading_skipped", + service=self.service, + context={"reason": "cdm_only flag set"}, + ) + else: + with console.status("Loading Key Vaults...", spinner="dots"): + self.vaults = Vaults(self.service) + total_vaults = len(config.key_vaults) + failed_vaults = [] + + for vault in config.key_vaults: + vault_type = vault["type"] + vault_name = vault.get("name", vault_type) + vault_copy = vault.copy() + del vault_copy["type"] + + if vault_type.lower() == "api" and "decrypt_labs" in vault_name.lower(): + if "token" not in vault_copy or not vault_copy["token"]: + if config.decrypt_labs_api_key: + vault_copy["token"] = config.decrypt_labs_api_key + else: + self.log.warning( + f"No token provided for DecryptLabs vault '{vault_name}' and no global " + "decrypt_labs_api_key configured" + ) + + if vault_type.lower() == "sqlite": + try: + self.vaults.load_critical(vault_type, **vault_copy) + self.log.debug(f"Successfully loaded vault: {vault_name} ({vault_type})") + except Exception as e: + self.log.error(f"vault failure: {vault_name} ({vault_type}) - {e}") + raise + else: + # Other vaults (MySQL, HTTP, API) - soft fail + if not self.vaults.load(vault_type, **vault_copy): + failed_vaults.append(vault_name) + self.log.debug(f"Failed to load vault: {vault_name} ({vault_type})") + else: + self.log.debug(f"Successfully loaded vault: {vault_name} ({vault_type})") + + loaded_count = len(self.vaults) + if failed_vaults: + self.log.warning(f"Failed to load {len(failed_vaults)} vault(s): {', '.join(failed_vaults)}") + self.log.info(f"Loaded {loaded_count}/{total_vaults} Vaults") + + # Debug: Show detailed vault status + if loaded_count > 0: + vault_names = [vault.name for vault in self.vaults] + self.log.debug(f"Active vaults: {', '.join(vault_names)}") + else: + self.log.debug("No vaults are currently active") + + with console.status("Loading DRM CDM...", spinner="dots"): + try: + self.cdm = self.get_cdm(self.service, self.profile) + except ValueError as e: + self.log.error(f"Failed to load CDM, {e}") + if self.debug_logger: + self.debug_logger.log_error("load_cdm", e, service=self.service) + sys.exit(1) + + if self.cdm: + cdm_info = {} + if isinstance(self.cdm, DecryptLabsRemoteCDM): + drm_type = "PlayReady" if self.cdm.is_playready else "Widevine" + self.log.info(f"Loaded {drm_type} Remote CDM: DecryptLabs (L{self.cdm.security_level})") + cdm_info = {"type": "DecryptLabs", "drm_type": drm_type, "security_level": self.cdm.security_level} + elif hasattr(self.cdm, "device_type") and self.cdm.device_type.name in ["ANDROID", "CHROME"]: + self.log.info(f"Loaded Widevine CDM: {self.cdm.system_id} (L{self.cdm.security_level})") + cdm_info = { + "type": "Widevine", + "system_id": self.cdm.system_id, + "security_level": self.cdm.security_level, + "device_type": self.cdm.device_type.name, + } + else: + # Handle both local PlayReady CDM and RemoteCdm (which has certificate_chain=None) + is_remote = self.cdm.certificate_chain is None and hasattr(self.cdm, "device_name") + if is_remote: + cdm_name = self.cdm.device_name + self.log.info(f"Loaded PlayReady Remote CDM: {cdm_name} (L{self.cdm.security_level})") + else: + cdm_name = self.cdm.certificate_chain.get_name() if self.cdm.certificate_chain else "Unknown" + self.log.info(f"Loaded PlayReady CDM: {cdm_name} (L{self.cdm.security_level})") + cdm_info = { + "type": "PlayReady", + "certificate": cdm_name, + "security_level": self.cdm.security_level, + } + + if self.debug_logger and cdm_info: + self.debug_logger.log( + level="INFO", operation="load_cdm", service=self.service, context={"cdm": cdm_info} + ) + + self.proxy_providers = [] + if no_proxy: + ctx.params["proxy"] = None + else: + with console.status("Loading Proxy Providers...", spinner="dots"): + if config.proxy_providers.get("basic"): + self.proxy_providers.append(Basic(**config.proxy_providers["basic"])) + if config.proxy_providers.get("nordvpn"): + self.proxy_providers.append(NordVPN(**config.proxy_providers["nordvpn"])) + if config.proxy_providers.get("surfsharkvpn"): + self.proxy_providers.append(SurfsharkVPN(**config.proxy_providers["surfsharkvpn"])) + if config.proxy_providers.get("windscribevpn"): + self.proxy_providers.append(WindscribeVPN(**config.proxy_providers["windscribevpn"])) + if config.proxy_providers.get("gluetun"): + self.proxy_providers.append(Gluetun(**config.proxy_providers["gluetun"])) + if binaries.HolaProxy: + self.proxy_providers.append(Hola()) + for proxy_provider in self.proxy_providers: + self.log.info(f"Loaded {proxy_provider.__class__.__name__}: {proxy_provider}") + + if proxy: + requested_provider = None + if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE): + # requesting proxy from a specific proxy provider + requested_provider, proxy = proxy.split(":", maxsplit=1) + # Match simple region codes (us, ca, uk1) or provider:region format (nordvpn:ca, windscribe:us) + if re.match(r"^[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE) or re.match( + r"^[a-z]+:[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE + ): + proxy = proxy.lower() + # Preserve the original user query (region code) for service-specific proxy_map overrides. + # NOTE: `proxy` may be overwritten with the resolved proxy URI later. + proxy_query = proxy + status_msg = ( + f"Connecting to VPN ({proxy})..." + if requested_provider == "gluetun" + else f"Getting a Proxy to {proxy}..." + ) + with console.status(status_msg, spinner="dots"): + if requested_provider: + proxy_provider = next( + (x for x in self.proxy_providers if x.__class__.__name__.lower() == requested_provider), + None, + ) + if not proxy_provider: + self.log.error(f"The proxy provider '{requested_provider}' was not recognised.") + sys.exit(1) + proxy_uri = proxy_provider.get_proxy(proxy) + if not proxy_uri: + self.log.error(f"The proxy provider {requested_provider} had no proxy for {proxy}") + sys.exit(1) + proxy = ctx.params["proxy"] = proxy_uri + # Show connection info for Gluetun (IP, location) instead of proxy URL + if hasattr(proxy_provider, "get_connection_info"): + conn_info = proxy_provider.get_connection_info(proxy_query) + if conn_info and conn_info.get("public_ip"): + location_parts = [conn_info.get("city"), conn_info.get("country")] + location = ", ".join(p for p in location_parts if p) + self.log.info(f"VPN Connected: {conn_info['public_ip']} ({location})") + else: + self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") + else: + self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") + else: + for proxy_provider in self.proxy_providers: + proxy_uri = proxy_provider.get_proxy(proxy) + if proxy_uri: + proxy = ctx.params["proxy"] = proxy_uri + # Show connection info for Gluetun (IP, location) instead of proxy URL + if hasattr(proxy_provider, "get_connection_info"): + conn_info = proxy_provider.get_connection_info(proxy_query) + if conn_info and conn_info.get("public_ip"): + location_parts = [conn_info.get("city"), conn_info.get("country")] + location = ", ".join(p for p in location_parts if p) + self.log.info(f"VPN Connected: {conn_info['public_ip']} ({location})") + else: + self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") + else: + self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") + break + # Store proxy query info for service-specific overrides + ctx.params["proxy_query"] = proxy_query + ctx.params["proxy_provider"] = requested_provider + else: + self.log.info(f"Using explicit Proxy: {proxy}") + # For explicit proxies, store None for query/provider + ctx.params["proxy_query"] = None + ctx.params["proxy_provider"] = None + + ctx.obj = ContextData( + config=self.service_config, cdm=self.cdm, proxy_providers=self.proxy_providers, profile=self.profile + ) + + if tag: + config.tag = tag + + # needs to be added this way instead of @cli.result_callback to be + # able to keep `self` as the first positional + self.cli._result_callback = self.result + + def result( + self, + service: Service, + quality: list[int], + vcodec: Optional[Video.Codec], + acodec: list[Audio.Codec], + vbitrate: int, + abitrate: int, + range_: list[Video.Range], + channels: float, + no_atmos: bool, + wanted: list[str], + latest_episode: bool, + lang: list[str], + v_lang: list[str], + a_lang: list[str], + s_lang: list[str], + require_subs: list[str], + forced_subs: bool, + exact_lang: bool, + sub_format: Optional[Subtitle.Codec], + video_only: bool, + audio_only: bool, + subs_only: bool, + chapters_only: bool, + no_subs: bool, + no_audio: bool, + no_chapters: bool, + no_video: bool, + audio_description: bool, + slow: bool, + list_: bool, + list_titles: bool, + skip_dl: bool, + export: Optional[Path], + cdm_only: Optional[bool], + no_proxy: bool, + no_folder: bool, + no_source: bool, + no_mux: bool, + workers: Optional[int], + downloads: int, + best_available: bool, + split_audio: Optional[bool] = None, + *_: Any, + **__: Any, + ) -> None: + self.tmdb_searched = False + self.search_source = None + start_time = time.time() + + if skip_dl: + DOWNLOAD_LICENCE_ONLY.set() + + if not acodec: + acodec = [] + elif isinstance(acodec, Audio.Codec): + acodec = [acodec] + elif isinstance(acodec, str) or ( + isinstance(acodec, list) and not all(isinstance(v, Audio.Codec) for v in acodec) + ): + acodec = AUDIO_CODEC_LIST.convert(acodec) + + if require_subs and s_lang != ["all"]: + self.log.error("--require-subs and --s-lang cannot be used together") + sys.exit(1) + + # Check if dovi_tool is available when hybrid mode is requested + if any(r == Video.Range.HYBRID for r in range_): + from envied.core.binaries import DoviTool + + if not DoviTool: + self.log.error("Unable to run hybrid mode: dovi_tool not detected") + self.log.error("Please install dovi_tool from https://github.com/quietvoid/dovi_tool") + sys.exit(1) + + if cdm_only is None: + vaults_only = None + else: + vaults_only = not cdm_only + + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="drm_mode_config", + service=self.service, + context={ + "cdm_only": cdm_only, + "vaults_only": vaults_only, + "mode": "CDM only" if cdm_only else ("Vaults only" if vaults_only else "Both CDM and Vaults"), + }, + ) + + with console.status("Authenticating with Service...", spinner="dots"): + try: + cookies = self.get_cookie_jar(self.service, self.profile) + credential = self.get_credentials(self.service, self.profile) + service.authenticate(cookies, credential) + if cookies or credential: + self.log.info("Authenticated with Service") + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="authenticate", + service=self.service, + context={ + "has_cookies": bool(cookies), + "has_credentials": bool(credential), + "profile": self.profile, + }, + ) + except Exception as e: + if self.debug_logger: + self.debug_logger.log_error( + "authenticate", e, service=self.service, context={"profile": self.profile} + ) + raise + + with console.status("Fetching Title Metadata...", spinner="dots"): + try: + titles = service.get_titles_cached() + if not titles: + self.log.error("No titles returned, nothing to download...") + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="get_titles", + service=self.service, + message="No titles returned from service", + success=False, + ) + sys.exit(1) + except Exception as e: + if self.debug_logger: + self.debug_logger.log_error("get_titles", e, service=self.service) + raise + + if self.debug_logger: + titles_info = { + "type": titles.__class__.__name__, + "count": len(titles) if hasattr(titles, "__len__") else 1, + "title": str(titles), + } + if hasattr(titles, "seasons"): + titles_info["seasons"] = len(titles.seasons) if hasattr(titles, "seasons") else 0 + self.debug_logger.log( + level="INFO", operation="get_titles", service=self.service, context={"titles": titles_info} + ) + + title_cacher = service.title_cache if hasattr(service, "title_cache") else None + cache_title_id = None + if hasattr(service, "title"): + cache_title_id = service.title + elif hasattr(service, "title_id"): + cache_title_id = service.title_id + cache_region = service.current_region if hasattr(service, "current_region") else None + cache_account_hash = get_account_hash(service.credential) if hasattr(service, "credential") else None + + if (self.tmdb_year or self.tmdb_name) and self.tmdb_id: + sample_title = titles[0] if hasattr(titles, "__getitem__") else titles + kind = "tv" if isinstance(sample_title, Episode) else "movie" + + tmdb_year_val = None + tmdb_name_val = None + + if self.tmdb_year: + tmdb_year_val = tags.get_year( + self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash + ) + + if self.tmdb_name: + tmdb_name_val = tags.get_title( + self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash + ) + + if isinstance(titles, (Series, Movies)): + for t in titles: + if tmdb_year_val: + t.year = tmdb_year_val + if tmdb_name_val: + if isinstance(t, Episode): + t.title = tmdb_name_val + else: + t.name = tmdb_name_val + else: + if tmdb_year_val: + titles.year = tmdb_year_val + if tmdb_name_val: + if isinstance(titles, Episode): + titles.title = tmdb_name_val + else: + titles.name = tmdb_name_val + + console.print(Padding(Rule(f"[rule.text]{titles.__class__.__name__}: {titles}"), (1, 2))) + + console.print(Padding(titles.tree(verbose=list_titles), (0, 5))) + if list_titles: + return + + # Determine the latest episode if --latest-episode is set + latest_episode_id = None + if latest_episode and isinstance(titles, Series) and len(titles) > 0: + # Series is already sorted by (season, number, year) + # The last episode in the sorted list is the latest + latest_ep = titles[-1] + latest_episode_id = f"{latest_ep.season}x{latest_ep.number}" + self.log.info(f"Latest episode mode: Selecting S{latest_ep.season:02}E{latest_ep.number:02}") + + for i, title in enumerate(titles): + if isinstance(title, Episode) and latest_episode and latest_episode_id: + # If --latest-episode is set, only process the latest episode + if f"{title.season}x{title.number}" != latest_episode_id: + continue + elif isinstance(title, Episode) and wanted and f"{title.season}x{title.number}" not in wanted: + continue + + console.print(Padding(Rule(f"[rule.text]{title}"), (1, 2))) + temp_font_files = [] + + if isinstance(title, Episode) and not self.tmdb_searched: + kind = "tv" + if self.tmdb_id: + tmdb_title = tags.get_title( + self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash + ) + else: + self.tmdb_id, tmdb_title, self.search_source = tags.search_show_info( + title.title, title.year, kind, title_cacher, cache_title_id, cache_region, cache_account_hash + ) + if not (self.tmdb_id and tmdb_title and tags.fuzzy_match(tmdb_title, title.title)): + self.tmdb_id = None + if list_ or list_titles: + if self.tmdb_id: + console.print( + Padding( + f"Search -> {tmdb_title or '?'} [bright_black](ID {self.tmdb_id})", + (0, 5), + ) + ) + else: + console.print(Padding("Search -> [bright_black]No match found[/]", (0, 5))) + self.tmdb_searched = True + + if isinstance(title, Movie) and (list_ or list_titles) and not self.tmdb_id: + movie_id, movie_title, _ = tags.search_show_info( + title.name, title.year, "movie", title_cacher, cache_title_id, cache_region, cache_account_hash + ) + if movie_id: + console.print( + Padding( + f"Search -> {movie_title or '?'} [bright_black](ID {movie_id})", + (0, 5), + ) + ) + else: + console.print(Padding("Search -> [bright_black]No match found[/]", (0, 5))) + + if self.tmdb_id and getattr(self, "search_source", None) != "simkl": + kind = "tv" if isinstance(title, Episode) else "movie" + tags.external_ids(self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash) + + if slow and i != 0: + delay = random.randint(60, 120) + with console.status(f"Delaying by {delay} seconds..."): + time.sleep(delay) + + with console.status("Subscribing to events...", spinner="dots"): + events.reset() + events.subscribe(events.Types.SEGMENT_DOWNLOADED, service.on_segment_downloaded) + events.subscribe(events.Types.TRACK_DOWNLOADED, service.on_track_downloaded) + events.subscribe(events.Types.TRACK_DECRYPTED, service.on_track_decrypted) + events.subscribe(events.Types.TRACK_REPACKED, service.on_track_repacked) + events.subscribe(events.Types.TRACK_MULTIPLEX, service.on_track_multiplex) + + if hasattr(service, "NO_SUBTITLES") and service.NO_SUBTITLES: + console.log("Skipping subtitles - service does not support subtitle downloads") + no_subs = True + s_lang = None + title.tracks.subtitles = [] + elif no_subs: + console.log("Skipped subtitles as --no-subs was used...") + s_lang = None + title.tracks.subtitles = [] + + if no_video: + console.log("Skipped video as --no-video was used...") + v_lang = None + title.tracks.videos = [] + + with console.status("Getting tracks...", spinner="dots"): + try: + title.tracks.add(service.get_tracks(title), warn_only=True) + title.tracks.chapters = service.get_chapters(title) + except Exception as e: + if self.debug_logger: + self.debug_logger.log_error( + "get_tracks", e, service=self.service, context={"title": str(title)} + ) + raise + + if self.debug_logger: + tracks_info = { + "title": str(title), + "video_tracks": len(title.tracks.videos), + "audio_tracks": len(title.tracks.audio), + "subtitle_tracks": len(title.tracks.subtitles), + "has_chapters": bool(title.tracks.chapters), + "videos": [ + { + "codec": str(v.codec), + "resolution": f"{v.width}x{v.height}" if v.width and v.height else "unknown", + "bitrate": v.bitrate, + "range": str(v.range), + "language": str(v.language) if v.language else None, + "drm": [str(type(d).__name__) for d in v.drm] if v.drm else [], + } + for v in title.tracks.videos + ], + "audio": [ + { + "codec": str(a.codec), + "bitrate": a.bitrate, + "channels": a.channels, + "language": str(a.language) if a.language else None, + "descriptive": a.descriptive, + "drm": [str(type(d).__name__) for d in a.drm] if a.drm else [], + } + for a in title.tracks.audio + ], + "subtitles": [ + { + "codec": str(s.codec), + "language": str(s.language) if s.language else None, + "forced": s.forced, + "sdh": s.sdh, + } + for s in title.tracks.subtitles + ], + } + self.debug_logger.log( + level="INFO", operation="get_tracks", service=self.service, context=tracks_info + ) + + # strip SDH subs to non-SDH if no equivalent same-lang non-SDH is available + # uses a loose check, e.g, wont strip en-US SDH sub if a non-SDH en-GB is available + # Check if automatic SDH stripping is enabled in config + if config.subtitle.get("strip_sdh", True): + for subtitle in title.tracks.subtitles: + if subtitle.sdh and not any( + is_close_match(subtitle.language, [x.language]) + for x in title.tracks.subtitles + if not x.sdh and not x.forced + ): + non_sdh_sub = deepcopy(subtitle) + non_sdh_sub.id += "_stripped" + non_sdh_sub.sdh = False + title.tracks.add(non_sdh_sub) + events.subscribe( + events.Types.TRACK_MULTIPLEX, + lambda track, sub_id=non_sdh_sub.id: (track.strip_hearing_impaired()) + if track.id == sub_id + else None, + ) + + with console.status("Sorting tracks by language and bitrate...", spinner="dots"): + video_sort_lang = v_lang or lang + processed_video_sort_lang = [] + for language in video_sort_lang: + if language == "orig": + if title.language: + orig_lang = str(title.language) if hasattr(title.language, "__str__") else title.language + if orig_lang not in processed_video_sort_lang: + processed_video_sort_lang.append(orig_lang) + else: + if language not in processed_video_sort_lang: + processed_video_sort_lang.append(language) + + audio_sort_lang = a_lang or lang + processed_audio_sort_lang = [] + for language in audio_sort_lang: + if language == "orig": + if title.language: + orig_lang = str(title.language) if hasattr(title.language, "__str__") else title.language + if orig_lang not in processed_audio_sort_lang: + processed_audio_sort_lang.append(orig_lang) + else: + if language not in processed_audio_sort_lang: + processed_audio_sort_lang.append(language) + + title.tracks.sort_videos(by_language=processed_video_sort_lang) + title.tracks.sort_audio(by_language=processed_audio_sort_lang) + title.tracks.sort_subtitles(by_language=s_lang) + + if list_: + available_tracks, _ = title.tracks.tree() + console.print(Padding(Panel(available_tracks, title="Available Tracks"), (0, 5))) + continue + + with console.status("Selecting tracks...", spinner="dots"): + if isinstance(title, (Movie, Episode)): + # filter video tracks + if vcodec: + title.tracks.select_video(lambda x: x.codec == vcodec) + if not title.tracks.videos: + self.log.error(f"There's no {vcodec.name} Video Track...") + sys.exit(1) + + if range_: + # Special handling for HYBRID - don't filter, keep all HDR10 and DV tracks + if Video.Range.HYBRID not in range_: + title.tracks.select_video(lambda x: x.range in range_) + missing_ranges = [r for r in range_ if not any(x.range == r for x in title.tracks.videos)] + for color_range in missing_ranges: + self.log.warning(f"Skipping {color_range.name} video tracks as none are available.") + + if vbitrate: + title.tracks.select_video(lambda x: x.bitrate and x.bitrate // 1000 == vbitrate) + if not title.tracks.videos: + self.log.error(f"There's no {vbitrate}kbps Video Track...") + sys.exit(1) + + video_languages = [lang for lang in (v_lang or lang) if lang != "best"] + if video_languages and "all" not in video_languages: + processed_video_lang = [] + for language in video_languages: + if language == "orig": + if title.language: + orig_lang = ( + str(title.language) if hasattr(title.language, "__str__") else title.language + ) + if orig_lang not in processed_video_lang: + processed_video_lang.append(orig_lang) + else: + self.log.warning( + "Original language not available for title, skipping 'orig' selection for video" + ) + else: + if language not in processed_video_lang: + processed_video_lang.append(language) + title.tracks.videos = title.tracks.by_language( + title.tracks.videos, processed_video_lang, exact_match=exact_lang + ) + if not title.tracks.videos: + self.log.error(f"There's no {processed_video_lang} Video Track...") + sys.exit(1) + + if quality: + missing_resolutions = [] + if any(r == Video.Range.HYBRID for r in range_): + title.tracks.select_video(title.tracks.select_hybrid(title.tracks.videos, quality)) + else: + title.tracks.by_resolutions(quality) + + for resolution in quality: + if any(v.height == resolution for v in title.tracks.videos): + continue + if any(int(v.width * 9 / 16) == resolution for v in title.tracks.videos): + continue + missing_resolutions.append(resolution) + + if missing_resolutions: + res_list = "" + if len(missing_resolutions) > 1: + res_list = ", ".join([f"{x}p" for x in missing_resolutions[:-1]]) + " or " + res_list = f"{res_list}{missing_resolutions[-1]}p" + plural = "s" if len(missing_resolutions) > 1 else "" + + if best_available: + self.log.warning( + f"There's no {res_list} Video Track{plural}, continuing with available qualities..." + ) + else: + self.log.error(f"There's no {res_list} Video Track{plural}...") + sys.exit(1) + + # choose best track by range and quality + if any(r == Video.Range.HYBRID for r in range_): + # For hybrid mode, always apply hybrid selection + # If no quality specified, use only the best (highest) resolution + if not quality: + # Get the highest resolution available + best_resolution = max((v.height for v in title.tracks.videos), default=None) + if best_resolution: + # Use the hybrid selection logic with only the best resolution + title.tracks.select_video( + title.tracks.select_hybrid(title.tracks.videos, [best_resolution]) + ) + # If quality was specified, hybrid selection was already applied above + else: + selected_videos: list[Video] = [] + for resolution, color_range in product(quality or [None], range_ or [None]): + match = next( + ( + t + for t in title.tracks.videos + if ( + not resolution + or t.height == resolution + or int(t.width * (9 / 16)) == resolution + ) + and (not color_range or t.range == color_range) + ), + None, + ) + if match and match not in selected_videos: + selected_videos.append(match) + title.tracks.videos = selected_videos + + # validate hybrid mode requirements + if any(r == Video.Range.HYBRID for r in range_): + hdr10_tracks = [v for v in title.tracks.videos if v.range == Video.Range.HDR10] + dv_tracks = [v for v in title.tracks.videos if v.range == Video.Range.DV] + + if not hdr10_tracks and not dv_tracks: + available_ranges = sorted(set(v.range.name for v in title.tracks.videos)) + self.log.error("HYBRID mode requires both HDR10 and DV tracks, but neither is available") + self.log.error( + f"Available ranges: {', '.join(available_ranges) if available_ranges else 'none'}" + ) + sys.exit(1) + elif not hdr10_tracks: + available_ranges = sorted(set(v.range.name for v in title.tracks.videos)) + self.log.error("HYBRID mode requires both HDR10 and DV tracks, but only DV is available") + self.log.error(f"Available ranges: {', '.join(available_ranges)}") + sys.exit(1) + elif not dv_tracks: + available_ranges = sorted(set(v.range.name for v in title.tracks.videos)) + self.log.error("HYBRID mode requires both HDR10 and DV tracks, but only HDR10 is available") + self.log.error(f"Available ranges: {', '.join(available_ranges)}") + sys.exit(1) + + # filter subtitle tracks + if require_subs: + missing_langs = [ + lang + for lang in require_subs + if not any(is_close_match(lang, [sub.language]) for sub in title.tracks.subtitles) + ] + + if missing_langs: + self.log.error(f"Required subtitle language(s) not found: {', '.join(missing_langs)}") + sys.exit(1) + + self.log.info( + f"Required languages found ({', '.join(require_subs)}), downloading all available subtitles" + ) + elif s_lang and "all" not in s_lang: + from envied.core.utilities import is_exact_match + + match_func = is_exact_match if exact_lang else is_close_match + + missing_langs = [ + lang_ + for lang_ in s_lang + if not any(match_func(lang_, [sub.language]) for sub in title.tracks.subtitles) + ] + if missing_langs: + self.log.error(", ".join(missing_langs) + " not found in tracks") + sys.exit(1) + + title.tracks.select_subtitles(lambda x: match_func(x.language, s_lang)) + if not title.tracks.subtitles: + self.log.error(f"There's no {s_lang} Subtitle Track...") + sys.exit(1) + + if not forced_subs: + title.tracks.select_subtitles(lambda x: not x.forced) + + # filter audio tracks + # might have no audio tracks if part of the video, e.g. transport stream hls + if len(title.tracks.audio) > 0: + if not audio_description: + title.tracks.select_audio(lambda x: not x.descriptive) # exclude descriptive audio + if acodec: + title.tracks.select_audio(lambda x: x.codec in acodec) + if not title.tracks.audio: + codec_names = ", ".join(c.name for c in acodec) + self.log.error(f"No audio tracks matching codecs: {codec_names}") + sys.exit(1) + if channels: + title.tracks.select_audio(lambda x: math.ceil(x.channels) == math.ceil(channels)) + if not title.tracks.audio: + self.log.error(f"There's no {channels} Audio Track...") + sys.exit(1) + if no_atmos: + title.tracks.audio = [x for x in title.tracks.audio if not x.atmos] + if not title.tracks.audio: + self.log.error("No non-Atmos audio tracks available...") + sys.exit(1) + if abitrate: + title.tracks.select_audio(lambda x: x.bitrate and x.bitrate // 1000 == abitrate) + if not title.tracks.audio: + self.log.error(f"There's no {abitrate}kbps Audio Track...") + sys.exit(1) + audio_languages = a_lang or lang + if audio_languages: + processed_lang = [] + for language in audio_languages: + if language == "orig": + if title.language: + orig_lang = ( + str(title.language) if hasattr(title.language, "__str__") else title.language + ) + if orig_lang not in processed_lang: + processed_lang.append(orig_lang) + else: + self.log.warning( + "Original language not available for title, skipping 'orig' selection" + ) + else: + if language not in processed_lang: + processed_lang.append(language) + + if "best" in processed_lang: + unique_languages = {track.language for track in title.tracks.audio} + selected_audio = [] + for language in unique_languages: + codecs_to_check = acodec if (acodec and len(acodec) > 1) else [None] + for codec in codecs_to_check: + base_candidates = [ + t + for t in title.tracks.audio + if t.language == language and (codec is None or t.codec == codec) + ] + if not base_candidates: + continue + if audio_description: + standards = [t for t in base_candidates if not t.descriptive] + if standards: + selected_audio.append(max(standards, key=lambda x: x.bitrate or 0)) + descs = [t for t in base_candidates if t.descriptive] + if descs: + selected_audio.append(max(descs, key=lambda x: x.bitrate or 0)) + else: + selected_audio.append(max(base_candidates, key=lambda x: x.bitrate or 0)) + title.tracks.audio = selected_audio + elif "all" not in processed_lang: + # If multiple codecs were explicitly requested, pick the best track per codec per + # requested language instead of selecting *all* bitrate variants of a codec. + if acodec and len(acodec) > 1: + selected_audio: list[Audio] = [] + + for language in processed_lang: + for codec in acodec: + codec_tracks = [a for a in title.tracks.audio if a.codec == codec] + if not codec_tracks: + continue + + candidates = title.tracks.by_language( + codec_tracks, [language], per_language=0, exact_match=exact_lang + ) + if not candidates: + continue + + if audio_description: + standards = [t for t in candidates if not t.descriptive] + if standards: + selected_audio.append(max(standards, key=lambda x: x.bitrate or 0)) + descs = [t for t in candidates if t.descriptive] + if descs: + selected_audio.append(max(descs, key=lambda x: x.bitrate or 0)) + else: + selected_audio.append(max(candidates, key=lambda x: x.bitrate or 0)) + + title.tracks.audio = selected_audio + else: + per_language = 1 + if audio_description: + standard_audio = [a for a in title.tracks.audio if not a.descriptive] + selected_standards = title.tracks.by_language( + standard_audio, processed_lang, per_language=per_language, exact_match=exact_lang + ) + desc_audio = [a for a in title.tracks.audio if a.descriptive] + # Include all descriptive tracks for the requested languages. + selected_descs = title.tracks.by_language( + desc_audio, processed_lang, per_language=0, exact_match=exact_lang + ) + title.tracks.audio = selected_standards + selected_descs + else: + title.tracks.audio = title.tracks.by_language( + title.tracks.audio, + processed_lang, + per_language=per_language, + exact_match=exact_lang, + ) + if not title.tracks.audio: + self.log.error(f"There's no {processed_lang} Audio Track, cannot continue...") + sys.exit(1) + + if ( + video_only + or audio_only + or subs_only + or chapters_only + or no_subs + or no_audio + or no_chapters + or no_video + ): + keep_videos = False + keep_audio = False + keep_subtitles = False + keep_chapters = False + + if video_only or audio_only or subs_only or chapters_only: + if video_only: + keep_videos = True + if audio_only: + keep_audio = True + if subs_only: + keep_subtitles = True + if chapters_only: + keep_chapters = True + else: + keep_videos = True + keep_audio = True + keep_subtitles = True + keep_chapters = True + + if no_subs: + keep_subtitles = False + if no_audio: + keep_audio = False + if no_chapters: + keep_chapters = False + if no_video: + keep_videos = False + + kept_tracks = [] + if keep_videos: + kept_tracks.extend(title.tracks.videos) + if keep_audio: + kept_tracks.extend(title.tracks.audio) + if keep_subtitles: + kept_tracks.extend(title.tracks.subtitles) + if keep_chapters: + kept_tracks.extend(title.tracks.chapters) + kept_tracks.extend(title.tracks.attachments) + + title.tracks = Tracks(kept_tracks) + + selected_tracks, tracks_progress_callables = title.tracks.tree(add_progress=True) + + for track in title.tracks: + if hasattr(track, "needs_drm_loading") and track.needs_drm_loading: + track.load_drm_if_needed(service) + + download_table = Table.grid() + download_table.add_row(selected_tracks) + + video_tracks = title.tracks.videos + if video_tracks: + highest_quality = max((track.height for track in video_tracks if track.height), default=0) + if highest_quality > 0: + if is_widevine_cdm(self.cdm): + quality_based_cdm = self.get_cdm( + self.service, self.profile, drm="widevine", quality=highest_quality + ) + if quality_based_cdm and quality_based_cdm != self.cdm: + self.log.debug( + f"Pre-selecting Widevine CDM based on highest quality {highest_quality}p across all video tracks" + ) + self.cdm = quality_based_cdm + elif is_playready_cdm(self.cdm): + quality_based_cdm = self.get_cdm( + self.service, self.profile, drm="playready", quality=highest_quality + ) + if quality_based_cdm and quality_based_cdm != self.cdm: + self.log.debug( + f"Pre-selecting PlayReady CDM based on highest quality {highest_quality}p across all video tracks" + ) + self.cdm = quality_based_cdm + + dl_start_time = time.time() + + try: + with Live(Padding(download_table, (1, 5)), console=console, refresh_per_second=5): + with ThreadPoolExecutor(downloads) as pool: + for download in futures.as_completed( + ( + pool.submit( + track.download, + session=service.session, + prepare_drm=partial( + partial(self.prepare_drm, table=download_table), + track=track, + title=title, + certificate=partial( + service.get_widevine_service_certificate, + title=title, + track=track, + ), + licence=partial( + service.get_playready_license + if ( + is_playready_cdm(self.cdm) + ) + and hasattr(service, "get_playready_license") + else service.get_widevine_license, + title=title, + track=track, + ), + cdm_only=cdm_only, + vaults_only=vaults_only, + export=export, + ), + cdm=self.cdm, + max_workers=workers, + progress=tracks_progress_callables[i], + ) + for i, track in enumerate(title.tracks) + ) + ): + download.result() + + except KeyboardInterrupt: + console.print(Padding(":x: Download Cancelled...", (0, 5, 1, 5))) + if self.debug_logger: + self.debug_logger.log( + level="WARNING", + operation="download_tracks", + service=self.service, + message="Download cancelled by user", + context={"title": str(title)}, + ) + return + except Exception as e: # noqa + error_messages = [ + ":x: Download Failed...", + ] + if isinstance(e, EnvironmentError): + error_messages.append(f" {e}") + if isinstance(e, ValueError): + error_messages.append(f" {e}") + if isinstance(e, (AttributeError, TypeError)): + console.print_exception() + else: + error_messages.append( + " An unexpected error occurred in one of the download workers.", + ) + if hasattr(e, "returncode"): + error_messages.append(f" Binary call failed, Process exit code: {e.returncode}") + error_messages.append(" See the error trace above for more information.") + if isinstance(e, subprocess.CalledProcessError): + # CalledProcessError already lists the exception trace + console.print_exception() + console.print(Padding(Group(*error_messages), (1, 5))) + + if self.debug_logger: + self.debug_logger.log_error( + "download_tracks", + e, + service=self.service, + context={ + "title": str(title), + "error_type": type(e).__name__, + "tracks_count": len(title.tracks), + "returncode": getattr(e, "returncode", None), + }, + ) + return + + if skip_dl: + console.log("Skipped downloads as --skip-dl was used...") + else: + dl_time = time_elapsed_since(dl_start_time) + console.print(Padding(f"Track downloads finished in [progress.elapsed]{dl_time}[/]", (0, 5))) + + video_track_n = 0 + + while ( + not title.tracks.subtitles + and not no_subs + and not (hasattr(service, "NO_SUBTITLES") and service.NO_SUBTITLES) + and not video_only + and not no_video + and len(title.tracks.videos) > video_track_n + and any( + x.get("codec_name", "").startswith("eia_") + for x in ffprobe(title.tracks.videos[video_track_n].path).get("streams", []) + ) + ): + with console.status(f"Checking Video track {video_track_n + 1} for Closed Captions..."): + try: + # TODO: Figure out the real language, it might be different + # EIA-CC tracks sadly don't carry language information :( + # TODO: Figure out if the CC language is original lang or not. + # Will need to figure out above first to do so. + video_track = title.tracks.videos[video_track_n] + track_id = f"ccextractor-{video_track.id}" + cc_lang = title.language or video_track.language + cc = video_track.ccextractor( + track_id=track_id, + out_path=config.directories.temp + / config.filenames.subtitle.format(id=track_id, language=cc_lang), + language=cc_lang, + original=False, + ) + if cc: + # will not appear in track listings as it's added after all times it lists + title.tracks.add(cc) + self.log.info(f"Extracted a Closed Caption from Video track {video_track_n + 1}") + else: + self.log.info(f"No Closed Captions were found in Video track {video_track_n + 1}") + except EnvironmentError: + self.log.error( + "Cannot extract Closed Captions as the ccextractor executable was not found..." + ) + break + video_track_n += 1 + + # Subtitle output mode configuration (for sidecar originals) + subtitle_output_mode = config.subtitle.get("output_mode", "mux") + sidecar_format = config.subtitle.get("sidecar_format", "srt") + skip_subtitle_mux = ( + subtitle_output_mode == "sidecar" and (title.tracks.videos or title.tracks.audio) + ) + sidecar_subtitles: list[Subtitle] = [] + sidecar_original_paths: dict[str, Path] = {} + if subtitle_output_mode in ("sidecar", "both") and not no_mux: + sidecar_subtitles = [s for s in title.tracks.subtitles if s.path and s.path.exists()] + if sidecar_format == "original": + config.directories.temp.mkdir(parents=True, exist_ok=True) + for subtitle in sidecar_subtitles: + original_path = ( + config.directories.temp / f"sidecar_original_{subtitle.id}{subtitle.path.suffix}" + ) + shutil.copy2(subtitle.path, original_path) + sidecar_original_paths[subtitle.id] = original_path + + with console.status("Converting Subtitles..."): + for subtitle in title.tracks.subtitles: + if sub_format: + if subtitle.codec != sub_format: + subtitle.convert(sub_format) + elif subtitle.codec == Subtitle.Codec.TimedTextMarkupLang: + # MKV does not support TTML, VTT is the next best option + subtitle.convert(Subtitle.Codec.WebVTT) + + with console.status("Checking Subtitles for Fonts..."): + font_names = [] + for subtitle in title.tracks.subtitles: + if subtitle.codec == Subtitle.Codec.SubStationAlphav4: + for line in subtitle.path.read_text("utf8").splitlines(): + if line.startswith("Style: "): + font_names.append(line.removeprefix("Style: ").split(",")[1].strip()) + + font_count, missing_fonts = self.attach_subtitle_fonts(font_names, title, temp_font_files) + + if font_count: + self.log.info(f"Attached {font_count} fonts for the Subtitles") + + if missing_fonts and sys.platform != "win32": + self.suggest_missing_fonts(missing_fonts) + + # Handle DRM decryption BEFORE repacking (must decrypt first!) + service_name = service.__class__.__name__.upper() + decryption_method = config.decryption_map.get(service_name, config.decryption) + decrypt_tool = "mp4decrypt" if decryption_method.lower() == "mp4decrypt" else "Shaka Packager" + + drm_tracks = [track for track in title.tracks if track.drm] + if drm_tracks: + with console.status(f"Decrypting tracks with {decrypt_tool}..."): + has_decrypted = False + for track in drm_tracks: + drm = track.get_drm_for_cdm(self.cdm) + if drm and hasattr(drm, "decrypt"): + drm.decrypt(track.path) + if not isinstance(drm, MonaLisa): + has_decrypted = True + events.emit(events.Types.TRACK_REPACKED, track=track) + else: + self.log.warning( + f"No matching DRM found for track {track} with CDM type {type(self.cdm).__name__}" + ) + if has_decrypted: + self.log.info(f"Decrypted tracks with {decrypt_tool}") + + # Now repack the decrypted tracks + with console.status("Repackaging tracks with FFMPEG..."): + has_repacked = False + for track in title.tracks: + if track.needs_repack: + track.repackage() + has_repacked = True + events.emit(events.Types.TRACK_REPACKED, track=track) + if has_repacked: + # we don't want to fill up the log with "Repacked x track" + self.log.info("Repacked one or more tracks with FFMPEG") + + muxed_paths = [] + muxed_audio_codecs: dict[Path, Optional[Audio.Codec]] = {} + append_audio_codec_suffix = True + + if no_mux: + # Skip muxing, handle individual track files + for track in title.tracks: + if track.path and track.path.exists(): + muxed_paths.append(track.path) + elif isinstance(title, (Movie, Episode)): + progress = Progress( + TextColumn("[progress.description]{task.description}"), + SpinnerColumn(finished_text=""), + BarColumn(), + "•", + TimeRemainingColumn(compact=True, elapsed_when_finished=True), + console=console, + ) + + merge_audio = ( + (not split_audio) if split_audio is not None else config.muxing.get("merge_audio", True) + ) + # When we split audio (merge_audio=False), multiple outputs may exist per title, so suffix codec. + append_audio_codec_suffix = not merge_audio + + multiplex_tasks: list[tuple[TaskID, Tracks, Optional[Audio.Codec]]] = [] + # Track hybrid-processing outputs explicitly so we can always clean them up, + # even if muxing fails early (e.g. SystemExit) before the normal delete loop. + hybrid_temp_paths: list[Path] = [] + + def clone_tracks_for_audio(base_tracks: Tracks, audio_tracks: list[Audio]) -> Tracks: + task_tracks = Tracks() + task_tracks.videos = list(base_tracks.videos) + task_tracks.audio = audio_tracks + task_tracks.subtitles = list(base_tracks.subtitles) + task_tracks.chapters = base_tracks.chapters + task_tracks.attachments = list(base_tracks.attachments) + return task_tracks + + def enqueue_mux_tasks(task_description: str, base_tracks: Tracks) -> None: + if merge_audio or not base_tracks.audio: + task_id = progress.add_task(f"{task_description}...", total=None, start=False) + multiplex_tasks.append((task_id, base_tracks, None)) + return + + audio_by_codec: dict[Optional[Audio.Codec], list[Audio]] = {} + for audio_track in base_tracks.audio: + audio_by_codec.setdefault(audio_track.codec, []).append(audio_track) + + for audio_codec, codec_audio_tracks in audio_by_codec.items(): + description = task_description + if audio_codec: + description = f"{task_description} {audio_codec.name}" + + task_id = progress.add_task(f"{description}...", total=None, start=False) + task_tracks = clone_tracks_for_audio(base_tracks, codec_audio_tracks) + multiplex_tasks.append((task_id, task_tracks, audio_codec)) + + # Check if we're in hybrid mode + if any(r == Video.Range.HYBRID for r in range_) and title.tracks.videos: + # Hybrid mode: process DV and HDR10 tracks separately for each resolution + self.log.info("Processing Hybrid HDR10+DV tracks...") + + # Group video tracks by resolution + resolutions_processed = set() + hdr10_tracks = [v for v in title.tracks.videos if v.range == Video.Range.HDR10] + dv_tracks = [v for v in title.tracks.videos if v.range == Video.Range.DV] + + for hdr10_track in hdr10_tracks: + resolution = hdr10_track.height + if resolution in resolutions_processed: + continue + resolutions_processed.add(resolution) + + # Find matching DV track for this resolution (use the lowest DV resolution) + matching_dv = min(dv_tracks, key=lambda v: v.height) if dv_tracks else None + + if matching_dv: + # Create track pair for this resolution + resolution_tracks = [hdr10_track, matching_dv] + + for track in resolution_tracks: + track.needs_duration_fix = True + + # Run the hybrid processing for this resolution + Hybrid(resolution_tracks, self.service) + + # Create unique output filename for this resolution + hybrid_filename = f"HDR10-DV-{resolution}p.hevc" + hybrid_output_path = config.directories.temp / hybrid_filename + hybrid_temp_paths.append(hybrid_output_path) + + # The Hybrid class creates HDR10-DV.hevc, rename it for this resolution + default_output = config.directories.temp / "HDR10-DV.hevc" + if default_output.exists(): + # If a previous run left this behind, replace it to avoid move() failures. + hybrid_output_path.unlink(missing_ok=True) + shutil.move(str(default_output), str(hybrid_output_path)) + + # Create tracks with the hybrid video output for this resolution + task_description = f"Multiplexing Hybrid HDR10+DV {resolution}p" + task_tracks = Tracks(title.tracks) + title.tracks.chapters + title.tracks.attachments + + # Create a new video track for the hybrid output + hybrid_track = deepcopy(hdr10_track) + hybrid_track.id = f"hybrid_{hdr10_track.id}_{resolution}" + hybrid_track.path = hybrid_output_path + hybrid_track.range = Video.Range.DV # It's now a DV track + hybrid_track.needs_duration_fix = True + title.tracks.add(hybrid_track) + task_tracks.videos = [hybrid_track] + + enqueue_mux_tasks(task_description, task_tracks) + + console.print() + else: + # Normal mode: process each video track separately + for video_track in title.tracks.videos or [None]: + task_description = "Multiplexing" + if video_track: + if len(quality) > 1: + task_description += f" {video_track.height}p" + if len(range_) > 1: + task_description += f" {video_track.range.name}" + + task_tracks = Tracks(title.tracks) + title.tracks.chapters + title.tracks.attachments + if video_track: + task_tracks.videos = [video_track] + + enqueue_mux_tasks(task_description, task_tracks) + + try: + with Live(Padding(progress, (0, 5, 1, 5)), console=console): + mux_index = 0 + for task_id, task_tracks, audio_codec in multiplex_tasks: + progress.start_task(task_id) # TODO: Needed? + audio_expected = not video_only and not no_audio + muxed_path, return_code, errors = task_tracks.mux( + str(title), + progress=partial(progress.update, task_id=task_id), + delete=False, + audio_expected=audio_expected, + title_language=title.language, + skip_subtitles=skip_subtitle_mux, + ) + if muxed_path.exists(): + mux_index += 1 + unique_path = muxed_path.with_name( + f"{muxed_path.stem}.{mux_index}{muxed_path.suffix}" + ) + if unique_path != muxed_path: + shutil.move(muxed_path, unique_path) + muxed_path = unique_path + muxed_paths.append(muxed_path) + muxed_audio_codecs[muxed_path] = audio_codec + if return_code >= 2: + self.log.error(f"Failed to Mux video to Matroska file ({return_code}):") + elif return_code == 1 or errors: + self.log.warning("mkvmerge had at least one warning or error, continuing anyway...") + for line in errors: + if line.startswith("#GUI#error"): + self.log.error(line) + else: + self.log.warning(line) + if return_code >= 2: + sys.exit(1) + + # Output sidecar subtitles before deleting track files + if sidecar_subtitles and not no_mux: + media_info = MediaInfo.parse(muxed_paths[0]) if muxed_paths else None + if media_info: + base_filename = title.get_filename(media_info, show_service=not no_source) + else: + base_filename = str(title) + + sidecar_dir = config.directories.downloads + if not no_folder and isinstance(title, (Episode, Song)) and media_info: + sidecar_dir /= title.get_filename(media_info, show_service=not no_source, folder=True) + sidecar_dir.mkdir(parents=True, exist_ok=True) + + with console.status("Saving subtitle sidecar files..."): + created = self.output_subtitle_sidecars( + sidecar_subtitles, + base_filename, + sidecar_dir, + sidecar_format, + original_paths=sidecar_original_paths or None, + ) + if created: + self.log.info(f"Saved {len(created)} sidecar subtitle files") + + for track in title.tracks: + track.delete() + + # Clear temp font attachment paths and delete other attachments + for attachment in title.tracks.attachments: + if attachment.path and attachment.path in temp_font_files: + attachment.path = None + else: + attachment.delete() + + # Clean up temp fonts + for temp_path in temp_font_files: + temp_path.unlink(missing_ok=True) + for temp_path in sidecar_original_paths.values(): + temp_path.unlink(missing_ok=True) + finally: + # Hybrid() produces a temp HEVC output we rename; make sure it's never left behind. + # Also attempt to remove the default hybrid output name if it still exists. + for temp_path in hybrid_temp_paths: + try: + temp_path.unlink(missing_ok=True) + except PermissionError: + self.log.warning(f"Failed to delete temp file (in use?): {temp_path}") + try: + (config.directories.temp / "HDR10-DV.hevc").unlink(missing_ok=True) + except PermissionError: + self.log.warning( + f"Failed to delete temp file (in use?): {config.directories.temp / 'HDR10-DV.hevc'}" + ) + + else: + # dont mux + muxed_paths.append(title.tracks.audio[0].path) + + if no_mux: + # Handle individual track files without muxing + final_dir = config.directories.downloads + if not no_folder and isinstance(title, (Episode, Song)): + # Create folder based on title + # Use first available track for filename generation + sample_track = ( + title.tracks.videos[0] + if title.tracks.videos + else ( + title.tracks.audio[0] + if title.tracks.audio + else (title.tracks.subtitles[0] if title.tracks.subtitles else None) + ) + ) + if sample_track and sample_track.path: + media_info = MediaInfo.parse(sample_track.path) + final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True) + + final_dir.mkdir(parents=True, exist_ok=True) + + for track_path in muxed_paths: + # Generate appropriate filename for each track + media_info = MediaInfo.parse(track_path) + base_filename = title.get_filename(media_info, show_service=not no_source) + + # Add track type suffix to filename + track = next((t for t in title.tracks if t.path == track_path), None) + if track: + if isinstance(track, Video): + track_suffix = f".{track.codec.name if hasattr(track.codec, 'name') else 'video'}" + elif isinstance(track, Audio): + lang_suffix = f".{track.language}" if track.language else "" + track_suffix = ( + f"{lang_suffix}.{track.codec.name if hasattr(track.codec, 'name') else 'audio'}" + ) + elif isinstance(track, Subtitle): + lang_suffix = f".{track.language}" if track.language else "" + forced_suffix = ".forced" if track.forced else "" + sdh_suffix = ".sdh" if track.sdh else "" + track_suffix = f"{lang_suffix}{forced_suffix}{sdh_suffix}" + else: + track_suffix = "" + + final_path = final_dir / f"{base_filename}{track_suffix}{track_path.suffix}" + else: + final_path = final_dir / f"{base_filename}{track_path.suffix}" + + shutil.move(track_path, final_path) + self.log.debug(f"Saved: {final_path.name}") + else: + # Handle muxed files + for muxed_path in muxed_paths: + media_info = MediaInfo.parse(muxed_path) + final_dir = config.directories.downloads + final_filename = title.get_filename(media_info, show_service=not no_source) + audio_codec_suffix = muxed_audio_codecs.get(muxed_path) + + if not no_folder and isinstance(title, (Episode, Song)): + final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True) + + final_dir.mkdir(parents=True, exist_ok=True) + final_path = final_dir / f"{final_filename}{muxed_path.suffix}" + if final_path.exists() and audio_codec_suffix and append_audio_codec_suffix: + sep = "." if config.scene_naming else " " + final_filename = f"{final_filename.rstrip()}{sep}{audio_codec_suffix.name}" + final_path = final_dir / f"{final_filename}{muxed_path.suffix}" + + if final_path.exists(): + sep = "." if config.scene_naming else " " + i = 2 + while final_path.exists(): + final_path = final_dir / f"{final_filename.rstrip()}{sep}{i}{muxed_path.suffix}" + i += 1 + + shutil.move(muxed_path, final_path) + tags.tag_file(final_path, title, self.tmdb_id) + + title_dl_time = time_elapsed_since(dl_start_time) + console.print( + Padding(f":tada: Title downloaded in [progress.elapsed]{title_dl_time}[/]!", (0, 5, 1, 5)) + ) + + # update cookies + cookie_file = self.get_cookie_path(self.service, self.profile) + if cookie_file: + self.save_cookies(cookie_file, service.session.cookies) + + dl_time = time_elapsed_since(start_time) + + console.print(Padding(f"Processed all titles in [progress.elapsed]{dl_time}", (0, 5, 1, 5))) + + def prepare_drm( + self, + drm: DRM_T, + track: AnyTrack, + title: Title_T, + certificate: Callable, + licence: Callable, + track_kid: Optional[UUID] = None, + table: Table = None, + cdm_only: bool = False, + vaults_only: bool = False, + export: Optional[Path] = None, + ) -> None: + """ + Prepare the DRM by getting decryption data like KIDs, Keys, and such. + The DRM object should be ready for decryption once this function ends. + """ + if not drm: + return + + track_quality = None + if isinstance(track, Video) and track.height: + track_quality = track.height + + if isinstance(drm, Widevine): + if not is_widevine_cdm(self.cdm): + widevine_cdm = self.get_cdm(self.service, self.profile, drm="widevine", quality=track_quality) + if widevine_cdm: + if track_quality: + self.log.info(f"Switching to Widevine CDM for Widevine {track_quality}p content") + else: + self.log.info("Switching to Widevine CDM for Widevine content") + self.cdm = widevine_cdm + + elif isinstance(drm, PlayReady): + if not is_playready_cdm(self.cdm): + playready_cdm = self.get_cdm(self.service, self.profile, drm="playready", quality=track_quality) + if playready_cdm: + if track_quality: + self.log.info(f"Switching to PlayReady CDM for PlayReady {track_quality}p content") + else: + self.log.info("Switching to PlayReady CDM for PlayReady content") + self.cdm = playready_cdm + + if isinstance(drm, Widevine): + if self.debug_logger: + self.debug_logger.log_drm_operation( + drm_type="Widevine", + operation="prepare_drm", + service=self.service, + context={ + "track": str(track), + "title": str(title), + "pssh": drm.pssh.dumps() if drm.pssh else None, + "kids": [k.hex for k in drm.kids], + "track_kid": track_kid.hex if track_kid else None, + }, + ) + + with self.DRM_TABLE_LOCK: + pssh_display = self.truncate_pssh_for_display(drm.pssh.dumps(), "Widevine") + cek_tree = Tree(Text.assemble(("Widevine", "cyan"), (f"({pssh_display})", "text"), overflow="fold")) + pre_existing_tree = next( + (x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None + ) + if pre_existing_tree: + cek_tree = pre_existing_tree + + need_license = False + all_kids = list(drm.kids) + if track_kid and track_kid not in all_kids: + all_kids.append(track_kid) + + for kid in all_kids: + if kid in drm.content_keys: + continue + + is_track_kid = ["", "*"][kid == track_kid] + + if not cdm_only: + content_key, vault_used = self.vaults.get_key(kid) + if content_key: + drm.content_keys[kid] = content_key + label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}" + if not any(f"{kid.hex}:{content_key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + self.vaults.add_key(kid, content_key, excluding=vault_used) + + if self.debug_logger: + self.debug_logger.log_vault_query( + vault_name=vault_used, + operation="get_key_success", + service=self.service, + context={ + "kid": kid.hex, + "content_key": content_key, + "track": str(track), + "from_cache": True, + }, + ) + elif vaults_only: + msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="vault_key_not_found", + service=self.service, + message=msg, + context={"kid": kid.hex, "track": str(track)}, + ) + raise Widevine.Exceptions.CEKNotFound(msg) + else: + need_license = True + + if kid not in drm.content_keys and cdm_only: + need_license = True + + if need_license and not vaults_only: + from_vaults = drm.content_keys.copy() + + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="get_license", + service=self.service, + message="Requesting Widevine license from service", + context={ + "track": str(track), + "kids_needed": [k.hex for k in all_kids if k not in drm.content_keys], + }, + ) + + try: + if self.service == "NF": + drm.get_NF_content_keys(cdm=self.cdm, licence=licence, certificate=certificate) + else: + drm.get_content_keys(cdm=self.cdm, licence=licence, certificate=certificate) + except Exception as e: + if isinstance(e, (Widevine.Exceptions.EmptyLicense, Widevine.Exceptions.CEKNotFound)): + msg = str(e) + else: + msg = f"An exception occurred in the Service's license function: {e}" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + if self.debug_logger: + self.debug_logger.log_error( + "get_license", + e, + service=self.service, + context={"track": str(track), "exception_type": type(e).__name__}, + ) + raise e + + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="license_keys_retrieved", + service=self.service, + context={ + "track": str(track), + "keys_count": len(drm.content_keys), + "kids": [k.hex for k in drm.content_keys.keys()], + }, + ) + + for kid_, key in drm.content_keys.items(): + if key == "0" * 32: + key = f"[red]{key}[/]" + is_track_kid_marker = ["", "*"][kid_ == track_kid] + label = f"[text2]{kid_.hex}:{key}{is_track_kid_marker}" + if not any(f"{kid_.hex}:{key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + + drm.content_keys = { + kid_: key for kid_, key in drm.content_keys.items() if key and key.count("0") != len(key) + } + + # The CDM keys may have returned blank content keys for KIDs we got from vaults. + # So we re-add the keys from vaults earlier overwriting blanks or removed KIDs data. + drm.content_keys.update(from_vaults) + + successful_caches = self.vaults.add_keys(drm.content_keys) + self.log.info( + f"Cached {len(drm.content_keys)} Key{'' if len(drm.content_keys) == 1 else 's'} to " + f"{successful_caches}/{len(self.vaults)} Vaults" + ) + + if track_kid and track_kid not in drm.content_keys: + msg = f"No Content Key for KID {track_kid.hex} was returned in the License" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + raise Widevine.Exceptions.CEKNotFound(msg) + + if cek_tree.children and not pre_existing_tree: + table.add_row() + table.add_row(cek_tree) + + if export: + keys = {} + if export.is_file(): + keys = jsonpickle.loads(export.read_text(encoding="utf8")) or {} + if str(title) not in keys: + keys[str(title)] = {} + if str(track) not in keys[str(title)]: + keys[str(title)][str(track)] = {} + + track_data = keys[str(title)][str(track)] + track_data["url"] = track.url + track_data["descriptor"] = track.descriptor.name + + if "keys" not in track_data: + track_data["keys"] = {} + for kid, key in drm.content_keys.items(): + track_data["keys"][kid.hex] = key + + export.write_text(jsonpickle.dumps(keys, indent=4), encoding="utf8") + + elif isinstance(drm, PlayReady): + if self.debug_logger: + self.debug_logger.log_drm_operation( + drm_type="PlayReady", + operation="prepare_drm", + service=self.service, + context={ + "track": str(track), + "title": str(title), + "pssh": drm.pssh_b64 or "", + "kids": [k.hex for k in drm.kids], + "track_kid": track_kid.hex if track_kid else None, + }, + ) + + with self.DRM_TABLE_LOCK: + pssh_display = self.truncate_pssh_for_display(drm.pssh_b64 or "", "PlayReady") + cek_tree = Tree( + Text.assemble( + ("PlayReady", "cyan"), + (f"({pssh_display})", "text"), + overflow="fold", + ) + ) + pre_existing_tree = next( + (x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None + ) + if pre_existing_tree: + cek_tree = pre_existing_tree + + need_license = False + all_kids = list(drm.kids) + if track_kid and track_kid not in all_kids: + all_kids.append(track_kid) + + for kid in all_kids: + if kid in drm.content_keys: + continue + + is_track_kid = ["", "*"][kid == track_kid] + + if not cdm_only: + content_key, vault_used = self.vaults.get_key(kid) + if content_key: + drm.content_keys[kid] = content_key + label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}" + if not any(f"{kid.hex}:{content_key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + self.vaults.add_key(kid, content_key, excluding=vault_used) + + if self.debug_logger: + self.debug_logger.log_vault_query( + vault_name=vault_used, + operation="get_key_success", + service=self.service, + context={ + "kid": kid.hex, + "content_key": content_key, + "track": str(track), + "from_cache": True, + "drm_type": "PlayReady", + }, + ) + elif vaults_only: + msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="vault_key_not_found", + service=self.service, + message=msg, + context={"kid": kid.hex, "track": str(track), "drm_type": "PlayReady"}, + ) + raise PlayReady.Exceptions.CEKNotFound(msg) + else: + need_license = True + + if kid not in drm.content_keys and cdm_only: + need_license = True + + if need_license and not vaults_only: + from_vaults = drm.content_keys.copy() + + try: + drm.get_content_keys(cdm=self.cdm, licence=licence, certificate=certificate) + except Exception as e: + if isinstance(e, (PlayReady.Exceptions.EmptyLicense, PlayReady.Exceptions.CEKNotFound)): + msg = str(e) + else: + msg = f"An exception occurred in the Service's license function: {e}" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + if self.debug_logger: + self.debug_logger.log_error( + "get_license_playready", + e, + service=self.service, + context={ + "track": str(track), + "exception_type": type(e).__name__, + "drm_type": "PlayReady", + }, + ) + raise e + + for kid_, key in drm.content_keys.items(): + is_track_kid_marker = ["", "*"][kid_ == track_kid] + label = f"[text2]{kid_.hex}:{key}{is_track_kid_marker}" + if not any(f"{kid_.hex}:{key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + + drm.content_keys.update(from_vaults) + + successful_caches = self.vaults.add_keys(drm.content_keys) + self.log.info( + f"Cached {len(drm.content_keys)} Key{'' if len(drm.content_keys) == 1 else 's'} to " + f"{successful_caches}/{len(self.vaults)} Vaults" + ) + + if track_kid and track_kid not in drm.content_keys: + msg = f"No Content Key for KID {track_kid.hex} was returned in the License" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + raise PlayReady.Exceptions.CEKNotFound(msg) + + if cek_tree.children and not pre_existing_tree: + table.add_row() + table.add_row(cek_tree) + + if export: + keys = {} + if export.is_file(): + keys = jsonpickle.loads(export.read_text(encoding="utf8")) or {} + if str(title) not in keys: + keys[str(title)] = {} + if str(track) not in keys[str(title)]: + keys[str(title)][str(track)] = {} + + track_data = keys[str(title)][str(track)] + track_data["url"] = track.url + track_data["descriptor"] = track.descriptor.name + + if "keys" not in track_data: + track_data["keys"] = {} + for kid, key in drm.content_keys.items(): + track_data["keys"][kid.hex] = key + + export.write_text(jsonpickle.dumps(keys, indent=4), encoding="utf8") + + elif isinstance(drm, MonaLisa): + with self.DRM_TABLE_LOCK: + display_id = drm.content_id or drm.pssh + pssh_display = self.truncate_pssh_for_display(display_id, "MonaLisa") + cek_tree = Tree(Text.assemble(("MonaLisa", "cyan"), (f"({pssh_display})", "text"), overflow="fold")) + pre_existing_tree = next( + (x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None + ) + if pre_existing_tree: + cek_tree = pre_existing_tree + + for kid_, key in drm.content_keys.items(): + label = f"[text2]{kid_.hex}:{key}" + if not any(f"{kid_.hex}:{key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + + if cek_tree.children and not pre_existing_tree: + table.add_row() + table.add_row(cek_tree) + + @staticmethod + def get_cookie_path(service: str, profile: Optional[str]) -> Optional[Path]: + """Get Service Cookie File Path for Profile.""" + direct_cookie_file = config.directories.cookies / f"{service}.txt" + profile_cookie_file = config.directories.cookies / service / f"{profile}.txt" + default_cookie_file = config.directories.cookies / service / "default.txt" + + if direct_cookie_file.exists(): + return direct_cookie_file + elif profile_cookie_file.exists(): + return profile_cookie_file + elif default_cookie_file.exists(): + return default_cookie_file + + @staticmethod + def get_cookie_jar(service: str, profile: Optional[str]) -> Optional[MozillaCookieJar]: + """Get Service Cookies for Profile.""" + cookie_file = dl.get_cookie_path(service, profile) + if cookie_file: + cookie_jar = MozillaCookieJar(cookie_file) + cookie_data = html.unescape(cookie_file.read_text("utf8")).splitlines(keepends=False) + for i, line in enumerate(cookie_data): + if line and not line.startswith("#"): + line_data = line.lstrip().split("\t") + # Disable client-side expiry checks completely across everywhere + # Even though the cookies are loaded under ignore_expires=True, stuff + # like python-requests may not use them if they are expired + line_data[4] = "" + cookie_data[i] = "\t".join(line_data) + cookie_data = "\n".join(cookie_data) + cookie_file.write_text(cookie_data, "utf8") + cookie_jar.load(ignore_discard=True, ignore_expires=True) + return cookie_jar + + @staticmethod + def save_cookies(path: Path, cookies: CookieJar): + if hasattr(cookies, "jar"): + cookies = cookies.jar + + cookie_jar = MozillaCookieJar(path) + cookie_jar.load() + for cookie in cookies: + cookie_jar.set_cookie(cookie) + cookie_jar.save(ignore_discard=True) + + @staticmethod + def get_credentials(service: str, profile: Optional[str]) -> Optional[Credential]: + """Get Service Credentials for Profile.""" + credentials = config.credentials.get(service) + if credentials: + if isinstance(credentials, dict): + if profile: + credentials = credentials.get(profile) or credentials.get("default") + else: + credentials = credentials.get("default") + if credentials: + if isinstance(credentials, list): + return Credential(*credentials) + return Credential.loads(credentials) # type: ignore + + def get_cdm( + self, + service: str, + profile: Optional[str] = None, + drm: Optional[str] = None, + quality: Optional[int] = None, + ) -> Optional[object]: + """ + Get CDM for a specified service (either Local or Remote CDM). + Now supports quality-based selection when quality is provided. + Raises a ValueError if there's a problem getting a CDM. + """ + cdm_name = config.cdm.get(service) or config.cdm.get("default") + if not cdm_name: + return None + + if isinstance(cdm_name, dict): + if quality: + quality_match = None + quality_keys = [] + + for key in cdm_name.keys(): + if ( + isinstance(key, str) + and any(op in key for op in [">=", ">", "<=", "<"]) + or (isinstance(key, str) and key.isdigit()) + ): + quality_keys.append(key) + + def sort_quality_key(key): + if key.isdigit(): + return (0, int(key)) # Exact matches first + elif key.startswith(">="): + return (1, -int(key[2:])) # >= descending + elif key.startswith(">"): + return (1, -int(key[1:])) # > descending + elif key.startswith("<="): + return (2, int(key[2:])) # <= ascending + elif key.startswith("<"): + return (2, int(key[1:])) # < ascending + return (3, 0) # Other keys last + + quality_keys.sort(key=sort_quality_key) + + for key in quality_keys: + if key.isdigit() and quality == int(key): + quality_match = cdm_name[key] + self.log.debug(f"Selected CDM based on exact quality match {quality}p: {quality_match}") + break + elif key.startswith(">="): + threshold = int(key[2:]) + if quality >= threshold: + quality_match = cdm_name[key] + self.log.debug(f"Selected CDM based on quality {quality}p >= {threshold}p: {quality_match}") + break + elif key.startswith(">"): + threshold = int(key[1:]) + if quality > threshold: + quality_match = cdm_name[key] + self.log.debug(f"Selected CDM based on quality {quality}p > {threshold}p: {quality_match}") + break + elif key.startswith("<="): + threshold = int(key[2:]) + if quality <= threshold: + quality_match = cdm_name[key] + self.log.debug(f"Selected CDM based on quality {quality}p <= {threshold}p: {quality_match}") + break + elif key.startswith("<"): + threshold = int(key[1:]) + if quality < threshold: + quality_match = cdm_name[key] + self.log.debug(f"Selected CDM based on quality {quality}p < {threshold}p: {quality_match}") + break + + if quality_match: + cdm_name = quality_match + + if isinstance(cdm_name, dict): + lower_keys = {k.lower(): v for k, v in cdm_name.items()} + if {"widevine", "playready"} & lower_keys.keys(): + drm_key = None + if drm: + drm_key = { + "wv": "widevine", + "widevine": "widevine", + "pr": "playready", + "playready": "playready", + }.get(drm.lower()) + cdm_name = lower_keys.get(drm_key or "widevine") or lower_keys.get("playready") + else: + cdm_name = cdm_name.get(profile) or cdm_name.get("default") or config.cdm.get("default") + if not cdm_name: + return None + + cdm_api = next(iter(x.copy() for x in config.remote_cdm if x["name"] == cdm_name), None) + if cdm_api: + cdm_type = cdm_api.get("type") + + if cdm_type == "decrypt_labs": + del cdm_api["name"] + del cdm_api["type"] + + if "secret" not in cdm_api or not cdm_api["secret"]: + if config.decrypt_labs_api_key: + cdm_api["secret"] = config.decrypt_labs_api_key + else: + raise ValueError( + f"No secret provided for DecryptLabs CDM '{cdm_name}' and no global " + "decrypt_labs_api_key configured" + ) + + # All DecryptLabs CDMs use DecryptLabsRemoteCDM + return DecryptLabsRemoteCDM(service_name=service, vaults=self.vaults, **cdm_api) + + elif cdm_type == "custom_api": + del cdm_api["name"] + del cdm_api["type"] + + # All Custom API CDMs use CustomRemoteCDM + return CustomRemoteCDM(service_name=service, vaults=self.vaults, **cdm_api) + + else: + device_type = cdm_api.get("Device Type", cdm_api.get("device_type", "")) + if str(device_type).upper() == "PLAYREADY": + return PlayReadyRemoteCdm( + security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)), + host=cdm_api.get("Host", cdm_api.get("host")), + secret=cdm_api.get("Secret", cdm_api.get("secret")), + device_name=cdm_api.get("Device Name", cdm_api.get("device_name")), + ) + else: + return RemoteCdm( + device_type=cdm_api.get("Device Type", cdm_api.get("device_type", "")), + system_id=cdm_api.get("System ID", cdm_api.get("system_id", "")), + security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)), + host=cdm_api.get("Host", cdm_api.get("host")), + secret=cdm_api.get("Secret", cdm_api.get("secret")), + device_name=cdm_api.get("Device Name", cdm_api.get("device_name")), + ) + + prd_path = config.directories.prds / f"{cdm_name}.prd" + if not prd_path.is_file(): + prd_path = config.directories.wvds / f"{cdm_name}.prd" + if prd_path.is_file(): + device = PlayReadyDevice.load(prd_path) + return PlayReadyCdm.from_device(device) + + cdm_path = config.directories.wvds / f"{cdm_name}.wvd" + if not cdm_path.is_file(): + raise ValueError(f"{cdm_name} does not exist or is not a file") + + try: + device = Device.load(cdm_path) + except ConstError as e: + if "expected 2 but parsed 1" in str(e): + raise ValueError( + f"{cdm_name}.wvd seems to be a v1 WVD file, use `pywidevine migrate --help` to migrate it to v2." + ) + raise ValueError(f"{cdm_name}.wvd is an invalid or corrupt Widevine Device file, {e}") + + return WidevineCdm.from_device(device) diff --git a/reset/packages/envied/src/envied/commands/dl.py b/reset/packages/envied/src/envied/commands/dl.py new file mode 100644 index 0000000..a0342ec --- /dev/null +++ b/reset/packages/envied/src/envied/commands/dl.py @@ -0,0 +1,3453 @@ +from __future__ import annotations + +import html +import json +import logging +import math +import os +import random +import re +import shutil +import subprocess +import sys +import time +from concurrent import futures +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +from functools import partial +from http.cookiejar import CookieJar, MozillaCookieJar +from itertools import product +from pathlib import Path +from threading import Lock +from typing import Any, Callable, Optional +from uuid import UUID + +import click +import yaml +from langcodes import Language +from pymediainfo import MediaInfo +from rich.console import Group +from rich.live import Live +from rich.padding import Padding +from rich.panel import Panel +from rich.progress import BarColumn, Progress, SpinnerColumn, TaskID, TextColumn, TimeRemainingColumn +from rich.rule import Rule +from rich.spinner import Spinner +from rich.table import Table +from rich.text import Text +from rich.tree import Tree + +from envied.core import binaries, providers +from envied.core.cdm import DecryptLabsRemoteCDM +from envied.core.cdm.detect import is_playready_cdm, is_widevine_cdm +from envied.core.config import config +from envied.core.console import console +from envied.core.constants import DOWNLOAD_LICENCE_ONLY, AnyTrack, context_settings +from envied.core.credential import Credential +from envied.core.drm import DRM_T, MonaLisa, PlayReady, Widevine +from envied.core.events import events +from envied.core.proxies import Basic, Gluetun, Hola, NordVPN, SurfsharkVPN, WindscribeVPN +from envied.core.service import Service +from envied.core.services import Services +from envied.core.title_cacher import get_account_hash +from envied.core.titles import Movie, Movies, Series, Song, Title_T +from envied.core.titles.episode import Episode +from envied.core.tracks import Audio, Subtitle, Tracks, Video +from envied.core.tracks.attachment import Attachment +from envied.core.tracks.dv_fixup import apply_dv_fixup +from envied.core.tracks.hybrid import Hybrid +from envied.core.utilities import (find_font_with_fallbacks, find_missing_langs, get_debug_logger, + get_system_fonts, init_debug_logger, is_close_match, suggest_font_packages, + time_elapsed_since) +from envied.core.utils import tags +from envied.core.utils.bitrate import apply_real_bitrates +from envied.core.utils.click_types import (AUDIO_CODEC_LIST, LANGUAGE_RANGE, QUALITY_LIST, SEASON_RANGE, + SLOW_DELAY_RANGE, ContextData, MultipleChoice, MultipleVideoCodecChoice, + SubtitleCodecChoice) +from envied.core.utils.collections import merge_dict +from envied.core.utils.selector import select_multiple +from envied.core.utils.subprocess import ffprobe +from envied.core.vaults import Vaults + + +class dl: + @staticmethod + def truncate_pssh_for_display(pssh_string: str, drm_type: str) -> str: + """Truncate PSSH string for display when not in debug mode.""" + if logging.root.level == logging.DEBUG or not pssh_string: + return pssh_string + + max_width = console.width - len(drm_type) - 12 + if len(pssh_string) <= max_width: + return pssh_string + + return pssh_string[: max_width - 3] + "..." + + def find_custom_font(self, font_name: str) -> Optional[Path]: + """ + Find font in custom fonts directory. + + Args: + font_name: Font family name to find + + Returns: + Path to font file, or None if not found + """ + family_dir = Path(config.directories.fonts, font_name) + if family_dir.exists(): + fonts = list(family_dir.glob("*.*tf")) + return fonts[0] if fonts else None + return None + + def prepare_temp_font( + self, font_name: str, matched_font: Path, system_fonts: dict[str, Path], temp_font_files: list[Path] + ) -> Path: + """ + Copy system font to temp and log if using fallback. + + Args: + font_name: Requested font name + matched_font: Path to matched system font + system_fonts: Dictionary of available system fonts + temp_font_files: List to track temp files for cleanup + + Returns: + Path to temp font file + """ + # Find the matched name for logging + matched_name = next((name for name, path in system_fonts.items() if path == matched_font), None) + + if matched_name and matched_name.lower() != font_name.lower(): + self.log.info(f"Using '{matched_name}' as fallback for '{font_name}'") + + # Create unique temp file path + safe_name = font_name.replace(" ", "_").replace("/", "_") + temp_path = config.directories.temp / f"font_{safe_name}{matched_font.suffix}" + + # Copy if not already exists + if not temp_path.exists(): + shutil.copy2(matched_font, temp_path) + temp_font_files.append(temp_path) + + return temp_path + + def attach_subtitle_fonts( + self, font_names: list[str], title: Title_T, temp_font_files: list[Path] + ) -> tuple[int, list[str]]: + """ + Attach fonts for subtitle rendering. + + Args: + font_names: List of font names requested by subtitles + title: Title object to attach fonts to + temp_font_files: List to track temp files for cleanup + + Returns: + Tuple of (fonts_attached_count, missing_fonts_list) + """ + system_fonts = get_system_fonts() + + font_count = 0 + missing_fonts = [] + + for font_name in set(font_names): + # Try custom fonts first + if custom_font := self.find_custom_font(font_name): + title.tracks.add(Attachment(path=custom_font, name=f"{font_name} ({custom_font.stem})")) + font_count += 1 + continue + + # Try system fonts with fallback + if system_font := find_font_with_fallbacks(font_name, system_fonts): + temp_path = self.prepare_temp_font(font_name, system_font, system_fonts, temp_font_files) + title.tracks.add(Attachment(path=temp_path, name=f"{font_name} ({system_font.stem})")) + font_count += 1 + else: + self.log.warning(f"Subtitle uses font '{font_name}' but it could not be found") + missing_fonts.append(font_name) + + return font_count, missing_fonts + + def suggest_missing_fonts(self, missing_fonts: list[str]) -> None: + """ + Show package installation suggestions for missing fonts. + + Args: + missing_fonts: List of font names that couldn't be found + """ + if suggestions := suggest_font_packages(missing_fonts): + self.log.info("Install font packages to improve subtitle rendering:") + for package_cmd, fonts in suggestions.items(): + self.log.info(f" $ sudo apt install {package_cmd}") + self.log.info(f" → Provides: {', '.join(fonts)}") + + def generate_sidecar_subtitle_path( + self, + subtitle: Subtitle, + base_filename: str, + output_dir: Path, + target_codec: Optional[Subtitle.Codec] = None, + source_path: Optional[Path] = None, + ) -> Path: + """Generate sidecar path: {base}.{lang}[.forced][.sdh].{ext}""" + lang_suffix = str(subtitle.language) if subtitle.language else "und" + forced_suffix = ".forced" if subtitle.forced else "" + sdh_suffix = ".sdh" if (subtitle.sdh or subtitle.cc) else "" + + extension = (target_codec or subtitle.codec or Subtitle.Codec.SubRip).extension + if not target_codec and not subtitle.codec and source_path and source_path.suffix: + extension = source_path.suffix.lstrip(".") + + filename = f"{base_filename}.{lang_suffix}{forced_suffix}{sdh_suffix}.{extension}" + return output_dir / filename + + def output_subtitle_sidecars( + self, + subtitles: list[Subtitle], + base_filename: str, + output_dir: Path, + sidecar_format: str, + original_paths: Optional[dict[str, Path]] = None, + ) -> list[Path]: + """Output subtitles as sidecar files, converting if needed.""" + created_paths: list[Path] = [] + config.directories.temp.mkdir(parents=True, exist_ok=True) + + for subtitle in subtitles: + source_path = subtitle.path + if sidecar_format == "original" and original_paths and subtitle.id in original_paths: + source_path = original_paths[subtitle.id] + + if not source_path or not source_path.exists(): + continue + + # Determine target codec + if sidecar_format == "original": + target_codec = None + if source_path.suffix: + try: + target_codec = Subtitle.Codec.from_mime(source_path.suffix.lstrip(".")) + except ValueError: + target_codec = None + else: + target_codec = Subtitle.Codec.from_mime(sidecar_format) + + sidecar_path = self.generate_sidecar_subtitle_path( + subtitle, base_filename, output_dir, target_codec, source_path=source_path + ) + + # Copy or convert + if not target_codec or subtitle.codec == target_codec: + shutil.copy2(source_path, sidecar_path) + else: + # Create temp copy for conversion to preserve original + temp_path = config.directories.temp / f"sidecar_{subtitle.id}{source_path.suffix}" + shutil.copy2(source_path, temp_path) + + temp_sub = Subtitle( + subtitle.url, + subtitle.language, + is_original_lang=subtitle.is_original_lang, + descriptor=subtitle.descriptor, + codec=subtitle.codec, + forced=subtitle.forced, + sdh=subtitle.sdh, + cc=subtitle.cc, + id_=f"{subtitle.id}_sc", + ) + temp_sub.path = temp_path + try: + temp_sub.convert(target_codec) + if temp_sub.path and temp_sub.path.exists(): + shutil.copy2(temp_sub.path, sidecar_path) + finally: + if temp_sub.path and temp_sub.path.exists(): + temp_sub.path.unlink(missing_ok=True) + temp_path.unlink(missing_ok=True) + + created_paths.append(sidecar_path) + + return created_paths + + @click.command( + short_help="Download, Decrypt, and Mux tracks for titles from a Service.", + cls=Services, + context_settings=dict(**context_settings, default_map=config.dl, token_normalize_func=Services.get_tag), + ) + @click.option( + "-p", "--profile", type=str, default=None, help="Profile to use for Credentials and Cookies (if available)." + ) + @click.option( + "-q", + "--quality", + type=QUALITY_LIST, + default=[], + help="Download Resolution(s), defaults to the best available resolution.", + ) + @click.option( + "-v", + "--vcodec", + type=MultipleVideoCodecChoice(Video.Codec), + default=[], + help="Video Codec(s) to download, defaults to any codec.", + ) + @click.option( + "-a", + "--acodec", + type=AUDIO_CODEC_LIST, + default=[], + help="Audio Codec(s) to download (comma-separated), e.g., 'AAC,EC3'. Defaults to any.", + ) + @click.option( + "-vb", + "--vbitrate", + type=int, + default=None, + help="Video Bitrate to download (in kbps), defaults to highest available.", + ) + @click.option( + "-ab", + "--abitrate", + type=int, + default=None, + help="Audio Bitrate to download (in kbps), defaults to highest available.", + ) + @click.option( + "-vb-range", + "--vbitrate-range", + type=str, + default=None, + help="Video Bitrate range in kbps (e.g., '6000-7000'). Selects the highest bitrate within the range.", + ) + @click.option( + "-ab-range", + "--abitrate-range", + type=str, + default=None, + help="Audio Bitrate range in kbps (e.g., '128-256'). Selects the highest bitrate within the range.", + ) + @click.option( + "-r", + "--range", + "range_", + type=MultipleChoice(Video.Range, case_sensitive=False), + default=[Video.Range.SDR], + help="Video Color Range(s) to download, defaults to SDR.", + ) + @click.option( + "-c", + "--channels", + type=float, + default=None, + help="Audio Channel(s) to download. Matches sub-channel layouts like 5.1 with 6.0 implicitly.", + ) + @click.option( + "-naa", + "--noatmos", + "no_atmos", + is_flag=True, + default=False, + help="Exclude Dolby Atmos audio tracks when selecting audio.", + ) + @click.option( + "--split-audio", + "split_audio", + is_flag=True, + default=None, + help="Create separate output files per audio codec instead of merging all audio.", + ) + @click.option( + "--select-titles", + is_flag=True, + default=False, + help="Interactively select downloads from a list. Only use with Series to select Episodes.", + ) + @click.option( + "-w", + "--wanted", + type=SEASON_RANGE, + default=None, + help="Wanted episodes, e.g. `S01-S05,S07`, `S01E01-S02E03`, `S02-S02E03`, etc., defaults to all.", + ) + @click.option( + "-l", + "--lang", + type=LANGUAGE_RANGE, + default="orig", + help="Language(s) wanted for Video and Audio (comma-separated). Use 'orig' to select the original language, e.g. 'orig,en' for both original and English.", + ) + @click.option( + "--latest-episode", + is_flag=True, + default=False, + help="Download only the single most recent episode available.", + ) + @click.option( + "-vl", + "--v-lang", + type=LANGUAGE_RANGE, + default=[], + help="Language wanted for Video. You would use this if the video language doesn't match the audio.", + ) + @click.option( + "-al", + "--a-lang", + type=LANGUAGE_RANGE, + default=[], + help="Language wanted for Audio, overrides -l/--lang for audio tracks.", + ) + @click.option("-sl", "--s-lang", type=LANGUAGE_RANGE, default=["all"], help="Language wanted for Subtitles.") + @click.option( + "--require-subs", + type=LANGUAGE_RANGE, + default=[], + help="Required subtitle languages. Downloads all subtitles only if these languages exist. Cannot be used with --s-lang.", + ) + @click.option("-fs", "--forced-subs", is_flag=True, default=False, help="Include forced subtitle tracks.") + @click.option( + "--exact-lang", + is_flag=True, + default=False, + help="Use exact language matching (no variants). With this flag, -l es-419 matches ONLY es-419, not es-ES or other variants.", + ) + @click.option( + "--proxy", + type=str, + default=None, + help="Proxy URI to use. If a 2-letter country is provided, it will try to get a proxy from the config.", + ) + @click.option( + "--tag", type=str, default=None, help="Set the Group Tag to be used, overriding the one in config if any." + ) + @click.option("--repack", is_flag=True, default=False, help="Add REPACK tag to the output filename.") + @click.option( + "-rvb", + "--real-video-bitrate", + is_flag=True, + default=False, + help="Probe actual media size to compute true video bitrates (top renditions per codec/range), " + "overriding the manifest's declared bitrate.", + ) + @click.option( + "-rab", + "--real-audio-bitrate", + is_flag=True, + default=False, + help="Probe actual media size to compute true audio bitrates (top renditions per codec/channels/language), " + "overriding the manifest's declared bitrate. Slower than --real-video-bitrate (more renditions).", + ) + @click.option( + "--tmdb", + "tmdb_id", + type=int, + default=None, + help="Use this TMDB ID for tagging instead of automatic lookup.", + ) + @click.option( + "--animeapi", + "animeapi_id", + type=str, + default=None, + help="Anime database ID via AnimeAPI (e.g. mal:12345, anilist:98765). Defaults to MAL if no prefix.", + ) + @click.option( + "--enrich", + is_flag=True, + default=False, + help="Override show title and year from external source. Requires --tmdb, --imdb, or --animeapi.", + ) + @click.option( + "--imdb", + "imdb_id", + type=str, + default=None, + help="Use this IMDB ID (e.g. tt1375666) for tagging instead of automatic lookup.", + ) + @click.option( + "--sub-format", + type=SubtitleCodecChoice(Subtitle.Codec), + default=None, + help="Set Output Subtitle Format, only converting if necessary.", + ) + @click.option("-V", "--video-only", is_flag=True, default=False, help="Only download video tracks.") + @click.option("-A", "--audio-only", is_flag=True, default=False, help="Only download audio tracks.") + @click.option("-S", "--subs-only", is_flag=True, default=False, help="Only download subtitle tracks.") + @click.option("-C", "--chapters-only", is_flag=True, default=False, help="Only download chapter markers.") + @click.option("-ns", "--no-subs", is_flag=True, default=False, help="Do not download subtitle tracks.") + @click.option("-na", "--no-audio", is_flag=True, default=False, help="Do not download audio tracks.") + @click.option("-nc", "--no-chapters", is_flag=True, default=False, help="Do not download chapter markers.") + @click.option("-nv", "--no-video", is_flag=True, default=False, help="Do not download video tracks.") + @click.option("-ad", "--audio-description", is_flag=True, default=False, help="Download audio description tracks.") + @click.option( + "--slow", + type=SLOW_DELAY_RANGE, + default=None, + help="Add a delay between each Title download to act more like a real device. " + "Use --slow for a 60-120s delay, or --slow MIN-MAX (e.g., --slow 20-40) for a custom range. " + "Minimum delay is 20 seconds.", + ) + @click.option( + "--list", + "list_", + is_flag=True, + default=False, + help="Skip downloading and list available tracks and what tracks would have been downloaded.", + ) + @click.option( + "--list-titles", + is_flag=True, + default=False, + help="Skip downloading, only list available titles that would have been downloaded.", + ) + @click.option( + "--skip-dl", is_flag=True, default=False, help="Skip downloading while still retrieving the decryption keys." + ) + @click.option( + "--export", + is_flag=True, + default=False, + help="Export track info and decryption keys to a JSON file in the exports directory.", + ) + @click.option( + "--import", + "import_file", + type=str, + default=None, + hidden=True, + help="Internal: path to an export JSON to reconstruct a download from (used by 'envied.import').", + ) + @click.option( + "--cdm-only/--vaults-only", + is_flag=True, + default=None, + help="Only use CDM, or only use Key Vaults for retrieval of Decryption Keys.", + ) + @click.option("--no-proxy", is_flag=True, default=False, help="Force disable all proxy use.") + @click.option( + "--no-proxy-download", + is_flag=True, + default=False, + help="Bypass proxy for segment downloads only. Manifest, license, and auth still use proxy.", + ) + @click.option("--no-folder", is_flag=True, default=False, help="Disable folder creation for TV Shows.") + @click.option( + "--no-source", is_flag=True, default=False, help="Disable the source tag from the output file name and path." + ) + @click.option("--no-mux", is_flag=True, default=False, help="Do not mux tracks into a container file.") + @click.option( + "--workers", + type=int, + default=None, + help="Max workers/threads to download with per-track. Default depends on the downloader.", + ) + @click.option("--downloads", type=int, default=1, help="Amount of tracks to download concurrently.") + @click.option( + "-o", + "--output", + "output_dir", + type=Path, + default=None, + help="Override the output directory for this download, instead of the one in config.", + ) + @click.option("--no-cache", "no_cache", is_flag=True, default=False, help="Bypass title cache for this download.") + @click.option( + "--reset-cache", "reset_cache", is_flag=True, default=False, help="Clear title cache before fetching." + ) + @click.option( + "--worst", + is_flag=True, + default=False, + help="Select the lowest bitrate track within the specified quality. Requires -q/--quality.", + ) + @click.option( + "--best-available", + "best_available", + is_flag=True, + default=False, + help="Continue with best available quality if requested resolutions are not available.", + ) + @click.option( + "--remote", + is_flag=True, + default=False, + is_eager=True, + help="Use a remote envied.server instead of local service code.", + ) + @click.option( + "--server", + type=str, + default=None, + is_eager=True, + help="Name of the remote server from remote_services config (if multiple configured).", + ) + @click.pass_context + def cli(ctx: click.Context, **kwargs: Any) -> dl: + return dl(ctx, **kwargs) + + DRM_TABLE_LOCK = Lock() + EXPORT_LOCK = Lock() + LICENSE_KEY_CACHE: dict[UUID, str] = {} + + def __init__( + self, + ctx: click.Context, + no_proxy: bool, + profile: Optional[str] = None, + proxy: Optional[str] = None, + repack: bool = False, + tag: Optional[str] = None, + tmdb_id: Optional[int] = None, + imdb_id: Optional[str] = None, + animeapi_id: Optional[str] = None, + enrich: bool = False, + output_dir: Optional[Path] = None, + *_: Any, + **__: Any, + ): + if not ctx.invoked_subcommand: + raise ValueError("A subcommand to invoke was not specified, the main code cannot continue.") + + self.log = logging.getLogger("download") + self.completed_files: list[Path] = [] + + if not config.output_template: + raise click.ClickException( + "No 'output_template' configured in your envied.yaml.\n" + "Please add an 'output_template' section with movies, series, and songs templates.\n" + "See envied.example.yaml for examples." + ) + + self.service = Services.get_tag(ctx.invoked_subcommand) + service_dl_config = config.services.get(self.service, {}).get("dl", {}) + if service_dl_config: + param_types = {param.name: param.type for param in ctx.command.params if param.name} + + for param_name, service_value in service_dl_config.items(): + if param_name not in ctx.params: + continue + + current_value = ctx.params[param_name] + global_default = config.dl.get(param_name) + param_type = param_types.get(param_name) + + try: + if param_type and global_default is not None: + global_default = param_type.convert(global_default, None, ctx) + except Exception as e: + self.log.debug(f"Failed to convert global default for '{param_name}': {e}") + + if current_value == global_default or (current_value is None and global_default is None): + try: + converted_value = service_value + if param_type and service_value is not None: + converted_value = param_type.convert(service_value, None, ctx) + + ctx.params[param_name] = converted_value + self.log.debug(f"Applied service-specific '{param_name}' override: {converted_value}") + except Exception as e: + self.log.warning( + f"Failed to apply service-specific '{param_name}' override: {e}. " + f"Check that the value '{service_value}' is valid for this parameter." + ) + + self.profile = profile + self.proxy_requested = bool(proxy) + self.tmdb_id = tmdb_id + self.imdb_id = imdb_id + self.enrich = enrich + self.animeapi_title: Optional[str] = None + self.output_dir = output_dir + + if animeapi_id: + from envied.core.utils.animeapi import resolve_animeapi + + anime_title, anime_ids = resolve_animeapi(animeapi_id) + self.animeapi_title = anime_title + if not self.tmdb_id and anime_ids.tmdb_id: + self.tmdb_id = anime_ids.tmdb_id + if not self.imdb_id and anime_ids.imdb_id: + self.imdb_id = anime_ids.imdb_id + + if self.enrich and not (self.tmdb_id or self.imdb_id or self.animeapi_title): + raise click.UsageError("--enrich requires --tmdb, --imdb, or --animeapi to provide a metadata source.") + + # Initialize debug logger with service name if debug logging is enabled + if config.debug or logging.root.level == logging.DEBUG: + from collections import defaultdict + from datetime import datetime + + debug_log_path = config.directories.logs / config.filenames.debug_log.format_map( + defaultdict(str, service=self.service, time=datetime.now().strftime("%Y%m%d-%H%M%S")) + ) + init_debug_logger(log_path=debug_log_path, enabled=True, log_keys=config.debug_keys) + self.debug_logger = get_debug_logger() + + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="download_init", + message=f"Download command initialized for service {self.service}", + service=self.service, + context={ + "profile": profile, + "proxy": proxy, + "tag": tag, + "tmdb_id": self.tmdb_id, + "imdb_id": self.imdb_id, + "animeapi_id": animeapi_id, + "enrich": enrich, + "cli_params": { + k: v + for k, v in ctx.params.items() + if k + not in [ + "profile", + "proxy", + "tag", + "tmdb_id", + "imdb_id", + "animeapi_id", + "enrich", + ] + }, + }, + ) + + # Log binary versions for diagnostics + binary_versions = {} + for name, binary in [ + ("shaka_packager", binaries.ShakaPackager), + ("mp4decrypt", binaries.Mp4decrypt), + ("mkvmerge", binaries.MKVToolNix), + ("ffmpeg", binaries.FFMPEG), + ("ffprobe", binaries.FFProbe), + ]: + if binary: + version = None + try: + if name == "shaka_packager": + r = subprocess.run( + [str(binary), "--version"], capture_output=True, text=True, timeout=5 + ) + version = (r.stdout or r.stderr or "").strip() + elif name in ("ffmpeg", "ffprobe"): + r = subprocess.run([str(binary), "-version"], capture_output=True, text=True, timeout=5) + version = (r.stdout or "").split("\n")[0].strip() + elif name == "mkvmerge": + r = subprocess.run( + [str(binary), "--version"], capture_output=True, text=True, timeout=5 + ) + version = (r.stdout or "").strip() + elif name == "mp4decrypt": + r = subprocess.run([str(binary)], capture_output=True, text=True, timeout=5) + output = (r.stdout or "") + (r.stderr or "") + lines = [line.strip() for line in output.split("\n") if line.strip()] + version = " | ".join(lines[:2]) if lines else None + except Exception: + version = "" + binary_versions[name] = {"path": str(binary), "version": version} + else: + binary_versions[name] = None + + self.debug_logger.log( + level="DEBUG", + operation="binary_versions", + message="Binary tool versions", + context=binary_versions, + ) + else: + self.debug_logger = None + + if self.profile: + self.log.info(f"Using profile: '{self.profile}'") + + self.is_remote = bool(ctx.params.get("remote")) + + with console.status("Loading Service Config...", spinner="dots"): + self.service_config = {} + if not self.is_remote: + try: + service_config_path = Services.get_path(self.service) / config.filenames.config + if service_config_path.exists(): + self.service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) + self.log.info("Service Config loaded") + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="load_service_config", + service=self.service, + context={"config_path": str(service_config_path), "config": self.service_config}, + ) + except KeyError: + pass + merge_dict(config.services.get(self.service), self.service_config) + + if getattr(config, "decryption_map", None): + config.decryption = config.decryption_map.get(self.service, config.decryption) + + service_config = config.services.get(self.service, {}) + if service_config: + reserved_keys = { + "profiles", + "api_key", + "certificate", + "api_endpoint", + "region", + "device", + "endpoints", + "client", + "dl", + } + + for config_key, override_value in service_config.items(): + if config_key in reserved_keys or not isinstance(override_value, dict): + continue + + if hasattr(config, config_key): + current_config = getattr(config, config_key, {}) + + if isinstance(current_config, dict): + merged_config = deepcopy(current_config) + merge_dict(override_value, merged_config) + setattr(config, config_key, merged_config) + + self.log.debug( + f"Applied service-specific '{config_key}' overrides for {self.service}: {override_value}" + ) + + cdm_only = ctx.params.get("cdm_only") + + if cdm_only: + self.vaults = Vaults(self.service) + self.log.info("CDM-only mode: Skipping vault loading") + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="vault_loading_skipped", + service=self.service, + context={"reason": "cdm_only flag set"}, + ) + else: + with console.status("Loading Key Vaults...", spinner="dots"): + self.vaults = Vaults(self.service) + total_vaults = len(config.key_vaults) + failed_vaults = [] + + for vault in config.key_vaults: + vault_type = vault["type"] + vault_name = vault.get("name", vault_type) + vault_copy = vault.copy() + del vault_copy["type"] + + if vault_type.lower() == "api" and "decrypt_labs" in vault_name.lower(): + if "token" not in vault_copy or not vault_copy["token"]: + if config.decrypt_labs_api_key: + vault_copy["token"] = config.decrypt_labs_api_key + else: + self.log.warning( + f"No token provided for DecryptLabs vault '{vault_name}' and no global " + "decrypt_labs_api_key configured" + ) + + if vault_type.lower() == "sqlite": + try: + self.vaults.load_critical(vault_type, **vault_copy) + self.log.debug(f"Successfully loaded vault: {vault_name} ({vault_type})") + except Exception as e: + self.log.error(f"vault failure: {vault_name} ({vault_type}) - {e}") + raise + else: + # Other vaults (MySQL, HTTP, API) - soft fail + if not self.vaults.load(vault_type, **vault_copy): + failed_vaults.append(vault_name) + self.log.debug(f"Failed to load vault: {vault_name} ({vault_type})") + else: + self.log.debug(f"Successfully loaded vault: {vault_name} ({vault_type})") + + loaded_count = len(self.vaults) + if failed_vaults: + self.log.warning(f"Failed to load {len(failed_vaults)} vault(s): {', '.join(failed_vaults)}") + self.log.info(f"Loaded {loaded_count}/{total_vaults} Vaults") + + # Debug: Show detailed vault status + if loaded_count > 0: + vault_names = [vault.name for vault in self.vaults] + self.log.debug(f"Active vaults: {', '.join(vault_names)}") + else: + self.log.debug("No vaults are currently active") + + with console.status("Loading DRM CDM...", spinner="dots"): + try: + self.cdm = self.get_cdm(self.service, self.profile) + except ValueError as e: + self.log.error(f"Failed to load CDM, {e}") + if self.debug_logger: + self.debug_logger.log_error("load_cdm", e, service=self.service) + sys.exit(1) + + if self.cdm: + cdm_info = {} + if isinstance(self.cdm, DecryptLabsRemoteCDM): + drm_type = "PlayReady" if self.cdm.is_playready else "Widevine" + self.log.info(f"Loaded {drm_type} Remote CDM: DecryptLabs (L{self.cdm.security_level})") + cdm_info = {"type": "DecryptLabs", "drm_type": drm_type, "security_level": self.cdm.security_level} + elif hasattr(self.cdm, "device_type") and self.cdm.device_type.name in ["ANDROID", "CHROME"]: + self.log.info(f"Loaded Widevine CDM: {self.cdm.system_id} (L{self.cdm.security_level})") + cdm_info = { + "type": "Widevine", + "system_id": self.cdm.system_id, + "security_level": self.cdm.security_level, + "device_type": self.cdm.device_type.name, + } + else: + # Handle both local PlayReady CDM and RemoteCdm (which has certificate_chain=None) + is_remote = self.cdm.certificate_chain is None and hasattr(self.cdm, "device_name") + if is_remote: + cdm_name = self.cdm.device_name + self.log.info(f"Loaded PlayReady Remote CDM: {cdm_name} (L{self.cdm.security_level})") + else: + cdm_name = self.cdm.certificate_chain.get_name() if self.cdm.certificate_chain else "Unknown" + self.log.info(f"Loaded PlayReady CDM: {cdm_name} (L{self.cdm.security_level})") + cdm_info = { + "type": "PlayReady", + "certificate": cdm_name, + "security_level": self.cdm.security_level, + } + + if self.debug_logger and cdm_info: + self.debug_logger.log( + level="INFO", operation="load_cdm", service=self.service, context={"cdm": cdm_info} + ) + + self.proxy_providers = [] + if no_proxy: + ctx.params["proxy"] = None + else: + with console.status("Loading Proxy Providers...", spinner="dots"): + if config.proxy_providers.get("basic"): + self.proxy_providers.append(Basic(**config.proxy_providers["basic"])) + if config.proxy_providers.get("nordvpn"): + self.proxy_providers.append(NordVPN(**config.proxy_providers["nordvpn"])) + if config.proxy_providers.get("surfsharkvpn"): + self.proxy_providers.append(SurfsharkVPN(**config.proxy_providers["surfsharkvpn"])) + if config.proxy_providers.get("windscribevpn"): + self.proxy_providers.append(WindscribeVPN(**config.proxy_providers["windscribevpn"])) + if config.proxy_providers.get("gluetun"): + self.proxy_providers.append(Gluetun(**config.proxy_providers["gluetun"])) + if binaries.HolaProxy: + self.proxy_providers.append(Hola()) + for proxy_provider in self.proxy_providers: + self.log.info(f"Loaded {proxy_provider.__class__.__name__}: {proxy_provider}") + + if proxy: + requested_provider = None + if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE): + # requesting proxy from a specific proxy provider + requested_provider, proxy = proxy.split(":", maxsplit=1) + # Match simple region codes (us, ca, uk1) or provider:region format (nordvpn:ca, windscribe:us) + if re.match(r"^[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE) or re.match( + r"^[a-z]+:[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE + ): + proxy = proxy.lower() + # Preserve the original user query (region code) for service-specific proxy_map overrides. + # NOTE: `proxy` may be overwritten with the resolved proxy URI later. + proxy_query = proxy + status_msg = ( + f"Connecting to VPN ({proxy})..." + if requested_provider == "gluetun" + else f"Getting a Proxy to {proxy}..." + ) + with console.status(status_msg, spinner="dots"): + if requested_provider: + proxy_provider = next( + (x for x in self.proxy_providers if x.__class__.__name__.lower() == requested_provider), + None, + ) + if not proxy_provider: + self.log.error(f"The proxy provider '{requested_provider}' was not recognised.") + sys.exit(1) + proxy_uri = proxy_provider.get_proxy(proxy) + if not proxy_uri: + self.log.error(f"The proxy provider {requested_provider} had no proxy for {proxy}") + sys.exit(1) + proxy = ctx.params["proxy"] = proxy_uri + # Show connection info for Gluetun (IP, location) instead of proxy URL + if hasattr(proxy_provider, "get_connection_info"): + conn_info = proxy_provider.get_connection_info(proxy_query) + if conn_info and conn_info.get("public_ip"): + location_parts = [conn_info.get("city"), conn_info.get("country")] + location = ", ".join(p for p in location_parts if p) + self.log.info(f"VPN Connected: {conn_info['public_ip']} ({location})") + else: + self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") + else: + self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") + else: + for proxy_provider in self.proxy_providers: + proxy_uri = proxy_provider.get_proxy(proxy) + if proxy_uri: + proxy = ctx.params["proxy"] = proxy_uri + # Show connection info for Gluetun (IP, location) instead of proxy URL + if hasattr(proxy_provider, "get_connection_info"): + conn_info = proxy_provider.get_connection_info(proxy_query) + if conn_info and conn_info.get("public_ip"): + location_parts = [conn_info.get("city"), conn_info.get("country")] + location = ", ".join(p for p in location_parts if p) + self.log.info(f"VPN Connected: {conn_info['public_ip']} ({location})") + else: + self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") + else: + self.log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") + break + # Store proxy query info for service-specific overrides + ctx.params["proxy_query"] = proxy_query + ctx.params["proxy_provider"] = requested_provider + else: + self.log.info(f"Using explicit Proxy: {proxy}") + # For explicit proxies, store None for query/provider + ctx.params["proxy_query"] = None + ctx.params["proxy_provider"] = None + + ctx.obj = ContextData( + config=self.service_config, cdm=self.cdm, proxy_providers=self.proxy_providers, profile=self.profile + ) + + if repack: + config.repack = True + + if tag: + config.tag = tag + + # needs to be added this way instead of @cli.result_callback to be + # able to keep `self` as the first positional + self.cli._result_callback = self.result + + def result( + self, + service: Service, + quality: list[int], + vcodec: list[Video.Codec], + acodec: list[Audio.Codec], + vbitrate: int, + abitrate: int, + vbitrate_range: Optional[str], + abitrate_range: Optional[str], + range_: list[Video.Range], + channels: float, + no_atmos: bool, + select_titles: bool, + wanted: list[str], + latest_episode: bool, + lang: list[str], + v_lang: list[str], + a_lang: list[str], + s_lang: list[str], + require_subs: list[str], + forced_subs: bool, + exact_lang: bool, + sub_format: Optional[Subtitle.Codec], + video_only: bool, + audio_only: bool, + subs_only: bool, + chapters_only: bool, + no_subs: bool, + no_audio: bool, + no_chapters: bool, + no_video: bool, + audio_description: bool, + slow: Optional[tuple[int, int]], + list_: bool, + list_titles: bool, + skip_dl: bool, + export: bool, + cdm_only: Optional[bool], + no_proxy: bool, + no_proxy_download: bool, + no_folder: bool, + no_source: bool, + no_mux: bool, + workers: Optional[int], + downloads: int, + worst: bool, + best_available: bool, + split_audio: Optional[bool] = None, + real_video_bitrate: bool = False, + real_audio_bitrate: bool = False, + *_: Any, + **__: Any, + ) -> None: + self.tmdb_searched = False + self.search_source = None + self.server_cdm = getattr(service, "_server_cdm", False) + self._remote_service = service if self.server_cdm else None + start_time = time.time() + + if skip_dl: + DOWNLOAD_LICENCE_ONLY.set() + + if export: + config.directories.exports.mkdir(parents=True, exist_ok=True) + export_path = config.directories.exports / f"export_{self.service}_{int(time.time())}.json" + self.export_service = service + else: + export_path = None + + # Parse bitrate range options + vbitrate_min, vbitrate_max = None, None + if vbitrate_range: + if vbitrate and vbitrate_range: + self.log.error("Cannot use both --vbitrate and --vbitrate-range at the same time.") + sys.exit(1) + try: + parts = vbitrate_range.split("-") + if len(parts) != 2: + raise ValueError + vbitrate_min, vbitrate_max = int(parts[0]), int(parts[1]) + if vbitrate_min > vbitrate_max: + vbitrate_min, vbitrate_max = vbitrate_max, vbitrate_min + except (ValueError, IndexError): + self.log.error("Invalid --vbitrate-range format. Use 'MIN-MAX' (e.g., '6000-7000').") + sys.exit(1) + + abitrate_min, abitrate_max = None, None + if abitrate_range: + if abitrate and abitrate_range: + self.log.error("Cannot use both --abitrate and --abitrate-range at the same time.") + sys.exit(1) + try: + parts = abitrate_range.split("-") + if len(parts) != 2: + raise ValueError + abitrate_min, abitrate_max = int(parts[0]), int(parts[1]) + if abitrate_min > abitrate_max: + abitrate_min, abitrate_max = abitrate_max, abitrate_min + except (ValueError, IndexError): + self.log.error("Invalid --abitrate-range format. Use 'MIN-MAX' (e.g., '128-256').") + sys.exit(1) + + if not acodec: + acodec = [] + elif isinstance(acodec, Audio.Codec): + acodec = [acodec] + elif isinstance(acodec, str) or ( + isinstance(acodec, list) and not all(isinstance(v, Audio.Codec) for v in acodec) + ): + acodec = AUDIO_CODEC_LIST.convert(acodec) + + if require_subs and s_lang != ["all"]: + self.log.error("--require-subs and --s-lang cannot be used together") + sys.exit(1) + + if worst and not quality: + self.log.error("--worst requires -q/--quality to be specified") + sys.exit(1) + + if select_titles and wanted: + self.log.error("--select-titles and -w/--wanted cannot be used together") + sys.exit(1) + + # Check if dovi_tool is available when hybrid mode is requested + if any(r == Video.Range.HYBRID for r in range_): + from envied.core.binaries import DoviTool + + if not DoviTool: + self.log.error("Unable to run hybrid mode: dovi_tool not detected") + self.log.error("Please install dovi_tool from https://github.com/quietvoid/dovi_tool") + sys.exit(1) + + if cdm_only is None: + vaults_only = None + else: + vaults_only = not cdm_only + + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="drm_mode_config", + service=self.service, + context={ + "cdm_only": cdm_only, + "vaults_only": vaults_only, + "mode": "CDM only" if cdm_only else ("Vaults only" if vaults_only else "Both CDM and Vaults"), + }, + ) + + with console.status( + "Authenticating with Remote Service..." if self.is_remote else "Authenticating with Service...", + spinner="dots", + ): + try: + cookies = self.get_cookie_jar(self.service, self.profile) + credential = self.get_credentials(self.service, self.profile) + service.authenticate(cookies, credential) + if cookies or credential: + self.log.info("Authenticated with Service") + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="authenticate", + service=self.service, + context={ + "has_cookies": bool(cookies), + "has_credentials": bool(credential), + "profile": self.profile, + }, + ) + except Exception as e: + if self.debug_logger: + self.debug_logger.log_error( + "authenticate", e, service=self.service, context={"profile": self.profile} + ) + raise + + with console.status( + "Fetching Remote Title Metadata..." if self.is_remote else "Fetching Title Metadata...", spinner="dots" + ): + try: + titles = service.get_titles_cached() + if not titles: + self.log.error("No titles returned, nothing to download...") + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="get_titles", + service=self.service, + message="No titles returned from service", + success=False, + ) + sys.exit(1) + except Exception as e: + if self.debug_logger: + self.debug_logger.log_error("get_titles", e, service=self.service) + raise + + if self.debug_logger: + titles_info = { + "type": titles.__class__.__name__, + "count": len(titles) if hasattr(titles, "__len__") else 1, + "title": str(titles), + } + if hasattr(titles, "seasons"): + titles_info["seasons"] = len(titles.seasons) if hasattr(titles, "seasons") else 0 + self.debug_logger.log( + level="INFO", operation="get_titles", service=self.service, context={"titles": titles_info} + ) + + title_cacher = service.title_cache if hasattr(service, "title_cache") else None + cache_title_id = None + if hasattr(service, "title"): + cache_title_id = service.title + elif hasattr(service, "title_id"): + cache_title_id = service.title_id + cache_region = service.current_region if hasattr(service, "current_region") else None + cache_account_hash = get_account_hash(service.credential) if hasattr(service, "credential") else None + + if self.enrich: + sample_title = titles[0] if hasattr(titles, "__getitem__") else titles + kind = "tv" if isinstance(sample_title, Episode) else "movie" + + enrich_title: Optional[str] = None + enrich_year: Optional[int] = None + + if self.animeapi_title: + enrich_title = self.animeapi_title + + if self.tmdb_id: + if not enrich_title: + enrich_title = providers.get_title_by_id( + self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash + ) + enrich_year = providers.get_year_by_id( + self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash + ) + elif self.imdb_id: + imdbapi = providers.get_provider("imdbapi") + if imdbapi: + imdb_result = imdbapi.get_by_id(self.imdb_id, kind) + if imdb_result: + if not enrich_title: + enrich_title = imdb_result.title + enrich_year = imdb_result.year + + if enrich_title or enrich_year: + if isinstance(titles, (Series, Movies)): + for t in titles: + if enrich_title: + if isinstance(t, Episode): + t.title = enrich_title + else: + t.name = enrich_title + if enrich_year and not t.year: + t.year = enrich_year + else: + if enrich_title: + if isinstance(titles, Episode): + titles.title = enrich_title + else: + titles.name = enrich_title + if enrich_year and not titles.year: + titles.year = enrich_year + + console.print(Padding(Rule(f"[rule.text]{titles.__class__.__name__}: {titles}"), (1, 2))) + + console.print(Padding(titles.tree(verbose=list_titles), (0, 5))) + if list_titles: + return + + # Enables manual selection for Series when --select-titles is set + if select_titles and isinstance(titles, Series): + console.print(Padding(Rule("[rule.text]Select Titles"), (1, 2))) + + selection_titles = [] + dependencies = {} + original_indices = {} + + current_season = None + current_season_header_idx = -1 + + unique_seasons = {t.season for t in titles} + multiple_seasons = len(unique_seasons) > 1 + + # Build selection options + for i, t in enumerate(titles): + # Insert season header only if multiple seasons exist + if multiple_seasons and t.season != current_season: + current_season = t.season + header_text = f"Season {t.season}" + selection_titles.append(header_text) + current_season_header_idx = len(selection_titles) - 1 + dependencies[current_season_header_idx] = [] + # Note: Headers are not mapped to actual title indices + + # Format display name + display_name = ((t.name[:30].rstrip() + "…") if len(t.name) > 30 else t.name) if t.name else None + + # Apply indentation only for multiple seasons + prefix = " " if multiple_seasons else "" + option_text = f"{prefix}{t.number}" + (f". {display_name}" if t.name else "") + + selection_titles.append(option_text) + current_ui_idx = len(selection_titles) - 1 + + # Map UI index to actual title index + original_indices[current_ui_idx] = i + + # Link episode to season header for group selection + if current_season_header_idx != -1: + dependencies[current_season_header_idx].append(current_ui_idx) + + selection_start = time.time() + + # Execute selector with dependencies (headers select all children) + selected_ui_idx = select_multiple( + selection_titles, + minimal_count=1, + page_size=8, + return_indices=True, + dependencies=dependencies, + collapse_on_start=multiple_seasons, + ) + + if not selected_ui_idx: + console.print(Padding(":x: Selection Cancelled...", (0, 5, 1, 5))) + return + + selection_end = time.time() + start_time += selection_end - selection_start + + # Map UI indices back to title indices (excluding headers) + selected_idx = [] + for idx in selected_ui_idx: + if idx in original_indices: + selected_idx.append(original_indices[idx]) + + # Ensure indices are unique and ordered + selected_idx = sorted(set(selected_idx)) + keep = set(selected_idx) + + # In-place filter: remove unselected items (iterate backwards) + for i in range(len(titles) - 1, -1, -1): + if i not in keep: + del titles[i] + + # Show selected count + if titles: + count = len(titles) + console.print(Padding(f"[text]Total selected: {count}[/]", (0, 5))) + + # Determine the latest episode if --latest-episode is set + latest_episode_id = None + if latest_episode and isinstance(titles, Series) and len(titles) > 0: + # Series is already sorted by (season, number, year) + # The last episode in the sorted list is the latest + latest_ep = titles[-1] + latest_episode_id = f"{latest_ep.season}x{latest_ep.number}" + self.log.info(f"Latest episode mode: Selecting S{latest_ep.season:02}E{latest_ep.number:02}") + + for i, title in enumerate(titles): + if isinstance(title, Episode) and latest_episode and latest_episode_id: + # If --latest-episode is set, only process the latest episode + if f"{title.season}x{title.number}" != latest_episode_id: + continue + elif isinstance(title, Episode) and wanted and f"{title.season}x{title.number}" not in wanted: + continue + + console.print(Padding(Rule(f"[rule.text]{title}"), (1, 2))) + temp_font_files = [] + + if isinstance(title, Episode) and not self.tmdb_searched: + kind = "tv" + tmdb_title: Optional[str] = None + if self.tmdb_id: + tmdb_title = providers.get_title_by_id( + self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash + ) + else: + result = providers.search_metadata( + title.title, title.year, kind, title_cacher, cache_title_id, cache_region, cache_account_hash + ) + if result and result.title and providers.fuzzy_match(result.title, title.title): + self.tmdb_id = result.external_ids.tmdb_id + tmdb_title = result.title + self.search_source = result.source + else: + self.tmdb_id = None + if list_ or list_titles: + if self.tmdb_id: + console.print( + Padding( + f"Search -> {tmdb_title or '?'} [bright_black](ID {self.tmdb_id})", + (0, 5), + ) + ) + else: + console.print(Padding("Search -> [bright_black]No match found[/]", (0, 5))) + self.tmdb_searched = True + + if isinstance(title, Movie) and (list_ or list_titles) and not self.tmdb_id: + movie_result = providers.search_metadata( + title.name, title.year, "movie", title_cacher, cache_title_id, cache_region, cache_account_hash + ) + if movie_result and movie_result.external_ids.tmdb_id: + console.print( + Padding( + f"Search -> {movie_result.title or '?'} " + f"[bright_black](ID {movie_result.external_ids.tmdb_id})", + (0, 5), + ) + ) + else: + console.print(Padding("Search -> [bright_black]No match found[/]", (0, 5))) + + if self.tmdb_id and getattr(self, "search_source", None) not in ("simkl", "imdbapi"): + kind = "tv" if isinstance(title, Episode) else "movie" + providers.fetch_external_ids( + self.tmdb_id, kind, title_cacher, cache_title_id, cache_region, cache_account_hash + ) + + if slow and i != 0: + delay = random.randint(slow[0], slow[1]) + spinner = Spinner("dots", text=f"Delaying by {delay} seconds...") + with Live(Padding(spinner, (0, 5)), console=console, refresh_per_second=12.5, transient=True): + for remaining in range(delay, 0, -1): + spinner.update(text=f"Delaying by {remaining} seconds...") + time.sleep(1) + + with console.status("Subscribing to events...", spinner="dots"): + events.reset() + events.subscribe(events.Types.SEGMENT_DOWNLOADED, service.on_segment_downloaded) + events.subscribe(events.Types.TRACK_DOWNLOADED, service.on_track_downloaded) + events.subscribe(events.Types.TRACK_DECRYPTED, service.on_track_decrypted) + events.subscribe(events.Types.TRACK_REPACKED, service.on_track_repacked) + events.subscribe(events.Types.TRACK_MULTIPLEX, service.on_track_multiplex) + + if hasattr(service, "NO_SUBTITLES") and service.NO_SUBTITLES: + console.log("Skipping subtitles - service does not support subtitle downloads") + no_subs = True + s_lang = None + title.tracks.subtitles = [] + elif no_subs: + console.log("Skipped subtitles as --no-subs was used...") + s_lang = None + title.tracks.subtitles = [] + + if no_video: + console.log("Skipped video as --no-video was used...") + v_lang = None + title.tracks.videos = [] + + if no_audio: + console.log("Skipped audio as --no-audio was used...") + a_lang = None + title.tracks.audio = [] + + if no_chapters: + console.log("Skipped chapters as --no-chapters was used...") + title.tracks.chapters = [] + + if no_proxy_download and any(service.session.proxies.values()): + console.log("Bypassing proxy for downloads as --no-proxy-download was used...") + + tracks_label = "Getting Remote Tracks..." if self.is_remote else "Getting Tracks..." + with console.status(tracks_label, spinner="dots"): + try: + title.tracks.add(service.get_tracks(title), warn_only=True) + title.tracks.chapters = service.get_chapters(title) + except Exception as e: + if self.debug_logger: + self.debug_logger.log_error( + "get_tracks", e, service=self.service, context={"title": str(title)} + ) + raise + + if self.debug_logger: + tracks_info = { + "title": str(title), + "video_tracks": len(title.tracks.videos), + "audio_tracks": len(title.tracks.audio), + "subtitle_tracks": len(title.tracks.subtitles), + "has_chapters": bool(title.tracks.chapters), + "videos": [ + { + "codec": str(v.codec), + "resolution": f"{v.width}x{v.height}" if v.width and v.height else "unknown", + "bitrate": v.bitrate, + "range": str(v.range), + "language": str(v.language) if v.language else None, + "drm": [str(type(d).__name__) for d in v.drm] if v.drm else [], + } + for v in title.tracks.videos + ], + "audio": [ + { + "codec": str(a.codec), + "bitrate": a.bitrate, + "channels": a.channels, + "language": str(a.language) if a.language else None, + "descriptive": a.descriptive, + "drm": [str(type(d).__name__) for d in a.drm] if a.drm else [], + } + for a in title.tracks.audio + ], + "subtitles": [ + { + "codec": str(s.codec), + "language": str(s.language) if s.language else None, + "forced": s.forced, + "sdh": s.sdh, + } + for s in title.tracks.subtitles + ], + } + self.debug_logger.log( + level="INFO", operation="get_tracks", service=self.service, context=tracks_info + ) + + # strip SDH subs to non-SDH if no equivalent same-lang non-SDH is available + # uses a loose check, e.g, wont strip en-US SDH sub if a non-SDH en-GB is available + # Check if automatic SDH stripping is enabled in config + if config.subtitle.get("strip_sdh", True): + for subtitle in title.tracks.subtitles: + if subtitle.sdh and not any( + is_close_match(subtitle.language, [x.language]) + for x in title.tracks.subtitles + if not x.sdh and not x.forced + ): + non_sdh_sub = deepcopy(subtitle) + non_sdh_sub.id += "_stripped" + non_sdh_sub.sdh = False + title.tracks.add(non_sdh_sub) + events.subscribe( + events.Types.TRACK_MULTIPLEX, + lambda track, sub_id=non_sdh_sub.id: ( + (track.strip_hearing_impaired()) if track.id == sub_id else None + ), + ) + + if real_video_bitrate: + with console.status("Probing real video bitrates...", spinner="dots"): + apply_real_bitrates( + title.tracks.videos, + service.session, + log=self.log, + group_key=lambda t: (t.codec, t.range), + ) + + if real_audio_bitrate: + with console.status("Probing real audio bitrates...", spinner="dots"): + apply_real_bitrates( + title.tracks.audio, + service.session, + log=self.log, + group_key=lambda t: (t.codec, t.channels, str(t.language), t.descriptive), + ) + + with console.status("Sorting tracks by language and bitrate...", spinner="dots"): + video_sort_lang = v_lang or lang + processed_video_sort_lang = [] + for language in video_sort_lang: + if language == "orig": + if title.language: + orig_lang = str(title.language) if hasattr(title.language, "__str__") else title.language + if orig_lang not in processed_video_sort_lang: + processed_video_sort_lang.append(orig_lang) + else: + if language not in processed_video_sort_lang: + processed_video_sort_lang.append(language) + + audio_sort_lang = a_lang or lang + processed_audio_sort_lang = [] + for language in audio_sort_lang: + if language == "orig": + if title.language: + orig_lang = str(title.language) if hasattr(title.language, "__str__") else title.language + if orig_lang not in processed_audio_sort_lang: + processed_audio_sort_lang.append(orig_lang) + else: + if language not in processed_audio_sort_lang: + processed_audio_sort_lang.append(language) + + title.tracks.sort_videos(by_language=processed_video_sort_lang) + title.tracks.sort_audio( + by_language=processed_audio_sort_lang, + codec_priority=config.audio.get("codec_priority"), + ) + title.tracks.sort_subtitles(by_language=s_lang) + + if list_: + available_tracks, _ = title.tracks.tree() + console.print(Padding(Panel(available_tracks, title="Available Tracks"), (0, 5))) + continue + + with console.status("Selecting tracks...", spinner="dots"): + if isinstance(title, (Movie, Episode)): + # filter video tracks + if vcodec: + title.tracks.select_video(lambda x: x.codec in vcodec) + missing_codecs = [c for c in vcodec if not any(x.codec == c for x in title.tracks.videos)] + for codec in missing_codecs: + self.log.warning(f"Skipping {codec.name} video tracks as none are available.") + if not title.tracks.videos: + self.log.error(f"There's no {', '.join(c.name for c in vcodec)} Video Track...") + sys.exit(1) + + if range_: + # Special handling for HYBRID - don't filter, keep all HDR10 and DV tracks + if Video.Range.HYBRID not in range_: + title.tracks.select_video(lambda x: x.range in range_) + missing_ranges = [r for r in range_ if not any(x.range == r for x in title.tracks.videos)] + for color_range in missing_ranges: + self.log.warning(f"Skipping {color_range.name} video tracks as none are available.") + if not title.tracks.videos: + self.log.error(f"There's no {', '.join(r.name for r in range_)} Video Track...") + sys.exit(1) + + if vbitrate: + if any(r == Video.Range.HYBRID for r in range_): + # In HYBRID mode, only apply bitrate filter to non-DV tracks + # DV tracks are kept regardless since they're only used for RPU metadata + title.tracks.select_video( + lambda x: x.range == Video.Range.DV or (x.bitrate and x.bitrate // 1000 == vbitrate) + ) + if not any(x.range != Video.Range.DV for x in title.tracks.videos): + self.log.error(f"There's no {vbitrate}kbps Video Track...") + sys.exit(1) + else: + title.tracks.select_video(lambda x: x.bitrate and x.bitrate // 1000 == vbitrate) + if not title.tracks.videos: + self.log.error(f"There's no {vbitrate}kbps Video Track...") + sys.exit(1) + + if vbitrate_min is not None and vbitrate_max is not None: + title.tracks.select_video( + lambda x: x.bitrate and vbitrate_min <= x.bitrate // 1000 <= vbitrate_max + ) + if not title.tracks.videos: + self.log.error(f"No Video Track in {vbitrate_min}-{vbitrate_max}kbps range...") + sys.exit(1) + + effective_video_lang = v_lang or lang + video_languages = [lang for lang in effective_video_lang if lang != "best"] + video_multi_lang = ( + "best" in effective_video_lang or "all" in effective_video_lang or len(video_languages) > 1 + ) + if video_languages and "all" not in video_languages: + processed_video_lang = [] + for language in video_languages: + if language == "orig": + if title.language: + orig_lang = ( + str(title.language) if hasattr(title.language, "__str__") else title.language + ) + if orig_lang not in processed_video_lang: + processed_video_lang.append(orig_lang) + else: + self.log.warning( + "Original language not available for title, skipping 'orig' selection for video" + ) + else: + if language not in processed_video_lang: + processed_video_lang.append(language) + title.tracks.videos = title.tracks.by_language( + title.tracks.videos, processed_video_lang, exact_match=exact_lang + ) + if not title.tracks.videos: + self.log.error(f"There's no {processed_video_lang} Video Track...") + sys.exit(1) + + has_hybrid = any(r == Video.Range.HYBRID for r in range_) + non_hybrid_ranges = [r for r in range_ if r != Video.Range.HYBRID] + if quality: + missing_resolutions = [] + if has_hybrid: + hybrid_candidate_tracks, non_hybrid_tracks = Tracks.partition_hybrid_videos( + title.tracks.videos, non_hybrid_ranges + ) + + hybrid_filter = title.tracks.select_hybrid(hybrid_candidate_tracks, quality, worst=worst) + hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks)) + + if non_hybrid_ranges and non_hybrid_tracks: + non_hybrid_selected = [ + v + for v in non_hybrid_tracks + if any(v.height == res or int(v.width * (9 / 16)) == res for res in quality) + ] + title.tracks.videos = Tracks.merge_video_selections(hybrid_selected, non_hybrid_selected) + else: + title.tracks.videos = hybrid_selected + else: + title.tracks.by_resolutions(quality) + + for resolution in quality: + if any(v.height == resolution for v in title.tracks.videos): + continue + if any(int(v.width * 9 / 16) == resolution for v in title.tracks.videos): + continue + missing_resolutions.append(resolution) + + if missing_resolutions: + res_list = "" + if len(missing_resolutions) > 1: + res_list = ", ".join([f"{x}p" for x in missing_resolutions[:-1]]) + " or " + res_list = f"{res_list}{missing_resolutions[-1]}p" + plural = "s" if len(missing_resolutions) > 1 else "" + + if best_available: + self.log.warning( + f"There's no {res_list} Video Track{plural}, continuing with available qualities..." + ) + else: + self.log.error(f"There's no {res_list} Video Track{plural}...") + sys.exit(1) + + # choose best track by range and quality + pre_hybrid_videos: list[Video] = list(title.tracks.videos) if has_hybrid else [] + if has_hybrid: + # Apply hybrid selection for HYBRID tracks + hybrid_candidate_tracks, non_hybrid_tracks = Tracks.partition_hybrid_videos( + title.tracks.videos, non_hybrid_ranges + ) + + if not quality: + best_resolution = max((v.height for v in hybrid_candidate_tracks), default=None) + if best_resolution: + hybrid_filter = title.tracks.select_hybrid( + hybrid_candidate_tracks, [best_resolution], worst=worst + ) + hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks)) + else: + hybrid_selected = [] + else: + hybrid_filter = title.tracks.select_hybrid(hybrid_candidate_tracks, quality, worst=worst) + hybrid_selected = list(filter(hybrid_filter, hybrid_candidate_tracks)) + + # For non-hybrid ranges, apply Cartesian product selection + non_hybrid_selected: list[Video] = [] + if non_hybrid_ranges and non_hybrid_tracks: + # Include language dimension when multiple video languages were requested + if video_multi_lang: + non_hybrid_langs = list(dict.fromkeys(str(v.language) for v in non_hybrid_tracks)) + else: + non_hybrid_langs = [None] + for resolution, color_range, codec, vlang in product( + quality or [None], non_hybrid_ranges, vcodec or [None], non_hybrid_langs + ): + candidates = [ + t + for t in non_hybrid_tracks + if ( + not resolution + or t.height == resolution + or int(t.width * (9 / 16)) == resolution + ) + and (not color_range or t.range == color_range) + and (not codec or t.codec == codec) + and (vlang is None or str(t.language) == vlang) + ] + match = candidates[-1] if worst and candidates else next(iter(candidates), None) + if match and match not in non_hybrid_selected: + non_hybrid_selected.append(match) + + title.tracks.videos = Tracks.merge_video_selections(hybrid_selected, non_hybrid_selected) + + # Flag tracks selected only as hybrid ingredients (the HDR base and/or + # the lowest DV) so the standalone mux loop skips them. Tracks also + # picked as explicit deliverables stay unflagged. + Tracks.flag_hybrid_ingredients(hybrid_selected, non_hybrid_selected) + else: + selected_videos: list[Video] = [] + if video_multi_lang: + unique_video_langs = list(dict.fromkeys(str(v.language) for v in title.tracks.videos)) + else: + unique_video_langs = [None] + for resolution, color_range, codec, vlang in product( + quality or [None], range_ or [None], vcodec or [None], unique_video_langs + ): + candidates = [ + t + for t in title.tracks.videos + if (not resolution or t.height == resolution or int(t.width * (9 / 16)) == resolution) + and (not color_range or t.range == color_range) + and (not codec or t.codec == codec) + and (vlang is None or str(t.language) == vlang) + ] + match = candidates[-1] if worst and candidates else next(iter(candidates), None) + if match and match not in selected_videos: + selected_videos.append(match) + title.tracks.videos = selected_videos + + # validate hybrid mode requirements + if any(r == Video.Range.HYBRID for r in range_): + base_tracks = [ + v for v in title.tracks.videos if v.range in (Video.Range.HDR10, Video.Range.HDR10P) + ] + dv_tracks = [v for v in title.tracks.videos if v.range == Video.Range.DV] + + hybrid_failed = False + if not base_tracks and not dv_tracks: + available_ranges = sorted(set(v.range.name for v in title.tracks.videos)) + msg = "HYBRID mode requires both HDR10/HDR10+ and DV tracks, but neither is available" + msg_detail = ( + f"Available ranges: {', '.join(available_ranges) if available_ranges else 'none'}" + ) + hybrid_failed = True + elif not base_tracks: + available_ranges = sorted(set(v.range.name for v in title.tracks.videos)) + msg = "HYBRID mode requires both HDR10/HDR10+ and DV tracks, but only DV is available" + msg_detail = f"Available ranges: {', '.join(available_ranges)}" + hybrid_failed = True + elif not dv_tracks: + available_ranges = sorted(set(v.range.name for v in title.tracks.videos)) + msg = "HYBRID mode requires both HDR10/HDR10+ and DV tracks, but only HDR10 is available" + msg_detail = f"Available ranges: {', '.join(available_ranges)}" + hybrid_failed = True + + if hybrid_failed: + other_ranges = [r for r in range_ if r != Video.Range.HYBRID] + if best_available and other_ranges: + self.log.warning(msg) + self.log.warning( + f"Continuing with remaining range(s): {', '.join(r.name for r in other_ranges)}" + ) + range_ = other_ranges + fallback_pool = pre_hybrid_videos + if video_multi_lang: + fallback_langs = list(dict.fromkeys(str(v.language) for v in fallback_pool)) + else: + fallback_langs = [None] + fallback_selected: list[Video] = [] + for resolution, color_range, codec, vlang in product( + quality or [None], other_ranges, vcodec or [None], fallback_langs + ): + candidates = [ + t + for t in fallback_pool + if ( + not resolution + or t.height == resolution + or int(t.width * (9 / 16)) == resolution + ) + and (not color_range or t.range == color_range) + and (not codec or t.codec == codec) + and (vlang is None or str(t.language) == vlang) + ] + match = candidates[-1] if worst and candidates else next(iter(candidates), None) + if match and match not in fallback_selected: + fallback_selected.append(match) + title.tracks.videos = fallback_selected + else: + self.log.error(msg) + self.log.error(msg_detail) + sys.exit(1) + + # filter subtitle tracks + if require_subs: + missing_langs = [ + lang + for lang in require_subs + if not any(is_close_match(lang, [sub.language]) for sub in title.tracks.subtitles) + ] + + if missing_langs: + self.log.error(f"Required subtitle language(s) not found: {', '.join(missing_langs)}") + sys.exit(1) + + self.log.info( + f"Required languages found ({', '.join(require_subs)}), downloading all available subtitles" + ) + elif s_lang and "all" not in s_lang: + from envied.core.utilities import is_exact_match + + match_func = is_exact_match if exact_lang else is_close_match + + missing_langs = find_missing_langs( + s_lang, + [sub.language for sub in title.tracks.subtitles], + exact=exact_lang, + ) + if missing_langs: + missing_str = ", ".join(missing_langs) + if best_available: + remaining = [tok for tok in s_lang if tok not in missing_langs] + if remaining: + self.log.warning( + f"{missing_str} not found in subtitle tracks, continuing with: {', '.join(remaining)}" + ) + s_lang = remaining + else: + self.log.warning( + f"{missing_str} not found in subtitle tracks, continuing without subtitles" + ) + title.tracks.subtitles = [] + else: + self.log.error(missing_str + " not found in tracks") + sys.exit(1) + + if s_lang and title.tracks.subtitles: + title.tracks.select_subtitles(lambda x: match_func(x.language, s_lang)) + if not title.tracks.subtitles and not best_available: + self.log.error(f"There's no {s_lang} Subtitle Track...") + sys.exit(1) + + if not forced_subs: + title.tracks.select_subtitles(lambda x: not x.forced) + + # filter audio tracks + # might have no audio tracks if part of the video, e.g. transport stream hls + if len(title.tracks.audio) > 0: + if not audio_description: + title.tracks.select_audio(lambda x: not x.descriptive) # exclude descriptive audio + if acodec: + title.tracks.select_audio(lambda x: x.codec in acodec) + if not title.tracks.audio: + codec_names = ", ".join(c.name for c in acodec) + self.log.error(f"No audio tracks matching codecs: {codec_names}") + sys.exit(1) + if channels: + title.tracks.select_audio(lambda x: math.ceil(x.channels) == math.ceil(channels)) + if not title.tracks.audio: + self.log.error(f"There's no {channels} Audio Track...") + sys.exit(1) + if no_atmos: + title.tracks.audio = [x for x in title.tracks.audio if not x.atmos] + if not title.tracks.audio: + self.log.error("No non-Atmos audio tracks available...") + sys.exit(1) + if abitrate: + title.tracks.select_audio(lambda x: x.bitrate and x.bitrate // 1000 == abitrate) + if not title.tracks.audio: + self.log.error(f"There's no {abitrate}kbps Audio Track...") + sys.exit(1) + if abitrate_min is not None and abitrate_max is not None: + title.tracks.select_audio( + lambda x: x.bitrate and abitrate_min <= x.bitrate // 1000 <= abitrate_max + ) + if not title.tracks.audio: + self.log.error(f"No Audio Track in {abitrate_min}-{abitrate_max}kbps range...") + sys.exit(1) + audio_languages = a_lang or lang + if audio_languages: + processed_lang = [] + for language in audio_languages: + if language == "orig": + if title.language: + orig_lang = ( + str(title.language) if hasattr(title.language, "__str__") else title.language + ) + if orig_lang not in processed_lang: + processed_lang.append(orig_lang) + else: + self.log.warning( + "Original language not available for title, skipping 'orig' selection" + ) + else: + if language not in processed_lang: + processed_lang.append(language) + + if not any(tok in processed_lang for tok in ("best", "all")): + missing_a_langs = find_missing_langs( + processed_lang, + [a.language for a in title.tracks.audio], + exact=exact_lang, + ) + if missing_a_langs: + missing_str = ", ".join(missing_a_langs) + if best_available: + remaining = [tok for tok in processed_lang if tok not in missing_a_langs] + if remaining: + self.log.warning( + f"{missing_str} not found in audio tracks, continuing with: {', '.join(remaining)}" + ) + processed_lang = remaining + else: + self.log.error( + f"{missing_str} not found in audio tracks and no fallback available" + ) + sys.exit(1) + else: + self.log.error(missing_str + " not found in audio tracks") + sys.exit(1) + + if "best" in processed_lang or "all" in processed_lang: + unique_languages = {track.language for track in title.tracks.audio} + selected_audio = [] + best_key = lambda x: (bool(x.atmos), x.bitrate or 0) # noqa: E731 + for language in unique_languages: + codecs_to_check = acodec if (acodec and len(acodec) > 1) else [None] + for codec in codecs_to_check: + base_candidates = [ + t + for t in title.tracks.audio + if t.language == language and (codec is None or t.codec == codec) + ] + if not base_candidates: + continue + if audio_description: + standards = [t for t in base_candidates if not t.descriptive] + if standards: + selected_audio.append(max(standards, key=best_key)) + descs = [t for t in base_candidates if t.descriptive] + if descs: + selected_audio.append(max(descs, key=best_key)) + else: + selected_audio.append(max(base_candidates, key=best_key)) + title.tracks.audio = selected_audio + else: + # If multiple codecs were explicitly requested, pick the best track per codec per + # requested language instead of selecting *all* bitrate variants of a codec. + if acodec and len(acodec) > 1: + selected_audio: list[Audio] = [] + best_key = lambda x: (bool(x.atmos), x.bitrate or 0) # noqa: E731 + + for language in processed_lang: + for codec in acodec: + codec_tracks = [a for a in title.tracks.audio if a.codec == codec] + if not codec_tracks: + continue + + candidates = title.tracks.by_language( + codec_tracks, [language], per_language=0, exact_match=exact_lang + ) + if not candidates: + continue + + if audio_description: + standards = [t for t in candidates if not t.descriptive] + if standards: + selected_audio.append(max(standards, key=best_key)) + descs = [t for t in candidates if t.descriptive] + if descs: + selected_audio.append(max(descs, key=best_key)) + else: + selected_audio.append(max(candidates, key=best_key)) + + title.tracks.audio = selected_audio + else: + per_language = 1 + if audio_description: + standard_audio = [a for a in title.tracks.audio if not a.descriptive] + selected_standards = title.tracks.by_language( + standard_audio, + processed_lang, + per_language=per_language, + exact_match=exact_lang, + ) + desc_audio = [a for a in title.tracks.audio if a.descriptive] + # Include all descriptive tracks for the requested languages. + selected_descs = title.tracks.by_language( + desc_audio, processed_lang, per_language=0, exact_match=exact_lang + ) + title.tracks.audio = selected_standards + selected_descs + else: + title.tracks.audio = title.tracks.by_language( + title.tracks.audio, + processed_lang, + per_language=per_language, + exact_match=exact_lang, + ) + if not title.tracks.audio: + self.log.error(f"There's no {processed_lang} Audio Track, cannot continue...") + sys.exit(1) + + if ( + video_only + or audio_only + or subs_only + or chapters_only + or no_subs + or no_audio + or no_chapters + or no_video + ): + keep_videos = False + keep_audio = False + keep_subtitles = False + keep_chapters = False + + if video_only or audio_only or subs_only or chapters_only: + if video_only: + keep_videos = True + if audio_only: + keep_audio = True + if subs_only: + keep_subtitles = True + if chapters_only: + keep_chapters = True + else: + keep_videos = True + keep_audio = True + keep_subtitles = True + keep_chapters = True + + if no_subs: + keep_subtitles = False + if no_audio: + keep_audio = False + if no_chapters: + keep_chapters = False + if no_video: + keep_videos = False + + kept_tracks = [] + if keep_videos: + kept_tracks.extend(title.tracks.videos) + if keep_audio: + kept_tracks.extend(title.tracks.audio) + if keep_subtitles: + kept_tracks.extend(title.tracks.subtitles) + if keep_chapters: + kept_tracks.extend(title.tracks.chapters) + kept_tracks.extend(title.tracks.attachments) + + title.tracks = Tracks(kept_tracks, manifest_url=title.tracks.manifest_url) + + selected_tracks, tracks_progress_callables = title.tracks.tree(add_progress=True) + + for track in title.tracks: + if hasattr(track, "needs_drm_loading") and track.needs_drm_loading: + track.load_drm_if_needed(service) + + download_table = Table.grid() + download_table.add_row(selected_tracks) + + video_tracks = title.tracks.videos + if video_tracks: + highest_quality = max((track.height for track in video_tracks if track.height), default=0) + if highest_quality > 0: + if is_widevine_cdm(self.cdm): + quality_based_cdm = self.get_cdm( + self.service, self.profile, drm="widevine", quality=highest_quality + ) + if quality_based_cdm and quality_based_cdm != self.cdm: + self.log.debug( + f"Pre-selecting Widevine CDM based on highest quality {highest_quality}p across all video tracks" + ) + self.cdm = quality_based_cdm + elif is_playready_cdm(self.cdm): + quality_based_cdm = self.get_cdm( + self.service, self.profile, drm="playready", quality=highest_quality + ) + if quality_based_cdm and quality_based_cdm != self.cdm: + self.log.debug( + f"Pre-selecting PlayReady CDM based on highest quality {highest_quality}p across all video tracks" + ) + self.cdm = quality_based_cdm + + if hasattr(service, "resolve_server_keys"): + service.resolve_server_keys(title) + + dl_start_time = time.time() + + try: + with Live(Padding(download_table, (1, 5)), console=console, refresh_per_second=5): + with ThreadPoolExecutor(downloads) as pool: + for download in futures.as_completed( + ( + pool.submit( + track.download, + session=track.session or service.session, + no_proxy_download=no_proxy_download, + prepare_drm=partial( + partial(self.prepare_drm, table=download_table), + track=track, + title=title, + certificate=partial( + service.get_widevine_service_certificate, + title=title, + track=track, + ), + licence=partial( + service.get_playready_license + if is_playready_cdm(self.cdm) + else service.get_widevine_license, + title=title, + track=track, + ), + cdm_only=cdm_only, + vaults_only=vaults_only, + export=export_path, + ), + cdm=self.cdm, + max_workers=workers, + progress=tracks_progress_callables[i], + ) + for i, track in enumerate(title.tracks) + ) + ): + download.result() + + except KeyboardInterrupt: + console.print(Padding(":x: Download Cancelled...", (0, 5, 1, 5))) + if self.debug_logger: + self.debug_logger.log( + level="WARNING", + operation="download_tracks", + service=self.service, + message="Download cancelled by user", + context={"title": str(title)}, + ) + return + except Exception as e: # noqa + error_messages = [ + ":x: Download Failed...", + f" {type(e).__name__}: {e}", + ] + if hasattr(e, "returncode"): + error_messages.append(f" Binary call failed, Process exit code: {e.returncode}") + error_messages.append( + " An unexpected error occurred in one of the download workers.", + ) + console.print(Padding(Group(*error_messages), (1, 5))) + console.print_exception() + + if self.debug_logger: + self.debug_logger.log_error( + "download_tracks", + e, + service=self.service, + context={ + "title": str(title), + "error_type": type(e).__name__, + "tracks_count": len(title.tracks), + "returncode": getattr(e, "returncode", None), + }, + ) + return + + if skip_dl: + console.log("Skipped downloads as --skip-dl was used...") + else: + dl_time = time_elapsed_since(dl_start_time) + console.print(Padding(f"Track downloads finished in [progress.elapsed]{dl_time}[/]", (0, 5))) + + # Subtitle output mode configuration (for sidecar originals) + subtitle_output_mode = config.subtitle.get("output_mode", "mux") + sidecar_format = config.subtitle.get("sidecar_format", "srt") + skip_subtitle_mux = subtitle_output_mode == "sidecar" and (title.tracks.videos or title.tracks.audio) + sidecar_subtitles: list[Subtitle] = [] + sidecar_original_paths: dict[str, Path] = {} + if subtitle_output_mode in ("sidecar", "both") and not no_mux: + sidecar_subtitles = [s for s in title.tracks.subtitles if s.path and s.path.exists()] + if sidecar_format == "original": + config.directories.temp.mkdir(parents=True, exist_ok=True) + for subtitle in sidecar_subtitles: + original_path = ( + config.directories.temp / f"sidecar_original_{subtitle.id}{subtitle.path.suffix}" + ) + shutil.copy2(subtitle.path, original_path) + sidecar_original_paths[subtitle.id] = original_path + + with console.status("Converting Subtitles..."): + for subtitle in title.tracks.subtitles: + if sub_format: + if subtitle.codec != sub_format: + subtitle.convert(sub_format) + elif subtitle.codec == Subtitle.Codec.TimedTextMarkupLang: + # MKV does not support TTML, VTT is the next best option + subtitle.convert(Subtitle.Codec.WebVTT) + + with console.status("Checking Subtitles for Fonts..."): + font_names = [] + for subtitle in title.tracks.subtitles: + if subtitle.codec == Subtitle.Codec.SubStationAlphav4: + for line in subtitle.path.read_text("utf8").splitlines(): + if line.startswith("Style: "): + font_names.append(line.removeprefix("Style: ").split(",")[1].strip()) + + font_count, missing_fonts = self.attach_subtitle_fonts(font_names, title, temp_font_files) + + if font_count: + self.log.info(f"Attached {font_count} fonts for the Subtitles") + + if missing_fonts and sys.platform != "win32": + self.suggest_missing_fonts(missing_fonts) + + # Handle DRM decryption BEFORE repacking (must decrypt first!) + service_name = service.__class__.__name__.upper() + decryption_method = config.decryption_map.get(service_name, config.decryption) + decrypt_tool = "mp4decrypt" if decryption_method.lower() == "mp4decrypt" else "Shaka Packager" + + drm_tracks = [track for track in title.tracks if track.drm] + if drm_tracks: + with console.status(f"Decrypting tracks with {decrypt_tool}..."): + has_decrypted = False + for track in drm_tracks: + drm = track.get_drm_for_cdm(self.cdm) + if drm and hasattr(drm, "decrypt"): + drm.decrypt(track.path) + if not isinstance(drm, MonaLisa): + has_decrypted = True + events.emit(events.Types.TRACK_REPACKED, track=track) + else: + self.log.warning( + f"No matching DRM found for track {track} with CDM type {type(self.cdm).__name__}" + ) + if has_decrypted: + self.log.info(f"Decrypted tracks with {decrypt_tool}") + + # Extract Closed Captions from decrypted video tracks + if ( + not no_subs + and not (hasattr(service, "NO_SUBTITLES") and service.NO_SUBTITLES) + and not video_only + and not no_video + ): + for video_track_n, video_track in enumerate(title.tracks.videos): + has_manifest_cc = bool(getattr(video_track, "closed_captions", None)) + has_eia_cc = ( + not has_manifest_cc + and not title.tracks.subtitles + and any( + x.get("codec_name", "").startswith("eia_") + for x in ffprobe(video_track.path).get("streams", []) + ) + ) + if not has_manifest_cc and not has_eia_cc: + continue + + with console.status(f"Checking Video track {video_track_n + 1} for Closed Captions..."): + try: + cc_lang = ( + Language.get(video_track.closed_captions[0]["language"]) + if has_manifest_cc and video_track.closed_captions[0].get("language") + else title.language or video_track.language + ) + track_id = f"ccextractor-{video_track.id}" + cc = video_track.ccextractor( + track_id=track_id, + out_path=config.directories.temp + / config.filenames.subtitle.format(id=track_id, language=cc_lang), + language=cc_lang, + original=False, + ) + if cc: + cc.cc = True + title.tracks.add(cc) + self.log.info(f"Extracted a Closed Caption from Video track {video_track_n + 1}") + else: + self.log.info(f"No Closed Captions were found in Video track {video_track_n + 1}") + except EnvironmentError: + self.log.error( + "Cannot extract Closed Captions as the ccextractor executable was not found..." + ) + break + + # Now repack the decrypted tracks + with console.status("Repackaging tracks with FFMPEG..."): + has_repacked = False + for track in title.tracks: + if track.needs_repack: + track.repackage() + has_repacked = True + events.emit(events.Types.TRACK_REPACKED, track=track) + if has_repacked: + # we don't want to fill up the log with "Repacked x track" + self.log.info("Repacked one or more tracks with FFMPEG") + + with console.status("Normalizing video VUI..."): + for track in title.tracks.videos: + try: + track.normalize_vui() + except Exception as e: # noqa: BLE001 + self.log.warning(f"VUI normalization skipped for {track.id}: {e}") + + muxed_paths = [] + muxed_audio_codecs: dict[Path, Optional[Audio.Codec]] = {} + append_audio_codec_suffix = True + + if no_mux: + # Skip muxing, handle individual track files + for track in title.tracks: + if track.path and track.path.exists(): + muxed_paths.append(track.path) + elif isinstance(title, (Movie, Episode)): + progress = Progress( + TextColumn("[progress.description]{task.description}"), + SpinnerColumn(finished_text=""), + BarColumn(), + "•", + TimeRemainingColumn(compact=True, elapsed_when_finished=True), + "•", + TextColumn("{task.fields[downloaded]}"), + console=console, + ) + + merge_audio = ( + (not split_audio) if split_audio is not None else config.muxing.get("merge_audio", True) + ) + # When we split audio (merge_audio=False), multiple outputs may exist per title, so suffix codec. + append_audio_codec_suffix = not merge_audio + + multiplex_tasks: list[tuple[TaskID, Tracks, Optional[Audio.Codec]]] = [] + # Track hybrid-processing outputs explicitly so we can always clean them up, + # even if muxing fails early (e.g. SystemExit) before the normal delete loop. + hybrid_temp_paths: list[Path] = [] + + def clone_tracks_for_audio(base_tracks: Tracks, audio_tracks: list[Audio]) -> Tracks: + task_tracks = Tracks() + task_tracks.videos = list(base_tracks.videos) + task_tracks.audio = audio_tracks + task_tracks.subtitles = list(base_tracks.subtitles) + task_tracks.chapters = base_tracks.chapters + task_tracks.attachments = list(base_tracks.attachments) + return task_tracks + + def enqueue_mux_tasks(task_description: str, base_tracks: Tracks) -> None: + if merge_audio or not base_tracks.audio: + task_id = progress.add_task( + f"{task_description}...", total=None, start=False, downloaded="" + ) + multiplex_tasks.append((task_id, base_tracks, None)) + return + + audio_by_codec: dict[Optional[Audio.Codec], list[Audio]] = {} + for audio_track in base_tracks.audio: + audio_by_codec.setdefault(audio_track.codec, []).append(audio_track) + + for audio_codec, codec_audio_tracks in audio_by_codec.items(): + description = task_description + if audio_codec: + description = f"{task_description} {audio_codec.name}" + + task_id = progress.add_task(f"{description}...", total=None, start=False, downloaded="") + task_tracks = clone_tracks_for_audio(base_tracks, codec_audio_tracks) + multiplex_tasks.append((task_id, task_tracks, audio_codec)) + + def mux_video_standalone(video_track: Optional[Video]) -> None: + if video_track and video_track.dv_compatible_bitstream: + apply_dv_fixup(video_track) + + task_description = "Multiplexing" + if video_track: + if len(quality) > 1: + task_description += f" {video_track.height}p" + if len(range_) > 1: + task_description += f" {video_track.range.name}" + if len(vcodec) > 1: + task_description += f" {video_track.codec.name}" + + task_tracks = Tracks(title.tracks) + title.tracks.chapters + title.tracks.attachments + if video_track: + task_tracks.videos = [video_track] + + enqueue_mux_tasks(task_description, task_tracks) + + if any(r == Video.Range.HYBRID for r in range_) and title.tracks.videos: + self.log.info("Processing Hybrid HDR10+DV tracks...") + + # Snapshot videos before hybrid tracks are added so the originals + # can still be muxed standalone afterwards. + original_videos = list(title.tracks.videos) + + # Prefer HDR10+ over HDR10 as the hybrid base layer. + resolutions_processed = set() + base_tracks_list = [ + v for v in title.tracks.videos if v.range in (Video.Range.HDR10P, Video.Range.HDR10) + ] + dv_tracks = [v for v in title.tracks.videos if v.range == Video.Range.DV] + + for hdr10_track in base_tracks_list: + resolution = hdr10_track.height + if resolution in resolutions_processed: + continue + + # DV layer only supplies RPU metadata, so the lowest resolution suffices. + matching_dv = min(dv_tracks, key=lambda v: v.height) if dv_tracks else None + + if matching_dv: + resolutions_processed.add(resolution) + + # Operate on copies so the originals stay muxable standalone. + resolution_tracks = [deepcopy(hdr10_track), deepcopy(matching_dv)] + for track in resolution_tracks: + track.needs_duration_fix = True + + Hybrid(resolution_tracks, self.service) + + hybrid_filename = f"HDR10-DV-{resolution}p.hevc" + hybrid_output_path = config.directories.temp / hybrid_filename + hybrid_temp_paths.append(hybrid_output_path) + + # Hybrid always writes HDR10-DV.hevc; rename it per resolution. + default_output = config.directories.temp / "HDR10-DV.hevc" + if default_output.exists(): + hybrid_output_path.unlink(missing_ok=True) + shutil.move(str(default_output), str(hybrid_output_path)) + + task_description = f"Multiplexing Hybrid HDR10+DV {resolution}p" + task_tracks = Tracks(title.tracks) + title.tracks.chapters + title.tracks.attachments + + hybrid_track = deepcopy(hdr10_track) + hybrid_track.id = f"hybrid_{hdr10_track.id}_{resolution}" + hybrid_track.path = hybrid_output_path + hybrid_track.range = Video.Range.DV # It's now a DV track + hybrid_track.needs_duration_fix = True + title.tracks.add(hybrid_track) + task_tracks.videos = [hybrid_track] + + enqueue_mux_tasks(task_description, task_tracks) + + # Mux every requested range standalone, skipping the ingredient-only DV. + for video_track in original_videos: + if video_track.hybrid_base_only: + continue + mux_video_standalone(video_track) + + console.print() + else: + # Normal mode: process each video track separately + for video_track in title.tracks.videos or [None]: + mux_video_standalone(video_track) + + try: + with Live(Padding(progress, (0, 5, 1, 5)), console=console): + mux_index = 0 + for task_id, task_tracks, audio_codec in multiplex_tasks: + progress.start_task(task_id) # TODO: Needed? + audio_expected = not video_only and not no_audio + muxed_path, return_code, errors = task_tracks.mux( + str(title), + progress=partial(progress.update, task_id=task_id), + delete=False, + audio_expected=audio_expected, + title_language=title.language, + skip_subtitles=skip_subtitle_mux, + ) + if muxed_path.exists(): + mux_index += 1 + unique_path = muxed_path.with_name( + f"{muxed_path.stem}.{mux_index}{muxed_path.suffix}" + ) + if unique_path != muxed_path: + shutil.move(muxed_path, unique_path) + muxed_path = unique_path + muxed_paths.append(muxed_path) + muxed_audio_codecs[muxed_path] = audio_codec + if return_code >= 2: + self.log.error(f"Failed to Mux video to Matroska file ({return_code}):") + elif return_code == 1 or errors: + self.log.warning("mkvmerge had at least one warning or error, continuing anyway...") + for line in errors: + if line.startswith("#GUI#error"): + self.log.error(line) + else: + self.log.warning(line) + if return_code >= 2: + sys.exit(1) + + # Output sidecar subtitles before deleting track files + if sidecar_subtitles and not no_mux: + media_info = MediaInfo.parse(muxed_paths[0]) if muxed_paths else None + if media_info: + base_filename = title.get_filename(media_info, show_service=not no_source) + else: + base_filename = str(title) + + sidecar_dir = self.output_dir or config.directories.downloads + if not no_folder and isinstance(title, (Episode, Song)) and media_info: + sidecar_dir /= title.get_filename( + media_info, show_service=not no_source, folder=True + ) + sidecar_dir.mkdir(parents=True, exist_ok=True) + + with console.status("Saving subtitle sidecar files..."): + created = self.output_subtitle_sidecars( + sidecar_subtitles, + base_filename, + sidecar_dir, + sidecar_format, + original_paths=sidecar_original_paths or None, + ) + if created: + self.log.info(f"Saved {len(created)} sidecar subtitle files") + + for track in title.tracks: + track.delete() + + # Clear temp font attachment paths and delete other attachments + for attachment in title.tracks.attachments: + if attachment.path and attachment.path in temp_font_files: + attachment.path = None + else: + attachment.delete() + + # Clean up temp fonts + for temp_path in temp_font_files: + temp_path.unlink(missing_ok=True) + for temp_path in sidecar_original_paths.values(): + temp_path.unlink(missing_ok=True) + finally: + # Hybrid() produces a temp HEVC output we rename; make sure it's never left behind. + # Also attempt to remove the default hybrid output name if it still exists. + for temp_path in hybrid_temp_paths: + try: + temp_path.unlink(missing_ok=True) + except PermissionError: + self.log.warning(f"Failed to delete temp file (in use?): {temp_path}") + try: + (config.directories.temp / "HDR10-DV.hevc").unlink(missing_ok=True) + except PermissionError: + self.log.warning( + f"Failed to delete temp file (in use?): {config.directories.temp / 'HDR10-DV.hevc'}" + ) + + else: + # dont mux + muxed_paths.append(title.tracks.audio[0].path) + + if no_mux: + # Handle individual track files without muxing + final_dir = self.output_dir or config.directories.downloads + if not no_folder and isinstance(title, (Episode, Song)): + # Create folder based on title + # Use first available track for filename generation + sample_track = ( + title.tracks.videos[0] + if title.tracks.videos + else ( + title.tracks.audio[0] + if title.tracks.audio + else (title.tracks.subtitles[0] if title.tracks.subtitles else None) + ) + ) + if sample_track and sample_track.path: + media_info = MediaInfo.parse(sample_track.path) + final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True) + + final_dir.mkdir(parents=True, exist_ok=True) + + for track_path in muxed_paths: + # Generate appropriate filename for each track + media_info = MediaInfo.parse(track_path) + base_filename = title.get_filename(media_info, show_service=not no_source) + + # Add track type suffix to filename + track = next((t for t in title.tracks if t.path == track_path), None) + if track: + if isinstance(track, Video): + track_suffix = f".{track.codec.name if hasattr(track.codec, 'name') else 'video'}" + elif isinstance(track, Audio): + lang_suffix = f".{track.language}" if track.language else "" + track_suffix = ( + f"{lang_suffix}.{track.codec.name if hasattr(track.codec, 'name') else 'audio'}" + ) + elif isinstance(track, Subtitle): + lang_suffix = f".{track.language}" if track.language else "" + forced_suffix = ".forced" if track.forced else "" + sdh_suffix = ".sdh" if track.sdh else "" + track_suffix = f"{lang_suffix}{forced_suffix}{sdh_suffix}" + else: + track_suffix = "" + + final_path = final_dir / f"{base_filename}{track_suffix}{track_path.suffix}" + else: + final_path = final_dir / f"{base_filename}{track_path.suffix}" + + shutil.move(track_path, final_path) + self.completed_files.append(final_path) + self.log.debug(f"Saved: {final_path.name}") + else: + # Handle muxed files + used_final_paths: set[Path] = set() + for muxed_path in muxed_paths: + media_info = MediaInfo.parse(muxed_path) + final_dir = self.output_dir or config.directories.downloads + final_filename = title.get_filename(media_info, show_service=not no_source) + audio_codec_suffix = muxed_audio_codecs.get(muxed_path) + + if not no_folder and isinstance(title, (Episode, Song)): + final_dir /= title.get_filename(media_info, show_service=not no_source, folder=True) + + final_dir.mkdir(parents=True, exist_ok=True) + final_path = final_dir / f"{final_filename}{muxed_path.suffix}" + template_type = ( + "series" if isinstance(title, Episode) else "songs" if isinstance(title, Song) else "movies" + ) + sep = config.get_template_separator(template_type) + + if final_path.exists() and audio_codec_suffix and append_audio_codec_suffix: + final_filename = f"{final_filename.rstrip()}{sep}{audio_codec_suffix.name}" + final_path = final_dir / f"{final_filename}{muxed_path.suffix}" + + if final_path in used_final_paths: + i = 2 + while final_path in used_final_paths: + final_path = final_dir / f"{final_filename.rstrip()}{sep}{i}{muxed_path.suffix}" + i += 1 + + try: + os.replace(muxed_path, final_path) + except OSError: + if final_path.exists(): + final_path.unlink() + shutil.move(muxed_path, final_path) + used_final_paths.add(final_path) + self.completed_files.append(final_path) + tags.tag_file(final_path, title, self.tmdb_id, self.imdb_id) + + title_dl_time = time_elapsed_since(dl_start_time) + console.print( + Padding(f":tada: Title downloaded in [progress.elapsed]{title_dl_time}[/]!", (0, 5, 1, 5)) + ) + + if not hasattr(service, "close"): + cookie_file = self.get_cookie_path(self.service, self.profile) + if cookie_file: + self.save_cookies(cookie_file, service.session.cookies) + + if hasattr(service, "close"): + service.close() + + dl_time = time_elapsed_since(start_time) + + console.print(Padding(f"Processed all titles in [progress.elapsed]{dl_time}", (0, 5, 1, 5))) + + @staticmethod + def title_to_meta(title: Title_T) -> dict[str, Any]: + """Capture the title fields needed to rebuild a Title for output naming on import.""" + from envied.core.titles import Episode, Movie + + meta: dict[str, Any] = { + "id": str(title.id), + "language": str(title.language) if getattr(title, "language", None) else None, + } + if isinstance(title, Episode): + meta.update( + type="episode", + series_title=title.title, + season=title.season, + number=title.number, + name=title.name, + year=title.year, + ) + elif isinstance(title, Movie): + meta.update(type="movie", name=title.name, year=title.year) + else: + meta.update(type="movie", name=str(title)) + return meta + + def write_export(self, export: Path, title: Title_T, track: AnyTrack, drm: Any) -> None: + """Write a shareable v2 export usable by ``envied.import``. + + Carries no session/cookies/dl-flags. Region (country code) is stored only when the + export used ``--proxy``, as an import geofence. Each track records only the licensed + DRM system; content keys live once under the track's ``keys``. + """ + with self.EXPORT_LOCK: + doc: dict[str, Any] = {} + if export.is_file(): + doc = json.loads(export.read_text(encoding="utf8")) or {} + + doc.setdefault("version", 2) + doc.setdefault("service", self.service) + if "region" not in doc and getattr(self, "proxy_requested", False): + region = getattr(getattr(self, "export_service", None), "current_region", None) + if region: + doc["region"] = region + + titles = doc.setdefault("titles", {}) + tinfo = titles.setdefault(str(title.id), {}) + tinfo.setdefault("meta", self.title_to_meta(title)) + + if title.tracks.manifest_url: + tinfo.setdefault("manifest_url", title.tracks.manifest_url) + + all_tracks = [*title.tracks.videos, *title.tracks.audio, *title.tracks.subtitles] + if "manifest_type" not in tinfo: + tinfo["manifest_type"] = next( + (t.descriptor.name for t in all_tracks if t.descriptor != Video.Descriptor.URL), None + ) + + tracks_map = tinfo.setdefault("tracks", {}) + if not tracks_map: + for t in all_tracks: + tracks_map[str(t.id)] = t.to_dict() + + track_data = tracks_map.setdefault(str(track.id), track.to_dict()) + if hasattr(drm, "to_dict"): + track_data["drm"] = [drm.to_dict()] + keys = track_data.setdefault("keys", {}) + for kid, key in drm.content_keys.items(): + keys[kid.hex] = key + + if "chapters" not in tinfo: + tinfo["chapters"] = [ + {"timestamp": chapter.timestamp, "name": chapter.name} for chapter in (title.tracks.chapters or []) + ] + + if "attachments" not in tinfo: + tinfo["attachments"] = [a.to_dict() for a in (title.tracks.attachments or []) if a.url] + + export.write_text(json.dumps(doc, indent=4, ensure_ascii=False), encoding="utf8") + + def prepare_drm( + self, + drm: DRM_T, + track: AnyTrack, + title: Title_T, + certificate: Callable, + licence: Callable, + track_kid: Optional[UUID] = None, + table: Table = None, + cdm_only: bool = False, + vaults_only: bool = False, + export: Optional[Path] = None, + ) -> None: + """ + Prepare the DRM by getting decryption data like KIDs, Keys, and such. + The DRM object should be ready for decryption once this function ends. + """ + if not drm: + return + + server_cdm = getattr(self, "server_cdm", False) + + if server_cdm: + if not drm.content_keys: + self.log.warning("Server CDM did not resolve any keys for this track") + return + svc = getattr(self, "_remote_service", None) + server_drm_type = getattr(svc, "_server_cdm_type", None) if svc else None + drm_name = {"widevine": "Widevine", "playready": "PlayReady"}.get( + server_drm_type or "", drm.__class__.__name__ + ) + with self.DRM_TABLE_LOCK: + pssh_str = "" + expected_class = "PlayReady" if server_drm_type == "playready" else "Widevine" + matching_drm = next( + (d for d in (track.drm or []) if d.__class__.__name__ == expected_class), + drm, + ) + if hasattr(matching_drm, "pssh") and matching_drm.pssh: + if hasattr(matching_drm.pssh, "dumps"): + pssh_str = self.truncate_pssh_for_display(matching_drm.pssh.dumps(), drm_name) + elif hasattr(matching_drm, "data") and matching_drm.data.get("pssh_b64"): + pssh_str = self.truncate_pssh_for_display(matching_drm.data["pssh_b64"], drm_name) + if pssh_str: + cek_tree = Tree(Text.assemble((drm_name, "cyan"), (f"({pssh_str})", "text"), overflow="fold")) + else: + cek_tree = Tree(Text.assemble((drm_name, "cyan"), overflow="fold")) + all_kids = list(getattr(drm, "kids", [])) + if track_kid and track_kid not in all_kids: + all_kids.append(track_kid) + for kid in all_kids: + if kid in drm.content_keys: + is_track_kid = ["", "*"][kid == track_kid] + key = drm.content_keys[kid] + cek_tree.add(f"[text2]{kid.hex}:{key}{is_track_kid}") + for kid, key in drm.content_keys.items(): + if kid not in all_kids: + cek_tree.add(f"[text2]{kid.hex}:{key}") + if not any(isinstance(x, Tree) and x.label == cek_tree.label for x in table.columns[0].cells): + table.add_row(cek_tree) + return + + track_quality = None + if isinstance(track, Video) and track.height: + track_quality = track.height + + if not server_cdm: + if isinstance(drm, Widevine): + if not is_widevine_cdm(self.cdm): + widevine_cdm = self.get_cdm(self.service, self.profile, drm="widevine", quality=track_quality) + if widevine_cdm: + if track_quality: + self.log.info(f"Switching to Widevine CDM for Widevine {track_quality}p content") + else: + self.log.info("Switching to Widevine CDM for Widevine content") + self.cdm = widevine_cdm + + elif isinstance(drm, PlayReady): + if not is_playready_cdm(self.cdm): + playready_cdm = self.get_cdm(self.service, self.profile, drm="playready", quality=track_quality) + if playready_cdm: + if track_quality: + self.log.info(f"Switching to PlayReady CDM for PlayReady {track_quality}p content") + else: + self.log.info("Switching to PlayReady CDM for PlayReady content") + self.cdm = playready_cdm + + if isinstance(drm, Widevine): + if self.debug_logger: + self.debug_logger.log_drm_operation( + drm_type="Widevine", + operation="prepare_drm", + service=self.service, + context={ + "track": str(track), + "title": str(title), + "pssh": drm.pssh.dumps() if drm.pssh else None, + "kids": [k.hex for k in drm.kids], + "track_kid": track_kid.hex if track_kid else None, + }, + ) + + with self.DRM_TABLE_LOCK: + pssh_display = self.truncate_pssh_for_display(drm.pssh.dumps(), "Widevine") + cek_tree = Tree(Text.assemble(("Widevine", "cyan"), (f"({pssh_display})", "text"), overflow="fold")) + pre_existing_tree = next( + (x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None + ) + if pre_existing_tree: + cek_tree = pre_existing_tree + + need_license = False + all_kids = list(drm.kids) + if track_kid and track_kid not in all_kids: + all_kids.append(track_kid) + + for kid in all_kids: + if kid in drm.content_keys: + is_track_kid = ["", "*"][kid == track_kid] + key = drm.content_keys[kid] + label = f"[text2]{kid.hex}:{key}{is_track_kid}" + if not any(f"{kid.hex}:{key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + continue + + is_track_kid = ["", "*"][kid == track_kid] + + cached_key = self.LICENSE_KEY_CACHE.get(kid) + if cached_key: + drm.content_keys[kid] = cached_key + label = f"[text2]{kid.hex}:{cached_key}{is_track_kid} from cache" + if not any(f"{kid.hex}:{cached_key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="license_cache_hit", + service=self.service, + context={ + "kid": kid.hex, + "content_key": cached_key, + "track": str(track), + "drm_type": "Widevine", + }, + ) + continue + + if not cdm_only: + content_key, vault_used = self.vaults.get_key(kid) + if content_key: + drm.content_keys[kid] = content_key + label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}" + if not any(f"{kid.hex}:{content_key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + self.vaults.add_key(kid, content_key, excluding=vault_used) + self.LICENSE_KEY_CACHE[kid] = content_key + + if self.debug_logger: + self.debug_logger.log_vault_query( + vault_name=vault_used, + operation="get_key_success", + service=self.service, + context={ + "kid": kid.hex, + "content_key": content_key, + "track": str(track), + "from_cache": True, + }, + ) + elif vaults_only: + msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="vault_key_not_found", + service=self.service, + message=msg, + context={"kid": kid.hex, "track": str(track)}, + ) + raise Widevine.Exceptions.CEKNotFound(msg) + else: + need_license = True + + if kid not in drm.content_keys and cdm_only: + need_license = True + + if need_license and all(kid in drm.content_keys for kid in all_kids): + need_license = False + + if need_license and not vaults_only: + from_vaults = drm.content_keys.copy() + + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="get_license", + service=self.service, + message="Requesting Widevine license from service", + context={ + "track": str(track), + "kids_needed": [k.hex for k in all_kids if k not in drm.content_keys], + }, + ) + + try: + if self.service == "NF": + drm.get_NF_content_keys(cdm=self.cdm, licence=licence, certificate=certificate) + else: + drm.get_content_keys(cdm=self.cdm, licence=licence, certificate=certificate) + except Exception as e: + if drm.content_keys: + self.log.debug(f"License call failed but keys already in content_keys: {e}") + else: + if isinstance(e, (Widevine.Exceptions.EmptyLicense, Widevine.Exceptions.CEKNotFound)): + msg = str(e) + else: + msg = f"An exception occurred in the Service's license function: {e}" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + if self.debug_logger: + self.debug_logger.log_error( + "get_license", + e, + service=self.service, + context={"track": str(track), "exception_type": type(e).__name__}, + ) + raise e + + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="license_keys_retrieved", + service=self.service, + context={ + "track": str(track), + "keys_count": len(drm.content_keys), + "kids": [k.hex for k in drm.content_keys.keys()], + }, + ) + + for kid_, key in drm.content_keys.items(): + if key == "0" * 32: + key = f"[red]{key}[/]" + is_track_kid_marker = ["", "*"][kid_ == track_kid] + label = f"[text2]{kid_.hex}:{key}{is_track_kid_marker}" + if not any(f"{kid_.hex}:{key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + + drm.content_keys = { + kid_: key for kid_, key in drm.content_keys.items() if key and key.count("0") != len(key) + } + + # The CDM keys may have returned blank content keys for KIDs we got from vaults. + # So we re-add the keys from vaults earlier overwriting blanks or removed KIDs data. + drm.content_keys.update(from_vaults) + + self.LICENSE_KEY_CACHE.update(drm.content_keys) + + successful_caches = self.vaults.add_keys(drm.content_keys) + self.log.info( + f"Cached {len(drm.content_keys)} Key{'' if len(drm.content_keys) == 1 else 's'} to " + f"{successful_caches}/{len(self.vaults)} Vaults" + ) + + if track_kid and track_kid not in drm.content_keys: + msg = f"No Content Key for KID {track_kid.hex} was returned in the License" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + raise Widevine.Exceptions.CEKNotFound(msg) + + if cek_tree.children and not pre_existing_tree: + table.add_row() + table.add_row(cek_tree) + + if export: + self.write_export(export, title, track, drm) + + elif isinstance(drm, PlayReady): + if self.debug_logger: + self.debug_logger.log_drm_operation( + drm_type="PlayReady", + operation="prepare_drm", + service=self.service, + context={ + "track": str(track), + "title": str(title), + "pssh": drm.pssh_b64 or "", + "kids": [k.hex for k in drm.kids], + "track_kid": track_kid.hex if track_kid else None, + }, + ) + + with self.DRM_TABLE_LOCK: + pssh_display = self.truncate_pssh_for_display(drm.pssh_b64 or "", "PlayReady") + cek_tree = Tree( + Text.assemble( + ("PlayReady", "cyan"), + (f"({pssh_display})", "text"), + overflow="fold", + ) + ) + pre_existing_tree = next( + (x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None + ) + if pre_existing_tree: + cek_tree = pre_existing_tree + + need_license = False + all_kids = list(drm.kids) + if track_kid and track_kid not in all_kids: + all_kids.append(track_kid) + + for kid in all_kids: + if kid in drm.content_keys: + is_track_kid = ["", "*"][kid == track_kid] + key = drm.content_keys[kid] + label = f"[text2]{kid.hex}:{key}{is_track_kid}" + if not any(f"{kid.hex}:{key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + continue + + is_track_kid = ["", "*"][kid == track_kid] + + cached_key = self.LICENSE_KEY_CACHE.get(kid) + if cached_key: + drm.content_keys[kid] = cached_key + label = f"[text2]{kid.hex}:{cached_key}{is_track_kid} from cache" + if not any(f"{kid.hex}:{cached_key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="license_cache_hit", + service=self.service, + context={ + "kid": kid.hex, + "content_key": cached_key, + "track": str(track), + "drm_type": "PlayReady", + }, + ) + continue + + if not cdm_only: + content_key, vault_used = self.vaults.get_key(kid) + if content_key: + drm.content_keys[kid] = content_key + label = f"[text2]{kid.hex}:{content_key}{is_track_kid} from {vault_used}" + if not any(f"{kid.hex}:{content_key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + self.vaults.add_key(kid, content_key, excluding=vault_used) + self.LICENSE_KEY_CACHE[kid] = content_key + + if self.debug_logger: + self.debug_logger.log_vault_query( + vault_name=vault_used, + operation="get_key_success", + service=self.service, + context={ + "kid": kid.hex, + "content_key": content_key, + "track": str(track), + "from_cache": True, + "drm_type": "PlayReady", + }, + ) + elif vaults_only: + msg = f"No Vault has a Key for {kid.hex} and --vaults-only was used" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="vault_key_not_found", + service=self.service, + message=msg, + context={"kid": kid.hex, "track": str(track), "drm_type": "PlayReady"}, + ) + raise PlayReady.Exceptions.CEKNotFound(msg) + else: + need_license = True + + if kid not in drm.content_keys and cdm_only: + need_license = True + + if need_license and all(kid in drm.content_keys for kid in all_kids): + need_license = False + + if need_license and not vaults_only: + from_vaults = drm.content_keys.copy() + + try: + drm.get_content_keys(cdm=self.cdm, licence=licence, certificate=certificate) + except Exception as e: + if drm.content_keys: + self.log.debug(f"License call failed but keys already in content_keys: {e}") + else: + if isinstance(e, (PlayReady.Exceptions.EmptyLicense, PlayReady.Exceptions.CEKNotFound)): + msg = str(e) + else: + msg = f"An exception occurred in the Service's license function: {e}" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + if self.debug_logger: + self.debug_logger.log_error( + "get_license_playready", + e, + service=self.service, + context={ + "track": str(track), + "exception_type": type(e).__name__, + "drm_type": "PlayReady", + }, + ) + raise e + + for kid_, key in drm.content_keys.items(): + is_track_kid_marker = ["", "*"][kid_ == track_kid] + label = f"[text2]{kid_.hex}:{key}{is_track_kid_marker}" + if not any(f"{kid_.hex}:{key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + + drm.content_keys.update(from_vaults) + + self.LICENSE_KEY_CACHE.update(drm.content_keys) + + successful_caches = self.vaults.add_keys(drm.content_keys) + self.log.info( + f"Cached {len(drm.content_keys)} Key{'' if len(drm.content_keys) == 1 else 's'} to " + f"{successful_caches}/{len(self.vaults)} Vaults" + ) + + if track_kid and track_kid not in drm.content_keys: + msg = f"No Content Key for KID {track_kid.hex} was returned in the License" + cek_tree.add(f"[logging.level.error]{msg}") + if not pre_existing_tree: + table.add_row(cek_tree) + raise PlayReady.Exceptions.CEKNotFound(msg) + + if cek_tree.children and not pre_existing_tree: + table.add_row() + table.add_row(cek_tree) + + if export: + self.write_export(export, title, track, drm) + + elif isinstance(drm, MonaLisa): + with self.DRM_TABLE_LOCK: + display_id = drm.content_id or drm.pssh + pssh_display = self.truncate_pssh_for_display(display_id, "MonaLisa") + cek_tree = Tree(Text.assemble(("MonaLisa", "cyan"), (f"({pssh_display})", "text"), overflow="fold")) + pre_existing_tree = next( + (x for x in table.columns[0].cells if isinstance(x, Tree) and x.label == cek_tree.label), None + ) + if pre_existing_tree: + cek_tree = pre_existing_tree + + for kid_, key in drm.content_keys.items(): + label = f"[text2]{kid_.hex}:{key}" + if not any(f"{kid_.hex}:{key}" in x.label for x in cek_tree.children): + cek_tree.add(label) + + if cek_tree.children and not pre_existing_tree: + table.add_row() + table.add_row(cek_tree) + + @staticmethod + def get_cookie_path(service: str, profile: Optional[str]) -> Optional[Path]: + """Get Service Cookie File Path for Profile.""" + direct_cookie_file = config.directories.cookies / f"{service}.txt" + profile_cookie_file = config.directories.cookies / service / f"{profile}.txt" + default_cookie_file = config.directories.cookies / service / "default.txt" + + if direct_cookie_file.exists(): + return direct_cookie_file + elif profile_cookie_file.exists(): + return profile_cookie_file + elif default_cookie_file.exists(): + return default_cookie_file + + @staticmethod + def get_cookie_jar(service: str, profile: Optional[str]) -> Optional[MozillaCookieJar]: + """Get Service Cookies for Profile.""" + cookie_file = dl.get_cookie_path(service, profile) + if cookie_file: + cookie_jar = MozillaCookieJar(cookie_file) + cookie_data = html.unescape(cookie_file.read_text("utf8")).splitlines(keepends=False) + for i, line in enumerate(cookie_data): + if line and not line.startswith("#"): + line_data = line.lstrip().split("\t") + # Disable client-side expiry checks completely across everywhere + # Even though the cookies are loaded under ignore_expires=True, stuff + # like python-requests may not use them if they are expired + line_data[4] = "" + cookie_data[i] = "\t".join(line_data) + cookie_data = "\n".join(cookie_data) + cookie_file.write_text(cookie_data, "utf8") + cookie_jar.load(ignore_discard=True, ignore_expires=True) + return cookie_jar + + @staticmethod + def save_cookies(path: Path, cookies: CookieJar): + if hasattr(cookies, "jar"): + cookies = cookies.jar + + cookie_jar = MozillaCookieJar(path) + cookie_jar.load() + for cookie in cookies: + cookie_jar.set_cookie(cookie) + cookie_jar.save(ignore_discard=True) + + @staticmethod + def get_credentials(service: str, profile: Optional[str]) -> Optional[Credential]: + """Get Service Credentials for Profile.""" + credentials = config.credentials.get(service) + if credentials: + if isinstance(credentials, dict): + if profile: + credentials = credentials.get(profile) or credentials.get("default") + else: + credentials = credentials.get("default") + if credentials: + if isinstance(credentials, list): + return Credential(*credentials) + return Credential.loads(credentials) # type: ignore + + def get_cdm( + self, + service: str, + profile: Optional[str] = None, + drm: Optional[str] = None, + quality: Optional[int] = None, + ) -> Optional[object]: + """ + Get CDM for a specified service (either Local or Remote CDM). + Now supports quality-based selection when quality is provided. + Raises a ValueError if there's a problem getting a CDM. + """ + cdm_name = config.cdm.get(service) or config.cdm.get("default") + if not cdm_name: + return None + + if isinstance(cdm_name, dict): + if quality: + quality_match = None + quality_keys = [] + + for key in cdm_name.keys(): + if ( + isinstance(key, str) + and any(op in key for op in [">=", ">", "<=", "<"]) + or (isinstance(key, str) and key.isdigit()) + ): + quality_keys.append(key) + + def sort_quality_key(key): + if key.isdigit(): + return (0, int(key)) # Exact matches first + elif key.startswith(">="): + return (1, -int(key[2:])) # >= descending + elif key.startswith(">"): + return (1, -int(key[1:])) # > descending + elif key.startswith("<="): + return (2, int(key[2:])) # <= ascending + elif key.startswith("<"): + return (2, int(key[1:])) # < ascending + return (3, 0) # Other keys last + + quality_keys.sort(key=sort_quality_key) + + for key in quality_keys: + if key.isdigit() and quality == int(key): + quality_match = cdm_name[key] + self.log.debug(f"Selected CDM based on exact quality match {quality}p: {quality_match}") + break + elif key.startswith(">="): + threshold = int(key[2:]) + if quality >= threshold: + quality_match = cdm_name[key] + self.log.debug(f"Selected CDM based on quality {quality}p >= {threshold}p: {quality_match}") + break + elif key.startswith(">"): + threshold = int(key[1:]) + if quality > threshold: + quality_match = cdm_name[key] + self.log.debug(f"Selected CDM based on quality {quality}p > {threshold}p: {quality_match}") + break + elif key.startswith("<="): + threshold = int(key[2:]) + if quality <= threshold: + quality_match = cdm_name[key] + self.log.debug(f"Selected CDM based on quality {quality}p <= {threshold}p: {quality_match}") + break + elif key.startswith("<"): + threshold = int(key[1:]) + if quality < threshold: + quality_match = cdm_name[key] + self.log.debug(f"Selected CDM based on quality {quality}p < {threshold}p: {quality_match}") + break + + if quality_match: + cdm_name = quality_match + + if isinstance(cdm_name, dict): + lower_keys = {k.lower(): v for k, v in cdm_name.items()} + if {"widevine", "playready"} & lower_keys.keys(): + drm_key = None + if drm: + drm_key = { + "wv": "widevine", + "widevine": "widevine", + "pr": "playready", + "playready": "playready", + }.get(drm.lower()) + cdm_name = lower_keys.get(drm_key or "widevine") or lower_keys.get("playready") + else: + cdm_name = cdm_name.get(profile) or cdm_name.get("default") or config.cdm.get("default") + if not cdm_name: + return None + + from envied.core.cdm import load_cdm + + return load_cdm(cdm_name, service_name=service, vaults=self.vaults) diff --git a/reset/packages/envied/src/envied/commands/env.py b/reset/packages/envied/src/envied/commands/env.py new file mode 100644 index 0000000..ae58920 --- /dev/null +++ b/reset/packages/envied/src/envied/commands/env.py @@ -0,0 +1,236 @@ +import logging +import os +import shutil +import sys +from pathlib import Path +from typing import Optional + +import click +from rich.padding import Padding +from rich.table import Table +from rich.tree import Tree + +from envied.core import binaries +from envied.core.config import POSSIBLE_CONFIG_PATHS, config, config_path +from envied.core.console import console +from envied.core.constants import context_settings +from envied.core.services import Services + + +@click.group(short_help="Manage and configure the project environment.", context_settings=context_settings) +def env() -> None: + """Manage and configure the project environment.""" + + +@env.command() +def check() -> None: + """Checks environment for the required dependencies.""" + # Define all dependencies + all_deps = [ + # Core Media Tools + {"name": "FFmpeg", "binary": binaries.FFMPEG, "required": True, "desc": "Media processing", "cat": "Core"}, + {"name": "FFprobe", "binary": binaries.FFProbe, "required": True, "desc": "Media analysis", "cat": "Core"}, + {"name": "MKVToolNix", "binary": binaries.MKVToolNix, "required": True, "desc": "MKV muxing", "cat": "Core"}, + { + "name": "mkvpropedit", + "binary": binaries.Mkvpropedit, + "required": True, + "desc": "MKV metadata", + "cat": "Core", + }, + { + "name": "shaka-packager", + "binary": binaries.ShakaPackager, + "required": True, + "desc": "DRM decryption", + "cat": "DRM", + }, + { + "name": "mp4decrypt", + "binary": binaries.Mp4decrypt, + "required": False, + "desc": "DRM decryption", + "cat": "DRM", + }, + { + "name": "ML-Worker", + "binary": binaries.ML_Worker, + "required": False, + "desc": "DRM licensing", + "cat": "DRM", + }, + # HDR Processing + {"name": "dovi_tool", "binary": binaries.DoviTool, "required": False, "desc": "Dolby Vision", "cat": "HDR"}, + { + "name": "HDR10Plus_tool", + "binary": binaries.HDR10PlusTool, + "required": False, + "desc": "HDR10+ metadata", + "cat": "HDR", + }, + # Subtitle Tools + { + "name": "SubtitleEdit", + "binary": binaries.SubtitleEdit, + "required": False, + "desc": "Sub conversion", + "cat": "Subtitle", + }, + { + "name": "CCExtractor", + "binary": binaries.CCExtractor, + "required": False, + "desc": "CC extraction", + "cat": "Subtitle", + }, + # Media Players + {"name": "FFplay", "binary": binaries.FFPlay, "required": False, "desc": "Simple player", "cat": "Player"}, + {"name": "MPV", "binary": binaries.MPV, "required": False, "desc": "Advanced player", "cat": "Player"}, + # Network Tools + { + "name": "HolaProxy", + "binary": binaries.HolaProxy, + "required": False, + "desc": "Proxy service", + "cat": "Network", + }, + {"name": "Caddy", "binary": binaries.Caddy, "required": False, "desc": "Web server", "cat": "Network"}, + {"name": "Docker", "binary": binaries.Docker, "required": False, "desc": "Gluetun VPN", "cat": "Network"}, + ] + + # Track overall status + all_required_installed = True + total_installed = 0 + total_required = 0 + missing_required = [] + + # Create a single table + table = Table( + title="Environment Dependencies", title_style="bold", show_header=True, header_style="bold", expand=False + ) + table.add_column("Category", style="bold cyan", width=10) + table.add_column("Tool", width=16) + table.add_column("Status", justify="center", width=10) + table.add_column("Req", justify="center", width=4) + table.add_column("Purpose", style="bright_black", width=20) + + last_cat = None + for dep in all_deps: + path = dep["binary"] + + # Category column (only show when it changes) + category = dep["cat"] if dep["cat"] != last_cat else "" + last_cat = dep["cat"] + + # Status + if path: + status = "[green]✓[/green]" + total_installed += 1 + else: + status = "[red]✗[/red]" + if dep["required"]: + all_required_installed = False + missing_required.append(dep["name"]) + + if dep["required"]: + total_required += 1 + + # Required column (compact) + req = "[red]Y[/red]" if dep["required"] else "[bright_black]-[/bright_black]" + + # Add row + table.add_row(category, dep["name"], status, req, dep["desc"]) + + console.print(Padding(table, (1, 2))) + + # Compact summary + summary_parts = [f"[bold]Total:[/bold] {total_installed}/{len(all_deps)}"] + + if all_required_installed: + summary_parts.append("[green]All required tools installed ✓[/green]") + else: + summary_parts.append(f"[red]Missing required: {', '.join(missing_required)}[/red]") + + console.print(Padding(" ".join(summary_parts), (1, 2))) + + +@env.command() +def info() -> None: + """Displays information about the current environment.""" + log = logging.getLogger("env") + + if config_path: + log.info(f"Config loaded from {config_path}") + else: + tree = Tree("No config file found, you can use any of the following locations:") + for i, path in enumerate(POSSIBLE_CONFIG_PATHS, start=1): + tree.add(f"[repr.number]{i}.[/] [text2]{path.resolve()}[/]") + console.print(Padding(tree, (0, 5))) + + table = Table(title="Directories", title_style="bold", expand=True) + table.add_column("Name", no_wrap=True) + table.add_column("Path", no_wrap=False, overflow="fold") + + path_vars = { + x: Path(os.getenv(x)) + for x in ("TEMP", "APPDATA", "LOCALAPPDATA", "USERPROFILE") + if sys.platform == "win32" and os.getenv(x) + } + + for name in sorted(dir(config.directories)): + if name.startswith("__") or name == "app_dirs": + continue + attr_value = getattr(config.directories, name) + + # Handle both single Path objects and lists of Path objects + if isinstance(attr_value, list): + # For lists, show each path on a separate line + paths_str = "\n".join(str(path.resolve()) for path in attr_value) + table.add_row(name.title(), paths_str) + else: + # For single Path objects, use the original logic + path = attr_value.resolve() + for var, var_path in path_vars.items(): + if path.is_relative_to(var_path): + path = rf"%{var}%\{path.relative_to(var_path)}" + break + table.add_row(name.title(), str(path)) + + console.print(Padding(table, (1, 5))) + + +@env.group(name="clear", short_help="Clear an environment directory.", context_settings=context_settings) +def clear() -> None: + """Clear an environment directory.""" + + +@clear.command() +@click.argument("service", type=str, required=False) +def cache(service: Optional[str]) -> None: + """Clear the environment cache directory.""" + log = logging.getLogger("env") + cache_dir = config.directories.cache + if service: + cache_dir = cache_dir / Services.get_tag(service) + log.info(f"Clearing cache directory: {cache_dir}") + files_count = len(list(cache_dir.glob("**/*"))) + if not files_count: + log.info("No files to delete") + else: + log.info(f"Deleting {files_count} files...") + shutil.rmtree(cache_dir) + log.info("Cleared") + + +@clear.command() +def temp() -> None: + """Clear the environment temp directory.""" + log = logging.getLogger("env") + log.info(f"Clearing temp directory: {config.directories.temp}") + files_count = len(list(config.directories.temp.glob("**/*"))) + if not files_count: + log.info("No files to delete") + else: + log.info(f"Deleting {files_count} files...") + shutil.rmtree(config.directories.temp) + log.info("Cleared") diff --git a/reset/packages/envied/src/envied/commands/import.py b/reset/packages/envied/src/envied/commands/import.py new file mode 100644 index 0000000..98b65a7 --- /dev/null +++ b/reset/packages/envied/src/envied/commands/import.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import click + +from envied.commands.dl import dl +from envied.core.constants import context_settings + + +class ImportCommand: + @staticmethod + @click.command( + name="import", + short_help="Reconstruct a download (download, decrypt, mux) from an --export JSON file.", + context_settings={**context_settings, "ignore_unknown_options": True}, + ) + @click.argument("export_file", type=Path) + @click.argument("dl_args", nargs=-1, type=click.UNPROCESSED) + @click.pass_context + def cli(ctx: click.Context, export_file: Path, dl_args: tuple[str, ...]) -> None: + """ + Reconstruct an exported download without re-contacting the service. + + Re-fetches the manifest, injects the stored keys, then downloads/decrypts/muxes as a + normal `dl` run. Any `dl` options after the file are forwarded verbatim: + + envied.import export.json -r HDR10 --proxy US + """ + if not export_file.is_file(): + raise click.ClickException(f"Export file not found: {export_file}") + + try: + data: dict[str, Any] = json.loads(export_file.read_text(encoding="utf8")) + except json.JSONDecodeError as e: + raise click.ClickException(f"Export file is not valid JSON: {e}") + + if data.get("version") != 2: + raise click.ClickException( + f"Unsupported export version {data.get('version')!r}. " + "Re-create the export with a current build of envied." + ) + + service_tag = data.get("service") + if not service_tag: + raise click.ClickException("Export file is missing the 'service' tag.") + + args = [*dl_args, "--import", str(export_file), service_tag] + dl.cli.main(args=args, prog_name="envied.dl", standalone_mode=False) + + +globals()["import"] = ImportCommand diff --git a/reset/packages/envied/src/envied/commands/kv.py b/reset/packages/envied/src/envied/commands/kv.py new file mode 100644 index 0000000..13bf5a1 --- /dev/null +++ b/reset/packages/envied/src/envied/commands/kv.py @@ -0,0 +1,335 @@ +import logging +import re +from pathlib import Path +from typing import Optional + +import click +from rich.padding import Padding +from rich.text import Text +from rich.tree import Tree + +from envied.core.config import config +from envied.core.console import console +from envied.core.constants import context_settings +from envied.core.services import Services +from envied.core.vault import Vault +from envied.core.vaults import Vaults + + +def load_vaults(vault_names: list[str]) -> Vaults: + """Load and validate vaults by name.""" + vaults = Vaults() + for vault_name in vault_names: + vault_config = next((x for x in config.key_vaults if x["name"] == vault_name), None) + if not vault_config: + raise click.ClickException(f"Vault ({vault_name}) is not defined in the config.") + + vault_type = vault_config["type"] + vault_args = vault_config.copy() + del vault_args["type"] + + if not vaults.load(vault_type, **vault_args): + raise click.ClickException(f"Failed to load vault ({vault_name}).") + + return vaults + + +def process_service_keys(from_vault: Vault, service: str, log: logging.Logger) -> dict[str, str]: + """Get and validate keys from a vault for a specific service.""" + content_keys = list(from_vault.get_keys(service)) + + bad_keys = {kid: key for kid, key in content_keys if not key or key.count("0") == len(key)} + for kid, key in bad_keys.items(): + log.warning(f"Skipping NULL key: {kid}:{key}") + + return {kid: key for kid, key in content_keys if kid not in bad_keys} + + +def copy_service_data(to_vault: Vault, from_vault: Vault, service: str, log: logging.Logger) -> int: + """Copy data for a single service between vaults.""" + content_keys = process_service_keys(from_vault, service, log) + total_count = len(content_keys) + + if total_count == 0: + log.info(f"{service}: No keys found in {from_vault}") + return 0 + + try: + added = to_vault.add_keys(service, content_keys) + except PermissionError: + log.warning(f"{service}: No permission to create table in {to_vault}, skipped") + return 0 + + existed = total_count - added + + if added > 0 and existed > 0: + log.info(f"{service}: {added} added, {existed} skipped ({total_count} total)") + elif added > 0: + log.info(f"{service}: {added} added ({total_count} total)") + else: + log.info(f"{service}: {existed} skipped (all existed)") + + return added + + +@click.group(short_help="Manage and configure Key Vaults.", context_settings=context_settings) +def kv() -> None: + """Manage and configure Key Vaults.""" + + +@kv.command() +@click.argument("to_vault_name", type=str) +@click.argument("from_vault_names", nargs=-1, type=click.UNPROCESSED) +@click.option("-s", "--service", type=str, default=None, help="Only copy data to and from a specific service.") +@click.option( + "-l", + "--local-only", + is_flag=True, + default=False, + help="Only copy data for services installed locally (skip vault tables for services not present in the configured services path).", +) +def copy( + to_vault_name: str, + from_vault_names: list[str], + service: Optional[str] = None, + local_only: bool = False, +) -> None: + """ + Copy data from multiple Key Vaults into a single Key Vault. + Rows with matching KIDs are skipped unless there's no KEY set. + Existing data is not deleted or altered. + + The `to_vault_name` argument is the key vault you wish to copy data to. + It should be the name of a Key Vault defined in the config. + + The `from_vault_names` argument is the key vault(s) you wish to take + data from. You may supply multiple key vaults. + """ + if not from_vault_names: + raise click.ClickException("No Vaults were specified to copy data from.") + + log = logging.getLogger("kv") + + all_vault_names = [to_vault_name] + list(from_vault_names) + vaults = load_vaults(all_vault_names) + + to_vault = vaults.vaults[0] + from_vaults = vaults.vaults[1:] + + vault_names = ", ".join([v.name for v in from_vaults]) + log.info(f"Copying data from {vault_names} → {to_vault.name}") + + if service and local_only: + raise click.UsageError("--service and --local-only are mutually exclusive.") + + if service: + service = Services.get_tag(service) + log.info(f"Filtering by service: {service}") + + installed: Optional[set[str]] = None + if local_only: + installed = {t.upper() for t in Services.get_tags()} + log.info(f"Filtering by locally installed services ({len(installed)} found)") + + total_added = 0 + for from_vault in from_vaults: + services_to_copy = [service] if service else list(from_vault.get_services()) + + if installed is not None: + before = len(services_to_copy) + services_to_copy = [s for s in services_to_copy if s and s.upper() in installed] + skipped = before - len(services_to_copy) + if skipped: + log.info(f"{from_vault.name}: skipping {skipped} service(s) not installed locally") + + for service_tag in services_to_copy: + added = copy_service_data(to_vault, from_vault, service_tag, log) + total_added += added + + if total_added > 0: + log.info(f"Successfully added {total_added} new keys to {to_vault}") + else: + log.info("Copy completed - no new keys to add") + + +@kv.command() +@click.argument("vaults", nargs=-1, type=click.UNPROCESSED) +@click.option("-s", "--service", type=str, default=None, help="Only sync data to and from a specific service.") +@click.option( + "-l", + "--local-only", + is_flag=True, + default=False, + help="Only sync data for services installed locally (skip vault tables for services not present in the configured services path).", +) +@click.pass_context +def sync( + ctx: click.Context, + vaults: list[str], + service: Optional[str] = None, + local_only: bool = False, +) -> None: + """ + Ensure multiple Key Vaults copies of all keys as each other. + It's essentially just a bi-way copy between each vault. + To see the precise details of what it's doing between each + provided vault, see the documentation for the `copy` command. + """ + if not len(vaults) > 1: + raise click.ClickException("You must provide more than one Vault to sync.") + + ctx.invoke( + copy, + to_vault_name=vaults[0], + from_vault_names=vaults[1:], + service=service, + local_only=local_only, + ) + for i in range(1, len(vaults)): + ctx.invoke( + copy, + to_vault_name=vaults[i], + from_vault_names=[vaults[i - 1]], + service=service, + local_only=local_only, + ) + + +@kv.command() +@click.argument("file", type=Path) +@click.argument("service", type=str) +@click.argument("vaults", nargs=-1, type=click.UNPROCESSED) +def add(file: Path, service: str, vaults: list[str]) -> None: + """ + Add new Content Keys to Key Vault(s) by service. + + File should contain one key per line in the format KID:KEY (HEX:HEX). + Each line should have nothing else within it except for the KID:KEY. + Encoding is presumed to be UTF8. + """ + if not file.exists(): + raise click.ClickException(f"File provided ({file}) does not exist.") + if not file.is_file(): + raise click.ClickException(f"File provided ({file}) is not a file.") + if not service or not isinstance(service, str): + raise click.ClickException(f"Service provided ({service}) is invalid.") + if len(vaults) < 1: + raise click.ClickException("You must provide at least one Vault.") + + log = logging.getLogger("kv") + service = Services.get_tag(service) + + vaults_ = load_vaults(list(vaults)) + + data = file.read_text(encoding="utf8") + kid_keys: dict[str, str] = {} + for line in data.splitlines(keepends=False): + line = line.strip() + match = re.search(r"^(?P[0-9a-fA-F]{32}):(?P[0-9a-fA-F]{32})$", line) + if not match: + continue + kid = match.group("kid").lower() + key = match.group("key").lower() + kid_keys[kid] = key + + total_count = len(kid_keys) + + for vault in vaults_: + log.info(f"Adding {total_count} Content Keys to {vault}") + added_count = vault.add_keys(service, kid_keys) + existed_count = total_count - added_count + log.info(f"{vault}: {added_count} newly added, {existed_count} already existed (skipped)") + + log.info("Done!") + + +@kv.command() +@click.argument("kid", type=str) +@click.option("-s", "--service", type=str, default=None, help="Limit search to a specific service tag.") +@click.option( + "-v", "--vault", "vault_name", type=str, default=None, help="Limit search to a specific configured vault by name." +) +def search(kid: str, service: Optional[str], vault_name: Optional[str]) -> None: + """ + Search configured Key Vault(s) for a KID and report any matching KEY. + + KID must be 32 hex characters (no dashes). If --service is omitted, every + service table in each vault is scanned. If --vault is omitted, every + vault in the config is searched. + """ + log = logging.getLogger("kv") + + kid_norm = kid.replace("-", "").lower() + if not re.fullmatch(r"[0-9a-f]{32}", kid_norm): + raise click.ClickException(f"KID '{kid}' is not 32 hex characters.") + + if vault_name: + vault_names = [vault_name] + else: + vault_names = [v["name"] for v in config.key_vaults] + if not vault_names: + raise click.ClickException("No Key Vaults are configured.") + + vaults_ = load_vaults(vault_names) + + service_tag = Services.get_tag(service) if service else None + + hit: Optional[tuple[str, str, str]] = None + for vault in vaults_: + if service_tag: + services_to_check: list[str] = [service_tag] + else: + try: + services_to_check = list(vault.get_services()) + except Exception as e: + log.debug(f"{vault}: get_services() failed ({e})") + services_to_check = [] + if not services_to_check: + log.warning(f"{vault}: cannot search without a service (remote vault requires --service). Skipping.") + continue + + for svc in services_to_check: + try: + key = vault.get_key(kid_norm, svc) + except Exception as e: + log.debug(f"{vault} [{svc}]: lookup error ({e})") + continue + if key and key.count("0") != len(key): + hit = (vault.name, svc, key) + break + if hit: + break + + if hit: + vname, svc, key = hit + tree = Tree(Text.assemble((svc, "cyan"), (f"({vname})", "text"), overflow="fold")) + tree.add(f"[text2]{kid_norm}:{key}") + console.print(Padding(tree, (1, 5))) + else: + log.info(f"KID {kid_norm} not found in {len(vaults_)} vault(s).") + + +@kv.command() +@click.argument("vaults", nargs=-1, type=click.UNPROCESSED) +def prepare(vaults: list[str]) -> None: + """Create Service Tables on Vaults if not yet created.""" + log = logging.getLogger("kv") + + vaults_ = load_vaults(vaults) + + for vault in vaults_: + if hasattr(vault, "has_table") and hasattr(vault, "create_table"): + for service_tag in Services.get_tags(): + if vault.has_table(service_tag): + log.info(f"{vault} already has a {service_tag} Table") + else: + try: + vault.create_table(service_tag, commit=True) + log.info(f"{vault}: Created {service_tag} Table") + except PermissionError: + log.error(f"{vault} user has no create table permission, skipping...") + continue + else: + log.info(f"{vault} does not use tables, skipping...") + + log.info("Done!") diff --git a/reset/packages/envied/src/envied/commands/prd.py b/reset/packages/envied/src/envied/commands/prd.py new file mode 100644 index 0000000..d9d6883 --- /dev/null +++ b/reset/packages/envied/src/envied/commands/prd.py @@ -0,0 +1,271 @@ +import logging +from pathlib import Path +from typing import Optional + +import click +import requests +from Crypto.Random import get_random_bytes +from pyplayready import InvalidCertificateChain, OutdatedDevice +from pyplayready.cdm import Cdm +from pyplayready.crypto.ecc_key import ECCKey +from pyplayready.device import Device +from pyplayready.system.bcert import Certificate, CertificateChain +from pyplayready.system.pssh import PSSH + +from envied.core.config import config +from envied.core.constants import context_settings + + +@click.group( + short_help="Manage creation of PRD (Playready Device) files.", + context_settings=context_settings, +) +def prd() -> None: + """Manage creation of PRD (Playready Device) files.""" + + +@prd.command() +@click.argument("paths", type=Path, nargs=-1) +@click.option( + "-e", + "--encryption_key", + type=Path, + required=False, + help="Optional Device ECC private encryption key", +) +@click.option( + "-s", + "--signing_key", + type=Path, + required=False, + help="Optional Device ECC private signing key", +) +@click.option("-o", "--output", type=Path, default=None, help="Output Directory") +@click.pass_context +def new( + ctx: click.Context, + paths: tuple[Path, ...], + encryption_key: Optional[Path], + signing_key: Optional[Path], + output: Optional[Path], +) -> None: + """Create a new .PRD PlayReady Device file. + + Accepts either paths to a group key and certificate or a single directory + containing ``zgpriv.dat`` and ``bgroupcert.dat``. + """ + if len(paths) == 1 and paths[0].is_dir(): + device_dir = paths[0] + group_key = device_dir / "zgpriv.dat" + group_certificate = device_dir / "bgroupcert.dat" + if not group_key.is_file() or not group_certificate.is_file(): + raise click.UsageError("Folder must contain zgpriv.dat and bgroupcert.dat", ctx) + elif len(paths) == 2: + group_key, group_certificate = paths + if not group_key.is_file(): + raise click.UsageError("group_key: Not a path to a file, or it doesn't exist.", ctx) + if not group_certificate.is_file(): + raise click.UsageError("group_certificate: Not a path to a file, or it doesn't exist.", ctx) + device_dir = None + else: + raise click.UsageError( + "Provide either a folder path or paths to group_key and group_certificate", + ctx, + ) + if encryption_key and not encryption_key.is_file(): + raise click.UsageError("encryption_key: Not a path to a file, or it doesn't exist.", ctx) + if signing_key and not signing_key.is_file(): + raise click.UsageError("signing_key: Not a path to a file, or it doesn't exist.", ctx) + + log = logging.getLogger("prd") + + encryption_key_obj = ECCKey.load(encryption_key) if encryption_key else ECCKey.generate() + signing_key_obj = ECCKey.load(signing_key) if signing_key else ECCKey.generate() + + group_key_obj = ECCKey.load(group_key) + certificate_chain = CertificateChain.load(group_certificate) + + if certificate_chain.get(0).get_issuer_key() != group_key_obj.public_bytes(): + raise InvalidCertificateChain("Group key does not match this certificate") + + new_certificate = Certificate.new_leaf_cert( + cert_id=get_random_bytes(16), + security_level=certificate_chain.get_security_level(), + client_id=get_random_bytes(16), + signing_key=signing_key_obj, + encryption_key=encryption_key_obj, + group_key=group_key_obj, + parent=certificate_chain, + ) + certificate_chain.prepend(new_certificate) + certificate_chain.verify() + + device = Device( + group_key=group_key_obj.dumps(), + encryption_key=encryption_key_obj.dumps(), + signing_key=signing_key_obj.dumps(), + group_certificate=certificate_chain.dumps(), + ) + + if output and output.suffix: + if output.suffix.lower() != ".prd": + log.warning( + "Saving PRD with the file extension '%s' but '.prd' is recommended.", + output.suffix, + ) + out_path = output + else: + out_dir = output or (device_dir or config.directories.prds) + out_path = out_dir / f"{device.get_name()}.prd" + + if out_path.exists(): + log.error("A file already exists at the path '%s', cannot overwrite.", out_path) + return + + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_bytes(device.dumps()) + + log.info("Created Playready Device (.prd) file, %s", out_path.name) + log.info(" + Security Level: %s", device.security_level) + log.info(" + Group Key: %s bytes", len(device.group_key.dumps())) + log.info(" + Encryption Key: %s bytes", len(device.encryption_key.dumps())) + log.info(" + Signing Key: %s bytes", len(device.signing_key.dumps())) + log.info(" + Group Certificate: %s bytes", len(device.group_certificate.dumps())) + log.info(" + Saved to: %s", out_path.absolute()) + + +@prd.command(name="reprovision") +@click.argument("prd_path", type=Path) +@click.option( + "-e", + "--encryption_key", + type=Path, + required=False, + help="Optional Device ECC private encryption key", +) +@click.option( + "-s", + "--signing_key", + type=Path, + required=False, + help="Optional Device ECC private signing key", +) +@click.option("-o", "--output", type=Path, default=None, help="Output Path or Directory") +@click.pass_context +def reprovision_device( + ctx: click.Context, + prd_path: Path, + encryption_key: Optional[Path], + signing_key: Optional[Path], + output: Optional[Path] = None, +) -> None: + """Reprovision a Playready Device (.prd) file.""" + if not prd_path.is_file(): + raise click.UsageError("prd_path: Not a path to a file, or it doesn't exist.", ctx) + + log = logging.getLogger("prd") + log.info("Reprovisioning Playready Device (.prd) file, %s", prd_path.name) + + device = Device.load(prd_path) + + if device.group_key is None: + raise OutdatedDevice( + "Device does not support reprovisioning, re-create it or use a Device with a version of 3 or higher" + ) + + device.group_certificate.remove(0) + + encryption_key_obj = ECCKey.load(encryption_key) if encryption_key else ECCKey.generate() + signing_key_obj = ECCKey.load(signing_key) if signing_key else ECCKey.generate() + + device.encryption_key = encryption_key_obj + device.signing_key = signing_key_obj + + new_certificate = Certificate.new_leaf_cert( + cert_id=get_random_bytes(16), + security_level=device.group_certificate.get_security_level(), + client_id=get_random_bytes(16), + signing_key=signing_key_obj, + encryption_key=encryption_key_obj, + group_key=device.group_key, + parent=device.group_certificate, + ) + device.group_certificate.prepend(new_certificate) + + if output and output.suffix: + if output.suffix.lower() != ".prd": + log.warning( + "Saving PRD with the file extension '%s' but '.prd' is recommended.", + output.suffix, + ) + out_path = output + else: + out_path = prd_path + + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_bytes(device.dumps()) + + log.info("Reprovisioned Playready Device (.prd) file, %s", out_path.name) + + +@prd.command() +@click.argument("device", type=Path) +@click.option( + "-c", + "--ckt", + type=click.Choice(["aesctr", "aescbc"], case_sensitive=False), + default="aesctr", + help="Content Key Encryption Type", +) +@click.option( + "-sl", + "--security-level", + type=click.Choice(["150", "2000", "3000"], case_sensitive=False), + default="2000", + help="Minimum Security Level", +) +@click.pass_context +def test( + ctx: click.Context, + device: Path, + ckt: str, + security_level: str, +) -> None: + """Test a Playready Device on the Microsoft demo server.""" + + if not device.is_file(): + raise click.UsageError("device: Not a path to a file, or it doesn't exist.", ctx) + + log = logging.getLogger("prd") + + prd_device = Device.load(device) + log.info("Loaded Device: %s", prd_device.get_name()) + + cdm = Cdm.from_device(prd_device) + log.info("Loaded CDM") + + session_id = cdm.open() + log.info("Opened Session") + + pssh_b64 = "AAADfHBzc2gAAAAAmgTweZhAQoarkuZb4IhflQAAA1xcAwAAAQABAFIDPABXAFIATQBIAEUAQQBEAEUAUgAgAHgAbQBsAG4AcwA9ACIAaAB0AHQAcAA6AC8ALwBzAGMAaABlAG0AYQBzAC4AbQBpAGMAcgBvAHMAbwBmAHQALgBjAG8AbQAvAEQAUgBNAC8AMgAwADAANwAvADAAMwAvAFAAbABhAHkAUgBlAGEAZAB5AEgAZQBhAGQAZQByACIAIAB2AGUAcgBzAGkAbwBuAD0AIgA0AC4AMAAuADAALgAwACIAPgA8AEQAQQBUAEEAPgA8AFAAUgBPAFQARQBDAFQASQBOAEYATwA+ADwASwBFAFkATABFAE4APgAxADYAPAAvAEsARQBZAEwARQBOAD4APABBAEwARwBJAEQAPgBBAEUAUwBDAFQAUgA8AC8AQQBMAEcASQBEAD4APAAvAFAAUgBPAFQARQBDAFQASQBOAEYATwA+ADwASwBJAEQAPgA0AFIAcABsAGIAKwBUAGIATgBFAFMAOAB0AEcAawBOAEYAVwBUAEUASABBAD0APQA8AC8ASwBJAEQAPgA8AEMASABFAEMASwBTAFUATQA+AEsATABqADMAUQB6AFEAUAAvAE4AQQA9ADwALwBDAEgARQBDAEsAUwBVAE0APgA8AEwAQQBfAFUAUgBMAD4AaAB0AHQAcABzADoALwAvAHAAcgBvAGYAZgBpAGMAaQBhAGwAcwBpAHQAZQAuAGsAZQB5AGQAZQBsAGkAdgBlAHIAeQAuAG0AZQBkAGkAYQBzAGUAcgB2AGkAYwBlAHMALgB3AGkAbgBkAG8AdwBzAC4AbgBlAHQALwBQAGwAYQB5AFIAZQBhAGQAeQAvADwALwBMAEEAXwBVAFIATAA+ADwAQwBVAFMAVABPAE0AQQBUAFQAUgBJAEIAVQBUAEUAUwA+ADwASQBJAFMAXwBEAFIATQBfAFYARQBSAFMASQBPAE4APgA4AC4AMQAuADIAMwAwADQALgAzADEAPAAvAEkASQBTAF8ARABSAE0AXwBWAEUAUgBTAEkATwBOAD4APAAvAEMAVQBTAFQATwBNAEEAVABUAFIASQBCAFUAVABFAFMAPgA8AC8ARABBAFQAQQA+ADwALwBXAFIATQBIAEUAQQBEAEUAUgA+AA==" + pssh = PSSH(pssh_b64) + + challenge = cdm.get_license_challenge(session_id, pssh.wrm_headers[0]) + log.info("Created License Request") + + license_server = f"https://test.playready.microsoft.com/service/rightsmanager.asmx?cfg=(persist:false,sl:{security_level},ckt:{ckt})" + + response = requests.post( + url=license_server, + headers={"Content-Type": "text/xml; charset=UTF-8"}, + data=challenge, + ) + + cdm.parse_license(session_id, response.text) + log.info("License Parsed Successfully") + + for key in cdm.get_keys(session_id): + log.info(f"{key.key_id.hex}:{key.key.hex()}") + + cdm.close(session_id) + log.info("Closed Session") diff --git a/reset/packages/envied/src/envied/commands/search.py b/reset/packages/envied/src/envied/commands/search.py new file mode 100644 index 0000000..c687125 --- /dev/null +++ b/reset/packages/envied/src/envied/commands/search.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import logging +import re +import sys +from typing import Any, Optional + +import click +import yaml +from rich.padding import Padding +from rich.rule import Rule +from rich.tree import Tree + +from envied.commands.dl import dl +from envied.core import binaries +from envied.core.config import config +from envied.core.console import console +from envied.core.constants import context_settings +from envied.core.proxies import Basic, Gluetun, Hola, NordVPN, SurfsharkVPN, WindscribeVPN +from envied.core.service import Service +from envied.core.services import Services +from envied.core.utils.click_types import ContextData +from envied.core.utils.collections import merge_dict + + +@click.command( + short_help="Search for titles from a Service.", + cls=Services, + context_settings=dict(**context_settings, token_normalize_func=Services.get_tag), +) +@click.option( + "-p", "--profile", type=str, default=None, help="Profile to use for Credentials and Cookies (if available)." +) +@click.option( + "--proxy", + type=str, + default=None, + help="Proxy URI to use. If a 2-letter country is provided, it will try get a proxy from the config.", +) +@click.option("--no-proxy", is_flag=True, default=False, help="Force disable all proxy use.") +@click.pass_context +def search(ctx: click.Context, no_proxy: bool, profile: Optional[str] = None, proxy: Optional[str] = None): + if not ctx.invoked_subcommand: + raise ValueError("A subcommand to invoke was not specified, the main code cannot continue.") + + log = logging.getLogger("search") + + service = Services.get_tag(ctx.invoked_subcommand) + profile = profile + + if profile: + log.info(f"Using profile: '{profile}'") + + with console.status("Loading Service Config...", spinner="dots"): + service_config_path = Services.get_path(service) / config.filenames.config + if service_config_path.exists(): + service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) + log.info("Service Config loaded") + else: + service_config = {} + merge_dict(config.services.get(service), service_config) + + proxy_providers = [] + if no_proxy: + ctx.params["proxy"] = None + else: + with console.status("Loading Proxy Providers...", spinner="dots"): + if config.proxy_providers.get("basic"): + proxy_providers.append(Basic(**config.proxy_providers["basic"])) + if config.proxy_providers.get("nordvpn"): + proxy_providers.append(NordVPN(**config.proxy_providers["nordvpn"])) + if config.proxy_providers.get("surfsharkvpn"): + proxy_providers.append(SurfsharkVPN(**config.proxy_providers["surfsharkvpn"])) + if config.proxy_providers.get("windscribevpn"): + proxy_providers.append(WindscribeVPN(**config.proxy_providers["windscribevpn"])) + if config.proxy_providers.get("gluetun"): + proxy_providers.append(Gluetun(**config.proxy_providers["gluetun"])) + if binaries.HolaProxy: + proxy_providers.append(Hola()) + for proxy_provider in proxy_providers: + log.info(f"Loaded {proxy_provider.__class__.__name__}: {proxy_provider}") + + if proxy: + requested_provider = None + if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE): + # requesting proxy from a specific proxy provider + requested_provider, proxy = proxy.split(":", maxsplit=1) + # Match simple region codes (us, ca, uk1) or provider:region format (nordvpn:ca, windscribe:us) + if re.match(r"^[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE) or re.match( + r"^[a-z]+:[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE + ): + proxy = proxy.lower() + with console.status(f"Getting a Proxy to {proxy}...", spinner="dots"): + if requested_provider: + proxy_provider = next( + (x for x in proxy_providers if x.__class__.__name__.lower() == requested_provider), None + ) + if not proxy_provider: + log.error(f"The proxy provider '{requested_provider}' was not recognised.") + sys.exit(1) + proxy_uri = proxy_provider.get_proxy(proxy) + if not proxy_uri: + log.error(f"The proxy provider {requested_provider} had no proxy for {proxy}") + sys.exit(1) + proxy = ctx.params["proxy"] = proxy_uri + log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") + else: + for proxy_provider in proxy_providers: + proxy_uri = proxy_provider.get_proxy(proxy) + if proxy_uri: + proxy = ctx.params["proxy"] = proxy_uri + log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}") + break + else: + log.info(f"Using explicit Proxy: {proxy}") + + ctx.obj = ContextData(config=service_config, cdm=None, proxy_providers=proxy_providers, profile=profile) + + +@search.result_callback() +def result(service: Service, profile: Optional[str] = None, **_: Any) -> None: + log = logging.getLogger("search") + + service_tag = service.__class__.__name__ + + with console.status("Authenticating with Service...", spinner="dots"): + cookies = dl.get_cookie_jar(service_tag, profile) + credential = dl.get_credentials(service_tag, profile) + service.authenticate(cookies, credential) + if cookies or credential: + log.info("Authenticated with Service") + + search_results = Tree("Search Results", hide_root=True) + with console.status("Searching...", spinner="dots"): + for result in service.search(): + result_text = f"[bold text]{result.title}[/]" + if result.url: + result_text = f"[link={result.url}]{result_text}[/link]" + + if result.label: + result_text += f" [pink]{result.label}[/]" + if result.description: + result_text += f"\n[text2]{result.description}[/]" + result_text += f"\n[bright_black]id: {result.id}[/]" + search_results.add(result_text + "\n") + + # update cookie + cookie_file = dl.get_cookie_path(service_tag, profile) + if cookie_file: + dl.save_cookies(cookie_file, service.session.cookies) + + console.print(Padding(Rule(f"[rule.text]{len(search_results.children)} Search Results"), (1, 2))) + + if search_results.children: + console.print(Padding(search_results, (0, 5))) + else: + console.print( + Padding("[bold text]No matches[/]\n[bright_black]Please check spelling and search again....[/]", (0, 5)) + ) diff --git a/reset/packages/envied/src/envied/commands/serve.py b/reset/packages/envied/src/envied/commands/serve.py new file mode 100644 index 0000000..6ca5b0d --- /dev/null +++ b/reset/packages/envied/src/envied/commands/serve.py @@ -0,0 +1,327 @@ +import logging +import subprocess + +import click +from aiohttp import web + +from envied.core import binaries +from envied.core.api import cors_middleware, setup_routes, setup_swagger +from envied.core.api.compression import compression_middleware +from envied.core.config import config +from envied.core.constants import context_settings + + +@click.command( + short_help="Serve your Local Widevine/PlayReady Devices and REST API for Remote Access.", + context_settings=context_settings, +) +@click.option("-h", "--host", type=str, default="127.0.0.1", help="Host to serve from.") +@click.option("-p", "--port", type=int, default=8786, help="Port to serve from.") +@click.option("--caddy", is_flag=True, default=False, help="Also serve with Caddy.") +@click.option( + "--api-only", is_flag=True, default=False, help="Serve only the REST API, not pywidevine/pyplayready CDM." +) +@click.option("--no-widevine", is_flag=True, default=False, help="Disable Widevine CDM endpoints.") +@click.option("--no-playready", is_flag=True, default=False, help="Disable PlayReady CDM endpoints.") +@click.option("--no-key", is_flag=True, default=False, help="Disable API key authentication (allows all requests).") +@click.option( + "--debug-api", + is_flag=True, + default=False, + help="Include technical debug information (tracebacks, stderr) in API error responses.", +) +@click.option( + "--debug", + is_flag=True, + default=False, + help="Enable debug logging for API operations.", +) +@click.option( + "--remote-only", + is_flag=True, + default=False, + help="Only expose remote service session endpoints (health, services, search, session).", +) +def serve( + host: str, + port: int, + caddy: bool, + api_only: bool, + no_widevine: bool, + no_playready: bool, + no_key: bool, + debug_api: bool, + debug: bool, + remote_only: bool, +) -> None: + """ + Serve your Local Widevine and PlayReady Devices and REST API for Remote Access. + + \b + CDM ENDPOINTS: + - Widevine: /{device}/open, /{device}/close/{session_id}, etc. + - PlayReady: /playready/{device}/open, /playready/{device}/close/{session_id}, etc. + + \b + You may serve with Caddy at the same time with --caddy. You can use Caddy + as a reverse-proxy to serve with HTTPS. The config used will be the Caddyfile + next to the envied.config. + + \b + DEVICE CONFIGURATION: + WVD files are auto-loaded from the WVDs directory, PRD files from the PRDs directory. + Configure user access in envied.yaml: + + \b + serve: + api_secret: "your-api-secret" + users: + your-secret-key: + devices: ["device_name"] # Widevine devices + playready_devices: ["device_name"] # PlayReady devices + username: user + """ + from pyplayready.remote import serve as pyplayready_serve + from pywidevine import serve as pywidevine_serve + + log = logging.getLogger("serve") + + if debug: + logging.basicConfig(level=logging.DEBUG, format="%(name)s - %(levelname)s - %(message)s") + log.info("Debug logging enabled for API operations") + else: + logging.getLogger("api").setLevel(logging.WARNING) + logging.getLogger("api.remote").setLevel(logging.WARNING) + + if not no_key: + api_secret = config.serve.get("api_secret") + if not api_secret: + raise click.ClickException( + "API secret key is not configured. Please add 'api_secret' to the 'serve' section in your config." + ) + else: + api_secret = None + log.warning("Running with --no-key: Authentication is DISABLED for all API endpoints!") + + if debug_api: + log.warning("Running with --debug-api: Error responses will include technical debug information!") + + if api_only and (no_widevine or no_playready): + raise click.ClickException("Cannot use --api-only with --no-widevine or --no-playready.") + + if caddy: + if not binaries.Caddy: + raise click.ClickException('Caddy executable "caddy" not found but is required for --caddy.') + caddy_p = subprocess.Popen( + [binaries.Caddy, "run", "--config", str(config.directories.user_configs / "Caddyfile")] + ) + else: + caddy_p = None + + try: + if not config.serve.get("devices"): + config.serve["devices"] = [] + config.serve["devices"].extend(list(config.directories.wvds.glob("*.wvd"))) + + if not config.serve.get("playready_devices"): + config.serve["playready_devices"] = [] + config.serve["playready_devices"].extend(list(config.directories.prds.glob("*.prd"))) + + @web.middleware + async def api_key_authentication(request: web.Request, handler) -> web.Response: + """Authenticate API requests using X-Secret-Key header.""" + if request.path == "/api/health": + return await handler(request) + secret_key = request.headers.get("X-Secret-Key") + if not secret_key: + return web.json_response({"status": 401, "message": "Secret Key is Empty."}, status=401) + if secret_key not in request.app["config"]["users"]: + return web.json_response({"status": 401, "message": "Secret Key is Invalid."}, status=401) + return await handler(request) + + remote_only = remote_only or config.serve.get("remote_only", False) + if remote_only: + api_only = True + + global_services = config.serve.get("services") + if global_services: + log.info(f"Global service allowlist: {', '.join(global_services)}") + users = config.serve.get("users", {}) + for user_key, user_cfg in users.items() if isinstance(users, dict) else []: + user_services = user_cfg.get("services") if isinstance(user_cfg, dict) else None + if user_services: + username = user_cfg.get("username", user_key[:8] + "...") + log.info(f"User '{username}' restricted to services: {', '.join(user_services)}") + + if api_only: + log.info("Starting REST API server (pywidevine/pyplayready CDM disabled)") + if no_key: + app = web.Application(middlewares=[cors_middleware, compression_middleware]) + app["config"] = {"users": {}} + else: + app = web.Application(middlewares=[cors_middleware, api_key_authentication, compression_middleware]) + app["config"] = {"users": {api_secret: {"devices": [], "username": "api_user"}}} + app["debug_api"] = debug_api + + # Start session cleanup loop for remote-dl sessions + from envied.core.api.session_store import get_session_store + + session_store = get_session_store() + + async def start_session_cleanup(_app: web.Application) -> None: + await session_store.start_cleanup_loop() + + async def stop_session_cleanup(_app: web.Application) -> None: + await session_store.cancel_all_bridges() + await session_store.stop_cleanup_loop() + + app.on_startup.append(start_session_cleanup) + app.on_cleanup.append(stop_session_cleanup) + + setup_routes(app, remote_only=remote_only) + if not remote_only: + setup_swagger(app) + log.info(f"REST API endpoints available at http://{host}:{port}/api/") + log.info(f"Swagger UI available at http://{host}:{port}/api/docs/") + else: + log.info(f"Remote service endpoints available at http://{host}:{port}/api/session/") + log.info("(Press CTRL+C to quit)") + web.run_app(app, host=host, port=port, print=None) + else: + serve_widevine = not no_widevine + serve_playready = not no_playready + + serve_config = dict(config.serve) + wvd_devices = serve_config.get("devices", []) if serve_widevine else [] + prd_devices = serve_config.get("playready_devices", []) if serve_playready else [] + + cdm_parts = [] + if serve_widevine: + cdm_parts.append("pywidevine CDM") + if serve_playready: + cdm_parts.append("pyplayready CDM") + log.info(f"Starting integrated server ({' + '.join(cdm_parts)} + REST API)") + + wvd_device_names = [d.stem if hasattr(d, "stem") else str(d) for d in wvd_devices] + prd_device_names = [d.stem if hasattr(d, "stem") else str(d) for d in prd_devices] + + if not serve_config.get("users") or not isinstance(serve_config["users"], dict): + serve_config["users"] = {} + + if not no_key and api_secret not in serve_config["users"]: + serve_config["users"][api_secret] = { + "devices": wvd_device_names, + "playready_devices": prd_device_names, + "username": "api_user", + } + + for user_key, user_config in serve_config["users"].items(): + if "playready_devices" not in user_config: + # Require explicit PlayReady device access per user (default: no access). + user_config["playready_devices"] = [] + log.warning( + f'User "{user_key}" has no "playready_devices" configured; PlayReady access disabled for this user. ' + f"Available PlayReady devices: {prd_device_names}" + ) + + def create_serve_authentication(serve_playready_flag: bool): + @web.middleware + async def serve_authentication(request: web.Request, handler) -> web.Response: + if serve_playready_flag and request.path in ("/playready", "/playready/"): + response = await handler(request) + else: + response = await pywidevine_serve.authentication(request, handler) + + if serve_playready_flag and request.path.startswith("/playready"): + from pyplayready import __version__ as pyplayready_version + + response.headers["Server"] = ( + f"https://git.gay/ready-dl/pyplayready serve v{pyplayready_version}" + ) + + return response + + return serve_authentication + + if no_key: + app = web.Application(middlewares=[cors_middleware, compression_middleware]) + else: + serve_auth = create_serve_authentication(serve_playready and bool(prd_devices)) + app = web.Application(middlewares=[cors_middleware, serve_auth, compression_middleware]) + + app["config"] = serve_config + app["debug_api"] = debug_api + + # Start session cleanup loop for remote-dl sessions + from envied.core.api.session_store import get_session_store + + session_store = get_session_store() + + async def start_session_cleanup(_app: web.Application) -> None: + await session_store.start_cleanup_loop() + + async def stop_session_cleanup(_app: web.Application) -> None: + await session_store.cancel_all_bridges() + await session_store.stop_cleanup_loop() + + app.on_startup.append(start_session_cleanup) + app.on_cleanup.append(stop_session_cleanup) + + if serve_widevine: + app.on_startup.append(pywidevine_serve._startup) + app.on_cleanup.append(pywidevine_serve._cleanup) + app.add_routes(pywidevine_serve.routes) + + if serve_playready and prd_devices: + if no_key: + playready_app = web.Application() + else: + playready_app = web.Application(middlewares=[pyplayready_serve.authentication]) + + # PlayReady subapp config maps playready_devices to "devices" for pyplayready compatibility + playready_config = { + "devices": prd_devices, + "users": { + user_key: { + "devices": user_cfg.get("playready_devices", []), + "username": user_cfg.get("username", "user"), + } + for user_key, user_cfg in serve_config["users"].items() + } + if not no_key + else {}, + } + playready_app["config"] = playready_config + playready_app.on_startup.append(pyplayready_serve._startup) + playready_app.on_cleanup.append(pyplayready_serve._cleanup) + playready_app.add_routes(pyplayready_serve.routes) + + async def playready_ping(_: web.Request) -> web.Response: + from pyplayready import __version__ as pyplayready_version + + response = web.json_response({"message": "OK"}) + response.headers["Server"] = f"https://git.gay/ready-dl/pyplayready serve v{pyplayready_version}" + return response + + app.router.add_route("*", "/playready", playready_ping) + + app.add_subapp("/playready", playready_app) + log.info(f"PlayReady CDM endpoints available at http://{host}:{port}/playready/") + elif serve_playready: + log.info("No PlayReady devices found, skipping PlayReady CDM endpoints") + + setup_routes(app, remote_only=remote_only) + + if serve_widevine: + log.info(f"Widevine CDM endpoints available at http://{host}:{port}/{{device}}/open") + if remote_only: + log.info(f"Remote service endpoints available at http://{host}:{port}/api/session/") + else: + setup_swagger(app) + log.info(f"REST API endpoints available at http://{host}:{port}/api/") + log.info(f"Swagger UI available at http://{host}:{port}/api/docs/") + log.info("(Press CTRL+C to quit)") + web.run_app(app, host=host, port=port, print=None) + finally: + if caddy_p: + caddy_p.kill() diff --git a/reset/packages/envied/src/envied/commands/util.py b/reset/packages/envied/src/envied/commands/util.py new file mode 100644 index 0000000..47f9d59 --- /dev/null +++ b/reset/packages/envied/src/envied/commands/util.py @@ -0,0 +1,275 @@ +import re +import subprocess +from pathlib import Path + +import click +from pymediainfo import MediaInfo + +from envied.core import binaries +from envied.core.constants import context_settings + + +def _natural_sort_key(path: Path) -> list: + """Sort key for natural sorting (S01E01 before S01E10).""" + return [int(part) if part.isdigit() else part.lower() for part in re.split(r"(\d+)", path.name)] + + +@click.group(short_help="Various helper scripts and programs.", context_settings=context_settings) +def util() -> None: + """Various helper scripts and programs.""" + + +@util.command() +@click.argument("path", type=Path) +@click.argument("aspect", type=str) +@click.option( + "--letter/--pillar", + default=True, + help="Specify which direction to crop. Top and Bottom would be --letter, Sides would be --pillar.", +) +@click.option("-o", "--offset", type=int, default=0, help="Fine tune the computed crop area if not perfectly centered.") +@click.option( + "-p", + "--preview", + is_flag=True, + default=False, + help="Instantly preview the newly-set aspect crop in MPV (or ffplay if mpv is unavailable).", +) +def crop(path: Path, aspect: str, letter: bool, offset: int, preview: bool) -> None: + """ + Losslessly crop H.264 and H.265 video files at the bit-stream level. + You may provide a path to a file, or a folder of mkv and/or mp4 files. + + Note: If you notice that the values you put in are not quite working, try + tune -o/--offset. This may be necessary on videos with sub-sampled chroma. + + Do note that you may not get an ideal lossless cropping result on some + cases, again due to sub-sampled chroma. + + It's recommended that you try -o about 10 or so pixels and lower it until + you get as close in as possible. Do make sure it's not over-cropping either + as it may go from being 2px away from a perfect crop, to 20px over-cropping + again due to sub-sampled chroma. + """ + if not binaries.FFMPEG: + raise click.ClickException('FFmpeg executable "ffmpeg" not found but is required.') + + if path.is_dir(): + paths = sorted(list(path.glob("*.mkv")) + list(path.glob("*.mp4")), key=_natural_sort_key) + else: + paths = [path] + for video_path in paths: + try: + video_track = next(iter(MediaInfo.parse(video_path).video_tracks or [])) + except StopIteration: + raise click.ClickException("There's no video tracks in the provided file.") + + crop_filter = {"HEVC": "hevc_metadata", "AVC": "h264_metadata"}.get(video_track.commercial_name) + if not crop_filter: + raise click.ClickException(f"{video_track.commercial_name} Codec not supported.") + + aspect_w, aspect_h = list(map(float, aspect.split(":"))) + if letter: + crop_value = (video_track.height - (video_track.width / (aspect_w * aspect_h))) / 2 + left, top, right, bottom = map(int, [0, crop_value + offset, 0, crop_value - offset]) + else: + crop_value = (video_track.width - (video_track.height * (aspect_w / aspect_h))) / 2 + left, top, right, bottom = map(int, [crop_value + offset, 0, crop_value - offset, 0]) + crop_filter += f"=crop_left={left}:crop_top={top}:crop_right={right}:crop_bottom={bottom}" + + if min(left, top, right, bottom) < 0: + raise click.ClickException("Cannot crop less than 0, are you cropping in the right direction?") + + if preview: + out_path = ["-f", "mpegts", "-"] # pipe + else: + out_path = [ + str( + video_path.with_name( + ".".join( + filter( + bool, + [ + video_path.stem, + video_track.language, + "crop", + str(offset or ""), + { + # ffmpeg's MKV muxer does not yet support HDR + "HEVC": "h265", + "AVC": "h264", + }.get(video_track.commercial_name, ".mp4"), + ], + ) + ) + ) + ) + ] + + ffmpeg_call = subprocess.Popen( + [binaries.FFMPEG, "-y", "-i", str(video_path), "-map", "0:v:0", "-c", "copy", "-bsf:v", crop_filter] + + out_path, + stdout=subprocess.PIPE, + ) + try: + if preview: + previewer = binaries.MPV or binaries.FFPlay + if not previewer: + raise click.ClickException("MPV/FFplay executables weren't found but are required for previewing.") + subprocess.Popen((previewer, "-"), stdin=ffmpeg_call.stdout) + finally: + if ffmpeg_call.stdout: + ffmpeg_call.stdout.close() + ffmpeg_call.wait() + + +@util.command(name="range") +@click.argument("path", type=Path) +@click.option("--full/--limited", is_flag=True, help="Full: 0..255, Limited: 16..235 (16..240 YUV luma)") +@click.option( + "-p", + "--preview", + is_flag=True, + default=False, + help="Instantly preview the newly-set video range in MPV (or ffplay if mpv is unavailable).", +) +def range_(path: Path, full: bool, preview: bool) -> None: + """ + Losslessly set the Video Range flag to full or limited at the bit-stream level. + You may provide a path to a file, or a folder of mkv and/or mp4 files. + + If you ever notice blacks not being quite black, and whites not being quite white, + then you're video may have the range set to the wrong value. Flip its range to the + opposite value and see if that fixes it. + """ + if not binaries.FFMPEG: + raise click.ClickException('FFmpeg executable "ffmpeg" not found but is required.') + + if path.is_dir(): + paths = sorted(list(path.glob("*.mkv")) + list(path.glob("*.mp4")), key=_natural_sort_key) + else: + paths = [path] + for video_path in paths: + try: + video_track = next(iter(MediaInfo.parse(video_path).video_tracks or [])) + except StopIteration: + raise click.ClickException("There's no video tracks in the provided file.") + + metadata_key = {"HEVC": "hevc_metadata", "AVC": "h264_metadata"}.get(video_track.commercial_name) + if not metadata_key: + raise click.ClickException(f"{video_track.commercial_name} Codec not supported.") + + if preview: + out_path = ["-f", "mpegts", "-"] # pipe + else: + out_path = [ + str( + video_path.with_name( + ".".join( + filter( + bool, + [ + video_path.stem, + video_track.language, + "range", + ["limited", "full"][full], + { + # ffmpeg's MKV muxer does not yet support HDR + "HEVC": "h265", + "AVC": "h264", + }.get(video_track.commercial_name, ".mp4"), + ], + ) + ) + ) + ) + ] + + ffmpeg_call = subprocess.Popen( + [ + binaries.FFMPEG, + "-y", + "-i", + str(video_path), + "-map", + "0:v:0", + "-c", + "copy", + "-bsf:v", + f"{metadata_key}=video_full_range_flag={int(full)}", + ] + + out_path, + stdout=subprocess.PIPE, + ) + try: + if preview: + previewer = binaries.MPV or binaries.FFPlay + if not previewer: + raise click.ClickException("MPV/FFplay executables weren't found but are required for previewing.") + subprocess.Popen((previewer, "-"), stdin=ffmpeg_call.stdout) + finally: + if ffmpeg_call.stdout: + ffmpeg_call.stdout.close() + ffmpeg_call.wait() + + +@util.command() +@click.argument("path", type=Path) +@click.option( + "-m", "--map", "map_", type=str, default="0", help="Test specific streams by setting FFmpeg's -map parameter." +) +def test(path: Path, map_: str) -> None: + """ + Decode an entire video and check for any corruptions or errors using FFmpeg. + You may provide a path to a file, or a folder of mkv and/or mp4 files. + + Tests all streams within the file by default. Subtitles cannot be tested. + You may choose specific streams using the -m/--map parameter. E.g., + '0:v:0' to test the first video stream, or '0:a' to test all audio streams. + """ + if not binaries.FFMPEG: + raise click.ClickException('FFmpeg executable "ffmpeg" not found but is required.') + + if path.is_dir(): + paths = sorted(list(path.glob("*.mkv")) + list(path.glob("*.mp4")), key=_natural_sort_key) + else: + paths = [path] + for video_path in paths: + print(f"Testing: {video_path.name}") + p = subprocess.Popen( + [ + binaries.FFMPEG, + "-hide_banner", + "-benchmark", + "-err_detect", + "+crccheck+bitstream+buffer+careful+compliant+aggressive", + "-i", + str(video_path), + "-map", + map_, + "-sn", + "-f", + "null", + "-", + ], + stderr=subprocess.PIPE, + universal_newlines=True, + ) + reached_output = False + errors = 0 + for line in p.stderr: + line = line.strip() + if "speed=" in line: + reached_output = True + if not reached_output: + continue + if line.startswith("[") and not line.startswith("[out#"): + errors += 1 + stream, error = line.split("] ", maxsplit=1) + stream = stream.split(" @ ")[0] + line = f"{stream} ERROR: {error}" + print(line) + p.stderr.close() + print(f"Finished with {errors} error(s)") + p.terminate() + p.wait() diff --git a/reset/packages/envied/src/envied/commands/wvd.py b/reset/packages/envied/src/envied/commands/wvd.py new file mode 100644 index 0000000..0dac520 --- /dev/null +++ b/reset/packages/envied/src/envied/commands/wvd.py @@ -0,0 +1,272 @@ +import logging +import shutil +from pathlib import Path +from typing import Optional + +import click +import yaml +from google.protobuf.json_format import MessageToDict +from pywidevine.device import Device, DeviceTypes +from pywidevine.license_protocol_pb2 import FileHashes +from rich.prompt import Prompt +from unidecode import UnidecodeError, unidecode + +from envied.core.config import config +from envied.core.console import console +from envied.core.constants import context_settings + + +@click.group( + short_help="Manage configuration and creation of WVD (Widevine Device) files.", context_settings=context_settings +) +def wvd() -> None: + """Manage configuration and creation of WVD (Widevine Device) files.""" + + +@wvd.command() +@click.argument("paths", type=Path, nargs=-1) +def add(paths: list[Path]) -> None: + """Add one or more WVD (Widevine Device) files to the WVDs Directory.""" + log = logging.getLogger("wvd") + for path in paths: + dst_path = config.directories.wvds / path.name + + if not path.exists(): + log.error(f"The WVD path '{path}' does not exist...") + elif dst_path.exists(): + log.error(f"WVD named '{path.stem}' already exists...") + else: + # TODO: Check for and log errors + _ = Device.load(path) # test if WVD is valid + dst_path.parent.mkdir(parents=True, exist_ok=True) + shutil.move(path, dst_path) + log.info(f"Added {path.stem}") + + +@wvd.command() +@click.argument("names", type=str, nargs=-1) +def delete(names: list[str]) -> None: + """Delete one or more WVD (Widevine Device) files from the WVDs Directory.""" + log = logging.getLogger("wvd") + for name in names: + path = (config.directories.wvds / name).with_suffix(".wvd") + if not path.exists(): + log.error(f"No WVD file exists by the name '{name}'...") + continue + + answer = Prompt.ask( + f"[red]Deleting '{name}'[/], are you sure you want to continue?", + choices=["y", "n"], + default="n", + console=console, + ) + if answer == "n": + log.info("Aborting...") + continue + + Path.unlink(path) + log.info(f"Deleted {name}") + + +@wvd.command() +@click.argument("path", type=Path) +def parse(path: Path) -> None: + """ + Parse a .WVD Widevine Device file to check information. + Relative paths are relative to the WVDs directory. + """ + try: + named = not path.suffix and path.relative_to(Path("")) + except ValueError: + named = False + if named: + path = config.directories.wvds / f"{path.name}.wvd" + + log = logging.getLogger("wvd") + + if not path.exists(): + console.log(f"[bright_blue]{path.absolute()}[/] does not exist...") + return + + device = Device.load(path) + + log.info(f"System ID: {device.system_id}") + log.info(f"Security Level: {device.security_level}") + log.info(f"Type: {device.type}") + log.info(f"Flags: {device.flags}") + log.info(f"Private Key: {bool(device.private_key)}") + log.info(f"Client ID: {bool(device.client_id)}") + log.info(f"VMP: {bool(device.client_id.vmp_data)}") + + log.info("Client ID:") + log.info(device.client_id) + + log.info("VMP:") + if device.client_id.vmp_data: + file_hashes = FileHashes() + file_hashes.ParseFromString(device.client_id.vmp_data) + log.info(str(file_hashes)) + else: + log.info("None") + + +@wvd.command() +@click.argument("wvd_paths", type=Path, nargs=-1) +@click.argument("out_dir", type=Path, nargs=1) +def dump(wvd_paths: list[Path], out_dir: Path) -> None: + """ + Extract data from a .WVD Widevine Device file to a folder structure. + + If the path is relative, with no file extension, it will dump the WVD in the WVDs + directory. + """ + log = logging.getLogger("wvd") + + if wvd_paths == (): + if not config.directories.wvds.exists(): + console.log(f"[bright_blue]{config.directories.wvds.absolute()}[/] does not exist...") + wvd_paths = list(x for x in config.directories.wvds.iterdir() if x.is_file() and x.suffix.lower() == ".wvd") + if not wvd_paths: + console.log(f"[bright_blue]{config.directories.wvds.absolute()}[/] is empty...") + + for i, (wvd_path, out_path) in enumerate(zip(wvd_paths, (out_dir / x.stem for x in wvd_paths))): + if i > 0: + log.info("") + + try: + named = not wvd_path.suffix and wvd_path.relative_to(Path("")) + except ValueError: + named = False + if named: + wvd_path = config.directories.wvds / f"{wvd_path.stem}.wvd" + out_path.mkdir(parents=True, exist_ok=True) + + log.info(f"Dumping: {wvd_path}") + device = Device.load(wvd_path) + + log.info(f"L{device.security_level} {device.system_id} {device.type.name}") + log.info(f"Saving to: {out_path}") + + device_meta = { + "wvd": {"device_type": device.type.name, "security_level": device.security_level, **device.flags}, + "client_info": {}, + "capabilities": MessageToDict(device.client_id, preserving_proto_field_name=True)["client_capabilities"], + } + for client_info in device.client_id.client_info: + device_meta["client_info"][client_info.name] = client_info.value + + device_meta_path = out_path / "metadata.yml" + device_meta_path.write_text(yaml.dump(device_meta), encoding="utf8") + log.info(" + Device Metadata") + + if device.private_key: + private_key_path = out_path / "private_key.pem" + private_key_path.write_text(data=device.private_key.export_key().decode(), encoding="utf8") + private_key_path.with_suffix(".der").write_bytes(device.private_key.export_key(format="DER")) + log.info(" + Private Key") + else: + log.warning(" - No Private Key available") + + if device.client_id: + client_id_path = out_path / "client_id.bin" + client_id_path.write_bytes(device.client_id.SerializeToString()) + log.info(" + Client ID") + else: + log.warning(" - No Client ID available") + + if device.client_id.vmp_data: + vmp_path = out_path / "vmp.bin" + vmp_path.write_bytes(device.client_id.vmp_data) + log.info(" + VMP (File Hashes)") + else: + log.info(" - No VMP (File Hashes) available") + + +@wvd.command() +@click.argument("name", type=str) +@click.argument("private_key", type=Path) +@click.argument("client_id", type=Path) +@click.argument("file_hashes", type=Path, required=False) +@click.option( + "-t", + "--type", + "type_", + type=click.Choice([x.name for x in DeviceTypes], case_sensitive=False), + default="Android", + help="Device Type", +) +@click.option("-l", "--level", type=click.IntRange(1, 3), default=1, help="Device Security Level") +@click.option("-o", "--output", type=Path, default=None, help="Output Directory") +@click.pass_context +def new( + ctx: click.Context, + name: str, + private_key: Path, + client_id: Path, + file_hashes: Optional[Path], + type_: str, + level: int, + output: Optional[Path], +) -> None: + """ + Create a new .WVD Widevine provision file. + + name: The origin device name of the provided data. e.g. `Nexus 6P`. You do not need to + specify the security level, that will be done automatically. + private_key: A PEM file of a Device's private key. + client_id: A binary blob file which follows the Widevine ClientIdentification protobuf + schema. + file_hashes: A binary blob file with follows the Widevine FileHashes protobuf schema. + Also known as VMP as it's used for VMP (Verified Media Path) assurance. + """ + try: + # TODO: Remove need for name, create name based on Client IDs ClientInfo values + name = unidecode(name.strip().lower().replace(" ", "_")) + except UnidecodeError as e: + raise click.UsageError(f"name: Failed to sanitize name, {e}", ctx) + if not name: + raise click.UsageError("name: Empty after sanitizing, please make sure the name is valid.", ctx) + if not private_key.is_file(): + raise click.UsageError("private_key: Not a path to a file, or it doesn't exist.", ctx) + if not client_id.is_file(): + raise click.UsageError("client_id: Not a path to a file, or it doesn't exist.", ctx) + if file_hashes and not file_hashes.is_file(): + raise click.UsageError("file_hashes: Not a path to a file, or it doesn't exist.", ctx) + + device = Device( + type_=DeviceTypes[type_.upper()], + security_level=level, + flags=None, + private_key=private_key.read_bytes(), + client_id=client_id.read_bytes(), + ) + + if file_hashes: + device.client_id.vmp_data = file_hashes.read_bytes() + + out_path = (output or config.directories.wvds) / f"{name}_{device.system_id}_l{device.security_level}.wvd" + device.dump(out_path) + + log = logging.getLogger("wvd") + + log.info(f"Created binary WVD file, {out_path.name}") + log.info(f" + Saved to: {out_path.absolute()}") + + log.info(f"System ID: {device.system_id}") + log.info(f"Security Level: {device.security_level}") + log.info(f"Type: {device.type}") + log.info(f"Flags: {device.flags}") + log.info(f"Private Key: {bool(device.private_key)}") + log.info(f"Client ID: {bool(device.client_id)}") + log.info(f"VMP: {bool(device.client_id.vmp_data)}") + + log.info("Client ID:") + log.info(device.client_id) + + log.info("VMP:") + if device.client_id.vmp_data: + file_hashes = FileHashes() + file_hashes.ParseFromString(device.client_id.vmp_data) + log.info(str(file_hashes)) + else: + log.info("None") diff --git a/reset/packages/envied/src/envied/core/__init__.py b/reset/packages/envied/src/envied/core/__init__.py new file mode 100644 index 0000000..f575288 --- /dev/null +++ b/reset/packages/envied/src/envied/core/__init__.py @@ -0,0 +1 @@ +__version__ = "5.3.0" diff --git a/reset/packages/envied/src/envied/core/__main__.py b/reset/packages/envied/src/envied/core/__main__.py new file mode 100644 index 0000000..494e007 --- /dev/null +++ b/reset/packages/envied/src/envied/core/__main__.py @@ -0,0 +1,79 @@ +import atexit +import logging +from datetime import datetime + +import click +import urllib3 +from rich import traceback +from rich.console import Group +from rich.padding import Padding +from rich.text import Text +from urllib3.exceptions import InsecureRequestWarning + +from envied.core import __version__ +from envied.core.commands import Commands +from envied.core.config import config +from envied.core.console import ComfyRichHandler, console +from envied.core.constants import context_settings +from envied.core.update_checker import UpdateChecker +from envied.core.utilities import close_debug_logger, init_debug_logger + + +@click.command(cls=Commands, invoke_without_command=True, context_settings=context_settings) +@click.option("-v", "--version", is_flag=True, default=False, help="Print version information.") +@click.option("-d", "--debug", is_flag=True, default=False, help="Enable DEBUG level logs and JSON debug logging.") +def main(version: bool, debug: bool) -> None: + """envied.Modular Movie, TV, and Music Archival Software.""" + debug_logging_enabled = debug or config.debug + + logging.basicConfig( + level=logging.DEBUG if debug else logging.INFO, + format="%(message)s", + handlers=[ + ComfyRichHandler( + show_time=False, + show_path=debug, + console=console, + rich_tracebacks=True, + tracebacks_suppress=[click], + log_renderer=console._log_render, # noqa + ) + ], + ) + + if debug_logging_enabled: + init_debug_logger(enabled=True) + + urllib3.disable_warnings(InsecureRequestWarning) + + traceback.install(console=console, width=80, suppress=[click]) + + console.print( + Padding( + Group( + Text( + r"░█▀▀░█▀█░█░█░▀█▀░█▀▀░█▀▄" + "\n" + r"░█▀▀░█░█░▀▄▀░░█░░█▀▀░█░█" + "\n" + r"░▀▀▀░▀░▀░░▀░░▀▀▀░▀▀▀░▀▀░" + "\n" , + style="ascii.art", + ), + Text(" and more than envied....", style = "ascii.art"), + f"\nv [repr.number]{__version__}[/] - https://github.com/vinefeeder/envied", + ), + (1, 11, 1, 10), + expand=True, + ), + justify="center", + ) + + + + +@atexit.register +def cleanup(): + """Clean up resources on exit.""" + close_debug_logger() + + +if __name__ == "__main__": + main() diff --git a/reset/packages/envied/src/envied/core/api/__init__.py b/reset/packages/envied/src/envied/core/api/__init__.py new file mode 100644 index 0000000..18ae1fa --- /dev/null +++ b/reset/packages/envied/src/envied/core/api/__init__.py @@ -0,0 +1,3 @@ +from envied.core.api.routes import cors_middleware, setup_routes, setup_swagger + +__all__ = ["setup_routes", "setup_swagger", "cors_middleware"] diff --git a/reset/packages/envied/src/envied/core/api/compression.py b/reset/packages/envied/src/envied/core/api/compression.py new file mode 100644 index 0000000..df568ac --- /dev/null +++ b/reset/packages/envied/src/envied/core/api/compression.py @@ -0,0 +1,40 @@ +"""aiohttp middleware for gzip transport compression.""" + +from __future__ import annotations + +import gzip + +from aiohttp import web + + +@web.middleware +async def compression_middleware(request: web.Request, handler) -> web.Response: + """Compress JSON responses with gzip when the client supports it.""" + response = await handler(request) + + accept_encoding = request.headers.get("Accept-Encoding", "") + if "gzip" not in accept_encoding: + return response + + if response.content_type and "json" not in response.content_type: + return response + + body = response.body + if body is None or len(body) < 256: + return response + + from envied.core.config import config + + level = config.serve.get("compression_level", 1) + if not level: + return response + + compressed = gzip.compress(body, compresslevel=level) + headers = dict(response.headers) + headers["Content-Encoding"] = "gzip" + headers["Content-Length"] = str(len(compressed)) + return web.Response( + body=compressed, + status=response.status, + headers=headers, + ) diff --git a/reset/packages/envied/src/envied/core/api/download_manager.py b/reset/packages/envied/src/envied/core/api/download_manager.py new file mode 100644 index 0000000..401340c --- /dev/null +++ b/reset/packages/envied/src/envied/core/api/download_manager.py @@ -0,0 +1,752 @@ +import asyncio +import json +import logging +import os +import re +import sys +import tempfile +import threading +import uuid +from contextlib import suppress +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from enum import Enum +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional + +log = logging.getLogger("download_manager") + + +def _sanitize_log(value: object) -> str: + """Sanitize a value for safe logging by removing newlines and control characters.""" + return str(value).replace("\n", "").replace("\r", "").replace("\x00", "") + + +class JobStatus(Enum): + QUEUED = "queued" + DOWNLOADING = "downloading" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +@dataclass +class DownloadJob: + """Represents a download job with all its parameters and status.""" + + job_id: str + status: JobStatus + created_time: datetime + service: str + title_id: str + parameters: Dict[str, Any] + + # Progress tracking + started_time: Optional[datetime] = None + completed_time: Optional[datetime] = None + progress: float = 0.0 + + # Results and error info + output_files: List[str] = field(default_factory=list) + error_message: Optional[str] = None + error_details: Optional[str] = None + error_code: Optional[str] = None + error_traceback: Optional[str] = None + worker_stderr: Optional[str] = None + + # Cancellation support + cancel_event: threading.Event = field(default_factory=threading.Event) + + def to_dict(self, include_full_details: bool = False) -> Dict[str, Any]: + """Convert job to dictionary for JSON response.""" + result = { + "job_id": self.job_id, + "status": self.status.value, + "created_time": self.created_time.isoformat(), + "service": self.service, + "title_id": self.title_id, + "progress": self.progress, + } + + if include_full_details: + result.update( + { + "parameters": self.parameters, + "started_time": self.started_time.isoformat() if self.started_time else None, + "completed_time": self.completed_time.isoformat() if self.completed_time else None, + "output_files": self.output_files, + "error_message": self.error_message, + "error_details": self.error_details, + "error_code": self.error_code, + "error_traceback": self.error_traceback, + "worker_stderr": self.worker_stderr, + } + ) + + return result + + +def _perform_download( + job_id: str, + service: str, + title_id: str, + params: Dict[str, Any], + cancel_event: Optional[threading.Event] = None, + progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None, +) -> List[str]: + """Execute the synchronous download logic for a job.""" + + def _check_cancel(stage: str): + if cancel_event and cancel_event.is_set(): + raise Exception(f"Job was cancelled {stage}") + + from contextlib import redirect_stderr, redirect_stdout + from io import StringIO + + _check_cancel("before execution started") + + # Import dl.py components lazily to avoid circular deps during module import + import click + import yaml + + from envied.commands.dl import dl + from envied.core.config import config + from envied.core.services import Services + from envied.core.tracks import Subtitle, Video + from envied.core.utils.click_types import ContextData + from envied.core.utils.collections import merge_dict + + log.info(f"Starting sync download for job {job_id}") + + # Convert string parameters to enums (API receives strings, dl.result() expects enums) + vcodec_raw = params.get("vcodec") + if vcodec_raw: + if isinstance(vcodec_raw, str): + vcodec_raw = [vcodec_raw] + if isinstance(vcodec_raw, list) and vcodec_raw and not isinstance(vcodec_raw[0], Video.Codec): + codec_map = {c.name.upper(): c for c in Video.Codec} + codec_map.update({c.value.upper(): c for c in Video.Codec}) + params["vcodec"] = [codec_map[v.upper()] for v in vcodec_raw if v.upper() in codec_map] + else: + params["vcodec"] = [] + + range_raw = params.get("range") + if range_raw: + if isinstance(range_raw, str): + range_raw = [range_raw] + if isinstance(range_raw, list) and range_raw and not isinstance(range_raw[0], Video.Range): + range_map = {r.name.upper(): r for r in Video.Range} + range_map.update({r.value.upper(): r for r in Video.Range}) + params["range"] = [range_map[r.upper()] for r in range_raw if r.upper() in range_map] + else: + params["range"] = [Video.Range.SDR] + + sub_format_raw = params.get("sub_format") + if sub_format_raw and isinstance(sub_format_raw, str): + sub_map = {c.name.upper(): c for c in Subtitle.Codec} + sub_map.update({c.value.upper(): c for c in Subtitle.Codec}) + params["sub_format"] = sub_map.get(sub_format_raw.upper()) + + if params.get("export"): + params["export"] = bool(params["export"]) + + # Normalize slow: accept string "MIN-MAX", list/tuple, or True (default 60-120) + slow_raw = params.get("slow") + if slow_raw is not None and not isinstance(slow_raw, tuple): + if isinstance(slow_raw, bool): + params["slow"] = (60, 120) if slow_raw else None + elif isinstance(slow_raw, list) and len(slow_raw) == 2: + params["slow"] = (int(slow_raw[0]), int(slow_raw[1])) + elif isinstance(slow_raw, str): + from envied.core.utils.click_types import SLOW_DELAY_RANGE + + try: + params["slow"] = SLOW_DELAY_RANGE.convert(slow_raw, None, None) + except click.BadParameter as exc: + raise Exception(f"Invalid slow parameter: {exc}") + + # Convert wanted episode strings to internal "SxE" format + # Accepts: "S01E01", "S01-S03", "s1e1", "1x1", or already-parsed format + wanted_raw = params.get("wanted") + if wanted_raw: + from envied.core.utils.click_types import SeasonRange + + if isinstance(wanted_raw, str): + wanted_raw = [wanted_raw] + # Only convert if not already in internal "SxE" format + needs_conversion = any(not re.match(r"^\d+x\d+$", w) for w in wanted_raw) + if needs_conversion: + season_range = SeasonRange() + params["wanted"] = season_range.parse_tokens(*wanted_raw) + + # Load service configuration + service_config_path = Services.get_path(service) / config.filenames.config + if service_config_path.exists(): + service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) + else: + service_config = {} + merge_dict(config.services.get(service), service_config) + + from envied.commands.dl import dl as dl_command + + ctx = click.Context(dl_command.cli) + ctx.invoked_subcommand = service + from envied.core.api.handlers import load_full_cdm + + cdm = load_full_cdm(service, params.get("profile"), params.get("cdm_type")) + ctx.obj = ContextData(config=service_config, cdm=cdm, proxy_providers=[], profile=params.get("profile")) + ctx.params = { + "proxy": params.get("proxy"), + "no_proxy": params.get("no_proxy", False), + "no_proxy_download": params.get("no_proxy_download", False), + "profile": params.get("profile"), + "repack": params.get("repack", False), + "tag": params.get("tag"), + "tmdb_id": params.get("tmdb_id"), + "imdb_id": params.get("imdb_id"), + "animeapi_id": params.get("animeapi_id"), + "enrich": params.get("enrich", False), + "output_dir": Path(params["output_dir"]) if params.get("output_dir") else None, + "no_cache": params.get("no_cache", False), + "reset_cache": params.get("reset_cache", False), + } + + dl_instance = dl( + ctx=ctx, + no_proxy=params.get("no_proxy", False), + profile=params.get("profile"), + proxy=params.get("proxy"), + repack=params.get("repack", False), + tag=params.get("tag"), + tmdb_id=params.get("tmdb_id"), + imdb_id=params.get("imdb_id"), + animeapi_id=params.get("animeapi_id"), + enrich=params.get("enrich", False), + output_dir=Path(params["output_dir"]) if params.get("output_dir") else None, + ) + + service_module = Services.load(service) + + _check_cancel("before service instantiation") + + try: + import inspect + + service_init_params = inspect.signature(service_module.__init__).parameters + + service_ctx = click.Context(click.Command(service)) + service_ctx.parent = ctx + service_ctx.obj = ctx.obj + + service_kwargs = {} + + if "title" in service_init_params: + service_kwargs["title"] = title_id + + for key, value in params.items(): + if key in service_init_params and key not in ["service", "title_id"]: + service_kwargs[key] = value + + for param_name, param_info in service_init_params.items(): + if param_name not in service_kwargs and param_name not in ["self", "ctx"]: + if param_info.default is inspect.Parameter.empty: + if param_name == "movie": + service_kwargs[param_name] = "/movies/" in title_id + elif param_name == "meta_lang": + service_kwargs[param_name] = None + else: + log.warning(f"Unknown required parameter '{param_name}' for service {service}, using None") + service_kwargs[param_name] = None + + service_instance = service_module(service_ctx, **service_kwargs) + + except Exception as exc: # noqa: BLE001 - propagate meaningful failure + log.error(f"Failed to create service instance: {exc}") + raise + + original_download_dir = config.directories.downloads + + _check_cancel("before download execution") + + stdout_capture = StringIO() + stderr_capture = StringIO() + + # Simple progress tracking if callback provided + if progress_callback: + # Report initial progress + progress_callback({"progress": 0.0, "status": "starting"}) + + # Simple approach: report progress at key points + original_result = dl_instance.result + + def result_with_progress(*args, **kwargs): + try: + # Report that download started + progress_callback({"progress": 5.0, "status": "downloading"}) + + # Call original method + result = original_result(*args, **kwargs) + + # Report completion + progress_callback({"progress": 100.0, "status": "completed"}) + return result + except Exception as e: + progress_callback({"progress": 0.0, "status": "failed", "error": str(e)}) + raise + + dl_instance.result = result_with_progress + + try: + with redirect_stdout(stdout_capture), redirect_stderr(stderr_capture): + dl_instance.result( + service=service_instance, + quality=params.get("quality", []), + vcodec=params.get("vcodec", []), + acodec=params.get("acodec"), + vbitrate=params.get("vbitrate"), + abitrate=params.get("abitrate"), + vbitrate_range=params.get("vbitrate_range"), + abitrate_range=params.get("abitrate_range"), + range_=params.get("range", ["SDR"]), + channels=params.get("channels"), + no_atmos=params.get("no_atmos", False), + select_titles=False, + wanted=params.get("wanted", []), + latest_episode=params.get("latest_episode", False), + lang=params.get("lang", ["orig"]), + v_lang=params.get("v_lang", []), + a_lang=params.get("a_lang", []), + s_lang=params.get("s_lang", ["all"]), + require_subs=params.get("require_subs", []), + forced_subs=params.get("forced_subs", False), + exact_lang=params.get("exact_lang", False), + sub_format=params.get("sub_format"), + video_only=params.get("video_only", False), + audio_only=params.get("audio_only", False), + subs_only=params.get("subs_only", False), + chapters_only=params.get("chapters_only", False), + no_subs=params.get("no_subs", False), + no_audio=params.get("no_audio", False), + no_chapters=params.get("no_chapters", False), + no_video=params.get("no_video", False), + audio_description=params.get("audio_description", False), + slow=params.get("slow", None), + list_=False, + list_titles=False, + skip_dl=params.get("skip_dl", False), + export=params.get("export"), + cdm_only=params.get("cdm_only"), + no_proxy=params.get("no_proxy", False), + no_proxy_download=params.get("no_proxy_download", False), + no_folder=params.get("no_folder", False), + no_source=params.get("no_source", False), + no_mux=params.get("no_mux", False), + workers=params.get("workers"), + downloads=params.get("downloads", 1), + worst=params.get("worst", False), + best_available=params.get("best_available", False), + split_audio=params.get("split_audio"), + ) + + except SystemExit as exc: + if exc.code != 0: + stdout_str = stdout_capture.getvalue() + stderr_str = stderr_capture.getvalue() + log.error(f"Download exited with code {exc.code}") + log.error(f"Stdout: {stdout_str}") + log.error(f"Stderr: {stderr_str}") + raise Exception(f"Download failed with exit code {exc.code}") + + except Exception as exc: # noqa: BLE001 - propagate to caller + stdout_str = stdout_capture.getvalue() + stderr_str = stderr_capture.getvalue() + log.error(f"Download execution failed: {exc}") + log.error(f"Stdout: {stdout_str}") + log.error(f"Stderr: {stderr_str}") + raise + + output_files = [str(p) for p in dl_instance.completed_files] + log.info(f"Download completed for job {job_id}, {len(output_files)} file(s) in {original_download_dir}") + + return output_files + + +class DownloadQueueManager: + """Manages download job queue with configurable concurrency limits.""" + + def __init__(self, max_concurrent_downloads: int = 2, job_retention_hours: int = 24): + self.max_concurrent_downloads = max_concurrent_downloads + self.job_retention_hours = job_retention_hours + + self._jobs: Dict[str, DownloadJob] = {} + self._job_queue: asyncio.Queue = asyncio.Queue() + self._active_downloads: Dict[str, asyncio.Task] = {} + self._download_processes: Dict[str, asyncio.subprocess.Process] = {} + self._job_temp_files: Dict[str, Dict[str, str]] = {} + self._workers_started = False + self._shutdown_event = asyncio.Event() + + log.info( + f"Initialized download queue manager: max_concurrent={max_concurrent_downloads}, retention_hours={job_retention_hours}" + ) + + def create_job(self, service: str, title_id: str, **parameters) -> DownloadJob: + """Create a new download job and add it to the queue.""" + job_id = str(uuid.uuid4()) + job = DownloadJob( + job_id=job_id, + status=JobStatus.QUEUED, + created_time=datetime.now(), + service=service, + title_id=title_id, + parameters=parameters, + ) + + self._jobs[job_id] = job + self._job_queue.put_nowait(job) + + log.info(f"Created download job {job_id} for {_sanitize_log(service)}:{_sanitize_log(title_id)}") + return job + + def get_job(self, job_id: str) -> Optional[DownloadJob]: + """Get job by ID.""" + return self._jobs.get(job_id) + + def list_jobs(self) -> List[DownloadJob]: + """List all jobs.""" + return list(self._jobs.values()) + + def cancel_job(self, job_id: str) -> bool: + """Cancel a job if it's queued or downloading.""" + job = self._jobs.get(job_id) + if not job: + return False + + if job.status == JobStatus.QUEUED: + job.status = JobStatus.CANCELLED + job.cancel_event.set() # Signal cancellation + log.info(f"Cancelled queued job {_sanitize_log(job_id)}") + return True + elif job.status == JobStatus.DOWNLOADING: + # Set the cancellation event first - this will be checked by the download thread + job.cancel_event.set() + job.status = JobStatus.CANCELLED + log.info(f"Signaled cancellation for downloading job {_sanitize_log(job_id)}") + + # Cancel the active download task + task = self._active_downloads.get(job_id) + if task: + task.cancel() + log.info(f"Cancelled download task for job {_sanitize_log(job_id)}") + + process = self._download_processes.get(job_id) + if process: + try: + process.terminate() + log.info(f"Terminated worker process for job {_sanitize_log(job_id)}") + except ProcessLookupError: + log.debug(f"Worker process for job {_sanitize_log(job_id)} already exited") + + return True + + return False + + def cleanup_old_jobs(self) -> int: + """Remove jobs older than retention period.""" + cutoff_time = datetime.now() - timedelta(hours=self.job_retention_hours) + jobs_to_remove = [] + + for job_id, job in self._jobs.items(): + if job.status in [JobStatus.COMPLETED, JobStatus.FAILED, JobStatus.CANCELLED]: + if job.completed_time and job.completed_time < cutoff_time: + jobs_to_remove.append(job_id) + elif not job.completed_time and job.created_time < cutoff_time: + jobs_to_remove.append(job_id) + + for job_id in jobs_to_remove: + del self._jobs[job_id] + + if jobs_to_remove: + log.info(f"Cleaned up {len(jobs_to_remove)} old jobs") + + return len(jobs_to_remove) + + async def start_workers(self): + """Start worker tasks to process the download queue.""" + if self._workers_started: + return + + self._workers_started = True + + # Start worker tasks + for i in range(self.max_concurrent_downloads): + asyncio.create_task(self._download_worker(f"worker-{i}")) + + # Start cleanup task + asyncio.create_task(self._cleanup_worker()) + + log.info(f"Started {self.max_concurrent_downloads} download workers") + + async def shutdown(self): + """Shutdown the queue manager and cancel all active downloads.""" + log.info("Shutting down download queue manager") + self._shutdown_event.set() + + # Cancel all active downloads + for task in self._active_downloads.values(): + task.cancel() + + # Terminate worker processes + for job_id, process in list(self._download_processes.items()): + try: + process.terminate() + except ProcessLookupError: + log.debug(f"Worker process for job {job_id} already exited during shutdown") + + for job_id, process in list(self._download_processes.items()): + try: + await asyncio.wait_for(process.wait(), timeout=5) + except asyncio.TimeoutError: + log.warning(f"Worker process for job {job_id} did not exit, killing") + process.kill() + await process.wait() + finally: + self._download_processes.pop(job_id, None) + + # Clean up any remaining temp files + for paths in self._job_temp_files.values(): + for path in paths.values(): + try: + os.remove(path) + except OSError: + pass + self._job_temp_files.clear() + + # Wait for workers to finish + if self._active_downloads: + await asyncio.gather(*self._active_downloads.values(), return_exceptions=True) + + async def _download_worker(self, worker_name: str): + """Worker task that processes jobs from the queue.""" + log.debug(f"Download worker {worker_name} started") + + while not self._shutdown_event.is_set(): + try: + # Wait for a job or shutdown signal + job = await asyncio.wait_for(self._job_queue.get(), timeout=1.0) + + if job.status == JobStatus.CANCELLED: + continue + + # Start processing the job + job.status = JobStatus.DOWNLOADING + job.started_time = datetime.now() + + log.info(f"Worker {worker_name} starting job {job.job_id}") + + # Create download task + download_task = asyncio.create_task(self._execute_download(job)) + self._active_downloads[job.job_id] = download_task + + try: + await download_task + except asyncio.CancelledError: + job.status = JobStatus.CANCELLED + log.info(f"Job {job.job_id} was cancelled") + except Exception as e: + job.status = JobStatus.FAILED + job.error_message = str(e) + log.error(f"Job {job.job_id} failed: {e}") + finally: + job.completed_time = datetime.now() + if job.job_id in self._active_downloads: + del self._active_downloads[job.job_id] + + except asyncio.TimeoutError: + continue + except Exception as e: + log.error(f"Worker {worker_name} error: {e}") + + async def _execute_download(self, job: DownloadJob): + """Execute the actual download for a job.""" + log.info(f"Executing download for job {job.job_id}") + + try: + output_files = await self._run_download_async(job) + job.status = JobStatus.COMPLETED + job.output_files = output_files + job.progress = 100.0 + log.info(f"Download completed for job {job.job_id}: {len(output_files)} files") + except Exception as e: + import traceback + + from envied.core.api.errors import categorize_exception + + job.status = JobStatus.FAILED + job.error_message = str(e) + job.error_details = str(e) + + api_error = categorize_exception( + e, context={"service": job.service, "title_id": job.title_id, "job_id": job.job_id} + ) + job.error_code = api_error.error_code.value + + job.error_traceback = traceback.format_exc() + + log.error(f"Download failed for job {job.job_id}: {e}") + raise + + async def _run_download_async(self, job: DownloadJob) -> List[str]: + """Invoke a worker subprocess to execute the download.""" + + payload = { + "job_id": job.job_id, + "service": job.service, + "title_id": job.title_id, + "parameters": job.parameters, + } + + payload_fd, payload_path = tempfile.mkstemp(prefix=f"envied.job_{job.job_id}_", suffix="_payload.json") + os.close(payload_fd) + result_fd, result_path = tempfile.mkstemp(prefix=f"envied.job_{job.job_id}_", suffix="_result.json") + os.close(result_fd) + progress_fd, progress_path = tempfile.mkstemp(prefix=f"envied.job_{job.job_id}_", suffix="_progress.json") + os.close(progress_fd) + + with open(payload_path, "w", encoding="utf-8") as handle: + json.dump(payload, handle) + + process = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "envied.core.api.download_worker", + payload_path, + result_path, + progress_path, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + self._download_processes[job.job_id] = process + self._job_temp_files[job.job_id] = {"payload": payload_path, "result": result_path, "progress": progress_path} + + communicate_task = asyncio.create_task(process.communicate()) + + stdout_bytes = b"" + stderr_bytes = b"" + + try: + while True: + done, _ = await asyncio.wait({communicate_task}, timeout=0.5) + if communicate_task in done: + stdout_bytes, stderr_bytes = communicate_task.result() + break + + # Check for progress updates + try: + if os.path.exists(progress_path): + with open(progress_path, "r", encoding="utf-8") as handle: + progress_data = json.load(handle) + if "progress" in progress_data: + new_progress = float(progress_data["progress"]) + if new_progress != job.progress: + job.progress = new_progress + log.info(f"Job {job.job_id} progress updated: {job.progress}%") + except (FileNotFoundError, json.JSONDecodeError, ValueError) as e: + log.debug(f"Could not read progress for job {job.job_id}: {e}") + + if job.cancel_event.is_set() or job.status == JobStatus.CANCELLED: + log.info(f"Cancellation detected for job {job.job_id}, terminating worker process") + process.terminate() + try: + await asyncio.wait_for(communicate_task, timeout=5) + except asyncio.TimeoutError: + log.warning(f"Worker process for job {job.job_id} did not terminate, killing") + process.kill() + await asyncio.wait_for(communicate_task, timeout=5) + raise asyncio.CancelledError("Job was cancelled") + + returncode = process.returncode + stdout = stdout_bytes.decode("utf-8", errors="ignore") + stderr = stderr_bytes.decode("utf-8", errors="ignore") + + if stdout.strip(): + log.debug(f"Worker stdout for job {job.job_id}: {stdout.strip()}") + if stderr.strip(): + job.worker_stderr = stderr.strip() + if returncode != 0: + log.warning(f"Worker stderr for job {job.job_id}: {stderr.strip()}") + else: + log.debug(f"Worker stderr for job {job.job_id}: {stderr.strip()}") + + result_data: Optional[Dict[str, Any]] = None + try: + with open(result_path, "r", encoding="utf-8") as handle: + result_data = json.load(handle) + except FileNotFoundError: + log.error(f"Result file missing for job {job.job_id}") + except json.JSONDecodeError as exc: + log.error(f"Failed to parse worker result for job {job.job_id}: {exc}") + + if returncode != 0: + message = result_data.get("message") if result_data else "unknown error" + if result_data: + job.error_details = result_data.get("error_details", message) + job.error_code = result_data.get("error_code") + raise Exception(f"Worker exited with code {returncode}: {message}") + + if not result_data or result_data.get("status") != "success": + message = result_data.get("message") if result_data else "worker did not report success" + if result_data: + job.error_details = result_data.get("error_details", message) + job.error_code = result_data.get("error_code") + raise Exception(f"Worker failure: {message}") + + return result_data.get("output_files", []) + + finally: + if not communicate_task.done(): + communicate_task.cancel() + with suppress(asyncio.CancelledError): + await communicate_task + + self._download_processes.pop(job.job_id, None) + + temp_paths = self._job_temp_files.pop(job.job_id, {}) + for path in temp_paths.values(): + try: + os.remove(path) + except OSError: + pass + + def _execute_download_sync(self, job: DownloadJob) -> List[str]: + """Execute download synchronously using existing dl.py logic.""" + return _perform_download(job.job_id, job.service, job.title_id, job.parameters.copy(), job.cancel_event) + + async def _cleanup_worker(self): + """Worker that periodically cleans up old jobs.""" + while not self._shutdown_event.is_set(): + try: + await asyncio.sleep(3600) # Run every hour + self.cleanup_old_jobs() + except Exception as e: + log.error(f"Cleanup worker error: {e}") + + +# Global instance +download_manager: Optional[DownloadQueueManager] = None + + +def get_download_manager() -> DownloadQueueManager: + """Get the global download manager instance.""" + global download_manager + if download_manager is None: + # Load configuration from envied.config + from envied.core.config import config + + max_concurrent = getattr(config, "max_concurrent_downloads", 2) + retention_hours = getattr(config, "download_job_retention_hours", 24) + + download_manager = DownloadQueueManager(max_concurrent, retention_hours) + + return download_manager diff --git a/reset/packages/envied/src/envied/core/api/download_worker.py b/reset/packages/envied/src/envied/core/api/download_worker.py new file mode 100644 index 0000000..008366f --- /dev/null +++ b/reset/packages/envied/src/envied/core/api/download_worker.py @@ -0,0 +1,102 @@ +"""Standalone worker process entry point for executing download jobs.""" + +from __future__ import annotations + +import json +import logging +import sys +import traceback +from pathlib import Path +from typing import Any, Dict + +from .download_manager import _perform_download + +log = logging.getLogger("download_worker") + + +def _read_payload(path: Path) -> Dict[str, Any]: + with path.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def _write_result(path: Path, payload: Dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as handle: + json.dump(payload, handle) + + +def main(argv: list[str]) -> int: + if len(argv) not in [3, 4]: + print( + "Usage: python -m envied.core.api.download_worker [progress_path]", + file=sys.stderr, + ) + return 2 + + payload_path = Path(argv[1]) + result_path = Path(argv[2]) + progress_path = Path(argv[3]) if len(argv) > 3 else None + + result: Dict[str, Any] = {} + exit_code = 0 + + try: + payload = _read_payload(payload_path) + job_id = payload["job_id"] + service = payload["service"] + title_id = payload["title_id"] + params = payload.get("parameters", {}) + + log.info(f"Worker starting job {job_id} ({service}:{title_id})") + + def progress_callback(progress_data: Dict[str, Any]) -> None: + """Write progress updates to file for main process to read.""" + if progress_path: + try: + log.info(f"Writing progress update: {progress_data}") + _write_result(progress_path, progress_data) + log.info(f"Progress update written to {progress_path}") + except Exception as e: + log.error(f"Failed to write progress update: {e}") + + output_files = _perform_download( + job_id, service, title_id, params, cancel_event=None, progress_callback=progress_callback + ) + + result = {"status": "success", "output_files": output_files} + + except Exception as exc: # noqa: BLE001 - capture for parent process + from envied.core.api.errors import categorize_exception + + exit_code = 1 + tb = traceback.format_exc() + log.error(f"Worker failed with error: {exc}") + + api_error = categorize_exception( + exc, + context={ + "service": payload.get("service") if "payload" in locals() else None, + "title_id": payload.get("title_id") if "payload" in locals() else None, + "job_id": payload.get("job_id") if "payload" in locals() else None, + }, + ) + + result = { + "status": "error", + "message": str(exc), + "error_details": api_error.message, + "error_code": api_error.error_code.value, + "traceback": tb, + } + + finally: + try: + _write_result(result_path, result) + except Exception as exc: # noqa: BLE001 - last resort logging + log.error(f"Failed to write worker result file: {exc}") + + return exit_code + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/reset/packages/envied/src/envied/core/api/errors.py b/reset/packages/envied/src/envied/core/api/errors.py new file mode 100644 index 0000000..3bc8ba4 --- /dev/null +++ b/reset/packages/envied/src/envied/core/api/errors.py @@ -0,0 +1,326 @@ +""" +API Error Handling System + +Provides structured error responses with error codes, categorization, +and optional debug information for the envied.REST API. +""" + +from __future__ import annotations + +import traceback +from datetime import datetime, timezone +from enum import Enum +from typing import Any + +from aiohttp import web + + +class APIErrorCode(str, Enum): + """Standard API error codes for programmatic error handling.""" + + # Client errors (4xx) + INVALID_INPUT = "INVALID_INPUT" # Missing or malformed request data + INVALID_SERVICE = "INVALID_SERVICE" # Unknown service name + INVALID_TITLE_ID = "INVALID_TITLE_ID" # Invalid or malformed title ID + INVALID_PROFILE = "INVALID_PROFILE" # Profile doesn't exist + INVALID_PROXY = "INVALID_PROXY" # Invalid proxy specification + INVALID_LANGUAGE = "INVALID_LANGUAGE" # Invalid language code + INVALID_PARAMETERS = "INVALID_PARAMETERS" # Invalid download parameters + + AUTH_FAILED = "AUTH_FAILED" # Authentication failure (invalid credentials/cookies) + AUTH_REQUIRED = "AUTH_REQUIRED" # Missing authentication + FORBIDDEN = "FORBIDDEN" # Action not allowed + GEOFENCE = "GEOFENCE" # Content not available in region + + NOT_FOUND = "NOT_FOUND" # Resource not found (title, job, etc.) + NO_CONTENT = "NO_CONTENT" # No titles/tracks/episodes found + JOB_NOT_FOUND = "JOB_NOT_FOUND" # Download job doesn't exist + SESSION_NOT_FOUND = "SESSION_NOT_FOUND" # Remote-dl session doesn't exist or expired + TRACK_NOT_FOUND = "TRACK_NOT_FOUND" # Track ID not found in session + + RATE_LIMITED = "RATE_LIMITED" # Service rate limiting + + # Server errors (5xx) + INTERNAL_ERROR = "INTERNAL_ERROR" # Unexpected server error + SERVICE_ERROR = "SERVICE_ERROR" # Streaming service API error + NETWORK_ERROR = "NETWORK_ERROR" # Network connectivity issue + DRM_ERROR = "DRM_ERROR" # DRM/license acquisition failure + DOWNLOAD_ERROR = "DOWNLOAD_ERROR" # Download process failure + SERVICE_UNAVAILABLE = "SERVICE_UNAVAILABLE" # Service temporarily unavailable + WORKER_ERROR = "WORKER_ERROR" # Download worker process error + + +class APIError(Exception): + """ + Structured API error with error code, message, and details. + + Attributes: + error_code: Standardized error code from APIErrorCode enum + message: User-friendly error message + details: Additional structured error information + retryable: Whether the operation can be retried + http_status: HTTP status code to return (default based on error_code) + """ + + def __init__( + self, + error_code: APIErrorCode, + message: str, + details: dict[str, Any] | None = None, + retryable: bool = False, + http_status: int | None = None, + ): + super().__init__(message) + self.error_code = error_code + self.message = message + self.details = details or {} + self.retryable = retryable + self.http_status = http_status or self._default_http_status(error_code) + + @staticmethod + def _default_http_status(error_code: APIErrorCode) -> int: + """Map error codes to default HTTP status codes.""" + status_map = { + # 400 Bad Request + APIErrorCode.INVALID_INPUT: 400, + APIErrorCode.INVALID_SERVICE: 400, + APIErrorCode.INVALID_TITLE_ID: 400, + APIErrorCode.INVALID_PROFILE: 400, + APIErrorCode.INVALID_PROXY: 400, + APIErrorCode.INVALID_LANGUAGE: 400, + APIErrorCode.INVALID_PARAMETERS: 400, + # 401 Unauthorized + APIErrorCode.AUTH_REQUIRED: 401, + APIErrorCode.AUTH_FAILED: 401, + # 403 Forbidden + APIErrorCode.FORBIDDEN: 403, + APIErrorCode.GEOFENCE: 403, + # 404 Not Found + APIErrorCode.NOT_FOUND: 404, + APIErrorCode.NO_CONTENT: 404, + APIErrorCode.JOB_NOT_FOUND: 404, + APIErrorCode.SESSION_NOT_FOUND: 404, + APIErrorCode.TRACK_NOT_FOUND: 404, + # 429 Too Many Requests + APIErrorCode.RATE_LIMITED: 429, + # 500 Internal Server Error + APIErrorCode.INTERNAL_ERROR: 500, + # 502 Bad Gateway + APIErrorCode.SERVICE_ERROR: 502, + APIErrorCode.DRM_ERROR: 502, + # 503 Service Unavailable + APIErrorCode.NETWORK_ERROR: 503, + APIErrorCode.SERVICE_UNAVAILABLE: 503, + APIErrorCode.DOWNLOAD_ERROR: 500, + APIErrorCode.WORKER_ERROR: 500, + } + return status_map.get(error_code, 500) + + +def build_error_response( + error: APIError | Exception, + debug_mode: bool = False, + extra_debug_info: dict[str, Any] | None = None, +) -> web.Response: + """ + Build a structured JSON error response. + + Args: + error: APIError or generic Exception to convert to response + debug_mode: Whether to include technical debug information + extra_debug_info: Additional debug info (stderr, stdout, etc.) + + Returns: + aiohttp JSON response with structured error data + """ + if isinstance(error, APIError): + error_code = error.error_code.value + message = error.message + details = error.details + http_status = error.http_status + retryable = error.retryable + else: + # Generic exception - convert to INTERNAL_ERROR + error_code = APIErrorCode.INTERNAL_ERROR.value + message = str(error) or "An unexpected error occurred" + details = {} + http_status = 500 + retryable = False + + response_data: dict[str, Any] = { + "status": "error", + "error_code": error_code, + "message": message, + "timestamp": datetime.now(timezone.utc).isoformat(), + } + + # Add details if present + if details: + response_data["details"] = details + + # Add retryable hint if specified + if retryable: + response_data["retryable"] = True + + # Add debug information if in debug mode + if debug_mode: + debug_info: dict[str, Any] = { + "exception_type": type(error).__name__, + } + + # Add traceback for debugging + if isinstance(error, Exception): + debug_info["traceback"] = traceback.format_exc() + + # Add any extra debug info provided + if extra_debug_info: + debug_info.update(extra_debug_info) + + response_data["debug_info"] = debug_info + + return web.json_response(response_data, status=http_status) + + +def categorize_exception( + exc: Exception, + context: dict[str, Any] | None = None, +) -> APIError: + """ + Categorize a generic exception into a structured APIError. + + This function attempts to identify the type of error based on the exception + type, message patterns, and optional context information. + + Args: + exc: The exception to categorize + context: Optional context (service name, operation type, etc.) + + Returns: + APIError with appropriate error code and details + """ + context = context or {} + exc_str = str(exc).lower() + exc_type = type(exc).__name__ + + # Authentication errors + if any(keyword in exc_str for keyword in ["auth", "login", "credential", "unauthorized", "forbidden", "token"]): + return APIError( + error_code=APIErrorCode.AUTH_FAILED, + message=f"Authentication failed: {exc}", + details={**context, "reason": "authentication_error"}, + retryable=False, + ) + + # Network errors + if any( + keyword in exc_str + for keyword in [ + "connection", + "timeout", + "network", + "unreachable", + "socket", + "dns", + "resolve", + ] + ) or exc_type in ["ConnectionError", "TimeoutError", "URLError", "SSLError"]: + return APIError( + error_code=APIErrorCode.NETWORK_ERROR, + message=f"Network error occurred: {exc}", + details={**context, "reason": "network_connectivity"}, + retryable=True, + http_status=503, + ) + + # Geofence/region errors + if any(keyword in exc_str for keyword in ["geofence", "region", "not available in", "territory"]): + return APIError( + error_code=APIErrorCode.GEOFENCE, + message=f"Content not available in your region: {exc}", + details={**context, "reason": "geofence_restriction"}, + retryable=False, + ) + + # Not found errors + if any(keyword in exc_str for keyword in ["not found", "404", "does not exist", "invalid id"]): + return APIError( + error_code=APIErrorCode.NOT_FOUND, + message=f"Resource not found: {exc}", + details={**context, "reason": "not_found"}, + retryable=False, + ) + + # Rate limiting + if any(keyword in exc_str for keyword in ["rate limit", "too many requests", "429", "throttle"]): + return APIError( + error_code=APIErrorCode.RATE_LIMITED, + message=f"Rate limit exceeded: {exc}", + details={**context, "reason": "rate_limited"}, + retryable=True, + http_status=429, + ) + + # DRM errors + if any(keyword in exc_str for keyword in ["drm", "license", "widevine", "playready", "decrypt"]): + return APIError( + error_code=APIErrorCode.DRM_ERROR, + message=f"DRM error: {exc}", + details={**context, "reason": "drm_failure"}, + retryable=False, + ) + + # Service unavailable + if any(keyword in exc_str for keyword in ["service unavailable", "503", "maintenance", "temporarily unavailable"]): + return APIError( + error_code=APIErrorCode.SERVICE_UNAVAILABLE, + message=f"Service temporarily unavailable: {exc}", + details={**context, "reason": "service_unavailable"}, + retryable=True, + http_status=503, + ) + + # Validation errors + if any(keyword in exc_str for keyword in ["invalid", "malformed", "validation"]) or exc_type in [ + "ValueError", + "ValidationError", + ]: + return APIError( + error_code=APIErrorCode.INVALID_INPUT, + message=f"Invalid input: {exc}", + details={**context, "reason": "validation_failed"}, + retryable=False, + ) + + # Default to internal error for unknown exceptions + return APIError( + error_code=APIErrorCode.INTERNAL_ERROR, + message=f"An unexpected error occurred: {exc}", + details={**context, "exception_type": exc_type}, + retryable=False, + ) + + +def handle_api_exception( + exc: Exception, + context: dict[str, Any] | None = None, + debug_mode: bool = False, + extra_debug_info: dict[str, Any] | None = None, +) -> web.Response: + """ + Convenience function to categorize an exception and build an error response. + + Args: + exc: The exception to handle + context: Optional context information + debug_mode: Whether to include debug information + extra_debug_info: Additional debug info + + Returns: + Structured JSON error response + """ + if isinstance(exc, APIError): + api_error = exc + else: + api_error = categorize_exception(exc, context) + + return build_error_response(api_error, debug_mode, extra_debug_info) diff --git a/reset/packages/envied/src/envied/core/api/handlers.py b/reset/packages/envied/src/envied/core/api/handlers.py new file mode 100644 index 0000000..a4cdaa4 --- /dev/null +++ b/reset/packages/envied/src/envied/core/api/handlers.py @@ -0,0 +1,2396 @@ +import asyncio +import enum +import logging +from typing import Any, Dict, List, Optional + +from aiohttp import web + +from envied.core.api.errors import APIError, APIErrorCode, handle_api_exception +from envied.core.api.input_bridge import AuthStatus, InputBridge +from envied.core.config import config +from envied.core.constants import AUDIO_CODEC_MAP, DYNAMIC_RANGE_MAP, VIDEO_CODEC_MAP +from envied.core.proxies.resolve import initialize_proxy_providers, resolve_proxy +from envied.core.services import Services +from envied.core.titles import Episode, Movie, Title_T +from envied.core.tracks import Audio, Subtitle, Video + +log = logging.getLogger("api") + + +def sanitize_log(value: object) -> str: + """Sanitize a value for safe logging by removing newlines and control characters.""" + return str(value).replace("\n", "").replace("\r", "").replace("\x00", "") + + +DEFAULT_DOWNLOAD_PARAMS = { + "profile": None, + "quality": [], + "vcodec": None, + "acodec": None, + "vbitrate": None, + "abitrate": None, + "vbitrate_range": None, + "abitrate_range": None, + "range": ["SDR"], + "channels": None, + "no_atmos": False, + "wanted": [], + "latest_episode": False, + "lang": ["orig"], + "v_lang": [], + "a_lang": [], + "s_lang": ["all"], + "require_subs": [], + "forced_subs": False, + "exact_lang": False, + "sub_format": None, + "video_only": False, + "audio_only": False, + "subs_only": False, + "chapters_only": False, + "no_subs": False, + "no_audio": False, + "no_chapters": False, + "no_video": False, + "audio_description": False, + "slow": None, + "split_audio": None, + "skip_dl": False, + "export": False, + "cdm_only": None, + "proxy": None, + "no_proxy": False, + "no_proxy_download": False, + "no_folder": False, + "no_source": False, + "no_mux": False, + "workers": None, + "downloads": 1, + "worst": False, + "best_available": False, + "repack": False, + "tag": None, + "tmdb_id": None, + "imdb_id": None, + "animeapi_id": None, + "enrich": False, + "output_dir": None, + "no_cache": False, + "reset_cache": False, +} + + +# Keys that are part of the API transport envelope, not service.cli options. +# Used by instantiate_service to avoid passing them as kwargs to a service. +LIST_HANDLER_TRANSPORT_KEYS = { + "service", + "title_id", + "profile", + "season", + "episode", + "wanted", + "proxy", + "no_proxy", + "query", +} + + +def load_full_cdm(service: str, profile: Optional[str], cdm_type: Optional[str] = None) -> Optional[Any]: + """Load a real CDM object for the given service. + + Services often touch ``ctx.obj.cdm.security_level`` / ``.device_type`` / ``.system_id`` + inside ``__init__``, so the lightweight ``_resolve_server_cdm`` stub is not enough + for list_titles / list_tracks / search. Mirrors ``dl.get_cdm`` selection logic but + skips the quality-tier shortcuts (no track context yet) and falls back to the stub + if no device is configured or loading fails. + """ + from envied.core.cdm import load_cdm + from envied.core.config import config as app_config + + cdm_name = app_config.cdm.get(service) or app_config.cdm.get("default") + if isinstance(cdm_name, dict): + lower_keys = {k.lower(): v for k, v in cdm_name.items()} + if {"widevine", "playready"} & lower_keys.keys(): + drm_key = None + if cdm_type: + drm_key = {"wv": "widevine", "widevine": "widevine", "pr": "playready", "playready": "playready"}.get( + cdm_type.lower() + ) + cdm_name = lower_keys.get(drm_key or "widevine") or lower_keys.get("playready") + else: + cdm_name = cdm_name.get(profile) or cdm_name.get("default") or app_config.cdm.get("default") + + if not cdm_name or not isinstance(cdm_name, str): + return _resolve_server_cdm(service, profile, cdm_type) + + try: + return load_cdm(cdm_name, service_name=service) + except Exception as exc: # noqa: BLE001 - fall back to stub on load failure + log.warning(f"load_cdm({cdm_name!r}) failed for {service}: {exc}; using lightweight stub") + return _resolve_server_cdm(service, profile, cdm_type) + + +def load_service_yaml(normalized_service: str) -> dict: + """Load a service's config.yaml and merge it with the global override block.""" + import yaml + + from envied.core.utils.collections import merge_dict + + service_config_path = Services.get_path(normalized_service) / config.filenames.config + if service_config_path.exists(): + service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8")) or {} + else: + service_config = {} + merge_dict(config.services.get(normalized_service), service_config) + return service_config + + +def build_parent_ctx( + profile: Optional[str], + cdm: Any, + proxy_param: Optional[str], + no_proxy: bool, + proxy_providers: list, + service_config: dict, + extra_params: Optional[Dict[str, Any]] = None, +) -> Any: + """Build a parent click Context for invoking a service.cli via ctx.invoke(). + + The service's CLI callback uses ``ctx.parent.params`` (proxy, range_, vcodec, etc.) + and ``ctx.obj`` (ContextData). Both flow through Click's parent chain. + """ + import click + + from envied.core.utils.click_types import ContextData + + @click.command() + @click.pass_context + def dummy(ctx: click.Context) -> None: + pass + + parent = click.Context(dummy) + parent.obj = ContextData(config=service_config, cdm=cdm, proxy_providers=proxy_providers, profile=profile) + params = {"proxy": proxy_param, "no_proxy": no_proxy} + if extra_params: + params.update(extra_params) + parent.params = params + return parent + + +def instantiate_service( + parent_ctx: Any, + service_module: Any, + title: str, + data: Optional[Dict[str, Any]] = None, + transport_keys: Optional[set] = None, +) -> Any: + """Instantiate a service by invoking its click cli through Click. + + Click fills option defaults via ``param.get_default()`` and runs type coercion, + so we no longer have to inspect ``__init__`` or stitch defaults by hand. Extra + kwargs are pulled from ``data`` when the key matches a cli option name and is + not in the transport-key blocklist. + """ + cli_params = getattr(getattr(service_module, "cli", None), "params", []) or [] + cli_param_names = {p.name for p in cli_params if hasattr(p, "name") and p.name} + transport_keys = transport_keys or set() + extras: Dict[str, Any] = {} + if data: + for k, v in data.items(): + if k in cli_param_names and k not in transport_keys and k != "title": + extras[k] = v + return parent_ctx.invoke(service_module.cli, title=title, **extras) + + +def get_allowed_services(request: Optional[web.Request] = None) -> Optional[List[str]]: + """Get effective service allowlist considering global + per-key config. + + Returns None if all services are allowed. + """ + global_allowed = config.serve.get("services") + global_set: Optional[set[str]] = None + if global_allowed: + global_set = {Services.get_tag(s) for s in global_allowed} + + key_set: Optional[set[str]] = None + if request: + secret_key = request.headers.get("X-Secret-Key") + if secret_key: + users = config.serve.get("users", {}) + user_config = users.get(secret_key, {}) + user_services = user_config.get("services") + if user_services: + key_set = {Services.get_tag(s) for s in user_services} + + if global_set and key_set: + result = global_set & key_set + elif global_set: + result = global_set + elif key_set: + result = key_set + else: + return None + + return list(result) + + +def validate_service(service_tag: str, request: Optional[web.Request] = None) -> Optional[str]: + """Validate, normalize, and check allowlist for service tag.""" + try: + normalized = Services.get_tag(service_tag) + service_path = Services.get_path(normalized) + if not service_path.exists(): + return None + allowed = get_allowed_services(request) + if allowed is not None and normalized not in allowed: + return None + return normalized + except Exception: + return None + + +def serialize_title(title: Title_T) -> Dict[str, Any]: + """Convert a title object to JSON-serializable dict.""" + title_language = str(title.language) if hasattr(title, "language") and title.language else None + + if isinstance(title, Episode): + episode_name = title.name if title.name else f"Episode {title.number:02d}" + result = { + "type": "episode", + "name": episode_name, + "series_title": str(title.title), + "season": title.season, + "number": title.number, + "year": title.year, + "id": str(title.id) if hasattr(title, "id") else None, + "language": title_language, + } + elif isinstance(title, Movie): + result = { + "type": "movie", + "name": str(title.name) if hasattr(title, "name") else str(title), + "year": title.year, + "id": str(title.id) if hasattr(title, "id") else None, + "language": title_language, + } + else: + result = { + "type": "other", + "name": str(title.name) if hasattr(title, "name") else str(title), + "id": str(title.id) if hasattr(title, "id") else None, + "language": title_language, + } + + return result + + +def _extract_manifests(tracks) -> List[Dict[str, Any]]: + """Extract manifest data from tracks for client-side re-parsing. + + Serializes DASH and ISM manifest XML as zlib-compressed base64 strings + so the client can reconstruct track.data locally. HLS tracks download + directly from their URL so no manifest serialization is needed. + """ + import base64 + import zlib + + from lxml import etree + + from envied.core.config import config as app_config + + compression_level = app_config.serve.get("compression_level", 1) + + seen: set[str] = set() + manifests: List[Dict[str, Any]] = [] + + for track in list(tracks.videos) + list(tracks.audio) + list(tracks.subtitles): + manifest_url = str(track.url) if track.url else None + if not manifest_url or manifest_url in seen: + continue + + if track.data.get("dash") and track.data["dash"].get("manifest"): + seen.add(manifest_url) + xml_bytes = etree.tostring(track.data["dash"]["manifest"], xml_declaration=True, encoding="UTF-8") + compressed = zlib.compress(xml_bytes, compression_level) if compression_level else xml_bytes + manifests.append( + { + "type": "dash", + "url": manifest_url, + "data": base64.b64encode(compressed).decode("ascii"), + } + ) + elif track.data.get("ism") and track.data["ism"].get("manifest"): + seen.add(manifest_url) + xml_bytes = etree.tostring(track.data["ism"]["manifest"], xml_declaration=True, encoding="UTF-8") + compressed = zlib.compress(xml_bytes, compression_level) if compression_level else xml_bytes + manifests.append( + { + "type": "ism", + "url": manifest_url, + "data": base64.b64encode(compressed).decode("ascii"), + } + ) + + return manifests + + +def serialize_drm(drm_list) -> Optional[List[Dict[str, Any]]]: + """Serialize DRM objects to JSON-serializable list.""" + if not drm_list: + return None + + if not isinstance(drm_list, list): + drm_list = [drm_list] + + result = [] + for drm in drm_list: + drm_info = {} + drm_class = drm.__class__.__name__ + drm_info["type"] = drm_class.lower() + + # Get PSSH - handle both Widevine and PlayReady + if hasattr(drm, "_pssh") and drm._pssh: + pssh_obj = None + try: + pssh_obj = drm._pssh + # Try to get base64 representation + if hasattr(pssh_obj, "dumps"): + # pywidevine PSSH has dumps() method + drm_info["pssh"] = pssh_obj.dumps() + elif hasattr(pssh_obj, "__bytes__"): + # Convert to base64 + import base64 + + drm_info["pssh"] = base64.b64encode(bytes(pssh_obj)).decode() + elif hasattr(pssh_obj, "to_base64"): + drm_info["pssh"] = pssh_obj.to_base64() + else: + # Fallback - str() works for pywidevine PSSH + pssh_str = str(pssh_obj) + # Check if it's already base64-like or an object repr + if not pssh_str.startswith("<"): + drm_info["pssh"] = pssh_str + except (ValueError, TypeError, KeyError): + # Some PSSH implementations can fail to parse/serialize; log and continue. + pssh_type = type(pssh_obj).__name__ if pssh_obj is not None else None + log.warning( + "Failed to extract/serialize PSSH for DRM type=%s pssh_type=%s", + drm_class, + pssh_type, + exc_info=True, + ) + except Exception: + # Don't silently swallow unexpected failures; make them visible and propagate. + pssh_type = type(pssh_obj).__name__ if pssh_obj is not None else None + log.exception( + "Unexpected error while extracting/serializing PSSH for DRM type=%s pssh_type=%s", + drm_class, + pssh_type, + ) + raise + + # Get KIDs + if hasattr(drm, "kids") and drm.kids: + drm_info["kids"] = [str(kid) for kid in drm.kids] + + # Get content keys if available + if hasattr(drm, "content_keys") and drm.content_keys: + drm_info["content_keys"] = {str(k): v for k, v in drm.content_keys.items()} + + # Get license URL - essential for remote licensing + if hasattr(drm, "license_url") and drm.license_url: + drm_info["license_url"] = str(drm.license_url) + elif hasattr(drm, "_license_url") and drm._license_url: + drm_info["license_url"] = str(drm._license_url) + + result.append(drm_info) + + return result if result else None + + +def serialize_video_track(track: Video, include_url: bool = False) -> Dict[str, Any]: + """Convert video track to JSON-serializable dict.""" + codec_name = track.codec.name if hasattr(track.codec, "name") else str(track.codec) + range_name = track.range.name if hasattr(track.range, "name") else str(track.range) + + # Serialize the manifest descriptor (HLS, DASH, URL, etc.) + descriptor_name = None + if hasattr(track, "descriptor") and track.descriptor: + descriptor_name = track.descriptor.name if hasattr(track.descriptor, "name") else str(track.descriptor) + + result = { + "id": str(track.id), + "codec": codec_name, + "codec_display": VIDEO_CODEC_MAP.get(codec_name, codec_name), + "bitrate": int(track.bitrate / 1000) if track.bitrate else None, + "width": track.width, + "height": track.height, + "resolution": f"{track.width}x{track.height}" if track.width and track.height else None, + "fps": track.fps if track.fps else None, + "range": range_name, + "range_display": DYNAMIC_RANGE_MAP.get(range_name, range_name), + "language": str(track.language) if track.language else None, + "drm": serialize_drm(track.drm) if hasattr(track, "drm") and track.drm else None, + "descriptor": descriptor_name, + } + if include_url and hasattr(track, "url") and track.url: + result["url"] = str(track.url) + return result + + +def serialize_audio_track(track: Audio, include_url: bool = False) -> Dict[str, Any]: + """Convert audio track to JSON-serializable dict.""" + codec_name = track.codec.name if hasattr(track.codec, "name") else str(track.codec) + + # Serialize the manifest descriptor (HLS, DASH, URL, etc.) + descriptor_name = None + if hasattr(track, "descriptor") and track.descriptor: + descriptor_name = track.descriptor.name if hasattr(track.descriptor, "name") else str(track.descriptor) + + result = { + "id": str(track.id), + "codec": codec_name, + "codec_display": AUDIO_CODEC_MAP.get(codec_name, codec_name), + "bitrate": int(track.bitrate / 1000) if track.bitrate else None, + "channels": track.channels if track.channels else None, + "language": str(track.language) if track.language else None, + "atmos": track.atmos if hasattr(track, "atmos") else False, + "descriptive": track.descriptive if hasattr(track, "descriptive") else False, + "drm": serialize_drm(track.drm) if hasattr(track, "drm") and track.drm else None, + "descriptor": descriptor_name, + } + if include_url and hasattr(track, "url") and track.url: + result["url"] = str(track.url) + return result + + +def serialize_subtitle_track(track: Subtitle, include_url: bool = False) -> Dict[str, Any]: + """Convert subtitle track to JSON-serializable dict.""" + # Get descriptor for compatibility + descriptor_name = None + if hasattr(track, "descriptor") and track.descriptor: + descriptor_name = track.descriptor.name if hasattr(track.descriptor, "name") else str(track.descriptor) + + result = { + "id": str(track.id), + "codec": track.codec.name if hasattr(track.codec, "name") else str(track.codec), + "language": str(track.language) if track.language else None, + "forced": track.forced if hasattr(track, "forced") else False, + "sdh": track.sdh if hasattr(track, "sdh") else False, + "cc": track.cc if hasattr(track, "cc") else False, + "descriptor": descriptor_name, + } + if include_url and hasattr(track, "url") and track.url: + result["url"] = str(track.url) + return result + + +async def search_handler(data: Dict[str, Any], request: Optional[web.Request] = None) -> web.Response: + """Handle search request.""" + from envied.commands.dl import dl + + service_tag = data.get("service") + query = data.get("query") + + if not service_tag: + raise APIError(APIErrorCode.MISSING_SERVICE, "Missing required 'service' field") + if not query: + raise APIError(APIErrorCode.INVALID_PARAMETERS, "Missing required 'query' field") + + normalized_service = Services.get_tag(service_tag) + if not normalized_service: + raise APIError( + APIErrorCode.INVALID_SERVICE, + f"Service '{service_tag}' not found", + details={"service": service_tag}, + ) + + allowed = get_allowed_services(request) + if allowed is not None and normalized_service not in allowed: + raise APIError( + APIErrorCode.INVALID_SERVICE, + f"Service '{service_tag}' not found", + details={"service": service_tag}, + ) + + profile = data.get("profile") + proxy_param = data.get("proxy") + no_proxy = data.get("no_proxy", False) + + service_config = load_service_yaml(normalized_service) + + proxy_providers = [] + if not no_proxy: + proxy_providers = initialize_proxy_providers() + + if proxy_param and not no_proxy: + try: + resolved_proxy = resolve_proxy(proxy_param, proxy_providers) + proxy_param = resolved_proxy + except ValueError as e: + raise APIError( + APIErrorCode.INVALID_PROXY, + f"Proxy error: {e}", + details={"proxy": proxy_param, "service": normalized_service}, + ) + + cdm = load_full_cdm(normalized_service, profile, data.get("cdm_type")) + parent_ctx = build_parent_ctx(profile, cdm, proxy_param, no_proxy, proxy_providers, service_config) + service_module = Services.load(normalized_service) + + try: + service_instance = instantiate_service(parent_ctx, service_module, query) + except Exception as exc: + raise APIError( + APIErrorCode.SERVICE_ERROR, + f"Failed to initialize service: {exc}", + details={"service": normalized_service}, + ) + + # Authenticate + cookies = dl.get_cookie_jar(normalized_service, profile) + credential = dl.get_credentials(normalized_service, profile) + service_instance.authenticate(cookies, credential) + + # Search + results = [] + try: + for result in service_instance.search(): + results.append( + { + "id": result.id, + "title": result.title, + "description": result.description, + "label": result.label, + "url": result.url, + } + ) + except NotImplementedError: + raise APIError( + APIErrorCode.SERVICE_ERROR, + f"Search is not supported by {normalized_service}", + details={"service": normalized_service}, + ) + + return web.json_response({"results": results, "count": len(results)}) + + +async def list_titles_handler(data: Dict[str, Any], request: Optional[web.Request] = None) -> web.Response: + """Handle list-titles request.""" + service_tag = data.get("service") + title_id = data.get("title_id") + profile = data.get("profile") + + if not service_tag: + raise APIError( + APIErrorCode.INVALID_INPUT, + "Missing required parameter: service", + details={"missing_parameter": "service"}, + ) + + if not title_id: + raise APIError( + APIErrorCode.INVALID_INPUT, + "Missing required parameter: title_id", + details={"missing_parameter": "title_id"}, + ) + + normalized_service = validate_service(service_tag, request) + if not normalized_service: + raise APIError( + APIErrorCode.INVALID_SERVICE, + f"Invalid or unavailable service: {service_tag}", + details={"service": service_tag}, + ) + + try: + from envied.commands.dl import dl + + service_config = load_service_yaml(normalized_service) + + proxy_param = data.get("proxy") + no_proxy = data.get("no_proxy", False) + proxy_providers = [] + + if not no_proxy: + proxy_providers = initialize_proxy_providers() + + if proxy_param and not no_proxy: + try: + resolved_proxy = resolve_proxy(proxy_param, proxy_providers) + proxy_param = resolved_proxy + except ValueError as e: + raise APIError( + APIErrorCode.INVALID_PROXY, + f"Proxy error: {e}", + details={"proxy": proxy_param, "service": normalized_service}, + ) + + cdm = load_full_cdm(normalized_service, profile, data.get("cdm_type")) + parent_ctx = build_parent_ctx(profile, cdm, proxy_param, no_proxy, proxy_providers, service_config) + service_module = Services.load(normalized_service) + service_instance = instantiate_service(parent_ctx, service_module, title_id, data, LIST_HANDLER_TRANSPORT_KEYS) + + cookies = dl.get_cookie_jar(normalized_service, profile) + credential = dl.get_credentials(normalized_service, profile) + service_instance.authenticate(cookies, credential) + + titles = service_instance.get_titles() + + if hasattr(titles, "__iter__") and not isinstance(titles, str): + title_list = [serialize_title(t) for t in titles] + else: + title_list = [serialize_title(titles)] + + return web.json_response({"titles": title_list}) + + except APIError: + raise + except (Exception, SystemExit) as e: + log.exception("Error listing titles") + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "list_titles", "service": normalized_service, "title_id": title_id}, + debug_mode=debug_mode, + ) + + +async def list_tracks_handler(data: Dict[str, Any], request: Optional[web.Request] = None) -> web.Response: + """Handle list-tracks request.""" + service_tag = data.get("service") + title_id = data.get("title_id") + profile = data.get("profile") + + if not service_tag: + raise APIError( + APIErrorCode.INVALID_INPUT, + "Missing required parameter: service", + details={"missing_parameter": "service"}, + ) + + if not title_id: + raise APIError( + APIErrorCode.INVALID_INPUT, + "Missing required parameter: title_id", + details={"missing_parameter": "title_id"}, + ) + + normalized_service = validate_service(service_tag, request) + if not normalized_service: + raise APIError( + APIErrorCode.INVALID_SERVICE, + f"Invalid or unavailable service: {service_tag}", + details={"service": service_tag}, + ) + + try: + from envied.commands.dl import dl + + service_config = load_service_yaml(normalized_service) + + proxy_param = data.get("proxy") + no_proxy = data.get("no_proxy", False) + proxy_providers = [] + + if not no_proxy: + proxy_providers = initialize_proxy_providers() + + if proxy_param and not no_proxy: + try: + resolved_proxy = resolve_proxy(proxy_param, proxy_providers) + proxy_param = resolved_proxy + except ValueError as e: + raise APIError( + APIErrorCode.INVALID_PROXY, + f"Proxy error: {e}", + details={"proxy": proxy_param, "service": normalized_service}, + ) + + cdm = load_full_cdm(normalized_service, profile, data.get("cdm_type")) + parent_ctx = build_parent_ctx(profile, cdm, proxy_param, no_proxy, proxy_providers, service_config) + service_module = Services.load(normalized_service) + service_instance = instantiate_service(parent_ctx, service_module, title_id, data, LIST_HANDLER_TRANSPORT_KEYS) + + cookies = dl.get_cookie_jar(normalized_service, profile) + credential = dl.get_credentials(normalized_service, profile) + service_instance.authenticate(cookies, credential) + + titles = service_instance.get_titles() + + wanted_param = data.get("wanted") + season = data.get("season") + episode = data.get("episode") + + if hasattr(titles, "__iter__") and not isinstance(titles, str): + titles_list = list(titles) + + wanted = None + if wanted_param: + from envied.core.utils.click_types import SeasonRange + + try: + season_range = SeasonRange() + if isinstance(wanted_param, list): + wanted = season_range.parse_tokens(*wanted_param) + else: + wanted = season_range.parse_tokens(wanted_param) + log.debug(f"Parsed wanted '{wanted_param}' into {len(wanted)} episodes: {wanted[:10]}...") + except (Exception, SystemExit) as e: + raise APIError( + APIErrorCode.INVALID_PARAMETERS, + f"Invalid wanted parameter: {e}", + details={"wanted": wanted_param, "service": normalized_service}, + ) + elif season is not None and episode is not None: + wanted = [f"{season}x{episode}"] + + if wanted: + # Filter titles based on wanted episodes, similar to how dl.py does it + matching_titles = [] + log.debug(f"Filtering {len(titles_list)} titles with {len(wanted)} wanted episodes") + for title in titles_list: + if isinstance(title, Episode): + episode_key = f"{title.season}x{title.number}" + if episode_key in wanted: + log.debug(f"Episode {episode_key} matches wanted list") + matching_titles.append(title) + else: + log.debug(f"Episode {episode_key} not in wanted list") + else: + matching_titles.append(title) + + log.debug(f"Found {len(matching_titles)} matching titles") + + if not matching_titles: + raise APIError( + APIErrorCode.NO_CONTENT, + "No episodes found matching wanted criteria", + details={ + "service": normalized_service, + "title_id": title_id, + "wanted": wanted_param or f"{season}x{episode}", + }, + ) + + # If multiple episodes match, return tracks for all episodes + if len(matching_titles) > 1 and all(isinstance(t, Episode) for t in matching_titles): + episodes_data = [] + failed_episodes = [] + + # Sort matching titles by season and episode number for consistent ordering + sorted_titles = sorted(matching_titles, key=lambda t: (t.season, t.number)) + + for title in sorted_titles: + try: + tracks = service_instance.get_tracks(title) + video_tracks = sorted(tracks.videos, key=lambda t: t.bitrate or 0, reverse=True) + audio_tracks = sorted(tracks.audio, key=lambda t: t.bitrate or 0, reverse=True) + + episode_data = { + "title": serialize_title(title), + "video": [serialize_video_track(t) for t in video_tracks], + "audio": [serialize_audio_track(t) for t in audio_tracks], + "subtitles": [serialize_subtitle_track(t) for t in tracks.subtitles], + } + episodes_data.append(episode_data) + log.debug(f"Successfully got tracks for {title.season}x{title.number}") + except SystemExit: + # Service calls sys.exit() for unavailable episodes - catch and skip + failed_episodes.append(f"S{title.season}E{title.number:02d}") + log.debug(f"Episode {title.season}x{title.number} not available, skipping") + continue + except (Exception, SystemExit) as e: + # Handle other errors gracefully + failed_episodes.append(f"S{title.season}E{title.number:02d}") + log.debug(f"Error getting tracks for {title.season}x{title.number}: {e}") + continue + + if episodes_data: + response = {"episodes": episodes_data} + if failed_episodes: + response["unavailable_episodes"] = failed_episodes + return web.json_response(response) + else: + raise APIError( + APIErrorCode.NO_CONTENT, + f"No available episodes found. Unavailable: {', '.join(failed_episodes)}", + details={ + "service": normalized_service, + "title_id": title_id, + "unavailable_episodes": failed_episodes, + }, + ) + else: + # Single episode or movie + first_title = matching_titles[0] + else: + first_title = titles_list[0] + else: + first_title = titles + + tracks = service_instance.get_tracks(first_title) + + video_tracks = sorted(tracks.videos, key=lambda t: t.bitrate or 0, reverse=True) + audio_tracks = sorted(tracks.audio, key=lambda t: t.bitrate or 0, reverse=True) + + response = { + "title": serialize_title(first_title), + "video": [serialize_video_track(t) for t in video_tracks], + "audio": [serialize_audio_track(t) for t in audio_tracks], + "subtitles": [serialize_subtitle_track(t) for t in tracks.subtitles], + } + + return web.json_response(response) + + except APIError: + raise + except (Exception, SystemExit) as e: + log.exception("Error listing tracks") + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "list_tracks", "service": normalized_service, "title_id": title_id}, + debug_mode=debug_mode, + ) + + +def validate_download_parameters(data: Dict[str, Any]) -> Optional[str]: + """ + Validate download parameters and return error message if invalid. + + Returns: + None if valid, error message string if invalid + """ + if "vcodec" in data and data["vcodec"]: + valid_vcodecs = ["H264", "H265", "H.264", "H.265", "AVC", "HEVC", "VC1", "VC-1", "VP8", "VP9", "AV1"] + if isinstance(data["vcodec"], str): + vcodec_values = [v.strip() for v in data["vcodec"].split(",") if v.strip()] + elif isinstance(data["vcodec"], list): + vcodec_values = [str(v).strip() for v in data["vcodec"] if str(v).strip()] + else: + return "vcodec must be a string or list" + + invalid = [value for value in vcodec_values if value.upper() not in valid_vcodecs] + if invalid: + return f"Invalid vcodec: {', '.join(invalid)}. Must be one of: {', '.join(valid_vcodecs)}" + + if "acodec" in data and data["acodec"]: + valid_acodecs = [ + "AAC", + "AC3", + "EC3", + "EAC3", + "DD", + "DD+", + "AC4", + "OPUS", + "FLAC", + "ALAC", + "VORBIS", + "OGG", + "DTS", + ] + if isinstance(data["acodec"], str): + acodec_values = [v.strip() for v in data["acodec"].split(",") if v.strip()] + elif isinstance(data["acodec"], list): + acodec_values = [str(v).strip() for v in data["acodec"] if str(v).strip()] + else: + return "acodec must be a string or list" + + invalid = [value for value in acodec_values if value.upper() not in valid_acodecs] + if invalid: + return f"Invalid acodec: {', '.join(invalid)}. Must be one of: {', '.join(valid_acodecs)}" + + if "sub_format" in data and data["sub_format"]: + valid_sub_formats = ["SRT", "VTT", "ASS", "SSA", "TTML", "STPP", "WVTT", "SMI", "SUB", "MPL2", "TMP"] + if data["sub_format"].upper() not in valid_sub_formats: + return f"Invalid sub_format: {data['sub_format']}. Must be one of: {', '.join(valid_sub_formats)}" + + if "vbitrate" in data and data["vbitrate"] is not None: + if not isinstance(data["vbitrate"], int) or data["vbitrate"] <= 0: + return "vbitrate must be a positive integer" + + if "abitrate" in data and data["abitrate"] is not None: + if not isinstance(data["abitrate"], int) or data["abitrate"] <= 0: + return "abitrate must be a positive integer" + + if "vbitrate_range" in data and data["vbitrate_range"] is not None: + if not isinstance(data["vbitrate_range"], str) or "-" not in data["vbitrate_range"]: + return "vbitrate_range must be a string in 'MIN-MAX' format (e.g., '6000-7000')" + + if "abitrate_range" in data and data["abitrate_range"] is not None: + if not isinstance(data["abitrate_range"], str) or "-" not in data["abitrate_range"]: + return "abitrate_range must be a string in 'MIN-MAX' format (e.g., '128-256')" + + if "channels" in data and data["channels"] is not None: + if not isinstance(data["channels"], (int, float)) or data["channels"] <= 0: + return "channels must be a positive number" + + if "workers" in data and data["workers"] is not None: + if not isinstance(data["workers"], int) or data["workers"] <= 0: + return "workers must be a positive integer" + + if "downloads" in data and data["downloads"] is not None: + if not isinstance(data["downloads"], int) or data["downloads"] <= 0: + return "downloads must be a positive integer" + + exclusive_flags = [] + if data.get("video_only"): + exclusive_flags.append("video_only") + if data.get("audio_only"): + exclusive_flags.append("audio_only") + if data.get("subs_only"): + exclusive_flags.append("subs_only") + if data.get("chapters_only"): + exclusive_flags.append("chapters_only") + + if len(exclusive_flags) > 1: + return f"Cannot use multiple exclusive flags: {', '.join(exclusive_flags)}" + + if data.get("no_subs") and data.get("subs_only"): + return "Cannot use both no_subs and subs_only" + if data.get("no_audio") and data.get("audio_only"): + return "Cannot use both no_audio and audio_only" + + if data.get("s_lang") and data.get("require_subs"): + return "Cannot use both s_lang and require_subs" + + if "range" in data and data["range"]: + valid_ranges = ["SDR", "HDR10", "HDR10+", "DV", "HLG", "HYBRID"] + if isinstance(data["range"], list): + for r in data["range"]: + if r.upper() not in valid_ranges: + return f"Invalid range value: {r}. Must be one of: {', '.join(valid_ranges)}" + elif data["range"].upper() not in valid_ranges: + return f"Invalid range value: {data['range']}. Must be one of: {', '.join(valid_ranges)}" + + return None + + +async def download_handler(data: Dict[str, Any], request: Optional[web.Request] = None) -> web.Response: + """Handle download request - create and queue a download job.""" + from envied.core.api.download_manager import get_download_manager + + service_tag = data.get("service") + title_id = data.get("title_id") + + if not service_tag: + raise APIError( + APIErrorCode.INVALID_INPUT, + "Missing required parameter: service", + details={"missing_parameter": "service"}, + ) + + if not title_id: + raise APIError( + APIErrorCode.INVALID_INPUT, + "Missing required parameter: title_id", + details={"missing_parameter": "title_id"}, + ) + + normalized_service = validate_service(service_tag, request) + if not normalized_service: + raise APIError( + APIErrorCode.INVALID_SERVICE, + f"Invalid or unavailable service: {service_tag}", + details={"service": service_tag}, + ) + + validation_error = validate_download_parameters(data) + if validation_error: + raise APIError( + APIErrorCode.INVALID_PARAMETERS, + validation_error, + details={"service": normalized_service, "title_id": title_id}, + ) + + try: + # Load service module to extract service-specific parameter defaults + service_module = Services.load(normalized_service) + service_specific_defaults = {} + + # Extract default values from the service's click command. + # Skip None defaults here: this dict overlays into job params; injecting + # None for keys like `profile` would clobber serve-config overrides. + # Missing required __init__ params are handled in download_manager._perform_download. + if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): + for param in service_module.cli.params: + if hasattr(param, "name") and param.default is not None and not isinstance(param.default, enum.Enum): + # Store service-specific defaults (e.g., drm_system, hydrate_track, profile for NF) + service_specific_defaults[param.name] = param.default + + # Get download manager and start workers if needed + manager = get_download_manager() + await manager.start_workers() + + # Create download job with filtered parameters (exclude service and title_id as they're already passed) + filtered_params = {k: v for k, v in data.items() if k not in ["service", "title_id"]} + # Overlay any dl-relevant keys from `serve:` config (e.g. downloads, workers) so the API + # respects server-side defaults without each client having to send them. + serve_overrides = { + k: v for k, v in (config.serve or {}).items() if k in DEFAULT_DOWNLOAD_PARAMS and v is not None + } + params_with_defaults = { + **DEFAULT_DOWNLOAD_PARAMS, + **serve_overrides, + **service_specific_defaults, + **filtered_params, + } + job = manager.create_job(normalized_service, title_id, **params_with_defaults) + + return web.json_response( + {"job_id": job.job_id, "status": job.status.value, "created_time": job.created_time.isoformat()}, status=202 + ) + + except APIError: + raise + except (Exception, SystemExit) as e: + log.exception("Error creating download job") + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "create_download_job", "service": normalized_service, "title_id": title_id}, + debug_mode=debug_mode, + ) + + +async def list_download_jobs_handler(data: Dict[str, Any], request: Optional[web.Request] = None) -> web.Response: + """Handle list download jobs request with optional filtering and sorting.""" + from envied.core.api.download_manager import get_download_manager + + try: + manager = get_download_manager() + jobs = manager.list_jobs() + + status_filter = data.get("status") + if status_filter: + jobs = [job for job in jobs if job.status.value == status_filter] + + service_filter = data.get("service") + if service_filter: + jobs = [job for job in jobs if job.service == service_filter] + + sort_by = data.get("sort_by", "created_time") + sort_order = data.get("sort_order", "desc") + + valid_sort_fields = ["created_time", "started_time", "completed_time", "progress", "status", "service"] + if sort_by not in valid_sort_fields: + raise APIError( + APIErrorCode.INVALID_PARAMETERS, + f"Invalid sort_by: {sort_by}. Must be one of: {', '.join(valid_sort_fields)}", + details={"sort_by": sort_by, "valid_values": valid_sort_fields}, + ) + + if sort_order not in ["asc", "desc"]: + raise APIError( + APIErrorCode.INVALID_PARAMETERS, + "Invalid sort_order: must be 'asc' or 'desc'", + details={"sort_order": sort_order, "valid_values": ["asc", "desc"]}, + ) + + reverse = sort_order == "desc" + + def get_sort_key(job): + """Get the sorting key value, handling None values.""" + value = getattr(job, sort_by, None) + if value is None: + if sort_by in ["created_time", "started_time", "completed_time"]: + from datetime import datetime + + return datetime.min if not reverse else datetime.max + elif sort_by == "progress": + return 0 + elif sort_by in ["status", "service"]: + return "" + return value + + jobs = sorted(jobs, key=get_sort_key, reverse=reverse) + + job_list = [job.to_dict(include_full_details=False) for job in jobs] + + return web.json_response({"jobs": job_list}) + + except APIError: + raise + except (Exception, SystemExit) as e: + log.exception("Error listing download jobs") + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "list_download_jobs"}, + debug_mode=debug_mode, + ) + + +async def get_download_job_handler(job_id: str, request: Optional[web.Request] = None) -> web.Response: + """Handle get specific download job request.""" + from envied.core.api.download_manager import get_download_manager + + try: + manager = get_download_manager() + job = manager.get_job(job_id) + + if not job: + raise APIError( + APIErrorCode.JOB_NOT_FOUND, + "Job not found", + details={"job_id": job_id}, + ) + + return web.json_response(job.to_dict(include_full_details=True)) + + except APIError: + raise + except (Exception, SystemExit) as e: + log.exception(f"Error getting download job {sanitize_log(job_id)}") + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "get_download_job", "job_id": job_id}, + debug_mode=debug_mode, + ) + + +async def cancel_download_job_handler(job_id: str, request: Optional[web.Request] = None) -> web.Response: + """Handle cancel download job request.""" + from envied.core.api.download_manager import get_download_manager + + try: + manager = get_download_manager() + + if not manager.get_job(job_id): + raise APIError( + APIErrorCode.JOB_NOT_FOUND, + "Job not found", + details={"job_id": job_id}, + ) + + success = manager.cancel_job(job_id) + + if success: + return web.json_response({"status": "success", "message": "Job cancelled"}) + else: + raise APIError( + APIErrorCode.INVALID_PARAMETERS, + "Job cannot be cancelled (already completed or failed)", + details={"job_id": job_id}, + ) + + except APIError: + raise + except (Exception, SystemExit) as e: + log.exception(f"Error cancelling download job {sanitize_log(job_id)}") + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "cancel_download_job", "job_id": job_id}, + debug_mode=debug_mode, + ) + + +# --------------------------------------------------------------------------- +# Remote-DL Session Handlers +# --------------------------------------------------------------------------- + + +SESSION_TRANSPORT_KEYS = { + "service", + "title_id", + "season", + "episode", + "wanted", + "proxy", + "no_proxy", + "credentials", + "cookies", + "cache", + "client_region", + "cdm_type", + "range_", + "vcodec", + "quality", + "best_available", +} + + +def _create_service_instance( + normalized_service: str, + title_id: str, + data: Dict[str, Any], + proxy_param: Optional[str], + proxy_providers: list, + profile: Optional[str], +) -> Any: + """Create and authenticate a service instance. + + Supports client-sent credentials/cookies (for remote-dl) with fallback + to server-local config (for backward compatibility). + """ + from envied.commands.dl import dl + from envied.core.credential import Credential + from envied.core.tracks import Video + + service_config = load_service_yaml(normalized_service) + cdm = load_full_cdm(normalized_service, profile, data.get("cdm_type")) + + # Reconstruct enum track-selection params from client data so service code that reads + # ctx.parent.params (Service.__init__ proxy/range/vcodec/best_available block) sees enums. + range_names = data.get("range_") + range_values: Optional[list] = None + if range_names: + range_values = [] + for name in range_names: + try: + range_values.append(Video.Range[name]) + except KeyError: + pass + range_values = range_values or None + + vcodec_names = data.get("vcodec") + vcodec_values: Optional[list] = None + if vcodec_names: + vcodec_values = [] + for name in vcodec_names: + try: + vcodec_values.append(Video.Codec[name]) + except KeyError: + pass + vcodec_values = vcodec_values or None + + extra_params = { + "range_": range_values, + "vcodec": vcodec_values, + "quality": data.get("quality"), + "best_available": data.get("best_available", False), + } + + parent_ctx = build_parent_ctx( + profile, + cdm, + proxy_param, + data.get("no_proxy", False), + proxy_providers, + service_config, + extra_params=extra_params, + ) + + service_module = Services.load(normalized_service) + service_instance = instantiate_service(parent_ctx, service_module, title_id, data, SESSION_TRANSPORT_KEYS) + + # Resolve credentials: client-sent > server-local + cred_data = data.get("credentials") + if cred_data and isinstance(cred_data, dict): + credential = Credential( + username=cred_data["username"], + password=cred_data["password"], + extra=cred_data.get("extra"), + ) + else: + credential = dl.get_credentials(normalized_service, profile) + + # Resolve cookies: client-sent > server-local + cookie_text = data.get("cookies") + if cookie_text and isinstance(cookie_text, str): + import base64 + import tempfile + import zlib + from http.cookiejar import MozillaCookieJar + + cookie_str = zlib.decompress(base64.b64decode(cookie_text)).decode("utf-8") + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False, encoding="utf-8") as f: + f.write(cookie_str) + tmp_path = f.name + try: + cookies = MozillaCookieJar(tmp_path) + cookies.load(ignore_discard=True, ignore_expires=True) + finally: + import os + + os.unlink(tmp_path) + else: + cookies = dl.get_cookie_jar(normalized_service, profile) + + return service_instance, cookies, credential + + +async def session_create_handler(data: Dict[str, Any], request: Optional[web.Request] = None) -> web.Response: + """Handle session creation: authenticate + get titles + get tracks + get chapters. + + This is the main entry point for remote-dl clients. It creates a persistent + session on the server with the authenticated service instance, fetches all + titles and tracks, and returns everything the client needs for track selection. + """ + from envied.core.api.session_store import get_session_store + + service_tag = data.get("service") + title_id = data.get("title_id") + profile = data.get("profile") + + if not service_tag: + raise APIError(APIErrorCode.INVALID_INPUT, "Missing required parameter: service") + if not title_id: + raise APIError(APIErrorCode.INVALID_INPUT, "Missing required parameter: title_id") + + normalized_service = validate_service(service_tag, request) + if not normalized_service: + raise APIError( + APIErrorCode.INVALID_SERVICE, + f"Invalid or unavailable service: {service_tag}", + details={"service": service_tag}, + ) + + try: + proxy_param, proxy_providers = _resolve_handler_proxy(data, normalized_service) + + import hashlib + import uuid as uuid_mod + + from envied.core.cacher import Cacher + from envied.core.config import config as app_config + + session_id = str(uuid_mod.uuid4()) + api_key = request.headers.get("X-Secret-Key", "anonymous") if request else "anonymous" + api_key_hash = hashlib.sha256(api_key.encode()).hexdigest()[:12] + session_cache_tag = f"_sessions/{api_key_hash}/{session_id}/{normalized_service}" + + service_instance, cookies, credential = _create_service_instance( + normalized_service, + title_id, + data, + proxy_param, + proxy_providers, + profile, + ) + + service_instance.cache = Cacher(session_cache_tag) + + cache_data = data.get("cache", {}) + if cache_data: + import base64 + import zlib + + cache_dir = app_config.directories.cache / session_cache_tag + cache_dir.mkdir(parents=True, exist_ok=True) + for key, content in cache_data.items(): + decompressed = zlib.decompress(base64.b64decode(content)).decode("utf-8") + (cache_dir / key).with_suffix(".json").write_text(decompressed, encoding="utf-8") + + bridge = InputBridge() + service_instance._input_bridge = bridge + + store = get_session_store() + session = await store.create( + normalized_service, + service_instance, + session_id=session_id, + ) + session.creator_ip = request.remote if request else None + session.cache_tag = session_cache_tag + session.input_bridge = bridge + session.auth_status = AuthStatus.AUTHENTICATING + + async def _run_auth() -> None: + try: + await asyncio.to_thread(service_instance.authenticate, cookies, credential) + session.auth_status = AuthStatus.AUTHENTICATED + bridge.status = AuthStatus.AUTHENTICATED + except (Exception, SystemExit) as e: + log.exception("Auth failed for session %s", session_id) + session.auth_status = AuthStatus.FAILED + session.auth_error = str(e) + bridge.status = AuthStatus.FAILED + bridge.error = str(e) + + asyncio.create_task(_run_auth()) + + return web.json_response( + { + "session_id": session.session_id, + "service": normalized_service, + "status": "authenticating", + } + ) + + except APIError: + raise + except (Exception, SystemExit) as e: + log.exception("Error creating session") + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "session_create", "service": service_tag, "title_id": title_id}, + debug_mode=debug_mode, + ) + + +async def session_titles_handler(session_id: str, request: Optional[web.Request] = None) -> web.Response: + """Get titles for the authenticated session. + + Called after session/create. This is separate from auth so that + interactive auth flows (OTP, captcha) can complete before titles + are fetched. + """ + session = await _get_validated_session(session_id, request) + _require_authenticated(session) + + try: + service_instance = session.service_instance + titles = service_instance.get_titles() + session.titles = titles + + # Serialize titles and build title map + if hasattr(titles, "__iter__") and not isinstance(titles, str): + titles_list = list(titles) + else: + titles_list = [titles] + + serialized_titles = [] + for t in titles_list: + tid = str(t.id) if hasattr(t, "id") else str(id(t)) + session.title_map[tid] = t + serialized_titles.append(serialize_title(t)) + + return web.json_response( + { + "session_id": session_id, + "titles": serialized_titles, + } + ) + + except (Exception, SystemExit) as e: + log.exception("Error getting titles") + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "session_titles", "session_id": session_id}, + debug_mode=debug_mode, + ) + + +async def session_tracks_handler( + data: Dict[str, Any], session_id: str, request: Optional[web.Request] = None +) -> web.Response: + """Get tracks and chapters for a specific title in the session. + + Called per-title by the client after session/create returns titles. + This keeps auth separate from track fetching, allowing interactive + auth flows (OTP, captcha) before any tracks are requested. + """ + session = await _get_validated_session(session_id, request) + _require_authenticated(session) + + title_id = data.get("title_id") + if not title_id: + raise APIError(APIErrorCode.INVALID_INPUT, "Missing required parameter: title_id") + + title = session.title_map.get(str(title_id)) + if not title: + raise APIError( + APIErrorCode.INVALID_INPUT, + f"Title not found in session: {title_id}", + details={"available_titles": list(session.title_map.keys())}, + ) + + try: + service_instance = session.service_instance + tracks = service_instance.get_tracks(title) + + title_tracks: Dict[str, Any] = {} + for track in tracks.videos: + title_tracks[str(track.id)] = track + session.tracks[str(track.id)] = track + for track in tracks.audio: + title_tracks[str(track.id)] = track + session.tracks[str(track.id)] = track + for track in tracks.subtitles: + title_tracks[str(track.id)] = track + session.tracks[str(track.id)] = track + session.tracks_by_title[str(title_id)] = title_tracks + + try: + chapters = service_instance.get_chapters(title) + session.chapters_by_title[str(title_id)] = chapters if chapters else [] + except (NotImplementedError, Exception): + session.chapters_by_title[str(title_id)] = [] + + video_tracks = sorted(tracks.videos, key=lambda t: t.bitrate or 0, reverse=True) + audio_tracks = sorted(tracks.audio, key=lambda t: t.bitrate or 0, reverse=True) + + manifests = _extract_manifests(tracks) + + svc_session = session.service_instance.session + session_headers = dict(svc_session.headers) if hasattr(svc_session, "headers") else {} + session_cookies = {} + if hasattr(svc_session, "cookies"): + for cookie in svc_session.cookies: + if hasattr(cookie, "name") and hasattr(cookie, "value"): + session_cookies[cookie.name] = cookie.value + + from envied.core.config import config as app_config + + api_key = request.headers.get("X-Secret-Key", "anonymous") if request else "anonymous" + user_cfg = app_config.serve.get("users", {}).get(api_key, {}) + has_wv = bool(user_cfg.get("devices")) + has_pr = bool(user_cfg.get("playready_devices")) + + service_tag = session.service_tag + config_cdm_type = _detect_cdm_type_for_service(service_tag, app_config) + + track_has_wv = any( + d.__class__.__name__ == "Widevine" for t in list(tracks.videos) + list(tracks.audio) if t.drm for d in t.drm + ) + track_has_pr = any( + d.__class__.__name__ == "PlayReady" + for t in list(tracks.videos) + list(tracks.audio) + if t.drm + for d in t.drm + ) + + if config_cdm_type: + server_cdm_type = config_cdm_type + elif track_has_pr and has_pr: + server_cdm_type = "playready" + elif track_has_wv and has_wv: + server_cdm_type = "widevine" + elif has_wv: + server_cdm_type = "widevine" + else: + server_cdm_type = "playready" + + return web.json_response( + { + "title": serialize_title(title), + "video": [serialize_video_track(t, include_url=True) for t in video_tracks], + "audio": [serialize_audio_track(t, include_url=True) for t in audio_tracks], + "subtitles": [serialize_subtitle_track(t, include_url=True) for t in tracks.subtitles], + "chapters": [ + {"timestamp": ch.timestamp, "name": ch.name} + for ch in session.chapters_by_title.get(str(title_id), []) + ], + "attachments": [ + {"url": a.url, "name": a.name, "mime_type": a.mime_type, "description": a.description} + for a in tracks.attachments + if hasattr(a, "url") and a.url + ], + "manifests": manifests, + "session_headers": session_headers, + "session_cookies": session_cookies, + "server_cdm_type": server_cdm_type, + } + ) + + except (Exception, SystemExit) as e: + log.exception(f"Error getting tracks for title {sanitize_log(title_id)}") + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "session_tracks", "session_id": session_id, "title_id": title_id}, + debug_mode=debug_mode, + ) + + +async def session_segments_handler( + data: Dict[str, Any], session_id: str, request: Optional[web.Request] = None +) -> web.Response: + """Resolve segment URLs for selected tracks. + + The client calls this after selecting which tracks to download. + Returns segment URLs, init data, DRM info, and any headers/cookies + needed for CDN download. + """ + session = await _get_validated_session(session_id, request) + + track_ids = data.get("track_ids", []) + if not track_ids: + raise APIError(APIErrorCode.INVALID_INPUT, "Missing required parameter: track_ids") + + try: + result: Dict[str, Any] = {} + + for track_id in track_ids: + track = session.tracks.get(track_id) + if not track: + raise APIError( + APIErrorCode.TRACK_NOT_FOUND, + f"Track not found in session: {track_id}", + details={"track_id": track_id, "session_id": session_id}, + ) + + descriptor_name = track.descriptor.name if hasattr(track.descriptor, "name") else str(track.descriptor) + + track_info: Dict[str, Any] = { + "descriptor": descriptor_name, + "url": str(track.url) if track.url else None, + "drm": serialize_drm(track.drm) if hasattr(track, "drm") and track.drm else None, + } + + # Extract session headers/cookies for CDN access + service_session = session.service_instance.session + if hasattr(service_session, "headers"): + # Only include relevant headers, not all session headers + headers = dict(service_session.headers) if service_session.headers else {} + track_info["headers"] = headers + else: + track_info["headers"] = {} + + if hasattr(service_session, "cookies"): + cookie_dict = {} + for cookie in service_session.cookies: + if hasattr(cookie, "name") and hasattr(cookie, "value"): + cookie_dict[cookie.name] = cookie.value + elif isinstance(cookie, str): + pass # Skip non-standard cookie objects + track_info["cookies"] = cookie_dict + else: + track_info["cookies"] = {} + + # Include manifest-specific data for segment resolution + if hasattr(track, "data") and track.data: + track_data = {} + for key, val in track.data.items(): + if isinstance(val, dict): + # Convert non-serializable values + serializable = {} + for k, v in val.items(): + try: + import json + + json.dumps(v) + serializable[k] = v + except (TypeError, ValueError): + serializable[k] = str(v) + track_data[key] = serializable + else: + try: + import json + + json.dumps(val) + track_data[key] = val + except (TypeError, ValueError): + track_data[key] = str(val) + track_info["data"] = track_data + else: + track_info["data"] = {} + + result[track_id] = track_info + + return web.json_response({"tracks": result}) + + except APIError: + raise + except (Exception, SystemExit) as e: + log.exception("Error resolving segments") + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={"operation": "session_segments", "session_id": session_id}, + debug_mode=debug_mode, + ) + + +class _CdmTypeStub: + """Lightweight CDM stub so ``is_playready_cdm()`` can detect CDM type. + + Used on the server when the client sends ``cdm_type`` but the server + does not need a full CDM (e.g. for cache key / device selection only). + """ + + def __init__(self, cdm_type: str) -> None: + self.is_playready = cdm_type == "playready" + + +def _resolve_server_cdm(service: str, profile: Optional[str], cdm_type: Optional[str]) -> Optional[Any]: + """Resolve CDM for the server context. + + Checks the server's own CDM config (``config.cdm[service]``) to + determine the CDM type without loading the full CDM object. This + ensures that when ``server_cdm: true`` is used, the server's CDM + determines device selection (e.g. PlayReady vs Widevine for AMZN). + + Falls back to a lightweight stub from *cdm_type* only if no server + CDM is configured for the service. + """ + from envied.core.config import config as app_config + + cdm_name = app_config.cdm.get(service) + if cdm_name: + if isinstance(cdm_name, dict): + lower_keys = {k.lower(): v for k, v in cdm_name.items()} + if {"widevine", "playready"} & lower_keys.keys(): + cdm_name = lower_keys.get("playready") or lower_keys.get("widevine") + else: + cdm_name = cdm_name.get("default") or next(iter(cdm_name.values()), None) + + if cdm_name and isinstance(cdm_name, str): + detected_type = _detect_cdm_type(cdm_name, app_config) + if detected_type: + return _CdmTypeStub(detected_type) + + if cdm_type: + return _CdmTypeStub(cdm_type) + return None + + +def _detect_cdm_type_for_service(service: str, app_config: Any) -> Optional[str]: + """Detect the CDM type configured for a service in config.cdm.""" + cdm_name = app_config.cdm.get(service) + if not cdm_name: + return None + if isinstance(cdm_name, dict): + lower_keys = {k.lower(): v for k, v in cdm_name.items()} + if {"widevine", "playready"} & lower_keys.keys(): + return "playready" if "playready" in lower_keys else "widevine" + cdm_name = cdm_name.get("default") or next(iter(cdm_name.values()), None) + if cdm_name and isinstance(cdm_name, str): + return _detect_cdm_type(cdm_name, app_config) + return None + + +def _detect_cdm_type(cdm_name: str, app_config: Any) -> Optional[str]: + """Detect CDM type (playready/widevine) from config without loading it. + + Checks remote_cdm entries and local file extensions to determine the type. + """ + for entry in getattr(app_config, "remote_cdm", []) or []: + if entry.get("name") == cdm_name: + device_type = str(entry.get("device_type", entry.get("Device Type", ""))).upper() + return "playready" if device_type == "PLAYREADY" else "widevine" + + prd_path = app_config.directories.prds / f"{cdm_name}.prd" + if not prd_path.is_file(): + prd_path = app_config.directories.wvds / f"{cdm_name}.prd" + if prd_path.is_file(): + return "playready" + + wvd_path = app_config.directories.wvds / f"{cdm_name}.wvd" + if wvd_path.is_file(): + return "widevine" + + return None + + +def _require_authenticated(session: Any) -> None: + """Raise if the session has not finished authenticating.""" + if session.auth_status == AuthStatus.FAILED: + raise APIError( + APIErrorCode.AUTH_FAILED, + f"Authentication failed: {session.auth_error or 'unknown error'}", + ) + if session.auth_status in (AuthStatus.AUTHENTICATING, AuthStatus.PENDING_INPUT): + raise APIError( + APIErrorCode.INVALID_INPUT, + f"Session authentication not complete (status: {session.auth_status.value})", + details={"auth_status": session.auth_status.value}, + ) + + +async def session_prompt_get_handler(session_id: str, request: Optional[web.Request] = None) -> web.Response: + """Poll for pending interactive prompts during authentication. + + Returns the current auth status and any pending prompt that the + remote client should display to the user. + """ + session = await _get_validated_session(session_id, request) + + if session.auth_status == AuthStatus.AUTHENTICATED: + return web.json_response({"status": "authenticated"}) + + if session.auth_status == AuthStatus.FAILED: + return web.json_response({"status": "failed", "error": session.auth_error or "unknown error"}) + + bridge = session.input_bridge + if bridge: + prompt = bridge.get_pending_prompt() + if prompt: + return web.json_response({"status": "pending_input", "prompt": prompt}) + + return web.json_response({"status": "authenticating"}) + + +async def session_prompt_post_handler( + data: Dict[str, Any], session_id: str, request: Optional[web.Request] = None +) -> web.Response: + """Submit a response to a pending interactive prompt. + + The remote client calls this after collecting user input (OTP code, + PIN, or device-code confirmation) to unblock the server auth thread. + """ + session = await _get_validated_session(session_id, request) + + response_text = data.get("response") + if response_text is None: + raise APIError(APIErrorCode.INVALID_INPUT, "Missing required field: response") + + bridge = session.input_bridge + if bridge is None or bridge.status != AuthStatus.PENDING_INPUT: + raise APIError(APIErrorCode.INVALID_INPUT, "No prompt pending for this session") + + bridge.submit_response(str(response_text)) + return web.json_response({"status": "accepted"}) + + +async def _get_validated_session(session_id: str, request: Optional[web.Request]) -> Any: + """Fetch a session and verify the requesting IP matches the creator.""" + from envied.core.api.session_store import get_session_store + + store = get_session_store() + session = await store.get(session_id) + if not session: + raise APIError( + APIErrorCode.SESSION_NOT_FOUND, + f"Session not found or expired: {session_id}", + details={"session_id": session_id}, + ) + if session.creator_ip and request and request.remote != session.creator_ip: + raise APIError( + APIErrorCode.FORBIDDEN, + "Session access denied", + ) + return session + + +def _resolve_handler_proxy(data: Dict[str, Any], normalized_service: str) -> tuple[Optional[str], list]: + """Resolve proxy and initialize providers from API request data. + + Handles explicit proxy param, provider:country format, and + client_region-based auto-proxy when server region differs. + """ + proxy_param = data.get("proxy") + no_proxy = data.get("no_proxy", False) + proxy_providers: list = [] + + if not no_proxy: + proxy_providers = initialize_proxy_providers() + + if proxy_param and not no_proxy: + try: + proxy_param = resolve_proxy(proxy_param, proxy_providers) + except ValueError as e: + raise APIError( + APIErrorCode.INVALID_PROXY, + f"Proxy error: {e}", + details={"proxy": data.get("proxy"), "service": normalized_service}, + ) + + client_region = data.get("client_region") + if not proxy_param and not no_proxy and client_region and proxy_providers: + try: + from envied.core.utils.ip_info import get_ip_info + + server_ip_info = get_ip_info(None, cached=True) + server_region = server_ip_info.get("country", "").lower() if server_ip_info else None + except Exception: + server_region = None + + if server_region and server_region == client_region.lower(): + log.info(f"Server already in client region '{client_region}', no proxy needed") + else: + try: + proxy_param = resolve_proxy(client_region, proxy_providers) + log.info(f"Using server proxy for client region '{client_region}'") + except ValueError: + log.debug(f"No server proxy available for client region '{client_region}'") + + return proxy_param, proxy_providers + + +def _find_title_for_track(track_id: str, session: Any) -> Any: + """Find the title object that owns a given track.""" + for t_id, tracks_dict in session.tracks_by_title.items(): + if track_id in tracks_dict: + return session.title_map.get(t_id) + if session.title_map: + return next(iter(session.title_map.values())) + return None + + +def _extract_pssh_from_track(track: Any, drm_type: str) -> Optional[str]: + """Extract PSSH base64 string from a track's DRM objects.""" + if not track.drm: + return None + pssh_b64 = None + for drm_obj in track.drm: + drm_class = drm_obj.__class__.__name__ + if drm_class == "Widevine" and hasattr(drm_obj, "_pssh") and drm_obj._pssh: + if hasattr(drm_obj._pssh, "dumps"): + pssh_b64 = drm_obj._pssh.dumps() + if drm_type == "widevine": + break + elif drm_class == "PlayReady": + if hasattr(drm_obj, "data") and drm_obj.data.get("pssh_b64"): + pssh_b64 = drm_obj.data["pssh_b64"] + if drm_type == "playready": + break + return pssh_b64 + + +def _ensure_track_drm(track: Any) -> None: + """Extract DRM from manifest data if track has none. + + Supports DASH (ContentProtection elements), HLS (EXT-X-KEY from + playlist fetch), and ISM (ProtectionHeader elements). + """ + if track.drm: + return + + # DASH: extract from ContentProtection elements + if track.data.get("dash"): + from envied.core.manifests import DASH as DASHManifest + + rep = track.data["dash"].get("representation") + ada = track.data["dash"].get("adaptation_set") + if rep is not None and ada is not None: + track.drm = DASHManifest.get_drm(rep.findall("ContentProtection") + ada.findall("ContentProtection")) + if track.drm: + return + + # HLS: fetch playlist and extract DRM from EXT-X-KEY + if track.data.get("hls") and track.url: + try: + import m3u8 + + from envied.core.drm import PlayReady, Widevine + from envied.core.manifests import HLS + + playlist = m3u8.load(track.url) + keys = [k for k in (playlist.keys or []) + (playlist.session_keys or []) if k is not None] + for key in keys: + try: + drm = HLS.get_drm(key) + if isinstance(drm, (Widevine, PlayReady)): + track.drm = [drm] + return + except Exception: + continue + except Exception: + pass + + # ISM: extract from ProtectionHeader elements + if track.data.get("ism"): + try: + from envied.core.manifests import ISM as ISMManifest + + stream_index = track.data["ism"].get("stream_index") + if stream_index is not None: + track.drm = ISMManifest.get_drm(stream_index) + except Exception: + pass + + +def _resolve_device_name(user_config: dict, drm_type: str, service_tag: str = "") -> str: + """Get the CDM device name, checking service-specific config.cdm first. + + Resolution order: + 1. config.cdm[service_tag] (service-specific CDM mapping) + 2. serve.users.{key}.devices / playready_devices (user device list) + """ + from envied.core.config import config as app_config + + cdm_name = app_config.cdm.get(service_tag) if service_tag else None + if isinstance(cdm_name, dict): + drm_key = {"widevine": "widevine", "playready": "playready"}.get(drm_type) + lower_keys = {k.lower(): v for k, v in cdm_name.items()} + cdm_name = lower_keys.get(drm_key) or lower_keys.get("default") or app_config.cdm.get("default") + if cdm_name and isinstance(cdm_name, str): + return cdm_name + + if drm_type == "playready": + device_name = (user_config.get("playready_devices") or [None])[0] + if not device_name: + raise APIError(APIErrorCode.INVALID_INPUT, "No PlayReady device configured for this API key") + else: + device_name = (user_config.get("devices") or [None])[0] + if not device_name: + raise APIError(APIErrorCode.INVALID_INPUT, "No Widevine device configured for this API key") + return device_name + + +def _load_server_vaults(service_name: str) -> Any: + """Load server vaults from config.key_vaults.""" + from envied.core.config import config as app_config + from envied.core.vaults import Vaults + + vaults = Vaults(service_name) + for vault_config in app_config.key_vaults: + cfg = vault_config.copy() + vault_type = cfg.pop("type", None) + if vault_type: + try: + vaults.load(vault_type, **cfg) + except (Exception, SystemExit) as e: + log.warning(f"Could not load vault '{vault_type}': {e}") + return vaults + + +def _check_vaults(kids: list, service_name: str) -> Optional[Dict[str, str]]: + """Check server vaults for existing keys matching all KIDs. + + Returns a KID:KEY dict if ALL KIDs are found, None otherwise. + """ + from uuid import UUID + + try: + vaults = _load_server_vaults(service_name) + if not vaults.vaults: + return None + keys: Dict[str, str] = {} + for kid in kids: + kid_uuid = kid if isinstance(kid, UUID) else UUID(hex=str(kid)) + content_key, vault_used = vaults.get_key(kid_uuid) + if content_key: + keys[kid_uuid.hex] = content_key + else: + return None + if keys: + log.info(f"Vault hit: {len(keys)} key(s) from server vaults, skipping CDM") + return keys + except Exception: + pass + return None + + +def _cache_to_vaults(keys: Dict[str, str], service_name: str) -> None: + """Cache newly obtained keys to server vaults.""" + from uuid import UUID + + try: + vaults = _load_server_vaults(service_name) + if not vaults.vaults: + return + + key_map = {UUID(hex=kid): key for kid, key in keys.items()} + cached = vaults.add_keys(key_map) + if cached: + log.info(f"Cached {cached} key(s) to {cached} server vault(s)") + except (Exception, SystemExit) as e: + log.warning(f"Failed to cache keys to vaults: {e}") + + +def _handle_single_server_cdm( + service: Any, + title: Any, + track: Any, + pssh_b64: Optional[str], + drm_type: str, + request: Optional[web.Request], +) -> Dict[str, str]: + """Handle single-track server_cdm licensing using the DRM class get_content_keys() flow.""" + import base64 + + from envied.core.cdm import load_cdm + from envied.core.config import config as app_config + + _ensure_track_drm(track) + + if not pssh_b64: + pssh_b64 = _extract_pssh_from_track(track, drm_type) + if not pssh_b64: + raise APIError(APIErrorCode.INVALID_INPUT, "No PSSH available for server_cdm licensing") + + api_key = request.headers.get("X-Secret-Key", "anonymous") if request else "anonymous" + user_config = app_config.serve.get("users", {}).get(api_key, {}) + + if drm_type == "playready": + from pyplayready.system.pssh import PSSH as PlayReadyPSSH + + from envied.core.drm import PlayReady + + pr_pssh = PlayReadyPSSH(base64.b64decode(pssh_b64)) + pr_drm = PlayReady(pssh=pr_pssh, pssh_b64=pssh_b64) + + vault_keys = _check_vaults(pr_drm.kids, service.__class__.__name__) + if vault_keys: + return vault_keys + + device_name = _resolve_device_name(user_config, drm_type, service.__class__.__name__) + cdm = load_cdm(device_name, service_name=service.__class__.__name__) + pr_drm.get_content_keys( + cdm=cdm, + certificate=lambda challenge, **_: None, + licence=lambda challenge, **_: service.get_playready_license(challenge=challenge, title=title, track=track), + ) + keys = {kid.hex: key for kid, key in pr_drm.content_keys.items()} + elif drm_type == "widevine": + from pywidevine.pssh import PSSH as WvPSSH + + from envied.core.drm import Widevine + + wv_pssh = WvPSSH(pssh_b64) + wv_drm = Widevine(pssh=wv_pssh) + + vault_keys = _check_vaults(wv_drm.kids, service.__class__.__name__) + if vault_keys: + return vault_keys + + device_name = _resolve_device_name(user_config, drm_type, service.__class__.__name__) + cdm = load_cdm(device_name, service_name=service.__class__.__name__) + wv_drm.get_content_keys( + cdm=cdm, + certificate=lambda challenge, **_: service.get_widevine_service_certificate( + challenge=challenge, title=title, track=track + ), + licence=lambda challenge, **_: service.get_widevine_license(challenge=challenge, title=title, track=track), + ) + keys = {kid.hex: key for kid, key in wv_drm.content_keys.items()} + else: + raise APIError( + APIErrorCode.INVALID_PARAMETERS, + f"Unsupported DRM type for server_cdm: {drm_type}", + ) + + if not keys: + raise APIError(APIErrorCode.NO_CONTENT, "Server CDM returned no content keys") + + _cache_to_vaults(keys, service.__class__.__name__) + return keys + + +def _handle_proxy_license( + service: Any, + title: Any, + track: Any, + challenge_b64: Optional[str], + drm_type: str, +) -> web.Response: + """Forward a client CDM challenge to the service license endpoint.""" + import base64 + + if not challenge_b64: + raise APIError(APIErrorCode.INVALID_INPUT, "Missing required parameter: challenge") + challenge_bytes = base64.b64decode(challenge_b64) + + if drm_type == "widevine": + license_response = service.get_widevine_license(challenge=challenge_bytes, title=title, track=track) + elif drm_type == "playready": + license_response = service.get_playready_license(challenge=challenge_bytes, title=title, track=track) + else: + raise APIError( + APIErrorCode.INVALID_PARAMETERS, + f"Unsupported DRM type: {drm_type}", + details={"drm_type": drm_type, "supported": ["widevine", "playready"]}, + ) + + if isinstance(license_response, str): + license_response = license_response.encode("utf-8") + + return web.json_response({"license": base64.b64encode(license_response).decode("ascii")}) + + +async def session_license_handler( + data: Dict[str, Any], session_id: str, request: Optional[web.Request] = None +) -> web.Response: + """Handle DRM licensing in proxy or server_cdm mode. + + Proxy mode (default): forwards client CDM challenge to the service's + license endpoint, returns raw license bytes for client-side parsing. + + Server-CDM mode (mode="server_cdm"): server uses its own CDM to generate + the challenge, obtain the license, and extract KID:KEY pairs. Supports + batch (track_ids list) and single-track requests. + """ + import base64 + + session = await _get_validated_session(session_id, request) + + track_id = data.get("track_id") + track_ids = data.get("track_ids") + challenge_b64 = data.get("challenge") + drm_type = data.get("drm_type", "widevine") + mode = data.get("mode", "proxy") + + if mode == "server_cdm" and track_ids: + from envied.core.config import config as app_config + + api_key = request.headers.get("X-Secret-Key", "anonymous") if request else "anonymous" + user_config = app_config.serve.get("users", {}).get(api_key, {}) + service = session.service_instance + has_wv_device = bool(user_config.get("devices")) + has_pr_device = bool(user_config.get("playready_devices")) + + service_tag = session.service_tag + config_cdm_type = _detect_cdm_type_for_service(service_tag, app_config) + + all_keys: Dict[str, Dict[str, str]] = {} + seen_pssh: set[str] = set() + actual_drm_type: Optional[str] = None + + for tid in track_ids: + track = session.tracks.get(tid) + if not track: + continue + + _ensure_track_drm(track) + if not track.drm: + continue + + title = _find_title_for_track(tid, session) + + track_drm_type = None + pssh_str = None + if config_cdm_type == "playready": + pssh_str = _extract_pssh_from_track(track, "playready") + if pssh_str: + track_drm_type = "playready" + if not pssh_str: + pssh_str = _extract_pssh_from_track(track, "widevine") + if pssh_str: + track_drm_type = "widevine" + elif config_cdm_type == "widevine": + pssh_str = _extract_pssh_from_track(track, "widevine") + if pssh_str: + track_drm_type = "widevine" + if not pssh_str: + pssh_str = _extract_pssh_from_track(track, "playready") + if pssh_str: + track_drm_type = "playready" + else: + if has_wv_device: + pssh_str = _extract_pssh_from_track(track, "widevine") + if pssh_str: + track_drm_type = "widevine" + if not pssh_str and has_pr_device: + pssh_str = _extract_pssh_from_track(track, "playready") + if pssh_str: + track_drm_type = "playready" + + if not pssh_str or not track_drm_type: + continue + + if pssh_str in seen_pssh: + for prev_keys in all_keys.values(): + if prev_keys: + all_keys[tid] = prev_keys + break + continue + seen_pssh.add(pssh_str) + + try: + keys = _handle_single_server_cdm(service, title, track, pssh_str, track_drm_type, request) + if keys: + all_keys[tid] = keys + if track_drm_type: + actual_drm_type = track_drm_type + except SystemExit: + log.warning(f"Service exited while resolving keys for track {tid[:12]}, skipping") + except (Exception, SystemExit) as e: + log.warning(f"Failed to resolve keys for track {tid[:12]}: {e}") + + response: Dict[str, Any] = {"keys": all_keys} + if actual_drm_type: + response["drm_type"] = actual_drm_type + return web.json_response(response) + + if not track_id: + raise APIError(APIErrorCode.INVALID_INPUT, "Missing required parameter: track_id") + + track = session.tracks.get(track_id) + if not track: + raise APIError( + APIErrorCode.TRACK_NOT_FOUND, + f"Track not found in session: {track_id}", + details={"track_id": track_id, "session_id": session_id}, + ) + + try: + title = _find_title_for_track(track_id, session) + service = session.service_instance + + pssh_b64 = data.get("pssh") + if pssh_b64: + if not track.drm: + track.drm = [] + if drm_type == "playready": + track.pr_pssh = pssh_b64 + from pyplayready.system.pssh import PSSH as PlayReadyPSSH + + from envied.core.drm import PlayReady + + pr_pssh = PlayReadyPSSH(base64.b64decode(pssh_b64)) + pr_drm = PlayReady(pssh=pr_pssh, pssh_b64=pssh_b64) + track.drm.append(pr_drm) + elif drm_type == "widevine": + from pywidevine.pssh import PSSH as WidevinePSSH + + from envied.core.drm import Widevine + + wv_pssh = WidevinePSSH(pssh_b64) + wv_drm = Widevine(pssh=wv_pssh) + track.drm.append(wv_drm) + + if mode == "server_cdm": + keys = _handle_single_server_cdm(service, title, track, pssh_b64, drm_type, request) + log.info(f"Server CDM resolved {len(keys)} key(s) for track {track_id[:12]}") + return web.json_response({"keys": keys}) + + return _handle_proxy_license(service, title, track, challenge_b64, drm_type) + + except APIError: + raise + except SystemExit: + raise APIError(APIErrorCode.SERVICE_ERROR, "Service exited during license request") + except (Exception, SystemExit) as e: + log.exception(f"Error proxying license for track {track_id}") + debug_mode = request.app.get("debug_api", False) if request else False + return handle_api_exception( + e, + context={ + "operation": "session_license", + "session_id": session_id, + "track_id": track_id, + "drm_type": drm_type, + }, + debug_mode=debug_mode, + ) + + +async def session_info_handler(session_id: str, request: Optional[web.Request] = None) -> web.Response: + """Check session validity and get session info.""" + session = await _get_validated_session(session_id, request) + + from envied.core.api.session_store import get_session_store + + return web.json_response( + { + "session_id": session.session_id, + "service": session.service_tag, + "valid": True, + "expires_in": get_session_store()._ttl, + "track_count": len(session.tracks), + "title_count": len(session.title_map), + } + ) + + +async def session_delete_handler(session_id: str, request: Optional[web.Request] = None) -> web.Response: + """Delete a session, return updated cache files, and clean up server-side data.""" + import base64 + import zlib + + from envied.core.api.session_store import get_session_store + from envied.core.config import config as app_config + + session = await _get_validated_session(session_id, request) + store = get_session_store() + + if session.input_bridge: + session.input_bridge.cancel() + + cache_tag = session.cache_tag + cache_data: Dict[str, str] = {} + if cache_tag: + cache_dir = app_config.directories.cache / cache_tag + if cache_dir.is_dir(): + for f in cache_dir.glob("*.json"): + if not f.stem.startswith("titles_"): + try: + cache_data[f.stem] = base64.b64encode(zlib.compress(f.read_bytes())).decode("ascii") + except Exception: + pass + + await store.delete(session_id) + + response: Dict[str, Any] = {"status": "ok"} + if cache_data: + response["cache"] = cache_data + return web.json_response(response) diff --git a/reset/packages/envied/src/envied/core/api/input_bridge.py b/reset/packages/envied/src/envied/core/api/input_bridge.py new file mode 100644 index 0000000..fd3e3e9 --- /dev/null +++ b/reset/packages/envied/src/envied/core/api/input_bridge.py @@ -0,0 +1,139 @@ +"""Thread-safe bridge for interactive input during remote authentication. + +When a service calls ``request_input()`` during ``authenticate()`` on the +server, the InputBridge pauses the auth thread and exposes the prompt to +the HTTP layer so a remote client can poll for it, collect the user's +response, and submit it back. The auth thread then resumes with the +response value. +""" + +from __future__ import annotations + +import threading +import time +from dataclasses import dataclass, field +from enum import Enum +from typing import Optional + + +class AuthStatus(Enum): + """Authentication lifecycle states for a remote session.""" + + AUTHENTICATING = "authenticating" + PENDING_INPUT = "pending_input" + AUTHENTICATED = "authenticated" + FAILED = "failed" + + +class BridgeCancelledError(Exception): + """Raised when the bridge is cancelled (client disconnected, session deleted).""" + + +@dataclass +class InputBridge: + """Thread-safe bridge between a sync auth thread and the async HTTP layer. + + The auth thread calls :meth:`request_input` which blocks until the + remote client submits a response via the HTTP prompt endpoints. + """ + + _prompt: Optional[str] = field(default=None, init=False, repr=False) + _response: Optional[str] = field(default=None, init=False, repr=False) + _status: AuthStatus = field(default=AuthStatus.AUTHENTICATING, init=False) + _error: Optional[str] = field(default=None, init=False, repr=False) + _cancelled: bool = field(default=False, init=False, repr=False) + _response_ready: threading.Event = field(default_factory=threading.Event, init=False, repr=False) + _lock: threading.Lock = field(default_factory=threading.Lock, init=False, repr=False) + + def request_input(self, prompt: str, timeout: float = 600.0) -> str: + """Block until the remote client submits a response for *prompt*. + + Args: + prompt: The message to display to the remote user. + timeout: Maximum seconds to wait for a response. + + Returns: + The string response from the remote client. + + Raises: + TimeoutError: If no response is received within *timeout*. + BridgeCancelledError: If the bridge was cancelled. + """ + with self._lock: + if self._cancelled: + raise BridgeCancelledError("Session was cancelled") + self._prompt = prompt + self._response = None + self._status = AuthStatus.PENDING_INPUT + self._response_ready.clear() + + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if self._response_ready.wait(timeout=0.5): + break + with self._lock: + if self._cancelled: + raise BridgeCancelledError("Session was cancelled") + else: + with self._lock: + self._status = AuthStatus.FAILED + self._error = "Input request timed out waiting for client response" + raise TimeoutError(f"No client response for prompt within {timeout}s") + + with self._lock: + if self._cancelled: + raise BridgeCancelledError("Session was cancelled") + response = self._response or "" + self._prompt = None + self._response = None + self._status = AuthStatus.AUTHENTICATING + return response + + def get_pending_prompt(self) -> Optional[str]: + """Return the current prompt if the auth thread is waiting for input.""" + with self._lock: + if self._status == AuthStatus.PENDING_INPUT: + return self._prompt + return None + + def submit_response(self, response: str) -> bool: + """Deliver the client's response and unblock the auth thread. + + Returns: + ``True`` if a prompt was pending and the response was accepted, + ``False`` otherwise. + """ + with self._lock: + if self._status != AuthStatus.PENDING_INPUT: + return False + self._response = response + self._response_ready.set() + return True + + def cancel(self) -> None: + """Cancel the bridge, unblocking any waiting auth thread.""" + with self._lock: + self._cancelled = True + self._status = AuthStatus.FAILED + self._error = "Session cancelled" + self._response_ready.set() + + @property + def status(self) -> AuthStatus: + with self._lock: + return self._status + + @status.setter + def status(self, value: AuthStatus) -> None: + with self._lock: + self._status = value + + @property + def error(self) -> Optional[str]: + with self._lock: + return self._error + + @error.setter + def error(self, value: Optional[str]) -> None: + with self._lock: + self._error = value diff --git a/reset/packages/envied/src/envied/core/api/routes.py b/reset/packages/envied/src/envied/core/api/routes.py new file mode 100644 index 0000000..2368ad7 --- /dev/null +++ b/reset/packages/envied/src/envied/core/api/routes.py @@ -0,0 +1,1360 @@ +import logging +import re + +import click +from aiohttp import web +from aiohttp_swagger3 import SwaggerDocs, SwaggerInfo, SwaggerUiSettings + +from envied.core import __version__ +from envied.core.api.errors import APIError, APIErrorCode, build_error_response, handle_api_exception +from envied.core.api.handlers import (cancel_download_job_handler, download_handler, get_allowed_services, + get_download_job_handler, list_download_jobs_handler, list_titles_handler, + list_tracks_handler, search_handler, session_create_handler, + session_delete_handler, session_info_handler, session_license_handler, + session_prompt_get_handler, session_prompt_post_handler, + session_segments_handler, session_titles_handler, session_tracks_handler) +from envied.core.services import Services +from envied.core.update_checker import UpdateChecker + + +@web.middleware +async def cors_middleware(request: web.Request, handler): + """Add CORS headers to all responses.""" + # Handle preflight requests + if request.method == "OPTIONS": + response = web.Response() + else: + response = await handler(request) + + # Add CORS headers + response.headers["Access-Control-Allow-Origin"] = "*" + response.headers["Access-Control-Allow-Methods"] = "GET, POST, PUT, DELETE, OPTIONS" + response.headers["Access-Control-Allow-Headers"] = "Content-Type, X-Secret-Key, Authorization" + response.headers["Access-Control-Max-Age"] = "3600" + + return response + + +log = logging.getLogger("api") + + +async def health(request: web.Request) -> web.Response: + """ + Health check endpoint. + --- + summary: Health check + description: Get server health status, version info, and update availability + responses: + '200': + description: Health status + content: + application/json: + schema: + type: object + properties: + status: + type: string + example: ok + version: + type: string + example: "2.0.0" + update_check: + type: object + properties: + update_available: + type: boolean + nullable: true + current_version: + type: string + latest_version: + type: string + nullable: true + """ + try: + latest_version = await UpdateChecker.check_for_updates(__version__) + update_info = { + "update_available": latest_version is not None, + "current_version": __version__, + "latest_version": latest_version, + } + except Exception as e: + log.warning(f"Failed to check for updates: {e}") + update_info = {"update_available": None, "current_version": __version__, "latest_version": None} + + return web.json_response({"status": "ok", "version": __version__, "update_check": update_info}) + + +async def services(request: web.Request) -> web.Response: + """ + List available services. + --- + summary: List services + description: Get all available streaming services with their details + responses: + '200': + description: List of services + content: + application/json: + schema: + type: object + properties: + services: + type: array + items: + type: object + properties: + tag: + type: string + aliases: + type: array + items: + type: string + geofence: + type: array + items: + type: string + title_regex: + oneOf: + - type: string + - type: array + items: + type: string + nullable: true + url: + type: string + nullable: true + description: Service URL from short_help + help: + type: string + nullable: true + description: Full service documentation + '500': + description: Server error + content: + application/json: + schema: + type: object + properties: + status: + type: string + example: error + error_code: + type: string + example: INTERNAL_ERROR + message: + type: string + example: An unexpected error occurred + details: + type: object + timestamp: + type: string + format: date-time + debug_info: + type: object + description: Only present when --debug-api flag is enabled + """ + try: + service_tags = Services.get_tags() + allowed = get_allowed_services(request) + if allowed is not None: + service_tags = [t for t in service_tags if t in allowed] + services_info = [] + + for tag in service_tags: + service_data = {"tag": tag, "aliases": [], "geofence": [], "title_regex": None, "url": None, "help": None} + + try: + service_module = Services.load(tag) + + if hasattr(service_module, "ALIASES"): + service_data["aliases"] = list(service_module.ALIASES) + + if hasattr(service_module, "GEOFENCE"): + service_data["geofence"] = list(service_module.GEOFENCE) + + if hasattr(service_module, "TITLE_RE"): + title_re = service_module.TITLE_RE + # Handle different types of TITLE_RE + if isinstance(title_re, re.Pattern): + service_data["title_regex"] = title_re.pattern + elif isinstance(title_re, str): + service_data["title_regex"] = title_re + elif isinstance(title_re, (list, tuple)): + # Convert list/tuple of patterns to list of strings + patterns = [] + for item in title_re: + if isinstance(item, re.Pattern): + patterns.append(item.pattern) + elif isinstance(item, str): + patterns.append(item) + service_data["title_regex"] = patterns if patterns else None + + if hasattr(service_module, "cli") and hasattr(service_module.cli, "short_help"): + service_data["url"] = service_module.cli.short_help + + if hasattr(service_module, "cli") and hasattr(service_module.cli, "params"): + cli_params = [] + for param in service_module.cli.params: + param_info: dict = {"name": getattr(param, "name", None)} + if isinstance(param, click.Argument): + param_info["kind"] = "argument" + param_info["required"] = param.required + else: + param_info["kind"] = "option" + param_info["opts"] = list(param.opts) if hasattr(param, "opts") else [] + param_info["is_flag"] = getattr(param, "is_flag", False) + default = param.default + if default is None: + pass + elif callable(default) or type(default).__name__ == "Sentinel": + default = None + elif hasattr(default, "name"): + default = default.name + elif not isinstance(default, (str, int, float, bool, list)): + default = str(default) + param_info["default"] = default + param_info["help"] = getattr(param, "help", None) + param_info["type"] = param.type.name if hasattr(param.type, "name") else str(param.type) + cli_params.append(param_info) + service_data["cli_params"] = cli_params + + if service_module.__doc__: + service_data["help"] = service_module.__doc__.strip() + + except Exception as e: + log.warning(f"Could not load details for service {tag}: {e}") + + services_info.append(service_data) + + return web.json_response({"services": services_info}) + except Exception as e: + log.exception("Error listing services") + debug_mode = request.app.get("debug_api", False) + return handle_api_exception(e, context={"operation": "list_services"}, debug_mode=debug_mode) + + +async def search(request: web.Request) -> web.Response: + """ + Search for titles from a service. + --- + summary: Search for titles + description: Search for titles by query string from a service + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - service + - query + properties: + service: + type: string + description: Service tag + query: + type: string + description: Search query string + profile: + type: string + description: Profile to use for credentials and cookies (default - None) + proxy: + type: string + description: Proxy URI or country code (default - None) + no_proxy: + type: boolean + description: Force disable all proxy use (default - false) + responses: + '200': + description: Search results + content: + application/json: + schema: + type: object + properties: + results: + type: array + items: + type: object + properties: + id: + type: string + description: Title ID for use with other endpoints + title: + type: string + description: Title name + description: + type: string + description: Title description + label: + type: string + description: Informative label (e.g., availability, region) + url: + type: string + description: URL to the title page + count: + type: integer + description: Number of results returned + '400': + description: Invalid request + """ + try: + data = await request.json() + except Exception as e: + return build_error_response( + APIError( + APIErrorCode.INVALID_INPUT, + "Invalid JSON request body", + details={"error": str(e)}, + ), + request.app.get("debug_api", False), + ) + + try: + return await search_handler(data, request) + except APIError as e: + return build_error_response(e, request.app.get("debug_api", False)) + except Exception as e: + log.exception("Error in search") + debug_mode = request.app.get("debug_api", False) + return handle_api_exception(e, context={"operation": "search"}, debug_mode=debug_mode) + + +async def list_titles(request: web.Request) -> web.Response: + """ + List titles for a service and title ID. + --- + summary: List titles + description: Get available titles for a service and title ID + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - service + - title_id + properties: + service: + type: string + description: Service tag + title_id: + type: string + description: Title identifier + responses: + '200': + description: List of titles + '400': + description: Invalid request (missing parameters, invalid service) + content: + application/json: + schema: + type: object + properties: + status: + type: string + example: error + error_code: + type: string + example: INVALID_INPUT + message: + type: string + example: Missing required parameter + details: + type: object + timestamp: + type: string + format: date-time + '401': + description: Authentication failed + content: + application/json: + schema: + type: object + properties: + status: + type: string + example: error + error_code: + type: string + example: AUTH_FAILED + message: + type: string + details: + type: object + timestamp: + type: string + format: date-time + '404': + description: Title not found + content: + application/json: + schema: + type: object + properties: + status: + type: string + example: error + error_code: + type: string + example: NOT_FOUND + message: + type: string + details: + type: object + timestamp: + type: string + format: date-time + '500': + description: Server error + content: + application/json: + schema: + type: object + properties: + status: + type: string + example: error + error_code: + type: string + example: INTERNAL_ERROR + message: + type: string + details: + type: object + timestamp: + type: string + format: date-time + """ + try: + data = await request.json() + except Exception as e: + return build_error_response( + APIError( + APIErrorCode.INVALID_INPUT, + "Invalid JSON request body", + details={"error": str(e)}, + ), + request.app.get("debug_api", False), + ) + + try: + return await list_titles_handler(data, request) + except APIError as e: + debug_mode = request.app.get("debug_api", False) + return build_error_response(e, debug_mode) + + +async def list_tracks(request: web.Request) -> web.Response: + """ + List tracks for a title, separated by type. + --- + summary: List tracks + description: Get available video, audio, and subtitle tracks for a title + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - service + - title_id + properties: + service: + type: string + description: Service tag + title_id: + type: string + description: Title identifier + wanted: + type: string + description: Specific episode/season (optional) + proxy: + type: string + description: Proxy configuration (optional) + responses: + '200': + description: Track information + '400': + description: Invalid request + """ + try: + data = await request.json() + except Exception as e: + return build_error_response( + APIError( + APIErrorCode.INVALID_INPUT, + "Invalid JSON request body", + details={"error": str(e)}, + ), + request.app.get("debug_api", False), + ) + + try: + return await list_tracks_handler(data, request) + except APIError as e: + debug_mode = request.app.get("debug_api", False) + return build_error_response(e, debug_mode) + + +async def download(request: web.Request) -> web.Response: + """ + Download content based on provided parameters. + --- + summary: Download content + description: Download video content based on specified parameters + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - service + - title_id + properties: + service: + type: string + description: Service tag + title_id: + type: string + description: Title identifier + profile: + type: string + description: Profile to use for credentials and cookies (default - None) + quality: + type: array + items: + type: integer + description: Download resolution(s) (default - best available) + vcodec: + oneOf: + - type: string + - type: array + items: + type: string + description: Video codec(s) to download (e.g., "H265" or ["H264", "H265"]) - accepts H264, H265, AVC, HEVC, VP8, VP9, AV1, VC1 (default - None) + acodec: + oneOf: + - type: string + - type: array + items: + type: string + description: Audio codec(s) to download (e.g., "AAC" or ["AAC", "EC3"]) - accepts AAC, AC3, EC3, AC4, OPUS, FLAC, ALAC, DTS, OGG (default - None) + vbitrate: + type: integer + description: Video bitrate in kbps (default - None) + abitrate: + type: integer + description: Audio bitrate in kbps (default - None) + range: + type: array + items: + type: string + description: Video color range (SDR, HDR10, HDR10+, HLG, DV, HYBRID) (default - ["SDR"]) + channels: + type: number + description: Audio channels (e.g., 2.0, 5.1, 7.1) (default - None) + no_atmos: + type: boolean + description: Exclude Dolby Atmos audio tracks (default - false) + wanted: + type: array + items: + type: string + description: Wanted episodes (e.g., ["S01E01", "S01E02"]) (default - all) + latest_episode: + type: boolean + description: Download only the single most recent episode (default - false) + lang: + type: array + items: + type: string + description: Language for video and audio (use 'orig' for original) (default - ["orig"]) + v_lang: + type: array + items: + type: string + description: Language for video tracks only (default - []) + a_lang: + type: array + items: + type: string + description: Language for audio tracks only (default - []) + s_lang: + type: array + items: + type: string + description: Language for subtitle tracks (default - ["all"]) + require_subs: + type: array + items: + type: string + description: Required subtitle languages (default - []) + forced_subs: + type: boolean + description: Include forced subtitle tracks (default - false) + exact_lang: + type: boolean + description: Use exact language matching (no variants) (default - false) + sub_format: + type: string + description: Output subtitle format (SRT, VTT, etc.) (default - None) + video_only: + type: boolean + description: Only download video tracks (default - false) + audio_only: + type: boolean + description: Only download audio tracks (default - false) + subs_only: + type: boolean + description: Only download subtitle tracks (default - false) + chapters_only: + type: boolean + description: Only download chapters (default - false) + no_subs: + type: boolean + description: Do not download subtitle tracks (default - false) + no_audio: + type: boolean + description: Do not download audio tracks (default - false) + no_chapters: + type: boolean + description: Do not download chapters (default - false) + no_video: + type: boolean + description: Do not download video tracks (default - false) + audio_description: + type: boolean + description: Download audio description tracks (default - false) + slow: + oneOf: + - type: boolean + - type: string + description: Add randomized delay between downloads. `true` for default 60-120s, or `"MIN-MAX"` string (e.g., `"20-40"`). Min must be >= 20 (default - null) + split_audio: + type: boolean + description: Create separate output files per audio codec instead of merging all audio (default - null) + skip_dl: + type: boolean + description: Skip downloading, only retrieve decryption keys (default - false) + export: + type: boolean + description: Export manifest, track URLs, keys, and subtitles to JSON in the exports directory (default - false) + cdm_only: + type: boolean + description: Only use CDM for key retrieval (true) or only vaults (false) (default - None) + proxy: + type: string + description: Proxy URI or country code (default - None) + no_proxy: + type: boolean + description: Force disable all proxy use (default - false) + no_proxy_download: + type: boolean + description: Bypass proxy for segment downloads only. Manifest, license, and auth still use proxy (default - false) + tag: + type: string + description: Set the group tag to be used (default - None) + tmdb_id: + type: integer + description: Use this TMDB ID for tagging (default - None) + animeapi_id: + type: string + description: Anime database ID via AnimeAPI, e.g. mal:12345 (default - None) + enrich: + type: boolean + description: Override show title and year from external source (default - false) + no_folder: + type: boolean + description: Disable folder creation for TV shows (default - false) + no_source: + type: boolean + description: Disable source tag from output file name (default - false) + no_mux: + type: boolean + description: Do not mux tracks into a container file (default - false) + workers: + type: integer + description: Max workers/threads per track download (default - None) + downloads: + type: integer + description: Amount of tracks to download concurrently (default - 1) + best_available: + type: boolean + description: Continue with best available if requested quality unavailable (default - false) + worst: + type: boolean + description: Select the lowest bitrate track within the specified quality. Requires `quality` (default - false) + repack: + type: boolean + description: Add REPACK tag to the output filename (default - false) + imdb_id: + type: string + description: Use this IMDB ID (e.g. tt1375666) for tagging (default - None) + output_dir: + type: string + description: Override the output directory for this download (default - None) + no_cache: + type: boolean + description: Bypass title cache for this download (default - false) + reset_cache: + type: boolean + description: Clear title cache before fetching (default - false) + responses: + '202': + description: Download job created + content: + application/json: + schema: + type: object + properties: + job_id: + type: string + status: + type: string + created_time: + type: string + '400': + description: Invalid request + """ + try: + data = await request.json() + except Exception as e: + return build_error_response( + APIError( + APIErrorCode.INVALID_INPUT, + "Invalid JSON request body", + details={"error": str(e)}, + ), + request.app.get("debug_api", False), + ) + + try: + return await download_handler(data, request) + except APIError as e: + debug_mode = request.app.get("debug_api", False) + return build_error_response(e, debug_mode) + + +async def download_jobs(request: web.Request) -> web.Response: + """ + List all download jobs with optional filtering and sorting. + --- + summary: List download jobs + description: Get list of all download jobs with their status, with optional filtering by status/service and sorting + parameters: + - name: status + in: query + required: false + schema: + type: string + enum: [queued, downloading, completed, failed, cancelled] + description: Filter jobs by status + - name: service + in: query + required: false + schema: + type: string + description: Filter jobs by service tag + - name: sort_by + in: query + required: false + schema: + type: string + enum: [created_time, started_time, completed_time, progress, status, service] + default: created_time + description: Field to sort by + - name: sort_order + in: query + required: false + schema: + type: string + enum: [asc, desc] + default: desc + description: Sort order (ascending or descending) + responses: + '200': + description: List of download jobs + content: + application/json: + schema: + type: object + properties: + jobs: + type: array + items: + type: object + properties: + job_id: + type: string + status: + type: string + created_time: + type: string + service: + type: string + title_id: + type: string + progress: + type: number + '400': + description: Invalid query parameters + '500': + description: Server error + """ + # Extract query parameters + query_params = { + "status": request.query.get("status"), + "service": request.query.get("service"), + "sort_by": request.query.get("sort_by", "created_time"), + "sort_order": request.query.get("sort_order", "desc"), + } + try: + return await list_download_jobs_handler(query_params, request) + except APIError as e: + debug_mode = request.app.get("debug_api", False) + return build_error_response(e, debug_mode) + + +async def download_job_detail(request: web.Request) -> web.Response: + """ + Get download job details. + --- + summary: Get download job + description: Get detailed information about a specific download job + parameters: + - name: job_id + in: path + required: true + schema: + type: string + responses: + '200': + description: Download job details + '404': + description: Job not found + '500': + description: Server error + """ + job_id = request.match_info["job_id"] + try: + return await get_download_job_handler(job_id, request) + except APIError as e: + debug_mode = request.app.get("debug_api", False) + return build_error_response(e, debug_mode) + + +async def cancel_download_job(request: web.Request) -> web.Response: + """ + Cancel download job. + --- + summary: Cancel download job + description: Cancel a queued or running download job + parameters: + - name: job_id + in: path + required: true + schema: + type: string + responses: + '200': + description: Job cancelled successfully + '400': + description: Job cannot be cancelled + '404': + description: Job not found + '500': + description: Server error + """ + job_id = request.match_info["job_id"] + try: + return await cancel_download_job_handler(job_id, request) + except APIError as e: + debug_mode = request.app.get("debug_api", False) + return build_error_response(e, debug_mode) + + +async def session_create(request: web.Request) -> web.Response: + """ + Create a remote-dl session. + --- + summary: Create session + description: Authenticate with a service, get titles, tracks, and chapters in one call + requestBody: + required: true + content: + application/json: + schema: + type: object + additionalProperties: true + required: + - service + - title_id + properties: + service: + type: string + title_id: + type: string + credentials: + type: object + additionalProperties: true + cookies: + type: string + proxy: + type: string + no_proxy: + type: boolean + profile: + type: string + cache: + type: object + additionalProperties: true + responses: + '200': + description: Session created with titles, tracks, and chapters + '400': + description: Invalid request + '401': + description: Authentication failed + """ + try: + data = await request.json() + except Exception as e: + return build_error_response( + APIError(APIErrorCode.INVALID_INPUT, "Invalid JSON request body", details={"error": str(e)}), + request.app.get("debug_api", False), + ) + try: + return await session_create_handler(data, request) + except APIError as e: + return build_error_response(e, request.app.get("debug_api", False)) + except Exception as e: + log.exception("Error in session create") + return handle_api_exception( + e, context={"operation": "session_create"}, debug_mode=request.app.get("debug_api", False) + ) + + +async def session_titles(request: web.Request) -> web.Response: + """ + Get titles for an authenticated session. + --- + summary: Get titles + description: Fetch titles from the authenticated service session + parameters: + - name: session_id + in: path + required: true + schema: + type: string + responses: + '200': + description: List of titles + '404': + description: Session not found + """ + session_id = request.match_info["session_id"] + try: + return await session_titles_handler(session_id, request) + except APIError as e: + return build_error_response(e, request.app.get("debug_api", False)) + except Exception as e: + log.exception("Error in session titles") + return handle_api_exception( + e, context={"operation": "session_titles"}, debug_mode=request.app.get("debug_api", False) + ) + + +async def session_tracks(request: web.Request) -> web.Response: + """ + Get tracks and chapters for a specific title. + --- + summary: Get tracks + description: Fetch tracks and chapters for a title in the session + parameters: + - name: session_id + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - title_id + properties: + title_id: + type: string + description: ID of the title to get tracks for + responses: + '200': + description: Tracks and chapters for the title + '404': + description: Session or title not found + """ + session_id = request.match_info["session_id"] + try: + data = await request.json() + except Exception as e: + return build_error_response( + APIError(APIErrorCode.INVALID_INPUT, "Invalid JSON request body", details={"error": str(e)}), + request.app.get("debug_api", False), + ) + try: + return await session_tracks_handler(data, session_id, request) + except APIError as e: + return build_error_response(e, request.app.get("debug_api", False)) + except Exception as e: + log.exception("Error in session tracks") + return handle_api_exception( + e, context={"operation": "session_tracks"}, debug_mode=request.app.get("debug_api", False) + ) + + +async def session_segments(request: web.Request) -> web.Response: + """ + Resolve segment URLs for selected tracks. + --- + summary: Resolve segments + description: Get download URLs, DRM info, and headers for selected tracks + parameters: + - name: session_id + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - track_ids + properties: + track_ids: + type: array + items: + type: string + description: List of track IDs to resolve + responses: + '200': + description: Segment URLs and DRM info for each track + '404': + description: Session or track not found + """ + session_id = request.match_info["session_id"] + try: + data = await request.json() + except Exception as e: + return build_error_response( + APIError(APIErrorCode.INVALID_INPUT, "Invalid JSON request body", details={"error": str(e)}), + request.app.get("debug_api", False), + ) + try: + return await session_segments_handler(data, session_id, request) + except APIError as e: + return build_error_response(e, request.app.get("debug_api", False)) + except Exception as e: + log.exception("Error in session segments") + return handle_api_exception( + e, context={"operation": "session_segments"}, debug_mode=request.app.get("debug_api", False) + ) + + +async def session_license(request: web.Request) -> web.Response: + """ + Proxy DRM license through authenticated service. + --- + summary: Proxy license + description: Forward a CDM challenge to the service's license endpoint + parameters: + - name: session_id + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - track_id + - challenge + properties: + track_id: + type: string + description: Track ID this license is for + challenge: + type: string + description: Base64-encoded CDM challenge + drm_type: + type: string + enum: [widevine, playready] + description: DRM type (default widevine) + responses: + '200': + description: License response + '404': + description: Session or track not found + """ + session_id = request.match_info["session_id"] + try: + data = await request.json() + except Exception as e: + return build_error_response( + APIError(APIErrorCode.INVALID_INPUT, "Invalid JSON request body", details={"error": str(e)}), + request.app.get("debug_api", False), + ) + try: + return await session_license_handler(data, session_id, request) + except APIError as e: + return build_error_response(e, request.app.get("debug_api", False)) + except Exception as e: + log.exception("Error in session license") + return handle_api_exception( + e, context={"operation": "session_license"}, debug_mode=request.app.get("debug_api", False) + ) + + +async def session_info(request: web.Request) -> web.Response: + """ + Get session info. + --- + summary: Session info + description: Check session validity and get metadata + parameters: + - name: session_id + in: path + required: true + schema: + type: string + responses: + '200': + description: Session info + '404': + description: Session not found + """ + session_id = request.match_info["session_id"] + try: + return await session_info_handler(session_id, request) + except APIError as e: + return build_error_response(e, request.app.get("debug_api", False)) + + +async def session_delete(request: web.Request) -> web.Response: + """ + Delete a session. + --- + summary: Delete session + description: Clean up a remote-dl session + parameters: + - name: session_id + in: path + required: true + schema: + type: string + responses: + '200': + description: Session deleted + '404': + description: Session not found + """ + session_id = request.match_info["session_id"] + try: + return await session_delete_handler(session_id, request) + except APIError as e: + return build_error_response(e, request.app.get("debug_api", False)) + + +async def session_prompt_get(request: web.Request) -> web.Response: + """ + Poll for pending interactive prompts during authentication. + --- + summary: Get auth prompt + description: Poll for pending interactive prompts (OTP, device code, PIN) during session authentication + parameters: + - name: session_id + in: path + required: true + schema: + type: string + responses: + '200': + description: Auth status and optional prompt + content: + application/json: + schema: + type: object + properties: + status: + type: string + enum: [authenticating, pending_input, authenticated, failed] + prompt: + type: string + description: Prompt to display to the user (only when status is pending_input) + error: + type: string + description: Error message (only when status is failed) + '404': + description: Session not found + """ + session_id = request.match_info["session_id"] + try: + return await session_prompt_get_handler(session_id, request) + except APIError as e: + return build_error_response(e, request.app.get("debug_api", False)) + except Exception as e: + log.exception("Error in session prompt get") + return handle_api_exception( + e, context={"operation": "session_prompt_get"}, debug_mode=request.app.get("debug_api", False) + ) + + +async def session_prompt_submit(request: web.Request) -> web.Response: + """ + Submit a response to a pending interactive prompt. + --- + summary: Submit prompt response + description: Submit user input (OTP code, PIN, device code confirmation) to unblock server authentication + parameters: + - name: session_id + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - response + properties: + response: + type: string + description: User's response to the prompt + responses: + '200': + description: Response accepted + content: + application/json: + schema: + type: object + properties: + status: + type: string + example: accepted + '400': + description: No prompt pending or invalid request + '404': + description: Session not found + """ + session_id = request.match_info["session_id"] + try: + data = await request.json() + except Exception as e: + return build_error_response( + APIError(APIErrorCode.INVALID_INPUT, "Invalid JSON request body", details={"error": str(e)}), + request.app.get("debug_api", False), + ) + try: + return await session_prompt_post_handler(data, session_id, request) + except APIError as e: + return build_error_response(e, request.app.get("debug_api", False)) + except Exception as e: + log.exception("Error in session prompt submit") + return handle_api_exception( + e, context={"operation": "session_prompt_submit"}, debug_mode=request.app.get("debug_api", False) + ) + + +def _setup_remote_session_routes(app: web.Application) -> None: + """Setup remote-DL session endpoints only.""" + app.router.add_get("/api/health", health) + app.router.add_get("/api/services", services) + app.router.add_post("/api/search", search) + app.router.add_post("/api/session/create", session_create) + app.router.add_get("/api/session/{session_id}/titles", session_titles) + app.router.add_post("/api/session/{session_id}/tracks", session_tracks) + app.router.add_post("/api/session/{session_id}/segments", session_segments) + app.router.add_post("/api/session/{session_id}/license", session_license) + app.router.add_get("/api/session/{session_id}/prompt", session_prompt_get) + app.router.add_post("/api/session/{session_id}/prompt", session_prompt_submit) + app.router.add_get("/api/session/{session_id}", session_info) + app.router.add_delete("/api/session/{session_id}", session_delete) + + +def setup_routes(app: web.Application, remote_only: bool = False) -> None: + """Setup API routes. When remote_only=True, only expose remote session endpoints.""" + if remote_only: + _setup_remote_session_routes(app) + return + + app.router.add_get("/api/health", health) + app.router.add_get("/api/services", services) + app.router.add_post("/api/search", search) + app.router.add_post("/api/list-titles", list_titles) + app.router.add_post("/api/list-tracks", list_tracks) + app.router.add_post("/api/download", download) + app.router.add_get("/api/download/jobs", download_jobs) + app.router.add_get("/api/download/jobs/{job_id}", download_job_detail) + app.router.add_delete("/api/download/jobs/{job_id}", cancel_download_job) + + # Remote-DL session endpoints + app.router.add_post("/api/session/create", session_create) + app.router.add_get("/api/session/{session_id}/titles", session_titles) + app.router.add_post("/api/session/{session_id}/tracks", session_tracks) + app.router.add_post("/api/session/{session_id}/segments", session_segments) + app.router.add_post("/api/session/{session_id}/license", session_license) + app.router.add_get("/api/session/{session_id}/prompt", session_prompt_get) + app.router.add_post("/api/session/{session_id}/prompt", session_prompt_submit) + app.router.add_get("/api/session/{session_id}", session_info) + app.router.add_delete("/api/session/{session_id}", session_delete) + + +def setup_swagger(app: web.Application) -> None: + """Setup Swagger UI documentation.""" + swagger = SwaggerDocs( + app, + swagger_ui_settings=SwaggerUiSettings(path="/api/docs/"), + info=SwaggerInfo( + title="envied.REST API", + version=__version__, + description="REST API for envied.- Modular Movie, TV, and Music Archival Software", + ), + ) + + # Add routes with OpenAPI documentation + swagger.add_routes( + [ + web.get("/api/health", health), + web.get("/api/services", services), + web.post("/api/search", search), + web.post("/api/list-titles", list_titles), + web.post("/api/list-tracks", list_tracks), + web.post("/api/download", download), + web.get("/api/download/jobs", download_jobs), + web.get("/api/download/jobs/{job_id}", download_job_detail), + web.delete("/api/download/jobs/{job_id}", cancel_download_job), + # Remote-DL session endpoints + web.post("/api/session/create", session_create), + web.get("/api/session/{session_id}/titles", session_titles), + web.post("/api/session/{session_id}/tracks", session_tracks), + web.post("/api/session/{session_id}/segments", session_segments), + web.post("/api/session/{session_id}/license", session_license), + web.get("/api/session/{session_id}/prompt", session_prompt_get), + web.post("/api/session/{session_id}/prompt", session_prompt_submit), + web.get("/api/session/{session_id}", session_info), + web.delete("/api/session/{session_id}", session_delete), + ] + ) diff --git a/reset/packages/envied/src/envied/core/api/session_store.py b/reset/packages/envied/src/envied/core/api/session_store.py new file mode 100644 index 0000000..e57dc8d --- /dev/null +++ b/reset/packages/envied/src/envied/core/api/session_store.py @@ -0,0 +1,209 @@ +"""Server-side session store for remote-dl client-server architecture. + +Maintains authenticated service instances between API calls so that +a client can authenticate once and then make multiple requests (list tracks, +resolve segments, proxy license) using the same session. +""" + +from __future__ import annotations + +import asyncio +import logging +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +from envied.core.api.input_bridge import AuthStatus, InputBridge +from envied.core.config import config +from envied.core.tracks import Track + +log = logging.getLogger("api.session") + + +@dataclass +class SessionEntry: + """A single authenticated session with a service.""" + + session_id: str + service_tag: str + service_instance: Any # Service instance (authenticated) + titles: Any = None # Titles_T from get_titles() + title_map: Dict[str, Any] = field(default_factory=dict) # title_id -> Title object + tracks: Dict[str, Track] = field(default_factory=dict) # track_id -> Track object + tracks_by_title: Dict[str, Dict[str, Track]] = field(default_factory=dict) # title_key -> {track_id -> Track} + chapters_by_title: Dict[str, List[Any]] = field(default_factory=dict) # title_key -> [Chapter] + creator_ip: Optional[str] = None + cache_tag: Optional[str] = None # per-session cache directory tag + input_bridge: Optional[InputBridge] = None + auth_status: AuthStatus = AuthStatus.AUTHENTICATED + auth_error: Optional[str] = None + created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + last_accessed: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + + def touch(self) -> None: + """Update last_accessed timestamp.""" + self.last_accessed = datetime.now(timezone.utc) + + +class SessionStore: + """Thread-safe session store with TTL-based expiration.""" + + def __init__(self) -> None: + self._sessions: Dict[str, SessionEntry] = {} + self._lock = asyncio.Lock() + self._cleanup_task: Optional[asyncio.Task] = None + + @property + def _ttl(self) -> int: + """Session TTL in seconds from config.""" + return config.serve.get("session_ttl", 300) # 5 min default + + @property + def _max_sessions(self) -> int: + """Max concurrent sessions from config.""" + return config.serve.get("max_sessions", 100) + + async def create( + self, + service_tag: str, + service_instance: Any, + session_id: Optional[str] = None, + ) -> SessionEntry: + """Create a new session with an authenticated service instance.""" + async with self._lock: + if len(self._sessions) >= self._max_sessions: + oldest_id = min(self._sessions, key=lambda k: self._sessions[k].last_accessed) + log.warning(f"Max sessions reached ({self._max_sessions}), evicting oldest: {oldest_id}") + del self._sessions[oldest_id] + + session_id = session_id or str(uuid.uuid4()) + entry = SessionEntry( + session_id=session_id, + service_tag=service_tag, + service_instance=service_instance, + ) + self._sessions[session_id] = entry + log.info(f"Created session {session_id} for service {service_tag}") + return entry + + async def get(self, session_id: str) -> Optional[SessionEntry]: + """Get a session by ID, returns None if not found or expired.""" + async with self._lock: + entry = self._sessions.get(session_id) + if entry is None: + return None + + if entry.auth_status not in (AuthStatus.AUTHENTICATING, AuthStatus.PENDING_INPUT): + elapsed = (datetime.now(timezone.utc) - entry.last_accessed).total_seconds() + if elapsed > self._ttl: + log.info(f"Session {session_id} expired (elapsed={elapsed:.0f}s, ttl={self._ttl}s)") + del self._sessions[session_id] + return None + + entry.touch() + return entry + + async def delete(self, session_id: str) -> bool: + """Delete a session. Returns True if it existed.""" + async with self._lock: + entry = self._sessions.pop(session_id, None) + if entry: + if entry.input_bridge: + entry.input_bridge.cancel() + self._cleanup_cache_dir(entry.cache_tag) + log.info(f"Deleted session {session_id}") + return True + return False + + async def cleanup_expired(self) -> int: + """Remove all expired sessions. Returns count of removed sessions.""" + async with self._lock: + now = datetime.now(timezone.utc) + expired = [] + for sid, entry in self._sessions.items(): + elapsed = (now - entry.last_accessed).total_seconds() + if entry.auth_status in (AuthStatus.AUTHENTICATING, AuthStatus.PENDING_INPUT): + if elapsed > 600: + expired.append(sid) + elif elapsed > self._ttl: + expired.append(sid) + for sid in expired: + entry = self._sessions.pop(sid) + if entry.input_bridge: + entry.input_bridge.cancel() + self._cleanup_cache_dir(entry.cache_tag) + if expired: + log.info(f"Cleaned up {len(expired)} expired sessions") + return len(expired) + + async def start_cleanup_loop(self) -> None: + """Start periodic cleanup of expired sessions.""" + if self._cleanup_task is not None: + return + + async def _loop() -> None: + while True: + await asyncio.sleep(60) # Check every minute + try: + await self.cleanup_expired() + except Exception: + log.exception("Error during session cleanup") + + self._cleanup_task = asyncio.create_task(_loop()) + log.info("Session cleanup loop started") + + async def stop_cleanup_loop(self) -> None: + """Stop the periodic cleanup task.""" + if self._cleanup_task is not None: + self._cleanup_task.cancel() + self._cleanup_task = None + + async def cancel_all_bridges(self) -> None: + """Cancel all active input bridges (called on server shutdown).""" + async with self._lock: + for entry in self._sessions.values(): + if entry.input_bridge: + entry.input_bridge.cancel() + count = len(self._sessions) + if count: + log.info(f"Cancelled bridges for {count} active session(s)") + + @staticmethod + def _cleanup_cache_dir(cache_tag: Optional[str]) -> None: + """Remove session cache directory and empty parents.""" + if not cache_tag: + return + import shutil + + cache_dir = config.directories.cache / cache_tag + if cache_dir.is_dir(): + try: + shutil.rmtree(cache_dir) + except Exception as e: + log.warning(f"Failed to remove session cache {cache_dir}: {e}") + for parent in cache_dir.parents: + if parent == config.directories.cache: + break + try: + if parent.is_dir() and not any(parent.iterdir()): + parent.rmdir() + except Exception: + break + + @property + def session_count(self) -> int: + """Number of active sessions.""" + return len(self._sessions) + + +# Singleton instance +_session_store: Optional[SessionStore] = None + + +def get_session_store() -> SessionStore: + """Get or create the global session store singleton.""" + global _session_store + if _session_store is None: + _session_store = SessionStore() + return _session_store diff --git a/reset/packages/envied/src/envied/core/binaries.py b/reset/packages/envied/src/envied/core/binaries.py new file mode 100644 index 0000000..dd0a114 --- /dev/null +++ b/reset/packages/envied/src/envied/core/binaries.py @@ -0,0 +1,79 @@ +import shutil +import sys +from pathlib import Path +from typing import Optional + +__shaka_platform = {"win32": "win", "darwin": "osx"}.get(sys.platform, sys.platform) + + +def find(*names: str) -> Optional[Path]: + """Find the path of the first found binary name.""" + current_dir = Path(__file__).resolve().parent.parent + local_binaries_dir = current_dir / "binaries" + + ext = ".exe" if sys.platform == "win32" else "" + + for name in names: + if local_binaries_dir.exists(): + candidate_paths = [local_binaries_dir / f"{name}{ext}", local_binaries_dir / name / f"{name}{ext}"] + + for subdir in local_binaries_dir.iterdir(): + if subdir.is_dir(): + candidate_paths.append(subdir / f"{name}{ext}") + + for path in candidate_paths: + if path.is_file(): + # On Unix-like systems, check if file is executable + if sys.platform == "win32" or (path.stat().st_mode & 0o111): + return path + + # Fall back to system PATH + path = shutil.which(name) + if path: + return Path(path) + return None + + +FFMPEG = find("ffmpeg") +FFProbe = find("ffprobe") +FFPlay = find("ffplay") +SubtitleEdit = find("SubtitleEdit") +ShakaPackager = find( + "shaka-packager", + "packager", + f"packager-{__shaka_platform}", + f"packager-{__shaka_platform}-arm64", + f"packager-{__shaka_platform}-x64", +) +CCExtractor = find("ccextractor", "ccextractorwin", "ccextractorwinfull") +HolaProxy = find("hola-proxy") +MPV = find("mpv") +Caddy = find("caddy") +MKVToolNix = find("mkvmerge") +Mkvpropedit = find("mkvpropedit") +DoviTool = find("dovi_tool") +HDR10PlusTool = find("hdr10plus_tool", "HDR10Plus_tool") +Mp4decrypt = find("mp4decrypt") +Docker = find("docker") +ML_Worker = find("ML-Worker") + + +__all__ = ( + "FFMPEG", + "FFProbe", + "FFPlay", + "SubtitleEdit", + "ShakaPackager", + "CCExtractor", + "HolaProxy", + "MPV", + "Caddy", + "MKVToolNix", + "Mkvpropedit", + "DoviTool", + "HDR10PlusTool", + "Mp4decrypt", + "Docker", + "ML_Worker", + "find", +) diff --git a/reset/packages/envied/src/envied/core/cacher.py b/reset/packages/envied/src/envied/core/cacher.py new file mode 100644 index 0000000..0bf6b2a --- /dev/null +++ b/reset/packages/envied/src/envied/core/cacher.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +import zlib +from datetime import datetime, timedelta +from os import stat_result +from pathlib import Path +from typing import Any, Optional, Union + +import jsonpickle +import jwt + +from envied.core.config import config + +EXP_T = Union[datetime, str, int, float] + + +class Cacher: + """Cacher for Services to get and set arbitrary data with expiration dates.""" + + def __init__( + self, + service_tag: str, + key: Optional[str] = None, + version: Optional[int] = 1, + data: Optional[Any] = None, + expiration: Optional[datetime] = None, + ) -> None: + self.service_tag = service_tag + self.key = key + self.version = version + self.data = data or {} + self.expiration = expiration + + if self.expiration and self.expired: + # if its expired, remove the data for safety and delete cache file + self.data = None + self.path.unlink() + + def __bool__(self) -> bool: + return bool(self.data) + + @property + def path(self) -> Path: + """Get the path at which the cache will be read and written.""" + return (config.directories.cache / self.service_tag / self.key).with_suffix(".json") + + @property + def expired(self) -> bool: + return self.expiration and self.expiration < datetime.now() + + def get(self, key: str, version: int = 1) -> Cacher: + """ + Get Cached data for the Service by Key. + :param key: the filename to save the data to, should be url-safe. + :param version: the config data version you expect to use. + :returns: Cache object containing the cached data or None if the file does not exist. + """ + cache = Cacher(self.service_tag, key, version) + if cache.path.is_file(): + data = jsonpickle.loads(cache.path.read_text(encoding="utf8")) + payload = data.copy() + del payload["crc32"] + checksum = data["crc32"] + calculated = zlib.crc32(jsonpickle.dumps(payload).encode("utf8")) + if calculated != checksum: + raise ValueError( + f"The checksum of the Cache payload mismatched. Checksum: {checksum} !== Calculated: {calculated}" + ) + cache.data = data["data"] + cache.expiration = data["expiration"] + cache.version = data["version"] + if cache.version != version: + raise ValueError( + f"The version of your {self.service_tag} {key} cache is outdated. Please delete: {cache.path}" + ) + return cache + + def set(self, data: Any, expiration: Optional[EXP_T] = None) -> Any: + """ + Set Cached data for the Service by Key. + :param data: absolutely anything including None. + :param expiration: when the data expires, optional. Can be ISO 8601, seconds + til expiration, unix timestamp, or a datetime object. + :returns: the data provided for quick wrapping of functions or vars. + """ + self.data = data + + if not expiration: + try: + expiration = jwt.decode(self.data, options={"verify_signature": False})["exp"] + except jwt.DecodeError: + pass + + self.expiration = self.resolve_datetime(expiration) if expiration else None + + payload = {"data": self.data, "expiration": self.expiration, "version": self.version} + payload["crc32"] = zlib.crc32(jsonpickle.dumps(payload).encode("utf8")) + + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text(jsonpickle.dumps(payload)) + + return self.data + + def stat(self) -> stat_result: + """ + Get Cache file OS Stat data like Creation Time, Modified Time, and such. + :returns: an os.stat_result tuple + """ + return self.path.stat() + + @staticmethod + def resolve_datetime(timestamp: EXP_T) -> datetime: + """ + Resolve multiple formats of a Datetime or Timestamp to an absolute Datetime. + + Examples: + >>> now = datetime.now() + datetime.datetime(2022, 6, 27, 9, 49, 13, 657208) + >>> iso8601 = now.isoformat() + '2022-06-27T09:49:13.657208' + >>> Cacher.resolve_datetime(iso8601) + datetime.datetime(2022, 6, 27, 9, 49, 13, 657208) + >>> Cacher.resolve_datetime(iso8601 + "Z") + datetime.datetime(2022, 6, 27, 9, 49, 13, 657208) + >>> Cacher.resolve_datetime(3600) + datetime.datetime(2022, 6, 27, 10, 52, 50, 657208) + >>> Cacher.resolve_datetime('3600') + datetime.datetime(2022, 6, 27, 10, 52, 51, 657208) + >>> Cacher.resolve_datetime(7800.113) + datetime.datetime(2022, 6, 27, 11, 59, 13, 770208) + + In the int/float examples you may notice that it did not return now + 3600 seconds + but rather something a bit more than that. This is because it did not resolve 3600 + seconds from the `now` variable but from right now as the function was called. + """ + if isinstance(timestamp, datetime): + return timestamp + if isinstance(timestamp, str): + if timestamp.endswith("Z"): + # fromisoformat doesn't accept the final Z + timestamp = timestamp.split("Z")[0] + try: + return datetime.fromisoformat(timestamp) + except ValueError: + timestamp = float(timestamp) + try: + if len(str(int(timestamp))) == 13: # JS-style timestamp + timestamp /= 1000 + timestamp = datetime.fromtimestamp(timestamp) + except ValueError: + raise ValueError(f"Unrecognized Timestamp value {timestamp!r}") + if timestamp < datetime.now(): + # timestamp is likely an amount of seconds til expiration + # or, it's an already expired timestamp which is unlikely + timestamp = timestamp + timedelta(seconds=datetime.now().timestamp()) + return timestamp diff --git a/reset/packages/envied/src/envied/core/cacher.py.orig b/reset/packages/envied/src/envied/core/cacher.py.orig new file mode 100644 index 0000000..c88b9ed --- /dev/null +++ b/reset/packages/envied/src/envied/core/cacher.py.orig @@ -0,0 +1,156 @@ +from __future__ import annotations + +import zlib +from datetime import datetime, timedelta +from os import stat_result +from pathlib import Path +from typing import Any, Optional, Union + +import jsonpickle +import jwt + +from envied.core.config import config + +EXP_T = Union[datetime, str, int, float] + + +class Cacher: + """Cacher for Services to get and set arbitrary data with expiration dates.""" + + def __init__( + self, + service_tag: str, + key: Optional[str] = None, + version: Optional[int] = 1, + data: Optional[Any] = None, + expiration: Optional[datetime] = None, + ) -> None: + self.service_tag = service_tag + self.key = key + self.version = version + self.data = data or {} + self.expiration = expiration + + if self.expiration and self.expired: + # if its expired, remove the data for safety and delete cache file + self.data = None + self.path.unlink() + + def __bool__(self) -> bool: + return bool(self.data) + + @property + def path(self) -> Path: + """Get the path at which the cache will be read and written.""" + return (config.directories.cache / self.service_tag / self.key).with_suffix(".json") + + @property + def expired(self) -> bool: + return self.expiration and self.expiration < datetime.now() + + def get(self, key: str, version: int = 1) -> Cacher: + """ + Get Cached data for the Service by Key. + :param key: the filename to save the data to, should be url-safe. + :param version: the config data version you expect to use. + :returns: Cache object containing the cached data or None if the file does not exist. + """ + cache = Cacher(self.service_tag, key, version) + if cache.path.is_file(): + data = jsonpickle.loads(cache.path.read_text(encoding="utf8")) + payload = data.copy() + del payload["crc32"] + checksum = data["crc32"] + calculated = zlib.crc32(jsonpickle.dumps(payload).encode("utf8")) + if calculated != checksum: + raise ValueError( + f"The checksum of the Cache payload mismatched. Checksum: {checksum} !== Calculated: {calculated}" + ) + cache.data = data["data"] + cache.expiration = data["expiration"] + cache.version = data["version"] + if cache.version != version: + raise ValueError( + f"The version of your {self.service_tag} {key} cache is outdated. Please delete: {cache.path}" + ) + return cache + + def set(self, data: Any, expiration: Optional[EXP_T] = None) -> Any: + """ + Set Cached data for the Service by Key. + :param data: absolutely anything including None. + :param expiration: when the data expires, optional. Can be ISO 8601, seconds + til expiration, unix timestamp, or a datetime object. + :returns: the data provided for quick wrapping of functions or vars. + """ + self.data = data + + if not expiration: + try: + expiration = jwt.decode(self.data, options={"verify_signature": False})["exp"] + except jwt.DecodeError: + pass + + self.expiration = self._resolve_datetime(expiration) if expiration else None + + payload = {"data": self.data, "expiration": self.expiration, "version": self.version} + payload["crc32"] = zlib.crc32(jsonpickle.dumps(payload).encode("utf8")) + + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text(jsonpickle.dumps(payload)) + + return self.data + + def stat(self) -> stat_result: + """ + Get Cache file OS Stat data like Creation Time, Modified Time, and such. + :returns: an os.stat_result tuple + """ + return self.path.stat() + + @staticmethod + def _resolve_datetime(timestamp: EXP_T) -> datetime: + """ + Resolve multiple formats of a Datetime or Timestamp to an absolute Datetime. + + Examples: + >>> now = datetime.now() + datetime.datetime(2022, 6, 27, 9, 49, 13, 657208) + >>> iso8601 = now.isoformat() + '2022-06-27T09:49:13.657208' + >>> Cacher._resolve_datetime(iso8601) + datetime.datetime(2022, 6, 27, 9, 49, 13, 657208) + >>> Cacher._resolve_datetime(iso8601 + "Z") + datetime.datetime(2022, 6, 27, 9, 49, 13, 657208) + >>> Cacher._resolve_datetime(3600) + datetime.datetime(2022, 6, 27, 10, 52, 50, 657208) + >>> Cacher._resolve_datetime('3600') + datetime.datetime(2022, 6, 27, 10, 52, 51, 657208) + >>> Cacher._resolve_datetime(7800.113) + datetime.datetime(2022, 6, 27, 11, 59, 13, 770208) + + In the int/float examples you may notice that it did not return now + 3600 seconds + but rather something a bit more than that. This is because it did not resolve 3600 + seconds from the `now` variable but from right now as the function was called. + """ + if isinstance(timestamp, datetime): + return timestamp + if isinstance(timestamp, str): + if timestamp.endswith("Z"): + # fromisoformat doesn't accept the final Z + timestamp = timestamp.split("Z")[0] + try: + return datetime.fromisoformat(timestamp) + except ValueError: + timestamp = float(timestamp) + try: + if len(str(int(timestamp))) == 13: # JS-style timestamp + timestamp /= 1000 + timestamp = datetime.fromtimestamp(timestamp) + except ValueError: + raise ValueError(f"Unrecognized Timestamp value {timestamp!r}") + if timestamp < datetime.now(): + # timestamp is likely an amount of seconds til expiration + # or, it's an already expired timestamp which is unlikely + timestamp = timestamp + timedelta(seconds=datetime.now().timestamp()) + return timestamp diff --git a/reset/packages/envied/src/envied/core/cdm/__init__.py b/reset/packages/envied/src/envied/core/cdm/__init__.py new file mode 100644 index 0000000..f705992 --- /dev/null +++ b/reset/packages/envied/src/envied/core/cdm/__init__.py @@ -0,0 +1,62 @@ +""" +CDM helpers and implementations. + +Keep this module import-light: downstream code frequently imports helpers from +`envied.core.cdm.detect`, which requires importing this package first. +Some CDM implementations pull in optional/heavy dependencies, so we lazily +import them via `__getattr__` (PEP 562). +""" + +from __future__ import annotations + +from typing import Any + +__all__ = [ + "DecryptLabsRemoteCDM", + "CustomRemoteCDM", + "MonaLisaCDM", + "load_cdm", + "is_remote_cdm", + "is_local_cdm", + "cdm_location", + "is_playready_cdm", + "is_widevine_cdm", +] + + +def __getattr__(name: str) -> Any: + if name == "DecryptLabsRemoteCDM": + from .decrypt_labs_remote_cdm import DecryptLabsRemoteCDM + + return DecryptLabsRemoteCDM + if name == "CustomRemoteCDM": + from .custom_remote_cdm import CustomRemoteCDM + + return CustomRemoteCDM + if name == "MonaLisaCDM": + from .monalisa import MonaLisaCDM + + return MonaLisaCDM + if name == "load_cdm": + from .loader import load_cdm + + return load_cdm + + if name in { + "is_remote_cdm", + "is_local_cdm", + "cdm_location", + "is_playready_cdm", + "is_widevine_cdm", + }: + from .detect import cdm_location, is_local_cdm, is_playready_cdm, is_remote_cdm, is_widevine_cdm + + return { + "is_remote_cdm": is_remote_cdm, + "is_local_cdm": is_local_cdm, + "cdm_location": cdm_location, + "is_playready_cdm": is_playready_cdm, + "is_widevine_cdm": is_widevine_cdm, + }[name] + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/reset/packages/envied/src/envied/core/cdm/custom_remote_cdm.py b/reset/packages/envied/src/envied/core/cdm/custom_remote_cdm.py new file mode 100644 index 0000000..be3c902 --- /dev/null +++ b/reset/packages/envied/src/envied/core/cdm/custom_remote_cdm.py @@ -0,0 +1,1092 @@ +from __future__ import annotations + +import base64 +import secrets +from typing import Any, Dict, List, Optional, Union +from uuid import UUID + +import requests +from pywidevine.cdm import Cdm as WidevineCdm +from pywidevine.device import DeviceTypes +from requests import Session + +from envied.core import __version__ +from envied.core.vaults import Vaults + + +class MockCertificateChain: + """Mock certificate chain for PlayReady compatibility.""" + + def __init__(self, name: str): + self._name = name + + def get_name(self) -> str: + return self._name + + +class Key: + """Key object compatible with pywidevine.""" + + def __init__(self, kid: str, key: str, type_: str = "CONTENT"): + if isinstance(kid, str): + clean_kid = kid.replace("-", "") + if len(clean_kid) == 32: + self.kid = UUID(hex=clean_kid) + else: + self.kid = UUID(hex=clean_kid.ljust(32, "0")) + else: + self.kid = kid + + if isinstance(key, str): + self.key = bytes.fromhex(key) + else: + self.key = key + + self.type = type_ + + +class CustomRemoteCDMExceptions: + """Exception classes for compatibility with pywidevine CDM.""" + + class InvalidSession(Exception): + """Raised when session ID is invalid.""" + + class TooManySessions(Exception): + """Raised when session limit is reached.""" + + class InvalidInitData(Exception): + """Raised when PSSH/init data is invalid.""" + + class InvalidLicenseType(Exception): + """Raised when license type is invalid.""" + + class InvalidLicenseMessage(Exception): + """Raised when license message is invalid.""" + + class InvalidContext(Exception): + """Raised when session has no context data.""" + + class SignatureMismatch(Exception): + """Raised when signature verification fails.""" + + +class CustomRemoteCDM: + """ + Highly Configurable Custom Remote CDM implementation. + + This class provides a maximally flexible CDM interface that can adapt to + ANY CDM API format through YAML configuration alone. It's designed to support + both current and future CDM providers without requiring code changes. + + Key Features: + - Fully configuration-driven behavior (all logic controlled via YAML) + - Pluggable authentication strategies (header, body, bearer, basic, custom) + - Flexible endpoint configuration (custom paths, methods, timeouts) + - Advanced parameter mapping (rename, add static, conditional, nested) + - Powerful response parsing (deep field access, type detection, transforms) + - Transform engine (base64, hex, JSON, custom key formats) + - Condition evaluation (response type detection, success validation) + - Compatible with both Widevine and PlayReady DRM schemes + - Vault integration for intelligent key caching + + Configuration Philosophy: + - 90% of new CDM providers: YAML config only + - 9% of cases: Add new transform type (minimal code) + - 1% of cases: Add new auth strategy (minimal code) + - 0% need to modify core request/response logic + + The class is designed to handle diverse API patterns including: + - Different authentication mechanisms (headers vs body vs tokens) + - Custom endpoint paths and HTTP methods + - Parameter name variations (scheme vs device, init_data vs pssh) + - Nested JSON structures in requests/responses + - Various key formats (JSON objects, colon-separated strings, etc.) + - Different response success indicators and error messages + - Conditional parameters based on device type or other factors + """ + + service_certificate_challenge = b"\x08\x04" + + def __init__( + self, + host: str, + service_name: Optional[str] = None, + vaults: Optional[Vaults] = None, + device: Optional[Dict[str, Any]] = None, + auth: Optional[Dict[str, Any]] = None, + endpoints: Optional[Dict[str, Any]] = None, + request_mapping: Optional[Dict[str, Any]] = None, + response_mapping: Optional[Dict[str, Any]] = None, + caching: Optional[Dict[str, Any]] = None, + legacy: Optional[Dict[str, Any]] = None, + timeout: int = 30, + **kwargs, + ): + """ + Initialize Custom Remote CDM with highly configurable options. + + Args: + host: Base URL for the CDM API + service_name: Service name for key caching and vault operations + vaults: Vaults instance for local key caching + device: Device configuration (name, type, system_id, security_level) + auth: Authentication configuration (type, credentials, headers) + endpoints: Endpoint configuration (paths, methods, timeouts) + request_mapping: Request transformation rules (param names, static params, transforms) + response_mapping: Response parsing rules (field locations, type detection, success conditions) + caching: Caching configuration (enabled, use_vaults, etc.) + legacy: Legacy mode configuration + timeout: Default request timeout in seconds + **kwargs: Additional configuration options for future extensibility + """ + self.host = host.rstrip("/") + self.service_name = service_name or "" + self.vaults = vaults + self.timeout = timeout + + # Device configuration + device = device or {} + self.device_name = device.get("name", "ChromeCDM") + self.device_type_str = device.get("type", "CHROME") + self.system_id = device.get("system_id", 26830) + self.security_level = device.get("security_level", 3) + + # Determine if this is a PlayReady CDM + self._is_playready = self.device_type_str.upper() == "PLAYREADY" or self.device_name in ["SL2", "SL3"] + + # Get device type enum for compatibility + if self.device_type_str: + self.device_type = self._get_device_type_enum(self.device_type_str) + + # Authentication configuration + self.auth_config = auth or {"type": "header", "header_name": "Authorization", "key": ""} + + # Endpoints configuration with defaults + endpoints = endpoints or {} + self.endpoints = { + "get_request": { + "path": endpoints.get("get_request", {}).get("path", "/get-challenge") + if isinstance(endpoints.get("get_request"), dict) + else endpoints.get("get_request", "/get-challenge"), + "method": ( + endpoints.get("get_request", {}).get("method", "POST") + if isinstance(endpoints.get("get_request"), dict) + else "POST" + ), + "timeout": ( + endpoints.get("get_request", {}).get("timeout", self.timeout) + if isinstance(endpoints.get("get_request"), dict) + else self.timeout + ), + }, + "decrypt_response": { + "path": endpoints.get("decrypt_response", {}).get("path", "/get-keys") + if isinstance(endpoints.get("decrypt_response"), dict) + else endpoints.get("decrypt_response", "/get-keys"), + "method": ( + endpoints.get("decrypt_response", {}).get("method", "POST") + if isinstance(endpoints.get("decrypt_response"), dict) + else "POST" + ), + "timeout": ( + endpoints.get("decrypt_response", {}).get("timeout", self.timeout) + if isinstance(endpoints.get("decrypt_response"), dict) + else self.timeout + ), + }, + } + + # Request mapping configuration + self.request_mapping = request_mapping or {} + + # Response mapping configuration + self.response_mapping = response_mapping or {} + + # Caching configuration + caching = caching or {} + self.caching_enabled = caching.get("enabled", True) + self.use_vaults = caching.get("use_vaults", True) and self.vaults is not None + self.check_cached_first = caching.get("check_cached_first", True) + + # Legacy configuration + self.legacy_config = legacy or {"enabled": False} + + # Session management + self._sessions: Dict[bytes, Dict[str, Any]] = {} + self._pssh_b64 = None + self._required_kids: Optional[List[str]] = None + + # HTTP session setup + self._http_session = Session() + self._http_session.headers.update( + {"Content-Type": "application/json", "User-Agent": f"envied.custom-cdm/{__version__}"} + ) + + # Apply custom headers from auth config + custom_headers = self.auth_config.get("custom_headers", {}) + if custom_headers: + self._http_session.headers.update(custom_headers) + + def _get_device_type_enum(self, device_type: str): + """Convert device type string to enum for compatibility.""" + device_type_upper = device_type.upper() + if device_type_upper == "ANDROID": + return DeviceTypes.ANDROID + elif device_type_upper == "CHROME": + return DeviceTypes.CHROME + else: + return DeviceTypes.CHROME + + @property + def is_playready(self) -> bool: + """Check if this CDM is in PlayReady mode.""" + return self._is_playready + + @property + def certificate_chain(self) -> MockCertificateChain: + """Mock certificate chain for PlayReady compatibility.""" + return MockCertificateChain(f"{self.device_name}_Custom_Remote") + + def set_pssh_b64(self, pssh_b64: str) -> None: + """Store base64-encoded PSSH data for PlayReady compatibility.""" + self._pssh_b64 = pssh_b64 + + def set_required_kids(self, kids: List[Union[str, UUID]]) -> None: + """ + Set the required Key IDs for intelligent caching decisions. + + This method enables the CDM to make smart decisions about when to request + additional keys via license challenges. When cached keys are available, + the CDM will compare them against the required KIDs to determine if a + license request is still needed for missing keys. + + Args: + kids: List of required Key IDs as UUIDs or hex strings + + Note: + Should be called by DRM classes (PlayReady/Widevine) before making + license challenge requests to enable optimal caching behavior. + """ + self._required_kids = [] + for kid in kids: + if isinstance(kid, UUID): + self._required_kids.append(str(kid).replace("-", "").lower()) + else: + self._required_kids.append(str(kid).replace("-", "").lower()) + + def _generate_session_id(self) -> bytes: + """Generate a unique session ID.""" + return secrets.token_bytes(16) + + def _get_init_data_from_pssh(self, pssh: Any) -> str: + """Extract init data from various PSSH formats.""" + if self.is_playready and self._pssh_b64: + return self._pssh_b64 + + if hasattr(pssh, "dumps"): + dumps_result = pssh.dumps() + + if isinstance(dumps_result, str): + try: + base64.b64decode(dumps_result) + return dumps_result + except Exception: + return base64.b64encode(dumps_result.encode("utf-8")).decode("utf-8") + else: + return base64.b64encode(dumps_result).decode("utf-8") + elif hasattr(pssh, "raw"): + raw_data = pssh.raw + if isinstance(raw_data, str): + raw_data = raw_data.encode("utf-8") + return base64.b64encode(raw_data).decode("utf-8") + elif hasattr(pssh, "__class__") and "WrmHeader" in pssh.__class__.__name__: + if self.is_playready: + raise ValueError("PlayReady WRM header received but no PSSH B64 was set via set_pssh_b64()") + + if hasattr(pssh, "raw_bytes"): + return base64.b64encode(pssh.raw_bytes).decode("utf-8") + elif hasattr(pssh, "bytes"): + return base64.b64encode(pssh.bytes).decode("utf-8") + else: + raise ValueError(f"Cannot extract PSSH data from WRM header type: {type(pssh)}") + else: + raise ValueError(f"Unsupported PSSH type: {type(pssh)}") + + def _get_nested_field(self, data: Dict[str, Any], field_path: str, default: Any = None) -> Any: + """ + Get a nested field from a dictionary using dot notation. + + Args: + data: Dictionary to extract field from + field_path: Field path using dot notation (e.g., "data.cached_keys") + default: Default value if field not found + + Returns: + Field value or default + + Examples: + _get_nested_field({"data": {"keys": [1,2,3]}}, "data.keys") -> [1,2,3] + _get_nested_field({"message": "success"}, "message") -> "success" + """ + if not field_path: + return default + + keys = field_path.split(".") + current = data + + for key in keys: + if isinstance(current, dict) and key in current: + current = current[key] + else: + return default + + return current + + def _apply_transform(self, value: Any, transform_type: str) -> Any: + """ + Apply a transformation to a value. + + Args: + value: Value to transform + transform_type: Type of transformation to apply + + Returns: + Transformed value + + Supported transforms: + - base64_encode: Encode bytes/string to base64 + - base64_decode: Decode base64 string to bytes + - hex_encode: Encode bytes to hex string + - hex_decode: Decode hex string to bytes + - json_stringify: Convert object to JSON string + - json_parse: Parse JSON string to object + - parse_key_string: Parse "kid:key" format strings + """ + if transform_type == "base64_encode": + if isinstance(value, str): + value = value.encode("utf-8") + return base64.b64encode(value).decode("utf-8") + + elif transform_type == "base64_decode": + if isinstance(value, str): + return base64.b64decode(value) + return value + + elif transform_type == "hex_encode": + if isinstance(value, bytes): + return value.hex() + elif isinstance(value, str): + return value.encode("utf-8").hex() + return value + + elif transform_type == "hex_decode": + if isinstance(value, str): + return bytes.fromhex(value) + return value + + elif transform_type == "json_stringify": + import json + + return json.dumps(value) + + elif transform_type == "json_parse": + import json + + if isinstance(value, str): + return json.loads(value) + return value + + elif transform_type == "parse_key_string": + # Handle key formats like "kid:key" or "--key kid:key" + if isinstance(value, str): + keys = [] + for line in value.split("\n"): + line = line.strip() + if line.startswith("--key "): + line = line[6:] + if ":" in line: + kid, key = line.split(":", 1) + keys.append({"kid": kid.strip(), "key": key.strip(), "type": "CONTENT"}) + return keys + return value + + # Unknown transform type - return value unchanged + return value + + def _evaluate_condition(self, condition: str, context: Dict[str, Any]) -> bool: + """ + Evaluate a simple condition against a context. + + Args: + condition: Condition string (e.g., "message == 'success'") + context: Context dictionary with values to check + + Returns: + True if condition is met, False otherwise + + Supported conditions: + - "field == value": Equality check + - "field != value": Inequality check + - "field == null": Null check + - "field != null": Not null check + - "field exists": Existence check + """ + condition = condition.strip() + + # Check for existence + if " exists" in condition: + field = condition.replace(" exists", "").strip() + return self._get_nested_field(context, field) is not None + + # Check for null comparisons + if " == null" in condition: + field = condition.replace(" == null", "").strip() + return self._get_nested_field(context, field) is None + + if " != null" in condition: + field = condition.replace(" != null", "").strip() + return self._get_nested_field(context, field) is not None + + # Check for equality + if " == " in condition: + parts = condition.split(" == ", 1) + field = parts[0].strip() + expected_value = parts[1].strip().strip("'\"") + actual_value = self._get_nested_field(context, field) + return str(actual_value) == expected_value + + # Check for inequality + if " != " in condition: + parts = condition.split(" != ", 1) + field = parts[0].strip() + expected_value = parts[1].strip().strip("'\"") + actual_value = self._get_nested_field(context, field) + return str(actual_value) != expected_value + + # Unknown condition format - return False + return False + + def _build_request_params( + self, endpoint_name: str, base_params: Dict[str, Any], session: Optional[Dict[str, Any]] = None + ) -> Dict[str, Any]: + """ + Build request parameters with mapping and transformations. + + Args: + endpoint_name: Name of the endpoint (e.g., "get_request", "decrypt_response") + base_params: Base parameters to transform + session: Optional session data for context + + Returns: + Transformed parameters dictionary + + This method applies the following transformations in order: + 1. Parameter name mappings (rename parameters) + 2. Static parameters (add fixed values) + 3. Conditional parameters (add based on conditions) + 4. Parameter transforms (apply data transformations) + 5. Nested parameter structure (create nested objects) + 6. Parameter exclusions (remove unwanted params) + """ + # Get mapping config for this endpoint + mapping_config = self.request_mapping.get(endpoint_name, {}) + + # Start with base parameters + params = base_params.copy() + + # 1. Apply parameter name mappings + param_names = mapping_config.get("param_names", {}) + if param_names: + renamed_params = {} + for old_name, new_name in param_names.items(): + if old_name in params: + renamed_params[new_name] = params.pop(old_name) + params.update(renamed_params) + + # 2. Add static parameters + static_params = mapping_config.get("static_params", {}) + if static_params: + params.update(static_params) + + # 3. Add conditional parameters + conditional_params = mapping_config.get("conditional_params", []) + for condition_block in conditional_params: + condition = condition_block.get("condition", "") + # Create context for condition evaluation + context = { + "device_type": self.device_type_str, + "device_name": self.device_name, + "is_playready": self._is_playready, + } + if session: + context.update(session) + + if self._evaluate_condition(condition, context): + params.update(condition_block.get("params", {})) + + # 4. Apply parameter transforms + transforms = mapping_config.get("transforms", []) + for transform in transforms: + param_name = transform.get("param") + transform_type = transform.get("type") + if param_name in params: + params[param_name] = self._apply_transform(params[param_name], transform_type) + + # 5. Handle nested parameter structure + nested_params = mapping_config.get("nested_params", {}) + if nested_params: + for parent_key, child_keys in nested_params.items(): + nested_obj = {} + for child_key in child_keys: + if child_key in params: + nested_obj[child_key] = params.pop(child_key) + if nested_obj: + params[parent_key] = nested_obj + + # 6. Exclude unwanted parameters + exclude_params = mapping_config.get("exclude_params", []) + for param_name in exclude_params: + params.pop(param_name, None) + + return params + + def _apply_authentication(self, session: Session) -> None: + """ + Apply authentication to the HTTP session based on auth configuration. + + Args: + session: requests.Session to apply authentication to + + Supported auth types: + - header: Add authentication header (e.g., x-api-key, Authorization) + - body: Authentication will be added to request body (handled in request building) + - bearer: Add Bearer token to Authorization header + - basic: Add HTTP Basic authentication + - query: Authentication will be added to query string (handled in request) + """ + auth_type = self.auth_config.get("type", "header") + + if auth_type == "header": + header_name = self.auth_config.get("header_name", "Authorization") + key = self.auth_config.get("key", "") + if key: + session.headers[header_name] = key + + elif auth_type == "bearer": + token = self.auth_config.get("bearer_token") or self.auth_config.get("key", "") + if token: + session.headers["Authorization"] = f"Bearer {token}" + + elif auth_type == "basic": + username = self.auth_config.get("username", "") + password = self.auth_config.get("password", "") + if username and password: + from requests.auth import HTTPBasicAuth + + session.auth = HTTPBasicAuth(username, password) + + def _parse_response_data(self, endpoint_name: str, response_data: Dict[str, Any]) -> Dict[str, Any]: + """ + Parse response data based on response mapping configuration. + + Args: + endpoint_name: Name of the endpoint (e.g., "get_request", "decrypt_response") + response_data: Raw response data from API + + Returns: + Parsed response with standardized field names + + This method extracts fields from the response using the response_mapping + configuration, handling nested fields, type detection, and transformations. + """ + # Get mapping config for this endpoint + mapping_config = self.response_mapping.get(endpoint_name, {}) + + # Extract fields based on mapping + fields_config = mapping_config.get("fields", {}) + parsed = {} + + for standard_name, field_path in fields_config.items(): + value = self._get_nested_field(response_data, field_path) + if value is not None: + parsed[standard_name] = value + + # Apply response transforms + transforms = mapping_config.get("transforms", []) + for transform in transforms: + field_name = transform.get("field") + transform_type = transform.get("type") + if field_name in parsed: + parsed[field_name] = self._apply_transform(parsed[field_name], transform_type) + + # Determine response type + response_types = mapping_config.get("response_types", []) + for response_type_config in response_types: + condition = response_type_config.get("condition", "") + if self._evaluate_condition(condition, parsed): + parsed["_response_type"] = response_type_config.get("type") + break + + # Check success conditions + success_conditions = mapping_config.get("success_conditions", []) + is_success = True + if success_conditions: + is_success = all(self._evaluate_condition(cond, parsed) for cond in success_conditions) + parsed["_is_success"] = is_success + + # Extract error messages if not successful + if not is_success: + error_fields = mapping_config.get("error_fields", ["error", "message", "details"]) + error_messages = [] + for error_field in error_fields: + error_msg = self._get_nested_field(response_data, error_field) + if error_msg and error_msg not in error_messages: + error_messages.append(str(error_msg)) + parsed["_error_message"] = " - ".join(error_messages) if error_messages else "Unknown error" + + return parsed + + def _parse_keys_from_response(self, endpoint_name: str, response_data: Dict[str, Any]) -> List[Dict[str, Any]]: + """ + Parse keys from response data using key field mapping. + + Args: + endpoint_name: Name of the endpoint + response_data: Parsed response data + + Returns: + List of key dictionaries with standardized format + """ + mapping_config = self.response_mapping.get(endpoint_name, {}) + key_fields = mapping_config.get("key_fields", {"kid": "kid", "key": "key", "type": "type"}) + + keys = [] + keys_data = response_data.get("keys", []) + + if isinstance(keys_data, list): + for key_obj in keys_data: + if isinstance(key_obj, dict): + kid = key_obj.get(key_fields.get("kid", "kid")) + key = key_obj.get(key_fields.get("key", "key")) + key_type = key_obj.get(key_fields.get("type", "type"), "CONTENT") + + if kid and key: + keys.append({"kid": str(kid), "key": str(key), "type": str(key_type)}) + + # Handle string format keys (e.g., "kid:key" format) + elif isinstance(keys_data, str): + keys = self._apply_transform(keys_data, "parse_key_string") + + return keys + + def open(self) -> bytes: + """ + Open a new CDM session. + + Returns: + Session identifier as bytes + """ + session_id = self._generate_session_id() + self._sessions[session_id] = { + "service_certificate": None, + "keys": [], + "pssh": None, + "challenge": None, + "remote_session_id": None, + "tried_cache": False, + "cached_keys": None, + } + return session_id + + def close(self, session_id: bytes) -> None: + """ + Close a CDM session and perform comprehensive cleanup. + + Args: + session_id: Session identifier + + Raises: + ValueError: If session ID is invalid + """ + if session_id not in self._sessions: + raise CustomRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + session = self._sessions[session_id] + session.clear() + del self._sessions[session_id] + + def get_service_certificate(self, session_id: bytes) -> Optional[bytes]: + """ + Get the service certificate for a session. + + Args: + session_id: Session identifier + + Returns: + Service certificate if set, None otherwise + + Raises: + ValueError: If session ID is invalid + """ + if session_id not in self._sessions: + raise CustomRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + return self._sessions[session_id]["service_certificate"] + + def set_service_certificate(self, session_id: bytes, certificate: Optional[Union[bytes, str]]) -> str: + """ + Set the service certificate for a session. + + Args: + session_id: Session identifier + certificate: Service certificate (bytes or base64 string) + + Returns: + Certificate status message + + Raises: + ValueError: If session ID is invalid + """ + if session_id not in self._sessions: + raise CustomRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + if certificate is None: + if not self._is_playready and self.device_name == "L1": + certificate = WidevineCdm.common_privacy_cert + self._sessions[session_id]["service_certificate"] = base64.b64decode(certificate) + return "Using default Widevine common privacy certificate for L1" + else: + self._sessions[session_id]["service_certificate"] = None + return "No certificate set (not required for this device type)" + + if isinstance(certificate, str): + certificate = base64.b64decode(certificate) + + self._sessions[session_id]["service_certificate"] = certificate + return "Successfully set Service Certificate" + + def has_cached_keys(self, session_id: bytes) -> bool: + """ + Check if cached keys are available for the session. + + Args: + session_id: Session identifier + + Returns: + True if cached keys are available + + Raises: + ValueError: If session ID is invalid + """ + if session_id not in self._sessions: + raise CustomRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + session = self._sessions[session_id] + session_keys = session.get("keys", []) + return len(session_keys) > 0 + + def get_license_challenge( + self, session_id: bytes, pssh_or_wrm: Any, license_type: str = "STREAMING", privacy_mode: bool = True + ) -> bytes: + """ + Generate a license challenge using the custom CDM API. + + This method implements intelligent caching logic that checks vaults first, + then attempts to retrieve cached keys from the API, and only makes a + license request if keys are missing. + + Args: + session_id: Session identifier + pssh_or_wrm: PSSH object or WRM header (for PlayReady compatibility) + license_type: Type of license (STREAMING, OFFLINE, AUTOMATIC) - for compatibility only + privacy_mode: Whether to use privacy mode - for compatibility only + + Returns: + License challenge as bytes, or empty bytes if available keys satisfy requirements + + Raises: + InvalidSession: If session ID is invalid + requests.RequestException: If API request fails + """ + _ = license_type, privacy_mode + + if session_id not in self._sessions: + raise CustomRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + session = self._sessions[session_id] + session["pssh"] = pssh_or_wrm + init_data = self._get_init_data_from_pssh(pssh_or_wrm) + + # Check vaults for cached keys first + if self.use_vaults and self._required_kids: + vault_keys = [] + for kid_str in self._required_kids: + try: + clean_kid = kid_str.replace("-", "") + if len(clean_kid) == 32: + kid_uuid = UUID(hex=clean_kid) + else: + kid_uuid = UUID(hex=clean_kid.ljust(32, "0")) + key, _ = self.vaults.get_key(kid_uuid) + if key and key.count("0") != len(key): + vault_keys.append({"kid": kid_str, "key": key, "type": "CONTENT"}) + except (ValueError, TypeError): + continue + + if vault_keys: + vault_kids = set(k["kid"] for k in vault_keys) + required_kids = set(self._required_kids) + + if required_kids.issubset(vault_kids): + session["keys"] = vault_keys + return b"" + else: + session["vault_keys"] = vault_keys + + # Build request parameters + base_params = { + "scheme": self.device_name, + "init_data": init_data, + } + + if self.service_name: + base_params["service"] = self.service_name + + if session["service_certificate"]: + base_params["service_certificate"] = base64.b64encode(session["service_certificate"]).decode("utf-8") + + # Transform parameters based on configuration + request_params = self._build_request_params("get_request", base_params, session) + + # Apply authentication + self._apply_authentication(self._http_session) + + # Make API request + endpoint_config = self.endpoints["get_request"] + url = f"{self.host}{endpoint_config['path']}" + timeout = endpoint_config["timeout"] + + response = self._http_session.post(url, json=request_params, timeout=timeout) + + if response.status_code != 200: + raise requests.RequestException(f"API request failed: {response.status_code} {response.text}") + + # Parse response + response_data = response.json() + parsed_response = self._parse_response_data("get_request", response_data) + + # Check if request was successful + if not parsed_response.get("_is_success", False): + error_msg = parsed_response.get("_error_message", "Unknown error") + raise requests.RequestException(f"API error: {error_msg}") + + # Determine response type + response_type = parsed_response.get("_response_type") + + # Handle cached keys response + if response_type == "cached_keys" or "cached_keys" in parsed_response: + cached_keys = self._parse_keys_from_response("get_request", parsed_response) + + all_available_keys = list(cached_keys) + if "vault_keys" in session: + all_available_keys.extend(session["vault_keys"]) + + session["tried_cache"] = True + + # Check if we have all required keys + if self._required_kids: + available_kids = set() + for key in all_available_keys: + if isinstance(key, dict) and "kid" in key: + available_kids.add(key["kid"].replace("-", "").lower()) + + required_kids = set(self._required_kids) + missing_kids = required_kids - available_kids + + if missing_kids: + # Store cached keys separately - don't populate session["keys"] yet + # This allows parse_license() to properly combine cached + license keys + session["cached_keys"] = cached_keys + else: + # All required keys are available from cache + session["keys"] = all_available_keys + return b"" + else: + # No required KIDs specified - return cached keys + session["keys"] = all_available_keys + return b"" + + # Handle license request response or fetch license if keys missing + challenge = parsed_response.get("challenge") + remote_session_id = parsed_response.get("session_id") + + if challenge and remote_session_id: + # Decode challenge if it's base64 + if isinstance(challenge, str): + try: + challenge = base64.b64decode(challenge) + except Exception: + challenge = challenge.encode("utf-8") + + session["challenge"] = challenge + session["remote_session_id"] = remote_session_id + return challenge + + # If we have some keys but not all, return empty to skip license parsing + if session.get("keys"): + return b"" + + raise requests.RequestException("API response did not contain challenge or cached keys") + + def parse_license(self, session_id: bytes, license_message: Union[bytes, str]) -> None: + """ + Parse license response using the custom CDM API. + + This method intelligently combines cached keys with newly obtained license keys, + avoiding duplicates while ensuring all required keys are available. + + Args: + session_id: Session identifier + license_message: License response from license server + + Raises: + ValueError: If session ID is invalid or no challenge available + requests.RequestException: If API request fails + """ + if session_id not in self._sessions: + raise CustomRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + session = self._sessions[session_id] + + # Skip parsing if we already have final keys (no cached keys to combine) + # If cached_keys exist (Widevine or PlayReady), we need to combine them with license keys + if session["keys"] and "cached_keys" not in session: + return + + # Ensure we have a challenge and session ID + if not session.get("challenge") or not session.get("remote_session_id"): + raise ValueError("No challenge available - call get_license_challenge first") + + # Prepare license message + if isinstance(license_message, str): + if self.is_playready and license_message.strip().startswith(" List[Key]: + """ + Get keys from the session. + + Args: + session_id: Session identifier + type_: Optional key type filter (CONTENT, SIGNING, etc.) + + Returns: + List of Key objects + + Raises: + InvalidSession: If session ID is invalid + """ + if session_id not in self._sessions: + raise CustomRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + key_dicts = self._sessions[session_id]["keys"] + keys = [Key(kid=k["kid"], key=k["key"], type_=k["type"]) for k in key_dicts] + + if type_: + keys = [key for key in keys if key.type == type_] + + return keys + + +__all__ = ["CustomRemoteCDM"] diff --git a/reset/packages/envied/src/envied/core/cdm/decrypt_labs_remote_cdm.py b/reset/packages/envied/src/envied/core/cdm/decrypt_labs_remote_cdm.py new file mode 100644 index 0000000..ac8cdbc --- /dev/null +++ b/reset/packages/envied/src/envied/core/cdm/decrypt_labs_remote_cdm.py @@ -0,0 +1,749 @@ +from __future__ import annotations + +import base64 +import secrets +from typing import Any, Dict, List, Optional, Union +from uuid import UUID + +import requests +from pywidevine.cdm import Cdm as WidevineCdm +from pywidevine.device import DeviceTypes +from requests import Session + +from envied.core import __version__ +from envied.core.vaults import Vaults + + +class MockCertificateChain: + """Mock certificate chain for PlayReady compatibility.""" + + def __init__(self, name: str): + self._name = name + + def get_name(self) -> str: + return self._name + + +class Key: + """Key object compatible with pywidevine.""" + + def __init__(self, kid: str, key: str, type_: str = "CONTENT"): + if isinstance(kid, str): + clean_kid = kid.replace("-", "") + if len(clean_kid) == 32: + self.kid = UUID(hex=clean_kid) + else: + self.kid = UUID(hex=clean_kid.ljust(32, "0")) + else: + self.kid = kid + + if isinstance(key, str): + self.key = bytes.fromhex(key) + else: + self.key = key + + self.type = type_ + + +class DecryptLabsRemoteCDMExceptions: + """Exception classes for compatibility with pywidevine CDM.""" + + class InvalidSession(Exception): + """Raised when session ID is invalid.""" + + class TooManySessions(Exception): + """Raised when session limit is reached.""" + + class InvalidInitData(Exception): + """Raised when PSSH/init data is invalid.""" + + class InvalidLicenseType(Exception): + """Raised when license type is invalid.""" + + class InvalidLicenseMessage(Exception): + """Raised when license message is invalid.""" + + class InvalidContext(Exception): + """Raised when session has no context data.""" + + class SignatureMismatch(Exception): + """Raised when signature verification fails.""" + + +class DecryptLabsRemoteCDM: + """ + Decrypt Labs Remote CDM implementation with intelligent caching system. + + This class provides a drop-in replacement for pywidevine's local CDM using + Decrypt Labs' KeyXtractor API service, enhanced with smart caching logic + that minimizes unnecessary license requests. + + Key Features: + - Compatible with both Widevine and PlayReady DRM schemes + - Intelligent caching that compares required vs. available keys + - Optimized caching for L1/L2 devices (leverages API auto-optimization) + - Automatic key combination for mixed cache/license scenarios + - Seamless fallback to license requests when keys are missing + + Intelligent Caching System: + 1. DRM classes (PlayReady/Widevine) provide required KIDs via set_required_kids() + 2. get_license_challenge() first checks for cached keys + 3. For L1/L2 devices, always attempts cached keys first (API optimized) + 4. If cached keys satisfy requirements, returns empty challenge (no license needed) + 5. If keys are missing, makes targeted license request for remaining keys + 6. parse_license() combines cached and license keys intelligently + """ + + service_certificate_challenge = b"\x08\x04" + + def __init__( + self, + secret: str, + host: str = "https://keyxtractor.decryptlabs.com", + device_name: str = "ChromeCDM", + service_name: Optional[str] = None, + vaults: Optional[Vaults] = None, + device_type: Optional[str] = None, + system_id: Optional[int] = None, + security_level: Optional[int] = None, + **kwargs, + ): + """ + Initialize Decrypt Labs Remote CDM for Widevine and PlayReady schemes. + + Args: + secret: Decrypt Labs API key (matches config format) + host: Decrypt Labs API host URL (matches config format) + device_name: DRM scheme (ChromeCDM, L1, L2 for Widevine; SL2, SL3 for PlayReady) + service_name: Service name for key caching and vault operations + vaults: Vaults instance for local key caching + device_type: Device type (CHROME, ANDROID, PLAYREADY) - for compatibility + system_id: System ID - for compatibility + security_level: Security level - for compatibility + """ + _ = kwargs + + self.secret = secret + self.host = host.rstrip("/") + self.device_name = device_name + self.service_name = service_name or "" + self.vaults = vaults + self.uch = self.host != "https://keyxtractor.decryptlabs.com" + + self._device_type_str = device_type + if device_type: + self.device_type = self._get_device_type_enum(device_type) + + self._is_playready = (device_type and device_type.upper() == "PLAYREADY") or (device_name in ["SL2", "SL3"]) + + if self._is_playready: + self.system_id = system_id or 0 + self.security_level = security_level or (2000 if device_name == "SL2" else 3000) + else: + self.system_id = system_id or 26830 + self.security_level = security_level or 3 + + self._sessions: Dict[bytes, Dict[str, Any]] = {} + self._pssh_b64 = None + self._required_kids: Optional[List[str]] = None + self._http_session = Session() + self._http_session.headers.update( + { + "decrypt-labs-api-key": self.secret, + "Content-Type": "application/json", + "User-Agent": f"envied.decrypt-labs-cdm/{__version__}", + } + ) + + def _get_device_type_enum(self, device_type: str): + """Convert device type string to enum for compatibility.""" + device_type_upper = device_type.upper() + if device_type_upper == "ANDROID": + return DeviceTypes.ANDROID + elif device_type_upper == "CHROME": + return DeviceTypes.CHROME + else: + return DeviceTypes.CHROME + + @property + def is_playready(self) -> bool: + """Check if this CDM is in PlayReady mode.""" + return self._is_playready + + @property + def certificate_chain(self) -> MockCertificateChain: + """Mock certificate chain for PlayReady compatibility.""" + return MockCertificateChain(f"{self.device_name}_Remote") + + def set_pssh_b64(self, pssh_b64: str) -> None: + """Store base64-encoded PSSH data for PlayReady compatibility.""" + self._pssh_b64 = pssh_b64 + + def set_required_kids(self, kids: List[Union[str, UUID]]) -> None: + """ + Set the required Key IDs for intelligent caching decisions. + + This method enables the CDM to make smart decisions about when to request + additional keys via license challenges. When cached keys are available, + the CDM will compare them against the required KIDs to determine if a + license request is still needed for missing keys. + + Args: + kids: List of required Key IDs as UUIDs or hex strings + + Note: + Should be called by DRM classes (PlayReady/Widevine) before making + license challenge requests to enable optimal caching behavior. + """ + self._required_kids = [] + for kid in kids: + if isinstance(kid, UUID): + self._required_kids.append(str(kid).replace("-", "").lower()) + else: + self._required_kids.append(str(kid).replace("-", "").lower()) + + def _generate_session_id(self) -> bytes: + """Generate a unique session ID.""" + return secrets.token_bytes(16) + + def _get_init_data_from_pssh(self, pssh: Any) -> str: + """Extract init data from various PSSH formats.""" + if self.is_playready and self._pssh_b64: + return self._pssh_b64 + + if hasattr(pssh, "dumps"): + dumps_result = pssh.dumps() + + if isinstance(dumps_result, str): + try: + base64.b64decode(dumps_result) + return dumps_result + except Exception: + return base64.b64encode(dumps_result.encode("utf-8")).decode("utf-8") + else: + return base64.b64encode(dumps_result).decode("utf-8") + elif hasattr(pssh, "raw"): + raw_data = pssh.raw + if isinstance(raw_data, str): + raw_data = raw_data.encode("utf-8") + return base64.b64encode(raw_data).decode("utf-8") + elif hasattr(pssh, "__class__") and "WrmHeader" in pssh.__class__.__name__: + if self.is_playready: + raise ValueError("PlayReady WRM header received but no PSSH B64 was set via set_pssh_b64()") + + if hasattr(pssh, "raw_bytes"): + return base64.b64encode(pssh.raw_bytes).decode("utf-8") + elif hasattr(pssh, "bytes"): + return base64.b64encode(pssh.bytes).decode("utf-8") + else: + raise ValueError(f"Cannot extract PSSH data from WRM header type: {type(pssh)}") + else: + raise ValueError(f"Unsupported PSSH type: {type(pssh)}") + + def open(self) -> bytes: + """ + Open a new CDM session. + + Returns: + Session identifier as bytes + """ + session_id = self._generate_session_id() + self._sessions[session_id] = { + "service_certificate": None, + "keys": [], + "pssh": None, + "challenge": None, + "decrypt_labs_session_id": None, + "tried_cache": False, + "cached_keys": None, + } + return session_id + + def close(self, session_id: bytes) -> None: + """ + Close a CDM session and perform comprehensive cleanup. + + Args: + session_id: Session identifier + + Raises: + ValueError: If session ID is invalid + """ + if session_id not in self._sessions: + raise DecryptLabsRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + session = self._sessions[session_id] + session.clear() + del self._sessions[session_id] + + def get_service_certificate(self, session_id: bytes) -> Optional[bytes]: + """ + Get the service certificate for a session. + + Args: + session_id: Session identifier + + Returns: + Service certificate if set, None otherwise + + Raises: + ValueError: If session ID is invalid + """ + if session_id not in self._sessions: + raise DecryptLabsRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + return self._sessions[session_id]["service_certificate"] + + def set_service_certificate(self, session_id: bytes, certificate: Optional[Union[bytes, str]]) -> str: + """ + Set the service certificate for a session. + + Args: + session_id: Session identifier + certificate: Service certificate (bytes or base64 string) + + Returns: + Certificate status message + + Raises: + ValueError: If session ID is invalid + """ + if session_id not in self._sessions: + raise DecryptLabsRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + if certificate is None: + if not self._is_playready and self.device_name == "L1": + certificate = WidevineCdm.common_privacy_cert + self._sessions[session_id]["service_certificate"] = base64.b64decode(certificate) + return "Using default Widevine common privacy certificate for L1" + else: + self._sessions[session_id]["service_certificate"] = None + return "No certificate set (not required for this device type)" + + if isinstance(certificate, str): + certificate = base64.b64decode(certificate) + + self._sessions[session_id]["service_certificate"] = certificate + return "Successfully set Service Certificate" + + def has_cached_keys(self, session_id: bytes) -> bool: + """ + Check if cached keys are available for the session. + + Args: + session_id: Session identifier + + Returns: + True if cached keys are available + + Raises: + ValueError: If session ID is invalid + """ + if session_id not in self._sessions: + raise DecryptLabsRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + session = self._sessions[session_id] + session_keys = session.get("keys", []) + return len(session_keys) > 0 + + def get_license_challenge( + self, session_id: bytes, pssh_or_wrm: Any, license_type: str = "STREAMING", privacy_mode: bool = True + ) -> bytes: + """ + Generate a license challenge using Decrypt Labs API with intelligent caching. + + This method implements smart caching logic that: + 1. First checks local vaults for required keys + 2. Attempts to retrieve cached keys from the API + 3. If required KIDs are set, compares available keys (vault + cached) against requirements + 4. Only makes a license request if keys are missing + 5. Returns empty challenge if all required keys are available + + The intelligent caching works as follows: + - Local vaults: Always checked first if available + - For L1/L2 devices: Always prioritizes cached keys (API automatically optimizes) + - For other devices: Uses cache retry logic based on session state + - With required KIDs set: Only requests license for missing keys + - Without required KIDs: Returns any available cached keys + - For PlayReady: Combines vault, cached, and license keys seamlessly + + Args: + session_id: Session identifier + pssh_or_wrm: PSSH object or WRM header (for PlayReady compatibility) + license_type: Type of license (STREAMING, OFFLINE, AUTOMATIC) - for compatibility only + privacy_mode: Whether to use privacy mode - for compatibility only + + Returns: + License challenge as bytes, or empty bytes if available keys satisfy requirements + + Raises: + InvalidSession: If session ID is invalid + requests.RequestException: If API request fails + + Note: + Call set_required_kids() before this method for optimal caching behavior. + L1/L2 devices automatically use cached keys when available per API design. + Local vault keys are always checked first when vaults are available. + """ + _ = license_type, privacy_mode + + if session_id not in self._sessions: + raise DecryptLabsRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + session = self._sessions[session_id] + + session["pssh"] = pssh_or_wrm + init_data = self._get_init_data_from_pssh(pssh_or_wrm) + already_tried_cache = session.get("tried_cache", False) + + if self.vaults and self._required_kids: + vault_keys = [] + for kid_str in self._required_kids: + try: + clean_kid = kid_str.replace("-", "") + if len(clean_kid) == 32: + kid_uuid = UUID(hex=clean_kid) + else: + kid_uuid = UUID(hex=clean_kid.ljust(32, "0")) + key, _ = self.vaults.get_key(kid_uuid) + if key and key.count("0") != len(key): + vault_keys.append({"kid": kid_str, "key": key, "type": "CONTENT"}) + except (ValueError, TypeError): + continue + + if vault_keys: + vault_kids = set(k["kid"] for k in vault_keys) + required_kids = set(self._required_kids) + + if required_kids.issubset(vault_kids): + session["keys"] = vault_keys + return b"" + else: + session["vault_keys"] = vault_keys + + if self.device_name in ["L1", "L2"]: + get_cached_keys = True + else: + get_cached_keys = not already_tried_cache + + request_data = { + "scheme": self.device_name, + "init_data": init_data, + "get_cached_keys_if_exists": get_cached_keys, + } + + if self.service_name: + request_data["service"] = self.service_name + + if session["service_certificate"]: + request_data["service_certificate"] = base64.b64encode(session["service_certificate"]).decode("utf-8") + + response = self._http_session.post(f"{self.host}/get-request", json=request_data, timeout=30) + + if response.status_code != 200: + raise requests.RequestException(f"API request failed: {response.status_code} {response.text}") + + data = response.json() + + if data.get("message") != "success": + error_msg = data.get("message", "Unknown error") + if "details" in data: + error_msg += f" - Details: {data['details']}" + if "Error" in data: + error_msg += f" - Error: {data['Error']}" + + if "service_certificate is required" in str(data) and not session["service_certificate"]: + error_msg += " (No service certificate was provided to the CDM session)" + + raise requests.RequestException(f"API error: {error_msg}") + + message_type = data.get("message_type") + + if message_type == "cached-keys" or "cached_keys" in data: + """ + Handle cached keys response from API. + + When the API returns cached keys, we need to determine if they satisfy + our requirements or if we need to make an additional license request + for missing keys. + """ + cached_keys = data.get("cached_keys", []) + parsed_keys = self._parse_cached_keys(cached_keys) + + all_available_keys = list(parsed_keys) + if "vault_keys" in session: + all_available_keys.extend(session["vault_keys"]) + + session["tried_cache"] = True + + if self._required_kids: + available_kids = set() + for key in all_available_keys: + if isinstance(key, dict) and "kid" in key: + available_kids.add(key["kid"].replace("-", "").lower()) + + required_kids = set(self._required_kids) + missing_kids = required_kids - available_kids + + if missing_kids: + session["cached_keys"] = parsed_keys + + if self.device_name in ["L1", "L2"]: + license_request_data = { + "scheme": self.device_name, + "init_data": init_data, + "get_cached_keys_if_exists": False, + } + if self.service_name: + license_request_data["service"] = self.service_name + if session["service_certificate"]: + license_request_data["service_certificate"] = base64.b64encode( + session["service_certificate"] + ).decode("utf-8") + else: + license_request_data = request_data.copy() + license_request_data["get_cached_keys_if_exists"] = False + + # Make license request for missing keys + response = self._http_session.post( + f"{self.host}/get-request", json=license_request_data, timeout=30 + ) + if response.status_code == 200: + data = response.json() + if data.get("message") == "success" and "challenge" in data: + challenge = base64.b64decode(data["challenge"]) + session["challenge"] = challenge + session["decrypt_labs_session_id"] = data["session_id"] + return challenge + + return b"" + else: + # All required keys are available from cache + session["keys"] = all_available_keys + return b"" + else: + # No required KIDs specified - return cached keys + session["keys"] = all_available_keys + return b"" + + if message_type == "license-request" or "challenge" in data: + challenge = base64.b64decode(data["challenge"]) + session["challenge"] = challenge + session["decrypt_labs_session_id"] = data["session_id"] + return challenge + + error_msg = f"Unexpected API response format. message_type={message_type}, available_fields={list(data.keys())}" + if data.get("message"): + error_msg = f"API response: {data['message']} - {error_msg}" + if "details" in data: + error_msg += f" - Details: {data['details']}" + if "Error" in data: + error_msg += f" - Error: {data['Error']}" + + if already_tried_cache and data.get("message") == "success": + return b"" + + raise requests.RequestException(error_msg) + + def parse_license(self, session_id: bytes, license_message: Union[bytes, str]) -> None: + """ + Parse license response using Decrypt Labs API with intelligent key combination. + + For PlayReady content with partial cached keys, this method intelligently + combines the cached keys with newly obtained license keys, avoiding + duplicates while ensuring all required keys are available. + + The key combination process: + 1. Extracts keys from the license response + 2. If cached keys exist (PlayReady), combines them with license keys + 3. Removes duplicate keys by comparing normalized KIDs + 4. Updates the session with the complete key set + + Args: + session_id: Session identifier + license_message: License response from license server + + Raises: + ValueError: If session ID is invalid or no challenge available + requests.RequestException: If API request fails + """ + if session_id not in self._sessions: + raise DecryptLabsRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + session = self._sessions[session_id] + + # Skip parsing if we already have final keys (no cached keys to combine) + # If cached_keys exist (Widevine or PlayReady), we need to combine them with license keys + if session["keys"] and "cached_keys" not in session: + return + + if not session.get("challenge") or not session.get("decrypt_labs_session_id"): + raise ValueError("No challenge available - call get_license_challenge first") + + if isinstance(license_message, str): + if self.is_playready and license_message.strip().startswith(" List[Key]: + """ + Get keys from the session. + + Args: + session_id: Session identifier + type_: Optional key type filter (CONTENT, SIGNING, etc.) + + Returns: + List of Key objects + + Raises: + InvalidSession: If session ID is invalid + """ + if session_id not in self._sessions: + raise DecryptLabsRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}") + + key_dicts = self._sessions[session_id]["keys"] + keys = [Key(kid=k["kid"], key=k["key"], type_=k["type"]) for k in key_dicts] + + if type_: + keys = [key for key in keys if key.type == type_] + + return keys + + def _parse_cached_keys(self, cached_keys_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Parse cached keys from API response. + + Args: + cached_keys_data: List of cached key objects from API + + Returns: + List of key dictionaries + """ + keys = [] + + try: + if cached_keys_data and isinstance(cached_keys_data, list): + for key_data in cached_keys_data: + if "kid" in key_data and "key" in key_data: + keys.append({"kid": key_data["kid"], "key": key_data["key"], "type": "CONTENT"}) + except Exception: + pass + return keys + + def _parse_keys_response(self, data: Dict[str, Any]) -> List[Dict[str, Any]]: + """Parse keys from decrypt response.""" + keys = [] + + if "keys" in data and isinstance(data["keys"], str): + keys_string = data["keys"] + + for line in keys_string.split("\n"): + line = line.strip() + if line.startswith("--key "): + key_part = line[6:] + if ":" in key_part: + kid, key = key_part.split(":", 1) + keys.append({"kid": kid.strip(), "key": key.strip(), "type": "CONTENT"}) + elif "keys" in data and isinstance(data["keys"], list): + for key_data in data["keys"]: + keys.append( + {"kid": key_data.get("kid"), "key": key_data.get("key"), "type": key_data.get("type", "CONTENT")} + ) + + return keys + + +__all__ = ["DecryptLabsRemoteCDM"] diff --git a/reset/packages/envied/src/envied/core/cdm/detect.py b/reset/packages/envied/src/envied/core/cdm/detect.py new file mode 100644 index 0000000..681deda --- /dev/null +++ b/reset/packages/envied/src/envied/core/cdm/detect.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +from typing import Any + + +def is_remote_cdm(cdm: Any) -> bool: + """ + Return True if the CDM instance is backed by a remote/service CDM. + + This is useful for service logic that needs to know whether the CDM runs + locally (in-process) vs over HTTP/RPC (remote). + """ + + if cdm is None: + return False + + if hasattr(cdm, "is_remote_cdm"): + try: + return bool(getattr(cdm, "is_remote_cdm")) + except Exception: + pass + + try: + from pyplayready.remote.remotecdm import RemoteCdm as PlayReadyRemoteCdm + except Exception: + PlayReadyRemoteCdm = None + + if PlayReadyRemoteCdm is not None: + try: + if isinstance(cdm, PlayReadyRemoteCdm): + return True + except Exception: + pass + + try: + from pywidevine.remotecdm import RemoteCdm as WidevineRemoteCdm + except Exception: + WidevineRemoteCdm = None + + if WidevineRemoteCdm is not None: + try: + if isinstance(cdm, WidevineRemoteCdm): + return True + except Exception: + pass + + cls = getattr(cdm, "__class__", None) + mod = getattr(cls, "__module__", "") or "" + name = getattr(cls, "__name__", "") or "" + + if mod == "envied.core.cdm.decrypt_labs_remote_cdm" and name == "DecryptLabsRemoteCDM": + return True + if mod == "envied.core.cdm.custom_remote_cdm" and name == "CustomRemoteCDM": + return True + + if mod.startswith("pyplayready.remote") or mod.startswith("pywidevine.remote"): + return True + if "remote" in mod.lower() and name.lower().endswith("cdm"): + return True + if name.lower().endswith("remotecdm"): + return True + + return False + + +def is_local_cdm(cdm: Any) -> bool: + """ + Return True if the CDM instance is local/in-process. + + Unknown CDM types return False (use `cdm_location()` if you need 3-state). + """ + + if cdm is None: + return False + + if is_remote_cdm(cdm): + return False + + if is_playready_cdm(cdm) or is_widevine_cdm(cdm): + return True + + cls = getattr(cdm, "__class__", None) + mod = getattr(cls, "__module__", "") or "" + name = getattr(cls, "__name__", "") or "" + if mod == "envied.core.cdm.monalisa.monalisa_cdm" and name == "MonaLisaCDM": + return True + + return False + + +def cdm_location(cdm: Any) -> str: + """ + Return one of: "local", "remote", "unknown". + """ + + if is_remote_cdm(cdm): + return "remote" + if is_local_cdm(cdm): + return "local" + return "unknown" + + +def is_playready_cdm(cdm: Any) -> bool: + """ + Return True if the given CDM should be treated as PlayReady. + + This intentionally supports both: + - Local PlayReady CDMs (pyplayready.cdm.Cdm) + - Remote/wrapper CDMs (e.g. DecryptLabsRemoteCDM) that expose `is_playready` + """ + + if cdm is None: + return False + + if hasattr(cdm, "is_playready"): + try: + return bool(getattr(cdm, "is_playready")) + except Exception: + pass + + try: + from pyplayready.cdm import Cdm as PlayReadyCdm + except Exception: + PlayReadyCdm = None + + if PlayReadyCdm is not None: + try: + return isinstance(cdm, PlayReadyCdm) + except Exception: + pass + + try: + from pyplayready.remote.remotecdm import RemoteCdm as PlayReadyRemoteCdm + except Exception: + PlayReadyRemoteCdm = None + + if PlayReadyRemoteCdm is not None: + try: + return isinstance(cdm, PlayReadyRemoteCdm) + except Exception: + pass + + mod = getattr(getattr(cdm, "__class__", None), "__module__", "") or "" + return "pyplayready" in mod + + +def is_widevine_cdm(cdm: Any) -> bool: + """ + Return True if the given CDM should be treated as Widevine. + + Note: for remote/wrapper CDMs that expose `is_playready`, Widevine is treated + as the logical opposite. + """ + + if cdm is None: + return False + + if hasattr(cdm, "is_playready"): + try: + return not bool(getattr(cdm, "is_playready")) + except Exception: + pass + + try: + from pywidevine.cdm import Cdm as WidevineCdm + except Exception: + WidevineCdm = None + + if WidevineCdm is not None: + try: + return isinstance(cdm, WidevineCdm) + except Exception: + pass + + try: + from pywidevine.remotecdm import RemoteCdm as WidevineRemoteCdm + except Exception: + WidevineRemoteCdm = None + + if WidevineRemoteCdm is not None: + try: + return isinstance(cdm, WidevineRemoteCdm) + except Exception: + pass + + mod = getattr(getattr(cdm, "__class__", None), "__module__", "") or "" + return "pywidevine" in mod diff --git a/reset/packages/envied/src/envied/core/cdm/loader.py b/reset/packages/envied/src/envied/core/cdm/loader.py new file mode 100644 index 0000000..7ee5281 --- /dev/null +++ b/reset/packages/envied/src/envied/core/cdm/loader.py @@ -0,0 +1,128 @@ +"""Shared CDM loading utility. + +Instantiates a CDM object (local or remote) given a resolved device name. +Name resolution (quality-based, profile-based, DRM-type) is the caller's +responsibility — this module only handles the instantiation step. +""" + +from __future__ import annotations + +import logging +from typing import Any, Optional + +log = logging.getLogger("cdm") + + +def load_cdm( + cdm_name: str, + *, + service_name: str = "", + vaults: Optional[Any] = None, +) -> Any: + """Instantiate a CDM by device name. + + Looks up the name in config.remote_cdm first (for remote/API CDMs), + then falls back to local .prd / .wvd files. + + Returns a CDM object (WidevineCdm, PlayReadyCdm, RemoteCdm, + DecryptLabsRemoteCDM, CustomRemoteCDM, or PlayReadyRemoteCdm). + + Raises ValueError if the device cannot be found or loaded. + """ + from envied.core.config import config + + cdm_api = next(iter(x.copy() for x in config.remote_cdm if x["name"] == cdm_name), None) + if cdm_api: + return _load_remote_cdm(cdm_api, cdm_name, service_name, vaults) + + return _load_local_cdm(cdm_name) + + +def _load_remote_cdm( + cdm_api: dict, + cdm_name: str, + service_name: str, + vaults: Optional[Any], +) -> Any: + """Instantiate a remote CDM from a config.remote_cdm entry.""" + from envied.core.config import config + + cdm_type = cdm_api.get("type") + + if cdm_type == "decrypt_labs": + from envied.core.cdm.decrypt_labs_remote_cdm import DecryptLabsRemoteCDM + + del cdm_api["name"] + del cdm_api["type"] + + if "secret" not in cdm_api or not cdm_api["secret"]: + if config.decrypt_labs_api_key: + cdm_api["secret"] = config.decrypt_labs_api_key + else: + raise ValueError( + f"No secret provided for DecryptLabs CDM '{cdm_name}' and no global decrypt_labs_api_key configured" + ) + + return DecryptLabsRemoteCDM(service_name=service_name, vaults=vaults, **cdm_api) + + if cdm_type == "custom_api": + from envied.core.cdm.custom_remote_cdm import CustomRemoteCDM + + del cdm_api["name"] + del cdm_api["type"] + return CustomRemoteCDM(service_name=service_name, vaults=vaults, **cdm_api) + + device_type = cdm_api.get("Device Type", cdm_api.get("device_type", "")) + if str(device_type).upper() == "PLAYREADY": + from pyplayready.remote.remotecdm import RemoteCdm as PlayReadyRemoteCdm + + return PlayReadyRemoteCdm( + security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)), + host=cdm_api.get("Host", cdm_api.get("host")), + secret=cdm_api.get("Secret", cdm_api.get("secret")), + device_name=cdm_api.get("Device Name", cdm_api.get("device_name")), + ) + + from pywidevine.remotecdm import RemoteCdm + + return RemoteCdm( + device_type=cdm_api.get("Device Type", cdm_api.get("device_type", "")), + system_id=cdm_api.get("System ID", cdm_api.get("system_id", "")), + security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)), + host=cdm_api.get("Host", cdm_api.get("host")), + secret=cdm_api.get("Secret", cdm_api.get("secret")), + device_name=cdm_api.get("Device Name", cdm_api.get("device_name")), + ) + + +def _load_local_cdm(cdm_name: str) -> Any: + """Instantiate a local CDM from a .prd or .wvd file.""" + from envied.core.config import config + + prd_path = config.directories.prds / f"{cdm_name}.prd" + if not prd_path.is_file(): + prd_path = config.directories.wvds / f"{cdm_name}.prd" + if prd_path.is_file(): + from pyplayready.cdm import Cdm as PlayReadyCdm + from pyplayready.device import Device as PlayReadyDevice + + return PlayReadyCdm.from_device(PlayReadyDevice.load(prd_path)) + + cdm_path = config.directories.wvds / f"{cdm_name}.wvd" + if not cdm_path.is_file(): + raise ValueError(f"{cdm_name} does not exist or is not a file") + + from construct import ConstError + from pywidevine.cdm import Cdm as WidevineCdm + from pywidevine.device import Device + + try: + device = Device.load(cdm_path) + except ConstError as e: + if "expected 2 but parsed 1" in str(e): + raise ValueError( + f"{cdm_name}.wvd seems to be a v1 WVD file, use `pywidevine migrate --help` to migrate it to v2." + ) + raise ValueError(f"{cdm_name}.wvd is an invalid or corrupt Widevine Device file, {e}") + + return WidevineCdm.from_device(device) diff --git a/reset/packages/envied/src/envied/core/cdm/monalisa/__init__.py b/reset/packages/envied/src/envied/core/cdm/monalisa/__init__.py new file mode 100644 index 0000000..999975f --- /dev/null +++ b/reset/packages/envied/src/envied/core/cdm/monalisa/__init__.py @@ -0,0 +1,3 @@ +from .monalisa_cdm import MonaLisaCDM + +__all__ = ["MonaLisaCDM"] diff --git a/reset/packages/envied/src/envied/core/cdm/monalisa/monalisa_cdm.py b/reset/packages/envied/src/envied/core/cdm/monalisa/monalisa_cdm.py new file mode 100644 index 0000000..7b885a3 --- /dev/null +++ b/reset/packages/envied/src/envied/core/cdm/monalisa/monalisa_cdm.py @@ -0,0 +1,417 @@ +""" +MonaLisa CDM - WASM-based Content Decryption Module wrapper. + +This module provides key extraction from MonaLisa-protected content using +a WebAssembly module that runs locally via wasmtime. +""" + +import base64 +import ctypes +import hashlib +import json +import logging +import re +import sys +import uuid +from pathlib import Path +from typing import Dict, Optional, Union + +import wasmtime + +from envied.core import binaries + +logger = logging.getLogger(__name__) + + +class MonaLisaCDM: + """ + MonaLisa CDM wrapper for WASM-based key extraction. + + This CDM differs from Widevine/PlayReady in that it does not use a + challenge/response flow with a license server. Instead, the license + (ticket) is provided directly by the service API, and keys are extracted + locally via the WASM module. + """ + + DYNAMIC_BASE = 6065008 + DYNAMICTOP_PTR = 821968 + LICENSE_KEY_OFFSET = 0x5C8C0C + LICENSE_KEY_LENGTH = 16 + + ENV_STRINGS = ( + "USER=web_user", + "LOGNAME=web_user", + "PATH=/", + "PWD=/", + "HOME=/home/web_user", + "LANG=zh_CN.UTF-8", + "_=./this.program", + ) + + def __init__(self, device_path: Path): + """ + Initialize the MonaLisa CDM. + + Args: + device_path: Path to the device file (.mld). + """ + device_path = Path(device_path) + + self.device_path = device_path + self.base_dir = device_path.parent + + if not self.device_path.is_file(): + raise FileNotFoundError(f"Device file not found at: {self.device_path}") + + try: + data = json.loads(self.device_path.read_text(encoding="utf-8", errors="replace")) + except Exception as e: + raise ValueError(f"Invalid device file (JSON): {e}") + + wasm_path_str = data.get("wasm_path") + if not wasm_path_str: + raise ValueError("Device file missing 'wasm_path'") + + wasm_filename = Path(wasm_path_str).name + wasm_path = self.base_dir / wasm_filename + + if not wasm_path.exists(): + raise FileNotFoundError(f"WASM file not found at: {wasm_path}") + + try: + self.engine = wasmtime.Engine() + if wasm_path.suffix.lower() == ".wat": + self.module = wasmtime.Module.from_file(self.engine, str(wasm_path)) + else: + self.module = wasmtime.Module(self.engine, wasm_path.read_bytes()) + except Exception as e: + raise RuntimeError(f"Failed to load WASM module: {e}") + + self.store = None + self.memory = None + self.instance = None + self.exports = {} + self.ctx = None + + @staticmethod + def get_worker_path() -> Optional[Path]: + """Get ML-Worker binary path from the envied.binaries system.""" + if binaries.ML_Worker: + return Path(binaries.ML_Worker) + return None + + def open(self) -> int: + """ + Open a CDM session. + + Returns: + Session ID (always 1 for MonaLisa). + + Raises: + RuntimeError: If session initialization fails. + """ + try: + self.store = wasmtime.Store(self.engine) + memory_type = wasmtime.MemoryType(wasmtime.Limits(256, 256)) + self.memory = wasmtime.Memory(self.store, memory_type) + + self._write_i32(self.DYNAMICTOP_PTR, self.DYNAMIC_BASE) + imports = self._build_imports() + self.instance = wasmtime.Instance(self.store, self.module, imports) + + ex = self.instance.exports(self.store) + self.exports = { + "___wasm_call_ctors": ex["s"], + "_monalisa_context_alloc": ex["D"], + "monalisa_set_license": ex["F"], + "_monalisa_set_canvas_id": ex["t"], + "_monalisa_version_get": ex["A"], + "monalisa_get_line_number": ex["v"], + "stackAlloc": ex["N"], + "stackSave": ex["L"], + "stackRestore": ex["M"], + } + + self.exports["___wasm_call_ctors"](self.store) + ctx = self.exports["_monalisa_context_alloc"](self.store) + self.ctx = ctx + + # _monalisa_context_alloc is expected to return a positive pointer/handle. + # Treat 0/negative/non-int-like values as allocation failure. + try: + ctx_int = int(ctx) + except Exception: + ctx_int = None + + if ctx_int is None or ctx_int <= 0: + # Ensure we don't leave a partially-initialized instance around. + self.close() + raise RuntimeError(f"Failed to allocate MonaLisa context (ctx={ctx!r})") + return 1 + except Exception as e: + # Clean up partial state (e.g., store/memory/instance) before propagating failure. + self.close() + if isinstance(e, RuntimeError): + raise + raise RuntimeError(f"Failed to initialize session: {e}") from e + + def close(self, session_id: int = 1) -> None: + """ + Close the CDM session and release resources. + + Args: + session_id: The session ID to close (unused, for API compatibility). + """ + self.store = None + self.memory = None + self.instance = None + self.exports = {} + self.ctx = None + + def extract_keys(self, license_data: Union[str, bytes]) -> Dict: + """ + Extract decryption keys from license/ticket data. + + Args: + license_data: The license ticket, either as base64 string or raw bytes. + + Returns: + Dictionary with keys: kid (hex), key (hex), type ("CONTENT"). + + Raises: + RuntimeError: If session not open or license validation fails. + ValueError: If license_data is empty. + """ + if not self.instance or not self.memory or self.ctx is None: + raise RuntimeError("Session not open. Call open() first.") + + if not license_data: + raise ValueError("license_data is empty") + + if isinstance(license_data, bytes): + license_b64 = base64.b64encode(license_data).decode("utf-8") + else: + license_b64 = license_data + + ret = self._ccall( + "monalisa_set_license", + int, + self.ctx, + license_b64, + len(license_b64), + "0", + ) + + if ret != 0: + raise RuntimeError(f"License validation failed with code: {ret}") + + key_bytes = self._extract_license_key_bytes() + + # Extract DCID from license to generate KID + try: + decoded = base64.b64decode(license_b64).decode("ascii", errors="ignore") + except Exception as e: + # Avoid logging raw license content; log only safe metadata. + logger.exception("Failed to base64-decode MonaLisa license (len=%s): %s", len(license_b64), e) + decoded = "" + + m = re.search( + r"DCID-[A-Z0-9]+-[A-Z0-9]+-\d{8}-\d{6}-[A-Z0-9]+-\d{10}-[A-Z0-9]+", + decoded, + ) + if m: + kid_bytes = uuid.uuid5(uuid.NAMESPACE_DNS, m.group()).bytes + else: + # No DCID in the license: derive a deterministic per-license KID to avoid collisions. + try: + license_raw = base64.b64decode(license_b64) + except Exception: + license_raw = license_b64.encode("utf-8", errors="replace") + + license_hash = hashlib.sha256(license_raw).hexdigest() + kid_bytes = uuid.uuid5(uuid.NAMESPACE_DNS, f"monalisa:license:{license_hash}").bytes + + return {"kid": kid_bytes.hex(), "key": key_bytes.hex(), "type": "CONTENT"} + + def _extract_license_key_bytes(self) -> bytes: + """Extract the 16-byte decryption key from WASM memory.""" + data_ptr = self.memory.data_ptr(self.store) + data_len = self.memory.data_len(self.store) + + if self.LICENSE_KEY_OFFSET + self.LICENSE_KEY_LENGTH > data_len: + raise RuntimeError("License key offset beyond memory bounds") + + mem_ptr = ctypes.cast(data_ptr, ctypes.POINTER(ctypes.c_ubyte * data_len)) + start = self.LICENSE_KEY_OFFSET + end = self.LICENSE_KEY_OFFSET + self.LICENSE_KEY_LENGTH + + return bytes(mem_ptr.contents[start:end]) + + def _ccall(self, func_name: str, return_type: type, *args): + """Call a WASM function with automatic string conversion.""" + stack = 0 + converted_args = [] + + try: + for arg in args: + if isinstance(arg, str): + if stack == 0: + stack = self.exports["stackSave"](self.store) + max_length = (len(arg) << 2) + 1 + ptr = self.exports["stackAlloc"](self.store, max_length) + self._string_to_utf8(arg, ptr, max_length) + converted_args.append(ptr) + else: + converted_args.append(arg) + + result = self.exports[func_name](self.store, *converted_args) + finally: + # stackAlloc pointers live on the WASM stack; always restore even if the call throws. + if stack != 0: + exc = sys.exc_info()[1] + try: + self.exports["stackRestore"](self.store, stack) + except Exception: + # If we're already failing, don't mask the original exception. + if exc is None: + raise + + if return_type is bool: + return bool(result) + return result + + def _write_i32(self, addr: int, value: int) -> None: + """Write a 32-bit integer to WASM memory.""" + if addr % 4 != 0: + raise ValueError(f"Unaligned i32 write: addr={addr} (must be 4-byte aligned)") + + data_len = self.memory.data_len(self.store) + if addr < 0 or addr + 4 > data_len: + raise IndexError(f"i32 write out of bounds: addr={addr}, mem_len={data_len}") + + data = self.memory.data_ptr(self.store) + mem_ptr = ctypes.cast(data, ctypes.POINTER(ctypes.c_int32)) + mem_ptr[addr >> 2] = value + + def _string_to_utf8(self, data: str, ptr: int, max_length: int) -> int: + """Convert string to UTF-8 and write to WASM memory.""" + encoded = data.encode("utf-8") + write_length = min(len(encoded), max_length - 1) + + mem_data = self.memory.data_ptr(self.store) + mem_ptr = ctypes.cast(mem_data, ctypes.POINTER(ctypes.c_ubyte)) + + for i in range(write_length): + mem_ptr[ptr + i] = encoded[i] + mem_ptr[ptr + write_length] = 0 + return write_length + + def _write_ascii_to_memory(self, string: str, buffer: int, dont_add_null: int = 0) -> None: + """Write ASCII string to WASM memory.""" + mem_data = self.memory.data_ptr(self.store) + mem_ptr = ctypes.cast(mem_data, ctypes.POINTER(ctypes.c_ubyte)) + + encoded = string.encode("utf-8") + for i, byte_val in enumerate(encoded): + mem_ptr[buffer + i] = byte_val + + if dont_add_null == 0: + mem_ptr[buffer + len(encoded)] = 0 + + def _build_imports(self): + """Build the WASM import stubs required by the MonaLisa module.""" + + def sys_fcntl64(a, b, c): + return 0 + + def fd_write(a, b, c, d): + return 0 + + def fd_close(a): + return 0 + + def sys_ioctl(a, b, c): + return 0 + + def sys_open(a, b, c): + return 0 + + def sys_rmdir(a): + return 0 + + def sys_unlink(a): + return 0 + + def clock(): + return 0 + + def time(a): + return 0 + + def emscripten_run_script(a): + return None + + def fd_seek(a, b, c, d, e): + return 0 + + def emscripten_resize_heap(a): + return 0 + + def fd_read(a, b, c, d): + return 0 + + def emscripten_run_script_string(a): + return 0 + + def emscripten_run_script_int(a): + return 1 + + def emscripten_memcpy_big(dest, src, num): + mem_data = self.memory.data_ptr(self.store) + data_len = self.memory.data_len(self.store) + if num is None: + num = data_len - 1 + mem_ptr = ctypes.cast(mem_data, ctypes.POINTER(ctypes.c_ubyte)) + for i in range(num): + if dest + i < data_len and src + i < data_len: + mem_ptr[dest + i] = mem_ptr[src + i] + return dest + + def environ_get(environ_ptr, environ_buf): + buf_size = 0 + for index, string in enumerate(self.ENV_STRINGS): + ptr = environ_buf + buf_size + self._write_i32(environ_ptr + index * 4, ptr) + self._write_ascii_to_memory(string, ptr) + buf_size += len(string) + 1 + return 0 + + def environ_sizes_get(penviron_count, penviron_buf_size): + self._write_i32(penviron_count, len(self.ENV_STRINGS)) + buf_size = sum(len(s) + 1 for s in self.ENV_STRINGS) + self._write_i32(penviron_buf_size, buf_size) + return 0 + + i32 = wasmtime.ValType.i32() + + return [ + wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32], [i32]), sys_fcntl64), + wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32, i32], [i32]), fd_write), + wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), fd_close), + wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32], [i32]), sys_ioctl), + wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32], [i32]), sys_open), + wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), sys_rmdir), + wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), sys_unlink), + wasmtime.Func(self.store, wasmtime.FuncType([], [i32]), clock), + wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), time), + wasmtime.Func(self.store, wasmtime.FuncType([i32], []), emscripten_run_script), + wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32, i32, i32], [i32]), fd_seek), + wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32], [i32]), emscripten_memcpy_big), + wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), emscripten_resize_heap), + wasmtime.Func(self.store, wasmtime.FuncType([i32, i32], [i32]), environ_get), + wasmtime.Func(self.store, wasmtime.FuncType([i32, i32], [i32]), environ_sizes_get), + wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32, i32], [i32]), fd_read), + wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), emscripten_run_script_string), + wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), emscripten_run_script_int), + self.memory, + ] diff --git a/reset/packages/envied/src/envied/core/commands.py b/reset/packages/envied/src/envied/core/commands.py new file mode 100644 index 0000000..fa03f35 --- /dev/null +++ b/reset/packages/envied/src/envied/core/commands.py @@ -0,0 +1,84 @@ +import logging +from pathlib import Path +from typing import Optional + +import click + +from envied.core.config import config +from envied.core.utilities import import_module_by_path + +log = logging.getLogger("commands") + +_COMMANDS = sorted( + (path for path in config.directories.commands.glob("*.py") if path.stem.lower() != "__init__"), key=lambda x: x.stem +) + + +def load_command(path: Path) -> object: + """Load one command module, returning its stem-named attribute. + + Raises a concise, single-line error naming the command and the real cause so + a broken command never surfaces as a raw traceback pointing at the loader. + """ + try: + module = import_module_by_path(path) + except Exception as e: + raise RuntimeError(f"{path.stem}: failed to import — {type(e).__name__}: {e} ({path})") from e + try: + return getattr(module, path.stem) + except AttributeError as e: + raise RuntimeError( + f"{path.stem}: no object named '{path.stem}' found in {path} — it must match the filename" + ) from e + + +def load_commands(paths: list[Path]) -> tuple[dict[str, object], list[str]]: + """Load every command, returning the good ones plus a list of load errors. + + Importing this module must never raise (it runs at CLI startup, before Rich + is installed, so a raise here prints an ugly pre-setup traceback). Instead we + collect failures and surface them once, cleanly, when the CLI is used. + """ + modules: dict[str, object] = {} + errors: list[str] = [] + for path in paths: + try: + modules[path.stem] = load_command(path) + except Exception as e: + errors.append(str(e)) + return modules, errors + + +_MODULES, LOAD_ERRORS = load_commands(_COMMANDS) + + +def check_load_errors() -> None: + """Raise a single clean error if any command failed to load.""" + if LOAD_ERRORS: + joined = "\n".join(f" - {err}" for err in LOAD_ERRORS) + raise click.ClickException(f"Failed to load {len(LOAD_ERRORS)} command(s):\n{joined}") + + +class Commands(click.MultiCommand): + """Lazy-loaded command group of project commands.""" + + def list_commands(self, ctx: click.Context) -> list[str]: + """Returns a list of command names from the command filenames.""" + check_load_errors() + return [x.stem.replace("_", "-") for x in _COMMANDS] + + def get_command(self, ctx: click.Context, name: str) -> Optional[click.Command]: + """Load the command code and return the main click command function.""" + check_load_errors() + module = _MODULES.get(name) or _MODULES.get(name.replace("-", "_")) + if not module: + raise click.ClickException(f"Unable to find command by the name '{name}'") + + if hasattr(module, "cli"): + return module.cli + + return module + + +# Hide direct access to commands from quick import form, they shouldn't be accessed directly +__all__ = ("Commands",) diff --git a/reset/packages/envied/src/envied/core/config.py b/reset/packages/envied/src/envied/core/config.py new file mode 100644 index 0000000..7309f9c --- /dev/null +++ b/reset/packages/envied/src/envied/core/config.py @@ -0,0 +1,261 @@ +from __future__ import annotations + +import re +import warnings +from pathlib import Path +from typing import Any, Optional + +import yaml +from appdirs import AppDirs + + +class Config: + class _Directories: + # default directories, do not modify here, set via config + app_dirs = AppDirs("envied", False) + core_dir = Path(__file__).resolve().parent + namespace_dir = core_dir.parent + commands = namespace_dir / "commands" + services = [namespace_dir / "services"] + vaults = namespace_dir / "vaults" + fonts = namespace_dir / "fonts" + user_configs = core_dir.parent + data = core_dir.parent + downloads = core_dir.parent.parent / "downloads" + temp = core_dir.parent.parent / "temp" + cache = data / "cache" + cookies = data / "cookies" + logs = data / "logs" + exports = data / "exports" + wvds = data / "WVDs" + prds = data / "PRDs" + dcsl = data / "DCSL" + + class _Filenames: + # default filenames, do not modify here, set via config + log = "envied.{name}_{time}.log" # Directories.logs + debug_log = "envied.debug_{service}_{time}.jsonl" # Directories.logs + config = "config.yaml" # Directories.services / tag + root_config = "envied.yaml" # Directories.user_configs + chapters = "Chapters_{title}_{random}.txt" # Directories.temp + subtitle = "Subtitle_{id}_{language}.srt" # Directories.temp + + def __init__(self, **kwargs: Any): + self.dl: dict = kwargs.get("dl") or {} + self.cdm: dict = kwargs.get("cdm") or {} + self.chapter_fallback_name: str = kwargs.get("chapter_fallback_name") or "" + self.curl_impersonate: dict = kwargs.get("curl_impersonate") or {} + self.remote_cdm: list[dict] = kwargs.get("remote_cdm") or [] + self.credentials: dict = kwargs.get("credentials") or {} + self.subtitle: dict = kwargs.get("subtitle") or {} + + self.directories = self._Directories() + for name, path in (kwargs.get("directories") or {}).items(): + if name.lower() in ("app_dirs", "core_dir", "namespace_dir", "user_configs", "data"): + # these must not be modified by the user + continue + if name == "services" and isinstance(path, list): + setattr(self.directories, name, [Path(p).expanduser() for p in path]) + else: + setattr(self.directories, name, Path(path).expanduser()) + + downloader_cfg = kwargs.get("downloader") + if downloader_cfg and downloader_cfg != "requests": + warnings.warn( + f"downloader '{downloader_cfg}' is deprecated. The unified requests downloader is now used.", + DeprecationWarning, + stacklevel=2, + ) + + self.filenames = self._Filenames() + for name, filename in (kwargs.get("filenames") or {}).items(): + setattr(self.filenames, name, filename) + + self.audio: dict = kwargs.get("audio") or {} + self.headers: dict = kwargs.get("headers") or {} + self.key_vaults: list[dict[str, Any]] = kwargs.get("key_vaults", []) + self.muxing: dict = kwargs.get("muxing") or {} + self.proxy_providers: dict = kwargs.get("proxy_providers") or {} + self.remote_services: dict = kwargs.get("remote_services") or {} + self.serve: dict = kwargs.get("serve") or {} + self.services: dict = kwargs.get("services") or {} + decryption_cfg = kwargs.get("decryption") or {} + if isinstance(decryption_cfg, dict): + self.decryption_map = {k.upper(): v for k, v in decryption_cfg.items()} + self.decryption = self.decryption_map.get("DEFAULT", "shaka") + else: + self.decryption_map = {} + self.decryption = decryption_cfg or "shaka" + + self.set_terminal_bg: bool = kwargs.get("set_terminal_bg", False) + self.tag: str = kwargs.get("tag") or "" + self.tag_group_name: bool = kwargs.get("tag_group_name", True) + self.tag_imdb_tmdb: bool = kwargs.get("tag_imdb_tmdb", True) + self.tmdb_api_key: str = kwargs.get("tmdb_api_key") or "" + self.simkl_client_id: str = kwargs.get("simkl_client_id") or "" + self.decrypt_labs_api_key: str = kwargs.get("decrypt_labs_api_key") or "" + self.ipinfo_api_key: str = kwargs.get("ipinfo_api_key") or "" + self.update_checks: bool = kwargs.get("update_checks", True) + self.update_check_interval: int = kwargs.get("update_check_interval", 24) + + self.language_tags: dict = kwargs.get("language_tags") or {} + self.output_template: dict = kwargs.get("output_template") or {} + folder_cfg = self.output_template.pop("folder", "") + self.folder_template: str = "" + self.folder_templates: dict = {} + if isinstance(folder_cfg, dict): + self.folder_templates = {k: v for k, v in folder_cfg.items() if isinstance(v, str) and v} + elif isinstance(folder_cfg, str): + self.folder_template = folder_cfg or "" + + if kwargs.get("scene_naming") is not None: + raise SystemExit( + "ERROR: The 'scene_naming' option has been removed.\n" + "Please configure 'output_template' in your envied.yaml instead.\n" + "See envied.example.yaml for examples." + ) + + if self.output_template: + self._validate_output_templates() + + self.unicode_filenames: bool = kwargs.get("unicode_filenames", False) + + self.title_cache_time: int = kwargs.get("title_cache_time", 1800) # 30 minutes default + self.title_cache_max_retention: int = kwargs.get("title_cache_max_retention", 86400) # 24 hours default + self.title_cache_enabled: bool = kwargs.get("title_cache_enabled", True) + + self.debug: bool = kwargs.get("debug", False) + self.debug_keys: bool = kwargs.get("debug_keys", False) + + def _validate_output_templates(self) -> None: + """Validate output template configurations and warn about potential issues.""" + if not self.output_template: + return + + valid_variables = { + "title", + "year", + "season", + "episode", + "season_episode", + "episode_name", + "quality", + "resolution", + "source", + "tag", + "track_number", + "artist", + "album", + "disc", + "audio", + "audio_channels", + "audio_full", + "atmos", + "dual", + "multi", + "video", + "hdr", + "hfr", + "edition", + "repack", + "lang_tag", + } + + unsafe_chars = r'[<>:"/\\|?*]' + + all_templates = dict(self.output_template) + if self.folder_template: + all_templates["folder"] = self.folder_template + for kind, tmpl in self.folder_templates.items(): + if kind not in {"movies", "series", "songs"}: + warnings.warn(f"Unknown folder template kind '{kind}' (expected movies/series/songs)") + continue + all_templates[f"folder.{kind}"] = tmpl + + for template_type, template_str in all_templates.items(): + if not isinstance(template_str, str): + warnings.warn(f"Template '{template_type}' must be a string, got {type(template_str).__name__}") + continue + + variables = re.findall(r"\{([^}]+)\}", template_str) + + for var in variables: + var_clean = var.rstrip("?") + if var_clean not in valid_variables: + warnings.warn(f"Unknown template variable '{var}' in {template_type} template") + + test_template = re.sub(r"\{[^}]+\}", "TEST", template_str) + if re.search(unsafe_chars, test_template): + warnings.warn(f"Template '{template_type}' may contain filesystem-unsafe characters") + + if not template_str.strip(): + warnings.warn(f"Template '{template_type}' is empty") + + def get_folder_template(self, kind: str) -> str: + """Resolve the folder template for the given title kind. + + kind: one of "movies", "series", "songs". + Falls back to the legacy single-string folder template, then "". + """ + if self.folder_templates: + tmpl = self.folder_templates.get(kind) + if tmpl: + return tmpl + return self.folder_template or "" + + def get_template_separator(self, template_type: str = "movies") -> str: + """Get the filename separator for the given template type. + + Analyzes the active template to determine whether it uses dots or spaces + between variables. Falls back to dot separator (scene-style) by default. + + Args: + template_type: One of "movies", "series", or "songs". + """ + template = self.output_template[template_type] + between_vars = re.findall(r"\}([^{]*)\{", template) + separator_text = "".join(between_vars) + dot_count = separator_text.count(".") + space_count = separator_text.count(" ") + + return " " if space_count > dot_count else "." + + @classmethod + def from_yaml(cls, path: Path) -> Config: + if not path.exists(): + raise FileNotFoundError(f"Config file path ({path}) was not found") + if not path.is_file(): + raise FileNotFoundError(f"Config file path ({path}) is not to a file.") + return cls(**yaml.safe_load(path.read_text(encoding="utf8")) or {}) + + +# noinspection PyProtectedMember +POSSIBLE_CONFIG_PATHS = ( + # The envied.Namespace Folder (e.g., %appdata%/Python/Python311/site-packages/envied. + Config._Directories.namespace_dir / Config._Filenames.root_config, + # The Parent Folder to the envied.Namespace Folder (e.g., %appdata%/Python/Python311/site-packages) + Config._Directories.namespace_dir.parent / Config._Filenames.root_config, + # The AppDirs User Config Folder (e.g., ~/.config/envied.on Linux, %LOCALAPPDATA%\envied.on Windows) + Path(Config._Directories.app_dirs.user_config_dir) / Config._Filenames.root_config, +) + + +def get_config_path() -> Optional[Path]: + """ + Get Path to Config from any one of the possible locations. + + Returns None if no config file could be found. + """ + for path in POSSIBLE_CONFIG_PATHS: + if path.exists(): + return path + return None + + +config_path = get_config_path() +if config_path: + config = Config.from_yaml(config_path) +else: + config = Config() + +__all__ = ("config",) diff --git a/reset/packages/envied/src/envied/core/console.py b/reset/packages/envied/src/envied/core/console.py new file mode 100644 index 0000000..f139da6 --- /dev/null +++ b/reset/packages/envied/src/envied/core/console.py @@ -0,0 +1,351 @@ +import logging +from datetime import datetime +from types import ModuleType +from typing import IO, Callable, Iterable, List, Literal, Mapping, Optional, Union + +from rich._log_render import FormatTimeCallable, LogRender +from rich.console import Console, ConsoleRenderable, HighlighterType, RenderableType +from rich.emoji import EmojiVariant +from rich.highlighter import Highlighter, ReprHighlighter +from rich.live import Live +from rich.logging import RichHandler +from rich.padding import Padding, PaddingDimensions +from rich.status import Status +from rich.style import StyleType +from rich.table import Table +from rich.text import Text, TextType +from rich.theme import Theme + +from envied.core.config import config + + +class ComfyLogRenderer(LogRender): + def __call__( + self, + console: "Console", + renderables: Iterable["ConsoleRenderable"], + log_time: Optional[datetime] = None, + time_format: Optional[Union[str, FormatTimeCallable]] = None, + level: TextType = "", + path: Optional[str] = None, + line_no: Optional[int] = None, + link_path: Optional[str] = None, + ) -> "Table": + from rich.containers import Renderables + + output = Table.grid(padding=(0, 5), pad_edge=True) + output.expand = True + if self.show_time: + output.add_column(style="log.time") + if self.show_level: + output.add_column(style="log.level", width=self.level_width) + output.add_column(ratio=1, style="log.message", overflow="fold") + if self.show_path and path: + output.add_column(style="log.path") + row: List["RenderableType"] = [] + if self.show_time: + log_time = log_time or console.get_datetime() + time_format = time_format or self.time_format + if callable(time_format): + log_time_display = time_format(log_time) + else: + log_time_display = Text(log_time.strftime(time_format)) + if log_time_display == self._last_time and self.omit_repeated_times: + row.append(Text(" " * len(log_time_display))) + else: + row.append(log_time_display) + self._last_time = log_time_display + if self.show_level: + row.append(level) + + row.append(Renderables(renderables)) + if self.show_path and path: + path_text = Text() + path_text.append(path, style=f"link file://{link_path}" if link_path else "") + if line_no: + path_text.append(":") + path_text.append( + f"{line_no}", + style=f"link file://{link_path}#{line_no}" if link_path else "", + ) + row.append(path_text) + + output.add_row(*row) + return output + + +class ComfyRichHandler(RichHandler): + def __init__( + self, + level: Union[int, str] = logging.NOTSET, + console: Optional[Console] = None, + *, + show_time: bool = True, + omit_repeated_times: bool = True, + show_level: bool = True, + show_path: bool = True, + enable_link_path: bool = True, + highlighter: Optional[Highlighter] = None, + markup: bool = False, + rich_tracebacks: bool = False, + tracebacks_width: Optional[int] = None, + tracebacks_extra_lines: int = 3, + tracebacks_theme: Optional[str] = None, + tracebacks_word_wrap: bool = True, + tracebacks_show_locals: bool = False, + tracebacks_suppress: Iterable[Union[str, ModuleType]] = (), + locals_max_length: int = 10, + locals_max_string: int = 80, + log_time_format: Union[str, FormatTimeCallable] = "[%x %X]", + keywords: Optional[List[str]] = None, + log_renderer: Optional[LogRender] = None, + ) -> None: + super().__init__( + level=level, + console=console, + show_time=show_time, + omit_repeated_times=omit_repeated_times, + show_level=show_level, + show_path=show_path, + enable_link_path=enable_link_path, + highlighter=highlighter, + markup=markup, + rich_tracebacks=rich_tracebacks, + tracebacks_width=tracebacks_width, + tracebacks_extra_lines=tracebacks_extra_lines, + tracebacks_theme=tracebacks_theme, + tracebacks_word_wrap=tracebacks_word_wrap, + tracebacks_show_locals=tracebacks_show_locals, + tracebacks_suppress=tracebacks_suppress, + locals_max_length=locals_max_length, + locals_max_string=locals_max_string, + log_time_format=log_time_format, + keywords=keywords, + ) + if log_renderer: + self._log_render = log_renderer + + +class ComfyConsole(Console): + """A comfy high level console interface. + + Args: + color_system (str, optional): The color system supported by your terminal, + either ``"standard"``, ``"256"`` or ``"truecolor"``. Leave as ``"auto"`` to autodetect. + force_terminal (Optional[bool], optional): Enable/disable terminal control codes, or None to auto-detect + terminal. Defaults to None. + force_jupyter (Optional[bool], optional): Enable/disable Jupyter rendering, or None to auto-detect Jupyter. + Defaults to None. + force_interactive (Optional[bool], optional): Enable/disable interactive mode, or None to auto-detect. + Defaults to None. + soft_wrap (Optional[bool], optional): Set soft wrap default on print method. Defaults to False. + theme (Theme, optional): An optional style theme object, or ``None`` for default theme. + stderr (bool, optional): Use stderr rather than stdout if ``file`` is not specified. Defaults to False. + file (IO, optional): A file object where the console should write to. Defaults to stdout. + quiet (bool, Optional): Boolean to suppress all output. Defaults to False. + width (int, optional): The width of the terminal. Leave as default to auto-detect width. + height (int, optional): The height of the terminal. Leave as default to auto-detect height. + style (StyleType, optional): Style to apply to all output, or None for no style. Defaults to None. + no_color (Optional[bool], optional): Enabled no color mode, or None to auto-detect. Defaults to None. + tab_size (int, optional): Number of spaces used to replace a tab character. Defaults to 8. + record (bool, optional): Boolean to enable recording of terminal output, + required to call :meth:`export_html`, :meth:`export_svg`, and :meth:`export_text`. Defaults to False. + markup (bool, optional): Boolean to enable :ref:`console_markup`. Defaults to True. + emoji (bool, optional): Enable emoji code. Defaults to True. + emoji_variant (str, optional): Optional emoji variant, either "text" or "emoji". Defaults to None. + highlight (bool, optional): Enable automatic highlighting. Defaults to True. + log_time (bool, optional): Boolean to enable logging of time by :meth:`log` methods. Defaults to True. + log_path (bool, optional): Boolean to enable the logging of the caller by :meth:`log`. Defaults to True. + log_time_format (Union[str, TimeFormatterCallable], optional): If ``log_time`` is enabled, either string for + strftime or callable that formats the time. Defaults to "[%X] ". + highlighter (HighlighterType, optional): Default highlighter. + legacy_windows (bool, optional): Enable legacy Windows mode, or ``None`` to auto-detect. Defaults to ``None``. + safe_box (bool, optional): Restrict box options that don't render on legacy Windows. + get_datetime (Callable[[], datetime], optional): Callable that gets the current time as a datetime.datetime + object (used by Console.log), or None for datetime.now. + get_time (Callable[[], time], optional): Callable that gets the current time in seconds, default uses + time.monotonic. + """ + + def __init__( + self, + *, + color_system: Optional[Literal["auto", "standard", "256", "truecolor", "windows"]] = "auto", + force_terminal: Optional[bool] = None, + force_jupyter: Optional[bool] = None, + force_interactive: Optional[bool] = None, + soft_wrap: bool = False, + theme: Optional[Theme] = None, + stderr: bool = False, + file: Optional[IO[str]] = None, + quiet: bool = False, + width: Optional[int] = None, + height: Optional[int] = None, + style: Optional[StyleType] = None, + no_color: Optional[bool] = None, + tab_size: int = 8, + record: bool = False, + markup: bool = True, + emoji: bool = True, + emoji_variant: Optional[EmojiVariant] = None, + highlight: bool = True, + log_time: bool = True, + log_path: bool = True, + log_time_format: Union[str, FormatTimeCallable] = "[%X]", + highlighter: Optional["HighlighterType"] = ReprHighlighter(), + legacy_windows: Optional[bool] = None, + safe_box: bool = True, + get_datetime: Optional[Callable[[], datetime]] = None, + get_time: Optional[Callable[[], float]] = None, + _environ: Optional[Mapping[str, str]] = None, + log_renderer: Optional[LogRender] = None, + ): + super().__init__( + color_system=color_system, + force_terminal=force_terminal, + force_jupyter=force_jupyter, + force_interactive=force_interactive, + soft_wrap=soft_wrap, + theme=theme, + stderr=stderr, + file=file, + quiet=quiet, + width=width, + height=height, + style=style, + no_color=no_color, + tab_size=tab_size, + record=record, + markup=markup, + emoji=emoji, + emoji_variant=emoji_variant, + highlight=highlight, + log_time=log_time, + log_path=log_path, + log_time_format=log_time_format, + highlighter=highlighter, + legacy_windows=legacy_windows, + safe_box=safe_box, + get_datetime=get_datetime, + get_time=get_time, + _environ=_environ, + ) + if log_renderer: + self._log_render = log_renderer + + def status( + self, + status: RenderableType, + *, + spinner: str = "dots", + spinner_style: str = "status.spinner", + speed: float = 1.0, + refresh_per_second: float = 12.5, + pad: PaddingDimensions = (0, 5), + ) -> Union[Live, Status]: + """Display a comfy status and spinner. + + Args: + status (RenderableType): A status renderable (str or Text typically). + spinner (str, optional): Name of spinner animation (see python -m rich.spinner). Defaults to "dots". + spinner_style (StyleType, optional): Style of spinner. Defaults to "status.spinner". + speed (float, optional): Speed factor for spinner animation. Defaults to 1.0. + refresh_per_second (float, optional): Number of refreshes per second. Defaults to 12.5. + pad (Union[int, Tuple[int]]): Padding for top, right, bottom, and left borders. + May be specified with 1, 2, or 4 integers (CSS style). + + Returns: + Status: A Status object that may be used as a context manager. + """ + status_renderable = super().status( + status=status, + spinner=spinner, + spinner_style=spinner_style, + speed=speed, + refresh_per_second=refresh_per_second, + ) + + if pad: + top, right, bottom, left = Padding.unpack(pad) + + renderable_width = len(status_renderable.status) + spinner_width = len(status_renderable.renderable.text) + status_width = spinner_width + renderable_width + + available_width = self.width - status_width + if available_width > right: + # fill up the available width with padding to apply bg color + right = available_width - right + + padding = Padding(status_renderable, (top, right, bottom, left)) + + return Live(padding, console=self, transient=True) + + return status_renderable + + +catppuccin_mocha = { + # Colors based on "CatppuccinMocha" from Gogh themes + "bg": "rgb(30,30,46)", + "text": "rgb(205,214,244)", + "text2": "rgb(162,169,193)", # slightly darker + "black": "rgb(69,71,90)", + "bright_black": "rgb(88,91,112)", + "red": "rgb(243,139,168)", + "green": "rgb(166,227,161)", + "yellow": "rgb(249,226,175)", + "blue": "rgb(137,180,250)", + "pink": "rgb(245,194,231)", + "cyan": "rgb(148,226,213)", + "gray": "rgb(166,173,200)", + "bright_gray": "rgb(186,194,222)", + "dark_gray": "rgb(54,54,84)", +} + +primary_scheme = catppuccin_mocha +primary_scheme["none"] = primary_scheme["text"] +primary_scheme["grey23"] = primary_scheme["black"] +primary_scheme["magenta"] = primary_scheme["pink"] +primary_scheme["bright_red"] = primary_scheme["red"] +primary_scheme["bright_green"] = primary_scheme["green"] +primary_scheme["bright_yellow"] = primary_scheme["yellow"] +primary_scheme["bright_blue"] = primary_scheme["blue"] +primary_scheme["bright_magenta"] = primary_scheme["pink"] +primary_scheme["bright_cyan"] = primary_scheme["cyan"] +if config.set_terminal_bg: + primary_scheme["none"] += f" on {primary_scheme['bg']}" + +custom_colors = {"ascii.art": primary_scheme["pink"]} +if config.set_terminal_bg: + custom_colors["ascii.art"] += f" on {primary_scheme['bg']}" + + +console = ComfyConsole( + log_time=False, + log_path=False, + width=80, + theme=Theme( + { + "bar.back": primary_scheme["dark_gray"], + "bar.complete": primary_scheme["pink"], + "bar.finished": primary_scheme["green"], + "bar.pulse": primary_scheme["bright_black"], + "black": primary_scheme["black"], + "inspect.async_def": f"italic {primary_scheme['cyan']}", + "progress.data.speed": "dark_orange", + "repr.number": f"bold not italic {primary_scheme['cyan']}", + "repr.number_complex": f"bold not italic {primary_scheme['cyan']}", + "rule.line": primary_scheme["dark_gray"], + "rule.text": primary_scheme["pink"], + "tree.line": primary_scheme["dark_gray"], + "status.spinner": primary_scheme["pink"], + "progress.spinner": primary_scheme["pink"], + **primary_scheme, + **custom_colors, + } + ), + log_renderer=ComfyLogRenderer(show_time=False, show_path=False), +) + + +__all__ = ("ComfyLogRenderer", "ComfyRichHandler", "ComfyConsole", "console") diff --git a/reset/packages/envied/src/envied/core/constants.py b/reset/packages/envied/src/envied/core/constants.py new file mode 100644 index 0000000..65c6681 --- /dev/null +++ b/reset/packages/envied/src/envied/core/constants.py @@ -0,0 +1,32 @@ +from threading import Event +from typing import TypeVar, Union + +DOWNLOAD_CANCELLED = Event() +DOWNLOAD_LICENCE_ONLY = Event() + +DRM_SORT_MAP = ["ClearKey", "Widevine"] +LANGUAGE_MAX_DISTANCE = 5 # this is max to be considered "same", e.g., en, en-US, en-AU +LANGUAGE_EXACT_DISTANCE = 0 # exact match only, no variants +VIDEO_CODEC_MAP = {"AVC": "H.264", "HEVC": "H.265"} +DYNAMIC_RANGE_MAP = { + "HDR10": "HDR", + "HDR10+": "HDR10P", + "Dolby Vision": "DV", + "HDR10 / HDR10+": "HDR10P", + "HDR10 / HDR10": "HDR", +} +AUDIO_CODEC_MAP = {"E-AC-3": "DDP", "AC-3": "DD"} + +context_settings = dict( + help_option_names=["-?", "-h", "--help"], # default only has --help + max_content_width=116, # max PEP8 line-width, -4 to adjust for initial indent +) + +# For use in signatures of functions which take one specific type of track at a time +# (it can't be a list that contains e.g. both Video and Audio objects) +TrackT = TypeVar("TrackT", bound="Track") # noqa: F821 + +# For general use in lists that can contain mixed types of tracks. +# list[Track] won't work because list is invariant. +# TODO: Add Chapter? +AnyTrack = Union["Video", "Audio", "Subtitle"] # noqa: F821 diff --git a/reset/packages/envied/src/envied/core/credential.py b/reset/packages/envied/src/envied/core/credential.py new file mode 100644 index 0000000..b923d70 --- /dev/null +++ b/reset/packages/envied/src/envied/core/credential.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import base64 +import hashlib +import re +from pathlib import Path +from typing import Optional, Union + + +class Credential: + """Username (or Email) and Password Credential.""" + + def __init__(self, username: str, password: str, extra: Optional[str] = None): + self.username = username + self.password = password + self.extra = extra + self.sha1 = hashlib.sha1(self.dumps().encode()).hexdigest() + + def __bool__(self) -> bool: + return bool(self.username) and bool(self.password) + + def __str__(self) -> str: + return self.dumps() + + def __repr__(self) -> str: + return "{name}({items})".format( + name=self.__class__.__name__, items=", ".join([f"{k}={repr(v)}" for k, v in self.__dict__.items()]) + ) + + def dumps(self) -> str: + """Return credential data as a string.""" + return f"{self.username}:{self.password}" + (f":{self.extra}" if self.extra else "") + + def dump(self, path: Union[Path, str]) -> int: + """Write credential data to a file.""" + if isinstance(path, str): + path = Path(path) + return path.write_text(self.dumps(), encoding="utf8") + + def as_base64(self, with_extra: bool = False, encode_password: bool = False, encode_extra: bool = False) -> str: + """ + Dump Credential as a Base64-encoded string in Basic Authorization style. + encode_password and encode_extra will also Base64-encode the password and extra respectively. + """ + value = f"{self.username}:" + if encode_password: + value += base64.b64encode(self.password.encode()).decode() + else: + value += self.password + if with_extra and self.extra: + if encode_extra: + value += f":{base64.b64encode(self.extra.encode()).decode()}" + else: + value += f":{self.extra}" + return base64.b64encode(value.encode()).decode() + + @classmethod + def loads(cls, text: str) -> Credential: + """ + Load credential from a text string. + + Format: {username}:{password} + Rules: + Only one Credential must be in this text contents. + All whitespace before and after all text will be removed. + Any whitespace between text will be kept and used. + The credential can be spanned across one or multiple lines as long as it + abides with all the above rules and the format. + + Example that follows the format and rules: + `\tJohnd\noe@gm\n\rail.com\n:Pass1\n23\n\r \t \t` + >>>Credential(username='Johndoe@gmail.com', password='Pass123') + """ + text = "".join([x.strip() for x in text.splitlines(keepends=False)]).strip() + credential = re.fullmatch(r"^([^:]+?):([^:]+?)(?::(.+))?$", text) + if credential: + return cls(*credential.groups()) + raise ValueError("No credentials found in text string. Expecting the format `username:password`") + + @classmethod + def load(cls, path: Path) -> Credential: + """ + Load Credential from a file path. + Use Credential.loads() for loading from text content and seeing the rules and + format expected to be found in the URIs contents. + """ + return cls.loads(path.read_text("utf8")) diff --git a/reset/packages/envied/src/envied/core/downloaders/__init__.py b/reset/packages/envied/src/envied/core/downloaders/__init__.py new file mode 100644 index 0000000..66d3b42 --- /dev/null +++ b/reset/packages/envied/src/envied/core/downloaders/__init__.py @@ -0,0 +1,3 @@ +from .requests import requests + +__all__ = ("requests",) diff --git a/reset/packages/envied/src/envied/core/downloaders/aria2c.py b/reset/packages/envied/src/envied/core/downloaders/aria2c.py new file mode 100644 index 0000000..b319d42 --- /dev/null +++ b/reset/packages/envied/src/envied/core/downloaders/aria2c.py @@ -0,0 +1,543 @@ +import logging +import os +import subprocess +import textwrap +import threading +import time +from functools import partial +from http.cookiejar import CookieJar +from pathlib import Path +from typing import Any, Callable, Generator, MutableMapping, Optional, Union +from urllib.parse import urlparse + +import requests +from Crypto.Random import get_random_bytes +from requests import Session +from requests.cookies import cookiejar_from_dict, get_cookie_header +from rich import filesize +from rich.text import Text + +from envied.core import binaries +from envied.core.config import config +from envied.core.console import console +from envied.core.constants import DOWNLOAD_CANCELLED +from envied.core.utilities import get_debug_logger, get_extension, get_free_port + + +def rpc(caller: Callable, secret: str, method: str, params: Optional[list[Any]] = None) -> Any: + """Make a call to Aria2's JSON-RPC API.""" + try: + rpc_res = caller( + json={ + "jsonrpc": "2.0", + "id": get_random_bytes(16).hex(), + "method": method, + "params": [f"token:{secret}", *(params or [])], + } + ).json() + if rpc_res.get("code"): + # wrap to console width - padding - '[Aria2c]: ' + error_pretty = "\n ".join( + textwrap.wrap( + f"RPC Error: {rpc_res['message']} ({rpc_res['code']})".strip(), + width=console.width - 20, + initial_indent="", + ) + ) + console.log(Text.from_ansi("\n[Aria2c]: " + error_pretty)) + return rpc_res["result"] + except requests.exceptions.ConnectionError: + # absorb, process likely ended as it was calling RPC + return + + +class _Aria2Manager: + """Singleton manager to run one aria2c process and enqueue downloads via RPC.""" + + def __init__(self) -> None: + self._logger = logging.getLogger(__name__) + self._proc: Optional[subprocess.Popen] = None + self._rpc_port: Optional[int] = None + self._rpc_secret: Optional[str] = None + self._rpc_uri: Optional[str] = None + self._session: Session = Session() + self._max_workers: Optional[int] = None + self._max_concurrent_downloads: int = 0 + self._max_connection_per_server: int = 1 + self._split_default: int = 5 + self._file_allocation: str = "prealloc" + self._proxy: Optional[str] = None + self._lock: threading.Lock = threading.Lock() + + def _wait_for_rpc_ready(self, timeout_s: float = 8.0, interval_s: float = 0.1) -> None: + assert self._proc is not None + assert self._rpc_uri is not None + assert self._rpc_secret is not None + + deadline = time.monotonic() + timeout_s + + payload = { + "jsonrpc": "2.0", + "id": get_random_bytes(16).hex(), + "method": "aria2.getVersion", + "params": [f"token:{self._rpc_secret}"], + } + + while time.monotonic() < deadline: + if self._proc.poll() is not None: + raise RuntimeError( + f"aria2c exited before RPC became ready (exit code {self._proc.returncode})" + ) + try: + res = self._session.post(self._rpc_uri, json=payload, timeout=0.25) + data = res.json() + if isinstance(data, dict) and data.get("result") is not None: + return + except (requests.exceptions.RequestException, ValueError): + # Not ready yet (connection refused / bad response / etc.) + pass + time.sleep(interval_s) + + # Timed out: ensure we don't leave a zombie/stray aria2c process behind. + try: + self._proc.terminate() + self._proc.wait(timeout=2) + except Exception: + try: + self._proc.kill() + self._proc.wait(timeout=2) + except Exception: + pass + raise TimeoutError(f"aria2c RPC did not become ready within {timeout_s:.1f}s") + + def _build_args(self) -> list[str]: + args = [ + "--continue=true", + f"--max-concurrent-downloads={self._max_concurrent_downloads}", + f"--max-connection-per-server={self._max_connection_per_server}", + f"--split={self._split_default}", + "--max-file-not-found=5", + "--max-tries=5", + "--retry-wait=2", + "--allow-overwrite=true", + "--auto-file-renaming=false", + "--console-log-level=warn", + "--download-result=default", + f"--file-allocation={self._file_allocation}", + "--summary-interval=0", + "--enable-rpc=true", + f"--rpc-listen-port={self._rpc_port}", + f"--rpc-secret={self._rpc_secret}", + ] + if self._proxy: + args.extend(["--all-proxy", self._proxy]) + return args + + def ensure_started( + self, + proxy: Optional[str], + max_workers: Optional[int], + ) -> None: + with self._lock: + if not binaries.Aria2: + debug_logger = get_debug_logger() + if debug_logger: + debug_logger.log( + level="ERROR", + operation="downloader_aria2c_binary_missing", + message="Aria2c executable not found in PATH or local binaries directory", + context={"searched_names": ["aria2c", "aria2"]}, + ) + raise EnvironmentError("Aria2c executable not found...") + + effective_proxy = proxy or None + + if not max_workers: + effective_max_workers = min(32, (os.cpu_count() or 1) + 4) + elif not isinstance(max_workers, int): + raise TypeError(f"Expected max_workers to be {int}, not {type(max_workers)}") + else: + effective_max_workers = max_workers + + if self._proc and self._proc.poll() is None: + if effective_proxy != self._proxy or effective_max_workers != self._max_workers: + self._logger.warning( + "aria2c process is already running; requested proxy=%r, max_workers=%r, " + "but running process will continue with proxy=%r, max_workers=%r", + effective_proxy, + effective_max_workers, + self._proxy, + self._max_workers, + ) + return + + self._rpc_port = get_free_port() + self._rpc_secret = get_random_bytes(16).hex() + self._rpc_uri = f"http://127.0.0.1:{self._rpc_port}/jsonrpc" + + self._max_workers = effective_max_workers + self._max_concurrent_downloads = int( + config.aria2c.get("max_concurrent_downloads", effective_max_workers) + ) + self._max_connection_per_server = int(config.aria2c.get("max_connection_per_server", 1)) + self._split_default = int(config.aria2c.get("split", 5)) + self._file_allocation = config.aria2c.get("file_allocation", "prealloc") + self._proxy = effective_proxy + + args = self._build_args() + self._proc = subprocess.Popen( + [binaries.Aria2, *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + self._wait_for_rpc_ready() + + @property + def rpc_uri(self) -> str: + assert self._rpc_uri + return self._rpc_uri + + @property + def rpc_secret(self) -> str: + assert self._rpc_secret + return self._rpc_secret + + @property + def session(self) -> Session: + return self._session + + def add_uris(self, uris: list[str], options: dict[str, Any]) -> str: + """Add a single download with multiple URIs via RPC.""" + gid = rpc( + caller=partial(self._session.post, url=self.rpc_uri), + secret=self.rpc_secret, + method="aria2.addUri", + params=[uris, options], + ) + return gid or "" + + def get_global_stat(self) -> dict[str, Any]: + return rpc( + caller=partial(self.session.post, url=self.rpc_uri), + secret=self.rpc_secret, + method="aria2.getGlobalStat", + ) or {} + + def tell_status(self, gid: str) -> Optional[dict[str, Any]]: + return rpc( + caller=partial(self.session.post, url=self.rpc_uri), + secret=self.rpc_secret, + method="aria2.tellStatus", + params=[gid, ["status", "errorCode", "errorMessage", "files", "completedLength", "totalLength"]], + ) + + def remove(self, gid: str) -> None: + rpc( + caller=partial(self.session.post, url=self.rpc_uri), + secret=self.rpc_secret, + method="aria2.forceRemove", + params=[gid], + ) + + +_manager = _Aria2Manager() + + +def download( + urls: Union[str, list[str], dict[str, Any], list[dict[str, Any]]], + output_dir: Path, + filename: str, + headers: Optional[MutableMapping[str, Union[str, bytes]]] = None, + cookies: Optional[Union[MutableMapping[str, str], CookieJar]] = None, + proxy: Optional[str] = None, + max_workers: Optional[int] = None, +) -> Generator[dict[str, Any], None, None]: + """Enqueue downloads to the singleton aria2c instance via stdin and track per-call progress via RPC.""" + debug_logger = get_debug_logger() + + if not urls: + raise ValueError("urls must be provided and not empty") + elif not isinstance(urls, (str, dict, list)): + raise TypeError(f"Expected urls to be {str} or {dict} or a list of one of them, not {type(urls)}") + + if not output_dir: + raise ValueError("output_dir must be provided") + elif not isinstance(output_dir, Path): + raise TypeError(f"Expected output_dir to be {Path}, not {type(output_dir)}") + + if not filename: + raise ValueError("filename must be provided") + elif not isinstance(filename, str): + raise TypeError(f"Expected filename to be {str}, not {type(filename)}") + + if not isinstance(headers, (MutableMapping, type(None))): + raise TypeError(f"Expected headers to be {MutableMapping}, not {type(headers)}") + + if not isinstance(cookies, (MutableMapping, CookieJar, type(None))): + raise TypeError(f"Expected cookies to be {MutableMapping} or {CookieJar}, not {type(cookies)}") + + if not isinstance(proxy, (str, type(None))): + raise TypeError(f"Expected proxy to be {str}, not {type(proxy)}") + + if not max_workers: + max_workers = min(32, (os.cpu_count() or 1) + 4) + elif not isinstance(max_workers, int): + raise TypeError(f"Expected max_workers to be {int}, not {type(max_workers)}") + + if not isinstance(urls, list): + urls = [urls] + + if cookies and not isinstance(cookies, CookieJar): + cookies = cookiejar_from_dict(cookies) + + _manager.ensure_started(proxy=proxy, max_workers=max_workers) + + if debug_logger: + first_url = urls[0] if isinstance(urls[0], str) else urls[0].get("url", "") + url_display = first_url[:200] + "..." if len(first_url) > 200 else first_url + debug_logger.log( + level="DEBUG", + operation="downloader_aria2c_start", + message="Starting Aria2c download", + context={ + "binary_path": str(binaries.Aria2), + "url_count": len(urls), + "first_url": url_display, + "output_dir": str(output_dir), + "filename": filename, + "has_proxy": bool(proxy), + }, + ) + + # Build options for each URI and add via RPC + gids: list[str] = [] + + for i, url in enumerate(urls): + if isinstance(url, str): + url_data = {"url": url} + else: + url_data: dict[str, Any] = url + + url_filename = filename.format(i=i, ext=get_extension(url_data["url"])) + + opts: dict[str, Any] = { + "dir": str(output_dir), + "out": url_filename, + "split": str(1 if len(urls) > 1 else int(config.aria2c.get("split", 5))), + } + + # Cookies as header + if cookies: + mock_request = requests.Request(url=url_data["url"]) + cookie_header = get_cookie_header(cookies, mock_request) + if cookie_header: + opts.setdefault("header", []).append(f"Cookie: {cookie_header}") + + # Global headers + for header, value in (headers or {}).items(): + if header.lower() == "cookie": + raise ValueError("You cannot set Cookies as a header manually, please use the `cookies` param.") + if header.lower() == "accept-encoding": + continue + if header.lower() == "referer": + opts["referer"] = str(value) + continue + if header.lower() == "user-agent": + opts["user-agent"] = str(value) + continue + opts.setdefault("header", []).append(f"{header}: {value}") + + # Per-url extra args + for key, value in url_data.items(): + if key == "url": + continue + if key == "headers": + for header_name, header_value in value.items(): + opts.setdefault("header", []).append(f"{header_name}: {header_value}") + else: + opts[key] = str(value) + + # Add via RPC + gid = _manager.add_uris([url_data["url"]], opts) + if gid: + gids.append(gid) + + yield dict(total=len(gids)) + + completed: set[str] = set() + + try: + while len(completed) < len(gids): + if DOWNLOAD_CANCELLED.is_set(): + # Remove tracked downloads on cancel + for gid in gids: + if gid not in completed: + _manager.remove(gid) + yield dict(downloaded="[yellow]CANCELLED") + raise KeyboardInterrupt() + + stats = _manager.get_global_stat() + dl_speed = int(stats.get("downloadSpeed", -1)) + + # Aggregate progress across all GIDs for this call + total_completed = 0 + total_size = 0 + + # Check each tracked GID + for gid in gids: + if gid in completed: + continue + + status = _manager.tell_status(gid) + if not status: + continue + + completed_length = int(status.get("completedLength", 0)) + total_length = int(status.get("totalLength", 0)) + total_completed += completed_length + total_size += total_length + + state = status.get("status") + if state in ("complete", "error"): + completed.add(gid) + yield dict(completed=len(completed)) + + if state == "error": + used_uri = None + try: + used_uri = next( + uri["uri"] + for file in status.get("files", []) + for uri in file.get("uris", []) + if uri.get("status") == "used" + ) + except Exception: + used_uri = "unknown" + error = f"Download Error (#{gid}): {status.get('errorMessage')} ({status.get('errorCode')}), {used_uri}" + error_pretty = "\n ".join(textwrap.wrap(error, width=console.width - 20, initial_indent="")) + console.log(Text.from_ansi("\n[Aria2c]: " + error_pretty)) + if debug_logger: + debug_logger.log( + level="ERROR", + operation="downloader_aria2c_download_error", + message=f"Aria2c download failed: {status.get('errorMessage')}", + context={ + "gid": gid, + "error_code": status.get("errorCode"), + "error_message": status.get("errorMessage"), + "used_uri": used_uri[:200] + "..." if used_uri and len(used_uri) > 200 else used_uri, + "completed_length": status.get("completedLength"), + "total_length": status.get("totalLength"), + }, + ) + raise ValueError(error) + + # Yield aggregate progress for this call's downloads + progress_data = {"advance": 0} + + if len(gids) > 1: + # Multi-file mode (e.g., HLS): Return the count of completed segments + progress_data["completed"] = len(completed) + progress_data["total"] = len(gids) + else: + # Single-file mode: Return the total bytes downloaded + progress_data["completed"] = total_completed + if total_size > 0: + progress_data["total"] = total_size + else: + progress_data["total"] = None + + if dl_speed != -1: + progress_data["downloaded"] = f"{filesize.decimal(dl_speed)}/s" + + yield progress_data + + time.sleep(1) + except KeyboardInterrupt: + DOWNLOAD_CANCELLED.set() + raise + except Exception as e: + DOWNLOAD_CANCELLED.set() + yield dict(downloaded="[red]FAILED") + if debug_logger and not isinstance(e, ValueError): + debug_logger.log( + level="ERROR", + operation="downloader_aria2c_exception", + message=f"Unexpected error during Aria2c download: {e}", + error=e, + context={ + "url_count": len(urls), + "output_dir": str(output_dir), + }, + ) + raise + + +def aria2c( + urls: Union[str, list[str], dict[str, Any], list[dict[str, Any]]], + output_dir: Path, + filename: str, + headers: Optional[MutableMapping[str, Union[str, bytes]]] = None, + cookies: Optional[Union[MutableMapping[str, str], CookieJar]] = None, + proxy: Optional[str] = None, + max_workers: Optional[int] = None, +) -> Generator[dict[str, Any], None, None]: + """ + Download files using Aria2(c). + https://aria2.github.io + + Yields the following download status updates while chunks are downloading: + + - {total: 100} (100% download total) + - {completed: 1} (1% download progress out of 100%) + - {downloaded: "10.1 MB/s"} (currently downloading at a rate of 10.1 MB/s) + + The data is in the same format accepted by rich's progress.update() function. + + Parameters: + urls: Web URL(s) to file(s) to download. You can use a dictionary with the key + "url" for the URI, and other keys for extra arguments to use per-URL. + output_dir: The folder to save the file into. If the save path's directory does + not exist then it will be made automatically. + filename: The filename or filename template to use for each file. The variables + you can use are `i` for the URL index and `ext` for the URL extension. + headers: A mapping of HTTP Header Key/Values to use for all downloads. + cookies: A mapping of Cookie Key/Values or a Cookie Jar to use for all downloads. + proxy: An optional proxy URI to route connections through for all downloads. + max_workers: The maximum amount of threads to use for downloads. Defaults to + min(32,(cpu_count+4)). Use for the --max-concurrent-downloads option. + """ + if proxy and not proxy.lower().startswith("http://"): + # Only HTTP proxies are supported by aria2(c) + proxy = urlparse(proxy) + + port = get_free_port() + username, password = get_random_bytes(8).hex(), get_random_bytes(8).hex() + local_proxy = f"http://{username}:{password}@localhost:{port}" + + scheme = {"https": "http+ssl", "socks5h": "socks"}.get(proxy.scheme, proxy.scheme) + + remote_server = f"{scheme}://{proxy.hostname}" + if proxy.port: + remote_server += f":{proxy.port}" + if proxy.username or proxy.password: + remote_server += "#" + if proxy.username: + remote_server += proxy.username + if proxy.password: + remote_server += f":{proxy.password}" + + p = subprocess.Popen( + ["pproxy", "-l", f"http://:{port}#{username}:{password}", "-r", remote_server], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + try: + yield from download(urls, output_dir, filename, headers, cookies, local_proxy, max_workers) + finally: + p.kill() + p.wait() + return + yield from download(urls, output_dir, filename, headers, cookies, proxy, max_workers) + + +__all__ = ("aria2c",) diff --git a/reset/packages/envied/src/envied/core/downloaders/curl_impersonate.py b/reset/packages/envied/src/envied/core/downloaders/curl_impersonate.py new file mode 100644 index 0000000..4218b6f --- /dev/null +++ b/reset/packages/envied/src/envied/core/downloaders/curl_impersonate.py @@ -0,0 +1,308 @@ +import math +import time +from concurrent import futures +from concurrent.futures.thread import ThreadPoolExecutor +from http.cookiejar import CookieJar +from pathlib import Path +from typing import Any, Generator, MutableMapping, Optional, Union + +from curl_cffi.requests import Session +from rich import filesize + +from envied.core.config import config +from envied.core.constants import DOWNLOAD_CANCELLED +from envied.core.utilities import get_debug_logger, get_extension + +MAX_ATTEMPTS = 5 +RETRY_WAIT = 2 +CHUNK_SIZE = 1024 +PROGRESS_WINDOW = 5 +BROWSER = config.curl_impersonate.get("browser", "chrome124") + + +def download(url: str, save_path: Path, session: Session, **kwargs: Any) -> Generator[dict[str, Any], None, None]: + """ + Download files using Curl Impersonate. + https://github.com/lwthiker/curl-impersonate + + Yields the following download status updates while chunks are downloading: + + - {total: 123} (there are 123 chunks to download) + - {total: None} (there are an unknown number of chunks to download) + - {advance: 1} (one chunk was downloaded) + - {downloaded: "10.1 MB/s"} (currently downloading at a rate of 10.1 MB/s) + - {file_downloaded: Path(...), written: 1024} (download finished, has the save path and size) + + The data is in the same format accepted by rich's progress.update() function. The + `downloaded` key is custom and is not natively accepted by all rich progress bars. + + Parameters: + url: Web URL of a file to download. + save_path: The path to save the file to. If the save path's directory does not + exist then it will be made automatically. + session: The Requests or Curl-Impersonate Session to make HTTP requests with. + Useful to set Header, Cookie, and Proxy data. Connections are saved and + re-used with the session so long as the server keeps the connection alive. + kwargs: Any extra keyword arguments to pass to the session.get() call. Use this + for one-time request changes like a header, cookie, or proxy. For example, + to request Byte-ranges use e.g., `headers={"Range": "bytes=0-128"}`. + """ + save_dir = save_path.parent + control_file = save_path.with_name(f"{save_path.name}.!dev") + + save_dir.mkdir(parents=True, exist_ok=True) + + if control_file.exists(): + # consider the file corrupt if the control file exists + save_path.unlink(missing_ok=True) + control_file.unlink() + elif save_path.exists(): + # if it exists, and no control file, then it should be safe + yield dict(file_downloaded=save_path, written=save_path.stat().st_size) + + # TODO: Design a control file format so we know how much of the file is missing + control_file.write_bytes(b"") + + attempts = 1 + try: + while True: + written = 0 + download_sizes = [] + last_speed_refresh = time.time() + + try: + stream = session.get(url, stream=True, **kwargs) + stream.raise_for_status() + + try: + content_length = int(stream.headers.get("Content-Length", "0")) + + # Skip Content-Length validation for compressed responses since + # curl_impersonate automatically decompresses but Content-Length shows compressed size + if stream.headers.get("Content-Encoding", "").lower() in ["gzip", "deflate", "br"]: + content_length = 0 + except ValueError: + content_length = 0 + + if content_length > 0: + yield dict(total=math.ceil(content_length / CHUNK_SIZE)) + else: + # we have no data to calculate total chunks + yield dict(total=None) # indeterminate mode + + with open(save_path, "wb") as f: + for chunk in stream.iter_content(chunk_size=CHUNK_SIZE): + download_size = len(chunk) + f.write(chunk) + written += download_size + + yield dict(advance=1) + + now = time.time() + time_since = now - last_speed_refresh + + download_sizes.append(download_size) + if time_since > PROGRESS_WINDOW or download_size < CHUNK_SIZE: + data_size = sum(download_sizes) + download_speed = math.ceil(data_size / (time_since or 1)) + yield dict(downloaded=f"{filesize.decimal(download_speed)}/s") + last_speed_refresh = now + download_sizes.clear() + + if content_length and written < content_length: + raise IOError(f"Failed to read {content_length} bytes from the track URI.") + + yield dict(file_downloaded=save_path, written=written) + break + except Exception as e: + save_path.unlink(missing_ok=True) + if DOWNLOAD_CANCELLED.is_set() or attempts == MAX_ATTEMPTS: + raise e + time.sleep(RETRY_WAIT) + attempts += 1 + finally: + control_file.unlink() + + +def curl_impersonate( + urls: Union[str, list[str], dict[str, Any], list[dict[str, Any]]], + output_dir: Path, + filename: str, + headers: Optional[MutableMapping[str, Union[str, bytes]]] = None, + cookies: Optional[Union[MutableMapping[str, str], CookieJar]] = None, + proxy: Optional[str] = None, + max_workers: Optional[int] = None, +) -> Generator[dict[str, Any], None, None]: + """ + Download files using Curl Impersonate. + https://github.com/lwthiker/curl-impersonate + + Yields the following download status updates while chunks are downloading: + + - {total: 123} (there are 123 chunks to download) + - {total: None} (there are an unknown number of chunks to download) + - {advance: 1} (one chunk was downloaded) + - {downloaded: "10.1 MB/s"} (currently downloading at a rate of 10.1 MB/s) + - {file_downloaded: Path(...), written: 1024} (download finished, has the save path and size) + + The data is in the same format accepted by rich's progress.update() function. + However, The `downloaded`, `file_downloaded` and `written` keys are custom and not + natively accepted by rich progress bars. + + Parameters: + urls: Web URL(s) to file(s) to download. You can use a dictionary with the key + "url" for the URI, and other keys for extra arguments to use per-URL. + output_dir: The folder to save the file into. If the save path's directory does + not exist then it will be made automatically. + filename: The filename or filename template to use for each file. The variables + you can use are `i` for the URL index and `ext` for the URL extension. + headers: A mapping of HTTP Header Key/Values to use for all downloads. + cookies: A mapping of Cookie Key/Values or a Cookie Jar to use for all downloads. + proxy: An optional proxy URI to route connections through for all downloads. + max_workers: The maximum amount of threads to use for downloads. Defaults to + min(32,(cpu_count+4)). + """ + if not urls: + raise ValueError("urls must be provided and not empty") + elif not isinstance(urls, (str, dict, list)): + raise TypeError(f"Expected urls to be {str} or {dict} or a list of one of them, not {type(urls)}") + + if not output_dir: + raise ValueError("output_dir must be provided") + elif not isinstance(output_dir, Path): + raise TypeError(f"Expected output_dir to be {Path}, not {type(output_dir)}") + + if not filename: + raise ValueError("filename must be provided") + elif not isinstance(filename, str): + raise TypeError(f"Expected filename to be {str}, not {type(filename)}") + + if not isinstance(headers, (MutableMapping, type(None))): + raise TypeError(f"Expected headers to be {MutableMapping}, not {type(headers)}") + + if not isinstance(cookies, (MutableMapping, CookieJar, type(None))): + raise TypeError(f"Expected cookies to be {MutableMapping} or {CookieJar}, not {type(cookies)}") + + if not isinstance(proxy, (str, type(None))): + raise TypeError(f"Expected proxy to be {str}, not {type(proxy)}") + + if not isinstance(max_workers, (int, type(None))): + raise TypeError(f"Expected max_workers to be {int}, not {type(max_workers)}") + + debug_logger = get_debug_logger() + + if not isinstance(urls, list): + urls = [urls] + + urls = [ + dict(save_path=save_path, **url) if isinstance(url, dict) else dict(url=url, save_path=save_path) + for i, url in enumerate(urls) + for save_path in [ + output_dir / filename.format(i=i, ext=get_extension(url["url"] if isinstance(url, dict) else url)) + ] + ] + + session = Session(impersonate=BROWSER) + if headers: + headers = {k: v for k, v in headers.items() if k.lower() != "accept-encoding"} + session.headers.update(headers) + if cookies: + session.cookies.update(cookies) + if proxy: + session.proxies.update({"all": proxy}) + + if debug_logger: + first_url = urls[0].get("url", "") if urls else "" + url_display = first_url[:200] + "..." if len(first_url) > 200 else first_url + debug_logger.log( + level="DEBUG", + operation="downloader_curl_impersonate_start", + message="Starting curl_impersonate download", + context={ + "url_count": len(urls), + "first_url": url_display, + "output_dir": str(output_dir), + "filename": filename, + "max_workers": max_workers, + "browser": BROWSER, + "has_proxy": bool(proxy), + }, + ) + + yield dict(total=len(urls)) + + download_sizes = [] + last_speed_refresh = time.time() + + with ThreadPoolExecutor(max_workers=max_workers) as pool: + for i, future in enumerate( + futures.as_completed((pool.submit(download, session=session, **url) for url in urls)) + ): + file_path, download_size = None, None + try: + for status_update in future.result(): + if status_update.get("file_downloaded") and status_update.get("written"): + file_path = status_update["file_downloaded"] + download_size = status_update["written"] + elif len(urls) == 1: + # these are per-chunk updates, only useful if it's one big file + yield status_update + except KeyboardInterrupt: + DOWNLOAD_CANCELLED.set() # skip pending track downloads + yield dict(downloaded="[yellow]CANCELLING") + pool.shutdown(wait=True, cancel_futures=True) + yield dict(downloaded="[yellow]CANCELLED") + # tell dl that it was cancelled + # the pool is already shut down, so exiting loop is fine + raise + except Exception as e: + DOWNLOAD_CANCELLED.set() # skip pending track downloads + yield dict(downloaded="[red]FAILING") + pool.shutdown(wait=True, cancel_futures=True) + yield dict(downloaded="[red]FAILED") + if debug_logger: + debug_logger.log( + level="ERROR", + operation="downloader_curl_impersonate_failed", + message=f"curl_impersonate download failed: {e}", + error=e, + context={ + "url_count": len(urls), + "output_dir": str(output_dir), + "browser": BROWSER, + }, + ) + # tell dl that it failed + # the pool is already shut down, so exiting loop is fine + raise + else: + yield dict(file_downloaded=file_path) + yield dict(advance=1) + + now = time.time() + time_since = now - last_speed_refresh + + if download_size: # no size == skipped dl + download_sizes.append(download_size) + + if download_sizes and (time_since > PROGRESS_WINDOW or i == len(urls)): + data_size = sum(download_sizes) + download_speed = math.ceil(data_size / (time_since or 1)) + yield dict(downloaded=f"{filesize.decimal(download_speed)}/s") + last_speed_refresh = now + download_sizes.clear() + + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="downloader_curl_impersonate_complete", + message="curl_impersonate download completed successfully", + context={ + "url_count": len(urls), + "output_dir": str(output_dir), + "filename": filename, + }, + ) + + +__all__ = ("curl_impersonate",) diff --git a/reset/packages/envied/src/envied/core/downloaders/n_m3u8dl_re.py b/reset/packages/envied/src/envied/core/downloaders/n_m3u8dl_re.py new file mode 100644 index 0000000..738f9ce --- /dev/null +++ b/reset/packages/envied/src/envied/core/downloaders/n_m3u8dl_re.py @@ -0,0 +1,548 @@ +import os +import re +import subprocess +import warnings +from http.cookiejar import CookieJar +from pathlib import Path +from typing import Any, Generator, MutableMapping + +import requests +from requests.cookies import cookiejar_from_dict, get_cookie_header + +from envied.core import binaries +from envied.core.binaries import FFMPEG, Mp4decrypt, ShakaPackager +from envied.core.config import config +from envied.core.console import console +from envied.core.constants import DOWNLOAD_CANCELLED +from envied.core.utilities import get_debug_logger + +PERCENT_RE = re.compile(r"(\d+\.\d+%)") +SPEED_RE = re.compile(r"(\d+\.\d+(?:MB|KB)ps)") +SIZE_RE = re.compile(r"(\d+\.\d+(?:MB|GB|KB)/\d+\.\d+(?:MB|GB|KB))") +WARN_RE = re.compile(r"(WARN : Response.*|WARN : One or more errors occurred.*)") +ERROR_RE = re.compile(r"(\bERROR\b.*|\bFAILED\b.*|\bException\b.*)") + +DECRYPTION_ENGINE = { + "shaka": "SHAKA_PACKAGER", + "mp4decrypt": "MP4DECRYPT", +} + +# Ignore FutureWarnings +warnings.simplefilter(action="ignore", category=FutureWarning) + + +def get_track_selection_args(track: Any) -> list[str]: + """ + Generates track selection arguments for N_m3u8dl_RE. + + Args: + track: A track object with attributes like descriptor, data, and class name. + + Returns: + A list of strings for track selection. + + Raises: + ValueError: If the manifest type is unsupported or track selection fails. + """ + descriptor = track.descriptor.name + track_type = track.__class__.__name__ + + def _create_args(flag: str, parts: list[str], type_str: str, extra_args: list[str] | None = None) -> list[str]: + if not parts: + raise ValueError(f"[N_m3u8DL-RE]: Unable to select {type_str} track from {descriptor} manifest") + + final_args = [flag, ":".join(parts)] + if extra_args: + final_args.extend(extra_args) + + return final_args + + match descriptor: + case "HLS": + # HLS playlists are direct inputs; no selection arguments needed. + return [] + + case "DASH": + representation = track.data.get("dash", {}).get("representation", {}) + adaptation_set = track.data.get("dash", {}).get("adaptation_set", {}) + parts = [] + + if track_type == "Audio": + track_id = representation.get("id") or adaptation_set.get("audioTrackId") + lang = representation.get("lang") or adaptation_set.get("lang") + + if track_id: + parts.append(rf'"id=\b{track_id}\b"') + if lang: + parts.append(f"lang={lang}") + else: + if codecs := representation.get("codecs"): + parts.append(f"codecs={codecs}") + if lang: + parts.append(f"lang={lang}") + if bw := representation.get("bandwidth"): + bitrate = int(bw) // 1000 + parts.append(f"bwMin={bitrate}:bwMax={bitrate + 5}") + if roles := representation.findall("Role") + adaptation_set.findall("Role"): + if role := next((r.get("value") for r in roles if r.get("value", "").lower() == "main"), None): + parts.append(f"role={role}") + return _create_args("-sa", parts, "audio") + + if track_type == "Video": + if track_id := representation.get("id"): + parts.append(rf'"id=\b{track_id}\b"') + else: + if width := representation.get("width"): + parts.append(f"res={width}*") + if codecs := representation.get("codecs"): + parts.append(f"codecs={codecs}") + if bw := representation.get("bandwidth"): + bitrate = int(bw) // 1000 + parts.append(f"bwMin={bitrate}:bwMax={bitrate + 5}") + return _create_args("-sv", parts, "video") + + if track_type == "Subtitle": + if track_id := representation.get("id"): + parts.append(rf'"id=\b{track_id}\b"') + else: + if lang := representation.get("lang"): + parts.append(f"lang={lang}") + return _create_args("-ss", parts, "subtitle", extra_args=["--auto-subtitle-fix", "false"]) + + case "ISM": + quality_level = track.data.get("ism", {}).get("quality_level", {}) + stream_index = track.data.get("ism", {}).get("stream_index", {}) + parts = [] + + if track_type == "Audio": + if name := stream_index.get("Name") or quality_level.get("Index"): + parts.append(rf'"id=\b{name}\b"') + else: + if codecs := quality_level.get("FourCC"): + parts.append(f"codecs={codecs}") + if lang := stream_index.get("Language"): + parts.append(f"lang={lang}") + if br := quality_level.get("Bitrate"): + bitrate = int(br) // 1000 + parts.append(f"bwMin={bitrate}:bwMax={bitrate + 5}") + return _create_args("-sa", parts, "audio") + + if track_type == "Video": + if name := stream_index.get("Name") or quality_level.get("Index"): + parts.append(rf'"id=\b{name}\b"') + else: + if width := quality_level.get("MaxWidth"): + parts.append(f"res={width}*") + if codecs := quality_level.get("FourCC"): + parts.append(f"codecs={codecs}") + if br := quality_level.get("Bitrate"): + bitrate = int(br) // 1000 + parts.append(f"bwMin={bitrate}:bwMax={bitrate + 5}") + return _create_args("-sv", parts, "video") + + # I've yet to encounter a subtitle track in ISM manifests, so this is mostly theoretical. + if track_type == "Subtitle": + if name := stream_index.get("Name") or quality_level.get("Index"): + parts.append(rf'"id=\b{name}\b"') + else: + if lang := stream_index.get("Language"): + parts.append(f"lang={lang}") + return _create_args("-ss", parts, "subtitle", extra_args=["--auto-subtitle-fix", "false"]) + + case "URL": + raise ValueError( + f"[N_m3u8DL-RE]: Direct URL downloads are not supported for {track_type} tracks. " + f"The track should use a different downloader (e.g., 'requests', 'aria2c')." + ) + + raise ValueError(f"[N_m3u8DL-RE]: Unsupported manifest type: {descriptor}") + + +def build_download_args( + track_url: str, + filename: str, + output_dir: Path, + thread_count: int, + retry_count: int, + track_from_file: Path | None, + custom_args: dict[str, Any] | None, + headers: dict[str, Any] | None, + cookies: CookieJar | None, + proxy: str | None, + content_keys: dict[str, str] | None, + ad_keyword: str | None, + skip_merge: bool | None = False, +) -> list[str]: + """Constructs the CLI arguments for N_m3u8DL-RE.""" + + # Default arguments + args = { + "--save-name": filename, + "--save-dir": output_dir, + "--tmp-dir": output_dir, + "--thread-count": thread_count, + "--download-retry-count": retry_count, + } + if FFMPEG: + args["--ffmpeg-binary-path"] = str(FFMPEG) + if proxy: + args["--custom-proxy"] = proxy + if skip_merge: + args["--skip-merge"] = skip_merge + if ad_keyword: + args["--ad-keyword"] = ad_keyword + # Disable segment count validation to work around N_m3u8DL-RE's Math.Ceiling + # bug in duration-based SegmentTemplate calculation (see nilaoda/N_m3u8DL-RE#108) + args["--check-segments-count"] = False + + key_args = [] + if content_keys: + for kid, key in content_keys.items(): + key_args.extend(["--key", f"{kid.hex}:{key.lower()}"]) + + decryption_config = config.decryption.lower() + engine_name = DECRYPTION_ENGINE.get(decryption_config) or "SHAKA_PACKAGER" + args["--decryption-engine"] = engine_name + + binary_path = None + if engine_name == "SHAKA_PACKAGER": + if ShakaPackager: + binary_path = str(ShakaPackager) + elif engine_name == "MP4DECRYPT": + if Mp4decrypt: + binary_path = str(Mp4decrypt) + if binary_path: + args["--decryption-binary-path"] = binary_path + + if custom_args: + args.update(custom_args) + + command = [track_from_file or track_url] + for flag, value in args.items(): + if value is True: + command.append(flag) + elif value is False: + command.extend([flag, "false"]) + elif value is not False and value is not None: + command.extend([flag, str(value)]) + + # Append all content keys (multiple --key flags supported by N_m3u8DL-RE) + command.extend(key_args) + + if headers: + for key, value in headers.items(): + if key.lower() not in ("accept-encoding", "cookie"): + command.extend(["--header", f"{key}: {value}"]) + + if cookies: + req = requests.Request(method="GET", url=track_url) + cookie_header = get_cookie_header(cookies, req) + command.extend(["--header", f"Cookie: {cookie_header}"]) + + return command + + +def download( + urls: str | dict[str, Any] | list[str | dict[str, Any]], + track: Any, + output_dir: Path, + filename: str, + headers: MutableMapping[str, str | bytes] | None, + cookies: MutableMapping[str, str] | CookieJar | None, + proxy: str | None, + max_workers: int | None, + content_keys: dict[str, Any] | None, + skip_merge: bool | None = False, +) -> Generator[dict[str, Any], None, None]: + debug_logger = get_debug_logger() + + if not urls: + raise ValueError("urls must be provided and not empty") + if not isinstance(urls, (str, dict, list)): + raise TypeError(f"Expected urls to be str, dict, or list, not {type(urls)}") + if not isinstance(output_dir, Path): + raise TypeError(f"Expected output_dir to be Path, not {type(output_dir)}") + if not isinstance(filename, str) or not filename: + raise ValueError("filename must be a non-empty string") + if not isinstance(headers, (MutableMapping, type(None))): + raise TypeError(f"Expected headers to be a mapping or None, not {type(headers)}") + if not isinstance(cookies, (MutableMapping, CookieJar, type(None))): + raise TypeError(f"Expected cookies to be a mapping, CookieJar, or None, not {type(cookies)}") + if not isinstance(proxy, (str, type(None))): + raise TypeError(f"Expected proxy to be a str or None, not {type(proxy)}") + if not isinstance(max_workers, (int, type(None))): + raise TypeError(f"Expected max_workers to be an int or None, not {type(max_workers)}") + if not isinstance(content_keys, (dict, type(None))): + raise TypeError(f"Expected content_keys to be a dict or None, not {type(content_keys)}") + if not isinstance(skip_merge, (bool, type(None))): + raise TypeError(f"Expected skip_merge to be a bool or None, not {type(skip_merge)}") + + if cookies and not isinstance(cookies, CookieJar): + cookies = cookiejar_from_dict(cookies) + + if not binaries.N_m3u8DL_RE: + raise EnvironmentError("N_m3u8DL-RE executable not found...") + + effective_max_workers = max_workers or min(32, (os.cpu_count() or 1) + 4) + + if proxy and not config.n_m3u8dl_re.get("use_proxy", True): + proxy = None + + thread_count = config.n_m3u8dl_re.get("thread_count", effective_max_workers) + retry_count = config.n_m3u8dl_re.get("retry_count", 10) + ad_keyword = config.n_m3u8dl_re.get("ad_keyword") + + arguments = build_download_args( + track_url=track.url, + track_from_file=track.from_file, + filename=filename, + output_dir=output_dir, + thread_count=thread_count, + retry_count=retry_count, + custom_args=track.downloader_args, + headers=headers, + cookies=cookies, + proxy=proxy, + content_keys=content_keys, + skip_merge=skip_merge, + ad_keyword=ad_keyword, + ) + selection_args = get_track_selection_args(track) + arguments.extend(selection_args) + + log_file_path: Path | None = None + if debug_logger: + log_file_path = output_dir / f".n_m3u8dl_re_{filename}.log" + arguments.extend([ + "--log-file-path", str(log_file_path), + "--log-level", "DEBUG", + ]) + + track_url_display = track.url[:200] + "..." if len(track.url) > 200 else track.url + debug_logger.log( + level="DEBUG", + operation="downloader_n_m3u8dl_re_start", + message="Starting N_m3u8DL-RE download", + context={ + "binary_path": str(binaries.N_m3u8DL_RE), + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "track_url": track_url_display, + "output_dir": str(output_dir), + "filename": filename, + "thread_count": thread_count, + "retry_count": retry_count, + "has_content_keys": bool(content_keys), + "content_key_count": len(content_keys) if content_keys else 0, + "has_proxy": bool(proxy), + "skip_merge": skip_merge, + "has_custom_args": bool(track.downloader_args), + "selection_args": selection_args, + "descriptor": track.descriptor.name if hasattr(track, "descriptor") else None, + }, + ) + else: + arguments.extend(["--no-log", "true"]) + + yield {"total": 100} + yield {"downloaded": "Parsing streams..."} + + try: + with subprocess.Popen( + [binaries.N_m3u8DL_RE, *arguments], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + ) as process: + last_line = "" + track_type = track.__class__.__name__ + + for line in process.stdout: + output = line.strip() + if not output: + continue + last_line = output + + if warn_match := WARN_RE.search(output): + console.log(f"{track_type} {warn_match.group(1)}") + continue + + if speed_match := SPEED_RE.search(output): + size = size_match.group(1) if (size_match := SIZE_RE.search(output)) else "" + yield {"downloaded": f"{speed_match.group(1)} {size}"} + + if percent_match := PERCENT_RE.search(output): + progress = int(percent_match.group(1).split(".", 1)[0]) + yield {"completed": progress} if progress < 100 else {"downloaded": "Merging"} + + process.wait() + + if process.returncode != 0: + if debug_logger and log_file_path: + log_contents = "" + if log_file_path.exists(): + try: + log_contents = log_file_path.read_text(encoding="utf-8", errors="replace") + except Exception: + log_contents = "" + + debug_logger.log( + level="ERROR", + operation="downloader_n_m3u8dl_re_failed", + message=f"N_m3u8DL-RE exited with code {process.returncode}", + context={ + "returncode": process.returncode, + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "last_line": last_line, + "log_file_contents": log_contents, + }, + ) + if error_match := ERROR_RE.search(last_line): + raise ValueError(f"[N_m3u8DL-RE]: {error_match.group(1)}") + raise subprocess.CalledProcessError(process.returncode, arguments) + + if debug_logger: + output_dir_exists = output_dir.exists() + output_files = [] + if output_dir_exists: + try: + output_files = [f.name for f in output_dir.iterdir() if f.is_file()][:20] + except Exception: + output_files = [""] + + debug_logger.log( + level="DEBUG", + operation="downloader_n_m3u8dl_re_complete", + message="N_m3u8DL-RE download completed successfully", + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "output_dir": str(output_dir), + "output_dir_exists": output_dir_exists, + "output_files_count": len(output_files), + "output_files": output_files, + "filename": filename, + }, + ) + + # Warn if no output was produced - include N_m3u8DL-RE's logs for diagnosis + if not output_dir_exists or not output_files: + # Read N_m3u8DL-RE's log file for debugging + n_m3u8dl_log = "" + if log_file_path and log_file_path.exists(): + try: + n_m3u8dl_log = log_file_path.read_text(encoding="utf-8", errors="replace") + except Exception: + n_m3u8dl_log = "" + + debug_logger.log( + level="WARNING", + operation="downloader_n_m3u8dl_re_no_output", + message="N_m3u8DL-RE exited successfully but produced no output files", + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "output_dir": str(output_dir), + "output_dir_exists": output_dir_exists, + "selection_args": selection_args, + "track_url": track.url[:200] + "..." if len(track.url) > 200 else track.url, + "n_m3u8dl_re_log": n_m3u8dl_log, + }, + ) + + except ConnectionResetError: + # interrupted while passing URI to download + raise KeyboardInterrupt() + except KeyboardInterrupt: + DOWNLOAD_CANCELLED.set() # skip pending track downloads + yield {"downloaded": "[yellow]CANCELLED"} + raise + except Exception as e: + DOWNLOAD_CANCELLED.set() # skip pending track downloads + yield {"downloaded": "[red]FAILED"} + if debug_logger and log_file_path and not isinstance(e, (subprocess.CalledProcessError, ValueError)): + log_contents = "" + if log_file_path.exists(): + try: + log_contents = log_file_path.read_text(encoding="utf-8", errors="replace") + except Exception: + log_contents = "" + + debug_logger.log( + level="ERROR", + operation="downloader_n_m3u8dl_re_exception", + message=f"Unexpected error during N_m3u8DL-RE download: {e}", + error=e, + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "log_file_contents": log_contents, + }, + ) + raise + finally: + # Clean up temporary debug files + if log_file_path and log_file_path.exists(): + try: + log_file_path.unlink() + except Exception: + pass + + +def n_m3u8dl_re( + urls: str | list[str] | dict[str, Any] | list[dict[str, Any]], + track: Any, + output_dir: Path, + filename: str, + headers: MutableMapping[str, str | bytes] | None = None, + cookies: MutableMapping[str, str] | CookieJar | None = None, + proxy: str | None = None, + max_workers: int | None = None, + content_keys: dict[str, Any] | None = None, + skip_merge: bool | None = False, +) -> Generator[dict[str, Any], None, None]: + """ + Download files using N_m3u8DL-RE. + https://github.com/nilaoda/N_m3u8DL-RE + + Yields the following download status updates while chunks are downloading: + + - {total: 100} (100% download total) + - {completed: 1} (1% download progress out of 100%) + - {downloaded: "10.1 MB/s"} (currently downloading at a rate of 10.1 MB/s) + + The data is in the same format accepted by rich's progress.update() function. + + Parameters: + urls: Web URL(s) to file(s) to download. NOTE: This parameter is ignored for now. + track: The track to download. Used to get track attributes for the selection + process. Note that Track.Descriptor.URL is not supported by N_m3u8DL-RE. + output_dir: The folder to save the file into. If the save path's directory does + not exist then it will be made automatically. + filename: The filename or filename template to use for each file. + headers: A mapping of HTTP Header Key/Values to use for all downloads. + cookies: A mapping of Cookie Key/Values or a Cookie Jar to use for all downloads. + proxy: A proxy to use for all downloads. + max_workers: The maximum amount of threads to use for downloads. Defaults to + min(32,(cpu_count+4)). Can be set in config with --thread-count option. + content_keys: The content keys to use for decryption. + skip_merge: Whether to skip merging the downloaded chunks. + """ + + yield from download( + urls=urls, + track=track, + output_dir=output_dir, + filename=filename, + headers=headers, + cookies=cookies, + proxy=proxy, + max_workers=max_workers, + content_keys=content_keys, + skip_merge=skip_merge, + ) + + +__all__ = ("n_m3u8dl_re",) diff --git a/reset/packages/envied/src/envied/core/downloaders/requests.py b/reset/packages/envied/src/envied/core/downloaders/requests.py new file mode 100644 index 0000000..3da2815 --- /dev/null +++ b/reset/packages/envied/src/envied/core/downloaders/requests.py @@ -0,0 +1,663 @@ +import math +import os +import time +from concurrent.futures import FIRST_COMPLETED, wait +from concurrent.futures.thread import ThreadPoolExecutor +from http.cookiejar import CookieJar +from pathlib import Path +from queue import Empty, Queue +from typing import Any, Generator, MutableMapping, Optional, Union + +from requests import Session +from requests.adapters import HTTPAdapter +from rich import filesize + +from envied.core.constants import DOWNLOAD_CANCELLED +from envied.core.utilities import get_debug_logger, get_extension + +MAX_ATTEMPTS = 5 +RETRY_WAIT = 2 +PROGRESS_WINDOW = 2 + +# Adaptive chunk sizing — benchmarked optimal range +MIN_CHUNK = 524_288 # 512KB +MAX_CHUNK = 4_194_304 # 4MB +DEFAULT_CHUNK = 524_288 # 512KB +SPEED_ROLLING_WINDOW = 10 # seconds of history to keep for speed calculation + +RANGE_PARALLEL_MIN_SIZE = 64 * 1024 * 1024 +RANGE_PARALLEL_PART_SIZE = 16 * 1024 * 1024 + + +def _adaptive_chunk_size(content_length: int) -> int: + """Pick chunk size based on content length. Benchmarked sweet spot: 512KB-4MB.""" + if content_length <= 0: + return DEFAULT_CHUNK + return min(MAX_CHUNK, max(MIN_CHUNK, content_length // 4)) + + +def _is_requests_session(session: Any) -> bool: + """Check if the session is a standard requests.Session (supports resp.raw).""" + return isinstance(session, Session) + + +def _is_rnet_session(session: Any) -> bool: + """Check if the session is an RnetSession (uses resp.stream()).""" + from envied.core.session import RnetSession + + return isinstance(session, RnetSession) + + +def _probe_ranged(url: str, session: Any, **kwargs: Any) -> tuple[int, bool]: + headers = {**(kwargs.get("headers") or {}), "Range": "bytes=0-0"} + rest = {k: v for k, v in kwargs.items() if k != "headers"} + try: + resp = session.get(url, stream=True, headers=headers, **rest) + except Exception: + return 0, False + try: + if resp.status_code != 206: + return 0, False + ce = (resp.headers.get("Content-Encoding") or resp.headers.get("content-encoding") or "").lower() + if ce in ("gzip", "deflate", "br"): + return 0, False + content_range = resp.headers.get("Content-Range") or resp.headers.get("content-range") or "" + total = content_range.rsplit("/", 1)[-1].strip() + return (int(total), True) if total.isdigit() else (0, False) + finally: + try: + resp.close() + except Exception: + pass + + +def _dispatch_parts( + url: str, + save_path: Path, + session: Any, + total_size: int, + max_workers: int, + **kwargs: Any, +) -> Generator[dict[str, Any], None, None]: + save_path.parent.mkdir(parents=True, exist_ok=True) + control_file = save_path.with_name(f"{save_path.name}.!dev") + + n_parts = max(1, min(max_workers, math.ceil(total_size / RANGE_PARALLEL_PART_SIZE))) + part_size = math.ceil(total_size / n_parts) + parts = [ + (s, e) + for i in range(n_parts) + for s, e in [(i * part_size, min(total_size - 1, (i + 1) * part_size - 1))] + if s <= e + ] + + control_file.write_bytes(b"") + with open(save_path, "wb") as f: + f.truncate(total_size) + + events: Queue[dict[str, Any]] = Queue() + + def _worker(start: int, end: int) -> None: + for ev in download( + url=url, + save_path=save_path, + session=session, + part_offset=start, + part_end=end, + **kwargs, + ): + events.put(ev) + + pool = ThreadPoolExecutor(max_workers=len(parts)) + futures = [pool.submit(_worker, s, e) for s, e in parts] + pending = set(futures) + + yield {"total": total_size} + + total_bytes = 0 + start_time = last_report = time.time() + completed = False + worker_error = False + + try: + while pending: + advance = 0 + while not events.empty(): + try: + ev = events.get_nowait() + except Empty: + break + a = ev.get("advance") + if a: + advance += a + if advance: + total_bytes += advance + yield {"advance": advance} + + now = time.time() + if now - last_report > 0.5 and total_bytes > 0: + yield {"downloaded": f"{filesize.decimal(math.ceil(total_bytes / (now - start_time)))}/s"} + last_report = now + + done, pending = wait(pending, timeout=0.1, return_when=FIRST_COMPLETED) + for fut in done: + exc = fut.exception() + if exc: + worker_error = True + DOWNLOAD_CANCELLED.set() + raise exc + + advance = 0 + while not events.empty(): + try: + ev = events.get_nowait() + except Empty: + break + a = ev.get("advance") + if a: + advance += a + if advance: + total_bytes += advance + yield {"advance": advance} + + yield {"file_downloaded": save_path, "written": total_size} + completed = True + except KeyboardInterrupt: + DOWNLOAD_CANCELLED.set() + pool.shutdown(wait=False, cancel_futures=True) + raise + finally: + pool.shutdown(wait=worker_error, cancel_futures=True) + if completed: + control_file.unlink(missing_ok=True) + + +def download( + url: str, + save_path: Path, + session: Optional[Any] = None, + segmented: bool = False, + part_offset: Optional[int] = None, + part_end: Optional[int] = None, + **kwargs: Any, +) -> Generator[dict[str, Any], None, None]: + """ + Download a file with optimized I/O. + + Supports both requests.Session and RnetSession for TLS fingerprinting. + Uses raw socket reads for requests.Session and native rnet streaming for RnetSession. + + Yields the following download status updates while chunks are downloading: + + - {total: 123} (there are 123 chunks to download) + - {total: None} (there are an unknown number of chunks to download) + - {advance: 1} (one chunk was downloaded) + - {downloaded: "10.1 MB/s"} (currently downloading at a rate of 10.1 MB/s) + - {file_downloaded: Path(...), written: 1024} (download finished, has the save path and size) + + Parameters: + url: Web URL of a file to download. + save_path: The path to save the file to. If the save path's directory does not + exist then it will be made automatically. + session: A requests.Session or RnetSession to make HTTP requests with. + RnetSession preserves TLS fingerprinting for services that need it. + segmented: If downloads are segments or parts of one bigger file. + part_offset: Byte offset to write at within a pre-allocated file. When set + (with `part_end`), enables part mode for parallel ranged downloads — + no truncate, no skip-if-exists, no control file; emits only `advance` + events; retries resume mid-part via Range. + part_end: Inclusive end byte of the part. Required when `part_offset` is set. + kwargs: Any extra keyword arguments to pass to the session.get() call. Use this + for one-time request changes like a header, cookie, or proxy. For example, + to request Byte-ranges use e.g., `headers={"Range": "bytes=0-128"}`. + """ + session = session or Session() + part_mode = part_offset is not None and part_end is not None + + save_dir = save_path.parent + control_file = save_path.with_name(f"{save_path.name}.!dev") + save_dir.mkdir(parents=True, exist_ok=True) + + resume_offset = 0 + if not part_mode: + if control_file.exists() and save_path.exists(): + resume_offset = save_path.stat().st_size + elif control_file.exists(): + control_file.unlink() + elif save_path.exists(): + yield dict(file_downloaded=save_path, written=save_path.stat().st_size) + return + control_file.write_bytes(b"") + + _time = time.time + use_raw = _is_requests_session(session) + + attempts = 1 + completed = False + written = 0 + try: + while True: + if not part_mode: + written = 0 + last_speed_refresh = _time() + + try: + use_rnet = _is_rnet_session(session) + + request_kwargs = dict(kwargs) + if part_mode: + req_headers = dict(request_kwargs.get("headers", {}) or {}) + req_headers["Range"] = f"bytes={part_offset + written}-{part_end}" + request_kwargs["headers"] = req_headers + elif resume_offset > 0: + req_headers = dict(request_kwargs.get("headers", {}) or {}) + req_headers["Range"] = f"bytes={resume_offset}-" + request_kwargs["headers"] = req_headers + + stream = session.get(url, stream=True, **request_kwargs) + stream.raise_for_status() + + resumed = (not part_mode) and resume_offset > 0 and stream.status_code == 206 + if (not part_mode) and resume_offset > 0 and not resumed: + resume_offset = 0 + if part_mode and stream.status_code != 206: + raise IOError(f"expected 206 for ranged part, got {stream.status_code}") + if use_rnet: + content_length = stream.content_length or 0 + else: + try: + content_length = int(stream.headers.get("Content-Length", "0")) + if stream.headers.get("Content-Encoding", "").lower() in ["gzip", "deflate", "br"]: + content_length = 0 + except ValueError: + content_length = 0 + + chunk_size = _adaptive_chunk_size(content_length) + total_size = (resume_offset + content_length) if resumed and content_length > 0 else content_length + + if not segmented and not part_mode: + if total_size > 0: + yield dict(total=total_size) + else: + yield dict(total=None) + if resumed and resume_offset > 0: + yield dict(advance=resume_offset) + + if part_mode: + file_mode = "r+b" + file_buffering = 0 + else: + file_mode = "ab" if resumed else "wb" + file_buffering = 1_048_576 + with open(save_path, file_mode, buffering=file_buffering) as f: + if part_mode: + f.seek(part_offset + written) + elif not resumed and content_length > 0: + f.truncate(content_length) + f.seek(0) + + _write = f.write + + if use_rnet: + chunks = stream.stream() + elif use_raw: + chunks = iter(lambda: stream.raw.read(chunk_size), b"") + else: + chunks = stream.iter_content(chunk_size=chunk_size) + + _data_accumulated = 0 + _bytes_since_yield = 0 + emit_progress = (not segmented) or part_mode + for chunk in chunks: + if DOWNLOAD_CANCELLED.is_set(): + break + _write(chunk) + download_size = len(chunk) + written += download_size + + if emit_progress: + _bytes_since_yield += download_size + _data_accumulated += download_size + now = _time() + time_since = now - last_speed_refresh + if time_since > PROGRESS_WINDOW: + yield dict(advance=_bytes_since_yield) + _bytes_since_yield = 0 + if not part_mode: + download_speed = math.ceil(_data_accumulated / (time_since or 1)) + yield dict(downloaded=f"{filesize.decimal(download_speed)}/s") + last_speed_refresh = now + _data_accumulated = 0 + + if emit_progress and _bytes_since_yield > 0: + yield dict(advance=_bytes_since_yield) + + try: + stream.close() + except Exception: + pass + + if not part_mode and not resumed and content_length > 0 and written != content_length: + f.truncate(written) + + if part_mode: + expected = part_end - part_offset + 1 + if written < expected: + raise IOError(f"Failed to read part {part_offset}-{part_end}: got {written}/{expected}") + elif not segmented and content_length and written < content_length: + raise IOError(f"Failed to read {content_length} bytes from the track URI.") + + if not part_mode: + yield dict(file_downloaded=save_path, written=resume_offset + written) + if segmented: + yield dict(advance=1) + completed = True + break + except Exception: + try: + stream.close() + except Exception: + pass + if DOWNLOAD_CANCELLED.is_set() or attempts == MAX_ATTEMPTS: + if part_mode and not DOWNLOAD_CANCELLED.is_set(): + raise + return + if not part_mode and save_path.exists(): + resume_offset = save_path.stat().st_size + time.sleep(RETRY_WAIT) + attempts += 1 + finally: + if completed and not part_mode: + control_file.unlink(missing_ok=True) + + +def requests( + urls: Union[str, list[str], dict[str, Any], list[dict[str, Any]]], + output_dir: Path, + filename: str, + headers: Optional[MutableMapping[str, Union[str, bytes]]] = None, + cookies: Optional[Union[MutableMapping[str, str], CookieJar]] = None, + proxy: Optional[str] = None, + max_workers: Optional[int] = None, + session: Optional[Any] = None, +) -> Generator[dict[str, Any], None, None]: + """ + Download files with optimized I/O and adaptive chunk sizing. + + Supports both requests.Session and RnetSession. When a RnetSession is + provided (e.g. from a service's get_session()), TLS fingerprinting is preserved + on all segment downloads. + + Yields the following download status updates while chunks are downloading: + + - {total: 123} (there are 123 chunks to download) + - {total: None} (there are an unknown number of chunks to download) + - {advance: 1} (one chunk was downloaded) + - {downloaded: "10.1 MB/s"} (currently downloading at a rate of 10.1 MB/s) + - {file_downloaded: Path(...), written: 1024} (download finished, has the save path and size) + + The data is in the same format accepted by rich's progress.update() function. + However, The `downloaded`, `file_downloaded` and `written` keys are custom and not + natively accepted by rich progress bars. + + Parameters: + urls: Web URL(s) to file(s) to download. You can use a dictionary with the key + "url" for the URI, and other keys for extra arguments to use per-URL. + output_dir: The folder to save the file into. If the save path's directory does + not exist then it will be made automatically. + filename: The filename or filename template to use for each file. The variables + you can use are `i` for the URL index and `ext` for the URL extension. + headers: A mapping of HTTP Header Key/Values to use for all downloads. + cookies: A mapping of Cookie Key/Values or a Cookie Jar to use for all downloads. + proxy: An optional proxy URI to route connections through for all downloads. + max_workers: The maximum amount of threads to use for downloads. Defaults to + min(12,(cpu_count+4)). + session: An optional requests.Session or RnetSession to use. If provided, + it will be used directly (preserving TLS fingerprinting). If None, a new + requests.Session with HTTPAdapter connection pooling will be created. + """ + if not urls: + raise ValueError("urls must be provided and not empty") + elif not isinstance(urls, (str, dict, list)): + raise TypeError(f"Expected urls to be {str} or {dict} or a list of one of them, not {type(urls)}") + + if not output_dir: + raise ValueError("output_dir must be provided") + elif not isinstance(output_dir, Path): + raise TypeError(f"Expected output_dir to be {Path}, not {type(output_dir)}") + + if not filename: + raise ValueError("filename must be provided") + elif not isinstance(filename, str): + raise TypeError(f"Expected filename to be {str}, not {type(filename)}") + + if not isinstance(headers, (MutableMapping, type(None))): + raise TypeError(f"Expected headers to be {MutableMapping}, not {type(headers)}") + + if not isinstance(cookies, (MutableMapping, CookieJar, type(None))): + raise TypeError(f"Expected cookies to be {MutableMapping} or {CookieJar}, not {type(cookies)}") + + if not isinstance(proxy, (str, type(None))): + raise TypeError(f"Expected proxy to be {str}, not {type(proxy)}") + + if not isinstance(max_workers, (int, type(None))): + raise TypeError(f"Expected max_workers to be {int}, not {type(max_workers)}") + + debug_logger = get_debug_logger() + + if not isinstance(urls, list): + urls = [urls] + + if not max_workers: + max_workers = min(16, (os.cpu_count() or 1) + 4) + + urls = [ + dict(save_path=save_path, **url) if isinstance(url, dict) else dict(url=url, save_path=save_path) + for i, url in enumerate(urls) + for save_path in [ + output_dir / filename.format(i=i, ext=get_extension(url["url"] if isinstance(url, dict) else url)) + ] + ] + + # Use provided session or create a new optimized requests.Session + # When a session is provided (e.g., service's RnetSession), don't mutate headers/cookies/proxy — + # they're already set and the session may be shared across tracks. + if session is None: + session = Session() + if headers: + headers = {k: v for k, v in headers.items() if k.lower() != "accept-encoding"} + session.headers.update(headers) + if cookies: + session.cookies.update(cookies) + if proxy: + session.proxies.update({"all": proxy}) + + # Mount HTTPAdapter with connection pooling sized to worker count. + # Safe to do on any requests.Session — improves connection reuse for parallel downloads. + if _is_requests_session(session): + adapter = HTTPAdapter(pool_connections=max_workers, pool_maxsize=max_workers, pool_block=True) + session.mount("https://", adapter) + session.mount("http://", adapter) + + if debug_logger: + first_url = urls[0].get("url", "") if urls else "" + url_display = first_url[:200] + "..." if len(first_url) > 200 else first_url + debug_logger.log( + level="DEBUG", + operation="downloader_start", + message="Starting download", + context={ + "url_count": len(urls), + "first_url": url_display, + "output_dir": str(output_dir), + "filename": filename, + "max_workers": max_workers, + "has_proxy": bool(proxy), + "session_type": type(session).__name__, + }, + ) + + segmented_batch = len(urls) > 1 + + if len(urls) == 1: + url_item = urls[0] + try: + ranged_used = False + if max_workers > 1: + total_size, supports_ranges = _probe_ranged(url_item["url"], session) + if supports_ranges and total_size >= RANGE_PARALLEL_MIN_SIZE: + try: + yield from _dispatch_parts( + session=session, + total_size=total_size, + max_workers=max_workers, + **url_item, + ) + ranged_used = True + except KeyboardInterrupt: + raise + except Exception: + save_path = url_item.get("save_path") + if save_path: + sp = Path(save_path) + for target in (sp, sp.with_name(f"{sp.name}.!dev")): + try: + target.unlink(missing_ok=True) + except OSError: + pass + if not ranged_used: + yield from download( + session=session, + segmented=segmented_batch, + **url_item, + ) + except KeyboardInterrupt: + DOWNLOAD_CANCELLED.set() + yield dict(downloaded="[yellow]CANCELLED") + raise + else: + # Segmented download with thread pool + # Speed is tracked here on the main thread, not in workers + total_bytes = 0 + start_time = time.time() + last_speed_report = start_time + + pool = ThreadPoolExecutor(max_workers=max_workers) + event_queue: Queue[dict[str, Any]] = Queue() + + def _download_worker(url_item: dict[str, Any]) -> None: + for event in download( + session=session, + segmented=segmented_batch, + **url_item, + ): + event_queue.put(event) + + futures = [pool.submit(_download_worker, url) for url in urls] + pending = set(futures) + + pending_advance = 0 + + try: + while pending: + # Drain queued events — batch advances, track bytes for speed + while True: + try: + event = event_queue.get_nowait() + except Empty: + break + # Accumulate advance events for batched yield + advance = event.get("advance") + if advance: + pending_advance += advance + continue + # Track bytes from completed segments for speed calculation + written = event.get("written") + if written: + total_bytes += written + # Pass through other events (file_downloaded, total, etc.) + yield event + + # Yield batched advances every drain cycle for responsive progress bar + if pending_advance > 0: + yield dict(advance=pending_advance) + pending_advance = 0 + + # Yield speed every 0.5s (throttled to avoid spamming Rich) + now = time.time() + if now - last_speed_report > 0.5 and total_bytes > 0: + elapsed = now - start_time + if elapsed > 0: + download_speed = math.ceil(total_bytes / elapsed) + yield dict(downloaded=f"{filesize.decimal(download_speed)}/s") + last_speed_report = now + + # Wait efficiently for next future completion (OS condition variable) + completed, pending = wait(pending, timeout=0.1, return_when=FIRST_COMPLETED) + for future in completed: + exc = future.exception() + if isinstance(exc, KeyboardInterrupt): + raise KeyboardInterrupt() + elif exc: + DOWNLOAD_CANCELLED.set() + yield dict(downloaded="[red]FAILING") + pool.shutdown(wait=False, cancel_futures=True) + yield dict(downloaded="[red]FAILED") + if debug_logger: + debug_logger.log( + level="ERROR", + operation="downloader_failed", + message=f"Download failed: {exc}", + error=exc, + context={ + "url_count": len(urls), + "output_dir": str(output_dir), + }, + ) + raise exc + except KeyboardInterrupt: + DOWNLOAD_CANCELLED.set() + yield dict(downloaded="[yellow]CANCELLING") + pool.shutdown(wait=False, cancel_futures=True) + yield dict(downloaded="[yellow]CANCELLED") + raise + finally: + pool.shutdown(wait=False, cancel_futures=True) + + # Drain remaining events + while True: + try: + event = event_queue.get_nowait() + except Empty: + break + advance = event.get("advance") + if advance: + pending_advance += advance + continue + written = event.get("written") + if written: + total_bytes += written + yield event + + # Flush remaining advances and final speed + if pending_advance > 0: + yield dict(advance=pending_advance) + elapsed = time.time() - start_time + if elapsed > 0 and total_bytes > 0: + download_speed = math.ceil(total_bytes / elapsed) + yield dict(downloaded=f"{filesize.decimal(download_speed)}/s") + + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="downloader_complete", + message="Download completed successfully", + context={ + "url_count": len(urls), + "output_dir": str(output_dir), + "filename": filename, + }, + ) + + +__all__ = ("requests",) diff --git a/reset/packages/envied/src/envied/core/drm/__init__.py b/reset/packages/envied/src/envied/core/drm/__init__.py new file mode 100644 index 0000000..aa64295 --- /dev/null +++ b/reset/packages/envied/src/envied/core/drm/__init__.py @@ -0,0 +1,44 @@ +import base64 +from typing import Any, Union +from uuid import UUID + +from envied.core.drm.clearkey import ClearKey +from envied.core.drm.monalisa import MonaLisa +from envied.core.drm.playready import PlayReady +from envied.core.drm.widevine import Widevine + +DRM_T = Union[ClearKey, Widevine, PlayReady, MonaLisa] + + +def drm_from_dict(data: dict[str, Any]) -> Union[Widevine, PlayReady]: + """Reconstruct a Widevine/PlayReady DRM instance from its ``to_dict()`` form. + + Rebuilds the PSSH from the stored base64 and re-injects any saved content keys + so the resulting object can decrypt without contacting a license server. + """ + system = data.get("system") + pssh_b64 = data.get("pssh_b64") + kids = data.get("kids") or [] + content_keys = data.get("content_keys") or {} + + if not pssh_b64: + raise ValueError("Cannot reconstruct DRM without a stored PSSH.") + + if system == "PlayReady": + from pyplayready.system.pssh import PSSH as PlayReadyPSSH + + drm: Union[Widevine, PlayReady] = PlayReady(pssh=PlayReadyPSSH(base64.b64decode(pssh_b64)), pssh_b64=pssh_b64) + elif system == "Widevine": + from pywidevine.pssh import PSSH as WidevinePSSH + + drm = Widevine(pssh=WidevinePSSH(pssh_b64), kid=kids[0] if kids else None) + else: + raise ValueError(f"Unsupported DRM system for reconstruction: {system!r}") + + for kid_hex, key in content_keys.items(): + drm.content_keys[UUID(hex=kid_hex)] = key + + return drm + + +__all__ = ("ClearKey", "Widevine", "PlayReady", "MonaLisa", "DRM_T", "drm_from_dict") diff --git a/reset/packages/envied/src/envied/core/drm/clearkey.py b/reset/packages/envied/src/envied/core/drm/clearkey.py new file mode 100644 index 0000000..4e7d0e9 --- /dev/null +++ b/reset/packages/envied/src/envied/core/drm/clearkey.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import base64 +import shutil +from pathlib import Path +from typing import Optional, Union +from urllib.parse import urljoin + +from Cryptodome.Cipher import AES +from Cryptodome.Util.Padding import unpad +from m3u8.model import Key +from requests import Session + +from envied.core.session import RnetSession + + +class ClearKey: + """AES Clear Key DRM System.""" + + def __init__(self, key: Union[bytes, str], iv: Optional[Union[bytes, str]] = None): + """ + Generally IV should be provided where possible. If not provided, it will be + set to \x00 of the same bit-size of the key. + """ + if isinstance(key, str): + key = bytes.fromhex(key.replace("0x", "")) + if not isinstance(key, bytes): + raise ValueError(f"Expected AES Key to be bytes, not {key!r}") + if not iv: + iv = b"\x00" + if isinstance(iv, str): + iv = bytes.fromhex(iv.replace("0x", "")) + if not isinstance(iv, bytes): + raise ValueError(f"Expected IV to be bytes, not {iv!r}") + + if len(iv) < len(key): + iv = iv * (len(key) - len(iv) + 1) + + self.key: bytes = key + self.iv: bytes = iv + + def decrypt(self, path: Path) -> None: + """Decrypt a Track with AES Clear Key DRM.""" + if not path or not path.exists(): + raise ValueError("Tried to decrypt a file that does not exist.") + + decrypted = AES.new(self.key, AES.MODE_CBC, self.iv).decrypt(path.read_bytes()) + + try: + decrypted = unpad(decrypted, AES.block_size) + except ValueError: + # the decrypted data is likely already in the block size boundary + pass + + decrypted_path = path.with_suffix(f".decrypted{path.suffix}") + decrypted_path.write_bytes(decrypted) + + path.unlink() + shutil.move(decrypted_path, path) + + @classmethod + def from_m3u_key(cls, m3u_key: Key, session: Optional[Session] = None) -> ClearKey: + """ + Load a ClearKey from an M3U(8) Playlist's EXT-X-KEY. + + Parameters: + m3u_key: A Key object parsed from a m3u(8) playlist using + the `m3u8` library. + session: Optional session used to request external URIs with. + Useful to set headers, proxies, cookies, and so forth. + """ + if not isinstance(m3u_key, Key): + raise ValueError(f"Provided M3U Key is in an unexpected type {m3u_key!r}") + if not isinstance(session, (Session, RnetSession, type(None))): + raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not a {type(session)}") + + if not m3u_key.method.startswith("AES"): + raise ValueError(f"Provided M3U Key is not an AES Clear Key, {m3u_key.method}") + if not m3u_key.uri: + raise ValueError("No URI in M3U Key, unable to get Key.") + + if not session: + session = Session() + + if not session.headers.get("User-Agent"): + # commonly needed default for HLS playlists + session.headers["User-Agent"] = "smartexoplayer/1.1.0 (Linux;Android 8.0.0) ExoPlayerLib/2.13.3" + + if m3u_key.uri.startswith("data:"): + media_types, data = m3u_key.uri[5:].split(",") + media_types = media_types.split(";") + if "base64" in media_types: + data = base64.b64decode(data) + key = data + else: + url = urljoin(m3u_key.base_uri, m3u_key.uri) + res = session.get(url) + res.raise_for_status() + if not res.content: + raise EOFError("Unexpected Empty Response by M3U Key URI.") + if len(res.content) < 16: + raise EOFError(f"Unexpected Length of Key ({len(res.content)} bytes) in M3U Key.") + key = res.content + + if m3u_key.iv: + iv = bytes.fromhex(m3u_key.iv.replace("0x", "")) + else: + iv = None + + return cls(key=key, iv=iv) + + +__all__ = ("ClearKey",) diff --git a/reset/packages/envied/src/envied/core/drm/monalisa.py b/reset/packages/envied/src/envied/core/drm/monalisa.py new file mode 100644 index 0000000..44c1466 --- /dev/null +++ b/reset/packages/envied/src/envied/core/drm/monalisa.py @@ -0,0 +1,299 @@ +""" +MonaLisa DRM System. + +A WASM-based DRM system that uses local key extraction and two-stage +segment decryption (ML-Worker binary + AES-ECB). +""" + +from __future__ import annotations + +import logging +import os +import subprocess +import sys +from pathlib import Path +from typing import Any, Optional, Union +from uuid import UUID + +from Cryptodome.Cipher import AES +from Cryptodome.Util.Padding import unpad + +log = logging.getLogger(__name__) + + +class MonaLisa: + """ + MonaLisa DRM System. + + Unlike Widevine/PlayReady, MonaLisa does not use a challenge/response flow + with a license server. Instead, the PSSH value (ticket) is provided directly + by the service API, and keys are extracted locally via a WASM module. + + Decryption is performed in two stages: + 1. ML-Worker binary: Removes MonaLisa encryption layer (bbts -> ents) + 2. AES-ECB decryption: Final decryption with service-provided key + """ + + class Exceptions: + class TicketNotFound(Exception): + """Raised when no PSSH/ticket data is provided.""" + + class KeyExtractionFailed(Exception): + """Raised when key extraction from the ticket fails.""" + + class WorkerNotFound(Exception): + """Raised when the ML-Worker binary is not found.""" + + class DecryptionFailed(Exception): + """Raised when segment decryption fails.""" + + def __init__( + self, + ticket: Union[str, bytes], + aes_key: Union[str, bytes], + device_path: Path, + **kwargs: Any, + ): + """ + Initialize MonaLisa DRM. + + Args: + ticket: PSSH value from service API (base64 string or raw bytes). + aes_key: AES-ECB key for second-stage decryption (hex string or bytes). + device_path: Path to the CDM device file (.mld). + **kwargs: Additional metadata stored in self.data. + + Raises: + TicketNotFound: If ticket/PSSH is empty. + KeyExtractionFailed: If key extraction fails. + """ + if not ticket: + raise MonaLisa.Exceptions.TicketNotFound("No PSSH/ticket data provided.") + + self._ticket = ticket + + # Store AES key for second-stage decryption + if isinstance(aes_key, str): + self._aes_key = bytes.fromhex(aes_key) + else: + self._aes_key = aes_key + + self._device_path = device_path + self._kid: Optional[UUID] = None + self._key: Optional[str] = None + self.data: dict = kwargs or {} + + # Extract keys immediately + self._extract_keys() + + def _extract_keys(self) -> None: + """Extract keys from the ticket using the MonaLisa CDM.""" + # Import here to avoid circular import + from envied.core.cdm.monalisa import MonaLisaCDM + + try: + cdm = MonaLisaCDM(device_path=self._device_path) + session_id = cdm.open() + try: + keys = cdm.extract_keys(self._ticket) + if keys: + kid_hex = keys.get("kid") + if kid_hex: + self._kid = UUID(hex=kid_hex) + self._key = keys.get("key") + finally: + cdm.close(session_id) + except Exception as e: + raise MonaLisa.Exceptions.KeyExtractionFailed(f"Failed to extract keys: {e}") + + @classmethod + def from_ticket( + cls, + ticket: Union[str, bytes], + aes_key: Union[str, bytes], + device_path: Path, + ) -> MonaLisa: + """ + Create a MonaLisa DRM instance from a PSSH/ticket. + + Args: + ticket: PSSH value from service API. + aes_key: AES-ECB key for second-stage decryption. + device_path: Path to the CDM device file (.mld). + + Returns: + MonaLisa DRM instance with extracted keys. + """ + return cls(ticket=ticket, aes_key=aes_key, device_path=device_path) + + @property + def kid(self) -> Optional[UUID]: + """Get the Key ID.""" + return self._kid + + @property + def key(self) -> Optional[str]: + """Get the content key as hex string.""" + return self._key + + @property + def pssh(self) -> str: + """ + Get the raw PSSH/ticket value as a string. + + Returns: + The raw PSSH value as a base64 string. + """ + if isinstance(self._ticket, bytes): + try: + return self._ticket.decode("utf-8") + except UnicodeDecodeError: + # Tickets are typically base64, so ASCII is a reasonable fallback. + try: + return self._ticket.decode("ascii") + except UnicodeDecodeError as e: + raise ValueError( + f"Ticket bytes must be UTF-8 text or ASCII base64; got undecodable bytes (len={len(self._ticket)})" + ) from e + return self._ticket + + @property + def content_id(self) -> Optional[str]: + """ + Extract the Content ID from the PSSH for display. + + The PSSH contains an embedded Content ID at bytes 21-75 with format: + H5DCID-V3-P1-YYYYMMDD-HHMMSS-MEDIAID-TIMESTAMP-SUFFIX + + Returns: + The Content ID string if extractable, None otherwise. + """ + import base64 + + try: + # Decode base64 PSSH to get raw bytes + if isinstance(self._ticket, bytes): + data = self._ticket + else: + data = base64.b64decode(self._ticket) + + # Content ID is at bytes 21-75 (55 bytes) + if len(data) >= 76: + content_id = data[21:76].decode("ascii") + # Validate it looks like a content ID + if content_id.startswith("H5DCID-"): + return content_id + except Exception: + pass + + return None + + @property + def content_keys(self) -> dict[UUID, str]: + """ + Get content keys in the same format as Widevine/PlayReady. + + Returns: + Dictionary mapping KID to key hex string. + """ + if self._kid and self._key: + return {self._kid: self._key} + return {} + + def decrypt_segment(self, segment_path: Path) -> None: + """ + Decrypt a single segment using two-stage decryption. + + Stage 1: ML-Worker binary (bbts -> ents) + Stage 2: AES-ECB decryption (ents -> ts) + + Args: + segment_path: Path to the encrypted segment file. + + Raises: + WorkerNotFound: If ML-Worker binary is not available. + DecryptionFailed: If decryption fails at any stage. + """ + if not self._key: + return + + # Import here to avoid circular import + from envied.core.cdm.monalisa import MonaLisaCDM + + worker_path = MonaLisaCDM.get_worker_path() + if not worker_path or not worker_path.exists(): + raise MonaLisa.Exceptions.WorkerNotFound("ML-Worker not found.") + + bbts_path = segment_path.with_suffix(".bbts") + ents_path = segment_path.with_suffix(".ents") + + try: + if segment_path.exists(): + segment_path.replace(bbts_path) + else: + raise MonaLisa.Exceptions.DecryptionFailed(f"Segment file does not exist: {segment_path}") + + # Stage 1: ML-Worker decryption + cmd = [str(worker_path), str(self._key), str(bbts_path), str(ents_path)] + + startupinfo = None + if sys.platform == "win32": + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + + worker_timeout_s = 60 + process = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + startupinfo=startupinfo, + timeout=worker_timeout_s, + ) + + if process.returncode != 0: + raise MonaLisa.Exceptions.DecryptionFailed( + f"ML-Worker failed for {segment_path.name}: {process.stderr}" + ) + + if not ents_path.exists(): + raise MonaLisa.Exceptions.DecryptionFailed( + f"Decrypted .ents file was not created for {segment_path.name}" + ) + + # Stage 2: AES-ECB decryption + with open(ents_path, "rb") as f: + ents_data = f.read() + + crypto = AES.new(self._aes_key, AES.MODE_ECB) + decrypted_data = unpad(crypto.decrypt(ents_data), AES.block_size) + + # Write decrypted segment back to original path + with open(segment_path, "wb") as f: + f.write(decrypted_data) + + except MonaLisa.Exceptions.DecryptionFailed: + raise + except subprocess.TimeoutExpired as e: + log.error("ML-Worker timed out after %ss for %s", worker_timeout_s, segment_path.name) + raise MonaLisa.Exceptions.DecryptionFailed( + f"ML-Worker timed out after {worker_timeout_s}s for {segment_path.name}" + ) from e + except Exception as e: + raise MonaLisa.Exceptions.DecryptionFailed(f"Failed to decrypt segment {segment_path.name}: {e}") + finally: + if ents_path.exists(): + os.remove(ents_path) + if bbts_path != segment_path and bbts_path.exists(): + os.remove(bbts_path) + + def decrypt(self, _path: Path) -> None: + """ + MonaLisa uses per-segment decryption during download via the + on_segment_downloaded callback. By the time this method is called, + the content has already been decrypted and muxed into a container. + + Args: + path: Path to the file (ignored). + """ + pass diff --git a/reset/packages/envied/src/envied/core/drm/playready.py b/reset/packages/envied/src/envied/core/drm/playready.py new file mode 100644 index 0000000..8b3f260 --- /dev/null +++ b/reset/packages/envied/src/envied/core/drm/playready.py @@ -0,0 +1,494 @@ +from __future__ import annotations + +import base64 +import shutil +import subprocess +import textwrap +from pathlib import Path +from typing import Any, Callable, Optional, Union +from uuid import UUID + +import m3u8 +from construct import Container +from pymp4.parser import Box +from pyplayready.cdm import Cdm as PlayReadyCdm +from pyplayready.system.pssh import PSSH +from requests import Session +from rich.text import Text + +from envied.core import binaries +from envied.core.config import config +from envied.core.console import console +from envied.core.constants import AnyTrack +from envied.core.utilities import get_boxes +from envied.core.utils.subprocess import ffprobe + + +class PlayReady: + """PlayReady DRM System.""" + + def __init__( + self, + pssh: PSSH, + kid: Union[UUID, str, bytes, None] = None, + pssh_b64: Optional[str] = None, + **kwargs: Any, + ): + if not pssh: + raise ValueError("Provided PSSH is empty.") + if not isinstance(pssh, PSSH): + raise TypeError(f"Expected pssh to be a {PSSH}, not {pssh!r}") + + if pssh_b64: + kids = self._extract_kids_from_pssh_b64(pssh_b64) + else: + kids = [] + + # Extract KIDs using pyplayready's WrmHeader key_ids + if not kids: + for header in pssh.wrm_headers: + for signed_key_id in getattr(header, "key_ids", []): + try: + if isinstance(signed_key_id.value, UUID): + kids.append(signed_key_id.value) + else: + kids.append(UUID(bytes_le=base64.b64decode(signed_key_id.value))) + except Exception: + continue + + if kid: + if isinstance(kid, str): + kid = UUID(hex=kid) + elif isinstance(kid, bytes): + kid = UUID(bytes=kid) + if not isinstance(kid, UUID): + raise ValueError(f"Expected kid to be a {UUID}, str, or bytes, not {kid!r}") + if kid not in kids: + kids.append(kid) + + self._pssh = pssh + self._kids = kids + + if not self.kids: + raise PlayReady.Exceptions.KIDNotFound("No Key ID was found within PSSH and none were provided.") + + self.content_keys: dict[UUID, str] = {} + self.data: dict = kwargs or {} + if pssh_b64: + self.data.setdefault("pssh_b64", pssh_b64) + + def _extract_kids_from_pssh_b64(self, pssh_b64: str) -> list[UUID]: + """Extract all KIDs from base64-encoded PSSH data.""" + try: + import xml.etree.ElementTree as ET + + # Decode the PSSH + pssh_bytes = base64.b64decode(pssh_b64) + + # Try to find XML in the PSSH data + # PlayReady PSSH usually has XML embedded in it + pssh_str = pssh_bytes.decode("utf-16le", errors="ignore") + + # Find WRMHEADER + xml_start = pssh_str.find("") + len("") + clean_xml = clean_xml[:xml_end] + + root = ET.fromstring(clean_xml) + ns = {"pr": "http://schemas.microsoft.com/DRM/2007/03/PlayReadyHeader"} + + kids = [] + + # Extract from CUSTOMATTRIBUTES/KIDS + kid_elements = root.findall(".//pr:CUSTOMATTRIBUTES/pr:KIDS/pr:KID", ns) + for kid_elem in kid_elements: + value = kid_elem.get("VALUE") + if value: + try: + kid_bytes = base64.b64decode(value + "==") + kid_uuid = UUID(bytes_le=kid_bytes) + kids.append(kid_uuid) + except Exception: + pass + + # v4.2/v4.3: DATA/PROTECTINFO/KIDS/KID + protectinfo_kids = root.findall(".//pr:DATA/pr:PROTECTINFO/pr:KIDS/pr:KID", ns) + for kid_elem in protectinfo_kids: + value = kid_elem.get("VALUE") + if value: + try: + kid_bytes = base64.b64decode(value + "==") + kid_uuid = UUID(bytes_le=kid_bytes) + if kid_uuid not in kids: + kids.append(kid_uuid) + except Exception: + pass + + # v4.1: DATA/PROTECTINFO/KID + protectinfo_kid = root.findall(".//pr:DATA/pr:PROTECTINFO/pr:KID", ns) + for kid_elem in protectinfo_kid: + value = kid_elem.get("VALUE") + if value: + try: + kid_bytes = base64.b64decode(value + "==") + kid_uuid = UUID(bytes_le=kid_bytes) + if kid_uuid not in kids: + kids.append(kid_uuid) + except Exception: + pass + + # v4.0: DATA/KID + individual_kids = root.findall(".//pr:DATA/pr:KID", ns) + for kid_elem in individual_kids: + if kid_elem.text: + try: + kid_bytes = base64.b64decode(kid_elem.text.strip() + "==") + kid_uuid = UUID(bytes_le=kid_bytes) + if kid_uuid not in kids: + kids.append(kid_uuid) + except Exception: + pass + + return kids + + except Exception: + pass + + return [] + + @classmethod + def from_track(cls, track: AnyTrack, session: Optional[Session] = None) -> PlayReady: + if not session: + session = Session() + session.headers.update(config.headers) + + kid: Optional[UUID] = None + pssh_boxes: list[Container] = [] + tenc_boxes: list[Container] = [] + + if track.descriptor == track.Descriptor.HLS: + m3u_url = track.url + master = m3u8.loads(session.get(m3u_url).text, uri=m3u_url) + pssh_boxes.extend( + Box.parse(base64.b64decode(x.uri.split(",")[-1])) + for x in (master.session_keys or master.keys) + if x + and x.keyformat + and x.keyformat.lower() in {f"urn:uuid:{PSSH.SYSTEM_ID}", "com.microsoft.playready"} + ) + + init_data = track.get_init_segment(session=session) + if init_data: + probe = ffprobe(init_data) + if probe: + for stream in probe.get("streams") or []: + enc_key_id = stream.get("tags", {}).get("enc_key_id") + if enc_key_id: + kid = UUID(bytes=base64.b64decode(enc_key_id)) + pssh_boxes.extend(list(get_boxes(init_data, b"pssh"))) + tenc_boxes.extend(list(get_boxes(init_data, b"tenc"))) + + pssh = next((b for b in pssh_boxes if b.system_ID == PSSH.SYSTEM_ID), None) + if not pssh: + raise PlayReady.Exceptions.PSSHNotFound("PSSH was not found in track data.") + + tenc = next(iter(tenc_boxes), None) + if not kid and tenc and tenc.key_ID.int != 0: + kid = tenc.key_ID + + pssh_bytes = Box.build(pssh) + return cls(pssh=PSSH(pssh_bytes), kid=kid, pssh_b64=base64.b64encode(pssh_bytes).decode()) + + @classmethod + def from_init_data(cls, init_data: bytes) -> PlayReady: + if not init_data: + raise ValueError("Init data should be provided.") + if not isinstance(init_data, bytes): + raise TypeError(f"Expected init data to be bytes, not {init_data!r}") + + kid: Optional[UUID] = None + pssh_boxes: list[Container] = list(get_boxes(init_data, b"pssh")) + tenc_boxes: list[Container] = list(get_boxes(init_data, b"tenc")) + + probe = ffprobe(init_data) + if probe: + for stream in probe.get("streams") or []: + enc_key_id = stream.get("tags", {}).get("enc_key_id") + if enc_key_id: + kid = UUID(bytes=base64.b64decode(enc_key_id)) + + pssh = next((b for b in pssh_boxes if b.system_ID == PSSH.SYSTEM_ID), None) + if not pssh: + raise PlayReady.Exceptions.PSSHNotFound("PSSH was not found in track data.") + + tenc = next(iter(tenc_boxes), None) + if not kid and tenc and tenc.key_ID.int != 0: + kid = tenc.key_ID + + pssh_bytes = Box.build(pssh) + return cls(pssh=PSSH(pssh_bytes), kid=kid, pssh_b64=base64.b64encode(pssh_bytes).decode()) + + @property + def pssh(self) -> PSSH: + return self._pssh + + @property + def pssh_b64(self) -> Optional[str]: + return self.data.get("pssh_b64") + + @property + def kid(self) -> Optional[UUID]: + return next(iter(self.kids), None) + + def to_dict(self) -> dict[str, Any]: + """Serialise this DRM instance for export/import (PSSH + KIDs). + + Content keys are stored once at the export's track level, not duplicated here. + """ + return { + "system": "PlayReady", + "pssh_b64": self.pssh_b64, + "kids": [kid.hex for kid in self.kids], + } + + @property + def kids(self) -> list[UUID]: + return self._kids + + def _extract_keys_from_cdm(self, cdm: PlayReadyCdm, session_id: bytes) -> dict: + """Extract keys from CDM session with cross-library compatibility. + + Args: + cdm: CDM instance + session_id: Session identifier + + Returns: + Dictionary mapping KID UUIDs to hex keys + """ + keys = {} + for key in cdm.get_keys(session_id): + if hasattr(key, "key_id"): + kid = key.key_id + elif hasattr(key, "kid"): + kid = key.kid + else: + continue + + if hasattr(key, "key") and hasattr(key.key, "hex"): + key_hex = key.key.hex() + elif hasattr(key, "key") and isinstance(key.key, bytes): + key_hex = key.key.hex() + elif hasattr(key, "key") and isinstance(key.key, str): + key_hex = key.key + else: + continue + + keys[kid] = key_hex + return keys + + def get_content_keys(self, cdm: PlayReadyCdm, certificate: Callable, licence: Callable) -> None: + session_id = cdm.open() + try: + if hasattr(cdm, "set_pssh_b64") and self.pssh_b64: + cdm.set_pssh_b64(self.pssh_b64) + + if hasattr(cdm, "set_required_kids"): + cdm.set_required_kids(self.kids) + + challenge = cdm.get_license_challenge(session_id, self.pssh.wrm_headers[0]) + + if challenge: + try: + try: + license_res = licence(challenge=challenge, pssh_b64=self.pssh_b64) + except TypeError: + license_res = licence(challenge=challenge) + if isinstance(license_res, bytes): + license_str = license_res.decode(errors="ignore") + else: + license_str = str(license_res) + + if "" not in license_str: + try: + license_str = base64.b64decode(license_str + "===").decode() + except Exception: + pass + + cdm.parse_license(session_id, license_str) + except Exception: + raise + + keys = self._extract_keys_from_cdm(cdm, session_id) + self.content_keys.update(keys) + finally: + cdm.close(session_id) + + if not self.content_keys: + raise PlayReady.Exceptions.EmptyLicense("No Content Keys were within the License") + + def decrypt(self, path: Path) -> None: + """ + Decrypt a Track with PlayReady DRM. + Args: + path: Path to the encrypted file to decrypt + Raises: + EnvironmentError if the required decryption executable could not be found. + ValueError if the track has not yet been downloaded. + SubprocessError if the decryption process returned a non-zero exit code. + """ + if not self.content_keys: + raise ValueError("Cannot decrypt a Track without any Content Keys...") + + if not path or not path.exists(): + raise ValueError("Tried to decrypt a file that does not exist.") + + decrypter = str(getattr(config, "decryption", "")).lower() + + if decrypter == "mp4decrypt": + return self._decrypt_with_mp4decrypt(path) + else: + return self._decrypt_with_shaka_packager(path) + + def _decrypt_with_mp4decrypt(self, path: Path) -> None: + """Decrypt using mp4decrypt""" + if not binaries.Mp4decrypt: + raise EnvironmentError("mp4decrypt executable not found but is required.") + + output_path = path.with_stem(f"{path.stem}_decrypted") + + # Build key arguments + key_args = [] + for kid, key in self.content_keys.items(): + kid_hex = kid.hex if hasattr(kid, "hex") else str(kid).replace("-", "") + key_hex = key if isinstance(key, str) else key.hex() + key_args.extend(["--key", f"{kid_hex}:{key_hex}"]) + + # Fallback for tracks whose tenc default_KID is all-zero and whose real + # KID is signalled out-of-band: emit a zero-KID entry per content key. + zero_kid = "00" * 16 + existing_kids = {kid.hex if hasattr(kid, "hex") else str(kid).replace("-", "") for kid in self.content_keys} + if zero_kid not in existing_kids: + for key in self.content_keys.values(): + key_hex = key if isinstance(key, str) else key.hex() + key_args.extend(["--key", f"{zero_kid}:{key_hex}"]) + + cmd = [ + str(binaries.Mp4decrypt), + "--show-progress", + *key_args, + str(path), + str(output_path), + ] + + try: + subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8") + except subprocess.CalledProcessError as e: + error_msg = e.stderr if e.stderr else f"mp4decrypt failed with exit code {e.returncode}" + raise subprocess.CalledProcessError(e.returncode, cmd, output=e.stdout, stderr=error_msg) + + if not output_path.exists(): + raise RuntimeError(f"mp4decrypt failed: output file {output_path} was not created") + if output_path.stat().st_size == 0: + raise RuntimeError(f"mp4decrypt failed: output file {output_path} is empty") + + path.unlink() + shutil.move(output_path, path) + + def _decrypt_with_shaka_packager(self, path: Path) -> None: + """Decrypt using Shaka Packager (original method)""" + if not binaries.ShakaPackager: + raise EnvironmentError("Shaka Packager executable not found but is required.") + + output_path = path.with_stem(f"{path.stem}_decrypted") + config.directories.temp.mkdir(parents=True, exist_ok=True) + + try: + arguments = [ + f"input={path},stream=0,output={output_path},output_format=MP4", + "--enable_raw_key_decryption", + "--keys", + ",".join( + [ + *[ + f"label={i}:key_id={kid.hex}:key={key.lower()}" + for i, (kid, key) in enumerate(self.content_keys.items()) + ], + *[ + f"label={i}:key_id={'00' * 16}:key={key.lower()}" + for i, (kid, key) in enumerate(self.content_keys.items(), len(self.content_keys)) + ], + ] + ), + "--temp_dir", + config.directories.temp, + ] + + p = subprocess.Popen( + [binaries.ShakaPackager, *arguments], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + ) + + stream_skipped = False + had_error = False + shaka_log_buffer = "" + for line in iter(p.stderr.readline, ""): + line = line.strip() + if not line: + continue + if "Skip stream" in line: + stream_skipped = True + if ":INFO:" in line: + continue + if "I0" in line or "W0" in line: + continue + if ":ERROR:" in line: + had_error = True + if "Insufficient bits in bitstream for given AVC profile" in line: + continue + shaka_log_buffer += f"{line.strip()}\n" + + if shaka_log_buffer: + shaka_log_buffer = "\n ".join( + textwrap.wrap(shaka_log_buffer.rstrip(), width=console.width - 22, initial_indent="") + ) + console.log(Text.from_ansi("\n[PlayReady]: " + shaka_log_buffer)) + + p.wait() + + if p.returncode != 0 or had_error: + raise subprocess.CalledProcessError(p.returncode, [binaries.ShakaPackager, *arguments]) + + path.unlink() + if not stream_skipped: + shutil.move(output_path, path) + except subprocess.CalledProcessError as e: + if e.returncode == 0xC000013A: + raise KeyboardInterrupt() + raise + + class Exceptions: + class PSSHNotFound(Exception): + pass + + class KIDNotFound(Exception): + pass + + class CEKNotFound(Exception): + pass + + class EmptyLicense(Exception): + pass + + +__all__ = ("PlayReady",) diff --git a/reset/packages/envied/src/envied/core/drm/widevine.py b/reset/packages/envied/src/envied/core/drm/widevine.py new file mode 100644 index 0000000..f4d7a82 --- /dev/null +++ b/reset/packages/envied/src/envied/core/drm/widevine.py @@ -0,0 +1,434 @@ +from __future__ import annotations + +import base64 +import shutil +import subprocess +import textwrap +from pathlib import Path +from typing import Any, Callable, Optional, Union +from uuid import UUID + +import m3u8 +from construct import Container +from pymp4.parser import Box +from pywidevine.cdm import Cdm as WidevineCdm +from pywidevine.pssh import PSSH +from requests import Session +from rich.text import Text + +from envied.core import binaries +from envied.core.config import config +from envied.core.console import console +from envied.core.constants import AnyTrack +from envied.core.utilities import get_boxes +from envied.core.utils.subprocess import ffprobe + + +class Widevine: + """Widevine DRM System.""" + + PLACEHOLDER_KIDS = { + UUID("00000000-0000-0000-0000-000000000000"), # All zeros (key rotation default) + UUID("00010203-0405-0607-0809-0a0b0c0d0e0f"), # Sequential 0x00-0x0f + UUID("00010203-0405-0607-0809-101112131415"), # Shaka Packager test pattern + } + + def __init__(self, pssh: PSSH, kid: Union[UUID, str, bytes, None] = None, **kwargs: Any): + if not pssh: + raise ValueError("Provided PSSH is empty.") + if not isinstance(pssh, PSSH): + raise TypeError(f"Expected pssh to be a {PSSH}, not {pssh!r}") + + if pssh.system_id == PSSH.SystemId.PlayReady: + pssh.to_widevine() + + self._kid: Optional[UUID] = None + if kid: + if isinstance(kid, str): + kid = UUID(hex=kid) + elif isinstance(kid, bytes): + kid = UUID(bytes=kid) + if not isinstance(kid, UUID): + raise ValueError(f"Expected kid to be a {UUID}, str, or bytes, not {kid!r}") + self._kid = kid + if pssh.key_ids and all(k in self.PLACEHOLDER_KIDS for k in pssh.key_ids): + pssh.set_key_ids([kid]) + elif kid not in (pssh.key_ids or []): + pssh.set_key_ids([*(pssh.key_ids or []), kid]) + + self._pssh = pssh + + if not self.kids: + raise Widevine.Exceptions.KIDNotFound("No Key ID was found within PSSH and none were provided.") + + self.content_keys: dict[UUID, str] = {} + self.data: dict = kwargs or {} + + @classmethod + def from_track(cls, track: AnyTrack, session: Optional[Session] = None) -> Widevine: + """ + Get PSSH and KID from within the Initiation Segment of the Track Data. + It also tries to get PSSH and KID from other track data like M3U8 data + as well as through ffprobe. + + Create a Widevine DRM System object from a track's information. + This should only be used if a PSSH could not be provided directly. + It is *rare* to need to use this. + + You may provide your own requests session to be able to use custom + headers and more. + + Raises: + PSSHNotFound - If the PSSH was not found within the data. + KIDNotFound - If the KID was not found within the data or PSSH. + """ + if not session: + session = Session() + session.headers.update(config.headers) + + kid: Optional[UUID] = None + pssh_boxes: list[Container] = [] + tenc_boxes: list[Container] = [] + + if track.descriptor == track.Descriptor.HLS: + m3u_url = track.url + master = m3u8.loads(session.get(m3u_url).text, uri=m3u_url) + pssh_boxes.extend( + Box.parse(base64.b64decode(x.uri.split(",")[-1])) + for x in (master.session_keys or master.keys) + if x and x.keyformat and x.keyformat.lower() == WidevineCdm.urn + ) + + init_data = track.get_init_segment(session=session) + if init_data: + # try get via ffprobe, needed for non mp4 data e.g. WEBM from Google Play + probe = ffprobe(init_data) + if probe: + for stream in probe.get("streams") or []: + enc_key_id = stream.get("tags", {}).get("enc_key_id") + if enc_key_id: + kid = UUID(bytes=base64.b64decode(enc_key_id)) + pssh_boxes.extend(list(get_boxes(init_data, b"pssh"))) + tenc_boxes.extend(list(get_boxes(init_data, b"tenc"))) + + pssh = next((b for b in pssh_boxes if b.system_ID == PSSH.SystemId.Widevine), None) + if not pssh: + raise Widevine.Exceptions.PSSHNotFound("PSSH was not found in track data.") + + tenc = next(iter(tenc_boxes), None) + if not kid and tenc and tenc.key_ID.int != 0: + kid = tenc.key_ID + + return cls(pssh=PSSH(pssh), kid=kid) + + @classmethod + def from_init_data(cls, init_data: bytes) -> Widevine: + """ + Get PSSH and KID from within Initialization Segment Data. + + This should only be used if a PSSH could not be provided directly. + It is *rare* to need to use this. + + Raises: + PSSHNotFound - If the PSSH was not found within the data. + KIDNotFound - If the KID was not found within the data or PSSH. + """ + if not init_data: + raise ValueError("Init data should be provided.") + if not isinstance(init_data, bytes): + raise TypeError(f"Expected init data to be bytes, not {init_data!r}") + + kid: Optional[UUID] = None + pssh_boxes: list[Container] = list(get_boxes(init_data, b"pssh")) + tenc_boxes: list[Container] = list(get_boxes(init_data, b"tenc")) + + # try get via ffprobe, needed for non mp4 data e.g. WEBM from Google Play + probe = ffprobe(init_data) + if probe: + for stream in probe.get("streams") or []: + enc_key_id = stream.get("tags", {}).get("enc_key_id") + if enc_key_id: + kid = UUID(bytes=base64.b64decode(enc_key_id)) + + pssh = next((b for b in pssh_boxes if b.system_ID == PSSH.SystemId.Widevine), None) + if not pssh: + raise Widevine.Exceptions.PSSHNotFound("PSSH was not found in track data.") + + tenc = next(iter(tenc_boxes), None) + if not kid and tenc and tenc.key_ID.int != 0: + kid = tenc.key_ID + + return cls(pssh=PSSH(pssh), kid=kid) + + @property + def pssh(self) -> PSSH: + """Get Protection System Specific Header Box.""" + return self._pssh + + @property + def kid(self) -> Optional[UUID]: + """Get first Key ID, if any.""" + return next(iter(self.kids), None) + + @property + def kids(self) -> list[UUID]: + """Get all Key IDs from PSSH, falling back to the externally provided KID.""" + pssh_kids = self._pssh.key_ids + if pssh_kids: + return pssh_kids + if self._kid: + return [self._kid] + return [] + + def to_dict(self) -> dict[str, Any]: + """Serialise this DRM instance for export/import (PSSH + KIDs). + + Content keys are stored once at the export's track level, not duplicated here. + """ + return { + "system": "Widevine", + "pssh_b64": self.pssh.dumps(), + "kids": [kid.hex for kid in self.kids], + } + + def get_content_keys(self, cdm: WidevineCdm, certificate: Callable, licence: Callable) -> None: + """ + Create a CDM Session and obtain Content Keys for this DRM Instance. + The certificate and license params are expected to be a function and will + be provided with the challenge and session ID. + """ + for kid in self.kids: + if kid in self.content_keys: + continue + + session_id = cdm.open() + + try: + cert = certificate(challenge=cdm.service_certificate_challenge) + if cert and hasattr(cdm, "set_service_certificate"): + cdm.set_service_certificate(session_id, cert) + + if hasattr(cdm, "set_required_kids"): + cdm.set_required_kids(self.kids) + + challenge = cdm.get_license_challenge(session_id, self.pssh) + + if hasattr(cdm, "has_cached_keys") and cdm.has_cached_keys(session_id): + pass + else: + try: + license_res = licence(challenge=challenge, pssh=self.pssh) + except TypeError: + license_res = licence(challenge=challenge) + cdm.parse_license(session_id, license_res) + + self.content_keys = {key.kid: key.key.hex() for key in cdm.get_keys(session_id, "CONTENT")} + if not self.content_keys: + raise Widevine.Exceptions.EmptyLicense("No Content Keys were within the License") + + if kid not in self.content_keys: + raise Widevine.Exceptions.CEKNotFound(f"No Content Key for KID {kid.hex} within the License") + finally: + cdm.close(session_id) + + def get_NF_content_keys(self, cdm: WidevineCdm, certificate: Callable, licence: Callable) -> None: + """ + Create a CDM Session and obtain Content Keys for this DRM Instance. + The certificate and license params are expected to be a function and will + be provided with the challenge and session ID. + """ + for kid in self.kids: + if kid in self.content_keys: + continue + + session_id = cdm.open() + + try: + cert = certificate(challenge=cdm.service_certificate_challenge) + if cert and hasattr(cdm, "set_service_certificate"): + cdm.set_service_certificate(session_id, cert) + + if hasattr(cdm, "set_required_kids"): + cdm.set_required_kids(self.kids) + + challenge = cdm.get_license_challenge(session_id, self.pssh) + + if hasattr(cdm, "has_cached_keys") and cdm.has_cached_keys(session_id): + pass + else: + cdm.parse_license( + session_id, + licence(session_id=session_id, challenge=challenge), + ) + + self.content_keys = {key.kid: key.key.hex() for key in cdm.get_keys(session_id, "CONTENT")} + if not self.content_keys: + raise Widevine.Exceptions.EmptyLicense("No Content Keys were within the License") + + if kid not in self.content_keys: + raise Widevine.Exceptions.CEKNotFound(f"No Content Key for KID {kid.hex} within the License") + finally: + cdm.close(session_id) + + def decrypt(self, path: Path) -> None: + """ + Decrypt a Track with Widevine DRM. + Args: + path: Path to the encrypted file to decrypt + Raises: + EnvironmentError if the required decryption executable could not be found. + ValueError if the track has not yet been downloaded. + SubprocessError if the decryption process returned a non-zero exit code. + """ + if not self.content_keys: + raise ValueError("Cannot decrypt a Track without any Content Keys...") + + if not path or not path.exists(): + raise ValueError("Tried to decrypt a file that does not exist.") + + decrypter = str(getattr(config, "decryption", "")).lower() + + if decrypter == "mp4decrypt": + return self._decrypt_with_mp4decrypt(path) + else: + return self._decrypt_with_shaka_packager(path) + + def _decrypt_with_mp4decrypt(self, path: Path) -> None: + """Decrypt using mp4decrypt""" + if not binaries.Mp4decrypt: + raise EnvironmentError("mp4decrypt executable not found but is required.") + + output_path = path.with_stem(f"{path.stem}_decrypted") + + # Build key arguments + key_args = [] + for kid, key in self.content_keys.items(): + kid_hex = kid.hex if hasattr(kid, "hex") else str(kid).replace("-", "") + key_hex = key if isinstance(key, str) else key.hex() + key_args.extend(["--key", f"{kid_hex}:{key_hex}"]) + + # Fallback for tracks whose tenc default_KID is all-zero and whose real + # KID is signalled out-of-band: emit a zero-KID entry per content key. + zero_kid = "00" * 16 + existing_kids = {kid.hex if hasattr(kid, "hex") else str(kid).replace("-", "") for kid in self.content_keys} + if zero_kid not in existing_kids: + for key in self.content_keys.values(): + key_hex = key if isinstance(key, str) else key.hex() + key_args.extend(["--key", f"{zero_kid}:{key_hex}"]) + + cmd = [ + str(binaries.Mp4decrypt), + "--show-progress", + *key_args, + str(path), + str(output_path), + ] + + try: + subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8") + except subprocess.CalledProcessError as e: + error_msg = e.stderr if e.stderr else f"mp4decrypt failed with exit code {e.returncode}" + raise subprocess.CalledProcessError(e.returncode, cmd, output=e.stdout, stderr=error_msg) + + if not output_path.exists(): + raise RuntimeError(f"mp4decrypt failed: output file {output_path} was not created") + if output_path.stat().st_size == 0: + raise RuntimeError(f"mp4decrypt failed: output file {output_path} is empty") + + path.unlink() + shutil.move(output_path, path) + + def _decrypt_with_shaka_packager(self, path: Path) -> None: + """Decrypt using Shaka Packager (original method)""" + if not binaries.ShakaPackager: + raise EnvironmentError("Shaka Packager executable not found but is required.") + + output_path = path.with_stem(f"{path.stem}_decrypted") + config.directories.temp.mkdir(parents=True, exist_ok=True) + + try: + arguments = [ + f"input={path},stream=0,output={output_path},output_format=MP4", + "--enable_raw_key_decryption", + "--keys", + ",".join( + [ + *[ + "label={}:key_id={}:key={}".format(i, kid.hex, key.lower()) + for i, (kid, key) in enumerate(self.content_keys.items()) + ], + *[ + # some services use a blank KID on the file, but real KID for license server + "label={}:key_id={}:key={}".format(i, "00" * 16, key.lower()) + for i, (kid, key) in enumerate(self.content_keys.items(), len(self.content_keys)) + ], + ] + ), + "--temp_dir", + config.directories.temp, + ] + + p = subprocess.Popen( + [binaries.ShakaPackager, *arguments], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + + stream_skipped = False + had_error = False + + shaka_log_buffer = "" + for line in iter(p.stderr.readline, ""): + line = line.strip() + if not line: + continue + if "Skip stream" in line: + # file/segment was so small that it didn't have any actual data, ignore + stream_skipped = True + if ":INFO:" in line: + continue + if "I0" in line or "W0" in line: + continue + if ":ERROR:" in line: + had_error = True + if "Insufficient bits in bitstream for given AVC profile" in line: + # this is a warning and is something we don't have to worry about + continue + shaka_log_buffer += f"{line.strip()}\n" + + if shaka_log_buffer: + # wrap to console width - padding - '[Widevine]: ' + shaka_log_buffer = "\n ".join( + textwrap.wrap(shaka_log_buffer.rstrip(), width=console.width - 22, initial_indent="") + ) + console.log(Text.from_ansi("\n[Widevine]: " + shaka_log_buffer)) + + p.wait() + + if p.returncode != 0 or had_error: + raise subprocess.CalledProcessError(p.returncode, [binaries.ShakaPackager, *arguments]) + + path.unlink() + if not stream_skipped: + shutil.move(output_path, path) + except subprocess.CalledProcessError as e: + if e.returncode == 0xC000013A: # STATUS_CONTROL_C_EXIT + raise KeyboardInterrupt() + raise + + class Exceptions: + class PSSHNotFound(Exception): + """PSSH (Protection System Specific Header) was not found.""" + + class KIDNotFound(Exception): + """KID (Encryption Key ID) was not found.""" + + class CEKNotFound(Exception): + """CEK (Content Encryption Key) for KID was not found in License.""" + + class EmptyLicense(Exception): + """License returned no Content Encryption Keys.""" + + +__all__ = ("Widevine",) diff --git a/reset/packages/envied/src/envied/core/events.py b/reset/packages/envied/src/envied/core/events.py new file mode 100644 index 0000000..79316ea --- /dev/null +++ b/reset/packages/envied/src/envied/core/events.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from copy import deepcopy +from enum import Enum +from typing import Any, Callable + + +class Events: + class Types(Enum): + _reserved = 0 + # A Track's segment has finished downloading + SEGMENT_DOWNLOADED = 1 + # Track has finished downloading + TRACK_DOWNLOADED = 2 + # Track has finished decrypting + TRACK_DECRYPTED = 3 + # Track has finished repacking + TRACK_REPACKED = 4 + # Track is about to be Multiplexed into a Container + TRACK_MULTIPLEX = 5 + + def __init__(self): + self.__subscriptions: dict[Events.Types, list[Callable]] = {} + self.__ephemeral: dict[Events.Types, list[Callable]] = {} + self.reset() + + def reset(self): + """Reset Event Observer clearing all Subscriptions.""" + self.__subscriptions = {k: [] for k in Events.Types.__members__.values()} + self.__ephemeral = deepcopy(self.__subscriptions) + + def subscribe(self, event_type: Events.Types, callback: Callable, ephemeral: bool = False) -> None: + """ + Subscribe to an Event with a Callback. + + Parameters: + event_type: The Events.Type to subscribe to. + callback: The function or lambda to call on event emit. + ephemeral: Unsubscribe the callback from the event on first emit. + Note that this is not thread-safe and may be called multiple + times at roughly the same time. + """ + [self.__subscriptions, self.__ephemeral][ephemeral][event_type].append(callback) + + def unsubscribe(self, event_type: Events.Types, callback: Callable) -> None: + """ + Unsubscribe a Callback from an Event. + + Parameters: + event_type: The Events.Type to unsubscribe from. + callback: The function or lambda to remove from event emit. + """ + if callback in self.__subscriptions[event_type]: + self.__subscriptions[event_type].remove(callback) + if callback in self.__ephemeral[event_type]: + self.__ephemeral[event_type].remove(callback) + + def emit(self, event_type: Events.Types, *args: Any, **kwargs: Any) -> None: + """ + Emit an Event, executing all subscribed Callbacks. + + Parameters: + event_type: The Events.Type to emit. + args: Positional arguments to pass to callbacks. + kwargs: Keyword arguments to pass to callbacks. + """ + if event_type not in self.__subscriptions: + raise ValueError(f'Event type "{event_type}" is invalid') + + for callback in self.__subscriptions[event_type] + self.__ephemeral[event_type]: + callback(*args, **kwargs) + + self.__ephemeral[event_type].clear() + + +events = Events() diff --git a/reset/packages/envied/src/envied/core/import_service.py b/reset/packages/envied/src/envied/core/import_service.py new file mode 100644 index 0000000..5506269 --- /dev/null +++ b/reset/packages/envied/src/envied/core/import_service.py @@ -0,0 +1,304 @@ +from __future__ import annotations + +import json +import logging +from http.cookiejar import CookieJar +from pathlib import Path +from typing import Any, Optional, Union +from uuid import UUID + +import click +import requests + +from envied.core.config import config +from envied.core.constants import AnyTrack +from envied.core.credential import Credential +from envied.core.drm import drm_from_dict +from envied.core.manifests import DASH, HLS, ISM +from envied.core.remote_service import RemoteService, _build_title, _resolve_proxy +from envied.core.titles import Episode, Movies, Series, Title_T, Titles_T, remap_titles +from envied.core.tracks import Audio, Chapter, Chapters, Tracks, Video +from envied.core.tracks.attachment import Attachment +from envied.core.tracks.track import Track + +log = logging.getLogger("import") + +PARSERS = {"DASH": DASH, "HLS": HLS, "ISM": ISM} + + +class ImportService: + """Reconstructs a download from an export JSON. + + Auth and licensing are skipped; tracks are rebuilt from the export and keys injected + directly. ``_server_cdm``/``_server_cdm_type`` keep their underscores: dl.py reads them + via getattr as the server-CDM contract that skips client licensing. + """ + + ALIASES: tuple[str, ...] = () + GEOFENCE: tuple[str, ...] = () + NO_SUBTITLES: bool = False + + def __init__(self, ctx: click.Context, service_tag: str, title: str, import_file: Optional[str]) -> None: + self.__class__.__name__ = service_tag + self.service_tag = service_tag + self.title_id = title + self.ctx = ctx + self.log = logging.getLogger(service_tag) + self.credential: Optional[Credential] = None + self.current_region: Optional[str] = None + self.title_cache = None + + if not import_file: + raise click.ClickException("No export file was provided to import from.") + export_path = Path(import_file) + if not export_path.is_file(): + raise click.ClickException(f"Export file not found: {export_path}") + + self.data: dict[str, Any] = json.loads(export_path.read_text(encoding="utf8")) + version = self.data.get("version") + if version != 2: + raise click.ClickException( + f"Unsupported export version {version!r}. Re-create the export with a current build." + ) + + self.titles_data: dict[str, Any] = self.data.get("titles", {}) + self.region: Optional[str] = self.data.get("region") + self.titles: Optional[Titles_T] = None + self.tracks_by_title: dict[str, Tracks] = {} + + self._server_cdm = True + self._server_cdm_type = "widevine" + + self.session = self.build_session(ctx, self.region) + + @staticmethod + def build_session(ctx: click.Context, region: Optional[str] = None) -> requests.Session: + """Session for re-fetching the manifest. + + Honours the importer's ``--proxy``; otherwise falls back to the export region as a + geofence. An explicit proxy that fails to resolve raises; a region fallback warns. + """ + session = requests.Session() + session.headers.update(config.headers) + + params = ctx.parent.params if ctx.parent else {} + if params.get("no_proxy"): + return session + + explicit = params.get("proxy") + proxy_query = explicit or region + if not proxy_query: + return session + + try: + proxy = _resolve_proxy(proxy_query) + except Exception as e: + if explicit: + raise click.ClickException(f"Failed to resolve proxy '{proxy_query}': {e}") + log.warning(f"Could not auto-select a proxy for export region '{region}': {e}. Continuing without proxy.") + proxy = None + + if proxy: + session.proxies.update({"all": proxy}) + if not explicit: + log.info(f"No --proxy given; using export region '{region}' via your proxy provider.") + return session + + @property + def title(self) -> str: + return self.title_id + + def authenticate(self, cookies: Optional[CookieJar] = None, credential: Optional[Credential] = None) -> None: + self.credential = credential + + def get_titles(self) -> Titles_T: + if self.titles is not None: + return self.titles + titles_list = [ + _build_title(entry.get("meta", {}), self.service_tag, fallback_id=title_id) + for title_id, entry in self.titles_data.items() + ] + self.titles = ( + Series(titles_list) if titles_list and isinstance(titles_list[0], Episode) else Movies(titles_list) + ) + return self.titles + + def get_titles_cached(self, title_id: Optional[str] = None) -> Titles_T: + """Apply the service's title_map to titles reconstructed from the export sidecar.""" + title_map = (config.services.get(self.service_tag) or {}).get("title_map") or {} + return remap_titles(self.get_titles(), title_map) + + def get_tracks(self, title: Title_T) -> Tracks: + """Reconstruct the title's tracks from the export. + + DASH/ISM: re-fetch and re-parse the manifest and return the full ladder (the importer + picks quality with normal dl flags; keys are injected by KID later). HLS/URL: rebuild + from the stored per-track dicts, since the variant is re-fetched from track.url at + download time and ATV-style master playlists carry unstable per-fetch tokens. + """ + title_id = str(title.id) + if title_id in self.tracks_by_title: + return self.tracks_by_title[title_id] + + entry = self.titles_data.get(title_id, {}) + tracks_map: dict[str, Any] = entry.get("tracks") or {} + manifest_url = entry.get("manifest_url") + manifest_type = entry.get("manifest_type") + + tracks = Tracks() + tracks.manifest_url = manifest_url + + parser = PARSERS.get(manifest_type or "") + if manifest_url and parser is not None and manifest_type in ("DASH", "ISM"): + try: + parsed = parser.from_url(url=manifest_url, session=self.session).to_tracks(language=title.language) + except Exception as e: + raise click.ClickException( + f"Failed to re-fetch/parse the {manifest_type} manifest for '{title}'. " + f"The manifest URL may have expired since export. ({e})" + ) + for track in parsed: + tracks.add(track) + else: + for track_dict in tracks_map.values(): + track = Track.from_dict(track_dict) + drm = self.rebuild_drm(track_dict) + if drm: + track.drm = drm + tracks.add(track) + + for attachment in entry.get("attachments") or []: + url = attachment.get("url") + if not url: + continue + try: + tracks.attachments.append( + Attachment.from_url( + url, + name=attachment.get("name"), + mime_type=attachment.get("mime_type"), + description=attachment.get("description"), + session=self.session, + ) + ) + except Exception as e: + self.log.warning(f"Skipping attachment '{attachment.get('name')}': {e}") + + self.tracks_by_title[title_id] = tracks + return tracks + + def key_pool(self) -> dict[UUID, str]: + """All exported KID:KEY pairs across every title, as {UUID: key_hex}.""" + pool: dict[UUID, str] = {} + for entry in self.titles_data.values(): + for track_dict in (entry.get("tracks") or {}).values(): + for kid_hex, key in (track_dict.get("keys") or {}).items(): + pool[UUID(hex=kid_hex)] = key + return pool + + def rebuild_drm(self, track_dict: dict[str, Any]) -> Optional[list[Any]]: + """Rebuild a DRM object (from stored PSSH, falling back to a stub) with the exported keys.""" + keys = track_dict.get("keys") or {} + drm_dicts = track_dict.get("drm") or [] + if not drm_dicts and not keys: + return None + + drm_obj = None + if drm_dicts: + try: + drm_obj = drm_from_dict(drm_dicts[0]) + except Exception as e: + self.log.debug(f"Falling back to DRM stub (PSSH rebuild failed: {e})") + + if drm_obj is None and keys: + drm_type = (drm_dicts[0].get("system", "Widevine").lower()) if drm_dicts else "widevine" + drm_obj = RemoteService._create_drm_stub(drm_type, list(keys.keys())) + + if drm_obj is None: + return None + + for kid_hex, key in keys.items(): + drm_obj.content_keys[UUID(hex=kid_hex)] = key + return [drm_obj] + + def resolve_server_keys(self, title: Title_T) -> None: + """Inject exported keys into the selected encrypted tracks by KID (no network). + + Called by dl.py after selection. Only encrypted video/audio are touched; encrypted + DASH tracks (no DRM at parse time) get a stub holding the keys, which + DASH.download_track preserves. decrypt() applies the key whose KID matches the media. + """ + pool = self.key_pool() + if not pool: + return + + system = self.exported_drm_system() + kid_hexes = [kid.hex for kid in pool] + + for track in title.tracks: + if not isinstance(track, (Video, Audio)) or not self.track_is_encrypted(track): + continue + drm_obj = track.drm[0] if track.drm else RemoteService._create_drm_stub(system, kid_hexes) + for kid, key in pool.items(): + drm_obj.content_keys[kid] = key + track.drm = [drm_obj] + self._server_cdm_type = drm_obj.__class__.__name__.lower() + + @staticmethod + def track_is_encrypted(track: Any) -> bool: + """True if the track carries DRM or its DASH manifest declares ContentProtection.""" + if track.drm: + return True + dash = track.data.get("dash") if getattr(track, "data", None) else None + if dash: + for element in (dash.get("representation"), dash.get("adaptation_set")): + if element is not None and element.findall("ContentProtection"): + return True + return False + + def exported_drm_system(self) -> str: + """The DRM system the exporter licensed (e.g. 'playready'), defaulting to widevine.""" + for entry in self.titles_data.values(): + for track_dict in (entry.get("tracks") or {}).values(): + for drm_dict in track_dict.get("drm") or []: + if drm_dict.get("system"): + return drm_dict["system"].lower() + return "widevine" + + def get_chapters(self, title: Title_T) -> Chapters: + entry = self.titles_data.get(str(title.id), {}) + return Chapters( + [Chapter(ch["timestamp"], ch.get("name")) for ch in (entry.get("chapters") or []) if ch.get("timestamp")] + ) + + def get_widevine_service_certificate(self, **_: Any) -> Optional[str]: + return None + + def get_widevine_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[Union[bytes, str]]: + raise RuntimeError("ImportService should not request a license; keys come from the export.") + + def get_playready_license( + self, *, challenge: bytes, title: Title_T, track: AnyTrack + ) -> Optional[Union[bytes, str]]: + raise RuntimeError("ImportService should not request a license; keys come from the export.") + + def on_segment_downloaded(self, track: AnyTrack, segment: Any) -> None: + pass + + def on_track_downloaded(self, track: AnyTrack) -> None: + pass + + def on_track_decrypted(self, track: AnyTrack, drm: Any, segment: Any = None) -> None: + pass + + def on_track_repacked(self, track: AnyTrack) -> None: + pass + + def on_track_multiplex(self, track: AnyTrack) -> None: + pass + + def close(self) -> None: + try: + self.session.close() + except Exception: + pass diff --git a/reset/packages/envied/src/envied/core/manifests/__init__.py b/reset/packages/envied/src/envied/core/manifests/__init__.py new file mode 100644 index 0000000..713f2ac --- /dev/null +++ b/reset/packages/envied/src/envied/core/manifests/__init__.py @@ -0,0 +1,5 @@ +from .dash import DASH +from .hls import HLS +from .ism import ISM + +__all__ = ("DASH", "HLS", "ISM") diff --git a/reset/packages/envied/src/envied/core/manifests/dash.py b/reset/packages/envied/src/envied/core/manifests/dash.py new file mode 100644 index 0000000..2fae7a8 --- /dev/null +++ b/reset/packages/envied/src/envied/core/manifests/dash.py @@ -0,0 +1,1057 @@ +from __future__ import annotations + +import base64 +import html +import logging +import math +import re +import shutil +import sys +from copy import deepcopy +from functools import partial +from pathlib import Path +from typing import Any, Callable, Optional, Union +from urllib.parse import urljoin, urlparse +from uuid import UUID +from zlib import crc32 + +import requests +from langcodes import Language, tag_is_valid +from lxml.etree import Element, ElementTree +from pyplayready.system.pssh import PSSH as PR_PSSH +from pywidevine.cdm import Cdm as WidevineCdm +from pywidevine.pssh import PSSH +from requests import Session + +from envied.core.cdm.detect import is_playready_cdm +from envied.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack +from envied.core.drm import DRM_T, PlayReady, Widevine +from envied.core.events import events +from envied.core.session import RnetSession +from envied.core.tracks import Audio, Subtitle, Tracks, Video +from envied.core.utilities import get_debug_logger, is_close_match, try_ensure_utf8 +from envied.core.utils.xml import load_xml + + +class DASH: + def __init__(self, manifest, url: str): + if manifest is None: + raise ValueError("DASH manifest must be provided.") + if manifest.tag != "MPD": + raise TypeError(f"Expected 'MPD' document, but received a '{manifest.tag}' document instead.") + + if not url: + raise requests.URLRequired("DASH manifest URL must be provided for relative path computations.") + if not isinstance(url, str): + raise TypeError(f"Expected url to be a {str}, not {url!r}") + + self.manifest = manifest + self.url = url + + @classmethod + def from_url(cls, url: str, session: Optional[Union[Session, RnetSession]] = None, **args: Any) -> DASH: + if not url: + raise requests.URLRequired("DASH manifest URL must be provided for relative path computations.") + if not isinstance(url, str): + raise TypeError(f"Expected url to be a {str}, not {url!r}") + + if not session: + session = Session() + elif not isinstance(session, (Session, RnetSession)): + raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {session!r}") + + res = session.get(url, **args) + if res.url != url: + url = res.url + + if not res.ok: + raise requests.ConnectionError("Failed to request the MPD document.", response=res) + + return DASH.from_text(res.text, url) + + @classmethod + def from_text(cls, text: str, url: str) -> DASH: + if not text: + raise ValueError("DASH manifest Text must be provided.") + if not isinstance(text, str): + raise TypeError(f"Expected text to be a {str}, not {text!r}") + + if not url: + raise requests.URLRequired("DASH manifest URL must be provided for relative path computations.") + if not isinstance(url, str): + raise TypeError(f"Expected url to be a {str}, not {url!r}") + + manifest = load_xml(text) + + return cls(manifest, url) + + def to_tracks( + self, language: Optional[Union[str, Language]] = None, period_filter: Optional[Callable] = None + ) -> Tracks: + """ + Convert an MPEG-DASH document to Video, Audio and Subtitle Track objects. + + Parameters: + language: The Title's Original Recorded Language. It will also be used as a fallback + track language value if the manifest does not list language information. + period_filter: Filter out period's within the manifest. + + All Track URLs will be a list of segment URLs. + """ + tracks = Tracks() + + filtered_period_ids: list[str] = [] + + for period in self.manifest.findall("Period"): + if callable(period_filter) and period_filter(period): + if period_id := period.get("id"): + filtered_period_ids.append(period_id) + continue + if not DASH._is_content_period(period, []): + if period_id := period.get("id"): + filtered_period_ids.append(period_id) + continue + + for adaptation_set in period.findall("AdaptationSet"): + if self.is_trick_mode(adaptation_set): + # we don't want trick mode streams (they are only used for fast-forward/rewind) + continue + + for rep in adaptation_set.findall("Representation"): + get = partial(self._get, adaptation_set=adaptation_set, representation=rep) + findall = partial(self._findall, adaptation_set=adaptation_set, representation=rep, both=True) + segment_base = rep.find("SegmentBase") + + codecs = get("codecs") + content_type = get("contentType") + mime_type = get("mimeType") + + if not content_type and mime_type: + content_type = mime_type.split("/")[0] + if not content_type and not mime_type: + raise ValueError("Unable to determine the format of a Representation, cannot continue...") + + if mime_type == "application/mp4" or content_type == "application": + # likely mp4-boxed subtitles + # TODO: It may not actually be subtitles + try: + real_codec = Subtitle.Codec.from_mime(codecs) + content_type = "text" + mime_type = f"application/mp4; codecs='{real_codec.value.lower()}'" + except ValueError: + raise ValueError(f"Unsupported content type '{content_type}' with codecs of '{codecs}'") + + if content_type == "text" and mime_type and "/mp4" not in mime_type: + # mimeType likely specifies the subtitle codec better than `codecs` + codecs = mime_type.split("/")[1] + + if content_type == "video": + track_type = Video + track_codec = Video.Codec.from_codecs(codecs) + track_fps = get("frameRate") + if not track_fps and segment_base is not None: + track_fps = segment_base.get("timescale") + + scan_type = None + scan_type_str = get("scanType") + if scan_type_str and scan_type_str.lower() == "interlaced": + scan_type = Video.ScanType.INTERLACED + + track_args = dict( + range_=self.get_video_range( + codecs, findall("SupplementalProperty"), findall("EssentialProperty") + ), + bitrate=get("bandwidth") or None, + width=get("width") or 0, + height=get("height") or 0, + fps=track_fps or None, + scan_type=scan_type, + ) + elif content_type == "audio": + track_type = Audio + track_codec = Audio.Codec.from_codecs(codecs) + track_args = dict( + bitrate=get("bandwidth") or None, + channels=next( + iter( + rep.xpath("AudioChannelConfiguration/@value") + or adaptation_set.xpath("AudioChannelConfiguration/@value") + ), + None, + ), + joc=self.get_ddp_complexity_index(adaptation_set, rep), + descriptive=self.is_descriptive(adaptation_set), + ) + elif content_type == "text": + track_type = Subtitle + track_codec = Subtitle.Codec.from_codecs(codecs or "vtt") + track_args = dict( + cc=self.is_closed_caption(adaptation_set), + sdh=self.is_sdh(adaptation_set), + forced=self.is_forced(adaptation_set), + ) + elif content_type == "image": + # we don't want what's likely thumbnails for the seekbar + continue + else: + raise ValueError(f"Unknown Track Type '{content_type}'") + + track_lang = self.get_language(adaptation_set, rep, fallback=language) + if not track_lang: + msg = "Language information could not be derived from a Representation." + if language is None: + msg += " No fallback language was provided when calling DASH.to_tracks()." + elif not tag_is_valid((str(language) or "").strip()) or str(language).startswith("und"): + msg += f" The fallback language provided is also invalid: {language}" + raise ValueError(msg) + + # for some reason it's incredibly common for services to not provide + # a good and actually unique track ID, sometimes because of the lang + # dialect not being represented in the id, or the bitrate, or such. + # this combines all of them as one and hashes it to keep it small(ish). + track_id = hex( + crc32( + "{codec}-{lang}-{bitrate}-{base_url}-{ids}-{track_args}".format( + codec=codecs, + lang=track_lang, + bitrate=get("bitrate"), + base_url=(rep.findtext("BaseURL") or "").split("?")[0], + ids=[get("audioTrackId"), get("id"), period.get("id")], + track_args=track_args, + ).encode() + ) + )[2:] + + tracks.add( + track_type( + id_=track_id, + url=self.url, + codec=track_codec, + language=track_lang, + is_original_lang=bool(language and is_close_match(track_lang, [language])), + descriptor=Video.Descriptor.DASH, + data={ + "dash": { + "manifest": self.manifest, + "period": period, + "adaptation_set": adaptation_set, + "representation": rep, + "representation_id": rep.get("id"), + "filtered_period_ids": filtered_period_ids, + } + }, + **track_args, + ) + ) + + # only get tracks from the first main-content period + break + + tracks.manifest_url = self.url + return tracks + + @staticmethod + def download_track( + track: AnyTrack, + save_path: Path, + save_dir: Path, + progress: partial, + session: Optional[Session] = None, + proxy: Optional[str] = None, + max_workers: Optional[int] = None, + license_widevine: Optional[Callable] = None, + *, + cdm: Optional[object] = None, + ): + if not session: + session = Session() + elif not isinstance(session, (Session, RnetSession)): + raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {session!r}") + + if proxy: + session.proxies.update({"all": proxy}) + + log = logging.getLogger("DASH") + + manifest: ElementTree = track.data["dash"]["manifest"] + adaptation_set: Element = track.data["dash"]["adaptation_set"] + representation: Element = track.data["dash"]["representation"] + rep_id: Optional[str] = track.data["dash"].get("representation_id") or representation.get("id") + filtered_period_ids: list[str] = track.data["dash"].get("filtered_period_ids", []) + + # Preserve existing DRM if it was set by the service, especially when service set Widevine + # but manifest only contains PlayReady protection (common scenario for some services) + existing_drm = track.drm + manifest_drm = DASH.get_drm( + representation.findall("ContentProtection") + adaptation_set.findall("ContentProtection") + ) + + # Only override existing DRM if: + # 1. No existing DRM was set, OR + # 2. Existing DRM contains same type as manifest DRM, OR + # 3. Existing DRM is not Widevine (preserve Widevine when service explicitly set it) + should_override_drm = ( + not existing_drm + or ( + existing_drm + and manifest_drm + and any(isinstance(existing, type(manifest)) for existing in existing_drm for manifest in manifest_drm) + ) + or (existing_drm and not any(isinstance(drm, Widevine) for drm in existing_drm)) + ) + + pre_existing_keys = {} + if existing_drm: + for drm_obj in existing_drm: + if hasattr(drm_obj, "content_keys") and drm_obj.content_keys: + pre_existing_keys.update(drm_obj.content_keys) + + if should_override_drm: + track.drm = manifest_drm + else: + track.drm = existing_drm + + if pre_existing_keys and track.drm: + for drm_obj in track.drm: + if hasattr(drm_obj, "content_keys"): + for kid, key in pre_existing_keys.items(): + if kid not in drm_obj.content_keys: + drm_obj.content_keys[kid] = key + + # Collect segments from all content periods in the manifest + all_periods = manifest.findall("Period") + segments: list[tuple[str, Optional[str]]] = [] + segment_durations: list[int] = [] + segment_timescale: float = 0 + init_data: Optional[bytes] = None + track_kid: Optional[UUID] = None + + content_periods = [p for p in all_periods if DASH._is_content_period(p, filtered_period_ids)] + period_count = len(content_periods) + + if period_count > 1: + log.debug(f"Multi-period manifest detected with {period_count} content periods") + + for period_idx, content_period in enumerate(content_periods): + # Find the matching representation in this period + matched_rep = None + matched_as = None + for as_ in content_period.findall("AdaptationSet"): + if DASH.is_trick_mode(as_): + continue + for rep in as_.findall("Representation"): + if rep.get("id") == rep_id: + matched_rep = rep + matched_as = as_ + break + if matched_rep is not None: + break + + if matched_rep is None or matched_as is None: + period_id = content_period.get("id", period_idx) + log.warning(f"Representation '{rep_id}' not found in period '{period_id}', skipping") + continue + + p_init, p_segments, p_timescale, p_durations, p_kid = DASH._get_period_segments( + period=content_period, + adaptation_set=matched_as, + representation=matched_rep, + manifest=manifest, + track=track, + track_url=track.url, + session=session, + ) + + if period_idx == 0: + # First period: use its init data and KID for DRM licensing + init_data = p_init + track_kid = p_kid + segment_timescale = p_timescale + else: + if p_kid and track_kid and p_kid != track_kid: + log.debug(f"Period {content_period.get('id', period_idx)} has different KID: {p_kid}") + + for seg in p_segments: + if seg not in segments: + segments.append(seg) + segment_durations.extend(p_durations) + + if not segments: + log.error("Could not find a way to get segments from this MPD manifest.") + log.debug(track.url) + sys.exit(1) + + # TODO: Should we floor/ceil/round, or is int() ok? + track.data["dash"]["timescale"] = int(segment_timescale) + track.data["dash"]["segment_durations"] = segment_durations + + if not track.drm and init_data and isinstance(track, (Video, Audio)): + prefers_playready = is_playready_cdm(cdm) + if prefers_playready: + try: + track.drm = [PlayReady.from_init_data(init_data)] + except PlayReady.Exceptions.PSSHNotFound: + try: + track.drm = [Widevine.from_init_data(init_data)] + except Widevine.Exceptions.PSSHNotFound: + log.warning("No PlayReady or Widevine PSSH was found for this track, is it DRM free?") + else: + try: + track.drm = [Widevine.from_init_data(init_data)] + except Widevine.Exceptions.PSSHNotFound: + try: + track.drm = [PlayReady.from_init_data(init_data)] + except PlayReady.Exceptions.PSSHNotFound: + log.warning("No Widevine or PlayReady PSSH was found for this track, is it DRM free?") + + if track.drm: + track_kid = track_kid or track.get_key_id(url=segments[0][0], session=session) + drm = track.get_drm_for_cdm(cdm) + if isinstance(drm, (Widevine, PlayReady)): + # license and grab content keys + try: + if not license_widevine: + raise ValueError("license_widevine func must be supplied to use DRM") + progress(downloaded="LICENSING") + license_widevine(drm, track_kid=track_kid) + progress(downloaded="[yellow]LICENSED") + except Exception: # noqa + DOWNLOAD_CANCELLED.set() # skip pending track downloads + progress(downloaded="[red]FAILED") + raise + else: + drm = None + + if DOWNLOAD_LICENCE_ONLY.is_set(): + progress(downloaded="[yellow]SKIPPED") + return + + progress(total=len(segments)) + + downloader = track.downloader + + downloader_args = dict( + urls=[ + {"url": url, "headers": {"Range": f"bytes={bytes_range}"} if bytes_range else {}} + for url, bytes_range in segments + ], + output_dir=save_dir, + filename="{i:0%d}.mp4" % (len(str(len(segments)))), + headers=session.headers, + cookies=session.cookies, + proxy=proxy, + max_workers=max_workers, + session=session, + ) + + debug_logger = get_debug_logger() + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="manifest_dash_download_start", + message="Starting DASH manifest download", + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "total_segments": len(segments), + "has_drm": bool(track.drm), + "drm_types": [drm.__class__.__name__ for drm in (track.drm or [])], + "save_path": str(save_path), + "has_init_data": bool(init_data), + }, + ) + + for status_update in downloader(**downloader_args): + file_downloaded = status_update.get("file_downloaded") + if file_downloaded: + events.emit(events.Types.SEGMENT_DOWNLOADED, track=track, segment=file_downloaded) + else: + downloaded = status_update.get("downloaded") + if downloaded and downloaded.endswith("/s"): + status_update["downloaded"] = f"DASH {downloaded}" + progress(**status_update) + + # Verify output directory exists and contains files + if not save_dir.exists(): + error_msg = f"Output directory does not exist: {save_dir}" + if debug_logger: + debug_logger.log( + level="ERROR", + operation="manifest_dash_download_output_missing", + message=error_msg, + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "save_dir": str(save_dir), + "save_path": str(save_path), + "downloader": "requests", + }, + ) + raise FileNotFoundError(error_msg) + + for control_file in save_dir.glob("*.!dev"): + control_file.unlink(missing_ok=True) + + segments_to_merge = [x for x in sorted(save_dir.iterdir()) if x.is_file()] + + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="manifest_dash_download_complete", + message="DASH download complete, preparing to merge", + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "save_dir": str(save_dir), + "save_dir_exists": save_dir.exists(), + "segments_found": len(segments_to_merge), + "segment_files": [f.name for f in segments_to_merge[:10]], # Limit to first 10 + "downloader": "requests", + }, + ) + + if not segments_to_merge: + error_msg = f"No segment files found in output directory: {save_dir}" + if debug_logger: + # List all contents of the directory for debugging + all_contents = list(save_dir.iterdir()) if save_dir.exists() else [] + debug_logger.log( + level="ERROR", + operation="manifest_dash_download_no_segments", + message=error_msg, + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "save_dir": str(save_dir), + "directory_contents": [str(p) for p in all_contents], + "downloader": "requests", + }, + ) + raise FileNotFoundError(error_msg) + + with open(save_path, "wb") as f: + if init_data: + f.write(init_data) + if len(segments_to_merge) > 1: + progress(downloaded="Merging", completed=0, total=len(segments_to_merge)) + for segment_file in segments_to_merge: + segment_data = segment_file.read_bytes() + if ( + not drm + and isinstance(track, Subtitle) + and track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML) + ): + segment_data = try_ensure_utf8(segment_data) + segment_data = ( + segment_data.decode("utf8") + .replace("‎", html.unescape("‎")) + .replace("‏", html.unescape("‏")) + .encode("utf8") + ) + f.write(segment_data) + f.flush() + segment_file.unlink() + progress(advance=1) + + track.path = save_path + events.emit(events.Types.TRACK_DOWNLOADED, track=track) + + if drm: + progress(downloaded="Decrypting", completed=0, total=100) + drm.decrypt(save_path) + track.drm = None + events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=drm, segment=None) + progress(downloaded="Decrypting", advance=100) + + # Clean up empty segment directory + if save_dir.exists() and save_dir.name.endswith("_segments"): + try: + save_dir.rmdir() + except OSError: + # Directory might not be empty, try removing recursively + shutil.rmtree(save_dir, ignore_errors=True) + + progress(downloaded="Downloaded") + + @staticmethod + def _is_content_period(period: Element, filtered_period_ids: list[str]) -> bool: + """Check if a period is a valid content period (not an ad, not filtered, not trick mode).""" + period_id = period.get("id") + if period_id and period_id in filtered_period_ids: + return False + if next(iter(period.xpath("SegmentType/@value")), "content") != "content": + return False + if "urn:amazon:primevideo:cachingBreadth" in [ + x.get("schemeIdUri") for x in period.findall("SupplementalProperty") + ]: + return False + return True + + @staticmethod + def _merge_segment_templates(adaptation_set: Element, representation: Element) -> Optional[Element]: + """ + Build the effective SegmentTemplate for a Representation by cascading the + AdaptationSet > Representation levels (ISO/IEC 23009-1 5.3.9.1). + + The Representation-level node, when present, is the base; attributes and the + SegmentTimeline child it does not declare are inherited from the AdaptationSet-level + node. Returns None if no SegmentTemplate exists at either level. + """ + levels = [node.find("SegmentTemplate") for node in (adaptation_set, representation)] + present = [node for node in levels if node is not None] + if not present: + return None + + merged = deepcopy(present[-1]) + for ancestor in reversed(present[:-1]): + for attr, value in ancestor.attrib.items(): + if merged.get(attr) is None: + merged.set(attr, value) + if merged.find("SegmentTimeline") is None: + timeline = ancestor.find("SegmentTimeline") + if timeline is not None: + merged.append(deepcopy(timeline)) + return merged + + @staticmethod + def _get_period_segments( + period: Element, + adaptation_set: Element, + representation: Element, + manifest: ElementTree, + track: AnyTrack, + track_url: str, + session: Union[Session, RnetSession], + ) -> tuple[ + Optional[bytes], + list[tuple[str, Optional[str]]], + float, + list[int], + Optional[UUID], + ]: + """ + Extract segments from a single period's representation. + + Returns: + A tuple of (init_data, segments, segment_timescale, segment_durations, track_kid). + """ + manifest_base_url = manifest.findtext("BaseURL") + if not manifest_base_url: + manifest_base_url = track_url + elif not re.match("^https?://", manifest_base_url, re.IGNORECASE): + manifest_base_url = urljoin(track_url, f"./{manifest_base_url}") + period_base_url = urljoin(manifest_base_url, period.findtext("BaseURL") or "") + adaptation_set_base_url = urljoin(period_base_url, adaptation_set.findtext("BaseURL") or "") + rep_base_url = urljoin(adaptation_set_base_url, representation.findtext("BaseURL") or "") + + period_duration = period.get("duration") or manifest.get("mediaPresentationDuration") + init_data: Optional[bytes] = None + + segment_template = DASH._merge_segment_templates(adaptation_set, representation) + + segment_list = representation.find("SegmentList") + if segment_list is None: + segment_list = adaptation_set.find("SegmentList") + + segment_base = representation.find("SegmentBase") + if segment_base is None: + segment_base = adaptation_set.find("SegmentBase") + + segments: list[tuple[str, Optional[str]]] = [] + segment_timescale: float = 0 + segment_durations: list[int] = [] + track_kid: Optional[UUID] = None + + if segment_template is not None: + start_number = int(segment_template.get("startNumber") or 1) + end_number = int(segment_template.get("endNumber") or 0) or None + segment_timeline = segment_template.find("SegmentTimeline") + segment_timescale = float(segment_template.get("timescale") or 1) + + for item in ("initialization", "media"): + value = segment_template.get(item) + if not value: + continue + if not re.match("^https?://", value, re.IGNORECASE): + if not rep_base_url: + raise ValueError("Resolved Segment URL is not absolute, and no Base URL is available.") + value = urljoin(rep_base_url, value) + if not urlparse(value).query: + manifest_url_query = urlparse(track_url).query + if manifest_url_query: + value += f"?{manifest_url_query}" + segment_template.set(item, value) + + init_url = segment_template.get("initialization") + if init_url: + res = session.get( + DASH.replace_fields( + init_url, Bandwidth=representation.get("bandwidth"), RepresentationID=representation.get("id") + ) + ) + res.raise_for_status() + init_data = res.content + track_kid = track.get_key_id(init_data) + + if segment_timeline is not None: + current_time = 0 + for s in segment_timeline.findall("S"): + if s.get("t"): + current_time = int(s.get("t")) + for _ in range(1 + (int(s.get("r") or 0))): + segment_durations.append(current_time) + current_time += int(s.get("d")) + + if not end_number: + end_number = len(segment_durations) + # Handle high startNumber in DVR/catch-up manifests where startNumber > segment count + if start_number > end_number: + end_number = start_number + len(segment_durations) - 1 + + for t, n in zip(segment_durations, range(start_number, end_number + 1)): + segments.append( + ( + DASH.replace_fields( + segment_template.get("media"), + Bandwidth=representation.get("bandwidth"), + Number=n, + RepresentationID=representation.get("id"), + Time=t, + ), + None, + ) + ) + else: + if not period_duration: + raise ValueError("Duration of the Period was unable to be determined.") + period_duration = DASH.pt_to_sec(period_duration) + segment_duration = float(segment_template.get("duration")) or 1 + + if not end_number: + segment_count = math.ceil(period_duration / (segment_duration / segment_timescale)) + end_number = start_number + segment_count - 1 + + for s in range(start_number, end_number + 1): + segments.append( + ( + DASH.replace_fields( + segment_template.get("media"), + Bandwidth=representation.get("bandwidth"), + Number=s, + RepresentationID=representation.get("id"), + Time=s, + ), + None, + ) + ) + # TODO: Should we floor/ceil/round, or is int() ok? + segment_durations.append(int(segment_duration)) + elif segment_list is not None: + segment_timescale = float(segment_list.get("timescale") or 1) + + init_data = None + initialization = segment_list.find("Initialization") + if initialization is not None: + source_url = initialization.get("sourceURL") + if not source_url: + source_url = rep_base_url + elif not re.match("^https?://", source_url, re.IGNORECASE): + source_url = urljoin(rep_base_url, f"./{source_url}") + + if initialization.get("range"): + init_range_header = {"Range": f"bytes={initialization.get('range')}"} + else: + init_range_header = None + + res = session.get(url=source_url, headers=init_range_header) + res.raise_for_status() + init_data = res.content + track_kid = track.get_key_id(init_data) + + segment_urls = segment_list.findall("SegmentURL") + for segment_url in segment_urls: + media_url = segment_url.get("media") + if not media_url: + media_url = rep_base_url + elif not re.match("^https?://", media_url, re.IGNORECASE): + media_url = urljoin(rep_base_url, f"./{media_url}") + + segments.append((media_url, segment_url.get("mediaRange"))) + segment_durations.append(int(segment_url.get("duration") or 1)) + elif segment_base is not None: + media_range = None + init_data = None + initialization = segment_base.find("Initialization") + if initialization is not None: + if initialization.get("range"): + init_range_header = {"Range": f"bytes={initialization.get('range')}"} + else: + init_range_header = None + + res = session.get(url=rep_base_url, headers=init_range_header) + res.raise_for_status() + init_data = res.content + track_kid = track.get_key_id(init_data) + total_size = res.headers.get("Content-Range", "").split("/")[-1] + if total_size: + media_range = f"{len(init_data)}-{total_size}" + + segments.append((rep_base_url, media_range)) + elif rep_base_url: + segments.append((rep_base_url, None)) + + return init_data, segments, segment_timescale, segment_durations, track_kid + + @staticmethod + def _get(item: str, adaptation_set: Element, representation: Optional[Element] = None) -> Optional[Any]: + """Helper to get a requested item from the Representation, otherwise from the AdaptationSet.""" + adaptation_set_item = adaptation_set.get(item) + if representation is None: + return adaptation_set_item + + representation_item = representation.get(item) + if representation_item is not None: + return representation_item + + return adaptation_set_item + + @staticmethod + def _findall( + item: str, adaptation_set: Element, representation: Optional[Element] = None, both: bool = False + ) -> list[Any]: + """ + Helper to get all requested items from the Representation, otherwise from the AdaptationSet. + Optionally, you may pass both=True to keep both values (where available). + """ + adaptation_set_items = adaptation_set.findall(item) + if representation is None: + return adaptation_set_items + + representation_items = representation.findall(item) + + if both: + return representation_items + adaptation_set_items + + if representation_items: + return representation_items + + return adaptation_set_items + + @staticmethod + def get_language( + adaptation_set: Element, + representation: Optional[Element] = None, + fallback: Optional[Union[str, Language]] = None, + ) -> Optional[Language]: + """ + Get Language (if any) from the AdaptationSet or Representation. + + A fallback language may be provided if no language information could be + retrieved. + """ + options = [] + + if representation is not None: + options.append(representation.get("lang")) + # derive language from somewhat common id string format + # the format is typically "{rep_id}_{lang}={bitrate}" or similar + rep_id = representation.get("id") + if rep_id: + m = re.match(r"\w+_(\w+)=\d+", rep_id) + if m: + options.append(m.group(1)) + + options.append(adaptation_set.get("lang")) + + if fallback: + options.append(fallback) + + for option in options: + option = (str(option) or "").strip() + if not tag_is_valid(option) or option.startswith("und"): + continue + return Language.get(option) + + @staticmethod + def get_video_range( + codecs: str, all_supplemental_props: list[Element], all_essential_props: list[Element] + ) -> Video.Range: + # TODO: Detect Dolby Vision composite streams in DASH manifests (DV RPU embedded but + # primary codec is plain hvc1, signaled via a separate AdaptationSet/Representation + # with DV codec strings or DolbyVisionConfigurationBox). When found, mark the track + # with dv_compatible_bitstream=True so DVFixup runs pre-mux. No DASH samples seen yet. + if codecs.startswith(("dva1", "dvav", "dvhe", "dvh1")): + return Video.Range.DV + + return Video.Range.from_cicp( + primaries=next( + ( + int(x.get("value")) + for x in all_supplemental_props + all_essential_props + if x.get("schemeIdUri") == "urn:mpeg:mpegB:cicp:ColourPrimaries" + ), + 0, + ), + transfer=next( + ( + int(x.get("value")) + for x in all_supplemental_props + all_essential_props + if x.get("schemeIdUri") == "urn:mpeg:mpegB:cicp:TransferCharacteristics" + ), + 0, + ), + matrix=next( + ( + int(x.get("value")) + for x in all_supplemental_props + all_essential_props + if x.get("schemeIdUri") == "urn:mpeg:mpegB:cicp:MatrixCoefficients" + ), + 0, + ), + ) + + @staticmethod + def is_trick_mode(adaptation_set: Element) -> bool: + """Check if contents of Adaptation Set is a Trick-Mode stream.""" + essential_props = adaptation_set.findall("EssentialProperty") + supplemental_props = adaptation_set.findall("SupplementalProperty") + + return any( + prop.get("schemeIdUri") == "http://dashif.org/guidelines/trickmode" + for prop in essential_props + supplemental_props + ) + + @staticmethod + def is_descriptive(adaptation_set: Element) -> bool: + """Check if contents of Adaptation Set is Descriptive.""" + return any( + (x.get("schemeIdUri"), x.get("value")) + in (("urn:mpeg:dash:role:2011", "descriptive"), ("urn:tva:metadata:cs:AudioPurposeCS:2007", "1")) + for x in adaptation_set.findall("Accessibility") + ) + + @staticmethod + def is_forced(adaptation_set: Element) -> bool: + """Check if contents of Adaptation Set is a Forced Subtitle.""" + return any( + x.get("schemeIdUri") == "urn:mpeg:dash:role:2011" + and x.get("value") in ("forced-subtitle", "forced_subtitle") + for x in adaptation_set.findall("Role") + ) + + @staticmethod + def is_sdh(adaptation_set: Element) -> bool: + """Check if contents of Adaptation Set is for the Hearing Impaired.""" + return any( + (x.get("schemeIdUri"), x.get("value")) == ("urn:tva:metadata:cs:AudioPurposeCS:2007", "2") + for x in adaptation_set.findall("Accessibility") + ) + + @staticmethod + def is_closed_caption(adaptation_set: Element) -> bool: + """Check if contents of Adaptation Set is a Closed Caption Subtitle.""" + return any( + (x.get("schemeIdUri"), x.get("value")) == ("urn:mpeg:dash:role:2011", "caption") + for x in adaptation_set.findall("Role") + ) + + @staticmethod + def get_ddp_complexity_index(adaptation_set: Element, representation: Optional[Element]) -> Optional[int]: + """Get the DD+ Complexity Index (if any) from the AdaptationSet or Representation.""" + return next( + ( + int(x.get("value")) + for x in DASH._findall("SupplementalProperty", adaptation_set, representation, both=True) + if x.get("schemeIdUri") == "tag:dolby.com,2018:dash:EC3_ExtensionComplexityIndex:2018" + ), + None, + ) + + @staticmethod + def get_drm(protections: list[Element]) -> list[DRM_T]: + drm: list[DRM_T] = [] + PLACEHOLDER_KIDS = { + UUID("00000000-0000-0000-0000-000000000000"), # All zeros (key rotation default) + UUID("00010203-0405-0607-0809-0a0b0c0d0e0f"), # Sequential 0x00-0x0f + UUID("00010203-0405-0607-0809-101112131415"), # Shaka Packager test pattern + } + + for protection in protections: + urn = (protection.get("schemeIdUri") or "").lower() + + if urn == WidevineCdm.urn: + pssh_text = protection.findtext("pssh") or protection.findtext("{urn:mpeg:cenc:2013}pssh") + if not pssh_text: + continue + pssh = PSSH(pssh_text) + kid_attr = protection.get("kid") or protection.get("{urn:mpeg:cenc:2013}kid") + kid = UUID(bytes=base64.b64decode(kid_attr)) if kid_attr else None + + if not kid: + default_kid_attr = protection.get("default_KID") or protection.get( + "{urn:mpeg:cenc:2013}default_KID" + ) + kid = UUID(default_kid_attr) if default_kid_attr else None + + if not kid: + kid = next( + ( + UUID(p.get("default_KID") or p.get("{urn:mpeg:cenc:2013}default_KID")) + for p in protections + if p.get("default_KID") or p.get("{urn:mpeg:cenc:2013}default_KID") + ), + None, + ) + + if kid and pssh.key_ids and all(k.int == 0 or k in PLACEHOLDER_KIDS for k in pssh.key_ids): + pssh.set_key_ids([kid]) + + drm.append(Widevine(pssh=pssh, kid=kid)) + + elif urn in ("urn:uuid:9a04f079-9840-4286-ab92-e65be0885f95", "urn:microsoft:playready"): + pr_pssh_b64 = ( + protection.findtext("pssh") + or protection.findtext("{urn:mpeg:cenc:2013}pssh") + or protection.findtext("pro") + or protection.findtext("{urn:microsoft:playready}pro") + ) + if not pr_pssh_b64: + continue + pr_pssh = PR_PSSH(pr_pssh_b64) + kid_b64 = protection.findtext("kid") + kid = None + if kid_b64: + try: + kid = UUID(bytes=base64.b64decode(kid_b64)) + except Exception: + kid = None + + drm.append(PlayReady(pssh=pr_pssh, kid=kid, pssh_b64=pr_pssh_b64)) + + return drm + + @staticmethod + def pt_to_sec(d: Union[str, float]) -> float: + if isinstance(d, float): + return d + has_ymd = d[0:8] == "P0Y0M0DT" + if d[0:2] != "PT" and not has_ymd: + raise ValueError("Input data is not a valid time string.") + if has_ymd: + d = d[6:].upper() # skip `P0Y0M0DT` + else: + d = d[2:].upper() # skip `PT` + m = re.findall(r"([\d.]+.)", d) + return sum(float(x[0:-1]) * {"H": 60 * 60, "M": 60, "S": 1}[x[-1].upper()] for x in m) + + @staticmethod + def replace_fields(url: str, **kwargs: Any) -> str: + for field, value in kwargs.items(): + url = url.replace(f"${field}$", str(value)) + m = re.search(rf"\${re.escape(field)}%([a-z0-9]+)\$", url, flags=re.I) + if m: + url = url.replace(m.group(), f"{value:{m.group(1)}}") + return url + + +__all__ = ("DASH",) diff --git a/reset/packages/envied/src/envied/core/manifests/hls.py b/reset/packages/envied/src/envied/core/manifests/hls.py new file mode 100644 index 0000000..faa4c9e --- /dev/null +++ b/reset/packages/envied/src/envied/core/manifests/hls.py @@ -0,0 +1,1294 @@ +from __future__ import annotations + +import base64 +import html +import json +import logging +import os +import re +import shutil +import subprocess +import sys +from functools import partial +from pathlib import Path +from typing import Any, Callable, Optional, Union +from urllib.parse import urljoin +from uuid import UUID +from zlib import crc32 + +import m3u8 +import requests +from langcodes import Language, tag_is_valid +from m3u8 import M3U8 +from pyplayready.cdm import Cdm as PlayReadyCdm +from pyplayready.system.pssh import PSSH as PR_PSSH +from pywidevine.cdm import Cdm as WidevineCdm +from pywidevine.pssh import PSSH as WV_PSSH +from requests import Session + +from envied.core import binaries +from envied.core.cdm.detect import is_playready_cdm, is_widevine_cdm +from envied.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack +from envied.core.drm import DRM_T, ClearKey, MonaLisa, PlayReady, Widevine +from envied.core.events import events +from envied.core.session import RnetResponse, RnetSession +from envied.core.tracks import Audio, Subtitle, Tracks, Video +from envied.core.utilities import get_debug_logger, get_extension, is_close_match, try_ensure_utf8 + + +class HLS: + SUPP_CODECS_RE = re.compile(r'SUPPLEMENTAL-CODECS="([^"]+)"', re.IGNORECASE) + + def __init__( + self, + manifest: M3U8, + session: Optional[Union[Session, RnetSession]] = None, + url: Optional[str] = None, + raw_text: Optional[str] = None, + ): + if not manifest: + raise ValueError("HLS manifest must be provided.") + if not isinstance(manifest, M3U8): + raise TypeError(f"Expected manifest to be a {M3U8}, not {manifest!r}") + if not manifest.is_variant: + raise ValueError("Expected the M3U(8) manifest to be a Variant Playlist.") + + self.manifest = manifest + self.session = session or Session() + self.url = url + self.raw_text = raw_text + + @classmethod + def from_url(cls, url: str, session: Optional[Union[Session, RnetSession]] = None, **args: Any) -> HLS: + if not url: + raise requests.URLRequired("HLS manifest URL must be provided.") + if not isinstance(url, str): + raise TypeError(f"Expected url to be a {str}, not {url!r}") + + if not session: + session = Session() + elif not isinstance(session, (Session, RnetSession)): + raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {session!r}") + + res = session.get(url, **args) + + # Handle requests and rnet response objects + if isinstance(res, requests.Response): + if not res.ok: + raise requests.ConnectionError("Failed to request the M3U(8) document.", response=res) + content = res.text + elif isinstance(res, RnetResponse): + if not res.ok: + raise requests.ConnectionError("Failed to request the M3U(8) document.", response=res) + content = res.text + else: + raise TypeError(f"Expected response to be a requests.Response or rnet.Response, not {type(res)}") + + master = m3u8.loads(content, uri=url) + + return cls(master, session, url=url, raw_text=content) + + @classmethod + def from_text(cls, text: str, url: str) -> HLS: + if not text: + raise ValueError("HLS manifest Text must be provided.") + if not isinstance(text, str): + raise TypeError(f"Expected text to be a {str}, not {text!r}") + + if not url: + raise requests.URLRequired("HLS manifest URL must be provided for relative path computations.") + if not isinstance(url, str): + raise TypeError(f"Expected url to be a {str}, not {url!r}") + + master = m3u8.loads(text, uri=url) + + return cls(master, raw_text=text) + + def supplemental_codecs_by_uri(self) -> dict[str, str]: + """Map each variant URI to its SUPPLEMENTAL-CODECS value. + + python-m3u8 drops this attribute, so we re-parse the raw text to recover it for + Dolby Vision composite detection (dvh1.08.x advertised only in SUPPLEMENTAL-CODECS + while primary CODECS stays plain hvc1). + """ + if not self.raw_text: + return {} + out: dict[str, str] = {} + lines = self.raw_text.splitlines() + for i, line in enumerate(lines): + if not line.startswith("#EXT-X-STREAM-INF"): + continue + supp_match = self.SUPP_CODECS_RE.search(line) + if not supp_match: + continue + for j in range(i + 1, len(lines)): + uri = lines[j].strip() + if uri and not uri.startswith("#"): + out[uri] = supp_match.group(1) + break + return out + + def to_tracks(self, language: Union[str, Language]) -> Tracks: + """ + Convert a Variant Playlist M3U(8) document to Video, Audio and Subtitle Track objects. + + Parameters: + language: Language you expect the Primary Track to be in. + + All Track objects' URL will be to another M3U(8) document. However, these documents + will be Invariant Playlists and contain the list of segments URIs among other metadata. + """ + session_keys = list(self.manifest.session_keys or []) + if not session_keys: + session_keys = HLS.parse_session_data_keys(self.manifest, self.session) + + session_drm = HLS.get_all_drm(session_keys) + + audio_codecs_by_group_id: dict[str, Audio.Codec] = {} + cc_by_group_id: dict[str, list[dict[str, Any]]] = {} + for media in self.manifest.media: + if media.type == "CLOSED-CAPTIONS": + cc_by_group_id.setdefault(media.group_id, []).append( + { + "language": media.language, + "name": media.name, + "instream_id": media.instream_id, + "characteristics": media.characteristics, + } + ) + tracks = Tracks() + + supplemental_codecs = self.supplemental_codecs_by_uri() + dv_supp_prefixes = ("dva1", "dvav", "dvhe", "dvh1") + + for playlist in self.manifest.playlists: + audio_group = playlist.stream_info.audio + audio_codec: Optional[Audio.Codec] = None + if audio_group and playlist.stream_info.codecs: + try: + audio_codec = Audio.Codec.from_codecs(playlist.stream_info.codecs) + except ValueError: + audio_codec = None + if audio_codec: + audio_codecs_by_group_id[audio_group] = audio_codec + + try: + # TODO: Any better way to figure out the primary track type? + if playlist.stream_info.codecs: + Video.Codec.from_codecs(playlist.stream_info.codecs) + except ValueError: + primary_track_type = Audio + else: + primary_track_type = Video + + primary_codecs = (playlist.stream_info.codecs or "").lower() + primary_has_dv = any(codec.split(".")[0] in dv_supp_prefixes for codec in primary_codecs.split(",")) + + supp_codecs_str = supplemental_codecs.get(playlist.uri, "") + supp_dv_codec: Optional[str] = None + for codec in supp_codecs_str.lower().split(","): + token = codec.strip().split("/")[0] + if token.split(".")[0] in dv_supp_prefixes: + supp_dv_codec = token + break + + video_range = ( + Video.Range.DV if primary_has_dv else Video.Range.from_m3u_range_tag(playlist.stream_info.video_range) + ) + # DV-composite track: primary codec is plain HEVC but SUPPLEMENTAL-CODECS advertises + # a DV codec. Range stays whatever VIDEO-RANGE signaled (HDR10/HLG/SDR); DVFixup will + # restore DV signaling post-download. Services that know their encoder embeds HDR10+ + # SEI must override `range` themselves (see services/ATV). + dv_compatible_bitstream = primary_track_type is Video and not primary_has_dv and supp_dv_codec is not None + + tracks.add( + primary_track_type( + id_=hex(crc32(str(playlist).encode()))[2:], + url=urljoin(playlist.base_uri, playlist.uri), + codec=( + primary_track_type.Codec.from_codecs(playlist.stream_info.codecs) + if playlist.stream_info.codecs + else None + ), + language=language, # HLS manifests do not seem to have language info + is_original_lang=True, # TODO: All we can do is assume Yes + bitrate=playlist.stream_info.average_bandwidth or playlist.stream_info.bandwidth, + descriptor=Video.Descriptor.HLS, + drm=session_drm, + data={"hls": {"playlist": playlist}}, + # video track args + **( + dict( + range_=video_range, + width=playlist.stream_info.resolution[0] if playlist.stream_info.resolution else None, + height=playlist.stream_info.resolution[1] if playlist.stream_info.resolution else None, + fps=playlist.stream_info.frame_rate, + closed_captions=cc_by_group_id.get( + (playlist.stream_info.closed_captions or "").strip('"'), [] + ), + dv_compatible_bitstream=dv_compatible_bitstream, + ) + if primary_track_type is Video + else {} + ), + ) + ) + + for media in self.manifest.media: + if not media.uri: + continue + + joc = 0 + if media.type == "AUDIO": + track_type = Audio + codec = audio_codecs_by_group_id.get(media.group_id) + if media.channels and media.channels.endswith("/JOC"): + joc = int(media.channels.split("/JOC")[0]) + media.channels = "5.1" + else: + track_type = Subtitle + codec = Subtitle.Codec.WebVTT # assuming WebVTT, codec info isn't shown + + track_lang = next( + ( + Language.get(option) + for x in (media.language, language) + for option in [(str(x) or "").strip()] + if tag_is_valid(option) and not option.startswith("und") + ), + None, + ) + if not track_lang: + msg = "Language information could not be derived for a media." + if language is None: + msg += " No fallback language was provided when calling HLS.to_tracks()." + elif not tag_is_valid((str(language) or "").strip()) or str(language).startswith("und"): + msg += f" The fallback language provided is also invalid: {language}" + raise ValueError(msg) + + tracks.add( + track_type( + id_=hex(crc32(str(media).encode()))[2:], + url=urljoin(media.base_uri, media.uri), + codec=codec, + language=track_lang, # HLS media may not have language info, fallback if needed + is_original_lang=bool(language and is_close_match(track_lang, [language])), + descriptor=Audio.Descriptor.HLS, + drm=session_drm if media.type == "AUDIO" else None, + data={"hls": {"media": media}}, + # audio track args + **( + dict( + bitrate=0, # TODO: M3U doesn't seem to state bitrate? + channels=media.channels, + joc=joc, + descriptive="public.accessibility.describes-video" in (media.characteristics or ""), + ) + if track_type is Audio + else dict( + forced=media.forced == "YES", + sdh="public.accessibility.describes-music-and-sound" in (media.characteristics or ""), + ) + if track_type is Subtitle + else {} + ), + ) + ) + + for video in tracks.videos: + has_resolution = video.width and video.height + has_codec = video.codec is not None + if has_resolution and has_codec: + continue + try: + probe = HLS._probe_ts_info(video.url, self.session) + if probe: + width, height, codec = probe + if not has_resolution: + video.width, video.height = width, height + if not has_codec: + video.codec = codec + except Exception: + pass + + if self.url: + tracks.manifest_url = self.url + return tracks + + @staticmethod + def _probe_ts_info( + variant_url: str, session: Optional[Union[Session, RnetSession]] = None + ) -> Optional[tuple[int, int, Video.Codec]]: + """Probe the first TS segment of a variant playlist to extract resolution and codec.""" + if not session: + session = Session() + + res = session.get(variant_url) + variant = m3u8.loads(res.text if hasattr(res, "text") else res.text, uri=variant_url) + if not variant.segments: + return None + + seg_uri = urljoin(variant_url, variant.segments[0].uri) + + # Download only the first 8KB — SPS is always near the start of the first TS packet + res = session.get(seg_uri, headers={"Range": "bytes=0-8191"}) + data = res.content + + return HLS._parse_ts_video_info(data) + + @staticmethod + def _parse_ts_video_info(data: bytes) -> Optional[tuple[int, int, Video.Codec]]: + """Parse H.264/H.265 NAL units from TS segment data to extract resolution and codec.""" + + class _BitReader: + def __init__(self, buf: bytes) -> None: + self.data = buf + self.pos = 0 + + def bits(self, n: int) -> int: + val = 0 + for _ in range(n): + val = (val << 1) | ((self.data[self.pos >> 3] >> (7 - (self.pos & 7))) & 1) + self.pos += 1 + return val + + def ue(self) -> int: + zeros = 0 + while self.bits(1) == 0: + zeros += 1 + return (1 << zeros) - 1 + self.bits(zeros) if zeros else 0 + + def se(self) -> int: + val = self.ue() + return (val + 1) // 2 if val & 1 else -(val // 2) + + # Find SPS NAL unit via start code + # H.264: NAL type 7 (SPS), identified by byte & 0x1F == 7 + # H.265: NAL type 33 (SPS), identified by (byte >> 1) & 0x3F == 33 + for i in range(len(data) - 4): + start3 = data[i : i + 3] == b"\x00\x00\x01" + start4 = data[i : i + 4] == b"\x00\x00\x00\x01" + if not start3 and not start4: + continue + offset = i + (4 if start4 else 3) + if offset >= len(data): + continue + + nal_byte = data[offset] + h264_type = nal_byte & 0x1F + h265_type = (nal_byte >> 1) & 0x3F + + # H.264 SPS (NAL type 7) + if h264_type == 7: + sps = data[offset : offset + 64] + if len(sps) < 5: + continue + try: + r = _BitReader(sps[1:]) # skip NAL header byte + profile = r.bits(8) + r.bits(8) # constraint flags + r.bits(8) # level + r.ue() # sps_id + + if profile in (100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134): + chroma = r.ue() + if chroma == 3: + r.bits(1) + r.ue() # bit_depth_luma + r.ue() # bit_depth_chroma + r.bits(1) # qpprime_y_zero_transform_bypass + if r.bits(1): # scaling_matrix_present + for j in range(6 if chroma != 3 else 12): + if r.bits(1): + last = 8 + for _ in range(16 if j < 6 else 64): + if last != 0: + last = (last + r.se()) & 0xFF + + r.ue() # log2_max_frame_num + poc_type = r.ue() + if poc_type == 0: + r.ue() + elif poc_type == 1: + r.bits(1) + r.se() + r.se() + for _ in range(r.ue()): + r.se() + + r.ue() # max_num_ref_frames + r.bits(1) # gaps_in_frame_num + + w_mbs = r.ue() + 1 + h_map = r.ue() + 1 + frame_mbs_only = r.bits(1) + if not frame_mbs_only: + r.bits(1) + r.bits(1) # direct_8x8_inference + + cl = cr = ct = cb = 0 + if r.bits(1): # crop + cl, cr, ct, cb = r.ue(), r.ue(), r.ue(), r.ue() + + width = w_mbs * 16 - (cl + cr) * 2 + height = (2 - frame_mbs_only) * h_map * 16 - (ct + cb) * 2 + return (width, height, Video.Codec.AVC) + except (IndexError, ValueError): + continue + + # H.265 SPS (NAL type 33) + elif h265_type == 33: + sps = data[offset : offset + 128] + if len(sps) < 10: + continue + try: + r = _BitReader(sps[2:]) # skip 2-byte NAL header + r.bits(4) # sps_video_parameter_set_id + max_sub_layers = r.bits(3) + r.bits(1) # sps_temporal_id_nesting + + # profile_tier_level + r.bits(2) # general_profile_space + r.bits(1) # general_tier + r.bits(5) # general_profile_idc + r.bits(32) # general_profile_compatibility_flags + r.bits(48) # general_constraint_indicator_flags + r.bits(8) # general_level_idc + sub_layer_flags = [] + for _ in range(max_sub_layers - 1): + sub_layer_flags.append((r.bits(1), r.bits(1))) + if max_sub_layers - 1 > 0: + for _ in range(8 - (max_sub_layers - 1)): + r.bits(2) + for profile_present, level_present in sub_layer_flags: + if profile_present: + r.bits(2 + 1 + 5 + 32 + 48 + 8) + if level_present: + r.bits(8) + + r.ue() # sps_seq_parameter_set_id + chroma = r.ue() + if chroma == 3: + r.bits(1) + + width = r.ue() + height = r.ue() + + if r.bits(1): # conformance_window + cl = r.ue() + cr = r.ue() + ct = r.ue() + cb = r.ue() + sub_w = 2 if chroma in (1, 2) else 1 + sub_h = 2 if chroma == 1 else 1 + width -= (cl + cr) * sub_w + height -= (ct + cb) * sub_h + + return (width, height, Video.Codec.HEVC) + except (IndexError, ValueError): + continue + + return None + + @staticmethod + def download_track( + track: AnyTrack, + save_path: Path, + save_dir: Path, + progress: partial, + session: Optional[Union[Session, RnetSession]] = None, + proxy: Optional[str] = None, + max_workers: Optional[int] = None, + license_widevine: Optional[Callable] = None, + *, + cdm: Optional[object] = None, + ) -> None: + if not session: + session = Session() + elif not isinstance(session, (Session, RnetSession)): + raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {session!r}") + + if proxy: + # Handle proxies differently based on session type + if isinstance(session, Session): + session.proxies.update({"all": proxy}) + + log = logging.getLogger("HLS") + + if track.from_file: + master = m3u8.load(str(track.from_file)) + else: + # Get the playlist text and handle both session types + response = session.get(track.url) + if isinstance(response, requests.Response) or isinstance(response, RnetResponse): + if not response.ok: + log.error(f"Failed to request the invariant M3U8 playlist: {response.status_code}") + sys.exit(1) + playlist_text = response.text + else: + raise TypeError(f"Expected response to be a requests.Response or rnet.Response, not {type(response)}") + + master = m3u8.loads(playlist_text, uri=track.url) + + if not master.segments: + log.error("Track's HLS playlist has no segments, expecting an invariant M3U8 playlist.") + sys.exit(1) + + # Get session DRM as fallback but prefer media playlist keys for accurate KID matching + if track.drm: + session_drm = track.get_drm_for_cdm(cdm) + else: + session_drm = None + + initial_drm_licensed = False + initial_drm_key = None # Track the EXT-X-KEY used for initial licensing + media_keys = [k for k in (master.keys or []) if k is not None] + if media_keys: + cdm_media_keys = HLS.filter_keys_for_cdm(media_keys, cdm) + media_playlist_key = HLS.get_supported_key(cdm_media_keys) if cdm_media_keys else None + + if media_playlist_key: + media_drm = HLS.get_drm(media_playlist_key, session) + if isinstance(media_drm, (Widevine, PlayReady)): + track_kid = HLS.get_track_kid_from_init(master, track, session) or media_drm.kid + # Preserve pre-existing keys (e.g. from server_cdm) + if track.drm: + for existing_drm in track.drm: + if hasattr(existing_drm, "content_keys") and existing_drm.content_keys: + media_drm.content_keys.update(existing_drm.content_keys) + track.drm = [media_drm] + try: + if not license_widevine: + raise ValueError("license_widevine func must be supplied to use DRM") + progress(downloaded="LICENSING") + license_widevine(media_drm, track_kid=track_kid) + progress(downloaded="[yellow]LICENSED") + initial_drm_licensed = True + initial_drm_key = media_playlist_key + session_drm = media_drm + except Exception: # noqa + DOWNLOAD_CANCELLED.set() # skip pending track downloads + progress(downloaded="[red]FAILED") + raise + + # Fall back to session DRM if media playlist has no matching keys + if not initial_drm_licensed and session_drm and isinstance(session_drm, (Widevine, PlayReady)): + try: + if not license_widevine: + raise ValueError("license_widevine func must be supplied to use DRM") + track_kid = HLS.get_track_kid_from_init(master, track, session) or session_drm.kid + progress(downloaded="LICENSING") + license_widevine(session_drm, track_kid=track_kid) + progress(downloaded="[yellow]LICENSED") + except Exception: # noqa + DOWNLOAD_CANCELLED.set() # skip pending track downloads + progress(downloaded="[red]FAILED") + raise + + if not initial_drm_licensed and session_drm and isinstance(session_drm, MonaLisa): + try: + if not license_widevine: + raise ValueError("license_widevine func must be supplied to use DRM") + progress(downloaded="LICENSING") + license_widevine(session_drm) + progress(downloaded="[yellow]LICENSED") + except Exception: # noqa + DOWNLOAD_CANCELLED.set() # skip pending track downloads + progress(downloaded="[red]FAILED") + raise + + if DOWNLOAD_LICENCE_ONLY.is_set(): + progress(downloaded="[yellow]SKIPPED") + return + + unwanted_segments = [ + segment for segment in master.segments if callable(track.OnSegmentFilter) and track.OnSegmentFilter(segment) + ] + + total_segments = len(master.segments) - len(unwanted_segments) + progress(total=total_segments) + + downloader = track.downloader + + urls: list[dict[str, Any]] = [] + segment_durations: list[int] = [] + + range_offset = 0 + for segment in master.segments: + if segment in unwanted_segments: + continue + + segment_durations.append(int(segment.duration)) + + if segment.byterange: + byte_range = HLS.calculate_byte_range(segment.byterange, range_offset) + range_offset = int(byte_range.split("-")[0]) + else: + byte_range = None + + urls.append( + { + "url": urljoin(segment.base_uri, segment.uri), + "headers": {"Range": f"bytes={byte_range}"} if byte_range else {}, + } + ) + + track.data["hls"]["segment_durations"] = segment_durations + + segment_save_dir = save_dir / "segments" + + downloader_args = dict( + urls=urls, + output_dir=segment_save_dir, + filename="{i:0%d}{ext}" % len(str(len(urls))), + headers=session.headers, + cookies=session.cookies, + proxy=proxy, + max_workers=max_workers, + session=session, + ) + + debug_logger = get_debug_logger() + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="manifest_hls_download_start", + message="Starting HLS manifest download", + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "total_segments": total_segments, + "has_drm": bool(session_drm), + "drm_type": session_drm.__class__.__name__ if session_drm else None, + "save_path": str(save_path), + }, + ) + + for status_update in downloader(**downloader_args): + file_downloaded = status_update.get("file_downloaded") + if file_downloaded: + events.emit(events.Types.SEGMENT_DOWNLOADED, track=track, segment=file_downloaded) + else: + downloaded = status_update.get("downloaded") + if downloaded and downloaded.endswith("/s"): + status_update["downloaded"] = f"HLS {downloaded}" + progress(**status_update) + + for control_file in segment_save_dir.glob("*.!dev"): + control_file.unlink(missing_ok=True) + + progress(total=total_segments, completed=0, downloaded="Merging") + + name_len = len(str(total_segments)) + discon_i = 0 + range_offset = 0 + map_data: Optional[tuple[m3u8.model.InitializationSection, bytes]] = None + if session_drm: + encryption_data: Optional[tuple[Optional[m3u8.Key], DRM_T]] = (initial_drm_key, session_drm) + else: + encryption_data: Optional[tuple[Optional[m3u8.Key], DRM_T]] = None + + i = -1 + for real_i, segment in enumerate(master.segments): + if segment not in unwanted_segments: + i += 1 + + is_last_segment = (real_i + 1) == len(master.segments) + + def merge(to: Path, via: list[Path], delete: bool = False, include_map_data: bool = False): + """ + Merge all files to a given path, optionally including map data. + + Parameters: + to: The output file with all merged data. + via: List of files to merge, in sequence. + delete: Delete the file once it's been merged. + include_map_data: Whether to include the init map data. + """ + with open(to, "wb") as x: + if include_map_data and map_data and map_data[1]: + x.write(map_data[1]) + for file in via: + x.write(file.read_bytes()) + x.flush() + if delete: + file.unlink() + + def decrypt(include_this_segment: bool) -> Path: + """ + Decrypt all segments that uses the currently set DRM. + + All segments that will be decrypted with this DRM will be merged together + in sequence, prefixed with the init data (if any), and then deleted. Once + merged they will be decrypted. The merged and decrypted file names state + the range of segments that were used. + + Parameters: + include_this_segment: Whether to include the current segment in the + list of segments to merge and decrypt. This should be False if + decrypting on EXT-X-KEY changes, or True when decrypting on the + last segment. + + Returns the decrypted path. + """ + drm = encryption_data[1] + first_segment_i = next( + int(file.stem) for file in sorted(segment_save_dir.iterdir()) if file.stem.isdigit() + ) + last_segment_i = max(0, i - int(not include_this_segment)) + range_len = (last_segment_i - first_segment_i) + 1 + + segment_range = f"{str(first_segment_i).zfill(name_len)}-{str(last_segment_i).zfill(name_len)}" + merged_path = segment_save_dir / f"{segment_range}{get_extension(master.segments[last_segment_i].uri)}" + decrypted_path = segment_save_dir / f"{merged_path.stem}_decrypted{merged_path.suffix}" + + files = [ + file + for file in sorted(segment_save_dir.iterdir()) + if file.stem.isdigit() and first_segment_i <= int(file.stem) <= last_segment_i + ] + if not files: + raise ValueError(f"None of the segment files for {segment_range} exist...") + elif len(files) != range_len: + raise ValueError(f"Missing {range_len - len(files)} segment files for {segment_range}...") + + if isinstance(drm, (Widevine, PlayReady)): + # with widevine we can merge all segments and decrypt once + merge(to=merged_path, via=files, delete=True, include_map_data=True) + drm.decrypt(merged_path) + merged_path.rename(decrypted_path) + else: + # with other drm we must decrypt separately and then merge them + # for aes this is because each segment likely has 16-byte padding + for file in files: + drm.decrypt(file) + merge(to=merged_path, via=files, delete=True, include_map_data=True) + + events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=drm, segment=decrypted_path) + + return decrypted_path + + def merge_discontinuity(include_this_segment: bool, include_map_data: bool = True): + """ + Merge all segments of the discontinuity. + + All segment files for this discontinuity must already be downloaded and + already decrypted (if it needs to be decrypted). + + Parameters: + include_this_segment: Whether to include the current segment in the + list of segments to merge and decrypt. This should be False if + decrypting on EXT-X-KEY changes, or True when decrypting on the + last segment. + include_map_data: Whether to prepend the init map data before the + segment files when merging. + """ + last_segment_i = max(0, i - int(not include_this_segment)) + + files = [ + file + for file in sorted(segment_save_dir.iterdir()) + if int(file.stem.replace("_decrypted", "").split("-")[-1]) <= last_segment_i + ] + if files: + to_dir = segment_save_dir.parent + to_path = to_dir / f"{str(discon_i).zfill(name_len)}{files[-1].suffix}" + merge(to=to_path, via=files, delete=True, include_map_data=include_map_data) + + if segment not in unwanted_segments: + if isinstance(track, Subtitle): + segment_file_ext = get_extension(segment.uri) + segment_file_path = segment_save_dir / f"{str(i).zfill(name_len)}{segment_file_ext}" + segment_data = try_ensure_utf8(segment_file_path.read_bytes()) + if track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML): + segment_data = ( + segment_data.decode("utf8") + .replace("‎", html.unescape("‎")) + .replace("‏", html.unescape("‏")) + .encode("utf8") + ) + segment_file_path.write_bytes(segment_data) + + if segment.discontinuity and i != 0: + if encryption_data: + decrypt(include_this_segment=False) + merge_discontinuity( + include_this_segment=False, include_map_data=not encryption_data or not encryption_data[1] + ) + + discon_i += 1 + range_offset = 0 # TODO: Should this be reset or not? + map_data = None + + if segment.init_section and (not map_data or segment.init_section != map_data[0]): + if segment.init_section.byterange: + init_byte_range = HLS.calculate_byte_range(segment.init_section.byterange, range_offset) + range_offset = int(init_byte_range.split("-")[0]) + init_range_header = {"Range": f"bytes={init_byte_range}"} + else: + init_range_header = {} + + # Handle both session types for init section request + res = session.get( + url=urljoin(segment.init_section.base_uri, segment.init_section.uri), + headers=init_range_header, + ) + + # Check response based on session type + if isinstance(res, requests.Response) or isinstance(res, RnetResponse): + res.raise_for_status() + init_content = res.content + else: + raise TypeError(f"Expected response to be requests.Response or rnet.Response, not {type(res)}") + + map_data = (segment.init_section, init_content) + + segment_keys = getattr(segment, "keys", None) + if segment_keys: + if cdm: + cdm_segment_keys = HLS.filter_keys_for_cdm(segment_keys, cdm) + key = ( + HLS.get_supported_key(cdm_segment_keys) + if cdm_segment_keys + else HLS.get_supported_key(segment_keys) + ) + else: + key = HLS.get_supported_key(segment_keys) + if encryption_data and encryption_data[0] != key and i != 0 and segment not in unwanted_segments: + decrypt(include_this_segment=False) + + if key is None: + encryption_data = None + elif not encryption_data or encryption_data[0] != key: + drm = HLS.get_drm(key, session) + if isinstance(drm, (Widevine, PlayReady)): + try: + if map_data: + track_kid = track.get_key_id(map_data[1]) + else: + track_kid = None + if not track_kid: + track_kid = drm.kid + progress(downloaded="LICENSING") + license_widevine(drm, track_kid=track_kid) + progress(downloaded="[yellow]LICENSED") + except Exception: # noqa + DOWNLOAD_CANCELLED.set() # skip pending track downloads + progress(downloaded="[red]FAILED") + raise + if ( + encryption_data + and isinstance(drm, (Widevine, PlayReady)) + and isinstance(encryption_data[1], type(drm)) + and getattr(encryption_data[1], "content_keys", None) + ): + for prev_kid, prev_key in encryption_data[1].content_keys.items(): + drm.content_keys.setdefault(prev_kid, prev_key) + encryption_data = (key, drm) + + if DOWNLOAD_LICENCE_ONLY.is_set(): + continue + + if is_last_segment: + # required as it won't end with EXT-X-DISCONTINUITY nor a new key + if encryption_data: + decrypt(include_this_segment=True) + merge_discontinuity( + include_this_segment=True, include_map_data=not encryption_data or not encryption_data[1] + ) + + progress(advance=1) + + if DOWNLOAD_LICENCE_ONLY.is_set(): + return + + def find_segments_recursively(directory: Path) -> list[Path]: + """Find all segment files recursively in any directory structure created by downloaders.""" + segments = [] + + # First check direct files in the directory + if directory.exists(): + segments.extend([x for x in directory.iterdir() if x.is_file()]) + + # If no direct files, recursively search subdirectories + if not segments: + for subdir in directory.iterdir(): + if subdir.is_dir(): + segments.extend(find_segments_recursively(subdir)) + + return sorted(segments) + + # finally merge all the discontinuity save files together to the final path + segments_to_merge = find_segments_recursively(save_dir) + + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="manifest_hls_download_complete", + message="HLS download complete, preparing to merge", + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "save_dir": str(save_dir), + "save_dir_exists": save_dir.exists(), + "segments_found": len(segments_to_merge), + "segment_files": [f.name for f in segments_to_merge[:10]], # Limit to first 10 + "downloader": "requests", + }, + ) + + if not segments_to_merge: + error_msg = f"No segment files found in output directory: {save_dir}" + if debug_logger: + all_contents = list(save_dir.iterdir()) if save_dir.exists() else [] + debug_logger.log( + level="ERROR", + operation="manifest_hls_download_no_segments", + message=error_msg, + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "save_dir": str(save_dir), + "save_dir_exists": save_dir.exists(), + "directory_contents": [str(p) for p in all_contents], + "downloader": "requests", + }, + ) + raise FileNotFoundError(error_msg) + + if len(segments_to_merge) == 1: + shutil.move(segments_to_merge[0], save_path) + else: + progress(downloaded="Merging") + if isinstance(track, (Video, Audio)): + HLS.merge_segments(segments=segments_to_merge, save_path=save_path) + else: + with open(save_path, "wb") as f: + for discontinuity_file in segments_to_merge: + discontinuity_data = discontinuity_file.read_bytes() + f.write(discontinuity_data) + f.flush() + os.fsync(f.fileno()) + discontinuity_file.unlink() + + # Clean up empty segment directory + if save_dir.exists() and save_dir.name.endswith("_segments"): + try: + save_dir.rmdir() + except OSError: + # Directory might not be empty, try removing recursively + shutil.rmtree(save_dir, ignore_errors=True) + + progress(downloaded="Downloaded") + + track.path = save_path + + if session_drm: + track.drm = None + + events.emit(events.Types.TRACK_DOWNLOADED, track=track) + + @staticmethod + def merge_segments(segments: list[Path], save_path: Path) -> int: + """ + Concatenate Segments using FFmpeg concat with binary fallback. + + Returns the file size of the merged file. + """ + # Track segment directories for cleanup + segment_dirs = set() + for segment in segments: + # Track all parent directories that contain segments + current_dir = segment.parent + while current_dir.name and "_segments" in str(current_dir): + segment_dirs.add(current_dir) + current_dir = current_dir.parent + + def cleanup_segments_and_dirs(): + """Clean up segments and directories after successful merge.""" + for segment in segments: + segment.unlink(missing_ok=True) + for segment_dir in segment_dirs: + if segment_dir.exists(): + try: + shutil.rmtree(segment_dir) + except OSError: + pass # Directory cleanup failed, but merge succeeded + + # Try FFmpeg concat first (preferred method) + if binaries.FFMPEG: + try: + demuxer_file = save_path.parent / f"ffmpeg_concat_demuxer_{save_path.stem}.txt" + demuxer_file.write_text("\n".join([f"file '{segment.absolute()}'" for segment in segments])) + + subprocess.check_call( + [ + binaries.FFMPEG, + "-hide_banner", + "-loglevel", + "error", + "-f", + "concat", + "-safe", + "0", + "-i", + demuxer_file, + "-map", + "0", + "-c", + "copy", + save_path, + ], + timeout=300, # 5 minute timeout + ) + demuxer_file.unlink(missing_ok=True) + cleanup_segments_and_dirs() + return save_path.stat().st_size + + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, OSError) as e: + # FFmpeg failed, clean up demuxer file and fall back to binary concat + logging.getLogger("HLS").debug(f"FFmpeg concat failed ({e}), falling back to binary concatenation") + demuxer_file.unlink(missing_ok=True) + # Remove partial output file if it exists + save_path.unlink(missing_ok=True) + + # Fallback: Binary concatenation + logging.getLogger("HLS").debug(f"Using binary concatenation for {len(segments)} segments") + with open(save_path, "wb") as output_file: + for segment in segments: + with open(segment, "rb") as segment_file: + output_file.write(segment_file.read()) + + cleanup_segments_and_dirs() + return save_path.stat().st_size + + @staticmethod + def parse_session_data_keys( + manifest: M3U8, session: Optional[Union[Session, RnetSession]] = None + ) -> list[m3u8.model.Key]: + """Parse `com.apple.hls.keys` session data and return Key objects.""" + keys: list[m3u8.model.Key] = [] + + for data in getattr(manifest, "session_data", []) or []: + if getattr(data, "data_id", None) != "com.apple.hls.keys": + continue + + value = getattr(data, "value", None) + if not value and data.uri: + if not session: + session = Session() + res = session.get(urljoin(manifest.base_uri or "", data.uri)) + value = res.text + + if not value: + continue + + try: + decoded = base64.b64decode(value).decode() + except Exception: + decoded = value + + try: + items = json.loads(decoded) + except Exception: + continue + + for item in items if isinstance(items, list) else []: + if not isinstance(item, dict): + continue + key = m3u8.model.Key( + method=item.get("method"), + base_uri=manifest.base_uri or "", + uri=item.get("uri"), + keyformat=item.get("keyformat"), + keyformatversions=",".join(item.get("keyformatversion") or item.get("keyformatversions") or []), + ) + if key.method in {"AES-128", "ISO-23001-7"} or ( + key.keyformat + and key.keyformat.lower() + in { + WidevineCdm.urn, + PlayReadyCdm, + "com.microsoft.playready", + } + ): + keys.append(key) + + return keys + + @staticmethod + def filter_keys_for_cdm( + keys: list[Union[m3u8.model.SessionKey, m3u8.model.Key]], + cdm: object, + ) -> list[Union[m3u8.model.SessionKey, m3u8.model.Key]]: + """ + Filter EXT-X-KEY entries to only include those matching the CDM type. + + This ensures we select the correct DRM system (Widevine vs PlayReady) + based on what CDM is configured, avoiding license request failures. + """ + playready_urn = f"urn:uuid:{PR_PSSH.SYSTEM_ID}" + playready_keyformats = {playready_urn, "com.microsoft.playready"} + if is_widevine_cdm(cdm): + return [k for k in keys if k.keyformat and k.keyformat.lower() == WidevineCdm.urn] + elif is_playready_cdm(cdm): + return [k for k in keys if k.keyformat and k.keyformat.lower() in playready_keyformats] + return keys + + @staticmethod + def get_track_kid_from_init( + master: M3U8, + track: AnyTrack, + session: Union[Session, RnetSession], + ) -> Optional[UUID]: + """ + Extract the track's Key ID from its init segment (EXT-X-MAP). + + Returns None if no init segment exists or KID extraction fails. + The caller should fall back to drm.kid from the PSSH if this returns None. + """ + map_section = next((seg.init_section for seg in master.segments if seg.init_section), None) + if not map_section: + return None + + map_uri = urljoin(map_section.base_uri or master.base_uri or "", map_section.uri) + try: + if map_section.byterange: + byte_range = HLS.calculate_byte_range(map_section.byterange, 0) + headers = {"Range": f"bytes={byte_range}"} + else: + headers = {} + map_res = session.get(url=map_uri, headers=headers) + if map_res.ok: + return track.get_key_id(map_res.content) + except Exception: + pass + return None + + @staticmethod + def get_supported_key(keys: list[Union[m3u8.model.SessionKey, m3u8.model.Key]]) -> Optional[m3u8.Key]: + """ + Get a support Key System from a list of Key systems. + + Note that the key systems are chosen in an opinionated order. + + Returns None if one of the key systems is method=NONE, which means all segments + from hence forth should be treated as plain text until another key system is + encountered, unless it's also method=NONE. + + Raises NotImplementedError if none of the key systems are supported. + """ + if any(key.method == "NONE" for key in keys): + return None + + unsupported_systems = [] + for key in keys: + if not key: + continue + # TODO: Add a way to specify which supported key system to use + # TODO: Add support for 'SAMPLE-AES', 'AES-CTR', 'AES-CBC', 'ClearKey' + elif key.method == "AES-128": + return key + elif key.method == "ISO-23001-7": + return key + elif key.keyformat and key.keyformat.lower() == WidevineCdm.urn: + return key + elif key.keyformat and key.keyformat.lower() in { + f"urn:uuid:{PR_PSSH.SYSTEM_ID}", + "com.microsoft.playready", + }: + return key + else: + unsupported_systems.append(key.method + (f" ({key.keyformat})" if key.keyformat else "")) + else: + raise NotImplementedError(f"None of the key systems are supported: {', '.join(unsupported_systems)}") + + @staticmethod + def get_drm( + key: Union[m3u8.model.SessionKey, m3u8.model.Key], + session: Optional[Union[Session, RnetSession]] = None, + ) -> DRM_T: + """ + Convert HLS EXT-X-KEY data to an initialized DRM object. + + Parameters: + key: m3u8 key system (EXT-X-KEY) object. + session: Optional session used to request AES-128 URIs. + Useful to set headers, proxies, cookies, and so forth. + + Raises a NotImplementedError if the key system is not supported. + """ + if not isinstance(session, (Session, RnetSession, type(None))): + raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {type(session)}") + if not session: + session = Session() + + # TODO: Add support for 'SAMPLE-AES', 'AES-CTR', 'AES-CBC', 'ClearKey' + if key.method == "AES-128": + drm = ClearKey.from_m3u_key(key, session) + elif key.method == "ISO-23001-7": + drm = Widevine(pssh=WV_PSSH.new(key_ids=[key.uri.split(",")[-1]], system_id=WV_PSSH.SystemId.Widevine)) + elif key.keyformat and key.keyformat.lower() == WidevineCdm.urn: + drm = Widevine( + pssh=WV_PSSH(key.uri.split(",")[-1]), + **key._extra_params, # noqa + ) + elif key.keyformat and key.keyformat.lower() in {f"urn:uuid:{PR_PSSH.SYSTEM_ID}", "com.microsoft.playready"}: + drm = PlayReady( + pssh=PR_PSSH(key.uri.split(",")[-1]), + pssh_b64=key.uri.split(",")[-1], + ) + else: + raise NotImplementedError(f"The key system is not supported: {key}") + + return drm + + @staticmethod + def get_all_drm( + keys: list[Union[m3u8.model.SessionKey, m3u8.model.Key]], proxy: Optional[str] = None + ) -> list[DRM_T]: + """ + Convert HLS EXT-X-KEY data to initialized DRM objects. + + Parameters: + keys: m3u8 key system (EXT-X-KEY) objects. + proxy: Optional proxy string used for requesting AES-128 URIs. + + Raises a NotImplementedError if none of the key systems are supported. + """ + unsupported_keys: list[m3u8.Key] = [] + drm_objects: list[DRM_T] = [] + + if any(key.method == "NONE" for key in keys): + return [] + + for key in keys: + try: + drm = HLS.get_drm(key, proxy) + drm_objects.append(drm) + except NotImplementedError: + unsupported_keys.append(key) + + if not drm_objects and unsupported_keys: + logging.debug( + "Ignoring unsupported key systems: %s", + ", ".join([str(k.keyformat or k.method) for k in unsupported_keys]), + ) + return [] + + return drm_objects + + @staticmethod + def calculate_byte_range(m3u_range: str, fallback_offset: int = 0) -> str: + """ + Convert a HLS EXT-X-BYTERANGE value to a more traditional range value. + E.g., '1433@0' -> '0-1432', '357392@1433' -> '1433-358824'. + """ + parts = [int(x) for x in m3u_range.split("@")] + if len(parts) != 2: + parts.append(fallback_offset) + length, offset = parts + return f"{offset}-{offset + length - 1}" + + +__all__ = ("HLS",) diff --git a/reset/packages/envied/src/envied/core/manifests/ism.py b/reset/packages/envied/src/envied/core/manifests/ism.py new file mode 100644 index 0000000..2ba9378 --- /dev/null +++ b/reset/packages/envied/src/envied/core/manifests/ism.py @@ -0,0 +1,401 @@ +from __future__ import annotations + +import base64 +import hashlib +import html +import urllib.parse +from functools import partial +from pathlib import Path +from typing import Any, Callable, Optional, Union + +import requests +from langcodes import Language, tag_is_valid +from lxml.etree import Element +from pyplayready.system.pssh import PSSH as PR_PSSH +from pywidevine.pssh import PSSH +from requests import Session + +from envied.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack +from envied.core.drm import DRM_T, PlayReady, Widevine +from envied.core.events import events +from envied.core.session import RnetSession +from envied.core.tracks import Audio, Subtitle, Track, Tracks, Video +from envied.core.utilities import get_debug_logger, try_ensure_utf8 +from envied.core.utils.xml import load_xml + + +class ISM: + def __init__(self, manifest: Element, url: str) -> None: + if manifest.tag != "SmoothStreamingMedia": + raise TypeError(f"Expected 'SmoothStreamingMedia' document, got '{manifest.tag}'") + if not url: + raise requests.URLRequired("ISM manifest URL must be provided for relative paths") + self.manifest = manifest + self.url = url + + @classmethod + def from_url(cls, url: str, session: Optional[Union[Session, RnetSession]] = None, **kwargs: Any) -> "ISM": + if not url: + raise requests.URLRequired("ISM manifest URL must be provided") + if not session: + session = Session() + elif not isinstance(session, (Session, RnetSession)): + raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {session!r}") + res = session.get(url, **kwargs) + if res.url != url: + url = res.url + res.raise_for_status() + return cls(load_xml(res.content), url) + + @classmethod + def from_text(cls, text: str, url: str) -> "ISM": + if not text: + raise ValueError("ISM manifest text must be provided") + if not url: + raise requests.URLRequired("ISM manifest URL must be provided for relative paths") + return cls(load_xml(text), url) + + @staticmethod + def _get_drm(headers: list[Element]) -> list[DRM_T]: + drm: list[DRM_T] = [] + for header in headers: + system_id = (header.get("SystemID") or header.get("SystemId") or "").lower() + data = "".join(header.itertext()).strip() + if not data: + continue + if system_id == "edef8ba9-79d6-4ace-a3c8-27dcd51d21ed": + try: + pssh = PSSH(base64.b64decode(data)) + except Exception: + continue + kid = next(iter(pssh.key_ids), None) + drm.append(Widevine(pssh=pssh, kid=kid)) + elif system_id == "9a04f079-9840-4286-ab92-e65be0885f95": + try: + pr_pssh = PR_PSSH(data) + except Exception: + continue + drm.append(PlayReady(pssh=pr_pssh, pssh_b64=data)) + return drm + + def to_tracks(self, language: Optional[Union[str, Language]] = None) -> Tracks: + tracks = Tracks() + base_url = self.url + duration = int(self.manifest.get("Duration") or 0) + drm = self._get_drm(self.manifest.xpath(".//ProtectionHeader")) + + for stream_index in self.manifest.findall("StreamIndex"): + content_type = stream_index.get("Type") + if not content_type: + raise ValueError("No content type value could be found") + for ql in stream_index.findall("QualityLevel"): + codec = ql.get("FourCC") + if codec == "TTML": + codec = "STPP" + track_lang = None + lang = (stream_index.get("Language") or "").strip() + if lang and tag_is_valid(lang) and not lang.startswith("und"): + track_lang = Language.get(lang) + + track_urls: list[str] = [] + fragment_time = 0 + fragments = stream_index.findall("c") + # Some manifests omit the first fragment in the list but + # still expect a request for start time 0 which contains the + # initialization segment. If the first declared fragment is not + # at time 0, prepend the missing initialization URL. + if fragments: + first_time = int(fragments[0].get("t") or 0) + if first_time != 0: + track_urls.append( + urllib.parse.urljoin( + base_url, + stream_index.get("Url").format_map( + { + "bitrate": ql.get("Bitrate"), + "start time": "0", + } + ), + ) + ) + + for idx, frag in enumerate(fragments): + fragment_time = int(frag.get("t", fragment_time)) + repeat = int(frag.get("r", 1)) + duration_frag = int(frag.get("d") or 0) + if not duration_frag: + try: + next_time = int(fragments[idx + 1].get("t")) + except (IndexError, AttributeError): + next_time = duration + duration_frag = (next_time - fragment_time) / repeat + for _ in range(repeat): + track_urls.append( + urllib.parse.urljoin( + base_url, + stream_index.get("Url").format_map( + { + "bitrate": ql.get("Bitrate"), + "start time": str(fragment_time), + } + ), + ) + ) + fragment_time += duration_frag + + track_id = hashlib.md5( + "{codec}-{lang}-{bitrate}-{index}-{name}-{url}".format( + codec=codec, + lang=track_lang, + bitrate=ql.get("Bitrate") or 0, + index=ql.get("Index") or 0, + name=stream_index.get("Name") or "", + url=stream_index.get("Url") or "", + ).encode() + ).hexdigest() + + data = { + "ism": { + "manifest": self.manifest, + "stream_index": stream_index, + "quality_level": ql, + "segments": track_urls, + } + } + + if content_type == "video": + try: + vcodec = Video.Codec.from_mime(codec) if codec else None + except ValueError: + vcodec = None + tracks.add( + Video( + id_=track_id, + url=self.url, + codec=vcodec, + language=track_lang or language, + is_original_lang=bool(language and track_lang and str(track_lang) == str(language)), + bitrate=ql.get("Bitrate"), + width=int(ql.get("MaxWidth") or 0) or int(stream_index.get("MaxWidth") or 0), + height=int(ql.get("MaxHeight") or 0) or int(stream_index.get("MaxHeight") or 0), + descriptor=Video.Descriptor.ISM, + drm=drm, + data=data, + ) + ) + elif content_type == "audio": + try: + acodec = Audio.Codec.from_mime(codec) if codec else None + except ValueError: + acodec = None + tracks.add( + Audio( + id_=track_id, + url=self.url, + codec=acodec, + language=track_lang or language, + is_original_lang=bool(language and track_lang and str(track_lang) == str(language)), + bitrate=ql.get("Bitrate"), + channels=ql.get("Channels"), + descriptor=Track.Descriptor.ISM, + drm=drm, + data=data, + ) + ) + else: + try: + scodec = Subtitle.Codec.from_mime(codec) if codec else None + except ValueError: + scodec = None + tracks.add( + Subtitle( + id_=track_id, + url=self.url, + codec=scodec, + language=track_lang or language, + is_original_lang=bool(language and track_lang and str(track_lang) == str(language)), + descriptor=Track.Descriptor.ISM, + drm=drm, + data=data, + ) + ) + tracks.manifest_url = self.url + return tracks + + @staticmethod + def download_track( + track: AnyTrack, + save_path: Path, + save_dir: Path, + progress: partial, + session: Optional[Session] = None, + proxy: Optional[str] = None, + max_workers: Optional[int] = None, + license_widevine: Optional[Callable] = None, + *, + cdm: Optional[object] = None, + ) -> None: + if not session: + session = Session() + elif not isinstance(session, Session): + raise TypeError(f"Expected session to be a {Session}, not {session!r}") + + if proxy: + session.proxies.update({"all": proxy}) + + segments: list[str] = track.data["ism"]["segments"] + + session_drm = None + if track.drm: + # Mirror HLS.download_track: pick the DRM matching the provided CDM + # (or the first available) and license it if supported. + session_drm = track.get_drm_for_cdm(cdm) + if isinstance(session_drm, (Widevine, PlayReady)): + try: + if not license_widevine: + raise ValueError("license_widevine func must be supplied to use DRM") + progress(downloaded="LICENSING") + license_widevine(session_drm) + progress(downloaded="[yellow]LICENSED") + except Exception: + DOWNLOAD_CANCELLED.set() + progress(downloaded="[red]FAILED") + raise + + if DOWNLOAD_LICENCE_ONLY.is_set(): + progress(downloaded="[yellow]SKIPPED") + return + + progress(total=len(segments)) + + downloader = track.downloader + downloader_args = dict( + urls=[{"url": url} for url in segments], + output_dir=save_dir, + filename="{i:0%d}.mp4" % len(str(len(segments))), + headers=session.headers, + cookies=session.cookies, + proxy=proxy, + max_workers=max_workers, + session=session, + ) + + debug_logger = get_debug_logger() + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="manifest_ism_download_start", + message="Starting ISM manifest download", + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "total_segments": len(segments), + "downloader": "requests", + "has_drm": bool(session_drm), + "drm_type": session_drm.__class__.__name__ if session_drm else None, + "save_path": str(save_path), + }, + ) + + for status_update in downloader(**downloader_args): + file_downloaded = status_update.get("file_downloaded") + if file_downloaded: + events.emit(events.Types.SEGMENT_DOWNLOADED, track=track, segment=file_downloaded) + else: + downloaded = status_update.get("downloaded") + if downloaded and downloaded.endswith("/s"): + status_update["downloaded"] = f"ISM {downloaded}" + progress(**status_update) + + # Verify output directory exists and contains files + if not save_dir.exists(): + error_msg = f"Output directory does not exist: {save_dir}" + if debug_logger: + debug_logger.log( + level="ERROR", + operation="manifest_ism_download_output_missing", + message=error_msg, + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "save_dir": str(save_dir), + "save_path": str(save_path), + "downloader": "requests", + }, + ) + raise FileNotFoundError(error_msg) + + for control_file in save_dir.glob("*.!dev"): + control_file.unlink(missing_ok=True) + + segments_to_merge = [x for x in sorted(save_dir.iterdir()) if x.is_file()] + + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="manifest_ism_download_complete", + message="ISM download complete, preparing to merge", + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "save_dir": str(save_dir), + "save_dir_exists": save_dir.exists(), + "segments_found": len(segments_to_merge), + "segment_files": [f.name for f in segments_to_merge[:10]], # Limit to first 10 + "downloader": "requests", + }, + ) + + if not segments_to_merge: + error_msg = f"No segment files found in output directory: {save_dir}" + if debug_logger: + all_contents = list(save_dir.iterdir()) if save_dir.exists() else [] + debug_logger.log( + level="ERROR", + operation="manifest_ism_download_no_segments", + message=error_msg, + context={ + "track_id": getattr(track, "id", None), + "track_type": track.__class__.__name__, + "save_dir": str(save_dir), + "directory_contents": [str(p) for p in all_contents], + "downloader": "requests", + }, + ) + raise FileNotFoundError(error_msg) + + with open(save_path, "wb") as f: + for segment_file in segments_to_merge: + segment_data = segment_file.read_bytes() + if ( + not session_drm + and isinstance(track, Subtitle) + and track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML) + ): + segment_data = try_ensure_utf8(segment_data) + segment_data = ( + segment_data.decode("utf8") + .replace("‎", html.unescape("‎")) + .replace("‏", html.unescape("‏")) + .encode("utf8") + ) + f.write(segment_data) + f.flush() + segment_file.unlink() + progress(advance=1) + + track.path = save_path + events.emit(events.Types.TRACK_DOWNLOADED, track=track) + + if session_drm: + progress(downloaded="Decrypting", completed=0, total=100) + session_drm.decrypt(save_path) + track.drm = None + events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=session_drm, segment=None) + progress(downloaded="Decrypting", advance=100) + + save_dir.rmdir() + progress(downloaded="Downloaded") + + +__all__ = ("ISM",) diff --git a/reset/packages/envied/src/envied/core/manifests/m3u8.py b/reset/packages/envied/src/envied/core/manifests/m3u8.py new file mode 100644 index 0000000..a9111e3 --- /dev/null +++ b/reset/packages/envied/src/envied/core/manifests/m3u8.py @@ -0,0 +1,35 @@ +"""Utility functions for parsing M3U8 playlists.""" + +from __future__ import annotations + +from typing import Optional, Union + +import m3u8 +from requests import Session + +from envied.core.manifests.hls import HLS +from envied.core.session import RnetSession +from envied.core.tracks import Tracks + + +def parse( + master: m3u8.M3U8, + language: str, + *, + session: Optional[Union[Session, RnetSession]] = None, + url: Optional[str] = None, +) -> Tracks: + """Parse a variant playlist to ``Tracks`` with basic information, defer DRM loading.""" + tracks = HLS(master, session=session, url=url).to_tracks(language) + + bool(master.session_keys or HLS.parse_session_data_keys(master, session or Session())) + + if True: + for t in tracks.videos + tracks.audio: + t.needs_drm_loading = True + t.session = session + + return tracks + + +__all__ = ["parse"] diff --git a/reset/packages/envied/src/envied/core/providers/__init__.py b/reset/packages/envied/src/envied/core/providers/__init__.py new file mode 100644 index 0000000..10267b1 --- /dev/null +++ b/reset/packages/envied/src/envied/core/providers/__init__.py @@ -0,0 +1,428 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +import requests + +from envied.core.providers._base import ExternalIds, MetadataProvider, MetadataResult, fuzzy_match, log +from envied.core.providers.imdbapi import IMDBApiProvider +from envied.core.providers.simkl import SimklProvider +from envied.core.providers.tmdb import TMDBProvider + +if TYPE_CHECKING: + from envied.core.title_cacher import TitleCacher + +# Ordered by priority: IMDBApi (free), SIMKL, TMDB +ALL_PROVIDERS: list[type[MetadataProvider]] = [IMDBApiProvider, SimklProvider, TMDBProvider] + + +def get_available_providers() -> list[MetadataProvider]: + """Return instantiated providers that have valid credentials.""" + return [cls() for cls in ALL_PROVIDERS if cls().is_available()] + + +def get_provider(name: str) -> Optional[MetadataProvider]: + """Get a specific provider by name.""" + for cls in ALL_PROVIDERS: + if cls.NAME == name: + p = cls() + return p if p.is_available() else None + return None + + +# -- Public API (replaces tags.py functions) -- + + +def search_metadata( + title: str, + year: Optional[int], + kind: str, + title_cacher: Optional[TitleCacher] = None, + cache_title_id: Optional[str] = None, + cache_region: Optional[str] = None, + cache_account_hash: Optional[str] = None, +) -> Optional[MetadataResult]: + """Search all available providers for metadata. Returns best match.""" + # Check cache first + if title_cacher and cache_title_id: + for cls in ALL_PROVIDERS: + p = cls() + if not p.is_available(): + continue + cached = title_cacher.get_cached_provider(p.NAME, cache_title_id, kind, cache_region, cache_account_hash) + if cached: + result = _cached_to_result(cached, p.NAME, kind) + if result and result.title and fuzzy_match(result.title, title): + log.debug("Using cached %s data for %r", p.NAME, title) + return result + + # Search providers in priority order + for cls in ALL_PROVIDERS: + p = cls() + if not p.is_available(): + continue + try: + result = p.search(title, year, kind) + except (requests.RequestException, ValueError, KeyError) as exc: + log.debug("%s search failed: %s", p.NAME, exc) + continue + if result and result.title and fuzzy_match(result.title, title): + # Enrich with cross-referenced IDs if we have IMDB but missing TMDB/TVDB + enrich_ids(result) + # Cache the result (include enriched IDs so they survive round-trip) + if title_cacher and cache_title_id and result.raw: + try: + cache_data = result.raw + if result.external_ids.tmdb_id or result.external_ids.tvdb_id: + cache_data = { + **result.raw, + "_enriched_ids": _external_ids_to_dict(result.external_ids), + } + title_cacher.cache_provider( + p.NAME, cache_title_id, cache_data, kind, cache_region, cache_account_hash + ) + except Exception as exc: + log.debug("Failed to cache %s data: %s", p.NAME, exc) + return result + + return None + + +def get_title_by_id( + tmdb_id: int, + kind: str, + title_cacher: Optional[TitleCacher] = None, + cache_title_id: Optional[str] = None, + cache_region: Optional[str] = None, + cache_account_hash: Optional[str] = None, +) -> Optional[str]: + """Get title name by TMDB ID.""" + # Check cache first + if title_cacher and cache_title_id: + cached = title_cacher.get_cached_provider("tmdb", cache_title_id, kind, cache_region, cache_account_hash) + if cached and cached.get("detail"): + detail = cached["detail"] + tmdb_title = detail.get("title") or detail.get("name") + if tmdb_title: + log.debug("Using cached TMDB title: %r", tmdb_title) + return tmdb_title + + tmdb = get_provider("tmdb") + if not tmdb: + return None + result = tmdb.get_by_id(tmdb_id, kind) + if not result: + return None + + # Cache if possible + if title_cacher and cache_title_id and result.raw: + try: + ext_ids = tmdb.get_external_ids(tmdb_id, kind) + title_cacher.cache_provider( + "tmdb", + cache_title_id, + {"detail": result.raw, "external_ids": _external_ids_to_dict(ext_ids)}, + kind, + cache_region, + cache_account_hash, + ) + except Exception as exc: + log.debug("Failed to cache TMDB data: %s", exc) + + return result.title + + +def get_year_by_id( + tmdb_id: int, + kind: str, + title_cacher: Optional[TitleCacher] = None, + cache_title_id: Optional[str] = None, + cache_region: Optional[str] = None, + cache_account_hash: Optional[str] = None, +) -> Optional[int]: + """Get release year by TMDB ID.""" + # Check cache first + if title_cacher and cache_title_id: + cached = title_cacher.get_cached_provider("tmdb", cache_title_id, kind, cache_region, cache_account_hash) + if cached and cached.get("detail"): + detail = cached["detail"] + date = detail.get("release_date") or detail.get("first_air_date") + if date and len(date) >= 4 and date[:4].isdigit(): + year = int(date[:4]) + log.debug("Using cached TMDB year: %d", year) + return year + + tmdb = get_provider("tmdb") + if not tmdb: + return None + result = tmdb.get_by_id(tmdb_id, kind) + if not result: + return None + + # Cache if possible + if title_cacher and cache_title_id and result.raw: + try: + ext_ids = tmdb.get_external_ids(tmdb_id, kind) + title_cacher.cache_provider( + "tmdb", + cache_title_id, + {"detail": result.raw, "external_ids": _external_ids_to_dict(ext_ids)}, + kind, + cache_region, + cache_account_hash, + ) + except Exception as exc: + log.debug("Failed to cache TMDB data: %s", exc) + + return result.year + + +def fetch_external_ids( + tmdb_id: int, + kind: str, + title_cacher: Optional[TitleCacher] = None, + cache_title_id: Optional[str] = None, + cache_region: Optional[str] = None, + cache_account_hash: Optional[str] = None, +) -> ExternalIds: + """Get external IDs by TMDB ID.""" + # Check cache first + if title_cacher and cache_title_id: + cached = title_cacher.get_cached_provider("tmdb", cache_title_id, kind, cache_region, cache_account_hash) + if cached and cached.get("external_ids"): + log.debug("Using cached TMDB external IDs") + raw = cached["external_ids"] + return ExternalIds( + imdb_id=raw.get("imdb_id"), + tmdb_id=tmdb_id, + tmdb_kind=kind, + tvdb_id=raw.get("tvdb_id"), + ) + + tmdb = get_provider("tmdb") + if not tmdb: + return ExternalIds() + ext = tmdb.get_external_ids(tmdb_id, kind) + + # Cache if possible + if title_cacher and cache_title_id: + try: + detail = None + result = tmdb.get_by_id(tmdb_id, kind) + if result and result.raw: + detail = result.raw + if detail: + title_cacher.cache_provider( + "tmdb", + cache_title_id, + {"detail": detail, "external_ids": _external_ids_to_dict(ext)}, + kind, + cache_region, + cache_account_hash, + ) + except Exception as exc: + log.debug("Failed to cache TMDB data: %s", exc) + + return ext + + +# -- Internal helpers -- + + +# Provider authority ranking for tie-breaking (lower index = more authoritative) +_ENRICHMENT_PROVIDERS = ("tmdb", "simkl") +_ENRICHMENT_AUTHORITY: dict[str, int] = {name: i for i, name in enumerate(_ENRICHMENT_PROVIDERS)} + + +def enrich_ids(result: MetadataResult) -> None: + """Enrich a MetadataResult by cross-referencing IMDB ID with available providers. + + Queries all available providers, cross-validates tmdb_id as anchor. + If a provider returns a different tmdb_id than the authoritative source, + ALL of that provider's data is dropped (likely resolved to wrong title). + """ + ids = result.external_ids + if not ids.imdb_id: + return + if ids.tmdb_id and ids.tvdb_id: + return # already have everything + + kind = result.kind or "movie" + + # Step 1: Collect enrichment results from all available providers + enrichments: list[tuple[str, ExternalIds]] = [] + for provider_name in _ENRICHMENT_PROVIDERS: + p = get_provider(provider_name) + if not p: + continue + try: + enriched = p.find_by_imdb_id(ids.imdb_id, kind) # type: ignore[union-attr] + except Exception as exc: + log.debug("Enrichment via %s failed: %s", provider_name, exc) + continue + if enriched: + enrichments.append((provider_name, enriched)) + + if not enrichments: + return + + # Step 2: Cross-validate using tmdb_id as anchor — drop providers that disagree + validated = _validate_enrichments(enrichments) + + # Step 3: Merge validated data (fill gaps only) + for _provider_name, ext in validated: + if not ids.tmdb_id and ext.tmdb_id: + ids.tmdb_id = ext.tmdb_id + ids.tmdb_kind = ext.tmdb_kind or kind + if not ids.tvdb_id and ext.tvdb_id: + ids.tvdb_id = ext.tvdb_id + + +def _validate_enrichments( + enrichments: list[tuple[str, ExternalIds]], +) -> list[tuple[str, ExternalIds]]: + """Drop providers whose tmdb_id conflicts with the authoritative value. + + If providers disagree on tmdb_id, the more authoritative source wins + and ALL data from disagreeing providers is discarded (different tmdb_id + means the provider likely resolved to a different title entirely). + """ + from collections import Counter + + # Collect tmdb_id votes + tmdb_votes: dict[str, int] = {} + for provider_name, ext in enrichments: + if ext.tmdb_id is not None: + tmdb_votes[provider_name] = ext.tmdb_id + + if len(set(tmdb_votes.values())) <= 1: + return enrichments # all agree or only one voted — no conflict + + # Find the authoritative tmdb_id + value_counts = Counter(tmdb_votes.values()) + most_common_val, most_common_count = value_counts.most_common(1)[0] + + if most_common_count > 1: + anchor_tmdb_id = most_common_val + else: + # No majority — pick the most authoritative provider + best_provider = min( + tmdb_votes.keys(), + key=lambda name: _ENRICHMENT_AUTHORITY.get(name, 99), + ) + anchor_tmdb_id = tmdb_votes[best_provider] + + # Drop any provider that disagrees + validated: list[tuple[str, ExternalIds]] = [] + for provider_name, ext in enrichments: + if ext.tmdb_id is not None and ext.tmdb_id != anchor_tmdb_id: + log.debug( + "Dropping %s enrichment data: tmdb_id %s conflicts with " + "authoritative value %s (likely resolved to wrong title)", + provider_name, + ext.tmdb_id, + anchor_tmdb_id, + ) + continue + validated.append((provider_name, ext)) + + return validated + + +def _external_ids_to_dict(ext: ExternalIds) -> dict: + """Convert ExternalIds to a dict for caching.""" + result: dict = {} + if ext.imdb_id: + result["imdb_id"] = ext.imdb_id + if ext.tmdb_id: + result["tmdb_id"] = ext.tmdb_id + if ext.tmdb_kind: + result["tmdb_kind"] = ext.tmdb_kind + if ext.tvdb_id: + result["tvdb_id"] = ext.tvdb_id + return result + + +def _cached_to_result(cached: dict, provider_name: str, kind: str) -> Optional[MetadataResult]: + """Convert a cached provider dict back to a MetadataResult.""" + if provider_name == "tmdb": + detail = cached.get("detail", {}) + ext_raw = cached.get("external_ids", {}) + title = detail.get("title") or detail.get("name") + date = detail.get("release_date") or detail.get("first_air_date") + year = int(date[:4]) if date and len(date) >= 4 and date[:4].isdigit() else None + tmdb_id = detail.get("id") + return MetadataResult( + title=title, + year=year, + kind=kind, + external_ids=ExternalIds( + imdb_id=ext_raw.get("imdb_id"), + tmdb_id=tmdb_id, + tmdb_kind=kind, + tvdb_id=ext_raw.get("tvdb_id"), + ), + source="tmdb", + raw=cached, + ) + elif provider_name == "simkl": + response = cached.get("response", cached) + if response.get("type") == "episode" and "show" in response: + info = response["show"] + elif response.get("type") == "movie" and "movie" in response: + info = response["movie"] + else: + return None + ids = info.get("ids", {}) + tmdb_id = ids.get("tmdbtv") or ids.get("tmdb") or ids.get("moviedb") + if tmdb_id: + tmdb_id = int(tmdb_id) + return MetadataResult( + title=info.get("title"), + year=info.get("year"), + kind=kind, + external_ids=ExternalIds( + imdb_id=ids.get("imdb"), + tmdb_id=tmdb_id, + tmdb_kind=kind, + tvdb_id=ids.get("tvdb"), + ), + source="simkl", + raw=cached, + ) + elif provider_name == "imdbapi": + title = cached.get("primaryTitle") or cached.get("originalTitle") + year = cached.get("startYear") + imdb_id = cached.get("id") + # Restore enriched IDs that were saved alongside the raw data + enriched = cached.get("_enriched_ids", {}) + return MetadataResult( + title=title, + year=year, + kind=kind, + external_ids=ExternalIds( + imdb_id=imdb_id, + tmdb_id=enriched.get("tmdb_id"), + tmdb_kind=enriched.get("tmdb_kind"), + tvdb_id=enriched.get("tvdb_id"), + ), + source="imdbapi", + raw=cached, + ) + return None + + +__all__ = [ + "ALL_PROVIDERS", + "ExternalIds", + "MetadataProvider", + "MetadataResult", + "enrich_ids", + "fetch_external_ids", + "fuzzy_match", + "get_available_providers", + "get_provider", + "get_title_by_id", + "get_year_by_id", + "search_metadata", +] diff --git a/reset/packages/envied/src/envied/core/providers/_base.py b/reset/packages/envied/src/envied/core/providers/_base.py new file mode 100644 index 0000000..431b551 --- /dev/null +++ b/reset/packages/envied/src/envied/core/providers/_base.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import logging +import re +from abc import ABCMeta, abstractmethod +from dataclasses import dataclass, field +from difflib import SequenceMatcher +from typing import Optional, Union + +import requests +from requests.adapters import HTTPAdapter, Retry + +log = logging.getLogger("METADATA") + +HEADERS = {"User-Agent": "envied.tags/1.0"} + +STRIP_RE = re.compile(r"[^a-z0-9]+", re.I) +YEAR_RE = re.compile(r"\s*\(?[12][0-9]{3}\)?$") + + +@dataclass +class ExternalIds: + """Normalized external IDs across providers.""" + + imdb_id: Optional[str] = None + tmdb_id: Optional[int] = None + tmdb_kind: Optional[str] = None # "movie" or "tv" + tvdb_id: Optional[int] = None + + +@dataclass +class MetadataResult: + """Unified metadata result from any provider.""" + + title: Optional[str] = None + year: Optional[int] = None + kind: Optional[str] = None # "movie" or "tv" + external_ids: ExternalIds = field(default_factory=ExternalIds) + source: str = "" # provider name, e.g. "tmdb", "simkl", "imdbapi" + raw: Optional[dict] = None # original API response for caching + + +class MetadataProvider(metaclass=ABCMeta): + """Abstract base for metadata providers.""" + + NAME: str = "" + REQUIRES_KEY: bool = True + + def __init__(self) -> None: + self.log = logging.getLogger(f"METADATA.{self.NAME.upper()}") + self._session: Optional[requests.Session] = None + + @property + def session(self) -> requests.Session: + if self._session is None: + self._session = requests.Session() + self._session.headers.update(HEADERS) + retry = Retry( + total=3, + backoff_factor=1, + status_forcelist=[429, 500, 502, 503, 504], + allowed_methods=["GET", "POST"], + ) + adapter = HTTPAdapter(max_retries=retry) + self._session.mount("https://", adapter) + self._session.mount("http://", adapter) + return self._session + + @abstractmethod + def is_available(self) -> bool: + """Return True if this provider has the credentials/keys it needs.""" + + @abstractmethod + def search(self, title: str, year: Optional[int], kind: str) -> Optional[MetadataResult]: + """Search for a title and return metadata, or None on failure/no match.""" + + @abstractmethod + def get_by_id(self, provider_id: Union[int, str], kind: str) -> Optional[MetadataResult]: + """Fetch metadata by this provider's native ID.""" + + @abstractmethod + def get_external_ids(self, provider_id: Union[int, str], kind: str) -> ExternalIds: + """Fetch external IDs for a title by this provider's native ID.""" + + +def _clean(s: str) -> str: + return STRIP_RE.sub("", s).lower() + + +def _strip_year(s: str) -> str: + return YEAR_RE.sub("", s).strip() + + +def fuzzy_match(a: str, b: str, threshold: float = 0.8) -> bool: + """Return True if ``a`` and ``b`` are a close match.""" + ratio = SequenceMatcher(None, _clean(a), _clean(b)).ratio() + return ratio >= threshold diff --git a/reset/packages/envied/src/envied/core/providers/imdbapi.py b/reset/packages/envied/src/envied/core/providers/imdbapi.py new file mode 100644 index 0000000..929b917 --- /dev/null +++ b/reset/packages/envied/src/envied/core/providers/imdbapi.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +from difflib import SequenceMatcher +from typing import Optional, Union + +import requests + +from envied.core.providers._base import ExternalIds, MetadataProvider, MetadataResult, _clean, fuzzy_match + +# Mapping from our kind ("movie"/"tv") to imdbapi.dev title types +KIND_TO_TYPES: dict[str, list[str]] = { + "movie": ["movie"], + "tv": ["tvSeries", "tvMiniSeries"], +} + + +class IMDBApiProvider(MetadataProvider): + """IMDb metadata provider using imdbapi.dev (free, no API key).""" + + NAME = "imdbapi" + REQUIRES_KEY = False + BASE_URL = "https://api.imdbapi.dev" + + def is_available(self) -> bool: + return True # no key needed + + def search(self, title: str, year: Optional[int], kind: str) -> Optional[MetadataResult]: + self.log.debug("Searching IMDBApi for %r (%s, %s)", title, kind, year) + + try: + params: dict[str, str | int] = {"query": title, "limit": 20} + r = self.session.get( + f"{self.BASE_URL}/search/titles", + params=params, + timeout=30, + ) + r.raise_for_status() + data = r.json() + except (requests.RequestException, ValueError) as exc: + self.log.debug("IMDBApi search failed: %s", exc) + return None + + results = data.get("titles") or data.get("results") or [] + if not results: + self.log.debug("IMDBApi returned no results for %r", title) + return None + + # Filter by type if possible + type_filter = KIND_TO_TYPES.get(kind, []) + filtered = [r for r in results if r.get("type") in type_filter] if type_filter else results + candidates = filtered if filtered else results + + # Find best fuzzy match, optionally filtered by year + best_match: Optional[dict] = None + best_ratio = 0.0 + + for candidate in candidates: + primary = candidate.get("primaryTitle") or "" + original = candidate.get("originalTitle") or "" + + for name in [primary, original]: + if not name: + continue + ratio = SequenceMatcher(None, _clean(title), _clean(name)).ratio() + if ratio > best_ratio: + # If year provided, prefer matches within 1 year + candidate_year = candidate.get("startYear") + if year and candidate_year and abs(year - candidate_year) > 1: + continue + best_ratio = ratio + best_match = candidate + + if not best_match: + self.log.debug("No matching result found in IMDBApi for %r", title) + return None + + result_title = best_match.get("primaryTitle") or best_match.get("originalTitle") + if not result_title or not fuzzy_match(result_title, title): + self.log.debug("IMDBApi title mismatch: searched %r, got %r", title, result_title) + return None + + imdb_id = best_match.get("id") + result_year = best_match.get("startYear") + + self.log.debug("IMDBApi -> %s (ID %s)", result_title, imdb_id) + + return MetadataResult( + title=result_title, + year=result_year, + kind=kind, + external_ids=ExternalIds(imdb_id=imdb_id), + source="imdbapi", + raw=best_match, + ) + + def get_by_id(self, provider_id: Union[int, str], kind: str) -> Optional[MetadataResult]: + """Fetch metadata by IMDB ID (e.g. 'tt1375666').""" + imdb_id = str(provider_id) + self.log.debug("Fetching IMDBApi title %s", imdb_id) + + try: + r = self.session.get(f"{self.BASE_URL}/titles/{imdb_id}", timeout=30) + r.raise_for_status() + data = r.json() + except (requests.RequestException, ValueError) as exc: + self.log.debug("IMDBApi get_by_id failed: %s", exc) + return None + + title = data.get("primaryTitle") or data.get("originalTitle") + result_year = data.get("startYear") + + return MetadataResult( + title=title, + year=result_year, + kind=kind, + external_ids=ExternalIds(imdb_id=data.get("id")), + source="imdbapi", + raw=data, + ) + + def get_external_ids(self, provider_id: Union[int, str], kind: str) -> ExternalIds: + """Return external IDs. For IMDB, the provider_id IS the IMDB ID.""" + return ExternalIds(imdb_id=str(provider_id)) diff --git a/reset/packages/envied/src/envied/core/providers/simkl.py b/reset/packages/envied/src/envied/core/providers/simkl.py new file mode 100644 index 0000000..1ab41f3 --- /dev/null +++ b/reset/packages/envied/src/envied/core/providers/simkl.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +from typing import Optional, Union + +import requests + +from envied.core.config import config +from envied.core.providers._base import ExternalIds, MetadataProvider, MetadataResult, fuzzy_match + + +class SimklProvider(MetadataProvider): + """SIMKL metadata provider (filename-based search).""" + + NAME = "simkl" + REQUIRES_KEY = True + BASE_URL = "https://api.simkl.com" + + def is_available(self) -> bool: + return bool(config.simkl_client_id) + + def search(self, title: str, year: Optional[int], kind: str) -> Optional[MetadataResult]: + self.log.debug("Searching Simkl for %r (%s, %s)", title, kind, year) + + # Construct appropriate filename based on type + filename = f"{title}" + if year: + filename = f"{title} {year}" + if kind == "tv": + filename += " S01E01.mkv" + else: + filename += " 2160p.mkv" + + try: + headers = {"simkl-api-key": config.simkl_client_id} + resp = self.session.post( + f"{self.BASE_URL}/search/file", json={"file": filename}, headers=headers, timeout=30 + ) + resp.raise_for_status() + data = resp.json() + self.log.debug("Simkl API response received") + except (requests.RequestException, ValueError) as exc: + self.log.debug("Simkl search failed: %s", exc) + return None + + # Handle case where SIMKL returns empty list (no results) + if isinstance(data, list): + self.log.debug("Simkl returned list (no matches) for %r", filename) + return None + + return self._parse_response(data, title, year, kind) + + def get_by_id(self, provider_id: Union[int, str], kind: str) -> Optional[MetadataResult]: + return None # SIMKL has no direct ID lookup used here + + def get_external_ids(self, provider_id: Union[int, str], kind: str) -> ExternalIds: + return ExternalIds() # IDs come from search() response + + def find_by_imdb_id(self, imdb_id: str, kind: str) -> Optional[ExternalIds]: + """Look up TMDB/TVDB IDs from an IMDB ID using SIMKL's /search/id and detail endpoints.""" + self.log.debug("Looking up IMDB ID %s on SIMKL", imdb_id) + headers = {"simkl-api-key": config.simkl_client_id} + + try: + r = self.session.get(f"{self.BASE_URL}/search/id", params={"imdb": imdb_id}, headers=headers, timeout=30) + r.raise_for_status() + data = r.json() + except (requests.RequestException, ValueError) as exc: + self.log.debug("SIMKL search/id failed: %s", exc) + return None + + if not isinstance(data, list) or not data: + self.log.debug("No SIMKL results for IMDB ID %s", imdb_id) + return None + + entry = data[0] + simkl_id = entry.get("ids", {}).get("simkl") + if not simkl_id: + return None + + # Map SIMKL type to endpoint + simkl_type = entry.get("type", "") + endpoint = "tv" if simkl_type in ("tv", "anime") else "movies" + + # Fetch full details to get cross-referenced IDs + try: + r2 = self.session.get( + f"{self.BASE_URL}/{endpoint}/{simkl_id}", + params={"extended": "full"}, + headers=headers, + timeout=30, + ) + r2.raise_for_status() + detail = r2.json() + except (requests.RequestException, ValueError) as exc: + self.log.debug("SIMKL detail fetch failed: %s", exc) + return None + + ids = detail.get("ids", {}) + tmdb_id: Optional[int] = None + raw_tmdb = ids.get("tmdb") + if raw_tmdb: + tmdb_id = int(raw_tmdb) + + tvdb_id: Optional[int] = None + raw_tvdb = ids.get("tvdb") + if raw_tvdb: + tvdb_id = int(raw_tvdb) + + self.log.debug("SIMKL find -> TMDB %s, TVDB %s for IMDB %s", tmdb_id, tvdb_id, imdb_id) + + return ExternalIds( + imdb_id=imdb_id, + tmdb_id=tmdb_id, + tmdb_kind=kind, + tvdb_id=tvdb_id, + ) + + def _parse_response( + self, data: dict, search_title: str, search_year: Optional[int], kind: str + ) -> Optional[MetadataResult]: + """Parse a SIMKL response into a MetadataResult.""" + if data.get("type") == "episode" and "show" in data: + info = data["show"] + content_type = "tv" + elif data.get("type") == "movie" and "movie" in data: + info = data["movie"] + content_type = "movie" + else: + return None + + result_title = info.get("title") + result_year = info.get("year") + + # Verify title matches + if not result_title or not fuzzy_match(result_title, search_title): + self.log.debug("Simkl title mismatch: searched %r, got %r", search_title, result_title) + return None + + # Verify year if provided (allow 1 year difference) + if search_year and result_year and abs(search_year - result_year) > 1: + self.log.debug("Simkl year mismatch: searched %d, got %d", search_year, result_year) + return None + + ids = info.get("ids", {}) + tmdb_id: Optional[int] = None + if content_type == "tv": + raw_tmdb = ids.get("tmdbtv") + else: + raw_tmdb = ids.get("tmdb") or ids.get("moviedb") + if raw_tmdb: + tmdb_id = int(raw_tmdb) + + tvdb_id: Optional[int] = None + raw_tvdb = ids.get("tvdb") + if raw_tvdb: + tvdb_id = int(raw_tvdb) + + self.log.debug("Simkl -> %s (TMDB ID %s)", result_title, tmdb_id) + + return MetadataResult( + title=result_title, + year=result_year, + kind=kind, + external_ids=ExternalIds( + imdb_id=ids.get("imdb"), + tmdb_id=tmdb_id, + tmdb_kind=kind, + tvdb_id=tvdb_id, + ), + source="simkl", + raw=data, + ) diff --git a/reset/packages/envied/src/envied/core/providers/tmdb.py b/reset/packages/envied/src/envied/core/providers/tmdb.py new file mode 100644 index 0000000..e159d4a --- /dev/null +++ b/reset/packages/envied/src/envied/core/providers/tmdb.py @@ -0,0 +1,199 @@ +from __future__ import annotations + +from difflib import SequenceMatcher +from typing import Optional, Union + +import requests + +from envied.core.config import config +from envied.core.providers._base import ExternalIds, MetadataProvider, MetadataResult, _clean, _strip_year + + +class TMDBProvider(MetadataProvider): + """TMDB (The Movie Database) metadata provider.""" + + NAME = "tmdb" + REQUIRES_KEY = True + BASE_URL = "https://api.themoviedb.org/3" + + def is_available(self) -> bool: + return bool(config.tmdb_api_key) + + @property + def _api_key(self) -> str: + return config.tmdb_api_key + + def search(self, title: str, year: Optional[int], kind: str) -> Optional[MetadataResult]: + search_title = _strip_year(title) + self.log.debug("Searching TMDB for %r (%s, %s)", search_title, kind, year) + + params: dict[str, str | int] = {"api_key": self._api_key, "query": search_title} + if year is not None: + params["year" if kind == "movie" else "first_air_date_year"] = year + + try: + r = self.session.get(f"{self.BASE_URL}/search/{kind}", params=params, timeout=30) + r.raise_for_status() + results = r.json().get("results") or [] + self.log.debug("TMDB returned %d results", len(results)) + if not results: + return None + except requests.RequestException as exc: + self.log.warning("Failed to search TMDB for %s: %s", title, exc) + return None + + best_ratio = 0.0 + best_id: Optional[int] = None + best_title: Optional[str] = None + for result in results: + candidates = [ + result.get("title"), + result.get("name"), + result.get("original_title"), + result.get("original_name"), + ] + candidates = [c for c in candidates if c] + + for candidate in candidates: + ratio = SequenceMatcher(None, _clean(search_title), _clean(candidate)).ratio() + if ratio > best_ratio: + best_ratio = ratio + best_id = result.get("id") + best_title = candidate + + self.log.debug("Best candidate ratio %.2f for %r (ID %s)", best_ratio, best_title, best_id) + + if best_id is None: + first = results[0] + best_id = first.get("id") + best_title = first.get("title") or first.get("name") + + if best_id is None: + return None + + # Fetch full detail for caching + detail = self._fetch_detail(best_id, kind) + ext_raw = self._fetch_external_ids_raw(best_id, kind) + + date = (detail or {}).get("release_date") or (detail or {}).get("first_air_date") + result_year = int(date[:4]) if date and len(date) >= 4 and date[:4].isdigit() else None + + ext = ExternalIds( + imdb_id=ext_raw.get("imdb_id") if ext_raw else None, + tmdb_id=best_id, + tmdb_kind=kind, + tvdb_id=ext_raw.get("tvdb_id") if ext_raw else None, + ) + + return MetadataResult( + title=best_title, + year=result_year, + kind=kind, + external_ids=ext, + source="tmdb", + raw={"detail": detail or {}, "external_ids": ext_raw or {}}, + ) + + def get_by_id(self, provider_id: Union[int, str], kind: str) -> Optional[MetadataResult]: + detail = self._fetch_detail(int(provider_id), kind) + if not detail: + return None + + title = detail.get("title") or detail.get("name") + date = detail.get("release_date") or detail.get("first_air_date") + year = int(date[:4]) if date and len(date) >= 4 and date[:4].isdigit() else None + + return MetadataResult( + title=title, + year=year, + kind=kind, + external_ids=ExternalIds(tmdb_id=int(provider_id), tmdb_kind=kind), + source="tmdb", + raw=detail, + ) + + def get_external_ids(self, provider_id: Union[int, str], kind: str) -> ExternalIds: + raw = self._fetch_external_ids_raw(int(provider_id), kind) + if not raw: + return ExternalIds(tmdb_id=int(provider_id), tmdb_kind=kind) + return ExternalIds( + imdb_id=raw.get("imdb_id"), + tmdb_id=int(provider_id), + tmdb_kind=kind, + tvdb_id=raw.get("tvdb_id"), + ) + + def find_by_imdb_id(self, imdb_id: str, kind: str) -> Optional[ExternalIds]: + """Look up TMDB/TVDB IDs from an IMDB ID using TMDB's /find endpoint.""" + self.log.debug("Looking up IMDB ID %s on TMDB", imdb_id) + try: + r = self.session.get( + f"{self.BASE_URL}/find/{imdb_id}", + params={"api_key": self._api_key, "external_source": "imdb_id"}, + timeout=30, + ) + r.raise_for_status() + data = r.json() + except requests.RequestException as exc: + self.log.debug("TMDB find by IMDB ID failed: %s", exc) + return None + + # Check movie_results or tv_results based on kind + if kind == "movie": + results = data.get("movie_results") or [] + else: + results = data.get("tv_results") or [] + + if not results: + # Try the other type as fallback + fallback_key = "tv_results" if kind == "movie" else "movie_results" + results = data.get(fallback_key) or [] + if results: + kind = "tv" if kind == "movie" else "movie" + + if not results: + self.log.debug("No TMDB results found for IMDB ID %s", imdb_id) + return None + + match = results[0] + tmdb_id = match.get("id") + if not tmdb_id: + return None + + self.log.debug("TMDB find -> ID %s (%s) for IMDB %s", tmdb_id, kind, imdb_id) + + # Now fetch the full external IDs from TMDB to get TVDB etc. + ext_raw = self._fetch_external_ids_raw(tmdb_id, kind) + + return ExternalIds( + imdb_id=imdb_id, + tmdb_id=tmdb_id, + tmdb_kind=kind, + tvdb_id=ext_raw.get("tvdb_id") if ext_raw else None, + ) + + def _fetch_detail(self, tmdb_id: int, kind: str) -> Optional[dict]: + try: + r = self.session.get( + f"{self.BASE_URL}/{kind}/{tmdb_id}", + params={"api_key": self._api_key}, + timeout=30, + ) + r.raise_for_status() + return r.json() + except requests.RequestException as exc: + self.log.debug("Failed to fetch TMDB detail: %s", exc) + return None + + def _fetch_external_ids_raw(self, tmdb_id: int, kind: str) -> Optional[dict]: + try: + r = self.session.get( + f"{self.BASE_URL}/{kind}/{tmdb_id}/external_ids", + params={"api_key": self._api_key}, + timeout=30, + ) + r.raise_for_status() + return r.json() + except requests.RequestException as exc: + self.log.debug("Failed to fetch TMDB external IDs: %s", exc) + return None diff --git a/reset/packages/envied/src/envied/core/proxies/__init__.py b/reset/packages/envied/src/envied/core/proxies/__init__.py new file mode 100644 index 0000000..4a53298 --- /dev/null +++ b/reset/packages/envied/src/envied/core/proxies/__init__.py @@ -0,0 +1,8 @@ +from .basic import Basic +from .gluetun import Gluetun +from .hola import Hola +from .nordvpn import NordVPN +from .surfsharkvpn import SurfsharkVPN +from .windscribevpn import WindscribeVPN + +__all__ = ("Basic", "Gluetun", "Hola", "NordVPN", "SurfsharkVPN", "WindscribeVPN") diff --git a/reset/packages/envied/src/envied/core/proxies/basic.py b/reset/packages/envied/src/envied/core/proxies/basic.py new file mode 100644 index 0000000..3502c89 --- /dev/null +++ b/reset/packages/envied/src/envied/core/proxies/basic.py @@ -0,0 +1,54 @@ +import random +import re +from typing import Optional, Union + +from requests.utils import prepend_scheme_if_needed +from urllib3.util import parse_url + +from envied.core.proxies.proxy import Proxy + + +class Basic(Proxy): + def __init__(self, **countries: dict[str, Union[str, list[str]]]): + """Basic Proxy Service using Proxies specified in the config.""" + self.countries = {k.lower(): v for k, v in countries.items()} + + def __repr__(self) -> str: + countries = len(self.countries) + servers = len(self.countries.values()) + + return f"{countries} Countr{['ies', 'y'][countries == 1]} ({servers} Server{['s', ''][servers == 1]})" + + def get_proxy(self, query: str) -> Optional[str]: + """Get a proxy URI from the config.""" + query = query.lower() + + match = re.match(r"^([a-z]{2})(\d+)?$", query, re.IGNORECASE) + if not match: + raise ValueError(f'The query "{query}" was not recognized...') + + country_code = match.group(1) + entry = match.group(2) + + servers: Optional[Union[str, list[str]]] = self.countries.get(country_code) + if not servers: + return None + + if isinstance(servers, str): + proxy = servers + elif entry: + try: + proxy = servers[int(entry) - 1] + except IndexError: + raise ValueError( + f'There\'s only {len(servers)} prox{"y" if len(servers) == 1 else "ies"} for "{country_code}"...' + ) + else: + proxy = random.choice(servers) + + proxy = prepend_scheme_if_needed(proxy, "http") + parsed_proxy = parse_url(proxy) + if not parsed_proxy.host: + raise ValueError(f"The proxy '{proxy}' is not a valid proxy URI supported by Python-Requests.") + + return proxy diff --git a/reset/packages/envied/src/envied/core/proxies/gluetun.py b/reset/packages/envied/src/envied/core/proxies/gluetun.py new file mode 100644 index 0000000..f5331ce --- /dev/null +++ b/reset/packages/envied/src/envied/core/proxies/gluetun.py @@ -0,0 +1,1409 @@ +import atexit +import logging +import os +import re +import stat +import subprocess +import tempfile +import threading +import time +from typing import Optional + +import requests + +from envied.core import binaries +from envied.core.proxies.proxy import Proxy +from envied.core.utilities import get_country_code, get_country_name, get_debug_logger +from envied.core.utils.ip_info import get_ip_info + +# Global registry for cleanup on exit +_gluetun_instances: list["Gluetun"] = [] +_cleanup_lock = threading.Lock() +_cleanup_registered = False + + +def _cleanup_all_gluetun_containers(): + """Cleanup all Gluetun containers on exit.""" + # Get instances without holding the lock during cleanup + with _cleanup_lock: + instances = list(_gluetun_instances) + _gluetun_instances.clear() + + # Cleanup each instance (no lock held, so no deadlock possible) + for instance in instances: + try: + instance.cleanup() + except Exception: + pass + + +def _register_cleanup(): + """Register cleanup handlers (only once).""" + global _cleanup_registered + with _cleanup_lock: + if not _cleanup_registered: + # Only use atexit for cleanup - don't override signal handlers + # This allows Ctrl+C to work normally while still cleaning up on exit + atexit.register(_cleanup_all_gluetun_containers) + _cleanup_registered = True + + +class Gluetun(Proxy): + """ + Dynamic Gluetun VPN-to-HTTP Proxy Provider with multi-provider support. + + Automatically manages Docker containers running Gluetun for WireGuard/OpenVPN VPN connections. + Supports multiple VPN providers in a single configuration using query format: provider:region + + Supported VPN providers: windscribe, expressvpn, nordvpn, surfshark, protonvpn, mullvad, + privateinternetaccess, cyberghost, vyprvpn, torguard, and 50+ more. + + Configuration example in envied.yaml: + proxy_providers: + gluetun: + providers: + windscribe: + vpn_type: wireguard + credentials: + private_key: YOUR_KEY + addresses: YOUR_ADDRESS + server_countries: + us: US + uk: GB + nordvpn: + vpn_type: wireguard + credentials: + private_key: YOUR_KEY + addresses: YOUR_ADDRESS + server_countries: + us: US + de: DE + # Global settings (optional) + base_port: 8888 + auto_cleanup: true + container_prefix: "envied.gluetun" + + Usage: + --proxy gluetun:windscribe:us + --proxy gluetun:nordvpn:de + """ + + # Mapping of common VPN provider names to Gluetun identifiers + PROVIDER_MAPPING = { + "windscribe": "windscribe", + "expressvpn": "expressvpn", + "nordvpn": "nordvpn", + "surfshark": "surfshark", + "protonvpn": "protonvpn", + "mullvad": "mullvad", + "pia": "private internet access", + "privateinternetaccess": "private internet access", + "cyberghost": "cyberghost", + "vyprvpn": "vyprvpn", + "torguard": "torguard", + "ipvanish": "ipvanish", + "purevpn": "purevpn", + } + + # Windscribe uses specific region names instead of country codes + # See: https://github.com/qdm12/gluetun-wiki/blob/main/setup/providers/windscribe.md + WINDSCRIBE_REGION_MAP = { + # Country codes to Windscribe region names + "us": "US East", + "us-east": "US East", + "us-west": "US West", + "us-central": "US Central", + "ca": "Canada East", + "ca-east": "Canada East", + "ca-west": "Canada West", + "uk": "United Kingdom", + "gb": "United Kingdom", + "de": "Germany", + "fr": "France", + "nl": "Netherlands", + "au": "Australia", + "jp": "Japan", + "sg": "Singapore", + "hk": "Hong Kong", + "kr": "South Korea", + "in": "India", + "it": "Italy", + "es": "Spain", + "ch": "Switzerland", + "se": "Sweden", + "no": "Norway", + "dk": "Denmark", + "fi": "Finland", + "at": "Austria", + "be": "Belgium", + "ie": "Ireland", + "pl": "Poland", + "pt": "Portugal", + "cz": "Czech Republic", + "ro": "Romania", + "hu": "Hungary", + "gr": "Greece", + "tr": "Turkey", + "ru": "Russia", + "ua": "Ukraine", + "br": "Brazil", + "mx": "Mexico", + "ar": "Argentina", + "za": "South Africa", + "nz": "New Zealand", + "th": "Thailand", + "ph": "Philippines", + "id": "Indonesia", + "my": "Malaysia", + "vn": "Vietnam", + "tw": "Taiwan", + "ae": "United Arab Emirates", + "il": "Israel", + } + + def __init__( + self, + providers: Optional[dict] = None, + base_port: int = 8888, + auto_cleanup: bool = True, + container_prefix: str = "envied.gluetun", + auth_user: Optional[str] = None, + auth_password: Optional[str] = None, + verify_ip: bool = True, + **kwargs, + ): + """ + Initialize Gluetun proxy provider with multi-provider support. + + Args: + providers: Dict of VPN provider configurations + Format: { + "windscribe": { + "vpn_type": "wireguard", + "credentials": {"private_key": "...", "addresses": "..."}, + "server_countries": {"us": "US", "uk": "GB"} + }, + "nordvpn": {...} + } + base_port: Starting port for HTTP proxies (default: 8888) + auto_cleanup: Automatically remove stopped containers (default: True) + container_prefix: Docker container name prefix (default: "envied.gluetun") + auth_user: Optional HTTP proxy authentication username + auth_password: Optional HTTP proxy authentication password + verify_ip: Automatically verify IP and region after connection (default: True) + """ + # Check Docker availability using binaries module + if not binaries.Docker: + raise RuntimeError( + "Docker is not available. Please install Docker to use Gluetun proxy.\n" + "Visit: https://docs.docker.com/engine/install/" + ) + + self.providers = providers or {} + self.base_port = base_port + self.auto_cleanup = auto_cleanup + self.container_prefix = container_prefix + self.auth_user = auth_user + self.auth_password = auth_password + self.verify_ip = verify_ip + + # Track active containers: {query_key: {"container_name": ..., "port": ..., ...}} + self.active_containers = {} + + # Lock for thread-safe port allocation + self._port_lock = threading.Lock() + + # Validate provider configurations + for provider_name, config in self.providers.items(): + self._validate_provider_config(provider_name, config) + + # Register this instance for cleanup on exit + _register_cleanup() + with _cleanup_lock: + _gluetun_instances.append(self) + + # Log initialization + debug_logger = get_debug_logger() + if debug_logger: + debug_logger.log( + level="INFO", + operation="gluetun_init", + message=f"Gluetun proxy provider initialized with {len(self.providers)} provider(s)", + context={ + "providers": list(self.providers.keys()), + "base_port": base_port, + "auto_cleanup": auto_cleanup, + "verify_ip": verify_ip, + "container_prefix": container_prefix, + }, + ) + + def __repr__(self) -> str: + provider_count = len(self.providers) + return f"Gluetun ({provider_count} provider{['s', ''][provider_count == 1]})" + + def get_proxy(self, query: str) -> Optional[str]: + """ + Get an HTTP proxy URI for a Gluetun VPN connection. + + Args: + query: Query format: "provider:region" (e.g., "windscribe:us", "nordvpn:uk") + + Returns: + HTTP proxy URI or None if unavailable + """ + # Parse query + parts = query.split(":") + if len(parts) != 2: + raise ValueError(f"Invalid query format: '{query}'. Expected 'provider:region' (e.g., 'windscribe:us')") + + provider_name = parts[0].lower() + region = parts[1].lower() + + # Check if provider is configured + if provider_name not in self.providers: + available = ", ".join(self.providers.keys()) + raise ValueError(f"VPN provider '{provider_name}' not configured. Available providers: {available}") + + # Create query key for tracking + query_key = f"{provider_name}:{region}" + container_name = f"{self.container_prefix}-{provider_name}-{region}" + + debug_logger = get_debug_logger() + + # Check if container already exists (in memory OR in Docker) + # This handles multiple concurrent envied.sessions + if query_key in self.active_containers: + container = self.active_containers[query_key] + if self._is_container_running(container["container_name"]): + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="gluetun_container_reuse", + message=f"Reusing existing container (in-memory): {query_key}", + context={ + "query_key": query_key, + "container_name": container["container_name"], + "port": container["port"], + }, + ) + # Re-verify if needed + if self.verify_ip: + self._verify_container(query_key) + return self._build_proxy_uri(container["port"]) + else: + # Not in memory, but might exist in Docker (from another session) + existing_info = self._get_existing_container_info(container_name) + if existing_info: + # Container exists in Docker, reuse it + self.active_containers[query_key] = existing_info + if debug_logger: + debug_logger.log( + level="INFO", + operation="gluetun_container_reuse_docker", + message=f"Reusing existing Docker container: {query_key}", + context={ + "query_key": query_key, + "container_name": container_name, + "port": existing_info["port"], + }, + ) + # Re-verify if needed + if self.verify_ip: + self._verify_container(query_key) + return self._build_proxy_uri(existing_info["port"]) + + # Get provider configuration + provider_config = self.providers[provider_name] + + # Determine server location + server_countries = provider_config.get("server_countries", {}) + server_cities = provider_config.get("server_cities", {}) + server_hostnames = provider_config.get("server_hostnames", {}) + + country = server_countries.get(region) + city = server_cities.get(region) + hostname = server_hostnames.get(region) + + # Check if region is a specific server pattern (e.g., us1239, uk5678) + # Format: 2-letter country code + number + specific_server_match = re.match(r"^([a-z]{2})(\d+)$", region, re.IGNORECASE) + + if specific_server_match and not country and not city and not hostname: + # Specific server requested (e.g., us1239) + country_code = specific_server_match.group(1).upper() + server_num = specific_server_match.group(2) + + # Build hostname based on provider + hostname = self._build_server_hostname(provider_name, country_code, server_num) + country = country_code # Set country for verification + + # If not explicitly mapped and not a specific server, try to use query as country code + elif not country and not city and not hostname: + if re.match(r"^[a-z]{2}$", region): + # Convert country code to full name for Gluetun + country = get_country_name(region) + if not country: + raise ValueError( + f"Country code '{region}' not recognized. " + f"Configure it in server_countries or use a valid ISO 3166-1 alpha-2 code." + ) + else: + raise ValueError( + f"Region '{region}' not recognized for provider '{provider_name}'. " + f"Configure it in server_countries or server_cities, or use a 2-letter country code." + ) + + # Remove any stopped container with the same name + self._remove_stopped_container(container_name) + + # Find available port + port = self._get_available_port() + + # Create container (name already set above) + try: + self._create_container( + container_name=container_name, + port=port, + provider_name=provider_name, + provider_config=provider_config, + country=country, + city=city, + hostname=hostname, + ) + + # Store container info + self.active_containers[query_key] = { + "container_name": container_name, + "port": port, + "provider": provider_name, + "region": region, + "country": country, + "city": city, + "hostname": hostname, + } + + # Wait for container to be ready (60s timeout for VPN connection) + if not self._wait_for_container(container_name, timeout=60): + # Get container logs for better error message + logs = self._get_container_logs(container_name, tail=30) + error_msg = f"Gluetun container '{container_name}' failed to start" + if hasattr(self, "_last_wait_error") and self._last_wait_error: + error_msg += f": {self._last_wait_error}" + if logs: + # Extract last few relevant lines + log_lines = [line for line in logs.strip().split("\n") if line.strip()][-5:] + error_msg += "\nRecent logs:\n" + "\n".join(log_lines) + raise RuntimeError(error_msg) + + # Verify IP and region if enabled + if self.verify_ip: + self._verify_container(query_key) + + return self._build_proxy_uri(port) + + except Exception as e: + # Cleanup on failure + self._remove_container(container_name) + if query_key in self.active_containers: + del self.active_containers[query_key] + raise RuntimeError(f"Failed to create Gluetun container: {e}") + + def cleanup(self): + """Stop and remove all managed Gluetun containers.""" + debug_logger = get_debug_logger() + container_count = len(self.active_containers) + + if container_count > 0 and debug_logger: + debug_logger.log( + level="DEBUG", + operation="gluetun_cleanup_start", + message=f"Cleaning up {container_count} Gluetun container(s)", + context={ + "container_count": container_count, + "containers": list(self.active_containers.keys()), + }, + ) + + for query_key, container_info in list(self.active_containers.items()): + container_name = container_info["container_name"] + self._remove_container(container_name) + + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="gluetun_container_removed", + message=f"Removed Gluetun container: {container_name}", + context={ + "query_key": query_key, + "container_name": container_name, + }, + ) + + self.active_containers.clear() + + if container_count > 0 and debug_logger: + debug_logger.log( + level="INFO", + operation="gluetun_cleanup_complete", + message=f"Cleanup complete: removed {container_count} container(s)", + context={"container_count": container_count}, + success=True, + ) + + def get_connection_info(self, query: str) -> Optional[dict]: + """ + Get connection info for a proxy query. + + Args: + query: Query format "provider:region" (e.g., "windscribe:us") + + Returns: + Dict with connection info including public_ip, country, city, or None if not found. + """ + parts = query.split(":") + if len(parts) != 2: + return None + + provider_name = parts[0].lower() + region = parts[1].lower() + query_key = f"{provider_name}:{region}" + + container = self.active_containers.get(query_key) + if not container: + return None + + return { + "provider": container.get("provider"), + "region": container.get("region"), + "public_ip": container.get("public_ip"), + "country": container.get("ip_country"), + "city": container.get("ip_city"), + "org": container.get("ip_org"), + } + + def _validate_provider_config(self, provider_name: str, config: dict): + """Validate a provider's configuration.""" + vpn_type = config.get("vpn_type", "wireguard").lower() + credentials = config.get("credentials", {}) + + if vpn_type not in ["wireguard", "openvpn"]: + raise ValueError(f"Provider '{provider_name}': Invalid vpn_type '{vpn_type}'. Use 'wireguard' or 'openvpn'") + + if vpn_type == "wireguard": + # private_key is always required for WireGuard + if "private_key" not in credentials: + raise ValueError(f"Provider '{provider_name}': WireGuard requires 'private_key' in credentials") + + # Provider-specific WireGuard requirements based on Gluetun wiki: + # - NordVPN, ProtonVPN: only private_key required + # - Windscribe: private_key, addresses, AND preshared_key required (preshared_key MUST be set) + # - Surfshark, Mullvad, IVPN: private_key AND addresses required + provider_lower = provider_name.lower() + + # Windscribe requires preshared_key (can be empty string, but must be set) + if provider_lower == "windscribe": + if "preshared_key" not in credentials: + raise ValueError( + f"Provider '{provider_name}': Windscribe WireGuard requires 'preshared_key' in credentials " + "(can be empty string, but must be set). Get it from windscribe.com/getconfig/wireguard" + ) + if "addresses" not in credentials: + raise ValueError( + f"Provider '{provider_name}': Windscribe WireGuard requires 'addresses' in credentials. " + "Get it from windscribe.com/getconfig/wireguard" + ) + + # Providers that require addresses (but not preshared_key) + elif provider_lower in ["surfshark", "mullvad", "ivpn"]: + if "addresses" not in credentials: + raise ValueError(f"Provider '{provider_name}': WireGuard requires 'addresses' in credentials") + + elif vpn_type == "openvpn": + if "username" not in credentials or "password" not in credentials: + raise ValueError( + f"Provider '{provider_name}': OpenVPN requires 'username' and 'password' in credentials" + ) + + def _get_available_port(self) -> int: + """Find an available port starting from base_port (thread-safe).""" + with self._port_lock: + used_ports = {info["port"] for info in self.active_containers.values()} + port = self.base_port + while port in used_ports or self._is_port_in_use(port): + port += 1 + return port + + def _is_port_in_use(self, port: int) -> bool: + """Check if a port is in use on the system or by any Docker container.""" + import socket + + # First check if the port is available on the system + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", port)) + except OSError: + # Port is in use by something on the system + return True + + # Also check Docker containers (in case of port forwarding) + try: + result = subprocess.run( + ["docker", "ps", "--format", "{{.Ports}}"], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode == 0: + return f":{port}->" in result.stdout or f"0.0.0.0:{port}" in result.stdout + return False + except (subprocess.TimeoutExpired, FileNotFoundError): + return False + + def _build_server_hostname(self, provider_name: str, country_code: str, server_num: str) -> str: + """ + Build a server hostname for specific server selection. + + Args: + provider_name: VPN provider name (e.g., "nordvpn") + country_code: 2-letter country code (e.g., "US") + server_num: Server number (e.g., "1239") + + Returns: + Server hostname (e.g., "us1239.nordvpn.com") + """ + # Convert to lowercase for hostname + country_lower = country_code.lower() + + # Provider-specific hostname formats + hostname_formats = { + "nordvpn": f"{country_lower}{server_num}.nordvpn.com", + "surfshark": f"{country_lower}-{server_num}.prod.surfshark.com", + "expressvpn": f"{country_lower}-{server_num}.expressvpn.com", + "cyberghost": f"{country_lower}-s{server_num}.cg-dialup.net", + # Generic fallback for other providers + } + + # Get provider-specific format or use generic + if provider_name in hostname_formats: + return hostname_formats[provider_name] + else: + # Generic format: country_code + server_num + return f"{country_lower}{server_num}" + + def _ensure_image_available(self, image: str = "qmcgaw/gluetun:latest") -> bool: + """ + Ensure the Gluetun Docker image is available locally. + + If the image is not present, it will be pulled. This prevents + the container creation from timing out during the first run. + + Args: + image: Docker image name with tag + + Returns: + True if image is available, False otherwise + """ + log = logging.getLogger("Gluetun") + + # Check if image exists locally + try: + result = subprocess.run( + ["docker", "image", "inspect", image], + capture_output=True, + text=True, + timeout=10, + encoding="utf-8", + errors="replace", + ) + if result.returncode == 0: + return True + log.debug(f"Image inspect failed: {result.stderr}") + except subprocess.TimeoutExpired: + log.warning("Docker image inspect timed out") + except FileNotFoundError: + log.error("Docker command not found - is Docker installed and in PATH?") + return False + + # Image not found, pull it + log.info(f"Pulling Docker image {image}...") + try: + result = subprocess.run( + ["docker", "pull", image], + capture_output=True, + text=True, + timeout=300, # 5 minutes for pull + encoding="utf-8", + errors="replace", + ) + if result.returncode == 0: + return True + log.error(f"Docker pull failed: {result.stderr}") + return False + except subprocess.TimeoutExpired: + raise RuntimeError(f"Timed out pulling Docker image '{image}'") + + def _create_container( + self, + container_name: str, + port: int, + provider_name: str, + provider_config: dict, + country: Optional[str] = None, + city: Optional[str] = None, + hostname: Optional[str] = None, + ): + """Create and start a Gluetun Docker container.""" + debug_logger = get_debug_logger() + start_time = time.time() + + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="gluetun_container_create_start", + message=f"Creating Gluetun container: {container_name}", + context={ + "container_name": container_name, + "port": port, + "provider": provider_name, + "country": country, + "city": city, + "hostname": hostname, + }, + ) + + # Ensure the Gluetun image is available (pulls if needed) + gluetun_image = "qmcgaw/gluetun:latest" + if not self._ensure_image_available(gluetun_image): + if debug_logger: + debug_logger.log( + level="ERROR", + operation="gluetun_image_pull_failed", + message=f"Failed to pull Docker image: {gluetun_image}", + success=False, + ) + raise RuntimeError(f"Failed to ensure Gluetun Docker image '{gluetun_image}' is available") + + vpn_type = provider_config.get("vpn_type", "wireguard").lower() + credentials = provider_config.get("credentials", {}) + extra_env = provider_config.get("extra_env", {}) + + # Normalize provider name + gluetun_provider = self.PROVIDER_MAPPING.get(provider_name.lower(), provider_name.lower()) + + # Build environment variables + env_vars = { + "VPN_SERVICE_PROVIDER": gluetun_provider, + "VPN_TYPE": vpn_type, + "HTTPPROXY": "on", + "HTTPPROXY_LISTENING_ADDRESS": ":8888", + "HTTPPROXY_LOG": "on", + "TZ": os.environ.get("TZ", "UTC"), + "LOG_LEVEL": "info", + } + + # Add credentials + if vpn_type == "wireguard": + env_vars["WIREGUARD_PRIVATE_KEY"] = credentials["private_key"] + # addresses is optional - not needed for some providers like NordVPN + if "addresses" in credentials: + env_vars["WIREGUARD_ADDRESSES"] = credentials["addresses"] + # preshared_key is required for Windscribe, optional for others + if "preshared_key" in credentials: + env_vars["WIREGUARD_PRESHARED_KEY"] = credentials["preshared_key"] + elif vpn_type == "openvpn": + env_vars["OPENVPN_USER"] = credentials.get("username", "") + env_vars["OPENVPN_PASSWORD"] = credentials.get("password", "") + + # Add server location + # Priority: hostname > country + city > country only + # Note: Different providers support different server selection variables + # - Most providers: SERVER_COUNTRIES, SERVER_CITIES + # - Windscribe, VyprVPN, VPN Secure: SERVER_REGIONS, SERVER_CITIES (no SERVER_COUNTRIES) + if hostname: + # Specific server hostname requested (e.g., us1239.nordvpn.com) + env_vars["SERVER_HOSTNAMES"] = hostname + else: + # Providers that use SERVER_REGIONS instead of SERVER_COUNTRIES + region_only_providers = {"windscribe", "vyprvpn", "vpn secure"} + uses_regions = gluetun_provider in region_only_providers + + # Use country/city selection + if country: + if uses_regions: + # Convert country code to provider-specific region name + if gluetun_provider == "windscribe": + region_name = self.WINDSCRIBE_REGION_MAP.get(country.lower(), country) + env_vars["SERVER_REGIONS"] = region_name + else: + env_vars["SERVER_REGIONS"] = country + else: + env_vars["SERVER_COUNTRIES"] = country + if city: + env_vars["SERVER_CITIES"] = city + + # Add authentication if configured + if self.auth_user: + env_vars["HTTPPROXY_USER"] = self.auth_user + if self.auth_password: + env_vars["HTTPPROXY_PASSWORD"] = self.auth_password + + # Merge extra environment variables + env_vars.update(extra_env) + + # Debug log environment variables (redact sensitive values) + if debug_logger: + redact_markers = ("KEY", "PASSWORD", "PASS", "TOKEN", "SECRET", "USER") + safe_env = {k: ("***" if any(m in k for m in redact_markers) else v) for k, v in env_vars.items()} + debug_logger.log( + level="DEBUG", + operation="gluetun_env_vars", + message=f"Environment variables for {container_name}", + context={"env_vars": safe_env, "gluetun_provider": gluetun_provider}, + ) + + # Build docker run command + cmd = [ + "docker", + "run", + "-d", + "--name", + container_name, + "--cap-add=NET_ADMIN", + "--device=/dev/net/tun", + "-p", + f"127.0.0.1:{port}:8888/tcp", + ] + + # Avoid exposing credentials in process listings by using --env-file instead of many "-e KEY=VALUE". + env_file_path: str | None = None + try: + fd, env_file_path = tempfile.mkstemp(prefix=f"envied.{container_name}-", suffix=".env") + try: + # Best-effort restrictive permissions. + if os.name != "nt": + if hasattr(os, "fchmod"): + os.fchmod(fd, 0o600) + else: + os.chmod(env_file_path, 0o600) + else: + os.chmod(env_file_path, stat.S_IREAD | stat.S_IWRITE) + + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as f: + for key, value in env_vars.items(): + if "=" in key: + raise ValueError(f"Invalid env var name for docker env-file: {key!r}") + v = "" if value is None else str(value) + if "\n" in v or "\r" in v: + raise ValueError(f"Invalid env var value (contains newline) for {key!r}") + f.write(f"{key}={v}\n") + except Exception: + # If we fail before fdopen closes the descriptor, make sure it's not leaked. + try: + os.close(fd) + except Exception: + pass + raise + + cmd.extend(["--env-file", env_file_path]) + + # Add Gluetun image + cmd.append(gluetun_image) + + # Execute docker run + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=30, + encoding="utf-8", + errors="replace", + ) + except subprocess.TimeoutExpired: + if debug_logger: + debug_logger.log( + level="ERROR", + operation="gluetun_container_create_timeout", + message=f"Docker run timed out for {container_name}", + context={"container_name": container_name}, + success=False, + duration_ms=(time.time() - start_time) * 1000, + ) + raise RuntimeError("Docker run command timed out") + + if result.returncode != 0: + error_msg = result.stderr or "unknown error" + if debug_logger: + debug_logger.log( + level="ERROR", + operation="gluetun_container_create_failed", + message=f"Docker run failed for {container_name}", + context={ + "container_name": container_name, + "return_code": result.returncode, + "stderr": error_msg, + }, + success=False, + duration_ms=(time.time() - start_time) * 1000, + ) + raise RuntimeError(f"Docker run failed: {error_msg}") + + # Log successful container creation + if debug_logger: + duration_ms = (time.time() - start_time) * 1000 + debug_logger.log( + level="INFO", + operation="gluetun_container_created", + message=f"Gluetun container created: {container_name}", + context={ + "container_name": container_name, + "port": port, + "provider": provider_name, + "vpn_type": vpn_type, + "country": country, + "city": city, + "hostname": hostname, + "container_id": result.stdout.strip()[:12] if result.stdout else None, + }, + success=True, + duration_ms=duration_ms, + ) + finally: + if env_file_path: + # Best-effort "secure delete": overwrite then unlink (not guaranteed on all filesystems). + try: + with open(env_file_path, "r+b") as f: + try: + f.seek(0, os.SEEK_END) + length = f.tell() + f.seek(0) + if length > 0: + f.write(b"\x00" * length) + f.flush() + os.fsync(f.fileno()) + except Exception: + pass + except Exception: + pass + try: + os.remove(env_file_path) + except FileNotFoundError: + pass + except Exception: + pass + + def _is_container_running(self, container_name: str) -> bool: + """Check if a Docker container is running.""" + try: + result = subprocess.run( + [ + "docker", + "ps", + "--filter", + f"name=^{re.escape(container_name)}$", + "--format", + "{{.Names}}", + ], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode != 0: + return False + + names = [line.strip() for line in (result.stdout or "").splitlines() if line.strip()] + return any(name == container_name for name in names) + except (subprocess.TimeoutExpired, FileNotFoundError): + return False + + def _get_existing_container_info(self, container_name: str) -> Optional[dict]: + """ + Check if a container exists in Docker and get its info. + + This handles multiple envied.sessions - if another session already + created the container, we'll reuse it instead of trying to create a duplicate. + + Args: + container_name: Name of the container to check + + Returns: + Dict with container info if exists and running, None otherwise + """ + try: + # Check if container is running + if not self._is_container_running(container_name): + return None + + # Get container port mapping + # Format: "127.0.0.1:8888->8888/tcp" + result = subprocess.run( + ["docker", "inspect", container_name, "--format", "{{.NetworkSettings.Ports}}"], + capture_output=True, + text=True, + timeout=5, + ) + + if result.returncode != 0: + return None + + # Parse port from output like "map[8888/tcp:[{127.0.0.1 8888}]]" + port_match = re.search(r"127\.0\.0\.1\s+(\d+)", result.stdout) + if not port_match: + return None + + port = int(port_match.group(1)) + + # Extract provider and region from container name + # Format: envied.gluetun-provider-region + name_pattern = f"{self.container_prefix}-(.+)-([^-]+)$" + name_match = re.match(name_pattern, container_name) + if not name_match: + return None + + provider_name = name_match.group(1) + region = name_match.group(2) + + # Get expected country and hostname from config (if available) + country = None + hostname = None + + # Check if region is a specific server (e.g., us1239) + specific_server_match = re.match(r"^([a-z]{2})(\d+)$", region, re.IGNORECASE) + if specific_server_match: + country_code = specific_server_match.group(1).upper() + server_num = specific_server_match.group(2) + hostname = self._build_server_hostname(provider_name, country_code, server_num) + country = country_code + + # Otherwise check config + elif provider_name in self.providers: + provider_config = self.providers[provider_name] + server_countries = provider_config.get("server_countries", {}) + country = server_countries.get(region) + + if not country and re.match(r"^[a-z]{2}$", region): + country = region.upper() + + return { + "container_name": container_name, + "port": port, + "provider": provider_name, + "region": region, + "country": country, + "city": None, + "hostname": hostname, + } + + except (subprocess.TimeoutExpired, FileNotFoundError, ValueError): + return None + + def _wait_for_container(self, container_name: str, timeout: int = 60) -> bool: + """ + Wait for Gluetun container to be ready by checking logs for proxy readiness. + + Gluetun logs "http proxy listening" when the HTTP proxy is ready to accept connections. + + Args: + container_name: Name of the container to wait for + timeout: Maximum time to wait in seconds (default: 60) + + Returns: + True if container is ready, False if it failed or timed out + """ + debug_logger = get_debug_logger() + start_time = time.time() + last_error = None + + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="gluetun_container_wait_start", + message=f"Waiting for container to be ready: {container_name}", + context={"container_name": container_name, "timeout": timeout}, + ) + + while time.time() - start_time < timeout: + try: + # First check if container is still running + if not self._is_container_running(container_name): + # Container may have exited - check if it crashed + exit_info = self._get_container_exit_info(container_name) + if exit_info: + last_error = f"Container exited with code {exit_info.get('exit_code', 'unknown')}" + time.sleep(1) + continue + + # Check logs for readiness indicators + result = subprocess.run( + ["docker", "logs", container_name, "--tail", "100"], + capture_output=True, + text=True, + timeout=5, + encoding="utf-8", + errors="replace", + ) + + if result.returncode == 0: + # Combine stdout and stderr for checking (handle None values) + stdout = result.stdout or "" + stderr = result.stderr or "" + all_logs = (stdout + stderr).lower() + + # Gluetun needs both proxy listening AND VPN connected + # The proxy starts before VPN is ready, so we need to wait for VPN + proxy_ready = "[http proxy] listening" in all_logs + vpn_ready = "initialization sequence completed" in all_logs or "public ip address is" in all_logs + + if proxy_ready and vpn_ready: + # Give a brief moment for the proxy to fully initialize + time.sleep(1) + duration_ms = (time.time() - start_time) * 1000 + if debug_logger: + debug_logger.log( + level="INFO", + operation="gluetun_container_ready", + message=f"Gluetun container is ready: {container_name}", + context={ + "container_name": container_name, + "proxy_ready": proxy_ready, + "vpn_ready": vpn_ready, + }, + success=True, + duration_ms=duration_ms, + ) + return True + + # Check for fatal errors that indicate VPN connection failure + error_indicators = [ + "fatal", + "cannot connect", + "authentication failed", + "invalid credentials", + "connection refused", + "no valid servers", + ] + + for error in error_indicators: + if error in all_logs: + # Extract the error line for better messaging + for line in (stdout + stderr).split("\n"): + if error in line.lower(): + last_error = line.strip() + break + # Fatal errors mean we should stop waiting + if "fatal" in all_logs or "invalid credentials" in all_logs: + return False + + except subprocess.TimeoutExpired: + pass + + time.sleep(2) + + # Store the last error for potential logging + if last_error: + self._last_wait_error = last_error + + # Log timeout/failure + duration_ms = (time.time() - start_time) * 1000 + if debug_logger: + debug_logger.log( + level="ERROR", + operation="gluetun_container_wait_timeout", + message=f"Gluetun container failed to become ready: {container_name}", + context={ + "container_name": container_name, + "timeout": timeout, + "last_error": last_error, + }, + success=False, + duration_ms=duration_ms, + ) + return False + + def _get_container_exit_info(self, container_name: str) -> Optional[dict]: + """Get exit information for a stopped container.""" + try: + result = subprocess.run( + ["docker", "inspect", container_name, "--format", "{{.State.ExitCode}}:{{.State.Error}}"], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode == 0: + parts = result.stdout.strip().split(":", 1) + return { + "exit_code": int(parts[0]) if parts[0].isdigit() else -1, + "error": parts[1] if len(parts) > 1 else "", + } + return None + except (subprocess.TimeoutExpired, FileNotFoundError, ValueError): + return None + + def _get_container_logs(self, container_name: str, tail: int = 50) -> str: + """Get recent logs from a container for error reporting.""" + try: + result = subprocess.run( + ["docker", "logs", container_name, "--tail", str(tail)], + capture_output=True, + text=True, + timeout=10, + encoding="utf-8", + errors="replace", + ) + return (result.stdout or "") + (result.stderr or "") + except (subprocess.TimeoutExpired, FileNotFoundError): + return "" + + def _verify_container(self, query_key: str, max_retries: int = 3): + """ + Verify container's VPN IP and region using ipinfo.io lookup. + + Uses the shared get_ip_info function with a session configured to use + the Gluetun proxy. Retries with exponential backoff if the network + isn't ready immediately after the VPN connects. + + Args: + query_key: The container query key (provider:region) + max_retries: Maximum number of retry attempts (default: 3) + + Raises: + RuntimeError: If verification fails after all retries + """ + debug_logger = get_debug_logger() + start_time = time.time() + + if query_key not in self.active_containers: + return + + container = self.active_containers[query_key] + proxy_url = self._build_proxy_uri(container["port"]) + expected_country = container.get("country", "").upper() + + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="gluetun_verify_start", + message=f"Verifying VPN IP for: {query_key}", + context={ + "query_key": query_key, + "container_name": container.get("container_name"), + "expected_country": expected_country, + "max_retries": max_retries, + }, + ) + + last_error = None + + # Create a session with the proxy configured + session = requests.Session() + try: + session.proxies = {"http": proxy_url, "https": proxy_url} + + # Retry with exponential backoff + for attempt in range(max_retries): + try: + # Get external IP through the proxy using shared utility + ip_info = get_ip_info(session) + + if ip_info: + actual_country = ip_info.get("country", "").upper() + + # Check if country matches (if we have an expected country) + # ipinfo.io returns country codes (CA), but we may have full names (Canada) + # Normalize both to country codes for comparison using shared utility + if expected_country: + # Convert expected country name to code if it's a full name + expected_code = get_country_code(expected_country) or expected_country + expected_code = expected_code.upper() + + if actual_country != expected_code: + duration_ms = (time.time() - start_time) * 1000 + if debug_logger: + debug_logger.log( + level="ERROR", + operation="gluetun_verify_mismatch", + message=f"Region mismatch for {query_key}", + context={ + "query_key": query_key, + "expected_country": expected_code, + "actual_country": actual_country, + "ip": ip_info.get("ip"), + "city": ip_info.get("city"), + "org": ip_info.get("org"), + }, + success=False, + duration_ms=duration_ms, + ) + raise RuntimeError( + f"Region mismatch for {container['provider']}:{container['region']}: " + f"Expected '{expected_code}' but got '{actual_country}' " + f"(IP: {ip_info.get('ip')}, City: {ip_info.get('city')})" + ) + + # Verification successful - store IP info in container record + if query_key in self.active_containers: + self.active_containers[query_key]["public_ip"] = ip_info.get("ip") + self.active_containers[query_key]["ip_country"] = actual_country + self.active_containers[query_key]["ip_city"] = ip_info.get("city") + self.active_containers[query_key]["ip_org"] = ip_info.get("org") + + duration_ms = (time.time() - start_time) * 1000 + if debug_logger: + debug_logger.log( + level="INFO", + operation="gluetun_verify_success", + message=f"VPN IP verified for: {query_key}", + context={ + "query_key": query_key, + "ip": ip_info.get("ip"), + "country": actual_country, + "city": ip_info.get("city"), + "org": ip_info.get("org"), + "attempts": attempt + 1, + }, + success=True, + duration_ms=duration_ms, + ) + return + + # ip_info was None, retry + last_error = "Failed to get IP info from ipinfo.io" + + except RuntimeError: + raise # Re-raise region mismatch errors immediately + except Exception as e: + last_error = str(e) + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="gluetun_verify_retry", + message=f"Verification attempt {attempt + 1} failed, retrying", + context={ + "query_key": query_key, + "attempt": attempt + 1, + "error": last_error, + }, + ) + + # Wait before retry (exponential backoff) + if attempt < max_retries - 1: + wait_time = 2**attempt # 1, 2, 4 seconds + time.sleep(wait_time) + finally: + try: + session.close() + except Exception: + pass + + # All retries exhausted + duration_ms = (time.time() - start_time) * 1000 + if debug_logger: + debug_logger.log( + level="ERROR", + operation="gluetun_verify_failed", + message=f"VPN verification failed after {max_retries} attempts", + context={ + "query_key": query_key, + "max_retries": max_retries, + "last_error": last_error, + }, + success=False, + duration_ms=duration_ms, + ) + raise RuntimeError( + f"Failed to verify VPN IP for {container['provider']}:{container['region']} " + f"after {max_retries} attempts. Last error: {last_error}" + ) + + def _remove_stopped_container(self, container_name: str) -> bool: + """ + Remove a stopped container with the given name if it exists. + + This prevents "container name already in use" errors when a previous + container wasn't properly cleaned up. + + Args: + container_name: Name of the container to check and remove + + Returns: + True if a container was removed, False otherwise + """ + try: + # Check if container exists (running or stopped) + result = subprocess.run( + ["docker", "ps", "-a", "--filter", f"name=^{container_name}$", "--format", "{{.Names}}:{{.Status}}"], + capture_output=True, + text=True, + timeout=5, + ) + + if result.returncode != 0 or not result.stdout.strip(): + return False + + # Parse status - format is "name:Up 2 hours" or "name:Exited (0) 2 hours ago" + output = result.stdout.strip() + if container_name not in output: + return False + + # Check if container is stopped (not running) + if "Exited" in output or "Created" in output or "Dead" in output: + # Container exists but is stopped - remove it + subprocess.run( + ["docker", "rm", "-f", container_name], + capture_output=True, + text=True, + timeout=10, + ) + return True + + return False + + except (subprocess.TimeoutExpired, FileNotFoundError): + return False + + def _remove_container(self, container_name: str): + """Stop and remove a Docker container.""" + try: + if self.auto_cleanup: + # Use docker rm -f to force remove (stops and removes in one command) + subprocess.run( + ["docker", "rm", "-f", container_name], + capture_output=True, + text=True, + timeout=10, + ) + else: + # Just stop the container + subprocess.run( + ["docker", "stop", container_name], + capture_output=True, + text=True, + timeout=10, + ) + except subprocess.TimeoutExpired: + # Force kill if timeout + try: + subprocess.run( + ["docker", "rm", "-f", container_name], + capture_output=True, + text=True, + timeout=5, + ) + except subprocess.TimeoutExpired: + pass + + def _build_proxy_uri(self, port: int) -> str: + """Build HTTP proxy URI.""" + if self.auth_user and self.auth_password: + return f"http://{self.auth_user}:{self.auth_password}@localhost:{port}" + return f"http://localhost:{port}" + + def __del__(self): + """Cleanup containers on object destruction.""" + if hasattr(self, "auto_cleanup") and self.auto_cleanup: + try: + if hasattr(self, "active_containers") and self.active_containers: + self.cleanup() + except Exception: + pass diff --git a/reset/packages/envied/src/envied/core/proxies/hola.py b/reset/packages/envied/src/envied/core/proxies/hola.py new file mode 100644 index 0000000..ac50d5d --- /dev/null +++ b/reset/packages/envied/src/envied/core/proxies/hola.py @@ -0,0 +1,60 @@ +import random +import re +import subprocess +from typing import Optional + +from envied.core import binaries +from envied.core.proxies.proxy import Proxy + + +class Hola(Proxy): + def __init__(self): + """ + Proxy Service using Hola's direct connections via the hola-proxy project. + https://github.com/Snawoot/hola-proxy + """ + self.binary = binaries.HolaProxy + if not self.binary: + raise EnvironmentError("hola-proxy executable not found but is required for the Hola proxy provider.") + + self.countries = self.get_countries() + + def __repr__(self) -> str: + countries = len(self.countries) + + return f"{countries} Countr{['ies', 'y'][countries == 1]}" + + def get_proxy(self, query: str) -> Optional[str]: + """ + Get an HTTP proxy URI for a Datacenter ('direct') or Residential ('lum') Hola server. + + TODO: - Add ability to select 'lum' proxies (residential proxies). + - Return and use Proxy Authorization + """ + query = query.lower() + + p = subprocess.check_output( + [self.binary, "-country", query, "-list-proxies"], stderr=subprocess.STDOUT + ).decode() + + if "Transaction error: temporary ban detected." in p: + raise ConnectionError("Hola banned your IP temporarily from it's services. Try change your IP.") + + username, password, proxy_authorization = re.search( + r"Login: (.*)\nPassword: (.*)\nProxy-Authorization: (.*)", p + ).groups() + + servers = re.findall(r"(zagent.*)", p) + proxies = [] + for server in servers: + host, ip_address, direct, peer, hola, trial, trial_peer, vendor = server.split(",") + proxies.append(f"http://{username}:{password}@{ip_address}:{peer}") + + proxy = random.choice(proxies) + return proxy + + def get_countries(self) -> list[dict[str, str]]: + """Get a list of available Countries.""" + p = subprocess.check_output([self.binary, "-list-countries"]).decode("utf8") + + return [{code: name} for country in p.splitlines() for (code, name) in [country.split(" - ", maxsplit=1)]] diff --git a/reset/packages/envied/src/envied/core/proxies/nordvpn.py b/reset/packages/envied/src/envied/core/proxies/nordvpn.py new file mode 100644 index 0000000..bc987ed --- /dev/null +++ b/reset/packages/envied/src/envied/core/proxies/nordvpn.py @@ -0,0 +1,197 @@ +import json +import random +import re +from typing import Optional + +import requests + +from envied.core.proxies.proxy import Proxy + + +class NordVPN(Proxy): + def __init__(self, username: str, password: str, server_map: Optional[dict[str, int]] = None): + """ + Proxy Service using NordVPN Service Credentials. + + A username and password must be provided. These are Service Credentials, not your Login Credentials. + The Service Credentials can be found here: https://my.nordaccount.com/dashboard/nordvpn/ + """ + if not username: + raise ValueError("No Username was provided to the NordVPN Proxy Service.") + if not password: + raise ValueError("No Password was provided to the NordVPN Proxy Service.") + if not re.match(r"^[a-z0-9]{48}$", username + password, re.IGNORECASE) or "@" in username: + raise ValueError( + "The Username and Password must be NordVPN Service Credentials, not your Login Credentials. " + "The Service Credentials can be found here: https://my.nordaccount.com/dashboard/nordvpn/" + ) + + if server_map is not None and not isinstance(server_map, dict): + raise TypeError(f"Expected server_map to be a dict mapping a region to a server ID, not '{server_map!r}'.") + + self.username = username + self.password = password + self.server_map = server_map or {} + + self.countries = self.get_countries() + + def __repr__(self) -> str: + countries = len(self.countries) + servers = sum(x["serverCount"] for x in self.countries) + + return f"{countries} Countr{['ies', 'y'][countries == 1]} ({servers} Server{['s', ''][servers == 1]})" + + def get_proxy(self, query: str) -> Optional[str]: + """ + Get an HTTP(SSL) proxy URI for a NordVPN server. + + HTTP proxies under port 80 were disabled on the 15th of Feb, 2021: + https://nordvpn.com/blog/removing-http-proxies + + Supports: + - Country code: "us", "ca", "gb" + - Country ID: "228" + - Specific server: "us1234" + - City selection: "us:seattle", "ca:calgary" + """ + query = query.lower() + city = None + + # Check if query includes city specification (e.g., "ca:calgary") + if ":" in query: + query, city = query.split(":", maxsplit=1) + city = city.strip() + + if re.match(r"^[a-z]{2}\d+$", query): + # country and nordvpn server id, e.g., us1, fr1234 + hostname = f"{query}.proxy.nordvpn.com" + else: + if query.isdigit(): + # country id + country = self.get_country(by_id=int(query)) + elif re.match(r"^[a-z]+$", query): + # country code + country = self.get_country(by_code=query) + else: + raise ValueError(f"The query provided is unsupported and unrecognized: {query}") + if not country: + # NordVPN doesnt have servers in this region + return + + # Check server_map for pinned servers (can include city) + server_map_key = f"{country['code'].lower()}:{city}" if city else country["code"].lower() + server_mapping = self.server_map.get(server_map_key) or ( + self.server_map.get(country["code"].lower()) if not city else None + ) + + if server_mapping: + # country was set to a specific server ID in config + hostname = f"{country['code'].lower()}{server_mapping}.proxy.nordvpn.com" + else: + # get the recommended server ID + recommended_servers = self.get_recommended_servers(country["id"]) + if not recommended_servers: + raise ValueError( + f"The NordVPN Country {query} currently has no recommended servers. " + "Try again later. If the issue persists, double-check the query." + ) + + # Filter by city if specified + if city: + city_servers = self.filter_servers_by_city(recommended_servers, city) + if not city_servers: + raise ValueError( + f"No servers found in city '{city}' for country '{country['name']}'. " + "Try a different city or check the city name spelling." + ) + recommended_servers = city_servers + + # Pick a random server from the filtered list + hostname = random.choice(recommended_servers)["hostname"] + + if hostname.startswith("gb"): + # NordVPN uses the alpha2 of 'GB' in API responses, but 'UK' in the hostname + hostname = f"gb{hostname[2:]}" + + if hostname.endswith(".nordvpn.com") and not hostname.endswith(".proxy.nordvpn.com"): + hostname = hostname[: -len(".nordvpn.com")] + ".proxy.nordvpn.com" + + return f"https://{self.username}:{self.password}@{hostname}:89" + + def get_country(self, by_id: Optional[int] = None, by_code: Optional[str] = None) -> Optional[dict]: + """Search for a Country and it's metadata.""" + if all(x is None for x in (by_id, by_code)): + raise ValueError("At least one search query must be made.") + + for country in self.countries: + if all( + [by_id is None or country["id"] == int(by_id), by_code is None or country["code"] == by_code.upper()] + ): + return country + + @staticmethod + def filter_servers_by_city(servers: list[dict], city: str) -> list[dict]: + """ + Filter servers by city name. + + The API returns servers with location data that includes city information. + This method filters servers to only those in the specified city. + + Args: + servers: List of server dictionaries from the NordVPN API + city: City name to filter by (case-insensitive) + + Returns: + List of servers in the specified city + """ + city_lower = city.lower() + filtered = [] + + for server in servers: + # Each server has a 'locations' list with location data + locations = server.get("locations", []) + for location in locations: + # City data can be in different formats: + # - {"city": {"name": "Seattle", ...}} + # - {"city": "Seattle"} + city_data = location.get("city") + if city_data: + # Handle both dict and string formats + city_name = city_data.get("name") if isinstance(city_data, dict) else city_data + if city_name and city_name.lower() == city_lower: + filtered.append(server) + break # Found a match, no need to check other locations for this server + + return filtered + + @staticmethod + def get_recommended_servers(country_id: int) -> list[dict]: + """ + Get the list of recommended Servers for a Country. + + Note: There may not always be more than one recommended server. + """ + res = requests.get( + url="https://api.nordvpn.com/v1/servers/recommendations", params={"filters[country_id]": country_id} + ) + if not res.ok: + raise ValueError(f"Failed to get a list of NordVPN countries [{res.status_code}]") + + try: + return res.json() + except json.JSONDecodeError: + raise ValueError("Could not decode list of NordVPN countries, not JSON data.") + + @staticmethod + def get_countries() -> list[dict]: + """Get a list of available Countries and their metadata.""" + res = requests.get( + url="https://api.nordvpn.com/v1/servers/countries", + ) + if not res.ok: + raise ValueError(f"Failed to get a list of NordVPN countries [{res.status_code}]") + + try: + return res.json() + except json.JSONDecodeError: + raise ValueError("Could not decode list of NordVPN countries, not JSON data.") diff --git a/reset/packages/envied/src/envied/core/proxies/proxy.py b/reset/packages/envied/src/envied/core/proxies/proxy.py new file mode 100644 index 0000000..10e044a --- /dev/null +++ b/reset/packages/envied/src/envied/core/proxies/proxy.py @@ -0,0 +1,31 @@ +from abc import abstractmethod +from typing import Optional + + +class Proxy: + @abstractmethod + def __init__(self, **kwargs): + """ + The constructor initializes the Service using passed configuration data. + + Any authorization or pre-fetching of data should be done here. + """ + + @abstractmethod + def __repr__(self) -> str: + """Return a string denoting a list of Countries and Servers (if possible).""" + countries = ... + servers = ... + return f"{countries} Countr{['ies', 'y'][countries == 1]} ({servers} Server{['s', ''][servers == 1]})" + + @abstractmethod + def get_proxy(self, query: str) -> Optional[str]: + """ + Get a Proxy URI from the Proxy Service. + + Only return None if the query was accepted, but no proxy could be returned. + Otherwise, please use exceptions to denote any errors with the call or query. + + The returned Proxy URI must be a string supported by Python-Requests: + '{scheme}://[{user}:{pass}@]{host}:{port}' + """ diff --git a/reset/packages/envied/src/envied/core/proxies/resolve.py b/reset/packages/envied/src/envied/core/proxies/resolve.py new file mode 100644 index 0000000..faf5624 --- /dev/null +++ b/reset/packages/envied/src/envied/core/proxies/resolve.py @@ -0,0 +1,88 @@ +"""Shared proxy provider initialization and resolution. + +Used by both the REST API handlers and the remote service client. +""" + +from __future__ import annotations + +import logging +import re +from typing import Any, List, Optional + +log = logging.getLogger("proxies") + + +def initialize_proxy_providers() -> List[Any]: + """Initialize and return available proxy providers from config.""" + proxy_providers: list = [] + try: + from envied.core import binaries + from envied.core.config import config as main_config + from envied.core.proxies.basic import Basic + from envied.core.proxies.hola import Hola + from envied.core.proxies.nordvpn import NordVPN + from envied.core.proxies.surfsharkvpn import SurfsharkVPN + + proxy_config = getattr(main_config, "proxy_providers", {}) + + if proxy_config.get("basic"): + proxy_providers.append(Basic(**proxy_config["basic"])) + if proxy_config.get("nordvpn"): + proxy_providers.append(NordVPN(**proxy_config["nordvpn"])) + if proxy_config.get("surfsharkvpn"): + proxy_providers.append(SurfsharkVPN(**proxy_config["surfsharkvpn"])) + if hasattr(binaries, "HolaProxy") and binaries.HolaProxy: + proxy_providers.append(Hola()) + + for provider in proxy_providers: + log.info(f"Loaded {provider.__class__.__name__}: {provider}") + + if not proxy_providers: + log.warning("No proxy providers were loaded. Check your proxy provider configuration in envied.yaml") + + except Exception as e: + log.warning(f"Failed to initialize some proxy providers: {e}") + + return proxy_providers + + +def resolve_proxy(proxy: str, proxy_providers: List[Any]) -> Optional[str]: + """Resolve a proxy parameter to an actual proxy URI. + + Accepts: + - Direct URI: "https://...", "socks5://..." + - Country code: "us", "uk" + - Provider:country: "nordvpn:us" + """ + if not proxy: + return None + + if re.match(r"^(https?://|socks)", proxy): + return proxy + + requested_provider = None + query = proxy + if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE): + requested_provider, query = proxy.split(":", maxsplit=1) + + if requested_provider: + provider = next( + (x for x in proxy_providers if x.__class__.__name__.lower() == requested_provider.lower()), + None, + ) + if not provider: + available = [x.__class__.__name__ for x in proxy_providers] + raise ValueError(f"Proxy provider '{requested_provider}' not found. Available: {available}") + proxy_uri = provider.get_proxy(query) + if not proxy_uri: + raise ValueError(f"Proxy provider {requested_provider} had no proxy for {query}") + log.info(f"Using {provider.__class__.__name__} Proxy: {proxy_uri}") + return proxy_uri + + for provider in proxy_providers: + proxy_uri = provider.get_proxy(query) + if proxy_uri: + log.info(f"Using {provider.__class__.__name__} Proxy: {proxy_uri}") + return proxy_uri + + raise ValueError(f"No proxy provider had a proxy for {proxy}") diff --git a/reset/packages/envied/src/envied/core/proxies/surfsharkvpn.py b/reset/packages/envied/src/envied/core/proxies/surfsharkvpn.py new file mode 100644 index 0000000..e811360 --- /dev/null +++ b/reset/packages/envied/src/envied/core/proxies/surfsharkvpn.py @@ -0,0 +1,173 @@ +import json +import random +import re +from typing import Optional + +import requests + +from envied.core.proxies.proxy import Proxy + + +class SurfsharkVPN(Proxy): + def __init__(self, username: str, password: str, server_map: Optional[dict[str, int]] = None): + """ + Proxy Service using SurfsharkVPN Service Credentials. + + A username and password must be provided. These are Service Credentials, not your Login Credentials. + The Service Credentials can be found here: https://my.surfshark.com/vpn/manual-setup/main/openvpn + """ + if not username: + raise ValueError("No Username was provided to the SurfsharkVPN Proxy Service.") + if not password: + raise ValueError("No Password was provided to the SurfsharkVPN Proxy Service.") + if not re.match(r"^[a-z0-9]{48}$", username + password, re.IGNORECASE) or "@" in username: + raise ValueError( + "The Username and Password must be SurfsharkVPN Service Credentials, not your Login Credentials. " + "The Service Credentials can be found here: https://my.surfshark.com/vpn/manual-setup/main/openvpn" + ) + + if server_map is not None and not isinstance(server_map, dict): + raise TypeError(f"Expected server_map to be a dict mapping a region to a server ID, not '{server_map!r}'.") + + self.username = username + self.password = password + self.server_map = server_map or {} + + self.countries = self.get_countries() + + def __repr__(self) -> str: + countries = len(set(x.get("country") for x in self.countries if x.get("country"))) + servers = sum(1 for x in self.countries if x.get("connectionName")) + + return f"{countries} Countr{['ies', 'y'][countries == 1]} ({servers} Server{['s', ''][servers == 1]})" + + def get_proxy(self, query: str) -> Optional[str]: + """ + Get an HTTP(SSL) proxy URI for a SurfsharkVPN server. + + Supports: + - Country code: "us", "ca", "gb" + - Country ID: "228" + - Specific server: "us-bos" (Boston) + - City selection: "us:seattle", "ca:toronto" + """ + query = query.lower() + city = None + + # Check if query includes city specification (e.g., "us:seattle") + if ":" in query: + query, city = query.split(":", maxsplit=1) + city = city.strip() + + if re.match(r"^[a-z]{2}\d+$", query): + # country and surfsharkvpn server id, e.g., au-per, be-anr, us-bos + hostname = f"{query}.prod.surfshark.com" + else: + if query.isdigit(): + # country id + country = self.get_country(by_id=int(query)) + elif re.match(r"^[a-z]+$", query): + # country code + country = self.get_country(by_code=query) + else: + raise ValueError(f"The query provided is unsupported and unrecognized: {query}") + if not country: + # SurfsharkVPN doesnt have servers in this region + return + + # Check server_map for pinned servers (can include city) + server_map_key = f"{country['countryCode'].lower()}:{city}" if city else country["countryCode"].lower() + server_mapping = self.server_map.get(server_map_key) or ( + self.server_map.get(country["countryCode"].lower()) if not city else None + ) + + if server_mapping: + # country was set to a specific server ID in config + hostname = f"{country['code'].lower()}{server_mapping}.prod.surfshark.com" + else: + # get the random server ID + random_server = self.get_random_server(country["countryCode"], city) + if not random_server: + raise ValueError( + f"The SurfsharkVPN Country {query} currently has no random servers. " + "Try again later. If the issue persists, double-check the query." + ) + hostname = random_server + + return f"https://{self.username}:{self.password}@{hostname}:443" + + def get_country(self, by_id: Optional[int] = None, by_code: Optional[str] = None) -> Optional[dict]: + """Search for a Country and it's metadata.""" + if all(x is None for x in (by_id, by_code)): + raise ValueError("At least one search query must be made.") + + for country in self.countries: + if all( + [ + by_id is None or country["id"] == int(by_id), + by_code is None or country["countryCode"] == by_code.upper(), + ] + ): + return country + + def get_random_server(self, country_id: str, city: Optional[str] = None): + """ + Get a random server for a Country, optionally filtered by city. + + Args: + country_id: The country code (e.g., "US", "CA") + city: Optional city name to filter by (case-insensitive) + + Note: The API may include a 'location' field with city information. + If not available, this will return any server from the country. + """ + servers = [x for x in self.countries if x["countryCode"].lower() == country_id.lower()] + + # Filter by city if specified + if city: + city_lower = city.lower() + # Check if servers have a 'location' field for city filtering + city_servers = [ + x + for x in servers + if x.get("location", "").lower() == city_lower or x.get("city", "").lower() == city_lower + ] + + if city_servers: + servers = city_servers + else: + raise ValueError( + f"No servers found in city '{city}' for country '{country_id}'. " + "Try a different city or check the city name spelling." + ) + + # Get connection names from filtered servers + if not servers: + raise ValueError(f"Could not get random server for country '{country_id}': no servers found.") + + # Only include servers that actually have a connection name to avoid KeyError. + connection_names = [x["connectionName"] for x in servers if "connectionName" in x] + if not connection_names: + raise ValueError( + f"Could not get random server for country '{country_id}': no servers with connectionName found." + ) + + return random.choice(connection_names) + + @staticmethod + def get_countries() -> list[dict]: + """Get a list of available Countries and their metadata.""" + res = requests.get( + url="https://api.surfshark.com/v3/server/clusters/all", + headers={ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", + "Content-Type": "application/json", + }, + ) + if not res.ok: + raise ValueError(f"Failed to get a list of SurfsharkVPN countries [{res.status_code}]") + + try: + return res.json() + except json.JSONDecodeError: + raise ValueError("Could not decode list of SurfsharkVPN countries, not JSON data.") diff --git a/reset/packages/envied/src/envied/core/proxies/windscribevpn.py b/reset/packages/envied/src/envied/core/proxies/windscribevpn.py new file mode 100644 index 0000000..4a00192 --- /dev/null +++ b/reset/packages/envied/src/envied/core/proxies/windscribevpn.py @@ -0,0 +1,177 @@ +import json +import random +import re +from typing import Optional + +import requests + +from envied.core.proxies.proxy import Proxy + + +class WindscribeVPN(Proxy): + def __init__(self, username: str, password: str, server_map: Optional[dict[str, str]] = None): + """ + Proxy Service using WindscribeVPN Service Credentials. + + A username and password must be provided. These are Service Credentials, not your Login Credentials. + The Service Credentials can be found here: https://windscribe.com/getconfig/openvpn + """ + if not username: + raise ValueError("No Username was provided to the WindscribeVPN Proxy Service.") + if not password: + raise ValueError("No Password was provided to the WindscribeVPN Proxy Service.") + + if server_map is not None and not isinstance(server_map, dict): + raise TypeError(f"Expected server_map to be a dict mapping a region to a hostname, not '{server_map!r}'.") + + self.username = username + self.password = password + self.server_map = server_map or {} + + self.countries = self.get_countries() + + def __repr__(self) -> str: + countries = len(set(x.get("country_code") for x in self.countries if x.get("country_code"))) + servers = sum( + len(host) + for location in self.countries + for group in location.get("groups", []) + for host in group.get("hosts", []) + ) + + return f"{countries} Countr{['ies', 'y'][countries == 1]} ({servers} Server{['s', ''][servers == 1]})" + + def get_proxy(self, query: str) -> Optional[str]: + """ + Get an HTTPS proxy URI for a WindscribeVPN server. + + Supports: + - Country code: "us", "ca", "gb" + - Specific server: "sg007", "us150" + - City selection: "us:seattle", "ca:toronto" + """ + query = query.lower() + city = None + + # Check if query includes city specification (e.g., "ca:toronto") + if ":" in query: + query, city = query.split(":", maxsplit=1) + city = city.strip() + + # Check server_map for pinned servers (can include city) + server_map_key = f"{query}:{city}" if city else query + if server_map_key in self.server_map: + hostname = self.server_map[server_map_key] + elif query in self.server_map: + hostname = self.server_map[query] + else: + server_match = re.match(r"^([a-z]{2})(\d+)$", query) + if server_match: + # Specific server selection, e.g., sg007, us150 + country_code, server_num = server_match.groups() + hostname = self.get_specific_server(country_code, server_num) + if not hostname: + raise ValueError( + f"No WindscribeVPN server found matching '{query}'. " + f"Check the server number or use just '{country_code}' for a random server." + ) + elif re.match(r"^[a-z]+$", query): + hostname = self.get_random_server(query, city) + else: + raise ValueError(f"The query provided is unsupported and unrecognized: {query}") + + if not hostname: + return None + + hostname = hostname.split(":")[0] + return f"https://{self.username}:{self.password}@{hostname}:443" + + def get_specific_server(self, country_code: str, server_num: str) -> Optional[str]: + """ + Find a specific server by country code and server number. + + Matches against hostnames like "sg-007.totallyacdn.com" for query "sg007". + Tries both the raw number and zero-padded variants. + + Args: + country_code: Two-letter country code (e.g., "sg", "us") + server_num: Server number as string (e.g., "007", "7", "150") + + Returns: + The matching hostname, or None if not found. + """ + num_stripped = server_num.lstrip("0") or "0" + candidates = { + f"{country_code}-{server_num}.", + f"{country_code}-{num_stripped}.", + f"{country_code}-{server_num.zfill(3)}.", + } + + for location in self.countries: + if location.get("country_code", "").lower() != country_code: + continue + for group in location.get("groups", []): + for host in group.get("hosts", []): + hostname = host.get("hostname", "") + if any(hostname.startswith(prefix) for prefix in candidates): + return hostname + + return None + + def get_random_server(self, country_code: str, city: Optional[str] = None) -> Optional[str]: + """ + Get a random server hostname for a country, optionally filtered by city. + + Args: + country_code: The country code (e.g., "us", "ca") + city: Optional city name to filter by (case-insensitive) + + Returns: + A random hostname from matching servers, or None if none available. + """ + hostnames = [] + + # Collect hostnames from ALL locations matching the country code + for location in self.countries: + if location.get("country_code", "").lower() == country_code.lower(): + for group in location.get("groups", []): + # Filter by city if specified + if city: + group_city = group.get("city", "") + if group_city.lower() != city.lower(): + continue + + # Collect hostnames from this group + for host in group.get("hosts", []): + if hostname := host.get("hostname"): + hostnames.append(hostname) + + if hostnames: + return random.choice(hostnames) + elif city: + # No servers found for the specified city + raise ValueError( + f"No servers found in city '{city}' for country code '{country_code}'. " + "Try a different city or check the city name spelling." + ) + + return None + + @staticmethod + def get_countries() -> list[dict]: + """Get a list of available Countries and their metadata.""" + res = requests.get( + url="https://assets.windscribe.com/serverlist/firefox/1/1", + headers={ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", + "Content-Type": "application/json", + }, + ) + if not res.ok: + raise ValueError(f"Failed to get a list of WindscribeVPN locations [{res.status_code}]") + + try: + data = res.json() + return data.get("data", []) + except json.JSONDecodeError: + raise ValueError("Could not decode list of WindscribeVPN locations, not JSON data.") diff --git a/reset/packages/envied/src/envied/core/remote_service.py b/reset/packages/envied/src/envied/core/remote_service.py new file mode 100644 index 0000000..e53ad8b --- /dev/null +++ b/reset/packages/envied/src/envied/core/remote_service.py @@ -0,0 +1,853 @@ +"""Remote service adapter for envied. + +Implements the Service interface by proxying authenticate, get_titles, +get_tracks, get_chapters, and license methods to a remote envied.server. +Everything else (track selection, download, decrypt, mux) runs locally. +""" + +from __future__ import annotations + +import base64 +import logging +import time +from enum import Enum +from http.cookiejar import CookieJar +from typing import Any, Dict, Optional, Union + +import click +import requests +from langcodes import Language +from requests.adapters import HTTPAdapter, Retry +from rich.padding import Padding +from rich.rule import Rule + +from envied.core.config import config +from envied.core.console import console +from envied.core.constants import AnyTrack +from envied.core.credential import Credential +from envied.core.titles import Title_T, Titles_T, remap_titles +from envied.core.titles.episode import Episode, Series +from envied.core.titles.movie import Movie, Movies +from envied.core.tracks import Audio, Chapter, Chapters, Subtitle, Tracks, Video +from envied.core.tracks.attachment import Attachment +from envied.core.tracks.track import Track + +log = logging.getLogger("remote_service") + + +class RemoteClient: + """HTTP client for the envied.serve API.""" + + def __init__(self, server_url: str, api_key: str) -> None: + self.server_url = server_url.rstrip("/") + self.api_key = api_key + self._session: Optional[requests.Session] = None + + @property + def session(self) -> requests.Session: + if self._session is None: + from envied.core import __version__ + + self._session = requests.Session() + self._session.headers["User-Agent"] = f"envied.{__version__}" + if self.api_key: + self._session.headers["X-Secret-Key"] = self.api_key + return self._session + + def _request(self, method: str, endpoint: str, data: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + url = f"{self.server_url}{endpoint}" + try: + resp = getattr(self.session, method)(url, json=data, timeout=120 if method == "post" else 30) + except requests.ConnectionError: + log.error(f"Could not connect to remote server at {self.server_url}. Is it running? (envied.serve)") + raise SystemExit(1) + except requests.Timeout: + log.error(f"Request to remote server timed out: {endpoint}") + raise SystemExit(1) + result = resp.json() + if resp.status_code >= 400: + error_msg = result.get("message", resp.text) + error_code = result.get("error_code", "UNKNOWN") + log.error(f"Server error [{error_code}]: {error_msg}") + raise SystemExit(1) + return result + + def post(self, endpoint: str, data: Dict[str, Any]) -> Dict[str, Any]: + return self._request("post", endpoint, data) + + def get(self, endpoint: str) -> Dict[str, Any]: + return self._request("get", endpoint) + + def delete(self, endpoint: str) -> Dict[str, Any]: + return self._request("delete", endpoint) + + +def _enum_get(enum_cls: type[Enum], name: Optional[str], default: Any = None) -> Any: + """Safely get an enum value by name.""" + if not name: + return default + try: + return enum_cls[name] + except KeyError: + return default + + +def _deserialize_video(data: Dict[str, Any]) -> Video: + v = Video( + url=data.get("url") or "https://placeholder", + language=Language.get(data.get("language") or "und"), + descriptor=_enum_get(Track.Descriptor, data.get("descriptor"), Track.Descriptor.URL), + codec=_enum_get(Video.Codec, data.get("codec")), + range_=_enum_get(Video.Range, data.get("range"), Video.Range.SDR), + bitrate=data["bitrate"] * 1000 if data.get("bitrate") else 0, + width=data.get("width") or 0, + height=data.get("height") or 0, + fps=data.get("fps"), + id_=data.get("id"), + ) + return v + + +def _deserialize_audio(data: Dict[str, Any]) -> Audio: + a = Audio( + url=data.get("url") or "https://placeholder", + language=Language.get(data.get("language") or "und"), + descriptor=_enum_get(Track.Descriptor, data.get("descriptor"), Track.Descriptor.URL), + codec=_enum_get(Audio.Codec, data.get("codec")), + bitrate=data["bitrate"] * 1000 if data.get("bitrate") else 0, + channels=data.get("channels"), + joc=1 if data.get("atmos") else 0, + descriptive=data.get("descriptive", False), + id_=data.get("id"), + ) + return a + + +def _deserialize_subtitle(data: Dict[str, Any]) -> Subtitle: + return Subtitle( + url=data.get("url") or "https://placeholder", + language=Language.get(data.get("language") or "und"), + descriptor=_enum_get(Track.Descriptor, data.get("descriptor"), Track.Descriptor.URL), + codec=_enum_get(Subtitle.Codec, data.get("codec")), + cc=data.get("cc", False), + sdh=data.get("sdh", False), + forced=data.get("forced", False), + id_=data.get("id"), + ) + + +def _reconstruct_drm(drm_list: Optional[list]) -> list: + """Reconstruct DRM objects from serialized API data.""" + if not drm_list: + return [] + result = [] + for drm_info in drm_list: + drm_type = drm_info.get("type", "") + pssh_str = drm_info.get("pssh") + if not pssh_str: + continue + try: + if drm_type == "widevine": + from pywidevine.pssh import PSSH as WidevinePSSH + + from envied.core.drm import Widevine + + wv_pssh = WidevinePSSH(pssh_str) + result.append(Widevine(pssh=wv_pssh)) + elif drm_type == "playready": + import base64 as b64 + + from pyplayready.system.pssh import PSSH as PlayReadyPSSH + + from envied.core.drm import PlayReady + + pr_pssh = PlayReadyPSSH(b64.b64decode(pssh_str)) + result.append(PlayReady(pssh=pr_pssh, pssh_b64=pssh_str)) + except Exception: + continue + return result + + +def _build_tracks(data: Dict[str, Any]) -> Tracks: + tracks = Tracks() + tracks.videos = [_deserialize_video(v) for v in data.get("video", [])] + tracks.audio = [_deserialize_audio(a) for a in data.get("audio", [])] + tracks.subtitles = [_deserialize_subtitle(s) for s in data.get("subtitles", [])] + + for track_data, track_obj in [ + *zip(data.get("video", []), tracks.videos), + *zip(data.get("audio", []), tracks.audio), + ]: + drm_objs = _reconstruct_drm(track_data.get("drm")) + if drm_objs: + track_obj.drm = drm_objs + tracks.attachments = [ + Attachment(url=a["url"], name=a.get("name"), mime_type=a.get("mime_type"), description=a.get("description")) + for a in data.get("attachments", []) + ] + return tracks + + +def _resolve_manifest_data(tracks: Tracks, manifests: list, session: Any) -> None: + """Re-parse serialized manifests and populate track.data for downloading. + + The server serializes DASH and ISM manifest XML as zlib-compressed base64. + We decode and decompress locally, re-parse with the appropriate manifest + parser, then match each remote track to the locally-parsed track by ID + to copy track.data. HLS is skipped as it re-fetches from track.url. + """ + import base64 as b64 + import zlib + + if not manifests: + return + + log_m = logging.getLogger("remote_service") + all_tracks = list(tracks.videos) + list(tracks.audio) + list(tracks.subtitles) + + for manifest_info in manifests: + m_type = manifest_info.get("type") + m_url = manifest_info.get("url") + m_data = manifest_info.get("data") + if not m_data or not m_url: + continue + + try: + raw = zlib.decompress(b64.b64decode(m_data)) + + if m_type == "dash": + from lxml import etree + + from envied.core.manifests import DASH + + xml_tree = etree.fromstring(raw) + fallback_lang = next( + (t.language for t in all_tracks if t.language and str(t.language) != "und"), + None, + ) + local_tracks = DASH(xml_tree, m_url).to_tracks(language=fallback_lang) + elif m_type == "ism": + from lxml import etree + + from envied.core.manifests import ISM + + local_tracks = ISM(etree.fromstring(raw), m_url).to_tracks() + else: + continue + + local_all = list(local_tracks.videos) + list(local_tracks.audio) + list(local_tracks.subtitles) + for remote_track in all_tracks: + if remote_track.data.get(m_type): + continue + matched = _match_track(remote_track, local_all) + if matched and matched.data.get(m_type): + remote_track.data.update(matched.data) + remote_track.descriptor = matched.descriptor + if matched.drm and not remote_track.drm: + remote_track.drm = matched.drm + + except Exception as e: + log_m.warning("Failed to re-parse %s manifest from %s: %s", m_type, m_url, e) + + +def _match_track(remote_track: Track, local_tracks: list) -> Optional[Track]: + """Match a remote track to a locally-parsed track by ID or attributes.""" + remote_id = str(remote_track.id) + for lt in local_tracks: + if str(lt.id) == remote_id: + return lt + + for lt in local_tracks: + if type(lt).__name__ != type(remote_track).__name__: + continue + if lt.codec != remote_track.codec or str(lt.language) != str(remote_track.language): + continue + if hasattr(lt, "width") and hasattr(remote_track, "width"): + if lt.width == remote_track.width and lt.height == remote_track.height: + return lt + elif hasattr(lt, "channels") and hasattr(remote_track, "channels"): + if lt.bitrate == remote_track.bitrate: + return lt + elif hasattr(lt, "forced"): + if lt.forced == remote_track.forced and lt.sdh == remote_track.sdh: + return lt + return None + + +def _build_title(info: Dict[str, Any], service_tag: str, fallback_id: str) -> Union[Episode, Movie]: + svc_class = type(service_tag, (), {}) + lang = Language.get(info["language"]) if info.get("language") else None + if info.get("type") == "episode": + return Episode( + id_=info.get("id", fallback_id), + service=svc_class, + title=info.get("series_title", "Unknown"), + season=info.get("season", 0), + number=info.get("number", 0), + name=info.get("name"), + year=info.get("year"), + language=lang, + ) + return Movie( + id_=info.get("id", fallback_id), + service=svc_class, + name=info.get("name", "Unknown"), + year=info.get("year"), + language=lang, + ) + + +def resolve_server(server_name: Optional[str]) -> tuple[str, str, dict]: + """Resolve server URL, API key, and per-service config from remote_services.""" + remote_services = config.remote_services + if not remote_services: + raise click.ClickException( + "No remote services configured. Add 'remote_services' to your envied.yaml:\n\n" + " remote_services:\n" + " my_server:\n" + ' url: "https://server:8080"\n' + ' api_key: "your-api-key"' + ) + + if server_name: + svc = remote_services.get(server_name) + if not svc: + available = ", ".join(remote_services.keys()) + raise click.ClickException(f"Remote service '{server_name}' not found. Available: {available}") + services = svc.get("services", {}) + services["_server_cdm"] = svc.get("server_cdm", False) + return svc["url"], svc.get("api_key", ""), services + + if len(remote_services) == 1: + name, svc = next(iter(remote_services.items())) + log.info(f"Using remote service: {name}") + services = svc.get("services", {}) + services["_server_cdm"] = svc.get("server_cdm", False) + return svc["url"], svc.get("api_key", ""), services + + available = ", ".join(remote_services.keys()) + raise click.ClickException(f"Multiple remote services configured. Use --server to select one: {available}") + + +def _load_credentials_for_transport(service_tag: str, profile: Optional[str]) -> Optional[Dict[str, str]]: + from envied.commands.dl import dl + + credential = dl.get_credentials(service_tag, profile) + if credential: + result: Dict[str, str] = {"username": credential.username, "password": credential.password} + if credential.extra: + result["extra"] = credential.extra + return result + return None + + +def _load_cookies_for_transport(service_tag: str, profile: Optional[str]) -> Optional[str]: + import zlib + + from envied.commands.dl import dl + + cookie_path = dl.get_cookie_path(service_tag, profile) + if cookie_path and cookie_path.exists(): + return base64.b64encode(zlib.compress(cookie_path.read_bytes())).decode("ascii") + return None + + +def _resolve_proxy(proxy_arg: Optional[str]) -> Optional[str]: + if not proxy_arg: + return None + + from envied.core.proxies.resolve import initialize_proxy_providers, resolve_proxy + + try: + providers = initialize_proxy_providers() + return resolve_proxy(proxy_arg, providers) + except ValueError as e: + raise click.ClickException(str(e)) + + +class RemoteService: + """Service adapter that proxies to a remote envied.server. + + Implements the same interface dl.py's result() expects without + subclassing Service (avoids proxy/geofence setup in __init__). + """ + + ALIASES: tuple[str, ...] = () + GEOFENCE: tuple[str, ...] = () + NO_SUBTITLES: bool = False + + def __init__( + self, + ctx: click.Context, + service_tag: str, + title_id: str, + server_url: str, + api_key: str, + services_config: dict, + service_params: Optional[Dict[str, Any]] = None, + ) -> None: + self.__class__.__name__ = service_tag + console.print(Padding(Rule(f"[rule.text]Service: {service_tag} (Remote)"), (1, 2))) + + self.service_tag = service_tag + self.title_id = title_id + self.client = RemoteClient(server_url, api_key) + self.ctx = ctx + self._service_params = service_params or {} + self.log = logging.getLogger(service_tag) + self.credential: Optional[Credential] = None + self.current_region: Optional[str] = None + self.title_cache = None + self._titles: Optional[Titles_T] = None + self._tracks_by_title: Dict[str, Tracks] = {} + self._chapters_by_title: Dict[str, list] = {} + self._session_id: Optional[str] = None + self._server_cdm_type: str = "widevine" + + self._session = requests.Session() + self._session.headers.update(config.headers) + self._session.mount( + "https://", + HTTPAdapter( + max_retries=Retry(total=5, backoff_factor=0.2, status_forcelist=[429, 500, 502, 503, 504]), + pool_block=True, + ), + ) + self._session.mount("http://", self._session.adapters["https://"]) + + svc_config = services_config.get(service_tag, {}) + self._server_cdm = services_config.get("_server_cdm", False) + self._apply_service_config(svc_config) + + def _apply_service_config(self, svc_config: dict) -> None: + if not svc_config: + return + config_maps = { + "cdm": ("cdm", self.service_tag), + "decryption": ("decryption_map", self.service_tag), + "downloader": ("downloader_map", self.service_tag), + } + for key, (attr, tag) in config_maps.items(): + if svc_config.get(key): + target = getattr(config, attr, None) + if target is None: + setattr(config, attr, {}) + target = getattr(config, attr) + target[tag] = svc_config[key] + + if svc_config.get("downloader"): + config.downloader = svc_config["downloader"] + if svc_config.get("decryption"): + config.decryption = svc_config["decryption"] + + extra = {k: v for k, v in svc_config.items() if k not in config_maps} + if extra: + existing = config.services.get(self.service_tag, {}) + for key, value in extra.items(): + if key in existing and isinstance(existing[key], dict) and isinstance(value, dict): + existing[key].update(value) + else: + existing[key] = value + config.services[self.service_tag] = existing + + @property + def session(self) -> requests.Session: + return self._session + + @property + def title(self) -> str: + return self.title_id + + def authenticate(self, cookies: Optional[CookieJar] = None, credential: Optional[Credential] = None) -> None: + self.credential = credential + profile = self.ctx.parent.params.get("profile") if self.ctx.parent else None + proxy = self.ctx.parent.params.get("proxy") if self.ctx.parent else None + no_proxy = self.ctx.parent.params.get("no_proxy", False) if self.ctx.parent else False + + create_data: Dict[str, Any] = {"service": self.service_tag, "title_id": self.title_id} + + credentials = _load_credentials_for_transport(self.service_tag, profile) + if credentials: + create_data["credentials"] = credentials + + cookies_text = _load_cookies_for_transport(self.service_tag, profile) + if cookies_text: + create_data["cookies"] = cookies_text + + if not no_proxy and proxy: + resolved_proxy = _resolve_proxy(proxy) + if resolved_proxy: + create_data["proxy"] = resolved_proxy + + if not no_proxy and not proxy: + try: + from envied.core.utils.ip_info import get_ip_info + + ip_info = get_ip_info(self._session, cached=True) + if ip_info and ip_info.get("country"): + create_data["client_region"] = ip_info["country"].lower() + except Exception: + pass + + if profile: + create_data["profile"] = profile + if no_proxy: + create_data["no_proxy"] = True + # Forward track selection params so the server fetches the right manifests + if self.ctx.parent: + range_ = self.ctx.parent.params.get("range_") + if range_: + create_data["range_"] = [r.name for r in range_] + vcodec = self.ctx.parent.params.get("vcodec") + if vcodec: + create_data["vcodec"] = [c.name for c in vcodec] + quality = self.ctx.parent.params.get("quality") + if quality: + create_data["quality"] = list(quality) + if self.ctx.parent.params.get("best_available"): + create_data["best_available"] = True + + if self._service_params: + create_data.update(self._service_params) + + cdm = self.ctx.obj.cdm if self.ctx.obj else None + if cdm is not None: + from envied.core.cdm.detect import is_playready_cdm + + create_data["cdm_type"] = "playready" if is_playready_cdm(cdm) else "widevine" + + cache_data = self._load_cache_files() + if cache_data: + create_data["cache"] = cache_data + + result = self.client.post("/api/session/create", create_data) + self._session_id = result["session_id"] + + status = result.get("status", "authenticated") + if status == "authenticating": + self._poll_auth_completion() + + def _poll_auth_completion(self, poll_interval: float = 2.0, timeout: float = 600.0) -> None: + """Poll the server until authentication completes, handling interactive prompts. + + When the server needs user input (OTP, device code, PIN), it returns + ``pending_input`` with a prompt. We display it locally, collect the + response, and POST it back. The server resumes its auth flow. + """ + deadline = time.monotonic() + timeout + + while time.monotonic() < deadline: + resp = self.client.get(f"/api/session/{self._session_id}/prompt") + status = resp.get("status") + + if status == "authenticated": + return + + if status == "failed": + error = resp.get("error", "Authentication failed on server") + log.error(f"Remote auth failed: {error}") + raise SystemExit(1) + + if status == "pending_input": + prompt = resp.get("prompt", "Enter input: ") + user_response = click.prompt(prompt.rstrip("\n "), default="", show_default=False) + self.client.post( + f"/api/session/{self._session_id}/prompt", + {"response": user_response}, + ) + continue + + time.sleep(poll_interval) + + log.error("Remote authentication timed out") + raise SystemExit(1) + + def get_titles(self) -> Titles_T: + if self._titles is not None: + return self._titles + result = self.client.get(f"/api/session/{self._session_id}/titles") + titles_list = [_build_title(t, self.service_tag, self.title_id) for t in result.get("titles", [])] + self._titles = ( + Series(titles_list) if titles_list and isinstance(titles_list[0], Episode) else Movies(titles_list) + ) + return self._titles + + def get_titles_cached(self, title_id: str = None) -> Titles_T: + """Apply the client's local title_map to titles fetched from the remote server. + + Lets users rename titles for remote services they don't have installed locally. + The server sends raw titles; the client's own ``services..title_map`` wins. + """ + title_map = (config.services.get(self.service_tag) or {}).get("title_map") or {} + return remap_titles(self.get_titles(), title_map) + + def get_tracks(self, title: Title_T) -> Tracks: + title_id = str(title.id) + if title_id in self._tracks_by_title: + return self._tracks_by_title[title_id] + result = self.client.post(f"/api/session/{self._session_id}/tracks", {"title_id": title_id}) + tracks = _build_tracks(result) + + for k, v in result.get("session_headers", {}).items(): + if k.lower() not in ("host", "content-length", "content-type"): + self._session.headers[k] = v + for k, v in result.get("session_cookies", {}).items(): + self._session.cookies.set(k, v) + + _resolve_manifest_data(tracks, result.get("manifests", []), self._session) + + self._server_cdm_type = result.get("server_cdm_type", "widevine") + + self._tracks_by_title[title_id] = tracks + self._chapters_by_title[title_id] = result.get("chapters", []) + + return tracks + + def resolve_server_keys(self, title: Title_T) -> None: + """Resolve DRM keys via server CDM for all tracks on a title. + + Called by dl.py between track selection and download. The server + decides which CDM device to use and tells the client via + server_cdm_type. We send track IDs and the server does the full + CDM flow, returning KID:KEY pairs. + """ + if not self._server_cdm: + return + + from uuid import UUID + + track_ids = [str(t.id) for t in title.tracks.videos + title.tracks.audio] + if not track_ids: + return + + drm_type = getattr(self, "_server_cdm_type", "widevine") + self.log.debug(f"Requesting server CDM keys (server_cdm_type={drm_type})") + + try: + with console.status("Retrieving Remote License...", spinner="dots"): + resp = self.client.post( + f"/api/session/{self._session_id}/license", + { + "track_ids": track_ids, + "mode": "server_cdm", + "drm_type": drm_type, + }, + ) + keys_by_track = resp.get("keys", {}) + server_drm_type = resp.get("drm_type", drm_type) + self._server_cdm_type = server_drm_type + self.log.debug(f"Server responded with drm_type={server_drm_type}, keys for {len(keys_by_track)} track(s)") + + for track in title.tracks: + track_keys = keys_by_track.get(str(track.id), {}) + if not track_keys: + continue + + kid_list = list(track_keys.keys()) + drm_obj = self._create_drm_stub(server_drm_type, kid_list) + for kid_hex, key_hex in track_keys.items(): + drm_obj.content_keys[UUID(hex=kid_hex)] = key_hex + track.drm = [drm_obj] + self.log.debug( + f"Track {track.id}: set DRM to {drm_obj.__class__.__name__} with {len(track_keys)} key(s)" + ) + key_count = sum(len(v) for v in keys_by_track.values()) + if key_count: + self.log.debug(f"Server CDM resolved {key_count} key(s) using {server_drm_type.upper()}") + except Exception as e: + self.log.warning("Failed to resolve server CDM keys: %s", e) + + @staticmethod + def _create_drm_stub(drm_type: str, kid_hexes: list[str]) -> Any: + """Create a DRM object stub matching the type the server actually used. + + For server_cdm mode, this is only used for display — keys are already + resolved. We build a minimal DRM object that holds content_keys. + """ + from uuid import UUID + + if drm_type == "playready": + import base64 as b64 + import struct + + from pyplayready.system.pssh import PSSH as PlayReadyPSSH + + from envied.core.drm import PlayReady + + kid_uuids = [UUID(hex=k) for k in kid_hexes] + kid_b64 = b64.b64encode(kid_uuids[0].bytes_le).decode() + wrm_xml = ( + '' + f"16AESCTR" + f"{kid_b64}" + ) + wrm_bytes = wrm_xml.encode("utf-16-le") + record_length = len(wrm_bytes) + obj_length = 4 + 2 + 2 + 2 + record_length + pr_obj = struct.pack(" Chapters: + title_id = str(title.id) + if title_id not in self._chapters_by_title: + self.get_tracks(title) + raw = self._chapters_by_title.get(title_id, []) + return Chapters([Chapter(ch["timestamp"], ch.get("name")) for ch in raw]) + + def get_widevine_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[Union[bytes, str]]: + return self._proxy_license(challenge, track, "widevine") + + def get_playready_license( + self, *, challenge: bytes, title: Title_T, track: AnyTrack + ) -> Optional[Union[bytes, str]]: + return self._proxy_license(challenge, track, "playready") + + def get_widevine_service_certificate( + self, + *, + challenge: bytes, + title: Title_T, + track: AnyTrack, + ) -> Union[bytes, str]: + try: + resp = self.client.post( + f"/api/session/{self._session_id}/license", + { + "track_id": str(track.id), + "challenge": base64.b64encode(challenge).decode("ascii"), + "drm_type": "widevine", + "is_certificate": True, + }, + ) + return base64.b64decode(resp["license"]) + except Exception: + return None + + def _proxy_license(self, challenge: Union[bytes, str], track: AnyTrack, drm_type: str) -> bytes: + if isinstance(challenge, str): + challenge = challenge.encode("utf-8") + + pssh_b64 = None + if track.drm: + for drm_obj in track.drm: + drm_class = drm_obj.__class__.__name__ + if drm_type == "playready" and drm_class == "PlayReady": + pssh_b64 = drm_obj.data["pssh_b64"] + break + elif drm_type == "widevine" and drm_class == "Widevine": + pssh_b64 = drm_obj.pssh.dumps() + break + + if self._server_cdm: + from uuid import UUID + + if pssh_b64: + try: + resp = self.client.post( + f"/api/session/{self._session_id}/license", + { + "track_id": str(track.id), + "drm_type": drm_type, + "mode": "server_cdm", + "pssh": pssh_b64, + }, + ) + keys = resp.get("keys", {}) + if keys and track.drm: + for drm_obj in track.drm: + if hasattr(drm_obj, "content_keys"): + for kid_hex, key_hex in keys.items(): + drm_obj.content_keys[UUID(hex=kid_hex)] = key_hex + return challenge + except Exception as e: + self.log.warning("server_cdm license failed: %s", e) + return challenge + + payload = { + "track_id": str(track.id), + "challenge": base64.b64encode(challenge).decode("ascii"), + "drm_type": drm_type, + } + if pssh_b64: + payload["pssh"] = pssh_b64 + + resp = self.client.post(f"/api/session/{self._session_id}/license", payload) + return base64.b64decode(resp["license"]) + + def on_segment_downloaded(self, track: AnyTrack, segment: Any) -> None: + pass + + def on_track_downloaded(self, track: AnyTrack) -> None: + pass + + def on_track_decrypted(self, track: AnyTrack, drm: Any, segment: Any = None) -> None: + pass + + def on_track_repacked(self, track: AnyTrack) -> None: + pass + + def on_track_multiplex(self, track: AnyTrack) -> None: + pass + + def close(self) -> None: + if self._session_id: + try: + result = self.client.delete(f"/api/session/{self._session_id}") + self._save_returned_cache(result.get("cache", {})) + except Exception as e: + self.log.warning(f"Failed to clean up remote session: {e}") + self._session_id = None + + def _save_returned_cache(self, cache_data: Dict[str, str]) -> None: + """Save cache files returned by the server to the local cache directory. + + The server returns updated cache files (e.g. refreshed tokens) on + session close. Writing them locally means the next remote session + can forward them back, skipping interactive auth. + """ + if not cache_data: + return + + import zlib + + cache_dir = config.directories.cache / self.service_tag + cache_dir.mkdir(parents=True, exist_ok=True) + + for key, content in cache_data.items(): + try: + decompressed = zlib.decompress(base64.b64decode(content)) + (cache_dir / key).with_suffix(".json").write_bytes(decompressed) + except Exception as e: + self.log.warning(f"Failed to save returned cache file '{key}': {e}") + + self.log.info(f"Saved {len(cache_data)} cache file(s) from server") + + def _load_cache_files(self) -> Dict[str, str]: + import zlib + + cache_dir = config.directories.cache / self.service_tag + if not cache_dir.is_dir(): + return {} + return { + f.stem: base64.b64encode(zlib.compress(f.read_bytes())).decode("ascii") + for f in cache_dir.glob("*.json") + if not f.stem.startswith("titles_") + } + + +__all__ = ("RemoteClient", "RemoteService", "resolve_server") diff --git a/reset/packages/envied/src/envied/core/search_result.py b/reset/packages/envied/src/envied/core/search_result.py new file mode 100644 index 0000000..3bfe46c --- /dev/null +++ b/reset/packages/envied/src/envied/core/search_result.py @@ -0,0 +1,48 @@ +from typing import Optional, Union + + +class SearchResult: + def __init__( + self, + id_: Union[str, int], + title: str, + description: Optional[str] = None, + label: Optional[str] = None, + url: Optional[str] = None, + image: Optional[str] = None + ): + """ + A Search Result for any support Title Type. + + Parameters: + id_: The search result's Title ID. + title: The primary display text, e.g., the Title's Name. + description: The secondary display text, e.g., the Title's Description or + further title information. + label: The tertiary display text. This will typically be used to display + an informative label or tag to the result. E.g., "unavailable", the + title's price tag, region, etc. + url: A hyperlink to the search result or title's page. + """ + if not isinstance(id_, (str, int)): + raise TypeError(f"Expected id_ to be a {str} or {int}, not {type(id_)}") + if not isinstance(title, str): + raise TypeError(f"Expected title to be a {str}, not {type(title)}") + if not isinstance(description, (str, type(None))): + raise TypeError(f"Expected description to be a {str}, not {type(description)}") + if not isinstance(label, (str, type(None))): + raise TypeError(f"Expected label to be a {str}, not {type(label)}") + if not isinstance(url, (str, type(None))): + raise TypeError(f"Expected url to be a {str}, not {type(url)}") + if not isinstance(image, (str, type(None))): + raise TypeError(f"Expected image to be a {str}, not {type(image)}") + + self.id = id_ + self.title = title + self.description = description + self.label = label + self.url = url + self.image = image + + +__all__ = ("SearchResult",) diff --git a/reset/packages/envied/src/envied/core/service.py b/reset/packages/envied/src/envied/core/service.py new file mode 100644 index 0000000..5758404 --- /dev/null +++ b/reset/packages/envied/src/envied/core/service.py @@ -0,0 +1,612 @@ +import logging +from abc import ABCMeta, abstractmethod +from collections.abc import Callable, Generator +from dataclasses import dataclass, field +from http.cookiejar import CookieJar +from pathlib import Path +from typing import TYPE_CHECKING, Optional, Union +from urllib.parse import urlparse, urlunparse + +if TYPE_CHECKING: + from envied.core.api.input_bridge import InputBridge + +import click +import m3u8 +import requests +from requests.adapters import HTTPAdapter, Retry +from rich.padding import Padding +from rich.rule import Rule +from rich.text import Text + +from envied.core.cacher import Cacher +from envied.core.config import config +from envied.core.console import console +from envied.core.constants import AnyTrack +from envied.core.credential import Credential +from envied.core.drm import DRM_T +from envied.core.search_result import SearchResult +from envied.core.title_cacher import TitleCacher, get_account_hash, get_region_from_proxy +from envied.core.titles import Title_T, Titles_T, remap_titles +from envied.core.tracks import Chapters, Tracks +from envied.core.tracks.video import Video +from envied.core.utils.ip_info import get_ip_info + + +@dataclass +class TrackRequest: + """Holds what the user requested for video codec and range selection. + + Services read from this instead of ctx.parent.params for vcodec/range. + + Attributes: + codecs: Requested codecs from CLI. Empty list means no filter (accept any). + ranges: Requested ranges from CLI. Defaults to [SDR]. + """ + + codecs: list[Video.Codec] = field(default_factory=list) + ranges: list[Video.Range] = field(default_factory=lambda: [Video.Range.SDR]) + best_available: bool = False + + +def sanitize_proxy_for_log(uri: Optional[str]) -> Optional[str]: + """ + Sanitize a proxy URI for logs by redacting any embedded userinfo (username/password). + + Examples: + - http://user:pass@host:8080 -> http://REDACTED@host:8080 + - socks5h://user@host:1080 -> socks5h://REDACTED@host:1080 + """ + if uri is None: + return None + if not isinstance(uri, str): + return str(uri) + if not uri: + return uri + + try: + parsed = urlparse(uri) + + # Handle schemeless proxies like "user:pass@host:port" + if not parsed.scheme and not parsed.netloc and "@" in uri and "://" not in uri: + # Parse as netloc using a dummy scheme, then strip scheme back out. + dummy = urlparse(f"http://{uri}") + netloc = dummy.netloc + if "@" in netloc: + netloc = f"REDACTED@{netloc.split('@', 1)[1]}" + # urlparse("http://...") sets path to "" for typical netloc-only strings; keep it just in case. + return f"{netloc}{dummy.path}" + + netloc = parsed.netloc + if "@" in netloc: + netloc = f"REDACTED@{netloc.split('@', 1)[1]}" + + return urlunparse(parsed._replace(netloc=netloc)) + except Exception: + if "@" in uri: + return f"REDACTED@{uri.split('@', 1)[1]}" + return uri + + +class Service(metaclass=ABCMeta): + """The Service Base Class.""" + + # Abstract class variables + ALIASES: tuple[str, ...] = () # list of aliases for the service; alternatives to the service tag. + GEOFENCE: tuple[str, ...] = () # list of ip regions required to use the service. empty list == no specific region. + + def __init__(self, ctx: click.Context): + console.print(Padding(Rule(f"[rule.text]Service: {self.__class__.__name__}"), (1, 2))) + + self.config = ctx.obj.config + + self.log = logging.getLogger(self.__class__.__name__) + + self.session = self.get_session() + self.cache = Cacher(self.__class__.__name__) + self.title_cache = TitleCacher(self.__class__.__name__) + self.cache_dir = config.directories.cache / self.__class__.__name__ + + # Store context for cache control flags and credential + self.ctx = ctx + self.credential = None # Will be set in authenticate() + self.current_region = None # Will be set based on proxy/geolocation + self._input_bridge: Optional[InputBridge] = None + + # Set track request from CLI params - services can read/override in their __init__ + vcodec = ctx.parent.params.get("vcodec") if ctx.parent else None + range_ = ctx.parent.params.get("range_") if ctx.parent else None + best_available = ctx.parent.params.get("best_available", False) if ctx.parent else False + self.track_request = TrackRequest( + codecs=list(vcodec) if vcodec else [], + ranges=list(range_) if range_ else [Video.Range.SDR], + best_available=bool(best_available), + ) + + if not ctx.parent or not ctx.parent.params.get("no_proxy"): + if ctx.parent: + proxy = ctx.parent.params["proxy"] + proxy_query = ctx.parent.params.get("proxy_query") + proxy_provider_name = ctx.parent.params.get("proxy_provider") + else: + proxy = None + proxy_query = None + proxy_provider_name = None + + # Check for service-specific proxy mapping + service_name = self.__class__.__name__ + service_config_dict = config.services.get(service_name, {}) + proxy_map = service_config_dict.get("proxy_map", {}) + + if proxy_map and proxy_query: + # Build the full proxy query key (e.g., "nordvpn:ca" or "us") + if proxy_provider_name: + full_proxy_key = f"{proxy_provider_name}:{proxy_query}" + else: + full_proxy_key = proxy_query + + # Check if there's a mapping for this query + mapped_value = proxy_map.get(full_proxy_key) + if mapped_value: + self.log.info( + f"Found service-specific proxy mapping: {full_proxy_key} -> {sanitize_proxy_for_log(mapped_value)}" + ) + # Query the proxy provider with the mapped value + if proxy_provider_name: + # Specific provider requested + proxy_provider = next( + (x for x in ctx.obj.proxy_providers if x.__class__.__name__.lower() == proxy_provider_name), + None, + ) + if proxy_provider: + mapped_proxy_uri = proxy_provider.get_proxy(mapped_value) + if mapped_proxy_uri: + proxy = mapped_proxy_uri + self.log.info( + f"Using mapped proxy from {proxy_provider.__class__.__name__}: {sanitize_proxy_for_log(proxy)}" + ) + else: + self.log.warning( + f"Failed to get proxy for mapped value '{sanitize_proxy_for_log(mapped_value)}', using default" + ) + else: + self.log.warning(f"Proxy provider '{proxy_provider_name}' not found, using default proxy") + else: + # No specific provider, try all providers + for proxy_provider in ctx.obj.proxy_providers: + mapped_proxy_uri = proxy_provider.get_proxy(mapped_value) + if mapped_proxy_uri: + proxy = mapped_proxy_uri + self.log.info( + f"Using mapped proxy from {proxy_provider.__class__.__name__}: {sanitize_proxy_for_log(proxy)}" + ) + break + else: + self.log.warning( + f"No provider could resolve mapped value '{sanitize_proxy_for_log(mapped_value)}', using default" + ) + + if not proxy: + # don't override the explicit proxy set by the user, even if they may be geoblocked + with console.status("Checking if current region is Geoblocked...", spinner="dots"): + if self.GEOFENCE: + # Service has geofence - need fresh IP check to determine if proxy needed + try: + current_region = get_ip_info(self.session)["country"].lower() + if any(x.lower() == current_region for x in self.GEOFENCE): + self.log.info("Service is not Geoblocked in your region") + else: + requested_proxy = self.GEOFENCE[0] # first is likely main region + self.log.info( + f"Service is Geoblocked in your region, getting a Proxy to {requested_proxy}" + ) + for proxy_provider in ctx.obj.proxy_providers: + proxy = proxy_provider.get_proxy(requested_proxy) + if proxy: + self.log.info(f"Got Proxy from {proxy_provider.__class__.__name__}") + break + except Exception as e: + self.log.warning(f"Failed to check geofence: {e}") + current_region = None + else: + self.log.info("Service has no Geofence") + + if proxy: + self.session.proxies.update({"all": proxy}) + # Don't set Proxy-Authorization manually: both rnet (Proxy.all) and + # requests authenticate from the credentials embedded in the proxy URL. + # A manual header here was malformed (no "Basic " scheme) and broke + # plaintext-http forward-proxy requests with HTTP 407. + # Always verify proxy IP - proxies can change exit nodes + try: + proxy_ip_info = get_ip_info(self.session) + self.current_region = proxy_ip_info.get("country", "").lower() if proxy_ip_info else None + except Exception as e: + self.log.warning(f"Failed to verify proxy IP: {e}") + # Fallback to extracting region from proxy config + self.current_region = get_region_from_proxy(proxy) + else: + # No proxy, use cached IP info for title caching (non-critical) + try: + ip_info = get_ip_info(self.session, cached=True) + self.current_region = ip_info.get("country", "").lower() if ip_info else None + except Exception as e: + self.log.debug(f"Failed to get cached IP info: {e}") + self.current_region = None + + def _get_tracks_for_variants( + self, + title: Title_T, + fetch_fn: Callable[..., Tracks], + ) -> Tracks: + """Call fetch_fn for each codec/range combo in track_request, merge results. + + Services that need separate API calls per codec/range combo can use this + helper from their get_tracks() implementation. + + The fetch_fn signature should be: (title, codec, range_) -> Tracks + + For HYBRID range, fetch_fn is called with HDR10 and DV separately and + the DV video tracks are merged into the HDR10 result. + + Args: + title: The title being processed. + fetch_fn: A callable that fetches tracks for a specific codec/range. + """ + all_tracks = Tracks() + first = True + + codecs = self.track_request.codecs or [None] + ranges = self.track_request.ranges or [Video.Range.SDR] + + for range_val in ranges: + if range_val == Video.Range.HYBRID: + # HYBRID: fetch HDR10 first (full tracks), then DV (video only) + for codec_val in codecs: + try: + hdr_tracks = fetch_fn(title, codec=codec_val, range_=Video.Range.HDR10) + except (ValueError, SystemExit) as e: + if self.track_request.best_available: + self.log.warning(f" - HDR10 not available for HYBRID, skipping ({e})") + continue + raise + if first: + all_tracks.add(hdr_tracks, warn_only=True) + if hdr_tracks.manifest_url and not all_tracks.manifest_url: + all_tracks.manifest_url = hdr_tracks.manifest_url + first = False + else: + for video in hdr_tracks.videos: + all_tracks.add(video, warn_only=True) + + try: + dv_tracks = fetch_fn(title, codec=codec_val, range_=Video.Range.DV) + for video in dv_tracks.videos: + all_tracks.add(video, warn_only=True) + except (ValueError, SystemExit): + self.log.info(" - No DolbyVision manifest available for HYBRID") + else: + for codec_val in codecs: + try: + tracks = fetch_fn(title, codec=codec_val, range_=range_val) + except (ValueError, SystemExit) as e: + if self.track_request.best_available: + codec_name = codec_val.name if codec_val else "default" + self.log.warning(f" - {range_val.name}/{codec_name} not available, skipping ({e})") + continue + raise + if first: + all_tracks.add(tracks, warn_only=True) + if tracks.manifest_url and not all_tracks.manifest_url: + all_tracks.manifest_url = tracks.manifest_url + first = False + else: + for video in tracks.videos: + all_tracks.add(video, warn_only=True) + + return all_tracks + + # Optional Abstract functions + # The following functions may be implemented by the Service. + # Otherwise, the base service code (if any) of the function will be executed on call. + # The functions will be executed in shown order. + + @staticmethod + def get_session() -> requests.Session: + """ + Creates a Python-requests Session, adds common headers + from config, cookies, retry handler, and a proxy if available. + :returns: Prepared Python-requests Session + """ + session = requests.Session() + session.headers.update(config.headers) + session.mount( + "https://", + HTTPAdapter( + max_retries=Retry(total=5, backoff_factor=0.2, status_forcelist=[429, 500, 502, 503, 504]), + pool_block=True, + ), + ) + session.mount("http://", session.adapters["https://"]) + return session + + def authenticate(self, cookies: Optional[CookieJar] = None, credential: Optional[Credential] = None) -> None: + """ + Authenticate the Service with Cookies and/or Credentials (Email/Username and Password). + + This is effectively a login() function. Any API calls or object initializations + needing to be made, should be made here. This will be run before any of the + following abstract functions. + + You should avoid storing or using the Credential outside this function. + Make any calls you need for any Cookies, Tokens, or such, then use those. + + The Cookie jar should also not be stored outside this function. However, you may load + the Cookie jar into the service session. + """ + if cookies is not None: + if not isinstance(cookies, CookieJar): + raise TypeError(f"Expected cookies to be a {CookieJar}, not {cookies!r}.") + self.session.cookies.update(cookies) + + # Store credential for cache key generation + self.credential = credential + + def request_input(self, prompt: str) -> str: + """Request interactive input from the user. + + When running locally (CLI), prompts via the shared rich console so the + prompt renders correctly alongside Live progress / log handlers. + When running in serve mode with an :class:`InputBridge` attached, + delegates to the bridge which relays the prompt to the remote client. + """ + if self._input_bridge is not None: + return self._input_bridge.request_input(prompt) + indent = " " * 5 + padded = indent + prompt.replace("\n", "\n" + indent) + return console.input(Text(padded, style="text")) + + def search(self) -> Generator[SearchResult, None, None]: + """ + Search by query for titles from the Service. + + The query must be taken as a CLI argument by the Service class. + Ideally just re-use the title ID argument (i.e. self.title). + + Search results will be displayed in the order yielded. + """ + raise NotImplementedError(f"Search functionality has not been implemented by {self.__class__.__name__}") + + def get_widevine_service_certificate( + self, *, challenge: bytes, title: Title_T, track: AnyTrack + ) -> Union[bytes, str]: + """ + Get the Widevine Service Certificate used for Privacy Mode. + + :param challenge: The service challenge, providing this to a License endpoint should return the + privacy certificate that the service uses. + :param title: The current `Title` from get_titles that is being executed. This is provided in + case it has data needed to be used, e.g. for a HTTP request. + :param track: The current `Track` needing decryption. Provided for same reason as `title`. + :return: The Service Privacy Certificate as Bytes or a Base64 string. Don't Base64 Encode or + Decode the data, return as is to reduce unnecessary computations. + """ + + def get_widevine_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[Union[bytes, str]]: + """ + Get a Widevine License message by sending a License Request (challenge). + + This License message contains the encrypted Content Decryption Keys and will be + read by the Cdm and decrypted. + + This is a very important request to get correct. A bad, unexpected, or missing + value in the request can cause your key to be detected and promptly banned, + revoked, disabled, or downgraded. + + :param challenge: The license challenge from the Widevine CDM. + :param title: The current `Title` from get_titles that is being executed. This is provided in + case it has data needed to be used, e.g. for a HTTP request. + :param track: The current `Track` needing decryption. Provided for same reason as `title`. + :return: The License response as Bytes or a Base64 string. Don't Base64 Encode or + Decode the data, return as is to reduce unnecessary computations. + """ + + def get_playready_license( + self, *, challenge: bytes, title: Title_T, track: AnyTrack + ) -> Optional[Union[bytes, str]]: + """ + Get a PlayReady License message by sending a License Request (challenge). + + This License message contains the encrypted Content Decryption Keys and will be + read by the CDM and decrypted. + + This is a very important request to get correct. A bad, unexpected, or missing + value in the request can cause your key to be detected and promptly banned, + revoked, disabled, or downgraded. + + :param challenge: The license challenge from the PlayReady CDM. + :param title: The current `Title` from get_titles that is being executed. This is provided in + case it has data needed to be used, e.g. for a HTTP request. + :param track: The current `Track` needing decryption. Provided for same reason as `title`. + :return: The License response as Bytes or a Base64 string. Don't Base64 Encode or + Decode the data, return as is to reduce unnecessary computations. + """ + # Delegates license handling to the Widevine license method by default if a service-specific PlayReady implementation is not provided. + return self.get_widevine_license(challenge=challenge, title=title, track=track) + + # Required Abstract functions + # The following functions *must* be implemented by the Service. + # The functions will be executed in shown order. + + @abstractmethod + def get_titles(self) -> Titles_T: + """ + Get Titles for the provided title ID. + + Return a Movies, Series, or Album objects containing Movie, Episode, or Song title objects respectively. + The returned data must be for the given title ID, or a spawn of the title ID. + + At least one object is expected to be returned, or it will presume an invalid Title ID was + provided. + + You can use the `data` dictionary class instance attribute of each Title to store data you may need later on. + This can be useful to store information on each title that will be required like any sub-asset IDs, or such. + """ + + def get_titles_cached(self, title_id: str = None) -> Titles_T: + """ + Cached wrapper around get_titles() to reduce redundant API calls. + + This method checks the cache before calling get_titles() and handles + fallback to cached data when API calls fail. + + Args: + title_id: Optional title ID for cache key generation. + If not provided, will try to extract from service instance. + + Returns: + Titles object (Movies, Series, or Album) + """ + # Try to get title_id from service instance if not provided + if title_id is None: + # Different services store the title ID in different attributes + if hasattr(self, "title"): + title_id = self.title + elif hasattr(self, "title_id"): + title_id = self.title_id + else: + # If we can't determine title_id, just call get_titles directly + self.log.debug("Cannot determine title_id for caching, bypassing cache") + return self.apply_title_map(self.get_titles()) + + # Get cache control flags from context + no_cache = False + reset_cache = False + if self.ctx and self.ctx.parent: + no_cache = self.ctx.parent.params.get("no_cache", False) + reset_cache = self.ctx.parent.params.get("reset_cache", False) + + # Get account hash for cache key + account_hash = get_account_hash(self.credential) + + # Use title cache to get titles with fallback support + titles = self.title_cache.get_cached_titles( + title_id=str(title_id), + fetch_function=self.get_titles, + region=self.current_region, + account_hash=account_hash, + no_cache=no_cache, + reset_cache=reset_cache, + ) + return self.apply_title_map(titles) + + def apply_title_map(self, titles: Titles_T) -> Titles_T: + """ + Rewrite service-provided titles using the per-service ``title_map`` config. + + ``title_map`` lives under ``services.`` in envied.yaml. Applied after the + title cache so config edits take effect without a cache reset, and before any + ``--enrich`` override so enrich wins. See ``remap_titles`` for the match rules. + """ + return remap_titles(titles, (self.config or {}).get("title_map") or {}) + + @abstractmethod + def get_tracks(self, title: Title_T) -> Tracks: + """ + Get Track objects of the Title. + + Return a Tracks object, which itself can contain Video, Audio, Subtitle or even Chapters. + Tracks.videos, Tracks.audio, Tracks.subtitles, and Track.chapters should be a List of Track objects. + + Each Track in the Tracks should represent a Video/Audio Stream/Representation/Adaptation or + a Subtitle file. + + While one Track should only hold information for one stream/downloadable, try to get as many + unique Track objects per stream type so Stream selection by the root code can give you more + options in terms of Resolution, Bitrate, Codecs, Language, e.t.c. + + No decision making or filtering of which Tracks get returned should happen here. It can be + considered an error to filter for e.g. resolution, codec, and such. All filtering based on + arguments will be done by the root code automatically when needed. + + Make sure you correctly mark which Tracks are encrypted or not, and by which DRM System + via its `drm` property. + + If you are able to obtain the Track's KID (Key ID) as a 32 char (16 bit) HEX string, provide + it to the Track's `kid` variable as it will speed up the decryption process later on. It may + or may not be needed, that depends on the service. Generally if you can provide it, without + downloading any of the Track's stream data, then do. + + :param title: The current `Title` from get_titles that is being executed. + :return: Tracks object containing Video, Audio, Subtitles, and Chapters, if available. + """ + + @abstractmethod + def get_chapters(self, title: Title_T) -> Chapters: + """ + Get Chapters for the Title. + + Parameters: + title: The current Title from `get_titles` that is being processed. + + You must return a Chapters object containing 0 or more Chapter objects. + + You do not need to set a Chapter number or sort/order the chapters in any way as + the Chapters class automatically handles all of that for you. If there's no + descriptive name for a Chapter then do not set a name at all. + + You must not set Chapter names to "Chapter {n}" or such. If you (or the user) + wants "Chapter {n}" style Chapter names (or similar) then they can use the config + option `chapter_fallback_name`. For example, `"Chapter {i:02}"` for "Chapter 01". + """ + + # Optional Event methods + + def on_segment_downloaded(self, track: AnyTrack, segment: Path) -> None: + """ + Called when one of a Track's Segments has finished downloading. + + Parameters: + track: The Track object that had a Segment downloaded. + segment: The Path to the Segment that was downloaded. + """ + + def on_track_downloaded(self, track: AnyTrack) -> None: + """ + Called when a Track has finished downloading. + + Parameters: + track: The Track object that was downloaded. + """ + + def on_track_decrypted(self, track: AnyTrack, drm: DRM_T, segment: Optional[m3u8.Segment] = None) -> None: + """ + Called when a Track has finished decrypting. + + Parameters: + track: The Track object that was decrypted. + drm: The DRM object it decrypted with. + segment: The HLS segment information that was decrypted. + """ + + def on_track_repacked(self, track: AnyTrack) -> None: + """ + Called when a Track has finished repacking. + + Parameters: + track: The Track object that was repacked. + """ + + def on_track_multiplex(self, track: AnyTrack) -> None: + """ + Called when a Track is about to be Multiplexed into a Container. + + Note: Right now only MKV containers are multiplexed but in the future + this may also be called when multiplexing to other containers like + MP4 via ffmpeg/mp4box. + + Parameters: + track: The Track object that was repacked. + """ + + +__all__ = ("Service", "TrackRequest") diff --git a/reset/packages/envied/src/envied/core/services.py b/reset/packages/envied/src/envied/core/services.py new file mode 100644 index 0000000..c91e2a4 --- /dev/null +++ b/reset/packages/envied/src/envied/core/services.py @@ -0,0 +1,278 @@ +from __future__ import annotations + +import logging +import re +from pathlib import Path + +import click + +from envied.core.config import config +from envied.core.service import Service +from envied.core.utilities import import_module_by_path + +log = logging.getLogger("services") + +_service_dirs = config.directories.services +if not isinstance(_service_dirs, list): + _service_dirs = [_service_dirs] + +_SERVICES = sorted( + (path for service_dir in _service_dirs for path in service_dir.glob("*/__init__.py")), + key=lambda x: x.parent.stem, +) + + +def load_service(path: Path) -> object: + """Load one Service module, returning its tag-named class. + + Raises a concise, single-line error naming the Service and the real cause so + a broken Service never surfaces as a raw traceback pointing at the loader. + """ + tag = path.parent.stem + try: + module = import_module_by_path(path) + except Exception as e: + raise RuntimeError(f"{tag}: failed to import — {type(e).__name__}: {e} ({path})") from e + try: + return getattr(module, tag) + except AttributeError as e: + raise RuntimeError( + f"{tag}: no class named '{tag}' found in {path} — the class name must match the directory name" + ) from e + + +def load_services(paths: list[Path]) -> tuple[dict[str, object], list[str]]: + """Load every Service, returning the good ones plus a list of load errors. + + Importing this module must never raise: it is imported by several commands, + and a failed import is not cached by Python, so raising here would re-run and + re-report for every command. Instead we collect failures and let the caller + surface them once, cleanly, at the point services are actually used. + """ + modules: dict[str, object] = {} + errors: list[str] = [] + for path in paths: + try: + modules[path.parent.stem] = load_service(path) + except Exception as e: + errors.append(str(e)) + return modules, errors + + +_MODULES, LOAD_ERRORS = load_services(_SERVICES) + +_ALIASES = {tag: getattr(module, "ALIASES", ()) for tag, module in _MODULES.items()} + + +def check_load_errors() -> None: + """Raise a single clean error if any Service failed to load. + + Called when services are actually needed (listing/resolving) so the message + is rendered once by Click, without a traceback and without cascading through + every command that imports this module. + """ + if LOAD_ERRORS: + joined = "\n".join(f" - {err}" for err in LOAD_ERRORS) + raise click.ClickException(f"Failed to load {len(LOAD_ERRORS)} service(s):\n{joined}") + + +class Services(click.MultiCommand): + """Lazy-loaded command group of project services.""" + + _remote_services_cache: list[dict] | None = None + + # Click-specific methods + + def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]: + """Preprocess --slow to support optional range value before Click parses args.""" + processed = [] + i = 0 + while i < len(args): + if args[i] == "--slow": + if i + 1 < len(args) and re.match(r"^\d+-\d+$", args[i + 1]): + processed.append(f"--slow={args[i + 1]}") + i += 2 + else: + processed.append("--slow=60-120") + i += 1 + else: + processed.append(args[i]) + i += 1 + return super().parse_args(ctx, processed) + + def list_commands(self, ctx: click.Context) -> list[str]: + """Returns a list of all available Services as command names for Click. + + In remote mode, fetches the service list from the remote server + so the user sees exactly what's available remotely. + """ + remote = ctx.params.get("remote") or (ctx.parent and ctx.parent.params.get("remote")) + if remote: + remote_services = Services._fetch_remote_services(ctx) + if remote_services is not None: + return [s["tag"] for s in remote_services] + tags = Services.get_tags() + for svc_cfg in config.remote_services.values(): + for remote_tag in svc_cfg.get("services", {}).keys(): + if remote_tag not in tags: + tags.append(remote_tag) + return tags + check_load_errors() + return Services.get_tags() + + def get_command(self, ctx: click.Context, name: str) -> click.Command: + """Load the Service and return the Click CLI method.""" + check_load_errors() + tag = Services.get_tag(name) + + import_file = ctx.params.get("import_file") or (ctx.parent and ctx.parent.params.get("import_file")) + if import_file: + return Services._make_import_command(tag, ctx) + + remote = ctx.params.get("remote") or (ctx.parent and ctx.parent.params.get("remote")) + if remote: + return Services._make_remote_command(tag, ctx) + + try: + service = Services.load(tag) + except KeyError as e: + available_services = self.list_commands(ctx) + if not available_services: + raise click.ClickException( + f"There are no Services added yet, therefore the '{name}' Service could not be found." + ) + raise click.ClickException(f"{e}. Available Services: {', '.join(available_services)}") + + if hasattr(service, "cli"): + return service.cli + + raise click.ClickException(f"Service '{tag}' has no 'cli' method configured.") + + @staticmethod + def _fetch_remote_services(ctx: click.Context) -> list[dict] | None: + """Fetch the service list from the remote server (cached per process).""" + if Services._remote_services_cache is not None: + return Services._remote_services_cache + try: + from envied.core.remote_service import RemoteClient, resolve_server + + server_name = ctx.params.get("server") + server_url, api_key, _ = resolve_server(server_name) + client = RemoteClient(server_url, api_key) + result = client.get("/api/services") + Services._remote_services_cache = result.get("services", []) + return Services._remote_services_cache + except Exception: + return None + + @staticmethod + def _make_remote_command(tag: str, ctx: click.Context) -> click.Command: + """Create a Click command for a remote service with server-provided options.""" + svc_info = Services._fetch_remote_service_info(tag, ctx) + short_help = svc_info.get("url") if svc_info else None + cli_params = svc_info.get("cli_params") if svc_info else None + + @click.command(name=tag, short_help=short_help) + @click.argument("title", type=str) + @click.pass_context + def remote_cli(ctx: click.Context, title: str, **kwargs: object) -> object: + from envied.core.remote_service import RemoteService, resolve_server + + server_name = ctx.parent.params.get("server") if ctx.parent else None + server_url, api_key, services_config = resolve_server(server_name) + service_params = {k: v for k, v in kwargs.items() if v is not None and v is not False} + return RemoteService(ctx, tag, title, server_url, api_key, services_config, service_params=service_params) + + if cli_params: + for param in cli_params: + if param.get("kind") == "option": + opts = param.get("opts", [f"--{param['name']}"]) + kwargs: dict = {} + if param.get("is_flag"): + kwargs["is_flag"] = True + kwargs["default"] = param.get("default", False) + else: + kwargs["default"] = param.get("default") + kwargs["type"] = str + if param.get("help"): + kwargs["help"] = param["help"] + remote_cli = click.option(*opts, **kwargs)(remote_cli) + + return remote_cli + + @staticmethod + def _make_import_command(tag: str, ctx: click.Context) -> click.Command: + """Create a synthetic command that yields an ImportService from an export JSON. + + Mirrors how remote services are wired so dl.py's result() runs unchanged. + """ + + @click.command(name=tag, short_help="Reconstruct a download from an export JSON.") + @click.argument("title", type=str, required=False, default="") + @click.pass_context + def import_cli(ctx: click.Context, title: str, **kwargs: object) -> object: + from envied.core.import_service import ImportService + + import_file = ctx.params.get("import_file") or (ctx.parent and ctx.parent.params.get("import_file")) + return ImportService(ctx, tag, title, import_file) + + return import_cli + + @staticmethod + def _fetch_remote_service_info(tag: str, ctx: click.Context) -> dict | None: + """Fetch service info for a specific service from the remote server.""" + try: + services = Services._fetch_remote_services(ctx) + if services: + for svc in services: + if svc.get("tag") == tag: + return svc + except Exception: + pass + return None + + # Methods intended to be used anywhere + + @staticmethod + def get_tags() -> list[str]: + """Returns a list of service tags from all available Services.""" + return [x.parent.stem for x in _SERVICES] + + @staticmethod + def get_path(name: str) -> Path: + """Get the directory path of a command.""" + tag = Services.get_tag(name) + + for service in _SERVICES: + if service.parent.stem == tag: + return service.parent + raise KeyError(f"There is no Service added by the Tag '{name}'") + + @staticmethod + def get_tag(value: str) -> str: + """ + Get the Service Tag (e.g. DSNP, not DisneyPlus/Disney+, etc.) by an Alias. + Input value can be of any case-sensitivity. + Original input value is returned if it did not match a service tag. + """ + original_value = value + value = value.lower() + + for path in _SERVICES: + tag = path.parent.stem + if value in (tag.lower(), *_ALIASES.get(tag, [])): + return tag + + return original_value + + @staticmethod + def load(tag: str) -> Service: + """Load a Service module by Service tag.""" + module = _MODULES.get(tag) + if module: + return module + + raise KeyError(f"There is no Service added by the Tag '{tag}'") + + +__all__ = ("Services",) diff --git a/reset/packages/envied/src/envied/core/session.py b/reset/packages/envied/src/envied/core/session.py new file mode 100644 index 0000000..8b45ebc --- /dev/null +++ b/reset/packages/envied/src/envied/core/session.py @@ -0,0 +1,732 @@ +"""Session utilities for creating HTTP sessions with TLS fingerprinting via rnet (Rust/BoringSSL).""" + +from __future__ import annotations + +import http +import logging +import random +import time +from collections.abc import Iterator, MutableMapping +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +from http.cookiejar import CookieJar +from typing import Any, Optional +from urllib.parse import urlencode, urlparse, urlunparse + +import rnet +from requests import HTTPError, Request +from requests.structures import CaseInsensitiveDict + +from envied.core.config import config + +# --------------------------------------------------------------------------- +# Impersonate preset mapping — rnet uses named presets (no custom JA3/Akamai) +# --------------------------------------------------------------------------- + +DEFAULT_IMPERSONATE = rnet.Impersonate.Chrome131 + + +def _resolve_impersonate(browser: str) -> rnet.Impersonate: + """Resolve a browser string to an rnet.Impersonate preset. + + Accepts exact rnet preset names (e.g. "Chrome131", "OkHttp4_12", "Edge101"). + See https://github.com/0x676e67/rnet for the full list of available presets. + """ + preset = getattr(rnet.Impersonate, browser, None) + if preset is not None: + return preset + raise ValueError( + f"Unknown impersonate preset: {browser!r}. " + f"Use exact rnet preset names like 'Chrome131', 'OkHttp4_12', 'Edge101'. " + f"See rnet.Impersonate for all available presets." + ) + + +# Map string method names to rnet.Method enum +_METHOD_MAP: dict[str, rnet.Method] = { + "GET": rnet.Method.GET, + "POST": rnet.Method.POST, + "PUT": rnet.Method.PUT, + "DELETE": rnet.Method.DELETE, + "HEAD": rnet.Method.HEAD, + "OPTIONS": rnet.Method.OPTIONS, + "PATCH": rnet.Method.PATCH, + "TRACE": rnet.Method.TRACE, +} + + +# --------------------------------------------------------------------------- +# Response headers adapter — bytes → str +# --------------------------------------------------------------------------- + + +class RnetResponseHeaders(MutableMapping): + """Read-only str-based view over rnet's bytes-based HeaderMap.""" + + def __init__(self, header_map: Any) -> None: + self._map = header_map + + def _decode(self, val: Any) -> str: + return val.decode("utf-8", errors="replace") if isinstance(val, (bytes, bytearray)) else str(val) + + def __getitem__(self, key: str) -> str: + val = self._map[key] + return self._decode(val) + + def __setitem__(self, key: str, value: str) -> None: + raise TypeError("Response headers are read-only") + + def __delitem__(self, key: str) -> None: + raise TypeError("Response headers are read-only") + + def __contains__(self, key: object) -> bool: + if not isinstance(key, str): + return False + return self._map.contains_key(key) + + def __iter__(self) -> Iterator[str]: + seen: set[str] = set() + for k, _ in self._map.items(): + dk = self._decode(k) + if dk not in seen: + seen.add(dk) + yield dk + + def __len__(self) -> int: + return self._map.keys_len() + + def get(self, key: str, default: Optional[str] = None) -> Optional[str]: + val = self._map.get(key) + if val is None: + return default + return self._decode(val) + + def items(self) -> list[tuple[str, str]]: + return [(self._decode(k), self._decode(v)) for k, v in self._map.items()] + + +# --------------------------------------------------------------------------- +# Response wrapper — requests-compatible interface +# --------------------------------------------------------------------------- + + +class RnetResponse: + """Wraps rnet.BlockingResponse with a requests-compatible API.""" + + def __init__(self, resp: Any) -> None: + self._resp = resp + self._headers: Optional[RnetResponseHeaders] = None + self._content: Optional[bytes] = None + self._text: Optional[str] = None + self._streamed = False + + @property + def status_code(self) -> int: + return int(str(self._resp.status_code)) + + @property + def ok(self) -> bool: + return self._resp.ok + + @property + def headers(self) -> RnetResponseHeaders: + if self._headers is None: + self._headers = RnetResponseHeaders(self._resp.headers) + return self._headers + + @property + def url(self) -> str: + return str(self._resp.url) + + @property + def content_length(self) -> Optional[int]: + return self._resp.content_length + + @property + def content(self) -> bytes: + if self._content is None: + self._content = self._resp.bytes() + return self._content + + @property + def text(self) -> str: + if self._text is None: + encoding = self._resp.encoding or "utf-8" + self._text = self.content.decode(encoding, errors="replace") + return self._text + + @property + def reason(self) -> str: + try: + return http.HTTPStatus(self.status_code).phrase + except ValueError: + return "Unknown" + + @property + def cookies(self) -> Any: + return self._resp.cookies + + def json(self, **kwargs: Any) -> Any: + import json as _json + + return _json.loads(self.content) + + def raise_for_status(self) -> None: + if not self.ok: + raise HTTPError( + f"{self.status_code} {self.reason}: {self.url}", + response=self, + ) + + def iter_content(self, chunk_size: Optional[int] = None) -> Iterator[bytes]: + """Re-chunk rnet's variable-size stream into fixed-size pieces.""" + self._streamed = True + if chunk_size is None or chunk_size <= 0: + yield from self._resp.stream() + return + + buf = bytearray() + for chunk in self._resp.stream(): + buf.extend(chunk) + while len(buf) >= chunk_size: + yield bytes(buf[:chunk_size]) + buf = buf[chunk_size:] + if buf: + yield bytes(buf) + + def stream(self) -> Iterator[bytes]: + """Direct pass-through of rnet's native stream iterator.""" + self._streamed = True + yield from self._resp.stream() + + def close(self) -> None: + try: + self._resp.close() + except Exception: + pass + + +# --------------------------------------------------------------------------- +# Session headers adapter — persists via client.update() +# --------------------------------------------------------------------------- + + +class RnetSessionHeaders(CaseInsensitiveDict): + """Dict-like headers that persist to the rnet client via update().""" + + def __init__(self, client: Any) -> None: + self._client = client + super().__init__() + + def _sync(self) -> None: + """Push current headers to the rnet client.""" + if self._client is not None and hasattr(self, "_store") and self._store: + self._client.update(headers={k: v for k, v in self.items()}) + + def __setitem__(self, key: str, value: str) -> None: + super().__setitem__(key, value) + self._sync() + + def update(self, __m: Any = None, **kwargs: Any) -> None: + if __m: + if hasattr(__m, "items"): + for k, v in __m.items(): + super().__setitem__(k, v) + else: + for k, v in __m: + super().__setitem__(k, v) + for k, v in kwargs.items(): + super().__setitem__(k, v) + self._sync() + + def pop(self, key: str, *args: Any) -> Any: + result = super().pop(key, *args) + # rnet doesn't support removing individual headers, but we track locally + # and always send the full set on next update + return result + + def __delitem__(self, key: str) -> None: + super().__delitem__(key) + + +# --------------------------------------------------------------------------- +# Session cookies adapter +# --------------------------------------------------------------------------- + + +class RnetCookieAdapter(MutableMapping): + """Cookie adapter that bridges requests-style cookie access to rnet.""" + + def __init__(self, client: Any) -> None: + self._client = client + self._cookies: dict[str, dict[str, str]] = {} + self._flat: dict[str, str] = {} + self._original_cookies: list[Any] = [] + + def _set_cookie_on_client(self, url: str, name: str, value: str) -> None: + """Set a cookie on the rnet client, or buffer locally if the client is not yet created.""" + if self._client is not None: + try: + self._client.set_cookie(url, rnet.Cookie(name, value)) + except Exception: + pass + + def _flush_to_client(self) -> None: + """Push all buffered cookies to the rnet client once it is created.""" + if self._client is None: + return + for domain, cookies in self._cookies.items(): + url = f"https://{domain.lstrip('.')}" if domain else "https://localhost" + for name, value in cookies.items(): + try: + self._client.set_cookie(url, rnet.Cookie(name, value)) + except Exception: + pass + + @property + def jar(self) -> CookieJar: + """Return a CookieJar with original Cookie objects (requests compat). + + Used by ``save_cookies`` in dl.py to persist cookies back to disk. + """ + jar = CookieJar() + for cookie in self._original_cookies: + jar.set_cookie(cookie) + return jar + + def update(self, other: Any = None, **kwargs: Any) -> None: + if other is None: + other = {} + if isinstance(other, CookieJar): + for cookie in other: + domain = cookie.domain or "" + name = cookie.name + value = cookie.value or "" + self._flat[name] = value + self._cookies.setdefault(domain, {})[name] = value + self._original_cookies.append(cookie) + url = f"https://{domain.lstrip('.')}" if domain else "https://localhost" + self._set_cookie_on_client(url, name, value) + elif isinstance(other, dict): + for name, value in other.items(): + self._flat[name] = value + self._set_cookie_on_client("https://localhost", name, str(value)) + self._flat.update(other) + elif hasattr(other, "items"): + for name, value in other.items(): + self._flat[name] = str(value) + self._set_cookie_on_client("https://localhost", name, str(value)) + + for name, value in kwargs.items(): + self._flat[name] = value + self._set_cookie_on_client("https://localhost", name, value) + + def get( + self, name: str, default: Optional[str] = None, domain: Optional[str] = None, path: Optional[str] = None + ) -> Optional[str]: + if domain and domain in self._cookies: + return self._cookies[domain].get(name, default) + return self._flat.get(name, default) + + def set(self, name: str, value: str, domain: str = "localhost") -> None: + self._flat[name] = value + self._cookies.setdefault(domain, {})[name] = value + url = f"https://{domain.lstrip('.')}" + self._set_cookie_on_client(url, name, value) + + def __getitem__(self, name: str) -> str: + return self._flat[name] + + def __setitem__(self, name: str, value: str) -> None: + self.set(name, value) + + def __delitem__(self, name: str) -> None: + self._flat.pop(name, None) + for domain_cookies in self._cookies.values(): + domain_cookies.pop(name, None) + + def __contains__(self, name: object) -> bool: + return name in self._flat + + def __iter__(self) -> Iterator: + return iter(self._flat) + + def __len__(self) -> int: + return len(self._flat) + + def __bool__(self) -> bool: + return bool(self._flat) + + def get_dict(self, domain: Optional[str] = None, path: Optional[str] = None) -> dict[str, str]: + """Return cookies as a plain dict (requests RequestsCookieJar compat). + + If *domain* is given, only cookies for that domain are returned. + *path* is accepted for API compatibility but ignored (flat storage). + """ + if domain is not None: + return dict(self._cookies.get(domain, {})) + return dict(self._flat) + + def clear(self, domain: Optional[str] = None, path: Optional[str] = None, name: Optional[str] = None) -> None: + """Remove cookies (requests RequestsCookieJar compat). + + - ``clear()`` removes all cookies. + - ``clear(domain=..., path=..., name=...)`` removes a specific cookie. + """ + if name is not None: + self._flat.pop(name, None) + if domain is not None and domain in self._cookies: + self._cookies[domain].pop(name, None) + else: + for domain_cookies in self._cookies.values(): + domain_cookies.pop(name, None) + elif domain is not None: + removed = self._cookies.pop(domain, {}) + for k in removed: + # Only remove from flat if no other domain has same key + still_exists = any(k in dc for dc in self._cookies.values()) + if not still_exists: + self._flat.pop(k, None) + else: + self._flat.clear() + self._cookies.clear() + + def items(self) -> list[tuple[str, str]]: + return list(self._flat.items()) + + def keys(self) -> list[str]: + return list(self._flat.keys()) + + def values(self) -> list[str]: + return list(self._flat.values()) + + +# --------------------------------------------------------------------------- +# Session proxy adapter +# --------------------------------------------------------------------------- + + +class RnetProxyDict(dict): + """Dict-like proxy config that syncs to the rnet client. + + Accepts ``{"all": url}``, ``{"https": url}``, or ``{"http": url}`` + and applies via rnet's native ``proxies`` parameter (``List[rnet.Proxy]``). + Supports both lazy (pre-client) and live (post-client) proxy updates. + """ + + def __init__(self, session: "RnetSession") -> None: + super().__init__() + self._session = session + + def _sync(self) -> None: + proxy = self.get("all") or self.get("https") or self.get("http") + proxies = [rnet.Proxy.all(proxy)] if proxy else [] + self._session._client_kwargs["proxies"] = proxies or None + if self._session._client is not None: + self._session._client.update(proxies=proxies or None) + + def update(self, __m: Any = None, **kwargs: Any) -> None: + super().update(__m or {}, **kwargs) + self._sync() + + def __setitem__(self, key: str, value: str) -> None: + super().__setitem__(key, value) + self._sync() + + +# --------------------------------------------------------------------------- +# Exceptions +# --------------------------------------------------------------------------- + + +class MaxRetriesError(Exception): + def __init__(self, message: str, cause: Optional[Exception] = None) -> None: + super().__init__(message) + self.__cause__ = cause + + +# --------------------------------------------------------------------------- +# RnetSession — main session class +# --------------------------------------------------------------------------- + + +class RnetSession: + """TLS-fingerprinted HTTP session powered by rnet (Rust/BoringSSL). + + Drop-in replacement for CurlSession with requests-compatible API. + Supports browser impersonation (Chrome, Firefox, Edge, Safari, OkHttp), + retry with exponential backoff, cookie persistence, and proxy support. + + The client is created lazily on the first request so that headers, + cookies, and proxies can be configured freely before any connection + is established. + """ + + def __init__( + self, + max_retries: int = 5, + backoff_factor: float = 0.2, + max_backoff: float = 60.0, + status_forcelist: Optional[list[int]] = None, + allowed_methods: Optional[set[str]] = None, + catch_exceptions: Optional[tuple[type[Exception], ...]] = None, + **session_kwargs: Any, + ) -> None: + self.max_retries = max_retries + self.backoff_factor = backoff_factor + self.max_backoff = max_backoff + self.status_forcelist = status_forcelist or [429, 500, 502, 503, 504] + self.allowed_methods = allowed_methods or {"GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE"} + self.catch_exceptions = catch_exceptions or ( + rnet.ConnectionError, + rnet.TimeoutError, + rnet.RequestError, + ) + self.log = logging.getLogger(self.__class__.__name__) + + client_kwargs: dict[str, Any] = {} + for key in ("impersonate", "timeout", "proxies", "verify", "redirect"): + if key in session_kwargs: + client_kwargs[key] = session_kwargs.pop(key) + if "proxy" in session_kwargs: + proxy_url = session_kwargs.pop("proxy") + if proxy_url: + client_kwargs["proxies"] = [rnet.Proxy.all(proxy_url)] + + client_kwargs["cookie_store"] = True + + self.verify: bool = client_kwargs.pop("verify", True) + if not self.verify: + client_kwargs["danger_accept_invalid_certs"] = True + + self._client_kwargs = dict(client_kwargs) + self._client: Optional[rnet.BlockingClient] = None + + self.headers = RnetSessionHeaders(None) + self.cookies = RnetCookieAdapter(None) + self.proxies = RnetProxyDict(self) + + if "headers" in session_kwargs: + self.headers.update(session_kwargs.pop("headers")) + if "cookies" in session_kwargs: + self.cookies.update(session_kwargs.pop("cookies")) + if "proxies" in session_kwargs: + self.proxies.update(session_kwargs.pop("proxies")) + + def _ensure_client(self) -> rnet.BlockingClient: + """Lazily create the rnet client on first use, flushing any buffered state.""" + if self._client is None: + self._client = rnet.BlockingClient(**self._client_kwargs) + self.headers._client = self._client + self.headers._sync() + self.cookies._client = self._client + self.cookies._flush_to_client() + return self._client + + def _build_url(self, url: str, params: Optional[Any] = None) -> str: + """Encode params into the URL (rnet ignores the params kwarg). + + Accepts the same shapes as requests: a mapping, a sequence of pairs, or a + pre-built query string/bytes. A string is appended verbatim (already encoded); + urlencode() would raise TypeError on it. + """ + if not params: + return url + if isinstance(params, bytes): + extra = params.decode("utf-8") + elif isinstance(params, str): + extra = params + else: + extra = urlencode(params, doseq=True) + parsed = urlparse(url) + separator = "&" if parsed.query else "" + query = parsed.query + separator + extra if parsed.query else extra + return urlunparse(parsed._replace(query=query)) + + def get_sleep_time(self, response: Optional[RnetResponse], attempt: int) -> Optional[float]: + if response: + retry_after = response.headers.get("Retry-After") + if retry_after: + try: + return float(retry_after) + except ValueError: + if retry_date := parsedate_to_datetime(retry_after): + return (retry_date - datetime.now(timezone.utc)).total_seconds() + + if attempt == 0: + return 0.0 + + backoff_value = self.backoff_factor * (2 ** (attempt - 1)) + jitter = backoff_value * 0.1 + sleep_time = backoff_value + random.uniform(-jitter, jitter) + return min(sleep_time, self.max_backoff) + + def request(self, method: str, url: str, **kwargs: Any) -> RnetResponse: + client = self._ensure_client() + method_upper = method.upper() if isinstance(method, str) else str(method).upper() + + # Build URL with params + url = self._build_url(url, kwargs.pop("params", None)) + + # Default allow_redirects=True + kwargs.setdefault("allow_redirects", True) + + # Pass verify setting + if not self.verify: + kwargs.setdefault("verify", False) + + # Remove kwargs rnet doesn't understand + kwargs.pop("stream", None) # rnet responses are always lazy + + # Translate requests-compatible 'data' kwarg to rnet equivalents + data = kwargs.pop("data", None) + if data is not None: + if isinstance(data, dict): + kwargs["form"] = list(data.items()) + elif isinstance(data, (str, bytes)): + kwargs["body"] = data + else: + kwargs["body"] = data + + # Resolve method enum + rnet_method = _METHOD_MAP.get(method_upper) + if rnet_method is None: + raise ValueError(f"Unsupported HTTP method: {method}") + + # Convert headers to standard dict once to resolve PyO3 CaseInsensitiveDict rejection. + if kwargs.get("headers") is not None: + kwargs["headers"] = dict(kwargs["headers"]) + + # Skip retry for non-allowed methods + if method_upper not in self.allowed_methods: + raw_resp = client.request(rnet_method, url, **kwargs) + return RnetResponse(raw_resp) + + last_exception: Optional[Exception] = None + response: Optional[RnetResponse] = None + + for attempt in range(self.max_retries + 1): + try: + raw_resp = client.request(rnet_method, url, **kwargs) + response = RnetResponse(raw_resp) + if response.status_code not in self.status_forcelist: + return response + last_exception = HTTPError(f"Received status code: {response.status_code}") + self.log.warning( + f"{response.status_code} {response.reason}({urlparse(url).path}). Retrying... " + f"({attempt + 1}/{self.max_retries})" + ) + + except self.catch_exceptions as e: + last_exception = e + response = None + self.log.warning( + f"{e.__class__.__name__}({urlparse(url).path}). Retrying... ({attempt + 1}/{self.max_retries})" + ) + + if attempt < self.max_retries: + if sleep_duration := self.get_sleep_time(response, attempt + 1): + if sleep_duration > 0: + time.sleep(sleep_duration) + else: + break + + raise MaxRetriesError(f"Max retries exceeded for {method} {url}", cause=last_exception) + + def get(self, url: str, **kwargs: Any) -> RnetResponse: + return self.request("GET", url, **kwargs) + + def post(self, url: str, **kwargs: Any) -> RnetResponse: + return self.request("POST", url, **kwargs) + + def put(self, url: str, **kwargs: Any) -> RnetResponse: + return self.request("PUT", url, **kwargs) + + def delete(self, url: str, **kwargs: Any) -> RnetResponse: + return self.request("DELETE", url, **kwargs) + + def head(self, url: str, **kwargs: Any) -> RnetResponse: + return self.request("HEAD", url, **kwargs) + + def options(self, url: str, **kwargs: Any) -> RnetResponse: + return self.request("OPTIONS", url, **kwargs) + + def patch(self, url: str, **kwargs: Any) -> RnetResponse: + return self.request("PATCH", url, **kwargs) + + def prepare_request(self, req: Request) -> Request: + """Compatibility shim for services using prepared requests.""" + # Merge session headers into request headers + if req.headers: + merged = dict(self.headers) + merged.update(req.headers) + req.headers = merged + else: + req.headers = dict(self.headers) + return req + + def send(self, req: Request, **kwargs: Any) -> RnetResponse: + """Compatibility shim for services using prepared requests.""" + method = req.method or "GET" + url = req.url or "" + + send_kwargs: dict[str, Any] = {} + if req.headers: + send_kwargs["headers"] = dict(req.headers) + if req.body: + send_kwargs["data"] = req.body + if req.json: + send_kwargs["json"] = req.json + + send_kwargs.update(kwargs) + return self.request(method, url, **send_kwargs) + + def mount(self, prefix: str, adapter: Any) -> None: + """No-op — rnet handles TLS and connection pooling natively.""" + pass + + def close(self) -> None: + """No-op — rnet manages its own resources.""" + pass + + +# --------------------------------------------------------------------------- +# session() factory +# --------------------------------------------------------------------------- + + +def session( + browser: Optional[str] = None, + **kwargs: Any, +) -> RnetSession: + """ + Create an rnet session with TLS fingerprinting (browser/app impersonation). + + Args: + browser: Exact rnet.Impersonate preset name. Examples: + "Chrome131", "OkHttp4_12", "Edge101", "Firefox135", + "Safari18", "OkHttp5", "Opera118" + Uses the configured default from config if not specified. + See rnet.Impersonate for all available presets. + **kwargs: Additional arguments passed to RnetSession constructor. + + Returns: + RnetSession configured with browser impersonation and retry behavior. + + Examples: + session() # Default browser from config + session("OkHttp4_12") # OkHttp 4.12 fingerprint + session("Chrome131") # Chrome 131 + session("Edge101", max_retries=3) # Edge 101 with custom retry + """ + if browser is None: + browser = config.curl_impersonate.get("browser", "Chrome131") + + impersonate = _resolve_impersonate(browser) + + session_kwargs: dict[str, Any] = {"impersonate": impersonate} + session_kwargs.update(kwargs) + + session_obj = RnetSession(**session_kwargs) + session_obj.headers.update(config.headers) + return session_obj diff --git a/reset/packages/envied/src/envied/core/title_cacher.py b/reset/packages/envied/src/envied/core/title_cacher.py new file mode 100644 index 0000000..919ac30 --- /dev/null +++ b/reset/packages/envied/src/envied/core/title_cacher.py @@ -0,0 +1,344 @@ +from __future__ import annotations + +import hashlib +import logging +from datetime import datetime, timedelta +from typing import Optional + +from envied.core.cacher import Cacher +from envied.core.config import config +from envied.core.titles import Titles_T + + +class TitleCacher: + """ + Handles caching of Title objects to reduce redundant API calls. + + This wrapper provides: + - Region-aware caching to handle geo-restricted content + - Automatic fallback to cached data when API calls fail + - Cache lifetime extension during failures + - Cache hit/miss statistics for debugging + """ + + def __init__(self, service_name: str): + self.service_name = service_name + self.log = logging.getLogger(f"{service_name}.TitleCache") + self.cacher = Cacher(service_name) + self.stats = {"hits": 0, "misses": 0, "fallbacks": 0} + self.no_cache = False + + def _generate_cache_key( + self, title_id: str, region: Optional[str] = None, account_hash: Optional[str] = None + ) -> str: + """ + Generate a unique cache key for title data. + + Args: + title_id: The title identifier + region: The region/proxy identifier + account_hash: Hash of account credentials (if applicable) + + Returns: + A unique cache key string + """ + # Hash the title_id to handle complex IDs (URLs, dots, special chars) + # This ensures consistent length and filesystem-safe keys + title_hash = hashlib.sha256(title_id.encode()).hexdigest()[:16] + + # Start with base key using hash + key_parts = ["titles", title_hash] + + # Add region if available + if region: + key_parts.append(region.lower()) + + # Add account hash if available + if account_hash: + key_parts.append(account_hash[:8]) # Use first 8 chars of hash + + # Join with underscores + cache_key = "_".join(key_parts) + + return cache_key + + def get_cached_titles( + self, + title_id: str, + fetch_function, + region: Optional[str] = None, + account_hash: Optional[str] = None, + no_cache: bool = False, + reset_cache: bool = False, + ) -> Optional[Titles_T]: + """ + Get titles from cache or fetch from API with fallback support. + + Args: + title_id: The title identifier + fetch_function: Function to call to fetch fresh titles + region: The region/proxy identifier + account_hash: Hash of account credentials + no_cache: Bypass cache completely + reset_cache: Clear cache before fetching + + Returns: + Titles object (Movies, Series, or Album) + """ + # If caching is globally disabled or no_cache flag is set + if not config.title_cache_enabled or no_cache: + self.no_cache = True + self.log.debug("Cache bypassed, fetching fresh titles") + return fetch_function() + + # Generate cache key + cache_key = self._generate_cache_key(title_id, region, account_hash) + + # If reset_cache flag is set, clear the cache entry + if reset_cache: + self.log.info(f"Clearing cache for {cache_key}") + cache_path = (config.directories.cache / self.service_name / cache_key).with_suffix(".json") + if cache_path.exists(): + cache_path.unlink() + + # Try to get from cache + cache = self.cacher.get(cache_key, version=1) + + # Check if we have valid cached data + if cache and not cache.expired: + self.stats["hits"] += 1 + self.log.debug(f"Cache hit for {title_id} (hits: {self.stats['hits']}, misses: {self.stats['misses']})") + return cache.data + + # Cache miss or expired, try to fetch fresh data + self.stats["misses"] += 1 + self.log.debug(f"Cache miss for {title_id} fetching fresh data") + + try: + # Attempt to fetch fresh titles + titles = fetch_function() + + if titles: + # Successfully fetched, update cache + self.log.debug(f"Successfully fetched titles for {title_id}, updating cache") + cache = self.cacher.get(cache_key, version=1) + cache.set(titles, expiration=datetime.now() + timedelta(seconds=config.title_cache_time)) + + return titles + + except Exception as e: + # API call failed, check if we have fallback cached data + if cache and cache.data: + # We have expired cached data, use it as fallback + current_time = datetime.now() + max_retention_time = cache.expiration + timedelta( + seconds=config.title_cache_max_retention - config.title_cache_time + ) + + if current_time < max_retention_time: + self.stats["fallbacks"] += 1 + self.log.warning( + f"API call failed for {title_id}, using cached data as fallback " + f"(fallbacks: {self.stats['fallbacks']})" + ) + self.log.debug(f"Error was: {e}") + + # Extend cache lifetime + extended_expiration = current_time + timedelta(minutes=5) + if extended_expiration < max_retention_time: + cache.expiration = extended_expiration + cache.set(cache.data, expiration=extended_expiration) + + return cache.data + else: + self.log.error(f"API call failed and cached data for {title_id} exceeded maximum retention time") + + # Re-raise the exception if no fallback available + raise + + def clear_all_title_cache(self): + """Clear all title caches for this service.""" + cache_dir = config.directories.cache / self.service_name + if cache_dir.exists(): + for cache_file in cache_dir.glob("titles_*.json"): + cache_file.unlink() + self.log.info(f"Cleared cache file: {cache_file.name}") + + def get_cache_stats(self) -> dict: + """Get cache statistics.""" + total = sum(self.stats.values()) + if total > 0: + hit_rate = (self.stats["hits"] / total) * 100 + else: + hit_rate = 0 + + return { + "hits": self.stats["hits"], + "misses": self.stats["misses"], + "fallbacks": self.stats["fallbacks"], + "hit_rate": f"{hit_rate:.1f}%", + } + + # -- Generic provider cache methods -- + + def get_cached_provider( + self, + provider_name: str, + title_id: str, + kind: Optional[str] = None, + region: Optional[str] = None, + account_hash: Optional[str] = None, + ) -> Optional[dict]: + """Get cached metadata for any provider.""" + if not config.title_cache_enabled or self.no_cache: + return None + + cache_key = self._generate_cache_key(title_id, region, account_hash) + cache = self.cacher.get(cache_key, version=1) + + if not cache or not cache.data: + return None + + provider_data = getattr(cache.data, f"{provider_name}_data", None) + if not provider_data: + return None + + expiration = provider_data.get("expires_at") + if not expiration or datetime.now() >= expiration: + self.log.debug(f"{provider_name} cache expired for {title_id}") + return None + + if kind and provider_data.get("kind") != kind: + self.log.debug( + f"{provider_name} cache kind mismatch for {title_id}: " + f"cached {provider_data.get('kind')}, requested {kind}" + ) + return None + + self.log.debug(f"{provider_name} cache hit for {title_id}") + + # Return the inner data (provider-specific format) + response = provider_data.get("response") + if response is not None: + return response + + # For TMDB-style caches that store detail + external_ids at top level + result: dict = {} + if "detail" in provider_data: + result["detail"] = provider_data["detail"] + if "external_ids" in provider_data: + result["external_ids"] = provider_data["external_ids"] + if "fetched_at" in provider_data: + result["fetched_at"] = provider_data["fetched_at"] + return result if result else provider_data + + def cache_provider( + self, + provider_name: str, + title_id: str, + data: dict, + kind: Optional[str] = None, + region: Optional[str] = None, + account_hash: Optional[str] = None, + ttl_days: int = 7, + ) -> None: + """Cache metadata from any provider.""" + if not config.title_cache_enabled or self.no_cache: + return + + cache_key = self._generate_cache_key(title_id, region, account_hash) + cache = self.cacher.get(cache_key, version=1) + + if not cache or not cache.data: + self.log.debug(f"Cannot cache {provider_name} data: no title cache exists for {title_id}") + return + + now = datetime.now() + + # Build cache entry in a format compatible with legacy methods + if provider_name == "tmdb" and "detail" in data: + # TMDB stores detail + external_ids at top level + cache_entry = { + **data, + "kind": kind, + "fetched_at": now, + "expires_at": now + timedelta(days=ttl_days), + } + elif provider_name == "simkl": + # SIMKL wraps in a "response" key + cache_entry = { + "response": data, + "fetched_at": now, + "expires_at": now + timedelta(days=ttl_days), + } + else: + # Generic format: store data directly with metadata + cache_entry = { + "response": data, + "kind": kind, + "fetched_at": now, + "expires_at": now + timedelta(days=ttl_days), + } + + setattr(cache.data, f"{provider_name}_data", cache_entry) + cache.set(cache.data, expiration=cache.expiration) + self.log.debug(f"Cached {provider_name} data for {title_id}") + + +def get_region_from_proxy(proxy_url: Optional[str]) -> Optional[str]: + """ + Extract region identifier from proxy URL. + + Args: + proxy_url: The proxy URL string + + Returns: + Region identifier or None + """ + if not proxy_url: + return None + + # Try to extract region from common proxy patterns + # e.g., "us123.nordvpn.com", "gb-proxy.example.com" + import re + + # Pattern for NordVPN style + nord_match = re.search(r"([a-z]{2})\d+\.nordvpn", proxy_url.lower()) + if nord_match: + return nord_match.group(1) + + # Pattern for country code at start + cc_match = re.search(r"([a-z]{2})[-_]", proxy_url.lower()) + if cc_match: + return cc_match.group(1) + + # Pattern for country code subdomain + subdomain_match = re.search(r"://([a-z]{2})\.", proxy_url.lower()) + if subdomain_match: + return subdomain_match.group(1) + + return None + + +def get_account_hash(credential) -> Optional[str]: + """ + Generate a hash for account identification. + + Args: + credential: Credential object + + Returns: + SHA1 hash of the credential or None + """ + if not credential: + return None + + # Use existing sha1 property if available + if hasattr(credential, "sha1"): + return credential.sha1 + + # Otherwise generate hash from username + if hasattr(credential, "username"): + return hashlib.sha1(credential.username.encode()).hexdigest() + + return None diff --git a/reset/packages/envied/src/envied/core/titles/__init__.py b/reset/packages/envied/src/envied/core/titles/__init__.py new file mode 100644 index 0000000..9c95645 --- /dev/null +++ b/reset/packages/envied/src/envied/core/titles/__init__.py @@ -0,0 +1,47 @@ +from typing import Union + +from .episode import Episode, Series +from .movie import Movie, Movies +from .song import Album, Song + +Title_T = Union[Movie, Episode, Song] +Titles_T = Union[Movies, Series, Album] + + +def remap_titles(titles: Titles_T, title_map: dict) -> Titles_T: + """ + Rewrite titles in-place using an exact-match ``title_map``. + + Some services name a title differently from how the user wants it stored, which can + break library matching. ``title_map`` maps a source title string to the desired output + title. Episodes are matched on their ``title`` (the show name), Movies and Songs on + their ``name``. Returns the same collection for convenient chaining. + """ + if not title_map or not titles: + return titles + + def remap_one(title: Title_T) -> None: + attr = "title" if isinstance(title, Episode) else "name" + current = getattr(title, attr, None) + if current and current in title_map: + setattr(title, attr, title_map[current]) + + if hasattr(titles, "__iter__"): + for title in titles: + remap_one(title) + else: + remap_one(titles) + return titles + + +__all__ = ( + "Episode", + "Series", + "Movie", + "Movies", + "Album", + "Song", + "Title_T", + "Titles_T", + "remap_titles", +) diff --git a/reset/packages/envied/src/envied/core/titles/episode.py b/reset/packages/envied/src/envied/core/titles/episode.py new file mode 100644 index 0000000..a440ac3 --- /dev/null +++ b/reset/packages/envied/src/envied/core/titles/episode.py @@ -0,0 +1,185 @@ +import re +from abc import ABC +from collections import Counter +from typing import Any, Iterable, Optional, Union + +from langcodes import Language +from pymediainfo import MediaInfo +from rich.tree import Tree +from sortedcontainers import SortedKeyList + +from envied.core.config import config +from envied.core.titles.title import Title +from envied.core.utilities import sanitize_filename +from envied.core.utils.template_formatter import TemplateFormatter + + +class Episode(Title): + def __init__( + self, + id_: Any, + service: type, + title: str, + season: Union[int, str], + number: Union[int, str], + name: Optional[str] = None, + year: Optional[Union[int, str]] = None, + language: Optional[Union[str, Language]] = None, + data: Optional[Any] = None, + description: Optional[str] = None, + ) -> None: + super().__init__(id_, service, language, data) + + if not title: + raise ValueError("Episode title must be provided") + if not isinstance(title, str): + raise TypeError(f"Expected title to be a str, not {title!r}") + + if season != 0 and not season: + raise ValueError("Episode season must be provided") + if isinstance(season, str) and season.isdigit(): + season = int(season) + elif not isinstance(season, int): + raise TypeError(f"Expected season to be an int, not {season!r}") + + if number != 0 and not number: + raise ValueError("Episode number must be provided") + if isinstance(number, str) and number.isdigit(): + number = int(number) + elif not isinstance(number, int): + raise TypeError(f"Expected number to be an int, not {number!r}") + + if name is not None and not isinstance(name, str): + raise TypeError(f"Expected name to be a str, not {name!r}") + + if year is not None: + if isinstance(year, str) and year.isdigit(): + year = int(year) + elif not isinstance(year, int): + raise TypeError(f"Expected year to be an int, not {year!r}") + + title = title.strip() + + if name is not None: + name = name.strip() + # ignore episode names that are the episode number or title name + if re.match(r"Episode ?#?\d+", name, re.IGNORECASE): + name = None + elif name.lower() == title.lower(): + name = None + + if year is not None and year <= 0: + raise ValueError(f"Episode year cannot be {year}") + + self.title = title + self.season = season + self.number = number + self.name = name + self.year = year + self.description = description + + def _build_template_context(self, media_info: MediaInfo, show_service: bool = True) -> dict: + """Build template context dictionary from MediaInfo.""" + context = self._build_base_template_context(media_info, show_service) + context["title"] = self.title.replace("$", "S") + context["year"] = self.year or "" + context["season"] = f"S{self.season:02}" + context["episode"] = f"E{self.number:02}" + context["season_episode"] = f"S{self.season:02}E{self.number:02}" + context["episode_name"] = self.name or "" + return context + + def __str__(self) -> str: + return "{title}{year} S{season:02}E{number:02} {name}".format( + title=self.title, + year=f" {self.year}" if self.year else "", + season=self.season, + number=self.number, + name=self.name or "", + ).strip() + + def get_filename(self, media_info: MediaInfo, folder: bool = False, show_service: bool = True) -> str: + if folder: + template = config.get_folder_template("series") + if template: + formatter = TemplateFormatter(template) + context = self._build_template_context(media_info, show_service) + context["season"] = f"S{self.season:02}" + + folder_name = formatter.format(context) + + separators = re.sub(r"\{[^}]*\}", "", template) + spacer = "." if "." in separators and " " not in separators else " " + return sanitize_filename(folder_name, spacer) + + series_template = config.output_template.get("series") + if series_template: + derived_template = series_template + derived_template = re.sub(r"\{episode\}", "", derived_template) + derived_template = re.sub(r"\{episode_name\?\}", "", derived_template) + derived_template = re.sub(r"\{episode_name\}", "", derived_template) + derived_template = re.sub(r"\{season_episode\}", "{season}", derived_template) + + derived_template = re.sub(r"\.{2,}", ".", derived_template) + derived_template = re.sub(r"\s{2,}", " ", derived_template) + derived_template = re.sub(r"^[\.\s]+|[\.\s]+$", "", derived_template) + + formatter = TemplateFormatter(derived_template) + context = self._build_template_context(media_info, show_service) + context["season"] = f"S{self.season:02}" + + folder_name = formatter.format(context) + + separators = re.sub(r"\{[^}]*\}", "", derived_template) + spacer = "." if "." in separators and " " not in separators else " " + return sanitize_filename(folder_name, spacer) + else: + name = f"{self.title}" + if self.year: + name += f" {self.year}" + name += f" S{self.season:02}" + return sanitize_filename(name, " ") + + formatter = TemplateFormatter(config.output_template["series"]) + context = self._build_template_context(media_info, show_service) + return formatter.format(context) + + +class Series(SortedKeyList, ABC): + def __init__(self, iterable: Optional[Iterable] = None): + super().__init__(iterable, key=lambda x: (x.season, x.number, x.year or 0)) + + def __str__(self) -> str: + if not self: + return super().__str__() + return self[0].title + (f" ({self[0].year})" if self[0].year else "") + + def tree(self, verbose: bool = False) -> Tree: + seasons = Counter(x.season for x in self) + num_seasons = len(seasons) + sum(seasons.values()) + season_breakdown = ", ".join(f"S{season}({count})" for season, count in sorted(seasons.items())) + tree = Tree( + f"{num_seasons} season{'s'[:num_seasons^1]}, {season_breakdown}", + guide_style="bright_black", + ) + if verbose: + for season, episodes in seasons.items(): + season_tree = tree.add( + f"[bold]Season {str(season).zfill(len(str(num_seasons)))}[/]: [bright_black]{episodes} episodes", + guide_style="bright_black", + ) + for episode in self: + if episode.season == season: + if episode.name: + season_tree.add( + f"[bold]{str(episode.number).zfill(len(str(episodes)))}.[/] " + f"[bright_black]{episode.name}" + ) + else: + season_tree.add(f"[bright_black]Episode {str(episode.number).zfill(len(str(episodes)))}") + + return tree + + +__all__ = ("Episode", "Series") diff --git a/reset/packages/envied/src/envied/core/titles/movie.py b/reset/packages/envied/src/envied/core/titles/movie.py new file mode 100644 index 0000000..7a2e9f3 --- /dev/null +++ b/reset/packages/envied/src/envied/core/titles/movie.py @@ -0,0 +1,102 @@ +import re +from abc import ABC +from typing import Any, Iterable, Optional, Union + +from langcodes import Language +from pymediainfo import MediaInfo +from rich.tree import Tree +from sortedcontainers import SortedKeyList + +from envied.core.config import config +from envied.core.titles.title import Title +from envied.core.utilities import sanitize_filename +from envied.core.utils.template_formatter import TemplateFormatter + + +class Movie(Title): + def __init__( + self, + id_: Any, + service: type, + name: str, + year: Optional[Union[int, str]] = None, + language: Optional[Union[str, Language]] = None, + data: Optional[Any] = None, + description: Optional[str] = None, + ) -> None: + super().__init__(id_, service, language, data) + + if not name: + raise ValueError("Movie name must be provided") + if not isinstance(name, str): + raise TypeError(f"Expected name to be a str, not {name!r}") + + if year is not None: + if isinstance(year, str) and year.isdigit(): + year = int(year) + elif not isinstance(year, int): + raise TypeError(f"Expected year to be an int, not {year!r}") + + name = name.strip() + + if year is not None and year <= 0: + raise ValueError(f"Movie year cannot be {year}") + + self.name = name + self.year = year + self.description = description + + def _build_template_context(self, media_info: MediaInfo, show_service: bool = True) -> dict: + """Build template context dictionary from MediaInfo.""" + context = self._build_base_template_context(media_info, show_service) + context["title"] = self.name.replace("$", "S") + context["year"] = self.year or "" + return context + + def __str__(self) -> str: + if self.year: + return f"{self.name} ({self.year})" + return self.name + + def get_filename(self, media_info: MediaInfo, folder: bool = False, show_service: bool = True) -> str: + if folder: + template = config.get_folder_template("movies") + if template: + formatter = TemplateFormatter(template) + context = self._build_template_context(media_info, show_service) + folder_name = formatter.format(context) + + separators = re.sub(r"\{[^}]*\}", "", template) + spacer = "." if "." in separators and " " not in separators else " " + return sanitize_filename(folder_name, spacer) + name = f"{self.name}" + if self.year: + name += f" ({self.year})" + return sanitize_filename(name, " ") + + formatter = TemplateFormatter(config.output_template["movies"]) + context = self._build_template_context(media_info, show_service) + return formatter.format(context) + + +class Movies(SortedKeyList, ABC): + def __init__(self, iterable: Optional[Iterable] = None): + super().__init__(iterable, key=lambda x: x.year or 0) + + def __str__(self) -> str: + if not self: + return super().__str__() + # TODO: Assumes there's only one movie + return self[0].name + (f" ({self[0].year})" if self[0].year else "") + + def tree(self, verbose: bool = False) -> Tree: + num_movies = len(self) + tree = Tree(f"{num_movies} Movie{['s', ''][num_movies == 1]}", guide_style="bright_black") + if verbose: + for movie in self: + tree.add(f"[bold]{movie.name}[/] [bright_black]({movie.year or '?'})", guide_style="bright_black") + + return tree + + +__all__ = ("Movie", "Movies") diff --git a/reset/packages/envied/src/envied/core/titles/song.py b/reset/packages/envied/src/envied/core/titles/song.py new file mode 100644 index 0000000..59f91b5 --- /dev/null +++ b/reset/packages/envied/src/envied/core/titles/song.py @@ -0,0 +1,136 @@ +import re +from abc import ABC +from typing import Any, Iterable, Optional, Union + +from langcodes import Language +from pymediainfo import MediaInfo +from rich.tree import Tree +from sortedcontainers import SortedKeyList + +from envied.core.config import config +from envied.core.titles.title import Title +from envied.core.utilities import sanitize_filename +from envied.core.utils.template_formatter import TemplateFormatter + + +class Song(Title): + def __init__( + self, + id_: Any, + service: type, + name: str, + artist: str, + album: str, + track: int, + disc: int, + year: int, + language: Optional[Union[str, Language]] = None, + data: Optional[Any] = None, + ) -> None: + super().__init__(id_, service, language, data) + + if not name: + raise ValueError("Song name must be provided") + if not isinstance(name, str): + raise TypeError(f"Expected name to be a str, not {name!r}") + + if not artist: + raise ValueError("Song artist must be provided") + if not isinstance(artist, str): + raise TypeError(f"Expected artist to be a str, not {artist!r}") + + if not album: + raise ValueError("Song album must be provided") + if not isinstance(album, str): + raise TypeError(f"Expected album to be a str, not {album!r}") + + if not track: + raise ValueError("Song track must be provided") + if not isinstance(track, int): + raise TypeError(f"Expected track to be an int, not {track!r}") + + if not disc: + raise ValueError("Song disc must be provided") + if not isinstance(disc, int): + raise TypeError(f"Expected disc to be an int, not {disc!r}") + + if not year: + raise ValueError("Song year must be provided") + if not isinstance(year, int): + raise TypeError(f"Expected year to be an int, not {year!r}") + + name = name.strip() + artist = artist.strip() + album = album.strip() + + if track <= 0: + raise ValueError(f"Song track cannot be {track}") + if disc <= 0: + raise ValueError(f"Song disc cannot be {disc}") + if year <= 0: + raise ValueError(f"Song year cannot be {year}") + + self.name = name + self.artist = artist + self.album = album + self.track = track + self.disc = disc + self.year = year + + def __str__(self) -> str: + return "{artist} - {album} ({year}) / {track:02}. {name}".format( + artist=self.artist, album=self.album, year=self.year, track=self.track, name=self.name + ).strip() + + def _build_template_context(self, media_info: MediaInfo, show_service: bool = True) -> dict: + """Build template context dictionary from MediaInfo.""" + context = self._build_base_template_context(media_info, show_service) + context["title"] = self.name.replace("$", "S") + context["year"] = self.year or "" + context["track_number"] = f"{self.track:02}" + context["artist"] = self.artist.replace("$", "S") + context["album"] = self.album.replace("$", "S") + context["disc"] = f"{self.disc:02}" if self.disc > 1 else "" + return context + + def get_filename(self, media_info: MediaInfo, folder: bool = False, show_service: bool = True) -> str: + if folder: + template = config.get_folder_template("songs") + if template: + formatter = TemplateFormatter(template) + context = self._build_template_context(media_info, show_service) + folder_name = formatter.format(context) + + separators = re.sub(r"\{[^}]*\}", "", template) + spacer = "." if "." in separators and " " not in separators else " " + return sanitize_filename(folder_name, spacer) + name = f"{self.artist} - {self.album}" + if self.year: + name += f" ({self.year})" + return sanitize_filename(name, " ") + + formatter = TemplateFormatter(config.output_template["songs"]) + context = self._build_template_context(media_info, show_service) + return formatter.format(context) + + +class Album(SortedKeyList, ABC): + def __init__(self, iterable: Optional[Iterable] = None): + super().__init__(iterable, key=lambda x: (x.album, x.disc, x.track, x.year or 0)) + + def __str__(self) -> str: + if not self: + return super().__str__() + return f"{self[0].artist} - {self[0].album} ({self[0].year or '?'})" + + def tree(self, verbose: bool = False) -> Tree: + num_songs = len(self) + tree = Tree(f"{num_songs} Song{['s', ''][num_songs == 1]}", guide_style="bright_black") + if verbose: + for song in self: + tree.add(f"[bold]Track {song.track:02}.[/] [bright_black]({song.name})", guide_style="bright_black") + + return tree + + +__all__ = ("Song", "Album") diff --git a/reset/packages/envied/src/envied/core/titles/title.py b/reset/packages/envied/src/envied/core/titles/title.py new file mode 100644 index 0000000..0551b6f --- /dev/null +++ b/reset/packages/envied/src/envied/core/titles/title.py @@ -0,0 +1,220 @@ +from __future__ import annotations + +from abc import abstractmethod +from typing import Any, Optional, Union + +from langcodes import Language +from pymediainfo import MediaInfo + +from envied.core.config import config +from envied.core.constants import AUDIO_CODEC_MAP, DYNAMIC_RANGE_MAP, VIDEO_CODEC_MAP +from envied.core.tracks import Tracks + + +class Title: + def __init__( + self, id_: Any, service: type, language: Optional[Union[str, Language]] = None, data: Optional[Any] = None + ) -> None: + """ + Media Title from a Service. + + Parameters: + id_: An identifier for this specific title. It must be unique. Can be of any + value. + service: Service class that this title is from. + language: The original recorded language for the title. If that information + is not available, this should not be set to anything. + data: Arbitrary storage for the title. Often used to store extra metadata + information, IDs, URIs, and so on. + """ + if not id_: # includes 0, false, and similar values, this is intended + raise ValueError("A unique ID must be provided") + if hasattr(id_, "__len__") and len(id_) < 4: + raise ValueError("The unique ID is not large enough, clash likely.") + + if not service: + raise ValueError("Service class must be provided") + if not isinstance(service, type): + raise TypeError(f"Expected service to be a Class (type), not {service!r}") + + if language is not None: + if isinstance(language, str): + language = Language.get(language) + elif not isinstance(language, Language): + raise TypeError(f"Expected language to be a {Language} or str, not {language!r}") + + self.id = id_ + self.service = service + self.language = language + self.data = data + + self.tracks = Tracks() + + def __eq__(self, other: Title) -> bool: + return self.id == other.id + + def _build_base_template_context(self, media_info: MediaInfo, show_service: bool = True) -> dict: + """Build base template context dictionary from MediaInfo. + + Extracts video, audio, HDR, HFR, and multi-language information shared + across all title types. Subclasses should call this and extend the + returned dict with their specific fields (e.g., season/episode). + """ + primary_video_track = next(iter(media_info.video_tracks), None) + original_lang_tag = ( + str(self.language).split("-")[0].lower() if self.language else "" + ) + primary_audio_track = None + if original_lang_tag: + primary_audio_track = next( + ( + t + for t in media_info.audio_tracks + if t.language and t.language.split("-")[0].lower() == original_lang_tag + ), + None, + ) + if primary_audio_track is None: + primary_audio_track = next(iter(media_info.audio_tracks), None) + unique_audio_languages = len({x.language.split("-")[0] for x in media_info.audio_tracks if x.language}) + + context: dict[str, Any] = { + "source": self.service.__name__ if show_service else "", + "tag": config.tag or "", + "repack": "REPACK" if getattr(config, "repack", False) else "", + "quality": "", + "resolution": "", + "audio": "", + "audio_channels": "", + "audio_full": "", + "atmos": "", + "dual": "", + "multi": "", + "video": "", + "hdr": "", + "hfr": "", + "edition": "", + "lang_tag": "", + } + + if self.tracks: + first_track = next(iter(self.tracks), None) + if first_track and first_track.edition: + context["edition"] = " ".join(first_track.edition) + + if primary_video_track: + width = getattr(primary_video_track, "width", primary_video_track.height) + resolution = min(width, primary_video_track.height) + try: + dar = getattr(primary_video_track, "other_display_aspect_ratio", None) or [] + if dar and dar[0]: + aspect_ratio = [int(float(plane)) for plane in str(dar[0]).split(":")] + if len(aspect_ratio) == 1: + aspect_ratio.append(1) + ratio = aspect_ratio[0] / aspect_ratio[1] + if ratio not in (16 / 9, 4 / 3, 9 / 16, 3 / 4): + if abs(width - 3840) <= 50: + width = 3840 + elif abs(width - 2560) <= 50: + width = 2560 + elif abs(width - 1920) <= 50 or abs(width - 1620) <= 50: + width = 1920 + elif abs(width - 1280) <= 50 or abs(width - 1080) <= 50: + width = 1280 + + resolution = int(max(width, primary_video_track.height) * (9 / 16)) + + track_height = primary_video_track.height + if abs(resolution - track_height) <= 10 or track_height in (2160, 1440, 1080, 720, 480): + resolution = track_height + except Exception: + pass + + scan_suffix = "i" if str(getattr(primary_video_track, "scan_type", "")).lower() == "interlaced" else "p" + + context.update( + { + "quality": f"{resolution}{scan_suffix}", + "resolution": str(resolution), + "video": VIDEO_CODEC_MAP.get(primary_video_track.format, primary_video_track.format), + } + ) + + hdr_format = primary_video_track.hdr_format_commercial + trc = primary_video_track.transfer_characteristics or primary_video_track.transfer_characteristics_original + if hdr_format: + if (primary_video_track.hdr_format or "").startswith("Dolby Vision"): + context["hdr"] = "DV" + base_layer = DYNAMIC_RANGE_MAP.get(hdr_format) + if base_layer and base_layer != "DV": + context["hdr"] += f".{base_layer}" + elif (primary_video_track.hdr_format or "").startswith("HDR Vivid"): + context["hdr"] = "HDR" + else: + context["hdr"] = DYNAMIC_RANGE_MAP.get(hdr_format, "") + elif trc and "HLG" in trc: + context["hdr"] = "HLG" + else: + context["hdr"] = "" + + frame_rate = float(primary_video_track.frame_rate) if primary_video_track.frame_rate else 0.0 + context["hfr"] = "HFR" if frame_rate > 30 else "" + + if primary_audio_track: + codec = primary_audio_track.format + channel_layout = primary_audio_track.channel_layout or primary_audio_track.channellayout_original + + if channel_layout: + channels = float(sum({"LFE": 0.1}.get(position.upper(), 1) for position in channel_layout.split(" "))) + else: + channel_count = primary_audio_track.channel_s or primary_audio_track.channels or 0 + channels = float(channel_count) + + has_atmos = any( + "JOC" in (t.format_additionalfeatures or "") or t.joc for t in media_info.audio_tracks + ) + + context.update( + { + "audio": AUDIO_CODEC_MAP.get(codec, codec), + "audio_channels": f"{channels:.1f}", + "audio_full": f"{AUDIO_CODEC_MAP.get(codec, codec)}{channels:.1f}", + "atmos": "Atmos" if has_atmos else "", + } + ) + + if unique_audio_languages == 2: + context["dual"] = "DUAL" + context["multi"] = "" + elif unique_audio_languages > 2: + context["dual"] = "" + context["multi"] = "MULTi" + else: + context["dual"] = "" + context["multi"] = "" + + lang_tag_rules = config.language_tags.get("rules") if config.language_tags else None + if lang_tag_rules and self.tracks: + from envied.core.utils.language_tags import evaluate_language_tag + + audio_langs = [a.language for a in self.tracks.audio] + sub_langs = [s.language for s in self.tracks.subtitles] + context["lang_tag"] = evaluate_language_tag(lang_tag_rules, audio_langs, sub_langs) + + return context + + @abstractmethod + def get_filename(self, media_info: MediaInfo, folder: bool = False, show_service: bool = True) -> str: + """ + Get a Filename for this Title with the provided Media Info. + All filenames should be sanitized with the sanitize_filename() utility function. + + Parameters: + media_info: MediaInfo object of the file this name will be used for. + folder: This filename will be used as a folder name. Some changes may want to + be made if this is the case. + show_service: Show the service tag (e.g., iT, NF) in the filename. + """ + + +__all__ = ("Title",) diff --git a/reset/packages/envied/src/envied/core/tracks/__init__.py b/reset/packages/envied/src/envied/core/tracks/__init__.py new file mode 100644 index 0000000..37aca7f --- /dev/null +++ b/reset/packages/envied/src/envied/core/tracks/__init__.py @@ -0,0 +1,11 @@ +from .attachment import Attachment +from .audio import Audio +from .chapter import Chapter +from .chapters import Chapters +from .hybrid import Hybrid +from .subtitle import Subtitle +from .track import Track +from .tracks import Tracks +from .video import Video + +__all__ = ("Audio", "Attachment", "Chapter", "Chapters", "Hybrid", "Subtitle", "Track", "Tracks", "Video") diff --git a/reset/packages/envied/src/envied/core/tracks/attachment.py b/reset/packages/envied/src/envied/core/tracks/attachment.py new file mode 100644 index 0000000..7e94225 --- /dev/null +++ b/reset/packages/envied/src/envied/core/tracks/attachment.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import mimetypes +import os +import re +from pathlib import Path +from typing import Optional, Union +from urllib.parse import urlparse +from zlib import crc32 + +import requests + +from envied.core.config import config +from envied.core.constants import DOWNLOAD_LICENCE_ONLY + + +class Attachment: + def __init__( + self, + path: Union[Path, str, None] = None, + url: Optional[str] = None, + name: Optional[str] = None, + mime_type: Optional[str] = None, + description: Optional[str] = None, + session: Optional[requests.Session] = None, + ): + """ + Create a new Attachment. + + If providing a path, the file must already exist. + If providing a URL, the file will be downloaded to the temp directory. + Either path or url must be provided. + + If name is not provided it will use the file name (without extension). + If mime_type is not provided, it will try to guess it. + + Args: + path: Path to an existing file. + url: URL to download the attachment from. + name: Name of the attachment. + mime_type: MIME type of the attachment. + description: Description of the attachment. + session: Optional requests session to use for downloading. + """ + if path is None and url is None: + raise ValueError("Either path or url must be provided.") + + self.url = url + + if url: + if not isinstance(url, str): + raise ValueError("The attachment URL must be a string.") + + # If a URL is provided, download the file to the temp directory + parsed_url = urlparse(url) + file_name = os.path.basename(parsed_url.path) or "attachment" + + # Use provided name for the file if available + if name: + safe_name = re.sub(r'[<>:"/\\|?*]', "", name).replace(" ", "_") + file_name = f"{safe_name}{os.path.splitext(file_name)[1]}" + + download_path = config.directories.temp / file_name + + # Download the file unless we're in license-only mode + if DOWNLOAD_LICENCE_ONLY.is_set(): + path = None + else: + try: + if session is None: + with requests.Session() as session: + response = session.get(url, stream=True) + response.raise_for_status() + else: + response = session.get(url, stream=True) + response.raise_for_status() + config.directories.temp.mkdir(parents=True, exist_ok=True) + download_path.parent.mkdir(parents=True, exist_ok=True) + + with open(download_path, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + + path = download_path + except Exception as e: + raise ValueError(f"Failed to download attachment from URL: {e}") + + if path is not None and not isinstance(path, (str, Path)): + raise ValueError(f"Invalid attachment path type: expected str or Path, got {type(path).__name__}.") + + if path is not None: + path = Path(path) + if not path.exists(): + raise ValueError("The attachment file does not exist.") + + if path is not None: + name = (name or path.stem).strip() + else: + name = (name or Path(file_name).stem).strip() + mime_type = (mime_type or "").strip() or None + description = (description or "").strip() or None + + if not mime_type: + suffix = path.suffix.lower() if path is not None else Path(file_name).suffix.lower() + mime_type = { + ".ttf": "application/x-truetype-font", + ".otf": "application/vnd.ms-opentype", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".png": "image/png", + }.get(suffix, mimetypes.guess_type(file_name if path is None else path)[0]) + if not mime_type: + raise ValueError("The attachment mime-type could not be automatically detected.") + + self.path = path + self.name = name + self.mime_type = mime_type + self.description = description + + def __repr__(self) -> str: + return "{name}({items})".format( + name=self.__class__.__name__, items=", ".join([f"{k}={repr(v)}" for k, v in self.__dict__.items()]) + ) + + def __str__(self) -> str: + return " | ".join(filter(bool, ["ATT", self.name, self.mime_type, self.description])) + + def to_dict(self) -> dict[str, Optional[str]]: + """Serialise a URL-backed attachment for export/import.""" + return {"url": self.url, "name": self.name, "mime_type": self.mime_type, "description": self.description} + + @property + def id(self) -> str: + """Compute an ID from the attachment data.""" + if self.path and self.path.exists(): + checksum = crc32(self.path.read_bytes()) + elif self.url: + checksum = crc32(self.url.encode("utf8")) + else: + checksum = crc32(self.name.encode("utf8")) + return hex(checksum) + + def delete(self) -> None: + if self.path and self.path.exists(): + self.path.unlink() + self.path = None + + @classmethod + def from_url( + cls, + url: str, + name: Optional[str] = None, + mime_type: Optional[str] = None, + description: Optional[str] = None, + session: Optional[requests.Session] = None, + ) -> "Attachment": + """ + Create an attachment from a URL. + + Args: + url: URL to download the attachment from. + name: Name of the attachment. + mime_type: MIME type of the attachment. + description: Description of the attachment. + session: Optional requests session to use for downloading. + + Returns: + Attachment: A new attachment instance. + """ + return cls(url=url, name=name, mime_type=mime_type, description=description, session=session) + + +__all__ = ("Attachment",) diff --git a/reset/packages/envied/src/envied/core/tracks/audio.py b/reset/packages/envied/src/envied/core/tracks/audio.py new file mode 100644 index 0000000..03ee9cd --- /dev/null +++ b/reset/packages/envied/src/envied/core/tracks/audio.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +import math +from enum import Enum +from typing import Any, Optional, Union + +from envied.core.tracks.track import Track + + +class Audio(Track): + class Codec(str, Enum): + AAC = "AAC" # https://wikipedia.org/wiki/Advanced_Audio_Coding + AC3 = "DD" # https://wikipedia.org/wiki/Dolby_Digital + EC3 = "DD+" # https://wikipedia.org/wiki/Dolby_Digital_Plus + AC4 = "AC-4" # https://wikipedia.org/wiki/Dolby_AC-4 + OPUS = "OPUS" # https://wikipedia.org/wiki/Opus_(audio_format) + OGG = "VORB" # https://wikipedia.org/wiki/Vorbis + DTS = "DTS" # https://en.wikipedia.org/wiki/DTS_(company)#DTS_Digital_Surround + ALAC = "ALAC" # https://en.wikipedia.org/wiki/Apple_Lossless_Audio_Codec + FLAC = "FLAC" # https://en.wikipedia.org/wiki/FLAC + + @property + def extension(self) -> str: + return self.name.lower() + + @staticmethod + def from_mime(mime: str) -> Audio.Codec: + mime = mime.lower().strip().split(".")[0] + if mime == "mp4a": + return Audio.Codec.AAC + if mime == "ac-3": + return Audio.Codec.AC3 + if mime == "ec-3": + return Audio.Codec.EC3 + if mime == "ac-4": + return Audio.Codec.AC4 + if mime == "opus": + return Audio.Codec.OPUS + if mime == "dtsc": + return Audio.Codec.DTS + if mime == "alac": + return Audio.Codec.ALAC + if mime == "flac": + return Audio.Codec.FLAC + raise ValueError(f"The MIME '{mime}' is not a supported Audio Codec") + + @staticmethod + def from_codecs(codecs: str) -> Audio.Codec: + for codec in codecs.lower().split(","): + mime = codec.strip().split(".")[0] + try: + return Audio.Codec.from_mime(mime) + except ValueError: + pass + raise ValueError(f"No MIME types matched any supported Audio Codecs in '{codecs}'") + + @staticmethod + def from_netflix_profile(profile: str) -> Audio.Codec: + profile = profile.lower().strip() + if profile.startswith("heaac") or profile.startswith("xheaac"): + return Audio.Codec.AAC + if profile.startswith("dd-"): + return Audio.Codec.AC3 + if profile.startswith("ddplus"): + return Audio.Codec.EC3 + if profile.startswith("ac4"): + return Audio.Codec.AC4 + if profile.startswith("playready-oggvorbis"): + return Audio.Codec.OGG + raise ValueError(f"The Content Profile '{profile}' is not a supported Audio Codec") + + def __init__( + self, + *args: Any, + codec: Optional[Audio.Codec] = None, + bitrate: Optional[Union[str, int, float]] = None, + channels: Optional[Union[str, int, float]] = None, + joc: Optional[int] = None, + descriptive: Union[bool, int] = False, + **kwargs: Any, + ): + """ + Create a new Audio track object. + + Parameters: + codec: An Audio.Codec enum representing the audio codec. + If not specified, MediaInfo will be used to retrieve the codec + once the track has been downloaded. + bitrate: A number or float representing the average bandwidth in bytes/s. + Float values are rounded up to the nearest integer. + channels: A number, float, or string representing the number of audio channels. + Strings may represent numbers or floats. Expanded layouts like 7.1.1 is + not supported. All numbers and strings will be cast to float. + joc: The number of Joint-Object-Coding Channels/Objects in the audio stream. + descriptive: Mark this audio as being descriptive audio for the blind. + + Note: If codec, bitrate, channels, or joc is not specified some checks may be + skipped or assume a value. Specifying as much information as possible is highly + recommended. + """ + super().__init__(*args, **kwargs) + + if not isinstance(codec, (Audio.Codec, type(None))): + raise TypeError(f"Expected codec to be a {Audio.Codec}, not {codec!r}") + if not isinstance(bitrate, (str, int, float, type(None))): + raise TypeError(f"Expected bitrate to be a {str}, {int}, or {float}, not {bitrate!r}") + if not isinstance(channels, (str, int, float, type(None))): + raise TypeError(f"Expected channels to be a {str}, {int}, or {float}, not {channels!r}") + if not isinstance(joc, (int, type(None))): + raise TypeError(f"Expected joc to be a {int}, not {joc!r}") + if not isinstance(descriptive, (bool, int)) or (isinstance(descriptive, int) and descriptive not in (0, 1)): + raise TypeError(f"Expected descriptive to be a {bool} or bool-like {int}, not {descriptive!r}") + + self.codec = codec + + try: + self.bitrate = int(math.ceil(float(bitrate))) if bitrate else None + except (ValueError, TypeError) as e: + raise ValueError(f"Expected bitrate to be a number or float, {e}") + + try: + self.channels = self.parse_channels(channels) if channels else None + except (ValueError, NotImplementedError) as e: + raise ValueError(f"Expected channels to be a number, float, or a string, {e}") + + self.joc = joc + self.descriptive = bool(descriptive) + + def to_dict(self) -> dict[str, Any]: + data = super().to_dict() + data.update( + { + "codec": self.codec.name if self.codec else None, + "bitrate": self.bitrate, + "channels": self.channels, + "joc": self.joc, + "descriptive": self.descriptive, + } + ) + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> Audio: + kwargs = Track.base_kwargs_from_dict(data) + return cls( + **kwargs, + codec=Audio.Codec[data["codec"]] if data.get("codec") else None, + bitrate=data.get("bitrate"), + channels=data.get("channels"), + joc=data.get("joc"), + descriptive=data.get("descriptive", False), + ) + + @property + def atmos(self) -> bool: + """Return True if the audio track contains Dolby Atmos.""" + return bool(self.joc) + + def __str__(self) -> str: + return " | ".join( + filter( + bool, + [ + "AUD", + f"[{self.codec.value}]" if self.codec else None, + str(self.language), + ", ".join( + filter( + bool, + [ + str(self.channels) if self.channels else None, + "Atmos" if self.atmos else None, + f"JOC {self.joc}" if self.joc else None, + ], + ) + ), + f"{self.bitrate // 1000} kb/s" if self.bitrate else None, + self.get_track_name(), + ", ".join(self.edition) if self.edition else None, + ], + ) + ) + + @staticmethod + def parse_channels(channels: Union[str, int, float]) -> float: + """ + Converts a Channel string to a float representing audio channel count and layout. + E.g. "3" -> "3.0", "2.1" -> "2.1", ".1" -> "0.1". + + This does not validate channel strings as genuine channel counts or valid layouts. + It does not convert the value to assume a sub speaker channel layout, e.g. 5.1->6.0. + It also does not support expanded surround sound channel layout strings like 7.1.2. + """ + if isinstance(channels, str): + # TODO: Support all possible DASH channel configurations (https://datatracker.ietf.org/doc/html/rfc8216) + if channels.upper() == "A000": + return 2.0 + elif channels.upper() == "F801": + return 5.1 + elif channels.replace("ch", "").replace(".", "", 1).isdigit(): + # e.g., '2ch', '2', '2.0', '5.1ch', '5.1' + return float(channels.replace("ch", "")) + raise NotImplementedError(f"Unsupported Channels string value, '{channels}'") + + return float(channels) + + def get_track_name(self) -> Optional[str]: + """Return the base Track Name.""" + track_name = super().get_track_name() or "" + flag = self.descriptive and "Descriptive" + if flag: + if track_name: + flag = f" ({flag})" + track_name += flag + return track_name or None + + +__all__ = ("Audio",) diff --git a/reset/packages/envied/src/envied/core/tracks/chapter.py b/reset/packages/envied/src/envied/core/tracks/chapter.py new file mode 100644 index 0000000..b02c8ca --- /dev/null +++ b/reset/packages/envied/src/envied/core/tracks/chapter.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import re +from typing import Optional, Union +from zlib import crc32 + +TIMESTAMP_FORMAT = re.compile(r"^(?P\d{2}):(?P\d{2}):(?P\d{2})(?P.\d{3}|)$") + + +class Chapter: + def __init__(self, timestamp: Union[str, int, float], name: Optional[str] = None): + """ + Create a new Chapter with a Timestamp and optional name. + + The timestamp may be in the following formats: + - "HH:MM:SS" string, e.g., `25:05:23`. + - "HH:MM:SS.mss" string, e.g., `25:05:23.120`. + - a timecode integer in milliseconds, e.g., `90323120` is `25:05:23.120`. + - a timecode float in seconds, e.g., `90323.12` is `25:05:23.120`. + + If you have a timecode integer in seconds, just multiply it by 1000. + If you have a timecode float in milliseconds (no decimal value), just convert + it to an integer. + """ + if timestamp is None: + raise ValueError("The timestamp must be provided.") + + if not isinstance(timestamp, (str, int, float)): + raise TypeError(f"Expected timestamp to be {str}, {int} or {float}, not {type(timestamp)}") + if not isinstance(name, (str, type(None))): + raise TypeError(f"Expected name to be {str}, not {type(name)}") + + if not isinstance(timestamp, str): + if isinstance(timestamp, int): # ms + hours, remainder = divmod(timestamp, 1000 * 60 * 60) + minutes, remainder = divmod(remainder, 1000 * 60) + seconds, ms = divmod(remainder, 1000) + elif isinstance(timestamp, float): # seconds.ms + hours, remainder = divmod(timestamp, 60 * 60) + minutes, remainder = divmod(remainder, 60) + seconds, ms = divmod(int(remainder * 1000), 1000) + else: + raise TypeError + timestamp = f"{int(hours):02}:{int(minutes):02}:{int(seconds):02}.{str(ms).zfill(3)[:3]}" + + timestamp_m = TIMESTAMP_FORMAT.match(timestamp) + if not timestamp_m: + raise ValueError(f"The timestamp format is invalid: {timestamp}") + + hour, minute, second, ms = timestamp_m.groups() + if not ms: + timestamp += ".000" + + self.timestamp = timestamp + self.name = name + + def __repr__(self) -> str: + return "{name}({items})".format( + name=self.__class__.__name__, items=", ".join([f"{k}={repr(v)}" for k, v in self.__dict__.items()]) + ) + + def __str__(self) -> str: + return " | ".join(filter(bool, ["CHP", self.timestamp, self.name])) + + @property + def id(self) -> str: + """Compute an ID from the Chapter data.""" + checksum = crc32(str(self).encode("utf8")) + return hex(checksum) + + @property + def named(self) -> bool: + """Check if Chapter is named.""" + return bool(self.name) + + +__all__ = ("Chapter",) diff --git a/reset/packages/envied/src/envied/core/tracks/chapters.py b/reset/packages/envied/src/envied/core/tracks/chapters.py new file mode 100644 index 0000000..08ba0bf --- /dev/null +++ b/reset/packages/envied/src/envied/core/tracks/chapters.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import re +from abc import ABC +from pathlib import Path +from typing import Any, Iterable, Optional, Union +from zlib import crc32 + +from sortedcontainers import SortedKeyList + +from envied.core.tracks import Chapter + +OGM_SIMPLE_LINE_1_FORMAT = re.compile(r"^CHAPTER(?P\d+)=(?P\d{2,}:\d{2}:\d{2}\.\d{3})$") +OGM_SIMPLE_LINE_2_FORMAT = re.compile(r"^CHAPTER(?P\d+)NAME=(?P.*)$") + + +class Chapters(SortedKeyList, ABC): + def __init__(self, iterable: Optional[Iterable[Chapter]] = None): + super().__init__(key=lambda x: x.timestamp or 0) + for chapter in iterable or []: + self.add(chapter) + + def __repr__(self) -> str: + return "{name}({items})".format( + name=self.__class__.__name__, items=", ".join([f"{k}={repr(v)}" for k, v in self.__dict__.items()]) + ) + + def __str__(self) -> str: + return "\n".join( + [ + " | ".join(filter(bool, ["CHP", f"[{i:02}]", chapter.timestamp, chapter.name])) + for i, chapter in enumerate(self, start=1) + ] + ) + + @classmethod + def loads(cls, data: str) -> Chapters: + """Load chapter data from a string.""" + lines = [line.strip() for line in data.strip().splitlines(keepends=False)] + + if len(lines) % 2 != 0: + raise ValueError("The number of chapter lines must be even.") + + chapters = [] + + for line_1, line_2 in zip(lines[::2], lines[1::2]): + line_1_match = OGM_SIMPLE_LINE_1_FORMAT.match(line_1) + if not line_1_match: + raise SyntaxError(f"An unexpected syntax error occurred on: {line_1}") + line_2_match = OGM_SIMPLE_LINE_2_FORMAT.match(line_2) + if not line_2_match: + raise SyntaxError(f"An unexpected syntax error occurred on: {line_2}") + + line_1_number, timestamp = line_1_match.groups() + line_2_number, name = line_2_match.groups() + + if line_1_number != line_2_number: + raise SyntaxError( + f"The chapter numbers {line_1_number} and {line_2_number} do not match on:\n{line_1}\n{line_2}" + ) + + if not timestamp: + raise SyntaxError(f"The timestamp is missing on: {line_1}") + + chapters.append(Chapter(timestamp, name)) + + return cls(chapters) + + @classmethod + def load(cls, path: Union[Path, str]) -> Chapters: + """Load chapter data from a file.""" + if isinstance(path, str): + path = Path(path) + return cls.loads(path.read_text(encoding="utf8")) + + def dumps(self, fallback_name: str = "") -> str: + """ + Return chapter data in OGM-based Simple Chapter format. + https://mkvtoolnix.download/doc/mkvmerge.html#mkvmerge.chapters.simple + + Parameters: + fallback_name: Name used for Chapters without a Name set. + + The fallback name can use the following variables in f-string style: + + - {i}: The Chapter number starting at 1. + E.g., `"Chapter {i}"`: "Chapter 1", "Intro", "Chapter 3". + - {j}: A number starting at 1 that increments any time a Chapter has no name. + E.g., `"Chapter {j}"`: "Chapter 1", "Intro", "Chapter 2". + + These are formatted with f-strings, directives are supported. + For example, `"Chapter {i:02}"` will result in `"Chapter 01"`. + """ + chapters = [] + j = 0 + + for i, chapter in enumerate(self, start=1): + if not chapter.name: + j += 1 + chapters.append( + "CHAPTER{num}={time}\nCHAPTER{num}NAME={name}".format( + num=f"{i:02}", time=chapter.timestamp, name=chapter.name or fallback_name.format(i=i, j=j) + ) + ) + + return "\n".join(chapters) + + def dump(self, path: Union[Path, str], *args: Any, **kwargs: Any) -> int: + """ + Write chapter data in OGM-based Simple Chapter format to a file. + + Parameters: + path: The file path to write the Chapter data to, overwriting + any existing data. + + See `Chapters.dumps` for more parameter documentation. + """ + if isinstance(path, str): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + + ogm_text = self.dumps(*args, **kwargs) + return path.write_text(ogm_text, encoding="utf8") + + def add(self, value: Chapter) -> None: + if not isinstance(value, Chapter): + raise TypeError(f"Can only add {Chapter} objects, not {type(value)}") + + if any(chapter.timestamp == value.timestamp for chapter in self): + raise ValueError(f"A Chapter with the Timestamp {value.timestamp} already exists") + + super().add(value) + + if not any(chapter.timestamp == "00:00:00.000" for chapter in self): + self.add(Chapter(0)) + + @property + def id(self) -> str: + """Compute an ID from the Chapter data.""" + checksum = crc32("\n".join([chapter.id for chapter in self]).encode("utf8")) + return hex(checksum) + + +__all__ = ("Chapters", "Chapter") diff --git a/reset/packages/envied/src/envied/core/tracks/dv_fixup.py b/reset/packages/envied/src/envied/core/tracks/dv_fixup.py new file mode 100644 index 0000000..5810ea5 --- /dev/null +++ b/reset/packages/envied/src/envied/core/tracks/dv_fixup.py @@ -0,0 +1,117 @@ +""" +DV fixup for HLS composite HEVC streams. + +Some services deliver DV Profile 8.1 in a stream whose primary CODECS is plain +hvc1, with DV advertised only via SUPPLEMENTAL-CODECS. The fMP4 carries DV RPU NALs but +the container does not signal DV, so muxing produces an MKV that mediainfo and DV-capable +TVs see as plain HDR10/HDR10+. + +A dovi_tool extract-rpu / inject-rpu round-trip rewrites the bitstream so it is recognised +as DV after muxing. HDR10+ SEI NALs and HDR10 base layer signaling survive untouched. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import TYPE_CHECKING + +from rich.padding import Padding +from rich.rule import Rule + +from envied.core.binaries import FFMPEG, DoviTool +from envied.core.config import config +from envied.core.console import console +from envied.core.utilities import get_debug_logger +from envied.core.utils import dovi +from envied.core.utils.subprocess import run_step + +if TYPE_CHECKING: + from envied.core.tracks import Video + + +class DVFixup: + """Round-trip a DV-composite HEVC track through dovi_tool to restore DV signaling.""" + + def __init__(self, video: "Video") -> None: + self.log = logging.getLogger("dv-fixup") + self.debug_logger = get_debug_logger() + self.video = video + + if not DoviTool: + raise EnvironmentError("dovi_tool is required for DV-composite fixup but was not found.") + if not FFMPEG: + raise EnvironmentError("ffmpeg is required for DV-composite fixup but was not found.") + if not video.path or not Path(video.path).exists(): + raise ValueError(f"Video track {video.id} was not downloaded before DV fixup.") + + def run(self) -> Path: + """Execute the fixup. Returns the DV-signaled HEVC path, or the original + source path on any failure so muxing can proceed with the as-downloaded file.""" + source = Path(self.video.path) + height = self.video.height or 0 + console.print(Padding(Rule(f"[rule.text]DV Composite Fixup ({height}p)"), (1, 2))) + + fixed_hevc = source.with_name(f"{self.video.id}.dv.hevc") + if fixed_hevc.exists() and fixed_hevc.stat().st_size > 0: + self.log.info("✓ DV signaling already restored (reusing existing fixup)") + return fixed_hevc + + tmp = config.directories.temp + tmp.mkdir(parents=True, exist_ok=True) + suffix = f"{self.video.id}_{height or 'na'}" + raw_hevc = tmp / f"dvfix_{suffix}.hevc" + rpu = tmp / f"dvfix_{suffix}_rpu.bin" + + try: + run_step( + [FFMPEG, "-nostdin", "-y", "-i", source, "-c:v", "copy", "-f", "hevc", raw_hevc], + status="Demuxing HEVC bitstream...", + output=raw_hevc, + label="ffmpeg demux", + ) + dovi.extract_rpu_with_fallback(raw_hevc, rpu) + dovi.inject_rpu(raw_hevc, rpu, fixed_hevc, status="Re-injecting DV RPU with proper signaling...") + except Exception as e: + self.log.warning(f"DV fixup failed ({e}); muxing source as-is.") + if self.debug_logger: + self.debug_logger.log( + level="WARNING", + operation="dv_fixup", + message="DV fixup failed; falling back to source", + context={"error": str(e), "source": str(source)}, + ) + for leftover in (raw_hevc, rpu, fixed_hevc): + leftover.unlink(missing_ok=True) + return source + + for leftover in (raw_hevc, rpu): + leftover.unlink(missing_ok=True) + + self.log.info("✓ DV signaling restored") + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="dv_fixup", + message="DV fixup complete", + context={"source": str(source), "output": str(fixed_hevc)}, + success=True, + ) + return fixed_hevc + + +def apply_dv_fixup(video: "Video") -> None: + """Run DV fixup on `video` if flagged as DV-composite. Updates `video.path` in place + and deletes the original source file so the standard mux cleanup handles the new path.""" + if not getattr(video, "dv_compatible_bitstream", False): + return + if not video.path or not Path(video.path).exists(): + return + original = Path(video.path) + fixed = DVFixup(video).run() + if fixed != original: + video.path = fixed + original.unlink(missing_ok=True) + + +__all__ = ("DVFixup", "apply_dv_fixup") diff --git a/reset/packages/envied/src/envied/core/tracks/hybrid.py b/reset/packages/envied/src/envied/core/tracks/hybrid.py new file mode 100644 index 0000000..ccc4444 --- /dev/null +++ b/reset/packages/envied/src/envied/core/tracks/hybrid.py @@ -0,0 +1,714 @@ +import json +import logging +import os +import random +import re +import subprocess +import sys +from pathlib import Path +from typing import Optional + +from rich.padding import Padding +from rich.rule import Rule + +from envied.core.binaries import FFMPEG, FFProbe, HDR10PlusTool +from envied.core.config import config +from envied.core.console import console +from envied.core.utilities import get_debug_logger +from envied.core.utils import dovi +from envied.core.utils.subprocess import run_step + + +class Hybrid: + def __init__(self, videos, source) -> None: + self.log = logging.getLogger("hybrid") + self.debug_logger = get_debug_logger() + + """ + Takes the Dolby Vision and HDR10(+) streams out of the VideoTracks. + It will then attempt to inject the Dolby Vision metadata layer to the HDR10(+) stream. + If no DV track is available but HDR10+ is present, it will convert HDR10+ to DV. + """ + global directories + from envied.core.tracks import Video + + self.videos = videos + self.source = source + self.rpu_file = "RPU.bin" + self.hdr_type = "HDR10" + self.hevc_file = f"{self.hdr_type}-DV.hevc" + self.hdr10plus_to_dv = False + self.hdr10plus_file = "HDR10Plus.json" + + # Get resolution info from HDR10 track for display + hdr10_track = next((v for v in videos if v.range == Video.Range.HDR10), None) + hdr10p_track = next((v for v in videos if v.range == Video.Range.HDR10P), None) + track_for_res = hdr10_track or hdr10p_track + self.resolution = f"{track_for_res.height}p" if track_for_res and track_for_res.height else "Unknown" + + console.print(Padding(Rule(f"[rule.text]HDR10+DV Hybrid ({self.resolution})"), (1, 2))) + + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="hybrid_init", + message="Starting HDR10+DV hybrid processing", + context={ + "source": source, + "resolution": self.resolution, + "video_count": len(videos), + "video_ranges": [str(v.range) for v in videos], + }, + ) + + for video in self.videos: + if not video.path or not os.path.exists(video.path): + raise ValueError(f"Video track {video.id} was not downloaded before injection.") + + # Check if we have DV track available + has_dv = any(video.range == Video.Range.DV for video in self.videos) + has_hdr10 = any(video.range == Video.Range.HDR10 for video in self.videos) + has_hdr10p = any(video.range == Video.Range.HDR10P for video in self.videos) + + if not has_hdr10 and not has_hdr10p: + raise ValueError("No HDR10 or HDR10+ track available for hybrid processing.") + + # If we have HDR10+ but no DV, we can convert HDR10+ to DV + if not has_dv and has_hdr10p: + console.status("No DV track found, but HDR10+ is available. Will convert HDR10+ to DV.") + self.hdr10plus_to_dv = True + elif not has_dv: + raise ValueError("No DV track available and no HDR10+ to convert.") + + if os.path.isfile(config.directories.temp / self.hevc_file): + console.status("Already Injected") + return + + for video in videos: + # Use the actual path from the video track + save_path = video.path + if not save_path or not os.path.exists(save_path): + raise ValueError(f"Video track {video.id} was not downloaded or path not found: {save_path}") + + if video.range == Video.Range.HDR10: + self.extract_stream(save_path, "HDR10") + elif video.range == Video.Range.HDR10P: + self.extract_stream(save_path, "HDR10") + self.hdr_type = "HDR10+" + elif video.range == Video.Range.DV: + self.extract_stream(save_path, "DV") + + if self.hdr10plus_to_dv: + # Extract HDR10+ metadata and convert to DV + hdr10p_video = next(v for v in videos if v.range == Video.Range.HDR10P) + self.extract_hdr10plus(hdr10p_video) + self.convert_hdr10plus_to_dv() + else: + # Regular DV extraction + dv_video = next(v for v in videos if v.range == Video.Range.DV) + self.extract_rpu(dv_video) + if os.path.isfile(config.directories.temp / "RPU_UNT.bin"): + self.rpu_file = "RPU_UNT.bin" + # Mode 3 conversion already done during extraction when not untouched + elif os.path.isfile(config.directories.temp / "RPU.bin"): + # RPU already extracted with mode 3 + pass + + # Edit L6 with actual luminance values from RPU, then L5 active area + self.level_6() + base_video = next((v for v in videos if v.range in (Video.Range.HDR10, Video.Range.HDR10P)), None) + if base_video and base_video.path: + self.level_5(base_video.path) + + self.injecting() + + if self.debug_logger: + self.debug_logger.log( + level="INFO", + operation="hybrid_complete", + message="Injection Completed", + context={ + "hdr_type": self.hdr_type, + "resolution": self.resolution, + "hdr10plus_to_dv": self.hdr10plus_to_dv, + "rpu_file": self.rpu_file, + "output_file": self.hevc_file, + }, + ) + self.log.info("✓ Injection Completed") + if self.source == ("itunes" or "appletvplus"): + Path.unlink(config.directories.temp / "hdr10.mkv") + Path.unlink(config.directories.temp / "dv.mkv") + Path.unlink(config.directories.temp / "HDR10.hevc", missing_ok=True) + Path.unlink(config.directories.temp / "DV.hevc", missing_ok=True) + for rpu_name in ("RPU.bin", "RPU_UNT.bin", "RPU_L5.bin", "RPU_L6.bin"): + Path.unlink(config.directories.temp / rpu_name, missing_ok=True) + Path.unlink(config.directories.temp / "L5.json", missing_ok=True) + Path.unlink(config.directories.temp / "L6.json", missing_ok=True) + + def extract_stream(self, save_path, type_): + output = Path(config.directories.temp / f"{type_}.hevc") + try: + run_step( + [FFMPEG or "ffmpeg", "-nostdin", "-y", "-i", save_path, "-c:v", "copy", output], + status=f"Extracting {type_} stream...", + output=output, + label=f"ffmpeg extract {type_}", + ) + except RuntimeError as e: + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_extract_stream", + message=f"Failed extracting {type_} stream", + context={"type": type_, "input": str(save_path), "output": str(output), "error": str(e)}, + ) + self.log.error(f"x Failed extracting {type_} stream") + sys.exit(1) + + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="hybrid_extract_stream", + message=f"Extracted {type_} stream", + context={"type": type_, "input": str(save_path), "output": str(output)}, + success=True, + ) + + def extract_rpu(self, video, untouched=False): + if os.path.isfile(config.directories.temp / "RPU.bin") or os.path.isfile( + config.directories.temp / "RPU_UNT.bin" + ): + return + + rpu_name = "RPU_UNT" if untouched else "RPU" + rpu_path = config.directories.temp / f"{rpu_name}.bin" + dv_stream = config.directories.temp / "DV.hevc" + spinner = f"Extracting{' untouched ' if untouched else ' '}RPU from Dolby Vision stream..." + + try: + dovi.extract_rpu(dv_stream, rpu_path, mode=None if untouched else 3, status=spinner) + except RuntimeError as e: + stderr_text = str(e) + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_extract_rpu", + message=f"Failed extracting{' untouched ' if untouched else ' '}RPU", + context={"untouched": untouched, "error": stderr_text}, + ) + if "MAX_PQ_LUMINANCE" in stderr_text: + self.extract_rpu(video, untouched=True) + return + if "Invalid PPS index" in stderr_text: + raise ValueError("Dolby Vision VideoTrack seems to be corrupt") + raise ValueError(f"Failed extracting{' untouched ' if untouched else ' '}RPU from Dolby Vision stream") + + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="hybrid_extract_rpu", + message=f"Extracted{' untouched ' if untouched else ' '}RPU from Dolby Vision stream", + context={"untouched": untouched, "output": f"{rpu_name}.bin"}, + success=True, + ) + + def level_5(self, input_video): + """Generate Level 5 active area metadata via crop detection on the HDR10 stream. + + This resolves mismatches where DV has no black bars but HDR10 does (or vice versa) + by telling the display the correct active area. + """ + if os.path.isfile(config.directories.temp / "RPU_L5.bin"): + return + + ffprobe_bin = str(FFProbe) if FFProbe else "ffprobe" + ffmpeg_bin = str(FFMPEG) if FFMPEG else "ffmpeg" + + # Get video duration for random sampling + with console.status("Detecting active area (crop detection)...", spinner="dots"): + result_duration = subprocess.run( + [ffprobe_bin, "-v", "error", "-show_entries", "format=duration", "-of", "json", str(input_video)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + if result_duration.returncode != 0: + if self.debug_logger: + self.debug_logger.log( + level="WARNING", + operation="hybrid_level5", + message="Could not probe video duration", + context={"returncode": result_duration.returncode, "stderr": (result_duration.stderr or "")}, + ) + self.log.warning("Could not probe video duration, skipping L5 crop detection") + return + + duration_info = json.loads(result_duration.stdout) + duration = float(duration_info["format"]["duration"]) + + # Get video resolution for proper border calculation + result_streams = subprocess.run( + [ + ffprobe_bin, + "-v", + "error", + "-select_streams", + "v:0", + "-show_entries", + "stream=width,height", + "-of", + "json", + str(input_video), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + if result_streams.returncode != 0: + if self.debug_logger: + self.debug_logger.log( + level="WARNING", + operation="hybrid_level5", + message="Could not probe video resolution", + context={"returncode": result_streams.returncode, "stderr": (result_streams.stderr or "")}, + ) + self.log.warning("Could not probe video resolution, skipping L5 crop detection") + return + + stream_info = json.loads(result_streams.stdout) + original_width = int(stream_info["streams"][0]["width"]) + original_height = int(stream_info["streams"][0]["height"]) + + # Sample 10 random timestamps and run cropdetect on each + random_times = sorted(random.uniform(0, duration) for _ in range(10)) + + crop_results = [] + for t in random_times: + result_cropdetect = subprocess.run( + [ + ffmpeg_bin, + "-y", + "-nostdin", + "-loglevel", + "info", + "-ss", + f"{t:.2f}", + "-i", + str(input_video), + "-vf", + "cropdetect=round=2", + "-vframes", + "10", + "-f", + "null", + "-", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + # cropdetect outputs crop=w:h:x:y + crop_match = re.search( + r"crop=(\d+):(\d+):(\d+):(\d+)", + (result_cropdetect.stdout or "") + (result_cropdetect.stderr or ""), + ) + if crop_match: + w, h = int(crop_match.group(1)), int(crop_match.group(2)) + x, y = int(crop_match.group(3)), int(crop_match.group(4)) + # Calculate actual border sizes from crop geometry + left = x + top = y + right = original_width - w - x + bottom = original_height - h - y + crop_results.append((left, top, right, bottom)) + + if not crop_results: + if self.debug_logger: + self.debug_logger.log( + level="WARNING", + operation="hybrid_level5", + message="No crop data detected, skipping L5", + context={"samples": len(random_times)}, + ) + self.log.warning("No crop data detected, skipping L5") + return + + # Find the most common crop values + crop_counts = {} + for crop in crop_results: + crop_counts[crop] = crop_counts.get(crop, 0) + 1 + most_common = max(crop_counts, key=crop_counts.get) + left, top, right, bottom = most_common # frame instead of leaving phantom bars from the source. + + l5_json = { + "active_area": { + "crop": False, + "presets": [{"id": 0, "left": left, "right": right, "top": top, "bottom": bottom}], + "edits": {"all": 0}, + } + } + + l5_path = config.directories.temp / "L5.json" + with open(l5_path, "w") as f: + json.dump(l5_json, f, indent=4) + + try: + dovi.editor( + config.directories.temp / self.rpu_file, + l5_path, + config.directories.temp / "RPU_L5.bin", + status="Editing RPU Level 5 active area...", + label="dovi_tool editor (L5)", + ) + except RuntimeError as e: + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_level5", + message="Failed editing RPU Level 5 values", + context={"error": str(e)}, + ) + raise ValueError("Failed editing RPU Level 5 values") + + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="hybrid_level5", + message="Edited RPU Level 5 active area", + context={ + "crop": {"left": left, "right": right, "top": top, "bottom": bottom}, + "samples": len(crop_results), + }, + success=True, + ) + self.rpu_file = "RPU_L5.bin" + + @staticmethod + def sanitize_l6( + max_mdl: Optional[int], min_mdl: Optional[int], max_cll: Optional[int], max_fall: Optional[int] + ) -> tuple[Optional[int], Optional[int], Optional[int], Optional[int]]: + """Clamp static L6 values to a valid relationship. + + MaxCLL must not exceed the mastering-display peak (some sources, e.g. ATV + HDR10+, ship MaxCLL 10000 on a 1000-nit master), and MaxFALL must not exceed + MaxCLL. A value of 0 means "unknown" and is preserved as-is. + """ + if max_mdl and max_cll and max_cll > max_mdl: + max_cll = max_mdl + if max_cll and max_fall and max_fall > max_cll: + max_fall = max_cll + return max_mdl, min_mdl, max_cll, max_fall + + def level_6(self): + """Edit RPU Level 6 values using the static L6 luminance data from the RPU.""" + if os.path.isfile(config.directories.temp / "RPU_L6.bin"): + return + + try: + with console.status("Reading RPU luminance metadata...", spinner="dots"): + info_text = dovi.info_summary(config.directories.temp / self.rpu_file) + except RuntimeError as e: + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_level6", + message="Failed reading RPU metadata for Level 6 values", + context={"error": str(e)}, + ) + raise ValueError("Failed reading RPU metadata for Level 6 values") + + max_cll = None + max_fall = None + max_mdl = None + min_mdl = None + + in_l6 = False + for line in info_text.splitlines(): + stripped = line.strip() + if "L6 metadata" in stripped: + in_l6 = True + if stripped.startswith("RPU mastering display:"): + mastering = stripped.split(":", 1)[1].strip() + min_lum, max_lum = mastering.split("/")[0], mastering.split("/")[1].split(" ")[0] + min_mdl = int(float(min_lum) * 10000) + max_mdl = int(float(max_lum)) + elif in_l6 and "MaxCLL:" in stripped and max_cll is None: + max_cll = int(float(stripped.split("MaxCLL:")[1].split("nits")[0].strip().rstrip(","))) + if "MaxFALL:" in stripped: + max_fall = int(float(stripped.split("MaxFALL:")[1].split("nits")[0].strip().rstrip(","))) + + if any(v is None for v in (max_cll, max_fall, max_mdl, min_mdl)): + base_max_mdl, base_min_mdl, base_cll, base_fall = self._probe_hdr_metadata() + if max_cll is None: + max_cll = base_cll + if max_fall is None: + max_fall = base_fall + if max_mdl is None: + max_mdl = base_max_mdl + if min_mdl is None: + min_mdl = base_min_mdl + + max_mdl, min_mdl, max_cll, max_fall = self.sanitize_l6(max_mdl, min_mdl, max_cll, max_fall) + + if any(v is None for v in (max_cll, max_fall, max_mdl, min_mdl)): + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_level6", + message="Could not extract Level 6 luminance data from RPU", + context={"max_cll": max_cll, "max_fall": max_fall, "max_mdl": max_mdl, "min_mdl": min_mdl}, + ) + raise ValueError("Could not extract Level 6 luminance data from RPU") + + level6_data = { + "level6": { + "remove_cmv4": False, + "remove_mapping": False, + "max_display_mastering_luminance": max_mdl, + "min_display_mastering_luminance": min_mdl, + "max_content_light_level": max_cll, + "max_frame_average_light_level": max_fall, + } + } + + l6_path = config.directories.temp / "L6.json" + with open(l6_path, "w") as f: + json.dump(level6_data, f, indent=4) + + try: + dovi.editor( + config.directories.temp / self.rpu_file, + l6_path, + config.directories.temp / "RPU_L6.bin", + status="Editing RPU Level 6 values...", + label="dovi_tool editor (L6)", + ) + except RuntimeError as e: + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_level6", + message="Failed editing RPU Level 6 values", + context={"error": str(e)}, + ) + raise ValueError("Failed editing RPU Level 6 values") + + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="hybrid_level6", + message="Edited RPU Level 6 luminance values", + context={ + "max_cll": max_cll, + "max_fall": max_fall, + "max_mdl": max_mdl, + "min_mdl": min_mdl, + }, + success=True, + ) + self.rpu_file = "RPU_L6.bin" + + def injecting(self): + if os.path.isfile(config.directories.temp / self.hevc_file): + return + + try: + dovi.inject_rpu( + config.directories.temp / "HDR10.hevc", + config.directories.temp / self.rpu_file, + config.directories.temp / self.hevc_file, + status=f"Injecting Dolby Vision metadata into {self.hdr_type} stream...", + label="dovi_tool inject-rpu", + ) + except RuntimeError as e: + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_inject_rpu", + message="Failed injecting Dolby Vision metadata into HDR10 stream", + context={"error": str(e)}, + ) + raise ValueError("Failed injecting Dolby Vision metadata into HDR10 stream") + + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="hybrid_inject_rpu", + message=f"Injected Dolby Vision metadata into {self.hdr_type} stream", + context={ + "hdr_type": self.hdr_type, + "rpu_file": self.rpu_file, + "output": self.hevc_file, + "drop_hdr10plus": self.hdr10plus_to_dv, + }, + success=True, + ) + + def extract_hdr10plus(self, _video): + """Extract HDR10+ metadata from the video stream""" + if os.path.isfile(config.directories.temp / self.hdr10plus_file): + return + + if not HDR10PlusTool: + raise ValueError("HDR10Plus_tool not found. Please install it to use HDR10+ to DV conversion.") + + try: + run_step( + [ + HDR10PlusTool, + "extract", + config.directories.temp / "HDR10.hevc", + "-o", + config.directories.temp / self.hdr10plus_file, + ], + status="Extracting HDR10+ metadata...", + output=config.directories.temp / self.hdr10plus_file, + label="hdr10plus_tool extract", + ) + except RuntimeError as e: + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_extract_hdr10plus", + message="Failed extracting HDR10+ metadata", + context={"error": str(e)}, + ) + raise ValueError("Failed extracting HDR10+ metadata") + + file_size = os.path.getsize(config.directories.temp / self.hdr10plus_file) + if file_size == 0: + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_extract_hdr10plus", + message="No HDR10+ metadata found in the stream", + context={"file_size": 0}, + ) + raise ValueError("No HDR10+ metadata found in the stream") + + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="hybrid_extract_hdr10plus", + message="Extracted HDR10+ metadata", + context={"output": self.hdr10plus_file, "file_size": file_size}, + success=True, + ) + + def _probe_hdr_metadata(self): + """Extract mastering display and content light level metadata from the HDR10 stream via ffprobe. + + Returns (max_mdl, min_mdl, max_cll, max_fall) in dovi_tool level6 units: + - max_mdl: nits (integer) + - min_mdl: 0.0001 nit units (integer) + - max_cll / max_fall: nits (integer) + """ + ffprobe_bin = str(FFProbe) if FFProbe else "ffprobe" + result = subprocess.run( + [ + ffprobe_bin, + "-v", + "error", + "-select_streams", + "v:0", + "-show_entries", + "stream_side_data=max_luminance,min_luminance,max_content,max_average", + "-of", + "json", + str(config.directories.temp / "HDR10.hevc"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + max_mdl = 1000 + min_mdl = 1 + max_cll = 0 + max_fall = 0 + + if result.returncode == 0 and result.stdout: + try: + probe = json.loads(result.stdout) + for stream in probe.get("streams", []): + for sd in stream.get("side_data_list", []): + if "max_luminance" in sd: + num, den = sd["max_luminance"].split("/") + max_mdl = int(int(num) / int(den)) + if "min_luminance" in sd: + num, den = sd["min_luminance"].split("/") + min_mdl = int(int(num) / int(den) * 10000) + if "max_content" in sd: + max_cll = int(sd["max_content"]) + if "max_average" in sd: + max_fall = int(sd["max_average"]) + except (json.JSONDecodeError, KeyError, ValueError, ZeroDivisionError): + pass + + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="hybrid_probe_hdr_metadata", + message="Probed HDR metadata from source stream", + context={"max_mdl": max_mdl, "min_mdl": min_mdl, "max_cll": max_cll, "max_fall": max_fall}, + ) + + return max_mdl, min_mdl, max_cll, max_fall + + def convert_hdr10plus_to_dv(self): + """Convert HDR10+ metadata to Dolby Vision RPU""" + if os.path.isfile(config.directories.temp / "RPU.bin"): + return + + with console.status("Converting HDR10+ metadata to Dolby Vision...", spinner="dots"): + # Extract actual HDR metadata from the source stream + max_mdl, min_mdl, max_cll, max_fall = self._probe_hdr_metadata() + max_mdl, min_mdl, max_cll, max_fall = self.sanitize_l6(max_mdl, min_mdl, max_cll, max_fall) + + # First create the extra metadata JSON for dovi_tool + extra_metadata = { + "cm_version": "V29", + "length": 0, # dovi_tool will figure this out + "level6": { + "max_display_mastering_luminance": max_mdl, + "min_display_mastering_luminance": min_mdl, + "max_content_light_level": max_cll, + "max_frame_average_light_level": max_fall, + }, + } + + with open(config.directories.temp / "extra.json", "w") as f: + json.dump(extra_metadata, f, indent=2) + + try: + dovi.generate_from_hdr10plus( + config.directories.temp / "extra.json", + config.directories.temp / self.hdr10plus_file, + config.directories.temp / "RPU.bin", + label="dovi_tool generate", + ) + except RuntimeError as e: + if self.debug_logger: + self.debug_logger.log( + level="ERROR", + operation="hybrid_convert_hdr10plus", + message="Failed converting HDR10+ to Dolby Vision", + context={"error": str(e)}, + ) + raise ValueError("Failed converting HDR10+ to Dolby Vision") + + if self.debug_logger: + self.debug_logger.log( + level="DEBUG", + operation="hybrid_convert_hdr10plus", + message="Converted HDR10+ metadata to Dolby Vision Profile 8", + success=True, + ) + + # Clean up temporary files + Path.unlink(config.directories.temp / "extra.json") + Path.unlink(config.directories.temp / self.hdr10plus_file) diff --git a/reset/packages/envied/src/envied/core/tracks/subtitle.py b/reset/packages/envied/src/envied/core/tracks/subtitle.py new file mode 100644 index 0000000..7135692 --- /dev/null +++ b/reset/packages/envied/src/envied/core/tracks/subtitle.py @@ -0,0 +1,1359 @@ +from __future__ import annotations + +import logging +import re +import subprocess +from collections import defaultdict +from enum import Enum +from functools import partial +from io import BytesIO +from pathlib import Path +from typing import Any, Callable, Iterable, Optional, Union + +import pycaption +import pysubs2 +import requests +from construct import Container +from pycaption import Caption, CaptionList, CaptionNode, WebVTTReader +from pycaption.geometry import Layout +from pymp4.parser import MP4 +from subby import CommonIssuesFixer, SAMIConverter, SDHStripper, WebVTTConverter, WVTTConverter +from subtitle_filter import Subtitles + +from envied.core import binaries +from envied.core.config import config +from envied.core.tracks.track import Track +from envied.core.utilities import try_ensure_utf8 +from envied.core.utils.webvtt import merge_segmented_webvtt + +# silence srt library INFO logging +logging.getLogger("srt").setLevel(logging.ERROR) + + +class Subtitle(Track): + class Codec(str, Enum): + SubRip = "SRT" # https://wikipedia.org/wiki/SubRip + SubStationAlpha = "SSA" # https://wikipedia.org/wiki/SubStation_Alpha + SubStationAlphav4 = "ASS" # https://wikipedia.org/wiki/SubStation_Alpha#Advanced_SubStation_Alpha= + TimedTextMarkupLang = "TTML" # https://wikipedia.org/wiki/Timed_Text_Markup_Language + WebVTT = "VTT" # https://wikipedia.org/wiki/WebVTT + SAMI = "SMI" # https://wikipedia.org/wiki/SAMI + MicroDVD = "SUB" # https://wikipedia.org/wiki/MicroDVD + MPL2 = "MPL2" # MPL2 subtitle format + TMP = "TMP" # TMP subtitle format + # MPEG-DASH box-encapsulated subtitle formats + fTTML = "STPP" # https://www.w3.org/TR/2018/REC-ttml-imsc1.0.1-20180424 + fVTT = "WVTT" # https://www.w3.org/TR/webvtt1 + + @property + def extension(self) -> str: + return self.value.lower() + + @staticmethod + def from_mime(mime: str) -> Subtitle.Codec: + mime = mime.lower().strip().split(".")[0] + if mime == "srt": + return Subtitle.Codec.SubRip + elif mime == "ssa": + return Subtitle.Codec.SubStationAlpha + elif mime == "ass": + return Subtitle.Codec.SubStationAlphav4 + elif mime == "ttml": + return Subtitle.Codec.TimedTextMarkupLang + elif mime == "vtt": + return Subtitle.Codec.WebVTT + elif mime in ("smi", "sami"): + return Subtitle.Codec.SAMI + elif mime in ("sub", "microdvd"): + return Subtitle.Codec.MicroDVD + elif mime == "mpl2": + return Subtitle.Codec.MPL2 + elif mime == "tmp": + return Subtitle.Codec.TMP + elif mime == "stpp": + return Subtitle.Codec.fTTML + elif mime == "wvtt": + return Subtitle.Codec.fVTT + raise ValueError(f"The MIME '{mime}' is not a supported Subtitle Codec") + + @staticmethod + def from_codecs(codecs: str) -> Subtitle.Codec: + for codec in codecs.lower().split(","): + mime = codec.strip().split(".")[0] + try: + return Subtitle.Codec.from_mime(mime) + except ValueError: + pass + raise ValueError(f"No MIME types matched any supported Subtitle Codecs in '{codecs}'") + + @staticmethod + def from_netflix_profile(profile: str) -> Subtitle.Codec: + profile = profile.lower().strip() + if profile.startswith("webvtt"): + return Subtitle.Codec.WebVTT + if profile.startswith("dfxp"): + return Subtitle.Codec.TimedTextMarkupLang + raise ValueError(f"The Content Profile '{profile}' is not a supported Subtitle Codec") + + # WebVTT sanitization patterns (compiled once for performance) + _CUE_ID_PATTERN = re.compile(r"^[A-Za-z]+\d+$") + _TIMING_START_PATTERN = re.compile(r"^\d+:\d+[:\.]") + _TIMING_LINE_PATTERN = re.compile(r"^((?:\d+:)?\d+:\d+[.,]\d+)\s*-->\s*((?:\d+:)?\d+:\d+[.,]\d+)(.*)$") + _LINE_POS_PATTERN = re.compile(r"line:(\d+(?:\.\d+)?%?)") + + def __init__( + self, + *args: Any, + codec: Optional[Subtitle.Codec] = None, + cc: bool = False, + sdh: bool = False, + forced: bool = False, + **kwargs: Any, + ): + """ + Create a new Subtitle track object. + + Parameters: + codec: A Subtitle.Codec enum representing the subtitle format. + If not specified, MediaInfo will be used to retrieve the format + once the track has been downloaded. + cc: Closed Caption. + - Intended as if you couldn't hear the audio at all. + - Can have Sound as well as Dialogue, but doesn't have to. + - Original source would be from an EIA-CC encoded stream. Typically all + upper-case characters. + Indicators of it being CC without knowing original source: + - Extracted with CCExtractor, or + - >>> (or similar) being used at the start of some or all lines, or + - All text is uppercase or at least the majority, or + - Subtitles are Scrolling-text style (one line appears, oldest line + then disappears). + Just because you downloaded it as a SRT or VTT or such, doesn't mean it + isn't from an EIA-CC stream. And I wouldn't take the streaming services + (CC) as gospel either as they tend to get it wrong too. + sdh: Deaf or Hard-of-Hearing. Also known as HOH in the UK (EU?). + - Intended as if you couldn't hear the audio at all. + - MUST have Sound as well as Dialogue to be considered SDH. + - It has no "syntax" or "format" but is not transmitted using archaic + forms like EIA-CC streams, would be intended for transmission via + SubRip (SRT), WebVTT (VTT), TTML, etc. + If you can see important audio/sound transcriptions and not just dialogue + and it doesn't have the indicators of CC, then it's most likely SDH. + If it doesn't have important audio/sounds transcriptions it might just be + regular subtitling (you wouldn't mark as CC or SDH). This would be the + case for most translation subtitles. Like Anime for example. + forced: Typically used if there's important information at some point in time + like watching Dubbed content and an important Sign or Letter is shown + or someone talking in a different language. + Forced tracks are recommended by the Matroska Spec to be played if + the player's current playback audio language matches a subtitle + marked as "forced". + However, that doesn't mean every player works like this but there is + no other way to reliably work with Forced subtitles where multiple + forced subtitles may be in the output file. Just know what to expect + with "forced" subtitles. + + Note: If codec is not specified some checks may be skipped or assume a value. + Specifying as much information as possible is highly recommended. + + Information on Subtitle Types: + https://bit.ly/2Oe4fLC (3PlayMedia Blog on SUB vs CC vs SDH). + However, I wouldn't pay much attention to the claims about SDH needing to + be in the original source language. It's logically not true. + + CC == Closed Captions. Source: Basically every site. + SDH = Subtitles for the Deaf or Hard-of-Hearing. Source: Basically every site. + HOH = Exact same as SDH. Is a term used in the UK. Source: https://bit.ly/2PGJatz (ICO UK) + + More in-depth information, examples, and stuff to look for can be found in the Parameter + explanation list above. + """ + super().__init__(*args, **kwargs) + + if not isinstance(codec, (Subtitle.Codec, type(None))): + raise TypeError(f"Expected codec to be a {Subtitle.Codec}, not {codec!r}") + if not isinstance(cc, (bool, int)) or (isinstance(cc, int) and cc not in (0, 1)): + raise TypeError(f"Expected cc to be a {bool} or bool-like {int}, not {cc!r}") + if not isinstance(sdh, (bool, int)) or (isinstance(sdh, int) and sdh not in (0, 1)): + raise TypeError(f"Expected sdh to be a {bool} or bool-like {int}, not {sdh!r}") + if not isinstance(forced, (bool, int)) or (isinstance(forced, int) and forced not in (0, 1)): + raise TypeError(f"Expected forced to be a {bool} or bool-like {int}, not {forced!r}") + + self.codec = codec + + self.cc = bool(cc) + self.sdh = bool(sdh) + self.forced = bool(forced) + + if self.cc and self.sdh: + raise ValueError("A text track cannot be both CC and SDH.") + + if self.forced and (self.cc or self.sdh): + raise ValueError("A text track cannot be CC/SDH as well as Forced.") + + # TODO: Migrate to new event observer system + # Called after Track has been converted to another format + self.OnConverted: Optional[Callable[[Subtitle.Codec], None]] = None + + def to_dict(self) -> dict[str, Any]: + data = super().to_dict() + data.update( + { + "codec": self.codec.name if self.codec else None, + "cc": self.cc, + "sdh": self.sdh, + "forced": self.forced, + } + ) + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> Subtitle: + kwargs = Track.base_kwargs_from_dict(data) + return cls( + **kwargs, + codec=Subtitle.Codec[data["codec"]] if data.get("codec") else None, + cc=data.get("cc", False), + sdh=data.get("sdh", False), + forced=data.get("forced", False), + ) + + def __str__(self) -> str: + return " | ".join( + filter( + bool, + ["SUB", f"[{self.codec.value}]" if self.codec else None, str(self.language), self.get_track_name()], + ) + ) + + def get_track_name(self) -> Optional[str]: + """Return the base Track Name.""" + track_name = super().get_track_name() or "" + flag = self.cc and "CC" or self.sdh and "SDH" or self.forced and "Forced" + if flag: + if track_name: + flag = f" ({flag})" + track_name += flag + return track_name or None + + def download( + self, + session: requests.Session, + prepare_drm: partial, + max_workers: Optional[int] = None, + progress: Optional[partial] = None, + *, + cdm: Optional[object] = None, + no_proxy_download: bool = False, + ): + super().download(session, prepare_drm, max_workers, progress, cdm=cdm, no_proxy_download=no_proxy_download) + if not self.path: + return + + if self.codec == Subtitle.Codec.fTTML: + self.convert(Subtitle.Codec.TimedTextMarkupLang) + elif self.codec == Subtitle.Codec.fVTT: + self.convert(Subtitle.Codec.WebVTT) + elif self.codec == Subtitle.Codec.WebVTT: + text = self.path.read_text("utf8") + if self.descriptor == Track.Descriptor.DASH: + if len(self.data["dash"]["segment_durations"]) > 1: + text = merge_segmented_webvtt( + text, + segment_durations=self.data["dash"]["segment_durations"], + timescale=self.data["dash"]["timescale"], + ) + elif self.descriptor == Track.Descriptor.HLS: + if len(self.data["hls"]["segment_durations"]) > 1: + text = merge_segmented_webvtt( + text, + segment_durations=self.data["hls"]["segment_durations"], + timescale=1, # ? + ) + + # Sanitize WebVTT timestamps before parsing + text = Subtitle.sanitize_webvtt_timestamps(text) + # Remove cue identifiers that confuse parsers like pysubs2 + text = Subtitle.sanitize_webvtt_cue_identifiers(text) + # Merge overlapping cues with line positioning into single multi-line cues + text = Subtitle.merge_overlapping_webvtt_cues(text) + + preserve_formatting = config.subtitle.get("preserve_formatting", True) + + if preserve_formatting: + self.path.write_text(text, encoding="utf8") + else: + try: + caption_set = pycaption.WebVTTReader().read(text) + Subtitle.merge_same_cues(caption_set) + Subtitle.filter_unwanted_cues(caption_set) + subtitle_text = pycaption.WebVTTWriter().write(caption_set) + self.path.write_text(subtitle_text, encoding="utf8") + except pycaption.exceptions.CaptionReadSyntaxError: + # If first attempt fails, try more aggressive sanitization + text = Subtitle.sanitize_webvtt(text) + try: + caption_set = pycaption.WebVTTReader().read(text) + Subtitle.merge_same_cues(caption_set) + Subtitle.filter_unwanted_cues(caption_set) + subtitle_text = pycaption.WebVTTWriter().write(caption_set) + self.path.write_text(subtitle_text, encoding="utf8") + except Exception: + # Keep the sanitized version even if parsing failed + self.path.write_text(text, encoding="utf8") + + @staticmethod + def sanitize_webvtt_timestamps(text: str) -> str: + """ + Fix invalid timestamps in WebVTT files, particularly negative timestamps. + + Parameters: + text: The WebVTT content as string + + Returns: + Sanitized WebVTT content + """ + # Replace negative timestamps with 00:00:00.000 + return re.sub(r"(-\d+:\d+:\d+\.\d+)", "00:00:00.000", text) + + @staticmethod + def has_webvtt_cue_identifiers(text: str) -> bool: + """ + Check if WebVTT content has cue identifiers that need removal. + + Parameters: + text: The WebVTT content as string + + Returns: + True if cue identifiers are detected, False otherwise + """ + lines = text.split("\n") + + for i, line in enumerate(lines): + line = line.strip() + if Subtitle._CUE_ID_PATTERN.match(line): + # Look ahead to see if next non-empty line is a timing line + j = i + 1 + while j < len(lines) and not lines[j].strip(): + j += 1 + if j < len(lines) and ("-->" in lines[j] or Subtitle._TIMING_START_PATTERN.match(lines[j].strip())): + return True + return False + + @staticmethod + def sanitize_webvtt_cue_identifiers(text: str) -> str: + """ + Remove WebVTT cue identifiers that can confuse subtitle parsers. + + Some services use cue identifiers like "Q0", "Q1", etc. + that appear on their own line before the timing line. These can be + incorrectly parsed as part of the previous cue's text content by + some parsers (like pysubs2). + + Parameters: + text: The WebVTT content as string + + Returns: + Sanitized WebVTT content with cue identifiers removed + """ + if not Subtitle.has_webvtt_cue_identifiers(text): + return text + + lines = text.split("\n") + sanitized_lines = [] + + i = 0 + while i < len(lines): + line = lines[i].strip() + + # Check if this line is a cue identifier followed by a timing line + if Subtitle._CUE_ID_PATTERN.match(line): + # Look ahead to see if next non-empty line is a timing line + j = i + 1 + while j < len(lines) and not lines[j].strip(): + j += 1 + if j < len(lines) and ("-->" in lines[j] or Subtitle._TIMING_START_PATTERN.match(lines[j].strip())): + # This is a cue identifier, skip it + i += 1 + continue + + sanitized_lines.append(lines[i]) + i += 1 + + return "\n".join(sanitized_lines) + + @staticmethod + def _parse_vtt_time(t: str) -> int: + """Parse WebVTT timestamp to milliseconds. Returns 0 for malformed input.""" + try: + t = t.replace(",", ".") + parts = t.split(":") + if len(parts) == 2: + m, s = parts + h = "0" + elif len(parts) >= 3: + h, m, s = parts[:3] + else: + return 0 + sec_parts = s.split(".") + secs = int(sec_parts[0]) + # Handle variable millisecond digits (e.g., .5 = 500ms, .50 = 500ms, .500 = 500ms) + ms = int(sec_parts[1].ljust(3, "0")[:3]) if len(sec_parts) > 1 else 0 + return int(h) * 3600000 + int(m) * 60000 + secs * 1000 + ms + except (ValueError, IndexError): + return 0 + + @staticmethod + def has_overlapping_webvtt_cues(text: str) -> bool: + """ + Check if WebVTT content has overlapping cues that need merging. + + Detects cues with start times within 50ms of each other and the same end time, + which indicates multi-line subtitles split into separate cues. + + Parameters: + text: The WebVTT content as string + + Returns: + True if overlapping cues are detected, False otherwise + """ + timings = [] + for line in text.split("\n"): + match = Subtitle._TIMING_LINE_PATTERN.match(line) + if match: + start_str, end_str = match.group(1), match.group(2) + timings.append((Subtitle._parse_vtt_time(start_str), Subtitle._parse_vtt_time(end_str))) + + # Check for overlapping cues (within 50ms start, same end) + for i in range(len(timings) - 1): + curr_start, curr_end = timings[i] + next_start, next_end = timings[i + 1] + if abs(curr_start - next_start) <= 50 and curr_end == next_end: + return True + + return False + + @staticmethod + def merge_overlapping_webvtt_cues(text: str) -> str: + """ + Merge WebVTT cues that have overlapping/near-identical times but different line positions. + + Some services use separate cues for each line of a multi-line subtitle, with + slightly different start times (1ms apart) and different line: positions. + This merges them into single cues with proper line ordering based on the + line: position (lower percentage = higher on screen = first line). + + Parameters: + text: The WebVTT content as string + + Returns: + WebVTT content with overlapping cues merged + """ + if not Subtitle.has_overlapping_webvtt_cues(text): + return text + + lines = text.split("\n") + cues = [] + header_lines = [] + in_header = True + i = 0 + + while i < len(lines): + line = lines[i] + + if in_header: + if "-->" in line: + in_header = False + else: + header_lines.append(line) + i += 1 + continue + + match = Subtitle._TIMING_LINE_PATTERN.match(line) + if match: + start_str, end_str, settings = match.groups() + line_pos = 100.0 # Default to bottom + line_match = Subtitle._LINE_POS_PATTERN.search(settings) + if line_match: + pos_str = line_match.group(1).rstrip("%") + line_pos = float(pos_str) + + content_lines = [] + i += 1 + while i < len(lines) and lines[i].strip() and "-->" not in lines[i]: + content_lines.append(lines[i]) + i += 1 + + cues.append( + { + "start_ms": Subtitle._parse_vtt_time(start_str), + "end_ms": Subtitle._parse_vtt_time(end_str), + "start_str": start_str, + "end_str": end_str, + "line_pos": line_pos, + "content": "\n".join(content_lines), + "settings": settings, + } + ) + else: + i += 1 + + # Merge overlapping cues (within 50ms of each other with same end time) + merged_cues = [] + i = 0 + while i < len(cues): + current = cues[i] + group = [current] + + j = i + 1 + while j < len(cues): + other = cues[j] + if abs(current["start_ms"] - other["start_ms"]) <= 50 and current["end_ms"] == other["end_ms"]: + group.append(other) + j += 1 + else: + break + + if len(group) > 1: + # Sort by line position (lower % = higher on screen = first) + group.sort(key=lambda x: x["line_pos"]) + # Use the earliest start time from the group + earliest = min(group, key=lambda x: x["start_ms"]) + merged_cues.append( + { + "start_str": earliest["start_str"], + "end_str": group[0]["end_str"], + "content": "\n".join(c["content"] for c in group), + "settings": "", + } + ) + else: + merged_cues.append( + { + "start_str": current["start_str"], + "end_str": current["end_str"], + "content": current["content"], + "settings": current["settings"], + } + ) + + i = j if len(group) > 1 else i + 1 + + result_lines = header_lines[:] + if result_lines and result_lines[-1].strip(): + result_lines.append("") + + for cue in merged_cues: + result_lines.append(f"{cue['start_str']} --> {cue['end_str']}{cue['settings']}") + result_lines.append(cue["content"]) + result_lines.append("") + + return "\n".join(result_lines) + + @staticmethod + def sanitize_webvtt(text: str) -> str: + """ + More thorough sanitization of WebVTT files to handle multiple potential issues. + + Parameters: + text: The WebVTT content as string + + Returns: + Sanitized WebVTT content + """ + # Make sure we have a proper WEBVTT header + if not text.strip().startswith("WEBVTT"): + text = "WEBVTT\n\n" + text + + lines = text.split("\n") + sanitized_lines = [] + timestamp_pattern = re.compile(r"^((?:\d+:)?\d+:\d+\.\d+)\s+-->\s+((?:\d+:)?\d+:\d+\.\d+)") + + # Skip invalid headers - keep only WEBVTT + header_done = False + for line in lines: + if not header_done: + if line.startswith("WEBVTT"): + sanitized_lines.append("WEBVTT") + header_done = True + continue + + # Replace negative timestamps + if "-" in line and "-->" in line: + line = re.sub(r"(-\d+:\d+:\d+\.\d+)", "00:00:00.000", line) + + # Validate timestamp format + match = timestamp_pattern.match(line) + if match: + start_time = match.group(1) + end_time = match.group(2) + + # Ensure proper format with hours if missing + if start_time.count(":") == 1: + start_time = f"00:{start_time}" + if end_time.count(":") == 1: + end_time = f"00:{end_time}" + + line = f"{start_time} --> {end_time}" + + sanitized_lines.append(line) + + return "\n".join(sanitized_lines) + + def convert_with_subby(self, codec: Subtitle.Codec) -> Path: + """ + Convert subtitle using subby library for better format support and processing. + + This method leverages subby's advanced subtitle processing capabilities + including better WebVTT handling, SDH stripping, and common issue fixing. + """ + + if not self.path or not self.path.exists(): + raise ValueError("You must download the subtitle track first.") + + if self.codec == codec: + return self.path + + output_path = self.path.with_suffix(f".{codec.value.lower()}") + original_path = self.path + + try: + # Convert to SRT using subby first + srt_subtitles = None + + if self.codec == Subtitle.Codec.WebVTT: + converter = WebVTTConverter() + srt_subtitles = converter.from_file(self.path) + if self.codec == Subtitle.Codec.fVTT: + converter = WVTTConverter() + srt_subtitles = converter.from_file(self.path) + elif self.codec == Subtitle.Codec.SAMI: + converter = SAMIConverter() + srt_subtitles = converter.from_file(self.path) + + if srt_subtitles is not None: + # Apply common fixes + fixer = CommonIssuesFixer() + fixed_srt, _ = fixer.from_srt(srt_subtitles) + + # If target is SRT, we're done + if codec == Subtitle.Codec.SubRip: + fixed_srt.save(output_path, encoding="utf8") + else: + # Convert from SRT to target format using existing pycaption logic + temp_srt_path = self.path.with_suffix(".temp.srt") + fixed_srt.save(temp_srt_path, encoding="utf8") + + # Parse the SRT and convert to target format + caption_set = self.parse(temp_srt_path.read_bytes(), Subtitle.Codec.SubRip) + self.merge_same_cues(caption_set) + + writer = { + Subtitle.Codec.TimedTextMarkupLang: pycaption.DFXPWriter, + Subtitle.Codec.WebVTT: pycaption.WebVTTWriter, + }.get(codec) + + if writer: + subtitle_text = writer().write(caption_set) + output_path.write_text(subtitle_text, encoding="utf8") + else: + # Fall back to existing conversion method + temp_srt_path.unlink() + return self._convert_standard(codec) + + temp_srt_path.unlink() + + if original_path.exists() and original_path != output_path: + original_path.unlink() + + self.path = output_path + self.codec = codec + + if callable(self.OnConverted): + self.OnConverted(codec) + + return output_path + else: + # Fall back to existing conversion method + return self._convert_standard(codec) + + except Exception: + # Fall back to existing conversion method on any error + return self._convert_standard(codec) + + def convert_with_pysubs2(self, codec: Subtitle.Codec) -> Path: + """ + Convert subtitle using pysubs2 library for broad format support. + + pysubs2 is a pure-Python library supporting SubRip (SRT), SubStation Alpha + (SSA/ASS), WebVTT, TTML, SAMI, MicroDVD, MPL2, and TMP formats. + """ + if not self.path or not self.path.exists(): + raise ValueError("You must download the subtitle track first.") + + if self.codec == codec: + return self.path + + output_path = self.path.with_suffix(f".{codec.value.lower()}") + original_path = self.path + + codec_to_pysubs2_format = { + Subtitle.Codec.SubRip: "srt", + Subtitle.Codec.SubStationAlpha: "ssa", + Subtitle.Codec.SubStationAlphav4: "ass", + Subtitle.Codec.WebVTT: "vtt", + Subtitle.Codec.TimedTextMarkupLang: "ttml", + Subtitle.Codec.SAMI: "sami", + Subtitle.Codec.MicroDVD: "microdvd", + Subtitle.Codec.MPL2: "mpl2", + Subtitle.Codec.TMP: "tmp", + } + + pysubs2_output_format = codec_to_pysubs2_format.get(codec) + if pysubs2_output_format is None: + return self._convert_standard(codec) + + try: + subs = pysubs2.load(str(self.path), encoding="utf-8") + + subs.save(str(output_path), format_=pysubs2_output_format, encoding="utf-8") + + if original_path.exists() and original_path != output_path: + original_path.unlink() + + self.path = output_path + self.codec = codec + + if callable(self.OnConverted): + self.OnConverted(codec) + + return output_path + + except Exception: + return self._convert_standard(codec) + + def convert(self, codec: Subtitle.Codec) -> Path: + """ + Convert this Subtitle to another Format. + + The conversion method is determined by the 'conversion_method' setting in config: + - 'auto' (default): Uses subby for WebVTT/fVTT/SAMI; for SSA/ASS/MicroDVD/MPL2/TMP + uses SubtitleEdit if available, otherwise pysubs2; standard for others + - 'subby': Always uses subby with CommonIssuesFixer + - 'subtitleedit': Uses SubtitleEdit when available, falls back to pycaption + - 'pycaption': Uses only pycaption library + - 'pysubs2': Uses pysubs2 library + """ + # Check configuration for conversion method + conversion_method = config.subtitle.get("conversion_method", "auto") + + if conversion_method == "subby": + return self.convert_with_subby(codec) + elif conversion_method == "subtitleedit": + return self._convert_standard(codec) + elif conversion_method == "pycaption": + return self._convert_pycaption_only(codec) + elif conversion_method == "pysubs2": + return self.convert_with_pysubs2(codec) + elif conversion_method == "auto": + if self.codec in (Subtitle.Codec.WebVTT, Subtitle.Codec.fVTT, Subtitle.Codec.SAMI): + return self.convert_with_subby(codec) + elif self.codec in ( + Subtitle.Codec.SubStationAlpha, + Subtitle.Codec.SubStationAlphav4, + Subtitle.Codec.MicroDVD, + Subtitle.Codec.MPL2, + Subtitle.Codec.TMP, + ): + if binaries.SubtitleEdit: + return self._convert_standard(codec) + else: + return self.convert_with_pysubs2(codec) + else: + return self._convert_standard(codec) + else: + return self._convert_standard(codec) + + def _convert_pycaption_only(self, codec: Subtitle.Codec) -> Path: + """ + Convert subtitle using only pycaption library (no SubtitleEdit, no subby). + + This is the original conversion method that only uses pycaption. + """ + if not self.path or not self.path.exists(): + raise ValueError("You must download the subtitle track first.") + + if self.codec == codec: + return self.path + + output_path = self.path.with_suffix(f".{codec.value.lower()}") + original_path = self.path + + # Use only pycaption for conversion + writer = { + Subtitle.Codec.SubRip: pycaption.SRTWriter, + Subtitle.Codec.TimedTextMarkupLang: pycaption.DFXPWriter, + Subtitle.Codec.WebVTT: pycaption.WebVTTWriter, + }.get(codec) + + if writer is None: + raise NotImplementedError(f"Cannot convert {self.codec.name} to {codec.name} using pycaption only.") + + caption_set = self.parse(self.path.read_bytes(), self.codec) + Subtitle.merge_same_cues(caption_set) + if codec == Subtitle.Codec.WebVTT: + Subtitle.filter_unwanted_cues(caption_set) + subtitle_text = writer().write(caption_set) + + output_path.write_text(subtitle_text, encoding="utf8") + + if original_path.exists() and original_path != output_path: + original_path.unlink() + + self.path = output_path + self.codec = codec + + if callable(self.OnConverted): + self.OnConverted(codec) + + return output_path + + def _convert_standard(self, codec: Subtitle.Codec) -> Path: + """ + Convert this Subtitle to another Format. + + The file path location of the Subtitle data will be kept at the same + location but the file extension will be changed appropriately. + + Supported formats: + - SubRip - SubtitleEdit or pycaption.SRTWriter + - TimedTextMarkupLang - SubtitleEdit or pycaption.DFXPWriter + - WebVTT - SubtitleEdit or pycaption.WebVTTWriter + - SubStationAlphav4 - SubtitleEdit + - SAMI - subby.SAMIConverter (when available) + - fTTML* - custom code using some pycaption functions + - fVTT* - custom code using some pycaption functions + *: Can read from format, but cannot convert to format + + Note: It currently prioritizes using SubtitleEdit over PyCaption as + I have personally noticed more oddities with PyCaption parsing over + SubtitleEdit. Especially when working with TTML/DFXP where it would + often have timecodes and stuff mixed in/duplicated. + + Returns the new file path of the Subtitle. + """ + if not self.path or not self.path.exists(): + raise ValueError("You must download the subtitle track first.") + + if self.codec == codec: + return self.path + + output_path = self.path.with_suffix(f".{codec.value.lower()}") + original_path = self.path + + if binaries.SubtitleEdit and self.codec not in (Subtitle.Codec.fTTML, Subtitle.Codec.fVTT): + sub_edit_format = { + Subtitle.Codec.SubRip: "subrip", + Subtitle.Codec.SubStationAlpha: "substationalpha", + Subtitle.Codec.SubStationAlphav4: "advancedsubstationalpha", + Subtitle.Codec.TimedTextMarkupLang: "timedtext1.0", + Subtitle.Codec.WebVTT: "webvtt", + Subtitle.Codec.SAMI: "sami", + Subtitle.Codec.MicroDVD: "microdvd", + }.get(codec, codec.name.lower()) + sub_edit_args = [ + str(binaries.SubtitleEdit), + "/convert", + str(self.path), + sub_edit_format, + f"/outputfilename:{output_path.name}", + "/encoding:utf8", + ] + if codec == Subtitle.Codec.SubRip: + sub_edit_args.append("/ConvertColorsToDialog") + subprocess.run(sub_edit_args, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + else: + writer = { + # pycaption generally only supports these subtitle formats + Subtitle.Codec.SubRip: pycaption.SRTWriter, + Subtitle.Codec.TimedTextMarkupLang: pycaption.DFXPWriter, + Subtitle.Codec.WebVTT: pycaption.WebVTTWriter, + }.get(codec) + if writer is None: + raise NotImplementedError(f"Cannot yet convert {self.codec.name} to {codec.name}.") + + caption_set = self.parse(self.path.read_bytes(), self.codec) + Subtitle.merge_same_cues(caption_set) + if codec == Subtitle.Codec.WebVTT: + Subtitle.filter_unwanted_cues(caption_set) + subtitle_text = writer().write(caption_set) + + output_path.write_text(subtitle_text, encoding="utf8") + + if original_path.exists() and original_path != output_path: + original_path.unlink() + + self.path = output_path + self.codec = codec + + if callable(self.OnConverted): + self.OnConverted(codec) + + return output_path + + @staticmethod + def parse(data: bytes, codec: Subtitle.Codec) -> pycaption.CaptionSet: + if not isinstance(data, bytes): + raise ValueError(f"Subtitle data must be parsed as bytes data, not {type(data).__name__}") + + try: + if codec == Subtitle.Codec.SubRip: + text = try_ensure_utf8(data).decode("utf8") + caption_set = pycaption.SRTReader().read(text) + elif codec == Subtitle.Codec.fTTML: + caption_lists: dict[str, pycaption.CaptionList] = defaultdict(pycaption.CaptionList) + for segment in ( + Subtitle.parse(box.data, Subtitle.Codec.TimedTextMarkupLang) + for box in MP4.parse_stream(BytesIO(data)) + if box.type == b"mdat" + ): + for lang in segment.get_languages(): + caption_lists[lang].extend(segment.get_captions(lang)) + caption_set: pycaption.CaptionSet = pycaption.CaptionSet(caption_lists) + elif codec == Subtitle.Codec.TimedTextMarkupLang: + text = try_ensure_utf8(data).decode("utf8") + text = text.replace("tt:", "") + # negative size values aren't allowed in TTML/DFXP spec, replace with 0 + text = re.sub(r"-(\d+(?:\.\d+)?)(px|em|%|c|pt)", r"0\2", text) + caption_set = pycaption.DFXPReader().read(text) + elif codec == Subtitle.Codec.fVTT: + caption_lists: dict[str, pycaption.CaptionList] = defaultdict(pycaption.CaptionList) + caption_list, language = Subtitle.merge_segmented_wvtt(data) + caption_lists[language] = caption_list + caption_set: pycaption.CaptionSet = pycaption.CaptionSet(caption_lists) + elif codec == Subtitle.Codec.WebVTT: + text = try_ensure_utf8(data).decode("utf8") + text = Subtitle.sanitize_broken_webvtt(text) + text = Subtitle.space_webvtt_headers(text) + caption_set = pycaption.WebVTTReader().read(text) + elif codec == Subtitle.Codec.SAMI: + # Use subby for SAMI parsing + converter = SAMIConverter() + srt_subtitles = converter.from_bytes(data) + # Convert SRT back to CaptionSet for compatibility + srt_text = str(srt_subtitles).encode("utf8") + caption_set = Subtitle.parse(srt_text, Subtitle.Codec.SubRip) + else: + raise ValueError(f'Unknown Subtitle format "{codec}"...') + except pycaption.exceptions.CaptionReadSyntaxError as e: + raise SyntaxError(f'A syntax error has occurred when reading the "{codec}" subtitle: {e}') + except pycaption.exceptions.CaptionReadNoCaptions: + return pycaption.CaptionSet({"en": []}) + + # remove empty caption lists or some code breaks, especially if it's the first list + for language in caption_set.get_languages(): + if not caption_set.get_captions(language): + # noinspection PyProtectedMember + del caption_set._captions[language] + + return caption_set + + @staticmethod + def sanitize_broken_webvtt(text: str) -> str: + """ + Remove or fix corrupted WebVTT lines, particularly those with invalid timestamps. + + Parameters: + text: The WebVTT content as string + + Returns: + Sanitized WebVTT content with corrupted lines removed + """ + lines = text.splitlines() + sanitized_lines = [] + + i = 0 + while i < len(lines): + # Skip empty lines + if not lines[i].strip(): + sanitized_lines.append(lines[i]) + i += 1 + continue + + # Check for timestamp lines + if "-->" in lines[i]: + # Validate timestamp format + timestamp_parts = lines[i].split("-->") + if len(timestamp_parts) != 2 or not timestamp_parts[1].strip() or timestamp_parts[1].strip() == "0": + # Skip this timestamp and its content until next timestamp or end + j = i + 1 + while j < len(lines) and "-->" not in lines[j] and lines[j].strip(): + j += 1 + i = j + continue + + # Add valid timestamp line + sanitized_lines.append(lines[i]) + else: + # Add non-timestamp line + sanitized_lines.append(lines[i]) + + i += 1 + + return "\n".join(sanitized_lines) + + @staticmethod + def space_webvtt_headers(data: Union[str, bytes]): + """ + Space out the WEBVTT Headers from Captions. + + Segmented VTT when merged may have the WEBVTT headers part of the next caption + as they were not separated far enough from the previous caption and ended up + being considered as caption text rather than the header for the next segment. + """ + if isinstance(data, bytes): + data = try_ensure_utf8(data).decode("utf8") + elif not isinstance(data, str): + raise ValueError(f"Expecting data to be a str, not {data!r}") + + text = ( + data.replace("WEBVTT", "\n\nWEBVTT").replace("\r", "").replace("\n\n\n", "\n \n\n").replace("\n\n<", "\n<") + ) + + return text + + @staticmethod + def merge_same_cues(caption_set: pycaption.CaptionSet): + """Merge captions with the same timecodes and text as one in-place.""" + for lang in caption_set.get_languages(): + captions = caption_set.get_captions(lang) + last_caption = None + concurrent_captions = pycaption.CaptionList() + merged_captions = pycaption.CaptionList() + for caption in captions: + if last_caption: + if (caption.start, caption.end) == (last_caption.start, last_caption.end): + if caption.get_text() != last_caption.get_text(): + concurrent_captions.append(caption) + last_caption = caption + continue + else: + merged_captions.append(pycaption.base.merge(concurrent_captions)) + concurrent_captions = [caption] + last_caption = caption + + if concurrent_captions: + merged_captions.append(pycaption.base.merge(concurrent_captions)) + if merged_captions: + caption_set.set_captions(lang, merged_captions) + + @staticmethod + def filter_unwanted_cues(caption_set: pycaption.CaptionSet): + """ + Filter out subtitle cues containing only   or whitespace. + """ + for lang in caption_set.get_languages(): + captions = caption_set.get_captions(lang) + filtered_captions = pycaption.CaptionList() + + for caption in captions: + text = caption.get_text().strip() + if not text or text == " " or all(c in " \t\n\r\xa0" for c in text.replace(" ", "\xa0")): + continue + + filtered_captions.append(caption) + + caption_set.set_captions(lang, filtered_captions) + + @staticmethod + def merge_segmented_wvtt(data: bytes, period_start: float = 0.0) -> tuple[CaptionList, Optional[str]]: + """ + Convert Segmented DASH WebVTT cues into a pycaption Caption List. + Also returns an ISO 639-2 alpha-3 language code if available. + + Code ported originally by xhlove to Python from shaka-player. + Has since been improved upon by rlaphoenix using pymp4 and + pycaption functions. + """ + captions = CaptionList() + + # init: + saw_wvtt_box = False + timescale = None + language = None + + # media: + # > tfhd + default_duration = None + # > tfdt + saw_tfdt_box = False + base_time = 0 + # > trun + saw_trun_box = False + samples = [] + + def flatten_boxes(box: Container) -> Iterable[Container]: + for child in box: + if hasattr(child, "children"): + yield from flatten_boxes(child.children) + del child["children"] + if hasattr(child, "entries"): + yield from flatten_boxes(child.entries) + del child["entries"] + # some boxes (mainly within 'entries') uses format not type + child["type"] = child.get("type") or child.get("format") + yield child + + for box in flatten_boxes(MP4.parse_stream(BytesIO(data))): + # init + if box.type == b"mdhd": + timescale = box.timescale + language = box.language + + if box.type == b"wvtt": + saw_wvtt_box = True + + # media + if box.type == b"styp": + # essentially the start of each segment + # media var resets + # > tfhd + default_duration = None + # > tfdt + saw_tfdt_box = False + base_time = 0 + # > trun + saw_trun_box = False + samples = [] + + if box.type == b"tfhd": + if box.flags.default_sample_duration_present: + default_duration = box.default_sample_duration + + if box.type == b"tfdt": + saw_tfdt_box = True + base_time = box.baseMediaDecodeTime + + if box.type == b"trun": + saw_trun_box = True + samples = box.sample_info + + if box.type == b"mdat": + if not timescale: + raise ValueError("Timescale was not found in the Segmented WebVTT.") + if not saw_wvtt_box: + raise ValueError("The WVTT box was not found in the Segmented WebVTT.") + if not saw_tfdt_box: + raise ValueError("The TFDT box was not found in the Segmented WebVTT.") + if not saw_trun_box: + raise ValueError("The TRUN box was not found in the Segmented WebVTT.") + + vttc_boxes = MP4.parse_stream(BytesIO(box.data)) + current_time = base_time + period_start + + for sample, vttc_box in zip(samples, vttc_boxes): + duration = sample.sample_duration or default_duration + if sample.sample_composition_time_offsets: + current_time += sample.sample_composition_time_offsets + + start_time = current_time + end_time = current_time + (duration or 0) + current_time = end_time + + if vttc_box.type == b"vtte": + # vtte is a vttc that's empty, skip + continue + + layout: Optional[Layout] = None + nodes: list[CaptionNode] = [] + + for cue_box in vttc_box.children: + if cue_box.type == b"vsid": + # this is a V(?) Source ID box, we don't care + continue + if cue_box.type == b"sttg": + layout = Layout(webvtt_positioning=cue_box.settings) + elif cue_box.type == b"payl": + nodes.extend( + [ + node + for line in cue_box.cue_text.split("\n") + for node in [ + CaptionNode.create_text(WebVTTReader()._decode(line)), + CaptionNode.create_break(), + ] + ] + ) + nodes.pop() + + if nodes: + caption = Caption( + start=start_time * timescale, # as microseconds + end=end_time * timescale, + nodes=nodes, + layout_info=layout, + ) + p_caption = captions[-1] if captions else None + if p_caption and caption.start == p_caption.end and str(caption.nodes) == str(p_caption.nodes): + # it's a duplicate, but lets take its end time + p_caption.end = caption.end + continue + captions.append(caption) + + return captions, language + + def strip_hearing_impaired(self) -> None: + """ + Strip captions for hearing impaired (SDH). + + The SDH stripping method is determined by the 'sdh_method' setting in config: + - 'auto' (default): Tries subby first, then SubtitleEdit, then filter-subs + - 'subby': Uses subby's SDHStripper + - 'subtitleedit': Uses SubtitleEdit when available + - 'filter-subs': Uses subtitle-filter library + """ + if not self.path or not self.path.exists(): + raise ValueError("You must download the subtitle track first.") + + # Check configuration for SDH stripping method + sdh_method = config.subtitle.get("sdh_method", "auto") + + if sdh_method == "subby" and self.codec == Subtitle.Codec.SubRip: + # Use subby's SDHStripper directly on the file + fixer = CommonIssuesFixer() + stripper = SDHStripper() + srt, _ = fixer.from_file(self.path) + stripped, status = stripper.from_srt(srt) + if status is True: + stripped.save(self.path) + return + elif sdh_method == "subtitleedit" and binaries.SubtitleEdit: + # Force use of SubtitleEdit + pass # Continue to SubtitleEdit section below + elif sdh_method == "filter-subs": + # Force use of filter-subs + sub = Subtitles(self.path) + try: + sub.filter(rm_fonts=True, rm_ast=True, rm_music=True, rm_effects=True, rm_names=True, rm_author=True) + except ValueError as e: + if "too many values to unpack" in str(e): + # Retry without name removal if the error is due to multiple colons in time references + # This can happen with lines like "at 10:00 and 2:00" + sub = Subtitles(self.path) + sub.filter( + rm_fonts=True, rm_ast=True, rm_music=True, rm_effects=True, rm_names=False, rm_author=True + ) + else: + raise + sub.save() + return + elif sdh_method == "auto": + # Try subby first for SRT files, then fall back + if self.codec == Subtitle.Codec.SubRip: + try: + fixer = CommonIssuesFixer() + stripper = SDHStripper() + srt, _ = fixer.from_file(self.path) + stripped, status = stripper.from_srt(srt) + if status is True: + stripped.save(self.path) + return + except Exception: + pass # Fall through to other methods + + conversion_method = config.subtitle.get("conversion_method", "auto") + use_subtitleedit = sdh_method == "subtitleedit" or ( + sdh_method == "auto" and conversion_method in ("auto", "subtitleedit") + ) + + if binaries.SubtitleEdit and use_subtitleedit: + output_format = { + Subtitle.Codec.SubRip: "subrip", + Subtitle.Codec.SubStationAlpha: "substationalpha", + Subtitle.Codec.SubStationAlphav4: "advancedsubstationalpha", + Subtitle.Codec.TimedTextMarkupLang: "timedtext1.0", + Subtitle.Codec.WebVTT: "webvtt", + Subtitle.Codec.SAMI: "sami", + Subtitle.Codec.MicroDVD: "microdvd", + }.get(self.codec, self.codec.name.lower()) + subprocess.run( + [ + str(binaries.SubtitleEdit), + "/convert", + str(self.path), + output_format, + "/encoding:utf8", + "/overwrite", + "/RemoveTextForHI", + ], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + else: + if config.subtitle.get("convert_before_strip", True) and self.codec != Subtitle.Codec.SubRip: + self.path = self.convert(Subtitle.Codec.SubRip) + self.codec = Subtitle.Codec.SubRip + + try: + sub = Subtitles(self.path) + try: + sub.filter( + rm_fonts=True, rm_ast=True, rm_music=True, rm_effects=True, rm_names=True, rm_author=True + ) + except ValueError as e: + if "too many values to unpack" in str(e): + # Retry without name removal if the error is due to multiple colons in time references + # This can happen with lines like "at 10:00 and 2:00" + sub = Subtitles(self.path) + sub.filter( + rm_fonts=True, rm_ast=True, rm_music=True, rm_effects=True, rm_names=False, rm_author=True + ) + else: + raise + sub.save() + except (IOError, OSError) as e: + if "is not valid subtitle file" in str(e): + self.log.warning(f"Failed to strip SDH from {self.path.name}: {e}") + self.log.warning("Continuing without SDH stripping for this subtitle") + else: + raise + + def reverse_rtl(self) -> None: + """ + Reverse RTL (Right to Left) Start/End on Captions. + This can be used to fix the positioning of sentence-ending characters. + """ + if not self.path or not self.path.exists(): + raise ValueError("You must download the subtitle track first.") + + if not binaries.SubtitleEdit: + raise EnvironmentError("SubtitleEdit executable not found...") + + output_format = { + Subtitle.Codec.SubRip: "subrip", + Subtitle.Codec.SubStationAlpha: "substationalpha", + Subtitle.Codec.SubStationAlphav4: "advancedsubstationalpha", + Subtitle.Codec.TimedTextMarkupLang: "timedtext1.0", + Subtitle.Codec.WebVTT: "webvtt", + Subtitle.Codec.SAMI: "sami", + Subtitle.Codec.MicroDVD: "microdvd", + }.get(self.codec, self.codec.name.lower()) + + subprocess.run( + [ + str(binaries.SubtitleEdit), + "/convert", + str(self.path), + output_format, + "/ReverseRtlStartEnd", + "/encoding:utf8", + "/overwrite", + ], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + +__all__ = ("Subtitle",) diff --git a/reset/packages/envied/src/envied/core/tracks/track.py b/reset/packages/envied/src/envied/core/tracks/track.py new file mode 100644 index 0000000..bcc285a --- /dev/null +++ b/reset/packages/envied/src/envied/core/tracks/track.py @@ -0,0 +1,782 @@ +import base64 +import html +import logging +import re +import shutil +import subprocess +from collections import defaultdict +from copy import copy +from enum import Enum +from functools import partial +from pathlib import Path +from typing import Any, Callable, Iterable, Optional, Union +from uuid import UUID +from zlib import crc32 + +from langcodes import Language +from requests import Session + +from envied.core import binaries +from envied.core.cdm.detect import is_playready_cdm, is_widevine_cdm +from envied.core.config import config +from envied.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY +from envied.core.downloaders import requests +from envied.core.drm import DRM_T, PlayReady, Widevine +from envied.core.events import events +from envied.core.session import RnetSession +from envied.core.utilities import get_boxes, try_ensure_utf8 +from envied.core.utils.subprocess import ffprobe + + +def direct_session(session: Union[Session, "RnetSession"]) -> Session: + """Vanilla requests.Session with copied headers/cookies, no proxy.""" + new = Session() + headers = getattr(session, "headers", None) + if headers is not None: + try: + new.headers.update(dict(headers)) + except Exception: + pass + cookies = getattr(session, "cookies", None) + if cookies is not None: + jar = getattr(cookies, "jar", None) + try: + new.cookies.update(jar if jar is not None else cookies) + except Exception: + pass + return new + + +class Track: + class Descriptor(Enum): + URL = 1 # Direct URL, nothing fancy + HLS = 2 # https://en.wikipedia.org/wiki/HTTP_Live_Streaming + DASH = 3 # https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP + ISM = 4 # https://learn.microsoft.com/en-us/silverlight/smooth-streaming + + def __init__( + self, + url: Union[str, list[str]], + language: Union[Language, str], + is_original_lang: bool = False, + descriptor: Descriptor = Descriptor.URL, + needs_repack: bool = False, + name: Optional[str] = None, + drm: Optional[Iterable[DRM_T]] = None, + edition: Optional[str] = None, + session: Optional[Union[Session, "RnetSession"]] = None, + downloader: Optional[Callable] = None, + downloader_args: Optional[dict] = None, + from_file: Optional[Path] = None, + data: Optional[Union[dict, defaultdict]] = None, + id_: Optional[str] = None, + extra: Optional[Any] = None, + ) -> None: + if not isinstance(url, (str, list)): + raise TypeError(f"Expected url to be a {str}, or list of {str}, not {type(url)}") + if not isinstance(language, (Language, str)): + raise TypeError(f"Expected language to be a {Language} or {str}, not {type(language)}") + if not isinstance(is_original_lang, bool): + raise TypeError(f"Expected is_original_lang to be a {bool}, not {type(is_original_lang)}") + if not isinstance(descriptor, Track.Descriptor): + raise TypeError(f"Expected descriptor to be a {Track.Descriptor}, not {type(descriptor)}") + if not isinstance(needs_repack, bool): + raise TypeError(f"Expected needs_repack to be a {bool}, not {type(needs_repack)}") + if not isinstance(name, (str, type(None))): + raise TypeError(f"Expected name to be a {str}, not {type(name)}") + if not isinstance(id_, (str, type(None))): + raise TypeError(f"Expected id_ to be a {str}, not {type(id_)}") + if not isinstance(edition, (str, list, type(None))): + raise TypeError(f"Expected edition to be a {str}, {list}, or None, not {type(edition)}") + if not isinstance(downloader, (Callable, type(None))): + raise TypeError(f"Expected downloader to be a {Callable}, not {type(downloader)}") + if not isinstance(downloader_args, (dict, type(None))): + raise TypeError(f"Expected downloader_args to be a {dict}, not {type(downloader_args)}") + if not isinstance(from_file, (Path, type(None))): + raise TypeError(f"Expected from_file to be a {Path}, not {type(from_file)}") + if not isinstance(data, (dict, defaultdict, type(None))): + raise TypeError(f"Expected data to be a {dict} or {defaultdict}, not {type(data)}") + + invalid_urls = ", ".join(set(type(x) for x in url if not isinstance(x, str))) + if invalid_urls: + raise TypeError(f"Expected all items in url to be a {str}, but found {invalid_urls}") + + if drm is not None: + try: + iter(drm) + except TypeError: + raise TypeError(f"Expected drm to be an iterable, not {type(drm)}") + + if downloader is None: + downloader = requests + + self.path: Optional[Path] = None + self.url = url + self.language = Language.get(language) + self.is_original_lang = is_original_lang + self.descriptor = descriptor + self.needs_repack = needs_repack + self.name = name + self.drm = drm + self.edition: list[str] = [edition] if isinstance(edition, str) else (edition or []) + self.session = session + self.downloader = downloader + self.downloader_args = downloader_args + self.from_file = from_file + self._data: defaultdict[Any, Any] = defaultdict(dict) + self.data = data or {} + self.extra: Any = extra or {} # allow anything for extra, but default to a dict + + if self.name is None: + lang = Language.get(self.language) + if (lang.language or "").lower() == (lang.territory or "").lower(): + lang.territory = None # e.g. en-en, de-DE + reduced = lang.simplify_script() + extra_parts = [] + if reduced.script is not None: + script = reduced.script_name(max_distance=25) + if script and script != "Zzzz": + extra_parts.append(script) + if reduced.territory is not None: + territory = reduced.territory_name(max_distance=25) + if territory and territory != "ZZ": + territory = territory.removesuffix(" SAR China") + extra_parts.append(territory) + self.name = ", ".join(extra_parts) or None + + if not id_: + this = copy(self) + this.url = self.url.rsplit("?", maxsplit=1)[0] + checksum = crc32(repr(this).encode("utf8")) + id_ = hex(checksum)[2:] + + self.id = id_ + + # TODO: Currently using OnFoo event naming, change to just segment_filter + self.OnSegmentFilter: Optional[Callable] = None + + def __repr__(self) -> str: + return "{name}({items})".format( + name=self.__class__.__name__, items=", ".join([f"{k}={repr(v)}" for k, v in self.__dict__.items()]) + ) + + def __eq__(self, other: Any) -> bool: + return isinstance(other, Track) and self.id == other.id + + @property + def data(self) -> defaultdict[Any, Any]: + """ + Arbitrary track data dictionary. + + A defaultdict is used with a dict as the factory for easier + nested saving and safer exists-checks. + + Reserved keys: + + - "hls" used by the HLS class. + - playlist: m3u8.model.Playlist - The primary track information. + - media: m3u8.model.Media - The audio/subtitle track information. + - segment_durations: list[int] - A list of each segment's duration. + - "dash" used by the DASH class. + - manifest: lxml.ElementTree - DASH MPD manifest. + - period: lxml.Element - The period of this track. + - adaptation_set: lxml.Element - The adaptation set of this track. + - representation: lxml.Element - The representation of this track. + - timescale: int - The timescale of the track's segments. + - segment_durations: list[int] - A list of each segment's duration. + + You should not add, change, or remove any data within reserved keys. + You may use their data but do note that the values of them may change + or be removed at any point. + """ + return self._data + + @data.setter + def data(self, value: Union[dict, defaultdict]) -> None: + if not isinstance(value, (dict, defaultdict)): + raise TypeError(f"Expected data to be a {dict} or {defaultdict}, not {type(value)}") + if isinstance(value, dict): + value = defaultdict(dict, **value) + self._data = value + + def download( + self, + session: Union[Session, "RnetSession"], + prepare_drm: partial, + max_workers: Optional[int] = None, + progress: Optional[partial] = None, + *, + cdm: Optional[object] = None, + no_proxy_download: bool = False, + ): + """Download and optionally Decrypt this Track.""" + from envied.core.manifests import DASH, HLS, ISM + + if DOWNLOAD_LICENCE_ONLY.is_set(): + progress(downloaded="[yellow]SKIPPING") + + if DOWNLOAD_CANCELLED.is_set(): + progress(downloaded="[yellow]SKIPPED") + return + + log = logging.getLogger("track") + + proxy = next(iter(session.proxies.values()), None) + + dl_session = session + if no_proxy_download and proxy: + dl_session = direct_session(session) + proxy = None + + track_type = self.__class__.__name__ + save_path = config.directories.temp / f"{track_type}_{self.id}.mp4" + if track_type == "Subtitle": + save_path = save_path.with_suffix(f".{self.codec.extension}") + + if self.descriptor != self.Descriptor.URL: + save_dir = save_path.with_name(save_path.name + "_segments") + else: + save_dir = save_path.parent + + def cleanup(): + save_path.unlink(missing_ok=True) + if save_dir.exists() and save_dir.name.endswith("_segments"): + shutil.rmtree(save_dir) + + if not DOWNLOAD_LICENCE_ONLY.is_set(): + if config.directories.temp.is_file(): + raise ValueError(f"Temp Directory '{config.directories.temp}' must be a Directory, not a file") + + config.directories.temp.mkdir(parents=True, exist_ok=True) + + # Delete any pre-existing temp files matching this track. + # We can't re-use or continue downloading these tracks as they do not use a + # lock file. Or at least the majority don't. Even if they did I've encountered + # corruptions caused by sudden interruptions to the lock file. + cleanup() + + try: + if self.descriptor == self.Descriptor.HLS: + HLS.download_track( + track=self, + save_path=save_path, + save_dir=save_dir, + progress=progress, + session=dl_session, + proxy=proxy, + max_workers=max_workers, + license_widevine=prepare_drm, + cdm=cdm, + ) + elif self.descriptor == self.Descriptor.DASH: + DASH.download_track( + track=self, + save_path=save_path, + save_dir=save_dir, + progress=progress, + session=dl_session, + proxy=proxy, + max_workers=max_workers, + license_widevine=prepare_drm, + cdm=cdm, + ) + elif self.descriptor == self.Descriptor.ISM: + ISM.download_track( + track=self, + save_path=save_path, + save_dir=save_dir, + progress=progress, + session=dl_session, + proxy=proxy, + max_workers=max_workers, + license_widevine=prepare_drm, + cdm=cdm, + ) + elif self.descriptor == self.Descriptor.URL: + try: + if not self.drm and track_type in ("Video", "Audio"): + # the service might not have explicitly defined the `drm` property + # try find DRM information from the init data of URL based on CDM type + if is_playready_cdm(cdm): + try: + self.drm = [PlayReady.from_track(self, session)] + except PlayReady.Exceptions.PSSHNotFound: + try: + self.drm = [Widevine.from_track(self, session)] + except Widevine.Exceptions.PSSHNotFound: + log.warning( + "No PlayReady or Widevine PSSH was found for this track, is it DRM free?" + ) + else: + try: + self.drm = [Widevine.from_track(self, session)] + except Widevine.Exceptions.PSSHNotFound: + try: + self.drm = [PlayReady.from_track(self, session)] + except PlayReady.Exceptions.PSSHNotFound: + log.warning( + "No Widevine or PlayReady PSSH was found for this track, is it DRM free?" + ) + + if self.drm: + track_kid = self.get_key_id(session=session) + drm = self.get_drm_for_cdm(cdm) + if isinstance(drm, Widevine): + # license and grab content keys + if not prepare_drm: + raise ValueError("prepare_drm func must be supplied to use Widevine DRM") + progress(downloaded="LICENSING") + prepare_drm(drm, track_kid=track_kid) + progress(downloaded="[yellow]LICENSED") + elif isinstance(drm, PlayReady): + # license and grab content keys + if not prepare_drm: + raise ValueError("prepare_drm func must be supplied to use PlayReady DRM") + progress(downloaded="LICENSING") + prepare_drm(drm, track_kid=track_kid) + progress(downloaded="[yellow]LICENSED") + else: + drm = None + + if DOWNLOAD_LICENCE_ONLY.is_set(): + progress(downloaded="[yellow]SKIPPED") + else: + for status_update in self.downloader( + urls=self.url, + output_dir=save_path.parent, + filename=save_path.name, + headers=dl_session.headers, + cookies=dl_session.cookies, + proxy=proxy, + max_workers=max_workers, + session=dl_session, + ): + file_downloaded = status_update.get("file_downloaded") + if not file_downloaded: + downloaded = status_update.get("downloaded") + if downloaded and downloaded.endswith("/s"): + status_update["downloaded"] = f"URL {downloaded}" + progress(**status_update) + + self.path = save_path + events.emit(events.Types.TRACK_DOWNLOADED, track=self) + + if drm: + progress(downloaded="Decrypting", completed=0, total=100) + drm.decrypt(save_path) + self.drm = None + events.emit(events.Types.TRACK_DECRYPTED, track=self, drm=drm, segment=None) + progress(downloaded="Decrypted", completed=100) + + if track_type == "Subtitle" and self.codec.name not in ("fVTT", "fTTML"): + track_data = self.path.read_bytes() + track_data = try_ensure_utf8(track_data) + track_data = ( + track_data.decode("utf8") + .replace("‎", html.unescape("‎")) + .replace("‏", html.unescape("‏")) + .encode("utf8") + ) + self.path.write_bytes(track_data) + + progress(downloaded="Downloaded") + except KeyboardInterrupt: + DOWNLOAD_CANCELLED.set() + progress(downloaded="[yellow]CANCELLED") + raise + except Exception: + DOWNLOAD_CANCELLED.set() + progress(downloaded="[red]FAILED") + raise + except (Exception, KeyboardInterrupt): + if not DOWNLOAD_LICENCE_ONLY.is_set(): + cleanup() + raise + + if DOWNLOAD_CANCELLED.is_set(): + # we stopped during the download, let's exit + return + + if not DOWNLOAD_LICENCE_ONLY.is_set(): + if self.path.stat().st_size <= 3: # Empty UTF-8 BOM == 3 bytes + raise IOError("Download failed, the downloaded file is empty.") + + events.emit(events.Types.TRACK_DOWNLOADED, track=self) + + def delete(self) -> None: + if self.path: + self.path.unlink() + self.path = None + + def move(self, target: Union[Path, str]) -> Path: + """ + Move the Track's file from current location, to target location. + This will overwrite anything at the target path. + + Raises: + TypeError: If the target argument is not the expected type. + ValueError: If track has no file to move, or the target does not exist. + OSError: If the file somehow failed to move. + + Returns the new location of the track. + """ + if not isinstance(target, (str, Path)): + raise TypeError(f"Expected {target} to be a {Path} or {str}, not {type(target)}") + + if not self.path: + raise ValueError("Track has no file to move") + + if not isinstance(target, Path): + target = Path(target) + + if not target.exists(): + raise ValueError(f"Target file {repr(target)} does not exist") + + moved_to = Path(shutil.move(self.path, target)) + if moved_to.resolve() != target.resolve(): + raise OSError(f"Failed to move {self.path} to {target}") + + self.path = target + return target + + def to_dict(self) -> dict[str, Any]: + """Serialise the track for export/import (identity/URL/descriptor/language). + + DRM is not serialised here; the export writer attaches the licensed DRM + keys. + Subclasses add their own codec/quality fields. + """ + data: dict[str, Any] = { + "type": self.__class__.__name__, + "id": self.id, + "url": self.url, + "language": str(self.language), + "is_original_lang": self.is_original_lang, + "descriptor": self.descriptor.name, + "needs_repack": self.needs_repack, + "name": self.name, + "edition": self.edition, + } + return data + + @staticmethod + def base_kwargs_from_dict(data: dict[str, Any]) -> dict[str, Any]: + """Build the shared Track constructor kwargs from a ``to_dict()`` payload. + + DRM is not reconstructed here — ``to_dict`` does not serialise it, and the import + flow attaches the licensed DRM + content keys separately. + """ + return { + "url": data["url"], + "language": data.get("language") or "und", + "is_original_lang": data.get("is_original_lang", False), + "descriptor": Track.Descriptor[data.get("descriptor", "URL")], + "needs_repack": data.get("needs_repack", False), + "name": data.get("name"), + "edition": data.get("edition") or None, + "id_": data.get("id"), + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "Track": + """Reconstruct the correct Track subclass from a ``to_dict()`` payload.""" + from envied.core.tracks.audio import Audio + from envied.core.tracks.subtitle import Subtitle + from envied.core.tracks.video import Video + + track_type = data.get("type") + builders = {"Video": Video, "Audio": Audio, "Subtitle": Subtitle} + builder = builders.get(track_type) + if builder is None: + raise ValueError(f"Cannot reconstruct unsupported track type: {track_type!r}") + return builder.from_dict(data) + + def get_track_name(self) -> Optional[str]: + """Get the Track Name.""" + return self.name + + def get_drm_for_cdm(self, cdm: Optional[object]) -> Optional[DRM_T]: + """Return the DRM matching the provided CDM, if available.""" + if not self.drm: + return None + + if is_widevine_cdm(cdm): + for drm in self.drm: + if isinstance(drm, Widevine): + return drm + elif is_playready_cdm(cdm): + for drm in self.drm: + if isinstance(drm, PlayReady): + return drm + + return self.drm[0] + + def get_key_id(self, init_data: Optional[bytes] = None, *args, **kwargs) -> Optional[UUID]: + """ + Probe the DRM encryption Key ID (KID) for this specific track. + + It currently supports finding the Key ID by probing the track's stream + with ffprobe for `enc_key_id` data, as well as for mp4 `tenc` (Track + Encryption) boxes. + + It explicitly ignores PSSH information like the `PSSH` box, as the box + is likely to contain multiple Key IDs that may or may not be for this + specific track. + + To retrieve the initialization segment, this method calls :meth:`get_init_segment` + with the positional and keyword arguments. The return value of `get_init_segment` + is then used to determine the Key ID. + + Returns: + The Key ID as a UUID object, or None if the Key ID could not be determined. + """ + if not init_data: + init_data = self.get_init_segment(*args, **kwargs) + if not isinstance(init_data, bytes): + raise TypeError(f"Expected init_data to be bytes, not {init_data!r}") + + probe = ffprobe(init_data) + if probe: + for stream in probe.get("streams") or []: + enc_key_id = stream.get("tags", {}).get("enc_key_id") + if enc_key_id: + return UUID(bytes=base64.b64decode(enc_key_id)) + + for tenc in get_boxes(init_data, b"tenc"): + if tenc.key_ID.int != 0: + return tenc.key_ID + + for uuid_box in get_boxes(init_data, b"uuid"): + if uuid_box.extended_type == UUID("8974dbce-7be7-4c51-84f9-7148f9882554"): # tenc + tenc = uuid_box.data + if tenc.key_ID.int != 0: + return tenc.key_ID + + def load_drm_if_needed(self, service=None) -> bool: + """ + Load DRM information for this track if it was deferred during parsing. + + Args: + service: Service instance that can fetch track-specific DRM info + + Returns: + True if DRM was loaded or already present, False if failed + """ + if not getattr(self, "needs_drm_loading", False): + return bool(self.drm) + + if self.drm: + self.needs_drm_loading = False + return True + + if not service or not hasattr(service, "get_track_drm"): + return self.load_drm_from_playlist() + + try: + track_drm = service.get_track_drm(self) + if track_drm: + self.drm = track_drm if isinstance(track_drm, list) else [track_drm] + self.needs_drm_loading = False + return True + except Exception as e: + raise ValueError(f"Failed to load DRM from service for track {self.id}: {e}") + + return self.load_drm_from_playlist() + + def load_drm_from_playlist(self) -> bool: + """ + Fallback method to load DRM by fetching this track's individual playlist. + """ + if self.drm: + self.needs_drm_loading = False + return True + + try: + import m3u8 + from pyplayready.system.pssh import PSSH as PR_PSSH + from pywidevine.cdm import Cdm as WidevineCdm + from pywidevine.pssh import PSSH as WV_PSSH + + session = getattr(self, "session", None) or Session() + + response = session.get(self.url) + playlist = m3u8.loads(response.text, self.url) + + drm_list = [] + + for key in playlist.keys or []: + if not key or not key.keyformat: + continue + + fmt = key.keyformat.lower() + if fmt == WidevineCdm.urn: + pssh_b64 = key.uri.split(",")[-1] + drm = Widevine(pssh=WV_PSSH(pssh_b64)) + drm_list.append(drm) + elif fmt in {f"urn:uuid:{PR_PSSH.SYSTEM_ID}", "com.microsoft.playready"}: + pssh_b64 = key.uri.split(",")[-1] + drm = PlayReady(pssh=PR_PSSH(pssh_b64), pssh_b64=pssh_b64) + drm_list.append(drm) + + if drm_list: + self.drm = drm_list + self.needs_drm_loading = False + return True + + except Exception as e: + raise ValueError(f"Failed to load DRM from playlist for track {self.id}: {e}") + + return False + + def get_init_segment( + self, + maximum_size: int = 20000, + url: Optional[str] = None, + byte_range: Optional[str] = None, + session: Optional[Session] = None, + ) -> bytes: + """ + Get the Track's Initial Segment Data Stream. + + HLS and DASH tracks must explicitly provide a URL to the init segment or file. + Providing the byte-range for the init segment is recommended where possible. + + If `byte_range` is not set, it will make a HEAD request and check the size of + the file. If the size could not be determined, it will download up to the first + 20KB only, which should contain the entirety of the init segment. You may + override this by changing the `maximum_size`. + + The default maximum_size of 20000 (20KB) is a tried-and-tested value that + seems to work well across the board. + + Parameters: + maximum_size: Size to assume as the content length if byte-range is not + used, the content size could not be determined, or the content size + is larger than it. A value of 20000 (20KB) or higher is recommended. + url: Explicit init map or file URL to probe from. + byte_range: Range of bytes to download from the explicit or implicit URL. + session: Session context, e.g., authorization and headers. + """ + if not isinstance(maximum_size, int): + raise TypeError(f"Expected maximum_size to be an {int}, not {type(maximum_size)}") + if not isinstance(url, (str, type(None))): + raise TypeError(f"Expected url to be a {str}, not {type(url)}") + if not isinstance(byte_range, (str, type(None))): + raise TypeError(f"Expected byte_range to be a {str}, not {type(byte_range)}") + if not isinstance(session, (Session, RnetSession, type(None))): + raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {type(session)}") + + if not url: + if self.descriptor != self.Descriptor.URL: + raise ValueError(f"An explicit URL must be provided for {self.descriptor.name} tracks") + if not self.url: + raise ValueError("An explicit URL must be provided as the track has no URL") + url = self.url + + if not session: + session = Session() + + content_length = maximum_size + + if byte_range: + if not isinstance(byte_range, str): + raise TypeError(f"Expected byte_range to be a str, not {byte_range!r}") + if not re.match(r"^\d+-\d+$", byte_range): + raise ValueError(f"The value of byte_range is unrecognized: '{byte_range}'") + start, end = byte_range.split("-") + if start > end: + raise ValueError(f"The start range cannot be greater than the end range: {start}>{end}") + else: + size_test = session.head(url) + if "Content-Length" in size_test.headers: + content_length_header = int(size_test.headers["Content-Length"]) + if content_length_header > 0: + content_length = min(content_length_header, maximum_size) + range_test = session.head(url, headers={"Range": "bytes=0-1"}) + if range_test.status_code == 206: + byte_range = f"0-{content_length - 1}" + + if byte_range: + res = session.get(url=url, headers={"Range": f"bytes={byte_range}"}) + res.raise_for_status() + init_data = res.content + else: + init_data = None + s = session.get(url, stream=True) + for chunk in s.iter_content(content_length): + init_data = chunk + break + s.close() + if not init_data: + raise ValueError(f"Failed to read {content_length} bytes from the track URI.") + + return init_data + + def repackage(self) -> None: + if not self.path or not self.path.exists(): + raise ValueError("Cannot repackage a Track that has not been downloaded.") + + if not binaries.FFMPEG: + raise EnvironmentError('FFmpeg executable "ffmpeg" was not found but is required for this call.') + + original_path = self.path + output_path = original_path.with_stem(f"{original_path.stem}_repack") + + def _ffmpeg(extra_args: list[str] = None): + args = [ + binaries.FFMPEG, + "-hide_banner", + "-loglevel", + "error", + "-i", + original_path, + *(extra_args or []), + ] + + if hasattr(self, "data") and self.data.get("audio_language"): + audio_lang = self.data["audio_language"] + audio_name = self.data.get("audio_language_name", audio_lang) + args.extend( + [ + "-metadata:s:a:0", + f"language={audio_lang}", + "-metadata:s:a:0", + f"title={audio_name}", + "-metadata:s:a:0", + f"handler_name={audio_name}", + ] + ) + + args.extend( + [ + # Following are very important! + "-map_metadata", + "-1", # don't transfer metadata to output file + "-fflags", + "bitexact", # only have minimal tag data, reproducible mux + "-codec", + "copy", + str(output_path), + ] + ) + + subprocess.run( + args, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + try: + _ffmpeg() + except subprocess.CalledProcessError as e: + if b"Malformed AAC bitstream detected" in e.stderr: + # e.g., TruTV's dodgy encodes + _ffmpeg(["-y", "-bsf:a", "aac_adtstoasc"]) + else: + raise + + original_path.unlink() + self.path = output_path + + +__all__ = ("Track",) diff --git a/reset/packages/envied/src/envied/core/tracks/tracks.py b/reset/packages/envied/src/envied/core/tracks/tracks.py new file mode 100644 index 0000000..bbc0003 --- /dev/null +++ b/reset/packages/envied/src/envied/core/tracks/tracks.py @@ -0,0 +1,741 @@ +from __future__ import annotations + +import logging +import subprocess +from functools import partial +from pathlib import Path +from typing import Callable, Iterator, Optional, Sequence, Union + +from langcodes import Language, closest_supported_match +from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn, TimeRemainingColumn +from rich.table import Table +from rich.tree import Tree + +from envied.core import binaries +from envied.core.config import config +from envied.core.console import console +from envied.core.constants import LANGUAGE_EXACT_DISTANCE, LANGUAGE_MAX_DISTANCE, AnyTrack, TrackT +from envied.core.events import events +from envied.core.tracks.attachment import Attachment +from envied.core.tracks.audio import Audio +from envied.core.tracks.chapters import Chapter, Chapters +from envied.core.tracks.subtitle import Subtitle +from envied.core.tracks.track import Track +from envied.core.tracks.video import Video +from envied.core.utilities import get_debug_logger, is_close_match, sanitize_filename +from envied.core.utils.collections import as_list, flatten + + +class Tracks: + """ + Video, Audio, Subtitle, Chapter, and Attachment Track Store. + It provides convenience functions for listing, sorting, and selecting tracks. + """ + + TRACK_ORDER_MAP = {Video: 0, Audio: 1, Subtitle: 2, Chapter: 3, Attachment: 4} + + def __init__( + self, + *args: Union[ + Tracks, Sequence[Union[AnyTrack, Chapter, Chapters, Attachment]], Track, Chapter, Chapters, Attachment + ], + manifest_url: Optional[str] = None, + ): + self.videos: list[Video] = [] + self.audio: list[Audio] = [] + self.subtitles: list[Subtitle] = [] + self.chapters = Chapters() + self.attachments: list[Attachment] = [] + self.manifest_url: Optional[str] = manifest_url + + if args: + self.add(args) + + def __iter__(self) -> Iterator[AnyTrack]: + return iter(as_list(self.videos, self.audio, self.subtitles)) + + def __len__(self) -> int: + return len(self.videos) + len(self.audio) + len(self.subtitles) + + def __add__( + self, + other: Union[ + Tracks, Sequence[Union[AnyTrack, Chapter, Chapters, Attachment]], Track, Chapter, Chapters, Attachment + ], + ) -> Tracks: + self.add(other) + return self + + def __repr__(self) -> str: + return "{name}({items})".format( + name=self.__class__.__name__, items=", ".join([f"{k}={repr(v)}" for k, v in self.__dict__.items()]) + ) + + def __str__(self) -> str: + rep = {Video: [], Audio: [], Subtitle: [], Chapter: [], Attachment: []} + tracks = [*list(self), *self.chapters] + + for track in sorted(tracks, key=lambda t: self.TRACK_ORDER_MAP[type(t)]): + if not rep[type(track)]: + count = sum(type(x) is type(track) for x in tracks) + rep[type(track)].append( + "{count} {type} Track{plural}{colon}".format( + count=count, + type=track.__class__.__name__, + plural="s" if count != 1 else "", + colon=":" if count > 0 else "", + ) + ) + rep[type(track)].append(str(track)) + + for type_ in list(rep): + if not rep[type_]: + del rep[type_] + continue + rep[type_] = "\n".join([rep[type_][0]] + [f"├─ {x}" for x in rep[type_][1:-1]] + [f"└─ {rep[type_][-1]}"]) + rep = "\n".join(list(rep.values())) + + return rep + + def tree(self, add_progress: bool = False) -> tuple[Tree, list[Callable[..., None]]]: + all_tracks = [*list(self), *self.chapters, *self.attachments] + + progress_callables = [] + + tree = Tree("", hide_root=True) + for track_type in self.TRACK_ORDER_MAP: + tracks = list(x for x in all_tracks if isinstance(x, track_type)) + if tracks: + num_tracks = len(tracks) + track_type_plural = track_type.__name__ + ("s" if track_type != Audio and num_tracks != 1 else "") + tracks_tree = tree.add(f"[repr.number]{num_tracks}[/] {track_type_plural}") + for track in tracks: + if add_progress and track_type not in (Chapter, Attachment): + progress = Progress( + SpinnerColumn(finished_text=""), + BarColumn(), + "•", + TimeRemainingColumn(compact=True, elapsed_when_finished=True), + "•", + TextColumn("[progress.data.speed]{task.fields[downloaded]}"), + console=console, + speed_estimate_period=10, + ) + task = progress.add_task("", downloaded="-") + state = {"total": 100.0} + + def update_track_progress( + task_id: int = task, + _state: dict[str, float] = state, + _progress: Progress = progress, + **kwargs, + ) -> None: + """ + Ensure terminal status states render as a fully completed bar. + + Some downloaders can report completed slightly below total + before emitting the final "Downloaded" state. + """ + if "total" in kwargs and kwargs["total"] is not None: + _state["total"] = kwargs["total"] + + downloaded_state = kwargs.get("downloaded") + if downloaded_state in {"Downloaded", "Decrypted", "[yellow]SKIPPED"}: + kwargs["completed"] = _state["total"] + _progress.update(task_id=task_id, **kwargs) + + progress_callables.append(update_track_progress) + track_table = Table.grid() + track_table.add_row(str(track)[6:], style="text2") + track_table.add_row(progress) + tracks_tree.add(track_table) + else: + tracks_tree.add(str(track)[6:], style="text2") + + # Show Closed Captions right after Subtitles (even if no subtitle tracks exist) + if track_type is Subtitle: + seen_cc: set[str] = set() + unique_cc: list[str] = [] + for video in (x for x in all_tracks if isinstance(x, Video)): + for cc in getattr(video, "closed_captions", []): + lang = cc.get("language", "und") + name = cc.get("name", "") + instream_id = cc.get("instream_id", "") + key = f"{lang}|{instream_id}" + if key in seen_cc: + continue + seen_cc.add(key) + parts = [f"[CC] | {lang}"] + if name: + parts.append(name) + if instream_id: + parts.append(instream_id) + unique_cc.append(" | ".join(parts)) + if unique_cc: + cc_tree = tree.add( + f"[repr.number]{len(unique_cc)}[/] Closed Caption{'s' if len(unique_cc) != 1 else ''}" + ) + for cc_str in unique_cc: + cc_tree.add(cc_str, style="text2") + + return tree, progress_callables + + def exists(self, by_id: Optional[str] = None, by_url: Optional[Union[str, list[str]]] = None) -> bool: + """Check if a track already exists by various methods.""" + if by_id: # recommended + return any(x.id == by_id for x in self) + if by_url: + return any(x.url == by_url for x in self) + return False + + def add( + self, + tracks: Union[ + Tracks, Sequence[Union[AnyTrack, Chapter, Chapters, Attachment]], Track, Chapter, Chapters, Attachment + ], + warn_only: bool = False, + ) -> None: + """Add a provided track to its appropriate array and ensuring it's not a duplicate.""" + if isinstance(tracks, Tracks): + if tracks.manifest_url and not self.manifest_url: + self.manifest_url = tracks.manifest_url + tracks = [*list(tracks), *tracks.chapters, *tracks.attachments] + + duplicates = 0 + for track in flatten(tracks): + if self.exists(by_id=track.id): + if not warn_only: + raise ValueError( + "One or more of the provided Tracks is a duplicate. " + "Track IDs must be unique but accurate using static values. The " + "value should stay the same no matter when you request the same " + "content. Use a value that has relation to the track content " + "itself and is static or permanent and not random/RNG data that " + "wont change each refresh or conflict in edge cases." + ) + duplicates += 1 + continue + + if isinstance(track, Video): + self.videos.append(track) + elif isinstance(track, Audio): + self.audio.append(track) + elif isinstance(track, Subtitle): + self.subtitles.append(track) + elif isinstance(track, Chapter): + self.chapters.add(track) + elif isinstance(track, Attachment): + self.attachments.append(track) + else: + raise ValueError("Track type was not set or is invalid.") + + log = logging.getLogger("Tracks") + + if duplicates: + log.debug(f" - Found and skipped {duplicates} duplicate tracks...") + + def sort_videos(self, by_language: Optional[Sequence[Union[str, Language]]] = None) -> None: + """Sort video tracks by bitrate, and optionally language.""" + if not self.videos: + return + # bitrate + self.videos.sort(key=lambda x: float(x.bitrate or 0.0), reverse=True) + # language + for language in reversed(by_language or []): + if str(language) in ("all", "best"): + language = next((x.language for x in self.videos if x.is_original_lang), "") + if not language: + continue + self.videos.sort(key=lambda x: str(x.language)) + self.videos.sort(key=lambda x: not is_close_match(language, [x.language])) + + def sort_audio( + self, + by_language: Optional[Sequence[Union[str, Language]]] = None, + codec_priority: Optional[Sequence[str]] = None, + ) -> None: + """Sort audio tracks by bitrate, codec priority, Atmos, descriptive, and optionally language.""" + if not self.audio: + return + # bitrate (highest first) + self.audio.sort(key=lambda x: float(x.bitrate or 0.0), reverse=True) + # codec priority (listed codecs ranked in order; unlisted fall to end with bitrate order preserved) + if codec_priority: + rank = {str(c).upper(): i for i, c in enumerate(codec_priority)} + default_rank = len(rank) + self.audio.sort(key=lambda x: rank.get(x.codec.name if x.codec else "", default_rank)) + # Atmos tracks first (prioritize over higher bitrate non-Atmos) + self.audio.sort(key=lambda x: not x.atmos) + # descriptive tracks last + self.audio.sort(key=lambda x: x.descriptive) + # language + for language in reversed(by_language or []): + if str(language) in ("all", "best"): + language = next((x.language for x in self.audio if x.is_original_lang), "") + if not language: + continue + self.audio.sort(key=lambda x: not is_close_match(language, [x.language])) + + def sort_subtitles(self, by_language: Optional[Sequence[Union[str, Language]]] = None) -> None: + """ + Sort subtitle tracks by various track attributes to a common P2P standard. + You may optionally provide a sequence of languages to prioritize to the top. + + Section Order: + - by_language groups prioritized to top, and ascending alphabetically + - then rest ascending alphabetically after the prioritized groups + (Each section ascending alphabetically, but separated) + + Language Group Order: + - Forced + - Normal + - Hard of Hearing (SDH/CC) + (Least to most captions expected in the subtitle) + """ + if not self.subtitles: + return + # language groups + self.subtitles.sort(key=lambda x: str(x.language)) + self.subtitles.sort(key=lambda x: x.sdh or x.cc) + self.subtitles.sort(key=lambda x: x.forced, reverse=True) + # sections + for language in reversed(by_language or []): + if str(language) == "all": + language = next((x.language for x in self.subtitles if x.is_original_lang), "") + if not language: + continue + self.subtitles.sort(key=lambda x: is_close_match(language, [x.language]), reverse=True) + + def select_video(self, x: Callable[[Video], bool]) -> None: + self.videos = list(filter(x, self.videos)) + + def select_audio(self, x: Callable[[Audio], bool]) -> None: + self.audio = list(filter(x, self.audio)) + + def select_subtitles(self, x: Callable[[Subtitle], bool]) -> None: + self.subtitles = list(filter(x, self.subtitles)) + + def filter(self, predicate: Callable[[AnyTrack], bool]) -> Tracks: + """Return a new Tracks with tracks filtered by predicate, preserving metadata.""" + new_tracks = Tracks(manifest_url=self.manifest_url) + new_tracks.videos = [t for t in self.videos if predicate(t)] + new_tracks.audio = [t for t in self.audio if predicate(t)] + new_tracks.subtitles = [t for t in self.subtitles if predicate(t)] + new_tracks.chapters = self.chapters + new_tracks.attachments = list(self.attachments) + return new_tracks + + @staticmethod + def merge_video_selections(*groups: list[Video]) -> list[Video]: + """Concatenate video selections, dropping duplicates (by track id, order-preserving). + + A DV track can be chosen as both the hybrid ingredient (lowest) and an explicit + deliverable; without dedup the same track would be muxed/downloaded twice. + """ + merged: list[Video] = [] + for group in groups: + for video in group: + if video not in merged: + merged.append(video) + return merged + + @staticmethod + def partition_hybrid_videos( + videos: list[Video], non_hybrid_ranges: list[Video.Range] + ) -> tuple[list[Video], list[Video]]: + """Split videos into hybrid-ingredient candidates and the standalone-deliverable pool. + + HDR10/HDR10+/DV tracks are hybrid ingredients; they only enter the standalone + pool when their range was explicitly requested alongside HYBRID, so e.g. + `-r HYBRID` muxes only the hybrid while `-r HYBRID,HDR10P` also delivers HDR10+. + """ + ingredient_ranges = (Video.Range.HDR10, Video.Range.HDR10P, Video.Range.DV) + hybrid_candidates = [v for v in videos if v.range in ingredient_ranges] + non_hybrid = [v for v in videos if v.range not in ingredient_ranges or v.range in non_hybrid_ranges] + return hybrid_candidates, non_hybrid + + @staticmethod + def flag_hybrid_ingredients(hybrid_selected: list[Video], non_hybrid_selected: list[Video]) -> None: + """Mark tracks selected only as hybrid ingredients so the standalone mux loop skips them. + + A track that was also picked as an explicit deliverable (same track in both + selections) stays unflagged and is muxed standalone alongside the hybrid. + """ + for video in hybrid_selected: + if video not in non_hybrid_selected: + video.hybrid_base_only = True + + def select_hybrid(self, tracks, quality, worst: bool = False): + # Prefer HDR10+ over HDR10 as the base layer (preserves dynamic metadata) + base_ranges = (Video.Range.HDR10P, Video.Range.HDR10) + base_tracks = [] + for range_type in base_ranges: + base_tracks = [ + v for v in tracks if v.range == range_type and (v.height in quality or int(v.width * 9 / 16) in quality) + ] + if base_tracks: + break + + pick = min if worst else max + base_selected = [] + for res in quality: + candidates = [v for v in base_tracks if v.height == res or int(v.width * 9 / 16) == res] + if candidates: + chosen = pick(candidates, key=lambda v: v.bitrate) + base_selected.append(chosen) + + dv_tracks = [v for v in tracks if v.range == Video.Range.DV] + lowest_dv = min(dv_tracks, key=lambda v: v.height) if dv_tracks else None + + def select(x): + if x in base_selected: + return True + if lowest_dv and x is lowest_dv: + return True + return False + + return select + + def by_resolutions(self, resolutions: list[int], per_resolution: int = 0) -> None: + # Note: Do not merge these list comprehensions. They must be done separately so the results + # from the 16:9 canvas check is only used if there's no exact height resolution match. + selected = [] + for resolution in resolutions: + matches = [ # exact matches + x for x in self.videos if x.height == resolution + ] + if not matches: + matches = [ # 16:9 canvas matches + x for x in self.videos if int(x.width * (9 / 16)) == resolution + ] + selected.extend(matches[: per_resolution or None]) + self.videos = selected + + @staticmethod + def by_language( + tracks: list[TrackT], languages: list[str], per_language: int = 0, exact_match: bool = False + ) -> list[TrackT]: + distance = LANGUAGE_EXACT_DISTANCE if exact_match else LANGUAGE_MAX_DISTANCE + selected = [] + for language in languages: + selected.extend( + [x for x in tracks if closest_supported_match(str(x.language), [language], distance)][ + : per_language or None + ] + ) + return selected + + def mux( + self, + title: str, + delete: bool = True, + progress: Optional[partial] = None, + audio_expected: bool = True, + title_language: Optional[Language] = None, + skip_subtitles: bool = False, + ) -> tuple[Path, int, list[str]]: + """ + Multiplex all the Tracks into a Matroska Container file. + + Parameters: + title: Set the Matroska Container file title. Usually displayed in players + instead of the filename if set. + delete: Delete all track files after multiplexing. + progress: Update a rich progress bar via `completed=...`. This must be the + progress object's update() func, pre-set with task id via functools.partial. + audio_expected: Whether audio is expected in the output. Used to determine + if embedded audio metadata should be added. + title_language: The title's intended language. Used to select the best video track + for audio metadata when multiple video tracks exist. + skip_subtitles: Skip muxing subtitle tracks into the container. + """ + if self.videos and not self.audio and audio_expected: + video_track = None + if title_language: + video_track = next((v for v in self.videos if v.language == title_language), None) + if not video_track: + video_track = next((v for v in self.videos if v.is_original_lang), None) + + video_track = video_track or self.videos[0] + if video_track.language.is_valid(): + lang_code = str(video_track.language) + lang_name = video_track.language.display_name() + + for video in self.videos: + video.needs_repack = True + video.data["audio_language"] = lang_code + video.data["audio_language_name"] = lang_name + + if not binaries.MKVToolNix: + raise RuntimeError("MKVToolNix (mkvmerge) is required for muxing but was not found") + + cl = [ + str(binaries.MKVToolNix), + "--no-date", # remove dates from the output for security + ] + + if config.muxing.get("set_title", True): + cl.extend(["--title", title]) + + default_language = config.muxing.get("default_language") or {} + preferred_video_lang = default_language.get("video") + preferred_audio_lang = default_language.get("audio") + preferred_subtitle_lang = default_language.get("subtitle") + + preferred_video_idx: Optional[int] = None + if preferred_video_lang: + preferred_video_idx = next( + (idx for idx, v in enumerate(self.videos) if is_close_match(v.language, [preferred_video_lang])), + None, + ) + + preferred_audio_idx: Optional[int] = None + if preferred_audio_lang: + preferred_audio_idx = next( + (idx for idx, a in enumerate(self.audio) if is_close_match(a.language, [preferred_audio_lang])), + None, + ) + + preferred_subtitle_idx: Optional[int] = None + if preferred_subtitle_lang and not skip_subtitles: + preferred_subtitle_idx = next( + (idx for idx, s in enumerate(self.subtitles) if is_close_match(s.language, [preferred_subtitle_lang])), + None, + ) + + for i, vt in enumerate(self.videos): + if not vt.path or not vt.path.exists(): + raise ValueError("Video Track must be downloaded before muxing...") + events.emit(events.Types.TRACK_MULTIPLEX, track=vt) + + if preferred_video_idx is not None: + is_default = i == preferred_video_idx + elif title_language: + is_default = vt.language == title_language + if not any(v.language == title_language for v in self.videos): + is_default = vt.is_original_lang or i == 0 + else: + is_default = i == 0 + + # Prepare base arguments + video_args = [ + "--language", + f"0:{vt.language}", + "--default-track", + f"0:{is_default}", + "--original-flag", + f"0:{vt.is_original_lang}", + "--compression", + "0:none", # disable extra compression + ] + + # Add FPS fix if needed (typically for hybrid mode to prevent sync issues) + if hasattr(vt, "needs_duration_fix") and vt.needs_duration_fix and vt.fps: + video_args.extend( + [ + "--default-duration", + f"0:{vt.fps}fps" if isinstance(vt.fps, str) else f"0:{vt.fps:.3f}fps", + "--fix-bitstream-timing-information", + "0:1", + ] + ) + + if hasattr(vt, "range") and vt.range == Video.Range.HLG: + video_args.extend( + [ + "--color-transfer-characteristics", + "0:18", # ARIB STD-B67 (HLG) + ] + ) + + if hasattr(vt, "data") and vt.data.get("audio_language"): + audio_lang = vt.data["audio_language"] + audio_name = vt.data.get("audio_language_name", audio_lang) + video_args.extend( + [ + "--language", + f"1:{audio_lang}", + "--track-name", + f"1:{audio_name}", + ] + ) + + cl.extend(video_args + ["(", str(vt.path), ")"]) + + for i, at in enumerate(self.audio): + if not at.path or not at.path.exists(): + raise ValueError("Audio Track must be downloaded before muxing...") + events.emit(events.Types.TRACK_MULTIPLEX, track=at) + if preferred_audio_idx is not None: + audio_default = i == preferred_audio_idx + else: + audio_default = at.is_original_lang + cl.extend( + [ + "--track-name", + f"0:{at.get_track_name() or ''}", + "--language", + f"0:{at.language}", + "--default-track", + f"0:{audio_default}", + "--visual-impaired-flag", + f"0:{at.descriptive}", + "--original-flag", + f"0:{at.is_original_lang}", + "--compression", + "0:none", # disable extra compression + "(", + str(at.path), + ")", + ] + ) + + if not skip_subtitles: + for i, st in enumerate(self.subtitles): + if not st.path or not st.path.exists(): + raise ValueError("Text Track must be downloaded before muxing...") + events.emit(events.Types.TRACK_MULTIPLEX, track=st) + if preferred_subtitle_idx is not None: + default = i == preferred_subtitle_idx + else: + default = bool(self.audio and is_close_match(st.language, [self.audio[0].language]) and st.forced) + cl.extend( + [ + "--track-name", + f"0:{st.get_track_name() or ''}", + "--language", + f"0:{st.language}", + "--sub-charset", + "0:UTF-8", + "--forced-track", + f"0:{st.forced}", + "--default-track", + f"0:{default}", + "--hearing-impaired-flag", + f"0:{st.sdh}", + "--original-flag", + f"0:{st.is_original_lang}", + "--compression", + "0:none", # disable extra compression (probably zlib) + "(", + str(st.path), + ")", + ] + ) + + if self.chapters: + chapters_path = config.directories.temp / config.filenames.chapters.format( + title=sanitize_filename(title), random=self.chapters.id + ) + self.chapters.dump(chapters_path, fallback_name=config.chapter_fallback_name) + cl.extend(["--chapter-charset", "UTF-8", "--chapters", str(chapters_path)]) + else: + chapters_path = None + + for attachment in self.attachments: + if not attachment.path or not attachment.path.exists(): + raise ValueError("Attachment File was not found...") + cl.extend( + [ + "--attachment-description", + attachment.description or "", + "--attachment-mime-type", + attachment.mime_type, + "--attachment-name", + attachment.name, + "--attach-file", + str(attachment.path.resolve()), + ] + ) + + output_path = ( + self.videos[0].path.with_suffix(".muxed.mkv") + if self.videos + else self.audio[0].path.with_suffix(".muxed.mka") + if self.audio + else self.subtitles[0].path.with_suffix(".muxed.mks") + if self.subtitles + else chapters_path.with_suffix(".muxed.mkv") + if self.chapters + else None + ) + if not output_path: + raise ValueError("No tracks provided, at least one track must be provided.") + + debug_logger = get_debug_logger() + if debug_logger: + debug_logger.log( + level="DEBUG", + operation="mux_start", + message="Starting mkvmerge muxing", + context={ + "title": title, + "output_path": str(output_path), + "video_count": len(self.videos), + "audio_count": len(self.audio), + "subtitle_count": len(self.subtitles), + "attachment_count": len(self.attachments), + "has_chapters": bool(self.chapters), + "video_tracks": [ + {"id": v.id, "codec": getattr(v, "codec", None), "language": str(v.language)} + for v in self.videos + ], + "audio_tracks": [ + {"id": a.id, "codec": getattr(a, "codec", None), "language": str(a.language)} + for a in self.audio + ], + "subtitle_tracks": [ + {"id": s.id, "codec": getattr(s, "codec", None), "language": str(s.language)} + for s in self.subtitles + ], + }, + ) + + # let potential failures go to caller, caller should handle + try: + errors = [] + p = subprocess.Popen([*cl, "--output", str(output_path), "--gui-mode"], text=True, stdout=subprocess.PIPE) + for line in iter(p.stdout.readline, ""): + if line.startswith("#GUI#error") or line.startswith("#GUI#warning"): + errors.append(line) + if "progress" in line: + progress(total=100, completed=int(line.strip()[14:-1])) + + returncode = p.wait() + + if debug_logger: + if returncode != 0 or errors: + debug_logger.log( + level="ERROR", + operation="mux_failed", + message=f"mkvmerge exited with code {returncode}", + context={ + "returncode": returncode, + "output_path": str(output_path), + "errors": errors, + }, + ) + else: + debug_logger.log( + level="DEBUG", + operation="mux_complete", + message="mkvmerge muxing completed successfully", + context={ + "output_path": str(output_path), + "output_exists": output_path.exists() if output_path else False, + }, + ) + + return output_path, returncode, errors + finally: + if chapters_path: + chapters_path.unlink() + if delete: + for track in self: + track.delete() + for attachment in self.attachments: + if attachment.path and attachment.path.exists(): + attachment.path.unlink() + + +__all__ = ("Tracks",) diff --git a/reset/packages/envied/src/envied/core/tracks/video.py b/reset/packages/envied/src/envied/core/tracks/video.py new file mode 100644 index 0000000..fd8727d --- /dev/null +++ b/reset/packages/envied/src/envied/core/tracks/video.py @@ -0,0 +1,610 @@ +from __future__ import annotations + +import logging +import math +import re +import subprocess +from enum import Enum +from pathlib import Path +from typing import Any, Optional, Union + +from langcodes import Language + +from envied.core import binaries +from envied.core.config import config +from envied.core.tracks.subtitle import Subtitle +from envied.core.tracks.track import Track +from envied.core.utilities import FPS, get_boxes + + +class Video(Track): + class Codec(str, Enum): + AVC = "H.264" + HEVC = "H.265" + VC1 = "VC-1" + VP8 = "VP8" + VP9 = "VP9" + AV1 = "AV1" + + @property + def extension(self) -> str: + return self.value.lower().replace(".", "").replace("-", "") + + @staticmethod + def from_mime(mime: str) -> Video.Codec: + mime = mime.lower().strip().split(".")[0] + if mime in ( + "avc1", + "avc2", + "avc3", + "dva1", + "dvav", # Dolby Vision + ): + return Video.Codec.AVC + if mime in ( + "hev1", + "hev2", + "hev3", + "hvc1", + "hvc2", + "hvc3", + "dvh1", + "dvhe", # Dolby Vision + "lhv1", + "lhe1", # Layered + ): + return Video.Codec.HEVC + if mime == "vc-1": + return Video.Codec.VC1 + if mime in ("vp08", "vp8"): + return Video.Codec.VP8 + if mime in ("vp09", "vp9"): + return Video.Codec.VP9 + if mime == "av01": + return Video.Codec.AV1 + raise ValueError(f"The MIME '{mime}' is not a supported Video Codec") + + @staticmethod + def from_codecs(codecs: str) -> Video.Codec: + for codec in codecs.lower().split(","): + codec = codec.strip() + mime = codec.split(".")[0] + try: + return Video.Codec.from_mime(mime) + except ValueError: + pass + raise ValueError(f"No MIME types matched any supported Video Codecs in '{codecs}'") + + @staticmethod + def from_netflix_profile(profile: str) -> Video.Codec: + profile = profile.lower().strip() + if profile.startswith(("h264", "playready-h264")): + return Video.Codec.AVC + if profile.startswith("hevc"): + return Video.Codec.HEVC + if profile.startswith("vp9"): + return Video.Codec.VP9 + if profile.startswith("av1"): + return Video.Codec.AV1 + raise ValueError(f"The Content Profile '{profile}' is not a supported Video Codec") + + class Range(str, Enum): + SDR = "SDR" # No Dynamic Range + HLG = "HLG" # https://en.wikipedia.org/wiki/Hybrid_log%E2%80%93gamma + HDR10 = "HDR10" # https://en.wikipedia.org/wiki/HDR10 + HDR10P = "HDR10+" # https://en.wikipedia.org/wiki/HDR10%2B + DV = "DV" # https://en.wikipedia.org/wiki/Dolby_Vision + HYBRID = "HYBRID" # Selects both HDR10 and DV tracks for hybrid processing with DoviTool + + @staticmethod + def from_cicp(primaries: int, transfer: int, matrix: int) -> Video.Range: + """ + Convert CICP (Coding-Independent Code Points) values to Video Range. + + CICP is defined in ITU-T H.273 and ISO/IEC 23091-2 for signaling video + color properties independently of the compression codec. These values are + used across AVC (H.264), HEVC (H.265), VVC, AV1, and other modern codecs. + + The enum values (Primaries, Transfer, Matrix) match the official specifications: + - ITU-T H.273: Coding-independent code points for video signal type identification + - ISO/IEC 23091-2: Information technology — Coding-independent code points — Part 2: Video + - H.264 Table E-3 (Colour Primaries) and Table E-4 (Transfer Characteristics) + - H.265 Table E.3 and E.4 (identical to H.264) + + Note: Value 0 = "Reserved" and Value 2 = "Unspecified" per specification. + While both effectively mean "unknown" in practice, the distinction matters for + spec compliance. Value 2 was added based on user feedback (GitHub issue) and + verified against FFmpeg's AVColorPrimaries/AVColorTransferCharacteristic enums. + + Sources: + - https://www.itu.int/rec/T-REC-H.273 + - https://www.itu.int/rec/T-REC-H.Sup19-202104-I + - https://github.com/FFmpeg/FFmpeg/blob/master/libavutil/pixfmt.h + """ + + class Primaries(Enum): + Reserved = 0 + BT_709 = 1 + Unspecified = 2 + BT_470_M = 4 + BT_601_625 = 5 + BT_601_525 = 6 + SMPTE_240M = 7 + Generic_Film = 8 + BT_2020_and_2100 = 9 + SMPTE_ST_428_1 = 10 + SMPTE_RP_431_2 = 11 # P3DCI + SMPTE_ST_2113_and_EG_4321 = 12 # P3D65 + EBU_Tech_3213_E = 22 + + class Transfer(Enum): + Reserved = 0 + BT_709 = 1 + Unspecified = 2 + BT_470_M = 4 + BT_601 = 6 + SMPTE_240M = 7 + Linear = 8 + Log_100 = 9 + Log_316 = 10 + IEC_61966_2_4 = 11 + BT_1361 = 12 + IEC_61966_2_1 = 13 # sRGB / sYCC + BT_2020 = 14 + BT_2100 = 15 + BT_2100_PQ = 16 + SMPTE_ST_428_1 = 17 + BT_2100_HLG = 18 + + class Matrix(Enum): + RGB = 0 + YCbCr_BT_709 = 1 + Unspecified = 2 + YCbCr_FCC_73_682 = 4 + YCbCr_BT_601_625 = 5 + YCbCr_BT_601_525 = 6 + SMPTE_240M = 7 + YCgCo = 8 + YCbCr_BT_2020_and_2100 = 9 # YCbCr BT.2100 shares the same CP + YCbCr_BT_2020_CL = 10 + YCbCr_SMPTE_ST_2085 = 11 + ICtCp_BT_2100 = 14 + + if transfer == 5: + # While not part of any standard, it is typically used as a PAL variant of Transfer.BT_601=6. + # i.e. where Transfer 6 would be for BT.601-NTSC and Transfer 5 would be for BT.601-PAL. + # The codebase is currently agnostic to either, so a manual conversion to 6 is done. + transfer = 6 + + def _safe(enum_cls, value): + try: + return enum_cls(value) + except ValueError: + return enum_cls(2) # Unspecified for unknown/private-use codes + + primaries = _safe(Primaries, primaries) + transfer = _safe(Transfer, transfer) + matrix = _safe(Matrix, matrix) + + # primaries and matrix does not strictly correlate to a range + + if (primaries, transfer, matrix) == (Primaries.Reserved, Transfer.Reserved, Matrix.RGB): + return Video.Range.SDR + elif primaries in (Primaries.BT_601_625, Primaries.BT_601_525): + return Video.Range.SDR + elif transfer == Transfer.BT_2100_PQ: + return Video.Range.HDR10 + elif transfer == Transfer.BT_2100_HLG: + return Video.Range.HLG + else: + return Video.Range.SDR + + @staticmethod + def from_m3u_range_tag(tag: str) -> Optional[Video.Range]: + tag = (tag or "").upper().replace('"', "").strip() + if not tag: + return None + if tag == "SDR": + return Video.Range.SDR + elif tag == "PQ": + return Video.Range.HDR10 # technically could be any PQ-transfer range + elif tag == "HLG": + return Video.Range.HLG + # for some reason there's no Dolby Vision info tag + raise ValueError(f"The M3U Range Tag '{tag}' is not a supported Video Range") + + class ScanType(str, Enum): + PROGRESSIVE = "progressive" + INTERLACED = "interlaced" + + def __init__( + self, + *args: Any, + codec: Optional[Video.Codec] = None, + range_: Optional[Video.Range] = None, + bitrate: Optional[Union[str, int, float]] = None, + width: Optional[int] = None, + height: Optional[int] = None, + fps: Optional[Union[str, int, float]] = None, + scan_type: Optional[Video.ScanType] = None, + closed_captions: Optional[list[dict[str, Any]]] = None, + dv_compatible_bitstream: bool = False, + **kwargs: Any, + ) -> None: + """ + Create a new Video track object. + + Parameters: + codec: A Video.Codec enum representing the video codec. + If not specified, MediaInfo will be used to retrieve the codec + once the track has been downloaded. + range_: A Video.Range enum representing the video color range. + Defaults to SDR if not specified. + bitrate: A number or float representing the average bandwidth in bytes/s. + Float values are rounded up to the nearest integer. + width: The horizontal resolution of the video. + height: The vertical resolution of the video. + fps: A number, float, or string representing the frames/s of the video. + Strings may represent numbers, floats, or a fraction (num/den). + All strings will be cast to either a number or float. + + Note: If codec, bitrate, width, height, or fps is not specified some checks + may be skipped or assume a value. Specifying as much information as possible + is highly recommended. + """ + super().__init__(*args, **kwargs) + + if not isinstance(codec, (Video.Codec, type(None))): + raise TypeError(f"Expected codec to be a {Video.Codec}, not {codec!r}") + if not isinstance(range_, (Video.Range, type(None))): + raise TypeError(f"Expected range_ to be a {Video.Range}, not {range_!r}") + if not isinstance(bitrate, (str, int, float, type(None))): + raise TypeError(f"Expected bitrate to be a {str}, {int}, or {float}, not {bitrate!r}") + if not isinstance(width, (int, str, type(None))): + raise TypeError(f"Expected width to be a {int}, not {width!r}") + if not isinstance(height, (int, str, type(None))): + raise TypeError(f"Expected height to be a {int}, not {height!r}") + if not isinstance(fps, (str, int, float, type(None))): + raise TypeError(f"Expected fps to be a {str}, {int}, or {float}, not {fps!r}") + if not isinstance(scan_type, (Video.ScanType, type(None))): + raise TypeError(f"Expected scan_type to be a {Video.ScanType}, not {scan_type!r}") + + self.codec = codec + self.range = range_ or Video.Range.SDR + + try: + self.bitrate = int(math.ceil(float(bitrate))) if bitrate else None + except (ValueError, TypeError) as e: + raise ValueError(f"Expected bitrate to be a number or float, {e}") + + try: + self.width = int(width or 0) or None + except ValueError as e: + raise ValueError(f"Expected width to be a number, not {width!r}, {e}") + + try: + self.height = int(height or 0) or None + except ValueError as e: + raise ValueError(f"Expected height to be a number, not {height!r}, {e}") + + try: + self.fps = (FPS.parse(str(fps)) or None) if fps else None + except Exception as e: + raise ValueError("Expected fps to be a number, float, or a string as numerator/denominator form, " + str(e)) + + self.scan_type = scan_type + self.closed_captions: list[dict[str, Any]] = closed_captions or [] + self.needs_duration_fix = False + self.dv_compatible_bitstream = dv_compatible_bitstream + self.hybrid_base_only = False + + def to_dict(self) -> dict[str, Any]: + data = super().to_dict() + data.update( + { + "codec": self.codec.name if self.codec else None, + "range": self.range.name if self.range else None, + "bitrate": self.bitrate, + "width": self.width, + "height": self.height, + "fps": str(self.fps) if self.fps else None, + "scan_type": self.scan_type.name if self.scan_type else None, + "dv_compatible_bitstream": self.dv_compatible_bitstream, + } + ) + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> Video: + kwargs = Track.base_kwargs_from_dict(data) + return cls( + **kwargs, + codec=Video.Codec[data["codec"]] if data.get("codec") else None, + range_=Video.Range[data["range"]] if data.get("range") else None, + bitrate=data.get("bitrate"), + width=data.get("width"), + height=data.get("height"), + fps=data.get("fps"), + scan_type=Video.ScanType[data["scan_type"]] if data.get("scan_type") else None, + dv_compatible_bitstream=data.get("dv_compatible_bitstream", False), + ) + + def __str__(self) -> str: + return " | ".join( + filter( + bool, + [ + "VID", + "[" + (", ".join(filter(bool, [self.codec.value if self.codec else None, self.range.name]))) + "]", + str(self.language), + ", ".join( + filter( + bool, + [ + " @ ".join( + filter( + bool, + [ + f"{self.width}x{self.height}" if self.width and self.height else None, + f"{self.bitrate // 1000} kb/s" if self.bitrate else None, + ], + ) + ), + f"{self.fps:.3f} FPS" if self.fps else None, + ], + ) + ), + ", ".join(self.edition) if self.edition else None, + ], + ) + ) + + def change_color_range(self, range_: int) -> None: + """Change the Video's Color Range to Limited (0) or Full (1).""" + if not self.path or not self.path.exists(): + raise ValueError("Cannot change the color range flag on a Video that has not been downloaded.") + if not self.codec: + raise ValueError("Cannot change the color range flag on a Video that has no codec specified.") + if self.codec not in (Video.Codec.AVC, Video.Codec.HEVC): + raise NotImplementedError( + "Cannot change the color range flag on this Video as " + f"it's codec, {self.codec.value}, is not yet supported." + ) + + if not binaries.FFMPEG: + raise EnvironmentError('FFmpeg executable "ffmpeg" was not found but is required for this call.') + + filter_key = {Video.Codec.AVC: "h264_metadata", Video.Codec.HEVC: "hevc_metadata"}[self.codec] + + original_path = self.path + output_path = original_path.with_stem(f"{original_path.stem}_{['limited', 'full'][range_]}_range") + + subprocess.run( + [ + binaries.FFMPEG, + "-hide_banner", + "-loglevel", + "panic", + "-i", + original_path, + "-codec", + "copy", + "-bsf:v", + f"{filter_key}=video_full_range_flag={range_}", + str(output_path), + ], + check=True, + ) + + self.path = output_path + original_path.unlink() + + def normalize_vui(self) -> bool: + """Rewrite SPS VUI colour metadata to match ``self.range``. + + Some services ship HDR10/HLG bitstreams with stale BT.709 VUI, which makes + downstream tools mis-classify the file. The manifest-derived range is the + source of truth. Skips SDR, DV, and HYBRID. Returns True if the bitstream + was rewritten. + """ + if not self.path or not self.path.exists(): + return False + if self.codec not in (Video.Codec.AVC, Video.Codec.HEVC): + return False + if self.range in (Video.Range.SDR, Video.Range.DV, Video.Range.HYBRID): + return False + + vui = { + Video.Range.HDR10: (9, 16, 9), + Video.Range.HDR10P: (9, 16, 9), + Video.Range.HLG: (9, 18, 9), + }.get(self.range) + if not vui: + return False + + if not binaries.FFMPEG: + raise EnvironmentError('FFmpeg executable "ffmpeg" was not found but is required for this call.') + + primaries, transfer, matrix = vui + filter_key = {Video.Codec.AVC: "h264_metadata", Video.Codec.HEVC: "hevc_metadata"}[self.codec] + bsf = ( + f"{filter_key}=colour_primaries={primaries}" + f":transfer_characteristics={transfer}" + f":matrix_coefficients={matrix}" + ) + + original_path = self.path + output_path = original_path.with_stem(f"{original_path.stem}_vui") + try: + subprocess.run( + [ + binaries.FFMPEG, + "-hide_banner", + "-loglevel", + "error", + "-i", + str(original_path), + "-codec", + "copy", + "-bsf:v", + bsf, + str(output_path), + ], + check=True, + ) + except subprocess.CalledProcessError: + output_path.unlink(missing_ok=True) + return False + + self.path = output_path + original_path.unlink() + return True + + def ccextractor( + self, track_id: Any, out_path: Union[Path, str], language: Language, original: bool = False + ) -> Optional[Subtitle]: + """Return a TextTrack object representing CC track extracted by CCExtractor.""" + if not self.path: + raise ValueError("You must download the track first.") + + if not binaries.CCExtractor: + raise EnvironmentError("ccextractor executable was not found.") + + out_path = Path(out_path) + + def _run_ccextractor() -> bool: + try: + subprocess.run( + [binaries.CCExtractor, "-trim", "-nobom", "-noru", "-ru1", "-o", out_path, self.path], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + except subprocess.CalledProcessError as e: + out_path.unlink(missing_ok=True) + if e.returncode != 10: # 10 = No captions found + raise + return out_path.exists() + + # Try on the original file first (preserves container-level CC data like c608 boxes), + # then fall back to repacked file (ccextractor can fail on some container formats). + if not _run_ccextractor(): + self.repackage() + _run_ccextractor() + + if out_path.exists(): + cc_track = Subtitle( + id_=track_id, + url="", # doesn't need to be downloaded + codec=Subtitle.Codec.SubRip, + language=language, + is_original_lang=original, + cc=True, + ) + cc_track.path = out_path + return cc_track + + return None + + def extract_c608(self) -> list[Subtitle]: + """ + Extract Apple-Style c608 box (CEA-608) subtitle using ccextractor. + + This isn't much more than a wrapper to the track.ccextractor function. + All this does, is actually check if a c608 box exists and only if so + does it actually call ccextractor. + + Even though there is a possibility of more than one c608 box, only one + can actually be extracted. Not only that but it's very possible this + needs to be done before any decryption as the decryption may destroy + some of the metadata. + + TODO: Need a test file with more than one c608 box to add support for + more than one CEA-608 extraction. + """ + if not self.path: + raise ValueError("You must download the track first.") + with self.path.open("rb") as f: + # assuming 20KB is enough to contain the c608 box. + # ffprobe will fail, so a c608 box check must be done. + c608_count = len(list(get_boxes(f.read(20000), b"c608"))) + if c608_count > 0: + # TODO: Figure out the real language, it might be different + # CEA-608 boxes doesnt seem to carry language information :( + # TODO: Figure out if the CC language is original lang or not. + # Will need to figure out above first to do so. + track_id = f"ccextractor-{self.id}" + cc_lang = self.language + cc_track = self.ccextractor( + track_id=track_id, + out_path=config.directories.temp / config.filenames.subtitle.format(id=track_id, language=cc_lang), + language=cc_lang, + original=False, + ) + if not cc_track: + return [] + return [cc_track] + return [] + + def remove_eia_cc(self) -> bool: + """ + Remove EIA-CC data from Bitstream while keeping SEI data. + + This works by removing all NAL Unit's with the Type of 6 from the bistream + and then re-adding SEI data (effectively a new NAL Unit with just the SEI data). + Only bitstreams with x264 encoding information is currently supported due to the + obscurity on the MDAT mp4 box structure. Therefore, we need to use hacky regex. + """ + if not self.path or not self.path.exists(): + raise ValueError("Cannot clean a Track that has not been downloaded.") + + if not binaries.FFMPEG: + raise EnvironmentError('FFmpeg executable "ffmpeg" was not found but is required for this call.') + + log = logging.getLogger("x264-clean") + log.info("Removing EIA-CC from Video Track with FFMPEG") + + with open(self.path, "rb") as f: + file = f.read(60000) + + x264 = re.search(rb"(.{16})(x264)", file) + if not x264: + log.info(" - No x264 encode settings were found, unsupported...") + return False + + uuid = x264.group(1).hex() + i = file.index(b"x264") + encoding_settings = file[i : i + file[i:].index(b"\x00")].replace(b":", rb"\\:").replace(b",", rb"\,").decode() + + original_path = self.path + cleaned_path = original_path.with_suffix(f".cleaned{original_path.suffix}") + subprocess.run( + [ + binaries.FFMPEG, + "-hide_banner", + "-loglevel", + "panic", + "-i", + original_path, + "-map_metadata", + "-1", + "-fflags", + "bitexact", + "-bsf:v", + f"filter_units=remove_types=6,h264_metadata=sei_user_data={uuid}+{encoding_settings}", + "-codec", + "copy", + str(cleaned_path), + ], + check=True, + ) + + log.info(" + Removed") + + self.path = cleaned_path + original_path.unlink() + + return True + + +__all__ = ("Video",) diff --git a/reset/packages/envied/src/envied/core/update_checker.py b/reset/packages/envied/src/envied/core/update_checker.py new file mode 100644 index 0000000..e706624 --- /dev/null +++ b/reset/packages/envied/src/envied/core/update_checker.py @@ -0,0 +1,276 @@ +from __future__ import annotations + +import asyncio +import json +import time +from pathlib import Path +from typing import Optional + +import requests + + +class UpdateChecker: + """ + Check for available updates from the GitHub repository. + + This class provides functionality to check for newer versions of the application + by querying the GitHub releases API. It includes rate limiting, caching, and + both synchronous and asynchronous interfaces. + + Attributes: + REPO_URL: GitHub API URL for latest release + TIMEOUT: Request timeout in seconds + DEFAULT_CHECK_INTERVAL: Default time between checks in seconds (24 hours) + """ + + REPO_URL = "https://api.github.com/repos/envied.dl/envied.releases/latest" + TIMEOUT = 5 + DEFAULT_CHECK_INTERVAL = 24 * 60 * 60 + + @classmethod + def get_cache_file(cls) -> Path: + """Get the path to the update check cache file.""" + from envied.core.config import config + + return config.directories.cache / "update_check.json" + + @classmethod + def load_cache_data(cls) -> dict: + """ + Load cache data from file. + + Returns: + Cache data dictionary or empty dict if loading fails + """ + cache_file = cls.get_cache_file() + + if not cache_file.exists(): + return {} + + try: + with open(cache_file, "r") as f: + return json.load(f) + except (json.JSONDecodeError, OSError): + return {} + + @staticmethod + def parse_version(version_string: str) -> str: + """ + Parse and normalize version string by removing 'v' prefix. + + Args: + version_string: Raw version string from API + + Returns: + Cleaned version string + """ + return version_string.lstrip("v") + + @staticmethod + def _is_valid_version(version: str) -> bool: + """ + Validate version string format. + + Args: + version: Version string to validate + + Returns: + True if version string is valid semantic version, False otherwise + """ + if not version or not isinstance(version, str): + return False + + try: + parts = version.split(".") + if len(parts) < 2: + return False + + for part in parts: + int(part) + + return True + except (ValueError, AttributeError): + return False + + @classmethod + def _fetch_latest_version(cls) -> Optional[str]: + """ + Fetch the latest version from GitHub API. + + Returns: + Latest version string if successful, None otherwise + """ + try: + response = requests.get(cls.REPO_URL, timeout=cls.TIMEOUT) + + if response.status_code != 200: + return None + + data = response.json() + latest_version = cls.parse_version(data.get("tag_name", "")) + + return latest_version if cls._is_valid_version(latest_version) else None + + except Exception: + return None + + @classmethod + def _should_check_for_updates(cls, check_interval: int = DEFAULT_CHECK_INTERVAL) -> bool: + """ + Check if enough time has passed since the last update check. + + Args: + check_interval: Time in seconds between checks (default: 24 hours) + + Returns: + True if we should check for updates, False otherwise + """ + cache_data = cls.load_cache_data() + + if not cache_data: + return True + + last_check = cache_data.get("last_check", 0) + current_time = time.time() + + return (current_time - last_check) >= check_interval + + @classmethod + def _update_cache(cls, latest_version: Optional[str] = None, current_version: Optional[str] = None) -> None: + """ + Update the cache file with the current timestamp and version info. + + Args: + latest_version: The latest version found, if any + current_version: The current version being used + """ + cache_file = cls.get_cache_file() + + try: + cache_file.parent.mkdir(parents=True, exist_ok=True) + + cache_data = { + "last_check": time.time(), + "latest_version": latest_version, + "current_version": current_version, + } + + with open(cache_file, "w") as f: + json.dump(cache_data, f, indent=2) + + except (OSError, json.JSONEncodeError): + pass + + @staticmethod + def _compare_versions(current: str, latest: str) -> bool: + """ + Simple semantic version comparison. + + Args: + current: Current version string (e.g., "1.1.0") + latest: Latest version string (e.g., "1.2.0") + + Returns: + True if latest > current, False otherwise + """ + if not UpdateChecker._is_valid_version(current) or not UpdateChecker._is_valid_version(latest): + return False + + try: + current_parts = [int(x) for x in current.split(".")] + latest_parts = [int(x) for x in latest.split(".")] + + max_length = max(len(current_parts), len(latest_parts)) + current_parts.extend([0] * (max_length - len(current_parts))) + latest_parts.extend([0] * (max_length - len(latest_parts))) + + for current_part, latest_part in zip(current_parts, latest_parts): + if latest_part > current_part: + return True + elif latest_part < current_part: + return False + + return False + except (ValueError, AttributeError): + return False + + @classmethod + async def check_for_updates(cls, current_version: str) -> Optional[str]: + """ + Check if there's a newer version available on GitHub. + + Args: + current_version: The current version string (e.g., "1.1.0") + + Returns: + The latest version string if an update is available, None otherwise + """ + if not cls._is_valid_version(current_version): + return None + + try: + loop = asyncio.get_event_loop() + latest_version = await loop.run_in_executor(None, cls._fetch_latest_version) + + if latest_version and cls._compare_versions(current_version, latest_version): + return latest_version + + except Exception: + pass + + return None + + @classmethod + def _get_cached_update_info(cls, current_version: str) -> Optional[str]: + """ + Check if there's a cached update available for the current version. + + Args: + current_version: The current version string + + Returns: + The latest version string if an update is available from cache, None otherwise + """ + cache_data = cls.load_cache_data() + + if not cache_data: + return None + + cached_current = cache_data.get("current_version") + cached_latest = cache_data.get("latest_version") + + if cached_current == current_version and cached_latest: + if cls._compare_versions(current_version, cached_latest): + return cached_latest + + return None + + @classmethod + def check_for_updates_sync(cls, current_version: str, check_interval: Optional[int] = None) -> Optional[str]: + """ + Synchronous version of update check with rate limiting. + + Args: + current_version: The current version string (e.g., "1.1.0") + check_interval: Time in seconds between checks (default: from config) + + Returns: + The latest version string if an update is available, None otherwise + """ + if not cls._is_valid_version(current_version): + return None + + if check_interval is None: + from envied.core.config import config + + check_interval = config.update_check_interval * 60 * 60 + + if not cls._should_check_for_updates(check_interval): + return cls._get_cached_update_info(current_version) + + latest_version = cls._fetch_latest_version() + cls._update_cache(latest_version, current_version) + if latest_version and cls._compare_versions(current_version, latest_version): + return latest_version + + return None diff --git a/reset/packages/envied/src/envied/core/utilities.py b/reset/packages/envied/src/envied/core/utilities.py new file mode 100644 index 0000000..5c4c0dd --- /dev/null +++ b/reset/packages/envied/src/envied/core/utilities.py @@ -0,0 +1,1068 @@ +import ast +import contextlib +import gzip +import importlib.util +import json +import logging +import os +import re +import socket +import sys +import time +import traceback +import unicodedata +import zlib +from collections import defaultdict +from datetime import datetime, timezone +from pathlib import Path +from types import ModuleType +from typing import Any, Optional, Sequence, Union +from urllib.parse import ParseResult, urlparse +from uuid import uuid4 + +import chardet +import pycountry +from construct import ValidationError +from fontTools import ttLib +from langcodes import Language, closest_match +from pymp4.parser import Box +from unidecode import unidecode + +from envied.core.config import config +from envied.core.constants import LANGUAGE_EXACT_DISTANCE, LANGUAGE_MAX_DISTANCE + +""" +Utility functions for the envied.media archival tool. + +This module provides various utility functions including: +- Font discovery and fallback system for subtitle rendering +- Cross-platform system font scanning with Windows → Linux font family mapping +- Log file management and rotation +- IP geolocation with caching and provider rotation +- Language matching utilities +- MP4/ISOBMFF box parsing +- File sanitization and path handling +- Structured JSON debug logging + +Font System: + The font subsystem enables cross-platform font discovery for ASS/SSA subtitles. + On Linux, it scans standard font directories and maps Windows font names (Arial, + Times New Roman) to their Linux equivalents (Liberation Sans, Liberation Serif). + +Main Font Functions: + - get_system_fonts(): Discover installed fonts across platforms + - find_font_with_fallbacks(): Match fonts with intelligent fallback strategies + - suggest_font_packages(): Recommend packages to install for missing fonts +""" + + +def rotate_log_file(log_path: Path, keep: int = 20) -> Path: + """ + Update Log Filename and delete old log files. + It keeps only the 20 newest logs by default. + """ + if not log_path: + raise ValueError("A log path must be provided") + + try: + log_path.relative_to(Path("")) # file name only + except ValueError: + pass + else: + log_path = config.directories.logs / log_path + + log_path = log_path.parent / log_path.name.format_map( + defaultdict(str, name="root", time=datetime.now().strftime("%Y%m%d-%H%M%S")) + ) + + if log_path.parent.exists(): + log_files = [x for x in log_path.parent.iterdir() if x.suffix == log_path.suffix] + for log_file in log_files[::-1][keep - 1 :]: + # keep n newest files and delete the rest + log_file.unlink() + + log_path.parent.mkdir(parents=True, exist_ok=True) + return log_path + + +def import_module_by_path(path: Path) -> ModuleType: + """Import a Python file by Path as a Module.""" + if not path: + raise ValueError("Path must be provided") + if not isinstance(path, Path): + raise TypeError(f"Expected path to be a {Path}, not {path!r}") + if not path.exists(): + raise ValueError("Path does not exist") + + # compute package hierarchy for relative import support + if path.is_relative_to(config.directories.core_dir): + name = [] + _path = path.parent + while _path.stem != config.directories.core_dir.stem: + name.append(_path.stem) + _path = _path.parent + name = ".".join([config.directories.core_dir.stem] + name[::-1]) + else: + # is outside the src package + if str(path.parent.parent) not in sys.path: + sys.path.insert(1, str(path.parent.parent)) + name = path.parent.stem + + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + return module + + +def sanitize_filename(filename: str, spacer: str = ".") -> str: + """ + Sanitize a string to be filename safe. + + The spacer is safer to be a '.' for older DDL and p2p sharing spaces. + This includes web-served content via direct links and such. + + Set `unicode_filenames: true` in config to preserve native language + characters (Korean, Japanese, Chinese, etc.) instead of transliterating + them to ASCII equivalents. + """ + # optionally replace non-ASCII characters with ASCII equivalents + if not config.unicode_filenames: + filename = unidecode(filename) + filename = re.sub(r"\[\(+", "[", filename) + filename = re.sub(r"\)+\]", "]", filename) + + # remove or replace further characters as needed + filename = "".join(c for c in filename if unicodedata.category(c) != "Mn") # hidden characters + filename = filename.replace("/", " & ").replace(";", " & ") # e.g. multi-episode filenames + if spacer == ".": + filename = re.sub(r" - ", spacer, filename) # title separators to spacer (avoids .-. pattern) + filename = re.sub(r"[:; ]", spacer, filename) # structural chars to (spacer) + filename = re.sub(r"[\\*!?¿,'\"" "<>|$#~]", "", filename) # not filename safe chars + filename = re.sub(rf"[{spacer}]{{2,}}", spacer, filename) # remove extra neighbouring (spacer)s + + return filename + + +def is_close_match(language: Union[str, Language], languages: Sequence[Union[str, Language, None]]) -> bool: + """Check if a language is a close match to any of the provided languages.""" + languages = [x for x in languages if x] + if not languages: + return False + return closest_match(language, list(map(str, languages)))[1] <= LANGUAGE_MAX_DISTANCE + + +def is_exact_match(language: Union[str, Language], languages: Sequence[Union[str, Language, None]]) -> bool: + """Check if a language is an exact match to any of the provided languages.""" + languages = [x for x in languages if x] + if not languages: + return False + return closest_match(language, list(map(str, languages)))[1] <= LANGUAGE_EXACT_DISTANCE + + +def find_missing_langs( + requested: Sequence[str], + available: Sequence[Union[str, Language, None]], + *, + exact: bool = False, +) -> list[str]: + """Return requested language tokens with no match in available languages.""" + match_func = is_exact_match if exact else is_close_match + skip = {"all", "best", "orig"} + return [tok for tok in requested if tok not in skip and not match_func(tok, available)] + + +def get_boxes(data: bytes, box_type: bytes, as_bytes: bool = False) -> Box: # type: ignore + """ + Scan a byte array for a wanted MP4/ISOBMFF box, then parse and yield each find. + + This function searches through binary MP4 data to find and parse specific box types. + The MP4/ISOBMFF box format consists of: + - 4 bytes: size of the box (including size and type fields) + - 4 bytes: box type identifier (e.g., 'moov', 'trak', 'pssh') + - Remaining bytes: box data + + The function uses slicing to directly locate the requested box type in the data + rather than recursively traversing the box hierarchy. This is efficient when + looking for specific box types regardless of their position in the hierarchy. + + Parameters: + data: Binary data containing MP4/ISOBMFF boxes + box_type: 4-byte identifier of the box type to find (e.g., b'pssh') + as_bytes: If True, returns the box as bytes, otherwise returns parsed box object + + Yields: + Box objects of the requested type found in the data + + Notes: + - For each box found, the function updates the search offset to skip past + the current box to avoid finding the same box multiple times + - The function handles validation errors for certain box types (e.g., tenc) + - The size field is located 4 bytes before the box type identifier + """ + # using slicing to get to the wanted box is done because parsing the entire box and recursively + # scanning through each box and its children often wouldn't scan far enough to reach the wanted box. + # since it doesn't care what child box the wanted box is from, this works fine. + if not isinstance(data, (bytes, bytearray)): + raise ValueError("data must be bytes") + + offset = 0 + while offset < len(data): + try: + index = data[offset:].index(box_type) + except ValueError: + break + + pos = offset + index + + if pos < 4: + offset = pos + len(box_type) + continue + + box_start = pos - 4 + + try: + box = Box.parse(data[box_start:]) + if as_bytes: + box = Box.build(box) + + yield box + + box_size = len(Box.build(box)) + offset = box_start + box_size + + except IOError: + break + except ValidationError as e: + if box_type == b"tenc": + offset = pos + len(box_type) + continue + raise e + + +def ap_case(text: str, keep_spaces: bool = False, stop_words: tuple[str] = None) -> str: + """ + Convert a string to title case using AP/APA style. + Based on https://github.com/words/ap-style-title-case + + Parameters: + text: The text string to title case with AP/APA style. + keep_spaces: To keep the original whitespace, or to just use a normal space. + This would only be needed if you have special whitespace between words. + stop_words: Override the default stop words with your own ones. + """ + if not text: + return "" + + if not stop_words: + stop_words = ( + "a", + "an", + "and", + "at", + "but", + "by", + "for", + "in", + "nor", + "of", + "on", + "or", + "so", + "the", + "to", + "up", + "yet", + ) + + splitter = re.compile(r"(\s+|[-‑–—])") + words = splitter.split(text) + + return "".join( + [ + [" ", word][keep_spaces] + if re.match(r"\s+", word) + else word + if splitter.match(word) + else word.lower() + if i != 0 and i != len(words) - 1 and word.lower() in stop_words + else word.capitalize() + for i, word in enumerate(words) + ] + ) + + +# Common country code aliases that differ from ISO 3166-1 alpha-2 +COUNTRY_CODE_ALIASES = { + "uk": "gb", # United Kingdom -> Great Britain +} + + +def get_country_name(code: str) -> Optional[str]: + """ + Convert a 2-letter country code to full country name. + + Args: + code: ISO 3166-1 alpha-2 country code (e.g., 'ca', 'us', 'gb', 'uk') + + Returns: + Full country name (e.g., 'Canada', 'United States', 'United Kingdom') or None if not found + + Examples: + >>> get_country_name('ca') + 'Canada' + >>> get_country_name('US') + 'United States' + >>> get_country_name('uk') + 'United Kingdom' + """ + # Handle common aliases + code = COUNTRY_CODE_ALIASES.get(code.lower(), code.lower()) + + try: + country = pycountry.countries.get(alpha_2=code.upper()) + if country: + return country.name + except (KeyError, LookupError): + pass + return None + + +def get_country_code(name: str) -> Optional[str]: + """ + Convert a country name to its 2-letter ISO 3166-1 alpha-2 code. + + Args: + name: Full country name (e.g., 'Canada', 'United States', 'United Kingdom') + + Returns: + 2-letter country code in uppercase (e.g., 'CA', 'US', 'GB') or None if not found + + Examples: + >>> get_country_code('Canada') + 'CA' + >>> get_country_code('united states') + 'US' + >>> get_country_code('United Kingdom') + 'GB' + """ + try: + # Try exact name match first + country = pycountry.countries.get(name=name.title()) + if country: + return country.alpha_2.upper() + + # Try common name (e.g., "Bolivia" vs "Bolivia, Plurinational State of") + country = pycountry.countries.get(common_name=name.title()) + if country: + return country.alpha_2.upper() + + # Try fuzzy search as fallback + results = pycountry.countries.search_fuzzy(name) + if results: + return results[0].alpha_2.upper() + except (KeyError, LookupError): + pass + return None + + +def time_elapsed_since(start: float) -> str: + """ + Get time elapsed since a timestamp as a string. + E.g., `1h56m2s`, `15m12s`, `0m55s`, e.t.c. + """ + elapsed = int(time.time() - start) + + minutes, seconds = divmod(elapsed, 60) + hours, minutes = divmod(minutes, 60) + + time_string = f"{minutes:d}m{seconds:d}s" + if hours: + time_string = f"{hours:d}h{time_string}" + + return time_string + + +def try_ensure_utf8(data: bytes) -> bytes: + """ + Try to ensure that the given data is encoded in UTF-8. + + Automatically decompresses gzip/deflate/zlib data before encoding detection. + This handles cases where HTTP responses are saved with raw Content-Encoding + (e.g., when decode_content=False is used for performance). + + Parameters: + data: Input data that may or may not yet be UTF-8 or another encoding. + + Returns the input data encoded in UTF-8 if successful. If unable to detect the + encoding of the input data, then the original data is returned as-received. + """ + # Decompress gzip data (magic bytes: 1f 8b) + if data[:2] == b"\x1f\x8b": + try: + data = gzip.decompress(data) + except Exception: + pass + # Decompress raw deflate/zlib data (common zlib headers: 78 01, 78 5e, 78 9c, 78 da) + elif data[:1] == b"\x78" and len(data) > 1 and data[1:2] in (b"\x01", b"\x5e", b"\x9c", b"\xda"): + try: + data = zlib.decompress(data) + except Exception: + pass + + try: + data.decode("utf8") + return data + except UnicodeDecodeError: + try: + # CP-1252 is a superset of latin1 but has gaps. Replace unknown + # characters instead of failing on them. + return data.decode("cp1252", errors="replace").encode("utf8") + except UnicodeDecodeError: + try: + # last ditch effort to detect encoding + detection_result = chardet.detect(data) + if not detection_result["encoding"]: + return data.decode("utf-8", errors="replace").encode("utf-8") + return data.decode(detection_result["encoding"], errors="replace").encode("utf8") + except (UnicodeDecodeError, LookupError): + return data.decode("utf-8", errors="replace").encode("utf-8") + + +def get_free_port() -> int: + """ + Get an available port to use between a-b (inclusive). + + The port is freed as soon as this has returned, therefore, it + is possible for the port to be taken before you try to use it. + """ + with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + s.bind(("", 0)) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return s.getsockname()[1] + + +def get_extension(value: Union[str, Path, ParseResult]) -> Optional[str]: + """ + Get a URL or Path file extension/suffix. + + Note: The returned value will begin with `.`. + """ + if isinstance(value, ParseResult): + value_parsed = value + elif isinstance(value, (str, Path)): + value_parsed = urlparse(str(value)) + else: + raise TypeError(f"Expected {str}, {Path}, or {ParseResult}, got {type(value)}") + + if value_parsed.path: + ext = os.path.splitext(value_parsed.path)[1] + if ext and ext != ".": + return ext + + +def extract_font_family(font_path: Path) -> Optional[str]: + """ + Extract font family name from TTF/OTF file using fontTools. + + Args: + font_path: Path to the font file + + Returns: + Font family name if successfully extracted, None otherwise + """ + # Suppress verbose fontTools logging during font table parsing + import io + + logging.getLogger("fontTools").setLevel(logging.ERROR) + logging.getLogger("fontTools.ttLib").setLevel(logging.ERROR) + logging.getLogger("fontTools.ttLib.tables").setLevel(logging.ERROR) + logging.getLogger("fontTools.ttLib.tables._n_a_m_e").setLevel(logging.ERROR) + stderr_backup = sys.stderr + sys.stderr = io.StringIO() + + try: + font = ttLib.TTFont(font_path, lazy=True) + name_table = font["name"] + + # Try to get family name (nameID 1) for Windows platform (platformID 3) + # This matches the naming convention used in Windows registry + for record in name_table.names: + if record.nameID == 1 and record.platformID == 3: + return record.toUnicode() + + # Fallback to other platforms if Windows name not found + for record in name_table.names: + if record.nameID == 1: + return record.toUnicode() + + except Exception: + pass + finally: + sys.stderr = stderr_backup + + return None + + +def get_windows_fonts() -> dict[str, Path]: + """ + Get fonts from Windows registry. + + Returns: + Dictionary mapping font family names to their file paths + """ + import winreg + + with winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) as reg: + key = winreg.OpenKey(reg, r"SOFTWARE\Microsoft\Windows NT\CurrentVersion\Fonts", 0, winreg.KEY_READ) + total_fonts = winreg.QueryInfoKey(key)[1] + return { + name.replace(" (TrueType)", ""): Path(r"C:\Windows\Fonts", filename) + for n in range(0, total_fonts) + for name, filename, _ in [winreg.EnumValue(key, n)] + } + + +def scan_font_directory(font_dir: Path, fonts: dict[str, Path], log: logging.Logger) -> None: + """ + Scan a single directory for fonts. + + Args: + font_dir: Directory to scan + fonts: Dictionary to populate with found fonts + log: Logger instance for error reporting + """ + font_files = list(font_dir.rglob("*.ttf")) + list(font_dir.rglob("*.otf")) + + for font_file in font_files: + try: + if family_name := extract_font_family(font_file): + if family_name not in fonts: + fonts[family_name] = font_file + except Exception as e: + log.debug(f"Failed to process {font_file}: {e}") + + +def get_unix_fonts() -> dict[str, Path]: + """ + Get fonts from Linux/macOS standard directories. + + Returns: + Dictionary mapping font family names to their file paths + """ + log = logging.getLogger("get_system_fonts") + fonts = {} + + font_dirs = [ + Path("/usr/share/fonts"), + Path("/usr/local/share/fonts"), + Path.home() / ".fonts", + Path.home() / ".local/share/fonts", + ] + + for font_dir in font_dirs: + if not font_dir.exists(): + continue + + try: + scan_font_directory(font_dir, fonts, log) + except Exception as e: + log.warning(f"Failed to scan {font_dir}: {e}") + return fonts + + +def get_system_fonts() -> dict[str, Path]: + """ + Get system fonts as a mapping of font family names to font file paths. + + On Windows: Uses registry to get font display names + On Linux/macOS: Scans standard font directories and extracts family names using fontTools + + Returns: + Dictionary mapping font family names to their file paths + """ + if sys.platform == "win32": + return get_windows_fonts() + return get_unix_fonts() + + +# Common Windows font names mapped to their Linux equivalents +# Ordered by preference (first match is used) +FONT_ALIASES = { + "Arial": ["Liberation Sans", "DejaVu Sans", "Nimbus Sans", "FreeSans"], + "Arial Black": ["Liberation Sans", "DejaVu Sans", "Nimbus Sans"], + "Arial Bold": ["Liberation Sans", "DejaVu Sans"], + "Arial Unicode MS": ["DejaVu Sans", "Noto Sans", "FreeSans"], + "Times New Roman": ["Liberation Serif", "DejaVu Serif", "Nimbus Roman", "FreeSerif"], + "Courier New": ["Liberation Mono", "DejaVu Sans Mono", "Nimbus Mono PS", "FreeMono"], + "Comic Sans MS": ["Comic Neue", "Comic Relief", "DejaVu Sans"], + "Georgia": ["Gelasio", "DejaVu Serif", "Liberation Serif"], + "Impact": ["Impact", "Anton", "Liberation Sans"], + "Trebuchet MS": ["Ubuntu", "DejaVu Sans", "Liberation Sans"], + "Verdana": ["DejaVu Sans", "Bitstream Vera Sans", "Liberation Sans"], + "Tahoma": ["DejaVu Sans", "Liberation Sans"], + "Adobe Arabic": ["Noto Sans Arabic", "DejaVu Sans"], + "Noto Sans Thai": ["Noto Sans Thai", "Noto Sans"], +} + + +def find_case_insensitive(font_name: str, fonts: dict[str, Path]) -> Optional[Path]: + """ + Find font by case-insensitive name match. + + Args: + font_name: Font family name to find + fonts: Dictionary of available fonts + + Returns: + Path to matched font, or None if not found + """ + font_lower = font_name.lower() + for name, path in fonts.items(): + if name.lower() == font_lower: + return path + return None + + +def find_font_with_fallbacks(font_name: str, system_fonts: dict[str, Path]) -> Optional[Path]: + """ + Find a font by name with intelligent fallback matching. + + Tries multiple strategies in order: + 1. Exact match (case-sensitive) + 2. Case-insensitive match + 3. Alias lookup (Windows → Linux font equivalents) + 4. Partial/prefix match + + Args: + font_name: The requested font family name (e.g., "Arial", "Times New Roman") + system_fonts: Dictionary of available fonts (family name → path) + + Returns: + Path to the matched font file, or None if no match found + """ + if not system_fonts: + return None + + # Strategy 1: Exact match (case-sensitive) + if font_name in system_fonts: + return system_fonts[font_name] + + # Strategy 2: Case-insensitive match + if result := find_case_insensitive(font_name, system_fonts): + return result + + # Strategy 3: Alias lookup + if font_name in FONT_ALIASES: + for alias in FONT_ALIASES[font_name]: + # Try exact match for alias + if alias in system_fonts: + return system_fonts[alias] + # Try case-insensitive match for alias + if result := find_case_insensitive(alias, system_fonts): + return result + + # Strategy 4: Partial/prefix match as last resort + font_name_lower = font_name.lower() + for name, path in system_fonts.items(): + if name.lower().startswith(font_name_lower): + return path + + return None + + +# Mapping of font families to system packages that provide them +FONT_PACKAGES = { + "liberation": { + "debian": "fonts-liberation fonts-liberation2", + "fonts": ["Liberation Sans", "Liberation Serif", "Liberation Mono"], + }, + "dejavu": { + "debian": "fonts-dejavu fonts-dejavu-core fonts-dejavu-extra", + "fonts": ["DejaVu Sans", "DejaVu Serif", "DejaVu Sans Mono"], + }, + "noto": { + "debian": "fonts-noto fonts-noto-core", + "fonts": ["Noto Sans", "Noto Serif", "Noto Sans Mono", "Noto Sans Arabic", "Noto Sans Thai"], + }, + "ubuntu": { + "debian": "fonts-ubuntu", + "fonts": ["Ubuntu", "Ubuntu Mono"], + }, +} + + +def suggest_font_packages(missing_fonts: list[str]) -> dict[str, list[str]]: + """ + Suggest system packages to install for missing fonts. + + Args: + missing_fonts: List of font family names that couldn't be found + + Returns: + Dictionary mapping package names to lists of fonts they would provide + """ + suggestions = {} + + # Check which fonts from aliases would help + needed_aliases = set() + for font in missing_fonts: + if font in FONT_ALIASES: + needed_aliases.update(FONT_ALIASES[font]) + + # Map needed aliases to packages + for package_name, package_info in FONT_PACKAGES.items(): + provided_fonts = package_info["fonts"] + matching_fonts = [f for f in provided_fonts if f in needed_aliases] + if matching_fonts: + suggestions[package_info["debian"]] = matching_fonts + + return suggestions + + +class FPS(ast.NodeVisitor): + def visit_BinOp(self, node: ast.BinOp) -> float: + if isinstance(node.op, ast.Div): + return self.visit(node.left) / self.visit(node.right) + raise ValueError(f"Invalid operation: {node.op}") + + def visit_Num(self, node: ast.Num) -> complex: + return node.n + + def visit_Expr(self, node: ast.Expr) -> float: + return self.visit(node.value) + + @classmethod + def parse(cls, expr: str) -> float: + return cls().visit(ast.parse(expr).body[0]) + + +""" +Structured JSON debug logging for envied. + +Provides comprehensive debugging information for service developers and troubleshooting. +When enabled, logs all operations, requests, responses, DRM operations, and errors in JSON format. +""" + + +class DebugLogger: + """ + Structured JSON debug logger for envied. + + Outputs JSON Lines format where each line is a complete JSON object. + This makes it easy to parse, filter, and analyze logs programmatically. + """ + + def __init__(self, log_path: Optional[Path] = None, enabled: bool = False, log_keys: bool = False): + """ + Initialize the debug logger. + + Args: + log_path: Path to the log file. If None, logging is disabled. + enabled: Whether debug logging is enabled. + log_keys: Whether to log decryption keys (for debugging key issues). + """ + self.enabled = enabled and log_path is not None + self.log_path = log_path + self.session_id = str(uuid4())[:8] + self.file_handle = None + self.log_keys = log_keys + + if self.enabled: + self.log_path.parent.mkdir(parents=True, exist_ok=True) + self.file_handle = open(self.log_path, "a", encoding="utf-8") + self.log_session_start() + + def log_session_start(self): + """Log the start of a new session with environment information.""" + import platform + + from envied.core import __version__ + + self.log( + level="INFO", + operation="session_start", + message="Debug logging session started", + context={ + "envied.version": __version__, + "python_version": sys.version, + "platform": platform.platform(), + "platform_system": platform.system(), + "platform_release": platform.release(), + }, + ) + + def log( + self, + level: str = "DEBUG", + operation: str = "", + message: str = "", + context: Optional[dict[str, Any]] = None, + service: Optional[str] = None, + error: Optional[Exception] = None, + request: Optional[dict[str, Any]] = None, + response: Optional[dict[str, Any]] = None, + duration_ms: Optional[float] = None, + success: Optional[bool] = None, + **kwargs, + ): + """ + Log a structured JSON entry. + + Args: + level: Log level (DEBUG, INFO, WARNING, ERROR) + operation: Name of the operation being performed + message: Human-readable message + context: Additional context information + service: Service name (e.g., DSNP, NF) + error: Exception object if an error occurred + request: Request details (URL, method, headers, body) + response: Response details (status, headers, body) + duration_ms: Operation duration in milliseconds + success: Whether the operation succeeded + **kwargs: Additional fields to include in the log entry + """ + if not self.enabled or not self.file_handle: + return + + entry = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "session_id": self.session_id, + "level": level, + } + + if operation: + entry["operation"] = operation + if message: + entry["message"] = message + if service: + entry["service"] = service + if context: + entry["context"] = self.sanitize_data(context) + if request: + entry["request"] = self.sanitize_data(request) + if response: + entry["response"] = self.sanitize_data(response) + if duration_ms is not None: + entry["duration_ms"] = duration_ms + if success is not None: + entry["success"] = success + + if error: + entry["error"] = { + "type": type(error).__name__, + "message": str(error), + "traceback": traceback.format_exception(type(error), error, error.__traceback__), + } + + for key, value in kwargs.items(): + if key not in entry: + entry[key] = self.sanitize_data(value) + + try: + self.file_handle.write(json.dumps(entry, default=str) + "\n") + self.file_handle.flush() + except Exception as e: + print(f"Failed to write debug log: {e}", file=sys.stderr) + + def sanitize_data(self, data: Any) -> Any: + """ + Sanitize data for JSON serialization. + Handles complex objects and removes sensitive information. + """ + if data is None: + return None + + if isinstance(data, (str, int, float, bool)): + return data + + if isinstance(data, (list, tuple)): + return [self.sanitize_data(item) for item in data] + + if isinstance(data, dict): + sanitized = {} + for key, value in data.items(): + key_lower = str(key).lower() + has_prefix = key_lower.startswith("has_") + + is_always_sensitive = not has_prefix and any( + sensitive in key_lower for sensitive in ["password", "token", "secret", "auth", "cookie"] + ) + + is_key_field = ( + "key" in key_lower + and not has_prefix + and not any(safe in key_lower for safe in ["_count", "_id", "_type", "kid", "keys_", "key_found"]) + ) + + should_redact = is_always_sensitive or (is_key_field and not self.log_keys) + + if should_redact: + sanitized[key] = "[REDACTED]" + else: + sanitized[key] = self.sanitize_data(value) + return sanitized + + if isinstance(data, bytes): + try: + return data.hex() + except Exception: + return "[BINARY_DATA]" + + if isinstance(data, Path): + return str(data) + + try: + return str(data) + except Exception: + return f"[{type(data).__name__}]" + + def log_operation_start(self, operation: str, **kwargs) -> str: + """ + Log the start of an operation and return an operation ID. + + Args: + operation: Name of the operation + **kwargs: Additional context + + Returns: + Operation ID that can be used to log the end of the operation + """ + op_id = str(uuid4())[:8] + self.log( + level="DEBUG", + operation=f"{operation}_start", + message=f"Starting operation: {operation}", + operation_id=op_id, + **kwargs, + ) + return op_id + + def log_operation_end( + self, operation: str, operation_id: str, success: bool = True, duration_ms: Optional[float] = None, **kwargs + ): + """ + Log the end of an operation. + + Args: + operation: Name of the operation + operation_id: Operation ID from log_operation_start + success: Whether the operation succeeded + duration_ms: Operation duration in milliseconds + **kwargs: Additional context + """ + self.log( + level="INFO" if success else "ERROR", + operation=f"{operation}_end", + message=f"Finished operation: {operation}", + operation_id=operation_id, + success=success, + duration_ms=duration_ms, + **kwargs, + ) + + def log_service_call(self, method: str, url: str, **kwargs): + """ + Log a service API call. + + Args: + method: HTTP method (GET, POST, etc.) + url: Request URL + **kwargs: Additional request details (headers, body, etc.) + """ + self.log(level="DEBUG", operation="service_call", request={"method": method, "url": url, **kwargs}) + + def log_drm_operation(self, drm_type: str, operation: str, **kwargs): + """ + Log a DRM operation (PSSH extraction, license request, key retrieval). + + Args: + drm_type: DRM type (Widevine, PlayReady, etc.) + operation: DRM operation name + **kwargs: Additional context (PSSH, KIDs, keys, etc.) + """ + self.log( + level="DEBUG", operation=f"drm_{operation}", message=f"{drm_type} {operation}", drm_type=drm_type, **kwargs + ) + + def log_vault_query(self, vault_name: str, operation: str, **kwargs): + """ + Log a vault query operation. + + Args: + vault_name: Name of the vault + operation: Vault operation (get_key, add_key, etc.) + **kwargs: Additional context (KID, key, success, etc.) + """ + self.log( + level="DEBUG", + operation=f"vault_{operation}", + message=f"Vault {vault_name}: {operation}", + vault=vault_name, + **kwargs, + ) + + def log_error(self, operation: str, error: Exception, **kwargs): + """ + Log an error with full context. + + Args: + operation: Operation that failed + error: Exception that occurred + **kwargs: Additional context + """ + self.log( + level="ERROR", + operation=operation, + message=f"Error in {operation}: {str(error)}", + error=error, + success=False, + **kwargs, + ) + + def close(self): + """Close the log file and clean up resources.""" + if self.file_handle: + self.log(level="INFO", operation="session_end", message="Debug logging session ended") + self.file_handle.close() + self.file_handle = None + + +# Global debug logger instance +_debug_logger: Optional[DebugLogger] = None + + +def get_debug_logger() -> Optional[DebugLogger]: + """Get the global debug logger instance.""" + return _debug_logger + + +def init_debug_logger(log_path: Optional[Path] = None, enabled: bool = False, log_keys: bool = False): + """ + Initialize the global debug logger. + + Args: + log_path: Path to the log file + enabled: Whether debug logging is enabled + log_keys: Whether to log decryption keys (for debugging key issues) + """ + global _debug_logger + if _debug_logger: + _debug_logger.close() + _debug_logger = DebugLogger(log_path=log_path, enabled=enabled, log_keys=log_keys) + + +def close_debug_logger(): + """Close the global debug logger.""" + global _debug_logger + if _debug_logger: + _debug_logger.close() + _debug_logger = None + + +__all__ = ( + "DebugLogger", + "get_debug_logger", + "init_debug_logger", + "close_debug_logger", +) diff --git a/reset/packages/envied/src/envied/core/utils/__init__.py b/reset/packages/envied/src/envied/core/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/reset/packages/envied/src/envied/core/utils/animeapi.py b/reset/packages/envied/src/envied/core/utils/animeapi.py new file mode 100644 index 0000000..d9d5de9 --- /dev/null +++ b/reset/packages/envied/src/envied/core/utils/animeapi.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import logging +from typing import Optional + +from envied.core.providers._base import ExternalIds + +log = logging.getLogger("ANIMEAPI") + +PLATFORM_MAP: dict[str, str] = { + "mal": "myanimelist", + "anilist": "anilist", + "kitsu": "kitsu", + "tmdb": "themoviedb", + "trakt": "trakt", + "tvdb": "thetvdb", +} + + +def resolve_animeapi(value: str) -> tuple[Optional[str], ExternalIds]: + """Resolve an anime database ID via AnimeAPI to a title and external IDs. + + Accepts formats like 'mal:12345', 'anilist:98765', or just '12345' (defaults to MAL). + Returns (anime_title, ExternalIds) with any TMDB/IMDB/TVDB IDs found. + """ + import animeapi + + platform_str, id_str = _parse_animeapi_value(value) + + platform_enum = _get_platform(platform_str) + if platform_enum is None: + log.warning("Unknown AnimeAPI platform: %s (supported: %s)", platform_str, ", ".join(PLATFORM_MAP)) + return None, ExternalIds() + + log.info("Resolving AnimeAPI %s:%s", platform_str, id_str) + + try: + with animeapi.AnimeAPI() as api: + relation = api.get_anime_relations(id_str, platform_enum) + except Exception as exc: + log.warning("AnimeAPI lookup failed for %s:%s: %s", platform_str, id_str, exc) + return None, ExternalIds() + + title = getattr(relation, "title", None) + + tmdb_id = getattr(relation, "themoviedb", None) + tmdb_type = getattr(relation, "themoviedb_type", None) + imdb_id = getattr(relation, "imdb", None) + tvdb_id = getattr(relation, "thetvdb", None) + + tmdb_kind: Optional[str] = None + if tmdb_type is not None: + tmdb_kind = tmdb_type.value if hasattr(tmdb_type, "value") else str(tmdb_type).lower() + if tmdb_kind not in ("movie", "tv"): + tmdb_kind = "tv" + + external_ids = ExternalIds( + tmdb_id=int(tmdb_id) if tmdb_id is not None else None, + tmdb_kind=tmdb_kind, + imdb_id=str(imdb_id) if imdb_id is not None else None, + tvdb_id=int(tvdb_id) if tvdb_id is not None else None, + ) + + log.info( + "AnimeAPI resolved: title=%r, tmdb=%s, imdb=%s, tvdb=%s", + title, + external_ids.tmdb_id, + external_ids.imdb_id, + external_ids.tvdb_id, + ) + + return title, external_ids + + +def _parse_animeapi_value(value: str) -> tuple[str, str]: + """Parse 'platform:id' format. Defaults to 'mal' if no prefix.""" + if ":" in value: + platform, _, id_str = value.partition(":") + return platform.lower().strip(), id_str.strip() + return "mal", value.strip() + + +def _get_platform(platform_str: str) -> object | None: + """Map a platform string to an animeapi.Platform enum value.""" + import animeapi + + canonical = PLATFORM_MAP.get(platform_str) + if canonical is None: + return None + + platform_name = canonical.upper() + return getattr(animeapi.Platform, platform_name, None) diff --git a/reset/packages/envied/src/envied/core/utils/bitrate.py b/reset/packages/envied/src/envied/core/utils/bitrate.py new file mode 100644 index 0000000..046809f --- /dev/null +++ b/reset/packages/envied/src/envied/core/utils/bitrate.py @@ -0,0 +1,441 @@ +from __future__ import annotations + +import json +import logging +import subprocess +from collections import OrderedDict, defaultdict +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from typing import TYPE_CHECKING, Callable, Hashable, Optional, Union +from urllib.parse import urljoin + +from requests import Session + +from envied.core.binaries import FFProbe +from envied.core.session import RnetSession + +if TYPE_CHECKING: + from envied.core.tracks import Track + +# Default ISM timescale (ticks per second) per the Smooth Streaming spec. +ISM_DEFAULT_TIMESCALE = 10_000_000 + +# Bytes fetched to locate an mp4 moov box when probing duration via ffprobe. +MOOV_PROBE_BYTES = 4 * 1024 * 1024 + +# Network timeout (seconds) for probe requests. +PROBE_TIMEOUT = 15 + + +@dataclass +class Segment: + """One probe target: a media URL, optional byte range, its size, and duration.""" + + url: str + # The original byte-range string (e.g. "0-1023"), preserved as the segment's + # identity so distinct ranges of one file are never confused with each other. + byte_range: Optional[str] + # Size in bytes when derivable without a request (from a byte range); else None. + known_size: Optional[int] + duration: float + + +def measure_real_bitrate( + track: "Track", + session: Union[Session, RnetSession], + *, + samples: int = 40, + log: logging.Logger, +) -> Optional[int]: + """ + Probe a track's actual media size to compute its real average bitrate. + + Manifests often declare an inaccurate bandwidth (DASH ``@bandwidth`` is a + leaky-bucket ceiling, not an average). This measures the true bitrate + (bits/sec) from real media byte sizes and durations using ``bytes * 8 / sec``. + + Single-file tracks are measured exactly. Segmented tracks probe up to + ``samples`` segments spread across the track and extrapolate; byte-range + segments need no request. Returns bits/sec, or ``None`` if it cannot be + measured. Never raises — a probe failure must not abort a download. + """ + from envied.core.tracks.track import Track + + try: + if track.descriptor == Track.Descriptor.DASH: + segments = extract_dash(track, session) + elif track.descriptor == Track.Descriptor.HLS: + segments = extract_hls(track, session) + elif track.descriptor == Track.Descriptor.ISM: + segments = extract_ism(track, session) + else: + # Descriptor.URL: a single file. Some services (e.g. AMZN) parse a DASH + # manifest then collapse each representation to its single BaseURL and + # flip the descriptor to URL, leaving the manifest (and its duration) in + # track.data — recover the duration from there, else probe the file. + segments = extract_url(track, session, log=log) + if not segments: + log.debug(f"{track.id}: cannot measure real bitrate (no known duration)") + return None + except Exception as e: + log.warning(f"{track.id}: failed to derive segments for real bitrate ({e})") + return None + + if not segments: + return None + + items = dedupe(segments) + chosen = pick_samples(items, samples) + + total_bytes = 0 + total_seconds = 0.0 + for segment in chosen: + if segment.duration <= 0: + continue + size = segment.known_size if segment.known_size is not None else probe_size(segment, session) + if not size: + continue + total_bytes += size + total_seconds += segment.duration + + log.debug( + f"{track.id}: real-bitrate probe desc={track.descriptor.name} " + f"n_seg={len(segments)} n_unique={len(items)} n_chosen={len(chosen)} " + f"sampled_bytes={total_bytes} sampled_seconds={round(total_seconds, 4)}" + ) + + if total_seconds <= 0 or total_bytes <= 0: + log.warning(f"{track.id}: real bitrate probe returned no usable data") + return None + + return round(total_bytes * 8 / total_seconds) + + +def apply_real_bitrates( + tracks: list["Track"], + session: Union[Session, RnetSession], + *, + log: logging.Logger, + group_key: Callable[["Track"], Hashable], + per_group: int = 5, + workers: int = 8, +) -> None: + """ + Probe real bitrates and overwrite ``track.bitrate`` for the tracks worth probing. + + Probing every rendition is slow when a service exposes dozens. Tracks are + grouped by ``group_key`` (a quality tier), and only the ``per_group`` highest + declared-bitrate tracks per group are probed, in parallel. Each group is then + extended downward: while the lowest probed bitrate in a group sits below the + next unprobed track's declared bitrate (so that track could outrank a probed + one), the next track is probed too — until the probed set is safely above the + rest. Unprobed tracks keep their manifest-declared bitrate. + """ + groups: defaultdict[Hashable, list["Track"]] = defaultdict(list) + for track in tracks: + groups[group_key(track)].append(track) + for group in groups.values(): + group.sort(key=lambda t: getattr(t, "bitrate", None) or 0, reverse=True) + + # Initial pass: top per_group of every group, all probed concurrently. + initial = [track for group in groups.values() for track in group[:per_group]] + probe_batch(initial, session, log=log, workers=workers) + + # Extend each group downward until unprobed tracks can't outrank probed ones. + for group in groups.values(): + probed = min(per_group, len(group)) + while probed < len(group): + lowest_probed = min((getattr(t, "bitrate", None) or 0) for t in group[:probed]) + next_declared = getattr(group[probed], "bitrate", None) or 0 + if next_declared <= lowest_probed: + break + probe_batch([group[probed]], session, log=log, workers=workers) + probed += 1 + + +def probe_batch( + tracks: list["Track"], + session: Union[Session, RnetSession], + *, + log: logging.Logger, + workers: int, +) -> None: + """Probe each track concurrently and overwrite its bitrate with the measured value.""" + if not tracks: + return + + def probe_one(track: "Track") -> tuple["Track", Optional[int]]: + return track, measure_real_bitrate(track, track.session or session, log=log) + + with ThreadPoolExecutor(max_workers=min(workers, len(tracks))) as executor: + for track, measured in executor.map(probe_one, tracks): + if not measured: + continue + declared = getattr(track, "bitrate", None) + if declared and declared != measured: + log.debug(f"{track.id}: bitrate {declared // 1000} → {measured // 1000} kb/s (real)") + setattr(track, "bitrate", measured) + + +def dedupe(segments: list[Segment]) -> list[Segment]: + """ + Collapse segments that address the same bytes so each object is measured once. + + Manifests sometimes wrap a single file in several segment entries sharing one + URL — with no byte range (a ``SegmentTemplate`` whose media pattern has no + ``$Number$``) or with the same range. Each resolves to the whole file, so + counting them all would multiply the size by the segment count. Segments + sharing the same ``(url, byte_range)`` are merged into one entry whose duration + is the sum they cover. Distinct byte ranges of one file (different offsets) are + kept individual so their sizes still add up to the full track. + """ + merged: OrderedDict[tuple[str, Optional[str]], Segment] = OrderedDict() + for segment in segments: + key = (segment.url, segment.byte_range) + existing = merged.get(key) + if existing is None: + merged[key] = Segment(segment.url, segment.byte_range, segment.known_size, segment.duration) + else: + existing.duration += segment.duration + return list(merged.values()) + + +def pick_samples(segments: list[Segment], samples: int) -> list[Segment]: + """Pick up to ``samples`` segments spread evenly across the track.""" + count = len(segments) + if count <= samples: + return segments + step = count / samples + indices = sorted({int(i * step) for i in range(samples)}) + return [segments[i] for i in indices] + + +def probe_size(segment: Segment, session: Union[Session, RnetSession]) -> Optional[int]: + """Return a segment's byte size via HEAD, falling back to a ranged GET. Validates status.""" + try: + res = session.head(segment.url, allow_redirects=True, timeout=PROBE_TIMEOUT) + if getattr(res, "status_code", 0) in (200, 206): + content_length = res.headers.get("Content-Length") + if content_length: + return int(content_length) + except Exception: + pass + + # Some hosts block or mishandle HEAD; ask for a single byte and read the total. + # Require a 206 so a server that ignores Range (returning the whole 200 body) + # is not mistaken for a valid size or downloaded wholesale. + try: + res = session.get(segment.url, headers={"Range": "bytes=0-0"}, timeout=PROBE_TIMEOUT) + if getattr(res, "status_code", 0) == 206: + content_range = res.headers.get("Content-Range") + if content_range and "/" in content_range: + total = content_range.rsplit("/", 1)[-1].strip() + if total.isdigit(): + return int(total) + except Exception: + pass + + return None + + +def range_size(byte_range: Optional[str]) -> Optional[int]: + """Size in bytes of a ``start-end`` media range, inclusive.""" + if not byte_range or "-" not in byte_range: + return None + start_s, _, end_s = byte_range.partition("-") + try: + start = int(start_s) if start_s else 0 + if not end_s: + return None + return int(end_s) - start + 1 + except ValueError: + return None + + +def uniform_segments( + raw_segments: list[tuple[str, Optional[str]]], + total_duration: Optional[float], +) -> list[Segment]: + """ + Build Segments giving each an equal share of the total duration. + + Used for DASH: ``DASH._get_period_segments`` returns timeline *start times* + rather than per-segment durations, so they cannot be trusted. Segment lengths + are near-uniform in practice, so the track duration (from + ``mediaPresentationDuration``) split evenly is both correct and timeline-safe. + """ + count = len(raw_segments) + if not count or not total_duration or total_duration <= 0: + return [] + per_segment = total_duration / count + return [Segment(url, byte_range, range_size(byte_range), per_segment) for url, byte_range in raw_segments] + + +def extract_dash(track: "Track", session: Union[Session, RnetSession]) -> list[Segment]: + from envied.core.manifests import DASH + + data = track.data["dash"] + manifest = data["manifest"] + rep_id = data.get("representation_id") or data["representation"].get("id") + filtered_period_ids = data.get("filtered_period_ids", []) + track_url = track.url if isinstance(track.url, str) else track.url[0] + + content_periods = [p for p in manifest.findall("Period") if DASH._is_content_period(p, filtered_period_ids)] + + raw_segments: list[tuple[str, Optional[str]]] = [] + for period in content_periods: + matched_rep = matched_as = None + for as_ in period.findall("AdaptationSet"): + if DASH.is_trick_mode(as_): + continue + for rep in as_.findall("Representation"): + if rep.get("id") == rep_id: + matched_rep, matched_as = rep, as_ + break + if matched_rep is not None: + break + if matched_rep is None or matched_as is None: + continue + + _, period_segments, _, _, _ = DASH._get_period_segments( + period=period, + adaptation_set=matched_as, + representation=matched_rep, + manifest=manifest, + track=track, + track_url=track_url, + session=session, + ) + raw_segments.extend(period_segments) + + total_duration: Optional[float] = None + mpd_duration = manifest.get("mediaPresentationDuration") + if mpd_duration: + total_duration = DASH.pt_to_sec(mpd_duration) + + return uniform_segments(raw_segments, total_duration) + + +def extract_hls(track: "Track", session: Union[Session, RnetSession]) -> list[Segment]: + import m3u8 + + playlist_url = track.url if isinstance(track.url, str) else track.url[0] + res = session.get(playlist_url, timeout=PROBE_TIMEOUT) + playlist = m3u8.loads(res.text, uri=playlist_url) + + out: list[Segment] = [] + for segment in playlist.segments: + url = urljoin(segment.base_uri or "", segment.uri) + byte_range = segment.byterange # "[@]" + known_size: Optional[int] = None + if byte_range: + length = byte_range.split("@")[0].strip() + if length.isdigit(): + known_size = int(length) + # EXTINF durations are reliable, so they are used directly (unlike DASH). + out.append(Segment(url, byte_range, known_size, float(segment.duration or 0))) + return out + + +def extract_ism(track: "Track", session: Union[Session, RnetSession]) -> list[Segment]: + data = track.data["ism"] + segments: list[str] = data.get("segments") or [] + manifest = data["manifest"] + + timescale = int(manifest.get("TimeScale") or ISM_DEFAULT_TIMESCALE) + duration_ticks = int(manifest.get("Duration") or 0) + total_duration = (duration_ticks / timescale) if timescale else 0.0 + + return uniform_segments([(url, None) for url in segments], total_duration) + + +def extract_url(track: "Track", session: Union[Session, RnetSession], *, log: logging.Logger) -> list[Segment]: + """Single-file track: one whole-file URL with the duration from leftover manifest data.""" + url = track.url if isinstance(track.url, str) else (track.url[0] if track.url else None) + if not url: + return [] + + duration: Optional[float] = None + dash_data = track.data.get("dash") + if dash_data and dash_data.get("manifest") is not None: + from envied.core.manifests import DASH + + mpd_duration = dash_data["manifest"].get("mediaPresentationDuration") + if mpd_duration: + duration = DASH.pt_to_sec(mpd_duration) + else: + ism_data = track.data.get("ism") + if ism_data and ism_data.get("manifest") is not None: + manifest = ism_data["manifest"] + timescale = int(manifest.get("TimeScale") or ISM_DEFAULT_TIMESCALE) + duration_ticks = int(manifest.get("Duration") or 0) + if timescale and duration_ticks: + duration = duration_ticks / timescale + + if not duration or duration <= 0: + # Services like AMZN clear the manifest data after collapsing to a single + # file; fall back to reading the duration straight from the remote file. + duration = ffprobe_duration(url, session, log=log) + + if not duration or duration <= 0: + return [] + return [Segment(url, None, None, duration)] + + +def ffprobe_duration(url: str, session: Union[Session, RnetSession], *, log: logging.Logger) -> Optional[float]: + """ + Read a single-file track's duration (seconds) without a manifest. + + The bundled ffprobe segfaults on network input, so the file's ``moov`` box is + fetched over HTTP with the session (keeping the service's proxy/headers) and + piped to ffprobe as local bytes. The head of the file is tried first (VOD is + usually faststart), then the tail as a fallback for moov-at-end files. + """ + head = ranged_get(url, session, f"bytes=0-{MOOV_PROBE_BYTES - 1}") + duration = probe_bytes_duration(head, log) + if duration: + return duration + + size = probe_size(Segment(url, None, None, 0.0), session) + if size and size > MOOV_PROBE_BYTES: + tail = ranged_get(url, session, f"bytes={size - MOOV_PROBE_BYTES}-{size - 1}") + duration = probe_bytes_duration(tail, log) + return duration + + +def ranged_get(url: str, session: Union[Session, RnetSession], byte_range: str) -> Optional[bytes]: + """Fetch a byte range, only accepting a real 206 partial response (never a full 200 body).""" + try: + res = session.get(url, headers={"Range": byte_range}, timeout=PROBE_TIMEOUT) + if getattr(res, "status_code", 0) != 206: + return None + content = getattr(res, "content", None) + return content if content else None + except Exception: + return None + + +def probe_bytes_duration(data: Optional[bytes], log: logging.Logger) -> Optional[float]: + """Pipe media bytes to ffprobe and return the format/stream duration in seconds.""" + if not data: + return None + ffprobe_bin = str(FFProbe) if FFProbe else "ffprobe" + try: + result = subprocess.run( + [ffprobe_bin, "-v", "error", "-show_entries", "format=duration:stream=duration", "-of", "json", "pipe:"], + input=data, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=60, + ) + info = json.loads(result.stdout or b"{}") + candidates = [info.get("format", {}).get("duration")] + candidates += [s.get("duration") for s in info.get("streams", [])] + for value in candidates: + if value: + return float(value) + log.debug(f"ffprobe found no duration (rc={result.returncode}): {result.stderr.decode(errors='replace')[:160]}") + return None + except (subprocess.SubprocessError, ValueError, json.JSONDecodeError) as e: + log.debug(f"ffprobe duration error: {e}") + return None diff --git a/reset/packages/envied/src/envied/core/utils/click_types.py b/reset/packages/envied/src/envied/core/utils/click_types.py new file mode 100644 index 0000000..a01031b --- /dev/null +++ b/reset/packages/envied/src/envied/core/utils/click_types.py @@ -0,0 +1,391 @@ +import re +from typing import Any, Optional, Union + +import click +from click.shell_completion import CompletionItem +from pywidevine.cdm import Cdm as WidevineCdm + +from envied.core.tracks.audio import Audio + + +class VideoCodecChoice(click.Choice): + """ + A custom Choice type for video codecs that accepts both enum names and values. + + Accepts both: + - Enum names: avc, hevc, vc1, vp8, vp9, av1 + - Enum values: H.264, H.265, VC-1, VP8, VP9, AV1 + """ + + def __init__(self, codec_enum): + self.codec_enum = codec_enum + # Build choices from both enum names and values + choices = [] + for codec in codec_enum: + choices.append(codec.name.lower()) # e.g., "avc", "hevc" + choices.append(codec.value) # e.g., "H.264", "H.265" + super().__init__(choices, case_sensitive=False) + + def convert(self, value: Any, param: Optional[click.Parameter] = None, ctx: Optional[click.Context] = None): + if not value: + return None + + # First try to convert using the parent class + converted_value = super().convert(value, param, ctx) + + # Now map the converted value back to the enum + for codec in self.codec_enum: + if converted_value.lower() == codec.name.lower(): + return codec + if converted_value == codec.value: + return codec + + # This shouldn't happen if the parent conversion worked + self.fail(f"'{value}' is not a valid video codec", param, ctx) + + +class MultipleVideoCodecChoice(VideoCodecChoice): + """ + A multiple-value variant of VideoCodecChoice that accepts comma-separated codecs. + + Accepts both enum names and values, e.g.: ``-v hevc,avc`` or ``-v H.264,H.265`` + """ + + name = "multiple_video_codec_choice" + + def convert( + self, value: Any, param: Optional[click.Parameter] = None, ctx: Optional[click.Context] = None + ) -> list[Any]: + if not value: + return [] + if isinstance(value, list): + values = value + elif isinstance(value, str): + values = value.split(",") + else: + self.fail(f"{value!r} is not a supported value.", param, ctx) + + chosen_values: list[Any] = [] + for v in values: + chosen_values.append(super().convert(v.strip(), param, ctx)) + return chosen_values + + +class SubtitleCodecChoice(click.Choice): + """ + A custom Choice type for subtitle codecs that accepts both enum names, values, and common aliases. + + Accepts: + - Enum names: subrip, substationalpha, substationalphav4, timedtextmarkuplang, webvtt, ftml, fvtt + - Enum values: SRT, SSA, ASS, TTML, VTT, STPP, WVTT + - Common aliases: srt (for SubRip) + """ + + def __init__(self, codec_enum): + self.codec_enum = codec_enum + # Build choices from enum names, values, and common aliases + choices = [] + aliases = {} + + for codec in codec_enum: + choices.append(codec.name.lower()) # e.g., "subrip", "webvtt" + + # Only add the value if it's different from common aliases + value_lower = codec.value.lower() + + # Add common aliases and track them + if codec.name == "SubRip": + if "srt" not in choices: + choices.append("srt") + aliases["srt"] = codec + elif codec.name == "WebVTT": + if "vtt" not in choices: + choices.append("vtt") + aliases["vtt"] = codec + # Also add the enum value if different + if value_lower != "vtt" and value_lower not in choices: + choices.append(value_lower) + elif codec.name == "SubStationAlpha": + if "ssa" not in choices: + choices.append("ssa") + aliases["ssa"] = codec + # Also add the enum value if different + if value_lower != "ssa" and value_lower not in choices: + choices.append(value_lower) + elif codec.name == "SubStationAlphav4": + if "ass" not in choices: + choices.append("ass") + aliases["ass"] = codec + # Also add the enum value if different + if value_lower != "ass" and value_lower not in choices: + choices.append(value_lower) + elif codec.name == "TimedTextMarkupLang": + if "ttml" not in choices: + choices.append("ttml") + aliases["ttml"] = codec + # Also add the enum value if different + if value_lower != "ttml" and value_lower not in choices: + choices.append(value_lower) + else: + # For other codecs, just add the enum value + if value_lower not in choices: + choices.append(value_lower) + + self.aliases = aliases + super().__init__(choices, case_sensitive=False) + + def convert(self, value: Any, param: Optional[click.Parameter] = None, ctx: Optional[click.Context] = None): + if not value: + return None + + # First try to convert using the parent class + converted_value = super().convert(value, param, ctx) + + # Check aliases first + if converted_value.lower() in self.aliases: + return self.aliases[converted_value.lower()] + + # Now map the converted value back to the enum + for codec in self.codec_enum: + if converted_value.lower() == codec.name.lower(): + return codec + if converted_value.lower() == codec.value.lower(): + return codec + + # This shouldn't happen if the parent conversion worked + self.fail(f"'{value}' is not a valid subtitle codec", param, ctx) + + +class ContextData: + def __init__(self, config: dict, cdm: WidevineCdm, proxy_providers: list, profile: Optional[str] = None): + self.config = config + self.cdm = cdm + self.proxy_providers = proxy_providers + self.profile = profile + + +class SeasonRange(click.ParamType): + name = "ep_range" + + MIN_EPISODE = 0 + MAX_EPISODE = 999 + + def parse_tokens(self, *tokens: str) -> list[str]: + """ + Parse multiple tokens or ranged tokens as '{s}x{e}' strings. + + Supports exclusioning by putting a `-` before the token. + + Example: + >>> sr = SeasonRange() + >>> sr.parse_tokens("S01E01") + ["1x1"] + >>> sr.parse_tokens("S02E01", "S02E03-S02E05") + ["2x1", "2x3", "2x4", "2x5"] + >>> sr.parse_tokens("S01-S05", "-S03", "-S02E01") + ["1x0", "1x1", ..., "2x0", (...), "2x2", (...), "4x0", ..., "5x0", ...] + """ + if len(tokens) == 0: + return [] + computed: list = [] + exclusions: list = [] + for token in tokens: + exclude = token.startswith("-") + if exclude: + token = token[1:] + parsed = [ + re.match(r"^S(?P\d+)(E(?P\d+))?$", x, re.IGNORECASE) for x in re.split(r"[:-]", token) + ] + if len(parsed) > 2: + self.fail(f"Invalid token, only a left and right range is acceptable: {token}") + if len(parsed) == 1: + parsed.append(parsed[0]) + if any(x is None for x in parsed): + self.fail(f"Invalid token, syntax error occurred: {token}") + from_season, from_episode = [ + int(v) if v is not None else self.MIN_EPISODE + for k, v in parsed[0].groupdict().items() + if parsed[0] # type: ignore[union-attr] + ] + to_season, to_episode = [ + int(v) if v is not None else self.MAX_EPISODE + for k, v in parsed[1].groupdict().items() + if parsed[1] # type: ignore[union-attr] + ] + if from_season > to_season: + self.fail(f"Invalid range, left side season cannot be bigger than right side season: {token}") + if from_season == to_season and from_episode > to_episode: + self.fail(f"Invalid range, left side episode cannot be bigger than right side episode: {token}") + for s in range(from_season, to_season + 1): + for e in range( + from_episode if s == from_season else 0, (self.MAX_EPISODE if s < to_season else to_episode) + 1 + ): + (computed if not exclude else exclusions).append(f"{s}x{e}") + for exclusion in exclusions: + if exclusion in computed: + computed.remove(exclusion) + return list(set(computed)) + + def convert( + self, value: str, param: Optional[click.Parameter] = None, ctx: Optional[click.Context] = None + ) -> list[str]: + return self.parse_tokens(*re.split(r"\s*[,;]\s*", value)) + + +class LanguageRange(click.ParamType): + name = "lang_range" + + def convert( + self, value: Union[str, list], param: Optional[click.Parameter] = None, ctx: Optional[click.Context] = None + ) -> list[str]: + if isinstance(value, list): + return value + if not value: + return [] + return re.split(r"\s*[,;]\s*", value) + + +class QualityList(click.ParamType): + name = "quality_list" + + def convert( + self, value: Union[str, list[str]], param: Optional[click.Parameter] = None, ctx: Optional[click.Context] = None + ) -> list[int]: + if not value: + return [] + if not isinstance(value, list): + value = value.split(",") + resolutions = [] + for resolution in value: + try: + resolutions.append(int(resolution.lower().rstrip("p"))) + except TypeError: + self.fail( + f"Expected string for int() conversion, got {resolution!r} of type {type(resolution).__name__}", + param, + ctx, + ) + except ValueError: + self.fail(f"{resolution!r} is not a valid integer", param, ctx) + return sorted(resolutions, reverse=True) + + +class AudioCodecList(click.ParamType): + """Parses comma-separated audio codecs like 'AAC,EC3'.""" + + name = "audio_codec_list" + + def __init__(self, codec_enum): + self.codec_enum = codec_enum + self._name_to_codec: dict[str, Audio.Codec] = {} + for codec in codec_enum: + self._name_to_codec[codec.name.lower()] = codec + self._name_to_codec[codec.value.lower()] = codec + + aliases = { + "eac3": "EC3", + "ddp": "EC3", + "vorbis": "OGG", + } + for alias, target in aliases.items(): + if target in codec_enum.__members__: + self._name_to_codec[alias] = codec_enum[target] + + def convert(self, value: Any, param: Optional[click.Parameter] = None, ctx: Optional[click.Context] = None) -> list: + if not value: + return [] + if isinstance(value, self.codec_enum): + return [value] + if isinstance(value, list): + if all(isinstance(v, self.codec_enum) for v in value): + return value + values = [str(v).strip() for v in value] + else: + values = [v.strip() for v in str(value).split(",")] + + codecs = [] + for val in values: + if not val: + continue + key = val.lower() + if key in self._name_to_codec: + codecs.append(self._name_to_codec[key]) + else: + valid = sorted(set(self._name_to_codec.keys())) + self.fail(f"'{val}' is not valid. Choices: {', '.join(valid)}", param, ctx) + return list(dict.fromkeys(codecs)) # Remove duplicates, preserve order + + +class MultipleChoice(click.Choice): + """ + The multiple choice type allows multiple values to be checked against + a fixed set of supported values. + + It internally uses and is based off of click.Choice. + """ + + name = "multiple_choice" + + def __repr__(self) -> str: + return f"MultipleChoice({list(self.choices)})" + + def convert( + self, value: Any, param: Optional[click.Parameter] = None, ctx: Optional[click.Context] = None + ) -> list[Any]: + if not value: + return [] + if isinstance(value, str): + values = value.split(",") + elif isinstance(value, list): + values = value + else: + self.fail(f"{value!r} is not a supported value.", param, ctx) + + chosen_values: list[Any] = [] + for value in values: + chosen_values.append(super().convert(value, param, ctx)) + + return chosen_values + + def shell_complete(self, ctx: click.Context, param: click.Parameter, incomplete: str) -> list[CompletionItem]: + """ + Complete choices that start with the incomplete value. + + Parameters: + ctx: Invocation context for this command. + param: The parameter that is requesting completion. + incomplete: Value being completed. May be empty. + """ + incomplete = incomplete.rsplit(",")[-1] + return super(self).shell_complete(ctx, param, incomplete) + + +class SlowDelayRange(click.ParamType): + """Parses a delay range string like '20-40' into a tuple of (min, max) seconds.""" + + name = "delay_range" + + def convert(self, value: Any, param: Optional[click.Parameter], ctx: Optional[click.Context]) -> tuple[int, int]: + if isinstance(value, tuple): + return value + + match = re.match(r"^(\d+)-(\d+)$", str(value)) + if not match: + self.fail(f"'{value}' is not a valid range. Use format: MIN-MAX (e.g., 20-40)", param, ctx) + + low, high = int(match.group(1)), int(match.group(2)) + if low < 20: + self.fail(f"Minimum delay must be at least 20 seconds, got {low}", param, ctx) + if low > high: + self.fail(f"Min ({low}) cannot be greater than max ({high})", param, ctx) + + return (low, high) + + +SEASON_RANGE = SeasonRange() +LANGUAGE_RANGE = LanguageRange() +QUALITY_LIST = QualityList() +AUDIO_CODEC_LIST = AudioCodecList(Audio.Codec) +SLOW_DELAY_RANGE = SlowDelayRange() + +# VIDEO_CODEC_CHOICE will be created dynamically when imported diff --git a/reset/packages/envied/src/envied/core/utils/collections.py b/reset/packages/envied/src/envied/core/utils/collections.py new file mode 100644 index 0000000..ae01bb2 --- /dev/null +++ b/reset/packages/envied/src/envied/core/utils/collections.py @@ -0,0 +1,51 @@ +import itertools +from typing import Any, Iterable, Iterator, Sequence, Tuple, Type, Union + + +def as_lists(*args: Any) -> Iterator[Any]: + """Converts any input objects to list objects.""" + for item in args: + yield item if isinstance(item, list) else [item] + + +def as_list(*args: Any) -> list: + """ + Convert any input objects to a single merged list object. + + Example: + >>> as_list('foo', ['buzz', 'bizz'], 'bazz', 'bozz', ['bar'], ['bur']) + ['foo', 'buzz', 'bizz', 'bazz', 'bozz', 'bar', 'bur'] + """ + return list(itertools.chain.from_iterable(as_lists(*args))) + + +def flatten(items: Any, ignore_types: Union[Type, Tuple[Type, ...]] = str) -> Iterator: + """ + Flattens items recursively. + + Example: + >>> list(flatten(["foo", [["bar", ["buzz", [""]], "bee"]]])) + ['foo', 'bar', 'buzz', '', 'bee'] + >>> list(flatten("foo")) + ['foo'] + >>> list(flatten({1}, set)) + [{1}] + """ + if isinstance(items, (Iterable, Sequence)) and not isinstance(items, ignore_types): + for i in items: + yield from flatten(i, ignore_types) + else: + yield items + + +def merge_dict(source: dict, destination: dict) -> None: + """Recursively merge Source into Destination in-place.""" + if not source: + return + for key, value in source.items(): + if isinstance(value, dict): + # get node or create one + node = destination.setdefault(key, {}) + merge_dict(value, node) + else: + destination[key] = value diff --git a/reset/packages/envied/src/envied/core/utils/dovi.py b/reset/packages/envied/src/envied/core/utils/dovi.py new file mode 100644 index 0000000..157b578 --- /dev/null +++ b/reset/packages/envied/src/envied/core/utils/dovi.py @@ -0,0 +1,130 @@ +"""Thin wrappers around dovi_tool subcommands used by DVFixup and Hybrid. + +Centralises argv construction, status spinners, and error handling so callers do not +re-implement subprocess plumbing per call site. Each wrapper: + +- Resolves `binaries.DoviTool` and raises EnvironmentError if missing. +- Delegates to `core.utils.subprocess.run_step` for execution, output validation, and + stderr-tail RuntimeError on failure. +- Returns captured stderr so callers can inspect specific failure modes (e.g. the + MAX_PQ_LUMINANCE retry path in extract_rpu). +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from typing import Optional + +from envied.core import binaries +from envied.core.utils.subprocess import run_step + + +def _require_dovi_tool() -> str: + if not binaries.DoviTool: + raise EnvironmentError("dovi_tool executable was not found but is required.") + return str(binaries.DoviTool) + + +def extract_rpu( + source: Path, + output: Path, + *, + mode: Optional[int] = 3, + status: Optional[str] = "Extracting DV RPU...", + label: str = "dovi_tool extract-rpu", +) -> bytes: + """Extract DV RPU NALs from a raw HEVC stream. `mode=None` skips the -m flag (untouched).""" + tool = _require_dovi_tool() + args: list = [tool] + if mode is not None: + args += ["-m", str(mode)] + args += ["extract-rpu", source, "-o", output] + return run_step(args, status=status, output=output, label=label) + + +def inject_rpu( + source: Path, + rpu: Path, + output: Path, + *, + status: Optional[str] = "Re-injecting DV RPU...", + label: str = "dovi_tool inject-rpu", +) -> bytes: + """Inject a DV RPU back into a raw HEVC stream, producing DV-signaled output.""" + tool = _require_dovi_tool() + return run_step( + [tool, "inject-rpu", "-i", source, "--rpu-in", rpu, "-o", output], + status=status, + output=output, + label=label, + ) + + +def editor( + source: Path, + json_spec: Path, + output: Path, + *, + status: Optional[str] = "Editing DV RPU...", + label: str = "dovi_tool editor", +) -> bytes: + """Apply a JSON edit spec to an RPU file.""" + tool = _require_dovi_tool() + return run_step( + [tool, "editor", "-i", source, "-j", json_spec, "-o", output], + status=status, + output=output, + label=label, + ) + + +def info_summary(rpu: Path) -> str: + """Return the textual summary (`dovi_tool info -i ... -s`) for an RPU file.""" + tool = _require_dovi_tool() + p = subprocess.run([tool, "info", "-i", str(rpu), "-s"], capture_output=True, text=True) + if p.returncode != 0: + raise RuntimeError(f"dovi_tool info failed: {(p.stderr or '')[-400:]}") + return p.stdout + + +def generate_from_hdr10plus( + extra_json: Path, + hdr10plus_json: Path, + output: Path, + *, + status: Optional[str] = "Generating DV RPU from HDR10+ metadata...", + label: str = "dovi_tool generate", +) -> bytes: + """Build a DV RPU from extracted HDR10+ metadata + an extra JSON descriptor.""" + tool = _require_dovi_tool() + return run_step( + [tool, "generate", "-j", extra_json, "--hdr10plus-json", hdr10plus_json, "-o", output], + status=status, + output=output, + label=label, + ) + + +def extract_rpu_with_fallback(source: Path, output: Path, *, label: str = "dovi_tool extract-rpu") -> bytes: + """Try `-m 3` first; on MAX_PQ_LUMINANCE error, retry untouched (no -m). Returns stderr. + + Used when the caller wants automatic normalization but cannot abort if the source + rejects mode-3 conversion. + """ + try: + return extract_rpu(source, output, mode=3, label=label) + except RuntimeError as e: + if "MAX_PQ_LUMINANCE" not in str(e): + raise + return extract_rpu(source, output, mode=None, status="Extracting DV RPU (untouched)...", label=label) + + +__all__ = ( + "extract_rpu", + "extract_rpu_with_fallback", + "inject_rpu", + "editor", + "info_summary", + "generate_from_hdr10plus", +) diff --git a/reset/packages/envied/src/envied/core/utils/gen_esn.py b/reset/packages/envied/src/envied/core/utils/gen_esn.py new file mode 100644 index 0000000..e58c4a3 --- /dev/null +++ b/reset/packages/envied/src/envied/core/utils/gen_esn.py @@ -0,0 +1,30 @@ +import logging +import os +import random +from datetime import datetime, timedelta + +log = logging.getLogger("NF-ESN") + + +def chrome_esn_generator(): + ESN_GEN = "".join(random.choice("0123456789ABCDEF") for _ in range(30)) + esn_file = ".esn" + + def gen_file(): + with open(esn_file, "w") as file: + file.write(f"NFCDIE-03-{ESN_GEN}") + + if not os.path.isfile(esn_file): + log.warning("Generating a new Chrome ESN") + gen_file() + + file_datetime = datetime.fromtimestamp(os.path.getmtime(esn_file)) + time_diff = datetime.now() - file_datetime + if time_diff > timedelta(hours=6): + log.warning("Old ESN detected, Generating a new Chrome ESN") + gen_file() + + with open(esn_file, "r") as f: + esn = f.read() + + return esn diff --git a/reset/packages/envied/src/envied/core/utils/ip_info.py b/reset/packages/envied/src/envied/core/utils/ip_info.py new file mode 100644 index 0000000..2ad5f58 --- /dev/null +++ b/reset/packages/envied/src/envied/core/utils/ip_info.py @@ -0,0 +1,252 @@ +from __future__ import annotations + +import logging +import time +from typing import Any, Callable, Optional + +import requests + +from envied.core.cacher import Cacher + +CACHE_KEY = "ip_info_v3" +CACHE_TTL = 86400 # 24 hours +PROVIDER_STATE_KEY = "ip_provider_state" +RATE_LIMIT_COOLDOWN = 300 # 5 minutes +REQUEST_TIMEOUT = 10 + +# Only these keys are persisted to the global cache. +GEO_CACHE_KEYS = ("country", "country_code") + +Fetcher = Callable[[requests.Session], Optional[dict]] + +log = logging.getLogger("ip_info") + + +class RateLimited(Exception): + """Raised by a provider fetcher when the upstream returns 429.""" + + +def normalize( + *, + country_code: str, + ip: str = "", + region: str = "", + city: str = "", + org: str = "", + asn: str = "", + as_name: str = "", + continent_code: str = "", +) -> Optional[dict]: + """Build the canonical IP-info dict, or None if no country code is present.""" + code = country_code.strip() + if not code: + return None + return { + "ip": ip, + "country": code.lower(), + "country_code": code.upper(), + "region": region, + "city": city, + "org": org, + "asn": asn, + "as_name": as_name, + "continent_code": continent_code.upper(), + } + + +def parse_ipinfo_lite(data: dict) -> Optional[dict]: + asn = (data.get("asn") or "").strip() + as_name = (data.get("as_name") or "").strip() + return normalize( + country_code=data.get("country_code") or "", + ip=data.get("ip") or "", + org=f"{asn} {as_name}".strip(), + asn=asn, + as_name=as_name, + continent_code=data.get("continent_code") or "", + ) + + +def parse_ipinfo(data: dict) -> Optional[dict]: + return normalize( + country_code=data.get("country") or "", + ip=data.get("ip") or "", + region=data.get("region") or "", + city=data.get("city") or "", + org=data.get("org") or "", + ) + + +def parse_ip_api_in(data: dict) -> Optional[dict]: + asn = (data.get("asn") or "").strip() + org_name = (data.get("organization") or "").strip() + return normalize( + country_code=data.get("country_code") or "", + ip=data.get("ip") or "", + region=data.get("region") or "", + city=data.get("city") or "", + org=f"{asn} {org_name}".strip(), + asn=asn, + as_name=org_name, + continent_code=data.get("continent_code") or "", + ) + + +def lookup_session(source: Optional[requests.Session]) -> requests.Session: + """ + Build a plain, retry-free requests session for IP geolocation. + + Geolocation needs no TLS fingerprinting, so we skip the impersonated rnet + session and the base session's urllib3 retry loop — both retry 429 internally, + which hides the response and defeats fast provider handover. With a bare session + a 429 comes straight back so we can move to the next provider immediately. Only + the proxy is carried over so proxied lookups still report the proxy's exit IP. + """ + sess = requests.Session() + proxies = getattr(source, "proxies", None) + if proxies: + proxy = proxies.get("all") or proxies.get("https") or proxies.get("http") + if proxy: + sess.proxies.update({"http": proxy, "https": proxy}) + return sess + + +def json_or_raise(response: requests.Response) -> Optional[dict]: + """Raise RateLimited on 429, return parsed JSON on 200, else None.""" + if response.status_code == 429: + raise RateLimited() + if response.status_code != 200: + return None + try: + return response.json() + except ValueError: + return None + + +def fetch_ipinfo_lite(token: str) -> Fetcher: + headers = {"Authorization": f"Bearer {token}"} + + def fetch(session: requests.Session) -> Optional[dict]: + payload = json_or_raise(session.get("https://api.ipinfo.io/lite/me", headers=headers, timeout=REQUEST_TIMEOUT)) + return parse_ipinfo_lite(payload) if payload else None + + return fetch + + +def fetch_ipinfo(session: requests.Session) -> Optional[dict]: + payload = json_or_raise(session.get("https://ipinfo.io/json", timeout=REQUEST_TIMEOUT)) + return parse_ipinfo(payload) if payload else None + + +def fetch_ip_api_in(session: requests.Session) -> Optional[dict]: + """ip-api.in has no /me endpoint — resolve IP via ipify first, then look it up.""" + ip_resp = session.get("https://api.ipify.org", timeout=REQUEST_TIMEOUT) + if ip_resp.status_code == 429: + raise RateLimited() + ip = (ip_resp.text or "").strip() if ip_resp.status_code == 200 else "" + if not ip: + return None + payload = json_or_raise(session.get(f"https://ip-api.in/api/v1/ip/{ip}", timeout=REQUEST_TIMEOUT)) + if not payload or not payload.get("success"): + return None + return parse_ip_api_in(payload.get("data") or {}) + + +def build_providers() -> list[tuple[str, Fetcher]]: + """Return ordered (name, fetcher) pairs. Token is read at call time.""" + from envied.core.config import config + + providers: list[tuple[str, Fetcher]] = [] + token = (getattr(config, "ipinfo_api_key", "") or "").strip() + if token: + providers.append(("ipinfo_lite", fetch_ipinfo_lite(token))) + providers.append(("ipinfo", fetch_ipinfo)) + providers.append(("ip_api_in", fetch_ip_api_in)) + return providers + + +def purge_stale_cache() -> None: + """Delete superseded ip_info cache files (older CACHE_KEY versions).""" + from envied.core.config import config + + global_dir = config.directories.cache / "global" + for stale in global_dir.glob("ip_info_v*.json"): + if stale.stem != CACHE_KEY: + stale.unlink(missing_ok=True) + + +def load_provider_state(cacher: Cacher) -> dict[str, Any]: + return cacher.data if cacher and not cacher.expired and isinstance(cacher.data, dict) else {} + + +def get_ip_info( + session: Optional[requests.Session] = None, + *, + cached: bool = False, +) -> Optional[dict]: + """ + Look up IP/geolocation info via ipinfo.io (Lite when `ipinfo_api_key` configured) + with fallback to ip-api.in. + + Live lookups return a dict with `ip`, `country` (lowercase ISO2), `country_code` + (uppercase ISO2), `region`, `city`, `org`, `asn`, `as_name`, `continent_code` and + `_provider`. Cached lookups return only `country`/`country_code` (see GEO_CACHE_KEYS). + Returns None if every provider fails. + + Args: + session: Optional requests session. If a proxied session is passed, the + returned info reflects the proxy's exit IP. Auth headers for ipinfo + are sent per-request; never mutated onto session.headers. + cached: When True, read/write a 24h Cacher-backed entry. Use only for + local IP lookups — never with a proxied session. + """ + cache = None + if cached: + purge_stale_cache() + cache = Cacher("global").get(CACHE_KEY) + if cache and not cache.expired and cache.data: + return cache.data + + state_cache = Cacher("global").get(PROVIDER_STATE_KEY) + state = load_provider_state(state_cache) + now = time.time() + + def on_cooldown(item: tuple[str, Fetcher]) -> int: + rate_limited_at = (state.get(item[0]) or {}).get("rate_limited_at", 0) + return 1 if (now - rate_limited_at) < RATE_LIMIT_COOLDOWN else 0 + + providers = sorted(build_providers(), key=on_cooldown) + sess = lookup_session(session) + + for name, fetcher in providers: + log.debug(f"Trying IP provider: {name}") + try: + normalized = fetcher(sess) + except RateLimited: + log.warning(f"Provider {name} returned 429 (rate limited), trying next provider") + entry = state.setdefault(name, {}) + entry["rate_limited_at"] = now + entry["rate_limit_count"] = entry.get("rate_limit_count", 0) + 1 + state_cache.set(state, expiration=RATE_LIMIT_COOLDOWN) + continue + except Exception as e: + log.debug(f"Provider {name} failed with exception: {e}") + continue + + if not normalized: + log.debug(f"Provider {name} returned no usable data") + continue + + normalized["_provider"] = name + log.debug(f"Successfully got IP info from provider: {name}") + + if name in state and state[name].pop("rate_limited_at", None) is not None: + state_cache.set(state, expiration=RATE_LIMIT_COOLDOWN) + + if cache is not None: + cache.set({k: normalized.get(k, "") for k in GEO_CACHE_KEYS}, expiration=CACHE_TTL) + + return normalized + + log.warning("All IP geolocation providers failed") + return None diff --git a/reset/packages/envied/src/envied/core/utils/language_tags.py b/reset/packages/envied/src/envied/core/utils/language_tags.py new file mode 100644 index 0000000..d827273 --- /dev/null +++ b/reset/packages/envied/src/envied/core/utils/language_tags.py @@ -0,0 +1,79 @@ +"""Language tag rule engine for output filename templates.""" + +from __future__ import annotations + +import logging +from typing import Any, Sequence + +from langcodes import Language + +from envied.core.utilities import is_close_match + +log = logging.getLogger(__name__) + + +def evaluate_language_tag( + rules: list[dict[str, Any]], + audio_languages: Sequence[Language], + subtitle_languages: Sequence[Language], +) -> str: + """Evaluate language tag rules against selected tracks. + + Rules are evaluated in order; the first matching rule's tag is returned. + Returns empty string if no rules match. + + Args: + rules: List of rule dicts from config, each with conditions and a ``tag``. + audio_languages: Languages of the selected audio tracks. + subtitle_languages: Languages of the selected subtitle tracks. + + Returns: + The tag string from the first matching rule, or ``""`` if none match. + """ + for rule in rules: + tag = rule.get("tag") + if not tag: + log.warning("Language tag rule missing 'tag' field, skipping: %s", rule) + continue + + if _rule_matches(rule, audio_languages, subtitle_languages): + log.debug("Language tag rule matched: %s -> %s", rule, tag) + return str(tag) + + return "" + + +def _rule_matches( + rule: dict[str, Any], + audio_languages: Sequence[Language], + subtitle_languages: Sequence[Language], +) -> bool: + """Check if all conditions in a rule are satisfied.""" + has_condition = False + + audio_lang = rule.get("audio") + if audio_lang is not None: + has_condition = True + if not is_close_match(audio_lang, list(audio_languages)): + return False + + subs_contain = rule.get("subs_contain") + if subs_contain is not None: + has_condition = True + if not is_close_match(subs_contain, list(subtitle_languages)): + return False + + subs_contain_all = rule.get("subs_contain_all") + if subs_contain_all is not None: + has_condition = True + if not isinstance(subs_contain_all, list): + subs_contain_all = [subs_contain_all] + for lang in subs_contain_all: + if not is_close_match(lang, list(subtitle_languages)): + return False + + if not has_condition: + log.warning("Language tag rule has no conditions, skipping: %s", rule) + return False + + return True diff --git a/reset/packages/envied/src/envied/core/utils/osenvironment.py b/reset/packages/envied/src/envied/core/utils/osenvironment.py new file mode 100644 index 0000000..40b3ecc --- /dev/null +++ b/reset/packages/envied/src/envied/core/utils/osenvironment.py @@ -0,0 +1,24 @@ +import platform + + +def get_os_arch(name: str) -> str: + """Builds a name-os-arch based on the input name, system, architecture.""" + os_name = platform.system().lower() + os_arch = platform.machine().lower() + + # Map platform.system() output to desired OS name + if os_name == "windows": + os_name = "win" + elif os_name == "darwin": + os_name = "osx" + else: + os_name = "linux" + + # Map platform.machine() output to desired architecture + if os_arch in ["x86_64", "amd64"]: + os_arch = "x64" + elif os_arch == "arm64": + os_arch = "arm64" + + # Construct the dependency name in the desired format using the input name + return f"{name}-{os_name}-{os_arch}" diff --git a/reset/packages/envied/src/envied/core/utils/selector.py b/reset/packages/envied/src/envied/core/utils/selector.py new file mode 100644 index 0000000..50fb6e9 --- /dev/null +++ b/reset/packages/envied/src/envied/core/utils/selector.py @@ -0,0 +1,439 @@ +import sys + +import click +from rich.console import Group +from rich.live import Live +from rich.padding import Padding +from rich.table import Table +from rich.text import Text + +from envied.core.console import console + +IS_WINDOWS = sys.platform == "win32" +if IS_WINDOWS: + import msvcrt + + +class Selector: + """ + A custom interactive selector class using the Rich library. + Allows for multi-selection of items with pagination and collapsible headers. + """ + + def __init__( + self, + options: list[str], + cursor_style: str = "pink", + text_style: str = "text", + page_size: int = 8, + minimal_count: int = 0, + dependencies: dict[int, list[int]] = None, + collapse_on_start: bool = False, + ): + """ + Initialize the Selector. + + Args: + options: List of strings to select from. + cursor_style: Rich style for the highlighted cursor item. + text_style: Rich style for normal items. + page_size: Number of items to show per page. + minimal_count: Minimum number of items that must be selected. + dependencies: Dictionary mapping parent index to list of child indices. + collapse_on_start: If True, child items are hidden initially. + """ + self.options = options + self.cursor_style = cursor_style + self.text_style = text_style + self.page_size = page_size + self.minimal_count = minimal_count + self.dependencies = dependencies or {} + + # Parent-Child mapping for quick lookup + self.child_to_parent = {} + for parent, children in self.dependencies.items(): + for child in children: + self.child_to_parent[child] = parent + + self.cursor_index = 0 + self.selected_indices = set() + self.scroll_offset = 0 + + # Tree view state + self.expanded_headers = set() + if not collapse_on_start: + # Expand all by default + self.expanded_headers.update(self.dependencies.keys()) + + def get_visible_indices(self) -> list[int]: + """ + Returns a sorted list of indices that should be currently visible. + A child is visible only if its parent is in self.expanded_headers. + """ + visible = [] + for idx in range(len(self.options)): + # If it's a child, check if parent is expanded + if idx in self.child_to_parent: + parent = self.child_to_parent[idx] + if parent in self.expanded_headers: + visible.append(idx) + else: + # It's a header or independent item, always visible + visible.append(idx) + return visible + + def get_renderable(self): + """ + Constructs and returns the renderable object (Table + Info) for the current state. + """ + visible_indices = self.get_visible_indices() + + # Adjust scroll offset to ensure cursor is visible + if self.cursor_index not in visible_indices: + # Fallback if cursor got hidden (should be handled in move, but safety check) + self.cursor_index = visible_indices[0] if visible_indices else 0 + + try: + cursor_visual_pos = visible_indices.index(self.cursor_index) + except ValueError: + cursor_visual_pos = 0 + self.cursor_index = visible_indices[0] + + # Calculate logical page start/end based on VISIBLE items + start_idx = self.scroll_offset + end_idx = start_idx + self.page_size + + # Dynamic scroll adjustment + if cursor_visual_pos < start_idx: + self.scroll_offset = cursor_visual_pos + elif cursor_visual_pos >= end_idx: + self.scroll_offset = cursor_visual_pos - self.page_size + 1 + + # Re-calc render range + render_indices = visible_indices[self.scroll_offset : self.scroll_offset + self.page_size] + + table = Table(show_header=False, show_edge=False, box=None, pad_edge=False, padding=(0, 1, 0, 0)) + table.add_column("Indicator", justify="right", no_wrap=True) + table.add_column("Option", overflow="ellipsis", no_wrap=True) + + for idx in render_indices: + option = self.options[idx] + is_cursor = idx == self.cursor_index + is_selected = idx in self.selected_indices + + symbol = "[X]" if is_selected else "[ ]" + style = self.cursor_style if is_cursor else self.text_style + indicator_text = Text(f"{symbol}", style=style) + + content_text = Text.from_markup(f"{option}") + content_text.style = style + + table.add_row(indicator_text, content_text) + + # Fill empty rows to maintain height + rows_rendered = len(render_indices) + for _ in range(self.page_size - rows_rendered): + table.add_row(Text(" "), Text(" ")) + + total_visible = len(visible_indices) + total_pages = (total_visible + self.page_size - 1) // self.page_size + if total_pages == 0: + total_pages = 1 + current_page = (self.scroll_offset // self.page_size) + 1 + + if self.dependencies: + info_text = Text( + f"\n[Space]: Toggle [a]: All [e]: Fold/Unfold [E]: All Fold/Unfold\n[Enter]: Confirm [↑/↓]: Move [←/→]: Page (Page {current_page}/{total_pages})", + style="gray", + ) + else: + info_text = Text( + f"\n[Space]: Toggle [a]: All [←/→]: Page [Enter]: Confirm (Page {current_page}/{total_pages})", + style="gray", + ) + + return Padding(Group(table, info_text), (0, 5)) + + def move_cursor(self, delta: int): + """ + Moves the cursor up or down through VISIBLE items only. + """ + visible_indices = self.get_visible_indices() + if not visible_indices: + return + + try: + current_visual_idx = visible_indices.index(self.cursor_index) + except ValueError: + current_visual_idx = 0 + + new_visual_idx = (current_visual_idx + delta) % len(visible_indices) + self.cursor_index = visible_indices[new_visual_idx] + + def change_page(self, delta: int): + """ + Changes the current page view by the specified delta (previous/next page). + """ + visible_indices = self.get_visible_indices() + if not visible_indices: + return + + total_visible = len(visible_indices) + + # Calculate current logical page + current_page = self.scroll_offset // self.page_size + total_pages = (total_visible + self.page_size - 1) // self.page_size + + new_page = current_page + delta + + if 0 <= new_page < total_pages: + self.scroll_offset = new_page * self.page_size + + # Move cursor to top of new page + try: + # Calculate what visual index corresponds to the start of the new page + new_visual_cursor = self.scroll_offset + if new_visual_cursor < len(visible_indices): + self.cursor_index = visible_indices[new_visual_cursor] + else: + self.cursor_index = visible_indices[-1] + except IndexError: + pass + + def toggle_selection(self): + """ + Toggles the selection state of the item currently under the cursor. + """ + target_indices = {self.cursor_index} + + if self.cursor_index in self.dependencies: + target_indices.update(self.dependencies[self.cursor_index]) + + should_select = self.cursor_index not in self.selected_indices + + if should_select: + self.selected_indices.update(target_indices) + else: + self.selected_indices.difference_update(target_indices) + + def toggle_expand(self): + """ + Expands or collapses the current header. + """ + if self.cursor_index in self.dependencies: + if self.cursor_index in self.expanded_headers: + self.expanded_headers.remove(self.cursor_index) + else: + self.expanded_headers.add(self.cursor_index) + + def toggle_expand_all(self): + """ + Toggles expansion state of ALL headers. + If all are expanded -> Collapse all. + Otherwise -> Expand all. + """ + if not self.dependencies: + return + all_headers = set(self.dependencies.keys()) + if self.expanded_headers == all_headers: + self.expanded_headers.clear() + else: + self.expanded_headers = all_headers.copy() + + def toggle_all(self): + """ + Toggles the selection of all items. + """ + if len(self.selected_indices) == len(self.options): + self.selected_indices.clear() + else: + self.selected_indices = set(range(len(self.options))) + + def get_input_windows(self): + """ + Captures and parses keyboard input on Windows systems using msvcrt. + Returns command strings like 'UP', 'DOWN', 'ENTER', etc. + """ + key = msvcrt.getch() + # Ctrl+C (0x03) or ESC (0x1b) + if key == b"\x03" or key == b"\x1b": + return "CANCEL" + # Special keys prefix (Arrow keys, etc., send 0xe0 or 0x00 first) + if key == b"\xe0" or key == b"\x00": + try: + key = msvcrt.getch() + if key == b"H": # Arrow Up + return "UP" + if key == b"P": # Arrow Down + return "DOWN" + if key == b"K": # Arrow Left + return "LEFT" + if key == b"M": # Arrow Right + return "RIGHT" + except Exception: + pass + + try: + char = key.decode("utf-8", errors="ignore") + except Exception: + return None + + if char in ("\r", "\n"): + return "ENTER" + if char == " ": + return "SPACE" + if char in ("q", "Q"): + return "QUIT" + if char in ("a", "A"): + return "ALL" + if char == "e": + return "EXPAND" + if char == "E": + return "EXPAND_ALL" + if char in ("w", "W", "k", "K"): + return "UP" + if char in ("s", "S", "j", "J"): + return "DOWN" + if char in ("h", "H"): + return "LEFT" + if char in ("d", "D", "l", "L"): + return "RIGHT" + return None + + def get_input_unix(self): + """ + Captures and parses keyboard input on Unix/Linux systems using click.getchar(). + Returns command strings like 'UP', 'DOWN', 'ENTER', etc. + """ + char = click.getchar() + # Ctrl+C + if char == "\x03": + return "CANCEL" + + # ANSI Escape Sequences for Arrow Keys + mapping = { + "\x1b[A": "UP", # Escape + [ + A + "\x1b[B": "DOWN", # Escape + [ + B + "\x1b[C": "RIGHT", # Escape + [ + C + "\x1b[D": "LEFT", # Escape + [ + D + } + if char in mapping: + return mapping[char] + + # Handling manual Escape sequences + if char == "\x1b": # ESC + try: + next1 = click.getchar() + if next1 in ("[", "O"): # Sequence indicators + next2 = click.getchar() + if next2 == "A": # Arrow Up + return "UP" + if next2 == "B": # Arrow Down + return "DOWN" + if next2 == "C": # Arrow Right + return "RIGHT" + if next2 == "D": # Arrow Left + return "LEFT" + return "CANCEL" + except Exception: + return "CANCEL" + + if char in ("\r", "\n"): + return "ENTER" + if char == " ": + return "SPACE" + if char in ("q", "Q"): + return "QUIT" + if char in ("a", "A"): + return "ALL" + if char == "e": + return "EXPAND" + if char == "E": + return "EXPAND_ALL" + if char in ("w", "W", "k", "K"): + return "UP" + if char in ("s", "S", "j", "J"): + return "DOWN" + if char in ("h", "H"): + return "LEFT" + if char in ("d", "D", "l", "L"): + return "RIGHT" + return None + + def run(self) -> list[int]: + """ + Starts the main event loop for the selector. + Renders the UI and processes input until confirmed or cancelled. + + Returns: + list[int]: A sorted list of selected indices. + """ + try: + with Live(self.get_renderable(), console=console, auto_refresh=False, transient=True) as live: + while True: + live.update(self.get_renderable(), refresh=True) + if IS_WINDOWS: + action = self.get_input_windows() + else: + action = self.get_input_unix() + + if action == "UP": + self.move_cursor(-1) + elif action == "DOWN": + self.move_cursor(1) + elif action == "LEFT": + self.change_page(-1) + elif action == "RIGHT": + self.change_page(1) + elif action == "EXPAND": + self.toggle_expand() + elif action == "EXPAND_ALL": + self.toggle_expand_all() + elif action == "SPACE": + self.toggle_selection() + elif action == "ALL": + self.toggle_all() + elif action in ("ENTER", "QUIT"): + if len(self.selected_indices) >= self.minimal_count: + return sorted(list(self.selected_indices)) + elif action == "CANCEL": + raise KeyboardInterrupt + except KeyboardInterrupt: + return [] + + +def select_multiple( + options: list[str], + minimal_count: int = 1, + page_size: int = 8, + return_indices: bool = True, + cursor_style: str = "pink", + collapse_on_start: bool = False, + **kwargs, +) -> list[int]: + """ + Drop-in replacement using custom Selector with global console. + + Args: + options: List of options to display. + minimal_count: Minimum number of selections required. + page_size: Number of items per page. + return_indices: If True, returns indices; otherwise returns the option strings. + cursor_style: Style color for the cursor. + collapse_on_start: If True, child items are hidden initially. + """ + selector = Selector( + options=options, + cursor_style=cursor_style, + text_style="text", + page_size=page_size, + minimal_count=minimal_count, + collapse_on_start=collapse_on_start, + **kwargs, + ) + + selected_indices = selector.run() + + if return_indices: + return selected_indices + return [options[i] for i in selected_indices] diff --git a/reset/packages/envied/src/envied/core/utils/sslciphers.py b/reset/packages/envied/src/envied/core/utils/sslciphers.py new file mode 100644 index 0000000..e3eb210 --- /dev/null +++ b/reset/packages/envied/src/envied/core/utils/sslciphers.py @@ -0,0 +1,77 @@ +import ssl +from typing import Optional + +from requests.adapters import HTTPAdapter + + +class SSLCiphers(HTTPAdapter): + """ + Custom HTTP Adapter to change the TLS Cipher set and security requirements. + + Security Level may optionally be provided. A level above 0 must be used at all times. + A list of Security Levels and their security is listed below. Usually 2 is used by default. + Do not set the Security level via @SECLEVEL in the cipher list. + + Level 0: + Everything is permitted. This retains compatibility with previous versions of OpenSSL. + + Level 1: + The security level corresponds to a minimum of 80 bits of security. Any parameters + offering below 80 bits of security are excluded. As a result RSA, DSA and DH keys + shorter than 1024 bits and ECC keys shorter than 160 bits are prohibited. All export + cipher suites are prohibited since they all offer less than 80 bits of security. SSL + version 2 is prohibited. Any cipher suite using MD5 for the MAC is also prohibited. + + Level 2: + Security level set to 112 bits of security. As a result RSA, DSA and DH keys shorter + than 2048 bits and ECC keys shorter than 224 bits are prohibited. In addition to the + level 1 exclusions any cipher suite using RC4 is also prohibited. SSL version 3 is + also not allowed. Compression is disabled. + + Level 3: + Security level set to 128 bits of security. As a result RSA, DSA and DH keys shorter + than 3072 bits and ECC keys shorter than 256 bits are prohibited. In addition to the + level 2 exclusions cipher suites not offering forward secrecy are prohibited. TLS + versions below 1.1 are not permitted. Session tickets are disabled. + + Level 4: + Security level set to 192 bits of security. As a result RSA, DSA and DH keys shorter + than 7680 bits and ECC keys shorter than 384 bits are prohibited. Cipher suites using + SHA1 for the MAC are prohibited. TLS versions below 1.2 are not permitted. + + Level 5: + Security level set to 256 bits of security. As a result RSA, DSA and DH keys shorter + than 15360 bits and ECC keys shorter than 512 bits are prohibited. + """ + + def __init__(self, cipher_list: Optional[str] = None, security_level: int = 0, *args, **kwargs): + if cipher_list: + if not isinstance(cipher_list, str): + raise TypeError(f"Expected cipher_list to be a str, not {cipher_list!r}") + if "@SECLEVEL" in cipher_list: + raise ValueError("You must not specify the Security Level manually in the cipher list.") + if not isinstance(security_level, int): + raise TypeError(f"Expected security_level to be an int, not {security_level!r}") + if security_level not in range(6): + raise ValueError(f"The security_level must be a value between 0 and 5, not {security_level}") + + if not cipher_list: + # cpython's default cipher list differs to Python-requests cipher list + cipher_list = "DEFAULT" + + cipher_list += f":@SECLEVEL={security_level}" + + ctx = ssl.create_default_context() + ctx.check_hostname = False # For some reason this is needed to avoid a verification error + ctx.set_ciphers(cipher_list) + + self._ssl_context = ctx + super().__init__(*args, **kwargs) + + def init_poolmanager(self, *args, **kwargs): + kwargs["ssl_context"] = self._ssl_context + return super().init_poolmanager(*args, **kwargs) + + def proxy_manager_for(self, *args, **kwargs): + kwargs["ssl_context"] = self._ssl_context + return super().proxy_manager_for(*args, **kwargs) diff --git a/reset/packages/envied/src/envied/core/utils/subprocess.py b/reset/packages/envied/src/envied/core/utils/subprocess.py new file mode 100644 index 0000000..d712402 --- /dev/null +++ b/reset/packages/envied/src/envied/core/utils/subprocess.py @@ -0,0 +1,57 @@ +import json +import subprocess +from pathlib import Path +from typing import Optional, Sequence, Union + +from envied.core import binaries +from envied.core.console import console + + +def ffprobe(uri: Union[bytes, Path]) -> dict: + """Use ffprobe on the provided data to get stream information.""" + if not binaries.FFProbe: + raise EnvironmentError('FFProbe executable "ffprobe" not found but is required.') + + args = [binaries.FFProbe, "-v", "quiet", "-of", "json", "-show_streams"] + if isinstance(uri, Path): + args.extend( + ["-f", "lavfi", "-i", "movie={}[out+subcc]".format(str(uri).replace("\\", "/").replace(":", "\\\\:"))] + ) + elif isinstance(uri, bytes): + args.append("pipe:") + try: + ff = subprocess.run(args, input=uri if isinstance(uri, bytes) else None, check=True, capture_output=True) + except subprocess.CalledProcessError: + return {} + return json.loads(ff.stdout.decode("utf8")) + + +def run_step( + args: Sequence[Union[str, Path]], + *, + status: Optional[str] = None, + output: Optional[Path] = None, + label: str = "subprocess step", +) -> bytes: + """Run a CLI step that writes to `output` (when provided). Returns stderr bytes. + + Raises RuntimeError with the stderr tail when the process exits non-zero, or when + `output` is given and does not exist / is empty after the run. + """ + if output is not None: + output.unlink(missing_ok=True) + + str_args = [str(a) for a in args] + if status: + with console.status(status, spinner="dots"): + p = subprocess.run(str_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + else: + p = subprocess.run(str_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + stderr = p.stderr or b"" + bad_output = output is not None and (not output.exists() or output.stat().st_size == 0) + if p.returncode or bad_output: + if output is not None: + output.unlink(missing_ok=True) + raise RuntimeError(f"{label} failed: {stderr.decode(errors='replace')[-400:]}") + return stderr diff --git a/reset/packages/envied/src/envied/core/utils/tags.py b/reset/packages/envied/src/envied/core/utils/tags.py new file mode 100644 index 0000000..6836d06 --- /dev/null +++ b/reset/packages/envied/src/envied/core/utils/tags.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import logging +import subprocess +import tempfile +from pathlib import Path +from typing import Optional +from xml.sax.saxutils import escape + +from envied.core import binaries +from envied.core.config import config +from envied.core.providers import (ExternalIds, MetadataResult, enrich_ids, fetch_external_ids, fuzzy_match, + get_available_providers, get_provider, search_metadata) +from envied.core.titles.episode import Episode +from envied.core.titles.movie import Movie +from envied.core.titles.title import Title + +log = logging.getLogger("TAGS") + + +def apply_tags(path: Path, tags: dict[str, str]) -> None: + if not tags: + return + if not binaries.Mkvpropedit: + log.debug("mkvpropedit not found on PATH; skipping tags") + return + log.debug("Applying tags to %s: %s", path, tags) + xml_lines = ['', "", " ", " "] + for name, value in tags.items(): + xml_lines.append(f" {escape(name)}{escape(value)}") + xml_lines.extend([" ", ""]) + with tempfile.NamedTemporaryFile("w", suffix=".xml", delete=False, encoding="utf-8") as f: + f.write("\n".join(xml_lines)) + tmp_path = Path(f.name) + try: + result = subprocess.run( + [str(binaries.Mkvpropedit), str(path), "--tags", f"global:{tmp_path}"], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + log.warning("mkvpropedit failed (exit %d): %s", result.returncode, result.stderr.strip()) + else: + log.debug("Tags applied via mkvpropedit") + finally: + tmp_path.unlink(missing_ok=True) + + +def _build_tags_from_ids(ids: ExternalIds, kind: str) -> dict[str, str]: + """Build standard MKV tags from external IDs.""" + tags: dict[str, str] = {} + if ids.imdb_id: + tags["IMDB"] = ids.imdb_id + if ids.tmdb_id and ids.tmdb_kind: + tags["TMDB"] = f"{ids.tmdb_kind}/{ids.tmdb_id}" + if ids.tvdb_id: + prefix = "movies" if kind == "movie" else "series" + tags["TVDB2"] = f"{prefix}/{ids.tvdb_id}" + return tags + + +def tag_file( + path: Path, + title: Title, + tmdb_id: Optional[int] = None, + imdb_id: Optional[str] = None, +) -> None: + log.debug("Tagging file %s with title %r", path, title) + custom_tags: dict[str, str] = {} + + if config.tag and config.tag_group_name: + custom_tags["Group"] = config.tag + description = getattr(title, "description", None) + if description: + if len(description) > 255: + truncated = description[:255] + if " " in truncated: + truncated = truncated.rsplit(" ", 1)[0] + description = truncated + "..." + custom_tags["Description"] = description + + if isinstance(title, Movie): + kind = "movie" + name = title.name + year = title.year + elif isinstance(title, Episode): + kind = "tv" + name = title.title + year = title.year + else: + apply_tags(path, custom_tags) + return + + standard_tags: dict[str, str] = {} + + if config.tag_imdb_tmdb: + try: + providers = get_available_providers() + if not providers: + log.debug("No metadata providers available; skipping tag lookup") + apply_tags(path, custom_tags) + return + + result: Optional[MetadataResult] = None + + # Direct ID lookup path + if imdb_id: + imdbapi = get_provider("imdbapi") + if imdbapi: + result = imdbapi.get_by_id(imdb_id, kind) + if result: + result.external_ids.imdb_id = imdb_id + enrich_ids(result) + elif tmdb_id is not None: + tmdb = get_provider("tmdb") + if tmdb: + result = tmdb.get_by_id(tmdb_id, kind) + if result: + ext = tmdb.get_external_ids(tmdb_id, kind) + result.external_ids = ext + else: + # Search across providers in priority order + result = search_metadata(name, year, kind) + + # If we got a TMDB ID from search but no full external IDs, fetch them + if result and result.external_ids.tmdb_id and not result.external_ids.imdb_id: + ext = fetch_external_ids(result.external_ids.tmdb_id, kind) + if ext.imdb_id: + result.external_ids.imdb_id = ext.imdb_id + if ext.tvdb_id: + result.external_ids.tvdb_id = ext.tvdb_id + + if result and result.external_ids: + standard_tags = _build_tags_from_ids(result.external_ids, kind) + except Exception as e: + log.warning("Metadata lookup failed, applying custom tags only: %s", e) + + apply_tags(path, {**custom_tags, **standard_tags}) + + +__all__ = [ + "apply_tags", + "fuzzy_match", + "tag_file", +] diff --git a/reset/packages/envied/src/envied/core/utils/template_formatter.py b/reset/packages/envied/src/envied/core/utils/template_formatter.py new file mode 100644 index 0000000..1c36684 --- /dev/null +++ b/reset/packages/envied/src/envied/core/utils/template_formatter.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +import logging +import re +from typing import Any + +from envied.core.utilities import sanitize_filename + +log = logging.getLogger(__name__) + + +class TemplateFormatter: + """ + Template formatter for custom filename patterns. + + Supports variable substitution and conditional variables. + Example: '{title}.{year}.{quality?}.{source}-{tag}' + """ + + def __init__(self, template: str): + """Initialize the template formatter. + + Args: + template: Template string with variables in {variable} format + """ + self.template = template + self.variables = self._extract_variables() + + def _extract_variables(self) -> list[str]: + """Extract all variables from the template.""" + pattern = r"\{([^}]+)\}" + matches = re.findall(pattern, self.template) + return [match.strip() for match in matches] + + def format(self, context: dict[str, Any]) -> str: + """Format the template with the provided context. + + Args: + context: Dictionary containing variable values + + Returns: + Formatted filename string + + Raises: + ValueError: If required template variables are missing from context + """ + is_valid, missing_vars = self.validate(context) + if not is_valid: + error_msg = f"Missing required template variables: {', '.join(missing_vars)}" + log.error(error_msg) + raise ValueError(error_msg) + + try: + result = self.template + + for variable in self.variables: + placeholder = "{" + variable + "}" + is_conditional = variable.endswith("?") + + if is_conditional: + var_name = variable[:-1] + value = context.get(var_name, "") + + if value: + safe_value = str(value).strip() + result = result.replace(placeholder, safe_value) + else: + # Remove the placeholder and consume the adjacent separator on one side + # e.g. "{disc?}-{track}" → "{track}" when disc is empty + # e.g. "{title}.{edition?}.{quality}" → "{title}.{quality}" when edition is empty + def _remove_conditional(m: re.Match) -> str: + s = m.group(0) + has_left = s[0] in ".- " + has_right = s[-1] in ".- " + if has_left and has_right: + if s[-1] == "-": + return s[-1] + return s[0] + return "" + + result = re.sub( + rf"[\.\s\-]?{re.escape(placeholder)}[\.\s\-]?", + _remove_conditional, + result, + count=1, + ) + else: + value = context.get(variable, "") + if value is None: + log.warning(f"Template variable '{variable}' is None, using empty string") + value = "" + + safe_value = str(value).strip() + result = result.replace(placeholder, safe_value) + + # Clean up multiple consecutive dots/separators and other artifacts + result = re.sub(r"\.{2,}", ".", result) # Multiple dots -> single dot + result = re.sub(r"\s{2,}", " ", result) # Multiple spaces -> single space + result = re.sub(r"-{2,}", "-", result) # Multiple dashes -> single dash + result = re.sub(r"^[\.\s\-]+|[\.\s\-]+$", "", result) # Remove leading/trailing dots, spaces, dashes + result = re.sub(r"\.-", "-", result) # Remove dots before dashes (for dot-based templates) + result = re.sub(r"[\.\s]+\)", ")", result) # Remove dots/spaces before closing parentheses + result = re.sub(r"\(\s*\)", "", result) # Remove empty parentheses (empty conditional) + + # Determine the appropriate separator based on template style + # Count separator characters between variables (between } and {) + between_vars = re.findall(r"\}([^{]*)\{", self.template) + separator_text = "".join(between_vars) + dot_count = separator_text.count(".") + space_count = separator_text.count(" ") + + if space_count > dot_count: + result = sanitize_filename(result, spacer=" ") + else: + result = sanitize_filename(result, spacer=".") + + if not result or result.isspace(): + log.warning("Template formatting resulted in empty filename, using fallback") + return "untitled" + + log.debug(f"Template formatted successfully: '{self.template}' -> '{result}'") + return result + + except (KeyError, ValueError, re.error) as e: + log.error(f"Error formatting template '{self.template}': {e}") + fallback = f"error_formatting_{hash(self.template) % 10000}" + log.warning(f"Using fallback filename: {fallback}") + return fallback + + def validate(self, context: dict[str, Any]) -> tuple[bool, list[str]]: + """Validate that all required variables are present in context. + + Args: + context: Dictionary containing variable values + + Returns: + Tuple of (is_valid, missing_variables) + """ + missing = [] + + for variable in self.variables: + is_conditional = variable.endswith("?") + var_name = variable[:-1] if is_conditional else variable + + if not is_conditional and var_name not in context: + missing.append(var_name) + + return len(missing) == 0, missing + + def get_required_variables(self) -> list[str]: + """Get list of required (non-conditional) variables.""" + required = [] + for variable in self.variables: + if not variable.endswith("?"): + required.append(variable) + return required + + def get_optional_variables(self) -> list[str]: + """Get list of optional (conditional) variables.""" + optional = [] + for variable in self.variables: + if variable.endswith("?"): + optional.append(variable[:-1]) # Remove the ? + return optional diff --git a/reset/packages/envied/src/envied/core/utils/webvtt.py b/reset/packages/envied/src/envied/core/utils/webvtt.py new file mode 100644 index 0000000..562dd25 --- /dev/null +++ b/reset/packages/envied/src/envied/core/utils/webvtt.py @@ -0,0 +1,225 @@ +import re +import sys +import typing +from typing import Optional + +import pysubs2 +from pycaption import Caption, CaptionList, CaptionNode, CaptionReadError, WebVTTReader, WebVTTWriter + +from envied.core.config import config + + +class CaptionListExt(CaptionList): + @typing.no_type_check + def __init__(self, iterable=None, layout_info=None): + self.first_segment_mpegts = 0 + super().__init__(iterable, layout_info) + + +class CaptionExt(Caption): + @typing.no_type_check + def __init__(self, start, end, nodes, style=None, layout_info=None, segment_index=0, mpegts=0, cue_time=0.0): + style = style or {} + self.segment_index: int = segment_index + self.mpegts: float = mpegts + self.cue_time: float = cue_time + super().__init__(start, end, nodes, style, layout_info) + + +class WebVTTReaderExt(WebVTTReader): + # HLS extension support + RE_TIMESTAMP_MAP = re.compile(r"X-TIMESTAMP-MAP.*") + RE_MPEGTS = re.compile(r"MPEGTS:(\d+)") + RE_LOCAL = re.compile(r"LOCAL:((?:(\d{1,}):)?(\d{2}):(\d{2})\.(\d{3}))") + + def _parse(self, lines: list[str]) -> CaptionList: + captions = CaptionListExt() + start = None + end = None + nodes: list[CaptionNode] = [] + layout_info = None + found_timing = False + segment_index = -1 + mpegts = 0 + cue_time = 0.0 + + # The first segment MPEGTS is needed to calculate the rest. It is possible that + # the first segment contains no cue and is ignored by pycaption, this acts as a fallback. + captions.first_segment_mpegts = 0 + + for i, line in enumerate(lines): + if "-->" in line: + found_timing = True + timing_line = i + last_start_time = captions[-1].start if captions else 0 + try: + start, end, layout_info = self._parse_timing_line(line, last_start_time) + except CaptionReadError as e: + new_msg = f"{e.args[0]} (line {timing_line})" + tb = sys.exc_info()[2] + raise type(e)(new_msg).with_traceback(tb) from None + + elif "" == line: + if found_timing and nodes: + found_timing = False + caption = CaptionExt( + start, + end, + nodes, + layout_info=layout_info, + segment_index=segment_index, + mpegts=mpegts, + cue_time=cue_time, + ) + captions.append(caption) + nodes = [] + + elif "WEBVTT" in line: + # Merged segmented VTT doesn't have index information, track manually. + segment_index += 1 + mpegts = 0 + cue_time = 0.0 + elif m := self.RE_TIMESTAMP_MAP.match(line): + if r := self.RE_MPEGTS.search(m.group()): + mpegts = int(r.group(1)) + + cue_time = self._parse_local(m.group()) + + # Early assignment in case the first segment contains no cue. + if segment_index == 0: + captions.first_segment_mpegts = mpegts + + else: + if found_timing: + if nodes: + nodes.append(CaptionNode.create_break()) + nodes.append(CaptionNode.create_text(self._decode(line))) + else: + # it's a comment or some metadata; ignore it + pass + + # Add a last caption if there are remaining nodes + if nodes: + caption = CaptionExt(start, end, nodes, layout_info=layout_info, segment_index=segment_index, mpegts=mpegts) + captions.append(caption) + + return captions + + @staticmethod + def _parse_local(string: str) -> float: + """ + Parse WebVTT LOCAL time and convert it to seconds. + """ + m = WebVTTReaderExt.RE_LOCAL.search(string) + if not m: + return 0 + + parsed = m.groups() + if not parsed: + return 0 + hours = int(parsed[1]) + minutes = int(parsed[2]) + seconds = int(parsed[3]) + milliseconds = int(parsed[4]) + return (milliseconds / 1000) + seconds + (minutes * 60) + (hours * 3600) + + +def merge_segmented_webvtt(vtt_raw: str, segment_durations: Optional[list[int]] = None, timescale: int = 1) -> str: + """ + Merge Segmented WebVTT data. + + Parameters: + vtt_raw: The concatenated WebVTT files to merge. All WebVTT headers must be + appropriately spaced apart, or it may produce unwanted effects like + considering headers as captions, timestamp lines, etc. + segment_durations: A list of each segment's duration. If not provided it will try + to get it from the X-TIMESTAMP-MAP headers, specifically the MPEGTS number. + timescale: The number of time units per second. + + This parses the X-TIMESTAMP-MAP data to compute new absolute timestamps, replacing + the old start and end timestamp values. All X-TIMESTAMP-MAP header information will + be removed from the output as they are no longer of concern. Consider this function + the opposite of a WebVTT Segmenter, a WebVTT Joiner of sorts. + + Algorithm borrowed from N_m3u8DL-RE and shaka-player. + """ + MPEG_TIMESCALE = 90_000 + + # Check config for conversion method preference + conversion_method = config.subtitle.get("conversion_method", "auto") + use_pysubs2 = conversion_method in ("pysubs2", "auto") + + if use_pysubs2: + # Try using pysubs2 first for more lenient parsing + try: + # Use pysubs2 to parse and normalize the VTT + subs = pysubs2.SSAFile.from_string(vtt_raw) + # Convert back to WebVTT string for pycaption processing + normalized_vtt = subs.to_string("vtt") + vtt = WebVTTReaderExt().read(normalized_vtt) + except Exception: + # Fall back to direct pycaption parsing + vtt = WebVTTReaderExt().read(vtt_raw) + else: + # Use pycaption directly + vtt = WebVTTReaderExt().read(vtt_raw) + for lang in vtt.get_languages(): + prev_caption = None + duplicate_index: list[int] = [] + captions = vtt.get_captions(lang) + + # Some providers can produce "segment_index" values that are + # outside the provided segment_durations list after normalization/merge. + # This used to crash with IndexError and abort the entire download. + if segment_durations and captions: + max_idx = max(getattr(c, "segment_index", 0) for c in captions) + if max_idx >= len(segment_durations): + # Pad with the last known duration (or 0 if empty) so indexing is safe. + pad_val = segment_durations[-1] if segment_durations else 0 + segment_durations = segment_durations + [pad_val] * (max_idx - len(segment_durations) + 1) + + if captions[0].segment_index == 0: + first_segment_mpegts = captions[0].mpegts + else: + first_segment_mpegts = segment_durations[0] if segment_durations else captions.first_segment_mpegts + + caption: CaptionExt + for i, caption in enumerate(captions): + # DASH WebVTT doesn't have MPEGTS timestamp like HLS. Instead, + # calculate the timestamp from SegmentTemplate/SegmentList duration. + likely_dash = first_segment_mpegts == 0 and caption.mpegts == 0 + if likely_dash and segment_durations: + # Defensive: segment_index can still be out of range if captions are malformed. + if caption.segment_index < 0 or caption.segment_index >= len(segment_durations): + continue + duration = segment_durations[caption.segment_index] + caption.mpegts = MPEG_TIMESCALE * (duration / timescale) + + if caption.mpegts == 0: + continue + + # Commeted to fix DSNP subs being out of sync and mistimed. + # seconds = (caption.mpegts - first_segment_mpegts) / MPEG_TIMESCALE - caption.cue_time + # offset = seconds * 1_000_000 # pycaption use microseconds + + # if caption.start < offset: + # caption.start += offset + # caption.end += offset + + # If the difference between current and previous captions is <=1ms + # and the payload is equal then splice. + if ( + prev_caption + and not caption.is_empty() + and (caption.start - prev_caption.end) <= 1000 # 1ms in microseconds + and caption.get_text() == prev_caption.get_text() + ): + prev_caption.end = caption.end + duplicate_index.append(i) + + prev_caption = caption + + # Remove duplicate + captions[:] = [c for c_index, c in enumerate(captions) if c_index not in set(duplicate_index)] + + return WebVTTWriter().write(vtt) diff --git a/reset/packages/envied/src/envied/core/utils/xml.py b/reset/packages/envied/src/envied/core/utils/xml.py new file mode 100644 index 0000000..30eb83d --- /dev/null +++ b/reset/packages/envied/src/envied/core/utils/xml.py @@ -0,0 +1,24 @@ +from typing import Union + +from lxml import etree +from lxml.etree import ElementTree + + +def load_xml(xml: Union[str, bytes]) -> ElementTree: + """Safely parse XML data to an ElementTree, without namespaces in tags.""" + if not isinstance(xml, bytes): + xml = xml.encode("utf8") + root = etree.fromstring(xml) + for elem in root.getiterator(): + if not hasattr(elem.tag, "find"): + # e.g. comment elements + continue + elem.tag = etree.QName(elem).localname + for name, value in elem.attrib.items(): + local_name = etree.QName(name).localname + if local_name == name: + continue + del elem.attrib[name] + elem.attrib[local_name] = value + etree.cleanup_namespaces(root) + return root diff --git a/reset/packages/envied/src/envied/core/vault.py b/reset/packages/envied/src/envied/core/vault.py new file mode 100644 index 0000000..a5c8747 --- /dev/null +++ b/reset/packages/envied/src/envied/core/vault.py @@ -0,0 +1,49 @@ +from abc import ABCMeta, abstractmethod +from typing import Iterator, Optional, Union +from uuid import UUID + + +class Vault(metaclass=ABCMeta): + def __init__(self, name: str, no_push: bool = False): + self.name = name + self.no_push = no_push + + def __str__(self) -> str: + return f"{self.name} {type(self).__name__}" + + @abstractmethod + def get_key(self, kid: Union[UUID, str], service: str) -> Optional[str]: + """ + Get Key from Vault by KID (Key ID) and Service. + + It does not get Key by PSSH as the PSSH can be different depending on it's implementation, + or even how it was crafted. Some PSSH values may also actually be a CENC Header rather + than a PSSH MP4 Box too, which makes the value even more confusingly different. + + However, the KID never changes unless the video file itself has changed too, meaning the + key for the presumed-matching KID wouldn't work, further proving matching by KID is + superior. + """ + + @abstractmethod + def get_keys(self, service: str) -> Iterator[tuple[str, str]]: + """Get All Keys from Vault by Service.""" + + @abstractmethod + def add_key(self, service: str, kid: Union[UUID, str], key: str) -> bool: + """Add KID:KEY to the Vault.""" + + @abstractmethod + def add_keys(self, service: str, kid_keys: dict[Union[UUID, str], str]) -> int: + """ + Add Multiple Content Keys with Key IDs for Service to the Vault. + Pre-existing Content Keys are ignored/skipped. + Raises PermissionError if the user has no permission to create the table. + """ + + @abstractmethod + def get_services(self) -> Iterator[str]: + """Get a list of Service Tags from Vault.""" + + +__all__ = ("Vault",) diff --git a/reset/packages/envied/src/envied/core/vaults.py b/reset/packages/envied/src/envied/core/vaults.py new file mode 100644 index 0000000..ee672ce --- /dev/null +++ b/reset/packages/envied/src/envied/core/vaults.py @@ -0,0 +1,98 @@ +import logging +from typing import Any, Iterator, Optional, Union +from uuid import UUID + +from envied.core.config import config +from envied.core.utilities import import_module_by_path +from envied.core.vault import Vault + +log = logging.getLogger(__name__) + +_VAULTS = sorted( + (path for path in config.directories.vaults.glob("*.py") if path.stem.lower() != "__init__"), key=lambda x: x.stem +) + +_MODULES = {path.stem: getattr(import_module_by_path(path), path.stem) for path in _VAULTS} + + +class Vaults: + """Keeps hold of Key Vaults with convenience functions, e.g. searching all vaults.""" + + def __init__(self, service: Optional[str] = None): + self.service = service or "" + self.vaults = [] + + def __iter__(self) -> Iterator[Vault]: + return iter(self.vaults) + + def __len__(self) -> int: + return len(self.vaults) + + def load(self, type_: str, **kwargs: Any) -> bool: + """Load a Vault into the vaults list. Returns True if successful, False otherwise.""" + module = _MODULES.get(type_) + if not module: + raise ValueError(f"Unable to find vault command by the name '{type_}'.") + try: + vault = module(**kwargs) + self.vaults.append(vault) + return True + except Exception: + return False + + def load_critical(self, type_: str, **kwargs: Any) -> None: + """Load a critical Vault that must succeed or raise an exception.""" + module = _MODULES.get(type_) + if not module: + raise ValueError(f"Unable to find vault command by the name '{type_}'.") + vault = module(**kwargs) + self.vaults.append(vault) + + def get_key(self, kid: Union[UUID, str]) -> tuple[Optional[str], Optional[Vault]]: + """Get Key from the first Vault it can by KID (Key ID) and Service.""" + for vault in self.vaults: + try: + key = vault.get_key(kid, self.service) + except (PermissionError, NotImplementedError): + continue + except Exception as e: + log.warning(f"Failed to get key from Vault '{vault.name}': {e}") + continue + if key and key.count("0") != len(key): + return key, vault + return None, None + + def add_key(self, kid: Union[UUID, str], key: str, excluding: Optional[Vault] = None) -> int: + """Add a KID:KEY to all Vaults, optionally with an exclusion.""" + success = 0 + for vault in self.vaults: + if vault != excluding and not vault.no_push: + try: + success += vault.add_key(self.service, kid, key) + except (PermissionError, NotImplementedError): + pass + except Exception as e: + log.warning(f"Failed to add key to Vault '{vault.name}': {e}") + return success + + def add_keys(self, kid_keys: dict[Union[UUID, str], str]) -> int: + """ + Add multiple KID:KEYs to all Vaults. Duplicate Content Keys are skipped. + PermissionErrors when the user cannot create Tables are absorbed and ignored. + Vaults with no_push=True are skipped. + """ + success = 0 + for vault in self.vaults: + if not vault.no_push: + try: + # Count each vault that successfully processes the keys (whether new or existing) + vault.add_keys(self.service, kid_keys) + success += 1 + except (PermissionError, NotImplementedError): + pass + except Exception as e: + log.warning(f"Failed to add keys to Vault '{vault.name}': {e}") + return success + + +__all__ = ("Vaults",) diff --git a/reset/packages/envied/src/envied/envied-example.yaml b/reset/packages/envied/src/envied/envied-example.yaml new file mode 100644 index 0000000..789a774 --- /dev/null +++ b/reset/packages/envied/src/envied/envied-example.yaml @@ -0,0 +1,758 @@ +# API key for The Movie Database (TMDB) +tmdb_api_key: "" + +# Client ID for SIMKL API (optional, improves metadata matching) +# Get your free client ID at: https://simkl.com/settings/developer/ +simkl_client_id: "" + +# Optional ipinfo.io API token. When set, unshackle uses the free Lite endpoint +# which has higher rate limits and richer IP info (ASN, org, continent). +# Get a free token at: https://ipinfo.io/signup +ipinfo_api_key: "" + +# Group or Username to postfix to the end of all download filenames following a dash +tag: user_tag + +# Enable/disable tagging with group name (default: true) +tag_group_name: true + +# Enable/disable tagging with IMDB/TMDB/TVDB details (default: true) +tag_imdb_tmdb: true + +# Set terminal background color (custom option not in CONFIG.md) +set_terminal_bg: false + +# Custom output templates for filenames +# Configure output_template in your envied.yaml to control filename format. +# If not configured, default scene-style templates are used and a warning is shown. +# Available variables: {title}, {year}, {season}, {episode}, {season_episode}, {episode_name}, +# {quality}, {resolution}, {source}, {audio}, {audio_channels}, {audio_full}, +# {video}, {hdr}, {hfr}, {atmos}, {dual}, {multi}, {tag}, {edition}, {repack}, +# {lang_tag} +# Conditional variables (included only if present): Add ? suffix like {year?}, {episode_name?}, {hdr?} +# Customize the templates below: +# +# Example outputs: +# Scene movies: 'The.Matrix.1999.1080p.SERVICE.WEB-DL.DDP5.1.H.264-EXAMPLE' +# Scene movies (HDR): 'Dune.2021.2160p.SERVICE.WEB-DL.DDP5.1.HDR10.H.265-EXAMPLE' +# Scene movies (REPACK): 'Dune.2021.REPACK.2160p.SERVICE.WEB-DL.DDP5.1.H.265-EXAMPLE' +# Scene series: 'Breaking.Bad.2008.S01E01.Pilot.1080p.SERVICE.WEB-DL.DDP5.1.H.264-EXAMPLE' +# Plex movies: 'The Matrix (1999) 1080p' +# Plex series: 'Breaking Bad S01E01 Pilot' +output_template: + # Scene-style naming (dot-separated) + movies: '{title}.{year}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + series: '{title}.{year?}.{season_episode}.{episode_name?}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + songs: '{track_number}.{title}.{repack?}.{edition?}.{source?}.WEB-DL.{audio_full}.{atmos?}-{tag}' + # + # Plex-friendly naming (space-separated, clean format) + # movies: '{title} ({year}) {quality}' + # series: '{title} {season_episode} {episode_name?}' + # songs: '{track_number}. {title}' + # + # Minimal naming (basic info only) + # movies: '{title}.{year}.{quality}' + # series: '{title}.{season_episode}.{episode_name?}' + # + # Custom scene-style with specific elements + # movies: '{title}.{year}.{quality}.{hdr?}.{source}.WEB-DL.{audio_full}.{video}-{tag}' + # series: '{title}.{year?}.{season_episode}.{episode_name?}.{quality}.{hdr?}.{source}.WEB-DL.{audio_full}.{atmos?}.{video}-{tag}' + # + # Folder naming (optional). Controls the folder name for downloaded content. + # If not configured, series folders are derived from the series template (minus episode info), + # movie folders use "{title} ({year})", and song folders use "{artist} - {album} ({year})". + # Uses the same template variables as the file templates above. + # + # Scene-style folder: + # folder: '{title}.{year?}.{repack?}.{edition?}.{lang_tag?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + # + # Plex-friendly folder: + # folder: '{title} ({year?})' + # + # Per-title-type folder templates (optional). Override folder naming separately for + # movies, series, and songs. Useful when music libraries need artist/album-style folders + # while movies/series follow a different scheme. Any kind omitted falls back to the + # default for that title type. + # + # folder: + # movies: '{title} ({year})' + # series: '{title} ({year?})' + # songs: '{artist}/{album} ({year?})' + +# Language-based tagging for output filenames +# Automatically adds language identifiers (e.g., DANiSH, NORDiC, DKsubs) based on +# audio and subtitle track languages. Rules are evaluated in order; first match wins. +# Use {lang_tag?} in your output_template to place the tag in the filename. +# +# Conditions (all conditions in a rule must match): +# audio: - any audio track matches this language +# subs_contain: - any subtitle matches this language +# subs_contain_all: [lang, ...] - subtitles include ALL listed languages +# +# language_tags: +# rules: +# - audio: da +# tag: DANiSH +# - audio: sv +# tag: SWEDiSH +# - audio: nb +# tag: NORWEGiAN +# - audio: en +# subs_contain_all: [da, sv, nb] +# tag: NORDiC +# - audio: en +# subs_contain: da +# tag: DKsubs + +# Check for updates from GitHub repository on startup (default: true) +update_checks: true + +# How often to check for updates, in hours (default: 24) +update_check_interval: 24 + +# Title caching configuration +# Cache title metadata to reduce redundant API calls +title_cache_enabled: true # Enable/disable title caching globally (default: true) +title_cache_time: 1800 # Cache duration in seconds (default: 1800 = 30 minutes) +title_cache_max_retention: 86400 # Maximum cache retention for fallback when API fails (default: 86400 = 24 hours) + +# Filename Configuration +unicode_filenames: false # optionally replace non-ASCII characters with ASCII equivalents + +# Debug logging configuration +# Comprehensive JSON-based debug logging for troubleshooting and service development +debug: + false # Enable structured JSON debug logging (default: false) + # When enabled with --debug flag or set to true: + # - Creates JSON Lines (.jsonl) log files with complete debugging context + # - Logs: session info, CLI params, service config, CDM details, authentication, + # titles, tracks metadata, DRM operations, vault queries, errors with stack traces + # - File location: logs/unshackle_debug_{service}_{timestamp}.jsonl + # - Also creates text log: logs/unshackle_root_{timestamp}.log + +debug_keys: + false # Log decryption keys in debug logs (default: false) + # Set to true to include actual decryption keys in logs + # Useful for debugging key retrieval and decryption issues + # SECURITY NOTE: Passwords, tokens, cookies, and session tokens + # are ALWAYS redacted regardless of this setting + # Only affects: content_key, key fields (the actual CEKs) + # Never affects: kid, keys_count, key_id (metadata is always logged) + +# Muxing configuration +muxing: + set_title: false + # merge_audio: Merge all audio tracks into each output file + # true (default): All selected audio in one MKV per quality + # false: Separate MKV per (quality, audio_codec) combination + # Example: Title.1080p.AAC.mkv, Title.1080p.EC3.mkv + merge_audio: true + # default_language: Override which track is flagged as the default in the muxed MKV. + # audio: BCP-47 tag of the preferred default audio track (e.g. pl, en, pt-BR). + # Wins over the title's original_language. Falls back to is_original_lang + # if no matching track is present. + # video: BCP-47 tag of the preferred default video track. Falls back to the + # original-language / first-track rule if no match is found. + # subtitle: BCP-47 tag of the preferred default subtitle track. Falls back to + # the existing rule (forced sub matching the audio language) if no + # matching subtitle is present. + # default_language: + # audio: pl + # video: pl + # subtitle: pl + +# Login credentials for each Service +credentials: + # Direct credentials (no profile support) + EXAMPLE: email@example.com:password + + # Per-profile credentials with default fallback + SERVICE_NAME: + default: default@email.com:password # Used when no -p/--profile is specified + profile1: user1@email.com:password1 + profile2: user2@email.com:password2 + + # Per-profile credentials without default (requires -p/--profile) + SERVICE_NAME2: + john: john@example.com:johnspassword + jane: jane@example.com:janespassword + + # You can also use list format for passwords with special characters + SERVICE_NAME3: + default: ["user@email.com", ":PasswordWith:Colons"] + +# Override default directories used across unshackle +directories: + cache: Cache + cookies: Cookies + dcsl: DCSL # Device Certificate Status List + downloads: Downloads + logs: Logs + temp: Temp + wvds: WVDs + prds: PRDs + exports: Exports # JSON export output from --export flag + # Additional directories that can be configured: + # commands: Commands + services: + - /path/to/services + - /other/path/to/services + # vaults: Vaults + # fonts: Fonts + +# Pre-define which Widevine or PlayReady device to use for each Service +cdm: + # Global default CDM device (fallback for all services/profiles) + default: WVD_1 + + # Direct service-specific CDM + DIFFERENT_EXAMPLE: PRD_1 + + # Per-profile CDM configuration + EXAMPLE: + john_sd: chromecdm_903_l3 # Profile 'john_sd' uses Chrome CDM L3 + jane_uhd: nexus_5_l1 # Profile 'jane_uhd' uses Nexus 5 L1 + default: generic_android_l3 # Default CDM for this service + + # NEW: Quality-based CDM selection + # Use different CDMs based on video resolution + # Supports operators: >=, >, <=, <, or exact match + EXAMPLE_QUALITY: + "<=1080": generic_android_l3 # Use L3 for 1080p and below + ">1080": nexus_5_l1 # Use L1 for above 1080p (1440p, 2160p) + default: generic_android_l3 # Optional: fallback if no quality match + + # You can mix profiles and quality thresholds in the same service + NETFLIX: + # Profile-based selection (existing functionality) + john: netflix_l3_profile + jane: netflix_l1_profile + # Quality-based selection (new functionality) + "<=720": netflix_mobile_l3 + "1080": netflix_standard_l3 + ">=1440": netflix_premium_l1 + # Fallback + default: netflix_standard_l3 + +# Use pywidevine Serve-compliant Remote CDMs + +# Example: Custom CDM API Configuration +# This demonstrates the highly configurable custom_api type that can adapt to any CDM API format +# - name: "chrome" +# type: "custom_api" +# host: "http://remotecdm.test/" +# timeout: 30 +# device: +# name: "ChromeCDM" +# type: "CHROME" +# system_id: 34312 +# security_level: 3 +# auth: +# type: "header" +# header_name: "x-api-key" +# key: "YOUR_API_KEY_HERE" +# custom_headers: +# User-Agent: "Unshackle/2.0.0" +# endpoints: +# get_request: +# path: "/get-challenge" +# method: "POST" +# timeout: 30 +# decrypt_response: +# path: "/get-keys" +# method: "POST" +# timeout: 30 +# request_mapping: +# get_request: +# param_names: +# scheme: "device" +# init_data: "init_data" +# static_params: +# scheme: "Widevine" +# decrypt_response: +# param_names: +# scheme: "device" +# license_request: "license_request" +# license_response: "license_response" +# static_params: +# scheme: "Widevine" +# response_mapping: +# get_request: +# fields: +# challenge: "challenge" +# session_id: "session_id" +# message: "message" +# message_type: "message_type" +# response_types: +# - condition: "message_type == 'license-request'" +# type: "license_request" +# success_conditions: +# - "message == 'success'" +# decrypt_response: +# fields: +# keys: "keys" +# message: "message" +# key_fields: +# kid: "kid" +# key: "key" +# type: "type" +# success_conditions: +# - "message == 'success'" +# caching: +# enabled: true +# use_vaults: true +# check_cached_first: true + +remote_cdm: + - name: "chrome" + device_name: chrome + device_type: CHROME + system_id: 27175 + security_level: 3 + host: https://domain.com/api + secret: secret_key + - name: "chrome-2" + device_name: chrome + device_type: CHROME + system_id: 26830 + security_level: 3 + host: https://domain-2.com/api + secret: secret_key + + - name: "decrypt_labs_chrome" + type: "decrypt_labs" # Required to identify as DecryptLabs CDM + device_name: "ChromeCDM" # Scheme identifier - must match exactly + device_type: CHROME + system_id: 4464 # Doesn't matter + security_level: 3 + host: "https://keyxtractor.decryptlabs.com" + secret: "your_decrypt_labs_api_key_here" # Replace with your API key + - name: "decrypt_labs_l1" + type: "decrypt_labs" + device_name: "L1" # Scheme identifier - must match exactly + device_type: ANDROID + system_id: 4464 + security_level: 1 + host: "https://keyxtractor.decryptlabs.com" + secret: "your_decrypt_labs_api_key_here" + + - name: "decrypt_labs_l2" + type: "decrypt_labs" + device_name: "L2" # Scheme identifier - must match exactly + device_type: ANDROID + system_id: 4464 + security_level: 2 + host: "https://keyxtractor.decryptlabs.com" + secret: "your_decrypt_labs_api_key_here" + + - name: "decrypt_labs_playready_sl2" + type: "decrypt_labs" + device_name: "SL2" # Scheme identifier - must match exactly + device_type: PLAYREADY + system_id: 0 + security_level: 2000 + host: "https://keyxtractor.decryptlabs.com" + secret: "your_decrypt_labs_api_key_here" + + - name: "decrypt_labs_playready_sl3" + type: "decrypt_labs" + device_name: "SL3" # Scheme identifier - must match exactly + device_type: PLAYREADY + system_id: 0 + security_level: 3000 + host: "https://keyxtractor.decryptlabs.com" + secret: "your_decrypt_labs_api_key_here" + + # PyPlayReady RemoteCdm - connects to an unshackle serve instance + - name: "playready_remote" + device_name: "my_prd_device" # Device name on the serve instance + device_type: PLAYREADY + system_id: 0 + security_level: 3000 # 2000 for SL2000, 3000 for SL3000 + host: "http://127.0.0.1:8786/playready" # Include /playready path + secret: "your-api-secret-key" + +# Key Vaults store your obtained Content Encryption Keys (CEKs) +# Use 'no_push: true' to prevent a vault from receiving pushed keys +# while still allowing it to provide keys when requested +key_vaults: + - type: SQLite + name: Local + path: key_store.db + # Additional vault types: + # - type: API + # name: "Remote Vault" + # uri: "https://key-vault.example.com" + # token: "secret_token" + # no_push: true # This vault will only provide keys, not receive them + # - type: MySQL + # name: "MySQL Vault" + # host: "127.0.0.1" + # port: 3306 + # database: vault + # username: user + # password: pass + # no_push: false # Default behavior - vault both provides and receives keys + +# Choose what software to use to download data +downloader: requests +# Options: requests +# Downloading now uses the unified in-process requests/rnet downloader; the legacy +# aria2c, curl_impersonate, and n_m3u8dl_re backends have been removed. + +# rnet TLS impersonation preset (not a downloader). Selects the browser +# fingerprint the HTTP session impersonates. +curl_impersonate: + browser: chrome120 + +# Pre-define default options and switches of the dl command +# Audio track selection preferences +audio: + # Codec priority order used as a tiebreaker when multiple audio tracks share the same + # bitrate and language. Listed codecs are ranked in the order given; codecs not in the + # list keep their bitrate-based ordering and are placed after all listed codecs. + # Atmos still trumps codec priority. Valid names: AAC, AC3, EC3, AC4, OPUS, OGG, DTS, ALAC, FLAC. + # codec_priority: [FLAC, ALAC, AC4, EC3, DTS, AC3, OPUS, AAC, OGG] + +dl: + sub_format: srt + downloads: 4 + workers: 16 + lang: + - en + - fr + EXAMPLE: + bitrate: CBR + +# Chapter Name to use when exporting a Chapter without a Name +chapter_fallback_name: "Chapter {j:02}" + +# Case-Insensitive dictionary of headers for all Services +headers: + Accept-Language: "en-US,en;q=0.8" + User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36" + +# Override default filenames used across unshackle +filenames: + debug_log: "unshackle_debug_{service}_{time}.jsonl" # JSON Lines debug log file + config: "config.yaml" + root_config: "envied.yaml" + chapters: "Chapters_{title}_{random}.txt" + subtitle: "Subtitle_{id}_{language}.srt" + +# conversion_method: +# - auto (default): Smart routing - subby for WebVTT/SAMI, pycaption for others +# - subby: Always use subby with advanced processing +# - pycaption: Use only pycaption library (no SubtitleEdit, no subby) +# - subtitleedit: Prefer SubtitleEdit when available, fall back to pycaption +# - pysubs2: Use pysubs2 library (supports SRT/SSA/ASS/WebVTT/TTML/SAMI/MicroDVD/MPL2/TMP) +subtitle: + conversion_method: auto + # sdh_method: Method to use for SDH (hearing impaired) stripping + # - auto (default): Try subby (SRT only), then SubtitleEdit (if available), then subtitle-filter + # - subby: Use subby library (SRT only) + # - subtitleedit: Use SubtitleEdit tool (Windows only, falls back to subtitle-filter) + # - filter-subs: Use subtitle-filter library directly + sdh_method: auto + # strip_sdh: Automatically create stripped (non-SDH) versions of SDH subtitles + # Set to false to disable automatic SDH stripping entirely (default: true) + strip_sdh: true + # convert_before_strip: Auto-convert VTT/other formats to SRT before using subtitle-filter + # This ensures compatibility when subtitle-filter is used as fallback (default: true) + convert_before_strip: true + # preserve_formatting: Preserve original subtitle formatting (tags, positioning, styling) + # When true, skips pycaption processing for WebVTT files to keep tags like , , positioning intact + # Combined with no sub_format setting, ensures subtitles remain in their original format (default: true) + preserve_formatting: true + # output_mode: Output mode for subtitles + # - mux: Embed subtitles in MKV container only (default) + # - sidecar: Save subtitles as separate files only + # - both: Embed in MKV AND save as sidecar files + output_mode: mux + # sidecar_format: Format for sidecar subtitle files + # Options: srt, vtt, ass, original (keep current format) + sidecar_format: srt + +# Configuration for pywidevine and pyplayready's serve functionality +# Also used for remote services (unshackle serve) +serve: + api_secret: "your-secret-key-here" + + # Compression level for API payloads (manifests, cache, cookies) + # 0=off, 1=fast, 6=balanced, 9=max compression (default: 1) + compression_level: 1 + + # Session inactivity timeout in seconds (default: 300 = 5 minutes) + # Sessions are automatically deleted after this many seconds of inactivity + # Each API request resets the timer + session_ttl: 300 + + # Maximum concurrent sessions before oldest is evicted (default: 100) + max_sessions: 100 + + # Global service allowlist (optional) + # Only these services will be exposed on the API. If omitted, all services are available. + # services: + # - SERVICE_TAG_1 + # - SERVICE_TAG_2 + + users: + secret_key_for_user: + devices: # Widevine devices (WVDs) this user can access + - generic_nexus_4464_l3 + playready_devices: # PlayReady devices (PRDs) this user can access + - playready_device_sl3000 + username: user + # Per-user service allowlist (optional) + # Restricts this user to only the listed services. If omitted, user can access + # all globally-allowed services. Effective access is the intersection of global + # and per-user allowlists. + # services: + # - SERVICE_TAG_1 + # devices: # Widevine device paths (auto-populated from directories.wvds) + # - '/path/to/device.wvd' + # playready_devices: # PlayReady device paths (auto-populated from directories.prds) + # - '/path/to/device.prd' + + # Optional: any /api/download flag can be set here as a server-side default. + # Per-request body values still win. Useful for raising concurrency without + # changing every client call. Full list of accepted keys: see docs/API.md. + # downloads: 4 # parallel tracks per download job + # workers: 16 # threads per track segment fetch + # best_available: true + # no_proxy_download: false + +# Remote Services Configuration +# Connect to a remote unshackle server (unshackle serve) to use its services +# without needing the service code locally. Use with: unshackle dl --remote +# If multiple servers are configured, specify which with: --server +remote_services: + # Server name (used with --server flag if multiple configured) + my-server: + url: "http://192.168.1.100:8786" + api_key: "your-secret-key-here" + + # Server-CDM mode: server handles all DRM licensing using its own CDM devices + # When false (default), client uses its own CDM and proxies license requests through the server + server_cdm: false + + # Per-service overrides for remote services + # Override downloader, decryption tool, or CDM settings per service on the client + services: + # Example: Override the decryption tool for specific services + # EXAMPLE_SERVICE: + # decryption: mp4decrypt # Override decryption tool (shaka, mp4decrypt) + + # Example: Multiple servers + # us-server: + # url: "https://us.example.com:8786" + # api_key: "us-api-key" + # server_cdm: true + # services: + # EXAMPLE: + # decryption: mp4decrypt + # eu-server: + # url: "https://eu.example.com:8786" + # api_key: "eu-api-key" + # server_cdm: false + +# Configuration data for each Service +services: + # Service-specific configuration goes here + # Profile-specific configurations can be nested under service names + + # You can override ANY global configuration option on a per-service basis + # This allows fine-tuned control for services with special requirements + # Supported overrides: dl, curl_impersonate, subtitle, muxing, headers, etc. + + # Example: Comprehensive service configuration showing all features + EXAMPLE: + # Standard service config + api_key: "service_api_key" + + # Service certificate for Widevine L1/L2 (base64 encoded) + # This certificate is automatically used when L1/L2 schemes are selected + # Services obtain this from their DRM provider or license server + certificate: | + CAUSwwUKvQIIAxIQ5US6QAvBDzfTtjb4tU/7QxiH8c+TBSKOAjCCAQoCggEBAObzvlu2hZRsapAPx4Aa4GUZj4/GjxgXUtBH4THSkM40x63wQeyVxlEEo + # ... (full base64 certificate here) + + # Profile-specific device configurations + profiles: + john_sd: + device: + app_name: "AIV" + device_model: "SHIELD Android TV" + jane_uhd: + device: + app_name: "AIV" + device_model: "Fire TV Stick 4K" + + # Service-specific proxy mappings + # Override global proxy selection with specific servers for this service + # When --proxy matches a key in proxy_map, the mapped server will be used + # instead of the default/random server selection + proxy_map: + nordvpn:ca: ca1577 # Use ca1577 when --proxy nordvpn:ca is specified + nordvpn:us: us9842 # Use us9842 when --proxy nordvpn:us is specified + us: 123 # Use server 123 (from any provider) when --proxy us is specified + gb: 456 # Use server 456 (from any provider) when --proxy gb is specified + # Without this service, --proxy nordvpn:ca picks a random CA server + # With this config, --proxy nordvpn:ca EXAMPLE uses ca1577 specifically + # Other services or no service specified will still use random selection + + # NEW: Configuration overrides (can be combined with profiles and certificates) + # Override dl command defaults for this service + dl: + downloads: 4 # Limit concurrent track downloads (global default: 6) + workers: 8 # Reduce workers per track (global default: 16) + lang: ["en", "es-419"] # Different language priority for this service + sub_format: srt # Force SRT subtitle format + + # Override subtitle processing for this service + subtitle: + conversion_method: pycaption # Use specific subtitle converter + sdh_method: auto + + # Service-specific headers + headers: + User-Agent: "Service-specific user agent string" + Accept-Language: "en-US,en;q=0.9" + + # Override muxing options + muxing: + set_title: true + + # Remap service-provided titles before naming/output + # Keyed by the exact title the service returns -> desired output title. + title_map: + Service Title: Desired Title + + # Example: Service with different regions per profile + SERVICE_NAME: + profiles: + us_account: + region: "US" + api_endpoint: "https://api.us.service.com" + uk_account: + region: "GB" + api_endpoint: "https://api.uk.service.com" + + # Example: Rate-limited service + RATE_LIMITED_SERVICE: + dl: + downloads: 2 # Limit concurrent downloads + workers: 4 # Reduce workers to avoid rate limits + + # Notes on service-specific overrides: + # - Overrides are merged with global config, not replaced + # - Only specified keys are overridden, others use global defaults + # - Reserved keys (profiles, api_key, certificate, etc.) are NOT treated as overrides + # - Any dict-type config option can be overridden (dl, subtitle, muxing, headers, etc.) + # - CLI arguments always take priority over service-specific config + +# External proxy provider services +proxy_providers: + nordvpn: + username: username_from_service_credentials + password: password_from_service_credentials + # server_map: global mapping that applies to ALL services + # Difference from service-specific proxy_map: + # - server_map: applies to ALL services when --proxy nordvpn:us is used + # - proxy_map: only applies to the specific service configured (see services: EXAMPLE: proxy_map above) + # - proxy_map takes precedence over server_map for that service + server_map: + us: 12 # force US server #12 for US proxies + ca:calgary: 2534 # force CA server #2534 for Calgary proxies + us:seattle: 7890 # force US server #7890 for Seattle proxies + surfsharkvpn: + username: your_surfshark_service_username # Service credentials from https://my.surfshark.com/vpn/manual-setup/main/openvpn + password: your_surfshark_service_password # Service credentials (not your login password) + server_map: + us: 3844 # force US server #3844 for US proxies + gb: 2697 # force GB server #2697 for GB proxies + au: 4621 # force AU server #4621 for AU proxies + us:seattle: 5678 # force US server #5678 for Seattle proxies + ca:toronto: 1234 # force CA server #1234 for Toronto proxies + windscribevpn: + username: your_windscribe_username # Service credentials from https://windscribe.com/getconfig/openvpn + password: your_windscribe_password # Service credentials (not your login password) + server_map: + us: "us-central-096.totallyacdn.com" # force US server + gb: "uk-london-055.totallyacdn.com" # force GB server + us:seattle: "us-west-011.totallyacdn.com" # force US Seattle server + ca:toronto: "ca-toronto-012.totallyacdn.com" # force CA Toronto server + + # Gluetun: Dynamic Docker-based VPN proxy (supports 50+ VPN providers) + # Creates Docker containers running Gluetun to bridge VPN connections to HTTP proxies + # Requires Docker to be installed and running + # Usage: --proxy gluetun:windscribe:us or --proxy gluetun:nordvpn:de + gluetun: + # Global settings + base_port: 8888 # Starting port for HTTP proxies (increments for each container) + auto_cleanup: true # Automatically remove containers when done + container_prefix: "unshackle-gluetun" # Docker container name prefix + verify_ip: true # Verify VPN IP matches expected region + # Optional HTTP proxy authentication (for the proxy itself, not VPN) + # auth_user: proxy_user + # auth_password: proxy_password + + # VPN provider configurations + providers: + # Windscribe (WireGuard) - Get credentials from https://windscribe.com/getconfig/wireguard + windscribe: + vpn_type: wireguard + credentials: + private_key: "YOUR_WIREGUARD_PRIVATE_KEY" + addresses: "YOUR_WIREGUARD_ADDRESS" # e.g., "10.x.x.x/32" + # Map friendly names to country codes + server_countries: + us: US + uk: GB + ca: CA + de: DE + + # NordVPN (OpenVPN) - Get service credentials from https://my.nordaccount.com/dashboard/nordvpn/manual-configuration/ + # Note: Service credentials are NOT your email+password - generate them from the link above + # nordvpn: + # vpn_type: openvpn + # credentials: + # username: "YOUR_NORDVPN_SERVICE_USERNAME" + # password: "YOUR_NORDVPN_SERVICE_PASSWORD" + # server_countries: + # us: US + # uk: GB + + # ExpressVPN (OpenVPN) - Get credentials from ExpressVPN setup page + # expressvpn: + # vpn_type: openvpn + # credentials: + # username: "YOUR_EXPRESSVPN_USERNAME" + # password: "YOUR_EXPRESSVPN_PASSWORD" + # server_countries: + # us: US + # uk: GB + + # Surfshark (WireGuard) - Get credentials from https://my.surfshark.com/vpn/manual-setup/main/wireguard + # surfshark: + # vpn_type: wireguard + # credentials: + # private_key: "YOUR_SURFSHARK_PRIVATE_KEY" + # addresses: "YOUR_SURFSHARK_ADDRESS" + # server_countries: + # us: US + # uk: GB + + # Specific server selection: Use format like "us1239" to select specific servers + # Example: --proxy gluetun:nordvpn:us1239 connects to us1239.nordvpn.com + # Supported providers: nordvpn, surfshark, expressvpn, cyberghost + + basic: + GB: + - "socks5://username:password@bhx.socks.ipvanish.com:1080" # 1 (Birmingham) + - "socks5://username:password@gla.socks.ipvanish.com:1080" # 2 (Glasgow) + AU: + - "socks5://username:password@syd.socks.ipvanish.com:1080" # 1 (Sydney) + - "https://username:password@au-syd.prod.surfshark.com" # 2 (Sydney) + - "https://username:password@au-bne.prod.surfshark.com" # 3 (Brisbane) + BG: "https://username:password@bg-sof.prod.surfshark.com" diff --git a/reset/packages/envied/src/envied/envied-working-example.yaml b/reset/packages/envied/src/envied/envied-working-example.yaml new file mode 100644 index 0000000..8230fd6 --- /dev/null +++ b/reset/packages/envied/src/envied/envied-working-example.yaml @@ -0,0 +1,234 @@ +# Group or Username to postfix to the end of all download fienames following a dash +#tag: '' + +# Set terminal background color (custom option not in CONFIG.md) +set_terminal_bg: false + +# Muxing configuration +muxing: + set_title: false + +# shakapackager or mp4decrypt +decryption: shakapackager + +# Custom output templates for filenames +# Configure output_template in your envied.yaml to control filename format. +# If not configured, default scene-style templates are used and a warning is shown. +# Available variables: {title}, {year}, {season}, {episode}, {season_episode}, {episode_name}, +# {quality}, {resolution}, {source}, {audio}, {audio_channels}, {audio_full}, +# {video}, {hdr}, {hfr}, {atmos}, {dual}, {multi}, {tag}, {edition}, {repack}, +# {lang_tag} +# Conditional variables (included only if present): Add ? suffix like {year?}, {episode_name?}, {hdr?} +# Customize the templates below: +# +# Example outputs: +# Scene movies: 'The.Matrix.1999.1080p.SERVICE.WEB-DL.DDP5.1.H.264-EXAMPLE' +# Scene movies (HDR): 'Dune.2021.2160p.SERVICE.WEB-DL.DDP5.1.HDR10.H.265-EXAMPLE' +# Scene movies (REPACK): 'Dune.2021.REPACK.2160p.SERVICE.WEB-DL.DDP5.1.H.265-EXAMPLE' +# Scene series: 'Breaking.Bad.2008.S01E01.Pilot.1080p.SERVICE.WEB-DL.DDP5.1.H.264-EXAMPLE' +# Plex movies: 'The Matrix (1999)' +# Plex series: 'Breaking Bad S01E01 Pilot' +output_template: + # Scene-style naming (dot-separated) + #movies: '{title}.{year}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + #series: '{title}.{year?}.{season_episode}.{episode_name?}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + #songs: '{track_number}.{title}.{repack?}.{edition?}.{source?}.WEB-DL.{audio_full}.{atmos?}-{tag}' + # + # Plex-friendly naming (space-separated, clean format) + movies: '{title} ({year})' + series: '{title} {season_episode} {episode_name?}' + songs: '{track_number}. {title}' + # + # Minimal naming (basic info only) + # movies: '{title}.{year}.{quality}' + # series: '{title}.{season_episode}.{episode_name?}' + # + # Custom scene-style with specific elements + # movies: '{title}.{year}.{quality}.{hdr?}.{source}.WEB-DL.{audio_full}.{video}-{tag}' + # series: '{title}.{year?}.{season_episode}.{episode_name?}.{quality}.{hdr?}.{source}.WEB-DL.{audio_full}.{atmos?}.{video}-{tag}' + +# Language-based tagging for output filenames +# Automatically adds language identifiers (e.g., DANiSH, NORDiC, DKsubs) based on +# audio and subtitle track languages. Rules are evaluated in order; first match wins. +# Use {lang_tag?} in your output_template to place the tag in the filename. +# +# Conditions (all conditions in a rule must match): +# audio: - any audio track matches this language +# subs_contain: - any subtitle matches this language +# subs_contain_all: [lang, ...] - subtitles include ALL listed languages +# +# language_tags: +# rules: +# - audio: da +# tag: DANiSH +# - audio: sv +# tag: SWEDiSH +# - audio: nb +# tag: NORWEGiAN +# - audio: en +# subs_contain_all: [da, sv, nb] +# tag: NORDiC +# - audio: en +# subs_contain: da +# tag: DKsubs + +# Login credentials for each Service +credentials: + ALL4: email:password + ROKU: email:password + TVNZ: email:password + TPTV: email:password + CBC: email:password +# Override default directories used across envied +directories: + cache: Cache + cookies: Cookies + dcsl: DCSL # Device Certificate Status List + downloads: Downloads + logs: Logs + temp: Temp + wvds: WVDs + prds: PRDs + # Additional directories that can be configured: + # commands: Commands + #services: + # - ./envied/services/ + vaults: vaults/ + # fonts: Fonts + +# Pre-define which Widevine or PlayReady device to use for each Serviceuv run envied +cdm: + default: device + +# Use pywidevine Serve-compliant Remote CDMs +# Currently not working! +#remote_cdm: +# - name: "CDRM_Project_API" +# device_type: ANDROID +# system_id: 22590 +# security_level: 3 +# host: "https://cdrm-project.com/remotecdm/widevine" +# secret: "CDRM" +# device_name: "public" + + +# Key Vaults store your obtained Content Encryption Keys (CEKs) +key_vaults: +#- type: SQLite +# name: Local vault +# path: key_store.db + +- type: API + name: LibreDRM Vault + uri: https://api.mapiro.com/vault + token: 5fc8fcead3bc4c1403dcec54fc3a326fbd3ec95da3899d24 + +- type: HTTPAPI + name: drmlab + host: http://api.drmlab.io/vault/ + password: N7kQp4XzL2mV8cHrY9tFw5BjUa6DeGs3 + + + +# Choose what software to use to download data +downloader: n_m3u8dl_re +# Options: requests | aria2c | curl_impersonate | n_m3u8dl_re +# Can also be a mapping: +# downloader: +# NF: requests +# AMZN: n_m3u8dl_re +# DSNP: n_m3u8dl_re +# default: requests + +# aria2c downloader configuration +aria2c: + max_concurrent_downloads: 4 + max_connection_per_server: 3 + split: 5 + file_allocation: falloc # none | prealloc | falloc | trunc + +# N_m3u8DL-RE downloader configuration +n_m3u8dl_re: + thread_count: 16 + ad_keyword: "advertisement" + use_proxy: true + +# curl_impersonate downloader configuration +curl_impersonate: + browser: chrome120 + +# Pre-define default options and switches of the dl command +dl: + best: true + sub_format: srt + downloads: 4 + workers: 16 + +# Chapter Name to use when exporting a Chapter without a Name +chapter_fallback_name: "Chapter {j:02}" + +# Case-Insensitive dictionary of headers for all Services +headers: + Accept-Language: "en-US,en;q=0.8" + User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36" + +# Override default filenames used across envied +filenames: + log: "envied_{name}_{time}.log" + config: "config.yaml" + root_config: "envied.yaml" + chapters: "Chapters_{title}_{random}.txt" + subtitle: "Subtitle_{id}_{language}.srt" + +# API key for The Movie Database (TMDB) +tmdb_api_key: "" + +#conversion_method: +# - auto (default): Smart routing - subby for WebVTT/SAMI, pycaption for others +# - subby: Always use subby with advanced processing +# - pycaption: Use only pycaption library (no SubtitleEdit, no subby) +# - subtitleedit: Prefer SubtitleEdit when available, fall back to pycaption +# - pysubs2: Use pysubs2 library (supports SRT/SSA/ASS/WebVTT/TTML/SAMI/MicroDVD/MPL2/TMP) +subtitle: + conversion_method: auto + # sdh_method: Method to use for SDH (hearing impaired) stripping + # - auto (default): Try subby (SRT only), then SubtitleEdit (if available), then subtitle-filter + # - subby: Use subby library (SRT only) + # - subtitleedit: Use SubtitleEdit tool (Windows only, falls back to subtitle-filter) + # - filter-subs: Use subtitle-filter library directly + sdh_method: auto + # strip_sdh: Automatically create stripped (non-SDH) versions of SDH subtitles + # Set to false to disable automatic SDH stripping entirely (default: true) + strip_sdh: true + # convert_before_strip: Auto-convert VTT/other formats to SRT before using subtitle-filter + # This ensures compatibility when subtitle-filter is used as fallback (default: true) + convert_before_strip: true + # preserve_formatting: Preserve original subtitle formatting (tags, positioning, styling) + # When true, skips pycaption processing for WebVTT files to keep tags like , , positioning intact + # Combined with no sub_format setting, ensures subtitles remain in their original format (default: true) + preserve_formatting: true + # output_mode: Output mode for subtitles + # - mux: Embed subtitles in MKV container only (default) + # - sidecar: Save subtitles as separate files only + # - both: Embed in MKV AND save as sidecar files + output_mode: mux + # sidecar_format: Format for sidecar subtitle files + # Options: srt, vtt, ass, original (keep current format) + sidecar_format: srt + + +# Configuration data for each Service +services: + # Service-specific configuration goes here + # EXAMPLE: + # api_key: "service_specific_key" + +# Legacy NordVPN configuration (use proxy_providers instead) +#proxy_providers: +# nordvpn: +# username: username +# password: password +# server_map: +# us: 6918 +# uk: 2613 +# nz: 100 diff --git a/reset/packages/envied/src/envied/envied.yaml.save b/reset/packages/envied/src/envied/envied.yaml.save new file mode 100644 index 0000000..9ef8af3 --- /dev/null +++ b/reset/packages/envied/src/envied/envied.yaml.save @@ -0,0 +1,274 @@ +# Group or Username to postfix to the end of all download fienames following a dash +#tag: '' + +# Set terminal background color (custom option not in CONFIG.md) +set_terminal_bg: false + +# Muxing configuration +muxing: + set_title: false + +# shakapackager or mp4decrypt +decryption: shakapackager + +# Custom output templates for filenames +# Configure output_template in your unshackle.yaml to control filename format. +# If not configured, default scene-style templates are used and a warning is shown. +# Available variables: {title}, {year}, {season}, {episode}, {season_episode}, {episode_name}, +# {quality}, {resolution}, {source}, {audio}, {audio_channels}, {audio_full}, +# {video}, {hdr}, {hfr}, {atmos}, {dual}, {multi}, {tag}, {edition}, {repack}, +# {lang_tag} +# Conditional variables (included only if present): Add ? suffix like {year?}, {episode_name?}, {hdr?} +# Customize the templates below: +# +# Example outputs: +# Scene movies: 'The.Matrix.1999.1080p.SERVICE.WEB-DL.DDP5.1.H.264-EXAMPLE' +# Scene movies (HDR): 'Dune.2021.2160p.SERVICE.WEB-DL.DDP5.1.HDR10.H.265-EXAMPLE' +# Scene movies (REPACK): 'Dune.2021.REPACK.2160p.SERVICE.WEB-DL.DDP5.1.H.265-EXAMPLE' +# Scene series: 'Breaking.Bad.2008.S01E01.Pilot.1080p.SERVICE.WEB-DL.DDP5.1.H.264-EXAMPLE' +# Plex movies: 'The Matrix (1999)' +# Plex series: 'Breaking Bad S01E01 Pilot' +output_template: + # Scene-style naming (dot-separated) + #movies: '{title}.{year}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + #series: '{title}.{year?}.{season_episode}.{episode_name?}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}' + #songs: '{track_number}.{title}.{repack?}.{edition?}.{source?}.WEB-DL.{audio_full}.{atmos?}-{tag}' + # + # Plex-friendly naming (space-separated, clean format) + movies: '{title} ({year}) {quality}' + series: '{title} {season_episode} {episode_name?}' + songs: '{track_number}. {title}' + # + # Minimal naming (basic info only) + # movies: '{title}.{year}.{quality}' + # series: '{title}.{season_episode}.{episode_name?}' + # + # Custom scene-style with specific elements + # movies: '{title}.{year}.{quality}.{hdr?}.{source}.WEB-DL.{audio_full}.{video}-{tag}' + # series: '{title}.{year?}.{season_episode}.{episode_name?}.{quality}.{hdr?}.{source}.WEB-DL.{audio_full}.{atmos?}.{video}-{tag}' + +# Language-based tagging for output filenames +# Automatically adds language identifiers (e.g., DANiSH, NORDiC, DKsubs) based on +# audio and subtitle track languages. Rules are evaluated in order; first match wins. +# Use {lang_tag?} in your output_template to place the tag in the filename. +# +# Conditions (all conditions in a rule must match): +# audio: - any audio track matches this language +# subs_contain: - any subtitle matches this language +# subs_contain_all: [lang, ...] - subtitles include ALL listed languages +# +# language_tags: +# rules: +# - audio: da +# tag: DANiSH +# - audio: sv +# tag: SWEDiSH +# - audio: nb +# tag: NORWEGiAN +# - audio: en +# subs_contain_all: [da, sv, nb] +# tag: NORDiC +# - audio: en +# subs_contain: da +# tag: DKsubs + + +# Widevine pssh display; fold, crop or ellipsis +pssh_display: ellipsis + +# Login credentials for each Service +credentials: + ALL4: angela.slaney@gmail.com:Nood13sh + ROKU: moretonplumber@yandex.com:$WPJ~Mdh2:X@EtB + TVNZ: moretonplumber@yandex.com:penumbra + TPTV: moretonplumber@yandex.com:penumbra1234 + CBC: moretonplumber@yandex.com:penumbra + CRAV: moretonplumber@yandex.com:Penumbra1$ + +# Override default directories used across unshackle +directories: + cache: Cache + cookies: Cookies + dcsl: DCSL # Device Certificate Status List + downloads: /home/angela/Downloads/devine/ + logs: Logs + temp: Temp + wvds: WVDs + prds: PRDs + # Additional directories that can be configured: + # commands: Commands + #services: + # - ./envied/services/ + vaults: vaults/ + +# fonts: Fonts + +# Pre-define which Widevine or PlayReady device to use for each Service +cdm: + default: device +# itv: DRMLABL3 + +# ========================================== +# REMOTE CDM - not working +# ========================================== + +remote_cdm: + - name: "DRMLABL3" + "Device Type": ANDROID + "Device Name": "l3" + "System ID": 8159 + "Security Level": 3 + "Host": "http://api2.drmlab.io" + "Secret": "drmlab" + +remote_cdm: + - name: "DRMLAB_SL3K" + "Device Type": PLAYREADY + "Device Name": "sl3k" + "System ID": 0 + "Security Level": 3000 + "Host": "http://api2.drmlab.io/playready" + "Secret": "drmlab" + +# Key Vaults store your obtained Content Encryption Keys (CEKs) +key_vaults: +- type: SQLite + name: Local vault + path: /home/angela/Programming/gits/devine-services/key_store.db + +#- type: HTTPAPI +# name: drmlab +# host: http://api.drmlab.io/vault/ +# password: gEX75q7I5YVkvgF5SUkcNd41IbGrDtTT +- type: HTTP + name: DRMLab + host: http://api.drmlab.io/vault/ + password: Qm9XvT4pZ8kLs2HdRw6YnBc7Ju3EfGa1 + api_mode: json + +- type: HTTP + name: Gang Slodziakow Vault + host: http://217.147.172.43/ + username: admin + password: 9BfsPtho4KD7LHpXhI3ZNvTR8p3TQb9iLWYtjRu366Nx9od0PAp8C37gPoDIDxl5 + +- type: API + name: LibreDRM Vault + uri: https://api.mapiro.com/vault + token: 5fc8fcead3bc4c1403dcec54fc3a326fbd3ec95da3899d24 + +- type: HTTP + name: Tupac Vault + host: https://keys.drm.hk + password: a8fGY4LObPLrt6hhT5J6pqAEqya5BJji6J4Y809gdcysW2Mg7Wo4QkrrVgecv0zD + api_mode: json + +# Choose what software to use to download data +#downloader: n_m3u8dl_re +# Options: requests | aria2c | curl_impersonate | n_m3u8dl_re +# Can also be a mapping: +downloader: +# NF: requests +# AMZN: n_m3u8dl_re + ALL4: n_m3u8dl_re + BBC: requests + default: n_m3u8dl_re + +# aria2c downloader configuration +aria2c: + max_concurrent_downloads: 4 + max_connection_per_server: 3 + split: 5 + file_allocation: falloc # none | prealloc | falloc | trunc + +# N_m3u8DL-RE downloader configuration +n_m3u8dl_re: + thread_count: 16 + ad_keyword: "advertisement" + use_proxy: true + +# curl_impersonate downloader configuration +curl_impersonate: + browser: chrome120 + +# Pre-define default options and switches of the dl command +dl: + best: true + sub_format: srt + downloads: 4 + workers: 16 + + + +# Chapter Name to use when exporting a Chapter without a Name +chapter_fallback_name: "Chapter {j:02}" + +# Case-Insensitive dictionary of headers for all Services +headers: + Accept-Language: "en-US,en;q=0.8" + User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36" + +# Override default filenames used across unshackle +filenames: + log: "envied_{name}_{time}.log" + config: "config.yaml" + root_config: "envied.yaml" + chapters: "Chapters_{title}_{random}.txt" + subtitle: "Subtitle_{id}_{language}.srt" + +# API key for The Movie Database (TMDB) +#tmdb_api_key: "cc4e028567dd1bd804d650b891b86f6e" +tmdb_api_key: "502b935d7899aaaf64f0a1104b9bf488" +# tag_imdb_tmdb (bool) +tag_imdb_tmdb: true + + +#conversion_method: +# - auto (default): Smart routing - subby for WebVTT/SAMI, pycaption for others +# - subby: Always use subby with advanced processing +# - pycaption: Use only pycaption library (no SubtitleEdit, no subby) +# - subtitleedit: Prefer SubtitleEdit when available, fall back to pycaption +# - pysubs2: Use pysubs2 library (supports SRT/SSA/ASS/WebVTT/TTML/SAMI/MicroDVD/MPL2/TMP) +subtitle: + conversion_method: auto + # sdh_method: Method to use for SDH (hearing impaired) stripping + # - auto (default): Try subby (SRT only), then SubtitleEdit (if available), then subtitle-filter + # - subby: Use subby library (SRT only) + # - subtitleedit: Use SubtitleEdit tool (Windows only, falls back to subtitle-filter) + # - filter-subs: Use subtitle-filter library directly + sdh_method: auto + # strip_sdh: Automatically create stripped (non-SDH) versions of SDH subtitles + # Set to false to disable automatic SDH stripping entirely (default: true) + strip_sdh: true + # convert_before_strip: Auto-convert VTT/other formats to SRT before using subtitle-filter + # This ensures compatibility when subtitle-filter is used as fallback (default: true) + convert_before_strip: true + # preserve_formatting: Preserve original subtitle formatting (tags, positioning, styling) + # When true, skips pycaption processing for WebVTT files to keep tags like , , positioning intact + # Combined with no sub_format setting, ensures subtitles remain in their original format (default: true) + preserve_formatting: true + # output_mode: Output mode for subtitles + # - mux: Embed subtitles in MKV container only (default) + # - sidecar: Save subtitles as separate files only + # - both: Embed in MKV AND save as sidecar files + output_mode: mux + # sidecar_format: Format for sidecar subtitle files + # Options: srt, vtt, ass, original (keep current format) + sidecar_format: srt + + +# Configuration data for each Service +services: + # Service-specific configuration goes here + # EXAMPLE: + # api_key: "service_specific_key" + +# Legacy NordVPN configuration (use proxy_providers instead) +proxy_providers: + nordvpn: + username: gBM9ejjmY1J15jccH12VT7Qy + password: 1kmuCrRXnT55uAjiUTd8twv9 + server_map: + us: 6918 + uk: 2613 + nz: 100 diff --git a/reset/packages/envied/src/envied/services/ADN/__init__.py b/reset/packages/envied/src/envied/services/ADN/__init__.py new file mode 100644 index 0000000..dc9e2f2 --- /dev/null +++ b/reset/packages/envied/src/envied/services/ADN/__init__.py @@ -0,0 +1,1245 @@ +import base64 +import json +import re +import time +import uuid +import subprocess +import tempfile +import shutil +from typing import Generator, Optional, Union, List, Any +from pathlib import Path +from functools import partial + +import click +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import padding +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from langcodes import Language + +from envied.core import binaries +from envied.core.config import config +from envied.core.manifests import HLS +from envied.core.search_result import SearchResult +from envied.core.service import Service +from envied.core.session import session +from envied.core.titles import Episode, Series +from envied.core.tracks import Tracks, Chapters, Chapter +from envied.core.tracks.audio import Audio +from envied.core.tracks.video import Video +from envied.core.tracks.audio import Audio +from envied.core.tracks.subtitle import Subtitle + + +class VideoNoAudio(Video): + """ + Video track that automatically removes audio after recording. + Necessary because ADN provides HLS streams with muxed audio. + """ + + def download(self, session, prepare_drm, max_workers=None, progress=None, *, cdm=None): + """Override: download then demux to remove audio.""" + import logging + log = logging.getLogger('ADN.VideoNoAudio') + + # Normal download + super().download(session, prepare_drm, max_workers, progress, cdm=cdm) + + # If no path, download failed + if not self.path or not self.path.exists(): + return + + # Check FFmpeg available + if not binaries.FFMPEG: + log.warning("FFmpeg not found, cannot remove audio from video") + return + + # Demux: remove audio + if progress: + progress(downloaded="Removing audio") + + original_path = self.path + noaudio_path = original_path.with_stem(f"{original_path.stem}_noaudio") + + try: + log.debug(f"Removing audio from {original_path.name}") + + result = subprocess.run( + [ + binaries.FFMPEG, + '-i', str(original_path), + '-vcodec', 'copy', # Copy video without re-encoding + '-an', # Remove audio + '-y', + str(noaudio_path) + ], + capture_output=True, + text=True, + timeout=120 + ) + + if result.returncode != 0: + log.error(f"FFmpeg demux failed: {result.stderr}") + noaudio_path.unlink(missing_ok=True) + return + + if not noaudio_path.exists() or noaudio_path.stat().st_size < 1000: + log.error("Demuxed video is empty or too small") + noaudio_path.unlink(missing_ok=True) + return + + # Replace original file + log.debug(f"Video demuxed successfully: {noaudio_path.stat().st_size} bytes") + original_path.unlink() + noaudio_path.rename(original_path) + + if progress: + progress(downloaded="Downloaded") + + except subprocess.TimeoutExpired: + log.error("FFmpeg demux timeout") + noaudio_path.unlink(missing_ok=True) + except Exception as e: + log.error(f"Failed to demux video: {e}") + noaudio_path.unlink(missing_ok=True) + + +class AudioExtracted(Audio): + """ + Audio track already extracted from muxed HLS stream. + Override download() to copy the file instead of downloading. + """ + + def __init__(self, *args, extracted_path: Path, **kwargs): + # Empty URL to prevent curl from trying to download + super().__init__(*args, url="", **kwargs) + self.extracted_path = extracted_path + + def download(self, session, prepare_drm, max_workers=None, progress=None, *, cdm=None): + """Override: copies the extracted file instead of downloading.""" + if not self.extracted_path or not self.extracted_path.exists(): + if progress: + progress(downloaded="[red]FAILED") + raise ValueError(f"Extracted audio file not found: {self.extracted_path}") + + # Create destination path (same logic as Track.download) + track_type = self.__class__.__name__ + save_path = config.directories.temp / f"{track_type}_{self.id}.m4a" + + if progress: + progress(downloaded="Copying", total=100, completed=0) + + # Copy the extracted file to the final destination + config.directories.temp.mkdir(parents=True, exist_ok=True) + shutil.copy2(self.extracted_path, save_path) + + self.path = save_path + + if progress: + progress(downloaded="Downloaded", completed=100) + + +class SubtitleEmbedded(Subtitle): + """ + Subtitle with embedded content (data URI). + Override download() to write the content directly. + """ + + def __init__(self, *args, embedded_content: str, **kwargs): + # Empty URL to prevent curl from trying to download + super().__init__(*args, url="", **kwargs) + self.embedded_content = embedded_content + + def download(self, session, prepare_drm, max_workers=None, progress=None, *, cdm=None): + """Override: writes the embedded content instead of downloading.""" + if not self.embedded_content: + if progress: + progress(downloaded="[red]FAILED") + raise ValueError("No embedded content in subtitle") + + # Create destination path + track_type = "Subtitle" + save_path = config.directories.temp / f"{track_type}_{self.id}.{self.codec.extension}" + + if progress: + progress(downloaded="Writing", total=100, completed=0) + + # Write content + config.directories.temp.mkdir(parents=True, exist_ok=True) + save_path.write_text(self.embedded_content, encoding='utf-8') + + self.path = save_path + + if progress: + progress(downloaded="Downloaded", completed=100) + + +class ADN(Service): + """ + Service code for Animation Digital Network (ADN). + + \b + Version: 3.2.1 (FINAL - Full multi-audio/subtitle support with custom Track classes) + Authorization: Credentials + Robustness: + Video: Clear HLS (Highest Quality) + Audio: Pre-extracted from muxed streams with AudioExtracted class + Subs: AES-128 Encrypted JSON -> ASS format with SubtitleEmbedded class + + Technical Solution: + - ADN provides HLS streams with muxed video+audio (not separable) + - AudioExtracted: Extracts audio in get_tracks(), copies during download() + - SubtitleEmbedded: Decrypts and converts to ASS, writes during download() + - Result: MKV with 1 video + multiple audio tracks + subtitles + + Custom Track Classes: + - AudioExtracted: Bypasses curl file:// limitation with direct file copy + - SubtitleEmbedded: Bypasses requests data: limitation with direct write + Made by: guilara_tv + """ + + + + RSA_PUBLIC_KEY = """-----BEGIN PUBLIC KEY----- +MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCbQrCJBRmaXM4gJidDmcpWDssg +numHinCLHAgS4buMtdH7dEGGEUfBofLzoEdt1jqcrCDT6YNhM0aFCqbLOPFtx9cg +/X2G/G5bPVu8cuFM0L+ehp8s6izK1kjx3OOPH/kWzvstM5tkqgJkNyNEvHdeJl6 +KhS+IFEqwvZqgbBpKuwIDAQAB +-----END PUBLIC KEY-----""" + + TITLE_RE = r"^(?:https?://(?:www\.)?animationdigitalnetwork\.com/video/[^/]+/)?(?P\d+)" + + @staticmethod + def get_session(): + return session("okhttp4") + + @staticmethod + @click.command( + name="ADN", + short_help="https://animationdigitalnetwork.com", + help=( + "Downloads series or movies from ADN.\n\n" + "TITLE: Series URL or ID (eg. 1125).\n\n" + "SELECTION SYSTEM:\n" + " - Simple: '-e 1-5' (episodes 1 to 5)\n" + " - Seasons: '-e S2' or '-e S02' (all season 2) or '-e S2E1-12'\n" + " - Mixed: '-e 1,3,S2E5' or '-e 1,3,S02E05'\n" + " - Bonus: '-e NC1,OAV1'" + ) + ) + @click.argument("title", type=str, required=True) + @click.option( + "-e", "--episode", "select", type=str, + help="Selection: numbers, ranges (5-10), seasons (S1, S2) or combined (S1E5)." + ) + @click.option( + "--but", is_flag=True, + help="Invert selection: download everything EXCEPT episodes specified with -e." + ) + @click.option( + "--all", "all_eps", is_flag=True, + help="Ignore all restrictions and download the entire series." + ) + @click.pass_context + def cli(ctx, **kwargs) -> "ADN": + return ADN(ctx, **kwargs) + + def __init__(self, ctx, title: str, select: Optional[str] = None, but: bool = False, all_eps: bool = False): + self.title = title + self.select_str = select + self.but = but + self.all_eps = all_eps + self.access_token: Optional[str] = None + self.refresh_token: Optional[str] = None + self.token_expiration: Optional[int] = None + + super().__init__(ctx) + + self.locale = self.config.get("params", {}).get("locale", "fr") + self.session.headers.update(self.config.get("headers", {})) + self.session.headers["x-target-distribution"] = self.locale + + + @staticmethod + def _timecode_to_ms(tc: str) -> int: + """Convert HH:MM:SS timecode to milliseconds.""" + parts = tc.split(':') + hours = int(parts[0]) + minutes = int(parts[1]) + seconds = int(parts[2]) + return (hours * 3600 + minutes * 60 + seconds) * 1000 + + @property + def auth_header(self) -> dict: + return { + "Authorization": f"Bearer {self.access_token}", + "X-Access-Token": self.access_token + } + + def ensure_authenticated(self) -> None: + """Checks the token and refreshes if necessary.""" + current_time = int(time.time()) + + if self.access_token and self.token_expiration and current_time < (self.token_expiration - 60): + return + + cache_key = f"adn_auth_{self.credential.sha1 if self.credential else 'default'}" + cached = self.cache.get(cache_key) + + if cached and not cached.expired: + self.access_token = cached.data["access_token"] + self.refresh_token = cached.data["refresh_token"] + self.token_expiration = cached.data["token_expiration"] + self.session.headers.update(self.auth_header) + self.log.debug("Loaded authentication from cache") + else: + self.authenticate(credential=self.credential) + + def authenticate(self, cookies=None, credential=None) -> None: + super().authenticate(cookies, credential) + + if self.refresh_token: + try: + self._do_refresh() + return + except Exception: + self.log.warning("Refresh failed, proceeding to full login") + + if not credential: + raise ValueError("Credentials required for ADN") + + response = self.session.post( + url=self.config["endpoints"]["login"], + json={ + "username": credential.username, + "password": credential.password, + "source": "Web" + } + ) + + if response.status_code != 200: + self.log.error(f"Login failed: {response.status_code} - {response.text}") + response.raise_for_status() + + self._save_tokens(response.json()) + + def _do_refresh(self): + response = self.session.post( + url=self.config["endpoints"]["refresh"], + json={"refreshToken": self.refresh_token}, + headers=self.auth_header + ) + if response.status_code != 200: + raise ValueError("Token refresh failed") + self._save_tokens(response.json()) + + def _save_tokens(self, data: dict): + self.access_token = data["accessToken"] + self.refresh_token = data["refreshToken"] + expires_in = data.get("expires_in", 3600) + self.token_expiration = int(time.time()) + expires_in + self.session.headers.update(self.auth_header) + + def _parse_select(self, ep_id: str, short_number: str, season_num: int, relative_number: Optional[int] = None) -> bool: + """Returns True if the episode should be included.""" + if self.all_eps or not self.select_str: + return True + + # Preparing possible identifiers for this episode + # We test: "30353" (id), "1" (number), "S02E01" (full format), "S02" (entire season) + candidates = [ + str(ep_id), + str(short_number).lstrip("0"), + f"S{season_num:02d}E{int(short_number):02d}" if str(short_number).isdigit() else "", + f"S{season_num:02d}" + ] + + # Add the relative match (e.g. S03E06 for episode 30 which is the 6th ep in S3) + if relative_number is not None: + candidates.append(f"S{season_num:02d}E{relative_number:02d}") + + parts = re.split(r'[ ,]+', self.select_str.strip().upper()) + selection: set[str] = set() + + for part in parts: + if '-' in part: + start_p, end_p = part.split('-', 1) + # Handle ranges S02E01-S02E04 + m_start = re.match(r'^S(\d+)E(\d+)$', start_p) + m_end = re.match(r'^S(\d+)E(\d+)$', end_p) + + if m_start and m_end: + s_start, e_start = map(int, m_start.groups()) + s_end, e_end = map(int, m_end.groups()) + if s_start == s_end: # Same season + for i in range(e_start, e_end + 1): + selection.add(f"S{s_start:02d}E{i:02d}") + continue + + # Classic ranges (1-10) + nums = re.findall(r'\d+', part) + if len(nums) >= 2: + for i in range(int(nums[0]), int(nums[1]) + 1): + selection.add(str(i)) + else: + selection.add(part.lstrip("0")) + + included = any(c in selection for c in candidates if c) + return not included if self.but else included + + def get_titles(self) -> Series: + """Retrieves episodes with the actual title of the series.""" + show_id = self.parse_show_id(self.title) + + # 1. Fetch the overall show info first to get the proper title + show_url = self.config["endpoints"]["show"].format(show_id=show_id) + show_res = self.session.get(show_url).json() + + # We extract the series title (e.g. "Demon Slave") + # This title will be used as the unique folder name + series_title = show_res["videos"][0]["show"]["title"] if show_res.get("videos") else "ADN Show" + + # 2. Fetch the season structure afterwards + url_seasons = self.config["endpoints"].get("seasons") + if not url_seasons: + url_seasons = "https://gw.api.animationdigitalnetwork.com/video/show/{show_id}/seasons?maxAgeCategory=18&order=asc" + + res = self.session.get(url_seasons.format(show_id=show_id)).json() + + if not res.get("seasons"): + self.log.error(f"No seasons found for ID {show_id}") + return Series([]) + + episodes = [] + for season_data in res["seasons"]: + s_val = str(season_data.get("season", "1")) + season_num = int(s_val) if s_val.isdigit() else 1 + + for idx, vid in enumerate(season_data.get("videos", []), 1): + video_id = str(vid["id"]) + + # Clean the episode number (keep only digits) + num_match = re.search(r'\d+', str(vid.get("number", "0"))) + short_number = num_match.group() if num_match else "0" + + # Selection logic (SxxEyy) - Relative and absolute support + if not self._parse_select(video_id, short_number, season_num, relative_number=idx): + continue + + # Create the episode + episodes.append(Episode( + id_=video_id, + service=self.__class__, + title=series_title, # Folder: "Demon Slave" + season=season_num, # Season: 2 + number=idx, # Force relative number (e.g. 30 becomes 6 in S3) + name=vid.get("name") or "", # Name: "The big reunion..." + data=vid + )) + + episodes.sort(key=lambda x: (x.season, x.number)) + return Series(episodes) + + def get_discovery(self, n: int = 12) -> list[dict]: + """ + Fetch the latest releases (Series) via the catalog API. + """ + self.ensure_authenticated() + + try: + url = self.config["endpoints"]["search"] + response = self.session.get( + url, + params={ + "maxAgeCategory": 18, + "order": "new", + "limit": n + } + ) + + if response.status_code != 200: + self.log.error(f"Catalog fetch failed: {response.status_code}") + return [] + + data = response.json() + return data.get("shows", []) + + except Exception as e: + self.log.error(f"Error fetching discovery: {e}") + return [] + + def get_latest_releases_calendar(self, n: int = 12) -> list[dict]: + """ + Fetch the latest releases (Episodes) via the calendar API. + Returns recent episodes with their actual release dates. + """ + from datetime import datetime, timedelta + import requests + + # self.ensure_authenticated() + + try: + results = [] + seen_episodes = set() + + # Use a fresh session to avoid auth issues if calendar is public + cal_session = requests.Session() + cal_session.headers.update({ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" + }) + + # Fetch calendar for the last 7 days including today + today = datetime.now() + + for day_offset in range(15): + date = today - timedelta(days=day_offset) + date_str = date.strftime("%Y-%m-%d") + + # Calendar endpoint + calendar_url = f"https://gw.api.animationdigitalnetwork.com/video/calendar?maxAgeCategory=18&date={date_str}" + + try: + res = cal_session.get(calendar_url, timeout=5) + if res.status_code != 200: continue + + data = res.json() + # Handle dict response (API returns {"videos": [...]}) + if isinstance(data, dict): + data = data.get("videos", []) + + for video in data: + show = video.get("show", {}) + show_id = str(show.get("id", "")) + ep_id = str(video.get("id", "")) + + if (show_id, ep_id) in seen_episodes: continue + seen_episodes.add((show_id, ep_id)) + + results.append(video) + except: + pass + + # Sort by releaseDate + results.sort(key=lambda x: x.get("releaseDate", ""), reverse=True) + return results[:n] + + except Exception as e: + self.log.error(f"Error fetching calendar discovery: {e}") + return [] + + def get_latest_episode(self, show_id: str) -> Episode | None: + """ + Retrieves the absolute latest episode of a series without filtering. + Used by the GUI to display 'Latest' info. + """ + try: + # 1. Fetch seasons + url_seasons = self.config["endpoints"].get("seasons") + if not url_seasons: + url_seasons = "https://gw.api.animationdigitalnetwork.com/video/show/{show_id}/seasons?maxAgeCategory=18&order=asc" + + res = self.session.get(url_seasons.format(show_id=show_id)) + if res.status_code != 200: + return None + + data = res.json() + if not data.get("seasons"): + return None + + episodes = [] + for season_data in data["seasons"]: + s_val = str(season_data.get("season", "1")) + season_num = int(s_val) if s_val.isdigit() else 1 + + for vid in season_data.get("videos", []): + # Simplified parsing similar to get_titles + num_match = re.search(r'\d+', str(vid.get("number", "0"))) + short_number = int(num_match.group()) if num_match else 0 + + episodes.append(Episode( + id_=str(vid["id"]), + service=self.__class__, + title="", # We don't need the series title here for simple display + season=season_num, + number=short_number, + name=vid.get("name") or "", + data=vid + )) + + if not episodes: + return None + + # Sort: Season desc, Number desc to find absolute latest? + # Or Season asc, Number asc and take [-1] + episodes.sort(key=lambda x: (x.season, x.number)) + return episodes[-1] + + except Exception as e: + self.log.error(f"Error in get_latest_episode: {e}") + return None + + def get_tracks(self, title: Episode) -> Tracks: + """ + Fetches tracks by pre-extracting audio. + Audio is extracted now and will be copied during download(). + """ + self.ensure_authenticated() + vid_id = title.id + + # Player configuration + config_url = self.config["endpoints"]["player_config"].format(video_id=vid_id) + config_res = self.session.get(config_url).json() + + player_opts = config_res["player"]["options"] + if not player_opts["user"]["hasAccess"]: + raise PermissionError("No access to this video (Premium required?)") + + # Player token + refresh_url = player_opts["user"].get("refreshTokenUrl") or self.config["endpoints"]["player_refresh"] + token_res = self.session.post( + refresh_url, + headers={"X-Player-Refresh-Token": player_opts["user"]["refreshToken"]} + ).json() + + player_token = token_res["token"] + links_url = player_opts["video"].get("url") or self.config["endpoints"]["player_links"].format(video_id=vid_id) + + # RSA Encryption + rand_key = uuid.uuid4().hex[:16] + payload = json.dumps({"k": rand_key, "t": player_token}).encode('utf-8') + + public_key = serialization.load_pem_public_key( + self.RSA_PUBLIC_KEY.encode('utf-8'), + backend=default_backend() + ) + + encrypted = public_key.encrypt(payload, padding.PKCS1v15()) + auth_header_val = base64.b64encode(encrypted).decode('utf-8') + + # Fetching links + links_res = self.session.get( + links_url, + params={"freeWithAds": "true", "adaptive": "true", "withMetadata": "true", "source": "Web"}, + headers={"X-Player-Token": auth_header_val} + ).json() + + tracks = Tracks() + streaming_links = links_res.get("links", {}).get("streaming", {}) + + # Language map + lang_map = { + "vf": "fr", + "vostf": "ja", + "vde": "de", + "vostde": "ja", + } + + # Priority: VOSTF (original) for the main video + priority_order = ["vostf", "vf", "vde", "vostde"] + available_streams = {k: v for k, v in streaming_links.items() if k in lang_map} + + sorted_streams = sorted( + available_streams.keys(), + key=lambda x: priority_order.index(x) if x in priority_order else 999 + ) + + if not sorted_streams: + raise ValueError("No supported streams found") + + # Main video (VOSTF or first available) + primary_stream = sorted_streams[0] + primary_lang = lang_map[primary_stream] + + self.log.info(f"Primary video stream: {primary_stream} ({primary_lang})") + + video_track = self._get_video_track( + streaming_links[primary_stream], + primary_stream, + primary_lang, + is_original=(primary_stream in ["vostf", "vostde"]) + ) + + if video_track: + tracks.add(video_track) + self.log.info(f"Video track added: {video_track.width}x{video_track.height}") + + # Extract audio for all available languages + for stream_type in sorted_streams: + audio_lang = lang_map[stream_type] + is_original = stream_type in ["vostf", "vostde"] + + self.log.info(f"Processing audio for: {stream_type} ({audio_lang})") + + audio_track = self._extract_audio_track( + streaming_links[stream_type], + stream_type, + audio_lang, + is_original, + title + ) + + if audio_track: + tracks.add(audio_track, warn_only=True) + self.log.info(f"Audio track added: {audio_lang}") + + # Store chapter data for get_chapters() + if "video" in links_res: + title.data["chapter_data"] = links_res["video"] + self.log.debug(f"Stored chapter data: intro={links_res['video'].get('tcIntroStart')}, ending={links_res['video'].get('tcEndingStart')}") + + # Subtitles + self._process_subtitles(links_res, rand_key, title, tracks) + + if not tracks.videos: + raise ValueError("No video tracks were successfully added") + + return tracks + + def _get_video_track(self, stream_data: dict, stream_type: str, lang: str, is_original: bool): + """Fetches the main video track (without audio).""" + try: + m3u8_url = self._resolve_stream_url(stream_data, stream_type) + if not m3u8_url: + return None + + hls_manifest = HLS.from_url(url=m3u8_url, session=self.session) + hls_tracks = hls_manifest.to_tracks(language=lang) + + if not hls_tracks.videos: + self.log.warning(f"No video tracks found for {stream_type}") + return None + + # Best quality + best_video = max( + hls_tracks.videos, + key=lambda v: (v.height or 0, v.width or 0, v.bitrate or 0) + ) + + # Convert to VideoNoAudio to demux automatically + video_no_audio = VideoNoAudio( + id_=best_video.id, + url=best_video.url, + codec=best_video.codec, + language=Language.get(lang), + is_original_lang=is_original, + bitrate=best_video.bitrate, + descriptor=best_video.descriptor, + width=best_video.width, + height=best_video.height, + fps=best_video.fps, + range_=best_video.range, + data=best_video.data, + ) + + video_no_audio.data["stream_type"] = stream_type + + return video_no_audio + + except Exception as e: + self.log.error(f"Failed to get video track for {stream_type}: {e}") + return None + + def _extract_audio_track(self, stream_data: dict, stream_type: str, lang: str, is_original: bool, title: Episode): + """ + Extracts audio and returns an AudioExtracted. + Audio is extracted NOW and will be copied during download(). + """ + if not binaries.FFMPEG: + self.log.warning("FFmpeg not found, cannot extract audio") + return None + + try: + m3u8_url = self._resolve_stream_url(stream_data, stream_type, prioritize_auto=True) + if not m3u8_url: + return None + + # Create an ADN temp directory inside envied.s temp + adn_temp = config.directories.temp / "adn_audio_extracts" + adn_temp.mkdir(parents=True, exist_ok=True) + + # Unique filename based on video_id + language + audio_filename = f"audio_{title.id}_{stream_type}.m4a" + audio_path = adn_temp / audio_filename + + # If already extracted, reuse it + if audio_path.exists() and audio_path.stat().st_size > 1000: + self.log.debug(f"Reusing existing extracted audio: {audio_path}") + else: + + # Parse M3U8 to find best audio + best_m3u8_url = m3u8_url + try: + import m3u8 + variant_m3u8 = m3u8.load(m3u8_url) + + audio_playlists = [] + # Check for alternative audio in media + for media in variant_m3u8.media: + if media.type == "AUDIO" and media.uri: + audio_playlists.append(media.uri) + + # If no media, check strict playlists (variants) + if not audio_playlists and variant_m3u8.playlists: + # Sort by bandwidth descending + sorted_playlists = sorted(variant_m3u8.playlists, key=lambda x: x.stream_info.bandwidth or 0, reverse=True) + audio_playlists = [p.uri for p in sorted_playlists] + + if audio_playlists: + # Construct absolute URL if needed + from urllib.parse import urljoin + best_audio_uri = audio_playlists[0] + if not best_audio_uri.startswith("http"): + best_m3u8_url = urljoin(m3u8_url, best_audio_uri) + else: + best_m3u8_url = best_audio_uri + self.log.info(f"Selected best audio stream: {best_m3u8_url}") + + except Exception as e: + self.log.warning(f"Failed to parse M3U8 for best audio, using default: {e}") + + # Extract with FFmpeg + result = subprocess.run( + [ + binaries.FFMPEG, + '-i', best_m3u8_url, + '-vn', + '-acodec', 'copy', + '-y', + str(audio_path) + ], + capture_output=True, + text=True, + timeout=300 + ) + + if result.returncode != 0: + self.log.error(f"FFmpeg failed for {stream_type}: {result.stderr}") + audio_path.unlink(missing_ok=True) + return None + + if not audio_path.exists() or audio_path.stat().st_size < 1000: + self.log.error(f"Extracted audio is invalid for {stream_type}") + audio_path.unlink(missing_ok=True) + return None + + # Detect actual bitrate + detected_bitrate = 128000 + try: + from pymediainfo import MediaInfo + media_info = MediaInfo.parse(audio_path) + if media_info.audio_tracks: + track = media_info.audio_tracks[0] + if track.bit_rate: + detected_bitrate = int(track.bit_rate) + elif track.other_bit_rate: + # Fallback for some formats + try: + # other_bit_rate is typically list like ['128 kb/s'] + raw = track.other_bit_rate[0] + detected_bitrate = int(re.sub(r'[^\d]', '', raw)) * 1000 + except: + pass + self.log.debug(f"Detected audio bitrate: {detected_bitrate}") + except Exception as e: + self.log.warning(f"Failed to detect bitrate: {e}") + + # Create AudioExtracted with the pre-extracted file + audio_track = AudioExtracted( + id_=f"audio-{stream_type}-{lang}", + extracted_path=audio_path, + codec=Audio.Codec.AAC, + language=Language.get(lang), + is_original_lang=is_original, + bitrate=detected_bitrate, + channels=2.0, + ) + + return audio_track + + except subprocess.TimeoutExpired: + self.log.error(f"FFmpeg timeout for {stream_type}") + return None + except Exception as e: + self.log.error(f"Failed to extract audio for {stream_type}: {e}") + return None + + def _resolve_stream_url(self, stream_data: dict, stream_type: str, prioritize_auto: bool = False) -> Optional[str]: + """Resolves the stream URL.""" + if prioritize_auto: + preferred_keys = ["auto", "fhd", "hd", "sd", "mobile"] + else: + preferred_keys = ["fhd", "hd", "auto", "sd", "mobile"] + + m3u8_url = None + for key in preferred_keys: + if key in stream_data and stream_data[key]: + m3u8_url = stream_data[key] + break + + if not m3u8_url: + return None + + try: + resp = self.session.get(m3u8_url, timeout=12) + if resp.status_code != 200: + return None + + content_type = resp.headers.get("Content-Type", "") + resp_text = resp.text.strip() + + if "application/json" in content_type or resp_text.startswith("{"): + try: + json_data = resp.json() + real_location = json_data.get("location") + if real_location: + return real_location + except json.JSONDecodeError: + pass + + return m3u8_url + + except Exception as e: + self.log.error(f"Failed to resolve URL for {stream_type}: {e}") + return None + + def _process_subtitles(self, links_res: dict, rand_key: str, title: Episode, tracks: Tracks): + """Processes subtitles.""" + subs_root = links_res.get("links", {}).get("subtitles", {}) + if "all" not in subs_root: + self.log.debug("No subtitles available") + return + + aes_key_bytes = bytes.fromhex(rand_key + '7fac1178830cfe0c') + + try: + sub_loc_res = self.session.get(subs_root["all"]).json() + encrypted_sub_res = self.session.get(sub_loc_res["location"]).text + + self.log.debug(f"Encrypted subtitle length: {len(encrypted_sub_res)}") + + iv_b64 = encrypted_sub_res[:24] + payload_b64 = encrypted_sub_res[24:] + + iv = base64.b64decode(iv_b64) + ciphertext = base64.b64decode(payload_b64) + + self.log.debug(f"IV length: {len(iv)}, Ciphertext length: {len(ciphertext)}") + + cipher = Cipher(algorithms.AES(aes_key_bytes), modes.CBC(iv), backend=default_backend()) + decryptor = cipher.decryptor() + decrypted_padded = decryptor.update(ciphertext) + decryptor.finalize() + + # ALWAYS remove PKCS7 padding (Python doesn't do it automatically) + pad_len = decrypted_padded[-1] + if not (1 <= pad_len <= 16): + self.log.error(f"Invalid PKCS7 padding length: {pad_len}") + return + + # Ensure all padding bytes have the same value + padding = decrypted_padded[-pad_len:] + if not all(b == pad_len for b in padding): + self.log.error(f"Invalid PKCS7 padding bytes") + return + + decrypted_json = decrypted_padded[:-pad_len].decode('utf-8') + self.log.debug(f"Decrypted JSON length: {len(decrypted_json)}") + + + subs_data = json.loads(decrypted_json) + + + if not isinstance(subs_data, dict): + self.log.error(f"subs_data is not a dict! Type: {type(subs_data)}") + return + + if len(subs_data) == 0: + self.log.warning("subs_data is empty!") + return + + # Debug each key + for key in subs_data.keys(): + value = subs_data[key] + if isinstance(value, list) and len(value) > 0: + self.log.debug(f" First item type: {type(value[0])}") + self.log.debug(f" First item keys: {value[0].keys() if isinstance(value[0], dict) else 'NOT A DICT'}") + self.log.debug(f" First item sample: {str(value[0])[:200]}") + processed_langs = set() + + for sub_lang_key, cues in subs_data.items(): + + if not isinstance(cues, list): + self.log.warning(f"Cues for {sub_lang_key} is not a list! Type: {type(cues)}") + continue + + if len(cues) == 0: + self.log.debug(f"No subtitles for {sub_lang_key} (normal for dubbed versions)") + continue + + self.log.debug(f" Cues count: {len(cues)}") + self.log.debug(f" First cue: {cues[0]}") + + if "vf" in sub_lang_key.lower(): + target_lang = "fr" + is_forced = True + elif "vostf" in sub_lang_key.lower(): + target_lang = "fr" + is_forced = False + elif "vde" in sub_lang_key.lower(): + target_lang = "de" + is_forced = True + elif "vostde" in sub_lang_key.lower(): + target_lang = "de" + is_forced = False + else: + self.log.debug(f"Skipping subtitle language: {sub_lang_key}") + continue + + if (target_lang, is_forced) in processed_langs: + self.log.debug(f"Already processed {target_lang} (forced={is_forced}), skipping") + continue + + processed_langs.add((target_lang, is_forced)) + + # Convert to ASS + ass_content = self._json_to_ass(cues, title.title, title.number) + + # Check if ASS file has content + event_count = ass_content.count("Dialogue:") + self.log.debug(f"Generated ASS with {event_count} dialogue events") + + if event_count == 0: + self.log.warning(f"ASS file has no dialogue events!") + self.log.warning(f"First cue was: {cues[0] if cues else 'EMPTY LIST'}") + + # Create SubtitleEmbedded directly with ASS content + subtitle = SubtitleEmbedded( + id_=f"sub-{target_lang}-{sub_lang_key}", + embedded_content=ass_content, # ASS content directly + codec=Subtitle.Codec.SubStationAlphav4, + language=Language.get(target_lang), + forced=is_forced, + sdh=False, + ) + + tracks.add(subtitle, warn_only=True) + self.log.info(f"Subtitle added: {target_lang} ({event_count} events)") + + except json.JSONDecodeError as e: + self.log.error(f"Failed to decode JSON: {e}") + self.log.error(f"Decrypted data (first 500 chars): {decrypted_json[:500] if 'decrypted_json' in locals() else 'NOT DECRYPTED'}") + except Exception as e: + self.log.error(f"Failed to process subtitles: {e}") + import traceback + self.log.debug(traceback.format_exc()) + + def get_chapters(self, title: Episode) -> Chapters: + """ + Creates chapters from ADN timecodes. + - If tcIntroStart exists: + - If tcIntroStart != "00:00:00": add "Prologue" at 00:00:00 + - Add "Opening" at tcIntroStart + - Add "Episode" at tcIntroEnd + - Otherwise: add "Episode" at 00:00:00 + - If tcEndingStart exists: + - Add "Ending Start" at tcEndingStart + - Add "Ending End" at tcEndingEnd + """ + chapters = Chapters() + + # Retrieve chapter data stored in get_tracks() + chapter_data = title.data.get("chapter_data", {}) + if not chapter_data: + self.log.debug("No chapter data available") + return chapters + + tc_intro_start = chapter_data.get("tcIntroStart") + tc_intro_end = chapter_data.get("tcIntroEnd") + tc_ending_start = chapter_data.get("tcEndingStart") + tc_ending_end = chapter_data.get("tcEndingEnd") + + self.log.debug(f"Chapter timecodes: intro={tc_intro_start}->{tc_intro_end}, ending={tc_ending_start}->{tc_ending_end}") + + try: + chapter_num = 1 + + if tc_intro_start: + # If intro does not start at 00:00:00, add a prologue (Chapter 1) + if tc_intro_start != "00:00:00": + chapters.add(Chapter( + timestamp=0, + name=f"Chapter {chapter_num}" + )) + self.log.debug(f"Added Chapter {chapter_num} at 00:00:00") + chapter_num += 1 + + # Opening + chapters.add(Chapter( + timestamp=self._timecode_to_ms(tc_intro_start), + name="Opening" + )) + self.log.debug(f"Added Opening at {tc_intro_start}") + + # Episode (after intro) + if tc_intro_end: + chapters.add(Chapter( + timestamp=self._timecode_to_ms(tc_intro_end), + name=f"Chapter {chapter_num}" + )) + self.log.debug(f"Added Chapter {chapter_num} at {tc_intro_end}") + chapter_num += 1 + else: + # No intro, episode starts at 00:00:00 + chapters.add(Chapter( + timestamp=0, + name=f"Chapter {chapter_num}" + )) + self.log.debug(f"Added Chapter {chapter_num} at 00:00:00 (no intro)") + chapter_num += 1 + + # Ending + if tc_ending_start: + chapters.add(Chapter( + timestamp=self._timecode_to_ms(tc_ending_start), + name="Ending" + )) + self.log.debug(f"Added Ending at {tc_ending_start}") + + if tc_ending_end: + # Check if the remaining chapter has a significant duration (> 10s) + # to avoid micro-chapters of 2s, while keeping actual post-credits scenes. + + tc_end_ms = self._timecode_to_ms(tc_ending_end) + total_duration_s = title.data.get("duration", 0) + total_duration_ms = int(total_duration_s * 1000) + + # If duration is unknown or > 10 seconds remaining + should_add = True + if total_duration_ms > 0: + remaining_ms = total_duration_ms - tc_end_ms + if remaining_ms < 10000: # Less than 10s + should_add = False + self.log.debug(f"Skipping post-ending chapter (only {remaining_ms}ms remaining)") + + if should_add: + chapters.add(Chapter( + timestamp=tc_end_ms, + name=f"Chapter {chapter_num}" + )) + self.log.debug(f"Added Chapter {chapter_num} at {tc_ending_end}") + chapter_num += 1 + + self.log.info(f"✓ Created {len(chapters)} chapters") + + except Exception as e: + self.log.error(f"Failed to create chapters: {e}") + import traceback + self.log.debug(traceback.format_exc()) + + return chapters + + def search(self) -> Generator[SearchResult, None, None]: + res = self.session.get( + self.config["endpoints"]["search"], + params={"search": self.title, "limit": 20, "offset": 0} + ).json() + + for show in res.get("shows", []): + yield SearchResult( + id_=str(show["id"]), + title=show["title"], + label=show["type"], + description=show.get("summary", "")[:300], + url=f"https://animationdigitalnetwork.com/video/{show['id']}", + image=show.get("image") + ) + + def parse_show_id(self, input_str: str) -> str: + if input_str.isdigit(): + return input_str + match = re.match(self.TITLE_RE, input_str) + if match: + return match.group("id") + raise ValueError(f"Invalid ADN Show ID/URL: {input_str}") + + def _json_to_ass(self, cues: List[dict], title: str, ep_num: Union[int, str]) -> str: + """Converts JSON subtitles to ASS.""" + header = """[Script Info] +ScriptType: v4.00+ +WrapStyle: 0 +Collisions: Normal +PlayResX: 1920 +PlayResY: 1080 +Timer: 0.0000 +WrapStyle: 0 +ScaledBorderAndShadow: yes + +[V4+ Styles] +Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding +Style: Default,Trebuchet MS,66,&H00FFFFFF,&H000000FF,&H00000000,&H00000000,-1,0,0,0,100,100,0,0,1,3,3,2,75,75,75,1 + +[Events] +Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text +""" + events = [] + pos_align_map = {"start": 1, "end": 3} + line_align_map = {"middle": 8, "end": 4} + + def format_time(seconds: float) -> str: + """Exact ADN format: HH:MM:SS.CC (2-digit centiseconds)""" + secs = int(seconds) + centiseconds = round((seconds - secs) * 100) + + hours = secs // 3600 + minutes = (secs % 3600) // 60 + remaining_seconds = secs % 60 + + # 2-digit padding for EVERYTHING (including hours) + return f"{hours:02d}:{minutes:02d}:{remaining_seconds:02d}.{centiseconds:02d}" + + for cue in cues: + start_time = cue.get("startTime", 0) + end_time = cue.get("endTime", 0) + text = cue.get("text", "") + + # Skip if text is empty + if not text or not text.strip(): + continue + + # EXACT cleanup corresponding to ADN code + text = text.replace(' \\N', '\\N') # remove space before \\N at end + if text.endswith('\\N'): + text = text[:-2] # remove \\N at end + text = text.replace('\r', '') + text = text.replace('\n', '\\N') + text = re.sub(r'\\N +', r'\\N', text) # \\N followed by spaces + text = re.sub(r' +\\N', r'\\N', text) # spaces followed by \\N + text = re.sub(r'(\\N)+', r'\\N', text) # multiple \\N + text = re.sub(r']*>([^<]*)', r'{\\b1}\1{\\b0}', text) + text = re.sub(r']*>([^<]*)', r'{\\i1}\1{\\i0}', text) + text = re.sub(r']*>([^<]*)', r'{\\u1}\1{\\u0}', text) + text = text.replace('<', '<').replace('>', '>').replace('&', '&') + text = re.sub(r'<[^>]>', '', text) # remove any remaining single tags + if text.endswith('\\N'): + text = text[:-2] + text = text.rstrip() # remove trailing spaces + + # Skip after cleanup if empty + if not text.strip(): + continue + + p_align = pos_align_map.get(cue.get("positionAlign"), 2) + l_align = line_align_map.get(cue.get("lineAlign", ""), 0) + align_val = p_align + l_align + + start = format_time(start_time) + end = format_time(end_time) + + style_mod = f"{{\\a{align_val}}}" if align_val != 2 else "" + events.append(f"Dialogue: 0,{start},{end},Default,,0,0,0,,{style_mod}{text}") + + self.log.debug(f"Converted {len(events)} subtitle events from {len(cues)} cues") + + if not events: + self.log.warning(f"No subtitle events generated - all cues were empty or invalid (total cues: {len(cues)})") + + return header + "\n".join(events) diff --git a/reset/packages/envied/src/envied/services/ADN/config.yaml b/reset/packages/envied/src/envied/services/ADN/config.yaml new file mode 100644 index 0000000..ffe4f9a --- /dev/null +++ b/reset/packages/envied/src/envied/services/ADN/config.yaml @@ -0,0 +1,29 @@ +# API Configuration + +# API Endpoints +endpoints: + # Authentication + login: "https://gw.api.animationdigitalnetwork.com/authentication/login" + refresh: "https://gw.api.animationdigitalnetwork.com/authentication/refresh" + +# Catalog + search: "https://gw.api.animationdigitalnetwork.com/show/catalog" + show: "https://gw.api.animationdigitalnetwork.com/video/show/{show_id}?maxAgeCategory=18&limit=-1&order=asc" + seasons: "https://gw.api.animationdigitalnetwork.com/video/show/{show_id}/seasons?maxAgeCategory=18&order=asc" + +# Player & Playback + player_config: "https://gw.api.animationdigitalnetwork.com/player/video/{video_id}/configuration" + player_refresh: "https://gw.api.animationdigitalnetwork.com/player/refresh/token" + player_links: "https://gw.api.animationdigitalnetwork.com/player/video/{video_id}/link" + +# Default Headers +headers: + User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + Origin: "https://animationdigitalnetwork.com" + Referer: "https://animationdigitalnetwork.com/" + Content-Type: "application/json" + X-Target-Distribution: "fr" + +# Parameters +params: + locale: "fr" \ No newline at end of file diff --git a/reset/packages/envied/src/envied/services/ALL4/__init__.py b/reset/packages/envied/src/envied/services/ALL4/__init__.py new file mode 100644 index 0000000..5ef72de --- /dev/null +++ b/reset/packages/envied/src/envied/services/ALL4/__init__.py @@ -0,0 +1,423 @@ +from __future__ import annotations + +import base64 +import hashlib +import json +import re +import sys +from collections.abc import Generator +from datetime import datetime, timezone +from http.cookiejar import MozillaCookieJar +from typing import Any, Optional, Union + +import click +from click import Context +from Crypto.Util.Padding import unpad +from Cryptodome.Cipher import AES +from pywidevine.cdm import Cdm as WidevineCdm +from envied.core.credential import Credential +from envied.core.manifests.dash import DASH +from envied.core.search_result import SearchResult +from envied.core.service import Service +from envied.core.titles import Episode, Movie, Movies, Series +from envied.core.tracks import Chapter, Subtitle, Tracks + + +class ALL4(Service): + """ + Service code for Channel 4's All4 streaming service (https://channel4.com). + + \b + Version: 1.0.2 + Author: stabbedbybrick + Authorization: Credentials + Robustness: + L3: 1080p, AAC2.0 + + \b + Tips: + - Use complete title URL or slug as input: + https://www.channel4.com/programmes/taskmaster OR taskmaster + - Use on demand URL for directly downloading episodes: + https://www.channel4.com/programmes/taskmaster/on-demand/75588-002 + - Both android and web/pc endpoints are checked for quality profiles. + If android is missing 1080p, it automatically falls back to web. + """ + + GEOFENCE = ("gb", "ie") + TITLE_RE = r"^(?:https?://(?:www\.)?channel4\.com/programmes/)?(?P[a-z0-9-]+)(?:/on-demand/(?P[0-9-]+))?" + + @staticmethod + @click.command(name="ALL4", short_help="https://channel4.com", help=__doc__) + @click.argument("title", type=str) + @click.pass_context + def cli(ctx: Context, **kwargs: Any) -> ALL4: + return ALL4(ctx, **kwargs) + + def __init__(self, ctx: Context, title: str): + self.title = title + super().__init__(ctx) + + self.authorization: str + self.asset_id: int + self.license_token: str + self.manifest: str + + self.session.headers.update( + { + "X-C4-Platform-Name": self.config["device"]["platform_name"], + "X-C4-Device-Type": self.config["device"]["device_type"], + "X-C4-Device-Name": self.config["device"]["device_name"], + "X-C4-App-Version": self.config["device"]["app_version"], + "X-C4-Optimizely-Datafile": self.config["device"]["optimizely_datafile"], + } + ) + + def authenticate(self, cookies: Optional[MozillaCookieJar] = None, credential: Optional[Credential] = None) -> None: + super().authenticate(cookies, credential) + if not credential: + raise EnvironmentError("Service requires Credentials for Authentication.") + + cache = self.cache.get(f"tokens_{credential.sha1}") + + if cache and not cache.expired: + # cached + self.log.info(" + Using cached Tokens...") + tokens = cache.data + elif cache and cache.expired: + # expired, refresh + self.log.info("Refreshing cached Tokens") + r = self.session.post( + self.config["endpoints"]["login"], + headers={"authorization": f"Basic {self.config['android']['auth']}"}, + data={ + "grant_type": "refresh_token", + "username": credential.username, + "password": credential.password, + "refresh_token": cache.data["refreshToken"], + }, + ) + try: + res = r.json() + except json.JSONDecodeError: + raise ValueError(f"Failed to refresh tokens: {r.text}") + + if "error" in res: + self.log.error(f"Failed to refresh tokens: {res['errorMessage']}") + sys.exit(1) + + tokens = res + self.log.info(" + Refreshed") + else: + # new + headers = {"authorization": f"Basic {self.config['android']['auth']}"} + data = { + "grant_type": "password", + "username": credential.username, + "password": credential.password, + } + r = self.session.post(self.config["endpoints"]["login"], headers=headers, data=data) + try: + res = r.json() + except json.JSONDecodeError: + raise ValueError(f"Failed to log in: {r.text}") + + if "error" in res: + self.log.error(f"Failed to log in: {res['errorMessage']}") + sys.exit(1) + + tokens = res + self.log.info(" + Acquired tokens...") + + cache.set(tokens, expiration=tokens["expiresIn"]) + + self.authorization = f"Bearer {tokens['accessToken']}" + + def search(self) -> Generator[SearchResult, None, None]: + params = { + "expand": "default", + "q": self.title, + "limit": "100", + "offset": "0", + } + + r = self.session.get(self.config["endpoints"]["search"], params=params) + r.raise_for_status() + + results = r.json() + if isinstance(results["results"], list): + for result in results["results"]: + yield SearchResult( + id_=result["brand"].get("websafeTitle"), + title=result["brand"].get("title"), + description=result["brand"].get("description"), + label=result.get("label"), + url=result["brand"].get("href"), + ) + + def get_titles(self) -> Union[Movies, Series]: + title, on_demand = (re.match(self.TITLE_RE, self.title).group(i) for i in ("id", "vid")) + + r = self.session.get( + self.config["endpoints"]["title"].format(title=title), + params={"client": "android-mod", "deviceGroup": "mobile", "include": "extended-restart"}, + headers={"Authorization": self.authorization}, + ) + if not r.ok: + self.log.error(r.text) + sys.exit(1) + + data = r.json() + + if on_demand is not None: + episodes = [ + Episode( + id_=episode["programmeId"], + service=self.__class__, + title=data["brand"]["title"], + season=episode["seriesNumber"], + number=episode["episodeNumber"], + name=self._clean_episode(episode.get("originalTitle", "")), + language="en", + data=episode["assetInfo"].get("streaming") or episode["assetInfo"].get("download"), + ) + for episode in data["brand"]["episodes"] + if episode.get("assetInfo") and episode["programmeId"] == on_demand + ] + if not episodes: + # Parse HTML of episode page to find title + data = self.get_html(self.title) + episodes = [ + Episode( + id_=data["selectedEpisode"]["programmeId"], + service=self.__class__, + title=data["brand"]["title"], + season=data["selectedEpisode"]["seriesNumber"] or 0, + number=data["selectedEpisode"]["episodeNumber"] or 0, + name=self._clean_episode(data.get("selectedEpisode", {}).get("originalTitle", "")), + language="en", + data=data["selectedEpisode"], + ) + ] + + return Series(episodes) + + elif data["brand"]["programmeType"] == "FM": + return Movies( + [ + Movie( + id_=movie["programmeId"], + service=self.__class__, + name=data["brand"]["title"], + year=int(data["brand"]["summary"].split(" ")[0].strip().strip("()")), + language="en", + data=movie["assetInfo"].get("streaming") or movie["assetInfo"].get("download"), + ) + for movie in data["brand"]["episodes"] + ] + ) + else: + return Series( + [ + Episode( + id_=episode["programmeId"], + service=self.__class__, + title=data["brand"]["title"], + season=episode["seriesNumber"], + number=episode["episodeNumber"], + name=self._clean_episode(episode.get("originalTitle", "")), + language="en", + data=episode["assetInfo"].get("streaming") or episode["assetInfo"].get("download"), + ) + for episode in data["brand"]["episodes"] + if episode.get("assetInfo") + ] + ) + + def get_tracks(self, title: Union[Movie, Episode]) -> Tracks: + android_assets: tuple = self.android_playlist(title.id) + web_assets: tuple = self.web_playlist(title.id) + self.manifest, self.license_token, subtitle, data = self.sort_assets(title, android_assets, web_assets) + self.asset_id = int(title.data["assetId"]) + + tracks = DASH.from_url(self.manifest, self.session).to_tracks(title.language) + tracks.videos[0].data = data + + # manifest subtitles are sometimes empty even if they exist + # so we clear them and add the subtitles manually + tracks.subtitles.clear() + if subtitle is not None: + tracks.add( + Subtitle( + id_=hashlib.md5(subtitle.encode()).hexdigest()[0:6], + url=subtitle, + codec=Subtitle.Codec.from_mime(subtitle[-3:]), + language=title.language, + is_original_lang=True, + forced=False, + sdh=False, + ) + ) + else: + self.log.warning("- Subtitles are either missing or empty") + + for track in tracks.audio: + role = track.data["dash"]["representation"].find("Role") + if role is not None and role.get("value") in ["description", "alternative", "alternate"]: + track.descriptive = True + + return tracks + + def get_chapters(self, title: Union[Movie, Episode]) -> list[Chapter]: + track = title.tracks.videos[0] + + chapters = [ + Chapter( + name=f"Chapter {i + 1:02}", + timestamp=datetime.fromtimestamp((ms / 1000), tz=timezone.utc).strftime("%H:%M:%S.%f")[:-3], + ) + for i, ms in enumerate(x["breakOffset"] for x in track.data["adverts"]["breaks"]) + ] + + if track.data.get("endCredits", {}).get("squeezeIn"): + chapters.append( + Chapter( + name="Credits", + timestamp=datetime.fromtimestamp( + (track.data["endCredits"]["squeezeIn"] / 1000), tz=timezone.utc + ).strftime("%H:%M:%S.%f")[:-3], + ) + ) + + return chapters + + def get_widevine_service_certificate(self, **_: Any) -> str: + return WidevineCdm.common_privacy_cert + + def get_widevine_license(self, challenge: bytes, **_: Any) -> str: + payload = { + "message": base64.b64encode(challenge).decode("utf8"), + "token": self.license_token, + "request_id": self.asset_id, + "video": {"type": "ondemand", "url": self.manifest}, + } + + r = self.session.post(self.config["endpoints"]["license"], json=payload) + if not r.ok: + raise ConnectionError(f"License request failed: {r.json()['status']['type']}") + + return r.json()["license"] + + # Service-specific functions + + def sort_assets(self, title: Union[Movie, Episode], android_assets: tuple, web_assets: tuple) -> tuple: + android_heights = None + web_heights = None + + if android_assets is not None: + try: + a_manifest, a_token, a_subtitle, data = android_assets + android_tracks = DASH.from_url(a_manifest, self.session).to_tracks(title.language) + android_heights = sorted([int(track.height) for track in android_tracks.videos], reverse=True) + except Exception: + android_heights = None + + if web_assets is not None: + try: + b_manifest, b_token, b_subtitle, data = web_assets + session = self.session + session.headers.update(self.config["headers"]) + web_tracks = DASH.from_url(b_manifest, session).to_tracks(title.language) + web_heights = sorted([int(track.height) for track in web_tracks.videos], reverse=True) + except Exception: + web_heights = None + + if not android_heights and not web_heights: + self.log.error("Failed to request manifest data. If you're behind a VPN/proxy, you might be blocked") + sys.exit(1) + + if not android_heights or android_heights[0] < 1080: + lic_token = self.decrypt_token(b_token, client="WEB") + return b_manifest, lic_token, b_subtitle, data + else: + lic_token = self.decrypt_token(a_token, client="ANDROID") + return a_manifest, lic_token, a_subtitle, data + + def android_playlist(self, video_id: str) -> tuple: + url = self.config["android"]["vod"].format(video_id=video_id) + headers = {"authorization": self.authorization} + + r = self.session.get(url=url, headers=headers) + if not r.ok: + self.log.warning("Request for Android endpoint returned %s", r) + return None + + data = json.loads(r.content) + manifest = data["videoProfiles"][0]["streams"][0]["uri"] + token = data["videoProfiles"][0]["streams"][0]["token"] + subtitle = next( + (x["url"] for x in data["subtitlesAssets"] if x["url"].endswith(".vtt")), + None, + ) + + return manifest, token, subtitle, data + + def web_playlist(self, video_id: str) -> tuple: + url = self.config["web"]["vod"].format(programmeId=video_id) + r = self.session.get(url, headers=self.config["headers"]) + if not r.ok: + self.log.warning("Request for WEB endpoint returned %s", r) + return None + + data = json.loads(r.content) + + for item in data["videoProfiles"]: + if item["name"] == "dashwv-dyn-stream-1": + token = item["streams"][0]["token"] + manifest = item["streams"][0]["uri"] + + subtitle = next( + (x["url"] for x in data["subtitlesAssets"] if x["url"].endswith(".vtt")), + None, + ) + + return manifest, token, subtitle, data + + def decrypt_token(self, token: str, client: str) -> tuple: + if client == "ANDROID": + key = self.config["android"]["key"] + iv = self.config["android"]["iv"] + + if client == "WEB": + key = self.config["web"]["key"] + iv = self.config["web"]["iv"] + + if isinstance(token, str): + token = base64.b64decode(token) + cipher = AES.new( + key=base64.b64decode(key), + iv=base64.b64decode(iv), + mode=AES.MODE_CBC, + ) + data = unpad(cipher.decrypt(token), AES.block_size) + dec_token = data.decode().split("|")[1] + return dec_token.strip() + + def get_html(self, url: str) -> dict: + r = self.session.get(url=url, headers=self.config["headers"]) + r.raise_for_status() + + init_data = re.search( + "", + "".join(r.content.decode().replace("\u200c", "").replace("\r\n", "").replace("undefined", "null")), + ) + try: + data = json.loads(init_data.group(1)) + return data["initialData"] + except Exception: + self.log.error(f"Failed to get episode for {url}") + sys.exit(1) + + @staticmethod + def _clean_episode(text: str) -> str: + return re.sub(r"(?i)(?:series|season|episode|ep|s|e)\s*\d+\s*(?:episode|ep|e)?\s*\d*\s*:?\s*", "", text) diff --git a/reset/packages/envied/src/envied/services/ALL4/config.yaml b/reset/packages/envied/src/envied/services/ALL4/config.yaml new file mode 100644 index 0000000..1468e01 --- /dev/null +++ b/reset/packages/envied/src/envied/services/ALL4/config.yaml @@ -0,0 +1,27 @@ +headers: + Accept-Language: en-US,en;q=0.8 + User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36 + +endpoints: + login: https://api.channel4.com/online/v2/auth/token + title: https://api.channel4.com/online/v1/views/content-hubs/{title}.json + license: https://c4.eme.lp.aws.redbeemedia.com/wvlicenceproxy-service/widevine/acquire + search: https://all4nav.channel4.com/v1/api/search + +android: + key: QVlESUQ4U0RGQlA0TThESA==" + iv: MURDRDAzODNES0RGU0w4Mg==" + auth: MzZVVUN0OThWTVF2QkFnUTI3QXU4ekdIbDMxTjlMUTE6Sllzd3lIdkdlNjJWbGlrVw== + vod: https://api.channel4.com/online/v1/vod/stream/{video_id}?client=android-mod + +web: + key: bjljTGllWWtxd3pOQ3F2aQ== + iv: b2R6Y1UzV2RVaVhMdWNWZA== + vod: https://www.channel4.com/vod/stream/{programmeId} + +device: + platform_name: android + device_type: mobile + device_name: "Sony C6903 (C6903)" + app_version: "android_app:9.4.2" + optimizely_datafile: "2908" diff --git a/reset/packages/envied/src/envied/services/AMZN/CHANGELOG_AMZN.docx b/reset/packages/envied/src/envied/services/AMZN/CHANGELOG_AMZN.docx new file mode 100644 index 0000000..54b1328 Binary files /dev/null and b/reset/packages/envied/src/envied/services/AMZN/CHANGELOG_AMZN.docx differ diff --git a/reset/packages/envied/src/envied/services/AMZN/__init__.py b/reset/packages/envied/src/envied/services/AMZN/__init__.py new file mode 100644 index 0000000..65dadc4 --- /dev/null +++ b/reset/packages/envied/src/envied/services/AMZN/__init__.py @@ -0,0 +1,1243 @@ +import base64 +import hashlib +import json +import os +import re +import secrets +import string +import time +import random +from collections import defaultdict +from http.cookiejar import CookieJar +from pathlib import Path +from typing import Any, Optional, Literal, Union +from urllib.parse import quote, urlencode + +import click +import jsonpickle +import requests +from click.core import ParameterSource +from langcodes import Language +from tldextract import tldextract + +from envied.core.cacher import Cacher +from envied.core.credential import Credential +from envied.core.manifests import DASH, ISM +from envied.core.service import Service +from envied.core.titles import Episode, Movie, Movies, Series, Title_T, Titles_T +from envied.core.tracks import Chapter, Chapters, Subtitle, Tracks, Track, Video +from envied.core.utilities import is_close_match +from envied.core.utils.collections import as_list + + + + +def _build_ordered_lang_map_from_mpd(mpd_text: str) -> dict: + import xml.etree.ElementTree as ET + import re as _re + ns_strip = _re.compile(r'\{[^}]*\}') + rid_lang_re = _re.compile(r'^audio_([a-zA-Z]{2,3}-[a-zA-Z0-9]{2,5})') + result: dict = {} + try: + root = ET.fromstring(mpd_text) + for elem in root.iter(): + if ns_strip.sub('', elem.tag) != 'AdaptationSet': + continue + content_type = elem.get('contentType', '') or elem.get('mimeType', '') + if 'audio' not in content_type.lower(): + if not any('audio' in (r.get('mimeType', '')).lower() for r in elem): + continue + base_lang = elem.get('lang') or elem.get('language') or '' + if not base_lang: + continue + for r in elem: + if ns_strip.sub('', r.tag) != 'Representation': + continue + rid = r.get('id') or '' + m = rid_lang_re.match(rid) + precise = m.group(1) if m else base_lang + result.setdefault(base_lang, []).append(precise) + except Exception: + pass + return result + + +def _apply_ordered_lang_map(audio_tracks, lang_map: dict) -> None: + from langcodes import Language as _Lang + counters: dict = {} + for track in audio_tracks: + base = str(track.language) + if base not in lang_map: + continue + ordered = lang_map[base] + if not any('-' in p for p in ordered): + continue + idx = counters.get(base, 0) + if idx < len(ordered): + track.language = _Lang.get(ordered[idx]) + counters[base] = idx + 1 + +class AMZN(Service): + """ + Service code for Amazon VOD (https://amazon.com) and Amazon Prime Video (https://primevideo.com). + + \b + Authorization: Cookies + Security: UHD@L1 FHD@L3(ChromeCDM) SD@L3, Maintains their own license server like Netflix, be cautious. + + \b + Region is chosen automatically based on domain extension found in cookies. + Prime Video specific code will be run if the ASIN is detected to be a prime video variant. + """ + + ALIASES = ["AMZN", "amazon", "prime"] + TITLE_RE = [ + r"^(?:https?://(?:www\.)?(?Pamazon\.(?Pcom|co\.uk|de|co\.jp)|primevideo\.com)(?:/.+)?/)?(?P[A-Z0-9]{10,}|amzn1\.dv\.gti\.[a-f0-9-]+)", + r"^(?:https?://(?:www\.)?(?Pamazon\.(?Pcom|co\.uk|de|co\.jp)|primevideo\.com)(?:/[^?]*)?(?:\?gti=)?)(?P[A-Z0-9]{10,}|amzn1\.dv\.gti\.[a-f0-9-]+)" + ] + + REGION_TLD_MAP = { + "au": "com.au", + "br": "com.br", + "jp": "co.jp", + "mx": "com.mx", + "tr": "com.tr", + "gb": "co.uk", + "us": "com", + } + VIDEO_RANGE_MAP = { + "SDR": "None", + "HDR10": "Hdr10", + "DV": "DolbyVision", + } + + @staticmethod + @click.command(name="AMZN", short_help="https://amazon.com, https://primevideo.com", help=__doc__) + @click.argument("title", type=str, required=False) + @click.option("-b", "--bitrate", default="CBR", + type=click.Choice(["CVBR", "CBR", "CVBR+CBR"], case_sensitive=False), + help="Video Bitrate Mode to download in. CVBR=Constrained Variable Bitrate, CBR=Constant Bitrate.") + @click.option("-p", "--player", default="html5", + type=click.Choice(["html5", "xp"], case_sensitive=False), + help="Video playerType to download in. html5, xp.") + @click.option("-c", "--cdn", default=None, type=str, + help="CDN to download from, defaults to the CDN with the highest weight set by Amazon.") + @click.option("-vq", "--vquality", default="HD", + type=click.Choice(["SD", "HD", "UHD"], case_sensitive=False), + help="Manifest quality to request.") + @click.option("-s", "--single", is_flag=True, default=False, + help="Force single episode/season instead of getting series ASIN.") + @click.option("-am", "--amanifest", default="CVBR", + type=click.Choice(["CVBR", "CBR", "H265"], case_sensitive=False), + help="Manifest to use for audio. Defaults to H265 if the video manifest is missing 640k audio.") + @click.option("-aq", "--aquality", default="SD", + type=click.Choice(["SD", "HD", "UHD"], case_sensitive=False), + help="Manifest quality to request for audio. Defaults to the same as --quality.") + @click.option("-aa", "--atmos", is_flag=True, default=False, + help="Prefer Atmos audio if available.") + @click.option("-drm", "--drm-system", type=click.Choice(["widevine", "playready"], case_sensitive=False), + default="playready", + help="which drm system to use") + @click.pass_context + def cli(ctx, **kwargs): + return AMZN(ctx, **kwargs) + + def __init__(self, ctx, title, bitrate: str, player: str, cdn: str, vquality: str, single: bool, + amanifest: str, aquality: str, drm_system: str, atmos: bool): + super().__init__(ctx) + self.parse_title(ctx, title) + self.bitrate = bitrate + self.player = player + self.bitrate_source = ctx.get_parameter_source("bitrate") + self.cdn = cdn + self.vquality = vquality + self.vquality_source = ctx.get_parameter_source("vquality") + self.single = single + self.amanifest = amanifest + self.aquality = aquality + self.atmos = atmos + self.drm_system = drm_system + + assert ctx.parent is not None + + # envied.params + self.chapters_only = ctx.parent.params.get("chapters_only") + self.quality = ctx.parent.params.get("quality") or 1080 + + vcodec = ctx.parent.params.get("vcodec") + range_ = ctx.parent.params.get("range_") + + self.range = range_[0].name if range_ else "SDR" + # Mapping envied.video codec enum to string + self.vcodec = "H265" if vcodec and "HEVC" in str(vcodec) else "H264" + + self.cdm = ctx.obj.cdm + self.profile = ctx.obj.profile + self.playready = self.drm_system == "playready" + + self.region: dict[str, str] = {} + self.endpoints: dict[str, str] = {} + self.device: dict[str, str] = {} + + self.pv = False + self.event = False + self.device_token = None + self.device_id = None + self.customer_id = None + self.client_id = "f22dbddb-ef2c-48c5-8876-bed0d47594fd" # browser client id + self.playbackEnvelope = None + + # Logic from Vinetrimmer regarding quality overrides + if self.vquality_source != ParameterSource.COMMANDLINE: + # Check if quality is list (envied. or int + q_check = self.quality[0] if isinstance(self.quality, list) else self.quality + + if 0 < q_check <= 576 and self.range == "SDR": + self.log.info(" + Setting manifest quality to SD") + self.vquality = "SD" + + if q_check > 1080: + self.log.info(" + Setting manifest quality to UHD to be able to get 2160p video track") + self.vquality = "UHD" + + self.vquality = self.vquality or "HD" + + if self.vquality == "UHD": + self.vcodec = "H265" + + if self.bitrate_source != ParameterSource.COMMANDLINE: + if self.vcodec == "H265" and self.range == "SDR" and self.bitrate != "CVBR+CBR": + self.bitrate = "CVBR+CBR" + self.log.info(" + Changed bitrate mode to CVBR+CBR to be able to get H.265 SDR video track") + + if self.vquality == "UHD" and self.range != "SDR" and self.bitrate != "CBR": + self.bitrate = "CBR" + self.log.info(f" + Changed bitrate mode to CBR to be able to get highest quality UHD {self.range} video track") + + self.orig_bitrate = self.bitrate + + def authenticate(self, cookies: Optional[CookieJar] = None, credential: Optional[Credential] = None) -> None: + super().authenticate(cookies, credential) + if not cookies: + raise EnvironmentError("Service requires Cookies for Authentication.") + + self.session.cookies.update(cookies) + self.configure() + + def configure(self) -> None: + if len(self.title) > 10: + self.pv = True + self.pv = True # always use primevideo endpoints + + self.log.info("Getting Account Region") + self.region = self.get_region() + if not self.region: + self.log.error(" - Failed to get Amazon Account region"); raise SystemExit(1) + + self.log.info(f" + Region: {self.region['code']}") + + # endpoints must be prepared AFTER region data is retrieved + self.endpoints = self.prepare_endpoints(self.config["endpoints"], self.region) + + self.session.headers.update({ + "Origin": f"https://{self.region['base']}" + }) + + _profile = self.profile or "default" + self.device = (self.config.get("device") or {}).get(_profile, {}) + + # Logic to decide if we need a specific device registration + need_device = False + if (isinstance(self.quality, list) and self.quality[0] > 1080) or self.vquality == "UHD" or self.range != "SDR": + need_device = True + + if self.device: + if need_device and self.vcodec == "H265": + self.log.info(f"Using device to get UHD manifests") + else: + self.log.info(f"Using configured device for profile: {_profile}") + self.register_device() + else: + # Falling back to browser-based device ID + self.log.warning( + "No Device information was provided for %s, using browser device...", + self.profile + ) + self.device_id = hashlib.sha224( + ("CustomerID" + self.session.headers["User-Agent"]).encode("utf-8") + ).hexdigest() + self.device = {"device_type": self.config["device_types"]["browser"]} + + def get_titles(self) -> Titles_T: + res = self.session.get( + url=self.endpoints["details"], + params={ + "titleID": self.title, + "isElcano": "1", + "sections": ["Atf", "Btf"] + }, + headers={"Accept": "application/json"} + ) + + if not res.ok: + self.log.error(f"Unable to get title: {res.text} [{res.status_code}]"); raise SystemExit(1) + + data = res.json()["widgets"] + product_details = data.get("productDetails", {}).get("detail") + + if not product_details: + error = res.json()["degradations"][0] + self.log.error(f"Unable to get title: {error['message']} [{error['code']}]"); raise SystemExit(1) + + if data["pageContext"]["subPageType"] == "Event": + self.event = True + + if data["pageContext"]["subPageType"] in ["Movie", "Event"]: + card = data["productDetails"]["detail"] + return Movies([Movie( + id_=card["catalogId"], + name=product_details["title"], + year=card.get("releaseYear", ""), + service=self.__class__, + data=card + )]) + else: + # TV Show logic with pagination support from Vinetrimmer + episodes_list = [] + seasons = [x.get("titleID") for x in data["seasonSelector"]] + + # If single flag is set, logic to filter seasons happens in main loop, + # but for envied.structure we usually return the whole series or let user filter. + # Implementing the Vinetrimmer pagination logic: + + for season in seasons: + # If single mode and logic requires skipping, we should handle it here + # But strict Vinetrimmer logic handled title switching. + # For envied. we iterate all found seasons. + + res = self.session.get( + url=self.endpoints["details"], + params={"titleID": season, "isElcano": "1", "sections": "Btf"}, + headers={"Accept": "application/json"}, + ).json()["widgets"] + + try: + episode_data_list = res["episodeList"]["episodes"] + except KeyError: + continue + + product_details_season = res["productDetails"]["detail"] + # exit(product_details_season) + + # Process initial batch + for episode in episode_data_list: + details = episode["detail"] + episodes_list.append(Episode( + id_=details["catalogId"], + title=product_details["title"], + name=details["title"], + season=product_details_season["seasonNumber"], + number=episode["self"]["sequenceNumber"], + service=self.__class__, + data=episode + )) + + # Handle Pagination + pagination_data = res.get('episodeList', {}).get('actions', {}).get('pagination', []) + token = next((quote(item.get('token')) for item in pagination_data if item.get('tokenType') == 'NextPage'), None) + + while token: + res_page = self.session.get( + url=self.endpoints["getDetailWidgets"], + params={ + "titleID": self.title, + "isTvodOnRow": "1", + "widgets": f'[{{"widgetType":"EpisodeList","widgetToken":"{token}"}}]' + }, + headers={"Accept": "application/json"} + ).json() + + episodeListWidget = res_page['widgets'].get('episodeList', {}) + for item in episodeListWidget.get('episodes', []): + ep_num = int(item.get('self', {}).get('sequenceNumber', 0)) + episodes_list.append(Episode( + id_=item["detail"]["catalogId"], + name=item["detail"]["title"], + season=product_details_season["seasonNumber"], + number=ep_num, + service=self.__class__, + data=item + )) + + pagination_data = res_page['widgets'].get('episodeList', {}).get('actions', {}).get('pagination', []) + token = next((quote(item.get('token')) for item in pagination_data if item.get('tokenType') == 'NextPage'), None) + + return Series(episodes_list) + + def get_tracks(self, title: Title_T) -> Tracks: + if self.chapters_only: + return Tracks([]) + + # Main Video Manifest + # When DV is requested, declare HEVC_DOLBY_VISION codec so Amazon returns DV streams + # If the server rejects the DV request, fall back to HDR10 automatically + effective_vcodec = "HEVC_DOLBY_VISION" if self.range == "DV" and self.vcodec == "H265" else self.vcodec + effective_range = self.range + manifest = self.get_manifest( + title, + video_codec=effective_vcodec, + bitrate_mode=self.bitrate, + quality=self.vquality, + hdr=effective_range, + ignore_errors=self.range == "DV" + ) + if self.range == "DV" and not manifest.get("vodPlaybackUrls"): + self.log.warning(" - Dolby Vision request rejected by server, retrying with HDR10...") + effective_vcodec = self.vcodec + effective_range = "HDR10" + manifest = self.get_manifest( + title, + video_codec=effective_vcodec, + bitrate_mode=self.bitrate, + quality=self.vquality, + hdr=effective_range, + ignore_errors=False + ) + + if "rightsException" in manifest.get("returnedTitleRendition", {}).get("selectedEntitlement", {}): + self.log.error(" - The profile used does not have the rights to this title.") + return Tracks([]) + + chosen_manifest = self.choose_manifest(manifest, self.cdn) + if not chosen_manifest: + self.log.error(f"No manifests available"); raise SystemExit(1) + + manifest_url = self.clean_mpd_url(chosen_manifest["url"], False) + if self.event: + devicetype = self.device.get("device_type") + manifest_url = chosen_manifest["url"] + manifest_url = f"{manifest_url}?amznDtid={devicetype}&encoding=segmentBase" + + self.log.info(" + Downloading Manifest") + + _mpd_raw = self.session.get(manifest_url).text + _lang_order_map = _build_ordered_lang_map_from_mpd(_mpd_raw) + self.log.info(f" + MPD language map: {sum(len(v) for v in _lang_order_map.values())} representations indexed") + + streamingProtocol = manifest["vodPlaybackUrls"]["result"]["playbackUrls"]["urlMetadata"]["streamingProtocol"] + + # Base Tracks object + tracks = Tracks() + + if streamingProtocol == "DASH": + tracks = Tracks([ + x for x in iter(DASH.from_url(url=manifest_url, session=self.session).to_tracks(language="en")) + ]) + elif streamingProtocol == "SmoothStreaming": + _ism_tracks = Tracks() + for _t in iter(ISM.from_url(url=manifest_url, session=self.session).to_tracks(language="en")): + _ism_tracks.add(_t, warn_only=True) + tracks = _ism_tracks + else: + self.log.error(f"Unsupported manifest type: {streamingProtocol}"); raise SystemExit(1) + + if _lang_order_map: + _apply_ordered_lang_map(tracks.audio, _lang_order_map) + + # Logic for separate audio (Higher Quality / Different Codec) + need_separate_audio = ((self.aquality or self.vquality) != self.vquality + or self.amanifest == "CVBR" and (self.vcodec, self.bitrate) != ("H264", "CVBR") + or self.amanifest == "CBR" and (self.vcodec, self.bitrate) != ("H264", "CBR") + or self.amanifest == "H265" and self.vcodec != "H265" + or self.amanifest != "H265" and self.vcodec == "H265") + + if not need_separate_audio: + # Check for low bitrate audio + audios = defaultdict(list) + for audio in tracks.audio: + audios[audio.language].append(audio) + for lang in audios: + if not any((x.bitrate or 0) >= 640000 for x in audios[lang]): + need_separate_audio = True + break + + if need_separate_audio and not self.atmos: + manifest_type = self.amanifest + self.log.info(f"Getting audio from {manifest_type} manifest for potential higher bitrate or better codec") + + audio_manifest = self.get_manifest( + title=title, + video_codec="H264", + bitrate_mode="CVBR", + quality="HD", + hdr=None, + ignore_errors=True + ) + + if not audio_manifest: + self.log.warning(f" - Unable to get {manifest_type} audio manifests, skipping") + elif not (chosen_audio_manifest := self.choose_manifest(audio_manifest, self.cdn)): + self.log.warning(f" - No {manifest_type} audio manifests available, skipping") + else: + audio_mpd_url = self.clean_mpd_url(chosen_audio_manifest["url"], optimise=False) + if self.event: + devicetype = self.device.get("device_type") + audio_mpd_url = chosen_audio_manifest["url"] + audio_mpd_url = f"{audio_mpd_url}?amznDtid={devicetype}&encoding=segmentBase" + + self.log.info(" + Downloading Audio Manifest") + try: + audio_protocol = audio_manifest["vodPlaybackUrls"]["result"]["playbackUrls"]["urlMetadata"]["streamingProtocol"] + if audio_protocol == "DASH": + _a_raw = self.session.get(audio_mpd_url).text + _a_lang_order = _build_ordered_lang_map_from_mpd(_a_raw) + self.log.info(f" + Audio MPD language map: {sum(len(v) for v in _a_lang_order.values())} entries") + audio_tracks = DASH.from_url(url=audio_mpd_url, session=self.session).to_tracks(language="en") + if _a_lang_order: + _apply_ordered_lang_map(audio_tracks.audio, _a_lang_order) + elif audio_protocol == "SmoothStreaming": + audio_tracks = ISM.from_url(url=audio_mpd_url, session=self.session).to_tracks(language="en") + else: + audio_tracks = Tracks([]) + + tracks.add(audio_tracks.audio, warn_only=True) + except Exception as e: + self.log.warning(f" - Failed to parse audio manifest: {e}") + + # Logic for UHD/Atmos Audio + need_uhd_audio = self.atmos + if not self.amanifest and ((self.aquality == "UHD" and self.vquality != "UHD") or not self.aquality): + # Simple check if current tracks lack high bitrate + if all((x.bitrate or 0) < 640000 for x in tracks.audio): + need_uhd_audio = True + + if need_uhd_audio and (self.config.get("device") or {}).get(self.profile, None): + self.log.info("Getting audio from UHD manifest for potential higher bitrate or better codec") + temp_device = self.device + temp_token = self.device_token + temp_id = self.device_id + + # Switch to device if on browser/playready + if self.playready or self.cdm.device.type == "CHROME": + self.register_device() # Switch to registered device context + + try: + uhd_audio_manifest = self.get_manifest( + title=title, + video_codec="H265", + bitrate_mode="CVBR+CBR", + quality="UHD", + hdr="DV", # Needed for 576kbps Atmos + ignore_errors=True + ) + except: + uhd_audio_manifest = None + + # Restore context + self.device = temp_device + self.device_token = temp_token + self.device_id = temp_id + + if uhd_audio_manifest and (chosen_uhd := self.choose_manifest(uhd_audio_manifest, self.cdn)): + uhd_url = self.clean_mpd_url(chosen_uhd["url"], optimise=False) + self.log.info(" + Downloading UHD Manifest") + try: + uhd_prot = uhd_audio_manifest["vodPlaybackUrls"]["result"]["playbackUrls"]["urlMetadata"]["streamingProtocol"] + if uhd_prot == "DASH": + uhd_tracks = DASH.from_url(url=uhd_url, session=self.session).to_tracks(language="en") + elif uhd_prot == "SmoothStreaming": + uhd_tracks = ISM.from_url(url=uhd_url, session=self.session).to_tracks(language="en") + else: + uhd_tracks = Tracks([]) + + # If atmos found, replace + if any(x for x in uhd_tracks.audio if "atmos" in (x.codec or "").lower() or x.channels >= 6): + tracks.add(uhd_tracks.audio, warn_only=True) + except Exception: + pass + + # Post-process video tracks (HDR info) + actual_range = manifest["vodPlaybackUrls"]["result"]["playbackUrls"]["urlMetadata"]["dynamicRange"] + for video in tracks.videos: + video.hdr10 = actual_range == "Hdr10" + video.dv = actual_range == "DolbyVision" + + if self.range == "DV" and actual_range != "DolbyVision": + friendly = {"Hdr10": "HDR10", "None": "SDR"}.get(actual_range, actual_range) + self.log.warning(f" - Dolby Vision not available for this title/region. Server returned: {friendly}") + + # Subtitles + for sub in manifest.get("timedTextUrls", {}).get("result", {}).get("subtitleUrls", []) + manifest.get("timedTextUrls", {}).get("result", {}).get("forcedNarrativeUrls", []): + url = sub["url"] + + url_path = url.split("?")[0] + url_ext = os.path.splitext(url_path)[1].lstrip(".").lower() + codec_map = { + "ttml": Subtitle.Codec.TimedTextMarkupLang, + "dfxp": Subtitle.Codec.TimedTextMarkupLang, + "vtt": Subtitle.Codec.WebVTT, + "srt": Subtitle.Codec.SubRip, + } + detected_codec = codec_map.get(url_ext, Subtitle.Codec.TimedTextMarkupLang) + + sub_obj = Subtitle( + id_=f"{sub['trackGroupId']}_{sub['languageCode']}_{sub['type']}_{sub['subtype']}", + url=url, + codec=detected_codec, + language=sub["languageCode"], + forced="ForcedNarrative" in sub["type"], + sdh=sub["type"].lower() == "sdh" + ) + + tracks.add(sub_obj, warn_only=True) + + return tracks + + def get_chapters(self, title: Title_T) -> Chapters: + manifest = self.get_manifest( + title, + video_codec=self.vcodec, + bitrate_mode=self.bitrate, + quality=self.vquality, + hdr=self.range + ) + + if "vodXrayMetadata" in manifest: + if "error" in manifest["vodXrayMetadata"]: + return [] + + xray_params = { + "pageId": "fullScreen", + "pageType": "xray", + "serviceToken": json.dumps({ + "consumptionType": "Streaming", + "deviceClass": "normal", + "playbackMode": "playback", + "vcid": json.loads(manifest["vodXrayMetadata"]["result"]["parameters"]["serviceToken"])["vcid"] + }) + } + else: + return [] + + xray_params.update({ + "deviceID": self.device_id, + "deviceTypeID": self.config["device_types"]["browser"], + "marketplaceID": self.region["marketplace_id"], + "gascEnabled": str(self.pv).lower(), + "decorationScheme": "none", + "version": "inception-v2", + "uxLocale": "en-US", + "featureScheme": "XRAY_WEB_2020_V1" + }) + + try: + xray = self.session.get( + url=self.endpoints["xray"], + params=xray_params + ).json().get("page") + except: + return [] + + if not xray: + return [] + + try: + widgets = xray["sections"]["center"]["widgets"]["widgetList"] + scenes = next((x for x in widgets if x["tabType"] == "scenesTab"), None) + if not scenes: + return [] + scenes = scenes["widgets"]["widgetList"][0]["items"]["itemList"] + except (KeyError, IndexError): + return [] + + chapters = [] + for scene in scenes: + chapter_title = scene["textMap"]["PRIMARY"] + match = re.search(r"(\d+\. |)(.+)", chapter_title) + if match: + chapter_title = match.group(2) + + timecode = scene["textMap"]["TERTIARY"].replace("Starts at ", "") + chapters.append(Chapter(name=chapter_title, timestamp=timecode)) + + return chapters + + def playbackEnvelope_data(self, title): + try: + res = self.session.get( + url=self.endpoints["metadata"], + params={ + 'metadataToEnrich': json.dumps({"placement": "HOVER", "playback": "true", "preroll": "true", "trailer": "true", "watchlist": "true"}), + 'titleIDsToEnrich': json.dumps([title.id]) + }, + headers={'x-requested-with': 'XMLHttpRequest'} + ) + + if res.status_code == 200: + try: + data = res.json() + if ( + title.id in data["enrichments"] + and "focusMessage" in data["enrichments"][title.id].get("entitlementCues", {}) + and data["enrichments"][title.id]["entitlementCues"]["focusMessage"].get("message") == "Watch with a free Prime trial" + ): + self.log.error("Invalid Cookies"); raise SystemExit(1) + + try: + playbackEnvelope = data["enrichments"][title.id]["playbackActions"][0]["playbackExperienceMetadata"]["playbackEnvelope"] + except: + playbackEnvelope = data['enrichments'][title.id]['prerollsEnvelope']['playbackEnvelope'] + return playbackEnvelope + except Exception as e: + self.log.error(f"Unable to get playbackEnvelope: {e}"); raise SystemExit(1) + else: + self.log.error(f"Unable to get playbackEnvelope: {res.text}"); raise SystemExit(1) + except Exception: + self.log.error("Unable to get playbackEnvelope"); raise SystemExit(1) + + def get_manifest(self, title, video_codec, bitrate_mode, quality, hdr, ignore_errors=False): + self.playbackEnvelope = self.playbackEnvelope_data(title) + + # Construct Payload (Vinetrimmer Style) + data_dict = { + "globalParameters": { + "deviceCapabilityFamily": "WebPlayer" if not self.device_token else "AndroidPlayer", + "playbackEnvelope": self.playbackEnvelope, + "capabilityDiscriminators": { + "operatingSystem": {"name": "Windows", "version": "10.0"}, + "middleware": {"name": "EdgeNext", "version": "136.0.0.0"}, + "nativeApplication": {"name": "EdgeNext", "version": "136.0.0.0"}, + "hfrControlMode": "Legacy", + "displayResolution": {"height": 2304, "width": 4096} + } if not self.device_token else { + "discriminators": {"software": {}, "version": 1} + } + }, + "auditPingsRequest": { + **({"device": {"category": "Tv", "platform": "Android"}} if self.device_token else {}) + }, + "playbackDataRequest": {}, + "timedTextUrlsRequest": { + "supportedTimedTextFormats": ["TTMLv2", "DFXP"] + }, + "trickplayUrlsRequest": {}, + "transitionTimecodesRequest": {}, + "vodPlaybackUrlsRequest": { + "device": { + "hdcpLevel": "2.2" if quality == "UHD" else "1.4", + "maxVideoResolution": ("1080p" if quality == "HD" else "480p" if quality == "SD" else "2160p"), + "supportedStreamingTechnologies": ["DASH"], + "streamingTechnologies": { + "DASH": { + "bitrateAdaptations": ["CVBR", "CBR"] if bitrate_mode in ("CVBR+CBR", "CVBR,CBR") else [bitrate_mode], + "codecs": [video_codec], + "drmKeyScheme": "SingleKey" if self.playready else "DualKey", + "drmType": "PlayReady" if self.playready else "Widevine", + "dynamicRangeFormats": self.VIDEO_RANGE_MAP.get(hdr, "None"), + "fragmentRepresentations": ["ByteOffsetRange"], + "frameRates": ["Standard"], + "stitchType": "MultiPeriod", + "segmentInfoType": "Base", + "timedTextRepresentations": ["NotInManifestNorStream", "SeparateStreamInManifest"], + "trickplayRepresentations": ["NotInManifestNorStream"], + "variableAspectRatio": "supported" + } + }, + "displayWidth": 4096, + "displayHeight": 2304 + }, + "ads": { + "sitePageUrl": "", + "gdpr": {"enabled": "false", "consentMap": {}} + }, + "playbackCustomizations": {}, + "playbackSettingsRequest": { + "firmware": "UNKNOWN", + "playerType": self.player, + "responseFormatVersion": "1.0.0", + "titleId": title.id + } + } if not self.device_token else { + # Android/Device Payload + "ads": {}, + "device": { + "displayBasedVending": "supported", + "displayHeight": 2304, + "displayWidth": 4096, + "streamingTechnologies": { + "DASH": { + "fragmentRepresentations": ["ByteOffsetRange"], + "manifestThinningToSupportedResolution": "Forbidden", + "segmentInfoType": "List", + "stitchType": "MultiPeriod", + "timedTextRepresentations": ["BurnedIn", "NotInManifestNorStream", "SeparateStreamInManifest"], + "trickplayRepresentations": ["NotInManifestNorStream"], + "variableAspectRatio": "supported", + "vastTimelineType": "Absolute", + "bitrateAdaptations": ["CVBR", "CBR"] if bitrate_mode in ("CVBR+CBR", "CVBR,CBR") else [bitrate_mode], + "codecs": [video_codec], + "drmKeyScheme": "SingleKey", + "drmStrength": "L40", + "drmType": "PlayReady" if self.playready else "Widevine", + "dynamicRangeFormats": [self.VIDEO_RANGE_MAP.get(hdr, "None")], + "frameRates": ["Standard"] + }, + "SmoothStreaming": { + "fragmentRepresentations": ["ByteOffsetRange"], + "manifestThinningToSupportedResolution": "Forbidden", + "segmentInfoType": "List", + "stitchType": "MultiPeriod", + "timedTextRepresentations": ["BurnedIn", "NotInManifestNorStream", "SeparateStreamInManifest"], + "trickplayRepresentations": ["NotInManifestNorStream"], + "variableAspectRatio": "supported", + "vastTimelineType": "Absolute", + "bitrateAdaptations": ["CVBR", "CBR"] if bitrate_mode in ("CVBR+CBR", "CVBR,CBR") else [bitrate_mode], + "codecs": [video_codec], + "drmKeyScheme": "SingleKey", + "drmStrength": "L40", + "drmType": "PlayReady", + "dynamicRangeFormats": [self.VIDEO_RANGE_MAP.get(hdr, "None")], + "frameRates": ["Standard"] + } + }, + "acceptedCreativeApis": [], + "category": "Tv", + "hdcpLevel": "2.2", + "maxVideoResolution": "2160p", + "platform": "Android", + "supportedStreamingTechnologies": ["DASH", "SmoothStreaming"] + }, + "playbackCustomizations": {}, + "playbackSettingsRequest": { + "firmware": "UNKNOWN", + "playerType": self.player, + "responseFormatVersion": "1.0.0", + "titleId": title.id + } + }, + "vodXrayMetadataRequest": { + "xrayDeviceClass": "normal", + "xrayPlaybackMode": "playback", + "xrayToken": "XRAY_WEB_2023_V2" + } + } + + json_data = json.dumps(data_dict) + + res = self.session.post( + url=self.endpoints["playback"], + params={ + 'deviceID': self.device_id, + 'deviceTypeID': self.device["device_type"], + 'gascEnabled': str(self.pv).lower(), + 'marketplaceID': self.region["marketplace_id"], + 'uxLocale': 'en_EN', + 'firmware': 1, + 'titleId': title.id, + 'nerid': self.generate_nerid(), + }, + data=json_data, + headers={ + "Authorization": f"Bearer {self.device_token}" if self.device_token else None, + }, + ) + + try: + manifest = res.json() + except json.JSONDecodeError: + if ignore_errors: return {} + self.log.error(f" - Amazon reported an error when obtaining the Playback Manifest\n{res.text}"); raise SystemExit(1) + + if "error" in manifest.get("vodPlaybackUrls", {}): + if ignore_errors: return {} + message = manifest["vodPlaybackUrls"]["error"]["message"] + self.log.error(f" - Amazon reported an error when obtaining the Playback Manifest: {message}"); raise SystemExit(1) + + return manifest + + def get_widevine_service_certificate(self, **_: Any) -> str: + return self.config["certificate"] + + def get_widevine_license(self, challenge: bytes, title: Title_T, track: Tracks, **_) -> str: + return self._get_license(challenge, title, track, widevine=True) + + def get_playready_license(self, challenge: bytes, title: Title_T, track: Tracks, **_) -> str: + return self._get_license(challenge, title, track, widevine=False) + + def _get_license(self, challenge: bytes, title: Title_T, track: Tracks, widevine: bool): + # Determine SessionHandoffToken from tracks extra data or similar + # In Vinetrimmer this is passed in track.extra, but in envied.tracks are standard objects. + # We need to rely on the fact that envied.usually doesn't need the sessionHandoffToken for the license request + # UNLESS Amazon specifically enforces it for the payload. + # Vinetrimmer payload: + + # NOTE: envied.Track objects might not carry the sessionHandoffToken unless we subclassed DASH parser. + # For migration purposes, we will attempt without it or fetch from manifest if strictly needed. + # Vinetrimmer logic extracts it from manifest during track parsing. + # Ideally we would pass it via the track object. + + session_handoff_token = None + # Try to find it in track extras if stored (envied.doesn't natively store custom extras easily) + + challenge_bytes = challenge if isinstance(challenge, bytes) else challenge.encode("utf-8") + encoded_challenge = base64.b64encode(challenge_bytes).decode("utf-8") + + data_lic = { + "includeHdcpTestKeyInLicense": "true", + "licenseChallenge": encoded_challenge, + "playbackEnvelope": self.playbackEnvelope, + } + + endpoint = self.endpoints["licence_pr"] if not widevine else self.endpoints["licence"] + + res = self.session.post( + url=endpoint, + params={ + 'deviceID': self.device_id, + 'deviceTypeID': self.device["device_type"], + 'gascEnabled': str(self.pv).lower(), + 'marketplaceID': self.region["marketplace_id"], + 'uxLocale': 'en_EN', + 'firmware': 1, + 'titleId': title.id, + 'nerid': self.generate_nerid(), + }, + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + "Authorization": f"Bearer {self.device_token}" + }, + json=data_lic + ).json() + + req_preview = {k: (v[:80] if isinstance(v, str) else v) for k, v in data_lic.items()} + if "errorsByResource" in res: + error = res["errorsByResource"] + if "errorCode" in error: + code = error["errorCode"] + elif "type" in error: + code = error["type"] + else: + code = "Unknown" + + if code == "PRS.NoRights.AnonymizerIP": + self.log.error(" - Amazon detected a Proxy/VPN and refused to return a license!"); raise SystemExit(1) + + self.log.error(f" - Amazon reported an error during the License request: [{code}]"); raise SystemExit(1) + + if "error" in res: + self.log.error(f" - License Error: {res['error']['message']}"); raise SystemExit(1) + + if widevine: + return res["widevineLicense"]["license"] + else: + return res["playReadyLicense"]["license"] + + # --- Helpers --- + + def choose_manifest(self, manifest: dict, cdn=None): + if not manifest or "vodPlaybackUrls" not in manifest: + return {} + + url_sets = manifest["vodPlaybackUrls"]["result"]["playbackUrls"].get("urlSets", []) + if not url_sets: return {} + + if cdn: + cdn = cdn.lower() + return next((x for x in url_sets if x["cdn"].lower() == cdn), {}) + + return random.choice(url_sets) + + @staticmethod + def generate_nerid(length=24): + chars = string.ascii_letters + string.digits + return ''.join(secrets.choice(chars) for _ in range(length)) + + @staticmethod + def clean_mpd_url(mpd_url, optimise=False): + if optimise: + return mpd_url.replace("~", "") + "?encoding=segmentBase" + if match := re.match(r"(https?://.*/)d.?/.*~/(.*)", mpd_url): + mpd_url = "".join(match.groups()) + else: + try: + mpd_url = "".join( + re.split(r"(?i)(/)", mpd_url)[:5] + re.split(r"(?i)(/)", mpd_url)[9:] + ) + except IndexError: + pass + return mpd_url + + def get_region(self) -> dict: + domain_region = self.get_domain_region() + if not domain_region: + return {} + + region = self.config["regions"].get(domain_region) + if not region: + self.log.error(f" - There's no region configuration data for the region: {domain_region}"); raise SystemExit(1) + + region["code"] = domain_region + + if self.pv: + res = self.session.get("https://www.primevideo.com").text + match = re.search(r'ue_furl *= *([\'"])fls-(na|eu|fe)\.amazon\.[a-z.]+\1', res) + if match: + pv_region = match.group(2).lower() + else: + self.log.error(" - Failed to get PrimeVideo region"); raise SystemExit(1) + pv_region = {"na": "atv-ps"}.get(pv_region, f"atv-ps-{pv_region}") + region["base_manifest"] = f"{pv_region}.primevideo.com" + region["base"] = "www.primevideo.com" + + return region + + def get_domain_region(self): + tlds = [tldextract.extract(x.domain) for x in self.session.cookies if x.domain_specified] + tld = next((x.suffix for x in tlds if x.domain.lower() in ("amazon", "primevideo")), None) + if tld: + tld = tld.split(".")[-1] + region = {"com": "us", "uk": "gb"}.get(tld, tld) + + if region == "us": + lc_cookie = next( + (x.value for x in self.session.cookies + if x.name in ("lc-main-av", "lc-main") and x.domain_specified), + None + ) + if lc_cookie: + parts = lc_cookie.replace("-", "_").split("_") + if len(parts) >= 2: + country = parts[-1].lower() + if country not in ("us", ""): + mapped = {"uk": "gb"}.get(country, country) + if mapped in self.config.get("regions", {}): + region = mapped + + return region + + def prepare_endpoint(self, name: str, uri: str, region: dict) -> str: + if name in ("browse", "playback", "licence", "licence_pr", "xray"): + return f"https://{(region['base_manifest'])}{uri}" + if name in ("ontv", "devicelink", "details", "getDetailWidgets", "metadata"): + if self.pv: + host = "www.primevideo.com" + else: + if name in ("metadata"): + host = f"{region['base']}/gp/video" + else: + host = region["base"] + return f"https://{host}{uri}" + if name in ("codepair", "register", "token"): + base_api = region.get("base_api") or self.config["regions"]["us"]["base_api"] + return f"https://{base_api}{uri}" + raise ValueError(f"Unknown endpoint: {name}") + + def prepare_endpoints(self, endpoints: dict, region: dict) -> dict: + return {k: self.prepare_endpoint(k, v, region) for k, v in endpoints.items()} + + def register_device(self) -> None: + _profile = self.profile or "default" + self.device = dict((self.config.get("device") or {}).get(_profile, {})) + + # Resolve unique device identity from cache, or generate and persist it. + # This avoids collisions when multiple users share the same config values, + # which can cause Amazon to deregister devices that appear duplicated. + identity_cache = Cacher("AMZN") + identity_key = f"device_identity_{_profile}" + cached_identity = identity_cache.get(identity_key) + + if cached_identity and cached_identity.data: + identity = cached_identity.data + self.log.debug(" + Using cached device identity") + else: + # Generate a unique serial (16 hex chars, same format as real Android devices) + unique_serial = secrets.token_hex(8) + # Build a plausible device name: keep the base from config but make it unique + base_name = self.device.get("device_name", "%FIRST_NAME%'s Shield TV") + # Strip any existing DUPE_STRATEGY placeholder so we can add our suffix cleanly + clean_name = re.sub(r"%DUPE_STRATEGY[^%]*%", "", base_name).rstrip() + suffix = secrets.token_hex(2).upper() # e.g. "A3F1" — short, looks like a serial suffix + unique_name = f"{clean_name}-{suffix}" + identity = {"device_serial": unique_serial, "device_name": unique_name} + # Persist indefinitely (10 years TTL) — identity should never rotate on its own + cached_identity = identity_cache.get(identity_key) + cached_identity.set(identity, int(time.time()) + 60 * 60 * 24 * 3650) + self.log.info(f" + Generated unique device identity: serial={unique_serial}, name={unique_name!r}") + + self.device["device_serial"] = identity["device_serial"] + self.device["device_name"] = identity["device_name"] + + device_hash = hashlib.md5(json.dumps(self.device, sort_keys=True).encode()).hexdigest()[0:6] + device_cache_path = f"device_tokens_{_profile}_{device_hash}" + + self.device_token = self.DeviceRegistration( + device=self.device, + endpoints=self.endpoints, + log=self.log, + cache_path=device_cache_path, + session=self.session + ).bearer + + self.device_id = self.device.get("device_serial") + if not self.device_id: + self.log.error(f" - A device serial is required in the config, perhaps use: {os.urandom(8).hex()}"); raise SystemExit(1) + + class DeviceRegistration: + def __init__(self, device: dict, endpoints: dict, cache_path: str, session: requests.Session, log): + self.session = session + self.device = device + self.endpoints = endpoints + self.cache_path = cache_path + self.log = log + self.cache = Cacher('AMZN') + + self.device = {k: str(v) if not isinstance(v, str) else v for k, v in self.device.items()} + self.bearer = None + + # Retrieve from Cacher + cached_data = self.cache.get(self.cache_path) + + if cached_data: + # Check expiration + if cached_data.data.get("expires_in", 0) > int(time.time()): + self.log.info(" + Using cached device bearer") + self.bearer = cached_data.data["access_token"] + else: + self.log.info("Cached device bearer expired, refreshing...") + # Note: Vinetrimmer uses a specific refresh logic calling self.refresh + # We need to extract refresh token from cache + refresh_token = cached_data.data.get("refresh_token") + if refresh_token: + refreshed_tokens = self.refresh(self.device, refresh_token) + refreshed_tokens["refresh_token"] = refresh_token + # Fix: fallback to 3600s if expires_in is missing or zero + expires_seconds = int(refreshed_tokens.get("expires_in") or 3600) + refreshed_tokens["expires_in"] = int(time.time()) + expires_seconds + + # Fix: persist refreshed token to cache (was only updated in memory before) + cached_data.data = refreshed_tokens + cached_data.set(refreshed_tokens, refreshed_tokens["expires_in"]) + self.bearer = refreshed_tokens["access_token"] + else: + self.log.info(" + Registering new device bearer (No refresh token)") + self.bearer = self.register(self.device) + else: + self.log.info(" + Registering new device bearer") + self.bearer = self.register(self.device) + + def register(self, device: dict) -> str: + code_pair = self.get_code_pair(device) + public_code = code_pair["public_code"] + + self.log.info(f" + Visit https://www.primevideo.com/mytv and enter the code: {public_code}") + self.log.info(f" Waiting for authorisation (up to 5 minutes)...") + + interval = 10 # seconds between polls + deadline = int(time.time()) + 300 # 5 minute timeout + + while int(time.time()) < deadline: + res = self.session.post( + url=self.endpoints["register"], + headers={"Content-Type": "application/json", "Accept-Language": "en-US"}, + json={ + "auth_data": {"code_pair": code_pair}, + "registration_data": device, + "requested_token_type": ["bearer"], + "requested_extensions": ["device_info", "customer_info"] + }, + cookies=None + ) + data = res.json() + + if res.status_code == 200 and "success" in data.get("response", {}): + break + + error_code = data.get("response", {}).get("error", {}).get("code", "") + if error_code == "Unauthorized": + time.sleep(interval) + continue + else: + self.log.error(f"Unable to register: {res.text}"); raise SystemExit(1) + else: + self.log.error("Device registration timed out — code was not approved in time."); raise SystemExit(1) + + bearer = data["response"]["success"]["tokens"]["bearer"] + expires_val = bearer.get("expires_in", 3600) + if isinstance(expires_val, dict): + expires_val = expires_val.get("value", 3600) + bearer_data = { + "access_token": bearer["access_token"], + "refresh_token": bearer.get("refresh_token", ""), + "expires_in": int(time.time()) + int(expires_val), + } + keyed_cache = self.cache.get(self.cache_path) + keyed_cache.set(bearer_data, int(time.time()) + int(expires_val)) + + self.log.info(" + Device registered and token cached successfully") + return bearer_data["access_token"] + + def refresh(self, device: dict, refresh_token: str) -> dict: + res = self.session.post( + url=self.endpoints["token"], + json={ + "app_name": device["app_name"], + "app_version": device["app_version"], + "source_token_type": "refresh_token", + "source_token": refresh_token, + "requested_token_type": "access_token" + } + ).json() + + if "error" in res: + # Invalidate cache if error + # self.cache.delete(self.cache_path) # If method existed + self.log.error(f"Failed to refresh device token: {res.get('error_description')}"); raise SystemExit(1) + + return res + + def get_csrf_token(self) -> str: + res = self.session.get(self.endpoints["ontv"]) + if 'name="appAction" value="SIGNIN"' in res.text or 'SIGNIN_PWD_COLLECT' in res.text: + self.log.error("Cookies are signed out, cannot get ontv CSRF token.") + raise SystemExit(1) + for match in re.finditer(r'', res.text, re.DOTALL): + try: + prop = json.loads(match.group(1)) + token = prop.get("props", {}).get("codeEntry", {}).get("token") + if token: return token + token = prop.get("codeEntry", {}).get("token") + if token: return token + except Exception: + pass + ce_idx = res.text.find('"codeEntry"') + if ce_idx != -1: + snippet = res.text[ce_idx:ce_idx+2000] + m2 = re.search(r'"token"\s*:\s*"([^"]+)"', snippet) + if m2: return m2.group(1) + self.log.error("Unable to get ontv CSRF token") + raise SystemExit(1) + def get_code_pair(self, device: dict) -> dict: + res = self.session.post( + url=self.endpoints["codepair"], + headers={"Content-Type": "application/json", "Accept-Language": "en-US"}, + json={"code_data": device} + ).json() + if "error" in res: + self.log.error(f"Unable to get code pair: {res['error']}"); raise SystemExit(1) + return res + # Misc + def parse_title(self, ctx, title): + title = title or ctx.parent.params.get("title") + if not title: + self.log.error(" - No title ID specified") + if not getattr(self, "TITLE_RE"): + self.title = title + return {} + for regex in as_list(self.TITLE_RE): + m = re.search(regex, title) + if m: + self.title = m.group("id") + return m.groupdict() + self.log.warning(f" - Unable to parse title ID {title!r}, using as-is") + self.title = title diff --git a/reset/packages/envied/src/envied/services/AMZN/config.yaml b/reset/packages/envied/src/envied/services/AMZN/config.yaml new file mode 100644 index 0000000..970194b --- /dev/null +++ b/reset/packages/envied/src/envied/services/AMZN/config.yaml @@ -0,0 +1,124 @@ +certificate: | + CAUSwgUKvAIIAxIQCuQRtZRasVgFt7DIvVtVHBi17OSpBSKOAjCCAQoCggEBAKU2UrYVOSDlcXajWhpEgGhqGraJtFdUPgu6plJGy9ViaRn5mhyXON5PXm + w1krQdi0SLxf00FfIgnYFLpDfvNeItGn9rcx0RNPwP39PW7aW0Fbqi6VCaKWlR24kRpd7NQ4woyMXr7xlBWPwPNxK4xmR/6UuvKyYWEkroyeIjWHAqgCjC + mpfIpVcPsyrnMuPFGl82MMVnAhTweTKnEPOqJpxQ1bdQvVNCvkba5gjOTbEnJ7aXegwhmCdRQzXjTeEV2dO8oo5YfxW6pRBovzF6wYBMQYpSCJIA24ptAP + /2TkneyJuqm4hJNFvtF8fsBgTQQ4TIhnX4bZ9imuhivYLa6HsCAwEAAToPYW1hem9uLmNvbS1wcm9kEoADETQD6R0H/h9fyg0Hw7mj0M7T4s0bcBf4fMhA + Rpwk2X4HpvB49bJ5Yvc4t41mAnXGe/wiXbzsddKMiMffkSE1QWK1CFPBgziU23y1PjQToGiIv/sJIFRKRJ4qMBxIl95xlvSEzKdt68n7wqGa442+uAgk7C + XU3uTfVofYY76CrPBnEKQfad/CVqTh48geNTb4qRH1TX30NzCsB9NWlcdvg10pCnWSm8cSHu1d9yH+2yQgsGe52QoHHCqHNzG/wAxMYWTevXQW7EPTBeFy + SPY0xUN+2F2FhCf5/A7uFUHywd0zNTswh0QJc93LBTh46clRLO+d4RKBiBSj3rah6Y5iXMw9N9o58tCRc9gFHrjfMNubopWHjDOO3ATUgqXrTp+fKVCmsG + uGl1ComHxXV9i1AqHwzzY2JY2vFqo73jR3IElr6oChPIwcNokmNc0D4TXtjE0BoYkbWKJfHvJJihzMOvDicWUsemVHvua9/FBtpbHgpbgwijFPjtQF9Ldb + 8Swf + +# device: +# new: +# domain: Device +# app_name: 'com.amazon.amazonvideo.livingroom' # AIV, com.amazon.amazonvideo.livingroom +# app_version: '1.1' # AIV: 3.12.0, livingroom: 1.1 +# device_model: 'Nexus Player' # SHIELD Android TV, Nexus Player +# os_version: '8.0.0' # 25, 8.0.0 +# device_type: 'A2SNKIF736WF4T' +# device_name: '%FIRST_NAME%''s%DUPE_STRATEGY_1ST% Nexus Player' +# device_serial: 'a906a7f9bfd6a7ab' # f8eb5625fd608718 + + +device: + default: + domain: Device + app_name: 'com.amazon.amazonvideo.livingroom' + app_version: '7.3.4' + device_model: 'SHIELD Android TV' + device_name: "%FIRST_NAME%'s%DUPE_STRATEGY_1ST% Shield TV" + os_version: '28' + device_type: 'A1KAXIG6VXSG8Y' + device_serial: '9a4740e6fa227483' + software_version: '248' + +device_types: + browser: 'AOAGZA014O5RE' # all browsers? all platforms? + tv_generic: 'A2SNKIF736WF4T' # type is shared among various random smart tvs + pc_app: 'A1RTAM01W29CUP' + mobile_app: 'A43PXU4ZN2AL1' + echo: 'A7WXQPH584YP' # echo Gen2 + echo_dot: 'A32DOYMUN6DTXA' # echo dot Gen3 + echo_studio: 'A3RBAYBE7VM004' # for audio stuff, this is probably the one to use + fire_7: 'A2M4YX06LWP8WI' + fire_7_again: 'A1Q7QCGNMXAKYW' # not sure the difference + fire_hd_8: 'A1C66CX2XD756O' + fire_hd_8_again: 'A38EHHIB10L47V' # not sure the difference + fire_hd_8_plus_2020: 'AVU7CPPF2ZRAS' + fire_hd_10: 'A1ZB65LA390I4K' + fire_tv: 'A2E0SNTXJVT7WK' # this is not the stick, this is the older stick-like diamond shaped one + fire_tv_gen2: 'A12GXV8XMS007S' + fire_tv_cube: 'A2JKHJ0PX4J3L3' # this is the STB-style big bulky cube + fire_tv_stick_gen1: 'ADVBD696BHNV5' # non-4k fire tv stick + fire_tv_stick_gen2: 'A2LWARUGJLBYEW' + fire_tv_stick_with_alexa: 'A265XOI9586NML' + fire_tv_stick_4k: 'A2GFL5ZMWNE0PX' # 4k fire tv stick + fire_tv_stick_4k_gen3: 'AKPGW064GI9HE' + nvidia_shield: 'A1KAXIG6VXSG8Y' # nvidia shield, unknown which one or if all + +endpoints: + browse: '/cdp/catalog/Browse' + details: '/gp/video/api/getDetailPage' + getDetailWidgets: '/gp/video/api/getDetailWidgets' + playback: '/playback/prs/GetVodPlaybackResources' + metadata: '/api/enrichItemMetadata' + licence: '/playback/drm-vod/GetWidevineLicense' + licence_pr: '/playback/drm-vod/GetPlayReadyLicense' + # chapters/scenes + xray: '/swift/page/xray' + # device registration + ontv: '/gp/video/ontv/code' + devicelink: '/gp/video/api/codeBasedLinking' + codepair: '/auth/create/codepair' + register: '/auth/register' + token: '/auth/token' + +regions: + us: + base: 'www.amazon.com' + base_api: 'api.amazon.com' + base_manifest: 'atv-ps.amazon.com' + marketplace_id: 'ATVPDKIKX0DER' + + gb: + base: 'www.amazon.co.uk' + base_api: 'api.amazon.co.uk' + base_manifest: 'atv-ps-eu.amazon.co.uk' + marketplace_id: 'A2IR4J4NTCP2M5' # A1F83G8C2ARO7P is also another marketplace_id + + es: + base: 'www.amazon.es' + base_api: 'api.amazon.es' + base_manifest: 'atv-ps-eu.primevideo.com' + marketplace_id: 'A1RKKUPIHCS9HS' + + it: + base: 'www.amazon.it' + base_api: 'api.amazon.it' + base_manifest: 'atv-ps-eu.primevideo.com' + marketplace_id: 'A3K6Y4MI8GDYMT' + + de: + base: 'www.amazon.de' + base_api: 'api.amazon.de' + base_manifest: 'atv-ps-eu.amazon.de' + marketplace_id: 'A1PA6795UKMFR9' + + au: + base: 'www.amazon.com.au' + base_api: 'api.amazon.com.au' + base_manifest: 'atv-ps-fe.amazon.com.au' + marketplace_id: 'A3K6Y4MI8GDYMT' + + jp: + base: 'www.amazon.co.jp' + base_api: 'api.amazon.co.jp' + base_manifest: 'atv-ps-fe.amazon.co.jp' + marketplace_id: 'A1VC38T7YXB528' + + pl: + base: 'www.amazon.com' + base_api: 'api.amazon.com' + base_manifest: 'atv-ps-eu.primevideo.com' + marketplace_id: 'A3K6Y4MI8GDYMT' \ No newline at end of file diff --git a/reset/packages/envied/src/envied/services/AMZN/subtitle.py b/reset/packages/envied/src/envied/services/AMZN/subtitle.py new file mode 100644 index 0000000..30fd0d9 --- /dev/null +++ b/reset/packages/envied/src/envied/services/AMZN/subtitle.py @@ -0,0 +1,1349 @@ +from __future__ import annotations + +import logging +import re +import subprocess +from collections import defaultdict +from enum import Enum +from functools import partial +from io import BytesIO +from pathlib import Path +from typing import Any, Callable, Iterable, Optional, Union + +import pycaption +import pysubs2 +import requests +from construct import Container +from pycaption import Caption, CaptionList, CaptionNode, WebVTTReader +from pycaption.geometry import Layout +from pymp4.parser import MP4 +from subby import CommonIssuesFixer, SAMIConverter, SDHStripper, WebVTTConverter, WVTTConverter +from subtitle_filter import Subtitles + +from envied.core import binaries +from envied.core.config import config +from envied.core.tracks.track import Track +from envied.core.utilities import try_ensure_utf8 +from envied.core.utils.webvtt import merge_segmented_webvtt + +# silence srt library INFO logging +logging.getLogger("srt").setLevel(logging.ERROR) + + +class Subtitle(Track): + class Codec(str, Enum): + SubRip = "SRT" # https://wikipedia.org/wiki/SubRip + SubStationAlpha = "SSA" # https://wikipedia.org/wiki/SubStation_Alpha + SubStationAlphav4 = "ASS" # https://wikipedia.org/wiki/SubStation_Alpha#Advanced_SubStation_Alpha= + TimedTextMarkupLang = "TTML" # https://wikipedia.org/wiki/Timed_Text_Markup_Language + WebVTT = "VTT" # https://wikipedia.org/wiki/WebVTT + SAMI = "SMI" # https://wikipedia.org/wiki/SAMI + MicroDVD = "SUB" # https://wikipedia.org/wiki/MicroDVD + MPL2 = "MPL2" # MPL2 subtitle format + TMP = "TMP" # TMP subtitle format + # MPEG-DASH box-encapsulated subtitle formats + fTTML = "STPP" # https://www.w3.org/TR/2018/REC-ttml-imsc1.0.1-20180424 + fVTT = "WVTT" # https://www.w3.org/TR/webvtt1 + + @property + def extension(self) -> str: + return self.value.lower() + + @staticmethod + def from_mime(mime: str) -> Subtitle.Codec: + mime = mime.lower().strip().split(".")[0] + if mime == "srt": + return Subtitle.Codec.SubRip + elif mime == "ssa": + return Subtitle.Codec.SubStationAlpha + elif mime == "ass": + return Subtitle.Codec.SubStationAlphav4 + elif mime == "ttml": + return Subtitle.Codec.TimedTextMarkupLang + elif mime == "vtt": + return Subtitle.Codec.WebVTT + elif mime in ("smi", "sami"): + return Subtitle.Codec.SAMI + elif mime in ("sub", "microdvd"): + return Subtitle.Codec.MicroDVD + elif mime == "mpl2": + return Subtitle.Codec.MPL2 + elif mime == "tmp": + return Subtitle.Codec.TMP + elif mime == "stpp": + return Subtitle.Codec.fTTML + elif mime == "wvtt": + return Subtitle.Codec.fVTT + raise ValueError(f"The MIME '{mime}' is not a supported Subtitle Codec") + + @staticmethod + def from_codecs(codecs: str) -> Subtitle.Codec: + for codec in codecs.lower().split(","): + mime = codec.strip().split(".")[0] + try: + return Subtitle.Codec.from_mime(mime) + except ValueError: + pass + raise ValueError(f"No MIME types matched any supported Subtitle Codecs in '{codecs}'") + + @staticmethod + def from_netflix_profile(profile: str) -> Subtitle.Codec: + profile = profile.lower().strip() + if profile.startswith("webvtt"): + return Subtitle.Codec.WebVTT + if profile.startswith("dfxp"): + return Subtitle.Codec.TimedTextMarkupLang + raise ValueError(f"The Content Profile '{profile}' is not a supported Subtitle Codec") + + # WebVTT sanitization patterns (compiled once for performance) + _CUE_ID_PATTERN = re.compile(r"^[A-Za-z]+\d+$") + _TIMING_START_PATTERN = re.compile(r"^\d+:\d+[:\.]") + _TIMING_LINE_PATTERN = re.compile(r"^((?:\d+:)?\d+:\d+[.,]\d+)\s*-->\s*((?:\d+:)?\d+:\d+[.,]\d+)(.*)$") + _LINE_POS_PATTERN = re.compile(r"line:(\d+(?:\.\d+)?%?)") + + def __init__( + self, + *args: Any, + codec: Optional[Subtitle.Codec] = None, + cc: bool = False, + sdh: bool = False, + forced: bool = False, + **kwargs: Any, + ): + """ + Create a new Subtitle track object. + + Parameters: + codec: A Subtitle.Codec enum representing the subtitle format. + If not specified, MediaInfo will be used to retrieve the format + once the track has been downloaded. + cc: Closed Caption. + - Intended as if you couldn't hear the audio at all. + - Can have Sound as well as Dialogue, but doesn't have to. + - Original source would be from an EIA-CC encoded stream. Typically all + upper-case characters. + Indicators of it being CC without knowing original source: + - Extracted with CCExtractor, or + - >>> (or similar) being used at the start of some or all lines, or + - All text is uppercase or at least the majority, or + - Subtitles are Scrolling-text style (one line appears, oldest line + then disappears). + Just because you downloaded it as a SRT or VTT or such, doesn't mean it + isn't from an EIA-CC stream. And I wouldn't take the streaming services + (CC) as gospel either as they tend to get it wrong too. + sdh: Deaf or Hard-of-Hearing. Also known as HOH in the UK (EU?). + - Intended as if you couldn't hear the audio at all. + - MUST have Sound as well as Dialogue to be considered SDH. + - It has no "syntax" or "format" but is not transmitted using archaic + forms like EIA-CC streams, would be intended for transmission via + SubRip (SRT), WebVTT (VTT), TTML, etc. + If you can see important audio/sound transcriptions and not just dialogue + and it doesn't have the indicators of CC, then it's most likely SDH. + If it doesn't have important audio/sounds transcriptions it might just be + regular subtitling (you wouldn't mark as CC or SDH). This would be the + case for most translation subtitles. Like Anime for example. + forced: Typically used if there's important information at some point in time + like watching Dubbed content and an important Sign or Letter is shown + or someone talking in a different language. + Forced tracks are recommended by the Matroska Spec to be played if + the player's current playback audio language matches a subtitle + marked as "forced". + However, that doesn't mean every player works like this but there is + no other way to reliably work with Forced subtitles where multiple + forced subtitles may be in the output file. Just know what to expect + with "forced" subtitles. + + Note: If codec is not specified some checks may be skipped or assume a value. + Specifying as much information as possible is highly recommended. + + Information on Subtitle Types: + https://bit.ly/2Oe4fLC (3PlayMedia Blog on SUB vs CC vs SDH). + However, I wouldn't pay much attention to the claims about SDH needing to + be in the original source language. It's logically not true. + + CC == Closed Captions. Source: Basically every site. + SDH = Subtitles for the Deaf or Hard-of-Hearing. Source: Basically every site. + HOH = Exact same as SDH. Is a term used in the UK. Source: https://bit.ly/2PGJatz (ICO UK) + + More in-depth information, examples, and stuff to look for can be found in the Parameter + explanation list above. + """ + super().__init__(*args, **kwargs) + + if not isinstance(codec, (Subtitle.Codec, type(None))): + raise TypeError(f"Expected codec to be a {Subtitle.Codec}, not {codec!r}") + if not isinstance(cc, (bool, int)) or (isinstance(cc, int) and cc not in (0, 1)): + raise TypeError(f"Expected cc to be a {bool} or bool-like {int}, not {cc!r}") + if not isinstance(sdh, (bool, int)) or (isinstance(sdh, int) and sdh not in (0, 1)): + raise TypeError(f"Expected sdh to be a {bool} or bool-like {int}, not {sdh!r}") + if not isinstance(forced, (bool, int)) or (isinstance(forced, int) and forced not in (0, 1)): + raise TypeError(f"Expected forced to be a {bool} or bool-like {int}, not {forced!r}") + + self.codec = codec + + self.cc = bool(cc) + self.sdh = bool(sdh) + self.forced = bool(forced) + + if self.cc and self.sdh: + raise ValueError("A text track cannot be both CC and SDH.") + + if self.forced and (self.cc or self.sdh): + raise ValueError("A text track cannot be CC/SDH as well as Forced.") + + # TODO: Migrate to new event observer system + # Called after Track has been converted to another format + self.OnConverted: Optional[Callable[[Subtitle.Codec], None]] = None + + def __str__(self) -> str: + return " | ".join( + filter( + bool, + ["SUB", f"[{self.codec.value}]" if self.codec else None, str(self.language), self.get_track_name()], + ) + ) + + def get_track_name(self) -> Optional[str]: + """Return the base Track Name.""" + track_name = super().get_track_name() or "" + flag = self.cc and "CC" or self.sdh and "SDH" or self.forced and "Forced" + if flag: + if track_name: + flag = f" ({flag})" + track_name += flag + return track_name or None + + def download( + self, + session: requests.Session, + prepare_drm: partial, + max_workers: Optional[int] = None, + progress: Optional[partial] = None, + *, + cdm: Optional[object] = None, + ): + super().download(session, prepare_drm, max_workers, progress, cdm=cdm) + if not self.path: + return + + if self.codec == Subtitle.Codec.fTTML: + self.convert(Subtitle.Codec.TimedTextMarkupLang) + elif self.codec == Subtitle.Codec.fVTT: + self.convert(Subtitle.Codec.WebVTT) + elif self.codec == Subtitle.Codec.WebVTT: + text = self.path.read_text("utf8") + if self.descriptor == Track.Descriptor.DASH: + if len(self.data["dash"]["segment_durations"]) > 1: + text = merge_segmented_webvtt( + text, + segment_durations=self.data["dash"]["segment_durations"], + timescale=self.data["dash"]["timescale"], + ) + elif self.descriptor == Track.Descriptor.HLS: + if len(self.data["hls"]["segment_durations"]) > 1: + text = merge_segmented_webvtt( + text, + segment_durations=self.data["hls"]["segment_durations"], + timescale=1, # ? + ) + + # Sanitize WebVTT timestamps before parsing + text = Subtitle.sanitize_webvtt_timestamps(text) + # Remove cue identifiers that confuse parsers like pysubs2 + text = Subtitle.sanitize_webvtt_cue_identifiers(text) + # Merge overlapping cues with line positioning into single multi-line cues + text = Subtitle.merge_overlapping_webvtt_cues(text) + + preserve_formatting = config.subtitle.get("preserve_formatting", True) + + if preserve_formatting: + self.path.write_text(text, encoding="utf8") + else: + try: + caption_set = pycaption.WebVTTReader().read(text) + Subtitle.merge_same_cues(caption_set) + Subtitle.filter_unwanted_cues(caption_set) + subtitle_text = pycaption.WebVTTWriter().write(caption_set) + self.path.write_text(subtitle_text, encoding="utf8") + except pycaption.exceptions.CaptionReadSyntaxError: + # If first attempt fails, try more aggressive sanitization + text = Subtitle.sanitize_webvtt(text) + try: + caption_set = pycaption.WebVTTReader().read(text) + Subtitle.merge_same_cues(caption_set) + Subtitle.filter_unwanted_cues(caption_set) + subtitle_text = pycaption.WebVTTWriter().write(caption_set) + self.path.write_text(subtitle_text, encoding="utf8") + except Exception: + # Keep the sanitized version even if parsing failed + self.path.write_text(text, encoding="utf8") + + @staticmethod + def sanitize_webvtt_timestamps(text: str) -> str: + """ + Fix invalid timestamps in WebVTT files, particularly negative timestamps. + + Parameters: + text: The WebVTT content as string + + Returns: + Sanitized WebVTT content + """ + # Replace negative timestamps with 00:00:00.000 + return re.sub(r"(-\d+:\d+:\d+\.\d+)", "00:00:00.000", text) + + @staticmethod + def has_webvtt_cue_identifiers(text: str) -> bool: + """ + Check if WebVTT content has cue identifiers that need removal. + + Parameters: + text: The WebVTT content as string + + Returns: + True if cue identifiers are detected, False otherwise + """ + lines = text.split("\n") + + for i, line in enumerate(lines): + line = line.strip() + if Subtitle._CUE_ID_PATTERN.match(line): + # Look ahead to see if next non-empty line is a timing line + j = i + 1 + while j < len(lines) and not lines[j].strip(): + j += 1 + if j < len(lines) and ("-->" in lines[j] or Subtitle._TIMING_START_PATTERN.match(lines[j].strip())): + return True + return False + + @staticmethod + def sanitize_webvtt_cue_identifiers(text: str) -> str: + """ + Remove WebVTT cue identifiers that can confuse subtitle parsers. + + Some services use cue identifiers like "Q0", "Q1", etc. + that appear on their own line before the timing line. These can be + incorrectly parsed as part of the previous cue's text content by + some parsers (like pysubs2). + + Parameters: + text: The WebVTT content as string + + Returns: + Sanitized WebVTT content with cue identifiers removed + """ + if not Subtitle.has_webvtt_cue_identifiers(text): + return text + + lines = text.split("\n") + sanitized_lines = [] + + i = 0 + while i < len(lines): + line = lines[i].strip() + + # Check if this line is a cue identifier followed by a timing line + if Subtitle._CUE_ID_PATTERN.match(line): + # Look ahead to see if next non-empty line is a timing line + j = i + 1 + while j < len(lines) and not lines[j].strip(): + j += 1 + if j < len(lines) and ("-->" in lines[j] or Subtitle._TIMING_START_PATTERN.match(lines[j].strip())): + # This is a cue identifier, skip it + i += 1 + continue + + sanitized_lines.append(lines[i]) + i += 1 + + return "\n".join(sanitized_lines) + + @staticmethod + def _parse_vtt_time(t: str) -> int: + """Parse WebVTT timestamp to milliseconds. Returns 0 for malformed input.""" + try: + t = t.replace(",", ".") + parts = t.split(":") + if len(parts) == 2: + m, s = parts + h = "0" + elif len(parts) >= 3: + h, m, s = parts[:3] + else: + return 0 + sec_parts = s.split(".") + secs = int(sec_parts[0]) + # Handle variable millisecond digits (e.g., .5 = 500ms, .50 = 500ms, .500 = 500ms) + ms = int(sec_parts[1].ljust(3, "0")[:3]) if len(sec_parts) > 1 else 0 + return int(h) * 3600000 + int(m) * 60000 + secs * 1000 + ms + except (ValueError, IndexError): + return 0 + + @staticmethod + def has_overlapping_webvtt_cues(text: str) -> bool: + """ + Check if WebVTT content has overlapping cues that need merging. + + Detects cues with start times within 50ms of each other and the same end time, + which indicates multi-line subtitles split into separate cues. + + Parameters: + text: The WebVTT content as string + + Returns: + True if overlapping cues are detected, False otherwise + """ + timings = [] + for line in text.split("\n"): + match = Subtitle._TIMING_LINE_PATTERN.match(line) + if match: + start_str, end_str = match.group(1), match.group(2) + timings.append((Subtitle._parse_vtt_time(start_str), Subtitle._parse_vtt_time(end_str))) + + # Check for overlapping cues (within 50ms start, same end) + for i in range(len(timings) - 1): + curr_start, curr_end = timings[i] + next_start, next_end = timings[i + 1] + if abs(curr_start - next_start) <= 50 and curr_end == next_end: + return True + + return False + + @staticmethod + def merge_overlapping_webvtt_cues(text: str) -> str: + """ + Merge WebVTT cues that have overlapping/near-identical times but different line positions. + + Some services use separate cues for each line of a multi-line subtitle, with + slightly different start times (1ms apart) and different line: positions. + This merges them into single cues with proper line ordering based on the + line: position (lower percentage = higher on screen = first line). + + Parameters: + text: The WebVTT content as string + + Returns: + WebVTT content with overlapping cues merged + """ + if not Subtitle.has_overlapping_webvtt_cues(text): + return text + + lines = text.split("\n") + cues = [] + header_lines = [] + in_header = True + i = 0 + + while i < len(lines): + line = lines[i] + + if in_header: + if "-->" in line: + in_header = False + else: + header_lines.append(line) + i += 1 + continue + + match = Subtitle._TIMING_LINE_PATTERN.match(line) + if match: + start_str, end_str, settings = match.groups() + line_pos = 100.0 # Default to bottom + line_match = Subtitle._LINE_POS_PATTERN.search(settings) + if line_match: + pos_str = line_match.group(1).rstrip("%") + line_pos = float(pos_str) + + content_lines = [] + i += 1 + while i < len(lines) and lines[i].strip() and "-->" not in lines[i]: + content_lines.append(lines[i]) + i += 1 + + cues.append( + { + "start_ms": Subtitle._parse_vtt_time(start_str), + "end_ms": Subtitle._parse_vtt_time(end_str), + "start_str": start_str, + "end_str": end_str, + "line_pos": line_pos, + "content": "\n".join(content_lines), + "settings": settings, + } + ) + else: + i += 1 + + # Merge overlapping cues (within 50ms of each other with same end time) + merged_cues = [] + i = 0 + while i < len(cues): + current = cues[i] + group = [current] + + j = i + 1 + while j < len(cues): + other = cues[j] + if abs(current["start_ms"] - other["start_ms"]) <= 50 and current["end_ms"] == other["end_ms"]: + group.append(other) + j += 1 + else: + break + + if len(group) > 1: + # Sort by line position (lower % = higher on screen = first) + group.sort(key=lambda x: x["line_pos"]) + # Use the earliest start time from the group + earliest = min(group, key=lambda x: x["start_ms"]) + merged_cues.append( + { + "start_str": earliest["start_str"], + "end_str": group[0]["end_str"], + "content": "\n".join(c["content"] for c in group), + "settings": "", + } + ) + else: + merged_cues.append( + { + "start_str": current["start_str"], + "end_str": current["end_str"], + "content": current["content"], + "settings": current["settings"], + } + ) + + i = j if len(group) > 1 else i + 1 + + result_lines = header_lines[:] + if result_lines and result_lines[-1].strip(): + result_lines.append("") + + for cue in merged_cues: + result_lines.append(f"{cue['start_str']} --> {cue['end_str']}{cue['settings']}") + result_lines.append(cue["content"]) + result_lines.append("") + + return "\n".join(result_lines) + + @staticmethod + def sanitize_webvtt(text: str) -> str: + """ + More thorough sanitization of WebVTT files to handle multiple potential issues. + + Parameters: + text: The WebVTT content as string + + Returns: + Sanitized WebVTT content + """ + # Make sure we have a proper WEBVTT header + if not text.strip().startswith("WEBVTT"): + text = "WEBVTT\n\n" + text + + lines = text.split("\n") + sanitized_lines = [] + timestamp_pattern = re.compile(r"^((?:\d+:)?\d+:\d+\.\d+)\s+-->\s+((?:\d+:)?\d+:\d+\.\d+)") + + # Skip invalid headers - keep only WEBVTT + header_done = False + for line in lines: + if not header_done: + if line.startswith("WEBVTT"): + sanitized_lines.append("WEBVTT") + header_done = True + continue + + # Replace negative timestamps + if "-" in line and "-->" in line: + line = re.sub(r"(-\d+:\d+:\d+\.\d+)", "00:00:00.000", line) + + # Validate timestamp format + match = timestamp_pattern.match(line) + if match: + start_time = match.group(1) + end_time = match.group(2) + + # Ensure proper format with hours if missing + if start_time.count(":") == 1: + start_time = f"00:{start_time}" + if end_time.count(":") == 1: + end_time = f"00:{end_time}" + + line = f"{start_time} --> {end_time}" + + sanitized_lines.append(line) + + return "\n".join(sanitized_lines) + + def convert_with_subby(self, codec: Subtitle.Codec) -> Path: + """ + Convert subtitle using subby library for better format support and processing. + + This method leverages subby's advanced subtitle processing capabilities + including better WebVTT handling, SDH stripping, and common issue fixing. + """ + + if not self.path or not self.path.exists(): + raise ValueError("You must download the subtitle track first.") + + if self.codec == codec: + return self.path + + output_path = self.path.with_suffix(f".{codec.value.lower()}") + original_path = self.path + + try: + # Convert to SRT using subby first + srt_subtitles = None + + if self.codec == Subtitle.Codec.WebVTT: + converter = WebVTTConverter() + srt_subtitles = converter.from_file(self.path) + if self.codec == Subtitle.Codec.fVTT: + converter = WVTTConverter() + srt_subtitles = converter.from_file(self.path) + elif self.codec == Subtitle.Codec.SAMI: + converter = SAMIConverter() + srt_subtitles = converter.from_file(self.path) + + if srt_subtitles is not None: + # Apply common fixes + fixer = CommonIssuesFixer() + fixed_srt, _ = fixer.from_srt(srt_subtitles) + + # If target is SRT, we're done + if codec == Subtitle.Codec.SubRip: + fixed_srt.save(output_path, encoding="utf8") + else: + # Convert from SRT to target format using existing pycaption logic + temp_srt_path = self.path.with_suffix(".temp.srt") + fixed_srt.save(temp_srt_path, encoding="utf8") + + # Parse the SRT and convert to target format + caption_set = self.parse(temp_srt_path.read_bytes(), Subtitle.Codec.SubRip) + self.merge_same_cues(caption_set) + + writer = { + Subtitle.Codec.TimedTextMarkupLang: pycaption.DFXPWriter, + Subtitle.Codec.WebVTT: pycaption.WebVTTWriter, + }.get(codec) + + if writer: + subtitle_text = writer().write(caption_set) + output_path.write_text(subtitle_text, encoding="utf8") + else: + # Fall back to existing conversion method + temp_srt_path.unlink() + return self._convert_standard(codec) + + temp_srt_path.unlink() + + if original_path.exists() and original_path != output_path: + original_path.unlink() + + self.path = output_path + self.codec = codec + + if callable(self.OnConverted): + self.OnConverted(codec) + + return output_path + else: + # Fall back to existing conversion method + return self._convert_standard(codec) + + except Exception: + # Fall back to existing conversion method on any error + return self._convert_standard(codec) + + def convert_with_pysubs2(self, codec: Subtitle.Codec) -> Path: + """ + Convert subtitle using pysubs2 library for broad format support. + + pysubs2 is a pure-Python library supporting SubRip (SRT), SubStation Alpha + (SSA/ASS), WebVTT, TTML, SAMI, MicroDVD, MPL2, and TMP formats. + """ + if not self.path or not self.path.exists(): + raise ValueError("You must download the subtitle track first.") + + if self.codec == codec: + return self.path + + output_path = self.path.with_suffix(f".{codec.value.lower()}") + original_path = self.path + + codec_to_pysubs2_format = { + Subtitle.Codec.SubRip: "srt", + Subtitle.Codec.SubStationAlpha: "ssa", + Subtitle.Codec.SubStationAlphav4: "ass", + Subtitle.Codec.WebVTT: "vtt", + Subtitle.Codec.TimedTextMarkupLang: "ttml", + Subtitle.Codec.SAMI: "sami", + Subtitle.Codec.MicroDVD: "microdvd", + Subtitle.Codec.MPL2: "mpl2", + Subtitle.Codec.TMP: "tmp", + } + + pysubs2_output_format = codec_to_pysubs2_format.get(codec) + if pysubs2_output_format is None: + return self._convert_standard(codec) + + try: + subs = pysubs2.load(str(self.path), encoding="utf-8") + + subs.save(str(output_path), format_=pysubs2_output_format, encoding="utf-8") + + if original_path.exists() and original_path != output_path: + original_path.unlink() + + self.path = output_path + self.codec = codec + + if callable(self.OnConverted): + self.OnConverted(codec) + + return output_path + + except Exception: + return self._convert_standard(codec) + + def convert(self, codec: Subtitle.Codec) -> Path: + """ + Convert this Subtitle to another Format. + + The conversion method is determined by the 'conversion_method' setting in config: + - 'auto' (default): Uses subby for WebVTT/fVTT/SAMI; for SSA/ASS/MicroDVD/MPL2/TMP + uses SubtitleEdit if available, otherwise pysubs2; standard for others + - 'subby': Always uses subby with CommonIssuesFixer + - 'subtitleedit': Uses SubtitleEdit when available, falls back to pycaption + - 'pycaption': Uses only pycaption library + - 'pysubs2': Uses pysubs2 library + """ + # Check configuration for conversion method + conversion_method = config.subtitle.get("conversion_method", "auto") + + if conversion_method == "subby": + return self.convert_with_subby(codec) + elif conversion_method == "subtitleedit": + return self._convert_standard(codec) + elif conversion_method == "pycaption": + return self._convert_pycaption_only(codec) + elif conversion_method == "pysubs2": + return self.convert_with_pysubs2(codec) + elif conversion_method == "auto": + if self.codec in (Subtitle.Codec.WebVTT, Subtitle.Codec.fVTT, Subtitle.Codec.SAMI): + return self.convert_with_subby(codec) + elif self.codec in ( + Subtitle.Codec.SubStationAlpha, + Subtitle.Codec.SubStationAlphav4, + Subtitle.Codec.MicroDVD, + Subtitle.Codec.MPL2, + Subtitle.Codec.TMP, + ): + if binaries.SubtitleEdit: + return self._convert_standard(codec) + else: + return self.convert_with_pysubs2(codec) + else: + return self._convert_standard(codec) + else: + return self._convert_standard(codec) + + def _convert_pycaption_only(self, codec: Subtitle.Codec) -> Path: + """ + Convert subtitle using only pycaption library (no SubtitleEdit, no subby). + + This is the original conversion method that only uses pycaption. + """ + if not self.path or not self.path.exists(): + raise ValueError("You must download the subtitle track first.") + + if self.codec == codec: + return self.path + + output_path = self.path.with_suffix(f".{codec.value.lower()}") + original_path = self.path + + # Use only pycaption for conversion + writer = { + Subtitle.Codec.SubRip: pycaption.SRTWriter, + Subtitle.Codec.TimedTextMarkupLang: pycaption.DFXPWriter, + Subtitle.Codec.WebVTT: pycaption.WebVTTWriter, + }.get(codec) + + if writer is None: + raise NotImplementedError(f"Cannot convert {self.codec.name} to {codec.name} using pycaption only.") + + caption_set = self.parse(self.path.read_bytes(), self.codec) + Subtitle.merge_same_cues(caption_set) + if codec == Subtitle.Codec.WebVTT: + Subtitle.filter_unwanted_cues(caption_set) + subtitle_text = writer().write(caption_set) + + output_path.write_text(subtitle_text, encoding="utf8") + + if original_path.exists() and original_path != output_path: + original_path.unlink() + + self.path = output_path + self.codec = codec + + if callable(self.OnConverted): + self.OnConverted(codec) + + return output_path + + def _convert_standard(self, codec: Subtitle.Codec) -> Path: + """ + Convert this Subtitle to another Format. + + The file path location of the Subtitle data will be kept at the same + location but the file extension will be changed appropriately. + + Supported formats: + - SubRip - SubtitleEdit or pycaption.SRTWriter + - TimedTextMarkupLang - SubtitleEdit or pycaption.DFXPWriter + - WebVTT - SubtitleEdit or pycaption.WebVTTWriter + - SubStationAlphav4 - SubtitleEdit + - SAMI - subby.SAMIConverter (when available) + - fTTML* - custom code using some pycaption functions + - fVTT* - custom code using some pycaption functions + *: Can read from format, but cannot convert to format + + Note: It currently prioritizes using SubtitleEdit over PyCaption as + I have personally noticed more oddities with PyCaption parsing over + SubtitleEdit. Especially when working with TTML/DFXP where it would + often have timecodes and stuff mixed in/duplicated. + + Returns the new file path of the Subtitle. + """ + if not self.path or not self.path.exists(): + raise ValueError("You must download the subtitle track first.") + + if self.codec == codec: + return self.path + + output_path = self.path.with_suffix(f".{codec.value.lower()}") + original_path = self.path + + if binaries.SubtitleEdit and self.codec not in (Subtitle.Codec.fTTML, Subtitle.Codec.fVTT): + sub_edit_format = { + Subtitle.Codec.SubRip: "subrip", + Subtitle.Codec.SubStationAlpha: "substationalpha", + Subtitle.Codec.SubStationAlphav4: "advancedsubstationalpha", + Subtitle.Codec.TimedTextMarkupLang: "timedtext1.0", + Subtitle.Codec.WebVTT: "webvtt", + Subtitle.Codec.SAMI: "sami", + Subtitle.Codec.MicroDVD: "microdvd", + }.get(codec, codec.name.lower()) + sub_edit_args = [ + str(binaries.SubtitleEdit), + "/convert", + str(self.path), + sub_edit_format, + f"/outputfilename:{output_path.name}", + "/encoding:utf8", + ] + if codec == Subtitle.Codec.SubRip: + sub_edit_args.append("/ConvertColorsToDialog") + subprocess.run(sub_edit_args, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + if codec == Subtitle.Codec.SubRip and output_path.exists(): + _ALLOWED_SRT_TAGS = {"i", "b", "u", "s"} + _TAG_RE = re.compile(r"<(/?)([a-zA-Z][a-zA-Z0-9]*)(\s[^>]*)?>") + def _clean_tag(m: re.Match) -> str: + tag = m.group(2).lower() + attrs = (m.group(3) or "").strip() + if tag in _ALLOWED_SRT_TAGS and not attrs: + return m.group(0) + return "" + srt_text = output_path.read_text(encoding="utf-8", errors="replace") + cleaned = _TAG_RE.sub(_clean_tag, srt_text) + if cleaned != srt_text: + output_path.write_text(cleaned, encoding="utf-8") + else: + writer = { + # pycaption generally only supports these subtitle formats + Subtitle.Codec.SubRip: pycaption.SRTWriter, + Subtitle.Codec.TimedTextMarkupLang: pycaption.DFXPWriter, + Subtitle.Codec.WebVTT: pycaption.WebVTTWriter, + }.get(codec) + if writer is None: + raise NotImplementedError(f"Cannot yet convert {self.codec.name} to {codec.name}.") + + caption_set = self.parse(self.path.read_bytes(), self.codec) + Subtitle.merge_same_cues(caption_set) + if codec == Subtitle.Codec.WebVTT: + Subtitle.filter_unwanted_cues(caption_set) + subtitle_text = writer().write(caption_set) + + output_path.write_text(subtitle_text, encoding="utf8") + + if original_path.exists() and original_path != output_path: + original_path.unlink() + + self.path = output_path + self.codec = codec + + if callable(self.OnConverted): + self.OnConverted(codec) + + return output_path + + @staticmethod + def parse(data: bytes, codec: Subtitle.Codec) -> pycaption.CaptionSet: + if not isinstance(data, bytes): + raise ValueError(f"Subtitle data must be parsed as bytes data, not {type(data).__name__}") + + try: + if codec == Subtitle.Codec.SubRip: + text = try_ensure_utf8(data).decode("utf8") + caption_set = pycaption.SRTReader().read(text) + elif codec == Subtitle.Codec.fTTML: + caption_lists: dict[str, pycaption.CaptionList] = defaultdict(pycaption.CaptionList) + for segment in ( + Subtitle.parse(box.data, Subtitle.Codec.TimedTextMarkupLang) + for box in MP4.parse_stream(BytesIO(data)) + if box.type == b"mdat" + ): + for lang in segment.get_languages(): + caption_lists[lang].extend(segment.get_captions(lang)) + caption_set: pycaption.CaptionSet = pycaption.CaptionSet(caption_lists) + elif codec == Subtitle.Codec.TimedTextMarkupLang: + text = try_ensure_utf8(data).decode("utf8") + text = text.replace("tt:", "") + # negative size values aren't allowed in TTML/DFXP spec, replace with 0 + text = re.sub(r"-(\d+(?:\.\d+)?)(px|em|%|c|pt)", r"0\2", text) + caption_set = pycaption.DFXPReader().read(text) + elif codec == Subtitle.Codec.fVTT: + caption_lists: dict[str, pycaption.CaptionList] = defaultdict(pycaption.CaptionList) + caption_list, language = Subtitle.merge_segmented_wvtt(data) + caption_lists[language] = caption_list + caption_set: pycaption.CaptionSet = pycaption.CaptionSet(caption_lists) + elif codec == Subtitle.Codec.WebVTT: + text = try_ensure_utf8(data).decode("utf8") + text = Subtitle.sanitize_broken_webvtt(text) + text = Subtitle.space_webvtt_headers(text) + caption_set = pycaption.WebVTTReader().read(text) + elif codec == Subtitle.Codec.SAMI: + # Use subby for SAMI parsing + converter = SAMIConverter() + srt_subtitles = converter.from_bytes(data) + # Convert SRT back to CaptionSet for compatibility + srt_text = str(srt_subtitles).encode("utf8") + caption_set = Subtitle.parse(srt_text, Subtitle.Codec.SubRip) + else: + raise ValueError(f'Unknown Subtitle format "{codec}"...') + except pycaption.exceptions.CaptionReadSyntaxError as e: + raise SyntaxError(f'A syntax error has occurred when reading the "{codec}" subtitle: {e}') + except pycaption.exceptions.CaptionReadNoCaptions: + return pycaption.CaptionSet({"en": []}) + + # remove empty caption lists or some code breaks, especially if it's the first list + for language in caption_set.get_languages(): + if not caption_set.get_captions(language): + # noinspection PyProtectedMember + del caption_set._captions[language] + + return caption_set + + @staticmethod + def sanitize_broken_webvtt(text: str) -> str: + """ + Remove or fix corrupted WebVTT lines, particularly those with invalid timestamps. + + Parameters: + text: The WebVTT content as string + + Returns: + Sanitized WebVTT content with corrupted lines removed + """ + lines = text.splitlines() + sanitized_lines = [] + + i = 0 + while i < len(lines): + # Skip empty lines + if not lines[i].strip(): + sanitized_lines.append(lines[i]) + i += 1 + continue + + # Check for timestamp lines + if "-->" in lines[i]: + # Validate timestamp format + timestamp_parts = lines[i].split("-->") + if len(timestamp_parts) != 2 or not timestamp_parts[1].strip() or timestamp_parts[1].strip() == "0": + # Skip this timestamp and its content until next timestamp or end + j = i + 1 + while j < len(lines) and "-->" not in lines[j] and lines[j].strip(): + j += 1 + i = j + continue + + # Add valid timestamp line + sanitized_lines.append(lines[i]) + else: + # Add non-timestamp line + sanitized_lines.append(lines[i]) + + i += 1 + + return "\n".join(sanitized_lines) + + @staticmethod + def space_webvtt_headers(data: Union[str, bytes]): + """ + Space out the WEBVTT Headers from Captions. + + Segmented VTT when merged may have the WEBVTT headers part of the next caption + as they were not separated far enough from the previous caption and ended up + being considered as caption text rather than the header for the next segment. + """ + if isinstance(data, bytes): + data = try_ensure_utf8(data).decode("utf8") + elif not isinstance(data, str): + raise ValueError(f"Expecting data to be a str, not {data!r}") + + text = ( + data.replace("WEBVTT", "\n\nWEBVTT").replace("\r", "").replace("\n\n\n", "\n \n\n").replace("\n\n<", "\n<") + ) + + return text + + @staticmethod + def merge_same_cues(caption_set: pycaption.CaptionSet): + """Merge captions with the same timecodes and text as one in-place.""" + for lang in caption_set.get_languages(): + captions = caption_set.get_captions(lang) + last_caption = None + concurrent_captions = pycaption.CaptionList() + merged_captions = pycaption.CaptionList() + for caption in captions: + if last_caption: + if (caption.start, caption.end) == (last_caption.start, last_caption.end): + if caption.get_text() != last_caption.get_text(): + concurrent_captions.append(caption) + last_caption = caption + continue + else: + merged_captions.append(pycaption.base.merge(concurrent_captions)) + concurrent_captions = [caption] + last_caption = caption + + if concurrent_captions: + merged_captions.append(pycaption.base.merge(concurrent_captions)) + if merged_captions: + caption_set.set_captions(lang, merged_captions) + + @staticmethod + def filter_unwanted_cues(caption_set: pycaption.CaptionSet): + """ + Filter out subtitle cues containing only   or whitespace. + """ + for lang in caption_set.get_languages(): + captions = caption_set.get_captions(lang) + filtered_captions = pycaption.CaptionList() + + for caption in captions: + text = caption.get_text().strip() + if not text or text == " " or all(c in " \t\n\r\xa0" for c in text.replace(" ", "\xa0")): + continue + + filtered_captions.append(caption) + + caption_set.set_captions(lang, filtered_captions) + + @staticmethod + def merge_segmented_wvtt(data: bytes, period_start: float = 0.0) -> tuple[CaptionList, Optional[str]]: + """ + Convert Segmented DASH WebVTT cues into a pycaption Caption List. + Also returns an ISO 639-2 alpha-3 language code if available. + + Code ported originally by xhlove to Python from shaka-player. + Has since been improved upon by rlaphoenix using pymp4 and + pycaption functions. + """ + captions = CaptionList() + + # init: + saw_wvtt_box = False + timescale = None + language = None + + # media: + # > tfhd + default_duration = None + # > tfdt + saw_tfdt_box = False + base_time = 0 + # > trun + saw_trun_box = False + samples = [] + + def flatten_boxes(box: Container) -> Iterable[Container]: + for child in box: + if hasattr(child, "children"): + yield from flatten_boxes(child.children) + del child["children"] + if hasattr(child, "entries"): + yield from flatten_boxes(child.entries) + del child["entries"] + # some boxes (mainly within 'entries') uses format not type + child["type"] = child.get("type") or child.get("format") + yield child + + for box in flatten_boxes(MP4.parse_stream(BytesIO(data))): + # init + if box.type == b"mdhd": + timescale = box.timescale + language = box.language + + if box.type == b"wvtt": + saw_wvtt_box = True + + # media + if box.type == b"styp": + # essentially the start of each segment + # media var resets + # > tfhd + default_duration = None + # > tfdt + saw_tfdt_box = False + base_time = 0 + # > trun + saw_trun_box = False + samples = [] + + if box.type == b"tfhd": + if box.flags.default_sample_duration_present: + default_duration = box.default_sample_duration + + if box.type == b"tfdt": + saw_tfdt_box = True + base_time = box.baseMediaDecodeTime + + if box.type == b"trun": + saw_trun_box = True + samples = box.sample_info + + if box.type == b"mdat": + if not timescale: + raise ValueError("Timescale was not found in the Segmented WebVTT.") + if not saw_wvtt_box: + raise ValueError("The WVTT box was not found in the Segmented WebVTT.") + if not saw_tfdt_box: + raise ValueError("The TFDT box was not found in the Segmented WebVTT.") + if not saw_trun_box: + raise ValueError("The TRUN box was not found in the Segmented WebVTT.") + + vttc_boxes = MP4.parse_stream(BytesIO(box.data)) + current_time = base_time + period_start + + for sample, vttc_box in zip(samples, vttc_boxes): + duration = sample.sample_duration or default_duration + if sample.sample_composition_time_offsets: + current_time += sample.sample_composition_time_offsets + + start_time = current_time + end_time = current_time + (duration or 0) + current_time = end_time + + if vttc_box.type == b"vtte": + # vtte is a vttc that's empty, skip + continue + + layout: Optional[Layout] = None + nodes: list[CaptionNode] = [] + + for cue_box in vttc_box.children: + if cue_box.type == b"vsid": + # this is a V(?) Source ID box, we don't care + continue + if cue_box.type == b"sttg": + layout = Layout(webvtt_positioning=cue_box.settings) + elif cue_box.type == b"payl": + nodes.extend( + [ + node + for line in cue_box.cue_text.split("\n") + for node in [ + CaptionNode.create_text(WebVTTReader()._decode(line)), + CaptionNode.create_break(), + ] + ] + ) + nodes.pop() + + if nodes: + caption = Caption( + start=start_time * timescale, # as microseconds + end=end_time * timescale, + nodes=nodes, + layout_info=layout, + ) + p_caption = captions[-1] if captions else None + if p_caption and caption.start == p_caption.end and str(caption.nodes) == str(p_caption.nodes): + # it's a duplicate, but lets take its end time + p_caption.end = caption.end + continue + captions.append(caption) + + return captions, language + + def strip_hearing_impaired(self) -> None: + """ + Strip captions for hearing impaired (SDH). + + The SDH stripping method is determined by the 'sdh_method' setting in config: + - 'auto' (default): Tries subby first, then SubtitleEdit, then filter-subs + - 'subby': Uses subby's SDHStripper + - 'subtitleedit': Uses SubtitleEdit when available + - 'filter-subs': Uses subtitle-filter library + """ + if not self.path or not self.path.exists(): + raise ValueError("You must download the subtitle track first.") + + # Check configuration for SDH stripping method + sdh_method = config.subtitle.get("sdh_method", "auto") + + if sdh_method == "subby" and self.codec == Subtitle.Codec.SubRip: + # Use subby's SDHStripper directly on the file + fixer = CommonIssuesFixer() + stripper = SDHStripper() + srt, _ = fixer.from_file(self.path) + stripped, status = stripper.from_srt(srt) + if status is True: + stripped.save(self.path) + return + elif sdh_method == "subtitleedit" and binaries.SubtitleEdit: + # Force use of SubtitleEdit + pass # Continue to SubtitleEdit section below + elif sdh_method == "filter-subs": + # Force use of filter-subs + sub = Subtitles(self.path) + try: + sub.filter(rm_fonts=True, rm_ast=True, rm_music=True, rm_effects=True, rm_names=True, rm_author=True) + except ValueError as e: + if "too many values to unpack" in str(e): + # Retry without name removal if the error is due to multiple colons in time references + # This can happen with lines like "at 10:00 and 2:00" + sub = Subtitles(self.path) + sub.filter( + rm_fonts=True, rm_ast=True, rm_music=True, rm_effects=True, rm_names=False, rm_author=True + ) + else: + raise + sub.save() + return + elif sdh_method == "auto": + # Try subby first for SRT files, then fall back + if self.codec == Subtitle.Codec.SubRip: + try: + fixer = CommonIssuesFixer() + stripper = SDHStripper() + srt, _ = fixer.from_file(self.path) + stripped, status = stripper.from_srt(srt) + if status is True: + stripped.save(self.path) + return + except Exception: + pass # Fall through to other methods + + conversion_method = config.subtitle.get("conversion_method", "auto") + use_subtitleedit = sdh_method == "subtitleedit" or ( + sdh_method == "auto" and conversion_method in ("auto", "subtitleedit") + ) + + if binaries.SubtitleEdit and use_subtitleedit: + output_format = { + Subtitle.Codec.SubRip: "subrip", + Subtitle.Codec.SubStationAlpha: "substationalpha", + Subtitle.Codec.SubStationAlphav4: "advancedsubstationalpha", + Subtitle.Codec.TimedTextMarkupLang: "timedtext1.0", + Subtitle.Codec.WebVTT: "webvtt", + Subtitle.Codec.SAMI: "sami", + Subtitle.Codec.MicroDVD: "microdvd", + }.get(self.codec, self.codec.name.lower()) + subprocess.run( + [ + str(binaries.SubtitleEdit), + "/convert", + str(self.path), + output_format, + "/encoding:utf8", + "/overwrite", + "/RemoveTextForHI", + ], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + else: + if config.subtitle.get("convert_before_strip", True) and self.codec != Subtitle.Codec.SubRip: + self.path = self.convert(Subtitle.Codec.SubRip) + self.codec = Subtitle.Codec.SubRip + + try: + sub = Subtitles(self.path) + try: + sub.filter( + rm_fonts=True, rm_ast=True, rm_music=True, rm_effects=True, rm_names=True, rm_author=True + ) + except ValueError as e: + if "too many values to unpack" in str(e): + # Retry without name removal if the error is due to multiple colons in time references + # This can happen with lines like "at 10:00 and 2:00" + sub = Subtitles(self.path) + sub.filter( + rm_fonts=True, rm_ast=True, rm_music=True, rm_effects=True, rm_names=False, rm_author=True + ) + else: + raise + sub.save() + except (IOError, OSError) as e: + if "is not valid subtitle file" in str(e): + self.log.warning(f"Failed to strip SDH from {self.path.name}: {e}") + self.log.warning("Continuing without SDH stripping for this subtitle") + else: + raise + + def reverse_rtl(self) -> None: + """ + Reverse RTL (Right to Left) Start/End on Captions. + This can be used to fix the positioning of sentence-ending characters. + """ + if not self.path or not self.path.exists(): + raise ValueError("You must download the subtitle track first.") + + if not binaries.SubtitleEdit: + raise EnvironmentError("SubtitleEdit executable not found...") + + output_format = { + Subtitle.Codec.SubRip: "subrip", + Subtitle.Codec.SubStationAlpha: "substationalpha", + Subtitle.Codec.SubStationAlphav4: "advancedsubstationalpha", + Subtitle.Codec.TimedTextMarkupLang: "timedtext1.0", + Subtitle.Codec.WebVTT: "webvtt", + Subtitle.Codec.SAMI: "sami", + Subtitle.Codec.MicroDVD: "microdvd", + }.get(self.codec, self.codec.name.lower()) + + subprocess.run( + [ + str(binaries.SubtitleEdit), + "/convert", + str(self.path), + output_format, + "/ReverseRtlStartEnd", + "/encoding:utf8", + "/overwrite", + ], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + +__all__ = ("Subtitle",) diff --git a/reset/packages/envied/src/envied/services/ARD/__init__.py b/reset/packages/envied/src/envied/services/ARD/__init__.py new file mode 100644 index 0000000..a59587c --- /dev/null +++ b/reset/packages/envied/src/envied/services/ARD/__init__.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +from http.cookiejar import MozillaCookieJar +from typing import Any, Optional, Union +from functools import partial +from pathlib import Path +import sys +import re + +import click +import webvtt +import requests +from click import Context +from bs4 import BeautifulSoup + +from envied.core.credential import Credential +from envied.core.service import Service +from envied.core.titles import Movie, Movies, Episode, Series +from envied.core.tracks import Track, Chapter, Tracks, Video, Subtitle +from envied.core.manifests.hls import HLS +from envied.core.manifests.dash import DASH + + +class ARD(Service): + """ + Service code for ARD Mediathek (https://www.ardmediathek.de) + + \b + Version: 1.0.0 + Author: lambda + Authorization: None + Robustness: + Unencrypted: 2160p, AAC2.0 + """ + + GEOFENCE = ("de",) + TITLE_RE = r"^(https://www\.ardmediathek\.de/(?Pserie|video)/.+/)(?P[a-zA-Z0-9]{10,})(/[0-9]{1,3})?$" + EPISODE_NAME_RE = r"^(Folge [0-9]+:)?(?P[^\(]+) \(S(?P[0-9]+)/E(?P[0-9]+)\)$" + + @staticmethod + @click.command(name="ARD", short_help="https://www.ardmediathek.de", help=__doc__) + @click.argument("title", type=str) + @click.pass_context + def cli(ctx: Context, **kwargs: Any) -> ARD: + return ARD(ctx, **kwargs) + + def __init__(self, ctx: Context, title: str): + self.title = title + super().__init__(ctx) + + def authenticate(self, cookies: Optional[MozillaCookieJar] = None, credential: Optional[Credential] = None) -> None: + pass + + def get_titles(self) -> Union[Movies, Series]: + match = re.match(self.TITLE_RE, self.title) + if not match: + return + + item_id = match.group("item_id") + if match.group("item_type") == "video": + return self.load_player(item_id) + + r = self.session.get(self.config["endpoints"]["grouping"].format(item_id=item_id)) + item = r.json() + + for widget in item["widgets"]: + if widget["type"] == "gridlist" and widget.get("compilationType") == "itemsOfShow": + episodes = Series() + for teaser in widget["teasers"]: + if teaser["coreAssetType"] != "EPISODE": + continue + + if 'Hörfassung' in teaser['longTitle']: + continue + + episodes += self.load_player(teaser["id"]) + return episodes + + def get_tracks(self, title: Union[Episode, Movie]) -> Tracks: + if title.data["blockedByFsk"]: + self.log.error( + "This content is age-restricted and not currently available. " + "Try again after 10pm German time") + sys.exit(0) + + media_collection = title.data["mediaCollection"]["embedded"] + tracks = Tracks() + for stream_collection in media_collection["streams"]: + if stream_collection["kind"] != "main": + continue + + for stream in stream_collection["media"]: + if stream["mimeType"] == "application/vnd.apple.mpegurl": + tracks += Tracks(HLS.from_url(stream["url"]).to_tracks(stream["audios"][0]["languageCode"])) + break + + # Fetch tracks from HBBTV endpoint to check for potential H.265/2160p DASH + r = self.session.get(self.config["endpoints"]["hbbtv"].format(item_id=title.id)) + hbbtv = r.json() + for stream in hbbtv["video"]["streams"]: + for media in stream["media"]: + if media["mimeType"] == "application/dash+xml" and media["audios"][0]["kind"] == "standard": + tracks += Tracks(DASH.from_url(media["url"]).to_tracks(media["audios"][0]["languageCode"])) + break + + # for stream in title.data["video"]["streams"]: + # for media in stream["media"]: + # if media["mimeType"] != "video/mp4" or media["audios"][0]["kind"] != "standard": + # continue + + # tracks += Video( + # codec=Video.Codec.AVC, # Should check media["videoCodec"] + # range_=Video.Range.SDR, # Should check media["isHighDynamicRange"] + # width=media["maxHResolutionPx"], + # height=media["maxVResolutionPx"], + # url=media["url"], + # language=media["audios"][0]["languageCode"], + # fps=50, + # ) + + for sub in media_collection["subtitles"]: + for source in sub["sources"]: + if source["kind"] == "ebutt": + tracks.add(Subtitle( + codec=Subtitle.Codec.TimedTextMarkupLang, + language=sub["languageCode"], + url=source["url"] + )) + + return tracks + + def get_chapters(self, title: Union[Episode, Movie]) -> list[Chapter]: + return [] + + def load_player(self, item_id): + r = self.session.get(self.config["endpoints"]["item"].format(item_id=item_id)) + item = r.json() + + for widget in item["widgets"]: + if widget["type"] != "player_ondemand": + continue + + common_data = { + "id_": item_id, + "data": widget, + "service": self.__class__, + "language": "de", + "year": widget["broadcastedOn"][0:4], + } + + if widget["show"]["coreAssetType"] == "SINGLE" or not widget["show"].get("availableSeasons"): + return Movies([Movie( + name=widget["title"], + **common_data + )]) + else: + match = re.match(self.EPISODE_NAME_RE, widget["title"]) + if not match: + name = widget["title"] + season = 0 + episode = 0 + else: + name = match.group("name") + season = match.group("season") or 0 + episode = match.group("episode") or 0 + + return Series([Episode( + name=name, + title=widget["show"]["title"], + #season=widget["show"]["availableSeasons"][0], + season=season, + number=episode, + **common_data + )]) + diff --git a/reset/packages/envied/src/envied/services/ARD/config.yaml b/reset/packages/envied/src/envied/services/ARD/config.yaml new file mode 100644 index 0000000..e54c434 --- /dev/null +++ b/reset/packages/envied/src/envied/services/ARD/config.yaml @@ -0,0 +1,8 @@ +headers: + Accept-Language: de-DE,de;q=0.8 + User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0 + +endpoints: + item: https://api.ardmediathek.de/page-gateway/pages/ard/item/{item_id}?embedded=true&mcV6=true + grouping: https://api.ardmediathek.de/page-gateway/pages/ard/grouping/{item_id}?seasoned=true&embedded=true + hbbtv: https://tv.ardmediathek.de/dyn/get?id=video:{item_id} diff --git a/reset/packages/envied/src/envied/services/ATVP/__init__.py b/reset/packages/envied/src/envied/services/ATVP/__init__.py new file mode 100644 index 0000000..e78bde6 --- /dev/null +++ b/reset/packages/envied/src/envied/services/ATVP/__init__.py @@ -0,0 +1,354 @@ +import base64 +import json +import re +from datetime import datetime + +import click +import m3u8 +import requests + +from envied.core.downloaders import n_m3u8dl_re +from envied.core.manifests import m3u8 as m3u8_parser +from envied.core.service import Service +from envied.core.titles import Episode, Movie, Movies, Series +from envied.core.tracks import Audio, Subtitle, Tracks, Video +from envied.core.utils.collections import as_list +from pyplayready.cdm import Cdm as PlayReadyCdm + + +class ATVP(Service): + """ + Service code for Apple's TV Plus streaming service (https://tv.apple.com). + + \b + WIP: decrypt and removal of bumper/dub cards + + \b + Authorization: Cookies + Security: UHD@L1 FHD@L1 HD@L3 + """ + + ALIASES = ["ATVP", "appletvplus", "appletv+"] + TITLE_RE = ( + r"^(?:https?://tv\.apple\.com(?:/[a-z]{2})?/(?:movie|show|episode)/[a-z0-9-]+/)?(?Pumc\.cmc\.[a-z0-9]+)" # noqa: E501 + ) + + VIDEO_CODEC_MAP = {"H264": ["avc"], "H265": ["hvc", "hev", "dvh"]} + AUDIO_CODEC_MAP = {"AAC": ["HE", "stereo"], "AC3": ["ac3"], "EC3": ["ec3", "atmos"]} + + @staticmethod + @click.command(name="ATVP", short_help="https://tv.apple.com") + @click.argument("title", type=str, required=False) + @click.pass_context + def cli(ctx, **kwargs): + return ATVP(ctx, **kwargs) + + def __init__(self, ctx, title): + super().__init__(ctx) + self.title = title + self.cdm = ctx.obj.cdm + if not isinstance(self.cdm, PlayReadyCdm): + self.log.warning("PlayReady CDM not provided, exiting") + raise SystemExit(1) + self.vcodec = ctx.parent.params["vcodec"] + self.acodec = ctx.parent.params["acodec"] + self.alang = ctx.parent.params["lang"] + self.subs_only = ctx.parent.params["subs_only"] + self.quality = ctx.parent.params["quality"] + + self.extra_server_parameters = None + # initialize storefront with a default value. + self.storefront = 'us' # or any default value + + def get_titles(self): + self.configure() + r = None + for i in range(2): + try: + self.params = { + "utsk": "6e3013c6d6fae3c2::::::9318c17fb39d6b9c", + "caller": "web", + "sf": self.storefront, + "v": "46", + "pfm": "appletv", + "mfr": "Apple", + "locale": "en-US", + "l": "en", + "ctx_brand": "tvs.sbd.4000", + "count": "100", + "skip": "0", + } + r = self.session.get( + url=self.config["endpoints"]["title"].format(type={0: "shows", 1: "movies"}[i], id=self.title), + params=self.params, + ) + except requests.HTTPError as e: + if e.response.status_code != 404: + raise + else: + if r.ok: + break + if not r: + raise self.log.exit(f" - Title ID {self.title!r} could not be found.") + try: + title_information = r.json()["data"]["content"] + except json.JSONDecodeError: + raise ValueError(f"Failed to load title manifest: {r.text}") + + if title_information["type"] == "Movie": + movie = Movie( + id_=self.title, + service=self.__class__, + name=title_information["title"], + year=datetime.fromtimestamp(title_information["releaseDate"] / 1000).year, + language=title_information["originalSpokenLanguages"][0]["locale"], + data=title_information, + ) + return Movies([movie]) + else: + r = self.session.get( + url=self.config["endpoints"]["tv_episodes"].format(id=self.title), + params=self.params, + ) + try: + episodes = r.json()["data"]["episodes"] + except json.JSONDecodeError: + raise ValueError(f"Failed to load episodes list: {r.text}") + + episodes_list = [ + Episode( + id_=episode["id"], + service=self.__class__, + title=episode["showTitle"], + season=episode["seasonNumber"], + number=episode["episodeNumber"], + name=episode.get("title"), + year=datetime.fromtimestamp(title_information["releaseDate"] / 1000).year, + language=title_information["originalSpokenLanguages"][0]["locale"], + data={**episode, "originalSpokenLanguages": title_information["originalSpokenLanguages"]}, + ) + for episode in episodes + ] + return Series(episodes_list) + + def get_tracks(self, title): + # call configure() before using self.storefront + self.configure() + + self.params = { + "utsk": "6e3013c6d6fae3c2::::::9318c17fb39d6b9c", + "caller": "web", + "sf": self.storefront, + "v": "46", + "pfm": "appletv", + "mfr": "Apple", + "locale": "en-US", + "l": "en", + "ctx_brand": "tvs.sbd.4000", + "count": "100", + "skip": "0", + } + r = self.session.get( + url=self.config["endpoints"]["manifest"].format(id=title.data["id"]), + params=self.params, + ) + try: + stream_data = r.json() + except json.JSONDecodeError: + raise ValueError(f"Failed to load stream data: {r.text}") + stream_data = stream_data["data"]["content"]["playables"][0] + + if not stream_data["isEntitledToPlay"]: + raise self.log.exit(" - User is not entitled to play this title") + + self.extra_server_parameters = stream_data["assets"]["fpsKeyServerQueryParameters"] + r = requests.get( + url=stream_data["assets"]["hlsUrl"], + headers={"User-Agent": "AppleTV6,2/11.1"}, + ) + res = r.text + + master = m3u8.loads(res, r.url) + tracks = m3u8_parser.parse( + master=master, + language=title.data["originalSpokenLanguages"][0]["locale"] or "en", + session=self.session, + ) + + # Set track properties based on type + for track in tracks: + if isinstance(track, Video): + # Convert codec string to proper Video.Codec enum if needed + if isinstance(track.codec, str): + codec_str = track.codec.lower() + if codec_str in ["avc", "h264", "h.264"]: + track.codec = Video.Codec.AVC + elif codec_str in ["hvc", "hev", "hevc", "h265", "h.265", "dvh"]: + track.codec = Video.Codec.HEVC + else: + print(f"Unknown video codec '{track.codec}', keeping as string") + + # Set pr_pssh for PlayReady license requests + if track.drm: + for drm in track.drm: + if hasattr(drm, 'data') and 'pssh_b64' in drm.data: + track.pr_pssh = drm.data['pssh_b64'] + elif isinstance(track, Audio): + # Extract bitrate from URL + bitrate = re.search(r"&g=(\d+?)&", track.url) + if not bitrate: + bitrate = re.search(r"_gr(\d+)_", track.url) # alternative pattern + if bitrate: + track.bitrate = int(bitrate.group(1)[-3::]) * 1000 # e.g. 128->128,000, 2448->448,000 + else: + raise ValueError(f"Unable to get a bitrate value for Track {track.id}") + codec_str = track.codec.replace("_vod", "") if track.codec else "" + if codec_str == "DD+": + track.codec = Audio.Codec.EC3 + elif codec_str == "DD": + track.codec = Audio.Codec.AC3 + elif codec_str in ["HE", "stereo", "AAC"]: + track.codec = Audio.Codec.AAC + elif codec_str == "atmos": + track.codec = Audio.Codec.EC3 + else: + if not hasattr(track.codec, "value"): + print(f"Unknown audio codec '{codec_str}', defaulting to AAC") + track.codec = Audio.Codec.AAC + + # Set pr_pssh for PlayReady license requests + if track.drm: + for drm in track.drm: + if hasattr(drm, 'data') and 'pssh_b64' in drm.data: + track.pr_pssh = drm.data['pssh_b64'] + elif isinstance(track, Subtitle): + codec_str = track.codec if track.codec else "" + if codec_str.lower() in ["vtt", "webvtt"]: + track.codec = Subtitle.Codec.WebVTT + elif codec_str.lower() in ["srt", "subrip"]: + track.codec = Subtitle.Codec.SubRip + elif codec_str.lower() in ["ttml", "dfxp"]: + track.codec = Subtitle.Codec.TimedTextMarkupLang + elif codec_str.lower() in ["ass", "ssa"]: + track.codec = Subtitle.Codec.SubStationAlphav4 + else: + if not hasattr(track.codec, "value"): + print(f"Unknown subtitle codec '{codec_str}', defaulting to WebVTT") + track.codec = Subtitle.Codec.WebVTT + + # Set pr_pssh for PlayReady license requests + if track.drm: + for drm in track.drm: + if hasattr(drm, 'data') and 'pssh_b64' in drm.data: + track.pr_pssh = drm.data['pssh_b64'] + + # Try to filter by CDN, but fallback to all tracks if filtering fails + try: + filtered_tracks = [ + x + for x in tracks + if any( + param.startswith("cdn=vod-ap") or param == "cdn=ap" + for param in as_list(x.url)[0].split("?")[1].split("&") + ) + ] + + for track in tracks: + if track not in tracks.attachments: + track.downloader = n_m3u8dl_re + if isinstance(track, (Video, Audio)): + track.needs_repack = True + + if filtered_tracks: + return Tracks(filtered_tracks) + else: + return Tracks(tracks) + + except Exception: + return Tracks(tracks) + + def get_chapters(self, title): + return [] + + def certificate(self, **_): + return None # will use common privacy cert + + def get_pssh(self, track) -> None: + res = self.session.get(as_list(track.url)[0]) + playlist = m3u8.loads(res.text, uri=res.url) + keys = list(filter(None, (playlist.session_keys or []) + (playlist.keys or []))) + for key in keys: + if key.keyformat and "playready" in key.keyformat.lower(): + track.pr_pssh = key.uri.split(",")[-1] + return + + def get_playready_license(self, *, challenge: bytes, title, track) -> str: + if isinstance(challenge, str): + challenge = challenge.encode() + + self.get_pssh(track) + + res = self.session.post( + url=self.config["endpoints"]["license"], + json={ + "streaming-request": { + "version": 1, + "streaming-keys": [ + { + "challenge": base64.b64encode(challenge).decode("utf-8"), + "key-system": "com.microsoft.playready", + "uri": f"data:text/plain;charset=UTF-16;base64,{track.pr_pssh}", + "id": 0, + "lease-action": "start", + "adamId": self.extra_server_parameters["adamId"], + "isExternal": True, + "svcId": self.extra_server_parameters["svcId"], + }, + ], + }, + }, + ).json() + return res["streaming-response"]["streaming-keys"][0]["license"] + + # Service specific functions + + def configure(self): + cc = self.session.cookies.get_dict()["itua"] + r = self.session.get( + "https://gist.githubusercontent.com/BrychanOdlum/2208578ba151d1d7c4edeeda15b4e9b1/raw/8f01e4a4cb02cf97a48aba4665286b0e8de14b8e/storefrontmappings.json" + ).json() + for g in r: + if g["code"] == cc: + self.storefront = g["storefrontId"] + + environment = self.get_environment_config() + if not environment: + raise ValueError("Failed to get AppleTV+ WEB TV App Environment Configuration...") + self.session.headers.update( + { + "User-Agent": self.config["user_agent"], + "Authorization": f"Bearer {environment['developerToken']}", + "media-user-token": self.session.cookies.get_dict()["media-user-token"], + "x-apple-music-user-token": self.session.cookies.get_dict()["media-user-token"], + } + ) + + def get_environment_config(self): + """Loads environment config data from WEB App's serialized server data.""" + res = self.session.get("https://tv.apple.com").text + + script_match = re.search( + r']*id=["\']serialized-server-data["\'][^>]*>(.*?)', + res, + re.DOTALL, + ) + if script_match: + try: + script_content = script_match.group(1).strip() + data = json.loads(script_content) + if data and len(data) > 0 and "data" in data[0] and "configureParams" in data[0]["data"]: + return data[0]["data"]["configureParams"] + except (json.JSONDecodeError, KeyError, IndexError) as e: + print(f"Failed to parse serialized server data: {e}") + + return None diff --git a/reset/packages/envied/src/envied/services/ATVP/config.yaml b/reset/packages/envied/src/envied/services/ATVP/config.yaml new file mode 100644 index 0000000..cc4bdd9 --- /dev/null +++ b/reset/packages/envied/src/envied/services/ATVP/config.yaml @@ -0,0 +1,38 @@ +user_agent: 'ATVE/6.2.0 Android/10 build/6A226 maker/Google model/Chromecast FW/QTS2.200918.0337115981' +storefront_mapping_url: 'https://gist.githubusercontent.com/BrychanOdlum/2208578ba151d1d7c4edeeda15b4e9b1/raw/8f01e4a4cb02cf97a48aba4665286b0e8de14b8e/storefrontmappings.json' + +endpoints: + title: 'https://tv.apple.com/api/uts/v3/{type}/{id}' + tv_episodes: 'https://tv.apple.com/api/uts/v2/view/show/{id}/episodes' + manifest: 'https://tv.apple.com/api/uts/v2/view/product/{id}/personalized' + license: 'https://play.itunes.apple.com/WebObjects/MZPlay.woa/wa/fpsRequest' + environment: 'https://tv.apple.com' + +params: + utsk: '6e3013c6d6fae3c2::::::9318c17fb39d6b9c' + caller: 'web' + v: '46' + pfm: 'appletv' + mfr: 'Apple' + locale: 'en-US' + l: 'en' + ctx_brand: 'tvs.sbd.4000' + count: '100' + skip: '0' + +headers: + Accept: 'application/json' + Accept-Language: 'en-US,en;q=0.9' + Connection: 'keep-alive' + DNT: '1' + Origin: 'https://tv.apple.com' + Referer: 'https://tv.apple.com/' + Sec-Fetch-Dest: 'empty' + Sec-Fetch-Mode: 'cors' + Sec-Fetch-Site: 'same-origin' + +quality_map: + SD: 480 + HD720: 720 + HD: 1080 + UHD: 2160 \ No newline at end of file diff --git a/reset/packages/envied/src/envied/services/AUBC/__init__.py b/reset/packages/envied/src/envied/services/AUBC/__init__.py new file mode 100644 index 0000000..8f5b44f --- /dev/null +++ b/reset/packages/envied/src/envied/services/AUBC/__init__.py @@ -0,0 +1,260 @@ +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Generator +from typing import Any, Optional, Union +from urllib.parse import urljoin + +import click +from click import Context +from requests import Request +from envied.core.constants import AnyTrack +from envied.core.manifests.dash import DASH +from envied.core.search_result import SearchResult +from envied.core.service import Service +from envied.core.titles import Episode, Movie, Movies, Series +from envied.core.tracks import Chapter, Chapters, Subtitle, Tracks + + +class AUBC(Service): + """ + \b + Service code for ABC iView streaming service (https://iview.abc.net.au/). + + \b + Version: 1.0.5 + Author: stabbedbybrick + Authorization: None + Robustness: + L3: 1080p, AAC2.0 + + \b + Tips: + - Input should be complete URL: + SHOW: https://iview.abc.net.au/show/return-to-paradise + EPISODE: https://iview.abc.net.au/video/DR2314H001S00 + MOVIE: https://iview.abc.net.au/show/way-back / https://iview.abc.net.au/show/way-back/video/ZW3981A001S00 + + """ + + GEOFENCE = ("au",) + ALIASES = ("iview", "abciview", "iv",) + + @staticmethod + @click.command(name="AUBC", short_help="https://iview.abc.net.au/", help=__doc__) + @click.argument("title", type=str) + @click.pass_context + def cli(ctx: Context, **kwargs: Any) -> AUBC: + return AUBC(ctx, **kwargs) + + def __init__(self, ctx: Context, title: str): + self.title = title + super().__init__(ctx) + + self.session.headers.update(self.config["headers"]) + + def search(self) -> Generator[SearchResult, None, None]: + url = ( + "https://y63q32nvdl-1.algolianet.com/1/indexes/*/queries?x-algolia-agent=Algolia" + "%20for%20JavaScript%20(4.9.1)%3B%20Browser%20(lite)%3B%20react%20(17.0.2)%3B%20" + "react-instantsearch%20(6.30.2)%3B%20JS%20Helper%20(3.10.0)&x-" + "algolia-api-key=bcdf11ba901b780dc3c0a3ca677fbefc&x-algolia-application-id=Y63Q32NVDL" + ) + payload = { + "requests": [ + { + "indexName": "ABC_production_iview_web", + "params": f"query={self.title}&tagFilters=&userToken=anonymous-74be3cf1-1dc7-4fa1-9cff-19592162db1c", + } + ], + } + + results = self._request("POST", url, payload=payload)["results"] + hits = [x for x in results[0]["hits"] if x["docType"] == "Program"] + + for result in hits: + yield SearchResult( + id_="https://iview.abc.net.au/show/{}".format(result.get("slug")), + title=result.get("title"), + description=result.get("synopsis"), + label=result.get("subType"), + url="https://iview.abc.net.au/show/{}".format(result.get("slug")), + ) + + def get_titles(self) -> Union[Movies, Series]: + title_re = r"^(?:https?://(?:www.)?iview.abc.net.au/(?Pshow|video)/)?(?P[a-zA-Z0-9_-]+)" + try: + kind, title_id = (re.match(title_re, self.title).group(i) for i in ("type", "id")) + except Exception: + raise ValueError("- Could not parse ID from title") + + if kind == "show": + data = self._request("GET", "/v3/show/{}".format(title_id)) + label = data.get("type") + + if label.lower() in ("series", "program"): + episodes = self._series(title_id) + return Series(episodes) + + elif label.lower() in ("feature", "movie"): + movie = self._movie(data) + return Movies(movie) + + elif kind == "video": + episode = self._episode(title_id) + return Series([episode]) + + def get_tracks(self, title: Union[Movie, Episode]) -> Tracks: + video = self._request("GET", "/v3/video/{}".format(title.id)) + if not video.get("playable"): + raise ConnectionError(video.get("unavailableMessage")) + + playlist = video.get("_embedded", {}).get("playlist", {}) + if not playlist: + raise ConnectionError("Could not find a playlist for this title") + + streams = next(x["streams"]["mpegdash"] for x in playlist if x["type"] == "program") + captions = next((x.get("captions") for x in playlist if x["type"] == "program"), None) + title.data["protected"] = streams.get("protected", False) + + if hd := streams.get("720"): + streams["1080"] = hd.replace("720.mpd", "1080.mpd") + + manifest = next( + (url for key in ["1080", "720", "sd", "sd-low"] if key in streams + for url in [streams[key]] + if self.session.head(url).status_code == 200), + None + ) + if not manifest: + raise ValueError("Could not find a manifest for this title") + + tracks = DASH.from_url(manifest, self.session).to_tracks(title.language) + + for track in tracks.audio: + role = track.data["dash"]["adaptation_set"].find("Role") + if role is not None and role.get("value") in ["description", "alternative", "alternate"]: + track.descriptive = True + + if captions: + subtitles = captions.get("src-vtt") + tracks.add( + Subtitle( + id_=hashlib.md5(subtitles.encode()).hexdigest()[0:6], + url=subtitles, + codec=Subtitle.Codec.from_mime(subtitles[-3:]), + language=title.language, + forced=False, + ) + ) + + return tracks + + def get_chapters(self, title: Union[Movie, Episode]) -> Chapters: + if not title.data.get("cuePoints"): + return Chapters() + + credits = next((x.get("start") for x in title.data["cuePoints"] if x["type"] == "end-credits"), None) + if credits: + return Chapters([Chapter(name="Credits", timestamp=credits * 1000)]) + + return Chapters() + + def get_widevine_service_certificate(self, **_: Any) -> str: + return None + + def get_widevine_license(self, *, challenge: bytes, title: Union[Movies, Series], track: AnyTrack) -> Optional[Union[bytes, str]]: + if not title.data.get("protected"): + return None + + customdata = self._license(title.id) + headers = {"customdata": customdata} + + r = self.session.post(self.config["endpoints"]["license"], headers=headers, data=challenge) + r.raise_for_status() + return r.content + + # Service specific + + def _series(self, title: str) -> Episode: + data = self._request("GET", "/v3/series/{}".format(title)) + content = data if isinstance(data, list) else [data] + + # Remove duplicate season entries + seen = set() + seasons = [x for x in content if (key := x.get("id")) not in seen and not seen.add(key)] + + episodes = [ + self.create_episode(episode) + for season in seasons + for episode in season.get("_embedded", {}).get("videoEpisodes", {}).get("items", []) + ] + + return Series(episodes) + + def _movie(self, data: dict) -> Movie: + return [ + Movie( + id_=data["_embedded"]["highlightVideo"]["id"], + service=self.__class__, + name=data.get("title"), + year=data.get("productionYear"), + data=data, + language=data.get("analytics", {}).get("dataLayer", {}).get("d_language", "en"), + ) + ] + + def _episode(self, video_id: str) -> Episode: + data = self._request("GET", "/v3/video/{}".format(video_id)) + return self.create_episode(data) + + def _license(self, video_id: str): + token = self._request("POST", "/v3/token/jwt", data={"clientId": self.config["client"]})["token"] + response = self._request("GET", "/v3/token/drm/{}".format(video_id), headers={"bearer": token}) + + return response["license"] + + def create_episode(self, episode: dict) -> Episode: + title = episode["showTitle"] + episode_id = episode.get("id", "") + series_id = episode.get("analytics", {}).get("dataLayer", {}).get("d_series_id", "") + episode_name = episode.get("analytics", {}).get("dataLayer", {}).get("d_episode_name", "") + episode_number = re.search(r"Episode (\d+)", episode.get("displaySubtitle", "")) + name = re.search(r"S\d+\sEpisode\s\d+\s(.*)", episode_name) + language = episode.get("analytics", {}).get("dataLayer", {}).get("d_language", "en") + + season = int(series_id.split("-")[-1]) if series_id else 0 + number = int(episode_number.group(1)) if episode_number else 0 + + if not number: + if match := re.search(r"[A-Z](\d{3})(?=S\d{2})", episode_id): + number = int(match.group(1)) + + return Episode( + id_=episode["id"], + service=self.__class__, + title=title, + season=season, + number=number, + name=name.group(1) if name else episode_name, + data=episode, + language=language, + ) + + def _request(self, method: str, api: str, **kwargs: Any) -> Any[dict | str]: + url = urljoin(self.config["endpoints"]["base_url"], api) + + prep = self.session.prepare_request(Request(method, url, **kwargs)) + + response = self.session.send(prep) + if response.status_code != 200: + raise ConnectionError(f"{response.text}") + + try: + return json.loads(response.content) + + except json.JSONDecodeError as e: + raise ValueError(f"Failed to parse JSON: {response.text}") from e + diff --git a/reset/packages/envied/src/envied/services/AUBC/config.yaml b/reset/packages/envied/src/envied/services/AUBC/config.yaml new file mode 100644 index 0000000..604b739 --- /dev/null +++ b/reset/packages/envied/src/envied/services/AUBC/config.yaml @@ -0,0 +1,9 @@ +headers: + User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0 + accept-language: en-US,en;q=0.8 + +endpoints: + base_url: https://api.iview.abc.net.au + license: https://wv-keyos.licensekeyserver.com/ + +client: "1d4b5cba-42d2-403e-80e7-34565cdf772d" \ No newline at end of file diff --git a/reset/packages/envied/src/envied/services/BLAZ/__init__.py b/reset/packages/envied/src/envied/services/BLAZ/__init__.py new file mode 100644 index 0000000..5aa2dbd --- /dev/null +++ b/reset/packages/envied/src/envied/services/BLAZ/__init__.py @@ -0,0 +1,387 @@ +from __future__ import annotations + +import html +import json +import re +from collections.abc import Generator +from typing import Any +from urllib.parse import urljoin, urlparse + +import click +from bs4 import BeautifulSoup +from click import Context +from requests import Request +from envied.core.manifests import HLS +from envied.core.search_result import SearchResult +from envied.core.service import Service +from envied.core.titles import Episode, Series, Title_T +from envied.core.tracks import Chapters, Tracks + + +class BLAZ(Service): + """ + \b + Service code for BLAZE TV's free streaming service (https://watch.blaze.tv/). + + \b + Version: 1.0.0 + Author: billybanana + Authorization: None + Geofence: GB/UK + Robustness: + HLS: 720p, AAC2.0 + + \b + Tips: + - Search is supported: + envied.dl BLAZ "outback wreckers" + - Show URLs, public Blaze URLs, slugs, and episode URLs are supported: + https://www.blaze.tv/outback-wreckers + https://watch.blaze.tv/shows/4090b77a-538c-11f1-b4ab-021644e4b9e7/outback-wreckers + outback-wreckers + https://watch.blaze.tv/watch/vod/53361682/island-dreams-or-french-connection + + """ + + ALIASES = ("blaze",) + GEOFENCE = ("GB", "UK") + TITLE_RE = ( + r"^(?:https?://)?" + r"(?:(?:www\.)?blaze\.tv/|watch\.blaze\.tv/shows/[a-f0-9-]+/)?" + r"(?P[a-z0-9-]+)$" + ) + EPISODE_RE = r"^(?:https?://watch\.blaze\.tv)?/watch/vod/(?P\d+)(?:/(?P[a-z0-9-]+))?" + + @staticmethod + @click.command(name="BLAZ", short_help="https://watch.blaze.tv/", help=__doc__) + @click.argument("title", type=str) + @click.pass_context + def cli(ctx: Context, **kwargs: Any) -> BLAZ: + return BLAZ(ctx, **kwargs) + + def __init__(self, ctx: Context, title: str): + self.title = title + super().__init__(ctx) + + self.session.headers.update(self.config["headers"]) + + def search(self) -> Generator[SearchResult, None, None]: + query = self._clean_text(self.title).lower() + + for item in self._catalogue(): + haystack = f"{item['title']} {item['slug']}".lower() + if query and query not in haystack: + continue + + yield SearchResult( + id_=item["url"], + title=item["title"], + description=item.get("description"), + label="SERIES", + url=item["url"], + ) + + def get_titles(self) -> Series: + episode_match = re.match(self.EPISODE_RE, self.title) + if episode_match: + return Series(self._episode(episode_match.group("uvid"), self.title)) + + show_url = self._resolve_show_url(self.title) + html_text = self._request("GET", show_url) + return Series(self._series(show_url, html_text)) + + def get_tracks(self, title: Title_T) -> Tracks: + token = self._request("GET", self.config["endpoints"]["token"].format(uvid=title.id)) + if not token.get("token") or not token.get("expiry"): + raise ValueError(f"Failed to retrieve player token: {token}") + + data = title.data or {} + content_type = data.get("type", "replay").lower() + key = data.get("key") or self.config["playback"]["key"] + + stream = self._request( + "POST", + self.config["endpoints"]["streams"].format(type=content_type, uvid=title.id), + params={ + "key": key, + "platform": self.config["playback"]["platform"], + }, + headers={ + "Accept": "application/json", + "Origin": self.config["endpoints"]["base"], + "Referer": data.get("url") or self.config["endpoints"]["base"], + "Token": token["token"], + "Token-Expiry": str(token["expiry"]), + "Userid": "123456", + "Uvid": str(title.id), + }, + ) + + response = stream.get("response", {}) + if response.get("error"): + raise ValueError(f"Streams API error: {response['error']}") + manifest = response.get("stream") + if not manifest: + raise ValueError(f"Streams API did not return a manifest: {stream}") + + return HLS.from_url(manifest, self.session).to_tracks(language=title.language) + + def get_chapters(self, title: Title_T) -> Chapters: + return Chapters() + + def get_widevine_service_certificate(self, **_: Any) -> str | None: + return None + + def get_widevine_license(self, **_: Any) -> bytes | str | None: + return None + + # Service specific + + def _catalogue(self) -> list[dict[str, str]]: + html_text = self._request("GET", self.config["endpoints"]["series"]) + soup = BeautifulSoup(html_text, "html.parser") + by_url: dict[str, dict[str, str]] = {} + + for a in soup.find_all("a", href=True): + href = urljoin(self.config["endpoints"]["base"], a["href"]) + if "/shows/" not in urlparse(href).path: + continue + by_url[href] = { + "url": href, + "slug": self._slug_from_show_url(href), + "title": self._title_from_node(a) or self._title_from_slug(self._slug_from_show_url(href)), + "description": None, + } + + for match in re.finditer(r"https://watch\.blaze\.tv/shows/[^'\"<>\s]+", html_text): + href = html.unescape(match.group(0)) + if href in by_url: + continue + + snippet = html_text[max(0, match.start() - 1200): match.end() + 1200] + title = self._title_from_snippet(snippet) or self._title_from_slug(self._slug_from_show_url(href)) + by_url[href] = { + "url": href, + "slug": self._slug_from_show_url(href), + "title": title, + "description": None, + } + + return sorted(by_url.values(), key=lambda item: item["title"].lower()) + + def _resolve_show_url(self, title: str) -> str: + if re.match(r"^https?://watch\.blaze\.tv/shows/", title): + return title + + match = re.match(self.TITLE_RE, title) + if not match: + raise ValueError(f"Could not parse BLAZE title input: {title}") + + slug = match.group("slug") + for item in self._catalogue(): + if item["slug"] == slug: + return item["url"] + + raise ValueError(f"Could not resolve BLAZE show slug: {slug}") + + def _series(self, show_url: str, html_text: str) -> list[Episode]: + soup = BeautifulSoup(html_text, "html.parser") + show_title = self._show_title(soup) + + entries: list[dict[str, Any]] = [] + seen: set[str] = set() + for a in soup.find_all("a", href=True): + href = urljoin(show_url, a["href"]) + match = re.search(r"/watch/vod/(\d+)/([^/?#]+)", href) + if not match: + continue + + uvid, slug = match.groups() + if uvid in seen: + continue + seen.add(uvid) + + player = self._player_data(html_text, uvid) + entries.append( + { + "uvid": uvid, + "slug": slug, + "url": href, + "name": self._episode_title(a, slug), + "description": self._episode_description(a), + "player": player, + } + ) + + if not entries: + raise ValueError(f"No BLAZE VOD episodes found for {show_url}") + + total = len(entries) + episodes = [] + for index, item in enumerate(entries): + number = total - index + data = { + "url": item["url"], + "slug": item["slug"], + "type": item["player"].get("type", "replay"), + "key": item["player"].get("key") or self.config["playback"]["key"], + "poster": item["player"].get("poster"), + } + episodes.append( + Episode( + id_=item["uvid"], + service=self.__class__, + title=show_title, + season=1, + number=number, + name=item["name"], + language="en", + data=data, + description=item["description"], + ) + ) + + return episodes + + def _episode(self, uvid: str, episode_url: str) -> list[Episode]: + if episode_url.startswith("/"): + episode_url = urljoin(self.config["endpoints"]["base"], episode_url) + elif not episode_url.startswith("http"): + episode_url = f"{self.config['endpoints']['base']}/watch/vod/{uvid}/{episode_url}" + + html_text = self._request("GET", episode_url) + soup = BeautifulSoup(html_text, "html.parser") + player = self._player_data(html_text, uvid) + + return [ + Episode( + id_=uvid, + service=self.__class__, + title=self._show_title(soup), + season=1, + number=0, + name=self._page_title(soup) or self._title_from_slug(urlparse(episode_url).path.rstrip("/").split("/")[-1]), + language="en", + data={ + "url": episode_url, + "type": player.get("type", "replay"), + "key": player.get("key") or self.config["playback"]["key"], + "poster": player.get("poster"), + }, + ) + ] + + def _player_data(self, html_text: str, uvid: str) -> dict[str, str]: + soup = BeautifulSoup(html_text, "html.parser") + node = soup.find(attrs={"data-uvid": uvid}) + data: dict[str, str] = {} + if node: + for key, value in node.attrs.items(): + if key.startswith("data-"): + data[key[5:].replace("-", "_")] = str(value) + + data.setdefault("uvid", uvid) + data.setdefault("type", "replay") + data.setdefault("key", self.config["playback"]["key"]) + return data + + def _request(self, method: str, endpoint: str, **kwargs: Any) -> Any: + prep = self.session.prepare_request(Request(method, endpoint, **kwargs)) + response = self.session.send(prep) + + if response.status_code not in (200, 201): + raise ConnectionError(response.text) + + content_type = response.headers.get("content-type", "") + if "json" in content_type or response.text.strip().startswith(("{", "[")): + try: + return json.loads(response.content) + except json.JSONDecodeError as e: + raise ValueError(f"Failed to parse JSON: {response.text}") from e + + return response.text + + @staticmethod + def _clean_text(value: str | None) -> str: + return " ".join(html.unescape(value or "").split()) + + @classmethod + def _title_from_slug(cls, slug: str) -> str: + return cls._clean_text(slug.replace("-", " ").title()) + + @staticmethod + def _slug_from_show_url(url: str) -> str: + return urlparse(url).path.rstrip("/").split("/")[-1] + + @classmethod + def _title_from_node(cls, node: Any) -> str: + text = cls._clean_text(node.get_text(" ", strip=True)) + text = re.sub(r"\bPlay\b", "", text, flags=re.I) + text = re.sub(r"\b\d+\s+Episodes?\b", "", text, flags=re.I) + return cls._clean_text(text) + + @classmethod + def _title_from_snippet(cls, snippet: str) -> str | None: + alt = re.search(r'alt="([^"]+)"', snippet) + if alt: + title = cls._clean_text(alt.group(1)) + if title and title.lower() not in ("image", "menu"): + return title + + overlay = re.search(r'overlay-title[^>]*>\s*([^<]+)', snippet, re.I) + if overlay: + return cls._clean_text(overlay.group(1)) + + return None + + @classmethod + def _show_title(cls, soup: BeautifulSoup) -> str: + h1 = soup.find("h1") + if h1: + title = cls._clean_text(h1.get_text(" ", strip=True)) + if title and title.lower() not in ("series", "watch"): + return title + + og_title = soup.find("meta", property="og:title") + if og_title and og_title.get("content"): + title = re.sub(r"^BLAZE\s*::\s*", "", cls._clean_text(og_title["content"])).strip() + if title and title.lower() not in ("series", "watch"): + return title + + return "BLAZE" + + @classmethod + def _page_title(cls, soup: BeautifulSoup) -> str | None: + for selector in ({"property": "og:title"}, {"name": "twitter:title"}): + meta = soup.find("meta", attrs=selector) + if meta and meta.get("content"): + title = re.sub(r"^BLAZE\s*::\s*", "", cls._clean_text(meta["content"])).strip() + if title and title.lower() not in ("series", "watch"): + return title + + title_tag = soup.find("title") + if title_tag: + title = re.sub(r"^BLAZE\s*::\s*", "", cls._clean_text(title_tag.get_text())).strip() + if title and title.lower() not in ("series", "watch"): + return title + + return None + + @classmethod + def _episode_title(cls, link: Any, slug: str) -> str: + title = cls._title_from_node(link) + return title or cls._title_from_slug(slug) + + @classmethod + def _episode_description(cls, link: Any) -> str | None: + node = link + for _ in range(6): + if not node: + return None + text = cls._clean_text(node.get_text(" ", strip=True)) + if len(text) > 80: + title = cls._title_from_node(link) + text = cls._clean_text(text.replace(title, "", 1)) + return text or None + node = node.parent + return None diff --git a/reset/packages/envied/src/envied/services/BLAZ/config.yaml b/reset/packages/envied/src/envied/services/BLAZ/config.yaml new file mode 100644 index 0000000..501ecaf --- /dev/null +++ b/reset/packages/envied/src/envied/services/BLAZ/config.yaml @@ -0,0 +1,13 @@ +headers: + User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36" + Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + +endpoints: + base: "https://watch.blaze.tv" + series: "https://watch.blaze.tv/series" + token: "https://watch.blaze.tv/api/player/token/{uvid}" + streams: "https://v2-streams-elb.simplestreamcdn.com/api/{type}/stream/{uvid}" + +playback: + key: "6Pr3Uj2Fo3Ix7Rb9Ze9Ro4Ez2Eq7Sy" + platform: "chrome" diff --git a/reset/packages/envied/src/envied/services/CBC/__init__.py b/reset/packages/envied/src/envied/services/CBC/__init__.py new file mode 100644 index 0000000..a144e25 --- /dev/null +++ b/reset/packages/envied/src/envied/services/CBC/__init__.py @@ -0,0 +1,328 @@ +from __future__ import annotations + +import json +import re +import sys +from collections.abc import Generator +from http.cookiejar import CookieJar +from typing import Any, Optional, Union +from urllib.parse import urljoin + +import click +from click import Context +from requests import Request +from envied.core.constants import AnyTrack +from envied.core.credential import Credential +from envied.core.manifests import DASH, HLS +from envied.core.search_result import SearchResult +from envied.core.service import Service +from envied.core.titles import Episode, Movie, Movies, Series +from envied.core.tracks import Chapter, Chapters, Tracks + + +class CBC(Service): + """ + \b + Service code for CBC Gem streaming service (https://gem.cbc.ca/). + + \b + Version: 1.0.1 + Author: stabbedbybrick + Authorization: Credentials + Robustness: + AES-128: 1080p, DDP5.1 + Widevine L3: 720p, DDP5.1 + + \b + Tips: + - Input can be complete title URL or just the slug: + SHOW: https://gem.cbc.ca/murdoch-mysteries OR murdoch-mysteries + MOVIE: https://gem.cbc.ca/the-babadook OR the-babadook + + \b + Notes: + - DRM encrypted titles max out at 720p. + - CCExtrator v0.94 will likely fail to extract subtitles. It's recommended to downgrade to v0.93. + - Some audio tracks contain invalid data, causing warning messages from mkvmerge during muxing + These can be ignored. + + """ + + GEOFENCE = ("ca",) + ALIASES = ("gem", "cbcgem",) + + @staticmethod + @click.command(name="CBC", short_help="https://gem.cbc.ca/", help=__doc__) + @click.argument("title", type=str) + @click.pass_context + def cli(ctx: Context, **kwargs: Any) -> CBC: + return CBC(ctx, **kwargs) + + def __init__(self, ctx: Context, title: str): + self.title: str = title + super().__init__(ctx) + + self.base_url: str = self.config["endpoints"]["base_url"] + + def search(self) -> Generator[SearchResult, None, None]: + params = { + "device": "web", + "pageNumber": "1", + "pageSize": "20", + "term": self.title, + } + response: dict = self._request("GET", "/ott/catalog/v1/gem/search", params=params) + + for result in response.get("result", []): + yield SearchResult( + id_="https://gem.cbc.ca/{}".format(result.get("url")), + title=result.get("title"), + description=result.get("synopsis"), + label=result.get("type"), + url="https://gem.cbc.ca/{}".format(result.get("url")), + ) + + def authenticate(self, cookies: Optional[CookieJar] = None, credential: Optional[Credential] = None) -> None: + super().authenticate(cookies, credential) + if not credential: + raise EnvironmentError("Service requires Credentials for Authentication.") + + tokens: Optional[Any] = self.cache.get(f"tokens_{credential.sha1}") + + """ + All grant types for future reference: + PASSWORD("password"), + ACCESS_TOKEN("access_token"), + REFRESH_TOKEN("refresh_token"), + CLIENT_CREDENTIALS("client_credentials"), + AUTHORIZATION_CODE("authorization_code"), + CODE("code"); + """ + + if tokens and not tokens.expired: + # cached + self.log.info(" + Using cached tokens") + auth_token: str = tokens.data["access_token"] + + elif tokens and tokens.expired: + # expired, refresh + self.log.info("Refreshing cached tokens...") + auth_url, scopes = self.settings() + params = { + "client_id": self.config["client"]["id"], + "grant_type": "refresh_token", + "refresh_token": tokens.data["refresh_token"], + "scope": scopes, + } + + access: dict = self._request("POST", auth_url, params=params) + + # Shorten expiration by one hour to account for clock skew + tokens.set(access, expiration=int(access["expires_in"]) - 3600) + auth_token: str = access["access_token"] + + else: + # new + self.log.info("Requesting new tokens...") + auth_url, scopes = self.settings() + params = { + "client_id": self.config["client"]["id"], + "grant_type": "password", + "username": credential.username, + "password": credential.password, + "scope": scopes, + } + + access: dict = self._request("POST", auth_url, params=params) + + # Shorten expiration by one hour to account for clock skew + tokens.set(access, expiration=int(access["expires_in"]) - 3600) + auth_token: str = access["access_token"] + + claims_token: str = self.claims_token(auth_token) + self.session.headers.update({"x-claims-token": claims_token}) + + def get_titles(self) -> Union[Movies, Series]: + title_re: str = r"^(?:https?://(?:www.)?gem.cbc.ca/)?(?P[a-zA-Z0-9_-]+)" + try: + title_id: str = re.match(title_re, self.title).group("id") + except Exception: + raise ValueError("- Could not parse ID from title") + + params = {"device": "web"} + data: dict = self._request("GET", "/ott/catalog/v2/gem/show/{}".format(title_id), params=params) + label: str = data.get("contentType", "").lower() + + if label in ("film", "movie", "standalone"): + movies: list[Movie] = self._movie(data) + return Movies(movies) + + else: + episodes: list[Episode] = self._show(data) + return Series(episodes) + + def get_tracks(self, title: Union[Movie, Episode]) -> Tracks: + index: dict = self._request( + "GET", "/media/meta/v1/index.ashx", params={"appCode": "gem", "idMedia": title.id, "output": "jsonObject"} + ) + + title.data["extra"] = { + "chapters": index["Metas"].get("Chapitres"), + "credits": index["Metas"].get("CreditStartTime"), + } + + self.drm: bool = index["Metas"].get("isDrmActive") == "true" + if self.drm: + tech: str = next(tech["name"] for tech in index["availableTechs"] if "widevine" in tech["drm"]) + else: + tech: str = next(tech["name"] for tech in index["availableTechs"] if not tech["drm"]) + + response: dict = self._request( + "GET", self.config["endpoints"]["validation"].format("android", title.id, "smart-tv", tech) + ) + + manifest = response.get("url") + self.license = next((x["value"] for x in response["params"] if "widevineLicenseUrl" in x["name"]), None) + self.token = next((x["value"] for x in response["params"] if "widevineAuthToken" in x["name"]), None) + + stream_type: Union[HLS, DASH] = HLS if tech == "hls" else DASH + tracks: Tracks = stream_type.from_url(manifest, self.session).to_tracks(language=title.language) + + if stream_type == DASH: + for track in tracks.audio: + label = track.data["dash"]["adaptation_set"].find("Label") + if label is not None and "descriptive" in label.text.lower(): + track.descriptive = True + + for track in tracks: + track.language = title.language + + return tracks + + def get_chapters(self, title: Union[Movie, Episode]) -> Chapters: + extra: dict = title.data["extra"] + + chapters = [] + if extra.get("chapters"): + chapters = [Chapter(timestamp=x) for x in set(extra["chapters"].split(","))] + + if extra.get("credits"): + chapters.append(Chapter(name="Credits", timestamp=float(extra["credits"]))) + + return Chapters(chapters) + + def get_widevine_service_certificate(self, **_: Any) -> str: + return None + + def get_widevine_license( + self, *, challenge: bytes, title: Union[Movies, Series], track: AnyTrack + ) -> Optional[Union[bytes, str]]: + if not self.license or not self.token: + return None + + headers = {"x-dt-auth-token": self.token} + r = self.session.post(self.license, headers=headers, data=challenge) + r.raise_for_status() + return r.content + + # Service specific + + def _show(self, data: dict) -> list[Episode]: + + lineups = next( + ( + x["lineups"] + for x in data["content"] + if x.get("title", "").lower() in ("episodes", "parts") + ), + None + ) + + if not lineups: + self.log.warning("No episodes found for: {}".format(data.get("title"))) + return + + titles = [] + for season in lineups: + for episode in season["items"]: + if episode.get("mediaType", "").lower() == "episode": + parts = episode.get("title", "").split(".", 1) + episode_name = parts[1].strip() if len(parts) > 1 else parts[0].strip() + titles.append( + Episode( + id_=episode["idMedia"], + service=self.__class__, + title=data.get("title"), + season=int(season.get("seasonNumber", 0)), + number=int(episode.get("episodeNumber", 0)), + name=episode_name, + year=episode.get("metadata", {}).get("productionYear"), + language=data["structuredMetadata"].get("inLanguage", "en-CA"), + data=episode, + ) + ) + + return titles + + def _movie(self, data: dict) -> list[Movie]: + unwanted: tuple = ("episodes", "trailers", "extras") + lineups: list = next((x["lineups"] for x in data["content"] if x.get("title", "").lower() not in unwanted), None) + if not lineups: + self.log.warning("No movies found for: {}".format(data.get("title"))) + return + + titles = [] + for season in lineups: + for movie in season["items"]: + if movie.get("mediaType", "").lower() == "episode": + parts = movie.get("title", "").split(".", 1) + movie_name = parts[1].strip() if len(parts) > 1 else parts[0].strip() + titles.append( + Movie( + id_=movie.get("idMedia"), + service=self.__class__, + name=movie_name, + year=movie.get("metadata", {}).get("productionYear"), + language=data["structuredMetadata"].get("inLanguage", "en-CA"), + data=movie, + ) + ) + + return titles + + def settings(self) -> tuple: + settings = self._request("GET", "/ott/catalog/v1/gem/settings", params={"device": "web"}) + auth_url: str = settings["identityManagement"]["ropc"]["url"] + scopes: str = settings["identityManagement"]["ropc"]["scopes"] + return auth_url, scopes + + def claims_token(self, token: str) -> str: + headers = { + "Authorization": "Bearer " + token, + } + params = {"device": "web"} + response: dict = self._request( + "GET", "/ott/subscription/v2/gem/Subscriber/profile", headers=headers, params=params + ) + return response["claimsToken"] + + def _request(self, method: str, api: str, **kwargs: Any) -> Any[dict | str]: + url: str = urljoin(self.base_url, api) + + prep: Request = self.session.prepare_request(Request(method, url, **kwargs)) + response = self.session.send(prep) + if response.status_code not in (200, 426): + raise ConnectionError(f"{response.status_code} - {response.text}") + + try: + data = json.loads(response.content) + error_keys = ["errorMessage", "ErrorMessage", "ErrorCode", "errorCode", "error"] + error_message = next((data.get(key) for key in error_keys if key in data), None) + if error_message: + self.log.error(f"\n - Error: {error_message}\n") + sys.exit(1) + + return data + + except json.JSONDecodeError: + raise ConnectionError("Request for {} failed: {}".format(response.url, response.text)) diff --git a/reset/packages/envied/src/envied/services/CBC/config.yaml b/reset/packages/envied/src/envied/services/CBC/config.yaml new file mode 100644 index 0000000..31bd19a --- /dev/null +++ b/reset/packages/envied/src/envied/services/CBC/config.yaml @@ -0,0 +1,7 @@ +endpoints: + base_url: "https://services.radio-canada.ca" + validation: "/media/validation/v2?appCode=gem&&deviceType={}&idMedia={}&manifestType={}&output=json&tech={}" + api_key: "3f4beddd-2061-49b0-ae80-6f1f2ed65b37" + +client: + id: "fc05b0ee-3865-4400-a3cc-3da82c330c23" \ No newline at end of file diff --git a/reset/packages/envied/src/envied/services/CBS/__init__.py b/reset/packages/envied/src/envied/services/CBS/__init__.py new file mode 100644 index 0000000..ae6792d --- /dev/null +++ b/reset/packages/envied/src/envied/services/CBS/__init__.py @@ -0,0 +1,242 @@ +from __future__ import annotations + +import json +import re +import sys +from collections.abc import Generator +from typing import Any, Optional, Union +from urllib.parse import urljoin + +import click +from requests import Request +from envied.core.constants import AnyTrack +from envied.core.manifests import DASH +from envied.core.search_result import SearchResult +from envied.core.service import Service +from envied.core.titles import Episode, Series, Title_T, Titles_T +from envied.core.tracks import Chapter, Chapters, Tracks +from envied.core.utils.sslciphers import SSLCiphers +from envied.core.utils.xml import load_xml + + +class CBS(Service): + """ + \b + Service code for CBS.com streaming service (https://cbs.com). + Credit to @srpen6 for the tip on anonymous session + + \b + Version: 1.0.1 + Author: stabbedbybrick + Authorization: None + Robustness: + Widevine: + L3: 2160p, DDP5.1 + + \b + Tips: + - Input should be complete URLs: + SERIES: https://www.cbs.com/shows/tracker/ + EPISODE: https://www.cbs.com/shows/video/E0wG_ovVMkLlHOzv7KDpUV9bjeKFFG2v/ + + \b + Common VPN/proxy errors: + - SSLError(SSLEOFError(8, '[SSL: UNEXPECTED_EOF_WHILE_READING]')) + - ConnectionError: 406 Not Acceptable, 403 Forbidden + + """ + + GEOFENCE = ("us",) + + @staticmethod + @click.command(name="CBS", short_help="https://cbs.com", help=__doc__) + @click.argument("title", type=str, required=False) + @click.pass_context + def cli(ctx, **kwargs) -> CBS: + return CBS(ctx, **kwargs) + + def __init__(self, ctx, title): + self.title = title + super().__init__(ctx) + + def search(self) -> Generator[SearchResult, None, None]: + params = { + "term": self.title, + "termCount": 50, + "showCanVids": "true", + } + results = self._request("GET", "/apps-api/v3.1/androidphone/contentsearch/search.json", params=params)["terms"] + + for result in results: + yield SearchResult( + id_=result.get("path"), + title=result.get("title"), + description=None, + label=result.get("term_type"), + url=result.get("path"), + ) + + def get_titles(self) -> Titles_T: + title_re = r"https://www\.cbs\.com/shows/(?P