thegame_1980/TwinVine
1
0
Fork 0
mirror of https://github.com/vinefeeder/TwinVine.git synced 2026-07-15 18:10:04 +02:00
Code Issues Packages Projects Releases Wiki Activity

updates

Browse Source
This commit is contained in:
VineFeeder
2026-07-11 14:13:47 +01:00
parent acc7af1c02
commit 1572c47808
358 changed files with 91282 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1
View File
@@ -97,6 +97,7 @@ wget -qO- https://astral.sh/uv/install.sh | sh
6. Then initialize TwinVine:
```bash
git clone https://github.com/vinefeeder/TwinVine.git
cd TwinVine
uv lock
uv sync
UPDATES.md
+3
View File
@@ -1,4 +1,7 @@
**Updates**
**11 July 2026**
vinefeeder update to BBC service to better select UHD quality availability. Granularity is now at the episode level rather than series.
meaning that any series with mixed UHD and HD titles will no longer fail at the download stage
**11 May 2026**
Vinefeeder update to BBC UHD detection to enable automatic HLG selection.
packages/envied/src/envied/commands/search.py
+2 -1
View File
@@ -136,6 +136,7 @@ def result(service: Service, profile: Optional[str] = None, **_: Any) -> None:
result_text = f"[bold text]{result.title}[/]"
if result.url:
result_text = f"[link={result.url}]{result_text}[/link]"
if result.label:
result_text += f" [pink]{result.label}[/]"
if result.description:
@@ -143,7 +144,7 @@ def result(service: Service, profile: Optional[str] = None, **_: Any) -> None:
result_text += f"\n[bright_black]id: {result.id}[/]"
search_results.add(result_text + "\n")
# update cookies
# update cookie
cookie_file = dl.get_cookie_path(service_tag, profile)
if cookie_file:
dl.save_cookies(cookie_file, service.session.cookies)
packages/envied/src/envied/core/search_result.py
+4
View File
@@ -9,6 +9,7 @@ class SearchResult:
description: Optional[str] = None,
label: Optional[str] = None,
url: Optional[str] = None,
image: Optional[str] = None
):
"""
A Search Result for any support Title Type.
@@ -33,12 +34,15 @@ class SearchResult:
raise TypeError(f"Expected label to be a {str}, not {type(label)}")
if not isinstance(url, (str, type(None))):
raise TypeError(f"Expected url to be a {str}, not {type(url)}")
if not isinstance(image, (str, type(None))):
raise TypeError(f"Expected image to be a {str}, not {type(image)}")
self.id = id_
self.title = title
self.description = description
self.label = label
self.url = url
self.image = image
__all__ = ("SearchResult",)
packages/envied/src/envied/services/STV/__init__.py
+4 -2
View File
@@ -22,8 +22,8 @@ class STV(Service):
Service code for STV Player streaming service (https://player.stv.tv/).
\b
Version: 1.0.3
Author: stabbedbybrick
Version: 1.0.4
Author: stabbedbybrick; search corrected by Angela
Authorization: None
Robustness:
L3: 1080p
@@ -68,6 +68,7 @@ class STV(Service):
synopsis = result['attributes']["long_description"]
label = result["attributes"]["subGenre"]
#image = result["attributes"]["images"][0]["_filepath"].split("&w")[0] if result["attributes"]["images"] else None
yield SearchResult(
id_= self.config["endpoints"]["prefix"].strip("/") + result["attributes"]["permalink"],
@@ -75,6 +76,7 @@ class STV(Service):
description=synopsis,
label=label,
url=url,
#image=image
)
def get_titles(self) -> Union[Movies, Series]:
packages/envied/src/envied/services/TPTV/__init__.py
+25 -4
View File
@@ -43,8 +43,8 @@ class TPTV(Service):
Service code for TPTVencore streaming service (https://www.TPTVencore.co.uk/).
\b
version 1.1.0
Date: June 2026
version 1.3.0
Date: July 2026
Author: A_n_g_e_l_a
Authorization: email/password for service in envied.yaml
Robustness:
@@ -190,8 +190,29 @@ class TPTV(Service):
self.authorization = tokens
def search(self) -> Generator[SearchResult, None, None]:
print(f"Searching not implemented in Envied for TPTVencore. Please use VineFeeder instead.")
return None
search_term = self.title.replace(" ", "+")
response = self.session.get(self.config["endpoints"]["search"].format(query=search_term))
response.raise_for_status()
results = json.loads(response.content.decode("utf-8"))
for result in results["data"]:
url = result['video']['playback'].replace("api/core/play", "playback")
title = result['title']
synopsis = result['description'].replace('\n', ' ')
label = result["subtype"]
id = result["video"]["playback"].replace("api/core/play", "playback")
yield SearchResult(
id_= id,
title=title,
description=synopsis,
label=label,
url=url,
)
def get_titles(self) -> Union[Movies, Series]:
data = self.get_data(self.title)
packages/vinefeeder/src/vinefeeder/base_loader.py
+12 -6
View File
@@ -178,7 +178,7 @@ class BaseLoader:
return
def add_episode(self, series_name, episode):
def add_episode(self, series_name, episode, uhd=None):
"""Add an episode to the series in memory.
Episode may be Any"""
if series_name not in self.series_data:
@@ -246,13 +246,19 @@ class BaseLoader:
def get_final_episode_list(self):
return self.final_episode_data
def display_final_episode_list(self, final_episode_data):
def display_final_episode_list(self, final_episode_data, check_uhd=False):
"""Use beaupy to display episodes for a selected series."""
# episodes = self.series_data.get(series_name, [])
episode_list = [
f"{ep['series_no']}, {ep['title']}, {ep['url']}, \n\t {ep['synopsis']}"
for ep in final_episode_data
]
if check_uhd:
episode_list = [
f"{ep['series_no']}, {ep['title']}, {ep['url']}, UHD={ep['uhd']}, \n\t {ep['synopsis']}"
for ep in final_episode_data
]
else:
episode_list = [
f"{ep['series_no']}, {ep['title']}, {ep['url']}, \n\t {ep['synopsis']}"
for ep in final_episode_data
]
selected_episodes = select_multiple(
episode_list,
preprocessor=lambda val: prettify(val),
packages/vinefeeder/src/vinefeeder/services/BBC/__init__.py
+33 -24
View File
@@ -9,6 +9,7 @@ from vinefeeder.parsing_utils import extract_params_json, parse_json, split, spl
from rich.console import Console
import jmespath
from scrapy.selector import Selector
import json
console = Console()
@@ -151,7 +152,7 @@ class BbcLoader(BaseLoader):
print(
"Error downloading video:",
e,
"Is self.DOWNLOAD_ORCHESTRATOR installed correctly via 'pip install self.DOWNLOAD_ORCHESTRATOR?",
f"Is {self.DOWNLOAD_ORCHESTRATOR} installed correctly via 'pip install {self.DOWNLOAD_ORCHESTRATOR}?",
)
else:
@@ -166,7 +167,7 @@ class BbcLoader(BaseLoader):
print(
"Error downloading video:",
e,
"Is self.DOWNLOAD_ORCHESTRATOR installed correctly via 'pip install self.DOWNLOAD_ORCHESTRATOR?",
f"Is {self.DOWNLOAD_ORCHESTRATOR} installed correctly via 'pip install {self.DOWNLOAD_ORCHESTRATOR}?",
)
return
@@ -213,15 +214,11 @@ class BbcLoader(BaseLoader):
def fetch_videos(self, search_term):
"""Fetch videos from BBC using a search term."""
url = "https://ibl.api.bbc.co.uk/ibl/v1/new-search"
# url = f"https://search.api.bbci.co.uk/formula/iplayer-ibl-root?q={search_term}&apikey=D2FgtcTxGqqIgLsfBWTJdrQh2tVdeaAp&seqId=0582e0f0-b911-11ee-806c-11c6c885ab56"
params = {
"q": search_term,
"rights": "web",
#'mixin': 'live'
}
url = f"https://ibl.api.bbc.co.uk/ibl/v1/new-search?q={search_term}&rights=mobile&size=10&sort=relevance&start=0"
try:
html = self.get_data(url, self.headers, params)
html = self.get_data(url, self.headers)
if "new_search" not in html:
print("Nothing found for that search; try again.")
return
@@ -290,10 +287,10 @@ class BbcLoader(BaseLoader):
parsed_data = parse_json(myhtml)
# testing
#file = open("init_data.json", "w")
#file.write(json.dumps(parsed_data))
#file.close()
#console.print_json(data=parsed_data)
# file = open("init_data.json", "w")
# file.write(json.dumps(parsed_data))
# file.close()
# console.print_json(data=parsed_data)
self.clear_series_data() # Clear existing series data
self.options_list = split_options(self.options)
@@ -317,6 +314,8 @@ class BbcLoader(BaseLoader):
episodes = parsed_data.get("programme_episodes").get("elements")
for item in episodes:
try:
# programme_episodes►elements►0►versions►0►uhd
uhd = item.get("versions", [{}])[0].get("uhd", False)
series_no = item["subtitle"].split(":")[0].split(" ")[1]
if int(series_no):
pass
@@ -330,21 +329,23 @@ class BbcLoader(BaseLoader):
], # .split(' ')[-1] or '01',
"url": "https://www.bbc.co.uk/iplayer/episode/" + item["id"],
"synopsis": item["synopses"]["small"] or None,
"uhd": uhd or False,
}
except Exception:
try:
episode = {
"series_no": 100, # special or one-off'
"series_no": int(100), # special or one-off'
# 'title' is episode number here, some services use descriptive text
"title": item["subtitle"], # could be date
"url": "https://www.bbc.co.uk/iplayer/episode/"
+ item["id"],
"synopsis": item["synopses"]["small"] or None,
"uhd": item.get("versions", [{}])[0].get("uhd", False) or False,
}
except KeyError as e:
print(f"Error: {e}")
continue # Skip any episode that doesn't have the required information
self.add_episode(series_name, episode)
self.add_episode(series_name, episode, uhd=uhd)
# Single episode expedite download
else:
@@ -356,6 +357,9 @@ class BbcLoader(BaseLoader):
self.AVAILABLE_HLG = True
break
# self.options_list = split_options(self.options)
try:
if BbcLoader.HLG and self.AVAILABLE_HLG:
command = (
@@ -378,7 +382,7 @@ class BbcLoader(BaseLoader):
print(
"Error downloading video:",
e,
"Is self.DOWNLOAD_ORCHESTRATOR installed correctly via 'pip install self.DOWNLOAD_ORCHESTRATOR?",
f"Is {self.DOWNLOAD_ORCHESTRATOR} installed correctly via 'pip install {self.DOWNLOAD_ORCHESTRATOR}'?",
)
return
@@ -386,17 +390,22 @@ class BbcLoader(BaseLoader):
series_name
) # creates list of series; allows user selection of wanted series prepares an episode list over chosen series
selected_final_episodes = self.display_final_episode_list(
self.final_episode_data
self.final_episode_data, check_uhd=True
)
# specific to BBC
# self.options_list = split_options(self.options)
for item in selected_final_episodes:
# check for UHD content
for hlg_item in self.uhd_list:
if series_name.lower() in hlg_item:
self.AVAILABLE_HLG = True
break
#print(type(item))
#print(item)
for part in item.split(","):
if "UHD" in part:
uhd = part.split("UHD=")[1].strip()
# Sanity check for UHD content
if uhd=="True":
self.AVAILABLE_HLG = True
else:
self.AVAILABLE_HLG = False
for part in item.split(","):
if "https" in part:
url = part.strip()
@@ -419,7 +428,7 @@ class BbcLoader(BaseLoader):
Returns:
None
"""
beaupylist = []
#beaupylist = []
req = self.client.get(browse_url, headers=self.headers)
init_data = extract_params_json(req.content.decode(), "__IPLAYER_REDUX_STATE__")
reset/.gitignore
+218
View File
@@ -0,0 +1,218 @@
Cache/
Logs/
Downloads
Temp/
packages/envied/src/envied/logs/
packages/envied/src/envied/envied.yaml
packages/envied/src/envied/cache/TVNZ/local_storage.jsonv
images/*
tools/
batch.txt
*.json
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[codz]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py.cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# UV
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
#uv.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
#poetry.toml
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
#pdm.lock
#pdm.toml
.pdm-python
.pdm-build/
# pixi
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
#pixi.lock
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
# in the .venv directory. It is recommended not to include this directory in version control.
.pixi
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.envrc
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
# Abstra
# Abstra is an AI-powered process automation framework.
# Ignore directories containing user credentials, local state, and settings.
# Learn more at https://abstra.io/docs
.abstra/
# Visual Studio Code
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
# and can be added to the global gitignore or merged into this file. However, if you prefer,
# you could uncomment the following to ignore the entire vscode folder
.vscode/
# Ruff stuff:
.ruff_cache/
# PyPI configuration file
.pypirc
# Cursor
# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
# refer to https://docs.cursor.com/context/ignore-files
.cursorignore
.cursorindexingignore
# Marimo
marimo/_static/
marimo/_lsp/
__marimo__/
reset/Install-media-tools.ps1
+231
View File
@@ -0,0 +1,231 @@
# Install-media-tools.ps1
# Run in elevated PowerShell:
# Inside the VM:
# Open Start
# Type PowerShell
# Right-click Windows PowerShell → Run as administrator
# chdir to the location of this script
# powershell -ExecutionPolicy Bypass -File .\Install-media-tools.ps1
$ErrorActionPreference = "Stop"
# Where to install (bin folder on PATH)
$BinDir = "C:\Tools\bin"
$ToolsRoot = "C:\Tools"
$SubtitleEditDir = Join-Path $ToolsRoot "SubtitleEdit"
$WorkDir = Join-Path $env:TEMP ("media-tools-" + [guid]::NewGuid().ToString("N"))
New-Item -ItemType Directory -Force -Path $BinDir | Out-Null
New-Item -ItemType Directory -Force -Path $WorkDir | Out-Null
New-Item -ItemType Directory -Force -Path $SubtitleEditDir | Out-Null
function Download-File([string]$Url, [string]$OutFile) {
Write-Host "Downloading: $Url"
Invoke-WebRequest -Uri $Url -OutFile $OutFile -UseBasicParsing
}
function Add-ToMachinePath([string]$PathToAdd) {
$current = [Environment]::GetEnvironmentVariable("Path", "Machine")
if (-not $current) { $current = "" }
# Compare case-insensitively on Windows
$parts = $current.Split(";", [System.StringSplitOptions]::RemoveEmptyEntries)
$exists = $parts | Where-Object { $_.TrimEnd("\") -ieq $PathToAdd.TrimEnd("\") }
if (-not $exists) {
$newPath = ($current.TrimEnd(";") + ";" + $PathToAdd).TrimStart(";")
[Environment]::SetEnvironmentVariable("Path", $newPath, "Machine")
Write-Host "Added to MACHINE PATH: $PathToAdd"
Write-Host "Restart your terminal/session to pick it up."
} else {
Write-Host "MACHINE PATH already contains: $PathToAdd"
}
}
try {
Push-Location $WorkDir
# -----------------------------
# Install FFmpeg (full build, Windows x64) from GitHub ZIP
# -----------------------------
$ffmpegZip = Join-Path $WorkDir "ffmpeg-full_build.zip"
Download-File `
"https://github.com/GyanD/codexffmpeg/releases/download/2025-12-14-git-3332b2db84/ffmpeg-2025-12-14-git-3332b2db84-full_build.zip" `
$ffmpegZip
$ffmpegExtract = Join-Path $WorkDir "ffmpeg"
New-Item -ItemType Directory -Force -Path $ffmpegExtract | Out-Null
Expand-Archive -Path $ffmpegZip -DestinationPath $ffmpegExtract -Force
# Find the extracted "bin" folder (contains ffmpeg.exe etc.)
$ffmpegBin = Get-ChildItem $ffmpegExtract -Recurse -Directory |
Where-Object { $_.Name -ieq "bin" } |
Select-Object -First 1
if (-not $ffmpegBin) {
throw "FFmpeg bin directory not found after extraction."
}
Get-ChildItem $ffmpegBin.FullName -File |
Where-Object { $_.Name -match '^ff(mpeg|probe|play)\.exe$' } |
ForEach-Object {
Copy-Item -Force $_.FullName (Join-Path $BinDir $_.Name)
}
Write-Host "FFmpeg installed to $BinDir"
# -----------------------------
# Install Bento4 (mp4decrypt.exe, etc.)
# -----------------------------
$bentoZip = Join-Path $WorkDir "Bento4.zip"
Download-File `
"https://www.bok.net/Bento4/binaries/Bento4-SDK-1-6-0-641.x86_64-microsoft-win32.zip" `
$bentoZip
Expand-Archive -Path $bentoZip -DestinationPath (Join-Path $WorkDir "bento") -Force
Get-ChildItem -Path (Join-Path $WorkDir "bento") -Recurse -File |
Where-Object { $_.FullName -match "\\bin\\.*\.exe$" } |
ForEach-Object { Copy-Item -Force $_.FullName $BinDir }
# -----------------------------
# Install MKVToolNix (Windows x64)
# -----------------------------
$mkvInstaller = Join-Path $WorkDir "mkvtoolnix-64-bit-96.0-setup.exe"
Download-File `
"https://mkvtoolnix.download/windows/releases/96.0/mkvtoolnix-64-bit-96.0-setup.exe" `
$mkvInstaller
# Silent install (NSIS): /S = silent
# Note: no UI will appear; wait for it to finish.
Start-Process -FilePath $mkvInstaller -ArgumentList "/S" -Wait
# Optional: add MKVToolNix install dir to PATH if present
$mkvDefaultPath = "C:\Program Files\MKVToolNix"
if (Test-Path $mkvDefaultPath) {
# If you kept Add-ToMachinePath, use that:
Add-ToMachinePath $mkvDefaultPath
# Or if you switched to Add-ToBestPath, use that instead:
# Add-ToBestPath $mkvDefaultPath
} else {
Write-Host "MKVToolNix installed, but default path not found: $mkvDefaultPath (maybe different install location)"
}
# -----------------------------
# Install N_m3u8DL-RE (Windows x64)
# -----------------------------
$nreZip = Join-Path $WorkDir "N_m3u8DL-RE.zip"
Download-File `
"https://github.com/nilaoda/N_m3u8DL-RE/releases/download/v0.3.0-beta/N_m3u8DL-RE_v0.3.0-beta_win-x64_20241203.zip" `
$nreZip
Expand-Archive -Path $nreZip -DestinationPath (Join-Path $WorkDir "nre") -Force
Get-ChildItem (Join-Path $WorkDir "nre") -Recurse -File |
Where-Object { $_.Name -match "^N_m3u8DL-RE(\.exe)?$" } |
Select-Object -First 1 |
ForEach-Object { Copy-Item -Force $_.FullName (Join-Path $BinDir "N_m3u8DL-RE.exe") }
'''# -----------------------------
# Install Git for Windows (64-bit)
# -----------------------------
$gitInstaller = Join-Path $WorkDir "Git-2.52.0-64-bit.exe"
Download-File `
"https://github.com/git-for-windows/git/releases/download/v2.52.0.windows.1/Git-2.52.0-64-bit.exe" `
$gitInstaller
# Silent install (NSIS): /S = silent
Start-Process -FilePath $gitInstaller -ArgumentList "/S" -Wait
# Add the typical Git cmd folder to MACHINE PATH if present
$gitDefaultPath = "C:\Program Files\Git\cmd"
if (Test-Path $gitDefaultPath) {
Add-ToMachinePath $gitDefaultPath
} else {
Write-Host "Git installed, but default path not found: $gitDefaultPath (maybe different install location)"
}'''
# -----------------------------
# Install Shaka Packager (Windows x64)
# -----------------------------
$shakaExe = Join-Path $BinDir "shaka-packager.exe"
Download-File `
"https://github.com/shaka-project/shaka-packager/releases/download/v2.6.1/packager-win-x64.exe" `
$shakaExe
# -----------------------------
# Install dovi_tool (Windows x64)
# -----------------------------
$doviZip = Join-Path $WorkDir "dovi_tool.zip"
Download-File `
"https://github.com/quietvoid/dovi_tool/releases/download/2.3.1/dovi_tool-2.3.1-x86_64-pc-windows-msvc.zip" `
$doviZip
Expand-Archive -Path $doviZip -DestinationPath (Join-Path $WorkDir "dovi") -Force
Get-ChildItem (Join-Path $WorkDir "dovi") -Recurse -File |
Where-Object { $_.Name -ieq "dovi_tool.exe" } |
Select-Object -First 1 |
ForEach-Object { Copy-Item -Force $_.FullName (Join-Path $BinDir "dovi_tool.exe") }
# -----------------------------
# Install hdr10plus_tool (Windows x64)
# -----------------------------
$hdrZip = Join-Path $WorkDir "hdr10plus_tool.zip"
Download-File `
"https://github.com/quietvoid/hdr10plus_tool/releases/download/1.7.1/hdr10plus_tool-1.7.1-x86_64-pc-windows-msvc.zip" `
$hdrZip
Expand-Archive -Path $hdrZip -DestinationPath (Join-Path $WorkDir "hdr") -Force
Get-ChildItem (Join-Path $WorkDir "hdr") -Recurse -File |
Where-Object { $_.Name -ieq "hdr10plus_tool.exe" } |
Select-Object -First 1 |
ForEach-Object { Copy-Item -Force $_.FullName (Join-Path $BinDir "hdr10plus_tool.exe") }
# -----------------------------
# Install SubtitleEdit permanently (Portable zip)
# -----------------------------
$seZip = Join-Path $WorkDir "SubtitleEdit.zip"
Download-File `
"https://github.com/SubtitleEdit/subtitleedit/releases/download/4.0.14/SE4014.zip" `
$seZip
# Replace any previous install cleanly
if (Test-Path $SubtitleEditDir) { Remove-Item -Recurse -Force $SubtitleEditDir }
New-Item -ItemType Directory -Force -Path $SubtitleEditDir | Out-Null
Expand-Archive -Path $seZip -DestinationPath $SubtitleEditDir -Force
# Launcher in bin (stable path)
$launcher = Join-Path $BinDir "SubtitleEdit.cmd"
@(
'@echo off'
'"C:\Tools\SubtitleEdit\SubtitleEdit.exe" %*'
) | Set-Content -Encoding ASCII $launcher
# -----------------------------
# Install uv (Python required)
# -----------------------------
python -m pip install --upgrade uv
# PATH (Machine)
Add-ToMachinePath $BinDir
Write-Host "`nDone. Open a NEW terminal and try:"
Write-Host " mp4decrypt.exe --help"
Write-Host " N_m3u8DL-RE.exe --help"
Write-Host " shaka-packager.exe --help"
Write-Host " dovi_tool.exe --help"
Write-Host " hdr10plus_tool.exe --help"
Write-Host " SubtitleEdit.cmd"
Write-Host " uv --version"
} finally {
Pop-Location
# Clean up temp work dir
if (Test-Path $WorkDir) { Remove-Item -Recurse -Force $WorkDir }
}
reset/Install-media-tools.sh
+53
View File
@@ -0,0 +1,53 @@
# Thanks Akirainblack for providing this routine
# run script as superuser using command:-
# sudo bash Install-media-tools.sh
# Install MKVToolNix and ffmpeg from Ubuntu repos
apt-get update && \
apt-get install -y mkvtoolnix mkvtoolnix-gui ffmpeg && \
rm -rf /var/lib/apt/lists/*
# Install Bento4 (mp4decrypt)
wget https://www.bok.net/Bento4/binaries/Bento4-SDK-1-6-0-641.x86_64-unknown-linux.zip && \
unzip -j Bento4-SDK-1-6-0-641.x86_64-unknown-linux.zip \
'Bento4-SDK-1-6-0-641.x86_64-unknown-linux/bin/*' -d /usr/local/bin/ && \
rm Bento4-SDK-1-6-0-641.x86_64-unknown-linux.zip && \
chmod +x /usr/local/bin/*
# Install N_m3u8DL-RE
wget https://github.com/nilaoda/N_m3u8DL-RE/releases/download/v0.3.0-beta/N_m3u8DL-RE_v0.3.0-beta_linux-x64_20241203.tar.gz && \
tar -xzf N_m3u8DL-RE_v0.3.0-beta_linux-x64_20241203.tar.gz && \
find . -name "N_m3u8DL-RE" -type f -exec mv {} /usr/local/bin/ \; && \
chmod +x /usr/local/bin/N_m3u8DL-RE && \
rm -rf N_m3u8DL-RE_v0.3.0-beta_linux-x64_20241203.tar.gz
# Install Shaka Packager
wget https://github.com/shaka-project/shaka-packager/releases/download/v3.2.0/packager-linux-x64 && \
mv packager-linux-x64 /usr/local/bin/shaka-packager && \
chmod +x /usr/local/bin/shaka-packager
# Install dovi_tool
wget https://github.com/quietvoid/dovi_tool/releases/download/2.3.1/dovi_tool-2.3.1-x86_64-unknown-linux-musl.tar.gz && \
tar -xzf dovi_tool-2.3.1-x86_64-unknown-linux-musl.tar.gz && \
find . -name "dovi_tool" -type f -exec mv {} /usr/local/bin/ \; && \
chmod +x /usr/local/bin/dovi_tool && \
rm -rf dovi_tool-2.3.1-x86_64-unknown-linux-musl.tar.gz
# Install HDR10Plus
wget https://github.com/quietvoid/hdr10plus_tool/releases/download/1.7.1/hdr10plus_tool-1.7.1-x86_64-unknown-linux-musl.tar.gz && \
tar -xzf hdr10plus_tool-1.7.1-x86_64-unknown-linux-musl.tar.gz && \
find . -name "hdr10plus_tool" -type f -exec mv {} /usr/local/bin/ \; && \
chmod +x /usr/local/bin/hdr10plus_tool && \
rm -rf hdr10plus_tool-1.7.1-x86_64-unknown-linux-musl.tar.gz
# Install SubtitleEdit
wget https://github.com/SubtitleEdit/subtitleedit/releases/download/4.0.14/SE4014.zip && \
unzip SE4014.zip -d /SE && \
awk 'BEGIN{print "#!/bin/bash"}' >> /usr/local/bin/SubtitleEdit && \
echo "exec mono /SE/SubtitleEdit.exe \"\$@\"" >> /usr/local/bin/SubtitleEdit && \
chmod +x /usr/local/bin/SubtitleEdit && \
rm -rf SE4014.zip
# Install uv by copying from official image
# cp --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
python -m pip install uv
reset/LICENSE
+674
View File
@@ -0,0 +1,674 @@
GNU GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The GNU General Public License is a free, copyleft license for
software and other kinds of works.
The licenses for most software and other practical works are designed
to take away your freedom to share and change the works. By contrast,
the GNU General Public License is intended to guarantee your freedom to
share and change all versions of a program--to make sure it remains free
software for all its users. We, the Free Software Foundation, use the
GNU General Public License for most of our software; it applies also to
any other work released this way by its authors. You can apply it to
your programs, too.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
them if you wish), that you receive source code or can get it if you
want it, that you can change the software or use pieces of it in new
free programs, and that you know you can do these things.
To protect your rights, we need to prevent others from denying you
these rights or asking you to surrender the rights. Therefore, you have
certain responsibilities if you distribute copies of the software, or if
you modify it: responsibilities to respect the freedom of others.
For example, if you distribute copies of such a program, whether
gratis or for a fee, you must pass on to the recipients the same
freedoms that you received. You must make sure that they, too, receive
or can get the source code. And you must show them these terms so they
know their rights.
Developers that use the GNU GPL protect your rights with two steps:
(1) assert copyright on the software, and (2) offer you this License
giving you legal permission to copy, distribute and/or modify it.
For the developers' and authors' protection, the GPL clearly explains
that there is no warranty for this free software. For both users' and
authors' sake, the GPL requires that modified versions be marked as
changed, so that their problems will not be attributed erroneously to
authors of previous versions.
Some devices are designed to deny users access to install or run
modified versions of the software inside them, although the manufacturer
can do so. This is fundamentally incompatible with the aim of
protecting users' freedom to change the software. The systematic
pattern of such abuse occurs in the area of products for individuals to
use, which is precisely where it is most unacceptable. Therefore, we
have designed this version of the GPL to prohibit the practice for those
products. If such problems arise substantially in other domains, we
stand ready to extend this provision to those domains in future versions
of the GPL, as needed to protect the freedom of users.
Finally, every program is threatened constantly by software patents.
States should not allow patents to restrict development and use of
software on general-purpose computers, but in those that do, we wish to
avoid the special danger that patents applied to a free program could
make it effectively proprietary. To prevent this, the GPL assures that
patents cannot be used to render the program non-free.
The precise terms and conditions for copying, distribution and
modification follow.
TERMS AND CONDITIONS
0. Definitions.
"This License" refers to version 3 of the GNU General Public License.
"Copyright" also means copyright-like laws that apply to other kinds of
works, such as semiconductor masks.
"The Program" refers to any copyrightable work licensed under this
License. Each licensee is addressed as "you". "Licensees" and
"recipients" may be individuals or organizations.
To "modify" a work means to copy from or adapt all or part of the work
in a fashion requiring copyright permission, other than the making of an
exact copy. The resulting work is called a "modified version" of the
earlier work or a work "based on" the earlier work.
A "covered work" means either the unmodified Program or a work based
on the Program.
To "propagate" a work means to do anything with it that, without
permission, would make you directly or secondarily liable for
infringement under applicable copyright law, except executing it on a
computer or modifying a private copy. Propagation includes copying,
distribution (with or without modification), making available to the
public, and in some countries other activities as well.
To "convey" a work means any kind of propagation that enables other
parties to make or receive copies. Mere interaction with a user through
a computer network, with no transfer of a copy, is not conveying.
An interactive user interface displays "Appropriate Legal Notices"
to the extent that it includes a convenient and prominently visible
feature that (1) displays an appropriate copyright notice, and (2)
tells the user that there is no warranty for the work (except to the
extent that warranties are provided), that licensees may convey the
work under this License, and how to view a copy of this License. If
the interface presents a list of user commands or options, such as a
menu, a prominent item in the list meets this criterion.
1. Source Code.
The "source code" for a work means the preferred form of the work
for making modifications to it. "Object code" means any non-source
form of a work.
A "Standard Interface" means an interface that either is an official
standard defined by a recognized standards body, or, in the case of
interfaces specified for a particular programming language, one that
is widely used among developers working in that language.
The "System Libraries" of an executable work include anything, other
than the work as a whole, that (a) is included in the normal form of
packaging a Major Component, but which is not part of that Major
Component, and (b) serves only to enable use of the work with that
Major Component, or to implement a Standard Interface for which an
implementation is available to the public in source code form. A
"Major Component", in this context, means a major essential component
(kernel, window system, and so on) of the specific operating system
(if any) on which the executable work runs, or a compiler used to
produce the work, or an object code interpreter used to run it.
The "Corresponding Source" for a work in object code form means all
the source code needed to generate, install, and (for an executable
work) run the object code and to modify the work, including scripts to
control those activities. However, it does not include the work's
System Libraries, or general-purpose tools or generally available free
programs which are used unmodified in performing those activities but
which are not part of the work. For example, Corresponding Source
includes interface definition files associated with source files for
the work, and the source code for shared libraries and dynamically
linked subprograms that the work is specifically designed to require,
such as by intimate data communication or control flow between those
subprograms and other parts of the work.
The Corresponding Source need not include anything that users
can regenerate automatically from other parts of the Corresponding
Source.
The Corresponding Source for a work in source code form is that
same work.
2. Basic Permissions.
All rights granted under this License are granted for the term of
copyright on the Program, and are irrevocable provided the stated
conditions are met. This License explicitly affirms your unlimited
permission to run the unmodified Program. The output from running a
covered work is covered by this License only if the output, given its
content, constitutes a covered work. This License acknowledges your
rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not
convey, without conditions so long as your license otherwise remains
in force. You may convey covered works to others for the sole purpose
of having them make modifications exclusively for you, or provide you
with facilities for running those works, provided that you comply with
the terms of this License in conveying all material for which you do
not control copyright. Those thus making or running the covered works
for you must do so exclusively on your behalf, under your direction
and control, on terms that prohibit them from making any copies of
your copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under
the conditions stated below. Sublicensing is not allowed; section 10
makes it unnecessary.
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
No covered work shall be deemed part of an effective technological
measure under any applicable law fulfilling obligations under article
11 of the WIPO copyright treaty adopted on 20 December 1996, or
similar laws prohibiting or restricting circumvention of such
measures.
When you convey a covered work, you waive any legal power to forbid
circumvention of technological measures to the extent such circumvention
is effected by exercising rights under this License with respect to
the covered work, and you disclaim any intention to limit operation or
modification of the work as a means of enforcing, against the work's
users, your or third parties' legal rights to forbid circumvention of
technological measures.
4. Conveying Verbatim Copies.
You may convey verbatim copies of the Program's source code as you
receive it, in any medium, provided that you conspicuously and
appropriately publish on each copy an appropriate copyright notice;
keep intact all notices stating that this License and any
non-permissive terms added in accord with section 7 apply to the code;
keep intact all notices of the absence of any warranty; and give all
recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey,
and you may offer support or warranty protection for a fee.
5. Conveying Modified Source Versions.
You may convey a work based on the Program, or the modifications to
produce it from the Program, in the form of source code under the
terms of section 4, provided that you also meet all of these conditions:
a) The work must carry prominent notices stating that you modified
it, and giving a relevant date.
b) The work must carry prominent notices stating that it is
released under this License and any conditions added under section
7. This requirement modifies the requirement in section 4 to
"keep intact all notices".
c) You must license the entire work, as a whole, under this
License to anyone who comes into possession of a copy. This
License will therefore apply, along with any applicable section 7
additional terms, to the whole of the work, and all its parts,
regardless of how they are packaged. This License gives no
permission to license the work in any other way, but it does not
invalidate such permission if you have separately received it.
d) If the work has interactive user interfaces, each must display
Appropriate Legal Notices; however, if the Program has interactive
interfaces that do not display Appropriate Legal Notices, your
work need not make them do so.
A compilation of a covered work with other separate and independent
works, which are not by their nature extensions of the covered work,
and which are not combined with it such as to form a larger program,
in or on a volume of a storage or distribution medium, is called an
"aggregate" if the compilation and its resulting copyright are not
used to limit the access or legal rights of the compilation's users
beyond what the individual works permit. Inclusion of a covered work
in an aggregate does not cause this License to apply to the other
parts of the aggregate.
6. Conveying Non-Source Forms.
You may convey a covered work in object code form under the terms
of sections 4 and 5, provided that you also convey the
machine-readable Corresponding Source under the terms of this License,
in one of these ways:
a) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by the
Corresponding Source fixed on a durable physical medium
customarily used for software interchange.
b) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by a
written offer, valid for at least three years and valid for as
long as you offer spare parts or customer support for that product
model, to give anyone who possesses the object code either (1) a
copy of the Corresponding Source for all the software in the
product that is covered by this License, on a durable physical
medium customarily used for software interchange, for a price no
more than your reasonable cost of physically performing this
conveying of source, or (2) access to copy the
Corresponding Source from a network server at no charge.
c) Convey individual copies of the object code with a copy of the
written offer to provide the Corresponding Source. This
alternative is allowed only occasionally and noncommercially, and
only if you received the object code with such an offer, in accord
with subsection 6b.
d) Convey the object code by offering access from a designated
place (gratis or for a charge), and offer equivalent access to the
Corresponding Source in the same way through the same place at no
further charge. You need not require recipients to copy the
Corresponding Source along with the object code. If the place to
copy the object code is a network server, the Corresponding Source
may be on a different server (operated by you or a third party)
that supports equivalent copying facilities, provided you maintain
clear directions next to the object code saying where to find the
Corresponding Source. Regardless of what server hosts the
Corresponding Source, you remain obligated to ensure that it is
available for as long as needed to satisfy these requirements.
e) Convey the object code using peer-to-peer transmission, provided
you inform other peers where the object code and Corresponding
Source of the work are being offered to the general public at no
charge under subsection 6d.
A separable portion of the object code, whose source code is excluded
from the Corresponding Source as a System Library, need not be
included in conveying the object code work.
A "User Product" is either (1) a "consumer product", which means any
tangible personal property which is normally used for personal, family,
or household purposes, or (2) anything designed or sold for incorporation
into a dwelling. In determining whether a product is a consumer product,
doubtful cases shall be resolved in favor of coverage. For a particular
product received by a particular user, "normally used" refers to a
typical or common use of that class of product, regardless of the status
of the particular user or of the way in which the particular user
actually uses, or expects or is expected to use, the product. A product
is a consumer product regardless of whether the product has substantial
commercial, industrial or non-consumer uses, unless such uses represent
the only significant mode of use of the product.
"Installation Information" for a User Product means any methods,
procedures, authorization keys, or other information required to install
and execute modified versions of a covered work in that User Product from
a modified version of its Corresponding Source. The information must
suffice to ensure that the continued functioning of the modified object
code is in no case prevented or interfered with solely because
modification has been made.
If you convey an object code work under this section in, or with, or
specifically for use in, a User Product, and the conveying occurs as
part of a transaction in which the right of possession and use of the
User Product is transferred to the recipient in perpetuity or for a
fixed term (regardless of how the transaction is characterized), the
Corresponding Source conveyed under this section must be accompanied
by the Installation Information. But this requirement does not apply
if neither you nor any third party retains the ability to install
modified object code on the User Product (for example, the work has
been installed in ROM).
The requirement to provide Installation Information does not include a
requirement to continue to provide support service, warranty, or updates
for a work that has been modified or installed by the recipient, or for
the User Product in which it has been modified or installed. Access to a
network may be denied when the modification itself materially and
adversely affects the operation of the network or violates the rules and
protocols for communication across the network.
Corresponding Source conveyed, and Installation Information provided,
in accord with this section must be in a format that is publicly
documented (and with an implementation available to the public in
source code form), and must require no special password or key for
unpacking, reading or copying.
7. Additional Terms.
"Additional permissions" are terms that supplement the terms of this
License by making exceptions from one or more of its conditions.
Additional permissions that are applicable to the entire Program shall
be treated as though they were included in this License, to the extent
that they are valid under applicable law. If additional permissions
apply only to part of the Program, that part may be used separately
under those permissions, but the entire Program remains governed by
this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option
remove any additional permissions from that copy, or from any part of
it. (Additional permissions may be written to require their own
removal in certain cases when you modify the work.) You may place
additional permissions on material, added by you to a covered work,
for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you
add to a covered work, you may (if authorized by the copyright holders of
that material) supplement the terms of this License with terms:
a) Disclaiming warranty or limiting liability differently from the
terms of sections 15 and 16 of this License; or
b) Requiring preservation of specified reasonable legal notices or
author attributions in that material or in the Appropriate Legal
Notices displayed by works containing it; or
c) Prohibiting misrepresentation of the origin of that material, or
requiring that modified versions of such material be marked in
reasonable ways as different from the original version; or
d) Limiting the use for publicity purposes of names of licensors or
authors of the material; or
e) Declining to grant rights under trademark law for use of some
trade names, trademarks, or service marks; or
f) Requiring indemnification of licensors and authors of that
material by anyone who conveys the material (or modified versions of
it) with contractual assumptions of liability to the recipient, for
any liability that these contractual assumptions directly impose on
those licensors and authors.
All other non-permissive additional terms are considered "further
restrictions" within the meaning of section 10. If the Program as you
received it, or any part of it, contains a notice stating that it is
governed by this License along with a term that is a further
restriction, you may remove that term. If a license document contains
a further restriction but permits relicensing or conveying under this
License, you may add to a covered work material governed by the terms
of that license document, provided that the further restriction does
not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you
must place, in the relevant source files, a statement of the
additional terms that apply to those files, or a notice indicating
where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the
form of a separately written license, or stated as exceptions;
the above requirements apply either way.
8. Termination.
You may not propagate or modify a covered work except as expressly
provided under this License. Any attempt otherwise to propagate or
modify it is void, and will automatically terminate your rights under
this License (including any patent licenses granted under the third
paragraph of section 11).
However, if you cease all violation of this License, then your
license from a particular copyright holder is reinstated (a)
provisionally, unless and until the copyright holder explicitly and
finally terminates your license, and (b) permanently, if the copyright
holder fails to notify you of the violation by some reasonable means
prior to 60 days after the cessation.
Moreover, your license from a particular copyright holder is
reinstated permanently if the copyright holder notifies you of the
violation by some reasonable means, this is the first time you have
received notice of violation of this License (for any work) from that
copyright holder, and you cure the violation prior to 30 days after
your receipt of the notice.
Termination of your rights under this section does not terminate the
licenses of parties who have received copies or rights from you under
this License. If your rights have been terminated and not permanently
reinstated, you do not qualify to receive new licenses for the same
material under section 10.
9. Acceptance Not Required for Having Copies.
You are not required to accept this License in order to receive or
run a copy of the Program. Ancillary propagation of a covered work
occurring solely as a consequence of using peer-to-peer transmission
to receive a copy likewise does not require acceptance. However,
nothing other than this License grants you permission to propagate or
modify any covered work. These actions infringe copyright if you do
not accept this License. Therefore, by modifying or propagating a
covered work, you indicate your acceptance of this License to do so.
10. Automatic Licensing of Downstream Recipients.
Each time you convey a covered work, the recipient automatically
receives a license from the original licensors, to run, modify and
propagate that work, subject to this License. You are not responsible
for enforcing compliance by third parties with this License.
An "entity transaction" is a transaction transferring control of an
organization, or substantially all assets of one, or subdividing an
organization, or merging organizations. If propagation of a covered
work results from an entity transaction, each party to that
transaction who receives a copy of the work also receives whatever
licenses to the work the party's predecessor in interest had or could
give under the previous paragraph, plus a right to possession of the
Corresponding Source of the work from the predecessor in interest, if
the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the
rights granted or affirmed under this License. For example, you may
not impose a license fee, royalty, or other charge for exercise of
rights granted under this License, and you may not initiate litigation
(including a cross-claim or counterclaim in a lawsuit) alleging that
any patent claim is infringed by making, using, selling, offering for
sale, or importing the Program or any portion of it.
11. Patents.
A "contributor" is a copyright holder who authorizes use under this
License of the Program or a work on which the Program is based. The
work thus licensed is called the contributor's "contributor version".
A contributor's "essential patent claims" are all patent claims
owned or controlled by the contributor, whether already acquired or
hereafter acquired, that would be infringed by some manner, permitted
by this License, of making, using, or selling its contributor version,
but do not include claims that would be infringed only as a
consequence of further modification of the contributor version. For
purposes of this definition, "control" includes the right to grant
patent sublicenses in a manner consistent with the requirements of
this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free
patent license under the contributor's essential patent claims, to
make, use, sell, offer for sale, import and otherwise run, modify and
propagate the contents of its contributor version.
In the following three paragraphs, a "patent license" is any express
agreement or commitment, however denominated, not to enforce a patent
(such as an express permission to practice a patent or covenant not to
sue for patent infringement). To "grant" such a patent license to a
party means to make such an agreement or commitment not to enforce a
patent against the party.
If you convey a covered work, knowingly relying on a patent license,
and the Corresponding Source of the work is not available for anyone
to copy, free of charge and under the terms of this License, through a
publicly available network server or other readily accessible means,
then you must either (1) cause the Corresponding Source to be so
available, or (2) arrange to deprive yourself of the benefit of the
patent license for this particular work, or (3) arrange, in a manner
consistent with the requirements of this License, to extend the patent
license to downstream recipients. "Knowingly relying" means you have
actual knowledge that, but for the patent license, your conveying the
covered work in a country, or your recipient's use of the covered work
in a country, would infringe one or more identifiable patents in that
country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or
arrangement, you convey, or propagate by procuring conveyance of, a
covered work, and grant a patent license to some of the parties
receiving the covered work authorizing them to use, propagate, modify
or convey a specific copy of the covered work, then the patent license
you grant is automatically extended to all recipients of the covered
work and works based on it.
A patent license is "discriminatory" if it does not include within
the scope of its coverage, prohibits the exercise of, or is
conditioned on the non-exercise of one or more of the rights that are
specifically granted under this License. You may not convey a covered
work if you are a party to an arrangement with a third party that is
in the business of distributing software, under which you make payment
to the third party based on the extent of your activity of conveying
the work, and under which the third party grants, to any of the
parties who would receive the covered work from you, a discriminatory
patent license (a) in connection with copies of the covered work
conveyed by you (or copies made from those copies), or (b) primarily
for and in connection with specific products or compilations that
contain the covered work, unless you entered into that arrangement,
or that patent license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting
any implied license or other defenses to infringement that may
otherwise be available to you under applicable patent law.
12. No Surrender of Others' Freedom.
If conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot convey a
covered work so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you may
not convey it at all. For example, if you agree to terms that obligate you
to collect a royalty for further conveying from those to whom you convey
the Program, the only way you could satisfy both those terms and this
License would be to refrain entirely from conveying the Program.
13. Use with the GNU Affero General Public License.
Notwithstanding any other provision of this License, you have
permission to link or combine any covered work with a work licensed
under version 3 of the GNU Affero General Public License into a single
combined work, and to convey the resulting work. The terms of this
License will continue to apply to the part which is the covered work,
but the special requirements of the GNU Affero General Public License,
section 13, concerning interaction through a network will apply to the
combination as such.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of
the GNU General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the
Program specifies that a certain numbered version of the GNU General
Public License "or any later version" applies to it, you have the
option of following the terms and conditions either of that numbered
version or of any later version published by the Free Software
Foundation. If the Program does not specify a version number of the
GNU General Public License, you may choose any version ever published
by the Free Software Foundation.
If the Program specifies that a proxy can decide which future
versions of the GNU General Public License can be used, that proxy's
public statement of acceptance of a version permanently authorizes you
to choose that version for the Program.
Later license versions may give you additional or different
permissions. However, no additional obligations are imposed on any
author or copyright holder as a result of your choosing to follow a
later version.
15. Disclaimer of Warranty.
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. Limitation of Liability.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
SUCH DAMAGES.
17. Interpretation of Sections 15 and 16.
If the disclaimer of warranty and limitation of liability provided
above cannot be given local legal effect according to their terms,
reviewing courts shall apply local law that most closely approximates
an absolute waiver of all civil liability in connection with the
Program, unless a warranty or assumption of liability accompanies a
copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
state the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If the program does terminal interaction, make it output a short
notice like this when it starts in an interactive mode:
<program> Copyright (C) <year> <name of author>
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate
parts of the General Public License. Of course, your program's commands
might be different; for a GUI interface, you would use an "about box".
You should also get your employer (if you work as a programmer) or school,
if any, to sign a "copyright disclaimer" for the program, if necessary.
For more information on this, and how to apply and follow the GNU GPL, see
<https://www.gnu.org/licenses/>.
The GNU General Public License does not permit incorporating your program
into proprietary programs. If your program is a subroutine library, you
may consider it more useful to permit linking proprietary applications with
the library. If this is what you want to do, use the GNU Lesser General
Public License instead of this License. But first, please read
<https://www.gnu.org/licenses/why-not-lgpl.html>.
reset/README.md
+317
View File
@@ -0,0 +1,317 @@
# TwinVine
![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeederA.png)
TwinVine combines two Python packages:
- [Vinefeeder](https://github.com/vinefeeder/TwinVine/blob/main/packages/vinefeeder/src/vinefeeder/README.md)
- [Envied](https://github.com/vinefeeder/TwinVine/blob/main/packages/envied/README.md)
TwinVine helps you find, select, and download media. Use the graphical front end when you want search and selection assistance, and use the command-line downloader for exact URLs.
Envied is forked from unshackle github.com/unshackle-dl/unshackle and I thank the developers for their effort.
## Key workflows
- Use `envied` when you already have an exact program URL.
- Use `vinefeeder` search when you only know a program name.
- Use the browse feature when you want to explore categories like Film, Drama, or Sport.
- Use Batch Mode to select and download multiple items from several services.
## Usage
TwinVine runs through the Python package manager `uv`.
### Run the main tools
```bash
uv run vinefeeder
uv run envied dl --select-titles <service> <url>
```
### Access envied from the GUI
- On Linux: choose **run envied** after clicking the GUI `envied` button.
- On Windows: close the GUI or press `Ctrl+C` to return to the terminal.
## Installation
### Prerequisites
Install `uv` if needed:
```bash
pip install uv
python3 -m pip install uv
```
Or install `python-uv` through your system package manager.
### Install TwinVine
```bash
git clone https://github.com/vinefeeder/TwinVine.git
cd TwinVine
uv clean
uv lock
uv sync
uv run vinefeeder --help
uv run envied --help
uv run envied dl -?
```
## Windows installation
This section is aimed at novice Windows users.
A printer-friendly checklist is available here:
[WINDOWS_INSTALL_CHECKLIST.txt](WINDOWS_INSTALL_CHECKLIST.txt)
### Windows quick overview
1. Install Git for Windows.
2. Clone the TwinVine repository.
3. Run the Windows installer script.
4. Verify that `uv` is installed.
5. Initialize TwinVine and verify the commands.
## Linux installation
On Linux, use the bundled shell script to install required binaries.
1. Open `Install-media-tools.sh` in a text editor.
2. Change the package manager command on lines 6 and 7 from `apt-get` to your distro's package manager (`dnf`, `pacman`, `zypper`, etc.).
3. Save the file.
4. Run:
```bash
sudo bash ./Install-media-tools.sh
```
5. If `uv` is still not installed, run:
```bash
wget -qO- https://astral.sh/uv/install.sh | sh
```
6. Then initialize TwinVine:
```bash
git clone https://github.com/vinefeeder/TwinVine.git
cd TwinVine
uv lock
uv sync
cp ./packages/envied/src/envied/envied-working-example.yaml ./packages/envied/src/envied/envied.yaml
uv run vinefeeder --help
uv run envied --help
uv run envied dl -?
```
## Locations
By default, downloaded files are saved to:
- `TwinVine/packages/envied/src/downloads/`
Edit `packages/envied/src/envied/envied.yaml` to change the media-download location and other values like email:password for a service.
- Use a full path.
- On Windows, use forward slashes: `C:/Users/Downloads`.
- On Linux, use `/home/user/Downloads`.
Cookies live in `packages/envied/src/`. Each cookie file should be named exactly for the service, for example `DNSP.txt`, and contain the service login cookie.
WVD files are stored in `TwinVine/WVDs/`, such as `device.wvd`.
Vaults are not configured locally by default. TwinVine uses a remote vault for caching and license retrieval and may display `DRMLab` as the license source.
## Linux note
Linux terminals can sometimes freeze after `envied` completes a download. If that happens, ensure `TERMINAL_RESET: True` is set in:
- `TwinVine/packages/vinefeeder/src/vinefeeder/config.yaml`
## Services
### Vinefeeder services
Vinefeeder currently supports search, browse, and list-select for these services:
- ALL4
- BBC
- ITVX
- MY5
- PLEX
- RTE
- STV
- TPTV
- TVNZ
- U
### Envied services
Envied supports a broader set of direct-download services:
- ALL4
- AUBC
- CBS
- CWTV
- DSCP
- iP
- MAX
- MY5
- NF
- PCOK
- PLEX
- RTE
- ROKU
- SPOT
- TPTV
- TVNZ
- YTBE
- ARD
- CBC
- CTV
- DSNP
- ITV
- MTSP
- NBLA
- NRK
- PLUTO
- STV
- TUBI
- UKTV
- ZDF
These services have web origins and not all have been tested.
## You can use AI to create your own Vinefeeder service!!
The prompt below was written by ChatGPT after refactoring a Vinefeeder service to reuse Envied service logic.
```
I am working on the TwinVine project:
https://github.com/vinefeeder/TwinVine
TwinVine contains two Python packages:
* `envied`: a command-line video downloader.
* `vinefeeder`: a graphical / interactive front end that searches, lists, selects, and then calls `envied` by subprocess.
I want to create or rewrite a Vinefeeder service by reusing the matching envied service instead of duplicating API logic.
Target service:
* Envied service path:
`packages/envied/src/envied/services/<SERVICE>/`
* Vinefeeder service path:
`packages/vinefeeder/src/vinefeeder/services/<SERVICE>/`
Please inspect the current code in both locations.
Goal:
Rewrite the Vinefeeder `<SERVICE>` service so that it imports and reuses the existing envied `<SERVICE>` service class wherever possible.
The Vinefeeder service should:
1. Preserve the standard Vinefeeder service interface:
* `receive()`
* `fetch_videos()`
* `second_fetch()`
* `fetch_videos_by_category()`
2. Use the envied service for:
* authentication / token handling
* search, if the envied service provides `search()`
* programme/title expansion via `get_titles()` or `get_titles_cached()`
* service-specific API calls already implemented in envied
3. Avoid reimplementing API endpoints that already exist in envied.
4. Keep Vinefeeder responsible only for:
* presenting search results
* letting the user choose a title
* expanding a multi-episode series into selectable individual episodes
* building one envied subprocess command per selected single title
5. Use `beaupy.select_multiple()` when there are multiple episodes available.
6. If the selected item is a movie, sport event, clip, highlight, or single episode, skip the beaupy episode list and call envied directly.
7. Preserve Vinefeeder’s existing download behaviour by calling:
`self.runsubprocess(command)`
where command should normally be shaped like:
`["uv", "run", "envied", "dl", *self.options_list, "<SERVICE>", url]`
8. Preserve support for Vinefeeder service options from config, using `split_options()` as existing services do.
9. If an envied service object needs a Click context, create the smallest safe adapter context needed rather than copying envied command-line internals unnecessarily.
10. If the envied service requires credentials, token cache, proxy state, or service config, reuse envied’s existing config-loading patterns as closely as possible.
11. Be careful not to authenticate twice unnecessarily. If `fetch_videos()` and `second_fetch()` are sequential, prefer reusing an already-authenticated envied service instance where safe.
12. Produce a complete replacement `__init__.py` for the Vinefeeder service.
13. Also explain:
* which envied methods/classes are being reused
* what Vinefeeder-specific logic remains
* any assumptions or caveats
* how I should test the rewritten service
Please do not merely describe the approach. Provide the actual rewritten Vinefeeder service code.
```
## Linux users only: Adding a little automation
In .bashrc in your home directory you can add useful aliases and scripts to speed the start-up of TwinVine.
Add the following lines at the end of your .bashrc, (or your system's equivalent), to start Vinefeeder by typing 'v' into a terminal window;
and 'e' to start envied with a pre-filled command. BE SURE TO EDIT your/path/to/Twinvine to reflect your needs.
```
alias v="cd /your/path/to/TwinVine/;uv run vinefeeder"
unalias e 2>/dev/null
e() {
cd /your/path/to/TwinVine || return
local cmd
read -e -i "uv run envied dl --select-titles " -p "TwinVine> " cmd || return
[[ -n "$cmd" ]] && history -s "$cmd" && eval "$cmd"
}
```
## Other README's
TwinVine/packages/vinefeeder/src/vinefeeder/README.md
for details for confuring Envied download options on a service by service basis.
TwinVine/packages/envied/README.md links to wiki (envied - envied's parent)
Images
![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeederA1.png)
![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder1.png)
![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder2.png)
![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder4.png)
![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder5.png)
![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder6.png)
![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder7.png)
![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder8.png)
![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder9.png)
![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder10.png)
![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeeder11.png)
![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/vinefeederB.png)
![TwinVine GUI](https://github.com/vinefeeder/TwinVine/blob/main/images/hellyes.png)
reset/UPDATES.md
+48
View File
@@ -0,0 +1,48 @@
**Updates**
**11 July 2026**
vinefeeder update to BBC service to better select UHD quality availability. Granularity is now at the episode level rather than series.
meaning that any series with mixed UHD and HD titles will no longer fail at the download stage
**11 May 2026**
Vinefeeder update to BBC UHD detection to enable automatic HLG selection.
Some service updates.
**Mar 2026**
Envied updated to match upstream to version 4.0.0
service updates
**19 Jan 2026**
- upstream envied to 2.3.0
- correction RTE vinefeeder service
**31 Dec 2025**
**Altered:**
- **Colour Render:** of --select-titles option title display
- **Display Ttiles:** for --select-titles option now include episode title and standard SxxExx designation
**29 Nov 2025**
- Followed upstream changes to envied V2.1.0
- Updated some services
- removed incorrectly duplicated vaults folder within src/envied/
**13 Sept 2025:**
- Added selected service name to screen display with GUI use. Featured in the original but got lost!
- corrected uv.lock to uv lock for initializing project
**18 Sept 2025:**
- Fully implemented Plex service - categories now selectable.
**22 Sept 2025:**
- Added RTE service with search and direct url entry.
- updated README
**28 Sept 2025**
- added HellYes GUI
- added images to reflect GUI change
**15 Oct 2025**
- corrected keyword search when using menu in terminal window. Previously that mode of search only took a url striing; now it takes keywords.
- code additions to reflect upstream envied changes.
reset/WINDOWS_INSTALL_CHECKLIST.txt
+57
View File
@@ -0,0 +1,57 @@
TwinVine Windows Installation Checklist
1. Install Git for Windows:
- Download from https://github.com/git-for-windows/git/releases/download/v2.52.0.windows.1/Git-2.52.0-64-bit.exe
- Run the installer.
2. Restart your computer.
3. Open PowerShell.
4. Change directory to your chosen install location.
5. Clone TwinVine:
```powershell
git clone https://github.com/vinefeeder/TwinVine.git
```
6. Close PowerShell.
7. Open PowerShell as Administrator:
- Right-click Windows PowerShell → Run as administrator.
8. Change directory to TwinVine:
```powershell
cd TwinVine
```
9. Run the Windows installer script:
```powershell
powershell -ExecutionPolicy Bypass -File .\Install-media-tools.ps1
```
10. After installation completes, close PowerShell and restart your computer.
11. Open PowerShell and verify uv is installed:
```powershell
uv
```
12. If uv is missing, install it as Administrator:
```powershell
powershell -ExecutionPolicy Bypass -c "irm https://github.com/astral-sh/uv/releases/download/0.9.18/uv-installer.ps1 | iex"
```
13. Close PowerShell and restart your computer.
14. Open PowerShell, go to TwinVine, and run:
```powershell
uv lock
uv sync
cp .\packages\envied\src\envied\envied-working-example.yaml .\packages\envied\src\envied\envied.yaml
uv run vinefeeder --help
uv run envied --help
uv run envied dl -?
```
15. You are ready to use TwinVine on Windows.
reset/WVDs/device.wvd
BIN
View File
Binary file not shown.
reset/docs/ADVANCED_CONFIG.md
+232
View File
@@ -0,0 +1,232 @@
# Advanced & System Configuration
This document covers advanced features, debugging, and system-level configuration options.
## serve (dict)
Configuration for the integrated server that provides CDM endpoints (Widevine/PlayReady) and a REST API for remote downloading.
Start the server with:
```bash
envied serve # Default: localhost:8786
envied serve -h 0.0.0.0 -p 8888 # Listen on all interfaces
envied serve --no-key # Disable authentication
envied serve --api-only # REST API only, no CDM endpoints
envied serve --remote-only # Only expose remote service session endpoints
```
### CLI Options
| Option | Default | Description |
| --- | --- | --- |
| `-h, --host` | `127.0.0.1` | Host to serve from |
| `-p, --port` | `8786` | Port to serve from |
| `--caddy` | `false` | Also serve with Caddy reverse-proxy for HTTPS |
| `--api-only` | `false` | Serve only the REST API, disable CDM endpoints |
| `--no-widevine` | `false` | Disable Widevine CDM endpoints |
| `--no-playready` | `false` | Disable PlayReady CDM endpoints |
| `--no-key` | `false` | Disable API key authentication (allows all requests) |
| `--debug-api` | `false` | Include tracebacks and stderr in API error responses |
| `--debug` | `false` | Enable debug logging for API operations |
| `--remote-only` | `false` | Only expose remote service session endpoints (health, services, search, session) |
### Configuration
- `api_secret` - Secret key for REST API authentication. Required unless `--no-key` is used. All API requests must include this key via the `X-Secret-Key` header.
- `compression_level` - Compression level for API payloads (manifests, cache, cookies). `0`=off, `1`=fast, `6`=balanced, `9`=max. Default: `1`.
- `session_ttl` - Session inactivity timeout in seconds. Each request resets the timer. Default: `300`.
- `max_sessions` - Maximum concurrent sessions before the oldest is evicted. Default: `100`.
- `services` - Optional global service allowlist. Only these service tags are exposed. If omitted, all services are available.
- `devices` - List of Widevine device files (.wvd). If not specified, auto-populated from the WVDs directory.
- `playready_devices` - List of PlayReady device files (.prd). If not specified, auto-populated from the PRDs directory.
- `users` - Dictionary mapping user secret keys to their access configuration:
- `devices` - List of Widevine devices this user can access
- `playready_devices` - List of PlayReady devices this user can access
- `username` - Internal logging name for the user (not visible to users)
- `services` - Optional per-user service allowlist. Effective access is the intersection of global and per-user allowlists.
#### Server-side `dl` defaults
Any key accepted by `/api/download` (see `docs/API.md`) can also be declared directly under `serve:` and the REST API will treat it as a default. Per-request bodies still win. Use this to raise concurrency, force `best_available`, etc. without each client repeating the values:
```yaml
serve:
api_secret: "..."
users: { ... }
downloads: 4 # parallel tracks per job
workers: 16 # threads per track
best_available: true
no_proxy_download: false
```
Layering: built-in defaults < `serve.*` overrides < service-specific defaults < request body.
For example,
```yaml
serve:
api_secret: "your-secret-key-here"
compression_level: 1
session_ttl: 300
max_sessions: 100
# services: # global allowlist (optional)
# - EXAMPLE1
# - EXAMPLE2
users:
secret_key_for_jane: # 32bit hex recommended, case-sensitive
devices: # list of allowed Widevine devices for this user
- generic_nexus_4464_l3
playready_devices: # list of allowed PlayReady devices for this user
- my_playready_device
username: jane # only for internal logging, users will not see this name
# services: # per-user allowlist (optional)
# - EXAMPLE1
secret_key_for_james:
devices:
- generic_nexus_4464_l3
username: james
# devices can be manually specified by path if you don't want to add it to
# envied's WVDs directory for whatever reason
# devices:
# - 'C:\Users\john\Devices\test_devices_001.wvd'
# playready_devices:
# - '/path/to/device.prd'
```
### REST API
When the server is running, interactive API documentation is available at:
- **Swagger UI**: `http://localhost:8786/api/docs/`
See [API.md](API.md) for full REST API documentation with endpoints, parameters, and examples.
---
## max_concurrent_downloads (int)
Maximum number of `/api/download` jobs the serve queue manager will execute in parallel. Each job runs the full `dl` pipeline (authenticate, fetch tracks, decrypt, mux) in its own worker subprocess. This is independent of `serve.downloads`, which controls parallel tracks **inside** a single job. Default: `2`.
```yaml
max_concurrent_downloads: 4
```
---
## download_job_retention_hours (int)
How long completed, failed, or cancelled download jobs remain queryable via `/api/download/jobs/{job_id}` before the periodic cleanup loop drops them. Default: `24`.
```yaml
download_job_retention_hours: 48
```
---
## debug (bool)
Enables comprehensive debug logging. Default: `false`
When enabled (either via config or the `-d`/`--debug` CLI flag):
- Sets console log level to DEBUG for verbose output
- Creates JSON Lines (`.jsonl`) debug log files with structured logging
- Logs detailed information about sessions, service configuration, DRM operations, and errors with full stack traces
For example,
```yaml
debug: true
```
---
## debug_keys (bool)
Controls whether actual decryption keys (CEKs) are included in debug logs. Default: `false`
When enabled:
- Content encryption keys are logged in debug output
- Only affects `content_key` and `key` fields (the actual CEKs)
- Key metadata (`kid`, `keys_count`, `key_id`) is always logged regardless of this setting
- Passwords, tokens, cookies, and session tokens remain redacted even when enabled
For example,
```yaml
debug_keys: true
```
---
## set_terminal_bg (bool)
Controls whether envied should set the terminal background color. Default: `false`
For example,
```yaml
set_terminal_bg: true
```
---
## update_checks (bool)
Check for updates from the GitHub repository on startup. Default: `true`.
---
## update_check_interval (int)
How often to check for updates, in hours. Default: `24`.
---
## title_cache_enabled (bool)
Enable or disable title metadata caching globally. Default: `true`.
---
## title_cache_time (int)
Title cache duration in seconds. Default: `1800` (30 minutes).
---
## title_cache_max_retention (int)
Maximum cache retention in seconds, used as fallback when the upstream API fails. Default: `86400` (24 hours).
---
## unicode_filenames (bool)
When `false`, replaces non-ASCII characters in output filenames with ASCII equivalents. Default: `false`.
---
## ipinfo_api_key (str)
Optional ipinfo.io token. When set, envied uses the ipinfo.io Lite endpoint for IP/geolocation lookups instead of the unauthenticated fallback.
---
## tmdb_api_key (str)
Optional TMDB API key, used for metadata enrichment and IMDb/TMDb tagging.
---
## simkl_client_id (str)
Optional Simkl client ID for metadata lookups.
---
## decrypt_labs_api_key (str)
Optional Decrypt Labs API key, used by services that integrate with the service.
---
reset/docs/API.md
+781
View File
@@ -0,0 +1,781 @@
# REST API Documentation
The envied REST API allows you to control downloads, search services, drive remote downloads from a thin client, and (optionally) co-host the pywidevine/pyplayready CDM. Start the server with `envied serve` and access the interactive Swagger UI at `http://localhost:8786/api/docs/`.
The server is built on **aiohttp** (not FastAPI). Implementation lives in `envied/commands/serve.py` and `envied/core/api/` (`routes.py`, `handlers.py`, `session_store.py`, `input_bridge.py`, `download_manager.py`, `download_worker.py`).
## Quick Start
```bash
# Start the server (no authentication)
envied serve --no-key
# Start with authentication (api_secret in envied.yaml)
envied serve
# Serve only the REST API (no pywidevine/pyplayready CDM)
envied serve --api-only
# Serve only the remote-dl session endpoints (CORS/Cloudflare friendly)
envied serve --remote-only
# Disable just one CDM
envied serve --no-widevine
envied serve --no-playready
# Verbose error responses (tracebacks/stderr in JSON)
envied serve --debug-api
```
`serve` flags:
| Flag | Description |
| --- | --- |
| `-h, --host` | Bind host (default `127.0.0.1`) |
| `-p, --port` | Bind port (default `8786`) |
| `--caddy` | Also launch Caddy using `Caddyfile` next to the envied config |
| `--api-only` | REST API only; skip the bundled pywidevine/pyplayready CDM endpoints |
| `--no-widevine` | Disable Widevine CDM endpoints |
| `--no-playready` | Disable PlayReady CDM endpoints |
| `--no-key` | Disable API key authentication entirely |
| `--debug-api` | Include tracebacks/stderr in error responses |
| `--debug` | Enable DEBUG-level logging for API operations |
| `--remote-only` | Expose only `/api/health`, `/api/services`, `/api/search`, and `/api/session/*` (implies `--api-only`) |
## Authentication
When `api_secret` is set in `envied.yaml`, all API requests require the **`X-Secret-Key`** header. There is no query-parameter fallback. `/api/health` is always reachable without authentication. `--no-key` disables auth entirely (not recommended for public-facing servers).
```yaml
# envied.yaml
serve:
api_secret: "your-master-secret" # falls back to global users map below
remote_only: false # also toggleable via --remote-only
services: ["EXAMPLE1", "EXAMPLE2"] # optional global service allowlist
users:
user-secret-1:
username: alice
devices: ["my_widevine_l3"] # Widevine WVD names this user may use
playready_devices: ["my_pr_sl2000"] # PlayReady PRD names; defaults to [] (no access)
services: ["EXAMPLE1"] # optional per-user allowlist (intersected with global)
user-secret-2:
username: bob
devices: []
playready_devices: []
```
### Service allowlists
`config.serve.services` is the global allowlist; `users.<key>.services` further narrows it per key. The effective set is the intersection. Endpoints affected: `/api/services`, `/api/search`, `/api/list-titles`, `/api/list-tracks`, `/api/download`, and all `/api/session/*` routes.
### CDM access (server-side decryption)
There is no separate "tier" flag. Whether the server can return KID:KEY for a session-mode download depends solely on the device lists configured for the calling user key:
- Empty `devices` and `playready_devices` -> server can only proxy CDM challenges; the client must run its own CDM and parse the license.
- Populated lists -> the client may set `mode: "server_cdm"` on `/api/session/{id}/license` and receive `{ "keys": { "<track_id>": { "<KID>": "<KEY>" } } }` instead of raw license bytes.
Per-service CDM type can be pinned via `config.cdm` (`widevine`/`playready`) or per-service `cdm_type`; otherwise the server picks the type the user has devices for.
### Server-side `dl` defaults
Any flag accepted by `/api/download` (see the table below) can be declared under `serve:` in `envied.yaml` and the API will apply it as a default. Request-body values still win. Useful for raising concurrency without changing every client call:
```yaml
serve:
api_secret: "..."
users: { ... }
downloads: 4 # parallel tracks per download job
workers: 16 # threads per track segment fetch
best_available: true
no_proxy_download: false
```
Layering order: built-in defaults < `serve.*` overrides < service-specific click defaults < request body.
---
## Endpoint Map
Standard endpoints (suppressed in `--remote-only` mode are marked R):
| Method | Path | R |
| --- | --- | :-: |
| GET | `/api/health` | ok |
| GET | `/api/services` | ok |
| POST | `/api/search` | ok |
| POST | `/api/list-titles` | hidden |
| POST | `/api/list-tracks` | hidden |
| POST | `/api/download` | hidden |
| GET | `/api/download/jobs` | hidden |
| GET | `/api/download/jobs/{job_id}` | hidden |
| DELETE | `/api/download/jobs/{job_id}` | hidden |
| POST | `/api/session/create` | ok |
| GET | `/api/session/{session_id}` | ok |
| DELETE | `/api/session/{session_id}` | ok |
| GET | `/api/session/{session_id}/titles` | ok |
| POST | `/api/session/{session_id}/tracks` | ok |
| POST | `/api/session/{session_id}/segments` | ok |
| POST | `/api/session/{session_id}/license` | ok |
| GET | `/api/session/{session_id}/prompt` | ok |
| POST | `/api/session/{session_id}/prompt` | ok |
CDM endpoints (`/{wvd}/...`, `/playready/{prd}/...`) are exposed unless `--api-only` / `--remote-only` / `--no-widevine` / `--no-playready` is set, and use pywidevine / pyplayready's own auth scheme.
---
## Endpoints
### GET /api/health
Health check with version and update information. Always reachable without auth.
```bash
curl http://localhost:8786/api/health
```
```json
{
"status": "ok",
"version": "4.0.0",
"update_check": {
"update_available": false,
"current_version": "4.0.0",
"latest_version": null
}
}
```
---
### GET /api/services
List all available streaming services (filtered by the effective allowlist for the caller).
```bash
curl -H "X-Secret-Key: $KEY" http://localhost:8786/api/services
```
Returns `{"services": [...]}`. Each entry has `tag`, `aliases`, `geofence`, `title_regex`, `url` (from `cli.short_help`), `help` (full docstring), and `cli_params` describing the service-level Click parameters.
---
### POST /api/search
Search for titles from a streaming service.
**Required parameters:**
| Parameter | Type | Description |
| --- | --- | --- |
| `service` | string | Service tag |
| `query` | string | Search query |
**Optional parameters:**
| Parameter | Type | Default | Description |
| --- | --- | --- | --- |
| `profile` | string | `null` | Profile for credentials/cookies |
| `proxy` | string | `null` | Proxy URI or country code |
| `no_proxy` | boolean | `false` | Disable all proxy use |
```bash
curl -X POST http://localhost:8786/api/search \
-H "X-Secret-Key: $KEY" \
-H "Content-Type: application/json" \
-d '{"service": "EXAMPLE1", "query": "example show"}'
```
```json
{
"results": [
{
"id": "abc123def456",
"title": "Example Show",
"description": null,
"label": "TV Show",
"url": "https://example.com/show/abc123def456"
}
],
"count": 1
}
```
---
### POST /api/list-titles
Get available titles (seasons/episodes/movies) for a service and title ID. Disabled in `--remote-only` mode.
**Required parameters:**
| Parameter | Type | Description |
| --- | --- | --- |
| `service` | string | Service tag |
| `title_id` | string | Title ID or URL |
```bash
curl -X POST http://localhost:8786/api/list-titles \
-H "X-Secret-Key: $KEY" \
-H "Content-Type: application/json" \
-d '{"service": "EXAMPLE1", "title_id": "abc123def456"}'
```
---
### POST /api/list-tracks
Get video, audio, and subtitle tracks for a title. Disabled in `--remote-only` mode.
**Required parameters:**
| Parameter | Type | Description |
| --- | --- | --- |
| `service` | string | Service tag |
| `title_id` | string | Title ID or URL |
**Optional parameters:**
| Parameter | Type | Default | Description |
| --- | --- | --- | --- |
| `wanted` | array | all | Episode filter (e.g., `["S01E01"]`) |
| `profile` | string | `null` | Profile for credentials/cookies |
| `proxy` | string | `null` | Proxy URI or country code |
| `no_proxy` | boolean | `false` | Disable all proxy use |
Returns video, audio, and subtitle tracks with codec, bitrate, resolution, language, and DRM information.
---
### POST /api/download
Start a download job. Returns immediately with a job ID (HTTP 202). Disabled in `--remote-only` mode.
**Required parameters:**
| Parameter | Type | Description |
| --- | --- | --- |
| `service` | string | Service tag |
| `title_id` | string | Title ID or URL |
**Quality and codec parameters:**
| Parameter | Type | Default | Description |
| --- | --- | --- | --- |
| `quality` | array[int] | best | Resolution(s) (e.g., `[1080, 2160]`) |
| `vcodec` | string or array | any | Video codec(s): `H264`, `H265`/`HEVC`, `VP9`, `AV1`, `VC1`, `VP8` |
| `acodec` | string or array | any | Audio codec(s): `AAC`, `AC3`, `EC3`, `AC4`, `OPUS`, `FLAC`, `ALAC`, `DTS`, `OGG` |
| `vbitrate` | int | highest | Video bitrate in kbps |
| `abitrate` | int | highest | Audio bitrate in kbps |
| `range` | array[string] | `["SDR"]` | Color range(s): `SDR`, `HDR10`, `HDR10+`, `HLG`, `DV`, `HYBRID` |
| `channels` | float | any | Audio channels (e.g., `5.1`, `7.1`) |
| `no_atmos` | boolean | `false` | Exclude Dolby Atmos tracks |
| `split_audio` | boolean | `null` | Create separate output per audio codec |
| `sub_format` | string | `null` | Output subtitle format: `SRT`, `VTT`, `ASS`, `SSA`, `TTML` |
**Episode selection:**
| Parameter | Type | Default | Description |
| --- | --- | --- | --- |
| `wanted` | array[string] | all | Episodes (e.g., `["S01E01", "S01E02-S01E05"]`) |
| `latest_episode` | boolean | `false` | Download only the most recent episode |
**Language parameters:**
| Parameter | Type | Default | Description |
| --- | --- | --- | --- |
| `lang` | array[string] | `["orig"]` | Language for video and audio (`orig` = original) |
| `v_lang` | array[string] | `[]` | Language override for video tracks only |
| `a_lang` | array[string] | `[]` | Language override for audio tracks only |
| `s_lang` | array[string] | `["all"]` | Language for subtitles |
| `require_subs` | array[string] | `[]` | Required subtitle languages (skip if missing) |
| `forced_subs` | boolean | `false` | Include forced subtitle tracks |
| `exact_lang` | boolean | `false` | Exact language matching (no variants) |
**Track selection:**
| Parameter | Type | Default | Description |
| --- | --- | --- | --- |
| `video_only` | boolean | `false` | Only download video tracks |
| `audio_only` | boolean | `false` | Only download audio tracks |
| `subs_only` | boolean | `false` | Only download subtitle tracks |
| `chapters_only` | boolean | `false` | Only download chapters |
| `no_video` | boolean | `false` | Skip video tracks |
| `no_audio` | boolean | `false` | Skip audio tracks |
| `no_subs` | boolean | `false` | Skip subtitle tracks |
| `no_chapters` | boolean | `false` | Skip chapters |
| `audio_description` | boolean | `false` | Include audio description tracks |
**Output and tagging:**
| Parameter | Type | Default | Description |
| --- | --- | --- | --- |
| `tag` | string | `null` | Override group tag |
| `repack` | boolean | `false` | Add REPACK tag to filename |
| `tmdb_id` | int | `null` | Use specific TMDB ID for tagging |
| `imdb_id` | string | `null` | Use specific IMDB ID (e.g., `tt1375666`) |
| `animeapi_id` | string | `null` | Anime database ID via AnimeAPI (e.g., `mal:12345`) |
| `enrich` | boolean | `false` | Override show title and year from external source |
| `no_folder` | boolean | `false` | Disable folder creation for TV shows |
| `no_source` | boolean | `false` | Remove source tag from filename |
| `no_mux` | boolean | `false` | Do not mux tracks into container |
| `output_dir` | string | `null` | Override output directory |
**Download behavior:**
| Parameter | Type | Default | Description |
| --- | --- | --- | --- |
| `profile` | string | `null` | Profile for credentials/cookies |
| `proxy` | string | `null` | Proxy URI or country code |
| `no_proxy` | boolean | `false` | Disable all proxy use |
| `no_proxy_download` | boolean | `false` | Bypass proxy for segment downloads only. Manifest, license, and auth still use proxy |
| `workers` | int | `null` | Max threads per track download |
| `downloads` | int | `1` | Concurrent track downloads |
| `slow` | boolean or string | `null` | Add randomized delay between titles. `true` = 60-120s, or `"MIN-MAX"` string (e.g., `"20-40"`). Min must be >= 20 |
| `best_available` | boolean | `false` | Continue if requested quality unavailable |
| `worst` | boolean | `false` | Select the lowest bitrate track within the specified quality. Requires `quality` |
| `skip_dl` | boolean | `false` | Skip download, only get decryption keys |
| `export` | boolean | `false` | Export manifest, track URLs, keys, and subtitles to JSON in the exports directory |
| `cdm_only` | boolean | `null` | Only use CDM (`true`) or only vaults (`false`) |
| `no_cache` | boolean | `false` | Bypass title cache |
| `reset_cache` | boolean | `false` | Clear title cache before fetching |
**Example:**
```bash
curl -X POST http://localhost:8786/api/download \
-H "X-Secret-Key: $KEY" \
-H "Content-Type: application/json" \
-d '{
"service": "EXAMPLE1",
"title_id": "abc123def456",
"wanted": ["S01E01"],
"quality": [1080, 2160],
"vcodec": ["H265"],
"acodec": ["AAC", "EC3"],
"range": ["HDR10", "SDR"],
"split_audio": true,
"lang": ["en"]
}'
```
```json
{
"job_id": "504db959-80b0-446c-a764-7924b761d613",
"status": "queued",
"created_time": "2026-02-27T18:00:00.000000"
}
```
---
### GET /api/download/jobs
List all download jobs with optional filtering and sorting. Disabled in `--remote-only` mode.
**Query parameters:**
| Parameter | Type | Default | Description |
| --- | --- | --- | --- |
| `status` | string | all | Filter by status: `queued`, `downloading`, `completed`, `failed`, `cancelled` |
| `service` | string | all | Filter by service tag |
| `sort_by` | string | `created_time` | Sort field: `created_time`, `started_time`, `completed_time`, `progress`, `status`, `service` |
| `sort_order` | string | `desc` | Sort order: `asc`, `desc` |
```bash
curl -H "X-Secret-Key: $KEY" "http://localhost:8786/api/download/jobs?status=completed"
```
---
### GET /api/download/jobs/{job_id}
Get detailed information about a specific download job including progress, parameters, and error details.
```json
{
"job_id": "504db959-80b0-446c-a764-7924b761d613",
"status": "completed",
"created_time": "2026-02-27T18:00:00.000000",
"service": "EXAMPLE1",
"title_id": "abc123def456",
"progress": 100.0,
"parameters": { },
"started_time": "2026-02-27T18:00:01.000000",
"completed_time": "2026-02-27T18:00:15.000000",
"output_files": [],
"error_message": null,
"error_details": null
}
```
---
### DELETE /api/download/jobs/{job_id}
Cancel a queued or running download job. Returns 400 if the job has already terminated.
---
## Remote Service Sessions
These endpoints back the `RemoteService` adapter in `envied/core/remote_service.py`. They let a thin `dl` client (or any consumer) authenticate against a service on the server, fetch titles/tracks/manifests, and either proxy CDM challenges or have the server resolve KID:KEY directly. The `dl` command's `RemoteService` adapter replaces the old `remote_dl` command. These endpoints are the only `/api/*` routes available in `--remote-only` mode (in addition to `health`, `services`, and `search`).
### POST /api/session/create
Authenticate against a service and open a session. Body fields:
| Field | Type | Description |
| --- | --- | --- |
| `service` | string | Service tag (required) |
| `title_id` | string | Title ID/URL (required) |
| `credentials` | object | Auth credentials forwarded to `Service.authenticate` |
| `cookies` | string | Cookie blob (Netscape or JSON) |
| `proxy` | string | Proxy URI or country code |
| `no_proxy` | bool | Force-disable proxies |
| `profile` | string | Profile name |
| `cache` | object | Optional pre-warmed title cache payload |
If the service requires interactive input during authentication, poll `GET /api/session/{id}/prompt` and submit responses via `POST /api/session/{id}/prompt` until status is `authenticated`.
**Request:**
```json
{
"service": "EXAMPLE1",
"title_id": "abc123def456",
"credentials": {"username": "alice", "password": "hunter2"},
"cookies": "# Netscape HTTP Cookie File\n...",
"proxy": "us",
"no_proxy": false,
"profile": "default",
"cache": {}
}
```
**Response (202-style; auth runs asynchronously):**
```json
{
"session_id": "f1c4a8b2-9c7e-4d2a-bf91-2d3e4f5a6b7c",
"service": "EXAMPLE1",
"status": "authenticating"
}
```
### GET /api/session/{session_id}
Returns session metadata. 404 if expired or unknown.
```json
{
"session_id": "f1c4a8b2-9c7e-4d2a-bf91-2d3e4f5a6b7c",
"service": "EXAMPLE1",
"valid": true,
"expires_in": 3600,
"track_count": 0,
"title_count": 0
}
```
### DELETE /api/session/{session_id}
Tears down the session, cancels any pending prompts, and returns any updated per-session cache files (base64-encoded, zlib-compressed) so the client can re-warm next time.
```json
{
"status": "ok",
"cache": {
"tokens": "eJzLSM3JyVcozy/KSVGo5AIAGgQEvQ=="
}
}
```
### GET /api/session/{session_id}/titles
Returns the resolved titles list.
```json
{
"session_id": "f1c4a8b2-9c7e-4d2a-bf91-2d3e4f5a6b7c",
"titles": [
{
"type": "episode",
"name": "Pilot",
"series_title": "Example Show",
"season": 1,
"number": 1,
"year": 2024,
"id": "ep-0001",
"language": "en"
},
{
"type": "movie",
"name": "Example Movie",
"year": 2024,
"id": "mov-0001",
"language": "en"
}
]
}
```
### POST /api/session/{session_id}/tracks
**Request:**
```json
{"title_id": "ep-0001"}
```
**Response:**
```json
{
"title": {
"type": "episode",
"name": "Pilot",
"series_title": "Example Show",
"season": 1,
"number": 1,
"year": 2024,
"id": "ep-0001",
"language": "en"
},
"video": [
{
"id": "v-1080p-h264",
"codec": "H264",
"codec_display": "H.264",
"bitrate": 6000,
"width": 1920,
"height": 1080,
"resolution": "1920x1080",
"fps": "23.976",
"range": "SDR",
"range_display": "SDR",
"language": "en",
"drm": [
{
"type": "widevine",
"pssh": "AAAAW3Bzc2gAAAAA7e+...",
"kids": ["abcdef0123456789abcdef0123456789"],
"license_url": "https://license.example.com/widevine"
}
],
"descriptor": "DASH",
"url": "https://cdn.example.com/manifest.mpd"
}
],
"audio": [
{
"id": "a-en-eac3",
"codec": "EC3",
"codec_display": "Dolby Digital Plus",
"bitrate": 640,
"channels": "5.1",
"language": "en",
"atmos": false,
"descriptive": false,
"drm": null,
"descriptor": "DASH",
"url": "https://cdn.example.com/manifest.mpd"
}
],
"subtitles": [
{
"id": "s-en-vtt",
"codec": "WebVTT",
"language": "en",
"forced": false,
"sdh": false,
"cc": false,
"descriptor": "DASH",
"url": "https://cdn.example.com/subs/en.vtt"
}
],
"chapters": [
{"timestamp": "00:00:00.000", "name": "Chapter 1"}
],
"attachments": [],
"manifests": [
{
"type": "dash",
"url": "https://cdn.example.com/manifest.mpd",
"data": "eJzNVk1v2zAM/Ss..."
}
],
"session_headers": {
"User-Agent": "Mozilla/5.0 ..."
},
"session_cookies": {
"session": "abc123"
},
"server_cdm_type": "widevine"
}
```
### POST /api/session/{session_id}/segments
**Request:**
```json
{"track_ids": ["v-1080p-h264", "a-en-eac3"]}
```
**Response:**
```json
{
"tracks": {
"v-1080p-h264": {
"descriptor": "DASH",
"url": "https://cdn.example.com/manifest.mpd",
"drm": [
{
"type": "widevine",
"pssh": "AAAAW3Bzc2gAAAAA7e+...",
"kids": ["abcdef0123456789abcdef0123456789"],
"license_url": "https://license.example.com/widevine"
}
],
"headers": {"User-Agent": "Mozilla/5.0 ..."},
"cookies": {"session": "abc123"},
"data": {}
},
"a-en-eac3": {
"descriptor": "DASH",
"url": "https://cdn.example.com/manifest.mpd",
"drm": null,
"headers": {"User-Agent": "Mozilla/5.0 ..."},
"cookies": {"session": "abc123"},
"data": {}
}
}
}
```
### POST /api/session/{session_id}/license
Two modes, selected by the `mode` field.
**`mode: "proxy"` (default)** -- forward a client-built CDM challenge to the service's license endpoint.
Request:
```json
{
"mode": "proxy",
"track_id": "v-1080p-h264",
"challenge": "CAESxQEK...",
"drm_type": "widevine",
"pssh": "AAAAW3Bzc2gAAAAA7e+..."
}
```
Response:
```json
{"license": "CAIS3wIK..."}
```
**`mode: "server_cdm"`** -- the server uses its own CDM to license the track and extract keys. Single-track form takes `track_id`; batch form takes `track_ids`. Requires the calling user key to have a matching device (`devices` for Widevine, `playready_devices` for PlayReady) in `envied.yaml`.
Request (batch):
```json
{
"mode": "server_cdm",
"track_ids": ["v-1080p-h264", "a-en-eac3"],
"drm_type": "widevine"
}
```
Response:
```json
{
"keys": {
"v-1080p-h264": {
"abcdef0123456789abcdef0123456789": "00112233445566778899aabbccddeeff"
},
"a-en-eac3": {
"abcdef0123456789abcdef0123456789": "00112233445566778899aabbccddeeff"
}
},
"drm_type": "widevine"
}
```
### GET /api/session/{session_id}/prompt
Polled by the client during interactive authentication (OTP, PIN, device codes). Backed by the `InputBridge` in `envied/core/api/input_bridge.py`; `Service.request_input()` blocks server-side until the client posts a response.
Pending input:
```json
{"status": "pending_input", "prompt": "Enter OTP code: "}
```
Other states:
```json
{"status": "authenticating"}
```
```json
{"status": "authenticated"}
```
```json
{"status": "failed", "error": "Invalid credentials"}
```
### POST /api/session/{session_id}/prompt
Unblocks the server-side `request_input()` call.
Request:
```json
{"response": "123456"}
```
Response:
```json
{"status": "accepted"}
```
---
## Error Responses
All endpoints return consistent error responses:
```json
{
"status": "error",
"error_code": "INVALID_PARAMETERS",
"message": "Invalid vcodec: XYZ. Must be one of: H264, H265, VP9, AV1, VC1, VP8",
"timestamp": "2026-02-27T18:00:00.000000+00:00",
"details": { }
}
```
Common error codes:
- `INVALID_INPUT` -- malformed request body
- `INVALID_PARAMETERS` -- invalid parameter values
- `MISSING_SERVICE` -- service tag not provided
- `INVALID_SERVICE` -- service not found or not in the caller's allowlist
- `SERVICE_ERROR` -- service initialization or runtime error
- `AUTH_FAILED` -- authentication failure
- `NOT_FOUND` / `TRACK_NOT_FOUND` / session not found -- job/session/track/title missing
- `INTERNAL_ERROR` -- unexpected server error
When `--debug-api` is enabled, error responses include additional `debug_info` with tracebacks and stderr output.
Authentication errors from the auth middleware are returned as `{"status": 401, "message": "..."}` (not the standard error envelope).
---
## Download Job Lifecycle
```
queued -> downloading -> completed
\-> failed
queued -> cancelled
downloading -> cancelled
```
Jobs are retained for 24 hours after completion (override via top-level `download_job_retention_hours` in `envied.yaml`). The server runs up to 2 concurrent download jobs by default; override via top-level `max_concurrent_downloads`. This is independent of `serve.downloads`, which controls parallel tracks **within** a single job.
Remote sessions are managed by `SessionStore` (`envied/core/api/session_store.py`); idle sessions and their `InputBridge` instances are cleaned up by a background loop started/stopped with the app lifecycle.
reset/docs/DOWNLOAD_CONFIG.md
+322
View File
@@ -0,0 +1,322 @@
# Download & Processing Configuration
This document covers configuration options related to downloading and processing media content.
## downloader
envied ships a single unified downloader at `envied/core/downloaders/requests.py`. The legacy
`aria2c`, `curl_impersonate`, and `n_m3u8dl_re` backends have been removed; their config blocks no
longer have any effect.
The unified downloader:
- Works with both a standard `requests.Session` and `RnetSession` (rnet/BoringSSL TLS impersonation,
which replaces the previous `curl_cffi` backend). When a service exposes its own session via
`self.session`, TLS fingerprinting is preserved on every segment.
- Uses adaptive chunk sizing between **512 KB and 4 MB**, picked from the response `Content-Length`.
- Spawns **up to `min(16, cpu_count + 4)` worker threads** by default for segmented downloads
(override via `--workers` / `dl.workers`).
- Resumes interrupted downloads via HTTP `Range` requests (a sibling `<file>.!dev` control file
marks an in-progress download).
- Has a single-URL fast path: if the server supports byte ranges and the file is at least 64 MB,
the file is split into 16 MB parts and downloaded in parallel into a pre-allocated file.
- Is selected per-track via `track.downloader`, which defaults to this unified `requests` downloader.
There is no `downloader:` config key to set anymore. Setting one to a legacy value will emit a
`DeprecationWarning` and otherwise be ignored.
---
## dl (dict)
Pre-define default options and switches of the `dl` command.
The values will be ignored if explicitly set in the CLI call.
The Key must be the same value Python click would resolve it to as an argument.
E.g., `@click.option("-r", "--range", "range_", type=...` actually resolves as `range_` variable.
For example to set the default primary language to download to German,
```yaml
lang: de
```
You can also set multiple preferred languages using a list, e.g.,
```yaml
lang:
- en
- fr
```
to set how many tracks to download concurrently to 4 and download threads to 16,
```yaml
downloads: 4
workers: 16
```
to set `--bitrate=CVBR` for a specific service,
```yaml
lang: de
EXAMPLE:
bitrate: CVBR
```
or to change the output subtitle format from the default (original format) to WebVTT,
```yaml
sub_format: vtt
```
### All Available `dl` Keys
Below is a comprehensive list of keys that can be pre-defined in the `dl` section. Each corresponds
to a CLI option on the `dl` command. CLI arguments always take priority over config values.
**Quality and codec:**
| Key | Type | Default | Description |
| --- | --- | --- | --- |
| `quality` | int or list | best | Resolution(s) to download (e.g., `1080`, `[1080, 2160]`) |
| `vcodec` | str or list | any | Video codec(s): `H264`, `H265`, `VP9`, `AV1`, `VC1` |
| `acodec` | str or list | any | Audio codec(s): `AAC`, `AC3`, `EC3`, `AC4`, `OPUS`, `FLAC`, `ALAC`, `DTS` |
| `vbitrate` | int | highest | Video bitrate in kbps |
| `abitrate` | int | highest | Audio bitrate in kbps |
| `vbitrate_range` | str | none | Video bitrate window in kbps, format `MIN-MAX` (e.g., `6000-7000`) |
| `abitrate_range` | str | none | Audio bitrate window in kbps, format `MIN-MAX` |
| `real_video_bitrate` | bool | `false` | Probe actual media size to compute true video bitrates, overriding the manifest's declared value (`-rvb`). See [Real bitrate probing](#real-bitrate-probing) |
| `real_audio_bitrate` | bool | `false` | Same as above for audio tracks (`-rab`). Slower than video (more renditions) |
| `range_` | str or list | `SDR` | Color range(s): `SDR`, `HDR10`, `HDR10+`, `HLG`, `DV`, `HYBRID` |
| `channels` | float | any | Audio channels (e.g., `5.1`, `7.1`) |
| `worst` | bool | `false` | Select the lowest bitrate track within the specified quality. Requires `quality` |
| `best_available` | bool | `false` | Continue if requested quality is unavailable |
**Language:**
| Key | Type | Default | Description |
| --- | --- | --- | --- |
| `lang` | str or list | `orig` | Language for video and audio (`orig` = original language) |
| `v_lang` | list | `[]` | Language override for video tracks only |
| `a_lang` | list | `[]` | Language override for audio tracks only |
| `s_lang` | list | `["all"]` | Language for subtitles |
| `require_subs` | list | `[]` | Required subtitle languages (skip title if missing) |
| `forced_subs` | bool | `false` | Include forced subtitle tracks |
| `exact_lang` | bool | `false` | Exact language matching (no regional variants) |
| `latest_episode` | bool | `false` | Download only the single most recent episode of a series |
**Track selection:**
| Key | Type | Default | Description |
| --- | --- | --- | --- |
| `video_only` | bool | `false` | Only download video tracks |
| `audio_only` | bool | `false` | Only download audio tracks |
| `subs_only` | bool | `false` | Only download subtitle tracks |
| `chapters_only` | bool | `false` | Only download chapters |
| `no_video` | bool | `false` | Skip video tracks |
| `no_audio` | bool | `false` | Skip audio tracks |
| `no_subs` | bool | `false` | Skip subtitle tracks |
| `no_chapters` | bool | `false` | Skip chapters |
| `no_atmos` | bool | `false` | Exclude Dolby Atmos audio tracks |
| `audio_description` | bool | `false` | Include audio description tracks |
**Output and tagging:**
| Key | Type | Default | Description |
| --- | --- | --- | --- |
| `tag` | str | config default | Override group tag |
| `repack` | bool | `false` | Add REPACK tag to output filename |
| `sub_format` | str | original | Output subtitle format: `srt`, `vtt`, `ass`, `ssa`, `ttml` |
| `no_folder` | bool | `false` | Disable folder creation for TV shows |
| `no_source` | bool | `false` | Remove source tag from filename |
| `no_mux` | bool | `false` | Do not mux tracks into a container file |
| `split_audio` | bool | `false` | Create separate output files per audio codec |
| `export` | bool | `false` | Write a JSON sidecar with manifest URLs, subtitles, per-track KID:KEY, codec/track info |
**Metadata enrichment:**
| Key | Type | Default | Description |
| --- | --- | --- | --- |
| `tmdb_id` | int | `null` | Use specific TMDB ID for tagging |
| `imdb_id` | str | `null` | Use specific IMDB ID (e.g., `tt1375666`) |
| `animeapi_id` | str | `null` | Anime database ID via AnimeAPI (e.g., `mal:12345`, `anilist:98765`) |
| `enrich` | bool | `false` | Override show title and year from external source. Requires `tmdb_id`, `imdb_id`, or `animeapi_id` |
**Download behavior:**
| Key | Type | Default | Description |
| --- | --- | --- | --- |
| `downloads` | int | `1` | Concurrent track downloads |
| `workers` | int | `min(16, cpu_count + 4)` | Max threads per track download (segments / ranged parts) |
| `slow` | bool or `MIN-MAX` | `false` | Randomized delay between titles. `true` uses 60-120s; pass `MIN-MAX` (e.g., `20-40`) for a custom range |
| `no_proxy_download` | bool | `false` | Bypass proxy for segment downloads only. Manifest, license, and auth still use proxy |
| `skip_dl` | bool | `false` | Skip download, only get decryption keys |
| `cdm_only` | bool | `null` | Only use CDM (`true`) or only vaults (`false`) |
### Real bitrate probing
Some services declare inaccurate `bandwidth`/`BANDWIDTH` in their manifests — often
a peak or nominal figure that is far from the real average. Because `track.bitrate`
drives the track listing, sorting, and `--vbitrate` / `--vbitrate-range` selection,
a wrong value picks the wrong track.
`-rvb` / `--real-video-bitrate` (and `-rab` / `--real-audio-bitrate` for audio)
probe the actual media size and overwrite `track.bitrate` with the measured value
(`bytes * 8 / duration`) before listing and selection. So `-rvb --list` shows the
true numbers, and `-rvb --vbitrate-range 6000-7000` selects against them. Without
the flag, behaviour is unchanged (the manifest value is used).
How it works:
- **Single-file tracks** (one whole file per rendition — e.g. DASH `SegmentBase`
or services that collapse to a `BaseURL`) are measured **exactly**: the whole
file size over the track duration.
- **Multi-segment tracks** (most HLS) are a **sampled estimate** — a spread of
segments is probed and extrapolated, typically within a few percent. Segment
bytes include container overhead, so MPEG-TS HLS reads a few percent above the
demuxed stream (this is the real *delivered* size).
- Only the top renditions per quality tier are probed (video grouped by
codec + range, audio by codec + channels + language), in parallel, then extended
downward only as far as needed to keep ranking correct. This keeps the pass fast
even when a service exposes dozens of renditions.
- Tracks whose duration cannot be determined fall back to `ffprobe`; probe failures
are non-fatal and leave the manifest bitrate in place.
Per-track before→after values are logged at debug level (run with `-d`); the
corrected values always appear in the Available Tracks panel.
You can also set per-service `dl` overrides (see [Service Integration & Authentication Configuration](SERVICE_CONFIG.md)):
```yaml
dl:
lang: en
downloads: 4
workers: 16
EXAMPLE:
bitrate: CVBR
EXAMPLE2:
worst: true
quality: 1080
```
---
## audio (dict)
Configuration for audio track selection.
- `codec_priority`
Optional list of audio codec names defining the preferred order when multiple audio
tracks share the same bitrate and language. Listed codecs are ranked in the order given.
Codecs not in the list retain their bitrate-based ordering and are placed after all
listed codecs (i.e. soft priority — nothing is dropped).
Atmos tracks still take precedence over codec priority, and audio description tracks
are still moved to the end.
Valid codec names: `AAC`, `AC3`, `EC3`, `AC4`, `OPUS`, `OGG`, `DTS`, `ALAC`, `FLAC`.
For example,
```yaml
audio:
codec_priority: [FLAC, ALAC, AC4, EC3, DTS, AC3, OPUS, AAC, OGG]
```
Or to only prefer a subset (e.g. surround codecs first, everything else falls back to
bitrate order):
```yaml
audio:
codec_priority: [EC3, DTS, AC3, AAC]
```
When unset, audio tracks are sorted by bitrate alone (with Atmos/descriptive rules still
applied).
---
## subtitle (dict)
Configuration for subtitle processing and conversion.
- `conversion_method`
Method to use for converting subtitles between formats. Default: `"auto"`
- `"auto"` — Smart routing: uses subby for WebVTT/SAMI, pycaption for others.
- `"subby"` — Always use subby with advanced processing.
- `"pycaption"` — Use only pycaption library (no SubtitleEdit, no subby).
- `"subtitleedit"` — Prefer SubtitleEdit when available, fall back to pycaption.
- `"pysubs2"` — Use pysubs2 library (supports SRT/SSA/ASS/WebVTT/TTML/SAMI/MicroDVD/MPL2/TMP).
- `sdh_method`
Method to use for SDH (hearing impaired) stripping. Default: `"auto"`
- `"auto"` — Try subby (SRT only), then SubtitleEdit (if available), then subtitle-filter.
- `"subby"` — Use subby library (SRT only).
- `"subtitleedit"` — Use SubtitleEdit tool (Windows only, falls back to subtitle-filter).
- `"filter-subs"` — Use subtitle-filter library directly.
- `strip_sdh`
Automatically create stripped (non-SDH) versions of SDH subtitles. Default: `true`
- `convert_before_strip`
Auto-convert VTT/other formats to SRT before using subtitle-filter for SDH stripping.
Ensures compatibility when subtitle-filter is used as fallback. Default: `true`
- `preserve_formatting`
Preserve original subtitle formatting (tags, positioning, styling).
When `true`, skips pycaption processing for WebVTT files to keep tags like `<i>`, `<b>`,
positioning intact. Combined with no `sub_format` setting, ensures subtitles remain in
their original format. Default: `true`
- `output_mode`
Output mode for subtitles. Default: `"mux"`
- `"mux"` — Embed subtitles in MKV container only.
- `"sidecar"` — Save subtitles as separate files only.
- `"both"` — Embed in MKV and save as sidecar files.
- `sidecar_format`
Format for sidecar subtitle files when `output_mode` is `"sidecar"` or `"both"`. Default: `"srt"`
Options: `srt`, `vtt`, `ass`, `original` (keep current format).
For example,
```yaml
subtitle:
conversion_method: auto
sdh_method: auto
strip_sdh: true
convert_before_strip: true
preserve_formatting: true
output_mode: mux
sidecar_format: srt
```
---
## decryption (str | dict)
Choose what software to use to decrypt DRM-protected content throughout envied where needed.
You may provide a single decryption method globally or a mapping of service tags to
decryption methods.
Options:
- `shaka` (default) - Shaka Packager - <https://github.com/shaka-project/shaka-packager>
- `mp4decrypt` - mp4decrypt from Bento4 - <https://github.com/axiomatic-systems/Bento4>
Note that Shaka Packager is the traditional method and works with most services. mp4decrypt
is an alternative that may work better with certain services that have specific encryption formats.
Example mapping:
```yaml
decryption:
EXAMPLE: mp4decrypt
EXAMPLE2: shaka
default: shaka
```
The `default` entry is optional. If omitted, `shaka` will be used for services not listed.
Simple configuration (single method for all services):
```yaml
decryption: mp4decrypt
```
---
reset/docs/DRM_CONFIG.md
+481
View File
@@ -0,0 +1,481 @@
# DRM & CDM Configuration
This document covers Digital Rights Management (DRM) and Content Decryption Module (CDM) configuration options.
## cdm (dict)
Pre-define which Widevine or PlayReady device to use for each Service by Service Tag as Key (case-sensitive).
The value should be a WVD or PRD filename without the file extension. When
loading the device, envied will look in both the `WVDs` and `PRDs` directories
for a matching file.
For example,
```yaml
EXAMPLE: chromecdm_903_l3
EXAMPLE2: nexus_6_l1
```
You may also specify this device based on the profile used.
For example,
```yaml
EXAMPLE: chromecdm_903_l3
EXAMPLE2: nexus_6_l1
EXAMPLE3:
john_sd: chromecdm_903_l3
jane_uhd: nexus_5_l1
```
You can also specify a fallback value to predefine if a match was not made.
This can be done using `default` key. This can help reduce redundancy in your specifications.
For example, the following has the same result as the previous example, as well as all other
services and profiles being pre-defined to use `chromecdm_903_l3`.
```yaml
EXAMPLE2: nexus_6_l1
EXAMPLE3:
jane_uhd: nexus_5_l1
default: chromecdm_903_l3
```
You can also select CDMs based on video resolution using comparison operators (`>=`, `>`, `<=`, `<`)
or exact match on the resolution height.
For example,
```yaml
EXAMPLE:
"<=1080": generic_android_l3 # Use L3 for 1080p and below
">1080": nexus_5_l1 # Use L1 for above 1080p (1440p, 2160p)
default: generic_android_l3 # Fallback if no quality match
```
You can mix profiles and quality thresholds in the same service:
```yaml
EXAMPLE:
john: example_l3_profile # Profile-based selection
"<=720": example_mobile_l3 # Quality-based selection
"1080": example_standard_l3 # Exact match for 1080p
">=1440": example_premium_l1 # Quality-based selection
default: example_standard_l3 # Fallback
```
---
## remote_cdm (list\[dict])
Configure remote CDM (Content Decryption Module) APIs to use for decrypting DRM-protected content.
Remote CDMs allow you to use high-security CDMs (L1/L2 for Widevine, SL2000/SL3000 for PlayReady) without
having the physical device files locally.
envied supports multiple types of remote CDM providers:
1. **DecryptLabs CDM** - Official DecryptLabs KeyXtractor API with intelligent caching
2. **Custom API CDM** - Highly configurable adapter for any third-party CDM API
3. **Legacy PyWidevine Serve** - Standard pywidevine serve-compliant APIs
The name of each defined remote CDM can be referenced in the `cdm` configuration as if it was a local device file.
### DecryptLabs Remote CDM
DecryptLabs provides a professional CDM API service with support for multiple device types and intelligent key caching.
**Supported Devices:**
- **Widevine**: `ChromeCDM` (L3), `L1` (Security Level 1), `L2` (Security Level 2)
- **PlayReady**: `SL2` (SL2000), `SL3` (SL3000)
**Configuration:**
```yaml
remote_cdm:
# Widevine L1 Device
- name: decrypt_labs_l1
type: decrypt_labs # Required: identifies as DecryptLabs CDM
device_name: L1 # Required: must match exactly (L1, L2, ChromeCDM, SL2, SL3)
host: https://keyxtractor.decryptlabs.com
secret: YOUR_API_KEY # Your DecryptLabs API key
# Widevine L2 Device
- name: decrypt_labs_l2
type: decrypt_labs
device_name: L2
host: https://keyxtractor.decryptlabs.com
secret: YOUR_API_KEY
# Chrome CDM (L3)
- name: decrypt_labs_chrome
type: decrypt_labs
device_name: ChromeCDM
host: https://keyxtractor.decryptlabs.com
secret: YOUR_API_KEY
# PlayReady SL2000
- name: decrypt_labs_playready_sl2
type: decrypt_labs
device_name: SL2
device_type: PLAYREADY # Required for PlayReady
host: https://keyxtractor.decryptlabs.com
secret: YOUR_API_KEY
# PlayReady SL3000
- name: decrypt_labs_playready_sl3
type: decrypt_labs
device_name: SL3
device_type: PLAYREADY
host: https://keyxtractor.decryptlabs.com
secret: YOUR_API_KEY
```
**Features:**
- Intelligent key caching system (reduces API calls)
- Automatic integration with envied's vault system
- Support for both Widevine and PlayReady
- Multiple security levels (L1, L2, L3, SL2000, SL3000)
**Note:** The `device_type` field determines whether the CDM operates in PlayReady or Widevine mode.
Setting `device_type: PLAYREADY` (or using `device_name: SL2` / `SL3`) activates PlayReady mode.
The `security_level` field is auto-computed from `device_name` when not specified (e.g., SL2 defaults
to 2000, SL3 to 3000, and Widevine devices default to 3). You can override these if needed.
### Custom API Remote CDM
A highly configurable CDM adapter that can work with virtually any third-party CDM API through YAML configuration.
This allows you to integrate custom CDM services without writing code.
**Basic Example:**
```yaml
remote_cdm:
- name: custom_chrome_cdm
type: custom_api # Required: identifies as Custom API CDM
host: https://your-cdm-api.com
timeout: 30 # Optional: request timeout in seconds
device:
name: ChromeCDM
type: CHROME # CHROME, ANDROID, PLAYREADY
system_id: 27175
security_level: 3
auth:
type: bearer # bearer, header, basic, body
key: YOUR_API_TOKEN
endpoints:
get_request:
path: /get-challenge
method: POST
decrypt_response:
path: /get-keys
method: POST
caching:
enabled: true # Enable key caching
use_vaults: true # Integrate with vault system
```
**Advanced Example with Field Mapping:**
```yaml
remote_cdm:
- name: advanced_custom_api
type: custom_api
host: https://api.example.com
device:
name: L1
type: ANDROID
security_level: 1
# Authentication configuration
auth:
type: header
header_name: X-API-Key
key: YOUR_SECRET_KEY
custom_headers:
User-Agent: envied/3.1.0
X-Client-Version: "1.0"
# Endpoint configuration
endpoints:
get_request:
path: /v2/challenge
method: POST
timeout: 30
decrypt_response:
path: /v2/decrypt
method: POST
timeout: 30
# Request parameter mapping
request_mapping:
get_request:
param_names:
init_data: pssh # Rename 'init_data' to 'pssh'
scheme: device_type # Rename 'scheme' to 'device_type'
static_params:
api_version: "2.0" # Add static parameter
decrypt_response:
param_names:
license_request: challenge
license_response: license
# Response field mapping
response_mapping:
get_request:
fields:
challenge: data.challenge # Deep field access
session_id: session.id
success_conditions:
- status == 'ok' # Validate response
decrypt_response:
fields:
keys: data.keys
key_fields:
kid: key_id # Map 'kid' field
key: content_key # Map 'key' field
caching:
enabled: true
use_vaults: true
check_cached_first: true # Check cache before API calls
```
**Supported Authentication Types:**
- `bearer` - Bearer token authentication
- `header` - Custom header authentication
- `basic` - HTTP Basic authentication
- `body` - Credentials in request body
- `query` - Authentication added to query string parameters
### Legacy PyWidevine Serve Format
Standard [pywidevine] serve-compliant remote CDM configuration (backwards compatibility).
```yaml
remote_cdm:
- name: legacy_chrome_cdm
device_name: chrome
device_type: CHROME
system_id: 27175
security_level: 3
host: https://domain.com/api
secret: secret_key
```
**Note:** If the `type` field is not specified, the entry is treated as a legacy pywidevine serve CDM.
[pywidevine]: https://github.com/rlaphoenix/pywidevine
---
## decrypt_labs_api_key (str)
API key for DecryptLabs CDM service integration.
When set, enables the use of DecryptLabs remote CDM services in your `remote_cdm` configuration.
This is used specifically for `type: "decrypt_labs"` entries in the remote CDM list.
For example,
```yaml
decrypt_labs_api_key: "your_api_key_here"
```
**Note**: This is different from the per-CDM `secret` field in `remote_cdm` entries. This provides a global
API key that can be referenced across multiple DecryptLabs CDM configurations. If a `remote_cdm` entry with
`type: "decrypt_labs"` does not have a `secret` field specified, the global `decrypt_labs_api_key` will be
used as a fallback.
---
## decryption (str|dict)
Configure which decryption tool to use for DRM-protected content. Default: `shaka`.
Supported values:
- `shaka` - Shaka Packager (default)
- `mp4decrypt` - Bento4 mp4decrypt
You can specify a single decrypter for all services:
```yaml
decryption: shaka
```
Or configure per-service with a `DEFAULT` fallback:
```yaml
decryption:
DEFAULT: shaka
EXAMPLE: mp4decrypt
EXAMPLE2: shaka
```
Service keys are case-insensitive (normalized to uppercase internally).
---
## MonaLisa DRM
MonaLisa is a WASM-based DRM system that uses local key extraction and two-stage segment decryption.
Unlike Widevine and PlayReady, MonaLisa does not use a challenge/response flow with a license server.
Instead, the PSSH value (ticket) is provided directly by the service API, and keys are extracted
locally via a WASM module.
### Requirements
- **ML-Worker binary**: Must be available on your system `PATH` (discovered via `binaries.ML_Worker`).
This is the binary that performs stage-1 decryption.
### Decryption stages
1. **ML-Worker binary**: Removes MonaLisa encryption layer (bbts -> ents). The key is passed via command-line argument.
2. **AES-ECB decryption**: Final decryption with service-provided key.
MonaLisa uses per-segment decryption during download (not post-download like Widevine/PlayReady),
so segments are decrypted as they are downloaded.
**Note:** MonaLisa is configured per-service rather than through global config options. Services
that use MonaLisa handle ticket/key retrieval and CDM initialization internally.
---
## key_vaults (list\[dict])
Key Vaults store your obtained Content Encryption Keys (CEKs) and Key IDs per-service.
This can help reduce unnecessary License calls even during the first download. This is because a Service may
provide the same Key ID and CEK for both Video and Audio, as well as for multiple resolutions or bitrates.
You can have as many Key Vaults as you would like. It's nice to share Key Vaults or use a unified Vault on
Teams as sharing CEKs immediately can help reduce License calls drastically.
Four types of Vaults are in the Core codebase: API, SQLite, MySQL, and HTTP. API and HTTP make HTTP requests to a RESTful API,
whereas SQLite and MySQL directly connect to an SQLite or MySQL Database.
Note: SQLite and MySQL vaults have to connect directly to the Host/IP. It cannot be in front of a PHP API or such.
Beware that some Hosting Providers do not let you access the MySQL server outside their intranet and may not be
accessible outside their hosting platform.
Additional behavior:
- `no_push` (bool): Optional per-vault flag. When `true`, the vault will not receive pushed keys (writes) but
will still be queried and can provide keys for lookups. Useful for read-only/backup vaults.
### Using an API Vault
API vaults use a specific HTTP request format, therefore API or HTTP Key Vault APIs from other projects or services may
not work in envied. The API format can be seen in the [API Vault Code](envied/vaults/API.py).
```yaml
- type: API
name: "John#0001's Vault" # arbitrary vault name
uri: "https://key-vault.example.com" # api base uri (can also be an IP or IP:Port)
# uri: "127.0.0.1:80/key-vault"
# uri: "https://api.example.com/key-vault"
token: "random secret key" # authorization token
# no_push: true # optional; make this API vault read-only (lookups only)
```
### Using a MySQL Vault
MySQL vaults can be either MySQL or MariaDB servers. I recommend MariaDB.
A MySQL Vault can be on a local or remote network, but I recommend SQLite for local Vaults.
```yaml
- type: MySQL
name: "John#0001's Vault" # arbitrary vault name
host: "127.0.0.1" # host/ip
# port: 3306 # port (defaults to 3306)
database: vault # database used for envied
username: jane11
password: Doe123
# no_push: false # optional; defaults to false
```
I recommend giving only a trustable user (or yourself) CREATE permission and then use envied to cache at least one CEK
per Service to have it create the tables. If you don't give any user permissions to create tables, you will need to
make tables yourself.
- Use a password on all user accounts.
- Never use the root account with envied (even if it's you).
- Do not give multiple users the same username and/or password.
- Only give users access to the database used for envied.
- You may give trusted users CREATE permission so envied can create tables if needed.
- Other uses should only be given SELECT and INSERT permissions.
### Using an SQLite Vault
SQLite Vaults are usually only used for locally stored vaults. This vault may be stored on a mounted Cloud storage
drive, but I recommend using SQLite exclusively as an offline-only vault. Effectively this is your backup vault in
case something happens to your MySQL Vault.
```yaml
- type: SQLite
name: "My Local Vault" # arbitrary vault name
path: "C:/Users/Jane11/Documents/envied/data/key_vault.db"
# no_push: true # optional; commonly true for local backup vaults
```
**Note**: You do not need to create the file at the specified path.
SQLite will create a new SQLite database at that path if one does not exist.
Try not to accidentally move the `db` file once created without reflecting the change in the config, or you will end
up with multiple databases.
If you work on a Team I recommend every team member having their own SQLite Vault even if you all use a MySQL vault
together.
### Using an HTTP Vault
HTTP Vaults provide flexible HTTP-based key storage with support for multiple API modes. This vault type
is useful for integrating with various third-party key vault APIs.
```yaml
- type: HTTP
name: "My HTTP Vault"
host: "https://vault-api.example.com"
api_key: "your_api_key" # or use 'password' field
api_mode: "json" # query, json, or decrypt_labs
# username: "user" # required for query mode only
# no_push: false # optional; defaults to false
```
**Supported API Modes:**
- `query` - Uses GET requests with query parameters. Requires `username` field.
- `json` - Uses POST requests with JSON payloads. Token-based authentication.
- `decrypt_labs` - DecryptLabs API format. Read-only mode (`no_push` is forced to `true`).
**Example configurations:**
```yaml
# Query mode (requires username)
- type: HTTP
name: "Query Vault"
host: "https://api.example.com/keys"
username: "myuser"
password: "mypassword"
api_mode: "query"
# JSON mode
- type: HTTP
name: "JSON Vault"
host: "https://api.example.com/vault"
api_key: "secret_token"
api_mode: "json"
# DecryptLabs mode (read-only)
- type: HTTP
name: "DecryptLabs Cache"
host: "https://keyxtractor.decryptlabs.com/cache"
api_key: "your_decrypt_labs_api_key"
api_mode: "decrypt_labs"
```
**Note**: The `decrypt_labs` mode is always read-only and cannot receive pushed keys.
---
reset/docs/GLUETUN.md
+243
View File
@@ -0,0 +1,243 @@
# Gluetun VPN Proxy
Gluetun provides Docker-managed VPN proxies supporting 50+ VPN providers.
## Prerequisites
**Docker must be installed and running.**
```bash
# Linux
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER # Then log out/in
# Windows/Mac
# Install Docker Desktop: https://www.docker.com/products/docker-desktop/
```
## Quick Start
### 1. Configuration
Add to `~/.config/envied/envied.yaml`:
```yaml
proxy_providers:
gluetun:
providers:
windscribe:
vpn_type: openvpn
credentials:
username: "YOUR_OPENVPN_USERNAME"
password: "YOUR_OPENVPN_PASSWORD"
```
### 2. Usage
Use 2-letter country codes directly:
```bash
envied dl SERVICE CONTENT --proxy gluetun:windscribe:us
envied dl SERVICE CONTENT --proxy gluetun:windscribe:uk
```
Format: `gluetun:provider:region`
## Provider Credential Requirements
**OpenVPN (Recommended)**: Most providers support OpenVPN with just `username` and `password` - the simplest setup.
**WireGuard**: Requires private keys and varies by provider. See the [Gluetun Wiki](https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers) for provider-specific requirements. Note that `vpn_type` defaults to `wireguard` if not specified.
## Getting Your Credentials
### Windscribe (OpenVPN)
1. Go to [windscribe.com/getconfig/openvpn](https://windscribe.com/getconfig/openvpn)
2. Log in with your Windscribe account
3. Select any location and click "Get Config"
4. Copy the username and password shown
### NordVPN (OpenVPN)
1. Go to [NordVPN Service Credentials](https://my.nordaccount.com/dashboard/nordvpn/manual-configuration/service-credentials/)
2. Log in with your NordVPN account
3. Generate or view your service credentials
4. Copy the username and password
> **Note**: Use service credentials, NOT your account email/password.
### WireGuard Credentials (Advanced)
WireGuard requires private keys instead of username/password. See the [Gluetun Wiki](https://github.com/qdm12/gluetun-wiki/tree/main/setup/providers) for provider-specific WireGuard setup.
## Configuration Examples
**OpenVPN (Recommended)**
Most providers support OpenVPN with just username and password:
```yaml
providers:
windscribe:
vpn_type: openvpn
credentials:
username: YOUR_OPENVPN_USERNAME
password: YOUR_OPENVPN_PASSWORD
nordvpn:
vpn_type: openvpn
credentials:
username: YOUR_SERVICE_USERNAME
password: YOUR_SERVICE_PASSWORD
```
**WireGuard (Advanced)**
WireGuard can be faster but requires more complex credential setup:
```yaml
# NordVPN/ProtonVPN (only private_key needed)
providers:
nordvpn:
vpn_type: wireguard
credentials:
private_key: YOUR_PRIVATE_KEY
# Surfshark/Mullvad/IVPN (private_key AND addresses required)
surfshark:
vpn_type: wireguard
credentials:
private_key: YOUR_PRIVATE_KEY
addresses: 10.x.x.x/32
# Windscribe (all three credentials required)
windscribe:
vpn_type: wireguard
credentials:
private_key: YOUR_PRIVATE_KEY
addresses: 10.x.x.x/32
preshared_key: YOUR_PRESHARED_KEY
```
## Server Selection
Most providers use `SERVER_COUNTRIES`, but some use `SERVER_REGIONS`:
| Variable | Providers |
|----------|-----------|
| `SERVER_COUNTRIES` | NordVPN, ProtonVPN, Surfshark, Mullvad, ExpressVPN, and most others |
| `SERVER_REGIONS` | Windscribe, VyprVPN, VPN Secure |
envied handles this automatically - just use 2-letter country codes.
### Per-Provider Server Mapping
You can explicitly map region codes to country names, cities, or hostnames per provider:
```yaml
providers:
nordvpn:
vpn_type: openvpn
credentials:
username: YOUR_USERNAME
password: YOUR_PASSWORD
server_countries:
us: "United States"
uk: "United Kingdom"
server_cities:
us: "New York"
server_hostnames:
us: "us1239.nordvpn.com"
```
### Specific Server Selection
Use a `<country><number>` region (e.g. `us1239`) to target a specific server. envied builds the
hostname automatically per provider:
| Provider | Hostname format |
|----------|-----------------|
| NordVPN | `us1239.nordvpn.com` |
| Surfshark | `us-1239.prod.surfshark.com` |
| ExpressVPN | `us-1239.expressvpn.com` |
| CyberGhost | `us-s1239.cg-dialup.net` |
| Other | `us1239` (passed as-is to `SERVER_HOSTNAMES`) |
### Extra Environment Variables
You can pass additional Gluetun environment variables per provider using `extra_env`:
```yaml
providers:
nordvpn:
vpn_type: openvpn
credentials:
username: YOUR_USERNAME
password: YOUR_PASSWORD
extra_env:
LOG_LEVEL: debug
```
## Global Settings
```yaml
proxy_providers:
gluetun:
providers: {...}
base_port: 8888 # Starting port (default: 8888)
auto_cleanup: true # Remove containers on exit (default: true)
verify_ip: true # Verify IP matches region (default: true)
container_prefix: "envied-gluetun" # Docker container name prefix (default: "envied-gluetun")
auth_user: username # Proxy auth (optional)
auth_password: password # Proxy auth (optional)
```
## Features
- **Container Reuse**: First request takes 10-30s; subsequent requests are instant. Containers created by other envied processes are auto-detected via `docker inspect` and reused.
- **Ready Detection**: Waits up to 60s for both the HTTP proxy to listen (`[http proxy] listening`) and the VPN tunnel to come up (`initialization sequence completed` or `public ip address is`) before returning the proxy URI. Bails early on `fatal` or `invalid credentials` log lines.
- **IP Verification**: When `verify_ip: true` (default), looks up the exit IP via `ipinfo.io` through the proxy and compares country code to the requested region. Retries 3 times with exponential backoff (1s, 2s, 4s).
- **Concurrent Sessions**: Multiple downloads share the same container; ports are allocated thread-safely starting at `base_port`.
- **Specific Servers**: Use `--proxy gluetun:nordvpn:us1239` for specific server selection (see table above).
- **Automatic Image Pull**: The Gluetun Docker image (`qmcgaw/gluetun:latest`) is pulled automatically on first use (5 min timeout).
- **Secure Credentials**: Credentials are passed via temporary env files (mode 0600), then zero-overwritten and unlinked after `docker run`. They never appear in process listings.
- **Auto Cleanup**: Containers are removed via `atexit` (Ctrl+C still works normally). Disable with `auto_cleanup: false` to leave them stopped instead.
## Container Management
```bash
# View containers
docker ps | grep envied-gluetun
# Check logs
docker logs envied-gluetun-nordvpn-us
# Remove all containers
docker ps -a | grep envied-gluetun | awk '{print $1}' | xargs docker rm -f
```
## Troubleshooting
### Docker Permission Denied (Linux)
```bash
sudo usermod -aG docker $USER
# Then log out and log back in
```
### VPN Connection Failed
Check container logs for specific errors:
```bash
docker logs envied-gluetun-nordvpn-us
```
Common issues:
- Invalid/missing credentials
- Windscribe WireGuard requires `preshared_key` (can be empty string, but must be set in credentials)
- VPN provider server issues
- Container startup timeout (default 60 seconds)
## Resources
- [Gluetun Wiki](https://github.com/qdm12/gluetun-wiki) - Official provider documentation
- [Gluetun GitHub](https://github.com/qdm12/gluetun)
reset/docs/NETWORK_CONFIG.md
+178
View File
@@ -0,0 +1,178 @@
# Network & Proxy Configuration
This document covers network and proxy configuration options for bypassing geofencing and managing connections.
## proxy_providers (dict)
Enable external proxy provider services. These proxies will be used automatically where needed as defined by the
Service's GEOFENCE class property, but can also be explicitly used with `--proxy`. You can specify which provider
to use by prefixing it with the provider key name, e.g., `--proxy basic:de` or `--proxy nordvpn:de`. Some providers
support specific query formats for selecting a country/server.
### basic (dict[str, str|list])
Define a mapping of country to proxy to use where required.
The keys are region Alpha 2 Country Codes. Alpha 2 Country Codes are `[a-z]{2}` codes, e.g., `us`, `gb`, and `jp`.
Don't get this mixed up with language codes like `en` vs. `gb`, or `ja` vs. `jp`.
Do note that each key's value can be a list of strings, or a string. For example,
```yaml
us:
- "http://john%40email.tld:password123@proxy-us.domain.tld:8080"
- "http://jane%40email.tld:password456@proxy-us.domain2.tld:8080"
de: "https://127.0.0.1:8080"
```
Note that if multiple proxies are defined for a region, then by default one will be randomly chosen.
You can choose a specific one by specifying it's number, e.g., `--proxy basic:us2` will choose the
second proxy of the US list.
### nordvpn (dict)
Set your NordVPN Service credentials with `username` and `password` keys to automate the use of NordVPN as a Proxy
system where required.
You can also specify specific servers to use per-region with the `server_map` key.
Sometimes a specific server works best for a service than others, so hard-coding one for a day or two helps.
You can also select servers by city using the format `--proxy nordvpn:us:seattle` or `--proxy nordvpn:ca:calgary`.
For example,
```yaml
username: zxqsR7C5CyGwmGb6KSvk8qsZ # example of the login format
password: wXVHmht22hhRKUEQ32PQVjCZ
server_map:
us: 12 # force US server #12 for US proxies
```
The username and password should NOT be your normal NordVPN Account Credentials.
They should be the `Service credentials` which can be found on your Nord Account Dashboard.
Once set, you can also specifically opt in to use a NordVPN proxy by specifying `--proxy nordvpn:gb` or such.
You can even set a specific server number this way, e.g., `--proxy nordvpn:gb2366`.
Note that `gb` is used instead of `uk` to be more consistent across regional systems.
### surfsharkvpn (dict)
Enable Surfshark VPN proxy service using Surfshark Service credentials (not your login password).
You may pin specific server IDs per region using `server_map`.
You can also select servers by city using the format `--proxy surfsharkvpn:us:seattle`.
```yaml
username: your_surfshark_service_username # https://my.surfshark.com/vpn/manual-setup/main/openvpn
password: your_surfshark_service_password # service credentials, not account password
server_map:
us: 3844 # force US server #3844
gb: 2697 # force GB server #2697
au: 4621 # force AU server #4621
```
### hola
Enable Hola VPN proxy service. Requires the `hola-proxy` binary to be installed and available in your PATH.
No configuration is needed under `proxy_providers`. Hola is loaded automatically when the `hola-proxy` binary
is detected.
Once available, use `--proxy hola:us` or similar to connect through Hola.
### windscribevpn (dict)
Enable Windscribe VPN proxy service using static OpenVPN service credentials.
Use the service credentials from https://windscribe.com/getconfig/openvpn (not your account login credentials).
```yaml
proxy_providers:
windscribevpn:
username: openvpn_username # From https://windscribe.com/getconfig/openvpn
password: openvpn_password # Service credentials, NOT your account password
```
#### Server Mapping
You can optionally pin specific servers using `server_map`:
```yaml
proxy_providers:
windscribevpn:
username: openvpn_username
password: openvpn_password
server_map:
us: us-central-096.totallyacdn.com # Force specific US server
gb: uk-london-001.totallyacdn.com # Force specific UK server
```
Once configured, use `--proxy windscribevpn:us` or `--proxy windscribevpn:gb` etc. to connect through Windscribe.
You can also select specific servers by number (e.g., `--proxy windscribevpn:sg007`) or filter by city
(e.g., `--proxy windscribevpn:ca:toronto`).
### gluetun (dict)
Docker-managed VPN proxy supporting 50+ VPN providers via Gluetun. See [GLUETUN.md](GLUETUN.md) for full
configuration and usage details.
```yaml
proxy_providers:
gluetun:
providers:
windscribe:
vpn_type: openvpn
credentials:
username: "YOUR_OPENVPN_USERNAME"
password: "YOUR_OPENVPN_PASSWORD"
```
Usage: `--proxy gluetun:windscribe:us`
---
## headers (dict)
Case-Insensitive dictionary of headers that all Services begin their Request Session state with.
All requests will use these unless changed explicitly or implicitly via a Server response.
These should be sane defaults and anything that would only be useful for some Services should not
be put here.
Avoid headers like 'Accept-Encoding' as that would be a compatibility header that the underlying
HTTP backend (rnet) will set for you as part of its browser impersonation profile.
I recommend using,
```yaml
Accept-Language: "en-US,en;q=0.8"
User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
```
---
## HTTP Session Backend
envied uses [`rnet`](https://github.com/0x676e67/rnet) (Rust + BoringSSL) for HTTP with TLS
fingerprinting. `RnetSession` is a drop-in `requests.Session` replacement and is what
`self.session` exposes to services. It supports:
- Browser/app impersonation via named `rnet.Impersonate` presets (Chrome, Edge, Firefox, Safari,
OkHttp, etc.) — picks JA3, ALPN, HTTP/2 SETTINGS and header order to match the chosen client.
- Native rnet proxy support (HTTP, HTTPS, SOCKS5) — used by all proxy providers below.
- Cookie-jar and `requests`-style `data=` / `json=` / `headers=` kwargs for compatibility.
The legacy `curl_cffi` backend has been removed. The config key is still spelled
`curl_impersonate` for backward compatibility, but its value now selects an rnet preset.
### curl_impersonate (dict)
```yaml
curl_impersonate:
browser: Chrome131 # exact rnet.Impersonate preset name
```
`browser` must be an exact `rnet.Impersonate` preset name (e.g. `Chrome131`, `Chrome124`,
`Edge101`, `Firefox133`, `Safari18`, `OkHttp4_12`). See the rnet README for the full list.
Default when unset: `Chrome131`.
---
reset/docs/OUTPUT_CONFIG.md
+287
View File
@@ -0,0 +1,287 @@
# Output & Naming Configuration
This document covers output file organization and naming configuration options.
## filenames (dict)
Override the default filenames used across envied.
The filenames use various variables that are replaced during runtime.
The following filenames are available and may be overridden:
- `log` - Log filenames. Uses `{name}` and `{time}` variables.
- `debug_log` - Debug log filenames. Uses `{service}` and `{time}` variables.
- `config` - Service configuration filenames.
- `root_config` - Root configuration filename.
- `chapters` - Chapter export filenames. Uses `{title}` and `{random}` variables.
- `subtitle` - Subtitle export filenames. Uses `{id}` and `{language}` variables.
For example,
```yaml
filenames:
log: "envied_{name}_{time}.log"
debug_log: "envied_debug_{service}_{time}.jsonl"
config: "config.yaml"
root_config: "envied.yaml"
chapters: "Chapters_{title}_{random}.txt"
subtitle: "Subtitle_{id}_{language}.srt"
```
---
## output_template (dict)
Configure custom output filename templates for movies, series, and songs.
This is **required** in your `envied.yaml` — a warning is shown if not configured.
Available variables: `{title}`, `{year}`, `{season}`, `{episode}`, `{season_episode}`, `{episode_name}`,
`{quality}`, `{resolution}`, `{source}`, `{audio}`, `{audio_channels}`, `{audio_full}`,
`{video}`, `{hdr}`, `{hfr}`, `{atmos}`, `{dual}`, `{multi}`, `{tag}`, `{edition}`, `{repack}`,
`{lang_tag}`, `{track_number}`, `{artist}`, `{album}`, `{disc}`
Add `?` suffix to make a variable conditional (omitted when empty): `{year?}`, `{hdr?}`, `{repack?}`
```yaml
output_template:
# Scene-style (dot-separated)
movies: '{title}.{year}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}'
series: '{title}.{year?}.{season_episode}.{episode_name?}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}'
songs: '{track_number}.{title}.{repack?}.{edition?}.{source?}.WEB-DL.{audio_full}.{atmos?}-{tag}'
# Plex-friendly (space-separated)
# movies: '{title} ({year}) {quality}'
# series: '{title} {season_episode} {episode_name?}'
# songs: '{track_number}. {title}'
```
Example outputs:
- Scene movies: `Example.Movie.2024.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG`
- Scene movies (REPACK): `Example.Movie.2024.REPACK.2160p.EXAMPLE.WEB-DL.DDP5.1.H.265-TAG`
- Scene series: `Example.Show.2024.S01E01.Pilot.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG`
- Plex movies: `Example Movie (2024) 1080p`
### folder (optional)
Controls the folder name for downloaded content. Uses the same template variables as the file templates above.
If not configured, the default folder naming is used:
- Movies: `Title (Year)`
- Series: Derived from the `series` template with episode-specific variables removed
- Songs: `Artist - Album (Year)`
`folder` accepts either a single string (applies to all title kinds) or a mapping with per-kind
templates keyed by `movies`, `series`, and/or `songs`. Unknown keys are warned about and ignored.
```yaml
output_template:
movies: '{title}.{year}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}'
series: '{title}.{year?}.{season_episode}.{episode_name?}.{repack?}.{edition?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}'
songs: '{track_number}.{title}.{repack?}.{edition?}.{source?}.WEB-DL.{audio_full}.{atmos?}-{tag}'
# Scene-style folder (single template, applies to all kinds)
folder: '{title}.{year?}.{repack?}.{edition?}.{lang_tag?}.{quality}.{source}.WEB-DL.{dual?}.{multi?}.{audio_full}.{atmos?}.{hdr?}.{hfr?}.{video}-{tag}'
# Plex-friendly folder
# folder: '{title} ({year?})'
# Per-kind folder templates
# folder:
# movies: '{title} ({year})'
# series: '{title} ({year?})'
# songs: '{artist} - {album} ({year?})'
```
Example outputs:
- Scene folder: `Example.Show.2024.S01.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG/`
- Plex folder: `Example Show (2024)/`
---
---
## language_tags (dict)
Automatically adds language-based identifiers (e.g., `DANiSH`, `NORDiC`, `DKsubs`) to output filenames
based on audio and subtitle track languages. Use `{lang_tag?}` in your `output_template` to place the tag.
Rules are evaluated in order; the first matching rule wins. All conditions within a single rule
must match (AND logic). If no rules match, `{lang_tag?}` is cleanly removed from the filename.
### Conditions
| Condition | Type | Description |
|-----------|------|-------------|
| `audio` | string | Matches if any selected audio track has this language |
| `subs_contain` | string | Matches if any selected subtitle has this language |
| `subs_contain_all` | list | Matches if subtitles include ALL listed languages |
Language matching uses fuzzy matching (e.g., `en` matches `en-US`, `en-GB`).
### Example: Nordic tagging
```yaml
language_tags:
rules:
- audio: da
tag: DANiSH
- audio: sv
tag: SWEDiSH
- audio: nb
tag: NORWEGiAN
- audio: en
subs_contain_all: [da, sv, nb]
tag: NORDiC
- audio: en
subs_contain: da
tag: DKsubs
output_template:
movies: '{title}.{year?}.{lang_tag?}.{quality}.{source}.WEB-DL.{audio_full}.{video}-{tag}'
```
Example outputs:
- Danish audio: `Example.Show.S01E01.DANiSH.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG`
- English audio + multiple Nordic subs: `Example.Show.S01E01.NORDiC.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG`
- English audio + Danish subs only: `Example.Show.S01E01.DKsubs.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG`
- No matching languages: `Example.Show.S01E01.1080p.EXAMPLE.WEB-DL.DDP5.1.H.264-TAG`
### Example: Other regional tags
```yaml
language_tags:
rules:
- audio: nl
tag: DUTCH
- audio: de
tag: GERMAN
- audio: fr
subs_contain: en
tag: ENGFR
- audio: fr
tag: FRENCH
```
---
## unicode_filenames (bool)
Allow Unicode characters in output filenames. When `false`, Unicode characters are transliterated
to ASCII equivalents. Default: `false`.
---
## tag (str)
Group or Username to postfix to the end of download filenames following a dash.
Use `{tag}` in your output template to include it.
For example, `tag: "J0HN"` will have `-J0HN` at the end of all download filenames.
---
## tag_group_name (bool)
Enable/disable tagging downloads with your group name when `tag` is set. Default: `true`.
---
## tag_imdb_tmdb (bool)
Enable/disable tagging downloaded files with IMDB/TMDB/TVDB identifiers (when available). Default: `true`.
---
## muxing (dict)
- `set_title`
Set the container title to `Show SXXEXX Episode Name` or `Movie (Year)`. Default: `true`
- `merge_audio`
Merge all audio tracks into each output file. Default: `true`
- `true`: All selected audio tracks are muxed into one MKV per quality.
- `false`: Separate MKV per (quality, audio_codec) combination.
For example: `Title.1080p.AAC.mkv`, `Title.1080p.EC3.mkv`.
Note: The `--split-audio` CLI flag overrides this setting. When `--split-audio` is passed,
`merge_audio` is effectively set to `false` for that run.
- `default_language` (dict)
Override which track is flagged as the default in the muxed MKV, regardless
of the title's original language. Useful when you always want your player to
open on a specific language (e.g. always default to Polish audio even on
English originals). Only affects the MKV `--default-track` flag — track
selection (`-l`, `--alang`, etc.) is unchanged. All keys are optional; each
track type falls back to its previous default rule when the configured
language isn't present in the manifest.
- `audio`: BCP-47 tag (e.g. `pl`, `en`, `pt-BR`). Wins over `is_original_lang`.
The `--original-flag` continues to mark the true original-audio track.
- `video`: BCP-47 tag. Wins over the title-language / first-track rule.
- `subtitle`: BCP-47 tag. Wins over the "forced sub matching audio" rule.
Languages are matched with the same close-match logic used elsewhere
(`pt` matches `pt-BR`, etc.). Supports per-service overrides like the rest
of `muxing`.
```yaml
muxing:
default_language:
audio: pl
video: pl
subtitle: pl
```
---
## chapter_fallback_name (str)
The Chapter Name to use when exporting a Chapter without a Name.
The default is no fallback name at all and no Chapter name will be set.
The fallback name can use the following variables in f-string style:
- `{i}`: The Chapter number starting at 1.
E.g., `"Chapter {i}"`: "Chapter 1", "Intro", "Chapter 3".
- `{j}`: A number starting at 1 that increments any time a Chapter has no title.
E.g., `"Chapter {j}"`: "Chapter 1", "Intro", "Chapter 2".
These are formatted with f-strings, directives are supported.
For example, `"Chapter {i:02}"` will result in `"Chapter 01"`.
---
## directories (dict)
Override the default directories used across envied.
The directories are set to common values by default.
The following directories are available and may be overridden,
- `commands` - CLI Command Classes.
- `services` - Service Classes.
- `vaults` - Vault Classes.
- `fonts` - Font files (ttf or otf).
- `downloads` - Downloads.
- `temp` - Temporary files or conversions during download.
- `cache` - Expiring data like Authorization tokens, or other misc data.
- `cookies` - Expiring Cookie data.
- `logs` - Logs.
- `exports` - JSON sidecar exports written when `--export` is used on `dl`.
- `wvds` - Widevine Devices.
- `prds` - PlayReady Devices.
- `dcsl` - Device Certificate Status List.
Notes:
- `services` accepts either a single directory or a list of directories to search for service modules.
For example,
```yaml
directories:
downloads: "D:/Downloads/envied"
temp: "D:/Temp/envied"
```
There are directories not listed that cannot be modified as they are crucial to the operation of envied.
---
reset/docs/SERVICE_CONFIG.md
+246
View File
@@ -0,0 +1,246 @@
# Service Integration & Authentication Configuration
This document covers service-specific configuration, authentication, and metadata integration options.
## services (dict)
Configuration data for each Service. The Service will have the data within this section merged into the per-service
`config.yaml` (located in the service's directory) before being provided to the Service class.
Think of this config to be used for more sensitive configuration data, like user or device-specific API keys, IDs,
device attributes, and so on. A per-service `config.yaml` file is typically shared and not meant to be modified,
so use this for any sensitive configuration data.
The Key is the Service Tag, but can take any arbitrary form for its value. It's expected to begin as either a list or
a dictionary.
For example,
```yaml
EXAMPLE:
client:
auth_scheme: MESSO
# ... more sensitive data
```
### Per-Service Configuration Overrides
You can override many global configuration options on a per-service basis by nesting them under the
service tag in the `services` section. Supported override keys include: `dl`, `subtitle`, `muxing`,
`headers`, `proxy_map`, `title_map`, and more.
Overrides are merged with global config (not replaced) -- only specified keys are overridden, others
use global defaults. CLI arguments always take priority over service-specific config.
For example,
```yaml
services:
RATE_LIMITED_SERVICE:
dl:
downloads: 2 # Limit concurrent track downloads
workers: 4 # Reduce workers to avoid rate limits
headers:
User-Agent: "..." # Service-specific UA override
```
Note: envied uses a single unified `requests`-based downloader. The legacy `aria2c`,
`n_m3u8dl_re`, and `curl_impersonate` override sections have been removed.
### title_map (dict)
Rewrites service-provided titles before naming and output. Some services name a title differently
from how you want it stored, which can break library matching (e.g. a regional variant reusing the
international name). Keys are the exact title string the service returns; values are the desired
output title.
```yaml
services:
EXAMPLE:
title_map:
Service Title: Desired Title
```
Episodes are matched on their show title, Movies and Songs on their name. The remap is applied
after the title cache (so edits take effect without a cache reset) and before any `--enrich`
override (so an explicit enrich still wins).
It applies on the local `dl` path, the `import` command, and the remote client (`dl --remote`).
For remote services the **client's** `title_map` is applied to the titles returned by the server,
so you can rename titles for services you don't have installed locally. The server sends raw
titles and does not remap, leaving the final name fully under the client's control.
### Service Class Conventions
Each service directory under `envied/services/` exports a class extending
`envied.core.service.Service`. The class name must match the directory name (the service tag).
Key class variables (defined on `Service` or by service-level idiom):
- `ALIASES: tuple[str, ...]` — alternative tags accepted on the CLI. Empty by default.
- `GEOFENCE: tuple[str, ...]` — ISO country codes the service is available in. Empty == no geofence.
- `TITLE_RE: str` — regex (with named groups, e.g. `(?P<id>...)`, `(?P<type>...)`) used by the
service to parse the CLI title argument. Service-level idiom, not declared on the base class.
- `NO_SUBTITLES: bool` — service-level idiom indicating the service has no subtitle tracks.
`self.*` helpers available after `super().__init__(ctx)`:
- `self.session` — pre-configured HTTP session (`requests.Session`, or `RnetSession` when TLS
impersonation is active). Cookies, headers, proxies pre-applied.
- `self.config` — merged service config (per-service `config.yaml` plus the `services.<TAG>` block
from `envied.yaml`).
- `self.log` — `logging.Logger` named for the service class.
- `self.cache` — generic `Cacher` for arbitrary key/value persistence.
- `self.title_cache` — specialized `TitleCacher` for title metadata.
- `self.track_request` — `TrackRequest` built from CLI flags. Fields: `codecs: list[Video.Codec]`,
`ranges: list[Video.Range]` (defaults to `[SDR]`), `best_available: bool`. Services may
read or rewrite these (e.g. force HEVC for HDR ranges).
- `self.credential` — set during `authenticate()`; `None` if cookies-only.
- `self.current_region` — lowercase ISO country code from proxy/geolocation, or `None`.
- `self.request_input(prompt: str) -> str` — interactive prompt. Falls through to `input()`
locally; under `serve`, the attached `InputBridge` relays the prompt to the remote client.
Driving CLI flags (parsed into `self.track_request`):
- `-v` / `--vcodec` — comma-separated `Video.Codec` list (e.g. `H264,H265`).
- `-a` / `--acodec` — comma-separated audio codec list.
- `-r` / `--range` — comma-separated `Video.Range` list (`SDR`, `HDR10`, `HDR10+`, `DV`,
`HYBRID`). Defaults to `[SDR]`.
- `-q` / `--quality` — resolution list.
- `--vbitrate-range` / `--abitrate-range` — `MIN-MAX` kbps windows.
---
## credentials (dict[str, str|list|dict])
Specify login credentials to use for each Service, and optionally per-profile.
For example,
```yaml
EXAMPLE: jane@example.tld:LoremIpsum100 # directly
EXAMPLE2: # or per-profile, optionally with a default
default: jane@example.tld:LoremIpsum99 # <-- used by default if -p/--profile is not used
james: james@example.tld:TheFriend97
john: john@example.tld:LoremIpsum98
EXAMPLE3: # the `default` key is not necessary, but no credential will be used by default
john: john@example.tld:SecretPassword123
```
The value should be in string form, i.e. `john@example.tld:password123` or `john:password123`.
Any arbitrary values can be used on the left (username/password/phone) and right (password/secret).
You can also specify these in list form, i.e., `["john@example.tld", ":PasswordWithAColon"]`.
If you specify multiple credentials with keys like the `EXAMPLE2` and `EXAMPLE3` example above, then you should
use a `default` key or no credential will be loaded automatically unless you use `-p/--profile`. You
do not have to use a `default` key at all.
Please be aware that this information is sensitive and to keep it safe. Do not share your config.
---
## tmdb_api_key (str)
API key for The Movie Database (TMDB). This is used for tagging downloaded files with TMDB,
IMDB and TVDB identifiers. Leave empty to disable automatic lookups.
To obtain a TMDB API key:
1. Create an account at <https://www.themoviedb.org/>
2. Go to <https://www.themoviedb.org/settings/api> to register for API access
3. Fill out the API application form with your project details
4. Once approved, you'll receive your API key
For example,
```yaml
tmdb_api_key: cf66bf18956kca5311ada3bebb84eb9a # Not a real key
```
**Note**: Keep your API key secure and do not share it publicly. This key is used by the `core/providers/tmdb.py` metadata provider to fetch metadata from TMDB for proper file tagging and ID enrichment.
---
## simkl_client_id (str)
Client ID for SIMKL API integration. SIMKL is used as a metadata source for improved title matching and tagging,
especially when a TMDB API key is not configured.
To obtain a SIMKL Client ID:
1. Create an account at <https://simkl.com/>
2. Go to <https://simkl.com/settings/developer/>
3. Register a new application to receive your Client ID
For example,
```yaml
simkl_client_id: "your_client_id_here"
```
**Note**: While optional, having a SIMKL Client ID improves metadata lookup reliability. SIMKL serves as an alternative or fallback metadata source to TMDB. This is used by the `core/providers/simkl.py` metadata provider.
---
## ipinfo_api_key (str)
Optional API token for [ipinfo.io](https://ipinfo.io). When set, envied uses the free authenticated **Lite** endpoint (`https://api.ipinfo.io/lite/me`), which has substantially higher rate limits than the anonymous endpoint and returns richer fields (ASN, organization name, continent). Leave empty to use the anonymous ipinfo.io endpoint, with [ip-api.in](https://ip-api.in) as a final fallback.
To obtain an ipinfo.io token:
1. Sign up for a free account at <https://ipinfo.io/signup>
2. Copy the token from your dashboard
For example,
```yaml
ipinfo_api_key: "12a3b45cd678ef" # Not a real key
```
**Note**: The token is only ever sent to `api.ipinfo.io` as a per-request `Authorization` header — it is never attached to your session for service requests. Used by `core/utils/ip_info.py` for region detection and proxy verification.
---
## title_cache_enabled (bool)
Enable/disable caching of title metadata to reduce redundant API calls. Default: `true`.
---
## title_cache_time (int)
Cache duration in seconds for title metadata. Default: `1800` (30 minutes).
---
## title_cache_max_retention (int)
Maximum retention time in seconds for serving slightly stale cached title metadata when API calls fail.
Default: `86400` (24 hours). Effective retention is `min(title_cache_time + grace, title_cache_max_retention)`.
---
## debug (bool)
Enable structured JSON debug logging for troubleshooting and service development. Default: `false`.
When enabled (via config or the `--debug` CLI flag):
- Creates JSON Lines (`.jsonl`) log files with complete debugging context
- Logs: session info, CLI params, service config, CDM details, authentication, titles, tracks metadata,
DRM operations, vault queries, errors with stack traces
- File location: `logs/envied_debug_{service}_{timestamp}.jsonl`
---
## debug_keys (bool)
Log decryption keys in debug logs. Default: `false`.
When `true`, actual content encryption keys (CEKs) are included in debug log output. Useful for
debugging key retrieval and decryption issues.
**Security note:** Passwords, tokens, cookies, and session tokens are always redacted regardless
of this setting. Only content keys (`content_key`, `key` fields) are affected. Key IDs (`kid`),
key counts, and other metadata are always logged.
---
reset/docs/SUBTITLE_CONFIG.md
+73
View File
@@ -0,0 +1,73 @@
# Subtitle Processing Configuration
This document covers subtitle processing and formatting options under the top-level `subtitle:` key in `envied.yaml`.
For the canonical example, see `envied/envied-example.yaml`.
## subtitle (dict)
Control subtitle conversion, SDH (hearing-impaired) stripping, formatting preservation, and output behavior.
- `conversion_method`: How to convert subtitles between formats. Default: `auto`.
- `auto`: Smart routing - subby for WebVTT/fVTT/SAMI; for SSA/ASS/MicroDVD/MPL2/TMP use SubtitleEdit when available, otherwise pysubs2; standard pycaption/SubtitleEdit pipeline for everything else.
- `subby`: Always use subby with `CommonIssuesFixer` (falls back to standard if the source codec isn't supported by subby).
- `subtitleedit`: Prefer SubtitleEdit when available; otherwise fall back to the standard pycaption pipeline.
- `pycaption`: Use only the pycaption library (no SubtitleEdit, no subby). Limited to SRT, TTML, and WebVTT outputs.
- `pysubs2`: Use pysubs2 (supports SRT, SSA, ASS, WebVTT, TTML, SAMI, MicroDVD, MPL2, TMP).
- `sdh_method`: How to strip SDH cues. Default: `auto`.
- `auto`: Try subby for SRT first, then SubtitleEdit (when `conversion_method` is `auto`/`subtitleedit` and the binary is available), then subtitle-filter as the final fallback.
- `subby`: Use subby's `SDHStripper`. **Only operates on SRT**; for other codecs the call returns without stripping.
- `subtitleedit`: Use SubtitleEdit's `/RemoveTextForHI` when the binary is available; otherwise falls through to subtitle-filter.
- `filter-subs`: Use the `subtitle-filter` library directly (`rm_fonts`, `rm_ast`, `rm_music`, `rm_effects`, `rm_names`, `rm_author`).
- `strip_sdh`: Enable/disable automatic SDH stripping for tracks flagged as SDH. Default: `true`.
- `convert_before_strip`: When falling through to the subtitle-filter path, auto-convert non-SRT subtitles to SRT first for better compatibility. Default: `true`. Has no effect when SubtitleEdit handles stripping directly.
- `preserve_formatting`: Keep original subtitle tags and positioning during WebVTT processing. When `true`, sanitized WebVTT is written back without round-tripping through pycaption, preserving tags like `<i>`, `<b>`, and `line:` positioning. Default: `true`.
- `output_mode`: Controls how subtitles are included in the output. Default: `mux`.
- `mux`: Embed subtitles in the MKV container only.
- `sidecar`: Save subtitles as separate files only (not muxed).
- `both`: Embed in the MKV container and save as sidecar files.
- `sidecar_format`: Format for sidecar subtitle files (used when `output_mode` is `sidecar` or `both`). Default: `srt`.
- `srt`: SubRip.
- `vtt`: WebVTT.
- `ass`: Advanced SubStation Alpha.
- `original`: Keep the subtitle in its current format without conversion.
Example:
```yaml
subtitle:
conversion_method: auto
sdh_method: auto
strip_sdh: true
convert_before_strip: true
preserve_formatting: true
output_mode: mux
sidecar_format: srt
```
## WebVTT Sanitization (automatic, not configurable)
After download, WebVTT and segmented WebVTT (`fVTT`/`WVTT`) tracks pass through a fixed sanitization pipeline before any conversion or muxing:
1. **Segment merge** — segmented DASH/HLS WebVTT is stitched via `merge_segmented_webvtt` (uses pysubs2 for lenient parsing when `conversion_method` is `auto` or `pysubs2`, otherwise pycaption directly).
2. **Negative timestamps** — `sanitize_webvtt_timestamps` rewrites `-HH:MM:SS.mmm` cues to `00:00:00.000`.
3. **Cue identifiers** — `sanitize_webvtt_cue_identifiers` strips letter+digit IDs (e.g. `Q0`, `S12`) on their own line before a timing line, which otherwise confuse parsers like pysubs2.
4. **Overlapping cues** — `merge_overlapping_webvtt_cues` collapses cues with start times within 50 ms and matching end times into a single multi-line cue, ordered by `line:` percentage (lower % = higher on screen = first line).
5. **Fallback hardening** — when `preserve_formatting` is `false` and the first pycaption parse fails, `sanitize_webvtt` retries with a `WEBVTT` header guard, hour-padded timings, and another negative-timestamp pass; if that still fails, the sanitized text is written as-is.
`sanitize_broken_webvtt` and `space_webvtt_headers` additionally run inside `Subtitle.parse()` to drop malformed `-->` lines and reflow merged-segment headers. `merge_same_cues` and `filter_unwanted_cues` (drops `&nbsp;`/whitespace-only cues) run only on the pycaption path.
These behaviors are intentional and have no config knobs — they apply to every WebVTT track regardless of `conversion_method`.
## Related
- Filename sanitization (e.g. parenthesis handling, unidecode bracket artifacts from PR #105) lives in `envied/core/utilities.py::sanitize_filename` and is governed by `output_template`, not the `subtitle:` config block.
- Subtitle codec support and the conversion matrix are defined in `envied/core/tracks/subtitle.py`.
---
reset/envied_gui.py
+190
View File
@@ -0,0 +1,190 @@
# -*- coding: utf-8 -*-
from PyQt6.QtWidgets import QWidget, QVBoxLayout, QPushButton, QFrame, QLabel
from PyQt6.QtCore import QProcess
from PyQt6.QtCore import Qt
import sys
import os
import subprocess
import shutil
CFG = os.path.abspath("./packages/envied/src/envied/envied.yaml")
class EnviedPanel(QFrame):
def __init__(self, parent=None):
super().__init__(parent)
self.setWindowTitle("Envied Control ")
# Frame styling: 1px white border (no pink)
self.setObjectName("enviedFrame")
self.setStyleSheet("#enviedFrame { background-color: #2D2D2D; border: 1px solid pink; }")
# ("color: #f5c2e7; border: none; background-color:#1E1E2E;padding: 5px;")
# Layout
layout = QVBoxLayout(self)
layout.setContentsMargins(5, 5, 5, 5)
layout.setSpacing(5)
# Helper to make consistently styled buttons
def make_btn(text):
b = QPushButton(text)
b.setStyleSheet(
"background-color: #1E1E2E; \
color: #f5c2e7; \
border-width: 2px;\
border-color: pink;\
font: 14px;\
min-width: 10em;\
padding: 5px;"
)
b.setAutoFillBackground(True)
b.setAttribute(Qt.WidgetAttribute.WA_StyledBackground, True)
return b
# Buttons
btn_run = make_btn("run envied")
btn_check = make_btn("env check")
btn_info = make_btn("env info")
btn_cfg = make_btn("config")
layout.addWidget(btn_run)
layout.addWidget(btn_check)
layout.addWidget(btn_info)
layout.addWidget(btn_cfg)
# --- Wire up clicks ---
# Option A: non-blocking (recommended): use QProcess so the GUI stays responsive
btn_run.clicked.connect(lambda: open_prefilled_terminal())
btn_check.clicked.connect(lambda: self.run_cmd(["uv", "run", "envied", "env", "check"]))
btn_info.clicked.connect(lambda: self.run_cmd(["uv", "run", "envied", "env", "info"]))
'''btn_cfg.clicked.connect(
lambda: os.system('nano ./packages/envied/src/envied/envied.yaml') if os.name == 'posix'
else os.system('notepad.exe ./packages/envied/src/envied/envied.yaml')
)'''
btn_cfg.clicked.connect(open_config_detached)
# ---- Helpers ----
def run_cmd(self, argv):
"""
Non-blocking run using QProcess.
"""
proc = QProcess(self)
# Optional: forward output to your terminal
proc.setProgram(argv[0])
proc.setArguments(argv[1:])
# Show output in your launching terminal (detach these if not needed)
proc.readyReadStandardOutput.connect(lambda p=proc: print(p.readAllStandardOutput().data().decode(), end=""))
proc.readyReadStandardError.connect(lambda p=proc: print(p.readAllStandardError().data().decode(), end=""))
proc.start()
def run_script(self, what):
"""
Placeholder hook for your 'config' action.
e.g., open a config dialog or run a script.
"""
print(f"[EnviedPanel] run_script({what}) not implemented yet.")
def get_terminal():
terminals = ["gnome-terminal", "xterm", "konsole", "lxterminal", "xfce4-terminal"]
for term in terminals:
if shutil.which(term):
return term
elif os.__name__ =="nt":
TERMINALS = ["WindowsTerminal.exe", "OpenConsole.exe","powershell.exe", "Terminal.exe", "cmd.exe"]
for TERMINAL in TERMINALS:
if shutil.which(TERMINAL):
return TERMINAL
raise EnvironmentError("No suitable terminal emulator found.")
def open_prefilled_terminal():
prefill = 'uv run envied dl --select-titles '
term = get_terminal()
if term == "gnome-terminal":
# -l starts login shell; -c runs the command, then we keep the shell open with 'exec bash'
cmd = [
"gnome-terminal",
"--",
"bash", "-lc",
# read: -e enables readline, -i sets initial text
f'read -e -i "{prefill}" cmd; eval "$cmd"; exec bash'
]
elif term == "konsole":
cmd = [
"konsole", "-e",
"bash", "-lc",
f'read -e -i "{prefill}" cmd; eval "$cmd"; exec bash'
]
elif term =="lxterminal": # xterm fallback
cmd = [
"xterm", "-e",
"bash", "-lc",
f'read -e -i "{prefill}" cmd; eval "$cmd"; exec bash'
]
elif os.name == "nt":
# --- Windows: open PowerShell (or Windows Terminal -> PowerShell)
# and pre-fill the command line using PSReadLine.
# This requires PSReadLine (present by default on modern Windows).
ps_prefill = prefill.replace('"', r'`"') # escape quotes for PowerShell
if shutil.which("wt.exe"):
# Windows Terminal present: open a new tab running PowerShell
cmd = [
"wt.exe",
"powershell",
"-NoExit",
"-Command",
f"[Microsoft.PowerShell.PSConsoleReadLine]::Insert(\"{ps_prefill}\")"
]
elif shutil.which("powershell.exe"):
# Fallback: plain PowerShell
cmd = [
"powershell.exe",
"-NoExit",
"-Command",
f"[Microsoft.PowerShell.PSConsoleReadLine]::Insert(\"{ps_prefill}\")"
]
else:
# Last resort: cmd.exe can't truly prefill. Keep the window open and hint user.
cmd = [
"cmd.exe", "/K",
f'echo Please use PowerShell for prefill. Intended command: {prefill}'
]
subprocess.Popen(cmd)
def open_config_detached():
if os.name == "nt":
QProcess.startDetached("notepad.exe", [CFG])
return
# POSIX
# Easiest: open in the default GUI editor
if shutil.which("xdg-open"):
QProcess.startDetached("xdg-open", [CFG])
return
# If you insist on nano, run it *inside* a terminal emulator
# (xterm is very common and works out of the box)
if shutil.which("xterm"):
QProcess.startDetached("xterm", ["-e", "nano", CFG])
return
# Last resort (may fail without a TTY)
QProcess.startDetached("nano", [CFG])
if __name__ == "__main__":
from PyQt6.QtWidgets import QApplication
app = QApplication(sys.argv)
envied_panel = EnviedPanel()
window = envied_panel
window.show()
sys.exit(app.exec())
subprocess.Popen(cmd)
reset/gui.py
+449
View File
@@ -0,0 +1,449 @@
import sys
from PyQt6.QtWidgets import (
QApplication, QWidget, QVBoxLayout, QLabel, QLineEdit, QTextEdit, QPushButton, QCheckBox, QFrame, QMessageBox, QHBoxLayout
)
from PyQt6.QtCore import Qt, QThread, pyqtSignal
from PyQt6.QtGui import QPalette, QColor
import subprocess
import httpx
import base64
import re
import urllib.parse
import xml.etree.ElementTree as ET
from pywidevine.cdm import Cdm
from pywidevine.device import Device
from pywidevine.pssh import PSSH
from base64 import b64encode
import codecs
from pathlib import Path
import os
import shlex
WVD_PATH = "./WVDs/device.wvd"
WIDEVINE_SYSTEM_ID = 'EDEF8BA9-79D6-4ACE-A3C8-27DCD51D21ED'
class DownloadThread(QThread):
finished = pyqtSignal()
error = pyqtSignal(str)
def __init__(self, command):
super().__init__()
self.command = command
def run(self):
try:
subprocess.run(self.command, check=True)
self.finished.emit()
except subprocess.CalledProcessError as e:
self.error.emit(str(e))
class AllHell3App(QWidget):
def __init__(self):
super().__init__()
self.init_ui()
def init_ui(self):
self.setWindowTitle("HellYes!")
layout = QVBoxLayout()
self.mpd_url_label = QLabel("MPD URL")
layout.addWidget(self.mpd_url_label)
self.mpd_url_entry = QLineEdit()
layout.addWidget(self.mpd_url_entry)
self.curl_label = QLabel("cURL of License Request")
layout.addWidget(self.curl_label)
self.curl_text = QTextEdit()
layout.addWidget(self.curl_text)
highlighted_frame = QFrame()
highlighted_layout = QVBoxLayout()
highlighted_frame.setLayout(highlighted_layout)
highlighted_frame.setStyleSheet("border: 1px solid red;")
self.video_name_label = QLabel("Video Name")
highlighted_layout.addWidget(self.video_name_label)
self.video_name_entry = QLineEdit()
highlighted_layout.addWidget(self.video_name_entry)
self.fetch_keys_button = QPushButton("Get Keys")
self.fetch_keys_button.clicked.connect(self.fetch_keys)
highlighted_layout.addWidget(self.fetch_keys_button)
self.download_button = QPushButton("Download Nm~RE")
self.download_button.clicked.connect(self.download_video)
highlighted_layout.addWidget(self.download_button)
self.download_button2 = QPushButton("Download DASH")
self.download_button2.clicked.connect(self.download_dash)
highlighted_layout.addWidget(self.download_button2)
layout.addWidget(highlighted_frame)
self.keys_output = QTextEdit()
layout.addWidget(self.keys_output)
self.n_m3u8dl_re_label = QLabel("N_m3u8DL-RE command")
layout.addWidget(self.n_m3u8dl_re_label)
self.n_m3u8dl_re_output = QTextEdit()
layout.addWidget(self.n_m3u8dl_re_output)
self.dash_mpd_cli_label = QLabel("Dash-MPD-CLI command")
layout.addWidget(self.dash_mpd_cli_label)
self.dash_mpd_cli_output = QTextEdit()
layout.addWidget(self.dash_mpd_cli_output)
checkbox_layout = QHBoxLayout()
self.dark_mode_checkbox = QCheckBox("Dark Mode")
self.dark_mode_checkbox.setChecked(True)
self.dark_mode_checkbox.stateChanged.connect(self.toggle_dark_mode)
checkbox_layout.addWidget(self.dark_mode_checkbox)
self.reset_button = QPushButton("Reset")
self.reset_button.clicked.connect(self.reset_fields)
checkbox_layout.addWidget(self.reset_button)
layout.addLayout(checkbox_layout)
self.setLayout(layout)
self.toggle_dark_mode()
def reset_fields(self):
self.mpd_url_entry.clear()
self.curl_text.clear()
self.video_name_entry.clear()
self.keys_output.clear()
self.n_m3u8dl_re_output.clear()
self.dash_mpd_cli_output.clear()
def toggle_dark_mode(self):
if self.dark_mode_checkbox.isChecked():
palette = QPalette()
palette.setColor(QPalette.ColorRole.Window, QColor(53, 53, 53))
palette.setColor(QPalette.ColorRole.WindowText, Qt.GlobalColor.white)
palette.setColor(QPalette.ColorRole.Base, QColor(35, 35, 35))
palette.setColor(QPalette.ColorRole.AlternateBase, QColor(53, 53, 53))
palette.setColor(QPalette.ColorRole.ToolTipBase, Qt.GlobalColor.black)
palette.setColor(QPalette.ColorRole.ToolTipText, Qt.GlobalColor.white)
palette.setColor(QPalette.ColorRole.Text, Qt.GlobalColor.white)
palette.setColor(QPalette.ColorRole.Button, QColor(53, 53, 53))
palette.setColor(QPalette.ColorRole.ButtonText, Qt.GlobalColor.white)
palette.setColor(QPalette.ColorRole.BrightText, Qt.GlobalColor.red)
palette.setColor(QPalette.ColorRole.Link, QColor(42, 130, 218))
palette.setColor(QPalette.ColorRole.Highlight, QColor(42, 130, 218))
palette.setColor(QPalette.ColorRole.HighlightedText, Qt.GlobalColor.black)
self.setPalette(palette)
self.mpd_url_label.setStyleSheet("color: white;")
self.curl_label.setStyleSheet("color: white;")
self.video_name_label.setStyleSheet("color: white;")
self.n_m3u8dl_re_label.setStyleSheet("color: white;")
self.dash_mpd_cli_label.setStyleSheet("color: white;")
self.dark_mode_checkbox.setStyleSheet("color: white;")
self.fetch_keys_button.setStyleSheet("""color: white;
background-color: #4e4e4e;
border: 1px solid #6e6e6e;
padding: 5px;""")
self.download_button.setStyleSheet("""color: white;
background-color: #4e4e4e;
border: 1px solid #6e6e6e;
padding: 5px;""")
self.download_button2.setStyleSheet("""color: white;
background-color: #4e4e4e;
border: 1px solid #6e6e6e;
padding: 5px;""")
self.curl_text.setStyleSheet("""color: white; border: 1px solid red;
background-color: #232323;""")
self.video_name_label.setStyleSheet("""
color: white; border: None;""")
self.mpd_url_entry.setStyleSheet("""color: white; border: 1px solid red;
background-color: #232323;""")
else:
self.setPalette(QApplication.palette())
self.mpd_url_label.setStyleSheet("color: black;")
self.curl_label.setStyleSheet("color: black;")
self.video_name_label.setStyleSheet("color: black;")
self.n_m3u8dl_re_label.setStyleSheet("color: black;")
self.dash_mpd_cli_label.setStyleSheet("color: black;")
self.dark_mode_checkbox.setStyleSheet("color: black;")
self.fetch_keys_button.setStyleSheet("""color: black;
background-color: #aeaeae;
border: 1px solid #6e6e6e;
padding: 5px;""")
self.download_button.setStyleSheet("""color: black;
background-color: #aeaeae;
border: 1px solid #6e6e6e;
padding: 5px;""")
self.download_button2.setStyleSheet("""color: black;
background-color: #aeaeae;
border: 1px solid #6e6e6e;
padding: 5px;""")
self.curl_text.setStyleSheet("""color: black; border: 1px solid red;
background-color: white;""")
self.video_name_label.setStyleSheet("""
color: black; border: None;""")
self.mpd_url_entry.setStyleSheet("""color: black; border: 1px solid red;
background-color: white;""")
def fetch_mpd_content(self, mpd_url):
response = httpx.get(mpd_url)
response.raise_for_status()
return response.text
def find_default_kid_with_regex(self, mpd_content):
match = re.search(r'cenc:default_KID="([A-F0-9-]+)"', mpd_content)
if match:
return match.group(1)
return None
def extract_or_generate_pssh(self, mpd_content):
try:
tree = ET.ElementTree(ET.fromstring(mpd_content))
root = tree.getroot()
namespaces = {
'cenc': 'urn:mpeg:cenc:2013',
'': 'urn:mpeg:dash:schema:mpd:2011'
}
default_kid = None
for elem in root.findall('.//ContentProtection', namespaces):
scheme_id_uri = elem.attrib.get('schemeIdUri', '').upper()
if scheme_id_uri == 'URN:MPEG:DASH:MP4PROTECTION:2011':
default_kid = elem.attrib.get('cenc:default_KID')
if default_kid:
break
if not default_kid:
default_kid = self.find_default_kid_with_regex(mpd_content)
pssh = None
for elem in root.findall('.//ContentProtection', namespaces):
scheme_id_uri = elem.attrib.get('schemeIdUri', '').upper()
if scheme_id_uri == f'URN:UUID:{WIDEVINE_SYSTEM_ID}':
pssh_elem = elem.find('cenc:pssh', namespaces)
if pssh_elem is not None:
pssh = pssh_elem.text
break
if pssh is not None:
return pssh
elif default_kid is not None:
default_kid = default_kid.replace('-', '')
s = f'000000387073736800000000edef8ba979d64acea3c827dcd51d21ed000000181210{default_kid}48e3dc959b06'
return b64encode(bytes.fromhex(s)).decode()
else:
# No pssh or default_KID found
try:
pssh = self.get_pssh_from_mpd(self.mpd_url_entry.text()) # init.m4f method
if pssh:
return pssh
else:
return None
except:
return None
except ET.ParseError as e:
try:
pssh = self.get_pssh_from_mpd(self.mpd_url_entry.text()) # init.m4f method
return pssh
except:
pass
print(f"Error parsing MPD content: {e}")
return None
def get_pssh_from_mpd(self, mpd: str):
print("Extracting PSSH from MPD...")
yt_dl = 'yt-dlp'
init = 'init.m4f'
files_to_delete = [init]
for file_name in files_to_delete:
if os.path.exists(file_name):
os.remove(file_name)
print(f"{file_name} file successfully deleted.")
try:
subprocess.run([yt_dl, '-q', '--no-warning', '--test', '--allow-u', '-f', 'bestvideo[ext=mp4]/bestaudio[ext=m4a]/best', '-o', init, mpd])
except FileNotFoundError:
print("yt-dlp not found. Trying installing it...")
exit(0)
pssh_list = self.extract_pssh_from_file('init.m4f')
pssh = None
for target_pssh in pssh_list:
if 20 < len(target_pssh) < 220:
pssh = target_pssh
print(f'\n{pssh}\n')
for file_name in files_to_delete:
if os.path.exists(file_name):
os.remove(file_name)
print(f"{file_name} file successfully deleted.")
return pssh
def extract_pssh_from_file(self,file_path: str) -> list:
print('Extracting PSSHs from init file:', file_path)
return self.to_pssh(Path(file_path).read_bytes())
def find_wv_pssh_offsets(self, raw: bytes) -> list:
offsets = []
offset = 0
while True:
offset = raw.find(b'pssh', offset)
if offset == -1:
break
size = int.from_bytes(raw[offset-4:offset], byteorder='big')
pssh_offset = offset - 4
offsets.append(raw[pssh_offset:pssh_offset+size])
offset += size
return offsets
def to_pssh(self, content: bytes) -> list:
wv_offsets = self.find_wv_pssh_offsets(content)
return [base64.b64encode(wv_offset).decode() for wv_offset in wv_offsets]
def parse_curl(self, curl_command):
url_match = re.search(r"curl\s+'(.*?)'", curl_command)
url = url_match.group(1) if url_match else ""
method_match = re.search(r"-X\s+(\w+)", curl_command)
method = method_match.group(1) if method_match else "UNDEFINED"
headers = {}
headers_matches = re.findall(r"-H\s+'([^:]+):\s*(.*?)'", curl_command)
for header in headers_matches:
headers[header[0]] = header[1]
data_match = re.search(r"--data(?:-raw)?\s+(?:(\$?')|(\$?{?))(.*?)'", curl_command, re.DOTALL)
if data_match:
raw_prefix = data_match.group(1)
data = data_match.group(3)
if raw_prefix and raw_prefix.startswith('$'):
data = None
else:
data = data.replace('\\\\', '\\').replace('\\x', '\\\\x')
try:
data = codecs.decode(data, 'unicode_escape')
except Exception as e:
data = ""
else:
data = ""
return url, method, headers, data
def get_key(self, pssh, license_url, headers, data):
device = Device.load(WVD_PATH)
cdm = Cdm.from_device(device)
session_id = cdm.open()
challenge = cdm.get_license_challenge(session_id, PSSH(pssh))
if data:
if match := re.search(r'"(CAQ=.*?)"', data):
challenge = data.replace(match.group(1), base64.b64encode(challenge).decode())
elif match := re.search(r'"(CAES.*?)"', data):
challenge = data.replace(match.group(1), base64.b64encode(challenge).decode())
elif match := re.search(r'=(CAES.*?)(&.*)?$', data):
b64challenge = base64.b64encode(challenge).decode()
quoted = urllib.parse.quote_plus(b64challenge)
challenge = data.replace(match.group(1), quoted)
payload = challenge if data is None else challenge
license_response = httpx.post(url=license_url, data=payload, headers=headers)
try:
license_response.raise_for_status()
except httpx.HTTPStatusError as e:
raise e
license_content = license_response.content
try:
match = re.search(r'"(CAIS.*?)"', license_response.content.decode('utf-8'))
if match:
license_content = base64.b64decode(match.group(1))
except:
pass
if isinstance(license_content, str):
license_content = base64.b64decode(license_content)
cdm.parse_license(session_id, license_content)
keys = []
for key in cdm.get_keys(session_id):
if key.type == 'CONTENT':
keys.append(f"--key {key.kid.hex}:{key.key.hex()}")
cdm.close(session_id)
return keys
def fetch_keys(self):
mpd_url = self.mpd_url_entry.text()
try:
mpd_content = self.fetch_mpd_content(mpd_url)
pssh = self.extract_or_generate_pssh(mpd_content)
if not pssh:
QMessageBox.critical(self, "Error", "Failed to extract or generate PSSH\nAre you sure the video is Widevine encrypted?.")
return
curl_command = self.curl_text.toPlainText().strip()
license_url, method, headers, data = self.parse_curl(curl_command)
keys = self.get_key(pssh, license_url, headers, data)
self.keys_output.clear()
self.keys_output.append("\n".join( keys))
key_str = " ".join(keys)
self.n_m3u8dl_re_command = f"N_m3u8DL-RE '{mpd_url}' {key_str} --save-name {self.video_name_entry.text()} -mt -M:format=mkv:muxer=mkvmerge"
self.n_m3u8dl_re_output.setText(self.n_m3u8dl_re_command)
self.dash_mpd_cli_command = f"dash-mpd-cli --quality best --muxer-preference mkv:mkvmerge {key_str} \"{mpd_url}\" --write-subs --output '{self.video_name_entry.text()}.mkv'"
self.dash_mpd_cli_output.setText(str(self.dash_mpd_cli_command))
except Exception as e:
QMessageBox.critical(self, "Error", str(e))
def download_video(self):
try:
self.thread = DownloadThread(shlex.split(self.n_m3u8dl_re_command))
self.thread.finished.connect(self.on_download_finished)
self.thread.error.connect(self.on_download_error)
self.thread.start()
except Exception as e:
QMessageBox.critical(self, "Error", str(e))
def download_dash(self):
try:
self.thread = DownloadThread(shlex.split(self.dash_mpd_cli_command))
self.thread.finished.connect(self.on_download_finished)
self.thread.error.connect(self.on_download_error)
self.thread.start()
except Exception as e:
QMessageBox.critical(self, "Error", str(e))
def on_download_finished(self):
QMessageBox.information(self, "Success", "Download finished successfully.")
def on_download_error(self, error_message):
QMessageBox.critical(self, "Error", f"Download failed: {error_message}")
if __name__ == "__main__":
app = QApplication(sys.argv)
window = AllHell3App()
window.show()
sys.exit(app.exec())
reset/packages/envied/CONFIG.md
+582
View File
@@ -0,0 +1,582 @@
# Config Documentation
This page documents configuration values and what they do. You begin with an empty configuration file.
You may alter your configuration with `envied cfg --help`, or find the direct location with `envied env info`.
Configuration values are listed in alphabetical order.
Avoid putting comments in the config file as they may be removed. Comments are currently kept only thanks
to the usage of `ruamel.yaml` to parse and write YAML files. In the future `yaml` may be used instead,
which does not keep comments.
## aria2c (dict)
- `max_concurrent_downloads`
Maximum number of parallel downloads. Default: `min(32,(cpu_count+4))`
Note: Overrides the `max_workers` parameter of the aria2(c) downloader function.
- `max_connection_per_server`
Maximum number of connections to one server for each download. Default: `1`
- `split`
Split a file into N chunks and download each chunk on its own connection. Default: `5`
- `file_allocation`
Specify file allocation method. Default: `"prealloc"`
- `"none"` doesn't pre-allocate file space.
- `"prealloc"` pre-allocates file space before download begins. This may take some time depending on the size of the
file.
- `"falloc"` is your best choice if you are using newer file systems such as ext4 (with extents support), btrfs, xfs
or NTFS (MinGW build only). It allocates large(few GiB) files almost instantly. Don't use falloc with legacy file
systems such as ext3 and FAT32 because it takes almost same time as prealloc, and it blocks aria2 entirely until
allocation finishes. falloc may not be available if your system doesn't have posix_fallocate(3) function.
- `"trunc"` uses ftruncate(2) system call or platform-specific counterpart to truncate a file to a specified length.
## cdm (dict)
Pre-define which Widevine or PlayReady device to use for each Service by Service Tag as Key (case-sensitive).
The value should be a WVD or PRD filename without the file extension. When
loading the device, envied will look in both the `WVDs` and `PRDs` directories
for a matching file.
For example,
```yaml
AMZN: chromecdm_903_l3
NF: nexus_6_l1
```
You may also specify this device based on the profile used.
For example,
```yaml
AMZN: chromecdm_903_l3
NF: nexus_6_l1
DSNP:
john_sd: chromecdm_903_l3
jane_uhd: nexus_5_l1
```
You can also specify a fallback value to predefine if a match was not made.
This can be done using `default` key. This can help reduce redundancy in your specifications.
For example, the following has the same result as the previous example, as well as all other
services and profiles being pre-defined to use `chromecdm_903_l3`.
```yaml
NF: nexus_6_l1
DSNP:
jane_uhd: nexus_5_l1
default: chromecdm_903_l3
```
## chapter_fallback_name (str)
The Chapter Name to use when exporting a Chapter without a Name.
The default is no fallback name at all and no Chapter name will be set.
The fallback name can use the following variables in f-string style:
- `{i}`: The Chapter number starting at 1.
E.g., `"Chapter {i}"`: "Chapter 1", "Intro", "Chapter 3".
- `{j}`: A number starting at 1 that increments any time a Chapter has no title.
E.g., `"Chapter {j}"`: "Chapter 1", "Intro", "Chapter 2".
These are formatted with f-strings, directives are supported.
For example, `"Chapter {i:02}"` will result in `"Chapter 01"`.
## credentials (dict[str, str|list|dict])
Specify login credentials to use for each Service, and optionally per-profile.
For example,
```yaml
ALL4: jane@gmail.com:LoremIpsum100 # directly
AMZN: # or per-profile, optionally with a default
default: jane@example.tld:LoremIpsum99 # <-- used by default if -p/--profile is not used
james: james@gmail.com:TheFriend97
john: john@example.tld:LoremIpsum98
NF: # the `default` key is not necessary, but no credential will be used by default
john: john@gmail.com:TheGuyWhoPaysForTheNetflix69420
```
The value should be in string form, i.e. `john@gmail.com:password123` or `john:password123`.
Any arbitrary values can be used on the left (username/password/phone) and right (password/secret).
You can also specify these in list form, i.e., `["john@gmail.com", ":PasswordWithAColon"]`.
If you specify multiple credentials with keys like the `AMZN` and `NF` example above, then you should
use a `default` key or no credential will be loaded automatically unless you use `-p/--profile`. You
do not have to use a `default` key at all.
Please be aware that this information is sensitive and to keep it safe. Do not share your config.
## curl_impersonate (dict)
- `browser` - The Browser to impersonate as. A list of available Browsers and Versions are listed here:
<https://github.com/yifeikong/curl_cffi#sessions>
Default: `"chrome124"`
For example,
```yaml
curl_impersonate:
browser: "chrome120"
```
## directories (dict)
Override the default directories used across envied.
The directories are set to common values by default.
The following directories are available and may be overridden,
- `commands` - CLI Command Classes.
- `services` - Service Classes.
- `vaults` - Vault Classes.
- `fonts` - Font files (ttf or otf).
- `downloads` - Downloads.
- `temp` - Temporary files or conversions during download.
- `cache` - Expiring data like Authorization tokens, or other misc data.
- `cookies` - Expiring Cookie data.
- `logs` - Logs.
- `wvds` - Widevine Devices.
- `prds` - PlayReady Devices.
For example,
```yaml
downloads: "D:/Downloads/envied"
temp: "D:/Temp/envied"
```
There are directories not listed that cannot be modified as they are crucial to the operation of envied.
## dl (dict)
Pre-define default options and switches of the `dl` command.
The values will be ignored if explicitly set in the CLI call.
The Key must be the same value Python click would resolve it to as an argument.
E.g., `@click.option("-r", "--range", "range_", type=...` actually resolves as `range_` variable.
For example to set the default primary language to download to German,
```yaml
lang: de
```
to set how many tracks to download concurrently to 4 and download threads to 16,
```yaml
downloads: 4
workers: 16
```
to set `--bitrate=CVBR` for the AMZN service,
```yaml
lang: de
AMZN:
bitrate: CVBR
```
or to change the output subtitle format from the default (original format) to WebVTT,
```yaml
sub_format: vtt
```
## downloader (str | dict)
Choose what software to use to download data throughout envied where needed.
You may provide a single downloader globally or a mapping of service tags to
downloaders.
Options:
- `requests` (default) - <https://github.com/psf/requests>
- `aria2c` - <https://github.com/aria2/aria2>
- `curl_impersonate` - <https://github.com/yifeikong/curl-impersonate> (via <https://github.com/yifeikong/curl_cffi>)
- `n_m3u8dl_re` - <https://github.com/nilaoda/N_m3u8DL-RE>
Note that aria2c can reach the highest speeds as it utilizes threading and more connections than the other downloaders. However, aria2c can also be one of the more unstable downloaders. It will work one day, then not another day. It also does not support HTTP(S) proxies while the other downloaders do.
Example mapping:
```yaml
downloader:
NF: requests
AMZN: n_m3u8dl_re
DSNP: n_m3u8dl_re
default: requests
```
The `default` entry is optional. If omitted, `requests` will be used for services not listed.
## decryption (str | dict)
Choose what software to use to decrypt DRM-protected content throughout envied where needed.
You may provide a single decryption method globally or a mapping of service tags to
decryption methods.
Options:
- `shaka` (default) - Shaka Packager - <https://github.com/shaka-project/shaka-packager>
- `mp4decrypt` - mp4decrypt from Bento4 - <https://github.com/axiomatic-systems/Bento4>
Note that Shaka Packager is the traditional method and works with most services. mp4decrypt
is an alternative that may work better with certain services that have specific encryption formats.
Example mapping:
```yaml
decryption:
ATVP: mp4decrypt
AMZN: shaka
default: shaka
```
The `default` entry is optional. If omitted, `shaka` will be used for services not listed.
Simple configuration (single method for all services):
```yaml
decryption: mp4decrypt
```
## filenames (dict)
Override the default filenames used across envied.
The filenames use various variables that are replaced during runtime.
The following filenames are available and may be overridden:
- `log` - Log filenames. Uses `{name}` and `{time}` variables.
- `config` - Service configuration filenames.
- `root_config` - Root configuration filename.
- `chapters` - Chapter export filenames. Uses `{title}` and `{random}` variables.
- `subtitle` - Subtitle export filenames. Uses `{id}` and `{language}` variables.
For example,
```yaml
filenames:
log: "envied_{name}_{time}.log"
config: "config.yaml"
root_config: "envied.yaml"
chapters: "Chapters_{title}_{random}.txt"
subtitle: "Subtitle_{id}_{language}.srt"
```
## headers (dict)
Case-Insensitive dictionary of headers that all Services begin their Request Session state with.
All requests will use these unless changed explicitly or implicitly via a Server response.
These should be sane defaults and anything that would only be useful for some Services should not
be put here.
Avoid headers like 'Accept-Encoding' as that would be a compatibility header that Python-requests will
set for you.
I recommend using,
```yaml
Accept-Language: "en-US,en;q=0.8"
User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36"
```
## key_vaults (list\[dict])
Key Vaults store your obtained Content Encryption Keys (CEKs) and Key IDs per-service.
This can help reduce unnecessary License calls even during the first download. This is because a Service may
provide the same Key ID and CEK for both Video and Audio, as well as for multiple resolutions or bitrates.
You can have as many Key Vaults as you would like. It's nice to share Key Vaults or use a unified Vault on
Teams as sharing CEKs immediately can help reduce License calls drastically.
Three types of Vaults are in the Core codebase, API, SQLite and MySQL. API makes HTTP requests to a RESTful API,
whereas SQLite and MySQL directly connect to an SQLite or MySQL Database.
Note: SQLite and MySQL vaults have to connect directly to the Host/IP. It cannot be in front of a PHP API or such.
Beware that some Hosting Providers do not let you access the MySQL server outside their intranet and may not be
accessible outside their hosting platform.
### Using an API Vault
API vaults use a specific HTTP request format, therefore API or HTTP Key Vault APIs from other projects or services may
not work in envied.The API format can be seen in the [API Vault Code](envied/vaults/API.py).
```yaml
- type: API
name: "John#0001's Vault" # arbitrary vault name
uri: "https://key-vault.example.com" # api base uri (can also be an IP or IP:Port)
# uri: "127.0.0.1:80/key-vault"
# uri: "https://api.example.com/key-vault"
token: "random secret key" # authorization token
```
### Using a MySQL Vault
MySQL vaults can be either MySQL or MariaDB servers. I recommend MariaDB.
A MySQL Vault can be on a local or remote network, but I recommend SQLite for local Vaults.
```yaml
- type: MySQL
name: "John#0001's Vault" # arbitrary vault name
host: "127.0.0.1" # host/ip
# port: 3306 # port (defaults to 3306)
database: vault # database used for envied
username: jane11
password: Doe123
```
I recommend giving only a trustable user (or yourself) CREATE permission and then use envied to cache at least one CEK
per Service to have it create the tables. If you don't give any user permissions to create tables, you will need to
make tables yourself.
- Use a password on all user accounts.
- Never use the root account with envied (even if it's you).
- Do not give multiple users the same username and/or password.
- Only give users access to the database used for envied.
- You may give trusted users CREATE permission so envied can create tables if needed.
- Other uses should only be given SELECT and INSERT permissions.
### Using an SQLite Vault
SQLite Vaults are usually only used for locally stored vaults. This vault may be stored on a mounted Cloud storage
drive, but I recommend using SQLite exclusively as an offline-only vault. Effectively this is your backup vault in
case something happens to your MySQL Vault.
```yaml
- type: SQLite
name: "My Local Vault" # arbitrary vault name
path: "C:/Users/Jane11/Documents/envied/data/key_vault.db"
```
**Note**: You do not need to create the file at the specified path.
SQLite will create a new SQLite database at that path if one does not exist.
Try not to accidentally move the `db` file once created without reflecting the change in the config, or you will end
up with multiple databases.
If you work on a Team I recommend every team member having their own SQLite Vault even if you all use a MySQL vault
together.
## muxing (dict)
- `set_title`
Set the container title to `Show SXXEXX Episode Name` or `Movie (Year)`. Default: `true`
## n_m3u8dl_re (dict)
Configuration for N_m3u8DL-RE downloader. This downloader is particularly useful for HLS streams.
- `thread_count`
Number of threads to use for downloading. Default: Uses the same value as max_workers from the command.
- `ad_keyword`
Keyword to identify and potentially skip advertisement segments. Default: `None`
- `use_proxy`
Whether to use proxy when downloading. Default: `true`
For example,
```yaml
n_m3u8dl_re:
thread_count: 16
ad_keyword: "advertisement"
use_proxy: true
```
## nordvpn (dict)
**Legacy configuration. Use `proxy_providers.nordvpn` instead.**
Set your NordVPN Service credentials with `username` and `password` keys to automate the use of NordVPN as a Proxy
system where required.
You can also specify specific servers to use per-region with the `servers` key.
Sometimes a specific server works best for a service than others, so hard-coding one for a day or two helps.
For example,
```yaml
nordvpn:
username: zxqsR7C5CyGwmGb6KSvk8qsZ # example of the login format
password: wXVHmht22hhRKUEQ32PQVjCZ
servers:
- us: 12 # force US server #12 for US proxies
```
The username and password should NOT be your normal NordVPN Account Credentials.
They should be the `Service credentials` which can be found on your Nord Account Dashboard.
Note that `gb` is used instead of `uk` to be more consistent across regional systems.
## proxy_providers (dict)
Enable external proxy provider services. These proxies will be used automatically where needed as defined by the
Service's GEOFENCE class property, but can also be explicitly used with `--proxy`. You can specify which provider
to use by prefixing it with the provider key name, e.g., `--proxy basic:de` or `--proxy nordvpn:de`. Some providers
support specific query formats for selecting a country/server.
### basic (dict[str, str|list])
Define a mapping of country to proxy to use where required.
The keys are region Alpha 2 Country Codes. Alpha 2 Country Codes are `[a-z]{2}` codes, e.g., `us`, `gb`, and `jp`.
Don't get this mixed up with language codes like `en` vs. `gb`, or `ja` vs. `jp`.
Do note that each key's value can be a list of strings, or a string. For example,
```yaml
us:
- "http://john%40email.tld:password123@proxy-us.domain.tld:8080"
- "http://jane%40email.tld:password456@proxy-us.domain2.tld:8080"
de: "https://127.0.0.1:8080"
```
Note that if multiple proxies are defined for a region, then by default one will be randomly chosen.
You can choose a specific one by specifying it's number, e.g., `--proxy basic:us2` will choose the
second proxy of the US list.
### nordvpn (dict)
Set your NordVPN Service credentials with `username` and `password` keys to automate the use of NordVPN as a Proxy
system where required.
You can also specify specific servers to use per-region with the `servers` key.
Sometimes a specific server works best for a service than others, so hard-coding one for a day or two helps.
For example,
```yaml
username: zxqsR7C5CyGwmGb6KSvk8qsZ # example of the login format
password: wXVHmht22hhRKUEQ32PQVjCZ
servers:
- us: 12 # force US server #12 for US proxies
```
The username and password should NOT be your normal NordVPN Account Credentials.
They should be the `Service credentials` which can be found on your Nord Account Dashboard.
Once set, you can also specifically opt in to use a NordVPN proxy by specifying `--proxy=gb` or such.
You can even set a specific server number this way, e.g., `--proxy=gb2366`.
Note that `gb` is used instead of `uk` to be more consistent across regional systems.
### hola (dict)
Enable Hola VPN proxy service. This is a simple provider that doesn't require configuration.
For example,
```yaml
proxy_providers:
hola: {}
```
Note: Hola VPN is automatically enabled when proxy_providers is configured, no additional setup is required.
## remote_cdm (list\[dict])
Use [pywidevine] Serve-compliant Remote CDMs in envied as if it was a local widevine device file.
The name of each defined device maps as if it was a local device and should be used like a local device.
For example,
```yaml
- name: chromecdm_903_l3 # name must be unique for each remote CDM
# the device type, system id and security level must match the values of the device on the API
# if any of the information is wrong, it will raise an error, if you do not know it ask the API owner
device_type: CHROME
system_id: 1234
security_level: 3
host: "http://xxxxxxxxxxxxxxxx/the_cdm_endpoint"
secret: "secret/api key"
device_name: "remote device to use" # the device name from the API, usually a wvd filename
```
[pywidevine]: https://github.com/rlaphoenix/pywidevine
## serve (dict)
Configuration data for pywidevine's serve functionality run through envied.
This effectively allows you to run `envied serve` to start serving pywidevine Serve-compliant CDMs right from your
local widevine device files.
For example,
```yaml
users:
secret_key_for_jane: # 32bit hex recommended, case-sensitive
devices: # list of allowed devices for this user
- generic_nexus_4464_l3
username: jane # only for internal logging, users will not see this name
secret_key_for_james:
devices:
- generic_nexus_4464_l3
username: james
secret_key_for_john:
devices:
- generic_nexus_4464_l3
username: john
# devices can be manually specified by path if you don't want to add it to
# envied's WVDs directory for whatever reason
# devices:
# - 'C:\Users\john\Devices\test_devices_001.wvd'
```
## services (dict)
Configuration data for each Service. The Service will have the data within this section merged into the `config.yaml`
before provided to the Service class.
Think of this config to be used for more sensitive configuration data, like user or device-specific API keys, IDs,
device attributes, and so on. A `config.yaml` file is typically shared and not meant to be modified, so use this for
any sensitive configuration data.
The Key is the Service Tag, but can take any arbitrary form for its value. It's expected to begin as either a list or
a dictionary.
For example,
```yaml
NOW:
client:
auth_scheme: MESSO
# ... more sensitive data
```
## set_terminal_bg (bool)
Controls whether envied should set the terminal background color. Default: `false`
For example,
```yaml
set_terminal_bg: true
```
## tag (str)
Group or Username to postfix to the end of all download filenames following a dash.
For example, `tag: "J0HN"` will have `-J0HN` at the end of all download filenames.
## tmdb_api_key (str)
API key for The Movie Database (TMDB). This is used for tagging downloaded files with TMDB,
IMDB and TVDB identifiers. Leave empty to disable automatic lookups.
To obtain a TMDB API key:
1. Create an account at <https://www.themoviedb.org/>
2. Go to <https://www.themoviedb.org/settings/api> to register for API access
3. Fill out the API application form with your project details
4. Once approved, you'll receive your API key
For example,
```yaml
tmdb_api_key: cf66bf18956kca5311ada3bebb84eb9a # Not a real key
```
**Note**: Keep your API key secure and do not share it publicly. This key is used by the core/utils/tags.py module to fetch metadata from TMDB for proper file tagging.
reset/packages/envied/LICENSE
+674
View File
@@ -0,0 +1,674 @@
GNU GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The GNU General Public License is a free, copyleft license for
software and other kinds of works.
The licenses for most software and other practical works are designed
to take away your freedom to share and change the works. By contrast,
the GNU General Public License is intended to guarantee your freedom to
share and change all versions of a program--to make sure it remains free
software for all its users. We, the Free Software Foundation, use the
GNU General Public License for most of our software; it applies also to
any other work released this way by its authors. You can apply it to
your programs, too.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
them if you wish), that you receive source code or can get it if you
want it, that you can change the software or use pieces of it in new
free programs, and that you know you can do these things.
To protect your rights, we need to prevent others from denying you
these rights or asking you to surrender the rights. Therefore, you have
certain responsibilities if you distribute copies of the software, or if
you modify it: responsibilities to respect the freedom of others.
For example, if you distribute copies of such a program, whether
gratis or for a fee, you must pass on to the recipients the same
freedoms that you received. You must make sure that they, too, receive
or can get the source code. And you must show them these terms so they
know their rights.
Developers that use the GNU GPL protect your rights with two steps:
(1) assert copyright on the software, and (2) offer you this License
giving you legal permission to copy, distribute and/or modify it.
For the developers' and authors' protection, the GPL clearly explains
that there is no warranty for this free software. For both users' and
authors' sake, the GPL requires that modified versions be marked as
changed, so that their problems will not be attributed erroneously to
authors of previous versions.
Some devices are designed to deny users access to install or run
modified versions of the software inside them, although the manufacturer
can do so. This is fundamentally incompatible with the aim of
protecting users' freedom to change the software. The systematic
pattern of such abuse occurs in the area of products for individuals to
use, which is precisely where it is most unacceptable. Therefore, we
have designed this version of the GPL to prohibit the practice for those
products. If such problems arise substantially in other domains, we
stand ready to extend this provision to those domains in future versions
of the GPL, as needed to protect the freedom of users.
Finally, every program is threatened constantly by software patents.
States should not allow patents to restrict development and use of
software on general-purpose computers, but in those that do, we wish to
avoid the special danger that patents applied to a free program could
make it effectively proprietary. To prevent this, the GPL assures that
patents cannot be used to render the program non-free.
The precise terms and conditions for copying, distribution and
modification follow.
TERMS AND CONDITIONS
0. Definitions.
"This License" refers to version 3 of the GNU General Public License.
"Copyright" also means copyright-like laws that apply to other kinds of
works, such as semiconductor masks.
"The Program" refers to any copyrightable work licensed under this
License. Each licensee is addressed as "you". "Licensees" and
"recipients" may be individuals or organizations.
To "modify" a work means to copy from or adapt all or part of the work
in a fashion requiring copyright permission, other than the making of an
exact copy. The resulting work is called a "modified version" of the
earlier work or a work "based on" the earlier work.
A "covered work" means either the unmodified Program or a work based
on the Program.
To "propagate" a work means to do anything with it that, without
permission, would make you directly or secondarily liable for
infringement under applicable copyright law, except executing it on a
computer or modifying a private copy. Propagation includes copying,
distribution (with or without modification), making available to the
public, and in some countries other activities as well.
To "convey" a work means any kind of propagation that enables other
parties to make or receive copies. Mere interaction with a user through
a computer network, with no transfer of a copy, is not conveying.
An interactive user interface displays "Appropriate Legal Notices"
to the extent that it includes a convenient and prominently visible
feature that (1) displays an appropriate copyright notice, and (2)
tells the user that there is no warranty for the work (except to the
extent that warranties are provided), that licensees may convey the
work under this License, and how to view a copy of this License. If
the interface presents a list of user commands or options, such as a
menu, a prominent item in the list meets this criterion.
1. Source Code.
The "source code" for a work means the preferred form of the work
for making modifications to it. "Object code" means any non-source
form of a work.
A "Standard Interface" means an interface that either is an official
standard defined by a recognized standards body, or, in the case of
interfaces specified for a particular programming language, one that
is widely used among developers working in that language.
The "System Libraries" of an executable work include anything, other
than the work as a whole, that (a) is included in the normal form of
packaging a Major Component, but which is not part of that Major
Component, and (b) serves only to enable use of the work with that
Major Component, or to implement a Standard Interface for which an
implementation is available to the public in source code form. A
"Major Component", in this context, means a major essential component
(kernel, window system, and so on) of the specific operating system
(if any) on which the executable work runs, or a compiler used to
produce the work, or an object code interpreter used to run it.
The "Corresponding Source" for a work in object code form means all
the source code needed to generate, install, and (for an executable
work) run the object code and to modify the work, including scripts to
control those activities. However, it does not include the work's
System Libraries, or general-purpose tools or generally available free
programs which are used unmodified in performing those activities but
which are not part of the work. For example, Corresponding Source
includes interface definition files associated with source files for
the work, and the source code for shared libraries and dynamically
linked subprograms that the work is specifically designed to require,
such as by intimate data communication or control flow between those
subprograms and other parts of the work.
The Corresponding Source need not include anything that users
can regenerate automatically from other parts of the Corresponding
Source.
The Corresponding Source for a work in source code form is that
same work.
2. Basic Permissions.
All rights granted under this License are granted for the term of
copyright on the Program, and are irrevocable provided the stated
conditions are met. This License explicitly affirms your unlimited
permission to run the unmodified Program. The output from running a
covered work is covered by this License only if the output, given its
content, constitutes a covered work. This License acknowledges your
rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not
convey, without conditions so long as your license otherwise remains
in force. You may convey covered works to others for the sole purpose
of having them make modifications exclusively for you, or provide you
with facilities for running those works, provided that you comply with
the terms of this License in conveying all material for which you do
not control copyright. Those thus making or running the covered works
for you must do so exclusively on your behalf, under your direction
and control, on terms that prohibit them from making any copies of
your copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under
the conditions stated below. Sublicensing is not allowed; section 10
makes it unnecessary.
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
No covered work shall be deemed part of an effective technological
measure under any applicable law fulfilling obligations under article
11 of the WIPO copyright treaty adopted on 20 December 1996, or
similar laws prohibiting or restricting circumvention of such
measures.
When you convey a covered work, you waive any legal power to forbid
circumvention of technological measures to the extent such circumvention
is effected by exercising rights under this License with respect to
the covered work, and you disclaim any intention to limit operation or
modification of the work as a means of enforcing, against the work's
users, your or third parties' legal rights to forbid circumvention of
technological measures.
4. Conveying Verbatim Copies.
You may convey verbatim copies of the Program's source code as you
receive it, in any medium, provided that you conspicuously and
appropriately publish on each copy an appropriate copyright notice;
keep intact all notices stating that this License and any
non-permissive terms added in accord with section 7 apply to the code;
keep intact all notices of the absence of any warranty; and give all
recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey,
and you may offer support or warranty protection for a fee.
5. Conveying Modified Source Versions.
You may convey a work based on the Program, or the modifications to
produce it from the Program, in the form of source code under the
terms of section 4, provided that you also meet all of these conditions:
a) The work must carry prominent notices stating that you modified
it, and giving a relevant date.
b) The work must carry prominent notices stating that it is
released under this License and any conditions added under section
7. This requirement modifies the requirement in section 4 to
"keep intact all notices".
c) You must license the entire work, as a whole, under this
License to anyone who comes into possession of a copy. This
License will therefore apply, along with any applicable section 7
additional terms, to the whole of the work, and all its parts,
regardless of how they are packaged. This License gives no
permission to license the work in any other way, but it does not
invalidate such permission if you have separately received it.
d) If the work has interactive user interfaces, each must display
Appropriate Legal Notices; however, if the Program has interactive
interfaces that do not display Appropriate Legal Notices, your
work need not make them do so.
A compilation of a covered work with other separate and independent
works, which are not by their nature extensions of the covered work,
and which are not combined with it such as to form a larger program,
in or on a volume of a storage or distribution medium, is called an
"aggregate" if the compilation and its resulting copyright are not
used to limit the access or legal rights of the compilation's users
beyond what the individual works permit. Inclusion of a covered work
in an aggregate does not cause this License to apply to the other
parts of the aggregate.
6. Conveying Non-Source Forms.
You may convey a covered work in object code form under the terms
of sections 4 and 5, provided that you also convey the
machine-readable Corresponding Source under the terms of this License,
in one of these ways:
a) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by the
Corresponding Source fixed on a durable physical medium
customarily used for software interchange.
b) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by a
written offer, valid for at least three years and valid for as
long as you offer spare parts or customer support for that product
model, to give anyone who possesses the object code either (1) a
copy of the Corresponding Source for all the software in the
product that is covered by this License, on a durable physical
medium customarily used for software interchange, for a price no
more than your reasonable cost of physically performing this
conveying of source, or (2) access to copy the
Corresponding Source from a network server at no charge.
c) Convey individual copies of the object code with a copy of the
written offer to provide the Corresponding Source. This
alternative is allowed only occasionally and noncommercially, and
only if you received the object code with such an offer, in accord
with subsection 6b.
d) Convey the object code by offering access from a designated
place (gratis or for a charge), and offer equivalent access to the
Corresponding Source in the same way through the same place at no
further charge. You need not require recipients to copy the
Corresponding Source along with the object code. If the place to
copy the object code is a network server, the Corresponding Source
may be on a different server (operated by you or a third party)
that supports equivalent copying facilities, provided you maintain
clear directions next to the object code saying where to find the
Corresponding Source. Regardless of what server hosts the
Corresponding Source, you remain obligated to ensure that it is
available for as long as needed to satisfy these requirements.
e) Convey the object code using peer-to-peer transmission, provided
you inform other peers where the object code and Corresponding
Source of the work are being offered to the general public at no
charge under subsection 6d.
A separable portion of the object code, whose source code is excluded
from the Corresponding Source as a System Library, need not be
included in conveying the object code work.
A "User Product" is either (1) a "consumer product", which means any
tangible personal property which is normally used for personal, family,
or household purposes, or (2) anything designed or sold for incorporation
into a dwelling. In determining whether a product is a consumer product,
doubtful cases shall be resolved in favor of coverage. For a particular
product received by a particular user, "normally used" refers to a
typical or common use of that class of product, regardless of the status
of the particular user or of the way in which the particular user
actually uses, or expects or is expected to use, the product. A product
is a consumer product regardless of whether the product has substantial
commercial, industrial or non-consumer uses, unless such uses represent
the only significant mode of use of the product.
"Installation Information" for a User Product means any methods,
procedures, authorization keys, or other information required to install
and execute modified versions of a covered work in that User Product from
a modified version of its Corresponding Source. The information must
suffice to ensure that the continued functioning of the modified object
code is in no case prevented or interfered with solely because
modification has been made.
If you convey an object code work under this section in, or with, or
specifically for use in, a User Product, and the conveying occurs as
part of a transaction in which the right of possession and use of the
User Product is transferred to the recipient in perpetuity or for a
fixed term (regardless of how the transaction is characterized), the
Corresponding Source conveyed under this section must be accompanied
by the Installation Information. But this requirement does not apply
if neither you nor any third party retains the ability to install
modified object code on the User Product (for example, the work has
been installed in ROM).
The requirement to provide Installation Information does not include a
requirement to continue to provide support service, warranty, or updates
for a work that has been modified or installed by the recipient, or for
the User Product in which it has been modified or installed. Access to a
network may be denied when the modification itself materially and
adversely affects the operation of the network or violates the rules and
protocols for communication across the network.
Corresponding Source conveyed, and Installation Information provided,
in accord with this section must be in a format that is publicly
documented (and with an implementation available to the public in
source code form), and must require no special password or key for
unpacking, reading or copying.
7. Additional Terms.
"Additional permissions" are terms that supplement the terms of this
License by making exceptions from one or more of its conditions.
Additional permissions that are applicable to the entire Program shall
be treated as though they were included in this License, to the extent
that they are valid under applicable law. If additional permissions
apply only to part of the Program, that part may be used separately
under those permissions, but the entire Program remains governed by
this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option
remove any additional permissions from that copy, or from any part of
it. (Additional permissions may be written to require their own
removal in certain cases when you modify the work.) You may place
additional permissions on material, added by you to a covered work,
for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you
add to a covered work, you may (if authorized by the copyright holders of
that material) supplement the terms of this License with terms:
a) Disclaiming warranty or limiting liability differently from the
terms of sections 15 and 16 of this License; or
b) Requiring preservation of specified reasonable legal notices or
author attributions in that material or in the Appropriate Legal
Notices displayed by works containing it; or
c) Prohibiting misrepresentation of the origin of that material, or
requiring that modified versions of such material be marked in
reasonable ways as different from the original version; or
d) Limiting the use for publicity purposes of names of licensors or
authors of the material; or
e) Declining to grant rights under trademark law for use of some
trade names, trademarks, or service marks; or
f) Requiring indemnification of licensors and authors of that
material by anyone who conveys the material (or modified versions of
it) with contractual assumptions of liability to the recipient, for
any liability that these contractual assumptions directly impose on
those licensors and authors.
All other non-permissive additional terms are considered "further
restrictions" within the meaning of section 10. If the Program as you
received it, or any part of it, contains a notice stating that it is
governed by this License along with a term that is a further
restriction, you may remove that term. If a license document contains
a further restriction but permits relicensing or conveying under this
License, you may add to a covered work material governed by the terms
of that license document, provided that the further restriction does
not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you
must place, in the relevant source files, a statement of the
additional terms that apply to those files, or a notice indicating
where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the
form of a separately written license, or stated as exceptions;
the above requirements apply either way.
8. Termination.
You may not propagate or modify a covered work except as expressly
provided under this License. Any attempt otherwise to propagate or
modify it is void, and will automatically terminate your rights under
this License (including any patent licenses granted under the third
paragraph of section 11).
However, if you cease all violation of this License, then your
license from a particular copyright holder is reinstated (a)
provisionally, unless and until the copyright holder explicitly and
finally terminates your license, and (b) permanently, if the copyright
holder fails to notify you of the violation by some reasonable means
prior to 60 days after the cessation.
Moreover, your license from a particular copyright holder is
reinstated permanently if the copyright holder notifies you of the
violation by some reasonable means, this is the first time you have
received notice of violation of this License (for any work) from that
copyright holder, and you cure the violation prior to 30 days after
your receipt of the notice.
Termination of your rights under this section does not terminate the
licenses of parties who have received copies or rights from you under
this License. If your rights have been terminated and not permanently
reinstated, you do not qualify to receive new licenses for the same
material under section 10.
9. Acceptance Not Required for Having Copies.
You are not required to accept this License in order to receive or
run a copy of the Program. Ancillary propagation of a covered work
occurring solely as a consequence of using peer-to-peer transmission
to receive a copy likewise does not require acceptance. However,
nothing other than this License grants you permission to propagate or
modify any covered work. These actions infringe copyright if you do
not accept this License. Therefore, by modifying or propagating a
covered work, you indicate your acceptance of this License to do so.
10. Automatic Licensing of Downstream Recipients.
Each time you convey a covered work, the recipient automatically
receives a license from the original licensors, to run, modify and
propagate that work, subject to this License. You are not responsible
for enforcing compliance by third parties with this License.
An "entity transaction" is a transaction transferring control of an
organization, or substantially all assets of one, or subdividing an
organization, or merging organizations. If propagation of a covered
work results from an entity transaction, each party to that
transaction who receives a copy of the work also receives whatever
licenses to the work the party's predecessor in interest had or could
give under the previous paragraph, plus a right to possession of the
Corresponding Source of the work from the predecessor in interest, if
the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the
rights granted or affirmed under this License. For example, you may
not impose a license fee, royalty, or other charge for exercise of
rights granted under this License, and you may not initiate litigation
(including a cross-claim or counterclaim in a lawsuit) alleging that
any patent claim is infringed by making, using, selling, offering for
sale, or importing the Program or any portion of it.
11. Patents.
A "contributor" is a copyright holder who authorizes use under this
License of the Program or a work on which the Program is based. The
work thus licensed is called the contributor's "contributor version".
A contributor's "essential patent claims" are all patent claims
owned or controlled by the contributor, whether already acquired or
hereafter acquired, that would be infringed by some manner, permitted
by this License, of making, using, or selling its contributor version,
but do not include claims that would be infringed only as a
consequence of further modification of the contributor version. For
purposes of this definition, "control" includes the right to grant
patent sublicenses in a manner consistent with the requirements of
this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free
patent license under the contributor's essential patent claims, to
make, use, sell, offer for sale, import and otherwise run, modify and
propagate the contents of its contributor version.
In the following three paragraphs, a "patent license" is any express
agreement or commitment, however denominated, not to enforce a patent
(such as an express permission to practice a patent or covenant not to
sue for patent infringement). To "grant" such a patent license to a
party means to make such an agreement or commitment not to enforce a
patent against the party.
If you convey a covered work, knowingly relying on a patent license,
and the Corresponding Source of the work is not available for anyone
to copy, free of charge and under the terms of this License, through a
publicly available network server or other readily accessible means,
then you must either (1) cause the Corresponding Source to be so
available, or (2) arrange to deprive yourself of the benefit of the
patent license for this particular work, or (3) arrange, in a manner
consistent with the requirements of this License, to extend the patent
license to downstream recipients. "Knowingly relying" means you have
actual knowledge that, but for the patent license, your conveying the
covered work in a country, or your recipient's use of the covered work
in a country, would infringe one or more identifiable patents in that
country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or
arrangement, you convey, or propagate by procuring conveyance of, a
covered work, and grant a patent license to some of the parties
receiving the covered work authorizing them to use, propagate, modify
or convey a specific copy of the covered work, then the patent license
you grant is automatically extended to all recipients of the covered
work and works based on it.
A patent license is "discriminatory" if it does not include within
the scope of its coverage, prohibits the exercise of, or is
conditioned on the non-exercise of one or more of the rights that are
specifically granted under this License. You may not convey a covered
work if you are a party to an arrangement with a third party that is
in the business of distributing software, under which you make payment
to the third party based on the extent of your activity of conveying
the work, and under which the third party grants, to any of the
parties who would receive the covered work from you, a discriminatory
patent license (a) in connection with copies of the covered work
conveyed by you (or copies made from those copies), or (b) primarily
for and in connection with specific products or compilations that
contain the covered work, unless you entered into that arrangement,
or that patent license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting
any implied license or other defenses to infringement that may
otherwise be available to you under applicable patent law.
12. No Surrender of Others' Freedom.
If conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot convey a
covered work so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you may
not convey it at all. For example, if you agree to terms that obligate you
to collect a royalty for further conveying from those to whom you convey
the Program, the only way you could satisfy both those terms and this
License would be to refrain entirely from conveying the Program.
13. Use with the GNU Affero General Public License.
Notwithstanding any other provision of this License, you have
permission to link or combine any covered work with a work licensed
under version 3 of the GNU Affero General Public License into a single
combined work, and to convey the resulting work. The terms of this
License will continue to apply to the part which is the covered work,
but the special requirements of the GNU Affero General Public License,
section 13, concerning interaction through a network will apply to the
combination as such.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of
the GNU General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the
Program specifies that a certain numbered version of the GNU General
Public License "or any later version" applies to it, you have the
option of following the terms and conditions either of that numbered
version or of any later version published by the Free Software
Foundation. If the Program does not specify a version number of the
GNU General Public License, you may choose any version ever published
by the Free Software Foundation.
If the Program specifies that a proxy can decide which future
versions of the GNU General Public License can be used, that proxy's
public statement of acceptance of a version permanently authorizes you
to choose that version for the Program.
Later license versions may give you additional or different
permissions. However, no additional obligations are imposed on any
author or copyright holder as a result of your choosing to follow a
later version.
15. Disclaimer of Warranty.
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. Limitation of Liability.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
SUCH DAMAGES.
17. Interpretation of Sections 15 and 16.
If the disclaimer of warranty and limitation of liability provided
above cannot be given local legal effect according to their terms,
reviewing courts shall apply local law that most closely approximates
an absolute waiver of all civil liability in connection with the
Program, unless a warranty or assumption of liability accompanies a
copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
state the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If the program does terminal interaction, make it output a short
notice like this when it starts in an interactive mode:
<program> Copyright (C) <year> <name of author>
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate
parts of the General Public License. Of course, your program's commands
might be different; for a GUI interface, you would use an "about box".
You should also get your employer (if you work as a programmer) or school,
if any, to sign a "copyright disclaimer" for the program, if necessary.
For more information on this, and how to apply and follow the GNU GPL, see
<https://www.gnu.org/licenses/>.
The GNU General Public License does not permit incorporating your program
into proprietary programs. If your program is a subroutine library, you
may consider it more useful to permit linking proprietary applications with
the library. If this is what you want to do, use the GNU Lesser General
Public License instead of this License. But first, please read
<https://www.gnu.org/licenses/why-not-lgpl.html>.
reset/packages/envied/README.md
+76
View File
@@ -0,0 +1,76 @@
# Envied in twinvine
**Note**
*Envied has already been installed by TwinVine. These notes are retained for information only*
## What is envied?
Envied is a fork of [Devine](https://github.com/devine-dl/devine/). The name 'envied' is an anagram of Devine, and as such, pays homage to the original author.
Is is based on v 1.4.7 of envied.It is a powerful archival tool for downloading movies, TV shows, and music from streaming services. Built with a focus on modularity and extensibility, it provides a robust framework for content acquisition with support for DRM-protected content.
No commands have been changed 'uv run envied' still works as usual.
The major difference is that envied comes complete and needs little configuration.
A basic CDM and services are taken care of.
The prime reason for the existence of envied is a --select-titles function.
If you already use envied you'll probably just want to replace envied/envied/envied.yaml
with your own. But the exisiting yaml is close to working - just needs a few directory locations.
Envied's existence helps keep download tools free. Unshackle is flirting with pay-for access, having a free, closley matched, aternative puts a break on any pay-me aspirations unshackle might have now or in the future.
## Divergence from Envied's Parent
- **select-titles option** avoid the uncertainty of -w S26E03 gobbledegook to get your video
- **Singleton Design Pattern** Good coding practice: where possible, a single instance of a Class is created and re-used. Saving time and resources.
- **Multitron Design Pattern** For those times when a Singleton will not do. Re-use Classes with care for the calling parameters.
- **Clear Branding** Clear presentation of the program name!
- **No Free Ride for Spammers** Envied will not implement any methods used to connect to Decrypt Labs or other payfor-CDM-access sites
**Recommended:** Use `uv run envied` instead of direct command execution to ensure proper virtual environment activation.
### Basic Usage
```shell
# Download content (requires configured services)
# from inside the TwinVine top level folder:-
uv run envied dl SERVICE_NAME CONTENT_ID
```
## Documentation
For comprehensive setup guides, configuration options, and advanced usage:
📖 **[Visit their WIKI](https://github.com/unshackle-dl/unshackle/wiki)**
The WIKI contains detailed information on:
- Service configuration
- DRM configuration
- Advanced features and troubleshooting
For guidance on creating services, see their [WIKI documentation](https://github.com/unshackle-dl/unshackle/wiki).
## End User License Agreement
Envied, and it's community pages, should be treated with the same kindness as other projects.
Please refrain from spam or asking for questions that infringe upon a Service's End User License Agreement.
1. Do not use envied for any purposes of which you do not have the rights to do so.
2. Do not share or request infringing content; this includes widevine Provision Keys, Content Encryption Keys,
or Service API Calls or Code.
3. The Core codebase is meant to stay Free and Open-Source while the Service code should be kept private.
4. Do not sell any part of this project, neither alone nor as part of a bundle.
If you paid for this software or received it as part of a bundle following payment, you should demand your money
back immediately.
5. Be kind to one another and do not single anyone out.
## Licensing
This software is licensed under the terms of [GNU General Public License, Version 3.0](LICENSE).
You can find a copy of the license in the LICENSE file in the root folder.
reset/packages/envied/img/envied1.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 126 KiB

reset/packages/envied/pyproject.toml
+133
View File
@@ -0,0 +1,133 @@
[build-system]
requires = ["uv_build>=0.8.12,<0.9.21"]
build-backend = "uv_build"
[project]
name = "envied"
version = "5.3.0"
description = "Modular Movie, TV, and Music Archival Software."
authors = [{ name = "rlaphoenix and others" }]
requires-python = ">=3.10,<=3.14"
readme = "README.md"
license = "GPL-3.0-only"
keywords = [
"python",
"downloader",
"drm",
"widevine",
]
classifiers = [
"Development Status :: 4 - Beta",
"Environment :: Console",
"Intended Audience :: End Users/Desktop",
"Natural Language :: English",
"Operating System :: OS Independent",
"Topic :: Multimedia :: Video",
"Topic :: Security :: Cryptography",
]
dependencies = [
"appdirs>=1.4.4,<2",
"Brotli>=1.2.0,<2",
"click>=8.1.8,<9",
"construct>=2.8.8,<3",
"crccheck>=1.3.0,<2",
"filelock>=3.20.3,<4",
"fonttools>=4.60.2,<5",
"jsonpickle>=3.0.4,<5",
"langcodes>=3.4.0,<4",
"lxml>=5.2.1,<7",
"protobuf>=4.25.3,<7",
"pycaption>=2.2.6,<3",
"pycryptodomex>=3.20.0,<4",
"pyjwt>=2.12.0,<3",
"pymediainfo>=6.1.0,<8",
"pymp4>=1.4.0,<2",
"pymysql>=1.1.0,<2",
"pywidevine[serve]>=1.8.0,<2",
"PyYAML>=6.0.1,<7",
"requests[socks]>=2.32.5,<3",
"rich>=13.7.1,<15.1",
"rlaphoenix.m3u8>=3.4.0,<4",
"ruamel.yaml>=0.18.6,<0.19",
"sortedcontainers>=2.4.0,<3",
"subtitle-filter>=1.4.9,<2",
"Unidecode>=1.3.8,<2",
"urllib3>=2.6.3,<3",
"chardet>=5.2.0,<6",
"pyplayready>=0.8.3,<0.9",
"cryptography>=45.0.0,<47",
"subby",
"aiohttp>=3.13.4,<4",
"aiohttp-swagger3>=0.9.0,<1",
"pysubs2>=1.7.0,<2",
"PyExecJS>=1.5.1,<2",
"pycountry>=24.6.1",
"language-data>=1.4.0",
"wasmtime>=41.0.0",
"animeapi-py>=0.6.0",
"rnet>=2.4.2",
"bandit>=1.9.4",
"defusedxml>=0.7.1",
"jsonpath-ng>=1.8.0",
"mutagen>=1.47.0",
# envied specific dependencies
"webvtt-py>=0.5.1,<0.6",
"isodate>=0.7.2,<0.8",
"scrapy>=2.14.0,<3",
"mypy>=1.19.1,<=2.1.0",
"httpx>=0.28.0,<0.35",
# Keeping your existing cap style here; latest visible is 0.10.0.
#"aiohttp-swagger3>=0.10.0,<1",
]
[project.urls]
Homepage = "https://github.com/envied-dl/envied"
Repository = "https://github.com/envied-dl/envied"
Issues = "https://github.com/envied-dl/envied/issues"
Discussions = "https://github.com/envied-dl/envied/discussions"
Changelog = "https://github.com/envied-dl/envied/blob/master/CHANGELOG.md"
[project.scripts]
envied = "envied.core.__main__:main"
[dependency-groups]
dev = [
"pre-commit>=3.7.0,<4",
"mypy>=1.18.2,<2",
"mypy-protobuf>=3.6.0,<4",
"types-protobuf>=4.24.0.20240408,<5",
"types-PyMySQL>=1.1.0.1,<2",
"types-requests>=2.31.0.20240406,<3",
"isort>=5.13.2,<6",
"ruff~=0.3.7"
]
#[tool.hatch.build.targets.wheel]
#packages = ["envied"]
[tool.ruff]
force-exclude = true
line-length = 120
[tool.ruff.lint]
select = ["E4", "E7", "E9", "F", "W"]
[tool.isort]
line_length = 118
[tool.mypy]
check_untyped_defs = true
disallow_incomplete_defs = true
disallow_untyped_defs = true
follow_imports = "silent"
ignore_missing_imports = true
no_implicit_optional = true
[tool.uv.sources]
#subby = { git = "https://github.com/vevv/subby.git" }
subby = { git = "https://github.com/vevv/subby.git", rev = "a057280f31acd3a0ba5c6dc69eaefbb88aea974e" }
reset/packages/envied/src/envied/CHANGELOG.md
+747
View File
@@ -0,0 +1,747 @@
# Changelog
All notable changes to this project will be documented in this file.
## 5.3.0 01-07-2026
Repository: unshackle-dl/unshackle · Tag: 5.3.0 · Commit: d4a5826 · Released by: imSp4rky
Features
--merge-video: merge video language variants into one MKV per (resolution, range, codec) group
Native ExpressVPN HTTPS proxy provider (full OAuth PKCE flow, no external tools)
Proton VPN proxy provider with country/city/server-number selection, plus ProtonVPN TV-code login
Load service plugins from git repos (clone + TTL-pull cache, refresh-services command, dirty-clone guard)
Python 3.14 support (supported range widened to 3.11-3.14)
Automatic Firefox cookie and localStorage extraction
Nested directories in folder templates (/ treated as path separator)
air_date for date-based episode naming (daily/sports content named by date instead of SxxExx)
Per-vault network timeout for remote API/HTTP vaults
Richer API job progress: active-track segment counts, transfer speed, title description/date/cover_url
Bug Fixes
drm: don't switch to a mismatched CDM type during licensing; raise a clear error instead
config: match per-service cdm keys case-insensitively
dl: apply per-service decryption tool override before decrypt
hls: stream segment merges to avoid OOM on large (multi-GB) tracks
console: pause active live/spinner contexts during terminal input so prompts stay visible
output: treat both / and \ as folder separators on any OS
env: only resolve Path items in directory lists (git repo specs pass through)
Performance & Changes
perf(hls): rename single decrypted range instead of re-copying a full track
perf(tracks): skip redundant DV/VUI bitstream passes when VUI already matches range
api: collapse route registration, dedup handlers, remove dead serve/remote code paths
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [2.3.0] - 2026-01-18
### Added
- **Unicode Filenames Option**: New `unicode_filenames` config option to preserve native characters
- Allows disabling ASCII transliteration in filenames
- Preserves Korean, Japanese, Chinese, and other native language characters
- Closes #49
### Fixed
- **WebVTT Cue Handling**: Handle WebVTT cue identifiers and overlapping multi-line cues
- Added detection and sanitization for cue identifiers (Q0, Q1, etc.) before timing lines
- Added merging of overlapping cues with different line positions into multi-line subtitles
- Fixes parsing issues with pysubs2/pycaption on certain WebVTT files
- **Widevine PSSH Filtering**: Filter Widevine PSSH by system ID instead of sorting
- Fixes KeyError crash when unsupported DRM systems are present in init segments
- **TTML Negative Values**: Handle negative values in multi-value TTML attributes
- Fixes pycaption parse errors for attributes like `tts:extent="-5% 7.5%"`
- Closes #47
- **ASS Font Names**: Strip whitespace from ASS font names
- Handles ASS subtitle files with spaces after commas in Style definitions
- Fixes #57
- **Shaka-Packager Error Messages**: Include shaka-packager binary path in error messages
- **N_m3u8DL-RE Merge and Decryption**: Handle merge and decryption properly
- Prevents audio corruption ("Box 'OG 2' size is too large") with DASH manifests
- Fixes duplicate init segment writing when using N_m3u8DL-RE
- **DASH Placeholder KIDs**: Handle placeholder KIDs and improve DRM init from segments
- Detects and replaces placeholder/test KIDs in Widevine PSSH
- Adds CENC namespace support for kid/default_KID attributes
- **PlayReady PSSH Comparison**: Correct PSSH system ID comparison in PlayReady
- Removes erroneous `.bytes` accessor from PSSH.SYSTEM_ID comparisons
## [2.2.0] - 2026-01-15
### Added
- **CDM-Aware PlayReady Fallback Detection**: Intelligent DRM fallback based on selected CDM
- Adds PlayReady PSSH/KID extraction from track and init data with CDM-aware ordering
- When PlayReady CDM is selected, tries PlayReady first then falls back to Widevine
- When Widevine CDM is selected (default), tries Widevine first then falls back to PlayReady
- **Comprehensive Debug Logging**: Enhanced debug logging for downloaders and muxing
- Added detailed debug logging to aria2c, curl_impersonate, n_m3u8dl_re, and requests downloaders
- Enhanced manifest parsers (DASH, HLS, ISM) with debug logging
- Added debug logging to track muxing operations
### Fixed
- **Hybrid DV+HDR10 Filename Detection**: Fixed HDR10 detection in hybrid Dolby Vision filenames
- Hybrid DV+HDR10 files were incorrectly named "DV.H.265" instead of "DV.HDR.H.265"
- Now checks both `hdr_format_full` and `hdr_format_commercial` fields for HDR10 indicators
- **Vault Adaptive Batch Sizing**: Improved bulk key operations with adaptive batch sizing
- Prevents query limit issues when retrieving large numbers of keys from vaults
- Dynamically adjusts batch sizes based on vault response characteristics
- **Test Command Improvements**: Enhanced test command error detection and sorting
- Improved error detection in test command output
- Added natural sorting for test results
## [2.1.0] - 2025-11-27
### Added
- **Per-Track Quality-Based CDM Selection**: Dynamic CDM switching during runtime DRM operations
- Enables quality-based CDM selection during runtime DRM switching
- Different CDMs can be used for different video quality levels within the same download session
- Example: Use Widevine L3 for SD/HD and PlayReady SL3 for 4K content
- **Enhanced Track Export**: Improved export functionality with additional metadata
- Added URL field to track export for easier identification
- Added descriptor information in export output
- Keys now exported in hex-formatted strings
### Changed
- **Dependencies**: Upgraded to latest compatible versions
- Updated various dependencies to their latest versions
### Fixed
- **Attachment Preservation**: Fixed attachments being dropped during track filtering
- Attachments (screenshots, fonts) were being lost when track list was rebuilt
- Fixes image files remaining in temp directory after muxing
- **DASH BaseURL Resolution**: Added AdaptationSet-level BaseURL support per DASH spec
- URL resolution chain now properly follows: MPD → Period → AdaptationSet → Representation
- **WindscribeVPN Region Support**: Restricted to supported regions with proper error handling
- Added error handling for unsupported regions in get_proxy method
- Prevents cryptic errors when using unsupported region codes
- **Filename Sanitization**: Fixed space-hyphen-space handling in filenames
- Pre-process space-hyphen-space patterns (e.g., "Title - Episode") before other replacements
- Made space-hyphen-space handling conditional on scene_naming setting
- Addresses PR #44 by fixing the root cause
- **CICP Enum Values**: Corrected values to match ITU-T H.273 specification
- Added Primaries.Unspecified (value 2) per H.273 spec
- Renamed Primaries/Transfer value 0 from Unspecified to Reserved for spec accuracy
- Simplified Transfer value 2 from Unspecified_Image to Unspecified
- Verified against ITU-T H.273, ISO/IEC 23091-2, H.264/H.265 specs, and FFmpeg enums
- **HLS Byte Range Parsing**: Fixed TypeError in range_offset conversion
- Converted range_offset to int to prevent TypeError in calculate_byte_range
- **pyplayready Compatibility**: Pinned to <0.7 to avoid KID extraction bug
## [2.0.0] - 2025-11-10
### Breaking Changes
- **REST API Integration**: Core architecture modified to support REST API functionality
- Changes to internal APIs for download management and tracking
- Title and track classes updated with API integration points
- Core component interfaces modified for queue management support
- **Configuration Changes**: New required configuration options for API and enhanced features
- Added `simkl_client_id` now required for Simkl functionality
- Service-specific configuration override structure introduced
- Debug logging configuration options added
- **Forced Subtitles**: Behavior change for forced subtitle inclusion
- Forced subs no longer auto-included, requires explicit `--forced-subs` flag
### Added
- **REST API Server**: Complete download management via REST API (early development)
- Implemented download queue management and worker system
- Added OpenAPI/Swagger documentation for easy API exploration
- Included download progress tracking and status endpoints
- API authentication and comprehensive error handling
- Updated core components to support API integration
- Early development work with more changes planned
- **CustomRemoteCDM**: Highly configurable custom CDM API support
- Support for third-party CDM API providers with maximum configurability
- Full configuration through YAML without code changes
- Addresses GitHub issue #26 for flexible CDM integration
- **WindscribeVPN Proxy Provider**: New VPN provider support
- Added WindscribeVPN following NordVPN and SurfsharkVPN patterns
- Fixes GitHub issue #29
- **Latest Episode Download**: New `--latest-episode` CLI option
- `-le, --latest-episode` flag to download only the most recent episode
- Automatically selects the single most recent episode regardless of season
- Fixes GitHub issue #28
- **Video Track Exclusion**: New `--no-video` CLI option
- `-nv, --no-video` flag to skip downloading video tracks
- Allows downloading only audio, subtitles, attachments, and chapters
- Useful for audio-only or subtitle extraction workflows
- Fixes GitHub issue #39
- **Service-Specific Configuration Overrides**: Per-service fine-tuned control
- Support for per-service configuration overrides in YAML
- Fine-tuned control of downloader and command options per service
- Fixes GitHub issue #13
- **Comprehensive JSON Debug Logging**: Structured logging for troubleshooting
- Binary toggle via `--debug` flag or `debug: true` in config
- JSON Lines (.jsonl) format for easy parsing and analysis
- Comprehensive logging of all operations (session info, CLI params, CDM details, auth status, title/track metadata, DRM operations, vault queries)
- Configurable key logging via `debug_keys` option with smart redaction
- Error logging for all critical operations
- Removed old text logging system
- **curl_cffi Retry Handler**: Enhanced session reliability
- Added automatic retry mechanism to curl_cffi Session
- Improved download reliability with configurable retries
- **Simkl API Configuration**: New API key support
- Added `simkl_client_id` configuration option
- Simkl now requires client_id from https://simkl.com/settings/developer/
- **Custom Session Fingerprints**: Enhanced browser impersonation capabilities
- Added custom fingerprint and preset support for better service compatibility
- Configurable fingerprint presets for different device types
- Improved success rate with services using advanced bot detection
- **TMDB and Simkl Metadata Caching**: Enhanced title cache system
- Added metadata caching to title cache to reduce API calls
- Caches movie/show metadata alongside title information
- Improves performance for repeated title lookups and reduces API rate limiting
- **API Enhancements**: Improved REST API functionality
- Added default parameter handling for better request processing
- Added URL field to services endpoint response for easier service identification
- Complete API enhancements for production readiness
- Improved error responses with better detail and debugging information
### Changed
- **Binary Search Enhancement**: Improved binary discovery
- Refactored to search for binaries in root of binary folder or subfolder named after the binary
- Better organization of binary dependencies
- **Type Annotations**: Modernized to PEP 604 syntax
- Updated session.py type annotations to use modern Python syntax
- Improved code readability and type checking
### Fixed
- **Audio Description Track Support**: Added option to download audio description tracks
- Added `--audio-description/-ad` flag to optionally include descriptive audio tracks
- Previously, audio description tracks were always filtered out
- Users can now choose to download AD tracks when needed
- Fixes GitHub issue #33
- **Config Directory Support**: Cross-platform user config directory support
- Fixed config loading to properly support user config directories across all platforms
- Fixes GitHub issue #23
- **HYBRID Mode Validation**: Pre-download validation for hybrid processing
- Added validation to check both HDR10 and DV tracks are available before download
- Prevents wasted downloads when hybrid processing would fail
- **TMDB/Simkl API Keys**: Graceful handling of missing API keys
- Improved error handling when TMDB or Simkl API keys are not configured
- Better user messaging for API configuration requirements
- **Forced Subtitles Behavior**: Correct forced subtitle filtering
- Fixed forced subtitles being incorrectly included without `--forced-subs` flag
- Forced subs now only included when explicitly requested
- **Font Attachment Constructor**: Fixed ASS/SSA font attachment
- Use keyword arguments for Attachment constructor in font attachment
- Fixes "Invalid URL: No scheme supplied" error
- Fixes GitHub issue #24
- **Binary Subdirectory Checking**: Enhanced binary location discovery (by @TPD94, PR #19)
- Updated binaries.py to check subdirectories in binaries folders named after the binary
- Improved binary detection and loading
- **HLS Manifest Processing**: Minor HLS parser fix (by @TPD94, PR #19)
- **lxml and pyplayready**: Updated dependencies (by @Sp5rky)
- Updated lxml constraint and pyplayready import path for compatibility
- **DASH Segment Calculation**: Corrected segment handling
- Fixed segment count calculation for DASH manifests with startNumber=0
- Ensures accurate segment processing for all DASH manifest configurations
- Prevents off-by-one errors in segment downloads
- **HDR Detection and Naming**: Comprehensive HDR format support
- Improved HDR detection with comprehensive transfer characteristics checks
- Added hybrid DV+HDR10 support for accurate file naming
- Better identification of HDR formats across different streaming services
- More accurate HDR/DV detection in filename generation
- **Subtitle Processing**: VTT subtitle handling improvements
- Resolved SDH (Subtitles for Deaf and Hard of hearing) stripping crash when processing VTT files
- More robust subtitle processing pipeline with better error handling
- Fixes crashes when filtering specific VTT subtitle formats
- **DRM Processing**: Enhanced encoding handling
- Added explicit UTF-8 encoding to mp4decrypt subprocess calls
- Prevents encoding issues on systems with non-UTF-8 default encodings
- Improves cross-platform compatibility for Windows and some Linux configurations
- **Session Fingerprints**: Updated OkHttp presets
- Updated OkHttp fingerprint presets for better Android TV compatibility
- Improved success rate with services using fingerprint-based detection
### Documentation
- **GitHub Issue Templates**: Enhanced issue reporting
- Improved bug report template with better structure and required fields
- Enhanced feature request template for clearer specifications
- Added helpful guidance for contributors to provide complete information
### Refactored
- **Import Cleanup**: Removed unused imports
- Removed unused mypy import from binaries.py
- Fixed import ordering in API download_manager and handlers
### Contributors
This release includes contributions from:
- @Sp5rky - REST API server implementation, dependency updates
- @stabbedbybrick - curl_cffi retry handler (PR #31)
- @stabbedbybrick - n_m3u8dl-re refactor (PR #38)
- @TPD94 - Binary search enhancements, manifest parser fixes (PR #19)
- @scene (Andy) - Core features, configuration system, bug fixes
## [1.4.8] - 2025-10-08
### Added
- **Exact Language Matching**: New `--exact-lang` flag for precise language matching
- Enables strict language code matching without fallbacks
- **No-Mux Flag**: New `--no-mux` flag to skip muxing tracks into container files
- Useful for keeping individual track files separate
- **DecryptLabs API Integration for HTTP Vault**: Enhanced vault support
- Added DecryptLabs API support to HTTP vault for improved key retrieval
- **AC4 Audio Codec Support**: Enhanced audio format handling
- Added AC4 codec support in Audio class with updated mime/profile handling
- **pysubs2 Subtitle Conversion**: Extended subtitle format support
- Added pysubs2 subtitle conversion with extended format support
- Configurable conversion method in configuration
### Changed
- **Audio Track Sorting**: Optimized audio track selection logic
- Improved audio track sorting by grouping descriptive tracks and sorting by bitrate
- Better identification of ATMOS and DD+ as highest quality for filenaming
- **pyplayready Update**: Upgraded to version 0.6.3
- Updated import paths to resolve compatibility issues
- Fixed lxml constraints for better dependency management
- **pysubs2 Conversion Method**: Moved from auto to manual configuration
- pysubs2 no longer auto-selected during testing phase
### Fixed
- **Remote CDM**: Fixed curl_cffi compatibility
- Added curl_cffi to instance checks in RemoteCDM
- **Temporary File Handling**: Improved encoding handling
- Specified UTF-8 encoding when opening temporary files
### Reverted
- **tinycss SyntaxWarning Suppression**: Removed ineffective warning filter
- Reverted warnings filter that didn't work as expected for suppressing tinycss warnings
## [1.4.7] - 2025-09-25
### Added
- **curl_cffi Session Support**: Enhanced anti-bot protection with browser impersonation
- Added new session utility with curl_cffi support for bypassing anti-bot measures
- Browser impersonation support for Chrome, Firefox, and Safari user agents
- Full backward compatibility with requests.Session maintained
- Suppressed HTTPS proxy warnings for improved user experience
- **Download Retry Functionality**: Configurable retry mechanism for failed downloads
- Added retry count option to download function for improved reliability
- **Subtitle Requirements Options**: Enhanced subtitle download control
- Added options for required subtitles in download command
- Better control over subtitle track selection and requirements
- **Quality Selection Enhancement**: Improved quality selection options
- Added best available quality option in download command for optimal track selection
- **DecryptLabs API Integration**: Enhanced remote CDM configuration
- Added decrypt_labs_api_key to Config initialization for better API integration
### Changed
- **Manifest Parser Updates**: Enhanced compatibility across all parsers
- Updated DASH, HLS, ISM, and M3U8 parsers to accept curl_cffi sessions
- Improved cookie handling compatibility between requests and curl_cffi
- **Logging Improvements**: Reduced log verbosity for better user experience
- Changed duplicate track log level to debug to reduce console noise
- Dynamic CDM selection messages moved to debug-only output
### Fixed
- **Remote CDM Reuse**: Fixed KeyError in dynamic CDM selection
- Prevents KeyError when reusing remote CDMs in dynamic selection process
- Creates copy of CDM dictionary before modification to prevent configuration mutation
- Allows same CDM to be selected multiple times within session without errors
## [1.4.6] - 2025-09-13
### Added
- **Quality-Based CDM Selection**: Dynamic CDM selection based on video resolution
- Automatically selects appropriate CDM (L3/L1) based on video track quality
- Supports quality thresholds in configuration (>=, >, <=, <, exact match)
- Pre-selects optimal CDM based on highest quality across all video tracks
- Maintains backward compatibility with existing CDM configurations
- **Automatic Audio Language Metadata**: Intelligent embedded audio language detection
- Automatically sets audio language metadata when no separate audio tracks exist
- Smart video track selection based on title language with fallbacks
- Enhanced FFmpeg repackaging with audio stream metadata injection
- **Lazy DRM Loading**: Deferred DRM loading for multi-track key retrieval optimization
- Add deferred DRM loading to M3U8 parser to mark tracks for later processing
- Just-in-time DRM loading during download process for better performance
### Changed
- **Enhanced CDM Management**: Improved CDM switching logic for multi-quality downloads
- CDM selection now based on highest quality track to avoid inefficient switching
- Quality-based selection only within same DRM type (Widevine-to-Widevine, PlayReady-to-PlayReady)
- Single CDM used per session for better performance and reliability
### Fixed
- **Vault Caching Issues**: Fixed vault count display and NoneType iteration errors
- Fix 'NoneType' object is not iterable error in DecryptLabsRemoteCDM
- Fix vault count display showing 0/3 instead of actual successful vault count
- **Service Name Transmission**: Resolved DecryptLabsRemoteCDM service name issues
- Fixed DecryptLabsRemoteCDM sending 'generic' instead of proper service names
- Added case-insensitive vault lookups for SQLite/MySQL vaults
- Added local vault integration to DecryptLabsRemoteCDM
- **Import Organization**: Improved import ordering and code formatting
- Reorder imports in decrypt_labs_remote_cdm.py for better organization
- Clean up trailing whitespace in vault files
### Configuration
- **New CDM Configuration Format**: Extended `cdm:` section supports quality-based selection
```yaml
cdm:
SERVICE_NAME:
"<=1080": l3_cdm_name
">1080": l1_cdm_name
default: l3_cdm_name
```
## [1.4.5] - 2025-09-09
### Added
- **Enhanced CDM Key Caching**: Improved key caching and session management for L1/L2 devices
- Optimized `get_cached_keys_if_exists` functionality for better performance with L1/L2 devices
- Enhanced cached key retrieval logic with improved session handling
- **Widevine Common Certificate Fallback**: Added fallback to Widevine common certificate for L1 devices
- Improved compatibility for L1 devices when service certificates are unavailable
- **Enhanced Vault Loading**: Improved vault loading and key copying logic
- Better error handling and key management in vault operations
- **PSSH Display Optimization**: Truncated PSSH string display in non-debug mode for cleaner output
- **CDM Error Messaging**: Added error messages for missing service certificates in CDM sessions
### Changed
- **Dynamic Version Headers**: Updated User-Agent headers to use dynamic version strings
- DecryptLabsRemoteCDM now uses dynamic version import instead of hardcoded version
- **Intelligent CDM Caching**: Implemented intelligent caching system for CDM license requests
- Enhanced caching logic reduces redundant license requests and improves performance
- **Enhanced Tag Handling**: Improved tag handling for TV shows and movies from Simkl data
- Better metadata processing and formatting for improved media tagging
### Fixed
- **CDM Session Management**: Clean up session data when retrieving cached keys
- Remove decrypt_labs_session_id and challenge from session when cached keys exist but there are missing kids
- Ensures clean state for subsequent requests and prevents session conflicts
- **Tag Formatting**: Fixed formatting issues in tag processing
- **Import Order**: Fixed import order issues in tags module
## [1.4.4] - 2025-09-02
### Added
- **Enhanced DecryptLabs CDM Support**: Comprehensive remote CDM functionality
- Full support for Widevine, PlayReady, and ChromeCDM through DecryptLabsRemoteCDM
- Enhanced session management and caching support for remote WV/PR operations
- Support for cached keys and improved license handling
- New CDM configurations for Chrome and PlayReady devices with updated User-Agent and service certificate
- **Advanced Configuration Options**: New device and language preferences
- Added configuration options for device certificate status list
- Enhanced language preference settings
### Changed
- **DRM Decryption Enhancements**: Streamlined decryption process
- Simplified decrypt method by removing unused parameter and streamlined logic
- Improved DecryptLabs CDM configurations with better device support
### Fixed
- **Matroska Tag Compliance**: Enhanced media container compatibility
- Fixed Matroska tag compliance with official specification
- **Application Branding**: Cleaned up version display
- Removed old devine version reference from banner to avoid developer confusion
- Updated branding while maintaining original GNU license compliance
- **IP Information Handling**: Improved geolocation services
- Enhanced get_ip_info functionality with better failover handling
- Added support for 429 error handling and multiple API provider fallback
- Implemented cached IP info retrieval with fallback tester to avoid rate limiting
- **Dependencies**: Streamlined package requirements
- Removed unnecessary data extra requirement from langcodes
### Removed
- Deprecated version references in application banner for clarity
## [1.4.3] - 2025-08-20
### Added
- Cached IP info helper for region detection
- New `get_cached_ip_info()` with 24h cache and provider rotation (ipinfo/ipapi) with 429 handling.
- Reduces external calls and stabilizes non-proxy region lookups for caching/logging.
### Changed
- DRM decryption selection is fully configuration-driven
- Widevine and PlayReady now select the decrypter based solely on `decryption` in YAML (including per-service mapping).
- Shaka Packager remains the default decrypter when not specified.
- `dl.py` logs the chosen tool based on the resolved configuration.
- Geofencing and proxy verification improvements
- Safer geofence checks with error handling and clearer logs.
- Always verify proxy exit region via live IP lookup; fallback to proxy parsing on failure.
- Example config updated to default to Shaka
- `envied.yaml`/example now sets `decryption.default: shaka` (service overrides still supported).
### Removed
- Deprecated parameter `use_mp4decrypt`
- Removed from `Widevine.decrypt()` and `PlayReady.decrypt()` and all callsites.
- Internal naming switched from mp4decrypt-specific flags to generic `decrypter` selection.
## [1.4.2] - 2025-08-14
### Added
- **Session Management for API Requests**: Enhanced API reliability with retry logic
- Implemented session management for tags functionality with automatic retry mechanisms
- Improved API request stability and error handling
- **Series Year Configuration**: New `series_year` option for title naming control
- Added configurable `series_year` option to control year inclusion in series titles
- Enhanced YAML configuration with series year handling options
- **Audio Language Override**: New audio language selection option
- Added `audio_language` option to override default language selection for audio tracks
- Provides more granular control over audio track selection
- **Vault Key Reception Control**: Enhanced vault security options
- Added `no_push` option to Vault and its subclasses to control key reception
- Improved key management security and flexibility
### Changed
- **HLS Segment Processing**: Enhanced segment retrieval and merging capabilities
- Enhanced segment retrieval to allow all file types for better compatibility
- Improved segment merging with recursive file search and fallback to binary concatenation
- Fixed issues with VTT files from HLS not being found correctly due to format changes
- Added cleanup of empty segment directories after processing
- **Documentation**: Updated README.md with latest information
### Fixed
- **Audio Track Selection**: Improved per-language logic for audio tracks
- Adjusted `per_language` logic to ensure correct audio track selection
- Fixed issue where all tracks for selected language were being downloaded instead of just the intended ones
## [1.4.1] - 2025-08-08
### Added
- **Title Caching System**: Intelligent title caching to reduce redundant API calls
- Configurable title caching with 30-minute default cache duration
- 24-hour fallback cache on API failures for improved reliability
- Region-aware caching to handle geo-restricted content properly
- SHA256 hashing for cache keys to handle complex title IDs
- Added `--no-cache` CLI flag to bypass caching when needed
- Added `--reset-cache` CLI flag to clear existing cache data
- New cache configuration variables in config system
- Documented caching options in example configuration file
- Significantly improves performance when debugging or modifying CLI parameters
- **Enhanced Tagging Configuration**: New options for customizing tag behavior
- Added `tag_group_name` config option to control group name inclusion in tags
- Added `tag_imdb_tmdb` config option to control IMDB/TMDB details in tags
- Added Simkl API endpoint support as fallback when no TMDB API key is provided
- Enhanced tag_file function to prioritize provided TMDB ID when `--tmdb` flag is used
- Improved TMDB ID handling with better prioritization logic
### Changed
- **Language Selection Enhancement**: Improved default language handling
- Updated language option default to 'orig' when no `-l` flag is set
- Avoids hardcoded 'en' default and respects original content language
- **Tagging Logic Improvements**: Simplified and enhanced tagging functionality
- Simplified Simkl search logic with soft-fail when no results found
- Enhanced tag_file function with better TMDB ID prioritization
- Improved error handling in tagging operations
### Fixed
- **Subtitle Processing**: Enhanced subtitle filtering for edge cases
- Fixed ValueError in subtitle filtering for multiple colons in time references
- Improved handling of subtitles containing complex time formatting
- Better error handling for malformed subtitle timestamps
### Removed
- **Docker Support**: Removed Docker configuration from repository
- Removed Dockerfile and .dockerignore files
- Cleaned up README.md Docker-related documentation
- Focuses on direct installation methods
## [1.4.0] - 2025-08-05
### Added
- **HLG Transfer Characteristics Preservation**: Enhanced video muxing to preserve HLG color metadata
- Added automatic detection of HLG video tracks during muxing process
- Implemented `--color-transfer-characteristics 0:18` argument for mkvmerge when processing HLG content
- Prevents incorrect conversion from HLG (18) to BT.2020 (14) transfer characteristics
- Ensures proper HLG playback support on compatible hardware without manual editing
- **Original Language Support**: Enhanced language selection with 'orig' keyword support
- Added support for 'orig' language selector for both video and audio tracks
- Automatically detects and uses the title's original language when 'orig' is specified
- Improved language processing logic with better duplicate handling
- Enhanced help text to document original language selection usage
- **Forced Subtitle Support**: Added option to include forced subtitle tracks
- New functionality to download and include forced subtitle tracks alongside regular subtitles
- **WebVTT Subtitle Filtering**: Enhanced subtitle processing capabilities
- Added filtering for unwanted cues in WebVTT subtitles
- Improved subtitle quality by removing unnecessary metadata
### Changed
- **DRM Track Decryption**: Improved DRM decryption track selection logic
- Enhanced `get_drm_for_cdm()` method usage for better DRM-CDM matching
- Added warning messages when no matching DRM is found for tracks
- Improved error handling and logging for DRM decryption failures
- **Series Tree Representation**: Enhanced episode tree display formatting
- Updated series tree to show season breakdown with episode counts
- Improved visual representation with "S{season}({count})" format
- Better organization of series information in console output
- **Hybrid Processing UI**: Enhanced extraction and conversion processes
- Added dynamic spinning bars to follow the rest of the codebase design
- Improved visual feedback during hybrid HDR processing operations
- **Track Selection Logic**: Enhanced multi-track selection capabilities
- Fixed track selection to support combining -V, -A, -S flags properly
- Improved flexibility in selecting multiple track types simultaneously
- **Service Subtitle Support**: Added configuration for services without subtitle support
- Services can now indicate if they don't support subtitle downloads
- Prevents unnecessary subtitle download attempts for unsupported services
- **Update Checker**: Enhanced update checking logic and cache handling
- Improved rate limiting and caching mechanisms for update checks
- Better performance and reduced API calls to GitHub
### Fixed
- **PlayReady KID Extraction**: Enhanced KID extraction from PSSH data
- Added base64 support and XML parsing for better KID detection
- Fixed issue where only one KID was being extracted for certain services
- Improved multi-KID support for PlayReady protected content
- **Dolby Vision Detection**: Improved DV codec detection across all formats
- Fixed detection of dvhe.05.06 codec which was not being recognized correctly
- Enhanced detection logic in Episode and Movie title classes
- Better support for various Dolby Vision codec variants
## [1.3.0] - 2025-08-03
### Added
- **mp4decrypt Support**: Alternative DRM decryption method using mp4decrypt from Bento4
- Added `mp4decrypt` binary detection and support in binaries module
- New `decryption` configuration option in envied.yaml for service-specific decryption methods
- Enhanced PlayReady and Widevine DRM classes with mp4decrypt decryption support
- Service-specific decryption mapping allows choosing between `shaka` and `mp4decrypt` per service
- Improved error handling and progress reporting for mp4decrypt operations
- **Scene Naming Configuration**: New `scene_naming` option for controlling file naming conventions
- Added scene naming logic to movie, episode, and song title classes
- Configurable through envied.yaml to enable/disable scene naming standards
- **Terminal Cleanup and Signal Handling**: Enhanced console management
- Implemented proper terminal cleanup on application exit
- Added signal handling for graceful shutdown in ComfyConsole
- **Configuration Template**: New `unshackle-example.yaml` template file
- Replaced main `envied.yaml` with example template to prevent git conflicts
- Users can now modify their local config without affecting repository updates
- **Enhanced Credential Management**: Improved CDM and vault configuration
- Expanded credential management documentation in configuration
- Enhanced CDM configuration examples and guidelines
- **Video Transfer Standards**: Added `Unspecified_Image` option to Transfer enum
- Implements ITU-T H.Sup19 standard value 2 for image characteristics
- Supports still image coding systems and unknown transfer characteristics
- **Update Check Rate Limiting**: Enhanced update checking system
- Added configurable update check intervals to prevent excessive API calls
- Improved rate limiting for GitHub API requests
### Changed
- **DRM Decryption Architecture**: Enhanced decryption system with dual method support
- Updated `dl.py` to handle service-specific decryption method selection
- Refactored `Config` class to manage decryption method mapping per service
- Enhanced DRM decrypt methods with `use_mp4decrypt` parameter for method selection
- **Error Handling**: Improved exception handling in Hybrid class
- Replaced log.exit calls with ValueError exceptions for better error propagation
- Enhanced error handling consistency across hybrid processing
### Fixed
- **Proxy Configuration**: Fixed proxy server mapping in configuration
- Renamed 'servers' to 'server_map' in proxy configuration to resolve Nord/Surfshark naming conflicts
- Updated configuration structure for better compatibility with proxy providers
- **HTTP Vault**: Improved URL handling and key retrieval logic
- Fixed URL processing issues in HTTP-based key vaults
- Enhanced key retrieval reliability and error handling
## [1.2.0] - 2025-07-30
### Added
- **Update Checker**: Automatic GitHub release version checking on startup
- Configurable update notifications via `update_checks` setting in envied.yaml
- Non-blocking HTTP requests with 5-second timeout for performance
- Smart semantic version comparison supporting all version formats (x.y.z, x.y, x)
- Graceful error handling for network issues and API failures
- User-friendly update notifications with current → latest version display
- Direct links to GitHub releases page for easy updates
- **HDR10+ Support**: Enhanced HDR10+ metadata processing for hybrid tracks
- HDR10+ tool binary support (`hdr10plus_tool`) added to binaries module
- HDR10+ to Dolby Vision conversion capabilities in hybrid processing
- Enhanced metadata extraction for HDR10+ content
- **Duration Fix Handling**: Added duration correction for video and hybrid tracks
- **Temporary Directory Management**: Automatic creation of temp directories for attachment downloads
### Changed
- Enhanced configuration system with new `update_checks` boolean option (defaults to true)
- Updated sample envied.yaml with update checker configuration documentation
- Improved console styling consistency using `bright_black` for dimmed text
- **Environment Dependency Check**: Complete overhaul with detailed categorization and status summary
- Organized dependencies by category (Core, HDR, Download, Subtitle, Player, Network)
- Enhanced status reporting with compact summary display
- Improved tool requirement tracking and missing dependency alerts
- **Hybrid Track Processing**: Significant improvements to HDR10+ and Dolby Vision handling
- Enhanced metadata extraction and processing workflows
- Better integration with HDR processing tools
### Removed
- **Docker Workflow**: Removed Docker build and publish GitHub Actions workflow for manual builds
## [1.1.0] - 2025-07-29
### Added
- **HDR10+DV Hybrid Processing**: New `-r HYBRID` command for processing HDR10 and Dolby Vision tracks
- Support for hybrid HDR processing and injection using dovi_tool
- New hybrid track processing module for seamless HDR10/DV conversion
- Automatic detection and handling of HDR10 and DV metadata
- Support for HDR10 and DV tracks in hybrid mode for EXAMPLE service
- Binary availability check for dovi_tool in hybrid mode operations
- Enhanced track processing capabilities for HDR content
### Fixed
- Import order issues and missing json import in hybrid processing
- UV installation process and error handling improvements
- Binary search functionality updated to use `binaries.find`
### Changed
- Updated package version from 1.0.2 to 1.1.0
- Enhanced dl.py command processing for hybrid mode support
- Improved core titles (episode/movie) processing for HDR content
- Extended tracks module with hybrid processing capabilities
reset/packages/envied/src/envied/__init__.py
View File
reset/packages/envied/src/envied/__main__.py
+4
View File
@@ -0,0 +1,4 @@
if __name__ == "__main__":
from envied.core.__main__ import main
main()
reset/packages/envied/src/envied/binaries/ML-Worker
BIN
View File
Binary file not shown.
reset/packages/envied/src/envied/binaries/placehere.txt
View File
reset/packages/envied/src/envied/commands/__init__.py
View File
reset/packages/envied/src/envied/commands/cfg.py
+90
View File
@@ -0,0 +1,90 @@
import ast
import logging
import sys
import click
from ruamel.yaml import YAML
from envied.core.config import config, get_config_path
from envied.core.constants import context_settings
@click.command(
short_help="Manage configuration values for the program and its services.", context_settings=context_settings
)
@click.argument("key", type=str, required=False)
@click.argument("value", type=str, required=False)
@click.option("--unset", is_flag=True, default=False, help="Unset/remove the configuration value.")
@click.option("--list", "list_", is_flag=True, default=False, help="List all set configuration values.")
@click.pass_context
def cfg(ctx: click.Context, key: str, value: str, unset: bool, list_: bool) -> None:
"""
Manage configuration values for the program and its services.
\b
Known Issues:
- Config changes remove all comments of the changed files, which may hold critical data. (#14)
"""
if not key and not value and not list_:
raise click.UsageError("Nothing to do.", ctx)
if value:
try:
value = ast.literal_eval(value)
except (ValueError, SyntaxError):
pass # probably a str without quotes or similar, assume it's a string value
log = logging.getLogger("cfg")
yaml, data = YAML(), None
yaml.default_flow_style = False
config_path = get_config_path() or config.directories.user_configs / config.filenames.root_config
if config_path.exists():
data = yaml.load(config_path)
if not data:
log.warning("No config file was found or it has no data, yet")
# yaml.load() returns `None` if the input data is blank instead of a usable object
# force a usable object by making one and removing the only item within it
data = yaml.load("""__TEMP__: null""")
del data["__TEMP__"]
if list_:
yaml.dump(data, sys.stdout)
return
key_items = key.split(".")
parent_key = key_items[:-1]
trailing_key = key_items[-1]
is_write = value is not None
is_delete = unset
if is_write and is_delete:
raise click.ClickException("You cannot set a value and use --unset at the same time.")
if not is_write and not is_delete:
data = data.mlget(key_items, default=KeyError)
if data is KeyError:
raise click.ClickException(f"Key '{key}' does not exist in the config.")
yaml.dump(data, sys.stdout)
else:
try:
parent_data = data
if parent_key:
parent_data = data.mlget(parent_key, default=data)
if parent_data == data:
for key in parent_key:
if not hasattr(parent_data, key):
parent_data[key] = {}
parent_data = parent_data[key]
if is_write:
parent_data[trailing_key] = value
log.info(f"Set {key} to {repr(value)}")
elif is_delete:
del parent_data[trailing_key]
log.info(f"Unset {key}")
except KeyError:
raise click.ClickException(f"Key '{key}' does not exist in the config.")
config_path.parent.mkdir(parents=True, exist_ok=True)
yaml.dump(data, config_path)
reset/packages/envied/src/envied/commands/dl.bak
+2774
View File
File diff suppressed because it is too large Load Diff
reset/packages/envied/src/envied/commands/dl.py
+3453
View File
File diff suppressed because it is too large Load Diff
reset/packages/envied/src/envied/commands/env.py
+236
View File
@@ -0,0 +1,236 @@
import logging
import os
import shutil
import sys
from pathlib import Path
from typing import Optional
import click
from rich.padding import Padding
from rich.table import Table
from rich.tree import Tree
from envied.core import binaries
from envied.core.config import POSSIBLE_CONFIG_PATHS, config, config_path
from envied.core.console import console
from envied.core.constants import context_settings
from envied.core.services import Services
@click.group(short_help="Manage and configure the project environment.", context_settings=context_settings)
def env() -> None:
"""Manage and configure the project environment."""
@env.command()
def check() -> None:
"""Checks environment for the required dependencies."""
# Define all dependencies
all_deps = [
# Core Media Tools
{"name": "FFmpeg", "binary": binaries.FFMPEG, "required": True, "desc": "Media processing", "cat": "Core"},
{"name": "FFprobe", "binary": binaries.FFProbe, "required": True, "desc": "Media analysis", "cat": "Core"},
{"name": "MKVToolNix", "binary": binaries.MKVToolNix, "required": True, "desc": "MKV muxing", "cat": "Core"},
{
"name": "mkvpropedit",
"binary": binaries.Mkvpropedit,
"required": True,
"desc": "MKV metadata",
"cat": "Core",
},
{
"name": "shaka-packager",
"binary": binaries.ShakaPackager,
"required": True,
"desc": "DRM decryption",
"cat": "DRM",
},
{
"name": "mp4decrypt",
"binary": binaries.Mp4decrypt,
"required": False,
"desc": "DRM decryption",
"cat": "DRM",
},
{
"name": "ML-Worker",
"binary": binaries.ML_Worker,
"required": False,
"desc": "DRM licensing",
"cat": "DRM",
},
# HDR Processing
{"name": "dovi_tool", "binary": binaries.DoviTool, "required": False, "desc": "Dolby Vision", "cat": "HDR"},
{
"name": "HDR10Plus_tool",
"binary": binaries.HDR10PlusTool,
"required": False,
"desc": "HDR10+ metadata",
"cat": "HDR",
},
# Subtitle Tools
{
"name": "SubtitleEdit",
"binary": binaries.SubtitleEdit,
"required": False,
"desc": "Sub conversion",
"cat": "Subtitle",
},
{
"name": "CCExtractor",
"binary": binaries.CCExtractor,
"required": False,
"desc": "CC extraction",
"cat": "Subtitle",
},
# Media Players
{"name": "FFplay", "binary": binaries.FFPlay, "required": False, "desc": "Simple player", "cat": "Player"},
{"name": "MPV", "binary": binaries.MPV, "required": False, "desc": "Advanced player", "cat": "Player"},
# Network Tools
{
"name": "HolaProxy",
"binary": binaries.HolaProxy,
"required": False,
"desc": "Proxy service",
"cat": "Network",
},
{"name": "Caddy", "binary": binaries.Caddy, "required": False, "desc": "Web server", "cat": "Network"},
{"name": "Docker", "binary": binaries.Docker, "required": False, "desc": "Gluetun VPN", "cat": "Network"},
]
# Track overall status
all_required_installed = True
total_installed = 0
total_required = 0
missing_required = []
# Create a single table
table = Table(
title="Environment Dependencies", title_style="bold", show_header=True, header_style="bold", expand=False
)
table.add_column("Category", style="bold cyan", width=10)
table.add_column("Tool", width=16)
table.add_column("Status", justify="center", width=10)
table.add_column("Req", justify="center", width=4)
table.add_column("Purpose", style="bright_black", width=20)
last_cat = None
for dep in all_deps:
path = dep["binary"]
# Category column (only show when it changes)
category = dep["cat"] if dep["cat"] != last_cat else ""
last_cat = dep["cat"]
# Status
if path:
status = "[green]✓[/green]"
total_installed += 1
else:
status = "[red]✗[/red]"
if dep["required"]:
all_required_installed = False
missing_required.append(dep["name"])
if dep["required"]:
total_required += 1
# Required column (compact)
req = "[red]Y[/red]" if dep["required"] else "[bright_black]-[/bright_black]"
# Add row
table.add_row(category, dep["name"], status, req, dep["desc"])
console.print(Padding(table, (1, 2)))
# Compact summary
summary_parts = [f"[bold]Total:[/bold] {total_installed}/{len(all_deps)}"]
if all_required_installed:
summary_parts.append("[green]All required tools installed ✓[/green]")
else:
summary_parts.append(f"[red]Missing required: {', '.join(missing_required)}[/red]")
console.print(Padding(" ".join(summary_parts), (1, 2)))
@env.command()
def info() -> None:
"""Displays information about the current environment."""
log = logging.getLogger("env")
if config_path:
log.info(f"Config loaded from {config_path}")
else:
tree = Tree("No config file found, you can use any of the following locations:")
for i, path in enumerate(POSSIBLE_CONFIG_PATHS, start=1):
tree.add(f"[repr.number]{i}.[/] [text2]{path.resolve()}[/]")
console.print(Padding(tree, (0, 5)))
table = Table(title="Directories", title_style="bold", expand=True)
table.add_column("Name", no_wrap=True)
table.add_column("Path", no_wrap=False, overflow="fold")
path_vars = {
x: Path(os.getenv(x))
for x in ("TEMP", "APPDATA", "LOCALAPPDATA", "USERPROFILE")
if sys.platform == "win32" and os.getenv(x)
}
for name in sorted(dir(config.directories)):
if name.startswith("__") or name == "app_dirs":
continue
attr_value = getattr(config.directories, name)
# Handle both single Path objects and lists of Path objects
if isinstance(attr_value, list):
# For lists, show each path on a separate line
paths_str = "\n".join(str(path.resolve()) for path in attr_value)
table.add_row(name.title(), paths_str)
else:
# For single Path objects, use the original logic
path = attr_value.resolve()
for var, var_path in path_vars.items():
if path.is_relative_to(var_path):
path = rf"%{var}%\{path.relative_to(var_path)}"
break
table.add_row(name.title(), str(path))
console.print(Padding(table, (1, 5)))
@env.group(name="clear", short_help="Clear an environment directory.", context_settings=context_settings)
def clear() -> None:
"""Clear an environment directory."""
@clear.command()
@click.argument("service", type=str, required=False)
def cache(service: Optional[str]) -> None:
"""Clear the environment cache directory."""
log = logging.getLogger("env")
cache_dir = config.directories.cache
if service:
cache_dir = cache_dir / Services.get_tag(service)
log.info(f"Clearing cache directory: {cache_dir}")
files_count = len(list(cache_dir.glob("**/*")))
if not files_count:
log.info("No files to delete")
else:
log.info(f"Deleting {files_count} files...")
shutil.rmtree(cache_dir)
log.info("Cleared")
@clear.command()
def temp() -> None:
"""Clear the environment temp directory."""
log = logging.getLogger("env")
log.info(f"Clearing temp directory: {config.directories.temp}")
files_count = len(list(config.directories.temp.glob("**/*")))
if not files_count:
log.info("No files to delete")
else:
log.info(f"Deleting {files_count} files...")
shutil.rmtree(config.directories.temp)
log.info("Cleared")
reset/packages/envied/src/envied/commands/import.py
+54
View File
@@ -0,0 +1,54 @@
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
import click
from envied.commands.dl import dl
from envied.core.constants import context_settings
class ImportCommand:
@staticmethod
@click.command(
name="import",
short_help="Reconstruct a download (download, decrypt, mux) from an --export JSON file.",
context_settings={**context_settings, "ignore_unknown_options": True},
)
@click.argument("export_file", type=Path)
@click.argument("dl_args", nargs=-1, type=click.UNPROCESSED)
@click.pass_context
def cli(ctx: click.Context, export_file: Path, dl_args: tuple[str, ...]) -> None:
"""
Reconstruct an exported download without re-contacting the service.
Re-fetches the manifest, injects the stored keys, then downloads/decrypts/muxes as a
normal `dl` run. Any `dl` options after the file are forwarded verbatim:
envied.import export.json -r HDR10 --proxy US
"""
if not export_file.is_file():
raise click.ClickException(f"Export file not found: {export_file}")
try:
data: dict[str, Any] = json.loads(export_file.read_text(encoding="utf8"))
except json.JSONDecodeError as e:
raise click.ClickException(f"Export file is not valid JSON: {e}")
if data.get("version") != 2:
raise click.ClickException(
f"Unsupported export version {data.get('version')!r}. "
"Re-create the export with a current build of envied."
)
service_tag = data.get("service")
if not service_tag:
raise click.ClickException("Export file is missing the 'service' tag.")
args = [*dl_args, "--import", str(export_file), service_tag]
dl.cli.main(args=args, prog_name="envied.dl", standalone_mode=False)
globals()["import"] = ImportCommand
reset/packages/envied/src/envied/commands/kv.py
+335
View File
@@ -0,0 +1,335 @@
import logging
import re
from pathlib import Path
from typing import Optional
import click
from rich.padding import Padding
from rich.text import Text
from rich.tree import Tree
from envied.core.config import config
from envied.core.console import console
from envied.core.constants import context_settings
from envied.core.services import Services
from envied.core.vault import Vault
from envied.core.vaults import Vaults
def load_vaults(vault_names: list[str]) -> Vaults:
"""Load and validate vaults by name."""
vaults = Vaults()
for vault_name in vault_names:
vault_config = next((x for x in config.key_vaults if x["name"] == vault_name), None)
if not vault_config:
raise click.ClickException(f"Vault ({vault_name}) is not defined in the config.")
vault_type = vault_config["type"]
vault_args = vault_config.copy()
del vault_args["type"]
if not vaults.load(vault_type, **vault_args):
raise click.ClickException(f"Failed to load vault ({vault_name}).")
return vaults
def process_service_keys(from_vault: Vault, service: str, log: logging.Logger) -> dict[str, str]:
"""Get and validate keys from a vault for a specific service."""
content_keys = list(from_vault.get_keys(service))
bad_keys = {kid: key for kid, key in content_keys if not key or key.count("0") == len(key)}
for kid, key in bad_keys.items():
log.warning(f"Skipping NULL key: {kid}:{key}")
return {kid: key for kid, key in content_keys if kid not in bad_keys}
def copy_service_data(to_vault: Vault, from_vault: Vault, service: str, log: logging.Logger) -> int:
"""Copy data for a single service between vaults."""
content_keys = process_service_keys(from_vault, service, log)
total_count = len(content_keys)
if total_count == 0:
log.info(f"{service}: No keys found in {from_vault}")
return 0
try:
added = to_vault.add_keys(service, content_keys)
except PermissionError:
log.warning(f"{service}: No permission to create table in {to_vault}, skipped")
return 0
existed = total_count - added
if added > 0 and existed > 0:
log.info(f"{service}: {added} added, {existed} skipped ({total_count} total)")
elif added > 0:
log.info(f"{service}: {added} added ({total_count} total)")
else:
log.info(f"{service}: {existed} skipped (all existed)")
return added
@click.group(short_help="Manage and configure Key Vaults.", context_settings=context_settings)
def kv() -> None:
"""Manage and configure Key Vaults."""
@kv.command()
@click.argument("to_vault_name", type=str)
@click.argument("from_vault_names", nargs=-1, type=click.UNPROCESSED)
@click.option("-s", "--service", type=str, default=None, help="Only copy data to and from a specific service.")
@click.option(
"-l",
"--local-only",
is_flag=True,
default=False,
help="Only copy data for services installed locally (skip vault tables for services not present in the configured services path).",
)
def copy(
to_vault_name: str,
from_vault_names: list[str],
service: Optional[str] = None,
local_only: bool = False,
) -> None:
"""
Copy data from multiple Key Vaults into a single Key Vault.
Rows with matching KIDs are skipped unless there's no KEY set.
Existing data is not deleted or altered.
The `to_vault_name` argument is the key vault you wish to copy data to.
It should be the name of a Key Vault defined in the config.
The `from_vault_names` argument is the key vault(s) you wish to take
data from. You may supply multiple key vaults.
"""
if not from_vault_names:
raise click.ClickException("No Vaults were specified to copy data from.")
log = logging.getLogger("kv")
all_vault_names = [to_vault_name] + list(from_vault_names)
vaults = load_vaults(all_vault_names)
to_vault = vaults.vaults[0]
from_vaults = vaults.vaults[1:]
vault_names = ", ".join([v.name for v in from_vaults])
log.info(f"Copying data from {vault_names} → {to_vault.name}")
if service and local_only:
raise click.UsageError("--service and --local-only are mutually exclusive.")
if service:
service = Services.get_tag(service)
log.info(f"Filtering by service: {service}")
installed: Optional[set[str]] = None
if local_only:
installed = {t.upper() for t in Services.get_tags()}
log.info(f"Filtering by locally installed services ({len(installed)} found)")
total_added = 0
for from_vault in from_vaults:
services_to_copy = [service] if service else list(from_vault.get_services())
if installed is not None:
before = len(services_to_copy)
services_to_copy = [s for s in services_to_copy if s and s.upper() in installed]
skipped = before - len(services_to_copy)
if skipped:
log.info(f"{from_vault.name}: skipping {skipped} service(s) not installed locally")
for service_tag in services_to_copy:
added = copy_service_data(to_vault, from_vault, service_tag, log)
total_added += added
if total_added > 0:
log.info(f"Successfully added {total_added} new keys to {to_vault}")
else:
log.info("Copy completed - no new keys to add")
@kv.command()
@click.argument("vaults", nargs=-1, type=click.UNPROCESSED)
@click.option("-s", "--service", type=str, default=None, help="Only sync data to and from a specific service.")
@click.option(
"-l",
"--local-only",
is_flag=True,
default=False,
help="Only sync data for services installed locally (skip vault tables for services not present in the configured services path).",
)
@click.pass_context
def sync(
ctx: click.Context,
vaults: list[str],
service: Optional[str] = None,
local_only: bool = False,
) -> None:
"""
Ensure multiple Key Vaults copies of all keys as each other.
It's essentially just a bi-way copy between each vault.
To see the precise details of what it's doing between each
provided vault, see the documentation for the `copy` command.
"""
if not len(vaults) > 1:
raise click.ClickException("You must provide more than one Vault to sync.")
ctx.invoke(
copy,
to_vault_name=vaults[0],
from_vault_names=vaults[1:],
service=service,
local_only=local_only,
)
for i in range(1, len(vaults)):
ctx.invoke(
copy,
to_vault_name=vaults[i],
from_vault_names=[vaults[i - 1]],
service=service,
local_only=local_only,
)
@kv.command()
@click.argument("file", type=Path)
@click.argument("service", type=str)
@click.argument("vaults", nargs=-1, type=click.UNPROCESSED)
def add(file: Path, service: str, vaults: list[str]) -> None:
"""
Add new Content Keys to Key Vault(s) by service.
File should contain one key per line in the format KID:KEY (HEX:HEX).
Each line should have nothing else within it except for the KID:KEY.
Encoding is presumed to be UTF8.
"""
if not file.exists():
raise click.ClickException(f"File provided ({file}) does not exist.")
if not file.is_file():
raise click.ClickException(f"File provided ({file}) is not a file.")
if not service or not isinstance(service, str):
raise click.ClickException(f"Service provided ({service}) is invalid.")
if len(vaults) < 1:
raise click.ClickException("You must provide at least one Vault.")
log = logging.getLogger("kv")
service = Services.get_tag(service)
vaults_ = load_vaults(list(vaults))
data = file.read_text(encoding="utf8")
kid_keys: dict[str, str] = {}
for line in data.splitlines(keepends=False):
line = line.strip()
match = re.search(r"^(?P<kid>[0-9a-fA-F]{32}):(?P<key>[0-9a-fA-F]{32})$", line)
if not match:
continue
kid = match.group("kid").lower()
key = match.group("key").lower()
kid_keys[kid] = key
total_count = len(kid_keys)
for vault in vaults_:
log.info(f"Adding {total_count} Content Keys to {vault}")
added_count = vault.add_keys(service, kid_keys)
existed_count = total_count - added_count
log.info(f"{vault}: {added_count} newly added, {existed_count} already existed (skipped)")
log.info("Done!")
@kv.command()
@click.argument("kid", type=str)
@click.option("-s", "--service", type=str, default=None, help="Limit search to a specific service tag.")
@click.option(
"-v", "--vault", "vault_name", type=str, default=None, help="Limit search to a specific configured vault by name."
)
def search(kid: str, service: Optional[str], vault_name: Optional[str]) -> None:
"""
Search configured Key Vault(s) for a KID and report any matching KEY.
KID must be 32 hex characters (no dashes). If --service is omitted, every
service table in each vault is scanned. If --vault is omitted, every
vault in the config is searched.
"""
log = logging.getLogger("kv")
kid_norm = kid.replace("-", "").lower()
if not re.fullmatch(r"[0-9a-f]{32}", kid_norm):
raise click.ClickException(f"KID '{kid}' is not 32 hex characters.")
if vault_name:
vault_names = [vault_name]
else:
vault_names = [v["name"] for v in config.key_vaults]
if not vault_names:
raise click.ClickException("No Key Vaults are configured.")
vaults_ = load_vaults(vault_names)
service_tag = Services.get_tag(service) if service else None
hit: Optional[tuple[str, str, str]] = None
for vault in vaults_:
if service_tag:
services_to_check: list[str] = [service_tag]
else:
try:
services_to_check = list(vault.get_services())
except Exception as e:
log.debug(f"{vault}: get_services() failed ({e})")
services_to_check = []
if not services_to_check:
log.warning(f"{vault}: cannot search without a service (remote vault requires --service). Skipping.")
continue
for svc in services_to_check:
try:
key = vault.get_key(kid_norm, svc)
except Exception as e:
log.debug(f"{vault} [{svc}]: lookup error ({e})")
continue
if key and key.count("0") != len(key):
hit = (vault.name, svc, key)
break
if hit:
break
if hit:
vname, svc, key = hit
tree = Tree(Text.assemble((svc, "cyan"), (f"({vname})", "text"), overflow="fold"))
tree.add(f"[text2]{kid_norm}:{key}")
console.print(Padding(tree, (1, 5)))
else:
log.info(f"KID {kid_norm} not found in {len(vaults_)} vault(s).")
@kv.command()
@click.argument("vaults", nargs=-1, type=click.UNPROCESSED)
def prepare(vaults: list[str]) -> None:
"""Create Service Tables on Vaults if not yet created."""
log = logging.getLogger("kv")
vaults_ = load_vaults(vaults)
for vault in vaults_:
if hasattr(vault, "has_table") and hasattr(vault, "create_table"):
for service_tag in Services.get_tags():
if vault.has_table(service_tag):
log.info(f"{vault} already has a {service_tag} Table")
else:
try:
vault.create_table(service_tag, commit=True)
log.info(f"{vault}: Created {service_tag} Table")
except PermissionError:
log.error(f"{vault} user has no create table permission, skipping...")
continue
else:
log.info(f"{vault} does not use tables, skipping...")
log.info("Done!")
reset/packages/envied/src/envied/commands/prd.py
+271
View File
@@ -0,0 +1,271 @@
import logging
from pathlib import Path
from typing import Optional
import click
import requests
from Crypto.Random import get_random_bytes
from pyplayready import InvalidCertificateChain, OutdatedDevice
from pyplayready.cdm import Cdm
from pyplayready.crypto.ecc_key import ECCKey
from pyplayready.device import Device
from pyplayready.system.bcert import Certificate, CertificateChain
from pyplayready.system.pssh import PSSH
from envied.core.config import config
from envied.core.constants import context_settings
@click.group(
short_help="Manage creation of PRD (Playready Device) files.",
context_settings=context_settings,
)
def prd() -> None:
"""Manage creation of PRD (Playready Device) files."""
@prd.command()
@click.argument("paths", type=Path, nargs=-1)
@click.option(
"-e",
"--encryption_key",
type=Path,
required=False,
help="Optional Device ECC private encryption key",
)
@click.option(
"-s",
"--signing_key",
type=Path,
required=False,
help="Optional Device ECC private signing key",
)
@click.option("-o", "--output", type=Path, default=None, help="Output Directory")
@click.pass_context
def new(
ctx: click.Context,
paths: tuple[Path, ...],
encryption_key: Optional[Path],
signing_key: Optional[Path],
output: Optional[Path],
) -> None:
"""Create a new .PRD PlayReady Device file.
Accepts either paths to a group key and certificate or a single directory
containing ``zgpriv.dat`` and ``bgroupcert.dat``.
"""
if len(paths) == 1 and paths[0].is_dir():
device_dir = paths[0]
group_key = device_dir / "zgpriv.dat"
group_certificate = device_dir / "bgroupcert.dat"
if not group_key.is_file() or not group_certificate.is_file():
raise click.UsageError("Folder must contain zgpriv.dat and bgroupcert.dat", ctx)
elif len(paths) == 2:
group_key, group_certificate = paths
if not group_key.is_file():
raise click.UsageError("group_key: Not a path to a file, or it doesn't exist.", ctx)
if not group_certificate.is_file():
raise click.UsageError("group_certificate: Not a path to a file, or it doesn't exist.", ctx)
device_dir = None
else:
raise click.UsageError(
"Provide either a folder path or paths to group_key and group_certificate",
ctx,
)
if encryption_key and not encryption_key.is_file():
raise click.UsageError("encryption_key: Not a path to a file, or it doesn't exist.", ctx)
if signing_key and not signing_key.is_file():
raise click.UsageError("signing_key: Not a path to a file, or it doesn't exist.", ctx)
log = logging.getLogger("prd")
encryption_key_obj = ECCKey.load(encryption_key) if encryption_key else ECCKey.generate()
signing_key_obj = ECCKey.load(signing_key) if signing_key else ECCKey.generate()
group_key_obj = ECCKey.load(group_key)
certificate_chain = CertificateChain.load(group_certificate)
if certificate_chain.get(0).get_issuer_key() != group_key_obj.public_bytes():
raise InvalidCertificateChain("Group key does not match this certificate")
new_certificate = Certificate.new_leaf_cert(
cert_id=get_random_bytes(16),
security_level=certificate_chain.get_security_level(),
client_id=get_random_bytes(16),
signing_key=signing_key_obj,
encryption_key=encryption_key_obj,
group_key=group_key_obj,
parent=certificate_chain,
)
certificate_chain.prepend(new_certificate)
certificate_chain.verify()
device = Device(
group_key=group_key_obj.dumps(),
encryption_key=encryption_key_obj.dumps(),
signing_key=signing_key_obj.dumps(),
group_certificate=certificate_chain.dumps(),
)
if output and output.suffix:
if output.suffix.lower() != ".prd":
log.warning(
"Saving PRD with the file extension '%s' but '.prd' is recommended.",
output.suffix,
)
out_path = output
else:
out_dir = output or (device_dir or config.directories.prds)
out_path = out_dir / f"{device.get_name()}.prd"
if out_path.exists():
log.error("A file already exists at the path '%s', cannot overwrite.", out_path)
return
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.write_bytes(device.dumps())
log.info("Created Playready Device (.prd) file, %s", out_path.name)
log.info(" + Security Level: %s", device.security_level)
log.info(" + Group Key: %s bytes", len(device.group_key.dumps()))
log.info(" + Encryption Key: %s bytes", len(device.encryption_key.dumps()))
log.info(" + Signing Key: %s bytes", len(device.signing_key.dumps()))
log.info(" + Group Certificate: %s bytes", len(device.group_certificate.dumps()))
log.info(" + Saved to: %s", out_path.absolute())
@prd.command(name="reprovision")
@click.argument("prd_path", type=Path)
@click.option(
"-e",
"--encryption_key",
type=Path,
required=False,
help="Optional Device ECC private encryption key",
)
@click.option(
"-s",
"--signing_key",
type=Path,
required=False,
help="Optional Device ECC private signing key",
)
@click.option("-o", "--output", type=Path, default=None, help="Output Path or Directory")
@click.pass_context
def reprovision_device(
ctx: click.Context,
prd_path: Path,
encryption_key: Optional[Path],
signing_key: Optional[Path],
output: Optional[Path] = None,
) -> None:
"""Reprovision a Playready Device (.prd) file."""
if not prd_path.is_file():
raise click.UsageError("prd_path: Not a path to a file, or it doesn't exist.", ctx)
log = logging.getLogger("prd")
log.info("Reprovisioning Playready Device (.prd) file, %s", prd_path.name)
device = Device.load(prd_path)
if device.group_key is None:
raise OutdatedDevice(
"Device does not support reprovisioning, re-create it or use a Device with a version of 3 or higher"
)
device.group_certificate.remove(0)
encryption_key_obj = ECCKey.load(encryption_key) if encryption_key else ECCKey.generate()
signing_key_obj = ECCKey.load(signing_key) if signing_key else ECCKey.generate()
device.encryption_key = encryption_key_obj
device.signing_key = signing_key_obj
new_certificate = Certificate.new_leaf_cert(
cert_id=get_random_bytes(16),
security_level=device.group_certificate.get_security_level(),
client_id=get_random_bytes(16),
signing_key=signing_key_obj,
encryption_key=encryption_key_obj,
group_key=device.group_key,
parent=device.group_certificate,
)
device.group_certificate.prepend(new_certificate)
if output and output.suffix:
if output.suffix.lower() != ".prd":
log.warning(
"Saving PRD with the file extension '%s' but '.prd' is recommended.",
output.suffix,
)
out_path = output
else:
out_path = prd_path
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.write_bytes(device.dumps())
log.info("Reprovisioned Playready Device (.prd) file, %s", out_path.name)
@prd.command()
@click.argument("device", type=Path)
@click.option(
"-c",
"--ckt",
type=click.Choice(["aesctr", "aescbc"], case_sensitive=False),
default="aesctr",
help="Content Key Encryption Type",
)
@click.option(
"-sl",
"--security-level",
type=click.Choice(["150", "2000", "3000"], case_sensitive=False),
default="2000",
help="Minimum Security Level",
)
@click.pass_context
def test(
ctx: click.Context,
device: Path,
ckt: str,
security_level: str,
) -> None:
"""Test a Playready Device on the Microsoft demo server."""
if not device.is_file():
raise click.UsageError("device: Not a path to a file, or it doesn't exist.", ctx)
log = logging.getLogger("prd")
prd_device = Device.load(device)
log.info("Loaded Device: %s", prd_device.get_name())
cdm = Cdm.from_device(prd_device)
log.info("Loaded CDM")
session_id = cdm.open()
log.info("Opened Session")
pssh_b64 = "AAADfHBzc2gAAAAAmgTweZhAQoarkuZb4IhflQAAA1xcAwAAAQABAFIDPABXAFIATQBIAEUAQQBEAEUAUgAgAHgAbQBsAG4AcwA9ACIAaAB0AHQAcAA6AC8ALwBzAGMAaABlAG0AYQBzAC4AbQBpAGMAcgBvAHMAbwBmAHQALgBjAG8AbQAvAEQAUgBNAC8AMgAwADAANwAvADAAMwAvAFAAbABhAHkAUgBlAGEAZAB5AEgAZQBhAGQAZQByACIAIAB2AGUAcgBzAGkAbwBuAD0AIgA0AC4AMAAuADAALgAwACIAPgA8AEQAQQBUAEEAPgA8AFAAUgBPAFQARQBDAFQASQBOAEYATwA+ADwASwBFAFkATABFAE4APgAxADYAPAAvAEsARQBZAEwARQBOAD4APABBAEwARwBJAEQAPgBBAEUAUwBDAFQAUgA8AC8AQQBMAEcASQBEAD4APAAvAFAAUgBPAFQARQBDAFQASQBOAEYATwA+ADwASwBJAEQAPgA0AFIAcABsAGIAKwBUAGIATgBFAFMAOAB0AEcAawBOAEYAVwBUAEUASABBAD0APQA8AC8ASwBJAEQAPgA8AEMASABFAEMASwBTAFUATQA+AEsATABqADMAUQB6AFEAUAAvAE4AQQA9ADwALwBDAEgARQBDAEsAUwBVAE0APgA8AEwAQQBfAFUAUgBMAD4AaAB0AHQAcABzADoALwAvAHAAcgBvAGYAZgBpAGMAaQBhAGwAcwBpAHQAZQAuAGsAZQB5AGQAZQBsAGkAdgBlAHIAeQAuAG0AZQBkAGkAYQBzAGUAcgB2AGkAYwBlAHMALgB3AGkAbgBkAG8AdwBzAC4AbgBlAHQALwBQAGwAYQB5AFIAZQBhAGQAeQAvADwALwBMAEEAXwBVAFIATAA+ADwAQwBVAFMAVABPAE0AQQBUAFQAUgBJAEIAVQBUAEUAUwA+ADwASQBJAFMAXwBEAFIATQBfAFYARQBSAFMASQBPAE4APgA4AC4AMQAuADIAMwAwADQALgAzADEAPAAvAEkASQBTAF8ARABSAE0AXwBWAEUAUgBTAEkATwBOAD4APAAvAEMAVQBTAFQATwBNAEEAVABUAFIASQBCAFUAVABFAFMAPgA8AC8ARABBAFQAQQA+ADwALwBXAFIATQBIAEUAQQBEAEUAUgA+AA=="
pssh = PSSH(pssh_b64)
challenge = cdm.get_license_challenge(session_id, pssh.wrm_headers[0])
log.info("Created License Request")
license_server = f"https://test.playready.microsoft.com/service/rightsmanager.asmx?cfg=(persist:false,sl:{security_level},ckt:{ckt})"
response = requests.post(
url=license_server,
headers={"Content-Type": "text/xml; charset=UTF-8"},
data=challenge,
)
cdm.parse_license(session_id, response.text)
log.info("License Parsed Successfully")
for key in cdm.get_keys(session_id):
log.info(f"{key.key_id.hex}:{key.key.hex()}")
cdm.close(session_id)
log.info("Closed Session")
reset/packages/envied/src/envied/commands/search.py
+159
View File
@@ -0,0 +1,159 @@
from __future__ import annotations
import logging
import re
import sys
from typing import Any, Optional
import click
import yaml
from rich.padding import Padding
from rich.rule import Rule
from rich.tree import Tree
from envied.commands.dl import dl
from envied.core import binaries
from envied.core.config import config
from envied.core.console import console
from envied.core.constants import context_settings
from envied.core.proxies import Basic, Gluetun, Hola, NordVPN, SurfsharkVPN, WindscribeVPN
from envied.core.service import Service
from envied.core.services import Services
from envied.core.utils.click_types import ContextData
from envied.core.utils.collections import merge_dict
@click.command(
short_help="Search for titles from a Service.",
cls=Services,
context_settings=dict(**context_settings, token_normalize_func=Services.get_tag),
)
@click.option(
"-p", "--profile", type=str, default=None, help="Profile to use for Credentials and Cookies (if available)."
)
@click.option(
"--proxy",
type=str,
default=None,
help="Proxy URI to use. If a 2-letter country is provided, it will try get a proxy from the config.",
)
@click.option("--no-proxy", is_flag=True, default=False, help="Force disable all proxy use.")
@click.pass_context
def search(ctx: click.Context, no_proxy: bool, profile: Optional[str] = None, proxy: Optional[str] = None):
if not ctx.invoked_subcommand:
raise ValueError("A subcommand to invoke was not specified, the main code cannot continue.")
log = logging.getLogger("search")
service = Services.get_tag(ctx.invoked_subcommand)
profile = profile
if profile:
log.info(f"Using profile: '{profile}'")
with console.status("Loading Service Config...", spinner="dots"):
service_config_path = Services.get_path(service) / config.filenames.config
if service_config_path.exists():
service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8"))
log.info("Service Config loaded")
else:
service_config = {}
merge_dict(config.services.get(service), service_config)
proxy_providers = []
if no_proxy:
ctx.params["proxy"] = None
else:
with console.status("Loading Proxy Providers...", spinner="dots"):
if config.proxy_providers.get("basic"):
proxy_providers.append(Basic(**config.proxy_providers["basic"]))
if config.proxy_providers.get("nordvpn"):
proxy_providers.append(NordVPN(**config.proxy_providers["nordvpn"]))
if config.proxy_providers.get("surfsharkvpn"):
proxy_providers.append(SurfsharkVPN(**config.proxy_providers["surfsharkvpn"]))
if config.proxy_providers.get("windscribevpn"):
proxy_providers.append(WindscribeVPN(**config.proxy_providers["windscribevpn"]))
if config.proxy_providers.get("gluetun"):
proxy_providers.append(Gluetun(**config.proxy_providers["gluetun"]))
if binaries.HolaProxy:
proxy_providers.append(Hola())
for proxy_provider in proxy_providers:
log.info(f"Loaded {proxy_provider.__class__.__name__}: {proxy_provider}")
if proxy:
requested_provider = None
if re.match(r"^[a-z]+:.+$", proxy, re.IGNORECASE):
# requesting proxy from a specific proxy provider
requested_provider, proxy = proxy.split(":", maxsplit=1)
# Match simple region codes (us, ca, uk1) or provider:region format (nordvpn:ca, windscribe:us)
if re.match(r"^[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE) or re.match(
r"^[a-z]+:[a-z]{2}(?:\d+)?$", proxy, re.IGNORECASE
):
proxy = proxy.lower()
with console.status(f"Getting a Proxy to {proxy}...", spinner="dots"):
if requested_provider:
proxy_provider = next(
(x for x in proxy_providers if x.__class__.__name__.lower() == requested_provider), None
)
if not proxy_provider:
log.error(f"The proxy provider '{requested_provider}' was not recognised.")
sys.exit(1)
proxy_uri = proxy_provider.get_proxy(proxy)
if not proxy_uri:
log.error(f"The proxy provider {requested_provider} had no proxy for {proxy}")
sys.exit(1)
proxy = ctx.params["proxy"] = proxy_uri
log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}")
else:
for proxy_provider in proxy_providers:
proxy_uri = proxy_provider.get_proxy(proxy)
if proxy_uri:
proxy = ctx.params["proxy"] = proxy_uri
log.info(f"Using {proxy_provider.__class__.__name__} Proxy: {proxy}")
break
else:
log.info(f"Using explicit Proxy: {proxy}")
ctx.obj = ContextData(config=service_config, cdm=None, proxy_providers=proxy_providers, profile=profile)
@search.result_callback()
def result(service: Service, profile: Optional[str] = None, **_: Any) -> None:
log = logging.getLogger("search")
service_tag = service.__class__.__name__
with console.status("Authenticating with Service...", spinner="dots"):
cookies = dl.get_cookie_jar(service_tag, profile)
credential = dl.get_credentials(service_tag, profile)
service.authenticate(cookies, credential)
if cookies or credential:
log.info("Authenticated with Service")
search_results = Tree("Search Results", hide_root=True)
with console.status("Searching...", spinner="dots"):
for result in service.search():
result_text = f"[bold text]{result.title}[/]"
if result.url:
result_text = f"[link={result.url}]{result_text}[/link]"
if result.label:
result_text += f" [pink]{result.label}[/]"
if result.description:
result_text += f"\n[text2]{result.description}[/]"
result_text += f"\n[bright_black]id: {result.id}[/]"
search_results.add(result_text + "\n")
# update cookie
cookie_file = dl.get_cookie_path(service_tag, profile)
if cookie_file:
dl.save_cookies(cookie_file, service.session.cookies)
console.print(Padding(Rule(f"[rule.text]{len(search_results.children)} Search Results"), (1, 2)))
if search_results.children:
console.print(Padding(search_results, (0, 5)))
else:
console.print(
Padding("[bold text]No matches[/]\n[bright_black]Please check spelling and search again....[/]", (0, 5))
)
reset/packages/envied/src/envied/commands/serve.py
+327
View File
@@ -0,0 +1,327 @@
import logging
import subprocess
import click
from aiohttp import web
from envied.core import binaries
from envied.core.api import cors_middleware, setup_routes, setup_swagger
from envied.core.api.compression import compression_middleware
from envied.core.config import config
from envied.core.constants import context_settings
@click.command(
short_help="Serve your Local Widevine/PlayReady Devices and REST API for Remote Access.",
context_settings=context_settings,
)
@click.option("-h", "--host", type=str, default="127.0.0.1", help="Host to serve from.")
@click.option("-p", "--port", type=int, default=8786, help="Port to serve from.")
@click.option("--caddy", is_flag=True, default=False, help="Also serve with Caddy.")
@click.option(
"--api-only", is_flag=True, default=False, help="Serve only the REST API, not pywidevine/pyplayready CDM."
)
@click.option("--no-widevine", is_flag=True, default=False, help="Disable Widevine CDM endpoints.")
@click.option("--no-playready", is_flag=True, default=False, help="Disable PlayReady CDM endpoints.")
@click.option("--no-key", is_flag=True, default=False, help="Disable API key authentication (allows all requests).")
@click.option(
"--debug-api",
is_flag=True,
default=False,
help="Include technical debug information (tracebacks, stderr) in API error responses.",
)
@click.option(
"--debug",
is_flag=True,
default=False,
help="Enable debug logging for API operations.",
)
@click.option(
"--remote-only",
is_flag=True,
default=False,
help="Only expose remote service session endpoints (health, services, search, session).",
)
def serve(
host: str,
port: int,
caddy: bool,
api_only: bool,
no_widevine: bool,
no_playready: bool,
no_key: bool,
debug_api: bool,
debug: bool,
remote_only: bool,
) -> None:
"""
Serve your Local Widevine and PlayReady Devices and REST API for Remote Access.
\b
CDM ENDPOINTS:
- Widevine: /{device}/open, /{device}/close/{session_id}, etc.
- PlayReady: /playready/{device}/open, /playready/{device}/close/{session_id}, etc.
\b
You may serve with Caddy at the same time with --caddy. You can use Caddy
as a reverse-proxy to serve with HTTPS. The config used will be the Caddyfile
next to the envied.config.
\b
DEVICE CONFIGURATION:
WVD files are auto-loaded from the WVDs directory, PRD files from the PRDs directory.
Configure user access in envied.yaml:
\b
serve:
api_secret: "your-api-secret"
users:
your-secret-key:
devices: ["device_name"] # Widevine devices
playready_devices: ["device_name"] # PlayReady devices
username: user
"""
from pyplayready.remote import serve as pyplayready_serve
from pywidevine import serve as pywidevine_serve
log = logging.getLogger("serve")
if debug:
logging.basicConfig(level=logging.DEBUG, format="%(name)s - %(levelname)s - %(message)s")
log.info("Debug logging enabled for API operations")
else:
logging.getLogger("api").setLevel(logging.WARNING)
logging.getLogger("api.remote").setLevel(logging.WARNING)
if not no_key:
api_secret = config.serve.get("api_secret")
if not api_secret:
raise click.ClickException(
"API secret key is not configured. Please add 'api_secret' to the 'serve' section in your config."
)
else:
api_secret = None
log.warning("Running with --no-key: Authentication is DISABLED for all API endpoints!")
if debug_api:
log.warning("Running with --debug-api: Error responses will include technical debug information!")
if api_only and (no_widevine or no_playready):
raise click.ClickException("Cannot use --api-only with --no-widevine or --no-playready.")
if caddy:
if not binaries.Caddy:
raise click.ClickException('Caddy executable "caddy" not found but is required for --caddy.')
caddy_p = subprocess.Popen(
[binaries.Caddy, "run", "--config", str(config.directories.user_configs / "Caddyfile")]
)
else:
caddy_p = None
try:
if not config.serve.get("devices"):
config.serve["devices"] = []
config.serve["devices"].extend(list(config.directories.wvds.glob("*.wvd")))
if not config.serve.get("playready_devices"):
config.serve["playready_devices"] = []
config.serve["playready_devices"].extend(list(config.directories.prds.glob("*.prd")))
@web.middleware
async def api_key_authentication(request: web.Request, handler) -> web.Response:
"""Authenticate API requests using X-Secret-Key header."""
if request.path == "/api/health":
return await handler(request)
secret_key = request.headers.get("X-Secret-Key")
if not secret_key:
return web.json_response({"status": 401, "message": "Secret Key is Empty."}, status=401)
if secret_key not in request.app["config"]["users"]:
return web.json_response({"status": 401, "message": "Secret Key is Invalid."}, status=401)
return await handler(request)
remote_only = remote_only or config.serve.get("remote_only", False)
if remote_only:
api_only = True
global_services = config.serve.get("services")
if global_services:
log.info(f"Global service allowlist: {', '.join(global_services)}")
users = config.serve.get("users", {})
for user_key, user_cfg in users.items() if isinstance(users, dict) else []:
user_services = user_cfg.get("services") if isinstance(user_cfg, dict) else None
if user_services:
username = user_cfg.get("username", user_key[:8] + "...")
log.info(f"User '{username}' restricted to services: {', '.join(user_services)}")
if api_only:
log.info("Starting REST API server (pywidevine/pyplayready CDM disabled)")
if no_key:
app = web.Application(middlewares=[cors_middleware, compression_middleware])
app["config"] = {"users": {}}
else:
app = web.Application(middlewares=[cors_middleware, api_key_authentication, compression_middleware])
app["config"] = {"users": {api_secret: {"devices": [], "username": "api_user"}}}
app["debug_api"] = debug_api
# Start session cleanup loop for remote-dl sessions
from envied.core.api.session_store import get_session_store
session_store = get_session_store()
async def start_session_cleanup(_app: web.Application) -> None:
await session_store.start_cleanup_loop()
async def stop_session_cleanup(_app: web.Application) -> None:
await session_store.cancel_all_bridges()
await session_store.stop_cleanup_loop()
app.on_startup.append(start_session_cleanup)
app.on_cleanup.append(stop_session_cleanup)
setup_routes(app, remote_only=remote_only)
if not remote_only:
setup_swagger(app)
log.info(f"REST API endpoints available at http://{host}:{port}/api/")
log.info(f"Swagger UI available at http://{host}:{port}/api/docs/")
else:
log.info(f"Remote service endpoints available at http://{host}:{port}/api/session/")
log.info("(Press CTRL+C to quit)")
web.run_app(app, host=host, port=port, print=None)
else:
serve_widevine = not no_widevine
serve_playready = not no_playready
serve_config = dict(config.serve)
wvd_devices = serve_config.get("devices", []) if serve_widevine else []
prd_devices = serve_config.get("playready_devices", []) if serve_playready else []
cdm_parts = []
if serve_widevine:
cdm_parts.append("pywidevine CDM")
if serve_playready:
cdm_parts.append("pyplayready CDM")
log.info(f"Starting integrated server ({' + '.join(cdm_parts)} + REST API)")
wvd_device_names = [d.stem if hasattr(d, "stem") else str(d) for d in wvd_devices]
prd_device_names = [d.stem if hasattr(d, "stem") else str(d) for d in prd_devices]
if not serve_config.get("users") or not isinstance(serve_config["users"], dict):
serve_config["users"] = {}
if not no_key and api_secret not in serve_config["users"]:
serve_config["users"][api_secret] = {
"devices": wvd_device_names,
"playready_devices": prd_device_names,
"username": "api_user",
}
for user_key, user_config in serve_config["users"].items():
if "playready_devices" not in user_config:
# Require explicit PlayReady device access per user (default: no access).
user_config["playready_devices"] = []
log.warning(
f'User "{user_key}" has no "playready_devices" configured; PlayReady access disabled for this user. '
f"Available PlayReady devices: {prd_device_names}"
)
def create_serve_authentication(serve_playready_flag: bool):
@web.middleware
async def serve_authentication(request: web.Request, handler) -> web.Response:
if serve_playready_flag and request.path in ("/playready", "/playready/"):
response = await handler(request)
else:
response = await pywidevine_serve.authentication(request, handler)
if serve_playready_flag and request.path.startswith("/playready"):
from pyplayready import __version__ as pyplayready_version
response.headers["Server"] = (
f"https://git.gay/ready-dl/pyplayready serve v{pyplayready_version}"
)
return response
return serve_authentication
if no_key:
app = web.Application(middlewares=[cors_middleware, compression_middleware])
else:
serve_auth = create_serve_authentication(serve_playready and bool(prd_devices))
app = web.Application(middlewares=[cors_middleware, serve_auth, compression_middleware])
app["config"] = serve_config
app["debug_api"] = debug_api
# Start session cleanup loop for remote-dl sessions
from envied.core.api.session_store import get_session_store
session_store = get_session_store()
async def start_session_cleanup(_app: web.Application) -> None:
await session_store.start_cleanup_loop()
async def stop_session_cleanup(_app: web.Application) -> None:
await session_store.cancel_all_bridges()
await session_store.stop_cleanup_loop()
app.on_startup.append(start_session_cleanup)
app.on_cleanup.append(stop_session_cleanup)
if serve_widevine:
app.on_startup.append(pywidevine_serve._startup)
app.on_cleanup.append(pywidevine_serve._cleanup)
app.add_routes(pywidevine_serve.routes)
if serve_playready and prd_devices:
if no_key:
playready_app = web.Application()
else:
playready_app = web.Application(middlewares=[pyplayready_serve.authentication])
# PlayReady subapp config maps playready_devices to "devices" for pyplayready compatibility
playready_config = {
"devices": prd_devices,
"users": {
user_key: {
"devices": user_cfg.get("playready_devices", []),
"username": user_cfg.get("username", "user"),
}
for user_key, user_cfg in serve_config["users"].items()
}
if not no_key
else {},
}
playready_app["config"] = playready_config
playready_app.on_startup.append(pyplayready_serve._startup)
playready_app.on_cleanup.append(pyplayready_serve._cleanup)
playready_app.add_routes(pyplayready_serve.routes)
async def playready_ping(_: web.Request) -> web.Response:
from pyplayready import __version__ as pyplayready_version
response = web.json_response({"message": "OK"})
response.headers["Server"] = f"https://git.gay/ready-dl/pyplayready serve v{pyplayready_version}"
return response
app.router.add_route("*", "/playready", playready_ping)
app.add_subapp("/playready", playready_app)
log.info(f"PlayReady CDM endpoints available at http://{host}:{port}/playready/")
elif serve_playready:
log.info("No PlayReady devices found, skipping PlayReady CDM endpoints")
setup_routes(app, remote_only=remote_only)
if serve_widevine:
log.info(f"Widevine CDM endpoints available at http://{host}:{port}/{{device}}/open")
if remote_only:
log.info(f"Remote service endpoints available at http://{host}:{port}/api/session/")
else:
setup_swagger(app)
log.info(f"REST API endpoints available at http://{host}:{port}/api/")
log.info(f"Swagger UI available at http://{host}:{port}/api/docs/")
log.info("(Press CTRL+C to quit)")
web.run_app(app, host=host, port=port, print=None)
finally:
if caddy_p:
caddy_p.kill()
reset/packages/envied/src/envied/commands/util.py
+275
View File
@@ -0,0 +1,275 @@
import re
import subprocess
from pathlib import Path
import click
from pymediainfo import MediaInfo
from envied.core import binaries
from envied.core.constants import context_settings
def _natural_sort_key(path: Path) -> list:
"""Sort key for natural sorting (S01E01 before S01E10)."""
return [int(part) if part.isdigit() else part.lower() for part in re.split(r"(\d+)", path.name)]
@click.group(short_help="Various helper scripts and programs.", context_settings=context_settings)
def util() -> None:
"""Various helper scripts and programs."""
@util.command()
@click.argument("path", type=Path)
@click.argument("aspect", type=str)
@click.option(
"--letter/--pillar",
default=True,
help="Specify which direction to crop. Top and Bottom would be --letter, Sides would be --pillar.",
)
@click.option("-o", "--offset", type=int, default=0, help="Fine tune the computed crop area if not perfectly centered.")
@click.option(
"-p",
"--preview",
is_flag=True,
default=False,
help="Instantly preview the newly-set aspect crop in MPV (or ffplay if mpv is unavailable).",
)
def crop(path: Path, aspect: str, letter: bool, offset: int, preview: bool) -> None:
"""
Losslessly crop H.264 and H.265 video files at the bit-stream level.
You may provide a path to a file, or a folder of mkv and/or mp4 files.
Note: If you notice that the values you put in are not quite working, try
tune -o/--offset. This may be necessary on videos with sub-sampled chroma.
Do note that you may not get an ideal lossless cropping result on some
cases, again due to sub-sampled chroma.
It's recommended that you try -o about 10 or so pixels and lower it until
you get as close in as possible. Do make sure it's not over-cropping either
as it may go from being 2px away from a perfect crop, to 20px over-cropping
again due to sub-sampled chroma.
"""
if not binaries.FFMPEG:
raise click.ClickException('FFmpeg executable "ffmpeg" not found but is required.')
if path.is_dir():
paths = sorted(list(path.glob("*.mkv")) + list(path.glob("*.mp4")), key=_natural_sort_key)
else:
paths = [path]
for video_path in paths:
try:
video_track = next(iter(MediaInfo.parse(video_path).video_tracks or []))
except StopIteration:
raise click.ClickException("There's no video tracks in the provided file.")
crop_filter = {"HEVC": "hevc_metadata", "AVC": "h264_metadata"}.get(video_track.commercial_name)
if not crop_filter:
raise click.ClickException(f"{video_track.commercial_name} Codec not supported.")
aspect_w, aspect_h = list(map(float, aspect.split(":")))
if letter:
crop_value = (video_track.height - (video_track.width / (aspect_w * aspect_h))) / 2
left, top, right, bottom = map(int, [0, crop_value + offset, 0, crop_value - offset])
else:
crop_value = (video_track.width - (video_track.height * (aspect_w / aspect_h))) / 2
left, top, right, bottom = map(int, [crop_value + offset, 0, crop_value - offset, 0])
crop_filter += f"=crop_left={left}:crop_top={top}:crop_right={right}:crop_bottom={bottom}"
if min(left, top, right, bottom) < 0:
raise click.ClickException("Cannot crop less than 0, are you cropping in the right direction?")
if preview:
out_path = ["-f", "mpegts", "-"] # pipe
else:
out_path = [
str(
video_path.with_name(
".".join(
filter(
bool,
[
video_path.stem,
video_track.language,
"crop",
str(offset or ""),
{
# ffmpeg's MKV muxer does not yet support HDR
"HEVC": "h265",
"AVC": "h264",
}.get(video_track.commercial_name, ".mp4"),
],
)
)
)
)
]
ffmpeg_call = subprocess.Popen(
[binaries.FFMPEG, "-y", "-i", str(video_path), "-map", "0:v:0", "-c", "copy", "-bsf:v", crop_filter]
+ out_path,
stdout=subprocess.PIPE,
)
try:
if preview:
previewer = binaries.MPV or binaries.FFPlay
if not previewer:
raise click.ClickException("MPV/FFplay executables weren't found but are required for previewing.")
subprocess.Popen((previewer, "-"), stdin=ffmpeg_call.stdout)
finally:
if ffmpeg_call.stdout:
ffmpeg_call.stdout.close()
ffmpeg_call.wait()
@util.command(name="range")
@click.argument("path", type=Path)
@click.option("--full/--limited", is_flag=True, help="Full: 0..255, Limited: 16..235 (16..240 YUV luma)")
@click.option(
"-p",
"--preview",
is_flag=True,
default=False,
help="Instantly preview the newly-set video range in MPV (or ffplay if mpv is unavailable).",
)
def range_(path: Path, full: bool, preview: bool) -> None:
"""
Losslessly set the Video Range flag to full or limited at the bit-stream level.
You may provide a path to a file, or a folder of mkv and/or mp4 files.
If you ever notice blacks not being quite black, and whites not being quite white,
then you're video may have the range set to the wrong value. Flip its range to the
opposite value and see if that fixes it.
"""
if not binaries.FFMPEG:
raise click.ClickException('FFmpeg executable "ffmpeg" not found but is required.')
if path.is_dir():
paths = sorted(list(path.glob("*.mkv")) + list(path.glob("*.mp4")), key=_natural_sort_key)
else:
paths = [path]
for video_path in paths:
try:
video_track = next(iter(MediaInfo.parse(video_path).video_tracks or []))
except StopIteration:
raise click.ClickException("There's no video tracks in the provided file.")
metadata_key = {"HEVC": "hevc_metadata", "AVC": "h264_metadata"}.get(video_track.commercial_name)
if not metadata_key:
raise click.ClickException(f"{video_track.commercial_name} Codec not supported.")
if preview:
out_path = ["-f", "mpegts", "-"] # pipe
else:
out_path = [
str(
video_path.with_name(
".".join(
filter(
bool,
[
video_path.stem,
video_track.language,
"range",
["limited", "full"][full],
{
# ffmpeg's MKV muxer does not yet support HDR
"HEVC": "h265",
"AVC": "h264",
}.get(video_track.commercial_name, ".mp4"),
],
)
)
)
)
]
ffmpeg_call = subprocess.Popen(
[
binaries.FFMPEG,
"-y",
"-i",
str(video_path),
"-map",
"0:v:0",
"-c",
"copy",
"-bsf:v",
f"{metadata_key}=video_full_range_flag={int(full)}",
]
+ out_path,
stdout=subprocess.PIPE,
)
try:
if preview:
previewer = binaries.MPV or binaries.FFPlay
if not previewer:
raise click.ClickException("MPV/FFplay executables weren't found but are required for previewing.")
subprocess.Popen((previewer, "-"), stdin=ffmpeg_call.stdout)
finally:
if ffmpeg_call.stdout:
ffmpeg_call.stdout.close()
ffmpeg_call.wait()
@util.command()
@click.argument("path", type=Path)
@click.option(
"-m", "--map", "map_", type=str, default="0", help="Test specific streams by setting FFmpeg's -map parameter."
)
def test(path: Path, map_: str) -> None:
"""
Decode an entire video and check for any corruptions or errors using FFmpeg.
You may provide a path to a file, or a folder of mkv and/or mp4 files.
Tests all streams within the file by default. Subtitles cannot be tested.
You may choose specific streams using the -m/--map parameter. E.g.,
'0:v:0' to test the first video stream, or '0:a' to test all audio streams.
"""
if not binaries.FFMPEG:
raise click.ClickException('FFmpeg executable "ffmpeg" not found but is required.')
if path.is_dir():
paths = sorted(list(path.glob("*.mkv")) + list(path.glob("*.mp4")), key=_natural_sort_key)
else:
paths = [path]
for video_path in paths:
print(f"Testing: {video_path.name}")
p = subprocess.Popen(
[
binaries.FFMPEG,
"-hide_banner",
"-benchmark",
"-err_detect",
"+crccheck+bitstream+buffer+careful+compliant+aggressive",
"-i",
str(video_path),
"-map",
map_,
"-sn",
"-f",
"null",
"-",
],
stderr=subprocess.PIPE,
universal_newlines=True,
)
reached_output = False
errors = 0
for line in p.stderr:
line = line.strip()
if "speed=" in line:
reached_output = True
if not reached_output:
continue
if line.startswith("[") and not line.startswith("[out#"):
errors += 1
stream, error = line.split("] ", maxsplit=1)
stream = stream.split(" @ ")[0]
line = f"{stream} ERROR: {error}"
print(line)
p.stderr.close()
print(f"Finished with {errors} error(s)")
p.terminate()
p.wait()
reset/packages/envied/src/envied/commands/wvd.py
+272
View File
@@ -0,0 +1,272 @@
import logging
import shutil
from pathlib import Path
from typing import Optional
import click
import yaml
from google.protobuf.json_format import MessageToDict
from pywidevine.device import Device, DeviceTypes
from pywidevine.license_protocol_pb2 import FileHashes
from rich.prompt import Prompt
from unidecode import UnidecodeError, unidecode
from envied.core.config import config
from envied.core.console import console
from envied.core.constants import context_settings
@click.group(
short_help="Manage configuration and creation of WVD (Widevine Device) files.", context_settings=context_settings
)
def wvd() -> None:
"""Manage configuration and creation of WVD (Widevine Device) files."""
@wvd.command()
@click.argument("paths", type=Path, nargs=-1)
def add(paths: list[Path]) -> None:
"""Add one or more WVD (Widevine Device) files to the WVDs Directory."""
log = logging.getLogger("wvd")
for path in paths:
dst_path = config.directories.wvds / path.name
if not path.exists():
log.error(f"The WVD path '{path}' does not exist...")
elif dst_path.exists():
log.error(f"WVD named '{path.stem}' already exists...")
else:
# TODO: Check for and log errors
_ = Device.load(path) # test if WVD is valid
dst_path.parent.mkdir(parents=True, exist_ok=True)
shutil.move(path, dst_path)
log.info(f"Added {path.stem}")
@wvd.command()
@click.argument("names", type=str, nargs=-1)
def delete(names: list[str]) -> None:
"""Delete one or more WVD (Widevine Device) files from the WVDs Directory."""
log = logging.getLogger("wvd")
for name in names:
path = (config.directories.wvds / name).with_suffix(".wvd")
if not path.exists():
log.error(f"No WVD file exists by the name '{name}'...")
continue
answer = Prompt.ask(
f"[red]Deleting '{name}'[/], are you sure you want to continue?",
choices=["y", "n"],
default="n",
console=console,
)
if answer == "n":
log.info("Aborting...")
continue
Path.unlink(path)
log.info(f"Deleted {name}")
@wvd.command()
@click.argument("path", type=Path)
def parse(path: Path) -> None:
"""
Parse a .WVD Widevine Device file to check information.
Relative paths are relative to the WVDs directory.
"""
try:
named = not path.suffix and path.relative_to(Path(""))
except ValueError:
named = False
if named:
path = config.directories.wvds / f"{path.name}.wvd"
log = logging.getLogger("wvd")
if not path.exists():
console.log(f"[bright_blue]{path.absolute()}[/] does not exist...")
return
device = Device.load(path)
log.info(f"System ID: {device.system_id}")
log.info(f"Security Level: {device.security_level}")
log.info(f"Type: {device.type}")
log.info(f"Flags: {device.flags}")
log.info(f"Private Key: {bool(device.private_key)}")
log.info(f"Client ID: {bool(device.client_id)}")
log.info(f"VMP: {bool(device.client_id.vmp_data)}")
log.info("Client ID:")
log.info(device.client_id)
log.info("VMP:")
if device.client_id.vmp_data:
file_hashes = FileHashes()
file_hashes.ParseFromString(device.client_id.vmp_data)
log.info(str(file_hashes))
else:
log.info("None")
@wvd.command()
@click.argument("wvd_paths", type=Path, nargs=-1)
@click.argument("out_dir", type=Path, nargs=1)
def dump(wvd_paths: list[Path], out_dir: Path) -> None:
"""
Extract data from a .WVD Widevine Device file to a folder structure.
If the path is relative, with no file extension, it will dump the WVD in the WVDs
directory.
"""
log = logging.getLogger("wvd")
if wvd_paths == ():
if not config.directories.wvds.exists():
console.log(f"[bright_blue]{config.directories.wvds.absolute()}[/] does not exist...")
wvd_paths = list(x for x in config.directories.wvds.iterdir() if x.is_file() and x.suffix.lower() == ".wvd")
if not wvd_paths:
console.log(f"[bright_blue]{config.directories.wvds.absolute()}[/] is empty...")
for i, (wvd_path, out_path) in enumerate(zip(wvd_paths, (out_dir / x.stem for x in wvd_paths))):
if i > 0:
log.info("")
try:
named = not wvd_path.suffix and wvd_path.relative_to(Path(""))
except ValueError:
named = False
if named:
wvd_path = config.directories.wvds / f"{wvd_path.stem}.wvd"
out_path.mkdir(parents=True, exist_ok=True)
log.info(f"Dumping: {wvd_path}")
device = Device.load(wvd_path)
log.info(f"L{device.security_level} {device.system_id} {device.type.name}")
log.info(f"Saving to: {out_path}")
device_meta = {
"wvd": {"device_type": device.type.name, "security_level": device.security_level, **device.flags},
"client_info": {},
"capabilities": MessageToDict(device.client_id, preserving_proto_field_name=True)["client_capabilities"],
}
for client_info in device.client_id.client_info:
device_meta["client_info"][client_info.name] = client_info.value
device_meta_path = out_path / "metadata.yml"
device_meta_path.write_text(yaml.dump(device_meta), encoding="utf8")
log.info(" + Device Metadata")
if device.private_key:
private_key_path = out_path / "private_key.pem"
private_key_path.write_text(data=device.private_key.export_key().decode(), encoding="utf8")
private_key_path.with_suffix(".der").write_bytes(device.private_key.export_key(format="DER"))
log.info(" + Private Key")
else:
log.warning(" - No Private Key available")
if device.client_id:
client_id_path = out_path / "client_id.bin"
client_id_path.write_bytes(device.client_id.SerializeToString())
log.info(" + Client ID")
else:
log.warning(" - No Client ID available")
if device.client_id.vmp_data:
vmp_path = out_path / "vmp.bin"
vmp_path.write_bytes(device.client_id.vmp_data)
log.info(" + VMP (File Hashes)")
else:
log.info(" - No VMP (File Hashes) available")
@wvd.command()
@click.argument("name", type=str)
@click.argument("private_key", type=Path)
@click.argument("client_id", type=Path)
@click.argument("file_hashes", type=Path, required=False)
@click.option(
"-t",
"--type",
"type_",
type=click.Choice([x.name for x in DeviceTypes], case_sensitive=False),
default="Android",
help="Device Type",
)
@click.option("-l", "--level", type=click.IntRange(1, 3), default=1, help="Device Security Level")
@click.option("-o", "--output", type=Path, default=None, help="Output Directory")
@click.pass_context
def new(
ctx: click.Context,
name: str,
private_key: Path,
client_id: Path,
file_hashes: Optional[Path],
type_: str,
level: int,
output: Optional[Path],
) -> None:
"""
Create a new .WVD Widevine provision file.
name: The origin device name of the provided data. e.g. `Nexus 6P`. You do not need to
specify the security level, that will be done automatically.
private_key: A PEM file of a Device's private key.
client_id: A binary blob file which follows the Widevine ClientIdentification protobuf
schema.
file_hashes: A binary blob file with follows the Widevine FileHashes protobuf schema.
Also known as VMP as it's used for VMP (Verified Media Path) assurance.
"""
try:
# TODO: Remove need for name, create name based on Client IDs ClientInfo values
name = unidecode(name.strip().lower().replace(" ", "_"))
except UnidecodeError as e:
raise click.UsageError(f"name: Failed to sanitize name, {e}", ctx)
if not name:
raise click.UsageError("name: Empty after sanitizing, please make sure the name is valid.", ctx)
if not private_key.is_file():
raise click.UsageError("private_key: Not a path to a file, or it doesn't exist.", ctx)
if not client_id.is_file():
raise click.UsageError("client_id: Not a path to a file, or it doesn't exist.", ctx)
if file_hashes and not file_hashes.is_file():
raise click.UsageError("file_hashes: Not a path to a file, or it doesn't exist.", ctx)
device = Device(
type_=DeviceTypes[type_.upper()],
security_level=level,
flags=None,
private_key=private_key.read_bytes(),
client_id=client_id.read_bytes(),
)
if file_hashes:
device.client_id.vmp_data = file_hashes.read_bytes()
out_path = (output or config.directories.wvds) / f"{name}_{device.system_id}_l{device.security_level}.wvd"
device.dump(out_path)
log = logging.getLogger("wvd")
log.info(f"Created binary WVD file, {out_path.name}")
log.info(f" + Saved to: {out_path.absolute()}")
log.info(f"System ID: {device.system_id}")
log.info(f"Security Level: {device.security_level}")
log.info(f"Type: {device.type}")
log.info(f"Flags: {device.flags}")
log.info(f"Private Key: {bool(device.private_key)}")
log.info(f"Client ID: {bool(device.client_id)}")
log.info(f"VMP: {bool(device.client_id.vmp_data)}")
log.info("Client ID:")
log.info(device.client_id)
log.info("VMP:")
if device.client_id.vmp_data:
file_hashes = FileHashes()
file_hashes.ParseFromString(device.client_id.vmp_data)
log.info(str(file_hashes))
else:
log.info("None")
reset/packages/envied/src/envied/core/__init__.py
+1
View File
@@ -0,0 +1 @@
__version__ = "5.3.0"
reset/packages/envied/src/envied/core/__main__.py
+79
View File
@@ -0,0 +1,79 @@
import atexit
import logging
from datetime import datetime
import click
import urllib3
from rich import traceback
from rich.console import Group
from rich.padding import Padding
from rich.text import Text
from urllib3.exceptions import InsecureRequestWarning
from envied.core import __version__
from envied.core.commands import Commands
from envied.core.config import config
from envied.core.console import ComfyRichHandler, console
from envied.core.constants import context_settings
from envied.core.update_checker import UpdateChecker
from envied.core.utilities import close_debug_logger, init_debug_logger
@click.command(cls=Commands, invoke_without_command=True, context_settings=context_settings)
@click.option("-v", "--version", is_flag=True, default=False, help="Print version information.")
@click.option("-d", "--debug", is_flag=True, default=False, help="Enable DEBUG level logs and JSON debug logging.")
def main(version: bool, debug: bool) -> None:
"""envied.Modular Movie, TV, and Music Archival Software."""
debug_logging_enabled = debug or config.debug
logging.basicConfig(
level=logging.DEBUG if debug else logging.INFO,
format="%(message)s",
handlers=[
ComfyRichHandler(
show_time=False,
show_path=debug,
console=console,
rich_tracebacks=True,
tracebacks_suppress=[click],
log_renderer=console._log_render, # noqa
)
],
)
if debug_logging_enabled:
init_debug_logger(enabled=True)
urllib3.disable_warnings(InsecureRequestWarning)
traceback.install(console=console, width=80, suppress=[click])
console.print(
Padding(
Group(
Text(
r"░█▀▀░█▀█░█░█░▀█▀░█▀▀░█▀▄" + "\n"
r"░█▀▀░█░█░▀▄▀░░█░░█▀▀░█░█" + "\n"
r"░▀▀▀░▀░▀░░▀░░▀▀▀░▀▀▀░▀▀░" + "\n" ,
style="ascii.art",
),
Text(" and more than envied....", style = "ascii.art"),
f"\nv [repr.number]{__version__}[/] - https://github.com/vinefeeder/envied",
),
(1, 11, 1, 10),
expand=True,
),
justify="center",
)
@atexit.register
def cleanup():
"""Clean up resources on exit."""
close_debug_logger()
if __name__ == "__main__":
main()
reset/packages/envied/src/envied/core/api/__init__.py
+3
View File
@@ -0,0 +1,3 @@
from envied.core.api.routes import cors_middleware, setup_routes, setup_swagger
__all__ = ["setup_routes", "setup_swagger", "cors_middleware"]
reset/packages/envied/src/envied/core/api/compression.py
+40
View File
@@ -0,0 +1,40 @@
"""aiohttp middleware for gzip transport compression."""
from __future__ import annotations
import gzip
from aiohttp import web
@web.middleware
async def compression_middleware(request: web.Request, handler) -> web.Response:
"""Compress JSON responses with gzip when the client supports it."""
response = await handler(request)
accept_encoding = request.headers.get("Accept-Encoding", "")
if "gzip" not in accept_encoding:
return response
if response.content_type and "json" not in response.content_type:
return response
body = response.body
if body is None or len(body) < 256:
return response
from envied.core.config import config
level = config.serve.get("compression_level", 1)
if not level:
return response
compressed = gzip.compress(body, compresslevel=level)
headers = dict(response.headers)
headers["Content-Encoding"] = "gzip"
headers["Content-Length"] = str(len(compressed))
return web.Response(
body=compressed,
status=response.status,
headers=headers,
)
reset/packages/envied/src/envied/core/api/download_manager.py
+752
View File
@@ -0,0 +1,752 @@
import asyncio
import json
import logging
import os
import re
import sys
import tempfile
import threading
import uuid
from contextlib import suppress
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from enum import Enum
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
log = logging.getLogger("download_manager")
def _sanitize_log(value: object) -> str:
"""Sanitize a value for safe logging by removing newlines and control characters."""
return str(value).replace("\n", "").replace("\r", "").replace("\x00", "")
class JobStatus(Enum):
QUEUED = "queued"
DOWNLOADING = "downloading"
COMPLETED = "completed"
FAILED = "failed"
CANCELLED = "cancelled"
@dataclass
class DownloadJob:
"""Represents a download job with all its parameters and status."""
job_id: str
status: JobStatus
created_time: datetime
service: str
title_id: str
parameters: Dict[str, Any]
# Progress tracking
started_time: Optional[datetime] = None
completed_time: Optional[datetime] = None
progress: float = 0.0
# Results and error info
output_files: List[str] = field(default_factory=list)
error_message: Optional[str] = None
error_details: Optional[str] = None
error_code: Optional[str] = None
error_traceback: Optional[str] = None
worker_stderr: Optional[str] = None
# Cancellation support
cancel_event: threading.Event = field(default_factory=threading.Event)
def to_dict(self, include_full_details: bool = False) -> Dict[str, Any]:
"""Convert job to dictionary for JSON response."""
result = {
"job_id": self.job_id,
"status": self.status.value,
"created_time": self.created_time.isoformat(),
"service": self.service,
"title_id": self.title_id,
"progress": self.progress,
}
if include_full_details:
result.update(
{
"parameters": self.parameters,
"started_time": self.started_time.isoformat() if self.started_time else None,
"completed_time": self.completed_time.isoformat() if self.completed_time else None,
"output_files": self.output_files,
"error_message": self.error_message,
"error_details": self.error_details,
"error_code": self.error_code,
"error_traceback": self.error_traceback,
"worker_stderr": self.worker_stderr,
}
)
return result
def _perform_download(
job_id: str,
service: str,
title_id: str,
params: Dict[str, Any],
cancel_event: Optional[threading.Event] = None,
progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
) -> List[str]:
"""Execute the synchronous download logic for a job."""
def _check_cancel(stage: str):
if cancel_event and cancel_event.is_set():
raise Exception(f"Job was cancelled {stage}")
from contextlib import redirect_stderr, redirect_stdout
from io import StringIO
_check_cancel("before execution started")
# Import dl.py components lazily to avoid circular deps during module import
import click
import yaml
from envied.commands.dl import dl
from envied.core.config import config
from envied.core.services import Services
from envied.core.tracks import Subtitle, Video
from envied.core.utils.click_types import ContextData
from envied.core.utils.collections import merge_dict
log.info(f"Starting sync download for job {job_id}")
# Convert string parameters to enums (API receives strings, dl.result() expects enums)
vcodec_raw = params.get("vcodec")
if vcodec_raw:
if isinstance(vcodec_raw, str):
vcodec_raw = [vcodec_raw]
if isinstance(vcodec_raw, list) and vcodec_raw and not isinstance(vcodec_raw[0], Video.Codec):
codec_map = {c.name.upper(): c for c in Video.Codec}
codec_map.update({c.value.upper(): c for c in Video.Codec})
params["vcodec"] = [codec_map[v.upper()] for v in vcodec_raw if v.upper() in codec_map]
else:
params["vcodec"] = []
range_raw = params.get("range")
if range_raw:
if isinstance(range_raw, str):
range_raw = [range_raw]
if isinstance(range_raw, list) and range_raw and not isinstance(range_raw[0], Video.Range):
range_map = {r.name.upper(): r for r in Video.Range}
range_map.update({r.value.upper(): r for r in Video.Range})
params["range"] = [range_map[r.upper()] for r in range_raw if r.upper() in range_map]
else:
params["range"] = [Video.Range.SDR]
sub_format_raw = params.get("sub_format")
if sub_format_raw and isinstance(sub_format_raw, str):
sub_map = {c.name.upper(): c for c in Subtitle.Codec}
sub_map.update({c.value.upper(): c for c in Subtitle.Codec})
params["sub_format"] = sub_map.get(sub_format_raw.upper())
if params.get("export"):
params["export"] = bool(params["export"])
# Normalize slow: accept string "MIN-MAX", list/tuple, or True (default 60-120)
slow_raw = params.get("slow")
if slow_raw is not None and not isinstance(slow_raw, tuple):
if isinstance(slow_raw, bool):
params["slow"] = (60, 120) if slow_raw else None
elif isinstance(slow_raw, list) and len(slow_raw) == 2:
params["slow"] = (int(slow_raw[0]), int(slow_raw[1]))
elif isinstance(slow_raw, str):
from envied.core.utils.click_types import SLOW_DELAY_RANGE
try:
params["slow"] = SLOW_DELAY_RANGE.convert(slow_raw, None, None)
except click.BadParameter as exc:
raise Exception(f"Invalid slow parameter: {exc}")
# Convert wanted episode strings to internal "SxE" format
# Accepts: "S01E01", "S01-S03", "s1e1", "1x1", or already-parsed format
wanted_raw = params.get("wanted")
if wanted_raw:
from envied.core.utils.click_types import SeasonRange
if isinstance(wanted_raw, str):
wanted_raw = [wanted_raw]
# Only convert if not already in internal "SxE" format
needs_conversion = any(not re.match(r"^\d+x\d+$", w) for w in wanted_raw)
if needs_conversion:
season_range = SeasonRange()
params["wanted"] = season_range.parse_tokens(*wanted_raw)
# Load service configuration
service_config_path = Services.get_path(service) / config.filenames.config
if service_config_path.exists():
service_config = yaml.safe_load(service_config_path.read_text(encoding="utf8"))
else:
service_config = {}
merge_dict(config.services.get(service), service_config)
from envied.commands.dl import dl as dl_command
ctx = click.Context(dl_command.cli)
ctx.invoked_subcommand = service
from envied.core.api.handlers import load_full_cdm
cdm = load_full_cdm(service, params.get("profile"), params.get("cdm_type"))
ctx.obj = ContextData(config=service_config, cdm=cdm, proxy_providers=[], profile=params.get("profile"))
ctx.params = {
"proxy": params.get("proxy"),
"no_proxy": params.get("no_proxy", False),
"no_proxy_download": params.get("no_proxy_download", False),
"profile": params.get("profile"),
"repack": params.get("repack", False),
"tag": params.get("tag"),
"tmdb_id": params.get("tmdb_id"),
"imdb_id": params.get("imdb_id"),
"animeapi_id": params.get("animeapi_id"),
"enrich": params.get("enrich", False),
"output_dir": Path(params["output_dir"]) if params.get("output_dir") else None,
"no_cache": params.get("no_cache", False),
"reset_cache": params.get("reset_cache", False),
}
dl_instance = dl(
ctx=ctx,
no_proxy=params.get("no_proxy", False),
profile=params.get("profile"),
proxy=params.get("proxy"),
repack=params.get("repack", False),
tag=params.get("tag"),
tmdb_id=params.get("tmdb_id"),
imdb_id=params.get("imdb_id"),
animeapi_id=params.get("animeapi_id"),
enrich=params.get("enrich", False),
output_dir=Path(params["output_dir"]) if params.get("output_dir") else None,
)
service_module = Services.load(service)
_check_cancel("before service instantiation")
try:
import inspect
service_init_params = inspect.signature(service_module.__init__).parameters
service_ctx = click.Context(click.Command(service))
service_ctx.parent = ctx
service_ctx.obj = ctx.obj
service_kwargs = {}
if "title" in service_init_params:
service_kwargs["title"] = title_id
for key, value in params.items():
if key in service_init_params and key not in ["service", "title_id"]:
service_kwargs[key] = value
for param_name, param_info in service_init_params.items():
if param_name not in service_kwargs and param_name not in ["self", "ctx"]:
if param_info.default is inspect.Parameter.empty:
if param_name == "movie":
service_kwargs[param_name] = "/movies/" in title_id
elif param_name == "meta_lang":
service_kwargs[param_name] = None
else:
log.warning(f"Unknown required parameter '{param_name}' for service {service}, using None")
service_kwargs[param_name] = None
service_instance = service_module(service_ctx, **service_kwargs)
except Exception as exc: # noqa: BLE001 - propagate meaningful failure
log.error(f"Failed to create service instance: {exc}")
raise
original_download_dir = config.directories.downloads
_check_cancel("before download execution")
stdout_capture = StringIO()
stderr_capture = StringIO()
# Simple progress tracking if callback provided
if progress_callback:
# Report initial progress
progress_callback({"progress": 0.0, "status": "starting"})
# Simple approach: report progress at key points
original_result = dl_instance.result
def result_with_progress(*args, **kwargs):
try:
# Report that download started
progress_callback({"progress": 5.0, "status": "downloading"})
# Call original method
result = original_result(*args, **kwargs)
# Report completion
progress_callback({"progress": 100.0, "status": "completed"})
return result
except Exception as e:
progress_callback({"progress": 0.0, "status": "failed", "error": str(e)})
raise
dl_instance.result = result_with_progress
try:
with redirect_stdout(stdout_capture), redirect_stderr(stderr_capture):
dl_instance.result(
service=service_instance,
quality=params.get("quality", []),
vcodec=params.get("vcodec", []),
acodec=params.get("acodec"),
vbitrate=params.get("vbitrate"),
abitrate=params.get("abitrate"),
vbitrate_range=params.get("vbitrate_range"),
abitrate_range=params.get("abitrate_range"),
range_=params.get("range", ["SDR"]),
channels=params.get("channels"),
no_atmos=params.get("no_atmos", False),
select_titles=False,
wanted=params.get("wanted", []),
latest_episode=params.get("latest_episode", False),
lang=params.get("lang", ["orig"]),
v_lang=params.get("v_lang", []),
a_lang=params.get("a_lang", []),
s_lang=params.get("s_lang", ["all"]),
require_subs=params.get("require_subs", []),
forced_subs=params.get("forced_subs", False),
exact_lang=params.get("exact_lang", False),
sub_format=params.get("sub_format"),
video_only=params.get("video_only", False),
audio_only=params.get("audio_only", False),
subs_only=params.get("subs_only", False),
chapters_only=params.get("chapters_only", False),
no_subs=params.get("no_subs", False),
no_audio=params.get("no_audio", False),
no_chapters=params.get("no_chapters", False),
no_video=params.get("no_video", False),
audio_description=params.get("audio_description", False),
slow=params.get("slow", None),
list_=False,
list_titles=False,
skip_dl=params.get("skip_dl", False),
export=params.get("export"),
cdm_only=params.get("cdm_only"),
no_proxy=params.get("no_proxy", False),
no_proxy_download=params.get("no_proxy_download", False),
no_folder=params.get("no_folder", False),
no_source=params.get("no_source", False),
no_mux=params.get("no_mux", False),
workers=params.get("workers"),
downloads=params.get("downloads", 1),
worst=params.get("worst", False),
best_available=params.get("best_available", False),
split_audio=params.get("split_audio"),
)
except SystemExit as exc:
if exc.code != 0:
stdout_str = stdout_capture.getvalue()
stderr_str = stderr_capture.getvalue()
log.error(f"Download exited with code {exc.code}")
log.error(f"Stdout: {stdout_str}")
log.error(f"Stderr: {stderr_str}")
raise Exception(f"Download failed with exit code {exc.code}")
except Exception as exc: # noqa: BLE001 - propagate to caller
stdout_str = stdout_capture.getvalue()
stderr_str = stderr_capture.getvalue()
log.error(f"Download execution failed: {exc}")
log.error(f"Stdout: {stdout_str}")
log.error(f"Stderr: {stderr_str}")
raise
output_files = [str(p) for p in dl_instance.completed_files]
log.info(f"Download completed for job {job_id}, {len(output_files)} file(s) in {original_download_dir}")
return output_files
class DownloadQueueManager:
"""Manages download job queue with configurable concurrency limits."""
def __init__(self, max_concurrent_downloads: int = 2, job_retention_hours: int = 24):
self.max_concurrent_downloads = max_concurrent_downloads
self.job_retention_hours = job_retention_hours
self._jobs: Dict[str, DownloadJob] = {}
self._job_queue: asyncio.Queue = asyncio.Queue()
self._active_downloads: Dict[str, asyncio.Task] = {}
self._download_processes: Dict[str, asyncio.subprocess.Process] = {}
self._job_temp_files: Dict[str, Dict[str, str]] = {}
self._workers_started = False
self._shutdown_event = asyncio.Event()
log.info(
f"Initialized download queue manager: max_concurrent={max_concurrent_downloads}, retention_hours={job_retention_hours}"
)
def create_job(self, service: str, title_id: str, **parameters) -> DownloadJob:
"""Create a new download job and add it to the queue."""
job_id = str(uuid.uuid4())
job = DownloadJob(
job_id=job_id,
status=JobStatus.QUEUED,
created_time=datetime.now(),
service=service,
title_id=title_id,
parameters=parameters,
)
self._jobs[job_id] = job
self._job_queue.put_nowait(job)
log.info(f"Created download job {job_id} for {_sanitize_log(service)}:{_sanitize_log(title_id)}")
return job
def get_job(self, job_id: str) -> Optional[DownloadJob]:
"""Get job by ID."""
return self._jobs.get(job_id)
def list_jobs(self) -> List[DownloadJob]:
"""List all jobs."""
return list(self._jobs.values())
def cancel_job(self, job_id: str) -> bool:
"""Cancel a job if it's queued or downloading."""
job = self._jobs.get(job_id)
if not job:
return False
if job.status == JobStatus.QUEUED:
job.status = JobStatus.CANCELLED
job.cancel_event.set() # Signal cancellation
log.info(f"Cancelled queued job {_sanitize_log(job_id)}")
return True
elif job.status == JobStatus.DOWNLOADING:
# Set the cancellation event first - this will be checked by the download thread
job.cancel_event.set()
job.status = JobStatus.CANCELLED
log.info(f"Signaled cancellation for downloading job {_sanitize_log(job_id)}")
# Cancel the active download task
task = self._active_downloads.get(job_id)
if task:
task.cancel()
log.info(f"Cancelled download task for job {_sanitize_log(job_id)}")
process = self._download_processes.get(job_id)
if process:
try:
process.terminate()
log.info(f"Terminated worker process for job {_sanitize_log(job_id)}")
except ProcessLookupError:
log.debug(f"Worker process for job {_sanitize_log(job_id)} already exited")
return True
return False
def cleanup_old_jobs(self) -> int:
"""Remove jobs older than retention period."""
cutoff_time = datetime.now() - timedelta(hours=self.job_retention_hours)
jobs_to_remove = []
for job_id, job in self._jobs.items():
if job.status in [JobStatus.COMPLETED, JobStatus.FAILED, JobStatus.CANCELLED]:
if job.completed_time and job.completed_time < cutoff_time:
jobs_to_remove.append(job_id)
elif not job.completed_time and job.created_time < cutoff_time:
jobs_to_remove.append(job_id)
for job_id in jobs_to_remove:
del self._jobs[job_id]
if jobs_to_remove:
log.info(f"Cleaned up {len(jobs_to_remove)} old jobs")
return len(jobs_to_remove)
async def start_workers(self):
"""Start worker tasks to process the download queue."""
if self._workers_started:
return
self._workers_started = True
# Start worker tasks
for i in range(self.max_concurrent_downloads):
asyncio.create_task(self._download_worker(f"worker-{i}"))
# Start cleanup task
asyncio.create_task(self._cleanup_worker())
log.info(f"Started {self.max_concurrent_downloads} download workers")
async def shutdown(self):
"""Shutdown the queue manager and cancel all active downloads."""
log.info("Shutting down download queue manager")
self._shutdown_event.set()
# Cancel all active downloads
for task in self._active_downloads.values():
task.cancel()
# Terminate worker processes
for job_id, process in list(self._download_processes.items()):
try:
process.terminate()
except ProcessLookupError:
log.debug(f"Worker process for job {job_id} already exited during shutdown")
for job_id, process in list(self._download_processes.items()):
try:
await asyncio.wait_for(process.wait(), timeout=5)
except asyncio.TimeoutError:
log.warning(f"Worker process for job {job_id} did not exit, killing")
process.kill()
await process.wait()
finally:
self._download_processes.pop(job_id, None)
# Clean up any remaining temp files
for paths in self._job_temp_files.values():
for path in paths.values():
try:
os.remove(path)
except OSError:
pass
self._job_temp_files.clear()
# Wait for workers to finish
if self._active_downloads:
await asyncio.gather(*self._active_downloads.values(), return_exceptions=True)
async def _download_worker(self, worker_name: str):
"""Worker task that processes jobs from the queue."""
log.debug(f"Download worker {worker_name} started")
while not self._shutdown_event.is_set():
try:
# Wait for a job or shutdown signal
job = await asyncio.wait_for(self._job_queue.get(), timeout=1.0)
if job.status == JobStatus.CANCELLED:
continue
# Start processing the job
job.status = JobStatus.DOWNLOADING
job.started_time = datetime.now()
log.info(f"Worker {worker_name} starting job {job.job_id}")
# Create download task
download_task = asyncio.create_task(self._execute_download(job))
self._active_downloads[job.job_id] = download_task
try:
await download_task
except asyncio.CancelledError:
job.status = JobStatus.CANCELLED
log.info(f"Job {job.job_id} was cancelled")
except Exception as e:
job.status = JobStatus.FAILED
job.error_message = str(e)
log.error(f"Job {job.job_id} failed: {e}")
finally:
job.completed_time = datetime.now()
if job.job_id in self._active_downloads:
del self._active_downloads[job.job_id]
except asyncio.TimeoutError:
continue
except Exception as e:
log.error(f"Worker {worker_name} error: {e}")
async def _execute_download(self, job: DownloadJob):
"""Execute the actual download for a job."""
log.info(f"Executing download for job {job.job_id}")
try:
output_files = await self._run_download_async(job)
job.status = JobStatus.COMPLETED
job.output_files = output_files
job.progress = 100.0
log.info(f"Download completed for job {job.job_id}: {len(output_files)} files")
except Exception as e:
import traceback
from envied.core.api.errors import categorize_exception
job.status = JobStatus.FAILED
job.error_message = str(e)
job.error_details = str(e)
api_error = categorize_exception(
e, context={"service": job.service, "title_id": job.title_id, "job_id": job.job_id}
)
job.error_code = api_error.error_code.value
job.error_traceback = traceback.format_exc()
log.error(f"Download failed for job {job.job_id}: {e}")
raise
async def _run_download_async(self, job: DownloadJob) -> List[str]:
"""Invoke a worker subprocess to execute the download."""
payload = {
"job_id": job.job_id,
"service": job.service,
"title_id": job.title_id,
"parameters": job.parameters,
}
payload_fd, payload_path = tempfile.mkstemp(prefix=f"envied.job_{job.job_id}_", suffix="_payload.json")
os.close(payload_fd)
result_fd, result_path = tempfile.mkstemp(prefix=f"envied.job_{job.job_id}_", suffix="_result.json")
os.close(result_fd)
progress_fd, progress_path = tempfile.mkstemp(prefix=f"envied.job_{job.job_id}_", suffix="_progress.json")
os.close(progress_fd)
with open(payload_path, "w", encoding="utf-8") as handle:
json.dump(payload, handle)
process = await asyncio.create_subprocess_exec(
sys.executable,
"-m",
"envied.core.api.download_worker",
payload_path,
result_path,
progress_path,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
self._download_processes[job.job_id] = process
self._job_temp_files[job.job_id] = {"payload": payload_path, "result": result_path, "progress": progress_path}
communicate_task = asyncio.create_task(process.communicate())
stdout_bytes = b""
stderr_bytes = b""
try:
while True:
done, _ = await asyncio.wait({communicate_task}, timeout=0.5)
if communicate_task in done:
stdout_bytes, stderr_bytes = communicate_task.result()
break
# Check for progress updates
try:
if os.path.exists(progress_path):
with open(progress_path, "r", encoding="utf-8") as handle:
progress_data = json.load(handle)
if "progress" in progress_data:
new_progress = float(progress_data["progress"])
if new_progress != job.progress:
job.progress = new_progress
log.info(f"Job {job.job_id} progress updated: {job.progress}%")
except (FileNotFoundError, json.JSONDecodeError, ValueError) as e:
log.debug(f"Could not read progress for job {job.job_id}: {e}")
if job.cancel_event.is_set() or job.status == JobStatus.CANCELLED:
log.info(f"Cancellation detected for job {job.job_id}, terminating worker process")
process.terminate()
try:
await asyncio.wait_for(communicate_task, timeout=5)
except asyncio.TimeoutError:
log.warning(f"Worker process for job {job.job_id} did not terminate, killing")
process.kill()
await asyncio.wait_for(communicate_task, timeout=5)
raise asyncio.CancelledError("Job was cancelled")
returncode = process.returncode
stdout = stdout_bytes.decode("utf-8", errors="ignore")
stderr = stderr_bytes.decode("utf-8", errors="ignore")
if stdout.strip():
log.debug(f"Worker stdout for job {job.job_id}: {stdout.strip()}")
if stderr.strip():
job.worker_stderr = stderr.strip()
if returncode != 0:
log.warning(f"Worker stderr for job {job.job_id}: {stderr.strip()}")
else:
log.debug(f"Worker stderr for job {job.job_id}: {stderr.strip()}")
result_data: Optional[Dict[str, Any]] = None
try:
with open(result_path, "r", encoding="utf-8") as handle:
result_data = json.load(handle)
except FileNotFoundError:
log.error(f"Result file missing for job {job.job_id}")
except json.JSONDecodeError as exc:
log.error(f"Failed to parse worker result for job {job.job_id}: {exc}")
if returncode != 0:
message = result_data.get("message") if result_data else "unknown error"
if result_data:
job.error_details = result_data.get("error_details", message)
job.error_code = result_data.get("error_code")
raise Exception(f"Worker exited with code {returncode}: {message}")
if not result_data or result_data.get("status") != "success":
message = result_data.get("message") if result_data else "worker did not report success"
if result_data:
job.error_details = result_data.get("error_details", message)
job.error_code = result_data.get("error_code")
raise Exception(f"Worker failure: {message}")
return result_data.get("output_files", [])
finally:
if not communicate_task.done():
communicate_task.cancel()
with suppress(asyncio.CancelledError):
await communicate_task
self._download_processes.pop(job.job_id, None)
temp_paths = self._job_temp_files.pop(job.job_id, {})
for path in temp_paths.values():
try:
os.remove(path)
except OSError:
pass
def _execute_download_sync(self, job: DownloadJob) -> List[str]:
"""Execute download synchronously using existing dl.py logic."""
return _perform_download(job.job_id, job.service, job.title_id, job.parameters.copy(), job.cancel_event)
async def _cleanup_worker(self):
"""Worker that periodically cleans up old jobs."""
while not self._shutdown_event.is_set():
try:
await asyncio.sleep(3600) # Run every hour
self.cleanup_old_jobs()
except Exception as e:
log.error(f"Cleanup worker error: {e}")
# Global instance
download_manager: Optional[DownloadQueueManager] = None
def get_download_manager() -> DownloadQueueManager:
"""Get the global download manager instance."""
global download_manager
if download_manager is None:
# Load configuration from envied.config
from envied.core.config import config
max_concurrent = getattr(config, "max_concurrent_downloads", 2)
retention_hours = getattr(config, "download_job_retention_hours", 24)
download_manager = DownloadQueueManager(max_concurrent, retention_hours)
return download_manager
reset/packages/envied/src/envied/core/api/download_worker.py
+102
View File
@@ -0,0 +1,102 @@
"""Standalone worker process entry point for executing download jobs."""
from __future__ import annotations
import json
import logging
import sys
import traceback
from pathlib import Path
from typing import Any, Dict
from .download_manager import _perform_download
log = logging.getLogger("download_worker")
def _read_payload(path: Path) -> Dict[str, Any]:
with path.open("r", encoding="utf-8") as handle:
return json.load(handle)
def _write_result(path: Path, payload: Dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as handle:
json.dump(payload, handle)
def main(argv: list[str]) -> int:
if len(argv) not in [3, 4]:
print(
"Usage: python -m envied.core.api.download_worker <payload_path> <result_path> [progress_path]",
file=sys.stderr,
)
return 2
payload_path = Path(argv[1])
result_path = Path(argv[2])
progress_path = Path(argv[3]) if len(argv) > 3 else None
result: Dict[str, Any] = {}
exit_code = 0
try:
payload = _read_payload(payload_path)
job_id = payload["job_id"]
service = payload["service"]
title_id = payload["title_id"]
params = payload.get("parameters", {})
log.info(f"Worker starting job {job_id} ({service}:{title_id})")
def progress_callback(progress_data: Dict[str, Any]) -> None:
"""Write progress updates to file for main process to read."""
if progress_path:
try:
log.info(f"Writing progress update: {progress_data}")
_write_result(progress_path, progress_data)
log.info(f"Progress update written to {progress_path}")
except Exception as e:
log.error(f"Failed to write progress update: {e}")
output_files = _perform_download(
job_id, service, title_id, params, cancel_event=None, progress_callback=progress_callback
)
result = {"status": "success", "output_files": output_files}
except Exception as exc: # noqa: BLE001 - capture for parent process
from envied.core.api.errors import categorize_exception
exit_code = 1
tb = traceback.format_exc()
log.error(f"Worker failed with error: {exc}")
api_error = categorize_exception(
exc,
context={
"service": payload.get("service") if "payload" in locals() else None,
"title_id": payload.get("title_id") if "payload" in locals() else None,
"job_id": payload.get("job_id") if "payload" in locals() else None,
},
)
result = {
"status": "error",
"message": str(exc),
"error_details": api_error.message,
"error_code": api_error.error_code.value,
"traceback": tb,
}
finally:
try:
_write_result(result_path, result)
except Exception as exc: # noqa: BLE001 - last resort logging
log.error(f"Failed to write worker result file: {exc}")
return exit_code
if __name__ == "__main__":
sys.exit(main(sys.argv))
reset/packages/envied/src/envied/core/api/errors.py
+326
View File
@@ -0,0 +1,326 @@
"""
API Error Handling System
Provides structured error responses with error codes, categorization,
and optional debug information for the envied.REST API.
"""
from __future__ import annotations
import traceback
from datetime import datetime, timezone
from enum import Enum
from typing import Any
from aiohttp import web
class APIErrorCode(str, Enum):
"""Standard API error codes for programmatic error handling."""
# Client errors (4xx)
INVALID_INPUT = "INVALID_INPUT" # Missing or malformed request data
INVALID_SERVICE = "INVALID_SERVICE" # Unknown service name
INVALID_TITLE_ID = "INVALID_TITLE_ID" # Invalid or malformed title ID
INVALID_PROFILE = "INVALID_PROFILE" # Profile doesn't exist
INVALID_PROXY = "INVALID_PROXY" # Invalid proxy specification
INVALID_LANGUAGE = "INVALID_LANGUAGE" # Invalid language code
INVALID_PARAMETERS = "INVALID_PARAMETERS" # Invalid download parameters
AUTH_FAILED = "AUTH_FAILED" # Authentication failure (invalid credentials/cookies)
AUTH_REQUIRED = "AUTH_REQUIRED" # Missing authentication
FORBIDDEN = "FORBIDDEN" # Action not allowed
GEOFENCE = "GEOFENCE" # Content not available in region
NOT_FOUND = "NOT_FOUND" # Resource not found (title, job, etc.)
NO_CONTENT = "NO_CONTENT" # No titles/tracks/episodes found
JOB_NOT_FOUND = "JOB_NOT_FOUND" # Download job doesn't exist
SESSION_NOT_FOUND = "SESSION_NOT_FOUND" # Remote-dl session doesn't exist or expired
TRACK_NOT_FOUND = "TRACK_NOT_FOUND" # Track ID not found in session
RATE_LIMITED = "RATE_LIMITED" # Service rate limiting
# Server errors (5xx)
INTERNAL_ERROR = "INTERNAL_ERROR" # Unexpected server error
SERVICE_ERROR = "SERVICE_ERROR" # Streaming service API error
NETWORK_ERROR = "NETWORK_ERROR" # Network connectivity issue
DRM_ERROR = "DRM_ERROR" # DRM/license acquisition failure
DOWNLOAD_ERROR = "DOWNLOAD_ERROR" # Download process failure
SERVICE_UNAVAILABLE = "SERVICE_UNAVAILABLE" # Service temporarily unavailable
WORKER_ERROR = "WORKER_ERROR" # Download worker process error
class APIError(Exception):
"""
Structured API error with error code, message, and details.
Attributes:
error_code: Standardized error code from APIErrorCode enum
message: User-friendly error message
details: Additional structured error information
retryable: Whether the operation can be retried
http_status: HTTP status code to return (default based on error_code)
"""
def __init__(
self,
error_code: APIErrorCode,
message: str,
details: dict[str, Any] | None = None,
retryable: bool = False,
http_status: int | None = None,
):
super().__init__(message)
self.error_code = error_code
self.message = message
self.details = details or {}
self.retryable = retryable
self.http_status = http_status or self._default_http_status(error_code)
@staticmethod
def _default_http_status(error_code: APIErrorCode) -> int:
"""Map error codes to default HTTP status codes."""
status_map = {
# 400 Bad Request
APIErrorCode.INVALID_INPUT: 400,
APIErrorCode.INVALID_SERVICE: 400,
APIErrorCode.INVALID_TITLE_ID: 400,
APIErrorCode.INVALID_PROFILE: 400,
APIErrorCode.INVALID_PROXY: 400,
APIErrorCode.INVALID_LANGUAGE: 400,
APIErrorCode.INVALID_PARAMETERS: 400,
# 401 Unauthorized
APIErrorCode.AUTH_REQUIRED: 401,
APIErrorCode.AUTH_FAILED: 401,
# 403 Forbidden
APIErrorCode.FORBIDDEN: 403,
APIErrorCode.GEOFENCE: 403,
# 404 Not Found
APIErrorCode.NOT_FOUND: 404,
APIErrorCode.NO_CONTENT: 404,
APIErrorCode.JOB_NOT_FOUND: 404,
APIErrorCode.SESSION_NOT_FOUND: 404,
APIErrorCode.TRACK_NOT_FOUND: 404,
# 429 Too Many Requests
APIErrorCode.RATE_LIMITED: 429,
# 500 Internal Server Error
APIErrorCode.INTERNAL_ERROR: 500,
# 502 Bad Gateway
APIErrorCode.SERVICE_ERROR: 502,
APIErrorCode.DRM_ERROR: 502,
# 503 Service Unavailable
APIErrorCode.NETWORK_ERROR: 503,
APIErrorCode.SERVICE_UNAVAILABLE: 503,
APIErrorCode.DOWNLOAD_ERROR: 500,
APIErrorCode.WORKER_ERROR: 500,
}
return status_map.get(error_code, 500)
def build_error_response(
error: APIError | Exception,
debug_mode: bool = False,
extra_debug_info: dict[str, Any] | None = None,
) -> web.Response:
"""
Build a structured JSON error response.
Args:
error: APIError or generic Exception to convert to response
debug_mode: Whether to include technical debug information
extra_debug_info: Additional debug info (stderr, stdout, etc.)
Returns:
aiohttp JSON response with structured error data
"""
if isinstance(error, APIError):
error_code = error.error_code.value
message = error.message
details = error.details
http_status = error.http_status
retryable = error.retryable
else:
# Generic exception - convert to INTERNAL_ERROR
error_code = APIErrorCode.INTERNAL_ERROR.value
message = str(error) or "An unexpected error occurred"
details = {}
http_status = 500
retryable = False
response_data: dict[str, Any] = {
"status": "error",
"error_code": error_code,
"message": message,
"timestamp": datetime.now(timezone.utc).isoformat(),
}
# Add details if present
if details:
response_data["details"] = details
# Add retryable hint if specified
if retryable:
response_data["retryable"] = True
# Add debug information if in debug mode
if debug_mode:
debug_info: dict[str, Any] = {
"exception_type": type(error).__name__,
}
# Add traceback for debugging
if isinstance(error, Exception):
debug_info["traceback"] = traceback.format_exc()
# Add any extra debug info provided
if extra_debug_info:
debug_info.update(extra_debug_info)
response_data["debug_info"] = debug_info
return web.json_response(response_data, status=http_status)
def categorize_exception(
exc: Exception,
context: dict[str, Any] | None = None,
) -> APIError:
"""
Categorize a generic exception into a structured APIError.
This function attempts to identify the type of error based on the exception
type, message patterns, and optional context information.
Args:
exc: The exception to categorize
context: Optional context (service name, operation type, etc.)
Returns:
APIError with appropriate error code and details
"""
context = context or {}
exc_str = str(exc).lower()
exc_type = type(exc).__name__
# Authentication errors
if any(keyword in exc_str for keyword in ["auth", "login", "credential", "unauthorized", "forbidden", "token"]):
return APIError(
error_code=APIErrorCode.AUTH_FAILED,
message=f"Authentication failed: {exc}",
details={**context, "reason": "authentication_error"},
retryable=False,
)
# Network errors
if any(
keyword in exc_str
for keyword in [
"connection",
"timeout",
"network",
"unreachable",
"socket",
"dns",
"resolve",
]
) or exc_type in ["ConnectionError", "TimeoutError", "URLError", "SSLError"]:
return APIError(
error_code=APIErrorCode.NETWORK_ERROR,
message=f"Network error occurred: {exc}",
details={**context, "reason": "network_connectivity"},
retryable=True,
http_status=503,
)
# Geofence/region errors
if any(keyword in exc_str for keyword in ["geofence", "region", "not available in", "territory"]):
return APIError(
error_code=APIErrorCode.GEOFENCE,
message=f"Content not available in your region: {exc}",
details={**context, "reason": "geofence_restriction"},
retryable=False,
)
# Not found errors
if any(keyword in exc_str for keyword in ["not found", "404", "does not exist", "invalid id"]):
return APIError(
error_code=APIErrorCode.NOT_FOUND,
message=f"Resource not found: {exc}",
details={**context, "reason": "not_found"},
retryable=False,
)
# Rate limiting
if any(keyword in exc_str for keyword in ["rate limit", "too many requests", "429", "throttle"]):
return APIError(
error_code=APIErrorCode.RATE_LIMITED,
message=f"Rate limit exceeded: {exc}",
details={**context, "reason": "rate_limited"},
retryable=True,
http_status=429,
)
# DRM errors
if any(keyword in exc_str for keyword in ["drm", "license", "widevine", "playready", "decrypt"]):
return APIError(
error_code=APIErrorCode.DRM_ERROR,
message=f"DRM error: {exc}",
details={**context, "reason": "drm_failure"},
retryable=False,
)
# Service unavailable
if any(keyword in exc_str for keyword in ["service unavailable", "503", "maintenance", "temporarily unavailable"]):
return APIError(
error_code=APIErrorCode.SERVICE_UNAVAILABLE,
message=f"Service temporarily unavailable: {exc}",
details={**context, "reason": "service_unavailable"},
retryable=True,
http_status=503,
)
# Validation errors
if any(keyword in exc_str for keyword in ["invalid", "malformed", "validation"]) or exc_type in [
"ValueError",
"ValidationError",
]:
return APIError(
error_code=APIErrorCode.INVALID_INPUT,
message=f"Invalid input: {exc}",
details={**context, "reason": "validation_failed"},
retryable=False,
)
# Default to internal error for unknown exceptions
return APIError(
error_code=APIErrorCode.INTERNAL_ERROR,
message=f"An unexpected error occurred: {exc}",
details={**context, "exception_type": exc_type},
retryable=False,
)
def handle_api_exception(
exc: Exception,
context: dict[str, Any] | None = None,
debug_mode: bool = False,
extra_debug_info: dict[str, Any] | None = None,
) -> web.Response:
"""
Convenience function to categorize an exception and build an error response.
Args:
exc: The exception to handle
context: Optional context information
debug_mode: Whether to include debug information
extra_debug_info: Additional debug info
Returns:
Structured JSON error response
"""
if isinstance(exc, APIError):
api_error = exc
else:
api_error = categorize_exception(exc, context)
return build_error_response(api_error, debug_mode, extra_debug_info)
reset/packages/envied/src/envied/core/api/handlers.py
+2396
View File
File diff suppressed because it is too large Load Diff
reset/packages/envied/src/envied/core/api/input_bridge.py
+139
View File
@@ -0,0 +1,139 @@
"""Thread-safe bridge for interactive input during remote authentication.
When a service calls ``request_input()`` during ``authenticate()`` on the
server, the InputBridge pauses the auth thread and exposes the prompt to
the HTTP layer so a remote client can poll for it, collect the user's
response, and submit it back. The auth thread then resumes with the
response value.
"""
from __future__ import annotations
import threading
import time
from dataclasses import dataclass, field
from enum import Enum
from typing import Optional
class AuthStatus(Enum):
"""Authentication lifecycle states for a remote session."""
AUTHENTICATING = "authenticating"
PENDING_INPUT = "pending_input"
AUTHENTICATED = "authenticated"
FAILED = "failed"
class BridgeCancelledError(Exception):
"""Raised when the bridge is cancelled (client disconnected, session deleted)."""
@dataclass
class InputBridge:
"""Thread-safe bridge between a sync auth thread and the async HTTP layer.
The auth thread calls :meth:`request_input` which blocks until the
remote client submits a response via the HTTP prompt endpoints.
"""
_prompt: Optional[str] = field(default=None, init=False, repr=False)
_response: Optional[str] = field(default=None, init=False, repr=False)
_status: AuthStatus = field(default=AuthStatus.AUTHENTICATING, init=False)
_error: Optional[str] = field(default=None, init=False, repr=False)
_cancelled: bool = field(default=False, init=False, repr=False)
_response_ready: threading.Event = field(default_factory=threading.Event, init=False, repr=False)
_lock: threading.Lock = field(default_factory=threading.Lock, init=False, repr=False)
def request_input(self, prompt: str, timeout: float = 600.0) -> str:
"""Block until the remote client submits a response for *prompt*.
Args:
prompt: The message to display to the remote user.
timeout: Maximum seconds to wait for a response.
Returns:
The string response from the remote client.
Raises:
TimeoutError: If no response is received within *timeout*.
BridgeCancelledError: If the bridge was cancelled.
"""
with self._lock:
if self._cancelled:
raise BridgeCancelledError("Session was cancelled")
self._prompt = prompt
self._response = None
self._status = AuthStatus.PENDING_INPUT
self._response_ready.clear()
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if self._response_ready.wait(timeout=0.5):
break
with self._lock:
if self._cancelled:
raise BridgeCancelledError("Session was cancelled")
else:
with self._lock:
self._status = AuthStatus.FAILED
self._error = "Input request timed out waiting for client response"
raise TimeoutError(f"No client response for prompt within {timeout}s")
with self._lock:
if self._cancelled:
raise BridgeCancelledError("Session was cancelled")
response = self._response or ""
self._prompt = None
self._response = None
self._status = AuthStatus.AUTHENTICATING
return response
def get_pending_prompt(self) -> Optional[str]:
"""Return the current prompt if the auth thread is waiting for input."""
with self._lock:
if self._status == AuthStatus.PENDING_INPUT:
return self._prompt
return None
def submit_response(self, response: str) -> bool:
"""Deliver the client's response and unblock the auth thread.
Returns:
``True`` if a prompt was pending and the response was accepted,
``False`` otherwise.
"""
with self._lock:
if self._status != AuthStatus.PENDING_INPUT:
return False
self._response = response
self._response_ready.set()
return True
def cancel(self) -> None:
"""Cancel the bridge, unblocking any waiting auth thread."""
with self._lock:
self._cancelled = True
self._status = AuthStatus.FAILED
self._error = "Session cancelled"
self._response_ready.set()
@property
def status(self) -> AuthStatus:
with self._lock:
return self._status
@status.setter
def status(self, value: AuthStatus) -> None:
with self._lock:
self._status = value
@property
def error(self) -> Optional[str]:
with self._lock:
return self._error
@error.setter
def error(self, value: Optional[str]) -> None:
with self._lock:
self._error = value
reset/packages/envied/src/envied/core/api/routes.py
+1360
View File
File diff suppressed because it is too large Load Diff
reset/packages/envied/src/envied/core/api/session_store.py
+209
View File
@@ -0,0 +1,209 @@
"""Server-side session store for remote-dl client-server architecture.
Maintains authenticated service instances between API calls so that
a client can authenticate once and then make multiple requests (list tracks,
resolve segments, proxy license) using the same session.
"""
from __future__ import annotations
import asyncio
import logging
import uuid
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional
from envied.core.api.input_bridge import AuthStatus, InputBridge
from envied.core.config import config
from envied.core.tracks import Track
log = logging.getLogger("api.session")
@dataclass
class SessionEntry:
"""A single authenticated session with a service."""
session_id: str
service_tag: str
service_instance: Any # Service instance (authenticated)
titles: Any = None # Titles_T from get_titles()
title_map: Dict[str, Any] = field(default_factory=dict) # title_id -> Title object
tracks: Dict[str, Track] = field(default_factory=dict) # track_id -> Track object
tracks_by_title: Dict[str, Dict[str, Track]] = field(default_factory=dict) # title_key -> {track_id -> Track}
chapters_by_title: Dict[str, List[Any]] = field(default_factory=dict) # title_key -> [Chapter]
creator_ip: Optional[str] = None
cache_tag: Optional[str] = None # per-session cache directory tag
input_bridge: Optional[InputBridge] = None
auth_status: AuthStatus = AuthStatus.AUTHENTICATED
auth_error: Optional[str] = None
created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
last_accessed: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
def touch(self) -> None:
"""Update last_accessed timestamp."""
self.last_accessed = datetime.now(timezone.utc)
class SessionStore:
"""Thread-safe session store with TTL-based expiration."""
def __init__(self) -> None:
self._sessions: Dict[str, SessionEntry] = {}
self._lock = asyncio.Lock()
self._cleanup_task: Optional[asyncio.Task] = None
@property
def _ttl(self) -> int:
"""Session TTL in seconds from config."""
return config.serve.get("session_ttl", 300) # 5 min default
@property
def _max_sessions(self) -> int:
"""Max concurrent sessions from config."""
return config.serve.get("max_sessions", 100)
async def create(
self,
service_tag: str,
service_instance: Any,
session_id: Optional[str] = None,
) -> SessionEntry:
"""Create a new session with an authenticated service instance."""
async with self._lock:
if len(self._sessions) >= self._max_sessions:
oldest_id = min(self._sessions, key=lambda k: self._sessions[k].last_accessed)
log.warning(f"Max sessions reached ({self._max_sessions}), evicting oldest: {oldest_id}")
del self._sessions[oldest_id]
session_id = session_id or str(uuid.uuid4())
entry = SessionEntry(
session_id=session_id,
service_tag=service_tag,
service_instance=service_instance,
)
self._sessions[session_id] = entry
log.info(f"Created session {session_id} for service {service_tag}")
return entry
async def get(self, session_id: str) -> Optional[SessionEntry]:
"""Get a session by ID, returns None if not found or expired."""
async with self._lock:
entry = self._sessions.get(session_id)
if entry is None:
return None
if entry.auth_status not in (AuthStatus.AUTHENTICATING, AuthStatus.PENDING_INPUT):
elapsed = (datetime.now(timezone.utc) - entry.last_accessed).total_seconds()
if elapsed > self._ttl:
log.info(f"Session {session_id} expired (elapsed={elapsed:.0f}s, ttl={self._ttl}s)")
del self._sessions[session_id]
return None
entry.touch()
return entry
async def delete(self, session_id: str) -> bool:
"""Delete a session. Returns True if it existed."""
async with self._lock:
entry = self._sessions.pop(session_id, None)
if entry:
if entry.input_bridge:
entry.input_bridge.cancel()
self._cleanup_cache_dir(entry.cache_tag)
log.info(f"Deleted session {session_id}")
return True
return False
async def cleanup_expired(self) -> int:
"""Remove all expired sessions. Returns count of removed sessions."""
async with self._lock:
now = datetime.now(timezone.utc)
expired = []
for sid, entry in self._sessions.items():
elapsed = (now - entry.last_accessed).total_seconds()
if entry.auth_status in (AuthStatus.AUTHENTICATING, AuthStatus.PENDING_INPUT):
if elapsed > 600:
expired.append(sid)
elif elapsed > self._ttl:
expired.append(sid)
for sid in expired:
entry = self._sessions.pop(sid)
if entry.input_bridge:
entry.input_bridge.cancel()
self._cleanup_cache_dir(entry.cache_tag)
if expired:
log.info(f"Cleaned up {len(expired)} expired sessions")
return len(expired)
async def start_cleanup_loop(self) -> None:
"""Start periodic cleanup of expired sessions."""
if self._cleanup_task is not None:
return
async def _loop() -> None:
while True:
await asyncio.sleep(60) # Check every minute
try:
await self.cleanup_expired()
except Exception:
log.exception("Error during session cleanup")
self._cleanup_task = asyncio.create_task(_loop())
log.info("Session cleanup loop started")
async def stop_cleanup_loop(self) -> None:
"""Stop the periodic cleanup task."""
if self._cleanup_task is not None:
self._cleanup_task.cancel()
self._cleanup_task = None
async def cancel_all_bridges(self) -> None:
"""Cancel all active input bridges (called on server shutdown)."""
async with self._lock:
for entry in self._sessions.values():
if entry.input_bridge:
entry.input_bridge.cancel()
count = len(self._sessions)
if count:
log.info(f"Cancelled bridges for {count} active session(s)")
@staticmethod
def _cleanup_cache_dir(cache_tag: Optional[str]) -> None:
"""Remove session cache directory and empty parents."""
if not cache_tag:
return
import shutil
cache_dir = config.directories.cache / cache_tag
if cache_dir.is_dir():
try:
shutil.rmtree(cache_dir)
except Exception as e:
log.warning(f"Failed to remove session cache {cache_dir}: {e}")
for parent in cache_dir.parents:
if parent == config.directories.cache:
break
try:
if parent.is_dir() and not any(parent.iterdir()):
parent.rmdir()
except Exception:
break
@property
def session_count(self) -> int:
"""Number of active sessions."""
return len(self._sessions)
# Singleton instance
_session_store: Optional[SessionStore] = None
def get_session_store() -> SessionStore:
"""Get or create the global session store singleton."""
global _session_store
if _session_store is None:
_session_store = SessionStore()
return _session_store
reset/packages/envied/src/envied/core/binaries.py
+79
View File
@@ -0,0 +1,79 @@
import shutil
import sys
from pathlib import Path
from typing import Optional
__shaka_platform = {"win32": "win", "darwin": "osx"}.get(sys.platform, sys.platform)
def find(*names: str) -> Optional[Path]:
"""Find the path of the first found binary name."""
current_dir = Path(__file__).resolve().parent.parent
local_binaries_dir = current_dir / "binaries"
ext = ".exe" if sys.platform == "win32" else ""
for name in names:
if local_binaries_dir.exists():
candidate_paths = [local_binaries_dir / f"{name}{ext}", local_binaries_dir / name / f"{name}{ext}"]
for subdir in local_binaries_dir.iterdir():
if subdir.is_dir():
candidate_paths.append(subdir / f"{name}{ext}")
for path in candidate_paths:
if path.is_file():
# On Unix-like systems, check if file is executable
if sys.platform == "win32" or (path.stat().st_mode & 0o111):
return path
# Fall back to system PATH
path = shutil.which(name)
if path:
return Path(path)
return None
FFMPEG = find("ffmpeg")
FFProbe = find("ffprobe")
FFPlay = find("ffplay")
SubtitleEdit = find("SubtitleEdit")
ShakaPackager = find(
"shaka-packager",
"packager",
f"packager-{__shaka_platform}",
f"packager-{__shaka_platform}-arm64",
f"packager-{__shaka_platform}-x64",
)
CCExtractor = find("ccextractor", "ccextractorwin", "ccextractorwinfull")
HolaProxy = find("hola-proxy")
MPV = find("mpv")
Caddy = find("caddy")
MKVToolNix = find("mkvmerge")
Mkvpropedit = find("mkvpropedit")
DoviTool = find("dovi_tool")
HDR10PlusTool = find("hdr10plus_tool", "HDR10Plus_tool")
Mp4decrypt = find("mp4decrypt")
Docker = find("docker")
ML_Worker = find("ML-Worker")
__all__ = (
"FFMPEG",
"FFProbe",
"FFPlay",
"SubtitleEdit",
"ShakaPackager",
"CCExtractor",
"HolaProxy",
"MPV",
"Caddy",
"MKVToolNix",
"Mkvpropedit",
"DoviTool",
"HDR10PlusTool",
"Mp4decrypt",
"Docker",
"ML_Worker",
"find",
)
reset/packages/envied/src/envied/core/cacher.py
+156
View File
@@ -0,0 +1,156 @@
from __future__ import annotations
import zlib
from datetime import datetime, timedelta
from os import stat_result
from pathlib import Path
from typing import Any, Optional, Union
import jsonpickle
import jwt
from envied.core.config import config
EXP_T = Union[datetime, str, int, float]
class Cacher:
"""Cacher for Services to get and set arbitrary data with expiration dates."""
def __init__(
self,
service_tag: str,
key: Optional[str] = None,
version: Optional[int] = 1,
data: Optional[Any] = None,
expiration: Optional[datetime] = None,
) -> None:
self.service_tag = service_tag
self.key = key
self.version = version
self.data = data or {}
self.expiration = expiration
if self.expiration and self.expired:
# if its expired, remove the data for safety and delete cache file
self.data = None
self.path.unlink()
def __bool__(self) -> bool:
return bool(self.data)
@property
def path(self) -> Path:
"""Get the path at which the cache will be read and written."""
return (config.directories.cache / self.service_tag / self.key).with_suffix(".json")
@property
def expired(self) -> bool:
return self.expiration and self.expiration < datetime.now()
def get(self, key: str, version: int = 1) -> Cacher:
"""
Get Cached data for the Service by Key.
:param key: the filename to save the data to, should be url-safe.
:param version: the config data version you expect to use.
:returns: Cache object containing the cached data or None if the file does not exist.
"""
cache = Cacher(self.service_tag, key, version)
if cache.path.is_file():
data = jsonpickle.loads(cache.path.read_text(encoding="utf8"))
payload = data.copy()
del payload["crc32"]
checksum = data["crc32"]
calculated = zlib.crc32(jsonpickle.dumps(payload).encode("utf8"))
if calculated != checksum:
raise ValueError(
f"The checksum of the Cache payload mismatched. Checksum: {checksum} !== Calculated: {calculated}"
)
cache.data = data["data"]
cache.expiration = data["expiration"]
cache.version = data["version"]
if cache.version != version:
raise ValueError(
f"The version of your {self.service_tag} {key} cache is outdated. Please delete: {cache.path}"
)
return cache
def set(self, data: Any, expiration: Optional[EXP_T] = None) -> Any:
"""
Set Cached data for the Service by Key.
:param data: absolutely anything including None.
:param expiration: when the data expires, optional. Can be ISO 8601, seconds
til expiration, unix timestamp, or a datetime object.
:returns: the data provided for quick wrapping of functions or vars.
"""
self.data = data
if not expiration:
try:
expiration = jwt.decode(self.data, options={"verify_signature": False})["exp"]
except jwt.DecodeError:
pass
self.expiration = self.resolve_datetime(expiration) if expiration else None
payload = {"data": self.data, "expiration": self.expiration, "version": self.version}
payload["crc32"] = zlib.crc32(jsonpickle.dumps(payload).encode("utf8"))
self.path.parent.mkdir(parents=True, exist_ok=True)
self.path.write_text(jsonpickle.dumps(payload))
return self.data
def stat(self) -> stat_result:
"""
Get Cache file OS Stat data like Creation Time, Modified Time, and such.
:returns: an os.stat_result tuple
"""
return self.path.stat()
@staticmethod
def resolve_datetime(timestamp: EXP_T) -> datetime:
"""
Resolve multiple formats of a Datetime or Timestamp to an absolute Datetime.
Examples:
>>> now = datetime.now()
datetime.datetime(2022, 6, 27, 9, 49, 13, 657208)
>>> iso8601 = now.isoformat()
'2022-06-27T09:49:13.657208'
>>> Cacher.resolve_datetime(iso8601)
datetime.datetime(2022, 6, 27, 9, 49, 13, 657208)
>>> Cacher.resolve_datetime(iso8601 + "Z")
datetime.datetime(2022, 6, 27, 9, 49, 13, 657208)
>>> Cacher.resolve_datetime(3600)
datetime.datetime(2022, 6, 27, 10, 52, 50, 657208)
>>> Cacher.resolve_datetime('3600')
datetime.datetime(2022, 6, 27, 10, 52, 51, 657208)
>>> Cacher.resolve_datetime(7800.113)
datetime.datetime(2022, 6, 27, 11, 59, 13, 770208)
In the int/float examples you may notice that it did not return now + 3600 seconds
but rather something a bit more than that. This is because it did not resolve 3600
seconds from the `now` variable but from right now as the function was called.
"""
if isinstance(timestamp, datetime):
return timestamp
if isinstance(timestamp, str):
if timestamp.endswith("Z"):
# fromisoformat doesn't accept the final Z
timestamp = timestamp.split("Z")[0]
try:
return datetime.fromisoformat(timestamp)
except ValueError:
timestamp = float(timestamp)
try:
if len(str(int(timestamp))) == 13: # JS-style timestamp
timestamp /= 1000
timestamp = datetime.fromtimestamp(timestamp)
except ValueError:
raise ValueError(f"Unrecognized Timestamp value {timestamp!r}")
if timestamp < datetime.now():
# timestamp is likely an amount of seconds til expiration
# or, it's an already expired timestamp which is unlikely
timestamp = timestamp + timedelta(seconds=datetime.now().timestamp())
return timestamp
reset/packages/envied/src/envied/core/cacher.py.orig
+156
View File
@@ -0,0 +1,156 @@
from __future__ import annotations
import zlib
from datetime import datetime, timedelta
from os import stat_result
from pathlib import Path
from typing import Any, Optional, Union
import jsonpickle
import jwt
from envied.core.config import config
EXP_T = Union[datetime, str, int, float]
class Cacher:
"""Cacher for Services to get and set arbitrary data with expiration dates."""
def __init__(
self,
service_tag: str,
key: Optional[str] = None,
version: Optional[int] = 1,
data: Optional[Any] = None,
expiration: Optional[datetime] = None,
) -> None:
self.service_tag = service_tag
self.key = key
self.version = version
self.data = data or {}
self.expiration = expiration
if self.expiration and self.expired:
# if its expired, remove the data for safety and delete cache file
self.data = None
self.path.unlink()
def __bool__(self) -> bool:
return bool(self.data)
@property
def path(self) -> Path:
"""Get the path at which the cache will be read and written."""
return (config.directories.cache / self.service_tag / self.key).with_suffix(".json")
@property
def expired(self) -> bool:
return self.expiration and self.expiration < datetime.now()
def get(self, key: str, version: int = 1) -> Cacher:
"""
Get Cached data for the Service by Key.
:param key: the filename to save the data to, should be url-safe.
:param version: the config data version you expect to use.
:returns: Cache object containing the cached data or None if the file does not exist.
"""
cache = Cacher(self.service_tag, key, version)
if cache.path.is_file():
data = jsonpickle.loads(cache.path.read_text(encoding="utf8"))
payload = data.copy()
del payload["crc32"]
checksum = data["crc32"]
calculated = zlib.crc32(jsonpickle.dumps(payload).encode("utf8"))
if calculated != checksum:
raise ValueError(
f"The checksum of the Cache payload mismatched. Checksum: {checksum} !== Calculated: {calculated}"
)
cache.data = data["data"]
cache.expiration = data["expiration"]
cache.version = data["version"]
if cache.version != version:
raise ValueError(
f"The version of your {self.service_tag} {key} cache is outdated. Please delete: {cache.path}"
)
return cache
def set(self, data: Any, expiration: Optional[EXP_T] = None) -> Any:
"""
Set Cached data for the Service by Key.
:param data: absolutely anything including None.
:param expiration: when the data expires, optional. Can be ISO 8601, seconds
til expiration, unix timestamp, or a datetime object.
:returns: the data provided for quick wrapping of functions or vars.
"""
self.data = data
if not expiration:
try:
expiration = jwt.decode(self.data, options={"verify_signature": False})["exp"]
except jwt.DecodeError:
pass
self.expiration = self._resolve_datetime(expiration) if expiration else None
payload = {"data": self.data, "expiration": self.expiration, "version": self.version}
payload["crc32"] = zlib.crc32(jsonpickle.dumps(payload).encode("utf8"))
self.path.parent.mkdir(parents=True, exist_ok=True)
self.path.write_text(jsonpickle.dumps(payload))
return self.data
def stat(self) -> stat_result:
"""
Get Cache file OS Stat data like Creation Time, Modified Time, and such.
:returns: an os.stat_result tuple
"""
return self.path.stat()
@staticmethod
def _resolve_datetime(timestamp: EXP_T) -> datetime:
"""
Resolve multiple formats of a Datetime or Timestamp to an absolute Datetime.
Examples:
>>> now = datetime.now()
datetime.datetime(2022, 6, 27, 9, 49, 13, 657208)
>>> iso8601 = now.isoformat()
'2022-06-27T09:49:13.657208'
>>> Cacher._resolve_datetime(iso8601)
datetime.datetime(2022, 6, 27, 9, 49, 13, 657208)
>>> Cacher._resolve_datetime(iso8601 + "Z")
datetime.datetime(2022, 6, 27, 9, 49, 13, 657208)
>>> Cacher._resolve_datetime(3600)
datetime.datetime(2022, 6, 27, 10, 52, 50, 657208)
>>> Cacher._resolve_datetime('3600')
datetime.datetime(2022, 6, 27, 10, 52, 51, 657208)
>>> Cacher._resolve_datetime(7800.113)
datetime.datetime(2022, 6, 27, 11, 59, 13, 770208)
In the int/float examples you may notice that it did not return now + 3600 seconds
but rather something a bit more than that. This is because it did not resolve 3600
seconds from the `now` variable but from right now as the function was called.
"""
if isinstance(timestamp, datetime):
return timestamp
if isinstance(timestamp, str):
if timestamp.endswith("Z"):
# fromisoformat doesn't accept the final Z
timestamp = timestamp.split("Z")[0]
try:
return datetime.fromisoformat(timestamp)
except ValueError:
timestamp = float(timestamp)
try:
if len(str(int(timestamp))) == 13: # JS-style timestamp
timestamp /= 1000
timestamp = datetime.fromtimestamp(timestamp)
except ValueError:
raise ValueError(f"Unrecognized Timestamp value {timestamp!r}")
if timestamp < datetime.now():
# timestamp is likely an amount of seconds til expiration
# or, it's an already expired timestamp which is unlikely
timestamp = timestamp + timedelta(seconds=datetime.now().timestamp())
return timestamp
reset/packages/envied/src/envied/core/cdm/__init__.py
+62
View File
@@ -0,0 +1,62 @@
"""
CDM helpers and implementations.
Keep this module import-light: downstream code frequently imports helpers from
`envied.core.cdm.detect`, which requires importing this package first.
Some CDM implementations pull in optional/heavy dependencies, so we lazily
import them via `__getattr__` (PEP 562).
"""
from __future__ import annotations
from typing import Any
__all__ = [
"DecryptLabsRemoteCDM",
"CustomRemoteCDM",
"MonaLisaCDM",
"load_cdm",
"is_remote_cdm",
"is_local_cdm",
"cdm_location",
"is_playready_cdm",
"is_widevine_cdm",
]
def __getattr__(name: str) -> Any:
if name == "DecryptLabsRemoteCDM":
from .decrypt_labs_remote_cdm import DecryptLabsRemoteCDM
return DecryptLabsRemoteCDM
if name == "CustomRemoteCDM":
from .custom_remote_cdm import CustomRemoteCDM
return CustomRemoteCDM
if name == "MonaLisaCDM":
from .monalisa import MonaLisaCDM
return MonaLisaCDM
if name == "load_cdm":
from .loader import load_cdm
return load_cdm
if name in {
"is_remote_cdm",
"is_local_cdm",
"cdm_location",
"is_playready_cdm",
"is_widevine_cdm",
}:
from .detect import cdm_location, is_local_cdm, is_playready_cdm, is_remote_cdm, is_widevine_cdm
return {
"is_remote_cdm": is_remote_cdm,
"is_local_cdm": is_local_cdm,
"cdm_location": cdm_location,
"is_playready_cdm": is_playready_cdm,
"is_widevine_cdm": is_widevine_cdm,
}[name]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
reset/packages/envied/src/envied/core/cdm/custom_remote_cdm.py
+1092
View File
File diff suppressed because it is too large Load Diff
reset/packages/envied/src/envied/core/cdm/decrypt_labs_remote_cdm.py
+749
View File
@@ -0,0 +1,749 @@
from __future__ import annotations
import base64
import secrets
from typing import Any, Dict, List, Optional, Union
from uuid import UUID
import requests
from pywidevine.cdm import Cdm as WidevineCdm
from pywidevine.device import DeviceTypes
from requests import Session
from envied.core import __version__
from envied.core.vaults import Vaults
class MockCertificateChain:
"""Mock certificate chain for PlayReady compatibility."""
def __init__(self, name: str):
self._name = name
def get_name(self) -> str:
return self._name
class Key:
"""Key object compatible with pywidevine."""
def __init__(self, kid: str, key: str, type_: str = "CONTENT"):
if isinstance(kid, str):
clean_kid = kid.replace("-", "")
if len(clean_kid) == 32:
self.kid = UUID(hex=clean_kid)
else:
self.kid = UUID(hex=clean_kid.ljust(32, "0"))
else:
self.kid = kid
if isinstance(key, str):
self.key = bytes.fromhex(key)
else:
self.key = key
self.type = type_
class DecryptLabsRemoteCDMExceptions:
"""Exception classes for compatibility with pywidevine CDM."""
class InvalidSession(Exception):
"""Raised when session ID is invalid."""
class TooManySessions(Exception):
"""Raised when session limit is reached."""
class InvalidInitData(Exception):
"""Raised when PSSH/init data is invalid."""
class InvalidLicenseType(Exception):
"""Raised when license type is invalid."""
class InvalidLicenseMessage(Exception):
"""Raised when license message is invalid."""
class InvalidContext(Exception):
"""Raised when session has no context data."""
class SignatureMismatch(Exception):
"""Raised when signature verification fails."""
class DecryptLabsRemoteCDM:
"""
Decrypt Labs Remote CDM implementation with intelligent caching system.
This class provides a drop-in replacement for pywidevine's local CDM using
Decrypt Labs' KeyXtractor API service, enhanced with smart caching logic
that minimizes unnecessary license requests.
Key Features:
- Compatible with both Widevine and PlayReady DRM schemes
- Intelligent caching that compares required vs. available keys
- Optimized caching for L1/L2 devices (leverages API auto-optimization)
- Automatic key combination for mixed cache/license scenarios
- Seamless fallback to license requests when keys are missing
Intelligent Caching System:
1. DRM classes (PlayReady/Widevine) provide required KIDs via set_required_kids()
2. get_license_challenge() first checks for cached keys
3. For L1/L2 devices, always attempts cached keys first (API optimized)
4. If cached keys satisfy requirements, returns empty challenge (no license needed)
5. If keys are missing, makes targeted license request for remaining keys
6. parse_license() combines cached and license keys intelligently
"""
service_certificate_challenge = b"\x08\x04"
def __init__(
self,
secret: str,
host: str = "https://keyxtractor.decryptlabs.com",
device_name: str = "ChromeCDM",
service_name: Optional[str] = None,
vaults: Optional[Vaults] = None,
device_type: Optional[str] = None,
system_id: Optional[int] = None,
security_level: Optional[int] = None,
**kwargs,
):
"""
Initialize Decrypt Labs Remote CDM for Widevine and PlayReady schemes.
Args:
secret: Decrypt Labs API key (matches config format)
host: Decrypt Labs API host URL (matches config format)
device_name: DRM scheme (ChromeCDM, L1, L2 for Widevine; SL2, SL3 for PlayReady)
service_name: Service name for key caching and vault operations
vaults: Vaults instance for local key caching
device_type: Device type (CHROME, ANDROID, PLAYREADY) - for compatibility
system_id: System ID - for compatibility
security_level: Security level - for compatibility
"""
_ = kwargs
self.secret = secret
self.host = host.rstrip("/")
self.device_name = device_name
self.service_name = service_name or ""
self.vaults = vaults
self.uch = self.host != "https://keyxtractor.decryptlabs.com"
self._device_type_str = device_type
if device_type:
self.device_type = self._get_device_type_enum(device_type)
self._is_playready = (device_type and device_type.upper() == "PLAYREADY") or (device_name in ["SL2", "SL3"])
if self._is_playready:
self.system_id = system_id or 0
self.security_level = security_level or (2000 if device_name == "SL2" else 3000)
else:
self.system_id = system_id or 26830
self.security_level = security_level or 3
self._sessions: Dict[bytes, Dict[str, Any]] = {}
self._pssh_b64 = None
self._required_kids: Optional[List[str]] = None
self._http_session = Session()
self._http_session.headers.update(
{
"decrypt-labs-api-key": self.secret,
"Content-Type": "application/json",
"User-Agent": f"envied.decrypt-labs-cdm/{__version__}",
}
)
def _get_device_type_enum(self, device_type: str):
"""Convert device type string to enum for compatibility."""
device_type_upper = device_type.upper()
if device_type_upper == "ANDROID":
return DeviceTypes.ANDROID
elif device_type_upper == "CHROME":
return DeviceTypes.CHROME
else:
return DeviceTypes.CHROME
@property
def is_playready(self) -> bool:
"""Check if this CDM is in PlayReady mode."""
return self._is_playready
@property
def certificate_chain(self) -> MockCertificateChain:
"""Mock certificate chain for PlayReady compatibility."""
return MockCertificateChain(f"{self.device_name}_Remote")
def set_pssh_b64(self, pssh_b64: str) -> None:
"""Store base64-encoded PSSH data for PlayReady compatibility."""
self._pssh_b64 = pssh_b64
def set_required_kids(self, kids: List[Union[str, UUID]]) -> None:
"""
Set the required Key IDs for intelligent caching decisions.
This method enables the CDM to make smart decisions about when to request
additional keys via license challenges. When cached keys are available,
the CDM will compare them against the required KIDs to determine if a
license request is still needed for missing keys.
Args:
kids: List of required Key IDs as UUIDs or hex strings
Note:
Should be called by DRM classes (PlayReady/Widevine) before making
license challenge requests to enable optimal caching behavior.
"""
self._required_kids = []
for kid in kids:
if isinstance(kid, UUID):
self._required_kids.append(str(kid).replace("-", "").lower())
else:
self._required_kids.append(str(kid).replace("-", "").lower())
def _generate_session_id(self) -> bytes:
"""Generate a unique session ID."""
return secrets.token_bytes(16)
def _get_init_data_from_pssh(self, pssh: Any) -> str:
"""Extract init data from various PSSH formats."""
if self.is_playready and self._pssh_b64:
return self._pssh_b64
if hasattr(pssh, "dumps"):
dumps_result = pssh.dumps()
if isinstance(dumps_result, str):
try:
base64.b64decode(dumps_result)
return dumps_result
except Exception:
return base64.b64encode(dumps_result.encode("utf-8")).decode("utf-8")
else:
return base64.b64encode(dumps_result).decode("utf-8")
elif hasattr(pssh, "raw"):
raw_data = pssh.raw
if isinstance(raw_data, str):
raw_data = raw_data.encode("utf-8")
return base64.b64encode(raw_data).decode("utf-8")
elif hasattr(pssh, "__class__") and "WrmHeader" in pssh.__class__.__name__:
if self.is_playready:
raise ValueError("PlayReady WRM header received but no PSSH B64 was set via set_pssh_b64()")
if hasattr(pssh, "raw_bytes"):
return base64.b64encode(pssh.raw_bytes).decode("utf-8")
elif hasattr(pssh, "bytes"):
return base64.b64encode(pssh.bytes).decode("utf-8")
else:
raise ValueError(f"Cannot extract PSSH data from WRM header type: {type(pssh)}")
else:
raise ValueError(f"Unsupported PSSH type: {type(pssh)}")
def open(self) -> bytes:
"""
Open a new CDM session.
Returns:
Session identifier as bytes
"""
session_id = self._generate_session_id()
self._sessions[session_id] = {
"service_certificate": None,
"keys": [],
"pssh": None,
"challenge": None,
"decrypt_labs_session_id": None,
"tried_cache": False,
"cached_keys": None,
}
return session_id
def close(self, session_id: bytes) -> None:
"""
Close a CDM session and perform comprehensive cleanup.
Args:
session_id: Session identifier
Raises:
ValueError: If session ID is invalid
"""
if session_id not in self._sessions:
raise DecryptLabsRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}")
session = self._sessions[session_id]
session.clear()
del self._sessions[session_id]
def get_service_certificate(self, session_id: bytes) -> Optional[bytes]:
"""
Get the service certificate for a session.
Args:
session_id: Session identifier
Returns:
Service certificate if set, None otherwise
Raises:
ValueError: If session ID is invalid
"""
if session_id not in self._sessions:
raise DecryptLabsRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}")
return self._sessions[session_id]["service_certificate"]
def set_service_certificate(self, session_id: bytes, certificate: Optional[Union[bytes, str]]) -> str:
"""
Set the service certificate for a session.
Args:
session_id: Session identifier
certificate: Service certificate (bytes or base64 string)
Returns:
Certificate status message
Raises:
ValueError: If session ID is invalid
"""
if session_id not in self._sessions:
raise DecryptLabsRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}")
if certificate is None:
if not self._is_playready and self.device_name == "L1":
certificate = WidevineCdm.common_privacy_cert
self._sessions[session_id]["service_certificate"] = base64.b64decode(certificate)
return "Using default Widevine common privacy certificate for L1"
else:
self._sessions[session_id]["service_certificate"] = None
return "No certificate set (not required for this device type)"
if isinstance(certificate, str):
certificate = base64.b64decode(certificate)
self._sessions[session_id]["service_certificate"] = certificate
return "Successfully set Service Certificate"
def has_cached_keys(self, session_id: bytes) -> bool:
"""
Check if cached keys are available for the session.
Args:
session_id: Session identifier
Returns:
True if cached keys are available
Raises:
ValueError: If session ID is invalid
"""
if session_id not in self._sessions:
raise DecryptLabsRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}")
session = self._sessions[session_id]
session_keys = session.get("keys", [])
return len(session_keys) > 0
def get_license_challenge(
self, session_id: bytes, pssh_or_wrm: Any, license_type: str = "STREAMING", privacy_mode: bool = True
) -> bytes:
"""
Generate a license challenge using Decrypt Labs API with intelligent caching.
This method implements smart caching logic that:
1. First checks local vaults for required keys
2. Attempts to retrieve cached keys from the API
3. If required KIDs are set, compares available keys (vault + cached) against requirements
4. Only makes a license request if keys are missing
5. Returns empty challenge if all required keys are available
The intelligent caching works as follows:
- Local vaults: Always checked first if available
- For L1/L2 devices: Always prioritizes cached keys (API automatically optimizes)
- For other devices: Uses cache retry logic based on session state
- With required KIDs set: Only requests license for missing keys
- Without required KIDs: Returns any available cached keys
- For PlayReady: Combines vault, cached, and license keys seamlessly
Args:
session_id: Session identifier
pssh_or_wrm: PSSH object or WRM header (for PlayReady compatibility)
license_type: Type of license (STREAMING, OFFLINE, AUTOMATIC) - for compatibility only
privacy_mode: Whether to use privacy mode - for compatibility only
Returns:
License challenge as bytes, or empty bytes if available keys satisfy requirements
Raises:
InvalidSession: If session ID is invalid
requests.RequestException: If API request fails
Note:
Call set_required_kids() before this method for optimal caching behavior.
L1/L2 devices automatically use cached keys when available per API design.
Local vault keys are always checked first when vaults are available.
"""
_ = license_type, privacy_mode
if session_id not in self._sessions:
raise DecryptLabsRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}")
session = self._sessions[session_id]
session["pssh"] = pssh_or_wrm
init_data = self._get_init_data_from_pssh(pssh_or_wrm)
already_tried_cache = session.get("tried_cache", False)
if self.vaults and self._required_kids:
vault_keys = []
for kid_str in self._required_kids:
try:
clean_kid = kid_str.replace("-", "")
if len(clean_kid) == 32:
kid_uuid = UUID(hex=clean_kid)
else:
kid_uuid = UUID(hex=clean_kid.ljust(32, "0"))
key, _ = self.vaults.get_key(kid_uuid)
if key and key.count("0") != len(key):
vault_keys.append({"kid": kid_str, "key": key, "type": "CONTENT"})
except (ValueError, TypeError):
continue
if vault_keys:
vault_kids = set(k["kid"] for k in vault_keys)
required_kids = set(self._required_kids)
if required_kids.issubset(vault_kids):
session["keys"] = vault_keys
return b""
else:
session["vault_keys"] = vault_keys
if self.device_name in ["L1", "L2"]:
get_cached_keys = True
else:
get_cached_keys = not already_tried_cache
request_data = {
"scheme": self.device_name,
"init_data": init_data,
"get_cached_keys_if_exists": get_cached_keys,
}
if self.service_name:
request_data["service"] = self.service_name
if session["service_certificate"]:
request_data["service_certificate"] = base64.b64encode(session["service_certificate"]).decode("utf-8")
response = self._http_session.post(f"{self.host}/get-request", json=request_data, timeout=30)
if response.status_code != 200:
raise requests.RequestException(f"API request failed: {response.status_code} {response.text}")
data = response.json()
if data.get("message") != "success":
error_msg = data.get("message", "Unknown error")
if "details" in data:
error_msg += f" - Details: {data['details']}"
if "Error" in data:
error_msg += f" - Error: {data['Error']}"
if "service_certificate is required" in str(data) and not session["service_certificate"]:
error_msg += " (No service certificate was provided to the CDM session)"
raise requests.RequestException(f"API error: {error_msg}")
message_type = data.get("message_type")
if message_type == "cached-keys" or "cached_keys" in data:
"""
Handle cached keys response from API.
When the API returns cached keys, we need to determine if they satisfy
our requirements or if we need to make an additional license request
for missing keys.
"""
cached_keys = data.get("cached_keys", [])
parsed_keys = self._parse_cached_keys(cached_keys)
all_available_keys = list(parsed_keys)
if "vault_keys" in session:
all_available_keys.extend(session["vault_keys"])
session["tried_cache"] = True
if self._required_kids:
available_kids = set()
for key in all_available_keys:
if isinstance(key, dict) and "kid" in key:
available_kids.add(key["kid"].replace("-", "").lower())
required_kids = set(self._required_kids)
missing_kids = required_kids - available_kids
if missing_kids:
session["cached_keys"] = parsed_keys
if self.device_name in ["L1", "L2"]:
license_request_data = {
"scheme": self.device_name,
"init_data": init_data,
"get_cached_keys_if_exists": False,
}
if self.service_name:
license_request_data["service"] = self.service_name
if session["service_certificate"]:
license_request_data["service_certificate"] = base64.b64encode(
session["service_certificate"]
).decode("utf-8")
else:
license_request_data = request_data.copy()
license_request_data["get_cached_keys_if_exists"] = False
# Make license request for missing keys
response = self._http_session.post(
f"{self.host}/get-request", json=license_request_data, timeout=30
)
if response.status_code == 200:
data = response.json()
if data.get("message") == "success" and "challenge" in data:
challenge = base64.b64decode(data["challenge"])
session["challenge"] = challenge
session["decrypt_labs_session_id"] = data["session_id"]
return challenge
return b""
else:
# All required keys are available from cache
session["keys"] = all_available_keys
return b""
else:
# No required KIDs specified - return cached keys
session["keys"] = all_available_keys
return b""
if message_type == "license-request" or "challenge" in data:
challenge = base64.b64decode(data["challenge"])
session["challenge"] = challenge
session["decrypt_labs_session_id"] = data["session_id"]
return challenge
error_msg = f"Unexpected API response format. message_type={message_type}, available_fields={list(data.keys())}"
if data.get("message"):
error_msg = f"API response: {data['message']} - {error_msg}"
if "details" in data:
error_msg += f" - Details: {data['details']}"
if "Error" in data:
error_msg += f" - Error: {data['Error']}"
if already_tried_cache and data.get("message") == "success":
return b""
raise requests.RequestException(error_msg)
def parse_license(self, session_id: bytes, license_message: Union[bytes, str]) -> None:
"""
Parse license response using Decrypt Labs API with intelligent key combination.
For PlayReady content with partial cached keys, this method intelligently
combines the cached keys with newly obtained license keys, avoiding
duplicates while ensuring all required keys are available.
The key combination process:
1. Extracts keys from the license response
2. If cached keys exist (PlayReady), combines them with license keys
3. Removes duplicate keys by comparing normalized KIDs
4. Updates the session with the complete key set
Args:
session_id: Session identifier
license_message: License response from license server
Raises:
ValueError: If session ID is invalid or no challenge available
requests.RequestException: If API request fails
"""
if session_id not in self._sessions:
raise DecryptLabsRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}")
session = self._sessions[session_id]
# Skip parsing if we already have final keys (no cached keys to combine)
# If cached_keys exist (Widevine or PlayReady), we need to combine them with license keys
if session["keys"] and "cached_keys" not in session:
return
if not session.get("challenge") or not session.get("decrypt_labs_session_id"):
raise ValueError("No challenge available - call get_license_challenge first")
if isinstance(license_message, str):
if self.is_playready and license_message.strip().startswith("<?xml"):
license_message = license_message.encode("utf-8")
else:
try:
license_message = base64.b64decode(license_message)
except Exception:
license_message = license_message.encode("utf-8")
pssh = session["pssh"]
init_data = self._get_init_data_from_pssh(pssh)
license_request_b64 = base64.b64encode(session["challenge"]).decode("utf-8")
license_response_b64 = base64.b64encode(license_message).decode("utf-8")
request_data = {
"scheme": self.device_name,
"session_id": session["decrypt_labs_session_id"],
"init_data": init_data,
"license_request": license_request_b64,
"license_response": license_response_b64,
}
response = self._http_session.post(f"{self.host}/decrypt-response", json=request_data, timeout=30)
if response.status_code != 200:
raise requests.RequestException(f"License decrypt failed: {response.status_code} {response.text}")
data = response.json()
if data.get("message") != "success":
error_msg = data.get("message", "Unknown error")
if "Error" in data:
error_msg += f" - Error: {data['Error']}"
if "details" in data:
error_msg += f" - Details: {data['details']}"
raise requests.RequestException(f"License decrypt error: {error_msg}")
license_keys = self._parse_keys_response(data)
all_keys = []
if "vault_keys" in session:
all_keys.extend(session["vault_keys"])
if "cached_keys" in session:
cached_keys = session.get("cached_keys", [])
if cached_keys:
for cached_key in cached_keys:
all_keys.append(cached_key)
for license_key in license_keys:
already_exists = False
license_kid = None
if isinstance(license_key, dict) and "kid" in license_key:
license_kid = license_key["kid"].replace("-", "").lower()
elif hasattr(license_key, "kid"):
license_kid = str(license_key.kid).replace("-", "").lower()
elif hasattr(license_key, "key_id"):
license_kid = str(license_key.key_id).replace("-", "").lower()
if license_kid:
for existing_key in all_keys:
existing_kid = None
if isinstance(existing_key, dict) and "kid" in existing_key:
existing_kid = existing_key["kid"].replace("-", "").lower()
elif hasattr(existing_key, "kid"):
existing_kid = str(existing_key.kid).replace("-", "").lower()
elif hasattr(existing_key, "key_id"):
existing_kid = str(existing_key.key_id).replace("-", "").lower()
if existing_kid == license_kid:
already_exists = True
break
if not already_exists:
all_keys.append(license_key)
session["keys"] = all_keys
session.pop("cached_keys", None)
session.pop("vault_keys", None)
if self.vaults and session["keys"]:
key_dict = {}
for key in session["keys"]:
if key["type"] == "CONTENT":
try:
clean_kid = key["kid"].replace("-", "")
if len(clean_kid) == 32:
kid_uuid = UUID(hex=clean_kid)
else:
kid_uuid = UUID(hex=clean_kid.ljust(32, "0"))
key_dict[kid_uuid] = key["key"]
except (ValueError, TypeError):
continue
if key_dict:
self.vaults.add_keys(key_dict)
def get_keys(self, session_id: bytes, type_: Optional[str] = None) -> List[Key]:
"""
Get keys from the session.
Args:
session_id: Session identifier
type_: Optional key type filter (CONTENT, SIGNING, etc.)
Returns:
List of Key objects
Raises:
InvalidSession: If session ID is invalid
"""
if session_id not in self._sessions:
raise DecryptLabsRemoteCDMExceptions.InvalidSession(f"Invalid session ID: {session_id.hex()}")
key_dicts = self._sessions[session_id]["keys"]
keys = [Key(kid=k["kid"], key=k["key"], type_=k["type"]) for k in key_dicts]
if type_:
keys = [key for key in keys if key.type == type_]
return keys
def _parse_cached_keys(self, cached_keys_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
"""Parse cached keys from API response.
Args:
cached_keys_data: List of cached key objects from API
Returns:
List of key dictionaries
"""
keys = []
try:
if cached_keys_data and isinstance(cached_keys_data, list):
for key_data in cached_keys_data:
if "kid" in key_data and "key" in key_data:
keys.append({"kid": key_data["kid"], "key": key_data["key"], "type": "CONTENT"})
except Exception:
pass
return keys
def _parse_keys_response(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse keys from decrypt response."""
keys = []
if "keys" in data and isinstance(data["keys"], str):
keys_string = data["keys"]
for line in keys_string.split("\n"):
line = line.strip()
if line.startswith("--key "):
key_part = line[6:]
if ":" in key_part:
kid, key = key_part.split(":", 1)
keys.append({"kid": kid.strip(), "key": key.strip(), "type": "CONTENT"})
elif "keys" in data and isinstance(data["keys"], list):
for key_data in data["keys"]:
keys.append(
{"kid": key_data.get("kid"), "key": key_data.get("key"), "type": key_data.get("type", "CONTENT")}
)
return keys
__all__ = ["DecryptLabsRemoteCDM"]
reset/packages/envied/src/envied/core/cdm/detect.py
+187
View File
@@ -0,0 +1,187 @@
from __future__ import annotations
from typing import Any
def is_remote_cdm(cdm: Any) -> bool:
"""
Return True if the CDM instance is backed by a remote/service CDM.
This is useful for service logic that needs to know whether the CDM runs
locally (in-process) vs over HTTP/RPC (remote).
"""
if cdm is None:
return False
if hasattr(cdm, "is_remote_cdm"):
try:
return bool(getattr(cdm, "is_remote_cdm"))
except Exception:
pass
try:
from pyplayready.remote.remotecdm import RemoteCdm as PlayReadyRemoteCdm
except Exception:
PlayReadyRemoteCdm = None
if PlayReadyRemoteCdm is not None:
try:
if isinstance(cdm, PlayReadyRemoteCdm):
return True
except Exception:
pass
try:
from pywidevine.remotecdm import RemoteCdm as WidevineRemoteCdm
except Exception:
WidevineRemoteCdm = None
if WidevineRemoteCdm is not None:
try:
if isinstance(cdm, WidevineRemoteCdm):
return True
except Exception:
pass
cls = getattr(cdm, "__class__", None)
mod = getattr(cls, "__module__", "") or ""
name = getattr(cls, "__name__", "") or ""
if mod == "envied.core.cdm.decrypt_labs_remote_cdm" and name == "DecryptLabsRemoteCDM":
return True
if mod == "envied.core.cdm.custom_remote_cdm" and name == "CustomRemoteCDM":
return True
if mod.startswith("pyplayready.remote") or mod.startswith("pywidevine.remote"):
return True
if "remote" in mod.lower() and name.lower().endswith("cdm"):
return True
if name.lower().endswith("remotecdm"):
return True
return False
def is_local_cdm(cdm: Any) -> bool:
"""
Return True if the CDM instance is local/in-process.
Unknown CDM types return False (use `cdm_location()` if you need 3-state).
"""
if cdm is None:
return False
if is_remote_cdm(cdm):
return False
if is_playready_cdm(cdm) or is_widevine_cdm(cdm):
return True
cls = getattr(cdm, "__class__", None)
mod = getattr(cls, "__module__", "") or ""
name = getattr(cls, "__name__", "") or ""
if mod == "envied.core.cdm.monalisa.monalisa_cdm" and name == "MonaLisaCDM":
return True
return False
def cdm_location(cdm: Any) -> str:
"""
Return one of: "local", "remote", "unknown".
"""
if is_remote_cdm(cdm):
return "remote"
if is_local_cdm(cdm):
return "local"
return "unknown"
def is_playready_cdm(cdm: Any) -> bool:
"""
Return True if the given CDM should be treated as PlayReady.
This intentionally supports both:
- Local PlayReady CDMs (pyplayready.cdm.Cdm)
- Remote/wrapper CDMs (e.g. DecryptLabsRemoteCDM) that expose `is_playready`
"""
if cdm is None:
return False
if hasattr(cdm, "is_playready"):
try:
return bool(getattr(cdm, "is_playready"))
except Exception:
pass
try:
from pyplayready.cdm import Cdm as PlayReadyCdm
except Exception:
PlayReadyCdm = None
if PlayReadyCdm is not None:
try:
return isinstance(cdm, PlayReadyCdm)
except Exception:
pass
try:
from pyplayready.remote.remotecdm import RemoteCdm as PlayReadyRemoteCdm
except Exception:
PlayReadyRemoteCdm = None
if PlayReadyRemoteCdm is not None:
try:
return isinstance(cdm, PlayReadyRemoteCdm)
except Exception:
pass
mod = getattr(getattr(cdm, "__class__", None), "__module__", "") or ""
return "pyplayready" in mod
def is_widevine_cdm(cdm: Any) -> bool:
"""
Return True if the given CDM should be treated as Widevine.
Note: for remote/wrapper CDMs that expose `is_playready`, Widevine is treated
as the logical opposite.
"""
if cdm is None:
return False
if hasattr(cdm, "is_playready"):
try:
return not bool(getattr(cdm, "is_playready"))
except Exception:
pass
try:
from pywidevine.cdm import Cdm as WidevineCdm
except Exception:
WidevineCdm = None
if WidevineCdm is not None:
try:
return isinstance(cdm, WidevineCdm)
except Exception:
pass
try:
from pywidevine.remotecdm import RemoteCdm as WidevineRemoteCdm
except Exception:
WidevineRemoteCdm = None
if WidevineRemoteCdm is not None:
try:
return isinstance(cdm, WidevineRemoteCdm)
except Exception:
pass
mod = getattr(getattr(cdm, "__class__", None), "__module__", "") or ""
return "pywidevine" in mod
reset/packages/envied/src/envied/core/cdm/loader.py
+128
View File
@@ -0,0 +1,128 @@
"""Shared CDM loading utility.
Instantiates a CDM object (local or remote) given a resolved device name.
Name resolution (quality-based, profile-based, DRM-type) is the caller's
responsibility — this module only handles the instantiation step.
"""
from __future__ import annotations
import logging
from typing import Any, Optional
log = logging.getLogger("cdm")
def load_cdm(
cdm_name: str,
*,
service_name: str = "",
vaults: Optional[Any] = None,
) -> Any:
"""Instantiate a CDM by device name.
Looks up the name in config.remote_cdm first (for remote/API CDMs),
then falls back to local .prd / .wvd files.
Returns a CDM object (WidevineCdm, PlayReadyCdm, RemoteCdm,
DecryptLabsRemoteCDM, CustomRemoteCDM, or PlayReadyRemoteCdm).
Raises ValueError if the device cannot be found or loaded.
"""
from envied.core.config import config
cdm_api = next(iter(x.copy() for x in config.remote_cdm if x["name"] == cdm_name), None)
if cdm_api:
return _load_remote_cdm(cdm_api, cdm_name, service_name, vaults)
return _load_local_cdm(cdm_name)
def _load_remote_cdm(
cdm_api: dict,
cdm_name: str,
service_name: str,
vaults: Optional[Any],
) -> Any:
"""Instantiate a remote CDM from a config.remote_cdm entry."""
from envied.core.config import config
cdm_type = cdm_api.get("type")
if cdm_type == "decrypt_labs":
from envied.core.cdm.decrypt_labs_remote_cdm import DecryptLabsRemoteCDM
del cdm_api["name"]
del cdm_api["type"]
if "secret" not in cdm_api or not cdm_api["secret"]:
if config.decrypt_labs_api_key:
cdm_api["secret"] = config.decrypt_labs_api_key
else:
raise ValueError(
f"No secret provided for DecryptLabs CDM '{cdm_name}' and no global decrypt_labs_api_key configured"
)
return DecryptLabsRemoteCDM(service_name=service_name, vaults=vaults, **cdm_api)
if cdm_type == "custom_api":
from envied.core.cdm.custom_remote_cdm import CustomRemoteCDM
del cdm_api["name"]
del cdm_api["type"]
return CustomRemoteCDM(service_name=service_name, vaults=vaults, **cdm_api)
device_type = cdm_api.get("Device Type", cdm_api.get("device_type", ""))
if str(device_type).upper() == "PLAYREADY":
from pyplayready.remote.remotecdm import RemoteCdm as PlayReadyRemoteCdm
return PlayReadyRemoteCdm(
security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)),
host=cdm_api.get("Host", cdm_api.get("host")),
secret=cdm_api.get("Secret", cdm_api.get("secret")),
device_name=cdm_api.get("Device Name", cdm_api.get("device_name")),
)
from pywidevine.remotecdm import RemoteCdm
return RemoteCdm(
device_type=cdm_api.get("Device Type", cdm_api.get("device_type", "")),
system_id=cdm_api.get("System ID", cdm_api.get("system_id", "")),
security_level=cdm_api.get("Security Level", cdm_api.get("security_level", 3000)),
host=cdm_api.get("Host", cdm_api.get("host")),
secret=cdm_api.get("Secret", cdm_api.get("secret")),
device_name=cdm_api.get("Device Name", cdm_api.get("device_name")),
)
def _load_local_cdm(cdm_name: str) -> Any:
"""Instantiate a local CDM from a .prd or .wvd file."""
from envied.core.config import config
prd_path = config.directories.prds / f"{cdm_name}.prd"
if not prd_path.is_file():
prd_path = config.directories.wvds / f"{cdm_name}.prd"
if prd_path.is_file():
from pyplayready.cdm import Cdm as PlayReadyCdm
from pyplayready.device import Device as PlayReadyDevice
return PlayReadyCdm.from_device(PlayReadyDevice.load(prd_path))
cdm_path = config.directories.wvds / f"{cdm_name}.wvd"
if not cdm_path.is_file():
raise ValueError(f"{cdm_name} does not exist or is not a file")
from construct import ConstError
from pywidevine.cdm import Cdm as WidevineCdm
from pywidevine.device import Device
try:
device = Device.load(cdm_path)
except ConstError as e:
if "expected 2 but parsed 1" in str(e):
raise ValueError(
f"{cdm_name}.wvd seems to be a v1 WVD file, use `pywidevine migrate --help` to migrate it to v2."
)
raise ValueError(f"{cdm_name}.wvd is an invalid or corrupt Widevine Device file, {e}")
return WidevineCdm.from_device(device)
reset/packages/envied/src/envied/core/cdm/monalisa/__init__.py
+3
View File
@@ -0,0 +1,3 @@
from .monalisa_cdm import MonaLisaCDM
__all__ = ["MonaLisaCDM"]
reset/packages/envied/src/envied/core/cdm/monalisa/monalisa_cdm.py
+417
View File
@@ -0,0 +1,417 @@
"""
MonaLisa CDM - WASM-based Content Decryption Module wrapper.
This module provides key extraction from MonaLisa-protected content using
a WebAssembly module that runs locally via wasmtime.
"""
import base64
import ctypes
import hashlib
import json
import logging
import re
import sys
import uuid
from pathlib import Path
from typing import Dict, Optional, Union
import wasmtime
from envied.core import binaries
logger = logging.getLogger(__name__)
class MonaLisaCDM:
"""
MonaLisa CDM wrapper for WASM-based key extraction.
This CDM differs from Widevine/PlayReady in that it does not use a
challenge/response flow with a license server. Instead, the license
(ticket) is provided directly by the service API, and keys are extracted
locally via the WASM module.
"""
DYNAMIC_BASE = 6065008
DYNAMICTOP_PTR = 821968
LICENSE_KEY_OFFSET = 0x5C8C0C
LICENSE_KEY_LENGTH = 16
ENV_STRINGS = (
"USER=web_user",
"LOGNAME=web_user",
"PATH=/",
"PWD=/",
"HOME=/home/web_user",
"LANG=zh_CN.UTF-8",
"_=./this.program",
)
def __init__(self, device_path: Path):
"""
Initialize the MonaLisa CDM.
Args:
device_path: Path to the device file (.mld).
"""
device_path = Path(device_path)
self.device_path = device_path
self.base_dir = device_path.parent
if not self.device_path.is_file():
raise FileNotFoundError(f"Device file not found at: {self.device_path}")
try:
data = json.loads(self.device_path.read_text(encoding="utf-8", errors="replace"))
except Exception as e:
raise ValueError(f"Invalid device file (JSON): {e}")
wasm_path_str = data.get("wasm_path")
if not wasm_path_str:
raise ValueError("Device file missing 'wasm_path'")
wasm_filename = Path(wasm_path_str).name
wasm_path = self.base_dir / wasm_filename
if not wasm_path.exists():
raise FileNotFoundError(f"WASM file not found at: {wasm_path}")
try:
self.engine = wasmtime.Engine()
if wasm_path.suffix.lower() == ".wat":
self.module = wasmtime.Module.from_file(self.engine, str(wasm_path))
else:
self.module = wasmtime.Module(self.engine, wasm_path.read_bytes())
except Exception as e:
raise RuntimeError(f"Failed to load WASM module: {e}")
self.store = None
self.memory = None
self.instance = None
self.exports = {}
self.ctx = None
@staticmethod
def get_worker_path() -> Optional[Path]:
"""Get ML-Worker binary path from the envied.binaries system."""
if binaries.ML_Worker:
return Path(binaries.ML_Worker)
return None
def open(self) -> int:
"""
Open a CDM session.
Returns:
Session ID (always 1 for MonaLisa).
Raises:
RuntimeError: If session initialization fails.
"""
try:
self.store = wasmtime.Store(self.engine)
memory_type = wasmtime.MemoryType(wasmtime.Limits(256, 256))
self.memory = wasmtime.Memory(self.store, memory_type)
self._write_i32(self.DYNAMICTOP_PTR, self.DYNAMIC_BASE)
imports = self._build_imports()
self.instance = wasmtime.Instance(self.store, self.module, imports)
ex = self.instance.exports(self.store)
self.exports = {
"___wasm_call_ctors": ex["s"],
"_monalisa_context_alloc": ex["D"],
"monalisa_set_license": ex["F"],
"_monalisa_set_canvas_id": ex["t"],
"_monalisa_version_get": ex["A"],
"monalisa_get_line_number": ex["v"],
"stackAlloc": ex["N"],
"stackSave": ex["L"],
"stackRestore": ex["M"],
}
self.exports["___wasm_call_ctors"](self.store)
ctx = self.exports["_monalisa_context_alloc"](self.store)
self.ctx = ctx
# _monalisa_context_alloc is expected to return a positive pointer/handle.
# Treat 0/negative/non-int-like values as allocation failure.
try:
ctx_int = int(ctx)
except Exception:
ctx_int = None
if ctx_int is None or ctx_int <= 0:
# Ensure we don't leave a partially-initialized instance around.
self.close()
raise RuntimeError(f"Failed to allocate MonaLisa context (ctx={ctx!r})")
return 1
except Exception as e:
# Clean up partial state (e.g., store/memory/instance) before propagating failure.
self.close()
if isinstance(e, RuntimeError):
raise
raise RuntimeError(f"Failed to initialize session: {e}") from e
def close(self, session_id: int = 1) -> None:
"""
Close the CDM session and release resources.
Args:
session_id: The session ID to close (unused, for API compatibility).
"""
self.store = None
self.memory = None
self.instance = None
self.exports = {}
self.ctx = None
def extract_keys(self, license_data: Union[str, bytes]) -> Dict:
"""
Extract decryption keys from license/ticket data.
Args:
license_data: The license ticket, either as base64 string or raw bytes.
Returns:
Dictionary with keys: kid (hex), key (hex), type ("CONTENT").
Raises:
RuntimeError: If session not open or license validation fails.
ValueError: If license_data is empty.
"""
if not self.instance or not self.memory or self.ctx is None:
raise RuntimeError("Session not open. Call open() first.")
if not license_data:
raise ValueError("license_data is empty")
if isinstance(license_data, bytes):
license_b64 = base64.b64encode(license_data).decode("utf-8")
else:
license_b64 = license_data
ret = self._ccall(
"monalisa_set_license",
int,
self.ctx,
license_b64,
len(license_b64),
"0",
)
if ret != 0:
raise RuntimeError(f"License validation failed with code: {ret}")
key_bytes = self._extract_license_key_bytes()
# Extract DCID from license to generate KID
try:
decoded = base64.b64decode(license_b64).decode("ascii", errors="ignore")
except Exception as e:
# Avoid logging raw license content; log only safe metadata.
logger.exception("Failed to base64-decode MonaLisa license (len=%s): %s", len(license_b64), e)
decoded = ""
m = re.search(
r"DCID-[A-Z0-9]+-[A-Z0-9]+-\d{8}-\d{6}-[A-Z0-9]+-\d{10}-[A-Z0-9]+",
decoded,
)
if m:
kid_bytes = uuid.uuid5(uuid.NAMESPACE_DNS, m.group()).bytes
else:
# No DCID in the license: derive a deterministic per-license KID to avoid collisions.
try:
license_raw = base64.b64decode(license_b64)
except Exception:
license_raw = license_b64.encode("utf-8", errors="replace")
license_hash = hashlib.sha256(license_raw).hexdigest()
kid_bytes = uuid.uuid5(uuid.NAMESPACE_DNS, f"monalisa:license:{license_hash}").bytes
return {"kid": kid_bytes.hex(), "key": key_bytes.hex(), "type": "CONTENT"}
def _extract_license_key_bytes(self) -> bytes:
"""Extract the 16-byte decryption key from WASM memory."""
data_ptr = self.memory.data_ptr(self.store)
data_len = self.memory.data_len(self.store)
if self.LICENSE_KEY_OFFSET + self.LICENSE_KEY_LENGTH > data_len:
raise RuntimeError("License key offset beyond memory bounds")
mem_ptr = ctypes.cast(data_ptr, ctypes.POINTER(ctypes.c_ubyte * data_len))
start = self.LICENSE_KEY_OFFSET
end = self.LICENSE_KEY_OFFSET + self.LICENSE_KEY_LENGTH
return bytes(mem_ptr.contents[start:end])
def _ccall(self, func_name: str, return_type: type, *args):
"""Call a WASM function with automatic string conversion."""
stack = 0
converted_args = []
try:
for arg in args:
if isinstance(arg, str):
if stack == 0:
stack = self.exports["stackSave"](self.store)
max_length = (len(arg) << 2) + 1
ptr = self.exports["stackAlloc"](self.store, max_length)
self._string_to_utf8(arg, ptr, max_length)
converted_args.append(ptr)
else:
converted_args.append(arg)
result = self.exports[func_name](self.store, *converted_args)
finally:
# stackAlloc pointers live on the WASM stack; always restore even if the call throws.
if stack != 0:
exc = sys.exc_info()[1]
try:
self.exports["stackRestore"](self.store, stack)
except Exception:
# If we're already failing, don't mask the original exception.
if exc is None:
raise
if return_type is bool:
return bool(result)
return result
def _write_i32(self, addr: int, value: int) -> None:
"""Write a 32-bit integer to WASM memory."""
if addr % 4 != 0:
raise ValueError(f"Unaligned i32 write: addr={addr} (must be 4-byte aligned)")
data_len = self.memory.data_len(self.store)
if addr < 0 or addr + 4 > data_len:
raise IndexError(f"i32 write out of bounds: addr={addr}, mem_len={data_len}")
data = self.memory.data_ptr(self.store)
mem_ptr = ctypes.cast(data, ctypes.POINTER(ctypes.c_int32))
mem_ptr[addr >> 2] = value
def _string_to_utf8(self, data: str, ptr: int, max_length: int) -> int:
"""Convert string to UTF-8 and write to WASM memory."""
encoded = data.encode("utf-8")
write_length = min(len(encoded), max_length - 1)
mem_data = self.memory.data_ptr(self.store)
mem_ptr = ctypes.cast(mem_data, ctypes.POINTER(ctypes.c_ubyte))
for i in range(write_length):
mem_ptr[ptr + i] = encoded[i]
mem_ptr[ptr + write_length] = 0
return write_length
def _write_ascii_to_memory(self, string: str, buffer: int, dont_add_null: int = 0) -> None:
"""Write ASCII string to WASM memory."""
mem_data = self.memory.data_ptr(self.store)
mem_ptr = ctypes.cast(mem_data, ctypes.POINTER(ctypes.c_ubyte))
encoded = string.encode("utf-8")
for i, byte_val in enumerate(encoded):
mem_ptr[buffer + i] = byte_val
if dont_add_null == 0:
mem_ptr[buffer + len(encoded)] = 0
def _build_imports(self):
"""Build the WASM import stubs required by the MonaLisa module."""
def sys_fcntl64(a, b, c):
return 0
def fd_write(a, b, c, d):
return 0
def fd_close(a):
return 0
def sys_ioctl(a, b, c):
return 0
def sys_open(a, b, c):
return 0
def sys_rmdir(a):
return 0
def sys_unlink(a):
return 0
def clock():
return 0
def time(a):
return 0
def emscripten_run_script(a):
return None
def fd_seek(a, b, c, d, e):
return 0
def emscripten_resize_heap(a):
return 0
def fd_read(a, b, c, d):
return 0
def emscripten_run_script_string(a):
return 0
def emscripten_run_script_int(a):
return 1
def emscripten_memcpy_big(dest, src, num):
mem_data = self.memory.data_ptr(self.store)
data_len = self.memory.data_len(self.store)
if num is None:
num = data_len - 1
mem_ptr = ctypes.cast(mem_data, ctypes.POINTER(ctypes.c_ubyte))
for i in range(num):
if dest + i < data_len and src + i < data_len:
mem_ptr[dest + i] = mem_ptr[src + i]
return dest
def environ_get(environ_ptr, environ_buf):
buf_size = 0
for index, string in enumerate(self.ENV_STRINGS):
ptr = environ_buf + buf_size
self._write_i32(environ_ptr + index * 4, ptr)
self._write_ascii_to_memory(string, ptr)
buf_size += len(string) + 1
return 0
def environ_sizes_get(penviron_count, penviron_buf_size):
self._write_i32(penviron_count, len(self.ENV_STRINGS))
buf_size = sum(len(s) + 1 for s in self.ENV_STRINGS)
self._write_i32(penviron_buf_size, buf_size)
return 0
i32 = wasmtime.ValType.i32()
return [
wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32], [i32]), sys_fcntl64),
wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32, i32], [i32]), fd_write),
wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), fd_close),
wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32], [i32]), sys_ioctl),
wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32], [i32]), sys_open),
wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), sys_rmdir),
wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), sys_unlink),
wasmtime.Func(self.store, wasmtime.FuncType([], [i32]), clock),
wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), time),
wasmtime.Func(self.store, wasmtime.FuncType([i32], []), emscripten_run_script),
wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32, i32, i32], [i32]), fd_seek),
wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32], [i32]), emscripten_memcpy_big),
wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), emscripten_resize_heap),
wasmtime.Func(self.store, wasmtime.FuncType([i32, i32], [i32]), environ_get),
wasmtime.Func(self.store, wasmtime.FuncType([i32, i32], [i32]), environ_sizes_get),
wasmtime.Func(self.store, wasmtime.FuncType([i32, i32, i32, i32], [i32]), fd_read),
wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), emscripten_run_script_string),
wasmtime.Func(self.store, wasmtime.FuncType([i32], [i32]), emscripten_run_script_int),
self.memory,
]
reset/packages/envied/src/envied/core/commands.py
+84
View File
@@ -0,0 +1,84 @@
import logging
from pathlib import Path
from typing import Optional
import click
from envied.core.config import config
from envied.core.utilities import import_module_by_path
log = logging.getLogger("commands")
_COMMANDS = sorted(
(path for path in config.directories.commands.glob("*.py") if path.stem.lower() != "__init__"), key=lambda x: x.stem
)
def load_command(path: Path) -> object:
"""Load one command module, returning its stem-named attribute.
Raises a concise, single-line error naming the command and the real cause so
a broken command never surfaces as a raw traceback pointing at the loader.
"""
try:
module = import_module_by_path(path)
except Exception as e:
raise RuntimeError(f"{path.stem}: failed to import — {type(e).__name__}: {e} ({path})") from e
try:
return getattr(module, path.stem)
except AttributeError as e:
raise RuntimeError(
f"{path.stem}: no object named '{path.stem}' found in {path} — it must match the filename"
) from e
def load_commands(paths: list[Path]) -> tuple[dict[str, object], list[str]]:
"""Load every command, returning the good ones plus a list of load errors.
Importing this module must never raise (it runs at CLI startup, before Rich
is installed, so a raise here prints an ugly pre-setup traceback). Instead we
collect failures and surface them once, cleanly, when the CLI is used.
"""
modules: dict[str, object] = {}
errors: list[str] = []
for path in paths:
try:
modules[path.stem] = load_command(path)
except Exception as e:
errors.append(str(e))
return modules, errors
_MODULES, LOAD_ERRORS = load_commands(_COMMANDS)
def check_load_errors() -> None:
"""Raise a single clean error if any command failed to load."""
if LOAD_ERRORS:
joined = "\n".join(f" - {err}" for err in LOAD_ERRORS)
raise click.ClickException(f"Failed to load {len(LOAD_ERRORS)} command(s):\n{joined}")
class Commands(click.MultiCommand):
"""Lazy-loaded command group of project commands."""
def list_commands(self, ctx: click.Context) -> list[str]:
"""Returns a list of command names from the command filenames."""
check_load_errors()
return [x.stem.replace("_", "-") for x in _COMMANDS]
def get_command(self, ctx: click.Context, name: str) -> Optional[click.Command]:
"""Load the command code and return the main click command function."""
check_load_errors()
module = _MODULES.get(name) or _MODULES.get(name.replace("-", "_"))
if not module:
raise click.ClickException(f"Unable to find command by the name '{name}'")
if hasattr(module, "cli"):
return module.cli
return module
# Hide direct access to commands from quick import form, they shouldn't be accessed directly
__all__ = ("Commands",)
reset/packages/envied/src/envied/core/config.py
+261
View File
@@ -0,0 +1,261 @@
from __future__ import annotations
import re
import warnings
from pathlib import Path
from typing import Any, Optional
import yaml
from appdirs import AppDirs
class Config:
class _Directories:
# default directories, do not modify here, set via config
app_dirs = AppDirs("envied", False)
core_dir = Path(__file__).resolve().parent
namespace_dir = core_dir.parent
commands = namespace_dir / "commands"
services = [namespace_dir / "services"]
vaults = namespace_dir / "vaults"
fonts = namespace_dir / "fonts"
user_configs = core_dir.parent
data = core_dir.parent
downloads = core_dir.parent.parent / "downloads"
temp = core_dir.parent.parent / "temp"
cache = data / "cache"
cookies = data / "cookies"
logs = data / "logs"
exports = data / "exports"
wvds = data / "WVDs"
prds = data / "PRDs"
dcsl = data / "DCSL"
class _Filenames:
# default filenames, do not modify here, set via config
log = "envied.{name}_{time}.log" # Directories.logs
debug_log = "envied.debug_{service}_{time}.jsonl" # Directories.logs
config = "config.yaml" # Directories.services / tag
root_config = "envied.yaml" # Directories.user_configs
chapters = "Chapters_{title}_{random}.txt" # Directories.temp
subtitle = "Subtitle_{id}_{language}.srt" # Directories.temp
def __init__(self, **kwargs: Any):
self.dl: dict = kwargs.get("dl") or {}
self.cdm: dict = kwargs.get("cdm") or {}
self.chapter_fallback_name: str = kwargs.get("chapter_fallback_name") or ""
self.curl_impersonate: dict = kwargs.get("curl_impersonate") or {}
self.remote_cdm: list[dict] = kwargs.get("remote_cdm") or []
self.credentials: dict = kwargs.get("credentials") or {}
self.subtitle: dict = kwargs.get("subtitle") or {}
self.directories = self._Directories()
for name, path in (kwargs.get("directories") or {}).items():
if name.lower() in ("app_dirs", "core_dir", "namespace_dir", "user_configs", "data"):
# these must not be modified by the user
continue
if name == "services" and isinstance(path, list):
setattr(self.directories, name, [Path(p).expanduser() for p in path])
else:
setattr(self.directories, name, Path(path).expanduser())
downloader_cfg = kwargs.get("downloader")
if downloader_cfg and downloader_cfg != "requests":
warnings.warn(
f"downloader '{downloader_cfg}' is deprecated. The unified requests downloader is now used.",
DeprecationWarning,
stacklevel=2,
)
self.filenames = self._Filenames()
for name, filename in (kwargs.get("filenames") or {}).items():
setattr(self.filenames, name, filename)
self.audio: dict = kwargs.get("audio") or {}
self.headers: dict = kwargs.get("headers") or {}
self.key_vaults: list[dict[str, Any]] = kwargs.get("key_vaults", [])
self.muxing: dict = kwargs.get("muxing") or {}
self.proxy_providers: dict = kwargs.get("proxy_providers") or {}
self.remote_services: dict = kwargs.get("remote_services") or {}
self.serve: dict = kwargs.get("serve") or {}
self.services: dict = kwargs.get("services") or {}
decryption_cfg = kwargs.get("decryption") or {}
if isinstance(decryption_cfg, dict):
self.decryption_map = {k.upper(): v for k, v in decryption_cfg.items()}
self.decryption = self.decryption_map.get("DEFAULT", "shaka")
else:
self.decryption_map = {}
self.decryption = decryption_cfg or "shaka"
self.set_terminal_bg: bool = kwargs.get("set_terminal_bg", False)
self.tag: str = kwargs.get("tag") or ""
self.tag_group_name: bool = kwargs.get("tag_group_name", True)
self.tag_imdb_tmdb: bool = kwargs.get("tag_imdb_tmdb", True)
self.tmdb_api_key: str = kwargs.get("tmdb_api_key") or ""
self.simkl_client_id: str = kwargs.get("simkl_client_id") or ""
self.decrypt_labs_api_key: str = kwargs.get("decrypt_labs_api_key") or ""
self.ipinfo_api_key: str = kwargs.get("ipinfo_api_key") or ""
self.update_checks: bool = kwargs.get("update_checks", True)
self.update_check_interval: int = kwargs.get("update_check_interval", 24)
self.language_tags: dict = kwargs.get("language_tags") or {}
self.output_template: dict = kwargs.get("output_template") or {}
folder_cfg = self.output_template.pop("folder", "")
self.folder_template: str = ""
self.folder_templates: dict = {}
if isinstance(folder_cfg, dict):
self.folder_templates = {k: v for k, v in folder_cfg.items() if isinstance(v, str) and v}
elif isinstance(folder_cfg, str):
self.folder_template = folder_cfg or ""
if kwargs.get("scene_naming") is not None:
raise SystemExit(
"ERROR: The 'scene_naming' option has been removed.\n"
"Please configure 'output_template' in your envied.yaml instead.\n"
"See envied.example.yaml for examples."
)
if self.output_template:
self._validate_output_templates()
self.unicode_filenames: bool = kwargs.get("unicode_filenames", False)
self.title_cache_time: int = kwargs.get("title_cache_time", 1800) # 30 minutes default
self.title_cache_max_retention: int = kwargs.get("title_cache_max_retention", 86400) # 24 hours default
self.title_cache_enabled: bool = kwargs.get("title_cache_enabled", True)
self.debug: bool = kwargs.get("debug", False)
self.debug_keys: bool = kwargs.get("debug_keys", False)
def _validate_output_templates(self) -> None:
"""Validate output template configurations and warn about potential issues."""
if not self.output_template:
return
valid_variables = {
"title",
"year",
"season",
"episode",
"season_episode",
"episode_name",
"quality",
"resolution",
"source",
"tag",
"track_number",
"artist",
"album",
"disc",
"audio",
"audio_channels",
"audio_full",
"atmos",
"dual",
"multi",
"video",
"hdr",
"hfr",
"edition",
"repack",
"lang_tag",
}
unsafe_chars = r'[<>:"/\\|?*]'
all_templates = dict(self.output_template)
if self.folder_template:
all_templates["folder"] = self.folder_template
for kind, tmpl in self.folder_templates.items():
if kind not in {"movies", "series", "songs"}:
warnings.warn(f"Unknown folder template kind '{kind}' (expected movies/series/songs)")
continue
all_templates[f"folder.{kind}"] = tmpl
for template_type, template_str in all_templates.items():
if not isinstance(template_str, str):
warnings.warn(f"Template '{template_type}' must be a string, got {type(template_str).__name__}")
continue
variables = re.findall(r"\{([^}]+)\}", template_str)
for var in variables:
var_clean = var.rstrip("?")
if var_clean not in valid_variables:
warnings.warn(f"Unknown template variable '{var}' in {template_type} template")
test_template = re.sub(r"\{[^}]+\}", "TEST", template_str)
if re.search(unsafe_chars, test_template):
warnings.warn(f"Template '{template_type}' may contain filesystem-unsafe characters")
if not template_str.strip():
warnings.warn(f"Template '{template_type}' is empty")
def get_folder_template(self, kind: str) -> str:
"""Resolve the folder template for the given title kind.
kind: one of "movies", "series", "songs".
Falls back to the legacy single-string folder template, then "".
"""
if self.folder_templates:
tmpl = self.folder_templates.get(kind)
if tmpl:
return tmpl
return self.folder_template or ""
def get_template_separator(self, template_type: str = "movies") -> str:
"""Get the filename separator for the given template type.
Analyzes the active template to determine whether it uses dots or spaces
between variables. Falls back to dot separator (scene-style) by default.
Args:
template_type: One of "movies", "series", or "songs".
"""
template = self.output_template[template_type]
between_vars = re.findall(r"\}([^{]*)\{", template)
separator_text = "".join(between_vars)
dot_count = separator_text.count(".")
space_count = separator_text.count(" ")
return " " if space_count > dot_count else "."
@classmethod
def from_yaml(cls, path: Path) -> Config:
if not path.exists():
raise FileNotFoundError(f"Config file path ({path}) was not found")
if not path.is_file():
raise FileNotFoundError(f"Config file path ({path}) is not to a file.")
return cls(**yaml.safe_load(path.read_text(encoding="utf8")) or {})
# noinspection PyProtectedMember
POSSIBLE_CONFIG_PATHS = (
# The envied.Namespace Folder (e.g., %appdata%/Python/Python311/site-packages/envied.
Config._Directories.namespace_dir / Config._Filenames.root_config,
# The Parent Folder to the envied.Namespace Folder (e.g., %appdata%/Python/Python311/site-packages)
Config._Directories.namespace_dir.parent / Config._Filenames.root_config,
# The AppDirs User Config Folder (e.g., ~/.config/envied.on Linux, %LOCALAPPDATA%\envied.on Windows)
Path(Config._Directories.app_dirs.user_config_dir) / Config._Filenames.root_config,
)
def get_config_path() -> Optional[Path]:
"""
Get Path to Config from any one of the possible locations.
Returns None if no config file could be found.
"""
for path in POSSIBLE_CONFIG_PATHS:
if path.exists():
return path
return None
config_path = get_config_path()
if config_path:
config = Config.from_yaml(config_path)
else:
config = Config()
__all__ = ("config",)
reset/packages/envied/src/envied/core/console.py
+351
View File
@@ -0,0 +1,351 @@
import logging
from datetime import datetime
from types import ModuleType
from typing import IO, Callable, Iterable, List, Literal, Mapping, Optional, Union
from rich._log_render import FormatTimeCallable, LogRender
from rich.console import Console, ConsoleRenderable, HighlighterType, RenderableType
from rich.emoji import EmojiVariant
from rich.highlighter import Highlighter, ReprHighlighter
from rich.live import Live
from rich.logging import RichHandler
from rich.padding import Padding, PaddingDimensions
from rich.status import Status
from rich.style import StyleType
from rich.table import Table
from rich.text import Text, TextType
from rich.theme import Theme
from envied.core.config import config
class ComfyLogRenderer(LogRender):
def __call__(
self,
console: "Console",
renderables: Iterable["ConsoleRenderable"],
log_time: Optional[datetime] = None,
time_format: Optional[Union[str, FormatTimeCallable]] = None,
level: TextType = "",
path: Optional[str] = None,
line_no: Optional[int] = None,
link_path: Optional[str] = None,
) -> "Table":
from rich.containers import Renderables
output = Table.grid(padding=(0, 5), pad_edge=True)
output.expand = True
if self.show_time:
output.add_column(style="log.time")
if self.show_level:
output.add_column(style="log.level", width=self.level_width)
output.add_column(ratio=1, style="log.message", overflow="fold")
if self.show_path and path:
output.add_column(style="log.path")
row: List["RenderableType"] = []
if self.show_time:
log_time = log_time or console.get_datetime()
time_format = time_format or self.time_format
if callable(time_format):
log_time_display = time_format(log_time)
else:
log_time_display = Text(log_time.strftime(time_format))
if log_time_display == self._last_time and self.omit_repeated_times:
row.append(Text(" " * len(log_time_display)))
else:
row.append(log_time_display)
self._last_time = log_time_display
if self.show_level:
row.append(level)
row.append(Renderables(renderables))
if self.show_path and path:
path_text = Text()
path_text.append(path, style=f"link file://{link_path}" if link_path else "")
if line_no:
path_text.append(":")
path_text.append(
f"{line_no}",
style=f"link file://{link_path}#{line_no}" if link_path else "",
)
row.append(path_text)
output.add_row(*row)
return output
class ComfyRichHandler(RichHandler):
def __init__(
self,
level: Union[int, str] = logging.NOTSET,
console: Optional[Console] = None,
*,
show_time: bool = True,
omit_repeated_times: bool = True,
show_level: bool = True,
show_path: bool = True,
enable_link_path: bool = True,
highlighter: Optional[Highlighter] = None,
markup: bool = False,
rich_tracebacks: bool = False,
tracebacks_width: Optional[int] = None,
tracebacks_extra_lines: int = 3,
tracebacks_theme: Optional[str] = None,
tracebacks_word_wrap: bool = True,
tracebacks_show_locals: bool = False,
tracebacks_suppress: Iterable[Union[str, ModuleType]] = (),
locals_max_length: int = 10,
locals_max_string: int = 80,
log_time_format: Union[str, FormatTimeCallable] = "[%x %X]",
keywords: Optional[List[str]] = None,
log_renderer: Optional[LogRender] = None,
) -> None:
super().__init__(
level=level,
console=console,
show_time=show_time,
omit_repeated_times=omit_repeated_times,
show_level=show_level,
show_path=show_path,
enable_link_path=enable_link_path,
highlighter=highlighter,
markup=markup,
rich_tracebacks=rich_tracebacks,
tracebacks_width=tracebacks_width,
tracebacks_extra_lines=tracebacks_extra_lines,
tracebacks_theme=tracebacks_theme,
tracebacks_word_wrap=tracebacks_word_wrap,
tracebacks_show_locals=tracebacks_show_locals,
tracebacks_suppress=tracebacks_suppress,
locals_max_length=locals_max_length,
locals_max_string=locals_max_string,
log_time_format=log_time_format,
keywords=keywords,
)
if log_renderer:
self._log_render = log_renderer
class ComfyConsole(Console):
"""A comfy high level console interface.
Args:
color_system (str, optional): The color system supported by your terminal,
either ``"standard"``, ``"256"`` or ``"truecolor"``. Leave as ``"auto"`` to autodetect.
force_terminal (Optional[bool], optional): Enable/disable terminal control codes, or None to auto-detect
terminal. Defaults to None.
force_jupyter (Optional[bool], optional): Enable/disable Jupyter rendering, or None to auto-detect Jupyter.
Defaults to None.
force_interactive (Optional[bool], optional): Enable/disable interactive mode, or None to auto-detect.
Defaults to None.
soft_wrap (Optional[bool], optional): Set soft wrap default on print method. Defaults to False.
theme (Theme, optional): An optional style theme object, or ``None`` for default theme.
stderr (bool, optional): Use stderr rather than stdout if ``file`` is not specified. Defaults to False.
file (IO, optional): A file object where the console should write to. Defaults to stdout.
quiet (bool, Optional): Boolean to suppress all output. Defaults to False.
width (int, optional): The width of the terminal. Leave as default to auto-detect width.
height (int, optional): The height of the terminal. Leave as default to auto-detect height.
style (StyleType, optional): Style to apply to all output, or None for no style. Defaults to None.
no_color (Optional[bool], optional): Enabled no color mode, or None to auto-detect. Defaults to None.
tab_size (int, optional): Number of spaces used to replace a tab character. Defaults to 8.
record (bool, optional): Boolean to enable recording of terminal output,
required to call :meth:`export_html`, :meth:`export_svg`, and :meth:`export_text`. Defaults to False.
markup (bool, optional): Boolean to enable :ref:`console_markup`. Defaults to True.
emoji (bool, optional): Enable emoji code. Defaults to True.
emoji_variant (str, optional): Optional emoji variant, either "text" or "emoji". Defaults to None.
highlight (bool, optional): Enable automatic highlighting. Defaults to True.
log_time (bool, optional): Boolean to enable logging of time by :meth:`log` methods. Defaults to True.
log_path (bool, optional): Boolean to enable the logging of the caller by :meth:`log`. Defaults to True.
log_time_format (Union[str, TimeFormatterCallable], optional): If ``log_time`` is enabled, either string for
strftime or callable that formats the time. Defaults to "[%X] ".
highlighter (HighlighterType, optional): Default highlighter.
legacy_windows (bool, optional): Enable legacy Windows mode, or ``None`` to auto-detect. Defaults to ``None``.
safe_box (bool, optional): Restrict box options that don't render on legacy Windows.
get_datetime (Callable[[], datetime], optional): Callable that gets the current time as a datetime.datetime
object (used by Console.log), or None for datetime.now.
get_time (Callable[[], time], optional): Callable that gets the current time in seconds, default uses
time.monotonic.
"""
def __init__(
self,
*,
color_system: Optional[Literal["auto", "standard", "256", "truecolor", "windows"]] = "auto",
force_terminal: Optional[bool] = None,
force_jupyter: Optional[bool] = None,
force_interactive: Optional[bool] = None,
soft_wrap: bool = False,
theme: Optional[Theme] = None,
stderr: bool = False,
file: Optional[IO[str]] = None,
quiet: bool = False,
width: Optional[int] = None,
height: Optional[int] = None,
style: Optional[StyleType] = None,
no_color: Optional[bool] = None,
tab_size: int = 8,
record: bool = False,
markup: bool = True,
emoji: bool = True,
emoji_variant: Optional[EmojiVariant] = None,
highlight: bool = True,
log_time: bool = True,
log_path: bool = True,
log_time_format: Union[str, FormatTimeCallable] = "[%X]",
highlighter: Optional["HighlighterType"] = ReprHighlighter(),
legacy_windows: Optional[bool] = None,
safe_box: bool = True,
get_datetime: Optional[Callable[[], datetime]] = None,
get_time: Optional[Callable[[], float]] = None,
_environ: Optional[Mapping[str, str]] = None,
log_renderer: Optional[LogRender] = None,
):
super().__init__(
color_system=color_system,
force_terminal=force_terminal,
force_jupyter=force_jupyter,
force_interactive=force_interactive,
soft_wrap=soft_wrap,
theme=theme,
stderr=stderr,
file=file,
quiet=quiet,
width=width,
height=height,
style=style,
no_color=no_color,
tab_size=tab_size,
record=record,
markup=markup,
emoji=emoji,
emoji_variant=emoji_variant,
highlight=highlight,
log_time=log_time,
log_path=log_path,
log_time_format=log_time_format,
highlighter=highlighter,
legacy_windows=legacy_windows,
safe_box=safe_box,
get_datetime=get_datetime,
get_time=get_time,
_environ=_environ,
)
if log_renderer:
self._log_render = log_renderer
def status(
self,
status: RenderableType,
*,
spinner: str = "dots",
spinner_style: str = "status.spinner",
speed: float = 1.0,
refresh_per_second: float = 12.5,
pad: PaddingDimensions = (0, 5),
) -> Union[Live, Status]:
"""Display a comfy status and spinner.
Args:
status (RenderableType): A status renderable (str or Text typically).
spinner (str, optional): Name of spinner animation (see python -m rich.spinner). Defaults to "dots".
spinner_style (StyleType, optional): Style of spinner. Defaults to "status.spinner".
speed (float, optional): Speed factor for spinner animation. Defaults to 1.0.
refresh_per_second (float, optional): Number of refreshes per second. Defaults to 12.5.
pad (Union[int, Tuple[int]]): Padding for top, right, bottom, and left borders.
May be specified with 1, 2, or 4 integers (CSS style).
Returns:
Status: A Status object that may be used as a context manager.
"""
status_renderable = super().status(
status=status,
spinner=spinner,
spinner_style=spinner_style,
speed=speed,
refresh_per_second=refresh_per_second,
)
if pad:
top, right, bottom, left = Padding.unpack(pad)
renderable_width = len(status_renderable.status)
spinner_width = len(status_renderable.renderable.text)
status_width = spinner_width + renderable_width
available_width = self.width - status_width
if available_width > right:
# fill up the available width with padding to apply bg color
right = available_width - right
padding = Padding(status_renderable, (top, right, bottom, left))
return Live(padding, console=self, transient=True)
return status_renderable
catppuccin_mocha = {
# Colors based on "CatppuccinMocha" from Gogh themes
"bg": "rgb(30,30,46)",
"text": "rgb(205,214,244)",
"text2": "rgb(162,169,193)", # slightly darker
"black": "rgb(69,71,90)",
"bright_black": "rgb(88,91,112)",
"red": "rgb(243,139,168)",
"green": "rgb(166,227,161)",
"yellow": "rgb(249,226,175)",
"blue": "rgb(137,180,250)",
"pink": "rgb(245,194,231)",
"cyan": "rgb(148,226,213)",
"gray": "rgb(166,173,200)",
"bright_gray": "rgb(186,194,222)",
"dark_gray": "rgb(54,54,84)",
}
primary_scheme = catppuccin_mocha
primary_scheme["none"] = primary_scheme["text"]
primary_scheme["grey23"] = primary_scheme["black"]
primary_scheme["magenta"] = primary_scheme["pink"]
primary_scheme["bright_red"] = primary_scheme["red"]
primary_scheme["bright_green"] = primary_scheme["green"]
primary_scheme["bright_yellow"] = primary_scheme["yellow"]
primary_scheme["bright_blue"] = primary_scheme["blue"]
primary_scheme["bright_magenta"] = primary_scheme["pink"]
primary_scheme["bright_cyan"] = primary_scheme["cyan"]
if config.set_terminal_bg:
primary_scheme["none"] += f" on {primary_scheme['bg']}"
custom_colors = {"ascii.art": primary_scheme["pink"]}
if config.set_terminal_bg:
custom_colors["ascii.art"] += f" on {primary_scheme['bg']}"
console = ComfyConsole(
log_time=False,
log_path=False,
width=80,
theme=Theme(
{
"bar.back": primary_scheme["dark_gray"],
"bar.complete": primary_scheme["pink"],
"bar.finished": primary_scheme["green"],
"bar.pulse": primary_scheme["bright_black"],
"black": primary_scheme["black"],
"inspect.async_def": f"italic {primary_scheme['cyan']}",
"progress.data.speed": "dark_orange",
"repr.number": f"bold not italic {primary_scheme['cyan']}",
"repr.number_complex": f"bold not italic {primary_scheme['cyan']}",
"rule.line": primary_scheme["dark_gray"],
"rule.text": primary_scheme["pink"],
"tree.line": primary_scheme["dark_gray"],
"status.spinner": primary_scheme["pink"],
"progress.spinner": primary_scheme["pink"],
**primary_scheme,
**custom_colors,
}
),
log_renderer=ComfyLogRenderer(show_time=False, show_path=False),
)
__all__ = ("ComfyLogRenderer", "ComfyRichHandler", "ComfyConsole", "console")
reset/packages/envied/src/envied/core/constants.py
+32
View File
@@ -0,0 +1,32 @@
from threading import Event
from typing import TypeVar, Union
DOWNLOAD_CANCELLED = Event()
DOWNLOAD_LICENCE_ONLY = Event()
DRM_SORT_MAP = ["ClearKey", "Widevine"]
LANGUAGE_MAX_DISTANCE = 5 # this is max to be considered "same", e.g., en, en-US, en-AU
LANGUAGE_EXACT_DISTANCE = 0 # exact match only, no variants
VIDEO_CODEC_MAP = {"AVC": "H.264", "HEVC": "H.265"}
DYNAMIC_RANGE_MAP = {
"HDR10": "HDR",
"HDR10+": "HDR10P",
"Dolby Vision": "DV",
"HDR10 / HDR10+": "HDR10P",
"HDR10 / HDR10": "HDR",
}
AUDIO_CODEC_MAP = {"E-AC-3": "DDP", "AC-3": "DD"}
context_settings = dict(
help_option_names=["-?", "-h", "--help"], # default only has --help
max_content_width=116, # max PEP8 line-width, -4 to adjust for initial indent
)
# For use in signatures of functions which take one specific type of track at a time
# (it can't be a list that contains e.g. both Video and Audio objects)
TrackT = TypeVar("TrackT", bound="Track") # noqa: F821
# For general use in lists that can contain mixed types of tracks.
# list[Track] won't work because list is invariant.
# TODO: Add Chapter?
AnyTrack = Union["Video", "Audio", "Subtitle"] # noqa: F821
reset/packages/envied/src/envied/core/credential.py
+87
View File
@@ -0,0 +1,87 @@
from __future__ import annotations
import base64
import hashlib
import re
from pathlib import Path
from typing import Optional, Union
class Credential:
"""Username (or Email) and Password Credential."""
def __init__(self, username: str, password: str, extra: Optional[str] = None):
self.username = username
self.password = password
self.extra = extra
self.sha1 = hashlib.sha1(self.dumps().encode()).hexdigest()
def __bool__(self) -> bool:
return bool(self.username) and bool(self.password)
def __str__(self) -> str:
return self.dumps()
def __repr__(self) -> str:
return "{name}({items})".format(
name=self.__class__.__name__, items=", ".join([f"{k}={repr(v)}" for k, v in self.__dict__.items()])
)
def dumps(self) -> str:
"""Return credential data as a string."""
return f"{self.username}:{self.password}" + (f":{self.extra}" if self.extra else "")
def dump(self, path: Union[Path, str]) -> int:
"""Write credential data to a file."""
if isinstance(path, str):
path = Path(path)
return path.write_text(self.dumps(), encoding="utf8")
def as_base64(self, with_extra: bool = False, encode_password: bool = False, encode_extra: bool = False) -> str:
"""
Dump Credential as a Base64-encoded string in Basic Authorization style.
encode_password and encode_extra will also Base64-encode the password and extra respectively.
"""
value = f"{self.username}:"
if encode_password:
value += base64.b64encode(self.password.encode()).decode()
else:
value += self.password
if with_extra and self.extra:
if encode_extra:
value += f":{base64.b64encode(self.extra.encode()).decode()}"
else:
value += f":{self.extra}"
return base64.b64encode(value.encode()).decode()
@classmethod
def loads(cls, text: str) -> Credential:
"""
Load credential from a text string.
Format: {username}:{password}
Rules:
Only one Credential must be in this text contents.
All whitespace before and after all text will be removed.
Any whitespace between text will be kept and used.
The credential can be spanned across one or multiple lines as long as it
abides with all the above rules and the format.
Example that follows the format and rules:
`\tJohnd\noe@gm\n\rail.com\n:Pass1\n23\n\r \t \t`
>>>Credential(username='Johndoe@gmail.com', password='Pass123')
"""
text = "".join([x.strip() for x in text.splitlines(keepends=False)]).strip()
credential = re.fullmatch(r"^([^:]+?):([^:]+?)(?::(.+))?$", text)
if credential:
return cls(*credential.groups())
raise ValueError("No credentials found in text string. Expecting the format `username:password`")
@classmethod
def load(cls, path: Path) -> Credential:
"""
Load Credential from a file path.
Use Credential.loads() for loading from text content and seeing the rules and
format expected to be found in the URIs contents.
"""
return cls.loads(path.read_text("utf8"))
reset/packages/envied/src/envied/core/downloaders/__init__.py
+3
View File
@@ -0,0 +1,3 @@
from .requests import requests
__all__ = ("requests",)
reset/packages/envied/src/envied/core/downloaders/aria2c.py
+543
View File
@@ -0,0 +1,543 @@
import logging
import os
import subprocess
import textwrap
import threading
import time
from functools import partial
from http.cookiejar import CookieJar
from pathlib import Path
from typing import Any, Callable, Generator, MutableMapping, Optional, Union
from urllib.parse import urlparse
import requests
from Crypto.Random import get_random_bytes
from requests import Session
from requests.cookies import cookiejar_from_dict, get_cookie_header
from rich import filesize
from rich.text import Text
from envied.core import binaries
from envied.core.config import config
from envied.core.console import console
from envied.core.constants import DOWNLOAD_CANCELLED
from envied.core.utilities import get_debug_logger, get_extension, get_free_port
def rpc(caller: Callable, secret: str, method: str, params: Optional[list[Any]] = None) -> Any:
"""Make a call to Aria2's JSON-RPC API."""
try:
rpc_res = caller(
json={
"jsonrpc": "2.0",
"id": get_random_bytes(16).hex(),
"method": method,
"params": [f"token:{secret}", *(params or [])],
}
).json()
if rpc_res.get("code"):
# wrap to console width - padding - '[Aria2c]: '
error_pretty = "\n ".join(
textwrap.wrap(
f"RPC Error: {rpc_res['message']} ({rpc_res['code']})".strip(),
width=console.width - 20,
initial_indent="",
)
)
console.log(Text.from_ansi("\n[Aria2c]: " + error_pretty))
return rpc_res["result"]
except requests.exceptions.ConnectionError:
# absorb, process likely ended as it was calling RPC
return
class _Aria2Manager:
"""Singleton manager to run one aria2c process and enqueue downloads via RPC."""
def __init__(self) -> None:
self._logger = logging.getLogger(__name__)
self._proc: Optional[subprocess.Popen] = None
self._rpc_port: Optional[int] = None
self._rpc_secret: Optional[str] = None
self._rpc_uri: Optional[str] = None
self._session: Session = Session()
self._max_workers: Optional[int] = None
self._max_concurrent_downloads: int = 0
self._max_connection_per_server: int = 1
self._split_default: int = 5
self._file_allocation: str = "prealloc"
self._proxy: Optional[str] = None
self._lock: threading.Lock = threading.Lock()
def _wait_for_rpc_ready(self, timeout_s: float = 8.0, interval_s: float = 0.1) -> None:
assert self._proc is not None
assert self._rpc_uri is not None
assert self._rpc_secret is not None
deadline = time.monotonic() + timeout_s
payload = {
"jsonrpc": "2.0",
"id": get_random_bytes(16).hex(),
"method": "aria2.getVersion",
"params": [f"token:{self._rpc_secret}"],
}
while time.monotonic() < deadline:
if self._proc.poll() is not None:
raise RuntimeError(
f"aria2c exited before RPC became ready (exit code {self._proc.returncode})"
)
try:
res = self._session.post(self._rpc_uri, json=payload, timeout=0.25)
data = res.json()
if isinstance(data, dict) and data.get("result") is not None:
return
except (requests.exceptions.RequestException, ValueError):
# Not ready yet (connection refused / bad response / etc.)
pass
time.sleep(interval_s)
# Timed out: ensure we don't leave a zombie/stray aria2c process behind.
try:
self._proc.terminate()
self._proc.wait(timeout=2)
except Exception:
try:
self._proc.kill()
self._proc.wait(timeout=2)
except Exception:
pass
raise TimeoutError(f"aria2c RPC did not become ready within {timeout_s:.1f}s")
def _build_args(self) -> list[str]:
args = [
"--continue=true",
f"--max-concurrent-downloads={self._max_concurrent_downloads}",
f"--max-connection-per-server={self._max_connection_per_server}",
f"--split={self._split_default}",
"--max-file-not-found=5",
"--max-tries=5",
"--retry-wait=2",
"--allow-overwrite=true",
"--auto-file-renaming=false",
"--console-log-level=warn",
"--download-result=default",
f"--file-allocation={self._file_allocation}",
"--summary-interval=0",
"--enable-rpc=true",
f"--rpc-listen-port={self._rpc_port}",
f"--rpc-secret={self._rpc_secret}",
]
if self._proxy:
args.extend(["--all-proxy", self._proxy])
return args
def ensure_started(
self,
proxy: Optional[str],
max_workers: Optional[int],
) -> None:
with self._lock:
if not binaries.Aria2:
debug_logger = get_debug_logger()
if debug_logger:
debug_logger.log(
level="ERROR",
operation="downloader_aria2c_binary_missing",
message="Aria2c executable not found in PATH or local binaries directory",
context={"searched_names": ["aria2c", "aria2"]},
)
raise EnvironmentError("Aria2c executable not found...")
effective_proxy = proxy or None
if not max_workers:
effective_max_workers = min(32, (os.cpu_count() or 1) + 4)
elif not isinstance(max_workers, int):
raise TypeError(f"Expected max_workers to be {int}, not {type(max_workers)}")
else:
effective_max_workers = max_workers
if self._proc and self._proc.poll() is None:
if effective_proxy != self._proxy or effective_max_workers != self._max_workers:
self._logger.warning(
"aria2c process is already running; requested proxy=%r, max_workers=%r, "
"but running process will continue with proxy=%r, max_workers=%r",
effective_proxy,
effective_max_workers,
self._proxy,
self._max_workers,
)
return
self._rpc_port = get_free_port()
self._rpc_secret = get_random_bytes(16).hex()
self._rpc_uri = f"http://127.0.0.1:{self._rpc_port}/jsonrpc"
self._max_workers = effective_max_workers
self._max_concurrent_downloads = int(
config.aria2c.get("max_concurrent_downloads", effective_max_workers)
)
self._max_connection_per_server = int(config.aria2c.get("max_connection_per_server", 1))
self._split_default = int(config.aria2c.get("split", 5))
self._file_allocation = config.aria2c.get("file_allocation", "prealloc")
self._proxy = effective_proxy
args = self._build_args()
self._proc = subprocess.Popen(
[binaries.Aria2, *args], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
)
self._wait_for_rpc_ready()
@property
def rpc_uri(self) -> str:
assert self._rpc_uri
return self._rpc_uri
@property
def rpc_secret(self) -> str:
assert self._rpc_secret
return self._rpc_secret
@property
def session(self) -> Session:
return self._session
def add_uris(self, uris: list[str], options: dict[str, Any]) -> str:
"""Add a single download with multiple URIs via RPC."""
gid = rpc(
caller=partial(self._session.post, url=self.rpc_uri),
secret=self.rpc_secret,
method="aria2.addUri",
params=[uris, options],
)
return gid or ""
def get_global_stat(self) -> dict[str, Any]:
return rpc(
caller=partial(self.session.post, url=self.rpc_uri),
secret=self.rpc_secret,
method="aria2.getGlobalStat",
) or {}
def tell_status(self, gid: str) -> Optional[dict[str, Any]]:
return rpc(
caller=partial(self.session.post, url=self.rpc_uri),
secret=self.rpc_secret,
method="aria2.tellStatus",
params=[gid, ["status", "errorCode", "errorMessage", "files", "completedLength", "totalLength"]],
)
def remove(self, gid: str) -> None:
rpc(
caller=partial(self.session.post, url=self.rpc_uri),
secret=self.rpc_secret,
method="aria2.forceRemove",
params=[gid],
)
_manager = _Aria2Manager()
def download(
urls: Union[str, list[str], dict[str, Any], list[dict[str, Any]]],
output_dir: Path,
filename: str,
headers: Optional[MutableMapping[str, Union[str, bytes]]] = None,
cookies: Optional[Union[MutableMapping[str, str], CookieJar]] = None,
proxy: Optional[str] = None,
max_workers: Optional[int] = None,
) -> Generator[dict[str, Any], None, None]:
"""Enqueue downloads to the singleton aria2c instance via stdin and track per-call progress via RPC."""
debug_logger = get_debug_logger()
if not urls:
raise ValueError("urls must be provided and not empty")
elif not isinstance(urls, (str, dict, list)):
raise TypeError(f"Expected urls to be {str} or {dict} or a list of one of them, not {type(urls)}")
if not output_dir:
raise ValueError("output_dir must be provided")
elif not isinstance(output_dir, Path):
raise TypeError(f"Expected output_dir to be {Path}, not {type(output_dir)}")
if not filename:
raise ValueError("filename must be provided")
elif not isinstance(filename, str):
raise TypeError(f"Expected filename to be {str}, not {type(filename)}")
if not isinstance(headers, (MutableMapping, type(None))):
raise TypeError(f"Expected headers to be {MutableMapping}, not {type(headers)}")
if not isinstance(cookies, (MutableMapping, CookieJar, type(None))):
raise TypeError(f"Expected cookies to be {MutableMapping} or {CookieJar}, not {type(cookies)}")
if not isinstance(proxy, (str, type(None))):
raise TypeError(f"Expected proxy to be {str}, not {type(proxy)}")
if not max_workers:
max_workers = min(32, (os.cpu_count() or 1) + 4)
elif not isinstance(max_workers, int):
raise TypeError(f"Expected max_workers to be {int}, not {type(max_workers)}")
if not isinstance(urls, list):
urls = [urls]
if cookies and not isinstance(cookies, CookieJar):
cookies = cookiejar_from_dict(cookies)
_manager.ensure_started(proxy=proxy, max_workers=max_workers)
if debug_logger:
first_url = urls[0] if isinstance(urls[0], str) else urls[0].get("url", "")
url_display = first_url[:200] + "..." if len(first_url) > 200 else first_url
debug_logger.log(
level="DEBUG",
operation="downloader_aria2c_start",
message="Starting Aria2c download",
context={
"binary_path": str(binaries.Aria2),
"url_count": len(urls),
"first_url": url_display,
"output_dir": str(output_dir),
"filename": filename,
"has_proxy": bool(proxy),
},
)
# Build options for each URI and add via RPC
gids: list[str] = []
for i, url in enumerate(urls):
if isinstance(url, str):
url_data = {"url": url}
else:
url_data: dict[str, Any] = url
url_filename = filename.format(i=i, ext=get_extension(url_data["url"]))
opts: dict[str, Any] = {
"dir": str(output_dir),
"out": url_filename,
"split": str(1 if len(urls) > 1 else int(config.aria2c.get("split", 5))),
}
# Cookies as header
if cookies:
mock_request = requests.Request(url=url_data["url"])
cookie_header = get_cookie_header(cookies, mock_request)
if cookie_header:
opts.setdefault("header", []).append(f"Cookie: {cookie_header}")
# Global headers
for header, value in (headers or {}).items():
if header.lower() == "cookie":
raise ValueError("You cannot set Cookies as a header manually, please use the `cookies` param.")
if header.lower() == "accept-encoding":
continue
if header.lower() == "referer":
opts["referer"] = str(value)
continue
if header.lower() == "user-agent":
opts["user-agent"] = str(value)
continue
opts.setdefault("header", []).append(f"{header}: {value}")
# Per-url extra args
for key, value in url_data.items():
if key == "url":
continue
if key == "headers":
for header_name, header_value in value.items():
opts.setdefault("header", []).append(f"{header_name}: {header_value}")
else:
opts[key] = str(value)
# Add via RPC
gid = _manager.add_uris([url_data["url"]], opts)
if gid:
gids.append(gid)
yield dict(total=len(gids))
completed: set[str] = set()
try:
while len(completed) < len(gids):
if DOWNLOAD_CANCELLED.is_set():
# Remove tracked downloads on cancel
for gid in gids:
if gid not in completed:
_manager.remove(gid)
yield dict(downloaded="[yellow]CANCELLED")
raise KeyboardInterrupt()
stats = _manager.get_global_stat()
dl_speed = int(stats.get("downloadSpeed", -1))
# Aggregate progress across all GIDs for this call
total_completed = 0
total_size = 0
# Check each tracked GID
for gid in gids:
if gid in completed:
continue
status = _manager.tell_status(gid)
if not status:
continue
completed_length = int(status.get("completedLength", 0))
total_length = int(status.get("totalLength", 0))
total_completed += completed_length
total_size += total_length
state = status.get("status")
if state in ("complete", "error"):
completed.add(gid)
yield dict(completed=len(completed))
if state == "error":
used_uri = None
try:
used_uri = next(
uri["uri"]
for file in status.get("files", [])
for uri in file.get("uris", [])
if uri.get("status") == "used"
)
except Exception:
used_uri = "unknown"
error = f"Download Error (#{gid}): {status.get('errorMessage')} ({status.get('errorCode')}), {used_uri}"
error_pretty = "\n ".join(textwrap.wrap(error, width=console.width - 20, initial_indent=""))
console.log(Text.from_ansi("\n[Aria2c]: " + error_pretty))
if debug_logger:
debug_logger.log(
level="ERROR",
operation="downloader_aria2c_download_error",
message=f"Aria2c download failed: {status.get('errorMessage')}",
context={
"gid": gid,
"error_code": status.get("errorCode"),
"error_message": status.get("errorMessage"),
"used_uri": used_uri[:200] + "..." if used_uri and len(used_uri) > 200 else used_uri,
"completed_length": status.get("completedLength"),
"total_length": status.get("totalLength"),
},
)
raise ValueError(error)
# Yield aggregate progress for this call's downloads
progress_data = {"advance": 0}
if len(gids) > 1:
# Multi-file mode (e.g., HLS): Return the count of completed segments
progress_data["completed"] = len(completed)
progress_data["total"] = len(gids)
else:
# Single-file mode: Return the total bytes downloaded
progress_data["completed"] = total_completed
if total_size > 0:
progress_data["total"] = total_size
else:
progress_data["total"] = None
if dl_speed != -1:
progress_data["downloaded"] = f"{filesize.decimal(dl_speed)}/s"
yield progress_data
time.sleep(1)
except KeyboardInterrupt:
DOWNLOAD_CANCELLED.set()
raise
except Exception as e:
DOWNLOAD_CANCELLED.set()
yield dict(downloaded="[red]FAILED")
if debug_logger and not isinstance(e, ValueError):
debug_logger.log(
level="ERROR",
operation="downloader_aria2c_exception",
message=f"Unexpected error during Aria2c download: {e}",
error=e,
context={
"url_count": len(urls),
"output_dir": str(output_dir),
},
)
raise
def aria2c(
urls: Union[str, list[str], dict[str, Any], list[dict[str, Any]]],
output_dir: Path,
filename: str,
headers: Optional[MutableMapping[str, Union[str, bytes]]] = None,
cookies: Optional[Union[MutableMapping[str, str], CookieJar]] = None,
proxy: Optional[str] = None,
max_workers: Optional[int] = None,
) -> Generator[dict[str, Any], None, None]:
"""
Download files using Aria2(c).
https://aria2.github.io
Yields the following download status updates while chunks are downloading:
- {total: 100} (100% download total)
- {completed: 1} (1% download progress out of 100%)
- {downloaded: "10.1 MB/s"} (currently downloading at a rate of 10.1 MB/s)
The data is in the same format accepted by rich's progress.update() function.
Parameters:
urls: Web URL(s) to file(s) to download. You can use a dictionary with the key
"url" for the URI, and other keys for extra arguments to use per-URL.
output_dir: The folder to save the file into. If the save path's directory does
not exist then it will be made automatically.
filename: The filename or filename template to use for each file. The variables
you can use are `i` for the URL index and `ext` for the URL extension.
headers: A mapping of HTTP Header Key/Values to use for all downloads.
cookies: A mapping of Cookie Key/Values or a Cookie Jar to use for all downloads.
proxy: An optional proxy URI to route connections through for all downloads.
max_workers: The maximum amount of threads to use for downloads. Defaults to
min(32,(cpu_count+4)). Use for the --max-concurrent-downloads option.
"""
if proxy and not proxy.lower().startswith("http://"):
# Only HTTP proxies are supported by aria2(c)
proxy = urlparse(proxy)
port = get_free_port()
username, password = get_random_bytes(8).hex(), get_random_bytes(8).hex()
local_proxy = f"http://{username}:{password}@localhost:{port}"
scheme = {"https": "http+ssl", "socks5h": "socks"}.get(proxy.scheme, proxy.scheme)
remote_server = f"{scheme}://{proxy.hostname}"
if proxy.port:
remote_server += f":{proxy.port}"
if proxy.username or proxy.password:
remote_server += "#"
if proxy.username:
remote_server += proxy.username
if proxy.password:
remote_server += f":{proxy.password}"
p = subprocess.Popen(
["pproxy", "-l", f"http://:{port}#{username}:{password}", "-r", remote_server],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
try:
yield from download(urls, output_dir, filename, headers, cookies, local_proxy, max_workers)
finally:
p.kill()
p.wait()
return
yield from download(urls, output_dir, filename, headers, cookies, proxy, max_workers)
__all__ = ("aria2c",)
reset/packages/envied/src/envied/core/downloaders/curl_impersonate.py
+308
View File
@@ -0,0 +1,308 @@
import math
import time
from concurrent import futures
from concurrent.futures.thread import ThreadPoolExecutor
from http.cookiejar import CookieJar
from pathlib import Path
from typing import Any, Generator, MutableMapping, Optional, Union
from curl_cffi.requests import Session
from rich import filesize
from envied.core.config import config
from envied.core.constants import DOWNLOAD_CANCELLED
from envied.core.utilities import get_debug_logger, get_extension
MAX_ATTEMPTS = 5
RETRY_WAIT = 2
CHUNK_SIZE = 1024
PROGRESS_WINDOW = 5
BROWSER = config.curl_impersonate.get("browser", "chrome124")
def download(url: str, save_path: Path, session: Session, **kwargs: Any) -> Generator[dict[str, Any], None, None]:
"""
Download files using Curl Impersonate.
https://github.com/lwthiker/curl-impersonate
Yields the following download status updates while chunks are downloading:
- {total: 123} (there are 123 chunks to download)
- {total: None} (there are an unknown number of chunks to download)
- {advance: 1} (one chunk was downloaded)
- {downloaded: "10.1 MB/s"} (currently downloading at a rate of 10.1 MB/s)
- {file_downloaded: Path(...), written: 1024} (download finished, has the save path and size)
The data is in the same format accepted by rich's progress.update() function. The
`downloaded` key is custom and is not natively accepted by all rich progress bars.
Parameters:
url: Web URL of a file to download.
save_path: The path to save the file to. If the save path's directory does not
exist then it will be made automatically.
session: The Requests or Curl-Impersonate Session to make HTTP requests with.
Useful to set Header, Cookie, and Proxy data. Connections are saved and
re-used with the session so long as the server keeps the connection alive.
kwargs: Any extra keyword arguments to pass to the session.get() call. Use this
for one-time request changes like a header, cookie, or proxy. For example,
to request Byte-ranges use e.g., `headers={"Range": "bytes=0-128"}`.
"""
save_dir = save_path.parent
control_file = save_path.with_name(f"{save_path.name}.!dev")
save_dir.mkdir(parents=True, exist_ok=True)
if control_file.exists():
# consider the file corrupt if the control file exists
save_path.unlink(missing_ok=True)
control_file.unlink()
elif save_path.exists():
# if it exists, and no control file, then it should be safe
yield dict(file_downloaded=save_path, written=save_path.stat().st_size)
# TODO: Design a control file format so we know how much of the file is missing
control_file.write_bytes(b"")
attempts = 1
try:
while True:
written = 0
download_sizes = []
last_speed_refresh = time.time()
try:
stream = session.get(url, stream=True, **kwargs)
stream.raise_for_status()
try:
content_length = int(stream.headers.get("Content-Length", "0"))
# Skip Content-Length validation for compressed responses since
# curl_impersonate automatically decompresses but Content-Length shows compressed size
if stream.headers.get("Content-Encoding", "").lower() in ["gzip", "deflate", "br"]:
content_length = 0
except ValueError:
content_length = 0
if content_length > 0:
yield dict(total=math.ceil(content_length / CHUNK_SIZE))
else:
# we have no data to calculate total chunks
yield dict(total=None) # indeterminate mode
with open(save_path, "wb") as f:
for chunk in stream.iter_content(chunk_size=CHUNK_SIZE):
download_size = len(chunk)
f.write(chunk)
written += download_size
yield dict(advance=1)
now = time.time()
time_since = now - last_speed_refresh
download_sizes.append(download_size)
if time_since > PROGRESS_WINDOW or download_size < CHUNK_SIZE:
data_size = sum(download_sizes)
download_speed = math.ceil(data_size / (time_since or 1))
yield dict(downloaded=f"{filesize.decimal(download_speed)}/s")
last_speed_refresh = now
download_sizes.clear()
if content_length and written < content_length:
raise IOError(f"Failed to read {content_length} bytes from the track URI.")
yield dict(file_downloaded=save_path, written=written)
break
except Exception as e:
save_path.unlink(missing_ok=True)
if DOWNLOAD_CANCELLED.is_set() or attempts == MAX_ATTEMPTS:
raise e
time.sleep(RETRY_WAIT)
attempts += 1
finally:
control_file.unlink()
def curl_impersonate(
urls: Union[str, list[str], dict[str, Any], list[dict[str, Any]]],
output_dir: Path,
filename: str,
headers: Optional[MutableMapping[str, Union[str, bytes]]] = None,
cookies: Optional[Union[MutableMapping[str, str], CookieJar]] = None,
proxy: Optional[str] = None,
max_workers: Optional[int] = None,
) -> Generator[dict[str, Any], None, None]:
"""
Download files using Curl Impersonate.
https://github.com/lwthiker/curl-impersonate
Yields the following download status updates while chunks are downloading:
- {total: 123} (there are 123 chunks to download)
- {total: None} (there are an unknown number of chunks to download)
- {advance: 1} (one chunk was downloaded)
- {downloaded: "10.1 MB/s"} (currently downloading at a rate of 10.1 MB/s)
- {file_downloaded: Path(...), written: 1024} (download finished, has the save path and size)
The data is in the same format accepted by rich's progress.update() function.
However, The `downloaded`, `file_downloaded` and `written` keys are custom and not
natively accepted by rich progress bars.
Parameters:
urls: Web URL(s) to file(s) to download. You can use a dictionary with the key
"url" for the URI, and other keys for extra arguments to use per-URL.
output_dir: The folder to save the file into. If the save path's directory does
not exist then it will be made automatically.
filename: The filename or filename template to use for each file. The variables
you can use are `i` for the URL index and `ext` for the URL extension.
headers: A mapping of HTTP Header Key/Values to use for all downloads.
cookies: A mapping of Cookie Key/Values or a Cookie Jar to use for all downloads.
proxy: An optional proxy URI to route connections through for all downloads.
max_workers: The maximum amount of threads to use for downloads. Defaults to
min(32,(cpu_count+4)).
"""
if not urls:
raise ValueError("urls must be provided and not empty")
elif not isinstance(urls, (str, dict, list)):
raise TypeError(f"Expected urls to be {str} or {dict} or a list of one of them, not {type(urls)}")
if not output_dir:
raise ValueError("output_dir must be provided")
elif not isinstance(output_dir, Path):
raise TypeError(f"Expected output_dir to be {Path}, not {type(output_dir)}")
if not filename:
raise ValueError("filename must be provided")
elif not isinstance(filename, str):
raise TypeError(f"Expected filename to be {str}, not {type(filename)}")
if not isinstance(headers, (MutableMapping, type(None))):
raise TypeError(f"Expected headers to be {MutableMapping}, not {type(headers)}")
if not isinstance(cookies, (MutableMapping, CookieJar, type(None))):
raise TypeError(f"Expected cookies to be {MutableMapping} or {CookieJar}, not {type(cookies)}")
if not isinstance(proxy, (str, type(None))):
raise TypeError(f"Expected proxy to be {str}, not {type(proxy)}")
if not isinstance(max_workers, (int, type(None))):
raise TypeError(f"Expected max_workers to be {int}, not {type(max_workers)}")
debug_logger = get_debug_logger()
if not isinstance(urls, list):
urls = [urls]
urls = [
dict(save_path=save_path, **url) if isinstance(url, dict) else dict(url=url, save_path=save_path)
for i, url in enumerate(urls)
for save_path in [
output_dir / filename.format(i=i, ext=get_extension(url["url"] if isinstance(url, dict) else url))
]
]
session = Session(impersonate=BROWSER)
if headers:
headers = {k: v for k, v in headers.items() if k.lower() != "accept-encoding"}
session.headers.update(headers)
if cookies:
session.cookies.update(cookies)
if proxy:
session.proxies.update({"all": proxy})
if debug_logger:
first_url = urls[0].get("url", "") if urls else ""
url_display = first_url[:200] + "..." if len(first_url) > 200 else first_url
debug_logger.log(
level="DEBUG",
operation="downloader_curl_impersonate_start",
message="Starting curl_impersonate download",
context={
"url_count": len(urls),
"first_url": url_display,
"output_dir": str(output_dir),
"filename": filename,
"max_workers": max_workers,
"browser": BROWSER,
"has_proxy": bool(proxy),
},
)
yield dict(total=len(urls))
download_sizes = []
last_speed_refresh = time.time()
with ThreadPoolExecutor(max_workers=max_workers) as pool:
for i, future in enumerate(
futures.as_completed((pool.submit(download, session=session, **url) for url in urls))
):
file_path, download_size = None, None
try:
for status_update in future.result():
if status_update.get("file_downloaded") and status_update.get("written"):
file_path = status_update["file_downloaded"]
download_size = status_update["written"]
elif len(urls) == 1:
# these are per-chunk updates, only useful if it's one big file
yield status_update
except KeyboardInterrupt:
DOWNLOAD_CANCELLED.set() # skip pending track downloads
yield dict(downloaded="[yellow]CANCELLING")
pool.shutdown(wait=True, cancel_futures=True)
yield dict(downloaded="[yellow]CANCELLED")
# tell dl that it was cancelled
# the pool is already shut down, so exiting loop is fine
raise
except Exception as e:
DOWNLOAD_CANCELLED.set() # skip pending track downloads
yield dict(downloaded="[red]FAILING")
pool.shutdown(wait=True, cancel_futures=True)
yield dict(downloaded="[red]FAILED")
if debug_logger:
debug_logger.log(
level="ERROR",
operation="downloader_curl_impersonate_failed",
message=f"curl_impersonate download failed: {e}",
error=e,
context={
"url_count": len(urls),
"output_dir": str(output_dir),
"browser": BROWSER,
},
)
# tell dl that it failed
# the pool is already shut down, so exiting loop is fine
raise
else:
yield dict(file_downloaded=file_path)
yield dict(advance=1)
now = time.time()
time_since = now - last_speed_refresh
if download_size: # no size == skipped dl
download_sizes.append(download_size)
if download_sizes and (time_since > PROGRESS_WINDOW or i == len(urls)):
data_size = sum(download_sizes)
download_speed = math.ceil(data_size / (time_since or 1))
yield dict(downloaded=f"{filesize.decimal(download_speed)}/s")
last_speed_refresh = now
download_sizes.clear()
if debug_logger:
debug_logger.log(
level="DEBUG",
operation="downloader_curl_impersonate_complete",
message="curl_impersonate download completed successfully",
context={
"url_count": len(urls),
"output_dir": str(output_dir),
"filename": filename,
},
)
__all__ = ("curl_impersonate",)
reset/packages/envied/src/envied/core/downloaders/n_m3u8dl_re.py
+548
View File
@@ -0,0 +1,548 @@
import os
import re
import subprocess
import warnings
from http.cookiejar import CookieJar
from pathlib import Path
from typing import Any, Generator, MutableMapping
import requests
from requests.cookies import cookiejar_from_dict, get_cookie_header
from envied.core import binaries
from envied.core.binaries import FFMPEG, Mp4decrypt, ShakaPackager
from envied.core.config import config
from envied.core.console import console
from envied.core.constants import DOWNLOAD_CANCELLED
from envied.core.utilities import get_debug_logger
PERCENT_RE = re.compile(r"(\d+\.\d+%)")
SPEED_RE = re.compile(r"(\d+\.\d+(?:MB|KB)ps)")
SIZE_RE = re.compile(r"(\d+\.\d+(?:MB|GB|KB)/\d+\.\d+(?:MB|GB|KB))")
WARN_RE = re.compile(r"(WARN : Response.*|WARN : One or more errors occurred.*)")
ERROR_RE = re.compile(r"(\bERROR\b.*|\bFAILED\b.*|\bException\b.*)")
DECRYPTION_ENGINE = {
"shaka": "SHAKA_PACKAGER",
"mp4decrypt": "MP4DECRYPT",
}
# Ignore FutureWarnings
warnings.simplefilter(action="ignore", category=FutureWarning)
def get_track_selection_args(track: Any) -> list[str]:
"""
Generates track selection arguments for N_m3u8dl_RE.
Args:
track: A track object with attributes like descriptor, data, and class name.
Returns:
A list of strings for track selection.
Raises:
ValueError: If the manifest type is unsupported or track selection fails.
"""
descriptor = track.descriptor.name
track_type = track.__class__.__name__
def _create_args(flag: str, parts: list[str], type_str: str, extra_args: list[str] | None = None) -> list[str]:
if not parts:
raise ValueError(f"[N_m3u8DL-RE]: Unable to select {type_str} track from {descriptor} manifest")
final_args = [flag, ":".join(parts)]
if extra_args:
final_args.extend(extra_args)
return final_args
match descriptor:
case "HLS":
# HLS playlists are direct inputs; no selection arguments needed.
return []
case "DASH":
representation = track.data.get("dash", {}).get("representation", {})
adaptation_set = track.data.get("dash", {}).get("adaptation_set", {})
parts = []
if track_type == "Audio":
track_id = representation.get("id") or adaptation_set.get("audioTrackId")
lang = representation.get("lang") or adaptation_set.get("lang")
if track_id:
parts.append(rf'"id=\b{track_id}\b"')
if lang:
parts.append(f"lang={lang}")
else:
if codecs := representation.get("codecs"):
parts.append(f"codecs={codecs}")
if lang:
parts.append(f"lang={lang}")
if bw := representation.get("bandwidth"):
bitrate = int(bw) // 1000
parts.append(f"bwMin={bitrate}:bwMax={bitrate + 5}")
if roles := representation.findall("Role") + adaptation_set.findall("Role"):
if role := next((r.get("value") for r in roles if r.get("value", "").lower() == "main"), None):
parts.append(f"role={role}")
return _create_args("-sa", parts, "audio")
if track_type == "Video":
if track_id := representation.get("id"):
parts.append(rf'"id=\b{track_id}\b"')
else:
if width := representation.get("width"):
parts.append(f"res={width}*")
if codecs := representation.get("codecs"):
parts.append(f"codecs={codecs}")
if bw := representation.get("bandwidth"):
bitrate = int(bw) // 1000
parts.append(f"bwMin={bitrate}:bwMax={bitrate + 5}")
return _create_args("-sv", parts, "video")
if track_type == "Subtitle":
if track_id := representation.get("id"):
parts.append(rf'"id=\b{track_id}\b"')
else:
if lang := representation.get("lang"):
parts.append(f"lang={lang}")
return _create_args("-ss", parts, "subtitle", extra_args=["--auto-subtitle-fix", "false"])
case "ISM":
quality_level = track.data.get("ism", {}).get("quality_level", {})
stream_index = track.data.get("ism", {}).get("stream_index", {})
parts = []
if track_type == "Audio":
if name := stream_index.get("Name") or quality_level.get("Index"):
parts.append(rf'"id=\b{name}\b"')
else:
if codecs := quality_level.get("FourCC"):
parts.append(f"codecs={codecs}")
if lang := stream_index.get("Language"):
parts.append(f"lang={lang}")
if br := quality_level.get("Bitrate"):
bitrate = int(br) // 1000
parts.append(f"bwMin={bitrate}:bwMax={bitrate + 5}")
return _create_args("-sa", parts, "audio")
if track_type == "Video":
if name := stream_index.get("Name") or quality_level.get("Index"):
parts.append(rf'"id=\b{name}\b"')
else:
if width := quality_level.get("MaxWidth"):
parts.append(f"res={width}*")
if codecs := quality_level.get("FourCC"):
parts.append(f"codecs={codecs}")
if br := quality_level.get("Bitrate"):
bitrate = int(br) // 1000
parts.append(f"bwMin={bitrate}:bwMax={bitrate + 5}")
return _create_args("-sv", parts, "video")
# I've yet to encounter a subtitle track in ISM manifests, so this is mostly theoretical.
if track_type == "Subtitle":
if name := stream_index.get("Name") or quality_level.get("Index"):
parts.append(rf'"id=\b{name}\b"')
else:
if lang := stream_index.get("Language"):
parts.append(f"lang={lang}")
return _create_args("-ss", parts, "subtitle", extra_args=["--auto-subtitle-fix", "false"])
case "URL":
raise ValueError(
f"[N_m3u8DL-RE]: Direct URL downloads are not supported for {track_type} tracks. "
f"The track should use a different downloader (e.g., 'requests', 'aria2c')."
)
raise ValueError(f"[N_m3u8DL-RE]: Unsupported manifest type: {descriptor}")
def build_download_args(
track_url: str,
filename: str,
output_dir: Path,
thread_count: int,
retry_count: int,
track_from_file: Path | None,
custom_args: dict[str, Any] | None,
headers: dict[str, Any] | None,
cookies: CookieJar | None,
proxy: str | None,
content_keys: dict[str, str] | None,
ad_keyword: str | None,
skip_merge: bool | None = False,
) -> list[str]:
"""Constructs the CLI arguments for N_m3u8DL-RE."""
# Default arguments
args = {
"--save-name": filename,
"--save-dir": output_dir,
"--tmp-dir": output_dir,
"--thread-count": thread_count,
"--download-retry-count": retry_count,
}
if FFMPEG:
args["--ffmpeg-binary-path"] = str(FFMPEG)
if proxy:
args["--custom-proxy"] = proxy
if skip_merge:
args["--skip-merge"] = skip_merge
if ad_keyword:
args["--ad-keyword"] = ad_keyword
# Disable segment count validation to work around N_m3u8DL-RE's Math.Ceiling
# bug in duration-based SegmentTemplate calculation (see nilaoda/N_m3u8DL-RE#108)
args["--check-segments-count"] = False
key_args = []
if content_keys:
for kid, key in content_keys.items():
key_args.extend(["--key", f"{kid.hex}:{key.lower()}"])
decryption_config = config.decryption.lower()
engine_name = DECRYPTION_ENGINE.get(decryption_config) or "SHAKA_PACKAGER"
args["--decryption-engine"] = engine_name
binary_path = None
if engine_name == "SHAKA_PACKAGER":
if ShakaPackager:
binary_path = str(ShakaPackager)
elif engine_name == "MP4DECRYPT":
if Mp4decrypt:
binary_path = str(Mp4decrypt)
if binary_path:
args["--decryption-binary-path"] = binary_path
if custom_args:
args.update(custom_args)
command = [track_from_file or track_url]
for flag, value in args.items():
if value is True:
command.append(flag)
elif value is False:
command.extend([flag, "false"])
elif value is not False and value is not None:
command.extend([flag, str(value)])
# Append all content keys (multiple --key flags supported by N_m3u8DL-RE)
command.extend(key_args)
if headers:
for key, value in headers.items():
if key.lower() not in ("accept-encoding", "cookie"):
command.extend(["--header", f"{key}: {value}"])
if cookies:
req = requests.Request(method="GET", url=track_url)
cookie_header = get_cookie_header(cookies, req)
command.extend(["--header", f"Cookie: {cookie_header}"])
return command
def download(
urls: str | dict[str, Any] | list[str | dict[str, Any]],
track: Any,
output_dir: Path,
filename: str,
headers: MutableMapping[str, str | bytes] | None,
cookies: MutableMapping[str, str] | CookieJar | None,
proxy: str | None,
max_workers: int | None,
content_keys: dict[str, Any] | None,
skip_merge: bool | None = False,
) -> Generator[dict[str, Any], None, None]:
debug_logger = get_debug_logger()
if not urls:
raise ValueError("urls must be provided and not empty")
if not isinstance(urls, (str, dict, list)):
raise TypeError(f"Expected urls to be str, dict, or list, not {type(urls)}")
if not isinstance(output_dir, Path):
raise TypeError(f"Expected output_dir to be Path, not {type(output_dir)}")
if not isinstance(filename, str) or not filename:
raise ValueError("filename must be a non-empty string")
if not isinstance(headers, (MutableMapping, type(None))):
raise TypeError(f"Expected headers to be a mapping or None, not {type(headers)}")
if not isinstance(cookies, (MutableMapping, CookieJar, type(None))):
raise TypeError(f"Expected cookies to be a mapping, CookieJar, or None, not {type(cookies)}")
if not isinstance(proxy, (str, type(None))):
raise TypeError(f"Expected proxy to be a str or None, not {type(proxy)}")
if not isinstance(max_workers, (int, type(None))):
raise TypeError(f"Expected max_workers to be an int or None, not {type(max_workers)}")
if not isinstance(content_keys, (dict, type(None))):
raise TypeError(f"Expected content_keys to be a dict or None, not {type(content_keys)}")
if not isinstance(skip_merge, (bool, type(None))):
raise TypeError(f"Expected skip_merge to be a bool or None, not {type(skip_merge)}")
if cookies and not isinstance(cookies, CookieJar):
cookies = cookiejar_from_dict(cookies)
if not binaries.N_m3u8DL_RE:
raise EnvironmentError("N_m3u8DL-RE executable not found...")
effective_max_workers = max_workers or min(32, (os.cpu_count() or 1) + 4)
if proxy and not config.n_m3u8dl_re.get("use_proxy", True):
proxy = None
thread_count = config.n_m3u8dl_re.get("thread_count", effective_max_workers)
retry_count = config.n_m3u8dl_re.get("retry_count", 10)
ad_keyword = config.n_m3u8dl_re.get("ad_keyword")
arguments = build_download_args(
track_url=track.url,
track_from_file=track.from_file,
filename=filename,
output_dir=output_dir,
thread_count=thread_count,
retry_count=retry_count,
custom_args=track.downloader_args,
headers=headers,
cookies=cookies,
proxy=proxy,
content_keys=content_keys,
skip_merge=skip_merge,
ad_keyword=ad_keyword,
)
selection_args = get_track_selection_args(track)
arguments.extend(selection_args)
log_file_path: Path | None = None
if debug_logger:
log_file_path = output_dir / f".n_m3u8dl_re_{filename}.log"
arguments.extend([
"--log-file-path", str(log_file_path),
"--log-level", "DEBUG",
])
track_url_display = track.url[:200] + "..." if len(track.url) > 200 else track.url
debug_logger.log(
level="DEBUG",
operation="downloader_n_m3u8dl_re_start",
message="Starting N_m3u8DL-RE download",
context={
"binary_path": str(binaries.N_m3u8DL_RE),
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"track_url": track_url_display,
"output_dir": str(output_dir),
"filename": filename,
"thread_count": thread_count,
"retry_count": retry_count,
"has_content_keys": bool(content_keys),
"content_key_count": len(content_keys) if content_keys else 0,
"has_proxy": bool(proxy),
"skip_merge": skip_merge,
"has_custom_args": bool(track.downloader_args),
"selection_args": selection_args,
"descriptor": track.descriptor.name if hasattr(track, "descriptor") else None,
},
)
else:
arguments.extend(["--no-log", "true"])
yield {"total": 100}
yield {"downloaded": "Parsing streams..."}
try:
with subprocess.Popen(
[binaries.N_m3u8DL_RE, *arguments],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
encoding="utf-8",
) as process:
last_line = ""
track_type = track.__class__.__name__
for line in process.stdout:
output = line.strip()
if not output:
continue
last_line = output
if warn_match := WARN_RE.search(output):
console.log(f"{track_type} {warn_match.group(1)}")
continue
if speed_match := SPEED_RE.search(output):
size = size_match.group(1) if (size_match := SIZE_RE.search(output)) else ""
yield {"downloaded": f"{speed_match.group(1)} {size}"}
if percent_match := PERCENT_RE.search(output):
progress = int(percent_match.group(1).split(".", 1)[0])
yield {"completed": progress} if progress < 100 else {"downloaded": "Merging"}
process.wait()
if process.returncode != 0:
if debug_logger and log_file_path:
log_contents = ""
if log_file_path.exists():
try:
log_contents = log_file_path.read_text(encoding="utf-8", errors="replace")
except Exception:
log_contents = "<failed to read log file>"
debug_logger.log(
level="ERROR",
operation="downloader_n_m3u8dl_re_failed",
message=f"N_m3u8DL-RE exited with code {process.returncode}",
context={
"returncode": process.returncode,
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"last_line": last_line,
"log_file_contents": log_contents,
},
)
if error_match := ERROR_RE.search(last_line):
raise ValueError(f"[N_m3u8DL-RE]: {error_match.group(1)}")
raise subprocess.CalledProcessError(process.returncode, arguments)
if debug_logger:
output_dir_exists = output_dir.exists()
output_files = []
if output_dir_exists:
try:
output_files = [f.name for f in output_dir.iterdir() if f.is_file()][:20]
except Exception:
output_files = ["<error listing files>"]
debug_logger.log(
level="DEBUG",
operation="downloader_n_m3u8dl_re_complete",
message="N_m3u8DL-RE download completed successfully",
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"output_dir": str(output_dir),
"output_dir_exists": output_dir_exists,
"output_files_count": len(output_files),
"output_files": output_files,
"filename": filename,
},
)
# Warn if no output was produced - include N_m3u8DL-RE's logs for diagnosis
if not output_dir_exists or not output_files:
# Read N_m3u8DL-RE's log file for debugging
n_m3u8dl_log = ""
if log_file_path and log_file_path.exists():
try:
n_m3u8dl_log = log_file_path.read_text(encoding="utf-8", errors="replace")
except Exception:
n_m3u8dl_log = "<failed to read log file>"
debug_logger.log(
level="WARNING",
operation="downloader_n_m3u8dl_re_no_output",
message="N_m3u8DL-RE exited successfully but produced no output files",
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"output_dir": str(output_dir),
"output_dir_exists": output_dir_exists,
"selection_args": selection_args,
"track_url": track.url[:200] + "..." if len(track.url) > 200 else track.url,
"n_m3u8dl_re_log": n_m3u8dl_log,
},
)
except ConnectionResetError:
# interrupted while passing URI to download
raise KeyboardInterrupt()
except KeyboardInterrupt:
DOWNLOAD_CANCELLED.set() # skip pending track downloads
yield {"downloaded": "[yellow]CANCELLED"}
raise
except Exception as e:
DOWNLOAD_CANCELLED.set() # skip pending track downloads
yield {"downloaded": "[red]FAILED"}
if debug_logger and log_file_path and not isinstance(e, (subprocess.CalledProcessError, ValueError)):
log_contents = ""
if log_file_path.exists():
try:
log_contents = log_file_path.read_text(encoding="utf-8", errors="replace")
except Exception:
log_contents = "<failed to read log file>"
debug_logger.log(
level="ERROR",
operation="downloader_n_m3u8dl_re_exception",
message=f"Unexpected error during N_m3u8DL-RE download: {e}",
error=e,
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"log_file_contents": log_contents,
},
)
raise
finally:
# Clean up temporary debug files
if log_file_path and log_file_path.exists():
try:
log_file_path.unlink()
except Exception:
pass
def n_m3u8dl_re(
urls: str | list[str] | dict[str, Any] | list[dict[str, Any]],
track: Any,
output_dir: Path,
filename: str,
headers: MutableMapping[str, str | bytes] | None = None,
cookies: MutableMapping[str, str] | CookieJar | None = None,
proxy: str | None = None,
max_workers: int | None = None,
content_keys: dict[str, Any] | None = None,
skip_merge: bool | None = False,
) -> Generator[dict[str, Any], None, None]:
"""
Download files using N_m3u8DL-RE.
https://github.com/nilaoda/N_m3u8DL-RE
Yields the following download status updates while chunks are downloading:
- {total: 100} (100% download total)
- {completed: 1} (1% download progress out of 100%)
- {downloaded: "10.1 MB/s"} (currently downloading at a rate of 10.1 MB/s)
The data is in the same format accepted by rich's progress.update() function.
Parameters:
urls: Web URL(s) to file(s) to download. NOTE: This parameter is ignored for now.
track: The track to download. Used to get track attributes for the selection
process. Note that Track.Descriptor.URL is not supported by N_m3u8DL-RE.
output_dir: The folder to save the file into. If the save path's directory does
not exist then it will be made automatically.
filename: The filename or filename template to use for each file.
headers: A mapping of HTTP Header Key/Values to use for all downloads.
cookies: A mapping of Cookie Key/Values or a Cookie Jar to use for all downloads.
proxy: A proxy to use for all downloads.
max_workers: The maximum amount of threads to use for downloads. Defaults to
min(32,(cpu_count+4)). Can be set in config with --thread-count option.
content_keys: The content keys to use for decryption.
skip_merge: Whether to skip merging the downloaded chunks.
"""
yield from download(
urls=urls,
track=track,
output_dir=output_dir,
filename=filename,
headers=headers,
cookies=cookies,
proxy=proxy,
max_workers=max_workers,
content_keys=content_keys,
skip_merge=skip_merge,
)
__all__ = ("n_m3u8dl_re",)
reset/packages/envied/src/envied/core/downloaders/requests.py
+663
View File
@@ -0,0 +1,663 @@
import math
import os
import time
from concurrent.futures import FIRST_COMPLETED, wait
from concurrent.futures.thread import ThreadPoolExecutor
from http.cookiejar import CookieJar
from pathlib import Path
from queue import Empty, Queue
from typing import Any, Generator, MutableMapping, Optional, Union
from requests import Session
from requests.adapters import HTTPAdapter
from rich import filesize
from envied.core.constants import DOWNLOAD_CANCELLED
from envied.core.utilities import get_debug_logger, get_extension
MAX_ATTEMPTS = 5
RETRY_WAIT = 2
PROGRESS_WINDOW = 2
# Adaptive chunk sizing — benchmarked optimal range
MIN_CHUNK = 524_288 # 512KB
MAX_CHUNK = 4_194_304 # 4MB
DEFAULT_CHUNK = 524_288 # 512KB
SPEED_ROLLING_WINDOW = 10 # seconds of history to keep for speed calculation
RANGE_PARALLEL_MIN_SIZE = 64 * 1024 * 1024
RANGE_PARALLEL_PART_SIZE = 16 * 1024 * 1024
def _adaptive_chunk_size(content_length: int) -> int:
"""Pick chunk size based on content length. Benchmarked sweet spot: 512KB-4MB."""
if content_length <= 0:
return DEFAULT_CHUNK
return min(MAX_CHUNK, max(MIN_CHUNK, content_length // 4))
def _is_requests_session(session: Any) -> bool:
"""Check if the session is a standard requests.Session (supports resp.raw)."""
return isinstance(session, Session)
def _is_rnet_session(session: Any) -> bool:
"""Check if the session is an RnetSession (uses resp.stream())."""
from envied.core.session import RnetSession
return isinstance(session, RnetSession)
def _probe_ranged(url: str, session: Any, **kwargs: Any) -> tuple[int, bool]:
headers = {**(kwargs.get("headers") or {}), "Range": "bytes=0-0"}
rest = {k: v for k, v in kwargs.items() if k != "headers"}
try:
resp = session.get(url, stream=True, headers=headers, **rest)
except Exception:
return 0, False
try:
if resp.status_code != 206:
return 0, False
ce = (resp.headers.get("Content-Encoding") or resp.headers.get("content-encoding") or "").lower()
if ce in ("gzip", "deflate", "br"):
return 0, False
content_range = resp.headers.get("Content-Range") or resp.headers.get("content-range") or ""
total = content_range.rsplit("/", 1)[-1].strip()
return (int(total), True) if total.isdigit() else (0, False)
finally:
try:
resp.close()
except Exception:
pass
def _dispatch_parts(
url: str,
save_path: Path,
session: Any,
total_size: int,
max_workers: int,
**kwargs: Any,
) -> Generator[dict[str, Any], None, None]:
save_path.parent.mkdir(parents=True, exist_ok=True)
control_file = save_path.with_name(f"{save_path.name}.!dev")
n_parts = max(1, min(max_workers, math.ceil(total_size / RANGE_PARALLEL_PART_SIZE)))
part_size = math.ceil(total_size / n_parts)
parts = [
(s, e)
for i in range(n_parts)
for s, e in [(i * part_size, min(total_size - 1, (i + 1) * part_size - 1))]
if s <= e
]
control_file.write_bytes(b"")
with open(save_path, "wb") as f:
f.truncate(total_size)
events: Queue[dict[str, Any]] = Queue()
def _worker(start: int, end: int) -> None:
for ev in download(
url=url,
save_path=save_path,
session=session,
part_offset=start,
part_end=end,
**kwargs,
):
events.put(ev)
pool = ThreadPoolExecutor(max_workers=len(parts))
futures = [pool.submit(_worker, s, e) for s, e in parts]
pending = set(futures)
yield {"total": total_size}
total_bytes = 0
start_time = last_report = time.time()
completed = False
worker_error = False
try:
while pending:
advance = 0
while not events.empty():
try:
ev = events.get_nowait()
except Empty:
break
a = ev.get("advance")
if a:
advance += a
if advance:
total_bytes += advance
yield {"advance": advance}
now = time.time()
if now - last_report > 0.5 and total_bytes > 0:
yield {"downloaded": f"{filesize.decimal(math.ceil(total_bytes / (now - start_time)))}/s"}
last_report = now
done, pending = wait(pending, timeout=0.1, return_when=FIRST_COMPLETED)
for fut in done:
exc = fut.exception()
if exc:
worker_error = True
DOWNLOAD_CANCELLED.set()
raise exc
advance = 0
while not events.empty():
try:
ev = events.get_nowait()
except Empty:
break
a = ev.get("advance")
if a:
advance += a
if advance:
total_bytes += advance
yield {"advance": advance}
yield {"file_downloaded": save_path, "written": total_size}
completed = True
except KeyboardInterrupt:
DOWNLOAD_CANCELLED.set()
pool.shutdown(wait=False, cancel_futures=True)
raise
finally:
pool.shutdown(wait=worker_error, cancel_futures=True)
if completed:
control_file.unlink(missing_ok=True)
def download(
url: str,
save_path: Path,
session: Optional[Any] = None,
segmented: bool = False,
part_offset: Optional[int] = None,
part_end: Optional[int] = None,
**kwargs: Any,
) -> Generator[dict[str, Any], None, None]:
"""
Download a file with optimized I/O.
Supports both requests.Session and RnetSession for TLS fingerprinting.
Uses raw socket reads for requests.Session and native rnet streaming for RnetSession.
Yields the following download status updates while chunks are downloading:
- {total: 123} (there are 123 chunks to download)
- {total: None} (there are an unknown number of chunks to download)
- {advance: 1} (one chunk was downloaded)
- {downloaded: "10.1 MB/s"} (currently downloading at a rate of 10.1 MB/s)
- {file_downloaded: Path(...), written: 1024} (download finished, has the save path and size)
Parameters:
url: Web URL of a file to download.
save_path: The path to save the file to. If the save path's directory does not
exist then it will be made automatically.
session: A requests.Session or RnetSession to make HTTP requests with.
RnetSession preserves TLS fingerprinting for services that need it.
segmented: If downloads are segments or parts of one bigger file.
part_offset: Byte offset to write at within a pre-allocated file. When set
(with `part_end`), enables part mode for parallel ranged downloads —
no truncate, no skip-if-exists, no control file; emits only `advance`
events; retries resume mid-part via Range.
part_end: Inclusive end byte of the part. Required when `part_offset` is set.
kwargs: Any extra keyword arguments to pass to the session.get() call. Use this
for one-time request changes like a header, cookie, or proxy. For example,
to request Byte-ranges use e.g., `headers={"Range": "bytes=0-128"}`.
"""
session = session or Session()
part_mode = part_offset is not None and part_end is not None
save_dir = save_path.parent
control_file = save_path.with_name(f"{save_path.name}.!dev")
save_dir.mkdir(parents=True, exist_ok=True)
resume_offset = 0
if not part_mode:
if control_file.exists() and save_path.exists():
resume_offset = save_path.stat().st_size
elif control_file.exists():
control_file.unlink()
elif save_path.exists():
yield dict(file_downloaded=save_path, written=save_path.stat().st_size)
return
control_file.write_bytes(b"")
_time = time.time
use_raw = _is_requests_session(session)
attempts = 1
completed = False
written = 0
try:
while True:
if not part_mode:
written = 0
last_speed_refresh = _time()
try:
use_rnet = _is_rnet_session(session)
request_kwargs = dict(kwargs)
if part_mode:
req_headers = dict(request_kwargs.get("headers", {}) or {})
req_headers["Range"] = f"bytes={part_offset + written}-{part_end}"
request_kwargs["headers"] = req_headers
elif resume_offset > 0:
req_headers = dict(request_kwargs.get("headers", {}) or {})
req_headers["Range"] = f"bytes={resume_offset}-"
request_kwargs["headers"] = req_headers
stream = session.get(url, stream=True, **request_kwargs)
stream.raise_for_status()
resumed = (not part_mode) and resume_offset > 0 and stream.status_code == 206
if (not part_mode) and resume_offset > 0 and not resumed:
resume_offset = 0
if part_mode and stream.status_code != 206:
raise IOError(f"expected 206 for ranged part, got {stream.status_code}")
if use_rnet:
content_length = stream.content_length or 0
else:
try:
content_length = int(stream.headers.get("Content-Length", "0"))
if stream.headers.get("Content-Encoding", "").lower() in ["gzip", "deflate", "br"]:
content_length = 0
except ValueError:
content_length = 0
chunk_size = _adaptive_chunk_size(content_length)
total_size = (resume_offset + content_length) if resumed and content_length > 0 else content_length
if not segmented and not part_mode:
if total_size > 0:
yield dict(total=total_size)
else:
yield dict(total=None)
if resumed and resume_offset > 0:
yield dict(advance=resume_offset)
if part_mode:
file_mode = "r+b"
file_buffering = 0
else:
file_mode = "ab" if resumed else "wb"
file_buffering = 1_048_576
with open(save_path, file_mode, buffering=file_buffering) as f:
if part_mode:
f.seek(part_offset + written)
elif not resumed and content_length > 0:
f.truncate(content_length)
f.seek(0)
_write = f.write
if use_rnet:
chunks = stream.stream()
elif use_raw:
chunks = iter(lambda: stream.raw.read(chunk_size), b"")
else:
chunks = stream.iter_content(chunk_size=chunk_size)
_data_accumulated = 0
_bytes_since_yield = 0
emit_progress = (not segmented) or part_mode
for chunk in chunks:
if DOWNLOAD_CANCELLED.is_set():
break
_write(chunk)
download_size = len(chunk)
written += download_size
if emit_progress:
_bytes_since_yield += download_size
_data_accumulated += download_size
now = _time()
time_since = now - last_speed_refresh
if time_since > PROGRESS_WINDOW:
yield dict(advance=_bytes_since_yield)
_bytes_since_yield = 0
if not part_mode:
download_speed = math.ceil(_data_accumulated / (time_since or 1))
yield dict(downloaded=f"{filesize.decimal(download_speed)}/s")
last_speed_refresh = now
_data_accumulated = 0
if emit_progress and _bytes_since_yield > 0:
yield dict(advance=_bytes_since_yield)
try:
stream.close()
except Exception:
pass
if not part_mode and not resumed and content_length > 0 and written != content_length:
f.truncate(written)
if part_mode:
expected = part_end - part_offset + 1
if written < expected:
raise IOError(f"Failed to read part {part_offset}-{part_end}: got {written}/{expected}")
elif not segmented and content_length and written < content_length:
raise IOError(f"Failed to read {content_length} bytes from the track URI.")
if not part_mode:
yield dict(file_downloaded=save_path, written=resume_offset + written)
if segmented:
yield dict(advance=1)
completed = True
break
except Exception:
try:
stream.close()
except Exception:
pass
if DOWNLOAD_CANCELLED.is_set() or attempts == MAX_ATTEMPTS:
if part_mode and not DOWNLOAD_CANCELLED.is_set():
raise
return
if not part_mode and save_path.exists():
resume_offset = save_path.stat().st_size
time.sleep(RETRY_WAIT)
attempts += 1
finally:
if completed and not part_mode:
control_file.unlink(missing_ok=True)
def requests(
urls: Union[str, list[str], dict[str, Any], list[dict[str, Any]]],
output_dir: Path,
filename: str,
headers: Optional[MutableMapping[str, Union[str, bytes]]] = None,
cookies: Optional[Union[MutableMapping[str, str], CookieJar]] = None,
proxy: Optional[str] = None,
max_workers: Optional[int] = None,
session: Optional[Any] = None,
) -> Generator[dict[str, Any], None, None]:
"""
Download files with optimized I/O and adaptive chunk sizing.
Supports both requests.Session and RnetSession. When a RnetSession is
provided (e.g. from a service's get_session()), TLS fingerprinting is preserved
on all segment downloads.
Yields the following download status updates while chunks are downloading:
- {total: 123} (there are 123 chunks to download)
- {total: None} (there are an unknown number of chunks to download)
- {advance: 1} (one chunk was downloaded)
- {downloaded: "10.1 MB/s"} (currently downloading at a rate of 10.1 MB/s)
- {file_downloaded: Path(...), written: 1024} (download finished, has the save path and size)
The data is in the same format accepted by rich's progress.update() function.
However, The `downloaded`, `file_downloaded` and `written` keys are custom and not
natively accepted by rich progress bars.
Parameters:
urls: Web URL(s) to file(s) to download. You can use a dictionary with the key
"url" for the URI, and other keys for extra arguments to use per-URL.
output_dir: The folder to save the file into. If the save path's directory does
not exist then it will be made automatically.
filename: The filename or filename template to use for each file. The variables
you can use are `i` for the URL index and `ext` for the URL extension.
headers: A mapping of HTTP Header Key/Values to use for all downloads.
cookies: A mapping of Cookie Key/Values or a Cookie Jar to use for all downloads.
proxy: An optional proxy URI to route connections through for all downloads.
max_workers: The maximum amount of threads to use for downloads. Defaults to
min(12,(cpu_count+4)).
session: An optional requests.Session or RnetSession to use. If provided,
it will be used directly (preserving TLS fingerprinting). If None, a new
requests.Session with HTTPAdapter connection pooling will be created.
"""
if not urls:
raise ValueError("urls must be provided and not empty")
elif not isinstance(urls, (str, dict, list)):
raise TypeError(f"Expected urls to be {str} or {dict} or a list of one of them, not {type(urls)}")
if not output_dir:
raise ValueError("output_dir must be provided")
elif not isinstance(output_dir, Path):
raise TypeError(f"Expected output_dir to be {Path}, not {type(output_dir)}")
if not filename:
raise ValueError("filename must be provided")
elif not isinstance(filename, str):
raise TypeError(f"Expected filename to be {str}, not {type(filename)}")
if not isinstance(headers, (MutableMapping, type(None))):
raise TypeError(f"Expected headers to be {MutableMapping}, not {type(headers)}")
if not isinstance(cookies, (MutableMapping, CookieJar, type(None))):
raise TypeError(f"Expected cookies to be {MutableMapping} or {CookieJar}, not {type(cookies)}")
if not isinstance(proxy, (str, type(None))):
raise TypeError(f"Expected proxy to be {str}, not {type(proxy)}")
if not isinstance(max_workers, (int, type(None))):
raise TypeError(f"Expected max_workers to be {int}, not {type(max_workers)}")
debug_logger = get_debug_logger()
if not isinstance(urls, list):
urls = [urls]
if not max_workers:
max_workers = min(16, (os.cpu_count() or 1) + 4)
urls = [
dict(save_path=save_path, **url) if isinstance(url, dict) else dict(url=url, save_path=save_path)
for i, url in enumerate(urls)
for save_path in [
output_dir / filename.format(i=i, ext=get_extension(url["url"] if isinstance(url, dict) else url))
]
]
# Use provided session or create a new optimized requests.Session
# When a session is provided (e.g., service's RnetSession), don't mutate headers/cookies/proxy —
# they're already set and the session may be shared across tracks.
if session is None:
session = Session()
if headers:
headers = {k: v for k, v in headers.items() if k.lower() != "accept-encoding"}
session.headers.update(headers)
if cookies:
session.cookies.update(cookies)
if proxy:
session.proxies.update({"all": proxy})
# Mount HTTPAdapter with connection pooling sized to worker count.
# Safe to do on any requests.Session — improves connection reuse for parallel downloads.
if _is_requests_session(session):
adapter = HTTPAdapter(pool_connections=max_workers, pool_maxsize=max_workers, pool_block=True)
session.mount("https://", adapter)
session.mount("http://", adapter)
if debug_logger:
first_url = urls[0].get("url", "") if urls else ""
url_display = first_url[:200] + "..." if len(first_url) > 200 else first_url
debug_logger.log(
level="DEBUG",
operation="downloader_start",
message="Starting download",
context={
"url_count": len(urls),
"first_url": url_display,
"output_dir": str(output_dir),
"filename": filename,
"max_workers": max_workers,
"has_proxy": bool(proxy),
"session_type": type(session).__name__,
},
)
segmented_batch = len(urls) > 1
if len(urls) == 1:
url_item = urls[0]
try:
ranged_used = False
if max_workers > 1:
total_size, supports_ranges = _probe_ranged(url_item["url"], session)
if supports_ranges and total_size >= RANGE_PARALLEL_MIN_SIZE:
try:
yield from _dispatch_parts(
session=session,
total_size=total_size,
max_workers=max_workers,
**url_item,
)
ranged_used = True
except KeyboardInterrupt:
raise
except Exception:
save_path = url_item.get("save_path")
if save_path:
sp = Path(save_path)
for target in (sp, sp.with_name(f"{sp.name}.!dev")):
try:
target.unlink(missing_ok=True)
except OSError:
pass
if not ranged_used:
yield from download(
session=session,
segmented=segmented_batch,
**url_item,
)
except KeyboardInterrupt:
DOWNLOAD_CANCELLED.set()
yield dict(downloaded="[yellow]CANCELLED")
raise
else:
# Segmented download with thread pool
# Speed is tracked here on the main thread, not in workers
total_bytes = 0
start_time = time.time()
last_speed_report = start_time
pool = ThreadPoolExecutor(max_workers=max_workers)
event_queue: Queue[dict[str, Any]] = Queue()
def _download_worker(url_item: dict[str, Any]) -> None:
for event in download(
session=session,
segmented=segmented_batch,
**url_item,
):
event_queue.put(event)
futures = [pool.submit(_download_worker, url) for url in urls]
pending = set(futures)
pending_advance = 0
try:
while pending:
# Drain queued events — batch advances, track bytes for speed
while True:
try:
event = event_queue.get_nowait()
except Empty:
break
# Accumulate advance events for batched yield
advance = event.get("advance")
if advance:
pending_advance += advance
continue
# Track bytes from completed segments for speed calculation
written = event.get("written")
if written:
total_bytes += written
# Pass through other events (file_downloaded, total, etc.)
yield event
# Yield batched advances every drain cycle for responsive progress bar
if pending_advance > 0:
yield dict(advance=pending_advance)
pending_advance = 0
# Yield speed every 0.5s (throttled to avoid spamming Rich)
now = time.time()
if now - last_speed_report > 0.5 and total_bytes > 0:
elapsed = now - start_time
if elapsed > 0:
download_speed = math.ceil(total_bytes / elapsed)
yield dict(downloaded=f"{filesize.decimal(download_speed)}/s")
last_speed_report = now
# Wait efficiently for next future completion (OS condition variable)
completed, pending = wait(pending, timeout=0.1, return_when=FIRST_COMPLETED)
for future in completed:
exc = future.exception()
if isinstance(exc, KeyboardInterrupt):
raise KeyboardInterrupt()
elif exc:
DOWNLOAD_CANCELLED.set()
yield dict(downloaded="[red]FAILING")
pool.shutdown(wait=False, cancel_futures=True)
yield dict(downloaded="[red]FAILED")
if debug_logger:
debug_logger.log(
level="ERROR",
operation="downloader_failed",
message=f"Download failed: {exc}",
error=exc,
context={
"url_count": len(urls),
"output_dir": str(output_dir),
},
)
raise exc
except KeyboardInterrupt:
DOWNLOAD_CANCELLED.set()
yield dict(downloaded="[yellow]CANCELLING")
pool.shutdown(wait=False, cancel_futures=True)
yield dict(downloaded="[yellow]CANCELLED")
raise
finally:
pool.shutdown(wait=False, cancel_futures=True)
# Drain remaining events
while True:
try:
event = event_queue.get_nowait()
except Empty:
break
advance = event.get("advance")
if advance:
pending_advance += advance
continue
written = event.get("written")
if written:
total_bytes += written
yield event
# Flush remaining advances and final speed
if pending_advance > 0:
yield dict(advance=pending_advance)
elapsed = time.time() - start_time
if elapsed > 0 and total_bytes > 0:
download_speed = math.ceil(total_bytes / elapsed)
yield dict(downloaded=f"{filesize.decimal(download_speed)}/s")
if debug_logger:
debug_logger.log(
level="DEBUG",
operation="downloader_complete",
message="Download completed successfully",
context={
"url_count": len(urls),
"output_dir": str(output_dir),
"filename": filename,
},
)
__all__ = ("requests",)
reset/packages/envied/src/envied/core/drm/__init__.py
+44
View File
@@ -0,0 +1,44 @@
import base64
from typing import Any, Union
from uuid import UUID
from envied.core.drm.clearkey import ClearKey
from envied.core.drm.monalisa import MonaLisa
from envied.core.drm.playready import PlayReady
from envied.core.drm.widevine import Widevine
DRM_T = Union[ClearKey, Widevine, PlayReady, MonaLisa]
def drm_from_dict(data: dict[str, Any]) -> Union[Widevine, PlayReady]:
"""Reconstruct a Widevine/PlayReady DRM instance from its ``to_dict()`` form.
Rebuilds the PSSH from the stored base64 and re-injects any saved content keys
so the resulting object can decrypt without contacting a license server.
"""
system = data.get("system")
pssh_b64 = data.get("pssh_b64")
kids = data.get("kids") or []
content_keys = data.get("content_keys") or {}
if not pssh_b64:
raise ValueError("Cannot reconstruct DRM without a stored PSSH.")
if system == "PlayReady":
from pyplayready.system.pssh import PSSH as PlayReadyPSSH
drm: Union[Widevine, PlayReady] = PlayReady(pssh=PlayReadyPSSH(base64.b64decode(pssh_b64)), pssh_b64=pssh_b64)
elif system == "Widevine":
from pywidevine.pssh import PSSH as WidevinePSSH
drm = Widevine(pssh=WidevinePSSH(pssh_b64), kid=kids[0] if kids else None)
else:
raise ValueError(f"Unsupported DRM system for reconstruction: {system!r}")
for kid_hex, key in content_keys.items():
drm.content_keys[UUID(hex=kid_hex)] = key
return drm
__all__ = ("ClearKey", "Widevine", "PlayReady", "MonaLisa", "DRM_T", "drm_from_dict")
reset/packages/envied/src/envied/core/drm/clearkey.py
+113
View File
@@ -0,0 +1,113 @@
from __future__ import annotations
import base64
import shutil
from pathlib import Path
from typing import Optional, Union
from urllib.parse import urljoin
from Cryptodome.Cipher import AES
from Cryptodome.Util.Padding import unpad
from m3u8.model import Key
from requests import Session
from envied.core.session import RnetSession
class ClearKey:
"""AES Clear Key DRM System."""
def __init__(self, key: Union[bytes, str], iv: Optional[Union[bytes, str]] = None):
"""
Generally IV should be provided where possible. If not provided, it will be
set to \x00 of the same bit-size of the key.
"""
if isinstance(key, str):
key = bytes.fromhex(key.replace("0x", ""))
if not isinstance(key, bytes):
raise ValueError(f"Expected AES Key to be bytes, not {key!r}")
if not iv:
iv = b"\x00"
if isinstance(iv, str):
iv = bytes.fromhex(iv.replace("0x", ""))
if not isinstance(iv, bytes):
raise ValueError(f"Expected IV to be bytes, not {iv!r}")
if len(iv) < len(key):
iv = iv * (len(key) - len(iv) + 1)
self.key: bytes = key
self.iv: bytes = iv
def decrypt(self, path: Path) -> None:
"""Decrypt a Track with AES Clear Key DRM."""
if not path or not path.exists():
raise ValueError("Tried to decrypt a file that does not exist.")
decrypted = AES.new(self.key, AES.MODE_CBC, self.iv).decrypt(path.read_bytes())
try:
decrypted = unpad(decrypted, AES.block_size)
except ValueError:
# the decrypted data is likely already in the block size boundary
pass
decrypted_path = path.with_suffix(f".decrypted{path.suffix}")
decrypted_path.write_bytes(decrypted)
path.unlink()
shutil.move(decrypted_path, path)
@classmethod
def from_m3u_key(cls, m3u_key: Key, session: Optional[Session] = None) -> ClearKey:
"""
Load a ClearKey from an M3U(8) Playlist's EXT-X-KEY.
Parameters:
m3u_key: A Key object parsed from a m3u(8) playlist using
the `m3u8` library.
session: Optional session used to request external URIs with.
Useful to set headers, proxies, cookies, and so forth.
"""
if not isinstance(m3u_key, Key):
raise ValueError(f"Provided M3U Key is in an unexpected type {m3u_key!r}")
if not isinstance(session, (Session, RnetSession, type(None))):
raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not a {type(session)}")
if not m3u_key.method.startswith("AES"):
raise ValueError(f"Provided M3U Key is not an AES Clear Key, {m3u_key.method}")
if not m3u_key.uri:
raise ValueError("No URI in M3U Key, unable to get Key.")
if not session:
session = Session()
if not session.headers.get("User-Agent"):
# commonly needed default for HLS playlists
session.headers["User-Agent"] = "smartexoplayer/1.1.0 (Linux;Android 8.0.0) ExoPlayerLib/2.13.3"
if m3u_key.uri.startswith("data:"):
media_types, data = m3u_key.uri[5:].split(",")
media_types = media_types.split(";")
if "base64" in media_types:
data = base64.b64decode(data)
key = data
else:
url = urljoin(m3u_key.base_uri, m3u_key.uri)
res = session.get(url)
res.raise_for_status()
if not res.content:
raise EOFError("Unexpected Empty Response by M3U Key URI.")
if len(res.content) < 16:
raise EOFError(f"Unexpected Length of Key ({len(res.content)} bytes) in M3U Key.")
key = res.content
if m3u_key.iv:
iv = bytes.fromhex(m3u_key.iv.replace("0x", ""))
else:
iv = None
return cls(key=key, iv=iv)
__all__ = ("ClearKey",)
reset/packages/envied/src/envied/core/drm/monalisa.py
+299
View File
@@ -0,0 +1,299 @@
"""
MonaLisa DRM System.
A WASM-based DRM system that uses local key extraction and two-stage
segment decryption (ML-Worker binary + AES-ECB).
"""
from __future__ import annotations
import logging
import os
import subprocess
import sys
from pathlib import Path
from typing import Any, Optional, Union
from uuid import UUID
from Cryptodome.Cipher import AES
from Cryptodome.Util.Padding import unpad
log = logging.getLogger(__name__)
class MonaLisa:
"""
MonaLisa DRM System.
Unlike Widevine/PlayReady, MonaLisa does not use a challenge/response flow
with a license server. Instead, the PSSH value (ticket) is provided directly
by the service API, and keys are extracted locally via a WASM module.
Decryption is performed in two stages:
1. ML-Worker binary: Removes MonaLisa encryption layer (bbts -> ents)
2. AES-ECB decryption: Final decryption with service-provided key
"""
class Exceptions:
class TicketNotFound(Exception):
"""Raised when no PSSH/ticket data is provided."""
class KeyExtractionFailed(Exception):
"""Raised when key extraction from the ticket fails."""
class WorkerNotFound(Exception):
"""Raised when the ML-Worker binary is not found."""
class DecryptionFailed(Exception):
"""Raised when segment decryption fails."""
def __init__(
self,
ticket: Union[str, bytes],
aes_key: Union[str, bytes],
device_path: Path,
**kwargs: Any,
):
"""
Initialize MonaLisa DRM.
Args:
ticket: PSSH value from service API (base64 string or raw bytes).
aes_key: AES-ECB key for second-stage decryption (hex string or bytes).
device_path: Path to the CDM device file (.mld).
**kwargs: Additional metadata stored in self.data.
Raises:
TicketNotFound: If ticket/PSSH is empty.
KeyExtractionFailed: If key extraction fails.
"""
if not ticket:
raise MonaLisa.Exceptions.TicketNotFound("No PSSH/ticket data provided.")
self._ticket = ticket
# Store AES key for second-stage decryption
if isinstance(aes_key, str):
self._aes_key = bytes.fromhex(aes_key)
else:
self._aes_key = aes_key
self._device_path = device_path
self._kid: Optional[UUID] = None
self._key: Optional[str] = None
self.data: dict = kwargs or {}
# Extract keys immediately
self._extract_keys()
def _extract_keys(self) -> None:
"""Extract keys from the ticket using the MonaLisa CDM."""
# Import here to avoid circular import
from envied.core.cdm.monalisa import MonaLisaCDM
try:
cdm = MonaLisaCDM(device_path=self._device_path)
session_id = cdm.open()
try:
keys = cdm.extract_keys(self._ticket)
if keys:
kid_hex = keys.get("kid")
if kid_hex:
self._kid = UUID(hex=kid_hex)
self._key = keys.get("key")
finally:
cdm.close(session_id)
except Exception as e:
raise MonaLisa.Exceptions.KeyExtractionFailed(f"Failed to extract keys: {e}")
@classmethod
def from_ticket(
cls,
ticket: Union[str, bytes],
aes_key: Union[str, bytes],
device_path: Path,
) -> MonaLisa:
"""
Create a MonaLisa DRM instance from a PSSH/ticket.
Args:
ticket: PSSH value from service API.
aes_key: AES-ECB key for second-stage decryption.
device_path: Path to the CDM device file (.mld).
Returns:
MonaLisa DRM instance with extracted keys.
"""
return cls(ticket=ticket, aes_key=aes_key, device_path=device_path)
@property
def kid(self) -> Optional[UUID]:
"""Get the Key ID."""
return self._kid
@property
def key(self) -> Optional[str]:
"""Get the content key as hex string."""
return self._key
@property
def pssh(self) -> str:
"""
Get the raw PSSH/ticket value as a string.
Returns:
The raw PSSH value as a base64 string.
"""
if isinstance(self._ticket, bytes):
try:
return self._ticket.decode("utf-8")
except UnicodeDecodeError:
# Tickets are typically base64, so ASCII is a reasonable fallback.
try:
return self._ticket.decode("ascii")
except UnicodeDecodeError as e:
raise ValueError(
f"Ticket bytes must be UTF-8 text or ASCII base64; got undecodable bytes (len={len(self._ticket)})"
) from e
return self._ticket
@property
def content_id(self) -> Optional[str]:
"""
Extract the Content ID from the PSSH for display.
The PSSH contains an embedded Content ID at bytes 21-75 with format:
H5DCID-V3-P1-YYYYMMDD-HHMMSS-MEDIAID-TIMESTAMP-SUFFIX
Returns:
The Content ID string if extractable, None otherwise.
"""
import base64
try:
# Decode base64 PSSH to get raw bytes
if isinstance(self._ticket, bytes):
data = self._ticket
else:
data = base64.b64decode(self._ticket)
# Content ID is at bytes 21-75 (55 bytes)
if len(data) >= 76:
content_id = data[21:76].decode("ascii")
# Validate it looks like a content ID
if content_id.startswith("H5DCID-"):
return content_id
except Exception:
pass
return None
@property
def content_keys(self) -> dict[UUID, str]:
"""
Get content keys in the same format as Widevine/PlayReady.
Returns:
Dictionary mapping KID to key hex string.
"""
if self._kid and self._key:
return {self._kid: self._key}
return {}
def decrypt_segment(self, segment_path: Path) -> None:
"""
Decrypt a single segment using two-stage decryption.
Stage 1: ML-Worker binary (bbts -> ents)
Stage 2: AES-ECB decryption (ents -> ts)
Args:
segment_path: Path to the encrypted segment file.
Raises:
WorkerNotFound: If ML-Worker binary is not available.
DecryptionFailed: If decryption fails at any stage.
"""
if not self._key:
return
# Import here to avoid circular import
from envied.core.cdm.monalisa import MonaLisaCDM
worker_path = MonaLisaCDM.get_worker_path()
if not worker_path or not worker_path.exists():
raise MonaLisa.Exceptions.WorkerNotFound("ML-Worker not found.")
bbts_path = segment_path.with_suffix(".bbts")
ents_path = segment_path.with_suffix(".ents")
try:
if segment_path.exists():
segment_path.replace(bbts_path)
else:
raise MonaLisa.Exceptions.DecryptionFailed(f"Segment file does not exist: {segment_path}")
# Stage 1: ML-Worker decryption
cmd = [str(worker_path), str(self._key), str(bbts_path), str(ents_path)]
startupinfo = None
if sys.platform == "win32":
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
worker_timeout_s = 60
process = subprocess.run(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
startupinfo=startupinfo,
timeout=worker_timeout_s,
)
if process.returncode != 0:
raise MonaLisa.Exceptions.DecryptionFailed(
f"ML-Worker failed for {segment_path.name}: {process.stderr}"
)
if not ents_path.exists():
raise MonaLisa.Exceptions.DecryptionFailed(
f"Decrypted .ents file was not created for {segment_path.name}"
)
# Stage 2: AES-ECB decryption
with open(ents_path, "rb") as f:
ents_data = f.read()
crypto = AES.new(self._aes_key, AES.MODE_ECB)
decrypted_data = unpad(crypto.decrypt(ents_data), AES.block_size)
# Write decrypted segment back to original path
with open(segment_path, "wb") as f:
f.write(decrypted_data)
except MonaLisa.Exceptions.DecryptionFailed:
raise
except subprocess.TimeoutExpired as e:
log.error("ML-Worker timed out after %ss for %s", worker_timeout_s, segment_path.name)
raise MonaLisa.Exceptions.DecryptionFailed(
f"ML-Worker timed out after {worker_timeout_s}s for {segment_path.name}"
) from e
except Exception as e:
raise MonaLisa.Exceptions.DecryptionFailed(f"Failed to decrypt segment {segment_path.name}: {e}")
finally:
if ents_path.exists():
os.remove(ents_path)
if bbts_path != segment_path and bbts_path.exists():
os.remove(bbts_path)
def decrypt(self, _path: Path) -> None:
"""
MonaLisa uses per-segment decryption during download via the
on_segment_downloaded callback. By the time this method is called,
the content has already been decrypted and muxed into a container.
Args:
path: Path to the file (ignored).
"""
pass
reset/packages/envied/src/envied/core/drm/playready.py
+494
View File
@@ -0,0 +1,494 @@
from __future__ import annotations
import base64
import shutil
import subprocess
import textwrap
from pathlib import Path
from typing import Any, Callable, Optional, Union
from uuid import UUID
import m3u8
from construct import Container
from pymp4.parser import Box
from pyplayready.cdm import Cdm as PlayReadyCdm
from pyplayready.system.pssh import PSSH
from requests import Session
from rich.text import Text
from envied.core import binaries
from envied.core.config import config
from envied.core.console import console
from envied.core.constants import AnyTrack
from envied.core.utilities import get_boxes
from envied.core.utils.subprocess import ffprobe
class PlayReady:
"""PlayReady DRM System."""
def __init__(
self,
pssh: PSSH,
kid: Union[UUID, str, bytes, None] = None,
pssh_b64: Optional[str] = None,
**kwargs: Any,
):
if not pssh:
raise ValueError("Provided PSSH is empty.")
if not isinstance(pssh, PSSH):
raise TypeError(f"Expected pssh to be a {PSSH}, not {pssh!r}")
if pssh_b64:
kids = self._extract_kids_from_pssh_b64(pssh_b64)
else:
kids = []
# Extract KIDs using pyplayready's WrmHeader key_ids
if not kids:
for header in pssh.wrm_headers:
for signed_key_id in getattr(header, "key_ids", []):
try:
if isinstance(signed_key_id.value, UUID):
kids.append(signed_key_id.value)
else:
kids.append(UUID(bytes_le=base64.b64decode(signed_key_id.value)))
except Exception:
continue
if kid:
if isinstance(kid, str):
kid = UUID(hex=kid)
elif isinstance(kid, bytes):
kid = UUID(bytes=kid)
if not isinstance(kid, UUID):
raise ValueError(f"Expected kid to be a {UUID}, str, or bytes, not {kid!r}")
if kid not in kids:
kids.append(kid)
self._pssh = pssh
self._kids = kids
if not self.kids:
raise PlayReady.Exceptions.KIDNotFound("No Key ID was found within PSSH and none were provided.")
self.content_keys: dict[UUID, str] = {}
self.data: dict = kwargs or {}
if pssh_b64:
self.data.setdefault("pssh_b64", pssh_b64)
def _extract_kids_from_pssh_b64(self, pssh_b64: str) -> list[UUID]:
"""Extract all KIDs from base64-encoded PSSH data."""
try:
import xml.etree.ElementTree as ET
# Decode the PSSH
pssh_bytes = base64.b64decode(pssh_b64)
# Try to find XML in the PSSH data
# PlayReady PSSH usually has XML embedded in it
pssh_str = pssh_bytes.decode("utf-16le", errors="ignore")
# Find WRMHEADER
xml_start = pssh_str.find("<WRMHEADER")
if xml_start == -1:
# Try UTF-8
pssh_str = pssh_bytes.decode("utf-8", errors="ignore")
xml_start = pssh_str.find("<WRMHEADER")
if xml_start != -1:
clean_xml = pssh_str[xml_start:]
xml_end = clean_xml.find("</WRMHEADER>") + len("</WRMHEADER>")
clean_xml = clean_xml[:xml_end]
root = ET.fromstring(clean_xml)
ns = {"pr": "http://schemas.microsoft.com/DRM/2007/03/PlayReadyHeader"}
kids = []
# Extract from CUSTOMATTRIBUTES/KIDS
kid_elements = root.findall(".//pr:CUSTOMATTRIBUTES/pr:KIDS/pr:KID", ns)
for kid_elem in kid_elements:
value = kid_elem.get("VALUE")
if value:
try:
kid_bytes = base64.b64decode(value + "==")
kid_uuid = UUID(bytes_le=kid_bytes)
kids.append(kid_uuid)
except Exception:
pass
# v4.2/v4.3: DATA/PROTECTINFO/KIDS/KID
protectinfo_kids = root.findall(".//pr:DATA/pr:PROTECTINFO/pr:KIDS/pr:KID", ns)
for kid_elem in protectinfo_kids:
value = kid_elem.get("VALUE")
if value:
try:
kid_bytes = base64.b64decode(value + "==")
kid_uuid = UUID(bytes_le=kid_bytes)
if kid_uuid not in kids:
kids.append(kid_uuid)
except Exception:
pass
# v4.1: DATA/PROTECTINFO/KID
protectinfo_kid = root.findall(".//pr:DATA/pr:PROTECTINFO/pr:KID", ns)
for kid_elem in protectinfo_kid:
value = kid_elem.get("VALUE")
if value:
try:
kid_bytes = base64.b64decode(value + "==")
kid_uuid = UUID(bytes_le=kid_bytes)
if kid_uuid not in kids:
kids.append(kid_uuid)
except Exception:
pass
# v4.0: DATA/KID
individual_kids = root.findall(".//pr:DATA/pr:KID", ns)
for kid_elem in individual_kids:
if kid_elem.text:
try:
kid_bytes = base64.b64decode(kid_elem.text.strip() + "==")
kid_uuid = UUID(bytes_le=kid_bytes)
if kid_uuid not in kids:
kids.append(kid_uuid)
except Exception:
pass
return kids
except Exception:
pass
return []
@classmethod
def from_track(cls, track: AnyTrack, session: Optional[Session] = None) -> PlayReady:
if not session:
session = Session()
session.headers.update(config.headers)
kid: Optional[UUID] = None
pssh_boxes: list[Container] = []
tenc_boxes: list[Container] = []
if track.descriptor == track.Descriptor.HLS:
m3u_url = track.url
master = m3u8.loads(session.get(m3u_url).text, uri=m3u_url)
pssh_boxes.extend(
Box.parse(base64.b64decode(x.uri.split(",")[-1]))
for x in (master.session_keys or master.keys)
if x
and x.keyformat
and x.keyformat.lower() in {f"urn:uuid:{PSSH.SYSTEM_ID}", "com.microsoft.playready"}
)
init_data = track.get_init_segment(session=session)
if init_data:
probe = ffprobe(init_data)
if probe:
for stream in probe.get("streams") or []:
enc_key_id = stream.get("tags", {}).get("enc_key_id")
if enc_key_id:
kid = UUID(bytes=base64.b64decode(enc_key_id))
pssh_boxes.extend(list(get_boxes(init_data, b"pssh")))
tenc_boxes.extend(list(get_boxes(init_data, b"tenc")))
pssh = next((b for b in pssh_boxes if b.system_ID == PSSH.SYSTEM_ID), None)
if not pssh:
raise PlayReady.Exceptions.PSSHNotFound("PSSH was not found in track data.")
tenc = next(iter(tenc_boxes), None)
if not kid and tenc and tenc.key_ID.int != 0:
kid = tenc.key_ID
pssh_bytes = Box.build(pssh)
return cls(pssh=PSSH(pssh_bytes), kid=kid, pssh_b64=base64.b64encode(pssh_bytes).decode())
@classmethod
def from_init_data(cls, init_data: bytes) -> PlayReady:
if not init_data:
raise ValueError("Init data should be provided.")
if not isinstance(init_data, bytes):
raise TypeError(f"Expected init data to be bytes, not {init_data!r}")
kid: Optional[UUID] = None
pssh_boxes: list[Container] = list(get_boxes(init_data, b"pssh"))
tenc_boxes: list[Container] = list(get_boxes(init_data, b"tenc"))
probe = ffprobe(init_data)
if probe:
for stream in probe.get("streams") or []:
enc_key_id = stream.get("tags", {}).get("enc_key_id")
if enc_key_id:
kid = UUID(bytes=base64.b64decode(enc_key_id))
pssh = next((b for b in pssh_boxes if b.system_ID == PSSH.SYSTEM_ID), None)
if not pssh:
raise PlayReady.Exceptions.PSSHNotFound("PSSH was not found in track data.")
tenc = next(iter(tenc_boxes), None)
if not kid and tenc and tenc.key_ID.int != 0:
kid = tenc.key_ID
pssh_bytes = Box.build(pssh)
return cls(pssh=PSSH(pssh_bytes), kid=kid, pssh_b64=base64.b64encode(pssh_bytes).decode())
@property
def pssh(self) -> PSSH:
return self._pssh
@property
def pssh_b64(self) -> Optional[str]:
return self.data.get("pssh_b64")
@property
def kid(self) -> Optional[UUID]:
return next(iter(self.kids), None)
def to_dict(self) -> dict[str, Any]:
"""Serialise this DRM instance for export/import (PSSH + KIDs).
Content keys are stored once at the export's track level, not duplicated here.
"""
return {
"system": "PlayReady",
"pssh_b64": self.pssh_b64,
"kids": [kid.hex for kid in self.kids],
}
@property
def kids(self) -> list[UUID]:
return self._kids
def _extract_keys_from_cdm(self, cdm: PlayReadyCdm, session_id: bytes) -> dict:
"""Extract keys from CDM session with cross-library compatibility.
Args:
cdm: CDM instance
session_id: Session identifier
Returns:
Dictionary mapping KID UUIDs to hex keys
"""
keys = {}
for key in cdm.get_keys(session_id):
if hasattr(key, "key_id"):
kid = key.key_id
elif hasattr(key, "kid"):
kid = key.kid
else:
continue
if hasattr(key, "key") and hasattr(key.key, "hex"):
key_hex = key.key.hex()
elif hasattr(key, "key") and isinstance(key.key, bytes):
key_hex = key.key.hex()
elif hasattr(key, "key") and isinstance(key.key, str):
key_hex = key.key
else:
continue
keys[kid] = key_hex
return keys
def get_content_keys(self, cdm: PlayReadyCdm, certificate: Callable, licence: Callable) -> None:
session_id = cdm.open()
try:
if hasattr(cdm, "set_pssh_b64") and self.pssh_b64:
cdm.set_pssh_b64(self.pssh_b64)
if hasattr(cdm, "set_required_kids"):
cdm.set_required_kids(self.kids)
challenge = cdm.get_license_challenge(session_id, self.pssh.wrm_headers[0])
if challenge:
try:
try:
license_res = licence(challenge=challenge, pssh_b64=self.pssh_b64)
except TypeError:
license_res = licence(challenge=challenge)
if isinstance(license_res, bytes):
license_str = license_res.decode(errors="ignore")
else:
license_str = str(license_res)
if "<License>" not in license_str:
try:
license_str = base64.b64decode(license_str + "===").decode()
except Exception:
pass
cdm.parse_license(session_id, license_str)
except Exception:
raise
keys = self._extract_keys_from_cdm(cdm, session_id)
self.content_keys.update(keys)
finally:
cdm.close(session_id)
if not self.content_keys:
raise PlayReady.Exceptions.EmptyLicense("No Content Keys were within the License")
def decrypt(self, path: Path) -> None:
"""
Decrypt a Track with PlayReady DRM.
Args:
path: Path to the encrypted file to decrypt
Raises:
EnvironmentError if the required decryption executable could not be found.
ValueError if the track has not yet been downloaded.
SubprocessError if the decryption process returned a non-zero exit code.
"""
if not self.content_keys:
raise ValueError("Cannot decrypt a Track without any Content Keys...")
if not path or not path.exists():
raise ValueError("Tried to decrypt a file that does not exist.")
decrypter = str(getattr(config, "decryption", "")).lower()
if decrypter == "mp4decrypt":
return self._decrypt_with_mp4decrypt(path)
else:
return self._decrypt_with_shaka_packager(path)
def _decrypt_with_mp4decrypt(self, path: Path) -> None:
"""Decrypt using mp4decrypt"""
if not binaries.Mp4decrypt:
raise EnvironmentError("mp4decrypt executable not found but is required.")
output_path = path.with_stem(f"{path.stem}_decrypted")
# Build key arguments
key_args = []
for kid, key in self.content_keys.items():
kid_hex = kid.hex if hasattr(kid, "hex") else str(kid).replace("-", "")
key_hex = key if isinstance(key, str) else key.hex()
key_args.extend(["--key", f"{kid_hex}:{key_hex}"])
# Fallback for tracks whose tenc default_KID is all-zero and whose real
# KID is signalled out-of-band: emit a zero-KID entry per content key.
zero_kid = "00" * 16
existing_kids = {kid.hex if hasattr(kid, "hex") else str(kid).replace("-", "") for kid in self.content_keys}
if zero_kid not in existing_kids:
for key in self.content_keys.values():
key_hex = key if isinstance(key, str) else key.hex()
key_args.extend(["--key", f"{zero_kid}:{key_hex}"])
cmd = [
str(binaries.Mp4decrypt),
"--show-progress",
*key_args,
str(path),
str(output_path),
]
try:
subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8")
except subprocess.CalledProcessError as e:
error_msg = e.stderr if e.stderr else f"mp4decrypt failed with exit code {e.returncode}"
raise subprocess.CalledProcessError(e.returncode, cmd, output=e.stdout, stderr=error_msg)
if not output_path.exists():
raise RuntimeError(f"mp4decrypt failed: output file {output_path} was not created")
if output_path.stat().st_size == 0:
raise RuntimeError(f"mp4decrypt failed: output file {output_path} is empty")
path.unlink()
shutil.move(output_path, path)
def _decrypt_with_shaka_packager(self, path: Path) -> None:
"""Decrypt using Shaka Packager (original method)"""
if not binaries.ShakaPackager:
raise EnvironmentError("Shaka Packager executable not found but is required.")
output_path = path.with_stem(f"{path.stem}_decrypted")
config.directories.temp.mkdir(parents=True, exist_ok=True)
try:
arguments = [
f"input={path},stream=0,output={output_path},output_format=MP4",
"--enable_raw_key_decryption",
"--keys",
",".join(
[
*[
f"label={i}:key_id={kid.hex}:key={key.lower()}"
for i, (kid, key) in enumerate(self.content_keys.items())
],
*[
f"label={i}:key_id={'00' * 16}:key={key.lower()}"
for i, (kid, key) in enumerate(self.content_keys.items(), len(self.content_keys))
],
]
),
"--temp_dir",
config.directories.temp,
]
p = subprocess.Popen(
[binaries.ShakaPackager, *arguments],
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
text=True,
encoding="utf-8",
errors="replace",
)
stream_skipped = False
had_error = False
shaka_log_buffer = ""
for line in iter(p.stderr.readline, ""):
line = line.strip()
if not line:
continue
if "Skip stream" in line:
stream_skipped = True
if ":INFO:" in line:
continue
if "I0" in line or "W0" in line:
continue
if ":ERROR:" in line:
had_error = True
if "Insufficient bits in bitstream for given AVC profile" in line:
continue
shaka_log_buffer += f"{line.strip()}\n"
if shaka_log_buffer:
shaka_log_buffer = "\n ".join(
textwrap.wrap(shaka_log_buffer.rstrip(), width=console.width - 22, initial_indent="")
)
console.log(Text.from_ansi("\n[PlayReady]: " + shaka_log_buffer))
p.wait()
if p.returncode != 0 or had_error:
raise subprocess.CalledProcessError(p.returncode, [binaries.ShakaPackager, *arguments])
path.unlink()
if not stream_skipped:
shutil.move(output_path, path)
except subprocess.CalledProcessError as e:
if e.returncode == 0xC000013A:
raise KeyboardInterrupt()
raise
class Exceptions:
class PSSHNotFound(Exception):
pass
class KIDNotFound(Exception):
pass
class CEKNotFound(Exception):
pass
class EmptyLicense(Exception):
pass
__all__ = ("PlayReady",)
reset/packages/envied/src/envied/core/drm/widevine.py
+434
View File
@@ -0,0 +1,434 @@
from __future__ import annotations
import base64
import shutil
import subprocess
import textwrap
from pathlib import Path
from typing import Any, Callable, Optional, Union
from uuid import UUID
import m3u8
from construct import Container
from pymp4.parser import Box
from pywidevine.cdm import Cdm as WidevineCdm
from pywidevine.pssh import PSSH
from requests import Session
from rich.text import Text
from envied.core import binaries
from envied.core.config import config
from envied.core.console import console
from envied.core.constants import AnyTrack
from envied.core.utilities import get_boxes
from envied.core.utils.subprocess import ffprobe
class Widevine:
"""Widevine DRM System."""
PLACEHOLDER_KIDS = {
UUID("00000000-0000-0000-0000-000000000000"), # All zeros (key rotation default)
UUID("00010203-0405-0607-0809-0a0b0c0d0e0f"), # Sequential 0x00-0x0f
UUID("00010203-0405-0607-0809-101112131415"), # Shaka Packager test pattern
}
def __init__(self, pssh: PSSH, kid: Union[UUID, str, bytes, None] = None, **kwargs: Any):
if not pssh:
raise ValueError("Provided PSSH is empty.")
if not isinstance(pssh, PSSH):
raise TypeError(f"Expected pssh to be a {PSSH}, not {pssh!r}")
if pssh.system_id == PSSH.SystemId.PlayReady:
pssh.to_widevine()
self._kid: Optional[UUID] = None
if kid:
if isinstance(kid, str):
kid = UUID(hex=kid)
elif isinstance(kid, bytes):
kid = UUID(bytes=kid)
if not isinstance(kid, UUID):
raise ValueError(f"Expected kid to be a {UUID}, str, or bytes, not {kid!r}")
self._kid = kid
if pssh.key_ids and all(k in self.PLACEHOLDER_KIDS for k in pssh.key_ids):
pssh.set_key_ids([kid])
elif kid not in (pssh.key_ids or []):
pssh.set_key_ids([*(pssh.key_ids or []), kid])
self._pssh = pssh
if not self.kids:
raise Widevine.Exceptions.KIDNotFound("No Key ID was found within PSSH and none were provided.")
self.content_keys: dict[UUID, str] = {}
self.data: dict = kwargs or {}
@classmethod
def from_track(cls, track: AnyTrack, session: Optional[Session] = None) -> Widevine:
"""
Get PSSH and KID from within the Initiation Segment of the Track Data.
It also tries to get PSSH and KID from other track data like M3U8 data
as well as through ffprobe.
Create a Widevine DRM System object from a track's information.
This should only be used if a PSSH could not be provided directly.
It is *rare* to need to use this.
You may provide your own requests session to be able to use custom
headers and more.
Raises:
PSSHNotFound - If the PSSH was not found within the data.
KIDNotFound - If the KID was not found within the data or PSSH.
"""
if not session:
session = Session()
session.headers.update(config.headers)
kid: Optional[UUID] = None
pssh_boxes: list[Container] = []
tenc_boxes: list[Container] = []
if track.descriptor == track.Descriptor.HLS:
m3u_url = track.url
master = m3u8.loads(session.get(m3u_url).text, uri=m3u_url)
pssh_boxes.extend(
Box.parse(base64.b64decode(x.uri.split(",")[-1]))
for x in (master.session_keys or master.keys)
if x and x.keyformat and x.keyformat.lower() == WidevineCdm.urn
)
init_data = track.get_init_segment(session=session)
if init_data:
# try get via ffprobe, needed for non mp4 data e.g. WEBM from Google Play
probe = ffprobe(init_data)
if probe:
for stream in probe.get("streams") or []:
enc_key_id = stream.get("tags", {}).get("enc_key_id")
if enc_key_id:
kid = UUID(bytes=base64.b64decode(enc_key_id))
pssh_boxes.extend(list(get_boxes(init_data, b"pssh")))
tenc_boxes.extend(list(get_boxes(init_data, b"tenc")))
pssh = next((b for b in pssh_boxes if b.system_ID == PSSH.SystemId.Widevine), None)
if not pssh:
raise Widevine.Exceptions.PSSHNotFound("PSSH was not found in track data.")
tenc = next(iter(tenc_boxes), None)
if not kid and tenc and tenc.key_ID.int != 0:
kid = tenc.key_ID
return cls(pssh=PSSH(pssh), kid=kid)
@classmethod
def from_init_data(cls, init_data: bytes) -> Widevine:
"""
Get PSSH and KID from within Initialization Segment Data.
This should only be used if a PSSH could not be provided directly.
It is *rare* to need to use this.
Raises:
PSSHNotFound - If the PSSH was not found within the data.
KIDNotFound - If the KID was not found within the data or PSSH.
"""
if not init_data:
raise ValueError("Init data should be provided.")
if not isinstance(init_data, bytes):
raise TypeError(f"Expected init data to be bytes, not {init_data!r}")
kid: Optional[UUID] = None
pssh_boxes: list[Container] = list(get_boxes(init_data, b"pssh"))
tenc_boxes: list[Container] = list(get_boxes(init_data, b"tenc"))
# try get via ffprobe, needed for non mp4 data e.g. WEBM from Google Play
probe = ffprobe(init_data)
if probe:
for stream in probe.get("streams") or []:
enc_key_id = stream.get("tags", {}).get("enc_key_id")
if enc_key_id:
kid = UUID(bytes=base64.b64decode(enc_key_id))
pssh = next((b for b in pssh_boxes if b.system_ID == PSSH.SystemId.Widevine), None)
if not pssh:
raise Widevine.Exceptions.PSSHNotFound("PSSH was not found in track data.")
tenc = next(iter(tenc_boxes), None)
if not kid and tenc and tenc.key_ID.int != 0:
kid = tenc.key_ID
return cls(pssh=PSSH(pssh), kid=kid)
@property
def pssh(self) -> PSSH:
"""Get Protection System Specific Header Box."""
return self._pssh
@property
def kid(self) -> Optional[UUID]:
"""Get first Key ID, if any."""
return next(iter(self.kids), None)
@property
def kids(self) -> list[UUID]:
"""Get all Key IDs from PSSH, falling back to the externally provided KID."""
pssh_kids = self._pssh.key_ids
if pssh_kids:
return pssh_kids
if self._kid:
return [self._kid]
return []
def to_dict(self) -> dict[str, Any]:
"""Serialise this DRM instance for export/import (PSSH + KIDs).
Content keys are stored once at the export's track level, not duplicated here.
"""
return {
"system": "Widevine",
"pssh_b64": self.pssh.dumps(),
"kids": [kid.hex for kid in self.kids],
}
def get_content_keys(self, cdm: WidevineCdm, certificate: Callable, licence: Callable) -> None:
"""
Create a CDM Session and obtain Content Keys for this DRM Instance.
The certificate and license params are expected to be a function and will
be provided with the challenge and session ID.
"""
for kid in self.kids:
if kid in self.content_keys:
continue
session_id = cdm.open()
try:
cert = certificate(challenge=cdm.service_certificate_challenge)
if cert and hasattr(cdm, "set_service_certificate"):
cdm.set_service_certificate(session_id, cert)
if hasattr(cdm, "set_required_kids"):
cdm.set_required_kids(self.kids)
challenge = cdm.get_license_challenge(session_id, self.pssh)
if hasattr(cdm, "has_cached_keys") and cdm.has_cached_keys(session_id):
pass
else:
try:
license_res = licence(challenge=challenge, pssh=self.pssh)
except TypeError:
license_res = licence(challenge=challenge)
cdm.parse_license(session_id, license_res)
self.content_keys = {key.kid: key.key.hex() for key in cdm.get_keys(session_id, "CONTENT")}
if not self.content_keys:
raise Widevine.Exceptions.EmptyLicense("No Content Keys were within the License")
if kid not in self.content_keys:
raise Widevine.Exceptions.CEKNotFound(f"No Content Key for KID {kid.hex} within the License")
finally:
cdm.close(session_id)
def get_NF_content_keys(self, cdm: WidevineCdm, certificate: Callable, licence: Callable) -> None:
"""
Create a CDM Session and obtain Content Keys for this DRM Instance.
The certificate and license params are expected to be a function and will
be provided with the challenge and session ID.
"""
for kid in self.kids:
if kid in self.content_keys:
continue
session_id = cdm.open()
try:
cert = certificate(challenge=cdm.service_certificate_challenge)
if cert and hasattr(cdm, "set_service_certificate"):
cdm.set_service_certificate(session_id, cert)
if hasattr(cdm, "set_required_kids"):
cdm.set_required_kids(self.kids)
challenge = cdm.get_license_challenge(session_id, self.pssh)
if hasattr(cdm, "has_cached_keys") and cdm.has_cached_keys(session_id):
pass
else:
cdm.parse_license(
session_id,
licence(session_id=session_id, challenge=challenge),
)
self.content_keys = {key.kid: key.key.hex() for key in cdm.get_keys(session_id, "CONTENT")}
if not self.content_keys:
raise Widevine.Exceptions.EmptyLicense("No Content Keys were within the License")
if kid not in self.content_keys:
raise Widevine.Exceptions.CEKNotFound(f"No Content Key for KID {kid.hex} within the License")
finally:
cdm.close(session_id)
def decrypt(self, path: Path) -> None:
"""
Decrypt a Track with Widevine DRM.
Args:
path: Path to the encrypted file to decrypt
Raises:
EnvironmentError if the required decryption executable could not be found.
ValueError if the track has not yet been downloaded.
SubprocessError if the decryption process returned a non-zero exit code.
"""
if not self.content_keys:
raise ValueError("Cannot decrypt a Track without any Content Keys...")
if not path or not path.exists():
raise ValueError("Tried to decrypt a file that does not exist.")
decrypter = str(getattr(config, "decryption", "")).lower()
if decrypter == "mp4decrypt":
return self._decrypt_with_mp4decrypt(path)
else:
return self._decrypt_with_shaka_packager(path)
def _decrypt_with_mp4decrypt(self, path: Path) -> None:
"""Decrypt using mp4decrypt"""
if not binaries.Mp4decrypt:
raise EnvironmentError("mp4decrypt executable not found but is required.")
output_path = path.with_stem(f"{path.stem}_decrypted")
# Build key arguments
key_args = []
for kid, key in self.content_keys.items():
kid_hex = kid.hex if hasattr(kid, "hex") else str(kid).replace("-", "")
key_hex = key if isinstance(key, str) else key.hex()
key_args.extend(["--key", f"{kid_hex}:{key_hex}"])
# Fallback for tracks whose tenc default_KID is all-zero and whose real
# KID is signalled out-of-band: emit a zero-KID entry per content key.
zero_kid = "00" * 16
existing_kids = {kid.hex if hasattr(kid, "hex") else str(kid).replace("-", "") for kid in self.content_keys}
if zero_kid not in existing_kids:
for key in self.content_keys.values():
key_hex = key if isinstance(key, str) else key.hex()
key_args.extend(["--key", f"{zero_kid}:{key_hex}"])
cmd = [
str(binaries.Mp4decrypt),
"--show-progress",
*key_args,
str(path),
str(output_path),
]
try:
subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8")
except subprocess.CalledProcessError as e:
error_msg = e.stderr if e.stderr else f"mp4decrypt failed with exit code {e.returncode}"
raise subprocess.CalledProcessError(e.returncode, cmd, output=e.stdout, stderr=error_msg)
if not output_path.exists():
raise RuntimeError(f"mp4decrypt failed: output file {output_path} was not created")
if output_path.stat().st_size == 0:
raise RuntimeError(f"mp4decrypt failed: output file {output_path} is empty")
path.unlink()
shutil.move(output_path, path)
def _decrypt_with_shaka_packager(self, path: Path) -> None:
"""Decrypt using Shaka Packager (original method)"""
if not binaries.ShakaPackager:
raise EnvironmentError("Shaka Packager executable not found but is required.")
output_path = path.with_stem(f"{path.stem}_decrypted")
config.directories.temp.mkdir(parents=True, exist_ok=True)
try:
arguments = [
f"input={path},stream=0,output={output_path},output_format=MP4",
"--enable_raw_key_decryption",
"--keys",
",".join(
[
*[
"label={}:key_id={}:key={}".format(i, kid.hex, key.lower())
for i, (kid, key) in enumerate(self.content_keys.items())
],
*[
# some services use a blank KID on the file, but real KID for license server
"label={}:key_id={}:key={}".format(i, "00" * 16, key.lower())
for i, (kid, key) in enumerate(self.content_keys.items(), len(self.content_keys))
],
]
),
"--temp_dir",
config.directories.temp,
]
p = subprocess.Popen(
[binaries.ShakaPackager, *arguments],
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
universal_newlines=True,
)
stream_skipped = False
had_error = False
shaka_log_buffer = ""
for line in iter(p.stderr.readline, ""):
line = line.strip()
if not line:
continue
if "Skip stream" in line:
# file/segment was so small that it didn't have any actual data, ignore
stream_skipped = True
if ":INFO:" in line:
continue
if "I0" in line or "W0" in line:
continue
if ":ERROR:" in line:
had_error = True
if "Insufficient bits in bitstream for given AVC profile" in line:
# this is a warning and is something we don't have to worry about
continue
shaka_log_buffer += f"{line.strip()}\n"
if shaka_log_buffer:
# wrap to console width - padding - '[Widevine]: '
shaka_log_buffer = "\n ".join(
textwrap.wrap(shaka_log_buffer.rstrip(), width=console.width - 22, initial_indent="")
)
console.log(Text.from_ansi("\n[Widevine]: " + shaka_log_buffer))
p.wait()
if p.returncode != 0 or had_error:
raise subprocess.CalledProcessError(p.returncode, [binaries.ShakaPackager, *arguments])
path.unlink()
if not stream_skipped:
shutil.move(output_path, path)
except subprocess.CalledProcessError as e:
if e.returncode == 0xC000013A: # STATUS_CONTROL_C_EXIT
raise KeyboardInterrupt()
raise
class Exceptions:
class PSSHNotFound(Exception):
"""PSSH (Protection System Specific Header) was not found."""
class KIDNotFound(Exception):
"""KID (Encryption Key ID) was not found."""
class CEKNotFound(Exception):
"""CEK (Content Encryption Key) for KID was not found in License."""
class EmptyLicense(Exception):
"""License returned no Content Encryption Keys."""
__all__ = ("Widevine",)
reset/packages/envied/src/envied/core/events.py
+76
View File
@@ -0,0 +1,76 @@
from __future__ import annotations
from copy import deepcopy
from enum import Enum
from typing import Any, Callable
class Events:
class Types(Enum):
_reserved = 0
# A Track's segment has finished downloading
SEGMENT_DOWNLOADED = 1
# Track has finished downloading
TRACK_DOWNLOADED = 2
# Track has finished decrypting
TRACK_DECRYPTED = 3
# Track has finished repacking
TRACK_REPACKED = 4
# Track is about to be Multiplexed into a Container
TRACK_MULTIPLEX = 5
def __init__(self):
self.__subscriptions: dict[Events.Types, list[Callable]] = {}
self.__ephemeral: dict[Events.Types, list[Callable]] = {}
self.reset()
def reset(self):
"""Reset Event Observer clearing all Subscriptions."""
self.__subscriptions = {k: [] for k in Events.Types.__members__.values()}
self.__ephemeral = deepcopy(self.__subscriptions)
def subscribe(self, event_type: Events.Types, callback: Callable, ephemeral: bool = False) -> None:
"""
Subscribe to an Event with a Callback.
Parameters:
event_type: The Events.Type to subscribe to.
callback: The function or lambda to call on event emit.
ephemeral: Unsubscribe the callback from the event on first emit.
Note that this is not thread-safe and may be called multiple
times at roughly the same time.
"""
[self.__subscriptions, self.__ephemeral][ephemeral][event_type].append(callback)
def unsubscribe(self, event_type: Events.Types, callback: Callable) -> None:
"""
Unsubscribe a Callback from an Event.
Parameters:
event_type: The Events.Type to unsubscribe from.
callback: The function or lambda to remove from event emit.
"""
if callback in self.__subscriptions[event_type]:
self.__subscriptions[event_type].remove(callback)
if callback in self.__ephemeral[event_type]:
self.__ephemeral[event_type].remove(callback)
def emit(self, event_type: Events.Types, *args: Any, **kwargs: Any) -> None:
"""
Emit an Event, executing all subscribed Callbacks.
Parameters:
event_type: The Events.Type to emit.
args: Positional arguments to pass to callbacks.
kwargs: Keyword arguments to pass to callbacks.
"""
if event_type not in self.__subscriptions:
raise ValueError(f'Event type "{event_type}" is invalid')
for callback in self.__subscriptions[event_type] + self.__ephemeral[event_type]:
callback(*args, **kwargs)
self.__ephemeral[event_type].clear()
events = Events()
reset/packages/envied/src/envied/core/import_service.py
+304
View File
@@ -0,0 +1,304 @@
from __future__ import annotations
import json
import logging
from http.cookiejar import CookieJar
from pathlib import Path
from typing import Any, Optional, Union
from uuid import UUID
import click
import requests
from envied.core.config import config
from envied.core.constants import AnyTrack
from envied.core.credential import Credential
from envied.core.drm import drm_from_dict
from envied.core.manifests import DASH, HLS, ISM
from envied.core.remote_service import RemoteService, _build_title, _resolve_proxy
from envied.core.titles import Episode, Movies, Series, Title_T, Titles_T, remap_titles
from envied.core.tracks import Audio, Chapter, Chapters, Tracks, Video
from envied.core.tracks.attachment import Attachment
from envied.core.tracks.track import Track
log = logging.getLogger("import")
PARSERS = {"DASH": DASH, "HLS": HLS, "ISM": ISM}
class ImportService:
"""Reconstructs a download from an export JSON.
Auth and licensing are skipped; tracks are rebuilt from the export and keys injected
directly. ``_server_cdm``/``_server_cdm_type`` keep their underscores: dl.py reads them
via getattr as the server-CDM contract that skips client licensing.
"""
ALIASES: tuple[str, ...] = ()
GEOFENCE: tuple[str, ...] = ()
NO_SUBTITLES: bool = False
def __init__(self, ctx: click.Context, service_tag: str, title: str, import_file: Optional[str]) -> None:
self.__class__.__name__ = service_tag
self.service_tag = service_tag
self.title_id = title
self.ctx = ctx
self.log = logging.getLogger(service_tag)
self.credential: Optional[Credential] = None
self.current_region: Optional[str] = None
self.title_cache = None
if not import_file:
raise click.ClickException("No export file was provided to import from.")
export_path = Path(import_file)
if not export_path.is_file():
raise click.ClickException(f"Export file not found: {export_path}")
self.data: dict[str, Any] = json.loads(export_path.read_text(encoding="utf8"))
version = self.data.get("version")
if version != 2:
raise click.ClickException(
f"Unsupported export version {version!r}. Re-create the export with a current build."
)
self.titles_data: dict[str, Any] = self.data.get("titles", {})
self.region: Optional[str] = self.data.get("region")
self.titles: Optional[Titles_T] = None
self.tracks_by_title: dict[str, Tracks] = {}
self._server_cdm = True
self._server_cdm_type = "widevine"
self.session = self.build_session(ctx, self.region)
@staticmethod
def build_session(ctx: click.Context, region: Optional[str] = None) -> requests.Session:
"""Session for re-fetching the manifest.
Honours the importer's ``--proxy``; otherwise falls back to the export region as a
geofence. An explicit proxy that fails to resolve raises; a region fallback warns.
"""
session = requests.Session()
session.headers.update(config.headers)
params = ctx.parent.params if ctx.parent else {}
if params.get("no_proxy"):
return session
explicit = params.get("proxy")
proxy_query = explicit or region
if not proxy_query:
return session
try:
proxy = _resolve_proxy(proxy_query)
except Exception as e:
if explicit:
raise click.ClickException(f"Failed to resolve proxy '{proxy_query}': {e}")
log.warning(f"Could not auto-select a proxy for export region '{region}': {e}. Continuing without proxy.")
proxy = None
if proxy:
session.proxies.update({"all": proxy})
if not explicit:
log.info(f"No --proxy given; using export region '{region}' via your proxy provider.")
return session
@property
def title(self) -> str:
return self.title_id
def authenticate(self, cookies: Optional[CookieJar] = None, credential: Optional[Credential] = None) -> None:
self.credential = credential
def get_titles(self) -> Titles_T:
if self.titles is not None:
return self.titles
titles_list = [
_build_title(entry.get("meta", {}), self.service_tag, fallback_id=title_id)
for title_id, entry in self.titles_data.items()
]
self.titles = (
Series(titles_list) if titles_list and isinstance(titles_list[0], Episode) else Movies(titles_list)
)
return self.titles
def get_titles_cached(self, title_id: Optional[str] = None) -> Titles_T:
"""Apply the service's title_map to titles reconstructed from the export sidecar."""
title_map = (config.services.get(self.service_tag) or {}).get("title_map") or {}
return remap_titles(self.get_titles(), title_map)
def get_tracks(self, title: Title_T) -> Tracks:
"""Reconstruct the title's tracks from the export.
DASH/ISM: re-fetch and re-parse the manifest and return the full ladder (the importer
picks quality with normal dl flags; keys are injected by KID later). HLS/URL: rebuild
from the stored per-track dicts, since the variant is re-fetched from track.url at
download time and ATV-style master playlists carry unstable per-fetch tokens.
"""
title_id = str(title.id)
if title_id in self.tracks_by_title:
return self.tracks_by_title[title_id]
entry = self.titles_data.get(title_id, {})
tracks_map: dict[str, Any] = entry.get("tracks") or {}
manifest_url = entry.get("manifest_url")
manifest_type = entry.get("manifest_type")
tracks = Tracks()
tracks.manifest_url = manifest_url
parser = PARSERS.get(manifest_type or "")
if manifest_url and parser is not None and manifest_type in ("DASH", "ISM"):
try:
parsed = parser.from_url(url=manifest_url, session=self.session).to_tracks(language=title.language)
except Exception as e:
raise click.ClickException(
f"Failed to re-fetch/parse the {manifest_type} manifest for '{title}'. "
f"The manifest URL may have expired since export. ({e})"
)
for track in parsed:
tracks.add(track)
else:
for track_dict in tracks_map.values():
track = Track.from_dict(track_dict)
drm = self.rebuild_drm(track_dict)
if drm:
track.drm = drm
tracks.add(track)
for attachment in entry.get("attachments") or []:
url = attachment.get("url")
if not url:
continue
try:
tracks.attachments.append(
Attachment.from_url(
url,
name=attachment.get("name"),
mime_type=attachment.get("mime_type"),
description=attachment.get("description"),
session=self.session,
)
)
except Exception as e:
self.log.warning(f"Skipping attachment '{attachment.get('name')}': {e}")
self.tracks_by_title[title_id] = tracks
return tracks
def key_pool(self) -> dict[UUID, str]:
"""All exported KID:KEY pairs across every title, as {UUID: key_hex}."""
pool: dict[UUID, str] = {}
for entry in self.titles_data.values():
for track_dict in (entry.get("tracks") or {}).values():
for kid_hex, key in (track_dict.get("keys") or {}).items():
pool[UUID(hex=kid_hex)] = key
return pool
def rebuild_drm(self, track_dict: dict[str, Any]) -> Optional[list[Any]]:
"""Rebuild a DRM object (from stored PSSH, falling back to a stub) with the exported keys."""
keys = track_dict.get("keys") or {}
drm_dicts = track_dict.get("drm") or []
if not drm_dicts and not keys:
return None
drm_obj = None
if drm_dicts:
try:
drm_obj = drm_from_dict(drm_dicts[0])
except Exception as e:
self.log.debug(f"Falling back to DRM stub (PSSH rebuild failed: {e})")
if drm_obj is None and keys:
drm_type = (drm_dicts[0].get("system", "Widevine").lower()) if drm_dicts else "widevine"
drm_obj = RemoteService._create_drm_stub(drm_type, list(keys.keys()))
if drm_obj is None:
return None
for kid_hex, key in keys.items():
drm_obj.content_keys[UUID(hex=kid_hex)] = key
return [drm_obj]
def resolve_server_keys(self, title: Title_T) -> None:
"""Inject exported keys into the selected encrypted tracks by KID (no network).
Called by dl.py after selection. Only encrypted video/audio are touched; encrypted
DASH tracks (no DRM at parse time) get a stub holding the keys, which
DASH.download_track preserves. decrypt() applies the key whose KID matches the media.
"""
pool = self.key_pool()
if not pool:
return
system = self.exported_drm_system()
kid_hexes = [kid.hex for kid in pool]
for track in title.tracks:
if not isinstance(track, (Video, Audio)) or not self.track_is_encrypted(track):
continue
drm_obj = track.drm[0] if track.drm else RemoteService._create_drm_stub(system, kid_hexes)
for kid, key in pool.items():
drm_obj.content_keys[kid] = key
track.drm = [drm_obj]
self._server_cdm_type = drm_obj.__class__.__name__.lower()
@staticmethod
def track_is_encrypted(track: Any) -> bool:
"""True if the track carries DRM or its DASH manifest declares ContentProtection."""
if track.drm:
return True
dash = track.data.get("dash") if getattr(track, "data", None) else None
if dash:
for element in (dash.get("representation"), dash.get("adaptation_set")):
if element is not None and element.findall("ContentProtection"):
return True
return False
def exported_drm_system(self) -> str:
"""The DRM system the exporter licensed (e.g. 'playready'), defaulting to widevine."""
for entry in self.titles_data.values():
for track_dict in (entry.get("tracks") or {}).values():
for drm_dict in track_dict.get("drm") or []:
if drm_dict.get("system"):
return drm_dict["system"].lower()
return "widevine"
def get_chapters(self, title: Title_T) -> Chapters:
entry = self.titles_data.get(str(title.id), {})
return Chapters(
[Chapter(ch["timestamp"], ch.get("name")) for ch in (entry.get("chapters") or []) if ch.get("timestamp")]
)
def get_widevine_service_certificate(self, **_: Any) -> Optional[str]:
return None
def get_widevine_license(self, *, challenge: bytes, title: Title_T, track: AnyTrack) -> Optional[Union[bytes, str]]:
raise RuntimeError("ImportService should not request a license; keys come from the export.")
def get_playready_license(
self, *, challenge: bytes, title: Title_T, track: AnyTrack
) -> Optional[Union[bytes, str]]:
raise RuntimeError("ImportService should not request a license; keys come from the export.")
def on_segment_downloaded(self, track: AnyTrack, segment: Any) -> None:
pass
def on_track_downloaded(self, track: AnyTrack) -> None:
pass
def on_track_decrypted(self, track: AnyTrack, drm: Any, segment: Any = None) -> None:
pass
def on_track_repacked(self, track: AnyTrack) -> None:
pass
def on_track_multiplex(self, track: AnyTrack) -> None:
pass
def close(self) -> None:
try:
self.session.close()
except Exception:
pass
reset/packages/envied/src/envied/core/manifests/__init__.py
+5
View File
@@ -0,0 +1,5 @@
from .dash import DASH
from .hls import HLS
from .ism import ISM
__all__ = ("DASH", "HLS", "ISM")
reset/packages/envied/src/envied/core/manifests/dash.py
+1057
View File
File diff suppressed because it is too large Load Diff
reset/packages/envied/src/envied/core/manifests/hls.py
+1294
View File
File diff suppressed because it is too large Load Diff
reset/packages/envied/src/envied/core/manifests/ism.py
+401
View File
@@ -0,0 +1,401 @@
from __future__ import annotations
import base64
import hashlib
import html
import urllib.parse
from functools import partial
from pathlib import Path
from typing import Any, Callable, Optional, Union
import requests
from langcodes import Language, tag_is_valid
from lxml.etree import Element
from pyplayready.system.pssh import PSSH as PR_PSSH
from pywidevine.pssh import PSSH
from requests import Session
from envied.core.constants import DOWNLOAD_CANCELLED, DOWNLOAD_LICENCE_ONLY, AnyTrack
from envied.core.drm import DRM_T, PlayReady, Widevine
from envied.core.events import events
from envied.core.session import RnetSession
from envied.core.tracks import Audio, Subtitle, Track, Tracks, Video
from envied.core.utilities import get_debug_logger, try_ensure_utf8
from envied.core.utils.xml import load_xml
class ISM:
def __init__(self, manifest: Element, url: str) -> None:
if manifest.tag != "SmoothStreamingMedia":
raise TypeError(f"Expected 'SmoothStreamingMedia' document, got '{manifest.tag}'")
if not url:
raise requests.URLRequired("ISM manifest URL must be provided for relative paths")
self.manifest = manifest
self.url = url
@classmethod
def from_url(cls, url: str, session: Optional[Union[Session, RnetSession]] = None, **kwargs: Any) -> "ISM":
if not url:
raise requests.URLRequired("ISM manifest URL must be provided")
if not session:
session = Session()
elif not isinstance(session, (Session, RnetSession)):
raise TypeError(f"Expected session to be a {Session} or {RnetSession}, not {session!r}")
res = session.get(url, **kwargs)
if res.url != url:
url = res.url
res.raise_for_status()
return cls(load_xml(res.content), url)
@classmethod
def from_text(cls, text: str, url: str) -> "ISM":
if not text:
raise ValueError("ISM manifest text must be provided")
if not url:
raise requests.URLRequired("ISM manifest URL must be provided for relative paths")
return cls(load_xml(text), url)
@staticmethod
def _get_drm(headers: list[Element]) -> list[DRM_T]:
drm: list[DRM_T] = []
for header in headers:
system_id = (header.get("SystemID") or header.get("SystemId") or "").lower()
data = "".join(header.itertext()).strip()
if not data:
continue
if system_id == "edef8ba9-79d6-4ace-a3c8-27dcd51d21ed":
try:
pssh = PSSH(base64.b64decode(data))
except Exception:
continue
kid = next(iter(pssh.key_ids), None)
drm.append(Widevine(pssh=pssh, kid=kid))
elif system_id == "9a04f079-9840-4286-ab92-e65be0885f95":
try:
pr_pssh = PR_PSSH(data)
except Exception:
continue
drm.append(PlayReady(pssh=pr_pssh, pssh_b64=data))
return drm
def to_tracks(self, language: Optional[Union[str, Language]] = None) -> Tracks:
tracks = Tracks()
base_url = self.url
duration = int(self.manifest.get("Duration") or 0)
drm = self._get_drm(self.manifest.xpath(".//ProtectionHeader"))
for stream_index in self.manifest.findall("StreamIndex"):
content_type = stream_index.get("Type")
if not content_type:
raise ValueError("No content type value could be found")
for ql in stream_index.findall("QualityLevel"):
codec = ql.get("FourCC")
if codec == "TTML":
codec = "STPP"
track_lang = None
lang = (stream_index.get("Language") or "").strip()
if lang and tag_is_valid(lang) and not lang.startswith("und"):
track_lang = Language.get(lang)
track_urls: list[str] = []
fragment_time = 0
fragments = stream_index.findall("c")
# Some manifests omit the first fragment in the <c> list but
# still expect a request for start time 0 which contains the
# initialization segment. If the first declared fragment is not
# at time 0, prepend the missing initialization URL.
if fragments:
first_time = int(fragments[0].get("t") or 0)
if first_time != 0:
track_urls.append(
urllib.parse.urljoin(
base_url,
stream_index.get("Url").format_map(
{
"bitrate": ql.get("Bitrate"),
"start time": "0",
}
),
)
)
for idx, frag in enumerate(fragments):
fragment_time = int(frag.get("t", fragment_time))
repeat = int(frag.get("r", 1))
duration_frag = int(frag.get("d") or 0)
if not duration_frag:
try:
next_time = int(fragments[idx + 1].get("t"))
except (IndexError, AttributeError):
next_time = duration
duration_frag = (next_time - fragment_time) / repeat
for _ in range(repeat):
track_urls.append(
urllib.parse.urljoin(
base_url,
stream_index.get("Url").format_map(
{
"bitrate": ql.get("Bitrate"),
"start time": str(fragment_time),
}
),
)
)
fragment_time += duration_frag
track_id = hashlib.md5(
"{codec}-{lang}-{bitrate}-{index}-{name}-{url}".format(
codec=codec,
lang=track_lang,
bitrate=ql.get("Bitrate") or 0,
index=ql.get("Index") or 0,
name=stream_index.get("Name") or "",
url=stream_index.get("Url") or "",
).encode()
).hexdigest()
data = {
"ism": {
"manifest": self.manifest,
"stream_index": stream_index,
"quality_level": ql,
"segments": track_urls,
}
}
if content_type == "video":
try:
vcodec = Video.Codec.from_mime(codec) if codec else None
except ValueError:
vcodec = None
tracks.add(
Video(
id_=track_id,
url=self.url,
codec=vcodec,
language=track_lang or language,
is_original_lang=bool(language and track_lang and str(track_lang) == str(language)),
bitrate=ql.get("Bitrate"),
width=int(ql.get("MaxWidth") or 0) or int(stream_index.get("MaxWidth") or 0),
height=int(ql.get("MaxHeight") or 0) or int(stream_index.get("MaxHeight") or 0),
descriptor=Video.Descriptor.ISM,
drm=drm,
data=data,
)
)
elif content_type == "audio":
try:
acodec = Audio.Codec.from_mime(codec) if codec else None
except ValueError:
acodec = None
tracks.add(
Audio(
id_=track_id,
url=self.url,
codec=acodec,
language=track_lang or language,
is_original_lang=bool(language and track_lang and str(track_lang) == str(language)),
bitrate=ql.get("Bitrate"),
channels=ql.get("Channels"),
descriptor=Track.Descriptor.ISM,
drm=drm,
data=data,
)
)
else:
try:
scodec = Subtitle.Codec.from_mime(codec) if codec else None
except ValueError:
scodec = None
tracks.add(
Subtitle(
id_=track_id,
url=self.url,
codec=scodec,
language=track_lang or language,
is_original_lang=bool(language and track_lang and str(track_lang) == str(language)),
descriptor=Track.Descriptor.ISM,
drm=drm,
data=data,
)
)
tracks.manifest_url = self.url
return tracks
@staticmethod
def download_track(
track: AnyTrack,
save_path: Path,
save_dir: Path,
progress: partial,
session: Optional[Session] = None,
proxy: Optional[str] = None,
max_workers: Optional[int] = None,
license_widevine: Optional[Callable] = None,
*,
cdm: Optional[object] = None,
) -> None:
if not session:
session = Session()
elif not isinstance(session, Session):
raise TypeError(f"Expected session to be a {Session}, not {session!r}")
if proxy:
session.proxies.update({"all": proxy})
segments: list[str] = track.data["ism"]["segments"]
session_drm = None
if track.drm:
# Mirror HLS.download_track: pick the DRM matching the provided CDM
# (or the first available) and license it if supported.
session_drm = track.get_drm_for_cdm(cdm)
if isinstance(session_drm, (Widevine, PlayReady)):
try:
if not license_widevine:
raise ValueError("license_widevine func must be supplied to use DRM")
progress(downloaded="LICENSING")
license_widevine(session_drm)
progress(downloaded="[yellow]LICENSED")
except Exception:
DOWNLOAD_CANCELLED.set()
progress(downloaded="[red]FAILED")
raise
if DOWNLOAD_LICENCE_ONLY.is_set():
progress(downloaded="[yellow]SKIPPED")
return
progress(total=len(segments))
downloader = track.downloader
downloader_args = dict(
urls=[{"url": url} for url in segments],
output_dir=save_dir,
filename="{i:0%d}.mp4" % len(str(len(segments))),
headers=session.headers,
cookies=session.cookies,
proxy=proxy,
max_workers=max_workers,
session=session,
)
debug_logger = get_debug_logger()
if debug_logger:
debug_logger.log(
level="DEBUG",
operation="manifest_ism_download_start",
message="Starting ISM manifest download",
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"total_segments": len(segments),
"downloader": "requests",
"has_drm": bool(session_drm),
"drm_type": session_drm.__class__.__name__ if session_drm else None,
"save_path": str(save_path),
},
)
for status_update in downloader(**downloader_args):
file_downloaded = status_update.get("file_downloaded")
if file_downloaded:
events.emit(events.Types.SEGMENT_DOWNLOADED, track=track, segment=file_downloaded)
else:
downloaded = status_update.get("downloaded")
if downloaded and downloaded.endswith("/s"):
status_update["downloaded"] = f"ISM {downloaded}"
progress(**status_update)
# Verify output directory exists and contains files
if not save_dir.exists():
error_msg = f"Output directory does not exist: {save_dir}"
if debug_logger:
debug_logger.log(
level="ERROR",
operation="manifest_ism_download_output_missing",
message=error_msg,
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"save_dir": str(save_dir),
"save_path": str(save_path),
"downloader": "requests",
},
)
raise FileNotFoundError(error_msg)
for control_file in save_dir.glob("*.!dev"):
control_file.unlink(missing_ok=True)
segments_to_merge = [x for x in sorted(save_dir.iterdir()) if x.is_file()]
if debug_logger:
debug_logger.log(
level="DEBUG",
operation="manifest_ism_download_complete",
message="ISM download complete, preparing to merge",
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"save_dir": str(save_dir),
"save_dir_exists": save_dir.exists(),
"segments_found": len(segments_to_merge),
"segment_files": [f.name for f in segments_to_merge[:10]], # Limit to first 10
"downloader": "requests",
},
)
if not segments_to_merge:
error_msg = f"No segment files found in output directory: {save_dir}"
if debug_logger:
all_contents = list(save_dir.iterdir()) if save_dir.exists() else []
debug_logger.log(
level="ERROR",
operation="manifest_ism_download_no_segments",
message=error_msg,
context={
"track_id": getattr(track, "id", None),
"track_type": track.__class__.__name__,
"save_dir": str(save_dir),
"directory_contents": [str(p) for p in all_contents],
"downloader": "requests",
},
)
raise FileNotFoundError(error_msg)
with open(save_path, "wb") as f:
for segment_file in segments_to_merge:
segment_data = segment_file.read_bytes()
if (
not session_drm
and isinstance(track, Subtitle)
and track.codec not in (Subtitle.Codec.fVTT, Subtitle.Codec.fTTML)
):
segment_data = try_ensure_utf8(segment_data)
segment_data = (
segment_data.decode("utf8")
.replace("&lrm;", html.unescape("&lrm;"))
.replace("&rlm;", html.unescape("&rlm;"))
.encode("utf8")
)
f.write(segment_data)
f.flush()
segment_file.unlink()
progress(advance=1)
track.path = save_path
events.emit(events.Types.TRACK_DOWNLOADED, track=track)
if session_drm:
progress(downloaded="Decrypting", completed=0, total=100)
session_drm.decrypt(save_path)
track.drm = None
events.emit(events.Types.TRACK_DECRYPTED, track=track, drm=session_drm, segment=None)
progress(downloaded="Decrypting", advance=100)
save_dir.rmdir()
progress(downloaded="Downloaded")
__all__ = ("ISM",)
reset/packages/envied/src/envied/core/manifests/m3u8.py
+35
View File
@@ -0,0 +1,35 @@
"""Utility functions for parsing M3U8 playlists."""
from __future__ import annotations
from typing import Optional, Union
import m3u8
from requests import Session
from envied.core.manifests.hls import HLS
from envied.core.session import RnetSession
from envied.core.tracks import Tracks
def parse(
master: m3u8.M3U8,
language: str,
*,
session: Optional[Union[Session, RnetSession]] = None,
url: Optional[str] = None,
) -> Tracks:
"""Parse a variant playlist to ``Tracks`` with basic information, defer DRM loading."""
tracks = HLS(master, session=session, url=url).to_tracks(language)
bool(master.session_keys or HLS.parse_session_data_keys(master, session or Session()))
if True:
for t in tracks.videos + tracks.audio:
t.needs_drm_loading = True
t.session = session
return tracks
__all__ = ["parse"]
reset/packages/envied/src/envied/core/providers/__init__.py
+428
View File
@@ -0,0 +1,428 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Optional
import requests
from envied.core.providers._base import ExternalIds, MetadataProvider, MetadataResult, fuzzy_match, log
from envied.core.providers.imdbapi import IMDBApiProvider
from envied.core.providers.simkl import SimklProvider
from envied.core.providers.tmdb import TMDBProvider
if TYPE_CHECKING:
from envied.core.title_cacher import TitleCacher
# Ordered by priority: IMDBApi (free), SIMKL, TMDB
ALL_PROVIDERS: list[type[MetadataProvider]] = [IMDBApiProvider, SimklProvider, TMDBProvider]
def get_available_providers() -> list[MetadataProvider]:
"""Return instantiated providers that have valid credentials."""
return [cls() for cls in ALL_PROVIDERS if cls().is_available()]
def get_provider(name: str) -> Optional[MetadataProvider]:
"""Get a specific provider by name."""
for cls in ALL_PROVIDERS:
if cls.NAME == name:
p = cls()
return p if p.is_available() else None
return None
# -- Public API (replaces tags.py functions) --
def search_metadata(
title: str,
year: Optional[int],
kind: str,
title_cacher: Optional[TitleCacher] = None,
cache_title_id: Optional[str] = None,
cache_region: Optional[str] = None,
cache_account_hash: Optional[str] = None,
) -> Optional[MetadataResult]:
"""Search all available providers for metadata. Returns best match."""
# Check cache first
if title_cacher and cache_title_id:
for cls in ALL_PROVIDERS:
p = cls()
if not p.is_available():
continue
cached = title_cacher.get_cached_provider(p.NAME, cache_title_id, kind, cache_region, cache_account_hash)
if cached:
result = _cached_to_result(cached, p.NAME, kind)
if result and result.title and fuzzy_match(result.title, title):
log.debug("Using cached %s data for %r", p.NAME, title)
return result
# Search providers in priority order
for cls in ALL_PROVIDERS:
p = cls()
if not p.is_available():
continue
try:
result = p.search(title, year, kind)
except (requests.RequestException, ValueError, KeyError) as exc:
log.debug("%s search failed: %s", p.NAME, exc)
continue
if result and result.title and fuzzy_match(result.title, title):
# Enrich with cross-referenced IDs if we have IMDB but missing TMDB/TVDB
enrich_ids(result)
# Cache the result (include enriched IDs so they survive round-trip)
if title_cacher and cache_title_id and result.raw:
try:
cache_data = result.raw
if result.external_ids.tmdb_id or result.external_ids.tvdb_id:
cache_data = {
**result.raw,
"_enriched_ids": _external_ids_to_dict(result.external_ids),
}
title_cacher.cache_provider(
p.NAME, cache_title_id, cache_data, kind, cache_region, cache_account_hash
)
except Exception as exc:
log.debug("Failed to cache %s data: %s", p.NAME, exc)
return result
return None
def get_title_by_id(
tmdb_id: int,
kind: str,
title_cacher: Optional[TitleCacher] = None,
cache_title_id: Optional[str] = None,
cache_region: Optional[str] = None,
cache_account_hash: Optional[str] = None,
) -> Optional[str]:
"""Get title name by TMDB ID."""
# Check cache first
if title_cacher and cache_title_id:
cached = title_cacher.get_cached_provider("tmdb", cache_title_id, kind, cache_region, cache_account_hash)
if cached and cached.get("detail"):
detail = cached["detail"]
tmdb_title = detail.get("title") or detail.get("name")
if tmdb_title:
log.debug("Using cached TMDB title: %r", tmdb_title)
return tmdb_title
tmdb = get_provider("tmdb")
if not tmdb:
return None
result = tmdb.get_by_id(tmdb_id, kind)
if not result:
return None
# Cache if possible
if title_cacher and cache_title_id and result.raw:
try:
ext_ids = tmdb.get_external_ids(tmdb_id, kind)
title_cacher.cache_provider(
"tmdb",
cache_title_id,
{"detail": result.raw, "external_ids": _external_ids_to_dict(ext_ids)},
kind,
cache_region,
cache_account_hash,
)
except Exception as exc:
log.debug("Failed to cache TMDB data: %s", exc)
return result.title
def get_year_by_id(
tmdb_id: int,
kind: str,
title_cacher: Optional[TitleCacher] = None,
cache_title_id: Optional[str] = None,
cache_region: Optional[str] = None,
cache_account_hash: Optional[str] = None,
) -> Optional[int]:
"""Get release year by TMDB ID."""
# Check cache first
if title_cacher and cache_title_id:
cached = title_cacher.get_cached_provider("tmdb", cache_title_id, kind, cache_region, cache_account_hash)
if cached and cached.get("detail"):
detail = cached["detail"]
date = detail.get("release_date") or detail.get("first_air_date")
if date and len(date) >= 4 and date[:4].isdigit():
year = int(date[:4])
log.debug("Using cached TMDB year: %d", year)
return year
tmdb = get_provider("tmdb")
if not tmdb:
return None
result = tmdb.get_by_id(tmdb_id, kind)
if not result:
return None
# Cache if possible
if title_cacher and cache_title_id and result.raw:
try:
ext_ids = tmdb.get_external_ids(tmdb_id, kind)
title_cacher.cache_provider(
"tmdb",
cache_title_id,
{"detail": result.raw, "external_ids": _external_ids_to_dict(ext_ids)},
kind,
cache_region,
cache_account_hash,
)
except Exception as exc:
log.debug("Failed to cache TMDB data: %s", exc)
return result.year
def fetch_external_ids(
tmdb_id: int,
kind: str,
title_cacher: Optional[TitleCacher] = None,
cache_title_id: Optional[str] = None,
cache_region: Optional[str] = None,
cache_account_hash: Optional[str] = None,
) -> ExternalIds:
"""Get external IDs by TMDB ID."""
# Check cache first
if title_cacher and cache_title_id:
cached = title_cacher.get_cached_provider("tmdb", cache_title_id, kind, cache_region, cache_account_hash)
if cached and cached.get("external_ids"):
log.debug("Using cached TMDB external IDs")
raw = cached["external_ids"]
return ExternalIds(
imdb_id=raw.get("imdb_id"),
tmdb_id=tmdb_id,
tmdb_kind=kind,
tvdb_id=raw.get("tvdb_id"),
)
tmdb = get_provider("tmdb")
if not tmdb:
return ExternalIds()
ext = tmdb.get_external_ids(tmdb_id, kind)
# Cache if possible
if title_cacher and cache_title_id:
try:
detail = None
result = tmdb.get_by_id(tmdb_id, kind)
if result and result.raw:
detail = result.raw
if detail:
title_cacher.cache_provider(
"tmdb",
cache_title_id,
{"detail": detail, "external_ids": _external_ids_to_dict(ext)},
kind,
cache_region,
cache_account_hash,
)
except Exception as exc:
log.debug("Failed to cache TMDB data: %s", exc)
return ext
# -- Internal helpers --
# Provider authority ranking for tie-breaking (lower index = more authoritative)
_ENRICHMENT_PROVIDERS = ("tmdb", "simkl")
_ENRICHMENT_AUTHORITY: dict[str, int] = {name: i for i, name in enumerate(_ENRICHMENT_PROVIDERS)}
def enrich_ids(result: MetadataResult) -> None:
"""Enrich a MetadataResult by cross-referencing IMDB ID with available providers.
Queries all available providers, cross-validates tmdb_id as anchor.
If a provider returns a different tmdb_id than the authoritative source,
ALL of that provider's data is dropped (likely resolved to wrong title).
"""
ids = result.external_ids
if not ids.imdb_id:
return
if ids.tmdb_id and ids.tvdb_id:
return # already have everything
kind = result.kind or "movie"
# Step 1: Collect enrichment results from all available providers
enrichments: list[tuple[str, ExternalIds]] = []
for provider_name in _ENRICHMENT_PROVIDERS:
p = get_provider(provider_name)
if not p:
continue
try:
enriched = p.find_by_imdb_id(ids.imdb_id, kind) # type: ignore[union-attr]
except Exception as exc:
log.debug("Enrichment via %s failed: %s", provider_name, exc)
continue
if enriched:
enrichments.append((provider_name, enriched))
if not enrichments:
return
# Step 2: Cross-validate using tmdb_id as anchor — drop providers that disagree
validated = _validate_enrichments(enrichments)
# Step 3: Merge validated data (fill gaps only)
for _provider_name, ext in validated:
if not ids.tmdb_id and ext.tmdb_id:
ids.tmdb_id = ext.tmdb_id
ids.tmdb_kind = ext.tmdb_kind or kind
if not ids.tvdb_id and ext.tvdb_id:
ids.tvdb_id = ext.tvdb_id
def _validate_enrichments(
enrichments: list[tuple[str, ExternalIds]],
) -> list[tuple[str, ExternalIds]]:
"""Drop providers whose tmdb_id conflicts with the authoritative value.
If providers disagree on tmdb_id, the more authoritative source wins
and ALL data from disagreeing providers is discarded (different tmdb_id
means the provider likely resolved to a different title entirely).
"""
from collections import Counter
# Collect tmdb_id votes
tmdb_votes: dict[str, int] = {}
for provider_name, ext in enrichments:
if ext.tmdb_id is not None:
tmdb_votes[provider_name] = ext.tmdb_id
if len(set(tmdb_votes.values())) <= 1:
return enrichments # all agree or only one voted — no conflict
# Find the authoritative tmdb_id
value_counts = Counter(tmdb_votes.values())
most_common_val, most_common_count = value_counts.most_common(1)[0]
if most_common_count > 1:
anchor_tmdb_id = most_common_val
else:
# No majority — pick the most authoritative provider
best_provider = min(
tmdb_votes.keys(),
key=lambda name: _ENRICHMENT_AUTHORITY.get(name, 99),
)
anchor_tmdb_id = tmdb_votes[best_provider]
# Drop any provider that disagrees
validated: list[tuple[str, ExternalIds]] = []
for provider_name, ext in enrichments:
if ext.tmdb_id is not None and ext.tmdb_id != anchor_tmdb_id:
log.debug(
"Dropping %s enrichment data: tmdb_id %s conflicts with "
"authoritative value %s (likely resolved to wrong title)",
provider_name,
ext.tmdb_id,
anchor_tmdb_id,
)
continue
validated.append((provider_name, ext))
return validated
def _external_ids_to_dict(ext: ExternalIds) -> dict:
"""Convert ExternalIds to a dict for caching."""
result: dict = {}
if ext.imdb_id:
result["imdb_id"] = ext.imdb_id
if ext.tmdb_id:
result["tmdb_id"] = ext.tmdb_id
if ext.tmdb_kind:
result["tmdb_kind"] = ext.tmdb_kind
if ext.tvdb_id:
result["tvdb_id"] = ext.tvdb_id
return result
def _cached_to_result(cached: dict, provider_name: str, kind: str) -> Optional[MetadataResult]:
"""Convert a cached provider dict back to a MetadataResult."""
if provider_name == "tmdb":
detail = cached.get("detail", {})
ext_raw = cached.get("external_ids", {})
title = detail.get("title") or detail.get("name")
date = detail.get("release_date") or detail.get("first_air_date")
year = int(date[:4]) if date and len(date) >= 4 and date[:4].isdigit() else None
tmdb_id = detail.get("id")
return MetadataResult(
title=title,
year=year,
kind=kind,
external_ids=ExternalIds(
imdb_id=ext_raw.get("imdb_id"),
tmdb_id=tmdb_id,
tmdb_kind=kind,
tvdb_id=ext_raw.get("tvdb_id"),
),
source="tmdb",
raw=cached,
)
elif provider_name == "simkl":
response = cached.get("response", cached)
if response.get("type") == "episode" and "show" in response:
info = response["show"]
elif response.get("type") == "movie" and "movie" in response:
info = response["movie"]
else:
return None
ids = info.get("ids", {})
tmdb_id = ids.get("tmdbtv") or ids.get("tmdb") or ids.get("moviedb")
if tmdb_id:
tmdb_id = int(tmdb_id)
return MetadataResult(
title=info.get("title"),
year=info.get("year"),
kind=kind,
external_ids=ExternalIds(
imdb_id=ids.get("imdb"),
tmdb_id=tmdb_id,
tmdb_kind=kind,
tvdb_id=ids.get("tvdb"),
),
source="simkl",
raw=cached,
)
elif provider_name == "imdbapi":
title = cached.get("primaryTitle") or cached.get("originalTitle")
year = cached.get("startYear")
imdb_id = cached.get("id")
# Restore enriched IDs that were saved alongside the raw data
enriched = cached.get("_enriched_ids", {})
return MetadataResult(
title=title,
year=year,
kind=kind,
external_ids=ExternalIds(
imdb_id=imdb_id,
tmdb_id=enriched.get("tmdb_id"),
tmdb_kind=enriched.get("tmdb_kind"),
tvdb_id=enriched.get("tvdb_id"),
),
source="imdbapi",
raw=cached,
)
return None
__all__ = [
"ALL_PROVIDERS",
"ExternalIds",
"MetadataProvider",
"MetadataResult",
"enrich_ids",
"fetch_external_ids",
"fuzzy_match",
"get_available_providers",
"get_provider",
"get_title_by_id",
"get_year_by_id",
"search_metadata",
]
reset/packages/envied/src/envied/core/providers/_base.py
+97
View File
@@ -0,0 +1,97 @@
from __future__ import annotations
import logging
import re
from abc import ABCMeta, abstractmethod
from dataclasses import dataclass, field
from difflib import SequenceMatcher
from typing import Optional, Union
import requests
from requests.adapters import HTTPAdapter, Retry
log = logging.getLogger("METADATA")
HEADERS = {"User-Agent": "envied.tags/1.0"}
STRIP_RE = re.compile(r"[^a-z0-9]+", re.I)
YEAR_RE = re.compile(r"\s*\(?[12][0-9]{3}\)?$")
@dataclass
class ExternalIds:
"""Normalized external IDs across providers."""
imdb_id: Optional[str] = None
tmdb_id: Optional[int] = None
tmdb_kind: Optional[str] = None # "movie" or "tv"
tvdb_id: Optional[int] = None
@dataclass
class MetadataResult:
"""Unified metadata result from any provider."""
title: Optional[str] = None
year: Optional[int] = None
kind: Optional[str] = None # "movie" or "tv"
external_ids: ExternalIds = field(default_factory=ExternalIds)
source: str = "" # provider name, e.g. "tmdb", "simkl", "imdbapi"
raw: Optional[dict] = None # original API response for caching
class MetadataProvider(metaclass=ABCMeta):
"""Abstract base for metadata providers."""
NAME: str = ""
REQUIRES_KEY: bool = True
def __init__(self) -> None:
self.log = logging.getLogger(f"METADATA.{self.NAME.upper()}")
self._session: Optional[requests.Session] = None
@property
def session(self) -> requests.Session:
if self._session is None:
self._session = requests.Session()
self._session.headers.update(HEADERS)
retry = Retry(
total=3,
backoff_factor=1,
status_forcelist=[429, 500, 502, 503, 504],
allowed_methods=["GET", "POST"],
)
adapter = HTTPAdapter(max_retries=retry)
self._session.mount("https://", adapter)
self._session.mount("http://", adapter)
return self._session
@abstractmethod
def is_available(self) -> bool:
"""Return True if this provider has the credentials/keys it needs."""
@abstractmethod
def search(self, title: str, year: Optional[int], kind: str) -> Optional[MetadataResult]:
"""Search for a title and return metadata, or None on failure/no match."""
@abstractmethod
def get_by_id(self, provider_id: Union[int, str], kind: str) -> Optional[MetadataResult]:
"""Fetch metadata by this provider's native ID."""
@abstractmethod
def get_external_ids(self, provider_id: Union[int, str], kind: str) -> ExternalIds:
"""Fetch external IDs for a title by this provider's native ID."""
def _clean(s: str) -> str:
return STRIP_RE.sub("", s).lower()
def _strip_year(s: str) -> str:
return YEAR_RE.sub("", s).strip()
def fuzzy_match(a: str, b: str, threshold: float = 0.8) -> bool:
"""Return True if ``a`` and ``b`` are a close match."""
ratio = SequenceMatcher(None, _clean(a), _clean(b)).ratio()
return ratio >= threshold
reset/packages/envied/src/envied/core/providers/imdbapi.py
+123
View File
@@ -0,0 +1,123 @@
from __future__ import annotations
from difflib import SequenceMatcher
from typing import Optional, Union
import requests
from envied.core.providers._base import ExternalIds, MetadataProvider, MetadataResult, _clean, fuzzy_match
# Mapping from our kind ("movie"/"tv") to imdbapi.dev title types
KIND_TO_TYPES: dict[str, list[str]] = {
"movie": ["movie"],
"tv": ["tvSeries", "tvMiniSeries"],
}
class IMDBApiProvider(MetadataProvider):
"""IMDb metadata provider using imdbapi.dev (free, no API key)."""
NAME = "imdbapi"
REQUIRES_KEY = False
BASE_URL = "https://api.imdbapi.dev"
def is_available(self) -> bool:
return True # no key needed
def search(self, title: str, year: Optional[int], kind: str) -> Optional[MetadataResult]:
self.log.debug("Searching IMDBApi for %r (%s, %s)", title, kind, year)
try:
params: dict[str, str | int] = {"query": title, "limit": 20}
r = self.session.get(
f"{self.BASE_URL}/search/titles",
params=params,
timeout=30,
)
r.raise_for_status()
data = r.json()
except (requests.RequestException, ValueError) as exc:
self.log.debug("IMDBApi search failed: %s", exc)
return None
results = data.get("titles") or data.get("results") or []
if not results:
self.log.debug("IMDBApi returned no results for %r", title)
return None
# Filter by type if possible
type_filter = KIND_TO_TYPES.get(kind, [])
filtered = [r for r in results if r.get("type") in type_filter] if type_filter else results
candidates = filtered if filtered else results
# Find best fuzzy match, optionally filtered by year
best_match: Optional[dict] = None
best_ratio = 0.0
for candidate in candidates:
primary = candidate.get("primaryTitle") or ""
original = candidate.get("originalTitle") or ""
for name in [primary, original]:
if not name:
continue
ratio = SequenceMatcher(None, _clean(title), _clean(name)).ratio()
if ratio > best_ratio:
# If year provided, prefer matches within 1 year
candidate_year = candidate.get("startYear")
if year and candidate_year and abs(year - candidate_year) > 1:
continue
best_ratio = ratio
best_match = candidate
if not best_match:
self.log.debug("No matching result found in IMDBApi for %r", title)
return None
result_title = best_match.get("primaryTitle") or best_match.get("originalTitle")
if not result_title or not fuzzy_match(result_title, title):
self.log.debug("IMDBApi title mismatch: searched %r, got %r", title, result_title)
return None
imdb_id = best_match.get("id")
result_year = best_match.get("startYear")
self.log.debug("IMDBApi -> %s (ID %s)", result_title, imdb_id)
return MetadataResult(
title=result_title,
year=result_year,
kind=kind,
external_ids=ExternalIds(imdb_id=imdb_id),
source="imdbapi",
raw=best_match,
)
def get_by_id(self, provider_id: Union[int, str], kind: str) -> Optional[MetadataResult]:
"""Fetch metadata by IMDB ID (e.g. 'tt1375666')."""
imdb_id = str(provider_id)
self.log.debug("Fetching IMDBApi title %s", imdb_id)
try:
r = self.session.get(f"{self.BASE_URL}/titles/{imdb_id}", timeout=30)
r.raise_for_status()
data = r.json()
except (requests.RequestException, ValueError) as exc:
self.log.debug("IMDBApi get_by_id failed: %s", exc)
return None
title = data.get("primaryTitle") or data.get("originalTitle")
result_year = data.get("startYear")
return MetadataResult(
title=title,
year=result_year,
kind=kind,
external_ids=ExternalIds(imdb_id=data.get("id")),
source="imdbapi",
raw=data,
)
def get_external_ids(self, provider_id: Union[int, str], kind: str) -> ExternalIds:
"""Return external IDs. For IMDB, the provider_id IS the IMDB ID."""
return ExternalIds(imdb_id=str(provider_id))
reset/packages/envied/src/envied/core/providers/simkl.py
+172
View File
@@ -0,0 +1,172 @@
from __future__ import annotations
from typing import Optional, Union
import requests
from envied.core.config import config
from envied.core.providers._base import ExternalIds, MetadataProvider, MetadataResult, fuzzy_match
class SimklProvider(MetadataProvider):
"""SIMKL metadata provider (filename-based search)."""
NAME = "simkl"
REQUIRES_KEY = True
BASE_URL = "https://api.simkl.com"
def is_available(self) -> bool:
return bool(config.simkl_client_id)
def search(self, title: str, year: Optional[int], kind: str) -> Optional[MetadataResult]:
self.log.debug("Searching Simkl for %r (%s, %s)", title, kind, year)
# Construct appropriate filename based on type
filename = f"{title}"
if year:
filename = f"{title} {year}"
if kind == "tv":
filename += " S01E01.mkv"
else:
filename += " 2160p.mkv"
try:
headers = {"simkl-api-key": config.simkl_client_id}
resp = self.session.post(
f"{self.BASE_URL}/search/file", json={"file": filename}, headers=headers, timeout=30
)
resp.raise_for_status()
data = resp.json()
self.log.debug("Simkl API response received")
except (requests.RequestException, ValueError) as exc:
self.log.debug("Simkl search failed: %s", exc)
return None
# Handle case where SIMKL returns empty list (no results)
if isinstance(data, list):
self.log.debug("Simkl returned list (no matches) for %r", filename)
return None
return self._parse_response(data, title, year, kind)
def get_by_id(self, provider_id: Union[int, str], kind: str) -> Optional[MetadataResult]:
return None # SIMKL has no direct ID lookup used here
def get_external_ids(self, provider_id: Union[int, str], kind: str) -> ExternalIds:
return ExternalIds() # IDs come from search() response
def find_by_imdb_id(self, imdb_id: str, kind: str) -> Optional[ExternalIds]:
"""Look up TMDB/TVDB IDs from an IMDB ID using SIMKL's /search/id and detail endpoints."""
self.log.debug("Looking up IMDB ID %s on SIMKL", imdb_id)
headers = {"simkl-api-key": config.simkl_client_id}
try:
r = self.session.get(f"{self.BASE_URL}/search/id", params={"imdb": imdb_id}, headers=headers, timeout=30)
r.raise_for_status()
data = r.json()
except (requests.RequestException, ValueError) as exc:
self.log.debug("SIMKL search/id failed: %s", exc)
return None
if not isinstance(data, list) or not data:
self.log.debug("No SIMKL results for IMDB ID %s", imdb_id)
return None
entry = data[0]
simkl_id = entry.get("ids", {}).get("simkl")
if not simkl_id:
return None
# Map SIMKL type to endpoint
simkl_type = entry.get("type", "")
endpoint = "tv" if simkl_type in ("tv", "anime") else "movies"
# Fetch full details to get cross-referenced IDs
try:
r2 = self.session.get(
f"{self.BASE_URL}/{endpoint}/{simkl_id}",
params={"extended": "full"},
headers=headers,
timeout=30,
)
r2.raise_for_status()
detail = r2.json()
except (requests.RequestException, ValueError) as exc:
self.log.debug("SIMKL detail fetch failed: %s", exc)
return None
ids = detail.get("ids", {})
tmdb_id: Optional[int] = None
raw_tmdb = ids.get("tmdb")
if raw_tmdb:
tmdb_id = int(raw_tmdb)
tvdb_id: Optional[int] = None
raw_tvdb = ids.get("tvdb")
if raw_tvdb:
tvdb_id = int(raw_tvdb)
self.log.debug("SIMKL find -> TMDB %s, TVDB %s for IMDB %s", tmdb_id, tvdb_id, imdb_id)
return ExternalIds(
imdb_id=imdb_id,
tmdb_id=tmdb_id,
tmdb_kind=kind,
tvdb_id=tvdb_id,
)
def _parse_response(
self, data: dict, search_title: str, search_year: Optional[int], kind: str
) -> Optional[MetadataResult]:
"""Parse a SIMKL response into a MetadataResult."""
if data.get("type") == "episode" and "show" in data:
info = data["show"]
content_type = "tv"
elif data.get("type") == "movie" and "movie" in data:
info = data["movie"]
content_type = "movie"
else:
return None
result_title = info.get("title")
result_year = info.get("year")
# Verify title matches
if not result_title or not fuzzy_match(result_title, search_title):
self.log.debug("Simkl title mismatch: searched %r, got %r", search_title, result_title)
return None
# Verify year if provided (allow 1 year difference)
if search_year and result_year and abs(search_year - result_year) > 1:
self.log.debug("Simkl year mismatch: searched %d, got %d", search_year, result_year)
return None
ids = info.get("ids", {})
tmdb_id: Optional[int] = None
if content_type == "tv":
raw_tmdb = ids.get("tmdbtv")
else:
raw_tmdb = ids.get("tmdb") or ids.get("moviedb")
if raw_tmdb:
tmdb_id = int(raw_tmdb)
tvdb_id: Optional[int] = None
raw_tvdb = ids.get("tvdb")
if raw_tvdb:
tvdb_id = int(raw_tvdb)
self.log.debug("Simkl -> %s (TMDB ID %s)", result_title, tmdb_id)
return MetadataResult(
title=result_title,
year=result_year,
kind=kind,
external_ids=ExternalIds(
imdb_id=ids.get("imdb"),
tmdb_id=tmdb_id,
tmdb_kind=kind,
tvdb_id=tvdb_id,
),
source="simkl",
raw=data,
)
reset/packages/envied/src/envied/core/providers/tmdb.py
+199
View File
@@ -0,0 +1,199 @@
from __future__ import annotations
from difflib import SequenceMatcher
from typing import Optional, Union
import requests
from envied.core.config import config
from envied.core.providers._base import ExternalIds, MetadataProvider, MetadataResult, _clean, _strip_year
class TMDBProvider(MetadataProvider):
"""TMDB (The Movie Database) metadata provider."""
NAME = "tmdb"
REQUIRES_KEY = True
BASE_URL = "https://api.themoviedb.org/3"
def is_available(self) -> bool:
return bool(config.tmdb_api_key)
@property
def _api_key(self) -> str:
return config.tmdb_api_key
def search(self, title: str, year: Optional[int], kind: str) -> Optional[MetadataResult]:
search_title = _strip_year(title)
self.log.debug("Searching TMDB for %r (%s, %s)", search_title, kind, year)
params: dict[str, str | int] = {"api_key": self._api_key, "query": search_title}
if year is not None:
params["year" if kind == "movie" else "first_air_date_year"] = year
try:
r = self.session.get(f"{self.BASE_URL}/search/{kind}", params=params, timeout=30)
r.raise_for_status()
results = r.json().get("results") or []
self.log.debug("TMDB returned %d results", len(results))
if not results:
return None
except requests.RequestException as exc:
self.log.warning("Failed to search TMDB for %s: %s", title, exc)
return None
best_ratio = 0.0
best_id: Optional[int] = None
best_title: Optional[str] = None
for result in results:
candidates = [
result.get("title"),
result.get("name"),
result.get("original_title"),
result.get("original_name"),
]
candidates = [c for c in candidates if c]
for candidate in candidates:
ratio = SequenceMatcher(None, _clean(search_title), _clean(candidate)).ratio()
if ratio > best_ratio:
best_ratio = ratio
best_id = result.get("id")
best_title = candidate
self.log.debug("Best candidate ratio %.2f for %r (ID %s)", best_ratio, best_title, best_id)
if best_id is None:
first = results[0]
best_id = first.get("id")
best_title = first.get("title") or first.get("name")
if best_id is None:
return None
# Fetch full detail for caching
detail = self._fetch_detail(best_id, kind)
ext_raw = self._fetch_external_ids_raw(best_id, kind)
date = (detail or {}).get("release_date") or (detail or {}).get("first_air_date")
result_year = int(date[:4]) if date and len(date) >= 4 and date[:4].isdigit() else None
ext = ExternalIds(
imdb_id=ext_raw.get("imdb_id") if ext_raw else None,
tmdb_id=best_id,
tmdb_kind=kind,
tvdb_id=ext_raw.get("tvdb_id") if ext_raw else None,
)
return MetadataResult(
title=best_title,
year=result_year,
kind=kind,
external_ids=ext,
source="tmdb",
raw={"detail": detail or {}, "external_ids": ext_raw or {}},
)
def get_by_id(self, provider_id: Union[int, str], kind: str) -> Optional[MetadataResult]:
detail = self._fetch_detail(int(provider_id), kind)
if not detail:
return None
title = detail.get("title") or detail.get("name")
date = detail.get("release_date") or detail.get("first_air_date")
year = int(date[:4]) if date and len(date) >= 4 and date[:4].isdigit() else None
return MetadataResult(
title=title,
year=year,
kind=kind,
external_ids=ExternalIds(tmdb_id=int(provider_id), tmdb_kind=kind),
source="tmdb",
raw=detail,
)
def get_external_ids(self, provider_id: Union[int, str], kind: str) -> ExternalIds:
raw = self._fetch_external_ids_raw(int(provider_id), kind)
if not raw:
return ExternalIds(tmdb_id=int(provider_id), tmdb_kind=kind)
return ExternalIds(
imdb_id=raw.get("imdb_id"),
tmdb_id=int(provider_id),
tmdb_kind=kind,
tvdb_id=raw.get("tvdb_id"),
)
def find_by_imdb_id(self, imdb_id: str, kind: str) -> Optional[ExternalIds]:
"""Look up TMDB/TVDB IDs from an IMDB ID using TMDB's /find endpoint."""
self.log.debug("Looking up IMDB ID %s on TMDB", imdb_id)
try:
r = self.session.get(
f"{self.BASE_URL}/find/{imdb_id}",
params={"api_key": self._api_key, "external_source": "imdb_id"},
timeout=30,
)
r.raise_for_status()
data = r.json()
except requests.RequestException as exc:
self.log.debug("TMDB find by IMDB ID failed: %s", exc)
return None
# Check movie_results or tv_results based on kind
if kind == "movie":
results = data.get("movie_results") or []
else:
results = data.get("tv_results") or []
if not results:
# Try the other type as fallback
fallback_key = "tv_results" if kind == "movie" else "movie_results"
results = data.get(fallback_key) or []
if results:
kind = "tv" if kind == "movie" else "movie"
if not results:
self.log.debug("No TMDB results found for IMDB ID %s", imdb_id)
return None
match = results[0]
tmdb_id = match.get("id")
if not tmdb_id:
return None
self.log.debug("TMDB find -> ID %s (%s) for IMDB %s", tmdb_id, kind, imdb_id)
# Now fetch the full external IDs from TMDB to get TVDB etc.
ext_raw = self._fetch_external_ids_raw(tmdb_id, kind)
return ExternalIds(
imdb_id=imdb_id,
tmdb_id=tmdb_id,
tmdb_kind=kind,
tvdb_id=ext_raw.get("tvdb_id") if ext_raw else None,
)
def _fetch_detail(self, tmdb_id: int, kind: str) -> Optional[dict]:
try:
r = self.session.get(
f"{self.BASE_URL}/{kind}/{tmdb_id}",
params={"api_key": self._api_key},
timeout=30,
)
r.raise_for_status()
return r.json()
except requests.RequestException as exc:
self.log.debug("Failed to fetch TMDB detail: %s", exc)
return None
def _fetch_external_ids_raw(self, tmdb_id: int, kind: str) -> Optional[dict]:
try:
r = self.session.get(
f"{self.BASE_URL}/{kind}/{tmdb_id}/external_ids",
params={"api_key": self._api_key},
timeout=30,
)
r.raise_for_status()
return r.json()
except requests.RequestException as exc:
self.log.debug("Failed to fetch TMDB external IDs: %s", exc)
return None
reset/packages/envied/src/envied/core/proxies/__init__.py
+8
View File
@@ -0,0 +1,8 @@
from .basic import Basic
from .gluetun import Gluetun
from .hola import Hola
from .nordvpn import NordVPN
from .surfsharkvpn import SurfsharkVPN
from .windscribevpn import WindscribeVPN
__all__ = ("Basic", "Gluetun", "Hola", "NordVPN", "SurfsharkVPN", "WindscribeVPN")
reset/packages/envied/src/envied/core/proxies/basic.py
+54
View File
@@ -0,0 +1,54 @@
import random
import re
from typing import Optional, Union
from requests.utils import prepend_scheme_if_needed
from urllib3.util import parse_url
from envied.core.proxies.proxy import Proxy
class Basic(Proxy):
def __init__(self, **countries: dict[str, Union[str, list[str]]]):
"""Basic Proxy Service using Proxies specified in the config."""
self.countries = {k.lower(): v for k, v in countries.items()}
def __repr__(self) -> str:
countries = len(self.countries)
servers = len(self.countries.values())
return f"{countries} Countr{['ies', 'y'][countries == 1]} ({servers} Server{['s', ''][servers == 1]})"
def get_proxy(self, query: str) -> Optional[str]:
"""Get a proxy URI from the config."""
query = query.lower()
match = re.match(r"^([a-z]{2})(\d+)?$", query, re.IGNORECASE)
if not match:
raise ValueError(f'The query "{query}" was not recognized...')
country_code = match.group(1)
entry = match.group(2)
servers: Optional[Union[str, list[str]]] = self.countries.get(country_code)
if not servers:
return None
if isinstance(servers, str):
proxy = servers
elif entry:
try:
proxy = servers[int(entry) - 1]
except IndexError:
raise ValueError(
f'There\'s only {len(servers)} prox{"y" if len(servers) == 1 else "ies"} for "{country_code}"...'
)
else:
proxy = random.choice(servers)
proxy = prepend_scheme_if_needed(proxy, "http")
parsed_proxy = parse_url(proxy)
if not parsed_proxy.host:
raise ValueError(f"The proxy '{proxy}' is not a valid proxy URI supported by Python-Requests.")
return proxy
reset/packages/envied/src/envied/core/proxies/gluetun.py
+1409
View File
File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More