From 36401d9b3d671fd4bb56189fed3fa657f5142d60 Mon Sep 17 00:00:00 2001 From: Nirvana Date: Sat, 17 Oct 2026 12:43:25 +0200 Subject: [PATCH] zdf auth --- lib/streaming_providers/providers/zdf/auth.py | 119 ++++++++++++++---- .../providers/zdf/channel_manager.py | 41 ++++-- .../providers/zdf/constants.py | 74 +++++++---- .../providers/zdf/models.py | 66 ++++++++-- .../providers/zdf/provider.py | 13 +- 5 files changed, 240 insertions(+), 73 deletions(-) diff --git a/lib/streaming_providers/providers/zdf/auth.py b/lib/streaming_providers/providers/zdf/auth.py index a3a798f..beffcb2 100644 --- a/lib/streaming_providers/providers/zdf/auth.py +++ b/lib/streaming_providers/providers/zdf/auth.py @@ -1,37 +1,112 @@ # streaming_providers/providers/zdf/auth.py """ -ZDF auth -- Variant A (README §9), degenerate on purpose. +ZDF auth -- Variant A (README §9): lazy, no network in __init__. -ZDF live TV needs no user account: the API only wants the public web-client -bearer tokens (constants.py / config override). There is nothing to log in to, -nothing to refresh and nothing persisted, so no lock, no expiry and no -retry/backoff exist here. If ZDF ever rotates the tokens, the symptom is an -HTTP 401/403 from the HTTP manager; the fix is a config override. +ZDF live TV needs no user account. The API wants two expiring, public client +tokens of the web player, which the start page embeds in its Next.js flight +data (`videoToken` for GraphQL POST + PTMD, `appToken` for the channel list). + +Behaviour: + * tokens are fetched on first use and cached with their expiry; they are + renewed TOKEN_REFRESH_BUFFER_SECONDS before they run out. One lock makes + concurrent callers share a single fetch. + * if the page fetch or parse fails, the static fallback tokens of the config + are used and no new fetch is tried for TOKEN_FETCH_BACKOFF_SECONDS (no + request storm while zdf.de is down or its page layout changed). + * `refresh_after_rejection()` is the one-shot recovery for a request the API + refused: drop the cache, fetch again, and say whether the tokens changed. + The manager retries exactly once, and only if they did. Satisfies AuthProtocol: get_access_token() / build_headers() / invalidate(). +Tokens are never logged. """ -from typing import Any, Dict +import threading +import time +from typing import Any, Dict, Tuple + +from ...base.utils.logger import logger +from .constants import ZdfDefaults +from .models import parse_flight_tokens class ZdfAuth: - def __init__(self, *, config: Any): + def __init__(self, *, http_manager: Any, config: Any): + self._http = http_manager self._config = config + self._lock = threading.RLock() + self._tokens: Dict[str, Tuple[str, float]] = {} # kind -> (token, expires_at) + self._retry_after = 0.0 + + # ------------------------------------------------------------------ + # Token handling + # ------------------------------------------------------------------ + def _is_fresh(self, kind: str) -> bool: + entry = self._tokens.get(kind) + return bool(entry) and entry[1] - time.time() > ZdfDefaults.TOKEN_REFRESH_BUFFER_SECONDS + + def _fetch_tokens(self) -> None: + resp = self._http.get( + ZdfDefaults.WEB_URL, + headers=self._config.page_headers(), + operation="api", + ) + tokens = parse_flight_tokens(resp.text) + if not tokens: + raise ValueError("no videoToken / appToken found in the zdf.de page") + self._tokens.update(tokens) + + def _token(self, kind: str) -> str: + with self._lock: + if self._is_fresh(kind): + return self._tokens[kind][0] + now = time.time() + if now >= self._retry_after: + try: + self._fetch_tokens() + except Exception as exc: # network, parse, anything: fall back + self._retry_after = now + ZdfDefaults.TOKEN_FETCH_BACKOFF_SECONDS + logger.warning( + f"{ZdfDefaults.PROVIDER_LABEL}: token fetch failed, " + f"using fallback tokens: {exc!r}" + ) + if self._is_fresh(kind): + return self._tokens[kind][0] + if kind == ZdfDefaults.TOKEN_KIND_LIST: + return self._config.list_token + return self._config.api_token def get_access_token(self) -> str: - return self._config.api_token - - def build_headers(self) -> Dict[str, str]: - """Default API headers (GraphQL POST).""" - return self._config.graphql_headers() - - def build_list_headers(self) -> Dict[str, str]: - """Persisted-query GET: it uses its own client token and app id.""" - return self._config.list_headers() - - def build_ptmd_headers(self) -> Dict[str, str]: - return self._config.ptmd_headers() + """The videoToken (GraphQL POST and PTMD).""" + return self._token(ZdfDefaults.TOKEN_KIND_API) def invalidate(self) -> None: - """Static public tokens: nothing in memory or in storage to clear.""" - return None + """Drop cached tokens (and the fetch back-off); the next use refetches.""" + with self._lock: + self._tokens.clear() + self._retry_after = 0.0 + + def refresh_after_rejection(self) -> bool: + """ + Called after the API refused a request. Refetch and report whether + either token actually changed (False = retrying would be pointless). + """ + with self._lock: + before = (self.get_access_token(), self._token(ZdfDefaults.TOKEN_KIND_LIST)) + self.invalidate() + after = (self.get_access_token(), self._token(ZdfDefaults.TOKEN_KIND_LIST)) + return before != after + + # ------------------------------------------------------------------ + # Headers + # ------------------------------------------------------------------ + def build_headers(self) -> Dict[str, str]: + """Default API headers (GraphQL POST).""" + return self._config.graphql_headers(self.get_access_token()) + + def build_list_headers(self) -> Dict[str, str]: + """Persisted-query GET: appToken + app id.""" + return self._config.list_headers(self._token(ZdfDefaults.TOKEN_KIND_LIST)) + + def build_ptmd_headers(self) -> Dict[str, str]: + return self._config.ptmd_headers(self.get_access_token()) diff --git a/lib/streaming_providers/providers/zdf/channel_manager.py b/lib/streaming_providers/providers/zdf/channel_manager.py index d23936d..b0de5a1 100644 --- a/lib/streaming_providers/providers/zdf/channel_manager.py +++ b/lib/streaming_providers/providers/zdf/channel_manager.py @@ -1,10 +1,10 @@ # streaming_providers/providers/zdf/channel_manager.py """ -ZDF channel manager -- the only manager (live TV, clear DASH, no DRM). +ZDF channel manager -- the only manager (live TV, clear HLS, no DRM). Contract (ChannelManager ABC; README §6.1): * get_channels() -> List[Channel] - * get_channel_manifest(id) -> DASH MPD url; raises on real failures + * get_channel_manifest(id) -> HLS master.m3u8 url; raises on real failures * get_channel_manifest_headers -> CDN headers (HEADERS_FROM_MANAGERS = True) * get_segment_headers -> inherited (= manifest headers) * get_channel_drm -> inherited ([] = no DRM; ZDF streams are clear) @@ -13,14 +13,17 @@ content_id == the video's `canonical` ("zdf-live-beitrag-100"). It resolves to a ptmd path via a small TTL cache that get_channels() fills; on a miss one VideoByCanonical request fills it. Failure semantics: * canonical unknown / no live media -> NotFoundError - * ptmd without a DASH stream, or encrypted -> ServerError (raised as + * ptmd without an HLS stream, or encrypted -> ServerError (raised as ValueError inside transport_errors); the message lists what was offered. + * API refuses a request (AuthError / EntitlementError, i.e. 401/403) -> + one refetch of the web-client tokens and exactly one retry, only if the + tokens really changed; otherwise the original error propagates. """ import time -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple -from ...base.errors import NotFoundError +from ...base.errors import AuthError, EntitlementError, NotFoundError from ...base.managers import ChannelManager from ...base.models import Channel from ...base.utils.logger import logger @@ -55,17 +58,33 @@ class ZdfChannelManager(ChannelManager): # Provider-owned dict, borrowed by reference: canonical -> (ptmd path, ts). self._ptmd_cache = ptmd_cache if ptmd_cache is not None else {} + # ------------------------------------------------------------------ + # Request helper: one retry after a token rejection + # ------------------------------------------------------------------ + def _send(self, send: Callable[[], Any]) -> Any: + """ + Run `send` (which must build its headers inside, so a retry picks up + the new token). A 401/403 may mean an expired web-client token: refetch + once and retry once, but only if the tokens actually changed. + """ + try: + return send() + except (AuthError, EntitlementError): + if not self.auth.refresh_after_rejection(): + raise + return send() + # ------------------------------------------------------------------ # Channels # ------------------------------------------------------------------ def get_channels(self, **kw: Any) -> List[Channel]: with transport_errors("get_channels", _PROVIDER): - resp = self.http_manager.get( + resp = self._send(lambda: self.http_manager.get( self.config.url(ZdfDefaults.PATH_GRAPHQL), params=self.config.list_query_params(), headers=self.auth.build_list_headers(), operation="api", - ) + )) nodes = extract_video_nodes(resp.json()) channels: List[ZdfChannel] = [] @@ -98,7 +117,7 @@ class ZdfChannelManager(ChannelManager): if cached and (now - cached[1]) < ZdfDefaults.PTMD_PATH_CACHE_TTL: return cached[0] with transport_errors(f"VideoByCanonical {canonical}", _PROVIDER): - resp = self.http_manager.post( + resp = self._send(lambda: self.http_manager.post( self.config.url(ZdfDefaults.PATH_GRAPHQL), json={ "operationName": "VideoByCanonical", @@ -107,7 +126,7 @@ class ZdfChannelManager(ChannelManager): }, headers=self.auth.build_headers(), operation="api", - ) + )) path = ptmd_path_from_video(resp.json()) if path is None: raise NotFoundError(f"{_PROVIDER}: no live stream for {canonical}") @@ -120,11 +139,11 @@ class ZdfChannelManager(ChannelManager): def get_channel_manifest(self, content_id: str, **kw: Any) -> Optional[str]: ptmd_path = self._ptmd_path(content_id) with transport_errors(f"ptmd for {content_id}", _PROVIDER): - resp = self.http_manager.get( + resp = self._send(lambda: self.http_manager.get( self.config.url(ptmd_path), headers=self.auth.build_ptmd_headers(), operation="api", - ) + )) playout = ZdfPlayout.from_api_response(resp.json()) return playout.manifest_url diff --git a/lib/streaming_providers/providers/zdf/constants.py b/lib/streaming_providers/providers/zdf/constants.py index 25ce403..9c769b1 100644 --- a/lib/streaming_providers/providers/zdf/constants.py +++ b/lib/streaming_providers/providers/zdf/constants.py @@ -9,15 +9,16 @@ Identity rule: ONE id, used everywhere Never rename it after release. Upstream flow (all against api.zdf.de, no user account, no DRM): + 0. GET https://www.zdf.de/ -> videoToken + appToken (auth.py, cached) 1. GET /graphql getActiveLiveStreams (persisted query) -> channel list 2. POST /graphql VideoByCanonical (only if the ptmd path is not cached) - 3. GET /tmd/2//live/ptmd/ -> DASH manifest URL + 3. GET /tmd/2//live/ptmd/ -> HLS manifest URL -The two ``api-auth`` bearer values are the PUBLIC client tokens of the ZDF web -player (visible in any browser capture), not user credentials. ZDF rotates them -from time to time: override via the provider config dict -(``api_token`` / ``list_token`` / ``app_id``) instead of editing this file. -They must never be logged. +The ``api-auth`` bearer values are the PUBLIC, expiring client tokens of the +ZDF web player, not user credentials. auth.py fetches them from the web page; +the static values below are only the FALLBACK when that fetch fails (they +stop working when ZDF rotates them). Override via the provider config dict +(``api_token`` / ``list_token`` / ``app_id``). Tokens must never be logged. """ import json @@ -53,17 +54,33 @@ class ZdfDefaults: # (sports, services) and test items do not, and are not channels. LIVE247_MARKER = "/live/ptmd/247onAir-" - # --- DASH selection ---------------------------------------------------- - # Only DASH is wanted (no HLS). A formitaet counts as DASH when its - # mimeType contains "dash" or its type contains "mpd". - DASH_MIME_MARKER = "dash" - DASH_TYPE_MARKER = "mpd" - # Preferred quality entries, best first ("auto" = the adaptive master). - DASH_QUALITY_ORDER = ("auto", "veryhigh", "high", "hd", "med", "low") + # --- HLS selection ------------------------------------------------------- + # The 24/7 channels were only seen with HLS formitaeten (no DASH), so + # HLS is returned. A formitaet counts as HLS when its mimeType contains + # "mpegurl" or its type contains "m3u8". Matching is case-insensitive. + HLS_MIME_MARKER = "mpegurl" + HLS_TYPE_MARKER = "m3u8" + # Preferred quality entries, best first. "auto" is the master ZDF itself + # lists first (and the one the web player / other clients use). + HLS_QUALITY_ORDER = ("auto", "veryhigh", "high", "hd", "med", "low") AUDIO_CLASS_PREFERRED = "main" # not "ad" (audio description) AUDIO_LANGUAGE_PREFERRED = "deu" - # --- Public web-client identity (override via config, never log) ----------- + # --- Token source --------------------------------------------------------- + # The page embeds the tokens in Next.js "flight" chunks: + # self.__next_f.push([1,"...\"videoToken\":{\"apiToken\":\"..\",\"expiresAt\":\"..\"}..."]) + # videoToken -> GraphQL POST + PTMD ("api"), appToken -> channel list ("app"). + WEB_URL = "https://www.zdf.de/" + TOKEN_KIND_API = "api" + TOKEN_KIND_LIST = "app" + FLIGHT_TOKEN_NAMES = {"videoToken": "api", "appToken": "app"} + # Refresh this long before the token really expires. + TOKEN_REFRESH_BUFFER_SECONDS = 300 + # After a failed token fetch use the static fallback and do not retry + # before this many seconds have passed. + TOKEN_FETCH_BACKOFF_SECONDS = 60 + + # --- Fallback client identity (override via config, never log) -------------- LIST_TOKEN = "aa3noh4ohz9eeboo8shiesheec9ciequ9Quah7el" API_TOKEN = "ahBaeMeekaiy5ohsai4bee4ki6Oopoi5quailieb" APP_ID = "ffw-mt-web-37e8d3ae" @@ -152,6 +169,7 @@ class ZdfConfig: } # ----- Header builders ------------------------------------------------ + # The token always comes from auth.py (fresh or fallback), never from here. def _base_headers(self) -> dict: return { "User-Agent": self.user_agent, @@ -160,33 +178,41 @@ class ZdfConfig: "Referer": ZdfDefaults.REFERER, } - def list_headers(self) -> dict: - """Persisted-query GET (channel list).""" + def page_headers(self) -> dict: + """Plain page fetch of the token source (no api-auth).""" + return { + "User-Agent": self.user_agent, + "Accept": "text/html,application/xhtml+xml", + "Accept-Language": ZdfDefaults.ACCEPT_LANGUAGE, + } + + def list_headers(self, token: str) -> dict: + """Persisted-query GET (channel list): appToken + app id.""" headers = self._base_headers() headers["Accept"] = ZdfDefaults.ACCEPT_LIST headers["Content-Type"] = "application/json" - headers["api-auth"] = f"Bearer {self.list_token}" + headers["api-auth"] = f"Bearer {token}" headers["zdf-app-id"] = self.app_id return headers - def graphql_headers(self) -> dict: - """VideoByCanonical POST.""" + def graphql_headers(self, token: str) -> dict: + """VideoByCanonical POST: videoToken.""" headers = self._base_headers() headers["Accept"] = "*/*" headers["Content-Type"] = "application/json" - headers["api-auth"] = f"Bearer {self.api_token}" + headers["api-auth"] = f"Bearer {token}" return headers - def ptmd_headers(self) -> dict: - """PTMD GET.""" + def ptmd_headers(self, token: str) -> dict: + """PTMD GET: videoToken.""" headers = self._base_headers() headers["Accept"] = ZdfDefaults.ACCEPT_PTMD - headers["api-auth"] = f"Bearer {self.api_token}" + headers["api-auth"] = f"Bearer {token}" return headers def stream_headers(self) -> dict: """ - Headers for MPD / segment fetches by the player (CDN). + Headers for manifest / segment fetches by the player (CDN). Deliberately WITHOUT api-auth / zdf-app-id: the CDN does not need them and sending them would leak the client token to the edge. diff --git a/lib/streaming_providers/providers/zdf/models.py b/lib/streaming_providers/providers/zdf/models.py index 383a0d6..ccc69e8 100644 --- a/lib/streaming_providers/providers/zdf/models.py +++ b/lib/streaming_providers/providers/zdf/models.py @@ -11,13 +11,59 @@ scheduled shows); `is_channel_node` is the one place that decides what counts as a channel. """ +import json +import re from dataclasses import dataclass +from datetime import datetime from typing import Any, Dict, Iterable, List, Optional, Tuple from ...base.models import Channel from .constants import ZdfDefaults +# ---------------------------------------------------------------------- +# Web-client tokens (parsed from the zdf.de page, see auth.py) +# ---------------------------------------------------------------------- +_FLIGHT_CHUNK = re.compile(r'self\.__next_f\.push\(\[1,\s*"((?:[^"\\]|\\.)*)"\]\)') + + +def _parse_expiry(value: str) -> float: + """ISO-8601 with offset ('2026-10-18T05:21:01+02:00', also 'Z') -> unix seconds.""" + return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() + + +def parse_flight_tokens(html: str) -> Dict[str, Tuple[str, float]]: + """ + {kind: (token, expires_at_unix)} from the zdf.de start page. + + Joins every ``self.__next_f.push([1,"..."])`` chunk (JSON-string + unescaped) and reads the ``videoToken`` / ``appToken`` objects out of the + result. Kinds that are missing or malformed are simply absent; the caller + decides what an incomplete result means. + """ + chunks: List[str] = [] + for raw in _FLIGHT_CHUNK.findall(html or ""): + try: + chunks.append(json.loads('"' + raw + '"')) + except ValueError: + continue + flight = "".join(chunks) + + tokens: Dict[str, Tuple[str, float]] = {} + for name, kind in ZdfDefaults.FLIGHT_TOKEN_NAMES.items(): + match = re.search( + r'"%s":\{"apiToken":"([^"]+)","expiresAt":"([^"]+)"\}' % re.escape(name), + flight, + ) + if not match: + continue + try: + tokens[kind] = (match.group(1), _parse_expiry(match.group(2))) + except ValueError: + continue + return tokens + + # ---------------------------------------------------------------------- # GraphQL helpers # ---------------------------------------------------------------------- @@ -132,33 +178,33 @@ def _attr(attributes: Dict[str, Any], name: str) -> Optional[str]: return entry.get("value") if isinstance(entry, dict) else None -def _is_dash(formitaet: Dict[str, Any]) -> bool: +def _is_hls(formitaet: Dict[str, Any]) -> bool: mime = (formitaet.get("mimeType") or "").lower() kind = (formitaet.get("type") or "").lower() - return ZdfDefaults.DASH_MIME_MARKER in mime or ZdfDefaults.DASH_TYPE_MARKER in kind + return ZdfDefaults.HLS_MIME_MARKER in mime or ZdfDefaults.HLS_TYPE_MARKER in kind def _rank(order: Tuple[str, ...], value: Optional[str]) -> int: return order.index(value) if value in order else len(order) -def pick_dash_url(priority_list: Iterable[Dict[str, Any]]) -> Tuple[Optional[str], List[str]]: +def pick_hls_url(priority_list: Iterable[Dict[str, Any]]) -> Tuple[Optional[str], List[str]]: """ - Choose ONE DASH manifest URL. Returns (url or None, offered formats). + Choose ONE HLS master playlist URL. Returns (url or None, offered formats). Order of preference: earlier priorityList item, then quality - (DASH_QUALITY_ORDER, "auto" first), then main-class German audio track, - then https. HLS and everything else is ignored. + (HLS_QUALITY_ORDER, "auto" first), then main-class German audio track, + then https. DASH and everything else is ignored. """ offered: List[str] = [] best: Optional[Tuple[Tuple[int, int, int, int], str]] = None for p_idx, item in enumerate(priority_list or []): for formitaet in item.get("formitaeten") or []: offered.append(formitaet.get("mimeType") or formitaet.get("type") or "?") - if not _is_dash(formitaet): + if not _is_hls(formitaet): continue for quality in formitaet.get("qualities") or []: - q_rank = _rank(ZdfDefaults.DASH_QUALITY_ORDER, quality.get("quality")) + q_rank = _rank(ZdfDefaults.HLS_QUALITY_ORDER, quality.get("quality")) for track in (quality.get("audio") or {}).get("tracks") or []: uri = track.get("uri") if not uri: @@ -189,10 +235,10 @@ class ZdfPlayout: encryption = _attr(attributes, "encryption") if encryption not in (None, "none"): raise ValueError(f"encrypted stream ({encryption}) is not supported") - url, offered = pick_dash_url(data.get("priorityList") or []) + url, offered = pick_hls_url(data.get("priorityList") or []) if not url: raise ValueError( - f"ptmd response offers no DASH stream (offered: {sorted(set(offered))})" + f"ptmd response offers no HLS stream (offered: {sorted(set(offered))})" ) return cls( manifest_url=url, diff --git a/lib/streaming_providers/providers/zdf/provider.py b/lib/streaming_providers/providers/zdf/provider.py index fbfedc4..afdb1da 100644 --- a/lib/streaming_providers/providers/zdf/provider.py +++ b/lib/streaming_providers/providers/zdf/provider.py @@ -2,15 +2,16 @@ """ ZDF provider (live TV of ZDF, ZDFneo, ZDFinfo, 3sat, phoenix, KiKA, ARTE). -Capabilities: channels only. Clear DASH streams, no account, no DRM, no EPG +Capabilities: channels only. Clear HLS streams, no account, no DRM, no EPG (the generic XMLTV path answers EPG), no catchup. Known limitations: * Streams are geo-limited to Germany; a ProxyConfig only affects the Python-side API calls. The manifest and segment fetches are made by inputstream.adaptive inside Kodi and do not use it. - * The api-auth client tokens are public web-player tokens and may be rotated - by ZDF (config keys api_token / list_token / app_id). + * The api-auth client tokens are public, expiring web-player tokens that + auth.py fetches from zdf.de; the static config values (api_token / + list_token / app_id) are only the fallback if that fetch fails. """ from typing import ClassVar, Dict, List, Optional, Tuple @@ -33,7 +34,7 @@ class ZdfProvider(ManagedProvider): # Decision 1 (§11): clear streams, no DRM at all -> no folded DRM, no _build_drm(). DRM_IN_MANAGERS: ClassVar[bool] = False # Decision 2 (§10): the manager's CDN headers (UA/Origin/Referer, no tokens) - # are what the player must send for MPD and segments. + # are what the player must send for the playlist and segments. HEADERS_FROM_MANAGERS: ClassVar[bool] = True @property @@ -60,8 +61,8 @@ class ZdfProvider(ManagedProvider): user_agent=self.provider_config.user_agent, timeout=self.provider_config.timeout, ) - # 3. auth (static public tokens, nothing lazy to do) - self.auth = ZdfAuth(config=self.provider_config) + # 3. auth: lazy, no network here (tokens are fetched on first use) + self.auth = ZdfAuth(http_manager=self.http_manager, config=self.provider_config) # 4. caches (provider-owned, borrowed by the manager) self._ptmd_cache: Dict[str, Tuple[str, float]] = {} # 5. managers -- must be the last line