zdf auth

lib/streaming_providers/providers/zdf/auth.py

@@ -1,37 +1,112 @@
 # streaming_providers/providers/zdf/auth.py
 """
-ZDF auth -- Variant A (README §9), degenerate on purpose.
+ZDF auth -- Variant A (README §9): lazy, no network in __init__.
 
-ZDF live TV needs no user account: the API only wants the public web-client
-bearer tokens (constants.py / config override). There is nothing to log in to,
-nothing to refresh and nothing persisted, so no lock, no expiry and no
-retry/backoff exist here. If ZDF ever rotates the tokens, the symptom is an
-HTTP 401/403 from the HTTP manager; the fix is a config override.
+ZDF live TV needs no user account. The API wants two expiring, public client
+tokens of the web player, which the start page embeds in its Next.js flight
+data (`videoToken` for GraphQL POST + PTMD, `appToken` for the channel list).
+
+Behaviour:
+  * tokens are fetched on first use and cached with their expiry; they are
+    renewed TOKEN_REFRESH_BUFFER_SECONDS before they run out. One lock makes
+    concurrent callers share a single fetch.
+  * if the page fetch or parse fails, the static fallback tokens of the config
+    are used and no new fetch is tried for TOKEN_FETCH_BACKOFF_SECONDS (no
+    request storm while zdf.de is down or its page layout changed).
+  * `refresh_after_rejection()` is the one-shot recovery for a request the API
+    refused: drop the cache, fetch again, and say whether the tokens changed.
+    The manager retries exactly once, and only if they did.
 
 Satisfies AuthProtocol: get_access_token() / build_headers() / invalidate().
+Tokens are never logged.
 """
 
-from typing import Any, Dict
+import threading
+import time
+from typing import Any, Dict, Tuple
+
+from ...base.utils.logger import logger
+from .constants import ZdfDefaults
+from .models import parse_flight_tokens
 
 
 class ZdfAuth:
-    def __init__(self, *, config: Any):
+    def __init__(self, *, http_manager: Any, config: Any):
+        self._http = http_manager
         self._config = config
+        self._lock = threading.RLock()
+        self._tokens: Dict[str, Tuple[str, float]] = {}   # kind -> (token, expires_at)
+        self._retry_after = 0.0
+
+    # ------------------------------------------------------------------
+    # Token handling
+    # ------------------------------------------------------------------
+    def _is_fresh(self, kind: str) -> bool:
+        entry = self._tokens.get(kind)
+        return bool(entry) and entry[1] - time.time() > ZdfDefaults.TOKEN_REFRESH_BUFFER_SECONDS
+
+    def _fetch_tokens(self) -> None:
+        resp = self._http.get(
+            ZdfDefaults.WEB_URL,
+            headers=self._config.page_headers(),
+            operation="api",
+        )
+        tokens = parse_flight_tokens(resp.text)
+        if not tokens:
+            raise ValueError("no videoToken / appToken found in the zdf.de page")
+        self._tokens.update(tokens)
+
+    def _token(self, kind: str) -> str:
+        with self._lock:
+            if self._is_fresh(kind):
+                return self._tokens[kind][0]
+            now = time.time()
+            if now >= self._retry_after:
+                try:
+                    self._fetch_tokens()
+                except Exception as exc:       # network, parse, anything: fall back
+                    self._retry_after = now + ZdfDefaults.TOKEN_FETCH_BACKOFF_SECONDS
+                    logger.warning(
+                        f"{ZdfDefaults.PROVIDER_LABEL}: token fetch failed, "
+                        f"using fallback tokens: {exc!r}"
+                    )
+            if self._is_fresh(kind):
+                return self._tokens[kind][0]
+            if kind == ZdfDefaults.TOKEN_KIND_LIST:
+                return self._config.list_token
+            return self._config.api_token
 
     def get_access_token(self) -> str:
-        return self._config.api_token
-
-    def build_headers(self) -> Dict[str, str]:
-        """Default API headers (GraphQL POST)."""
-        return self._config.graphql_headers()
-
-    def build_list_headers(self) -> Dict[str, str]:
-        """Persisted-query GET: it uses its own client token and app id."""
-        return self._config.list_headers()
-
-    def build_ptmd_headers(self) -> Dict[str, str]:
-        return self._config.ptmd_headers()
+        """The videoToken (GraphQL POST and PTMD)."""
+        return self._token(ZdfDefaults.TOKEN_KIND_API)
 
     def invalidate(self) -> None:
-        """Static public tokens: nothing in memory or in storage to clear."""
-        return None
+        """Drop cached tokens (and the fetch back-off); the next use refetches."""
+        with self._lock:
+            self._tokens.clear()
+            self._retry_after = 0.0
+
+    def refresh_after_rejection(self) -> bool:
+        """
+        Called after the API refused a request. Refetch and report whether
+        either token actually changed (False = retrying would be pointless).
+        """
+        with self._lock:
+            before = (self.get_access_token(), self._token(ZdfDefaults.TOKEN_KIND_LIST))
+            self.invalidate()
+            after = (self.get_access_token(), self._token(ZdfDefaults.TOKEN_KIND_LIST))
+            return before != after
+
+    # ------------------------------------------------------------------
+    # Headers
+    # ------------------------------------------------------------------
+    def build_headers(self) -> Dict[str, str]:
+        """Default API headers (GraphQL POST)."""
+        return self._config.graphql_headers(self.get_access_token())
+
+    def build_list_headers(self) -> Dict[str, str]:
+        """Persisted-query GET: appToken + app id."""
+        return self._config.list_headers(self._token(ZdfDefaults.TOKEN_KIND_LIST))
+
+    def build_ptmd_headers(self) -> Dict[str, str]:
+        return self._config.ptmd_headers(self.get_access_token())

lib/streaming_providers/providers/zdf/channel_manager.py

@@ -1,10 +1,10 @@
 # streaming_providers/providers/zdf/channel_manager.py
 """
-ZDF channel manager -- the only manager (live TV, clear DASH, no DRM).
+ZDF channel manager -- the only manager (live TV, clear HLS, no DRM).
 
 Contract (ChannelManager ABC; README §6.1):
   * get_channels()                -> List[Channel]
-  * get_channel_manifest(id)      -> DASH MPD url; raises on real failures
+  * get_channel_manifest(id)      -> HLS master.m3u8 url; raises on real failures
   * get_channel_manifest_headers  -> CDN headers (HEADERS_FROM_MANAGERS = True)
   * get_segment_headers           -> inherited (= manifest headers)
   * get_channel_drm               -> inherited ([] = no DRM; ZDF streams are clear)
@@ -13,14 +13,17 @@ content_id == the video's `canonical` ("zdf-live-beitrag-100"). It resolves to
 a ptmd path via a small TTL cache that get_channels() fills; on a miss one
 VideoByCanonical request fills it. Failure semantics:
   * canonical unknown / no live media  -> NotFoundError
-  * ptmd without a DASH stream, or encrypted -> ServerError (raised as
+  * ptmd without an HLS stream, or encrypted -> ServerError (raised as
     ValueError inside transport_errors); the message lists what was offered.
+  * API refuses a request (AuthError / EntitlementError, i.e. 401/403) ->
+    one refetch of the web-client tokens and exactly one retry, only if the
+    tokens really changed; otherwise the original error propagates.
 """
 
 import time
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
-from ...base.errors import NotFoundError
+from ...base.errors import AuthError, EntitlementError, NotFoundError
 from ...base.managers import ChannelManager
 from ...base.models import Channel
 from ...base.utils.logger import logger
@@ -55,17 +58,33 @@ class ZdfChannelManager(ChannelManager):
         # Provider-owned dict, borrowed by reference: canonical -> (ptmd path, ts).
         self._ptmd_cache = ptmd_cache if ptmd_cache is not None else {}
 
+    # ------------------------------------------------------------------
+    # Request helper: one retry after a token rejection
+    # ------------------------------------------------------------------
+    def _send(self, send: Callable[[], Any]) -> Any:
+        """
+        Run `send` (which must build its headers inside, so a retry picks up
+        the new token). A 401/403 may mean an expired web-client token: refetch
+        once and retry once, but only if the tokens actually changed.
+        """
+        try:
+            return send()
+        except (AuthError, EntitlementError):
+            if not self.auth.refresh_after_rejection():
+                raise
+            return send()
+
     # ------------------------------------------------------------------
     # Channels
     # ------------------------------------------------------------------
     def get_channels(self, **kw: Any) -> List[Channel]:
         with transport_errors("get_channels", _PROVIDER):
-            resp = self.http_manager.get(
+            resp = self._send(lambda: self.http_manager.get(
                 self.config.url(ZdfDefaults.PATH_GRAPHQL),
                 params=self.config.list_query_params(),
                 headers=self.auth.build_list_headers(),
                 operation="api",
-            )
+            ))
             nodes = extract_video_nodes(resp.json())
 
         channels: List[ZdfChannel] = []
@@ -98,7 +117,7 @@ class ZdfChannelManager(ChannelManager):
         if cached and (now - cached[1]) < ZdfDefaults.PTMD_PATH_CACHE_TTL:
             return cached[0]
         with transport_errors(f"VideoByCanonical {canonical}", _PROVIDER):
-            resp = self.http_manager.post(
+            resp = self._send(lambda: self.http_manager.post(
                 self.config.url(ZdfDefaults.PATH_GRAPHQL),
                 json={
                     "operationName": "VideoByCanonical",
@@ -107,7 +126,7 @@ class ZdfChannelManager(ChannelManager):
                 },
                 headers=self.auth.build_headers(),
                 operation="api",
-            )
+            ))
             path = ptmd_path_from_video(resp.json())
         if path is None:
             raise NotFoundError(f"{_PROVIDER}: no live stream for {canonical}")
@@ -120,11 +139,11 @@ class ZdfChannelManager(ChannelManager):
     def get_channel_manifest(self, content_id: str, **kw: Any) -> Optional[str]:
         ptmd_path = self._ptmd_path(content_id)
         with transport_errors(f"ptmd for {content_id}", _PROVIDER):
-            resp = self.http_manager.get(
+            resp = self._send(lambda: self.http_manager.get(
                 self.config.url(ptmd_path),
                 headers=self.auth.build_ptmd_headers(),
                 operation="api",
-            )
+            ))
             playout = ZdfPlayout.from_api_response(resp.json())
         return playout.manifest_url
 

lib/streaming_providers/providers/zdf/constants.py

@@ -9,15 +9,16 @@ Identity rule: ONE id, used everywhere
 Never rename it after release.
 
 Upstream flow (all against api.zdf.de, no user account, no DRM):
+    0. GET  https://www.zdf.de/  -> videoToken + appToken (auth.py, cached)
     1. GET  /graphql  getActiveLiveStreams (persisted query)  -> channel list
     2. POST /graphql  VideoByCanonical (only if the ptmd path is not cached)
-    3. GET  /tmd/2/<player>/live/ptmd/<id>                    -> DASH manifest URL
+    3. GET  /tmd/2/<player>/live/ptmd/<id>                    -> HLS manifest URL
 
-The two ``api-auth`` bearer values are the PUBLIC client tokens of the ZDF web
-player (visible in any browser capture), not user credentials. ZDF rotates them
-from time to time: override via the provider config dict
-(``api_token`` / ``list_token`` / ``app_id``) instead of editing this file.
-They must never be logged.
+The ``api-auth`` bearer values are the PUBLIC, expiring client tokens of the
+ZDF web player, not user credentials. auth.py fetches them from the web page;
+the static values below are only the FALLBACK when that fetch fails (they
+stop working when ZDF rotates them). Override via the provider config dict
+(``api_token`` / ``list_token`` / ``app_id``). Tokens must never be logged.
 """
 
 import json
@@ -53,17 +54,33 @@ class ZdfDefaults:
     # (sports, services) and test items do not, and are not channels.
     LIVE247_MARKER = "/live/ptmd/247onAir-"
 
-    # --- DASH selection ----------------------------------------------------
-    # Only DASH is wanted (no HLS). A formitaet counts as DASH when its
-    # mimeType contains "dash" or its type contains "mpd".
-    DASH_MIME_MARKER = "dash"
-    DASH_TYPE_MARKER = "mpd"
-    # Preferred quality entries, best first ("auto" = the adaptive master).
-    DASH_QUALITY_ORDER = ("auto", "veryhigh", "high", "hd", "med", "low")
+    # --- HLS selection -------------------------------------------------------
+    # The 24/7 channels were only seen with HLS formitaeten (no DASH), so
+    # HLS is returned. A formitaet counts as HLS when its mimeType contains
+    # "mpegurl" or its type contains "m3u8". Matching is case-insensitive.
+    HLS_MIME_MARKER = "mpegurl"
+    HLS_TYPE_MARKER = "m3u8"
+    # Preferred quality entries, best first. "auto" is the master ZDF itself
+    # lists first (and the one the web player / other clients use).
+    HLS_QUALITY_ORDER = ("auto", "veryhigh", "high", "hd", "med", "low")
     AUDIO_CLASS_PREFERRED = "main"      # not "ad" (audio description)
     AUDIO_LANGUAGE_PREFERRED = "deu"
 
-    # --- Public web-client identity (override via config, never log) -----------
+    # --- Token source ---------------------------------------------------------
+    # The page embeds the tokens in Next.js "flight" chunks:
+    #   self.__next_f.push([1,"...\"videoToken\":{\"apiToken\":\"..\",\"expiresAt\":\"..\"}..."])
+    # videoToken -> GraphQL POST + PTMD ("api"), appToken -> channel list ("app").
+    WEB_URL = "https://www.zdf.de/"
+    TOKEN_KIND_API = "api"
+    TOKEN_KIND_LIST = "app"
+    FLIGHT_TOKEN_NAMES = {"videoToken": "api", "appToken": "app"}
+    # Refresh this long before the token really expires.
+    TOKEN_REFRESH_BUFFER_SECONDS = 300
+    # After a failed token fetch use the static fallback and do not retry
+    # before this many seconds have passed.
+    TOKEN_FETCH_BACKOFF_SECONDS = 60
+
+    # --- Fallback client identity (override via config, never log) --------------
     LIST_TOKEN = "aa3noh4ohz9eeboo8shiesheec9ciequ9Quah7el"
     API_TOKEN = "ahBaeMeekaiy5ohsai4bee4ki6Oopoi5quailieb"
     APP_ID = "ffw-mt-web-37e8d3ae"
@@ -152,6 +169,7 @@ class ZdfConfig:
         }
 
     # ----- Header builders ------------------------------------------------
+    # The token always comes from auth.py (fresh or fallback), never from here.
     def _base_headers(self) -> dict:
         return {
             "User-Agent": self.user_agent,
@@ -160,33 +178,41 @@ class ZdfConfig:
             "Referer": ZdfDefaults.REFERER,
         }
 
-    def list_headers(self) -> dict:
-        """Persisted-query GET (channel list)."""
+    def page_headers(self) -> dict:
+        """Plain page fetch of the token source (no api-auth)."""
+        return {
+            "User-Agent": self.user_agent,
+            "Accept": "text/html,application/xhtml+xml",
+            "Accept-Language": ZdfDefaults.ACCEPT_LANGUAGE,
+        }
+
+    def list_headers(self, token: str) -> dict:
+        """Persisted-query GET (channel list): appToken + app id."""
         headers = self._base_headers()
         headers["Accept"] = ZdfDefaults.ACCEPT_LIST
         headers["Content-Type"] = "application/json"
-        headers["api-auth"] = f"Bearer {self.list_token}"
+        headers["api-auth"] = f"Bearer {token}"
         headers["zdf-app-id"] = self.app_id
         return headers
 
-    def graphql_headers(self) -> dict:
-        """VideoByCanonical POST."""
+    def graphql_headers(self, token: str) -> dict:
+        """VideoByCanonical POST: videoToken."""
         headers = self._base_headers()
         headers["Accept"] = "*/*"
         headers["Content-Type"] = "application/json"
-        headers["api-auth"] = f"Bearer {self.api_token}"
+        headers["api-auth"] = f"Bearer {token}"
         return headers
 
-    def ptmd_headers(self) -> dict:
-        """PTMD GET."""
+    def ptmd_headers(self, token: str) -> dict:
+        """PTMD GET: videoToken."""
         headers = self._base_headers()
         headers["Accept"] = ZdfDefaults.ACCEPT_PTMD
-        headers["api-auth"] = f"Bearer {self.api_token}"
+        headers["api-auth"] = f"Bearer {token}"
         return headers
 
     def stream_headers(self) -> dict:
         """
-        Headers for MPD / segment fetches by the player (CDN).
+        Headers for manifest / segment fetches by the player (CDN).
 
         Deliberately WITHOUT api-auth / zdf-app-id: the CDN does not need them
         and sending them would leak the client token to the edge.

lib/streaming_providers/providers/zdf/models.py

@@ -11,13 +11,59 @@ scheduled shows); `is_channel_node` is the one place that decides what counts
 as a channel.
 """
 
+import json
+import re
 from dataclasses import dataclass
+from datetime import datetime
 from typing import Any, Dict, Iterable, List, Optional, Tuple
 
 from ...base.models import Channel
 from .constants import ZdfDefaults
 
 
+# ----------------------------------------------------------------------
+# Web-client tokens (parsed from the zdf.de page, see auth.py)
+# ----------------------------------------------------------------------
+_FLIGHT_CHUNK = re.compile(r'self\.__next_f\.push\(\[1,\s*"((?:[^"\\]|\\.)*)"\]\)')
+
+
+def _parse_expiry(value: str) -> float:
+    """ISO-8601 with offset ('2026-10-18T05:21:01+02:00', also 'Z') -> unix seconds."""
+    return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
+
+
+def parse_flight_tokens(html: str) -> Dict[str, Tuple[str, float]]:
+    """
+    {kind: (token, expires_at_unix)} from the zdf.de start page.
+
+    Joins every ``self.__next_f.push([1,"..."])`` chunk (JSON-string
+    unescaped) and reads the ``videoToken`` / ``appToken`` objects out of the
+    result. Kinds that are missing or malformed are simply absent; the caller
+    decides what an incomplete result means.
+    """
+    chunks: List[str] = []
+    for raw in _FLIGHT_CHUNK.findall(html or ""):
+        try:
+            chunks.append(json.loads('"' + raw + '"'))
+        except ValueError:
+            continue
+    flight = "".join(chunks)
+
+    tokens: Dict[str, Tuple[str, float]] = {}
+    for name, kind in ZdfDefaults.FLIGHT_TOKEN_NAMES.items():
+        match = re.search(
+            r'"%s":\{"apiToken":"([^"]+)","expiresAt":"([^"]+)"\}' % re.escape(name),
+            flight,
+        )
+        if not match:
+            continue
+        try:
+            tokens[kind] = (match.group(1), _parse_expiry(match.group(2)))
+        except ValueError:
+            continue
+    return tokens
+
+
 # ----------------------------------------------------------------------
 # GraphQL helpers
 # ----------------------------------------------------------------------
@@ -132,33 +178,33 @@ def _attr(attributes: Dict[str, Any], name: str) -> Optional[str]:
     return entry.get("value") if isinstance(entry, dict) else None
 
 
-def _is_dash(formitaet: Dict[str, Any]) -> bool:
+def _is_hls(formitaet: Dict[str, Any]) -> bool:
     mime = (formitaet.get("mimeType") or "").lower()
     kind = (formitaet.get("type") or "").lower()
-    return ZdfDefaults.DASH_MIME_MARKER in mime or ZdfDefaults.DASH_TYPE_MARKER in kind
+    return ZdfDefaults.HLS_MIME_MARKER in mime or ZdfDefaults.HLS_TYPE_MARKER in kind
 
 
 def _rank(order: Tuple[str, ...], value: Optional[str]) -> int:
     return order.index(value) if value in order else len(order)
 
 
-def pick_dash_url(priority_list: Iterable[Dict[str, Any]]) -> Tuple[Optional[str], List[str]]:
+def pick_hls_url(priority_list: Iterable[Dict[str, Any]]) -> Tuple[Optional[str], List[str]]:
     """
-    Choose ONE DASH manifest URL. Returns (url or None, offered formats).
+    Choose ONE HLS master playlist URL. Returns (url or None, offered formats).
 
     Order of preference: earlier priorityList item, then quality
-    (DASH_QUALITY_ORDER, "auto" first), then main-class German audio track,
-    then https. HLS and everything else is ignored.
+    (HLS_QUALITY_ORDER, "auto" first), then main-class German audio track,
+    then https. DASH and everything else is ignored.
     """
     offered: List[str] = []
     best: Optional[Tuple[Tuple[int, int, int, int], str]] = None
     for p_idx, item in enumerate(priority_list or []):
         for formitaet in item.get("formitaeten") or []:
             offered.append(formitaet.get("mimeType") or formitaet.get("type") or "?")
-            if not _is_dash(formitaet):
+            if not _is_hls(formitaet):
                 continue
             for quality in formitaet.get("qualities") or []:
-                q_rank = _rank(ZdfDefaults.DASH_QUALITY_ORDER, quality.get("quality"))
+                q_rank = _rank(ZdfDefaults.HLS_QUALITY_ORDER, quality.get("quality"))
                 for track in (quality.get("audio") or {}).get("tracks") or []:
                     uri = track.get("uri")
                     if not uri:
@@ -189,10 +235,10 @@ class ZdfPlayout:
         encryption = _attr(attributes, "encryption")
         if encryption not in (None, "none"):
             raise ValueError(f"encrypted stream ({encryption}) is not supported")
-        url, offered = pick_dash_url(data.get("priorityList") or [])
+        url, offered = pick_hls_url(data.get("priorityList") or [])
         if not url:
             raise ValueError(
-                f"ptmd response offers no DASH stream (offered: {sorted(set(offered))})"
+                f"ptmd response offers no HLS stream (offered: {sorted(set(offered))})"
             )
         return cls(
             manifest_url=url,

lib/streaming_providers/providers/zdf/provider.py

@@ -2,15 +2,16 @@
 """
 ZDF provider (live TV of ZDF, ZDFneo, ZDFinfo, 3sat, phoenix, KiKA, ARTE).
 
-Capabilities: channels only. Clear DASH streams, no account, no DRM, no EPG
+Capabilities: channels only. Clear HLS streams, no account, no DRM, no EPG
 (the generic XMLTV path answers EPG), no catchup.
 
 Known limitations:
   * Streams are geo-limited to Germany; a ProxyConfig only affects the
     Python-side API calls. The manifest and segment fetches are made by
     inputstream.adaptive inside Kodi and do not use it.
-  * The api-auth client tokens are public web-player tokens and may be rotated
-    by ZDF (config keys api_token / list_token / app_id).
+  * The api-auth client tokens are public, expiring web-player tokens that
+    auth.py fetches from zdf.de; the static config values (api_token /
+    list_token / app_id) are only the fallback if that fetch fails.
 """
 
 from typing import ClassVar, Dict, List, Optional, Tuple
@@ -33,7 +34,7 @@ class ZdfProvider(ManagedProvider):
     # Decision 1 (§11): clear streams, no DRM at all -> no folded DRM, no _build_drm().
     DRM_IN_MANAGERS: ClassVar[bool] = False
     # Decision 2 (§10): the manager's CDN headers (UA/Origin/Referer, no tokens)
-    # are what the player must send for MPD and segments.
+    # are what the player must send for the playlist and segments.
     HEADERS_FROM_MANAGERS: ClassVar[bool] = True
 
     @property
@@ -60,8 +61,8 @@ class ZdfProvider(ManagedProvider):
             user_agent=self.provider_config.user_agent,
             timeout=self.provider_config.timeout,
         )
-        # 3. auth (static public tokens, nothing lazy to do)
-        self.auth = ZdfAuth(config=self.provider_config)
+        # 3. auth: lazy, no network here (tokens are fetched on first use)
+        self.auth = ZdfAuth(http_manager=self.http_manager, config=self.provider_config)
         # 4. caches (provider-owned, borrowed by the manager)
         self._ptmd_cache: Dict[str, Tuple[str, float]] = {}
         # 5. managers -- must be the last line