Redirect hls to proxy

lib/streaming_providers/base/utils/hls_rewriter.py

@@ -0,0 +1,195 @@
+# streaming_providers/base/utils/hls_rewriter.py
+"""
+HLS playlist rewriter (phase 2, Option A).
+
+Pure text transformation, no I/O and no knowledge of the proxy URL scheme or
+token signing: every reference in the playlist is resolved to an absolute
+upstream URL and handed to a caller-supplied `build_url(url, kind)`, which
+returns the URL to put into the playlist. Under Option A the caller maps
+  PLAYLIST            -> the backend's playlist endpoint
+  SEGMENT / INIT / KEY -> the media proxy (/api/proxy/<token>)
+Kind is decided from the TAG a reference sits in, never from the file
+extension (tokenised CDN URLs usually have none).
+
+Guarantees
+- Line order, comments, #EXT-X-MEDIA-SEQUENCE, discontinuity tags, BYTERANGE
+  and every attribute other than URI="..." are passed through untouched.
+- data: URIs and non-http(s) schemes (skd://, ...) are never rewritten.
+- Relative references resolve against `base_url` (the post-redirect playlist
+  URL), inheriting its query like the DASH path when `inherit_query` is true.
+
+Not supported (raises UnsupportedHLSFeature): low-latency HLS tags. Whether to
+support or reject LL-HLS is still an open decision (handoff question 5);
+rejecting is the safe default because blocking reloads and part requests
+cannot be served through a rewritten playlist.
+
+AES-128 key URIs are only routed through `build_url` when `proxy_keys=True`
+(and METHOD=AES-128 with an identity KEYFORMAT). Default is False, matching
+the current "AES-128 = unknown, pass through" policy (absolutised only).
+"""
+
+import re
+from enum import Enum
+from typing import Callable, Optional
+from urllib.parse import urlsplit
+
+from ..stream_format import join_hls_uri
+
+
+class HLSResource(str, Enum):
+    PLAYLIST = "playlist"  # variant / audio / subtitle / I-frame playlist
+    SEGMENT = "segment"    # media segment
+    INIT = "init"          # EXT-X-MAP initialisation segment
+    KEY = "key"            # AES-128 key file
+
+
+class HLSRewriteError(ValueError):
+    """The input is not a rewritable HLS playlist."""
+
+
+class UnsupportedHLSFeature(HLSRewriteError):
+    """The playlist uses a feature the proxied path does not support."""
+
+
+UrlBuilder = Callable[[str, HLSResource], str]
+
+# Same lookbehind as stream_format._HLS_URI_ATTR_RE: vendor attributes such as
+# X-ASSET-URI stay untouched.
+_URI_ATTR_RE = re.compile(r'(?<![A-Za-z0-9-])URI="([^"]*)"')
+_METHOD_RE = re.compile(r'(?<![A-Za-z0-9-])METHOD=([A-Z0-9-]+)')
+_KEYFORMAT_RE = re.compile(r'(?<![A-Za-z0-9-])KEYFORMAT="([^"]*)"')
+
+_LOW_LATENCY_TAGS = frozenset({
+    "EXT-X-PART",
+    "EXT-X-PART-INF",
+    "EXT-X-PRELOAD-HINT",
+    "EXT-X-RENDITION-REPORT",
+})
+
+_PLAYLIST_TAGS = frozenset({"EXT-X-MEDIA", "EXT-X-I-FRAME-STREAM-INF"})
+_KEY_TAGS = frozenset({"EXT-X-KEY", "EXT-X-SESSION-KEY"})
+
+
+def _is_rewritable_reference(uri: str) -> bool:
+    """Relative references and http(s) URLs only. data:, skd:, etc. are left."""
+    return urlsplit(uri).scheme.lower() in ("", "http", "https")
+
+
+def _kind_for_tag(name: str, line: str, proxy_keys: bool) -> Optional[HLSResource]:
+    """Resource kind of the URI="..." attribute of a tag, None = absolutise only."""
+    if name in _PLAYLIST_TAGS:
+        return HLSResource.PLAYLIST
+    if name == "EXT-X-MAP":
+        return HLSResource.INIT
+    if name in _KEY_TAGS and proxy_keys:
+        method = _METHOD_RE.search(line)
+        keyformat = _KEYFORMAT_RE.search(line)
+        if (
+            method and method.group(1) == "AES-128"
+            and (keyformat is None or keyformat.group(1).lower() == "identity")
+        ):
+            return HLSResource.KEY
+    return None
+
+
+_TARGET_DURATION_RE = re.compile(r"^#EXT-X-TARGETDURATION:(\d+)", re.MULTILINE)
+
+
+def hls_cache_ttl(
+    content: str,
+    *,
+    master_ttl: float = 30.0,
+    vod_ttl: float = 120.0,
+    live_min: float = 1.0,
+    live_max: float = 5.0,
+    live_default: float = 2.0,
+) -> float:
+    """How long a REWRITTEN playlist may be reused, in seconds.
+
+    master (multivariant)      -> master_ttl: stable within a session
+    media + #EXT-X-ENDLIST     -> vod_ttl: complete, does not change
+    live media playlist        -> half the target duration, clamped to
+                                  [live_min, live_max]: viewers share one
+                                  rewrite per refresh interval, but nobody is
+                                  served a playlist older than half a segment
+    The upstream Cache-Control header is deliberately not consulted: CDNs
+    often send none or a value unrelated to the refresh interval.
+    """
+    if "#EXT-X-STREAM-INF" in content:
+        return master_ttl
+    if "#EXT-X-ENDLIST" in content:
+        return vod_ttl
+    match = _TARGET_DURATION_RE.search(content)
+    if not match:
+        return live_default
+    return max(live_min, min(live_max, int(match.group(1)) / 2.0))
+
+
+def rewrite_hls_playlist(
+    content: str,
+    base_url: str,
+    build_url: UrlBuilder,
+    *,
+    inherit_query: bool = True,
+    proxy_keys: bool = False,
+) -> str:
+    """Rewrite every reference in an HLS playlist.
+
+    Args:
+        content: playlist text (master or media).
+        base_url: URL the playlist was actually fetched from (post-redirect).
+        build_url: maps (absolute upstream URL, kind) -> URL for the output.
+        inherit_query: relative references inherit base_url's query (see
+            stream_format.join_hls_uri); providers opt out through
+            `inherit_manifest_query_on_segments = False`.
+        proxy_keys: route AES-128 key URIs through build_url (default: only
+            absolutise them).
+
+    Raises:
+        HLSRewriteError: content is not an HLS playlist.
+        UnsupportedHLSFeature: low-latency HLS tags present.
+    """
+    content = content.lstrip("\ufeff")  # a BOM must not turn #EXTM3U into a "segment"
+    if not content.lstrip(" \t\r\n").startswith("#EXTM3U"):
+        raise HLSRewriteError("not an HLS playlist (missing #EXTM3U)")
+
+    def resolve(uri: str, kind: Optional[HLSResource]) -> str:
+        if not uri or not _is_rewritable_reference(uri):
+            return uri
+        # An absolute URL is taken exactly as the upstream wrote it (no query
+        # inheritance, as in MPDRewriter._urljoin_preserve_query); only relative
+        # references resolve against base_url.
+        absolute = uri if urlsplit(uri).scheme else join_hls_uri(base_url, uri, inherit_query)
+        return absolute if kind is None else build_url(absolute, kind)
+
+    out = []
+    expect_variant_uri = False  # the line after #EXT-X-STREAM-INF is a playlist
+
+    for raw in content.splitlines():
+        line = raw.strip()
+        if not line:
+            out.append(raw)
+            continue
+
+        if line.startswith("#"):
+            name = line[1:].split(":", 1)[0]
+            if name in _LOW_LATENCY_TAGS:
+                raise UnsupportedHLSFeature(
+                    f"low-latency HLS tag {name} is not supported on the proxied path"
+                )
+            if name == "EXT-X-STREAM-INF":
+                expect_variant_uri = True
+                out.append(line)
+                continue
+            kind = _kind_for_tag(name, line, proxy_keys)
+            out.append(_URI_ATTR_RE.sub(
+                lambda m, k=kind: f'URI="{resolve(m.group(1), k)}"', line
+            ))
+            continue
+
+        kind = HLSResource.PLAYLIST if expect_variant_uri else HLSResource.SEGMENT
+        expect_variant_uri = False
+        out.append(resolve(line, kind))
+
+    result = "\n".join(out)
+    return result + "\n" if content.endswith("\n") else result

lib/streaming_providers/base/utils/proxy_url.py

@@ -0,0 +1,65 @@
+# streaming_providers/base/utils/proxy_url.py
+"""
+Media-proxy URL construction shared by the HLS path.
+
+Produces exactly the URL form MPDRewriter.build_proxy_url emits for the plain
+(non-decrypt) case:
+
+    {media_proxy_url}/api/proxy/<token>
+
+    token = base64url(urlencode({"url": ..., "proxy": ..., "headers": ...}))
+
+`proxy` is the provider HTTP proxy (omitted when None), `headers` is
+base64url(JSON) of the provider's segment headers (omitted when empty). The
+parameter ORDER (url, proxy, headers) matters for byte-identical output and is
+pinned by test_proxy_url.py against MPDRewriter itself.
+
+Stdlib only. MPDRewriter keeps its own copy for now; the test guards against
+drift until it is switched over to these helpers.
+"""
+
+import base64
+import json
+from typing import Callable, Mapping, Optional
+from urllib.parse import urlencode
+
+
+def encode_url(value: str) -> str:
+    """base64url without padding (same as MPDRewriter.encode_url)."""
+    return base64.urlsafe_b64encode(value.encode("utf-8")).decode("utf-8").rstrip("=")
+
+
+def decode_url(token: str) -> str:
+    """Inverse of encode_url. Raises ValueError (binascii.Error and
+    UnicodeDecodeError are subclasses) for malformed input."""
+    # validate=True: urlsafe_b64decode would silently drop invalid characters.
+    raw = base64.b64decode(token + "=" * (-len(token) % 4), altchars=b"-_", validate=True)
+    return raw.decode("utf-8")
+
+
+def encode_segment_headers(headers: Mapping[str, str]) -> str:
+    """base64url(JSON) of a header mapping, as carried in the `headers` param."""
+    return base64.urlsafe_b64encode(json.dumps(headers).encode()).decode().rstrip("=")
+
+
+def make_media_proxy_url_builder(
+    media_proxy_url: str,
+    *,
+    provider_proxy_url: Optional[str] = None,
+    segment_headers: Optional[Mapping[str, str]] = None,
+    endpoint: str = "proxy",
+) -> Callable[[str], str]:
+    """Return build(url) -> media-proxy URL. The static params (provider proxy,
+    encoded headers) are computed once, so rewriting a playlist with thousands
+    of segments does not re-encode the headers per segment."""
+    base = f"{media_proxy_url.rstrip('/')}/api/{endpoint}"
+    static = {}
+    if provider_proxy_url:
+        static["proxy"] = provider_proxy_url
+    if segment_headers:
+        static["headers"] = encode_segment_headers(segment_headers)
+
+    def build(url: str) -> str:
+        return f"{base}/{encode_url(urlencode({'url': url, **static}))}"
+
+    return build

lib/streaming_providers/base/utils/single_flight.py

@@ -0,0 +1,54 @@
+# streaming_providers/base/utils/single_flight.py
+"""
+Single-flight: concurrent callers asking for the same key share ONE execution.
+
+Used for HLS playlist refreshes: N viewers polling the same live playlist at
+the same moment cause one upstream fetch + rewrite, not N. The first caller
+for a key (the leader) runs `fn`; callers arriving while it runs wait for its
+result (or exception). Nothing is cached here — the entry disappears as soon
+as the leader finishes; caching is the caller's job.
+"""
+
+import threading
+from typing import Callable, Dict, Hashable, TypeVar
+
+T = TypeVar("T")
+
+
+class _Call:
+    __slots__ = ("event", "result", "error")
+
+    def __init__(self):
+        self.event = threading.Event()
+        self.result = None
+        self.error = None
+
+
+class SingleFlight:
+    def __init__(self, wait_timeout: float = 60.0):
+        self._lock = threading.Lock()
+        self._calls: Dict[Hashable, _Call] = {}
+        self._wait_timeout = wait_timeout
+
+    def do(self, key: Hashable, fn: Callable[[], T]) -> T:
+        with self._lock:
+            call = self._calls.get(key)
+            leader = call is None
+            if leader:
+                call = self._calls[key] = _Call()
+
+        if leader:
+            try:
+                call.result = fn()
+            except BaseException as exc:  # shared with followers, re-raised below
+                call.error = exc
+            finally:
+                with self._lock:
+                    self._calls.pop(key, None)
+                call.event.set()
+        elif not call.event.wait(self._wait_timeout):
+            raise TimeoutError(f"single-flight wait for {key!r} timed out")
+
+        if call.error is not None:
+            raise call.error
+        return call.result

lib/streaming_providers/providers/_template_v2/README.md

@@ -1157,19 +1157,22 @@ thousands of manifests, so it uses the hint: HLS-hinted channels get the
 HLS), never per provider. Leave `None` for dynamic-manifest items — **unknown is not
 DASH**; without a hint nothing changes for existing DASH providers.
 
-**HLS limits** — state them in the provider docstring (phase 1, no HLS rewriter or
-media-proxy path yet):
+**HLS limits** — state them in the provider docstring. HLS can be served through the
+backend and the media proxy (phase 2); what is still missing is in this table and in §17.
 
 | Situation | Behaviour today |
 |---|---|
 | Clear HLS, CDN needs no proxy and no special headers | works: the backend answers with a redirect to the upstream playlist |
 | `requires_manifest_context = True` | the backend fetches the playlist and makes every relative URI absolute (HLS has no `<BaseURL>`); the manifest query (token) is inherited unless `inherit_manifest_query_on_segments = False` |
-| Provider needs an HTTP proxy (`needs_proxy`) | **not supported yet**: redirected anyway, a warning is logged; geo-proxied HLS will not play |
-| CDN needs headers for manifest/segments | **unverified**: a redirect carries none; headers reach the client only via `x-kodi-stream-headers` / KODIPROPs. Prove it on a device before you rely on it (§12.7) |
-| `METHOD=AES-128` | reported as unknown (never "clear"); passed through unchanged, the player fetches the key itself |
+| Provider needs an HTTP proxy (`needs_proxy`) | **works when `MEDIA_PROXY_URL` is set** and the request has no `no_proxy=true`: the backend fetches every playlist through the provider's HTTP manager and rewrites it. Segments and `EXT-X-MAP` init segments become `{MEDIA_PROXY_URL}/api/proxy/<token>` (provider segment headers and proxy inside the token, as for DASH); nested playlists (variants, audio/subtitle renditions) point back at the same stream URL as `…/index.m3u8?<query>&playlist=<base64url upstream URL>`. Without `MEDIA_PROXY_URL`: redirect plus a warning, geo-proxied HLS will not play |
+| CDN needs headers for manifest/segments | **unverified, and not proxied by default.** Unlike DASH, "needs headers" alone does not select the proxy path, so a redirect that already plays keeps playing. A provider whose CDN is header-gated sets `hls_requires_proxy = True` and must return the headers from `get_manifest_headers` / `get_segment_headers`. Header-only gating through `x-kodi-stream-headers` / KODIPROPs on the redirect is still unproven: prove it on a device before you rely on either (§12.7) |
+| `METHOD=AES-128` | reported as unknown (never "clear"); passed through unchanged, the player fetches the key itself. Key URIs are not routed through the media proxy yet (the rewriter can do it; the policy is undecided) |
 | `SAMPLE-AES`, Widevine/FairPlay HLS | detected as encrypted and passed through; no PSSH extraction, no server-side handling |
 | Only the first variant is probed | separately encrypted audio renditions are not checked |
-| `index.m3u8` alias | channels only; events, VOD and recordings still expose `index.mpd` |
+| Low-latency HLS (`EXT-X-PART`, preload hints) | rejected on the proxied path (502 with the reason); not affected on the redirect path |
+| `index.m3u8` alias | channels, events, VOD and recordings; `…/manifest` returns `stream_format` and points at the alias for HLS |
+| `playlist` query parameter | reserved on stream URLs: it carries a nested playlist of the proxied path and skips the format and catchup-window checks of the first request |
+| Proxied playlists: cache and trust | rewritten playlists are cached in memory only (about half the target duration for live, 30 s for a master, 120 s for VOD) and concurrent identical requests share one fetch. The `playlist` value and the media-proxy tokens are **unsigned**: backend and media proxy must only be reachable by trusted clients (LAN) |
 
 **Locally minted, short-lived manifest URLs (Eon).** When the provider builds the
 manifest URL itself (an AES-encrypted payload with a server `ctime` and a session id):
@@ -1757,9 +1760,10 @@ Things that already cost time. Each is covered by a test or the contract unless
     backend sniffs the content; never branch on the extension (§11).
 67. Setting `streaming_format` per provider, or `"dash"` as a "default" — unknown
     is `None`; a wrong hint sends playlists to the wrong alias (§13).
-68. HLS from a provider that needs an HTTP proxy or CDN headers: the redirect loses
-    both; it logs a warning and does not play. Document it, do not "fix" it in the
-    provider (§11).
+68. HLS from a provider whose CDN needs an HTTP proxy or headers: with `needs_proxy` and
+    `MEDIA_PROXY_URL` the backend proxies it; without `MEDIA_PROXY_URL`, or for a
+    header-gated CDN without `hls_requires_proxy`, the redirect loses both and the stream
+    does not play. Document it, do not "fix" it in the provider (§11).
 69. Treating `METHOD=AES-128` HLS as clear (it is *unknown* to the backend) or
     returning DRM configs for it (§11).
 70. Brand/operator modelled as country, or resolved by list position (§4).
@@ -1786,6 +1790,14 @@ Things that already cost time. Each is covered by a test or the contract unless
     `403` — as "wrong credentials" (§9).
 80. Persisting a re-derivable secret (stream key, anything the next token response
     returns) through `SessionManager`: unknown keys are logged in clear at INFO (§9).
+81. Exposing the backend or the media proxy beyond a trusted LAN: the `playlist` value and
+    the proxy tokens are unsigned base64, so a crafted one makes the proxy fetch any URL
+    and makes the backend send provider headers to any host. Signing comes first (§11,
+    §17).
+82. Assuming the backend and the player see the same playlist for a redirecting or
+    tokenised CDN (observed with ZDF/KiKA): the "clear" verdict and the proxied rewrite
+    come from the backend's own fetch, the redirect path hands the player the CDN's answer
+    (§11).
 
 ---
 
@@ -1810,10 +1822,14 @@ Known gaps in the base, so you don't hunt for them:
 * `README` v1 (`_template`) describes the old shape; it is not rewritten.
 * Provider-specific features outside the ABCs (restart, `get_restart`) are free-form;
   there is no `RestartInfo` type yet.
-* **HLS**: no HLS rewriter / media-proxy path (proxied or header-gated HLS and
-  geo-proxied providers do not play), AES-128 policy undecided, encrypted HLS only
-  passed through, only the first variant probed, `index.m3u8` alias for channels
-  only (§11).
+* **HLS**: the proxied path exists for `needs_proxy` providers and `hls_requires_proxy`
+  opt-ins (§11). Open: AES-128 key proxying (policy undecided, `_analyse_hls`
+  unchanged), low-latency HLS is rejected, encrypted HLS only passed through, only the
+  first variant probed, header-only gating unverified on a device. **No signing:**
+  `playlist` values and media-proxy tokens are plain base64, so backend and media proxy
+  stay LAN-only until signing is added. The media proxy still has to be adapted to the
+  HLS expectations (segments and keys, Range, redirects, content types). The VOD and
+  recordings M3U exports were not reviewed for the `index.m3u8` alias.
 * **`HTTPManager` has no `patch()`** (`_make_request("PATCH", …)` exists, only the public
   wrapper is missing). Eon needs `PATCH v2/devices`; until it is added the provider
   `hasattr`-checks and falls back to the `v1` POST (which mints a *new* device number).