Redirect hls to proxy

This commit is contained in:
Nirvana
2026-10-24 08:16:36 +02:00
parent e4e1b44f08
commit c947bf17f8
9 changed files with 771 additions and 53 deletions
@@ -0,0 +1,195 @@
# streaming_providers/base/utils/hls_rewriter.py
"""
HLS playlist rewriter (phase 2, Option A).
Pure text transformation, no I/O and no knowledge of the proxy URL scheme or
token signing: every reference in the playlist is resolved to an absolute
upstream URL and handed to a caller-supplied `build_url(url, kind)`, which
returns the URL to put into the playlist. Under Option A the caller maps
PLAYLIST -> the backend's playlist endpoint
SEGMENT / INIT / KEY -> the media proxy (/api/proxy/<token>)
Kind is decided from the TAG a reference sits in, never from the file
extension (tokenised CDN URLs usually have none).
Guarantees
- Line order, comments, #EXT-X-MEDIA-SEQUENCE, discontinuity tags, BYTERANGE
and every attribute other than URI="..." are passed through untouched.
- data: URIs and non-http(s) schemes (skd://, ...) are never rewritten.
- Relative references resolve against `base_url` (the post-redirect playlist
URL), inheriting its query like the DASH path when `inherit_query` is true.
Not supported (raises UnsupportedHLSFeature): low-latency HLS tags. Whether to
support or reject LL-HLS is still an open decision (handoff question 5);
rejecting is the safe default because blocking reloads and part requests
cannot be served through a rewritten playlist.
AES-128 key URIs are only routed through `build_url` when `proxy_keys=True`
(and METHOD=AES-128 with an identity KEYFORMAT). Default is False, matching
the current "AES-128 = unknown, pass through" policy (absolutised only).
"""
import re
from enum import Enum
from typing import Callable, Optional
from urllib.parse import urlsplit
from ..stream_format import join_hls_uri
class HLSResource(str, Enum):
PLAYLIST = "playlist" # variant / audio / subtitle / I-frame playlist
SEGMENT = "segment" # media segment
INIT = "init" # EXT-X-MAP initialisation segment
KEY = "key" # AES-128 key file
class HLSRewriteError(ValueError):
"""The input is not a rewritable HLS playlist."""
class UnsupportedHLSFeature(HLSRewriteError):
"""The playlist uses a feature the proxied path does not support."""
UrlBuilder = Callable[[str, HLSResource], str]
# Same lookbehind as stream_format._HLS_URI_ATTR_RE: vendor attributes such as
# X-ASSET-URI stay untouched.
_URI_ATTR_RE = re.compile(r'(?<![A-Za-z0-9-])URI="([^"]*)"')
_METHOD_RE = re.compile(r'(?<![A-Za-z0-9-])METHOD=([A-Z0-9-]+)')
_KEYFORMAT_RE = re.compile(r'(?<![A-Za-z0-9-])KEYFORMAT="([^"]*)"')
_LOW_LATENCY_TAGS = frozenset({
"EXT-X-PART",
"EXT-X-PART-INF",
"EXT-X-PRELOAD-HINT",
"EXT-X-RENDITION-REPORT",
})
_PLAYLIST_TAGS = frozenset({"EXT-X-MEDIA", "EXT-X-I-FRAME-STREAM-INF"})
_KEY_TAGS = frozenset({"EXT-X-KEY", "EXT-X-SESSION-KEY"})
def _is_rewritable_reference(uri: str) -> bool:
"""Relative references and http(s) URLs only. data:, skd:, etc. are left."""
return urlsplit(uri).scheme.lower() in ("", "http", "https")
def _kind_for_tag(name: str, line: str, proxy_keys: bool) -> Optional[HLSResource]:
"""Resource kind of the URI="..." attribute of a tag, None = absolutise only."""
if name in _PLAYLIST_TAGS:
return HLSResource.PLAYLIST
if name == "EXT-X-MAP":
return HLSResource.INIT
if name in _KEY_TAGS and proxy_keys:
method = _METHOD_RE.search(line)
keyformat = _KEYFORMAT_RE.search(line)
if (
method and method.group(1) == "AES-128"
and (keyformat is None or keyformat.group(1).lower() == "identity")
):
return HLSResource.KEY
return None
_TARGET_DURATION_RE = re.compile(r"^#EXT-X-TARGETDURATION:(\d+)", re.MULTILINE)
def hls_cache_ttl(
content: str,
*,
master_ttl: float = 30.0,
vod_ttl: float = 120.0,
live_min: float = 1.0,
live_max: float = 5.0,
live_default: float = 2.0,
) -> float:
"""How long a REWRITTEN playlist may be reused, in seconds.
master (multivariant) -> master_ttl: stable within a session
media + #EXT-X-ENDLIST -> vod_ttl: complete, does not change
live media playlist -> half the target duration, clamped to
[live_min, live_max]: viewers share one
rewrite per refresh interval, but nobody is
served a playlist older than half a segment
The upstream Cache-Control header is deliberately not consulted: CDNs
often send none or a value unrelated to the refresh interval.
"""
if "#EXT-X-STREAM-INF" in content:
return master_ttl
if "#EXT-X-ENDLIST" in content:
return vod_ttl
match = _TARGET_DURATION_RE.search(content)
if not match:
return live_default
return max(live_min, min(live_max, int(match.group(1)) / 2.0))
def rewrite_hls_playlist(
content: str,
base_url: str,
build_url: UrlBuilder,
*,
inherit_query: bool = True,
proxy_keys: bool = False,
) -> str:
"""Rewrite every reference in an HLS playlist.
Args:
content: playlist text (master or media).
base_url: URL the playlist was actually fetched from (post-redirect).
build_url: maps (absolute upstream URL, kind) -> URL for the output.
inherit_query: relative references inherit base_url's query (see
stream_format.join_hls_uri); providers opt out through
`inherit_manifest_query_on_segments = False`.
proxy_keys: route AES-128 key URIs through build_url (default: only
absolutise them).
Raises:
HLSRewriteError: content is not an HLS playlist.
UnsupportedHLSFeature: low-latency HLS tags present.
"""
content = content.lstrip("\ufeff") # a BOM must not turn #EXTM3U into a "segment"
if not content.lstrip(" \t\r\n").startswith("#EXTM3U"):
raise HLSRewriteError("not an HLS playlist (missing #EXTM3U)")
def resolve(uri: str, kind: Optional[HLSResource]) -> str:
if not uri or not _is_rewritable_reference(uri):
return uri
# An absolute URL is taken exactly as the upstream wrote it (no query
# inheritance, as in MPDRewriter._urljoin_preserve_query); only relative
# references resolve against base_url.
absolute = uri if urlsplit(uri).scheme else join_hls_uri(base_url, uri, inherit_query)
return absolute if kind is None else build_url(absolute, kind)
out = []
expect_variant_uri = False # the line after #EXT-X-STREAM-INF is a playlist
for raw in content.splitlines():
line = raw.strip()
if not line:
out.append(raw)
continue
if line.startswith("#"):
name = line[1:].split(":", 1)[0]
if name in _LOW_LATENCY_TAGS:
raise UnsupportedHLSFeature(
f"low-latency HLS tag {name} is not supported on the proxied path"
)
if name == "EXT-X-STREAM-INF":
expect_variant_uri = True
out.append(line)
continue
kind = _kind_for_tag(name, line, proxy_keys)
out.append(_URI_ATTR_RE.sub(
lambda m, k=kind: f'URI="{resolve(m.group(1), k)}"', line
))
continue
kind = HLSResource.PLAYLIST if expect_variant_uri else HLSResource.SEGMENT
expect_variant_uri = False
out.append(resolve(line, kind))
result = "\n".join(out)
return result + "\n" if content.endswith("\n") else result
@@ -0,0 +1,65 @@
# streaming_providers/base/utils/proxy_url.py
"""
Media-proxy URL construction shared by the HLS path.
Produces exactly the URL form MPDRewriter.build_proxy_url emits for the plain
(non-decrypt) case:
{media_proxy_url}/api/proxy/<token>
token = base64url(urlencode({"url": ..., "proxy": ..., "headers": ...}))
`proxy` is the provider HTTP proxy (omitted when None), `headers` is
base64url(JSON) of the provider's segment headers (omitted when empty). The
parameter ORDER (url, proxy, headers) matters for byte-identical output and is
pinned by test_proxy_url.py against MPDRewriter itself.
Stdlib only. MPDRewriter keeps its own copy for now; the test guards against
drift until it is switched over to these helpers.
"""
import base64
import json
from typing import Callable, Mapping, Optional
from urllib.parse import urlencode
def encode_url(value: str) -> str:
"""base64url without padding (same as MPDRewriter.encode_url)."""
return base64.urlsafe_b64encode(value.encode("utf-8")).decode("utf-8").rstrip("=")
def decode_url(token: str) -> str:
"""Inverse of encode_url. Raises ValueError (binascii.Error and
UnicodeDecodeError are subclasses) for malformed input."""
# validate=True: urlsafe_b64decode would silently drop invalid characters.
raw = base64.b64decode(token + "=" * (-len(token) % 4), altchars=b"-_", validate=True)
return raw.decode("utf-8")
def encode_segment_headers(headers: Mapping[str, str]) -> str:
"""base64url(JSON) of a header mapping, as carried in the `headers` param."""
return base64.urlsafe_b64encode(json.dumps(headers).encode()).decode().rstrip("=")
def make_media_proxy_url_builder(
media_proxy_url: str,
*,
provider_proxy_url: Optional[str] = None,
segment_headers: Optional[Mapping[str, str]] = None,
endpoint: str = "proxy",
) -> Callable[[str], str]:
"""Return build(url) -> media-proxy URL. The static params (provider proxy,
encoded headers) are computed once, so rewriting a playlist with thousands
of segments does not re-encode the headers per segment."""
base = f"{media_proxy_url.rstrip('/')}/api/{endpoint}"
static = {}
if provider_proxy_url:
static["proxy"] = provider_proxy_url
if segment_headers:
static["headers"] = encode_segment_headers(segment_headers)
def build(url: str) -> str:
return f"{base}/{encode_url(urlencode({'url': url, **static}))}"
return build
@@ -0,0 +1,54 @@
# streaming_providers/base/utils/single_flight.py
"""
Single-flight: concurrent callers asking for the same key share ONE execution.
Used for HLS playlist refreshes: N viewers polling the same live playlist at
the same moment cause one upstream fetch + rewrite, not N. The first caller
for a key (the leader) runs `fn`; callers arriving while it runs wait for its
result (or exception). Nothing is cached here — the entry disappears as soon
as the leader finishes; caching is the caller's job.
"""
import threading
from typing import Callable, Dict, Hashable, TypeVar
T = TypeVar("T")
class _Call:
__slots__ = ("event", "result", "error")
def __init__(self):
self.event = threading.Event()
self.result = None
self.error = None
class SingleFlight:
def __init__(self, wait_timeout: float = 60.0):
self._lock = threading.Lock()
self._calls: Dict[Hashable, _Call] = {}
self._wait_timeout = wait_timeout
def do(self, key: Hashable, fn: Callable[[], T]) -> T:
with self._lock:
call = self._calls.get(key)
leader = call is None
if leader:
call = self._calls[key] = _Call()
if leader:
try:
call.result = fn()
except BaseException as exc: # shared with followers, re-raised below
call.error = exc
finally:
with self._lock:
self._calls.pop(key, None)
call.event.set()
elif not call.event.wait(self._wait_timeout):
raise TimeoutError(f"single-flight wait for {key!r} timed out")
if call.error is not None:
raise call.error
return call.result
@@ -1157,19 +1157,22 @@ thousands of manifests, so it uses the hint: HLS-hinted channels get the
HLS), never per provider. Leave `None` for dynamic-manifest items — **unknown is not
DASH**; without a hint nothing changes for existing DASH providers.
**HLS limits** — state them in the provider docstring (phase 1, no HLS rewriter or
media-proxy path yet):
**HLS limits** — state them in the provider docstring. HLS can be served through the
backend and the media proxy (phase 2); what is still missing is in this table and in §17.
| Situation | Behaviour today |
|---|---|
| Clear HLS, CDN needs no proxy and no special headers | works: the backend answers with a redirect to the upstream playlist |
| `requires_manifest_context = True` | the backend fetches the playlist and makes every relative URI absolute (HLS has no `<BaseURL>`); the manifest query (token) is inherited unless `inherit_manifest_query_on_segments = False` |
| Provider needs an HTTP proxy (`needs_proxy`) | **not supported yet**: redirected anyway, a warning is logged; geo-proxied HLS will not play |
| CDN needs headers for manifest/segments | **unverified**: a redirect carries none; headers reach the client only via `x-kodi-stream-headers` / KODIPROPs. Prove it on a device before you rely on it (§12.7) |
| `METHOD=AES-128` | reported as unknown (never "clear"); passed through unchanged, the player fetches the key itself |
| Provider needs an HTTP proxy (`needs_proxy`) | **works when `MEDIA_PROXY_URL` is set** and the request has no `no_proxy=true`: the backend fetches every playlist through the provider's HTTP manager and rewrites it. Segments and `EXT-X-MAP` init segments become `{MEDIA_PROXY_URL}/api/proxy/<token>` (provider segment headers and proxy inside the token, as for DASH); nested playlists (variants, audio/subtitle renditions) point back at the same stream URL as `…/index.m3u8?<query>&playlist=<base64url upstream URL>`. Without `MEDIA_PROXY_URL`: redirect plus a warning, geo-proxied HLS will not play |
| CDN needs headers for manifest/segments | **unverified, and not proxied by default.** Unlike DASH, "needs headers" alone does not select the proxy path, so a redirect that already plays keeps playing. A provider whose CDN is header-gated sets `hls_requires_proxy = True` and must return the headers from `get_manifest_headers` / `get_segment_headers`. Header-only gating through `x-kodi-stream-headers` / KODIPROPs on the redirect is still unproven: prove it on a device before you rely on either (§12.7) |
| `METHOD=AES-128` | reported as unknown (never "clear"); passed through unchanged, the player fetches the key itself. Key URIs are not routed through the media proxy yet (the rewriter can do it; the policy is undecided) |
| `SAMPLE-AES`, Widevine/FairPlay HLS | detected as encrypted and passed through; no PSSH extraction, no server-side handling |
| Only the first variant is probed | separately encrypted audio renditions are not checked |
| `index.m3u8` alias | channels only; events, VOD and recordings still expose `index.mpd` |
| Low-latency HLS (`EXT-X-PART`, preload hints) | rejected on the proxied path (502 with the reason); not affected on the redirect path |
| `index.m3u8` alias | channels, events, VOD and recordings; `…/manifest` returns `stream_format` and points at the alias for HLS |
| `playlist` query parameter | reserved on stream URLs: it carries a nested playlist of the proxied path and skips the format and catchup-window checks of the first request |
| Proxied playlists: cache and trust | rewritten playlists are cached in memory only (about half the target duration for live, 30 s for a master, 120 s for VOD) and concurrent identical requests share one fetch. The `playlist` value and the media-proxy tokens are **unsigned**: backend and media proxy must only be reachable by trusted clients (LAN) |
**Locally minted, short-lived manifest URLs (Eon).** When the provider builds the
manifest URL itself (an AES-encrypted payload with a server `ctime` and a session id):
@@ -1757,9 +1760,10 @@ Things that already cost time. Each is covered by a test or the contract unless
backend sniffs the content; never branch on the extension (§11).
67. Setting `streaming_format` per provider, or `"dash"` as a "default" — unknown
is `None`; a wrong hint sends playlists to the wrong alias (§13).
68. HLS from a provider that needs an HTTP proxy or CDN headers: the redirect loses
both; it logs a warning and does not play. Document it, do not "fix" it in the
provider (§11).
68. HLS from a provider whose CDN needs an HTTP proxy or headers: with `needs_proxy` and
`MEDIA_PROXY_URL` the backend proxies it; without `MEDIA_PROXY_URL`, or for a
header-gated CDN without `hls_requires_proxy`, the redirect loses both and the stream
does not play. Document it, do not "fix" it in the provider (§11).
69. Treating `METHOD=AES-128` HLS as clear (it is *unknown* to the backend) or
returning DRM configs for it (§11).
70. Brand/operator modelled as country, or resolved by list position (§4).
@@ -1786,6 +1790,14 @@ Things that already cost time. Each is covered by a test or the contract unless
`403` — as "wrong credentials" (§9).
80. Persisting a re-derivable secret (stream key, anything the next token response
returns) through `SessionManager`: unknown keys are logged in clear at INFO (§9).
81. Exposing the backend or the media proxy beyond a trusted LAN: the `playlist` value and
the proxy tokens are unsigned base64, so a crafted one makes the proxy fetch any URL
and makes the backend send provider headers to any host. Signing comes first (§11,
§17).
82. Assuming the backend and the player see the same playlist for a redirecting or
tokenised CDN (observed with ZDF/KiKA): the "clear" verdict and the proxied rewrite
come from the backend's own fetch, the redirect path hands the player the CDN's answer
(§11).
---
@@ -1810,10 +1822,14 @@ Known gaps in the base, so you don't hunt for them:
* `README` v1 (`_template`) describes the old shape; it is not rewritten.
* Provider-specific features outside the ABCs (restart, `get_restart`) are free-form;
there is no `RestartInfo` type yet.
* **HLS**: no HLS rewriter / media-proxy path (proxied or header-gated HLS and
geo-proxied providers do not play), AES-128 policy undecided, encrypted HLS only
passed through, only the first variant probed, `index.m3u8` alias for channels
only (§11).
* **HLS**: the proxied path exists for `needs_proxy` providers and `hls_requires_proxy`
opt-ins (§11). Open: AES-128 key proxying (policy undecided, `_analyse_hls`
unchanged), low-latency HLS is rejected, encrypted HLS only passed through, only the
first variant probed, header-only gating unverified on a device. **No signing:**
`playlist` values and media-proxy tokens are plain base64, so backend and media proxy
stay LAN-only until signing is added. The media proxy still has to be adapted to the
HLS expectations (segments and keys, Range, redirects, content types). The VOD and
recordings M3U exports were not reviewed for the `index.m3u8` alias.
* **`HTTPManager` has no `patch()`** (`_make_request("PATCH", …)` exists, only the public
wrapper is missing). Eon needs `PATCH v2/devices`; until it is added the provider
`hasattr`-checks and falls back to the `v1` POST (which mints a *new* device number).
+191 -20
View File
@@ -30,7 +30,7 @@ import base64
import json
import re
import time
from urllib.parse import urljoin
from urllib.parse import urlencode, urljoin, urlsplit
from bottle import redirect, request, response
from streaming_providers.base.stream_format import (
@@ -40,6 +40,7 @@ from streaming_providers.base.stream_format import (
detect_format,
)
from streaming_providers.base.utils import logger
from streaming_providers.base.utils.proxy_url import decode_url, encode_url
from streaming_providers.base.utils.manifest_utils import ManifestUtils
# ---------------------------------------------------------------------------
@@ -584,6 +585,126 @@ def make_helpers(manager, service):
logger.debug(f"Redirecting to manifest: {manifest_url}")
return redirect(manifest_url)
def _header_context(
content_type: str,
country,
is_catchup: bool,
start_time,
end_time,
epg_id,
) -> dict:
"""Provider header context, the same keyword set _stream_needs_headers
passes to get_manifest_headers / get_segment_headers."""
kwargs = {}
if country:
kwargs["country"] = country
if is_catchup and content_type == CONTENT_TYPE_CHANNEL:
kwargs.update(start_time=start_time, end_time=end_time, epg_id=epg_id)
return kwargs
def _proxied_hls_playlist(
content_type: str,
provider: str,
content_id: str,
upstream_url: str,
country,
is_catchup: bool,
start_time,
end_time,
epg_id,
):
"""Serve one upstream playlist rewritten for the media proxy (Option A).
Nested playlists (variants, audio/subtitle renditions, I-frame playlists)
point back at THIS endpoint: the current URL with the query unchanged plus
`playlist=<base64url upstream URL>`. Everything that selects the stream
(country, catchup times, drm_variant, ...) therefore rides along in the
query and no extra state or route is needed. Segment, init and key URLs
become media-proxy URLs inside service.get_proxied_hls_playlist.
Assumes the backend is reachable only by trusted clients (LAN): the
`playlist` value is not signed, exactly like the DASH proxy tokens. If this
endpoint is ever exposed externally it needs signing, otherwise a crafted
value makes the backend send provider headers to an arbitrary host.
"""
parts = request.urlparts
# Whatever extension the client used, nested playlists are .m3u8.
path = re.sub(r"index\.mpd$", "index.m3u8", parts.path)
base = f"{parts.scheme}://{parts.netloc}{path}"
carried = [(k, v) for k, v in request.query.allitems() if k != "playlist"]
def build_playlist_url(url: str) -> str:
return f"{base}?{urlencode(carried + [('playlist', encode_url(url))])}"
return service.get_proxied_hls_playlist(
provider, content_id, upstream_url,
build_playlist_url=build_playlist_url,
# The output embeds `base` and the carried query, so they scope the cache.
cache_scope=f"{base}?{urlencode(carried)}",
header_kwargs=_header_context(
content_type, country, is_catchup, start_time, end_time, epg_id
),
)
def _serve_hls_variant(
content_type: str,
provider: str,
content_id: str,
playlist_ref: str,
country,
is_catchup: bool,
start_time,
end_time,
epg_id,
):
"""A `?playlist=` request: a nested playlist the player fetched from a
rewritten master. Skips format/DRM resolution and catchup-window
validation on purpose: the initial request already did both, and a live
playlist refresh must not fail because the window edge moved."""
try:
upstream_url = decode_url(playlist_ref)
except ValueError:
response.status = 400
return {"error": "Invalid playlist reference"}
if urlsplit(upstream_url).scheme not in ("http", "https"):
response.status = 400
return {"error": "Invalid playlist reference"}
return _proxied_hls_playlist(
content_type, provider, content_id, upstream_url,
country, is_catchup, start_time, end_time, epg_id,
)
def _hls_needs_media_proxy(
provider: str,
content_type: str,
content_id: str,
country,
is_catchup: bool,
start_time,
end_time,
epg_id,
) -> bool:
"""Should this HLS stream go through the backend + media proxy?
- provider with an HTTP/geo proxy (needs_proxy): yes — a redirect would
leave from a different IP.
- header-gated CDN: only for providers that opt in with
`hls_requires_proxy = True` AND actually return headers. Unlike DASH, a
plain "needs headers" is deliberately NOT enough: HLS playlists that work
through the redirect (ZDF/KiKA sends a Referer but plays fine) must keep
doing so until header KODIPROPs are proven insufficient on a device.
"""
if manager.needs_proxy(provider):
return True
provider_instance = manager.get_provider(provider)
if provider_instance and getattr(provider_instance, "hls_requires_proxy", False):
return _stream_needs_headers(
content_type, provider, content_id, country,
is_catchup=is_catchup, start_time=start_time,
end_time=end_time, epg_id=epg_id,
)
return False
def _serve_hls_passthrough(
content_type: str,
provider: str,
@@ -595,35 +716,74 @@ def make_helpers(manager, service):
end_time,
epg_id,
is_unencrypted: bool,
no_proxy: bool = False,
):
"""
HLS phase 1: hand the client the upstream playlist WITHOUT rewriting.
HLS entry point (the DASH routing in _resolve_stream_unified — ClearKey
rewrite, proxied manifest — never runs for HLS: MPDRewriter cannot parse a
playlist).
There is no HLS rewriter or proxy path yet — MPDRewriter and the media
proxy are DASH-only. The DASH routing in _resolve_stream_unified
(ClearKey rewrite, proxied manifest) must therefore never run for HLS:
MPDRewriter cannot parse a playlist and the request would end in a 502.
Instead the playlist is redirected to (or, for providers with
requires_manifest_context, fetched and absolutised by
_redirect_or_fetch).
Two outcomes:
* proxied (Option A): when _hls_needs_media_proxy says so, a media proxy
is configured and no_proxy is not set, the playlist is fetched through
the provider's HTTP manager and rewritten (service.get_proxied_hls_playlist).
Segments go through the media proxy, nested playlists come back to this
endpoint via `?playlist=`.
* passthrough (phase 1): otherwise the upstream playlist is redirected to
(or, for requires_manifest_context providers, fetched and absolutised by
_redirect_or_fetch).
Limits until an HLS proxy path exists: providers whose manifest or
segment fetches need an HTTP proxy cannot be served this way, and
provider headers only reach the client via x-kodi-stream-headers /
KODIPROPs. Encrypted or undecidable HLS (e.g. AES-128, SAMPLE-AES) is
passed through as-is; the client's player and the x-kodi-drm-configs
header have to cope with it.
Encrypted or undecidable HLS (AES-128, SAMPLE-AES, Widevine, FairPlay) is
passed on as it is; the client's player and the x-kodi-drm-configs header have
to cope with it.
"""
if not is_unencrypted:
logger.debug(
f"HLS {provider}/{content_id}: not verified clear — passing the "
f"playlist through unchanged"
)
if manager.needs_proxy(provider):
wants_proxy = not no_proxy and _hls_needs_media_proxy(
provider, content_type, content_id, country,
is_catchup, start_time, end_time, epg_id,
)
if wants_proxy and service.media_proxy_url:
if is_catchup:
manifest_url = manager.get_catchup_manifest(
provider_name=provider,
channel_id=content_id,
start_time=start_time,
end_time=end_time,
epg_id=epg_id,
country=country,
drm_variant=drm_variant,
)
if not manifest_url:
response.status = 404
return {
"error": f'Catchup manifest not available for channel "{content_id}"'
}
else:
manifest_url = _get_manifest_url(
content_type, provider, content_id,
country=country, drm_variant=drm_variant,
)
if not manifest_url:
response.status = 404
return {
"error": (
f'Manifest not available for {content_type} '
f'"{content_id}" from provider "{provider}"'
)
}
return _proxied_hls_playlist(
content_type, provider, content_id, manifest_url,
country, is_catchup, start_time, end_time, epg_id,
)
if wants_proxy:
logger.warning(
f"HLS {provider}/{content_id}: provider uses an HTTP proxy but HLS "
f"proxying is not implemented — redirecting to the upstream "
f"playlist; playback may fail"
f"HLS {provider}/{content_id}: needs the media proxy but MEDIA_PROXY_URL "
f"is not set — redirecting to the upstream playlist; playback may fail"
)
if is_catchup:
@@ -716,6 +876,15 @@ def make_helpers(manager, service):
catchup content that turns out unencrypted. Meaningless
when no_proxy=True (plain redirect, no rewriter involved).
"""
# --- Nested HLS playlist (?playlist=...): handled before everything else,
# see _serve_hls_variant. ---
playlist_ref = request.query.get("playlist")
if playlist_ref:
return _serve_hls_variant(
content_type, provider, content_id, playlist_ref,
country, is_catchup, start_time, end_time, epg_id,
)
# --- Catchup window validation (channels only), before anything else —
# previously duplicated between channels.py (for auto/noproxy) and
# _resolve_decrypted_stream (for decrypt); now the single source of truth
@@ -761,12 +930,14 @@ def make_helpers(manager, service):
is_unencrypted = "none" in drm_dict
# ==================================================================
# HLS: no rewriter exists, so none of the DASH routing below applies.
# HLS: own routing (proxied playlist rewrite or passthrough); none of the
# DASH routing below applies.
# ==================================================================
if stream_format is StreamFormat.HLS:
return _serve_hls_passthrough(
content_type, provider, content_id, country, drm_variant,
is_catchup, start_time, end_time, epg_id, is_unencrypted,
no_proxy=no_proxy,
)
# ==================================================================
+17 -3
View File
@@ -14,6 +14,7 @@ directly through _resolve_stream_unified.
"""
from bottle import HTTPResponse, request, response
from streaming_providers.base.stream_format import StreamFormat
from streaming_providers.base.utils import logger
@@ -22,6 +23,7 @@ def setup_event_routes(app, manager, service, helpers):
CONTENT_TYPE_EVENT = helpers["CONTENT_TYPE_EVENT"]
_build_drm_header = helpers["_build_drm_header"]
_attach_drm_header = helpers["_attach_drm_header"]
_build_stream_headers = helpers["_build_stream_headers"]
_resolve_stream_unified = helpers["_resolve_stream_unified"]
_get_drm_configs = helpers["_get_drm_configs"]
@@ -92,16 +94,24 @@ def setup_event_routes(app, manager, service, helpers):
country = request.query.get("country")
base_url = f"{request.urlparts.scheme}://{request.urlparts.netloc}"
qs = f"?country={country}" if country else ""
# Resolve first: the detected format decides which alias the URL
# points at. Both aliases hit the same handler — the extension only
# helps clients that pick their player from the URL. Unknown
# format (None) keeps index.mpd, i.e. the pre-HLS behaviour.
stream_info = _attach_drm_header(
CONTENT_TYPE_EVENT, provider, event_id, country=country
)
stream_format = stream_info.stream_format if stream_info else None
index_name = "index.m3u8" if stream_format is StreamFormat.HLS else "index.mpd"
stream_url = (
f"{base_url}/api/providers/{provider}/events/{event_id}"
f"/stream/index.mpd{qs}"
f"/stream/{index_name}{qs}"
)
sw_drm_qs_sep = "&" if qs else "?"
sw_drm_stream_url = (
f"{stream_url}{sw_drm_qs_sep}drm_variant=software&client_drm=true"
)
_build_drm_header(CONTENT_TYPE_EVENT, provider, event_id, country=country)
_build_stream_headers(CONTENT_TYPE_EVENT, provider, event_id, country=country)
return {
@@ -109,6 +119,7 @@ def setup_event_routes(app, manager, service, helpers):
"event_id": event_id,
"manifest_url": stream_url,
"sw_drm_manifest_url": sw_drm_stream_url,
"stream_format": stream_format.value if stream_format else None,
}
except ValueError as e:
@@ -121,9 +132,12 @@ def setup_event_routes(app, manager, service, helpers):
return {"error": f"Internal server error: {str(e)}"}
@app.route("/api/providers/<provider>/events/<event_id>/stream/index.mpd")
@app.route("/api/providers/<provider>/events/<event_id>/stream/index.m3u8")
def get_event_stream(provider, event_id):
"""
Single stream endpoint for event playback. Replaces the former
Single stream endpoint for event playback. index.m3u8 is an alias of
index.mpd served by the same handler; the format actually returned
follows the upstream manifest, not the extension. Replaces the former
/stream/sw-drm/, /stream/proxied/, and /stream/proxied/ffmpeg/
routes, which are removed. Every combination those routes used to
hardcode is now expressed via independent, freely-combinable query
+17 -5
View File
@@ -14,6 +14,7 @@ _resolve_stream_unified.
"""
from bottle import HTTPResponse, request, response
from streaming_providers.base.stream_format import StreamFormat
from streaming_providers.base.utils import logger
@@ -22,6 +23,7 @@ def setup_recording_routes(app, manager, service, helpers):
CONTENT_TYPE_RECORDING = helpers["CONTENT_TYPE_RECORDING"]
_build_drm_header = helpers["_build_drm_header"]
_attach_drm_header = helpers["_attach_drm_header"]
_build_stream_headers = helpers["_build_stream_headers"]
_resolve_stream_unified = helpers["_resolve_stream_unified"]
_get_drm_configs = helpers["_get_drm_configs"]
@@ -83,16 +85,22 @@ def setup_recording_routes(app, manager, service, helpers):
try:
country = request.query.get("country")
base_url = f"{request.urlparts.scheme}://{request.urlparts.netloc}"
# Resolve first: the detected format decides which alias the URL
# points at. Both aliases hit the same handler — the extension only
# helps clients that pick their player from the URL. Unknown
# format (None) keeps index.mpd, i.e. the pre-HLS behaviour.
stream_info = _attach_drm_header(
CONTENT_TYPE_RECORDING, provider, recording_id, country=country
)
stream_format = stream_info.stream_format if stream_info else None
index_name = "index.m3u8" if stream_format is StreamFormat.HLS else "index.mpd"
stream_url = (
f"{base_url}/api/providers/{provider}"
f"/recordings/{recording_id}/stream/index.mpd"
f"/recordings/{recording_id}/stream/{index_name}"
)
if country:
stream_url += f"?country={country}"
_build_drm_header(
CONTENT_TYPE_RECORDING, provider, recording_id, country=country
)
_build_stream_headers(
CONTENT_TYPE_RECORDING, provider, recording_id, country=country
)
@@ -101,6 +109,7 @@ def setup_recording_routes(app, manager, service, helpers):
"provider": provider,
"recording_id": recording_id,
"manifest_url": stream_url,
"stream_format": stream_format.value if stream_format else None,
}
except ValueError as e:
@@ -117,9 +126,12 @@ def setup_recording_routes(app, manager, service, helpers):
return {"error": f"Internal server error: {str(e)}"}
@app.route("/api/providers/<provider>/recordings/<recording_id>/stream/index.mpd")
@app.route("/api/providers/<provider>/recordings/<recording_id>/stream/index.m3u8")
def get_recording_stream(provider, recording_id):
"""
Single stream endpoint for recording playback. Replaces the former
Single stream endpoint for recording playback. index.m3u8 is an alias
of index.mpd served by the same handler; the format actually returned
follows the upstream manifest, not the extension. Replaces the former
/stream/proxied/ route, which is removed. client_drm=false now
reaches what that route used to do; see get_channel_stream in
channels.py for the full query-param list (client_drm, drm_variant,
+17 -3
View File
@@ -14,6 +14,7 @@ _resolve_stream_unified.
"""
from bottle import HTTPResponse, request, response
from streaming_providers.base.stream_format import StreamFormat
from streaming_providers.base.utils import logger
@@ -22,6 +23,7 @@ def setup_vod_routes(app, manager, service, helpers):
CONTENT_TYPE_VOD = helpers["CONTENT_TYPE_VOD"]
_build_drm_header = helpers["_build_drm_header"]
_attach_drm_header = helpers["_attach_drm_header"]
_build_stream_headers = helpers["_build_stream_headers"]
_resolve_stream_unified = helpers["_resolve_stream_unified"]
_get_drm_configs = helpers["_get_drm_configs"]
@@ -79,9 +81,12 @@ def setup_vod_routes(app, manager, service, helpers):
return {"error": f"Internal server error: {str(e)}"}
@app.route("/api/providers/<provider>/vod/<path:path>/stream/index.mpd")
@app.route("/api/providers/<provider>/vod/<path:path>/stream/index.m3u8")
def get_vod_stream(provider, path):
"""
Single stream endpoint for VOD playback. Replaces the former
Single stream endpoint for VOD playback. index.m3u8 is an alias of
index.mpd served by the same handler; the format actually returned
follows the upstream manifest, not the extension. Replaces the former
/stream/proxied/ and /stream/proxied/ffmpeg/ routes, which are
removed. Every combination those routes used to hardcode is now
expressed via independent, freely-combinable query params (see
@@ -99,19 +104,28 @@ def setup_vod_routes(app, manager, service, helpers):
try:
country = request.query.get("country")
base_url = f"{request.urlparts.scheme}://{request.urlparts.netloc}"
# Resolve first: the detected format decides which alias the URL
# points at. Both aliases hit the same handler — the extension only
# helps clients that pick their player from the URL. Unknown
# format (None) keeps index.mpd, i.e. the pre-HLS behaviour.
stream_info = _attach_drm_header(
CONTENT_TYPE_VOD, provider, vod_id, country=country
)
stream_format = stream_info.stream_format if stream_info else None
index_name = "index.m3u8" if stream_format is StreamFormat.HLS else "index.mpd"
stream_url = (
f"{base_url}/api/providers/{provider}/vod/{vod_id}/stream/index.mpd"
f"{base_url}/api/providers/{provider}/vod/{vod_id}/stream/{index_name}"
)
if country:
stream_url += f"?country={country}"
_build_drm_header(CONTENT_TYPE_VOD, provider, vod_id, country=country)
_build_stream_headers(CONTENT_TYPE_VOD, provider, vod_id, country=country)
return {
"provider": provider,
"vod_id": vod_id,
"manifest_url": stream_url,
"stream_format": stream_format.value if stream_format else None,
}
except ValueError as e:
+186 -9
View File
@@ -1,4 +1,5 @@
#!/usr/bin/env python3
import hashlib
import json
import math
import os
@@ -25,12 +26,23 @@ try:
# Add EPG Manager import
from streaming_providers.base.epg.epg_manager import EPGManager
from streaming_providers.base.models import StreamingChannel
from streaming_providers.base.stream_format import StreamFormat
from streaming_providers.base.stream_format import (
StreamFormat,
content_type_for,
detect_format,
)
from streaming_providers.base.settings.provider_enable_manager import (
ProviderEnableManager,
)
from streaming_providers.base.utils import MPDCacheManager, MPDRewriter, logger
from streaming_providers.base.utils.hls_rewriter import (
HLSResource,
hls_cache_ttl,
rewrite_hls_playlist,
)
from streaming_providers.base.utils.init_kid_resolver import get_init_kid_resolver
from streaming_providers.base.utils.proxy_url import make_media_proxy_url_builder
from streaming_providers.base.utils.single_flight import SingleFlight
from streaming_providers.base.utils.environment import (
get_environment_manager,
get_vfs_instance,
@@ -43,6 +55,29 @@ except ImportError as import_err:
raise
def _manifest_content_type(content: Optional[str]) -> str:
"""Content-Type for a manifest body, from the content itself (the source
of truth, same rule as detect_format). Unknown falls back to DASH, which is
what every manifest route served before HLS existed."""
# Cached manifests may come back as bytes, and a content type must never be
# the reason a manifest request fails: anything undecodable stays DASH.
if isinstance(content, (bytes, bytearray)):
content = bytes(content[:8192]).decode("utf-8", errors="ignore")
if not isinstance(content, str):
return content_type_for(None)
return content_type_for(detect_format(content))
def _require_dash(manifest_text: str, label: str) -> None:
"""The DASH rewrite paths must never be fed an HLS playlist: MPDRewriter
would fail with an opaque XML parse error. HLS goes through
UltimateService.get_proxied_hls_playlist."""
if detect_format(manifest_text) is StreamFormat.HLS:
raise ValueError(
f"{label}: upstream returned an HLS playlist on the DASH rewrite path"
)
class UltimateService:
def __init__(self, config_dir: str = None):
self.app = Bottle()
@@ -83,6 +118,11 @@ class UltimateService:
logger.info(f"MPD cache initialized: {self.mpd_cache.vfs.base_path}")
self._decrypted_cache = {} # key -> (content, expiry_ts) — memory-only, never persisted
# Rewritten HLS playlists: memory-only like the decrypted cache (they carry
# baked-in segment headers and session tokens, so never persisted).
self._hls_cache = {} # sha256 key -> (content, expiry_ts)
self._hls_cache_lock = threading.Lock()
self._hls_flight = SingleFlight()
# 1. Determine EPG URL FIRST (with proper precedence)
self.epg_url = self._determine_epg_url()
@@ -350,10 +390,15 @@ class UltimateService:
provider: str,
channel_id: str,
manifest_url: str,
header_kwargs: Optional[dict] = None,
) -> tuple:
"""
Fetch a manifest and collect everything MPDRewriter needs.
header_kwargs (country, start_time, end_time, epg_id, ...) are passed to the
provider's get_manifest_headers / get_segment_headers, as the route layer's
_build_stream_headers does. None keeps the historical behaviour (channel id only).
Returns:
(manifest_text, ttl, provider_proxy_url, segment_headers, effective_manifest_url)
effective_manifest_url is the post-redirect URL (may differ from manifest_url
@@ -370,12 +415,13 @@ class UltimateService:
# Resolve manifest headers from the provider so auth tokens etc. are sent
provider_instance = self.manager.get_provider(provider)
header_kwargs = header_kwargs or {}
manifest_headers = (
provider_instance.get_manifest_headers(channel_id)
provider_instance.get_manifest_headers(channel_id, **header_kwargs)
if provider_instance else {}
)
segment_headers = (
provider_instance.get_segment_headers(channel_id)
provider_instance.get_segment_headers(channel_id, **header_kwargs)
if provider_instance else {}
)
@@ -447,7 +493,7 @@ class UltimateService:
content, ttl = fetch_fn()
if cacheable:
cache_set(content, ttl)
response.content_type = "application/dash+xml; charset=utf-8"
response.content_type = _manifest_content_type(content)
return content
except ValueError as e:
# Config error (e.g. no HTTP manager) — not transient, don't mask with stale cache
@@ -461,7 +507,7 @@ class UltimateService:
stale = cache_get_stale(stale_window)
if stale:
logger.warning(f"Serving stale {label} due to fetch failure")
response.content_type = "application/dash+xml; charset=utf-8"
response.content_type = _manifest_content_type(stale)
return stale
logger.error(f"Failed to fetch {label}: {fetch_err}")
response.status = 502
@@ -474,7 +520,7 @@ class UltimateService:
if not highest_quality_only:
cached_mpd = self.mpd_cache.get(provider, channel_id)
if cached_mpd:
response.content_type = "application/dash+xml; charset=utf-8"
response.content_type = _manifest_content_type(cached_mpd)
return cached_mpd
if not self.media_proxy_url:
@@ -495,6 +541,7 @@ class UltimateService:
manifest_text, ttl, provider_proxy_url, segment_headers, effective_url = self.fetch_manifest_for_rewriter(
provider, channel_id, manifest_url
)
_require_dash(manifest_text, f"{provider}/{channel_id}")
rewriter = MPDRewriter(
self.media_proxy_url, provider_proxy_url, None, highest_quality_only,
provider=provider, channel=channel_id, segment_headers=segment_headers,
@@ -518,7 +565,7 @@ class UltimateService:
cached_mpd = self.mpd_cache.get(provider, cache_key)
if cached_mpd:
response.content_type = "application/dash+xml; charset=utf-8"
response.content_type = _manifest_content_type(cached_mpd)
return cached_mpd
logger.info(f"Cache miss for catchup {provider}/{channel_id}, fetching manifest")
@@ -541,6 +588,7 @@ class UltimateService:
manifest_text, ttl, provider_proxy_url, segment_headers, effective_url = self.fetch_manifest_for_rewriter(
provider, channel_id, manifest_url
)
_require_dash(manifest_text, f"{provider}/{channel_id} (catchup)")
rewriter = MPDRewriter(
self.media_proxy_url, provider_proxy_url, None, False,
provider=provider, channel=channel_id, segment_headers=segment_headers,
@@ -558,6 +606,135 @@ class UltimateService:
label=f"catchup manifest for {provider}/{channel_id}",
)
# ------------------------------------------------------------------
# HLS playlists through the media proxy (phase 2, Option A)
#
# The backend fetches, rewrites and serves EVERY playlist; the media proxy
# only carries segments / init segments / (later) keys. Cache is memory-only
# and keyed by a hash of everything the output depends on.
# ------------------------------------------------------------------
_HLS_CACHE_MAX_ENTRIES = 256
_HLS_STALE_KEEP = 30 # seconds an expired entry stays for the stale fallback
@staticmethod
def _hls_cache_key(provider: str, content_id: str, upstream_url: str,
header_kwargs: Optional[dict], cache_scope: str, proxy_keys: bool) -> str:
"""Hash, so session tokens inside upstream URLs never show up in logs."""
raw = json.dumps(
[provider, content_id, upstream_url, sorted((header_kwargs or {}).items()),
cache_scope, bool(proxy_keys)],
default=str,
)
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
def _get_hls_cached(self, key: str, max_stale: float = 0) -> Optional[str]:
entry = self._hls_cache.get(key)
if not entry:
return None
content, expiry = entry
overdue = time.time() - expiry
if overdue < 0:
return content
if max_stale and overdue <= max_stale:
logger.warning(f"Serving STALE HLS playlist ({overdue:.1f}s past expiry)")
return content
# Expired entries are NOT deleted here (unlike _get_decrypted_cached): the
# stale fallback runs after this lookup and still needs them. _set prunes.
return None
def _set_hls_cached(self, key: str, content: str, ttl: float) -> None:
now = time.time()
with self._hls_cache_lock:
self._hls_cache[key] = (content, now + ttl)
if len(self._hls_cache) <= self._HLS_CACHE_MAX_ENTRIES:
return
for k, (_, expiry) in list(self._hls_cache.items()):
if expiry < now - self._HLS_STALE_KEEP:
del self._hls_cache[k]
overflow = len(self._hls_cache) - self._HLS_CACHE_MAX_ENTRIES
if overflow > 0:
for k, _ in sorted(self._hls_cache.items(), key=lambda kv: kv[1][1])[:overflow]:
del self._hls_cache[k]
def get_proxied_hls_playlist(
self,
provider: str,
content_id: str,
upstream_url: str,
*,
build_playlist_url,
cache_scope: str = "",
header_kwargs: Optional[dict] = None,
proxy_keys: bool = False,
) -> str:
"""
Fetch one upstream HLS playlist (master or media) through the provider's
HTTP manager and return it rewritten for the media proxy.
build_playlist_url(upstream_url) -> URL of the backend playlist endpoint for a
nested playlist (variant, audio/subtitle rendition, I-frame). Supplied by
the endpoint, which owns token signing and the request's base URL.
cache_scope: anything else the OUTPUT depends on that is not in the other
arguments — in practice the request's base URL, because the playlist URLs
built above embed it.
header_kwargs: provider header context (country, start_time, end_time, epg_id);
forwarded to get_manifest_headers/get_segment_headers and part of the cache key.
proxy_keys: also route AES-128 key URIs through the media proxy.
Segment/init/key URLs become media-proxy URLs carrying the provider's segment
headers and provider proxy, exactly like the DASH path. Concurrent identical
requests share one fetch + rewrite (single-flight); the result is cached for
a fraction of the target duration (see hls_cache_ttl). Errors follow
_fetch_and_cache_manifest: config/rewrite errors -> 502 JSON, transient fetch
errors -> short stale fallback, else 502.
"""
if not self.media_proxy_url:
response.status = 503
response.content_type = "application/json"
return json.dumps({"error": "Media proxy not configured (MEDIA_PROXY_URL not set)"})
key = self._hls_cache_key(provider, content_id, upstream_url, header_kwargs, cache_scope, proxy_keys)
cached = self._get_hls_cached(key)
if cached:
response.content_type = _manifest_content_type(cached)
return cached
def produce():
# A previous flight may have finished between our cache check and becoming leader.
hit = self._get_hls_cached(key)
if hit:
return hit, 0
text, _http_ttl, provider_proxy_url, segment_headers, effective_url = self.fetch_manifest_for_rewriter(
provider, content_id, upstream_url, header_kwargs=header_kwargs
)
provider_instance = self.manager.get_provider(provider)
media_url = make_media_proxy_url_builder(
self.media_proxy_url,
provider_proxy_url=provider_proxy_url,
segment_headers=segment_headers,
)
def build_url(url: str, kind: HLSResource) -> str:
return build_playlist_url(url) if kind is HLSResource.PLAYLIST else media_url(url)
rewritten = rewrite_hls_playlist(
text, effective_url, build_url,
inherit_query=bool(getattr(provider_instance, "inherit_manifest_query_on_segments", True)),
proxy_keys=proxy_keys,
)
ttl = hls_cache_ttl(text)
self._set_hls_cached(key, rewritten, ttl)
return rewritten, ttl
return self._fetch_and_cache_manifest(
lambda: self._hls_flight.do(key, produce),
cache_set=lambda content, ttl: None, # produce() already stored it, once per flight
cache_get_stale=lambda max_stale: self._get_hls_cached(key, max_stale),
cacheable=True,
stale_window=6,
label=f"HLS playlist for {provider}/{content_id}",
)
def get_decrypted_manifest(self, provider: str, channel_id: str, keyids: dict,
highest_quality_only: bool = False, receiver_side: bool = False,
drm_variant: str = "auto") -> str:
@@ -567,7 +744,7 @@ class UltimateService:
if not highest_quality_only:
cached_mpd = self._get_decrypted_cached(cache_key)
if cached_mpd:
response.content_type = "application/dash+xml; charset=utf-8"
response.content_type = _manifest_content_type(cached_mpd)
return cached_mpd
if not self.media_proxy_url:
@@ -621,7 +798,7 @@ class UltimateService:
if not highest_quality_only:
cached_mpd = self._get_decrypted_cached(cache_key)
if cached_mpd:
response.content_type = "application/dash+xml; charset=utf-8"
response.content_type = _manifest_content_type(cached_mpd)
return cached_mpd
if not self.media_proxy_url: