From c947bf17f849ef29c0bbfd7ad7f7962b5fc9dc63 Mon Sep 17 00:00:00 2001 From: Nirvana Date: Sat, 24 Oct 2026 08:16:36 +0200 Subject: [PATCH] Redirect hls to proxy --- .../base/utils/hls_rewriter.py | 195 ++++++++++++++++ .../base/utils/proxy_url.py | 65 ++++++ .../base/utils/single_flight.py | 54 +++++ .../providers/_template_v2/README.md | 42 ++-- routes/streams/__init__.py | 211 ++++++++++++++++-- routes/streams/events.py | 20 +- routes/streams/recordings.py | 22 +- routes/streams/vod.py | 20 +- service.py | 195 +++++++++++++++- 9 files changed, 771 insertions(+), 53 deletions(-) create mode 100644 lib/streaming_providers/base/utils/hls_rewriter.py create mode 100644 lib/streaming_providers/base/utils/proxy_url.py create mode 100644 lib/streaming_providers/base/utils/single_flight.py diff --git a/lib/streaming_providers/base/utils/hls_rewriter.py b/lib/streaming_providers/base/utils/hls_rewriter.py new file mode 100644 index 0000000..6eaed95 --- /dev/null +++ b/lib/streaming_providers/base/utils/hls_rewriter.py @@ -0,0 +1,195 @@ +# streaming_providers/base/utils/hls_rewriter.py +""" +HLS playlist rewriter (phase 2, Option A). + +Pure text transformation, no I/O and no knowledge of the proxy URL scheme or +token signing: every reference in the playlist is resolved to an absolute +upstream URL and handed to a caller-supplied `build_url(url, kind)`, which +returns the URL to put into the playlist. Under Option A the caller maps + PLAYLIST -> the backend's playlist endpoint + SEGMENT / INIT / KEY -> the media proxy (/api/proxy/) +Kind is decided from the TAG a reference sits in, never from the file +extension (tokenised CDN URLs usually have none). + +Guarantees +- Line order, comments, #EXT-X-MEDIA-SEQUENCE, discontinuity tags, BYTERANGE + and every attribute other than URI="..." are passed through untouched. +- data: URIs and non-http(s) schemes (skd://, ...) are never rewritten. +- Relative references resolve against `base_url` (the post-redirect playlist + URL), inheriting its query like the DASH path when `inherit_query` is true. + +Not supported (raises UnsupportedHLSFeature): low-latency HLS tags. Whether to +support or reject LL-HLS is still an open decision (handoff question 5); +rejecting is the safe default because blocking reloads and part requests +cannot be served through a rewritten playlist. + +AES-128 key URIs are only routed through `build_url` when `proxy_keys=True` +(and METHOD=AES-128 with an identity KEYFORMAT). Default is False, matching +the current "AES-128 = unknown, pass through" policy (absolutised only). +""" + +import re +from enum import Enum +from typing import Callable, Optional +from urllib.parse import urlsplit + +from ..stream_format import join_hls_uri + + +class HLSResource(str, Enum): + PLAYLIST = "playlist" # variant / audio / subtitle / I-frame playlist + SEGMENT = "segment" # media segment + INIT = "init" # EXT-X-MAP initialisation segment + KEY = "key" # AES-128 key file + + +class HLSRewriteError(ValueError): + """The input is not a rewritable HLS playlist.""" + + +class UnsupportedHLSFeature(HLSRewriteError): + """The playlist uses a feature the proxied path does not support.""" + + +UrlBuilder = Callable[[str, HLSResource], str] + +# Same lookbehind as stream_format._HLS_URI_ATTR_RE: vendor attributes such as +# X-ASSET-URI stay untouched. +_URI_ATTR_RE = re.compile(r'(? bool: + """Relative references and http(s) URLs only. data:, skd:, etc. are left.""" + return urlsplit(uri).scheme.lower() in ("", "http", "https") + + +def _kind_for_tag(name: str, line: str, proxy_keys: bool) -> Optional[HLSResource]: + """Resource kind of the URI="..." attribute of a tag, None = absolutise only.""" + if name in _PLAYLIST_TAGS: + return HLSResource.PLAYLIST + if name == "EXT-X-MAP": + return HLSResource.INIT + if name in _KEY_TAGS and proxy_keys: + method = _METHOD_RE.search(line) + keyformat = _KEYFORMAT_RE.search(line) + if ( + method and method.group(1) == "AES-128" + and (keyformat is None or keyformat.group(1).lower() == "identity") + ): + return HLSResource.KEY + return None + + +_TARGET_DURATION_RE = re.compile(r"^#EXT-X-TARGETDURATION:(\d+)", re.MULTILINE) + + +def hls_cache_ttl( + content: str, + *, + master_ttl: float = 30.0, + vod_ttl: float = 120.0, + live_min: float = 1.0, + live_max: float = 5.0, + live_default: float = 2.0, +) -> float: + """How long a REWRITTEN playlist may be reused, in seconds. + + master (multivariant) -> master_ttl: stable within a session + media + #EXT-X-ENDLIST -> vod_ttl: complete, does not change + live media playlist -> half the target duration, clamped to + [live_min, live_max]: viewers share one + rewrite per refresh interval, but nobody is + served a playlist older than half a segment + The upstream Cache-Control header is deliberately not consulted: CDNs + often send none or a value unrelated to the refresh interval. + """ + if "#EXT-X-STREAM-INF" in content: + return master_ttl + if "#EXT-X-ENDLIST" in content: + return vod_ttl + match = _TARGET_DURATION_RE.search(content) + if not match: + return live_default + return max(live_min, min(live_max, int(match.group(1)) / 2.0)) + + +def rewrite_hls_playlist( + content: str, + base_url: str, + build_url: UrlBuilder, + *, + inherit_query: bool = True, + proxy_keys: bool = False, +) -> str: + """Rewrite every reference in an HLS playlist. + + Args: + content: playlist text (master or media). + base_url: URL the playlist was actually fetched from (post-redirect). + build_url: maps (absolute upstream URL, kind) -> URL for the output. + inherit_query: relative references inherit base_url's query (see + stream_format.join_hls_uri); providers opt out through + `inherit_manifest_query_on_segments = False`. + proxy_keys: route AES-128 key URIs through build_url (default: only + absolutise them). + + Raises: + HLSRewriteError: content is not an HLS playlist. + UnsupportedHLSFeature: low-latency HLS tags present. + """ + content = content.lstrip("\ufeff") # a BOM must not turn #EXTM3U into a "segment" + if not content.lstrip(" \t\r\n").startswith("#EXTM3U"): + raise HLSRewriteError("not an HLS playlist (missing #EXTM3U)") + + def resolve(uri: str, kind: Optional[HLSResource]) -> str: + if not uri or not _is_rewritable_reference(uri): + return uri + # An absolute URL is taken exactly as the upstream wrote it (no query + # inheritance, as in MPDRewriter._urljoin_preserve_query); only relative + # references resolve against base_url. + absolute = uri if urlsplit(uri).scheme else join_hls_uri(base_url, uri, inherit_query) + return absolute if kind is None else build_url(absolute, kind) + + out = [] + expect_variant_uri = False # the line after #EXT-X-STREAM-INF is a playlist + + for raw in content.splitlines(): + line = raw.strip() + if not line: + out.append(raw) + continue + + if line.startswith("#"): + name = line[1:].split(":", 1)[0] + if name in _LOW_LATENCY_TAGS: + raise UnsupportedHLSFeature( + f"low-latency HLS tag {name} is not supported on the proxied path" + ) + if name == "EXT-X-STREAM-INF": + expect_variant_uri = True + out.append(line) + continue + kind = _kind_for_tag(name, line, proxy_keys) + out.append(_URI_ATTR_RE.sub( + lambda m, k=kind: f'URI="{resolve(m.group(1), k)}"', line + )) + continue + + kind = HLSResource.PLAYLIST if expect_variant_uri else HLSResource.SEGMENT + expect_variant_uri = False + out.append(resolve(line, kind)) + + result = "\n".join(out) + return result + "\n" if content.endswith("\n") else result diff --git a/lib/streaming_providers/base/utils/proxy_url.py b/lib/streaming_providers/base/utils/proxy_url.py new file mode 100644 index 0000000..a716a93 --- /dev/null +++ b/lib/streaming_providers/base/utils/proxy_url.py @@ -0,0 +1,65 @@ +# streaming_providers/base/utils/proxy_url.py +""" +Media-proxy URL construction shared by the HLS path. + +Produces exactly the URL form MPDRewriter.build_proxy_url emits for the plain +(non-decrypt) case: + + {media_proxy_url}/api/proxy/ + + token = base64url(urlencode({"url": ..., "proxy": ..., "headers": ...})) + +`proxy` is the provider HTTP proxy (omitted when None), `headers` is +base64url(JSON) of the provider's segment headers (omitted when empty). The +parameter ORDER (url, proxy, headers) matters for byte-identical output and is +pinned by test_proxy_url.py against MPDRewriter itself. + +Stdlib only. MPDRewriter keeps its own copy for now; the test guards against +drift until it is switched over to these helpers. +""" + +import base64 +import json +from typing import Callable, Mapping, Optional +from urllib.parse import urlencode + + +def encode_url(value: str) -> str: + """base64url without padding (same as MPDRewriter.encode_url).""" + return base64.urlsafe_b64encode(value.encode("utf-8")).decode("utf-8").rstrip("=") + + +def decode_url(token: str) -> str: + """Inverse of encode_url. Raises ValueError (binascii.Error and + UnicodeDecodeError are subclasses) for malformed input.""" + # validate=True: urlsafe_b64decode would silently drop invalid characters. + raw = base64.b64decode(token + "=" * (-len(token) % 4), altchars=b"-_", validate=True) + return raw.decode("utf-8") + + +def encode_segment_headers(headers: Mapping[str, str]) -> str: + """base64url(JSON) of a header mapping, as carried in the `headers` param.""" + return base64.urlsafe_b64encode(json.dumps(headers).encode()).decode().rstrip("=") + + +def make_media_proxy_url_builder( + media_proxy_url: str, + *, + provider_proxy_url: Optional[str] = None, + segment_headers: Optional[Mapping[str, str]] = None, + endpoint: str = "proxy", +) -> Callable[[str], str]: + """Return build(url) -> media-proxy URL. The static params (provider proxy, + encoded headers) are computed once, so rewriting a playlist with thousands + of segments does not re-encode the headers per segment.""" + base = f"{media_proxy_url.rstrip('/')}/api/{endpoint}" + static = {} + if provider_proxy_url: + static["proxy"] = provider_proxy_url + if segment_headers: + static["headers"] = encode_segment_headers(segment_headers) + + def build(url: str) -> str: + return f"{base}/{encode_url(urlencode({'url': url, **static}))}" + + return build diff --git a/lib/streaming_providers/base/utils/single_flight.py b/lib/streaming_providers/base/utils/single_flight.py new file mode 100644 index 0000000..2e20f81 --- /dev/null +++ b/lib/streaming_providers/base/utils/single_flight.py @@ -0,0 +1,54 @@ +# streaming_providers/base/utils/single_flight.py +""" +Single-flight: concurrent callers asking for the same key share ONE execution. + +Used for HLS playlist refreshes: N viewers polling the same live playlist at +the same moment cause one upstream fetch + rewrite, not N. The first caller +for a key (the leader) runs `fn`; callers arriving while it runs wait for its +result (or exception). Nothing is cached here — the entry disappears as soon +as the leader finishes; caching is the caller's job. +""" + +import threading +from typing import Callable, Dict, Hashable, TypeVar + +T = TypeVar("T") + + +class _Call: + __slots__ = ("event", "result", "error") + + def __init__(self): + self.event = threading.Event() + self.result = None + self.error = None + + +class SingleFlight: + def __init__(self, wait_timeout: float = 60.0): + self._lock = threading.Lock() + self._calls: Dict[Hashable, _Call] = {} + self._wait_timeout = wait_timeout + + def do(self, key: Hashable, fn: Callable[[], T]) -> T: + with self._lock: + call = self._calls.get(key) + leader = call is None + if leader: + call = self._calls[key] = _Call() + + if leader: + try: + call.result = fn() + except BaseException as exc: # shared with followers, re-raised below + call.error = exc + finally: + with self._lock: + self._calls.pop(key, None) + call.event.set() + elif not call.event.wait(self._wait_timeout): + raise TimeoutError(f"single-flight wait for {key!r} timed out") + + if call.error is not None: + raise call.error + return call.result diff --git a/lib/streaming_providers/providers/_template_v2/README.md b/lib/streaming_providers/providers/_template_v2/README.md index 4a94078..f08ed3d 100644 --- a/lib/streaming_providers/providers/_template_v2/README.md +++ b/lib/streaming_providers/providers/_template_v2/README.md @@ -1157,19 +1157,22 @@ thousands of manifests, so it uses the hint: HLS-hinted channels get the HLS), never per provider. Leave `None` for dynamic-manifest items — **unknown is not DASH**; without a hint nothing changes for existing DASH providers. -**HLS limits** — state them in the provider docstring (phase 1, no HLS rewriter or -media-proxy path yet): +**HLS limits** — state them in the provider docstring. HLS can be served through the +backend and the media proxy (phase 2); what is still missing is in this table and in §17. | Situation | Behaviour today | |---|---| | Clear HLS, CDN needs no proxy and no special headers | works: the backend answers with a redirect to the upstream playlist | | `requires_manifest_context = True` | the backend fetches the playlist and makes every relative URI absolute (HLS has no ``); the manifest query (token) is inherited unless `inherit_manifest_query_on_segments = False` | -| Provider needs an HTTP proxy (`needs_proxy`) | **not supported yet**: redirected anyway, a warning is logged; geo-proxied HLS will not play | -| CDN needs headers for manifest/segments | **unverified**: a redirect carries none; headers reach the client only via `x-kodi-stream-headers` / KODIPROPs. Prove it on a device before you rely on it (§12.7) | -| `METHOD=AES-128` | reported as unknown (never "clear"); passed through unchanged, the player fetches the key itself | +| Provider needs an HTTP proxy (`needs_proxy`) | **works when `MEDIA_PROXY_URL` is set** and the request has no `no_proxy=true`: the backend fetches every playlist through the provider's HTTP manager and rewrites it. Segments and `EXT-X-MAP` init segments become `{MEDIA_PROXY_URL}/api/proxy/` (provider segment headers and proxy inside the token, as for DASH); nested playlists (variants, audio/subtitle renditions) point back at the same stream URL as `…/index.m3u8?&playlist=`. Without `MEDIA_PROXY_URL`: redirect plus a warning, geo-proxied HLS will not play | +| CDN needs headers for manifest/segments | **unverified, and not proxied by default.** Unlike DASH, "needs headers" alone does not select the proxy path, so a redirect that already plays keeps playing. A provider whose CDN is header-gated sets `hls_requires_proxy = True` and must return the headers from `get_manifest_headers` / `get_segment_headers`. Header-only gating through `x-kodi-stream-headers` / KODIPROPs on the redirect is still unproven: prove it on a device before you rely on either (§12.7) | +| `METHOD=AES-128` | reported as unknown (never "clear"); passed through unchanged, the player fetches the key itself. Key URIs are not routed through the media proxy yet (the rewriter can do it; the policy is undecided) | | `SAMPLE-AES`, Widevine/FairPlay HLS | detected as encrypted and passed through; no PSSH extraction, no server-side handling | | Only the first variant is probed | separately encrypted audio renditions are not checked | -| `index.m3u8` alias | channels only; events, VOD and recordings still expose `index.mpd` | +| Low-latency HLS (`EXT-X-PART`, preload hints) | rejected on the proxied path (502 with the reason); not affected on the redirect path | +| `index.m3u8` alias | channels, events, VOD and recordings; `…/manifest` returns `stream_format` and points at the alias for HLS | +| `playlist` query parameter | reserved on stream URLs: it carries a nested playlist of the proxied path and skips the format and catchup-window checks of the first request | +| Proxied playlists: cache and trust | rewritten playlists are cached in memory only (about half the target duration for live, 30 s for a master, 120 s for VOD) and concurrent identical requests share one fetch. The `playlist` value and the media-proxy tokens are **unsigned**: backend and media proxy must only be reachable by trusted clients (LAN) | **Locally minted, short-lived manifest URLs (Eon).** When the provider builds the manifest URL itself (an AES-encrypted payload with a server `ctime` and a session id): @@ -1757,9 +1760,10 @@ Things that already cost time. Each is covered by a test or the contract unless backend sniffs the content; never branch on the extension (§11). 67. Setting `streaming_format` per provider, or `"dash"` as a "default" — unknown is `None`; a wrong hint sends playlists to the wrong alias (§13). -68. HLS from a provider that needs an HTTP proxy or CDN headers: the redirect loses - both; it logs a warning and does not play. Document it, do not "fix" it in the - provider (§11). +68. HLS from a provider whose CDN needs an HTTP proxy or headers: with `needs_proxy` and + `MEDIA_PROXY_URL` the backend proxies it; without `MEDIA_PROXY_URL`, or for a + header-gated CDN without `hls_requires_proxy`, the redirect loses both and the stream + does not play. Document it, do not "fix" it in the provider (§11). 69. Treating `METHOD=AES-128` HLS as clear (it is *unknown* to the backend) or returning DRM configs for it (§11). 70. Brand/operator modelled as country, or resolved by list position (§4). @@ -1786,6 +1790,14 @@ Things that already cost time. Each is covered by a test or the contract unless `403` — as "wrong credentials" (§9). 80. Persisting a re-derivable secret (stream key, anything the next token response returns) through `SessionManager`: unknown keys are logged in clear at INFO (§9). +81. Exposing the backend or the media proxy beyond a trusted LAN: the `playlist` value and + the proxy tokens are unsigned base64, so a crafted one makes the proxy fetch any URL + and makes the backend send provider headers to any host. Signing comes first (§11, + §17). +82. Assuming the backend and the player see the same playlist for a redirecting or + tokenised CDN (observed with ZDF/KiKA): the "clear" verdict and the proxied rewrite + come from the backend's own fetch, the redirect path hands the player the CDN's answer + (§11). --- @@ -1810,10 +1822,14 @@ Known gaps in the base, so you don't hunt for them: * `README` v1 (`_template`) describes the old shape; it is not rewritten. * Provider-specific features outside the ABCs (restart, `get_restart`) are free-form; there is no `RestartInfo` type yet. -* **HLS**: no HLS rewriter / media-proxy path (proxied or header-gated HLS and - geo-proxied providers do not play), AES-128 policy undecided, encrypted HLS only - passed through, only the first variant probed, `index.m3u8` alias for channels - only (§11). +* **HLS**: the proxied path exists for `needs_proxy` providers and `hls_requires_proxy` + opt-ins (§11). Open: AES-128 key proxying (policy undecided, `_analyse_hls` + unchanged), low-latency HLS is rejected, encrypted HLS only passed through, only the + first variant probed, header-only gating unverified on a device. **No signing:** + `playlist` values and media-proxy tokens are plain base64, so backend and media proxy + stay LAN-only until signing is added. The media proxy still has to be adapted to the + HLS expectations (segments and keys, Range, redirects, content types). The VOD and + recordings M3U exports were not reviewed for the `index.m3u8` alias. * **`HTTPManager` has no `patch()`** (`_make_request("PATCH", …)` exists, only the public wrapper is missing). Eon needs `PATCH v2/devices`; until it is added the provider `hasattr`-checks and falls back to the `v1` POST (which mints a *new* device number). diff --git a/routes/streams/__init__.py b/routes/streams/__init__.py index 0bc109b..0c94505 100644 --- a/routes/streams/__init__.py +++ b/routes/streams/__init__.py @@ -30,7 +30,7 @@ import base64 import json import re import time -from urllib.parse import urljoin +from urllib.parse import urlencode, urljoin, urlsplit from bottle import redirect, request, response from streaming_providers.base.stream_format import ( @@ -40,6 +40,7 @@ from streaming_providers.base.stream_format import ( detect_format, ) from streaming_providers.base.utils import logger +from streaming_providers.base.utils.proxy_url import decode_url, encode_url from streaming_providers.base.utils.manifest_utils import ManifestUtils # --------------------------------------------------------------------------- @@ -584,6 +585,126 @@ def make_helpers(manager, service): logger.debug(f"Redirecting to manifest: {manifest_url}") return redirect(manifest_url) + def _header_context( + content_type: str, + country, + is_catchup: bool, + start_time, + end_time, + epg_id, + ) -> dict: + """Provider header context, the same keyword set _stream_needs_headers + passes to get_manifest_headers / get_segment_headers.""" + kwargs = {} + if country: + kwargs["country"] = country + if is_catchup and content_type == CONTENT_TYPE_CHANNEL: + kwargs.update(start_time=start_time, end_time=end_time, epg_id=epg_id) + return kwargs + + def _proxied_hls_playlist( + content_type: str, + provider: str, + content_id: str, + upstream_url: str, + country, + is_catchup: bool, + start_time, + end_time, + epg_id, + ): + """Serve one upstream playlist rewritten for the media proxy (Option A). + + Nested playlists (variants, audio/subtitle renditions, I-frame playlists) + point back at THIS endpoint: the current URL with the query unchanged plus + `playlist=`. Everything that selects the stream + (country, catchup times, drm_variant, ...) therefore rides along in the + query and no extra state or route is needed. Segment, init and key URLs + become media-proxy URLs inside service.get_proxied_hls_playlist. + + Assumes the backend is reachable only by trusted clients (LAN): the + `playlist` value is not signed, exactly like the DASH proxy tokens. If this + endpoint is ever exposed externally it needs signing, otherwise a crafted + value makes the backend send provider headers to an arbitrary host. + """ + parts = request.urlparts + # Whatever extension the client used, nested playlists are .m3u8. + path = re.sub(r"index\.mpd$", "index.m3u8", parts.path) + base = f"{parts.scheme}://{parts.netloc}{path}" + carried = [(k, v) for k, v in request.query.allitems() if k != "playlist"] + + def build_playlist_url(url: str) -> str: + return f"{base}?{urlencode(carried + [('playlist', encode_url(url))])}" + + return service.get_proxied_hls_playlist( + provider, content_id, upstream_url, + build_playlist_url=build_playlist_url, + # The output embeds `base` and the carried query, so they scope the cache. + cache_scope=f"{base}?{urlencode(carried)}", + header_kwargs=_header_context( + content_type, country, is_catchup, start_time, end_time, epg_id + ), + ) + + def _serve_hls_variant( + content_type: str, + provider: str, + content_id: str, + playlist_ref: str, + country, + is_catchup: bool, + start_time, + end_time, + epg_id, + ): + """A `?playlist=` request: a nested playlist the player fetched from a + rewritten master. Skips format/DRM resolution and catchup-window + validation on purpose: the initial request already did both, and a live + playlist refresh must not fail because the window edge moved.""" + try: + upstream_url = decode_url(playlist_ref) + except ValueError: + response.status = 400 + return {"error": "Invalid playlist reference"} + if urlsplit(upstream_url).scheme not in ("http", "https"): + response.status = 400 + return {"error": "Invalid playlist reference"} + return _proxied_hls_playlist( + content_type, provider, content_id, upstream_url, + country, is_catchup, start_time, end_time, epg_id, + ) + + def _hls_needs_media_proxy( + provider: str, + content_type: str, + content_id: str, + country, + is_catchup: bool, + start_time, + end_time, + epg_id, + ) -> bool: + """Should this HLS stream go through the backend + media proxy? + + - provider with an HTTP/geo proxy (needs_proxy): yes — a redirect would + leave from a different IP. + - header-gated CDN: only for providers that opt in with + `hls_requires_proxy = True` AND actually return headers. Unlike DASH, a + plain "needs headers" is deliberately NOT enough: HLS playlists that work + through the redirect (ZDF/KiKA sends a Referer but plays fine) must keep + doing so until header KODIPROPs are proven insufficient on a device. + """ + if manager.needs_proxy(provider): + return True + provider_instance = manager.get_provider(provider) + if provider_instance and getattr(provider_instance, "hls_requires_proxy", False): + return _stream_needs_headers( + content_type, provider, content_id, country, + is_catchup=is_catchup, start_time=start_time, + end_time=end_time, epg_id=epg_id, + ) + return False + def _serve_hls_passthrough( content_type: str, provider: str, @@ -595,35 +716,74 @@ def make_helpers(manager, service): end_time, epg_id, is_unencrypted: bool, + no_proxy: bool = False, ): """ - HLS phase 1: hand the client the upstream playlist WITHOUT rewriting. + HLS entry point (the DASH routing in _resolve_stream_unified — ClearKey + rewrite, proxied manifest — never runs for HLS: MPDRewriter cannot parse a + playlist). - There is no HLS rewriter or proxy path yet — MPDRewriter and the media - proxy are DASH-only. The DASH routing in _resolve_stream_unified - (ClearKey rewrite, proxied manifest) must therefore never run for HLS: - MPDRewriter cannot parse a playlist and the request would end in a 502. - Instead the playlist is redirected to (or, for providers with - requires_manifest_context, fetched and absolutised by - _redirect_or_fetch). + Two outcomes: + * proxied (Option A): when _hls_needs_media_proxy says so, a media proxy + is configured and no_proxy is not set, the playlist is fetched through + the provider's HTTP manager and rewritten (service.get_proxied_hls_playlist). + Segments go through the media proxy, nested playlists come back to this + endpoint via `?playlist=`. + * passthrough (phase 1): otherwise the upstream playlist is redirected to + (or, for requires_manifest_context providers, fetched and absolutised by + _redirect_or_fetch). - Limits until an HLS proxy path exists: providers whose manifest or - segment fetches need an HTTP proxy cannot be served this way, and - provider headers only reach the client via x-kodi-stream-headers / - KODIPROPs. Encrypted or undecidable HLS (e.g. AES-128, SAMPLE-AES) is - passed through as-is; the client's player and the x-kodi-drm-configs - header have to cope with it. + Encrypted or undecidable HLS (AES-128, SAMPLE-AES, Widevine, FairPlay) is + passed on as it is; the client's player and the x-kodi-drm-configs header have + to cope with it. """ if not is_unencrypted: logger.debug( f"HLS {provider}/{content_id}: not verified clear — passing the " f"playlist through unchanged" ) - if manager.needs_proxy(provider): + + wants_proxy = not no_proxy and _hls_needs_media_proxy( + provider, content_type, content_id, country, + is_catchup, start_time, end_time, epg_id, + ) + if wants_proxy and service.media_proxy_url: + if is_catchup: + manifest_url = manager.get_catchup_manifest( + provider_name=provider, + channel_id=content_id, + start_time=start_time, + end_time=end_time, + epg_id=epg_id, + country=country, + drm_variant=drm_variant, + ) + if not manifest_url: + response.status = 404 + return { + "error": f'Catchup manifest not available for channel "{content_id}"' + } + else: + manifest_url = _get_manifest_url( + content_type, provider, content_id, + country=country, drm_variant=drm_variant, + ) + if not manifest_url: + response.status = 404 + return { + "error": ( + f'Manifest not available for {content_type} ' + f'"{content_id}" from provider "{provider}"' + ) + } + return _proxied_hls_playlist( + content_type, provider, content_id, manifest_url, + country, is_catchup, start_time, end_time, epg_id, + ) + if wants_proxy: logger.warning( - f"HLS {provider}/{content_id}: provider uses an HTTP proxy but HLS " - f"proxying is not implemented — redirecting to the upstream " - f"playlist; playback may fail" + f"HLS {provider}/{content_id}: needs the media proxy but MEDIA_PROXY_URL " + f"is not set — redirecting to the upstream playlist; playback may fail" ) if is_catchup: @@ -716,6 +876,15 @@ def make_helpers(manager, service): catchup content that turns out unencrypted. Meaningless when no_proxy=True (plain redirect, no rewriter involved). """ + # --- Nested HLS playlist (?playlist=...): handled before everything else, + # see _serve_hls_variant. --- + playlist_ref = request.query.get("playlist") + if playlist_ref: + return _serve_hls_variant( + content_type, provider, content_id, playlist_ref, + country, is_catchup, start_time, end_time, epg_id, + ) + # --- Catchup window validation (channels only), before anything else — # previously duplicated between channels.py (for auto/noproxy) and # _resolve_decrypted_stream (for decrypt); now the single source of truth @@ -761,12 +930,14 @@ def make_helpers(manager, service): is_unencrypted = "none" in drm_dict # ================================================================== - # HLS: no rewriter exists, so none of the DASH routing below applies. + # HLS: own routing (proxied playlist rewrite or passthrough); none of the + # DASH routing below applies. # ================================================================== if stream_format is StreamFormat.HLS: return _serve_hls_passthrough( content_type, provider, content_id, country, drm_variant, is_catchup, start_time, end_time, epg_id, is_unencrypted, + no_proxy=no_proxy, ) # ================================================================== diff --git a/routes/streams/events.py b/routes/streams/events.py index 1886ac6..0e3ef58 100644 --- a/routes/streams/events.py +++ b/routes/streams/events.py @@ -14,6 +14,7 @@ directly through _resolve_stream_unified. """ from bottle import HTTPResponse, request, response +from streaming_providers.base.stream_format import StreamFormat from streaming_providers.base.utils import logger @@ -22,6 +23,7 @@ def setup_event_routes(app, manager, service, helpers): CONTENT_TYPE_EVENT = helpers["CONTENT_TYPE_EVENT"] _build_drm_header = helpers["_build_drm_header"] + _attach_drm_header = helpers["_attach_drm_header"] _build_stream_headers = helpers["_build_stream_headers"] _resolve_stream_unified = helpers["_resolve_stream_unified"] _get_drm_configs = helpers["_get_drm_configs"] @@ -92,16 +94,24 @@ def setup_event_routes(app, manager, service, helpers): country = request.query.get("country") base_url = f"{request.urlparts.scheme}://{request.urlparts.netloc}" qs = f"?country={country}" if country else "" + # Resolve first: the detected format decides which alias the URL + # points at. Both aliases hit the same handler — the extension only + # helps clients that pick their player from the URL. Unknown + # format (None) keeps index.mpd, i.e. the pre-HLS behaviour. + stream_info = _attach_drm_header( + CONTENT_TYPE_EVENT, provider, event_id, country=country + ) + stream_format = stream_info.stream_format if stream_info else None + index_name = "index.m3u8" if stream_format is StreamFormat.HLS else "index.mpd" stream_url = ( f"{base_url}/api/providers/{provider}/events/{event_id}" - f"/stream/index.mpd{qs}" + f"/stream/{index_name}{qs}" ) sw_drm_qs_sep = "&" if qs else "?" sw_drm_stream_url = ( f"{stream_url}{sw_drm_qs_sep}drm_variant=software&client_drm=true" ) - _build_drm_header(CONTENT_TYPE_EVENT, provider, event_id, country=country) _build_stream_headers(CONTENT_TYPE_EVENT, provider, event_id, country=country) return { @@ -109,6 +119,7 @@ def setup_event_routes(app, manager, service, helpers): "event_id": event_id, "manifest_url": stream_url, "sw_drm_manifest_url": sw_drm_stream_url, + "stream_format": stream_format.value if stream_format else None, } except ValueError as e: @@ -121,9 +132,12 @@ def setup_event_routes(app, manager, service, helpers): return {"error": f"Internal server error: {str(e)}"} @app.route("/api/providers//events//stream/index.mpd") + @app.route("/api/providers//events//stream/index.m3u8") def get_event_stream(provider, event_id): """ - Single stream endpoint for event playback. Replaces the former + Single stream endpoint for event playback. index.m3u8 is an alias of + index.mpd served by the same handler; the format actually returned + follows the upstream manifest, not the extension. Replaces the former /stream/sw-drm/, /stream/proxied/, and /stream/proxied/ffmpeg/ routes, which are removed. Every combination those routes used to hardcode is now expressed via independent, freely-combinable query diff --git a/routes/streams/recordings.py b/routes/streams/recordings.py index e8df8fe..3a72c88 100644 --- a/routes/streams/recordings.py +++ b/routes/streams/recordings.py @@ -14,6 +14,7 @@ _resolve_stream_unified. """ from bottle import HTTPResponse, request, response +from streaming_providers.base.stream_format import StreamFormat from streaming_providers.base.utils import logger @@ -22,6 +23,7 @@ def setup_recording_routes(app, manager, service, helpers): CONTENT_TYPE_RECORDING = helpers["CONTENT_TYPE_RECORDING"] _build_drm_header = helpers["_build_drm_header"] + _attach_drm_header = helpers["_attach_drm_header"] _build_stream_headers = helpers["_build_stream_headers"] _resolve_stream_unified = helpers["_resolve_stream_unified"] _get_drm_configs = helpers["_get_drm_configs"] @@ -83,16 +85,22 @@ def setup_recording_routes(app, manager, service, helpers): try: country = request.query.get("country") base_url = f"{request.urlparts.scheme}://{request.urlparts.netloc}" + # Resolve first: the detected format decides which alias the URL + # points at. Both aliases hit the same handler — the extension only + # helps clients that pick their player from the URL. Unknown + # format (None) keeps index.mpd, i.e. the pre-HLS behaviour. + stream_info = _attach_drm_header( + CONTENT_TYPE_RECORDING, provider, recording_id, country=country + ) + stream_format = stream_info.stream_format if stream_info else None + index_name = "index.m3u8" if stream_format is StreamFormat.HLS else "index.mpd" stream_url = ( f"{base_url}/api/providers/{provider}" - f"/recordings/{recording_id}/stream/index.mpd" + f"/recordings/{recording_id}/stream/{index_name}" ) if country: stream_url += f"?country={country}" - _build_drm_header( - CONTENT_TYPE_RECORDING, provider, recording_id, country=country - ) _build_stream_headers( CONTENT_TYPE_RECORDING, provider, recording_id, country=country ) @@ -101,6 +109,7 @@ def setup_recording_routes(app, manager, service, helpers): "provider": provider, "recording_id": recording_id, "manifest_url": stream_url, + "stream_format": stream_format.value if stream_format else None, } except ValueError as e: @@ -117,9 +126,12 @@ def setup_recording_routes(app, manager, service, helpers): return {"error": f"Internal server error: {str(e)}"} @app.route("/api/providers//recordings//stream/index.mpd") + @app.route("/api/providers//recordings//stream/index.m3u8") def get_recording_stream(provider, recording_id): """ - Single stream endpoint for recording playback. Replaces the former + Single stream endpoint for recording playback. index.m3u8 is an alias + of index.mpd served by the same handler; the format actually returned + follows the upstream manifest, not the extension. Replaces the former /stream/proxied/ route, which is removed. client_drm=false now reaches what that route used to do; see get_channel_stream in channels.py for the full query-param list (client_drm, drm_variant, diff --git a/routes/streams/vod.py b/routes/streams/vod.py index f0e9e83..198815d 100644 --- a/routes/streams/vod.py +++ b/routes/streams/vod.py @@ -14,6 +14,7 @@ _resolve_stream_unified. """ from bottle import HTTPResponse, request, response +from streaming_providers.base.stream_format import StreamFormat from streaming_providers.base.utils import logger @@ -22,6 +23,7 @@ def setup_vod_routes(app, manager, service, helpers): CONTENT_TYPE_VOD = helpers["CONTENT_TYPE_VOD"] _build_drm_header = helpers["_build_drm_header"] + _attach_drm_header = helpers["_attach_drm_header"] _build_stream_headers = helpers["_build_stream_headers"] _resolve_stream_unified = helpers["_resolve_stream_unified"] _get_drm_configs = helpers["_get_drm_configs"] @@ -79,9 +81,12 @@ def setup_vod_routes(app, manager, service, helpers): return {"error": f"Internal server error: {str(e)}"} @app.route("/api/providers//vod//stream/index.mpd") + @app.route("/api/providers//vod//stream/index.m3u8") def get_vod_stream(provider, path): """ - Single stream endpoint for VOD playback. Replaces the former + Single stream endpoint for VOD playback. index.m3u8 is an alias of + index.mpd served by the same handler; the format actually returned + follows the upstream manifest, not the extension. Replaces the former /stream/proxied/ and /stream/proxied/ffmpeg/ routes, which are removed. Every combination those routes used to hardcode is now expressed via independent, freely-combinable query params (see @@ -99,19 +104,28 @@ def setup_vod_routes(app, manager, service, helpers): try: country = request.query.get("country") base_url = f"{request.urlparts.scheme}://{request.urlparts.netloc}" + # Resolve first: the detected format decides which alias the URL + # points at. Both aliases hit the same handler — the extension only + # helps clients that pick their player from the URL. Unknown + # format (None) keeps index.mpd, i.e. the pre-HLS behaviour. + stream_info = _attach_drm_header( + CONTENT_TYPE_VOD, provider, vod_id, country=country + ) + stream_format = stream_info.stream_format if stream_info else None + index_name = "index.m3u8" if stream_format is StreamFormat.HLS else "index.mpd" stream_url = ( - f"{base_url}/api/providers/{provider}/vod/{vod_id}/stream/index.mpd" + f"{base_url}/api/providers/{provider}/vod/{vod_id}/stream/{index_name}" ) if country: stream_url += f"?country={country}" - _build_drm_header(CONTENT_TYPE_VOD, provider, vod_id, country=country) _build_stream_headers(CONTENT_TYPE_VOD, provider, vod_id, country=country) return { "provider": provider, "vod_id": vod_id, "manifest_url": stream_url, + "stream_format": stream_format.value if stream_format else None, } except ValueError as e: diff --git a/service.py b/service.py index ed23421..500d1ec 100644 --- a/service.py +++ b/service.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import hashlib import json import math import os @@ -25,12 +26,23 @@ try: # Add EPG Manager import from streaming_providers.base.epg.epg_manager import EPGManager from streaming_providers.base.models import StreamingChannel - from streaming_providers.base.stream_format import StreamFormat + from streaming_providers.base.stream_format import ( + StreamFormat, + content_type_for, + detect_format, + ) from streaming_providers.base.settings.provider_enable_manager import ( ProviderEnableManager, ) from streaming_providers.base.utils import MPDCacheManager, MPDRewriter, logger + from streaming_providers.base.utils.hls_rewriter import ( + HLSResource, + hls_cache_ttl, + rewrite_hls_playlist, + ) from streaming_providers.base.utils.init_kid_resolver import get_init_kid_resolver + from streaming_providers.base.utils.proxy_url import make_media_proxy_url_builder + from streaming_providers.base.utils.single_flight import SingleFlight from streaming_providers.base.utils.environment import ( get_environment_manager, get_vfs_instance, @@ -43,6 +55,29 @@ except ImportError as import_err: raise +def _manifest_content_type(content: Optional[str]) -> str: + """Content-Type for a manifest body, from the content itself (the source + of truth, same rule as detect_format). Unknown falls back to DASH, which is + what every manifest route served before HLS existed.""" + # Cached manifests may come back as bytes, and a content type must never be + # the reason a manifest request fails: anything undecodable stays DASH. + if isinstance(content, (bytes, bytearray)): + content = bytes(content[:8192]).decode("utf-8", errors="ignore") + if not isinstance(content, str): + return content_type_for(None) + return content_type_for(detect_format(content)) + + +def _require_dash(manifest_text: str, label: str) -> None: + """The DASH rewrite paths must never be fed an HLS playlist: MPDRewriter + would fail with an opaque XML parse error. HLS goes through + UltimateService.get_proxied_hls_playlist.""" + if detect_format(manifest_text) is StreamFormat.HLS: + raise ValueError( + f"{label}: upstream returned an HLS playlist on the DASH rewrite path" + ) + + class UltimateService: def __init__(self, config_dir: str = None): self.app = Bottle() @@ -83,6 +118,11 @@ class UltimateService: logger.info(f"MPD cache initialized: {self.mpd_cache.vfs.base_path}") self._decrypted_cache = {} # key -> (content, expiry_ts) — memory-only, never persisted + # Rewritten HLS playlists: memory-only like the decrypted cache (they carry + # baked-in segment headers and session tokens, so never persisted). + self._hls_cache = {} # sha256 key -> (content, expiry_ts) + self._hls_cache_lock = threading.Lock() + self._hls_flight = SingleFlight() # 1. Determine EPG URL FIRST (with proper precedence) self.epg_url = self._determine_epg_url() @@ -350,10 +390,15 @@ class UltimateService: provider: str, channel_id: str, manifest_url: str, + header_kwargs: Optional[dict] = None, ) -> tuple: """ Fetch a manifest and collect everything MPDRewriter needs. + header_kwargs (country, start_time, end_time, epg_id, ...) are passed to the + provider's get_manifest_headers / get_segment_headers, as the route layer's + _build_stream_headers does. None keeps the historical behaviour (channel id only). + Returns: (manifest_text, ttl, provider_proxy_url, segment_headers, effective_manifest_url) effective_manifest_url is the post-redirect URL (may differ from manifest_url @@ -370,12 +415,13 @@ class UltimateService: # Resolve manifest headers from the provider so auth tokens etc. are sent provider_instance = self.manager.get_provider(provider) + header_kwargs = header_kwargs or {} manifest_headers = ( - provider_instance.get_manifest_headers(channel_id) + provider_instance.get_manifest_headers(channel_id, **header_kwargs) if provider_instance else {} ) segment_headers = ( - provider_instance.get_segment_headers(channel_id) + provider_instance.get_segment_headers(channel_id, **header_kwargs) if provider_instance else {} ) @@ -447,7 +493,7 @@ class UltimateService: content, ttl = fetch_fn() if cacheable: cache_set(content, ttl) - response.content_type = "application/dash+xml; charset=utf-8" + response.content_type = _manifest_content_type(content) return content except ValueError as e: # Config error (e.g. no HTTP manager) — not transient, don't mask with stale cache @@ -461,7 +507,7 @@ class UltimateService: stale = cache_get_stale(stale_window) if stale: logger.warning(f"Serving stale {label} due to fetch failure") - response.content_type = "application/dash+xml; charset=utf-8" + response.content_type = _manifest_content_type(stale) return stale logger.error(f"Failed to fetch {label}: {fetch_err}") response.status = 502 @@ -474,7 +520,7 @@ class UltimateService: if not highest_quality_only: cached_mpd = self.mpd_cache.get(provider, channel_id) if cached_mpd: - response.content_type = "application/dash+xml; charset=utf-8" + response.content_type = _manifest_content_type(cached_mpd) return cached_mpd if not self.media_proxy_url: @@ -495,6 +541,7 @@ class UltimateService: manifest_text, ttl, provider_proxy_url, segment_headers, effective_url = self.fetch_manifest_for_rewriter( provider, channel_id, manifest_url ) + _require_dash(manifest_text, f"{provider}/{channel_id}") rewriter = MPDRewriter( self.media_proxy_url, provider_proxy_url, None, highest_quality_only, provider=provider, channel=channel_id, segment_headers=segment_headers, @@ -518,7 +565,7 @@ class UltimateService: cached_mpd = self.mpd_cache.get(provider, cache_key) if cached_mpd: - response.content_type = "application/dash+xml; charset=utf-8" + response.content_type = _manifest_content_type(cached_mpd) return cached_mpd logger.info(f"Cache miss for catchup {provider}/{channel_id}, fetching manifest") @@ -541,6 +588,7 @@ class UltimateService: manifest_text, ttl, provider_proxy_url, segment_headers, effective_url = self.fetch_manifest_for_rewriter( provider, channel_id, manifest_url ) + _require_dash(manifest_text, f"{provider}/{channel_id} (catchup)") rewriter = MPDRewriter( self.media_proxy_url, provider_proxy_url, None, False, provider=provider, channel=channel_id, segment_headers=segment_headers, @@ -558,6 +606,135 @@ class UltimateService: label=f"catchup manifest for {provider}/{channel_id}", ) + # ------------------------------------------------------------------ + # HLS playlists through the media proxy (phase 2, Option A) + # + # The backend fetches, rewrites and serves EVERY playlist; the media proxy + # only carries segments / init segments / (later) keys. Cache is memory-only + # and keyed by a hash of everything the output depends on. + # ------------------------------------------------------------------ + _HLS_CACHE_MAX_ENTRIES = 256 + _HLS_STALE_KEEP = 30 # seconds an expired entry stays for the stale fallback + + @staticmethod + def _hls_cache_key(provider: str, content_id: str, upstream_url: str, + header_kwargs: Optional[dict], cache_scope: str, proxy_keys: bool) -> str: + """Hash, so session tokens inside upstream URLs never show up in logs.""" + raw = json.dumps( + [provider, content_id, upstream_url, sorted((header_kwargs or {}).items()), + cache_scope, bool(proxy_keys)], + default=str, + ) + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + def _get_hls_cached(self, key: str, max_stale: float = 0) -> Optional[str]: + entry = self._hls_cache.get(key) + if not entry: + return None + content, expiry = entry + overdue = time.time() - expiry + if overdue < 0: + return content + if max_stale and overdue <= max_stale: + logger.warning(f"Serving STALE HLS playlist ({overdue:.1f}s past expiry)") + return content + # Expired entries are NOT deleted here (unlike _get_decrypted_cached): the + # stale fallback runs after this lookup and still needs them. _set prunes. + return None + + def _set_hls_cached(self, key: str, content: str, ttl: float) -> None: + now = time.time() + with self._hls_cache_lock: + self._hls_cache[key] = (content, now + ttl) + if len(self._hls_cache) <= self._HLS_CACHE_MAX_ENTRIES: + return + for k, (_, expiry) in list(self._hls_cache.items()): + if expiry < now - self._HLS_STALE_KEEP: + del self._hls_cache[k] + overflow = len(self._hls_cache) - self._HLS_CACHE_MAX_ENTRIES + if overflow > 0: + for k, _ in sorted(self._hls_cache.items(), key=lambda kv: kv[1][1])[:overflow]: + del self._hls_cache[k] + + def get_proxied_hls_playlist( + self, + provider: str, + content_id: str, + upstream_url: str, + *, + build_playlist_url, + cache_scope: str = "", + header_kwargs: Optional[dict] = None, + proxy_keys: bool = False, + ) -> str: + """ + Fetch one upstream HLS playlist (master or media) through the provider's + HTTP manager and return it rewritten for the media proxy. + + build_playlist_url(upstream_url) -> URL of the backend playlist endpoint for a + nested playlist (variant, audio/subtitle rendition, I-frame). Supplied by + the endpoint, which owns token signing and the request's base URL. + cache_scope: anything else the OUTPUT depends on that is not in the other + arguments — in practice the request's base URL, because the playlist URLs + built above embed it. + header_kwargs: provider header context (country, start_time, end_time, epg_id); + forwarded to get_manifest_headers/get_segment_headers and part of the cache key. + proxy_keys: also route AES-128 key URIs through the media proxy. + + Segment/init/key URLs become media-proxy URLs carrying the provider's segment + headers and provider proxy, exactly like the DASH path. Concurrent identical + requests share one fetch + rewrite (single-flight); the result is cached for + a fraction of the target duration (see hls_cache_ttl). Errors follow + _fetch_and_cache_manifest: config/rewrite errors -> 502 JSON, transient fetch + errors -> short stale fallback, else 502. + """ + if not self.media_proxy_url: + response.status = 503 + response.content_type = "application/json" + return json.dumps({"error": "Media proxy not configured (MEDIA_PROXY_URL not set)"}) + + key = self._hls_cache_key(provider, content_id, upstream_url, header_kwargs, cache_scope, proxy_keys) + cached = self._get_hls_cached(key) + if cached: + response.content_type = _manifest_content_type(cached) + return cached + + def produce(): + # A previous flight may have finished between our cache check and becoming leader. + hit = self._get_hls_cached(key) + if hit: + return hit, 0 + text, _http_ttl, provider_proxy_url, segment_headers, effective_url = self.fetch_manifest_for_rewriter( + provider, content_id, upstream_url, header_kwargs=header_kwargs + ) + provider_instance = self.manager.get_provider(provider) + media_url = make_media_proxy_url_builder( + self.media_proxy_url, + provider_proxy_url=provider_proxy_url, + segment_headers=segment_headers, + ) + + def build_url(url: str, kind: HLSResource) -> str: + return build_playlist_url(url) if kind is HLSResource.PLAYLIST else media_url(url) + + rewritten = rewrite_hls_playlist( + text, effective_url, build_url, + inherit_query=bool(getattr(provider_instance, "inherit_manifest_query_on_segments", True)), + proxy_keys=proxy_keys, + ) + ttl = hls_cache_ttl(text) + self._set_hls_cached(key, rewritten, ttl) + return rewritten, ttl + + return self._fetch_and_cache_manifest( + lambda: self._hls_flight.do(key, produce), + cache_set=lambda content, ttl: None, # produce() already stored it, once per flight + cache_get_stale=lambda max_stale: self._get_hls_cached(key, max_stale), + cacheable=True, + stale_window=6, + label=f"HLS playlist for {provider}/{content_id}", + ) + def get_decrypted_manifest(self, provider: str, channel_id: str, keyids: dict, highest_quality_only: bool = False, receiver_side: bool = False, drm_variant: str = "auto") -> str: @@ -567,7 +744,7 @@ class UltimateService: if not highest_quality_only: cached_mpd = self._get_decrypted_cached(cache_key) if cached_mpd: - response.content_type = "application/dash+xml; charset=utf-8" + response.content_type = _manifest_content_type(cached_mpd) return cached_mpd if not self.media_proxy_url: @@ -621,7 +798,7 @@ class UltimateService: if not highest_quality_only: cached_mpd = self._get_decrypted_cached(cache_key) if cached_mpd: - response.content_type = "application/dash+xml; charset=utf-8" + response.content_type = _manifest_content_type(cached_mpd) return cached_mpd if not self.media_proxy_url: