cache m3us realtive

routes/_url_utils.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+"""
+Shared helpers for deriving the base URL to hand out to clients.
+
+Kept separate from the individual route modules because the M3U route needs
+to absolutize URLs that were baked into a host-independent cache, and other
+routes (manifest endpoints) build absolute URLs the same way.
+"""
+
+from bottle import request
+
+
+def build_base_url() -> str:
+    """
+    Build the scheme://host[:port] prefix the current client used to reach us.
+
+    This is deliberately just the address the client itself put in its
+    request: whatever name/IP/port it used is, by definition, reachable from
+    where it is. Bottle's request.urlparts already resolves it in the right
+    order: X-Forwarded-Host / X-Forwarded-Proto (reverse proxy), then the
+    Host header, then SERVER_NAME:SERVER_PORT (HTTP/1.0 without Host).
+
+    Do NOT derive this from the local socket address: behind Docker bridge
+    networking or a reverse proxy that is the container/proxy-internal IP,
+    and dual-stack sockets report IPv4-mapped (::ffff:a.b.c.d) or scoped
+    (fe80::1%eth0) addresses that are not valid in URLs. (An earlier version
+    looked for a "waitress.client_socket" environ key; waitress never sets
+    one, and the server is bottle's default wsgiref anyway.)
+    """
+    parts = request.urlparts
+    return f"{parts.scheme}://{parts.netloc}"

routes/m3u.py

@@ -3,11 +3,14 @@
 M3U playlist route handlers
 """
 
+import re
 import time
 
-from bottle import request, response
+from bottle import response
 from streaming_providers.base.utils import logger
 
+from ._url_utils import build_base_url
+
 # TTL for the plain M3U cache ("/api/m3u", "/api/providers/<provider>/m3u")
 # only. Nothing embedded there goes stale from a security standpoint
 # (drm_directives="" — no keys/license URLs baked in), but the channel
@@ -29,6 +32,53 @@ M3U_MODE_PLAIN = "plain"
 M3U_MODE_CLIENTDRM_FALLBACK = "clientdrm-fallback"
 M3U_MODE_NOPROXY_RAW = "noproxy-raw"
 
+# ---------------------------------------------------------------------------
+# Serve-time absolutization of cached playlists.
+#
+# M3U files on disk carry RELATIVE stream URLs (`/api/providers/.../channels/
+# .../stream/index.mpd...`) so that one cached file serves every client,
+# regardless of which hostname/IP they used to reach the server. The absolute
+# base is derived per-request here, in the response path, from the current
+# connection (see _url_utils.build_base_url).
+#
+# Two shapes need rewriting:
+#   1. A standalone relative stream-URL line (the plain M3U format).
+#   2. The -i "<url>" argument inside an ffmpeg pipe:// command (the
+#      /proxied/ffmpeg variant wraps the stream URL in an ffmpeg invocation).
+#
+# External absolute URLs (tvg-logo="https://...") and catchup-source query
+# fragments ("?start_time=...") are deliberately untouched — they were never
+# host-dependent.
+# ---------------------------------------------------------------------------
+# A cache file written before the relative-URL change has an absolute
+# "http://oldhost:port/api/providers/..." baked in (standalone line or inside
+# the ffmpeg -i "..." argument). _absolutize deliberately leaves those alone,
+# so _serve_cached detects them and treats the file as a cache miss, which
+# regenerates it in the new (relative) format.
+_LEGACY_ABSOLUTE_STREAM_URL_RE = re.compile(
+    r'(?m)^https?://[^/\s]+/api/providers/'
+    r'|-i "https?://[^/"\s]+/api/providers/'
+)
+
+_RELATIVE_STREAM_URL_RE = re.compile(
+    r'(?m)^(?P<plain>/api/providers/\S+)$'
+    r'|'
+    r'(?P<ffmpeg_prefix>-i ")(?P<ffmpeg_url>/api/providers/[^"]+)'
+)
+
+
+def _absolutize(body: str, base_url: str) -> str:
+    """Prefix relative `/api/providers/...` stream URLs with `base_url`."""
+    if not body or "/api/providers/" not in body:
+        return body
+
+    def repl(m):
+        if m.group("plain"):
+            return f"{base_url}{m.group('plain')}"
+        return f"{m.group('ffmpeg_prefix')}{base_url}{m.group('ffmpeg_url')}"
+
+    return _RELATIVE_STREAM_URL_RE.sub(repl, body)
+
 
 def setup_m3u_routes(app, manager, service):
     """Setup M3U playlist-related routes"""
@@ -66,6 +116,9 @@ def setup_m3u_routes(app, manager, service):
         (or, when ttl_seconds is given, if the sidecar meta file shows the
         cache is older than that — triggering a regeneration the same as
         a cold cache would).
+
+        The returned body is the raw cached text with RELATIVE stream URLs;
+        _handle_m3u_route absolutizes it at the response boundary.
         """
         if ttl_seconds is not None:
             meta = service.vfs.read_json(_cache_meta_key(cache_key))
@@ -87,6 +140,12 @@ def setup_m3u_routes(app, manager, service):
         cached = service.vfs.read_text(cache_key)
         if not cached:
             return None
+        if _LEGACY_ABSOLUTE_STREAM_URL_RE.search(cached):
+            logger.info(
+                f"Cache {cache_key} holds legacy absolute stream URLs — "
+                f"discarding and regenerating with relative URLs"
+            )
+            return None
         logger.info(f"Serving cached M3U: {cache_key}")
         response.content_type = "audio/x-mpegurl; charset=utf-8"
         response.headers["Content-Disposition"] = f'attachment; filename="{filename}"'
@@ -95,12 +154,15 @@ def setup_m3u_routes(app, manager, service):
     def _handle_m3u_route(generate_fn, log_ctx, cache_key=None, filename=None,
                            ttl_seconds=None, force=False):
         """
-        Shared cache-check / error-handling wrapper for M3U routes.
+        Shared cache-check / error-handling / absolutization wrapper for M3U
+        routes.
 
         generate_fn: no-arg callable that returns the M3U string. It is
         responsible for its own success-path response headers (all the
         service.generate_* methods already do this) and, for the
-        proxy-gated "fast" endpoints, for their own 503 handling.
+        proxy-gated "fast" endpoints, for their own 503 handling. It MUST
+        emit relative stream URLs; the absolutization below is applied to
+        whatever string it returns.
 
         cache_key: pass None for routes that must always regenerate live
         (the uncached clientdrm endpoints, the ffmpeg endpoint, and the
@@ -120,7 +182,7 @@ def setup_m3u_routes(app, manager, service):
             if cache_key and not force:
                 cached = _serve_cached(cache_key, filename, ttl_seconds=ttl_seconds)
                 if cached is not None:
-                    return cached
+                    return _absolutize(cached, build_base_url())
                 logger.info(f"No valid cache found, generating M3U: {log_ctx}")
 
             result = generate_fn()
@@ -134,12 +196,22 @@ def setup_m3u_routes(app, manager, service):
             # A failed meta write must not turn a successful generation
             # into a 500: the worst case is a missing sidecar, which
             # _serve_cached already treats as "expired" and regenerates.
-            if cache_key and ttl_seconds is not None and isinstance(result, str):
+            #
+            # Only for a real playlist: not for an error body (status >= 400)
+            # and not for a 0-channel result, which the service does not
+            # write to the cache either. Touching the meta in that case would
+            # extend the life of the OLD cache file by another full TTL.
+            if (
+                cache_key and ttl_seconds is not None and isinstance(result, str)
+                and response.status_code < 400 and "#EXTINF" in result
+            ):
                 try:
                     _touch_cache_meta(cache_key)
                 except Exception as meta_err:
                     logger.warning(f"Could not write cache metadata for {cache_key}: {meta_err}")
 
+            if isinstance(result, str):
+                return _absolutize(result, build_base_url())
             return result
 
         except ValueError as val_err:
@@ -160,6 +232,19 @@ def setup_m3u_routes(app, manager, service):
             pass  # cache file may not exist yet - fine
         return _handle_m3u_route(generate_fn, log_ctx)
 
+    def _for_provider(provider: str, generate_fn):
+        """
+        Wrap a per-provider generator so an unknown provider name raises
+        ValueError (-> 404 in _handle_m3u_route) before anything is generated
+        or cached. Without this, any URL segment would generate a 0-channel
+        playlist and create `<name>.m3u` + meta files on disk.
+        """
+        def wrapped():
+            if provider not in manager.list_providers():
+                raise ValueError(f"Unknown provider '{provider}'")
+            return generate_fn()
+        return wrapped
+
     # ── Plain playlists (cached, 24h TTL) ──────────────────────────────────
     # Server-side decrypt, bare stream URLs (no "?client_drm=false" — that's
     # the route's own default). Nothing per-channel here is request-time-
@@ -272,8 +357,10 @@ def setup_m3u_routes(app, manager, service):
             cache_key=f"{provider}.m3u",
             filename=f"{provider}_playlist.m3u8",
             log_ctx=f"/api/providers/{provider}/m3u",
-            plain_fn=lambda: service.generate_m3u_plain_provider(provider, save_to_cache=True),
-            fallback_fn=lambda: service.generate_m3u_clientdrm_provider(provider),
+            plain_fn=_for_provider(
+                provider, lambda: service.generate_m3u_plain_provider(provider, save_to_cache=True)),
+            fallback_fn=_for_provider(
+                provider, lambda: service.generate_m3u_clientdrm_provider(provider)),
         )
 
     @app.route("/api/providers/<provider>/m3u/generate")
@@ -285,8 +372,10 @@ def setup_m3u_routes(app, manager, service):
             cache_key=f"{provider}.m3u",
             filename=f"{provider}_playlist.m3u8",
             log_ctx=f"/api/providers/{provider}/m3u/generate",
-            plain_fn=lambda: service.generate_m3u_plain_provider(provider, save_to_cache=True),
-            fallback_fn=lambda: service.generate_m3u_clientdrm_provider(provider),
+            plain_fn=_for_provider(
+                provider, lambda: service.generate_m3u_plain_provider(provider, save_to_cache=True)),
+            fallback_fn=_for_provider(
+                provider, lambda: service.generate_m3u_clientdrm_provider(provider)),
             force=True,
         )
 
@@ -344,7 +433,8 @@ def setup_m3u_routes(app, manager, service):
     def get_m3u_provider_noproxy(provider):
         """Generates no-proxy M3U playlist for a specific provider."""
         return _handle_m3u_route(
-            lambda: service.generate_m3u_noproxy_provider(provider, save_to_cache=True),
+            _for_provider(
+                provider, lambda: service.generate_m3u_noproxy_provider(provider, save_to_cache=True)),
             log_ctx=f"/api/providers/{provider}/m3u/noproxy",
             cache_key=f"{provider}_noproxy.m3u",
             filename=f"{provider}_playlist_noproxy.m3u8",
@@ -354,7 +444,8 @@ def setup_m3u_routes(app, manager, service):
     def generate_m3u_provider_noproxy(provider):
         """Force regeneration of no-proxy M3U playlist for a specific provider."""
         return _handle_m3u_route(
-            lambda: service.generate_m3u_noproxy_provider(provider, save_to_cache=True),
+            _for_provider(
+                provider, lambda: service.generate_m3u_noproxy_provider(provider, save_to_cache=True)),
             log_ctx=f"/api/providers/{provider}/m3u/noproxy/generate",
         )
 
@@ -372,6 +463,9 @@ def setup_m3u_routes(app, manager, service):
     # Unlike /api/m3u/noproxy, entries carry no catchup attributes: the raw
     # upstream live URL can't take the start/end times the redirect route
     # translates.
+    #
+    # These entries carry ABSOLUTE upstream URLs (never /api/providers/...),
+    # so the serve-time absolutizer leaves them alone. Good.
 
     @app.route("/api/m3u/noproxy/raw")
     def get_m3u_all_noproxy_raw():
@@ -432,7 +526,9 @@ def setup_m3u_routes(app, manager, service):
     def get_m3u_proxied_filtered_provider(provider):
         """Generates filtered proxied M3U for a specific provider."""
         return _handle_m3u_route(
-            lambda: service.generate_m3u_proxied_filtered_provider(provider, save_to_cache=True),
+            _for_provider(
+                provider,
+                lambda: service.generate_m3u_proxied_filtered_provider(provider, save_to_cache=True)),
             log_ctx=f"/api/providers/{provider}/m3u/proxied/filtered",
             cache_key=f"{provider}_proxied_filtered.m3u",
             filename=f"{provider}_proxied_filtered_playlist.m3u8",
@@ -443,7 +539,9 @@ def setup_m3u_routes(app, manager, service):
         """Force regeneration of filtered proxied M3U for a specific provider."""
         return _force_regenerate(
             f"{provider}_proxied_filtered.m3u",
-            lambda: service.generate_m3u_proxied_filtered_provider(provider, save_to_cache=True),
+            _for_provider(
+                provider,
+                lambda: service.generate_m3u_proxied_filtered_provider(provider, save_to_cache=True)),
             log_ctx=f"/api/providers/{provider}/m3u/proxied/filtered/generate",
         )
 
@@ -453,13 +551,17 @@ def setup_m3u_routes(app, manager, service):
     # M3U content directly (they were never moved into service.py) - only
     # the boilerplate around them is shared via the same helpers used
     # everywhere else.
+    #
+    # Stream URLs here are emitted RELATIVE so the cache is host-independent;
+    # _handle_m3u_route absolutizes at the response boundary, same as the
+    # service-side generators.
 
     def _generate_m3u_subscribed(proxied: bool = False):
         if proxied and not service.media_proxy_url:
             response.status = 503
             return {"error": "Media proxy not configured (MEDIA_PROXY_URL not set)"}
 
-        base_url = f"{request.urlparts.scheme}://{request.urlparts.netloc}"
+        # Stream URLs stay relative; _handle_m3u_route absolutizes.
         m3u_content = "#EXTM3U\n"
 
         for provider_name in manager.list_providers():
@@ -501,8 +603,9 @@ def setup_m3u_routes(app, manager, service):
                         else "stream/index.mpd?client_drm=true"
                     )
                     stream_path = service.stream_path_for(channel, stream_path)
+                    # RELATIVE: absolutized at the response boundary.
                     stream_url = (
-                        f"{base_url}/api/providers/{provider_name}/channels/{channel_id}/{stream_path}"
+                        f"/api/providers/{provider_name}/channels/{channel_id}/{stream_path}"
                     )
 
                     m3u_content += (
@@ -535,7 +638,7 @@ def setup_m3u_routes(app, manager, service):
         # Proxied variant is deliberately uncached — proxy/DRM session state
         # can shift between requests, same rationale as the "fast" proxied
         # playlists above.
-        if not proxied:
+        if not proxied and "#EXTINF" in m3u_content:
             if service.vfs.write_text("playlist_subscribed.m3u", m3u_content):
                 logger.info("Subscribed M3U playlist cached to playlist_subscribed.m3u")
 

service.py

@@ -880,6 +880,10 @@ class UltimateService:
         this just appends the stream URL built from stream_path.
 
         Args:
+            base_url: Prefix for the stream URL. Pass "" to emit a RELATIVE
+                      URL (/api/providers/...) — the M3U cache is
+                      host-independent; routes/m3u.py absolutizes at the
+                      response boundary using the current request's base URL.
             stream_path: Relative path (optionally with a query string) after
                          .../channels/{channel_id}/. There's only one real
                          channel stream route now — "stream/index.mpd" — so
@@ -897,7 +901,9 @@ class UltimateService:
             drm_directives=drm_directives,
             include_catchup=include_catchup,
         )
-        stream_url = f"{base_url}/api/providers/{provider_name}/channels/{channel_id}/{stream_path}"
+        stream_url = f"/api/providers/{provider_name}/channels/{channel_id}/{stream_path}"
+        if base_url:
+            stream_url = f"{base_url}{stream_url}"
         return header + f"{stream_url}\n"
 
     def _iter_m3u_provider_channels(self, providers_to_process):
@@ -952,6 +958,10 @@ class UltimateService:
         happening at generation time, unlike the dynamic-DRM "clientdrm"
         variant below.
 
+        Stream URLs are emitted RELATIVE and cached that way. routes/m3u.py
+        absolutizes at the response boundary so a single cached playlist
+        serves every client (IP or hostname) correctly.
+
         Args:
             providers: List of provider names, or None for all providers
             save_to_cache: Whether to save to cache
@@ -971,8 +981,8 @@ class UltimateService:
                 {"error": "Media proxy not configured (MEDIA_PROXY_URL not set)"}
             )
 
-        # Get base URL for absolute stream URLs
-        base_url = f"{request.urlparts.scheme}://{request.urlparts.netloc}"
+        # Stream URLs stay relative; routes/m3u.py absolutizes.
+        base_url = ""
 
         # Start M3U content
         m3u_content = "#EXTM3U\n"
@@ -1013,8 +1023,12 @@ class UltimateService:
         logger.info(f"Plain M3U: included {channels_included} channels")
 
         # Save to cache if requested — safe here (see docstring: nothing on
-        # this path is request-time-volatile).
-        if save_to_cache and cache_filename:
+        # this path is request-time-volatile, and URLs are relative).
+        # Never cache an empty playlist: a failed provider lookup (unknown
+        # name, upstream outage) would otherwise be served for the whole TTL.
+        if save_to_cache and cache_filename and not channels_included:
+            logger.warning(f"Plain M3U has 0 channels — not caching {cache_filename}")
+        elif save_to_cache and cache_filename:
             if self.vfs.write_text(cache_filename, m3u_content):
                 logger.info(f"M3U playlist cached to {cache_filename}")
             else:
@@ -1028,6 +1042,11 @@ class UltimateService:
         Includes ALL channels with highest quality video and ffmpeg-piped stream URLs.
         No DRM filtering, no caching - maximum speed.
 
+        Stream URLs (inside the pipe:// command) are emitted RELATIVE; the
+        serve-time absolutizer in routes/m3u.py rewrites them together with
+        the plain format, so this endpoint's URLs also follow the incoming
+        request's host.
+
         Args:
             providers: List of provider names, or None for all providers
 
@@ -1043,8 +1062,8 @@ class UltimateService:
                 {"error": "Media proxy not configured (MEDIA_PROXY_URL not set)"}
             )
 
-        # Get base URL for absolute stream URLs
-        base_url = f"{request.urlparts.scheme}://{request.urlparts.netloc}"
+        # Stream URLs stay relative; routes/m3u.py absolutizes.
+        base_url = ""
 
         # Start M3U content
         m3u_content = "#EXTM3U\n"
@@ -1073,7 +1092,9 @@ class UltimateService:
                     stream_path = self.stream_path_for(
                         channel, "stream/index.mpd?client_drm=false&highest_quality_only=true"
                     )
-                    stream_url = f"{base_url}/api/providers/{provider_name}/channels/{channel_id}/{stream_path}"
+                    stream_url = f"/api/providers/{provider_name}/channels/{channel_id}/{stream_path}"
+                    if base_url:
+                        stream_url = f"{base_url}{stream_url}"
 
                     # Build ffmpeg pipe command
                     ffmpeg_cmd = self._build_ffmpeg_pipe_command(stream_url, channel_name)
@@ -1126,6 +1147,8 @@ class UltimateService:
         Internal method to generate filtered decrypted M3U content for specified providers.
         Only includes channels with ClearKey DRM or unencrypted channels.
 
+        Stream URLs are emitted RELATIVE; routes/m3u.py absolutizes.
+
         Args:
             providers: List of provider names, or None for all providers
             save_to_cache: Whether to save to cache
@@ -1139,8 +1162,8 @@ class UltimateService:
             logger.error("Cannot generate decrypted M3U: MEDIA_PROXY_URL not set")
             return None
 
-        # Get base URL for absolute stream URLs
-        base_url = f"{request.urlparts.scheme}://{request.urlparts.netloc}"
+        # Stream URLs stay relative; routes/m3u.py absolutizes.
+        base_url = ""
 
         # Start M3U content
         m3u_content = "#EXTM3U\n"
@@ -1235,8 +1258,10 @@ class UltimateService:
             f"Filtered decrypted M3U: included {channels_included} channels, skipped {channels_skipped}"
         )
 
-        # Save to cache if requested
-        if save_to_cache and cache_filename:
+        # Save to cache if requested (never an empty playlist)
+        if save_to_cache and cache_filename and not channels_included:
+            logger.warning(f"Filtered M3U has 0 channels — not caching {cache_filename}")
+        elif save_to_cache and cache_filename:
             if self.vfs.write_text(cache_filename, m3u_content):
                 logger.info(f"Filtered decrypted M3U playlist cached to {cache_filename}")
             else:
@@ -1263,6 +1288,8 @@ class UltimateService:
         and the pre-existing "/m3u/noproxy" playlists, which combine
         client-side decrypt with a forced non-proxied stream route.
 
+        Stream URLs are emitted RELATIVE; routes/m3u.py absolutizes.
+
         Args:
             providers: List of provider names, or None for all providers
             save_to_cache: Whether to save to cache
@@ -1274,8 +1301,8 @@ class UltimateService:
         Returns:
             M3U content as string
         """
-        # Get base URL for absolute stream URLs
-        base_url = f"{request.urlparts.scheme}://{request.urlparts.netloc}"
+        # Stream URLs stay relative; routes/m3u.py absolutizes.
+        base_url = ""
 
         # Start M3U content
         m3u_content = "#EXTM3U\n"
@@ -1314,6 +1341,8 @@ class UltimateService:
         # every entry here would silently mismatch its own KODIPROP directives.
         stream_path = "stream/index.mpd?client_drm=true&no_proxy=true" if no_proxy else "stream/index.mpd?client_drm=true"
 
+        channels_included = 0
+
         for provider_name, provider_label, channels in self._iter_m3u_provider_channels(providers_to_process):
             try:
                 # Add each channel to M3U
@@ -1323,6 +1352,7 @@ class UltimateService:
                         stream_path=stream_path,
                         provider_label=provider_label,
                     )
+                    channels_included += 1
 
             except Exception as provider_err:
                 logger.warning(
@@ -1330,8 +1360,10 @@ class UltimateService:
                 )
                 continue
 
-        # Save to cache if requested
-        if save_to_cache and cache_filename:
+        # Save to cache if requested (never an empty playlist)
+        if save_to_cache and cache_filename and not channels_included:
+            logger.warning(f"M3U has 0 channels — not caching {cache_filename}")
+        elif save_to_cache and cache_filename:
             if self.vfs.write_text(cache_filename, m3u_content):
                 logger.info(f"M3U playlist cached to {cache_filename}")
             else:
@@ -1394,6 +1426,11 @@ class UltimateService:
         Deliberately UNCACHED — raw URLs and header tokens can both be
         short-lived. Never log header values.
 
+        NOTE: because this function emits ABSOLUTE UPSTREAM URLs (never
+        /api/providers/...), the serve-time absolutizer in routes/m3u.py
+        leaves the body alone. Intentional — the client should hit the
+        provider directly, not our server.
+
         Returns:
             (m3u_content, channels_included, channels_skipped)
         """