update readme

lib/streaming_providers/base/errors.py

@@ -48,6 +48,9 @@ that don't use codes leave it None.
 
 from __future__ import annotations
 
+import email.utils
+import math
+from datetime import datetime, timezone
 from typing import Any, Optional, Tuple
 
 
@@ -216,15 +219,41 @@ class BadRequestError(ProviderError):
 # ---------------------------------------------------------------------------
 
 class RateLimitError(ProviderError):
-    """429 -- caller should back off and retry."""
+    """429 -- caller should back off and retry.
+
+    ``retry_after`` is the server's Retry-After hint in seconds (already
+    capped by parse_retry_after), or None when the server sent none. It is a
+    class attribute default so provider subclasses with their own __init__
+    (e.g. a WAF-block exception) still have the attribute.
+    """
+
+    retry_after: Optional[float] = None
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        status: Optional[int] = None,
+        url: Optional[str] = None,
+        code: Optional[str] = None,
+        retry_after: Optional[float] = None,
+    ) -> None:
+        super().__init__(message, status=status, url=url, code=code)
+        self.retry_after = retry_after
 
 
 class ServerError(ProviderError):
     """5xx -- retryable."""
 
 
-class TransportError(ProviderError):
-    """Connection-level failure (DNS, TLS, timeout) -- no HTTP status."""
+class TransportError(ServerError):
+    """Connection-level failure (DNS, TLS, timeout) -- no HTTP status.
+
+    Deliberately a ServerError: the error contract (README section 8) maps
+    "network down" to ServerError and says retry loops retry only
+    ServerError/untyped errors. As a sibling instead of a subclass, a
+    TransportError would have escaped every such retry loop.
+    """
 
 
 # ---------------------------------------------------------------------------
@@ -265,38 +294,155 @@ class OperationFailedError(ProviderError, RuntimeError):
 
 
 # ---------------------------------------------------------------------------
-# HTTP status -> error class heuristic
+# HTTP status -> error class mapping
 # ---------------------------------------------------------------------------
 
+def parse_retry_after(
+    value: Any, *, max_seconds: float = 300.0
+) -> Optional[float]:
+    """
+    Parse a Retry-After header (delta-seconds or HTTP-date) into seconds.
+
+    Returns None when the value is missing or unparseable. Negative values
+    (a date in the past) become 0.0; results are capped at ``max_seconds`` so
+    a hostile or buggy header cannot park a worker thread for hours.
+    """
+    if value is None:
+        return None
+    text = str(value).strip()
+    if not text:
+        return None
+    try:
+        seconds = float(text)
+    except ValueError:
+        try:
+            when = email.utils.parsedate_to_datetime(text)
+        except (TypeError, ValueError):
+            return None
+        if when is None:
+            return None
+        if when.tzinfo is None:
+            when = when.replace(tzinfo=timezone.utc)
+        seconds = (when - datetime.now(timezone.utc)).total_seconds()
+    if math.isnan(seconds):
+        return None
+    return min(max(seconds, 0.0), max_seconds)
+
+
+_GEO_MARKERS = ("geo", "region")
+_WAF_MARKERS = (
+    "captcha",
+    "cf-chl",
+    "just a moment",
+    "attention required",
+    "cloudflare",
+)
+
+
 def default_error_for_status_heuristic(
     status: int,
     message: str,
     *,
     url: Optional[str] = None,
     body_snippet: str = "",
+    retry_after: Optional[float] = None,
 ) -> ProviderError:
     """
     Heuristic mapping from HTTP status to ProviderError subclass.
 
     This is a heuristic, not a contract. Providers with their own error
     classification should construct the specific error directly rather than
-    relying on this. The 403 branch in particular uses a substring match on
+    relying on this. The 403 branch in particular uses substring matches on
     body_snippet and is intentionally shallow so providers do not depend on
     it accidentally.
+
+    403 order: geo markers first (a Cloudflare "country or region" block is
+    permanent), then WAF/CAPTCHA markers (transient -> RateLimitError, as the
+    README error contract requires), else EntitlementError.
     """
     if status == 400:
         return BadRequestError(message, status=status, url=url)
     if status == 401:
         return AuthError(message, status=status, url=url)
+    if status == 402:
+        return EntitlementError(message, status=status, url=url)
     if status == 403:
         lower = body_snippet.lower()
-        if "geo" in lower or "region" in lower:
+        if any(marker in lower for marker in _GEO_MARKERS):
             return GeoBlockError(message, status=status, url=url)
+        if any(marker in lower for marker in _WAF_MARKERS):
+            return RateLimitError(
+                message, status=status, url=url, retry_after=retry_after
+            )
         return EntitlementError(message, status=status, url=url)
-    if status == 404:
+    if status in (404, 410):
         return NotFoundError(message, status=status, url=url)
     if status == 429:
-        return RateLimitError(message, status=status, url=url)
+        return RateLimitError(
+            message, status=status, url=url, retry_after=retry_after
+        )
+    if status == 451:
+        return GeoBlockError(message, status=status, url=url)
     if 500 <= status < 600:
         return ServerError(message, status=status, url=url)
-    return ProviderError(message, status=status, url=url)
+    return ProviderError(message, status=status, url=url)
+
+
+def error_from_response(
+    response: Any, message: Optional[str] = None, *, body_limit: int = 500
+) -> ProviderError:
+    """
+    Build a ProviderError from any response-like object (requests or
+    curl_cffi): needs ``status_code``; uses ``url``, ``reason``, ``headers``
+    and ``text`` when present. Applies default_error_for_status_heuristic and
+    fills ``retry_after`` from the Retry-After header.
+    """
+    try:
+        status = int(getattr(response, "status_code", 0) or 0)
+    except (TypeError, ValueError):
+        status = 0
+    raw_url = getattr(response, "url", None)
+    url = str(raw_url) if raw_url else None
+
+    try:
+        snippet = (response.text or "")[:body_limit]
+    except Exception:
+        snippet = ""
+
+    headers = getattr(response, "headers", None)
+    raw_retry = headers.get("Retry-After") if hasattr(headers, "get") else None
+    retry_after = parse_retry_after(raw_retry)
+
+    if message is None:
+        reason = getattr(response, "reason", None) or ""
+        message = f"HTTP {status} {reason}".strip()
+        if url:
+            message += f" for {url}"
+
+    return default_error_for_status_heuristic(
+        status, message, url=url, body_snippet=snippet, retry_after=retry_after
+    )
+
+
+def error_from_exception(exc: BaseException, *, body_limit: int = 500) -> ProviderError:
+    """
+    Convert an HTTP-layer exception into a ProviderError.
+
+    * already a ProviderError            -> returned unchanged
+    * has a non-None ``.response``       -> error_from_response()
+    * no response (timeout, DNS, proxy)  -> TransportError
+
+    Only pass exceptions raised by the HTTP layer (HTTPManager raises
+    requests exceptions on both of its backends). Usage::
+
+        try:
+            resp = http.get(url, operation="api")
+        except Exception as exc:
+            raise error_from_exception(exc) from exc
+    """
+    if isinstance(exc, ProviderError):
+        return exc
+    response = getattr(exc, "response", None)
+    if response is not None:  # NOT `if response`: a 4xx/5xx Response is falsy
+        return error_from_response(response, body_limit=body_limit)
+    return TransportError(str(exc) or exc.__class__.__name__)

lib/streaming_providers/base/models/proxy_models.py

@@ -1,4 +1,6 @@
 # streaming_providers/base/models/proxy_models.py
+import re
+import urllib.parse
 from dataclasses import dataclass, field
 from enum import Enum
 from typing import Any, Dict, Optional
@@ -10,7 +12,21 @@ class ProxyType(Enum):
     HTTP = "http"
     HTTPS = "https"
     SOCKS4 = "socks4"
+    SOCKS4A = "socks4a"  # hostname resolved by the proxy
     SOCKS5 = "socks5"
+    SOCKS5H = "socks5h"  # hostname resolved by the proxy
+
+
+# The `operation=` values HTTPManager.get/post/put/delete understand for proxy
+# scoping. Any other string (v1 providers pass ad-hoc names such as
+# "live_manifest") is NOT an error: it is proxied whenever the master switch
+# `ProxyScope.all` is on, regardless of the per-operation flags.
+KNOWN_OPERATIONS = ("api", "auth", "manifest", "license")
+
+
+def _redact_url_credentials(url: str) -> str:
+    """Hide everything between '//' and the last '@' (user:password)."""
+    return re.sub(r"//.*@", "//***@", url)
 
 
 @dataclass
@@ -27,7 +43,13 @@ class ProxyScope:
     all: bool = True  # Master switch - overrides all others
 
     def should_use_proxy_for(self, operation: str) -> bool:
-        """Check if proxy should be used for specific operation"""
+        """
+        Check if proxy should be used for specific operation.
+
+        Known operations: see KNOWN_OPERATIONS. Unknown operation names fall
+        back to True (proxied) as long as the master switch `all` is on --
+        they are never matched against the per-operation flags.
+        """
         if not self.all:
             return False
 
@@ -43,15 +65,21 @@ class ProxyScope:
 
 @dataclass
 class ProxyAuth:
-    """Proxy authentication details"""
+    """Proxy authentication details (password is hidden from repr/logs)"""
 
     username: str
-    password: str
+    password: str = field(repr=False)
 
     def to_auth_string(self) -> str:
-        """Convert to authentication string for proxy URL"""
+        """Raw 'username:password' (NOT URL-encoded, never log this)"""
         return f"{self.username}:{self.password}"
 
+    def to_url_userinfo(self) -> str:
+        """Percent-encoded 'username:password' for use inside a proxy URL"""
+        user = urllib.parse.quote(self.username, safe="")
+        password = urllib.parse.quote(self.password, safe="")
+        return f"{user}:{password}"
+
 
 @dataclass
 class ProxyConfig:
@@ -78,6 +106,12 @@ class ProxyConfig:
     # Provider-specific overrides
     provider_specific: Dict[str, Any] = field(default_factory=dict)
 
+    def _host_for_url(self) -> str:
+        """Host as it must appear in a URL (IPv6 literals need brackets)"""
+        if ":" in self.host and not self.host.startswith("["):
+            return f"[{self.host}]"
+        return self.host
+
     def to_proxy_dict(self) -> Dict[str, str]:
         """
         Convert to requests-compatible proxy dictionary
@@ -85,23 +119,32 @@ class ProxyConfig:
         Returns:
             Dict in format {'http': 'proxy_url', 'https': 'proxy_url'}
         """
-        # Build proxy URL
-        auth_part = ""
-        if self.auth:
-            auth_part = f"{self.auth.to_auth_string()}@"
-
-        proxy_url = f"{self.proxy_type.value}://{auth_part}{self.host}:{self.port}"
-
-        # Return both HTTP and HTTPS proxy settings
+        proxy_url = self.to_proxy_url()
         return {"http": proxy_url, "https": proxy_url}
 
     def to_proxy_url(self) -> str:
-        """Get single proxy URL string"""
-        auth_part = ""
-        if self.auth:
-            auth_part = f"{self.auth.to_auth_string()}@"
+        """
+        Get single proxy URL string.
 
-        return f"{self.proxy_type.value}://{auth_part}{self.host}:{self.port}"
+        Contains the password (percent-encoded). Use to_safe_url() for
+        anything that is logged or displayed.
+        """
+        userinfo = f"{self.auth.to_url_userinfo()}@" if self.auth else ""
+        return (
+            f"{self.proxy_type.value}://{userinfo}"
+            f"{self._host_for_url()}:{self.port}"
+        )
+
+    def to_safe_url(self) -> str:
+        """Proxy URL with the password masked -- safe for logs and UIs"""
+        userinfo = ""
+        if self.auth:
+            user = urllib.parse.quote(self.auth.username, safe="")
+            userinfo = f"{user}:***@"
+        return (
+            f"{self.proxy_type.value}://{userinfo}"
+            f"{self._host_for_url()}:{self.port}"
+        )
 
     def validate(self) -> bool:
         """Validate proxy configuration"""
@@ -113,8 +156,14 @@ class ProxyConfig:
             return False
         return True
 
-    def to_dict(self) -> Dict[str, Any]:
-        """Convert to dictionary for serialization"""
+    def to_dict(self, redact: bool = False) -> Dict[str, Any]:
+        """
+        Convert to dictionary for serialization.
+
+        Args:
+            redact: mask the proxy password ("***"). Use for logs/UI; the
+                default keeps the password so from_dict() round-trips.
+        """
         result = {
             "host": self.host,
             "port": self.port,
@@ -134,7 +183,7 @@ class ProxyConfig:
         if self.auth:
             result["auth"] = {
                 "username": self.auth.username,
-                "password": self.auth.password,
+                "password": "***" if redact else self.auth.password,
             }
 
         return result
@@ -176,34 +225,48 @@ class ProxyConfig:
         Create ProxyConfig from proxy URL string
 
         Args:
-            proxy_url: Proxy URL like "http://user:pass@proxy.example.com:8080"
+            proxy_url: Proxy URL like "http://user:pass@proxy.example.com:8080".
+                Credentials may be percent-encoded (they are decoded here).
             scope: Optional scope configuration
 
         Returns:
             ProxyConfig instance
-        """
-        import urllib.parse
 
+        Raises:
+            ValueError: missing host/port or unsupported scheme. Error
+                messages never contain the credentials.
+        """
+        safe = _redact_url_credentials(proxy_url)
         parsed = urllib.parse.urlparse(proxy_url)
 
-        if not parsed.hostname or not parsed.port:
-            raise ValueError(f"Invalid proxy URL: {proxy_url}")
+        try:
+            port = parsed.port
+        except ValueError:
+            raise ValueError(f"Invalid proxy URL (bad port): {safe}") from None
+        if not parsed.hostname or not port:
+            raise ValueError(f"Invalid proxy URL: {safe}")
 
         auth = None
-        if parsed.username and parsed.password:
-            auth = ProxyAuth(username=parsed.username, password=parsed.password)
+        if parsed.username:
+            auth = ProxyAuth(
+                username=urllib.parse.unquote(parsed.username),
+                password=urllib.parse.unquote(parsed.password or ""),
+            )
 
         proxy_type = ProxyType.HTTP
         if parsed.scheme:
             try:
                 proxy_type = ProxyType(parsed.scheme.lower())
             except ValueError:
-                # Default to HTTP if scheme not recognized
-                pass
+                # Silently falling back to HTTP turned e.g. socks5h:// into a
+                # broken HTTP proxy; refuse instead.
+                raise ValueError(
+                    f"Unsupported proxy scheme '{parsed.scheme}' in {safe}"
+                ) from None
 
         return cls(
             host=parsed.hostname,
-            port=parsed.port,
+            port=port,
             proxy_type=proxy_type,
             auth=auth,
             scope=scope or ProxyScope(),
@@ -221,7 +284,7 @@ class RequestConfig:
     proxy_config: Optional[ProxyConfig] = None
 
     # Request settings
-    timeout: int = 30
+    timeout: float = 30
     verify_ssl: bool = True
     max_retries: int = 3
     retry_delay: float = 1.0
@@ -247,6 +310,17 @@ class RequestConfig:
 
     use_tls_impersonation: bool = False  # curl_cffi Chrome impersonation; ignored under Kodi
 
+    # When True, HTTPManager sends the previous response URL as Referer.
+    # Opt-in: WAFs (e.g. Akamai) flag API calls that carry backend API URLs
+    # as Referer. (New fields are appended last so positional construction
+    # of the fields above keeps working.)
+    inject_referer: bool = False
+
+    # Retry POST requests on 429 / connection errors. Default True keeps the
+    # historical behaviour; set False for providers where a replayed POST
+    # (e.g. after a read timeout) would duplicate a write.
+    retry_post: bool = True
+
     def get_request_kwargs(self, operation: str = "api") -> Dict[str, Any]:
         """
         Get kwargs for requests library call
@@ -278,8 +352,8 @@ class RequestConfig:
             headers["User-Agent"] = self.user_agent
         return headers
 
-    def to_dict(self) -> Dict[str, Any]:
-        """Convert to dictionary"""
+    def to_dict(self, redact: bool = False) -> Dict[str, Any]:
+        """Convert to dictionary (redact=True masks the proxy password)"""
         result = {
             "timeout": self.timeout,
             "verify_ssl": self.verify_ssl,
@@ -287,10 +361,17 @@ class RequestConfig:
             "retry_delay": self.retry_delay,
             "default_headers": self.default_headers,
             "user_agent": self.user_agent,
+            "accept": self.accept,
+            "accept_language": self.accept_language,
+            "accept_encoding": self.accept_encoding,
+            "inject_origin": self.inject_origin,
+            "inject_referer": self.inject_referer,
+            "retry_post": self.retry_post,
+            "use_tls_impersonation": self.use_tls_impersonation,
             "provider": self.provider,
         }
 
         if self.proxy_config:
-            result["proxy_config"] = self.proxy_config.to_dict()
+            result["proxy_config"] = self.proxy_config.to_dict(redact=redact)
 
         return result

lib/streaming_providers/base/network/http_manager.py

@@ -9,6 +9,7 @@ import requests
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
+from ..errors import parse_retry_after
 from ..models.proxy_models import ProxyConfig, RequestConfig
 from ..utils.logger import logger
 
@@ -16,6 +17,25 @@ from ..utils.logger import logger
 # This affects all providers using HTTPManager
 logging.getLogger("urllib3.connectionpool").setLevel(logging.WARNING)
 
+# Methods retried on 429 / connection errors. POST is added on top of these
+# when RequestConfig.retry_post is True (the default).
+_IDEMPOTENT_METHODS = ("HEAD", "GET", "OPTIONS", "PUT", "DELETE")
+
+# Upper bound for any single wait caused by a Retry-After header. The server's
+# full hint is still reported to callers (RateLimitError.retry_after); we just
+# refuse to block a worker thread for minutes inside the transport layer.
+_MAX_RETRY_AFTER_SECONDS = 30.0
+
+
+class _CappedRetry(Retry):
+    """urllib3 Retry that never sleeps longer than _MAX_RETRY_AFTER_SECONDS."""
+
+    def get_retry_after(self, response):
+        value = super().get_retry_after(response)
+        if value is None:
+            return None
+        return min(value, _MAX_RETRY_AFTER_SECONDS)
+
 
 class HTTPManager:
     """
@@ -29,6 +49,18 @@ class HTTPManager:
     - Automatic Referer and Origin injection
     - Provider-specific configurations
     - Cookie jar access for external persistence (e.g. Kodi VFS)
+
+    Error contract (identical on the requests and the curl_cffi backend):
+    - HTTP status >= 400 RAISES requests.exceptions.HTTPError with `.response`
+      set (an error response is never returned) -- unless the call passes
+      `raise_for_status=False`.
+    - Network failures raise requests.exceptions.ProxyError / Timeout /
+      SSLError / ConnectionError / RequestException (curl_cffi's own
+      exceptions are translated, they do not subclass the requests ones).
+    - 429 is retried up to `config.max_retries` (Retry-After honoured, each
+      wait capped); when retries are exhausted the last 429 is raised as a
+      normal HTTPError, never as RetryError.
+    - Map to typed provider errors with base.errors.error_from_exception().
     """
 
     def __init__(self, config: Optional[RequestConfig] = None):
@@ -67,7 +99,11 @@ class HTTPManager:
                     self._using_cffi = True
                     logger.debug(
                         f"{self.config.provider}: using curl_cffi Chrome TLS impersonation"
-                        + (f" via proxy {proxies.get('https')}" if proxies else "")
+                        + (
+                            f" via proxy {self.config.proxy_config.to_safe_url()}"
+                            if proxies
+                            else ""
+                        )
                     )
                 except ImportError:
                     logger.warning(
@@ -77,11 +113,18 @@ class HTTPManager:
 
         if not self._using_cffi:
             self._session = requests.Session()
-            retry_strategy = Retry(
+            retry_methods = list(_IDEMPOTENT_METHODS)
+            if self.config.retry_post:
+                retry_methods.append("POST")
+            # raise_on_status=False: once retries are exhausted the last 429
+            # is returned and surfaces as a normal HTTPError (with .response)
+            # instead of requests' RetryError (which has no response).
+            retry_strategy = _CappedRetry(
                 total=self.config.max_retries,
                 backoff_factor=self.config.retry_delay,
                 status_forcelist=[429],
-                allowed_methods=["HEAD", "GET", "OPTIONS", "POST", "PUT", "DELETE"],
+                allowed_methods=retry_methods,
+                raise_on_status=False,
             )
             adapter = HTTPAdapter(max_retries=retry_strategy)
             self._session.mount("http://", adapter)
@@ -110,20 +153,34 @@ class HTTPManager:
         Args:
             config: New request configuration
         """
-        # Snapshot cookies before tearing down the old session
-        old_cookies = self._session.cookies if self._session else None
-
         self.config = config
         self._last_url = None
+        self._rebuild_session()
+
+    def _rebuild_session(self) -> None:
+        """
+        Build a fresh session, carry the cookies over and close the old one.
+
+        requests_cookies.update() accepts a RequestsCookieJar directly; we
+        pass the jar (not a plain dict) so domain/path/expiry metadata is
+        preserved rather than flattened. The old session is closed after the
+        cookies were copied, otherwise its connection pool (or curl handle)
+        leaks on every proxy/config change.
+        """
+        old_session = self._session
+        old_cookies = old_session.cookies if old_session else None
+
         self._setup_session()
 
-        # Restore cookies into the new session.
-        # requests_cookies.update() accepts a RequestsCookieJar directly;
-        # we pass the jar (not a plain dict) so domain/path/expiry metadata
-        # is preserved rather than flattened.
         if old_cookies is not None:
             self._session.cookies.update(old_cookies)  # type: ignore[arg-type]
 
+        if old_session is not None:
+            try:
+                old_session.close()
+            except Exception as e:  # never let cleanup break a config change
+                logger.debug(f"{self.config.provider}: closing old session failed: {e}")
+
     def update_proxy(self, proxy_config: Optional[ProxyConfig]) -> None:
         """
         Update just the proxy configuration
@@ -143,10 +200,7 @@ class HTTPManager:
         self.config.proxy_config = proxy_config
 
         if self._using_cffi:
-            old_cookies = self._session.cookies if self._session else None
-            self._setup_session()
-            if old_cookies is not None:
-                self._session.cookies.update(old_cookies)  # type: ignore[arg-type]
+            self._rebuild_session()
 
     def reset_referer(self) -> None:
         """
@@ -168,14 +222,20 @@ class HTTPManager:
 
         Args:
             url: Request URL
-            operation: Operation type (api, auth, manifest, license) for proxy scoping
-            **kwargs: Additional arguments for requests
+            operation: Operation type for proxy scoping: "api", "auth",
+                "manifest" or "license" (see proxy_models.KNOWN_OPERATIONS).
+                Any other string is proxied whenever the proxy master switch
+                is on.
+            **kwargs: Additional arguments for requests. The extra keyword
+                `raise_for_status=False` returns 4xx/5xx responses instead of
+                raising (use it when you need to read an error body).
 
         Returns:
             requests.Response object
 
         Raises:
-            requests.exceptions.RequestException: On request failure
+            requests.exceptions.HTTPError: status >= 400 (`.response` is set)
+            requests.exceptions.RequestException: any other request failure
         """
         return self._make_request("GET", url, operation, **kwargs)
 
@@ -229,13 +289,21 @@ class HTTPManager:
         """
         logger.debug(f"{self.config.provider}: _using_cffi={self._using_cffi} for {method} {url}")
 
+        # Not a requests kwarg: controls whether status >= 400 raises
+        raise_for_status = kwargs.pop("raise_for_status", True)
+
         # Get base request configuration
         request_kwargs = self.config.get_request_kwargs(operation)
 
         # Merge with any additional kwargs (caller overrides win)
         request_kwargs.update(kwargs)
 
-        headers = request_kwargs.setdefault("headers", {})
+        # Work on a copy: the injections below must never mutate a headers
+        # dict the caller keeps and reuses across requests (a stale Origin
+        # from the first host would otherwise stick to every later call).
+        headers = dict(request_kwargs.get("headers") or {})
+        request_kwargs["headers"] = headers
+        header_names = {name.lower() for name in headers}
 
         # Inject Referer from the last response URL, but only when the caller
         # opts in via config.inject_referer (mirrors inject_origin below).
@@ -243,18 +311,13 @@ class HTTPManager:
         # that carry backend API URLs as the Referer. Providers that want
         # this should set inject_referer=True in their RequestConfig, or
         # continue setting Referer explicitly in their own header factories.
-        #
-        # NOTE: `inject_referer` isn't (yet) a declared field on RequestConfig
-        # in proxy_models.py — using getattr so this doesn't break if it's
-        # absent. Add `inject_referer: bool = False` to RequestConfig to make
-        # this a first-class, documented option.
-        if getattr(self.config, "inject_referer", False) and self._last_url and "Referer" not in headers:
+        if self.config.inject_referer and self._last_url and "referer" not in header_names:
             headers["Referer"] = self._last_url
 
         # Inject Origin derived from target URL only when explicitly requested.
         # Origin injection is opt-in because some non-browser APIs reject
         # requests that carry an unexpected Origin header.
-        if self.config.inject_origin and "Origin" not in headers:
+        if self.config.inject_origin and "origin" not in header_names:
             parsed = urllib.parse.urlparse(url)
             if parsed.scheme and parsed.netloc:
                 headers["Origin"] = f"{parsed.scheme}://{parsed.netloc}"
@@ -263,12 +326,7 @@ class HTTPManager:
         self._log_request(method, url, operation, request_kwargs)
 
         try:
-            # curl_cffi session has proxy baked in at construction — strip it from
-            # per-request kwargs to prevent conflicts / double-proxy application
-            if self._using_cffi:
-                request_kwargs.pop("proxies", None)
-
-            response = self._session.request(method, url, **request_kwargs)
+            response = self._send(method, url, request_kwargs)
 
             # Log redirect chain — cookies set on intermediate responses are
             # otherwise invisible, making auth flow debugging very painful
@@ -286,8 +344,13 @@ class HTTPManager:
             # Track final URL (after redirects) for next request's Referer
             self._last_url = response.url
 
-            # Raise for 4xx/5xx
-            response.raise_for_status()
+            # Raise for 4xx/5xx. Done here (not via response.raise_for_status())
+            # so both backends raise the *requests* HTTPError; curl_cffi's own
+            # HTTPError does not subclass it and would bypass every handler.
+            if raise_for_status and response.status_code >= 400:
+                raise requests.exceptions.HTTPError(
+                    self._http_error_message(response), response=response
+                )
 
             return response
 
@@ -311,17 +374,17 @@ class HTTPManager:
             raise
 
         except requests.exceptions.HTTPError as e:
-            status = e.response.status_code if e.response else "unknown"
+            # NB: never `if e.response:` -- Response.__bool__ is `.ok`, which
+            # is False for every 4xx/5xx, so the status used to log "unknown".
+            response = e.response
+            status = response.status_code if response is not None else "unknown"
             logger.error(
                 f"{self.config.provider}: HTTP {status} error for {operation} request to {url}: {e}"
             )
-            if e.response is not None and 400 <= e.response.status_code < 500:
-                try:
-                    body = e.response.json()
-                except Exception:
-                    body = e.response.text
+            if response is not None:
                 logger.debug(
-                    f"{self.config.provider}: {status} response body: {body}"
+                    f"{self.config.provider}: {status} response body: "
+                    f"{self._body_snippet(response)}"
                 )
             raise
 
@@ -331,6 +394,114 @@ class HTTPManager:
             )
             raise
 
+    def _send(self, method: str, url: str, request_kwargs: Dict[str, Any]):
+        """
+        Send one request on the active backend.
+
+        requests: retries (429) are done by the mounted urllib3 adapter.
+        curl_cffi: no adapter exists, so 429 retries are done here with the
+        same limits, and curl_cffi exceptions are translated to their
+        requests equivalents.
+        """
+        if not self._using_cffi:
+            return self._session.request(method, url, **request_kwargs)
+
+        # curl_cffi session has proxy baked in at construction — strip it from
+        # per-request kwargs to prevent conflicts / double-proxy application
+        request_kwargs.pop("proxies", None)
+
+        attempt = 0
+        while True:
+            try:
+                response = self._session.request(method, url, **request_kwargs)
+            except requests.exceptions.RequestException:
+                raise
+            except Exception as exc:
+                translated = self._translate_cffi_exception(exc)
+                if translated is exc:
+                    raise
+                raise translated from exc
+
+            if (
+                response.status_code != 429
+                or attempt >= self.config.max_retries
+                or not self._method_is_retryable(method)
+            ):
+                return response
+
+            delay = self._retry_delay_seconds(response, attempt)
+            logger.warning(
+                f"{self.config.provider}: HTTP 429 for {method} {url}; "
+                f"retry {attempt + 1}/{self.config.max_retries} in {delay:.1f}s"
+            )
+            time.sleep(delay)
+            attempt += 1
+
+    def _method_is_retryable(self, method: str) -> bool:
+        method = method.upper()
+        if method in _IDEMPOTENT_METHODS:
+            return True
+        return method == "POST" and self.config.retry_post
+
+    def _retry_delay_seconds(self, response, attempt: int) -> float:
+        """Retry-After if present, else exponential backoff; always capped."""
+        hinted = parse_retry_after(
+            response.headers.get("Retry-After"), max_seconds=_MAX_RETRY_AFTER_SECONDS
+        )
+        if hinted is not None:
+            return hinted
+        return min(self.config.retry_delay * (2 ** attempt), _MAX_RETRY_AFTER_SECONDS)
+
+    @staticmethod
+    def _translate_cffi_exception(exc: Exception) -> Exception:
+        """
+        Map a curl_cffi exception to the matching requests exception.
+
+        curl_cffi's exception classes do not inherit from requests', so
+        without this every `except requests.exceptions.*` (here and in the
+        providers) misses them. Unknown exceptions are returned unchanged.
+        """
+        try:
+            from curl_cffi import CurlError
+            from curl_cffi.requests import exceptions as cffi_exc
+        except ImportError:
+            return exc
+
+        # Most specific first; names are looked up defensively because the
+        # exact set differs between curl_cffi versions.
+        pairs = (
+            ("ProxyError", requests.exceptions.ProxyError),
+            ("SSLError", requests.exceptions.SSLError),
+            ("Timeout", requests.exceptions.Timeout),
+            ("ConnectionError", requests.exceptions.ConnectionError),
+            ("RequestException", requests.exceptions.RequestException),
+        )
+        for name, target in pairs:
+            source = getattr(cffi_exc, name, None)
+            if isinstance(source, type) and isinstance(exc, source):
+                return target(str(exc))
+        if isinstance(exc, CurlError):
+            return requests.exceptions.RequestException(str(exc))
+        return exc
+
+    @staticmethod
+    def _http_error_message(response) -> str:
+        status = response.status_code
+        kind = "Client" if status < 500 else "Server"
+        reason = getattr(response, "reason", "") or ""
+        return f"{status} {kind} Error: {reason} for url: {response.url}"
+
+    @staticmethod
+    def _body_snippet(response, limit: int = 500) -> str:
+        """Truncated response text for debug logs (never raises)."""
+        try:
+            text = response.text or ""
+        except Exception:
+            return "<unreadable body>"
+        if len(text) > limit:
+            return f"{text[:limit]}... [{len(text) - limit} more chars]"
+        return text
+
     # ------------------------------------------------------------------
     # Logging helpers
     # ------------------------------------------------------------------
@@ -370,21 +541,36 @@ class HTTPManager:
         Uses Content-Length header instead of buffering response.content,
         so this is safe to call before consuming streaming responses.
         """
+        # `elapsed` is a timedelta on requests but a plain float (seconds) on
+        # some curl_cffi versions; a malformed Content-Length must not break
+        # the request either -- this is only logging.
         elapsed = ""
-        if hasattr(response, "elapsed"):
-            elapsed = f" [{int(response.elapsed.total_seconds() * 1000)}ms]"
+        elapsed_attr = getattr(response, "elapsed", None)
+        if elapsed_attr is not None:
+            try:
+                seconds = (
+                    elapsed_attr.total_seconds()
+                    if hasattr(elapsed_attr, "total_seconds")
+                    else float(elapsed_attr)
+                )
+                elapsed = f" [{int(seconds * 1000)}ms]"
+            except (TypeError, ValueError):
+                pass
 
+        size_display = "unknown size"
         content_length = response.headers.get("Content-Length")
         if content_length is not None:
-            size = int(content_length)
-            if size > 1024 * 1024:
-                size_display = f"{size / (1024 * 1024):.2f} MB"
-            elif size > 1024:
-                size_display = f"{size / 1024:.2f} KB"
-            else:
-                size_display = f"{size} bytes"
-        else:
-            size_display = "unknown size"
+            try:
+                size = int(content_length)
+            except (TypeError, ValueError):
+                size = None
+            if size is not None:
+                if size > 1024 * 1024:
+                    size_display = f"{size / (1024 * 1024):.2f} MB"
+                elif size > 1024:
+                    size_display = f"{size / 1024:.2f} KB"
+                else:
+                    size_display = f"{size} bytes"
 
         content_type = response.headers.get("Content-Type", "unknown")
 
@@ -567,7 +753,7 @@ class HTTPManagerFactory:
             # Add more providers as needed
         }
 
-        defaults = provider_defaults.get(provider_name, {})
+        defaults = dict(provider_defaults.get(provider_name, {}))
         defaults.update(config_kwargs)
         defaults["provider"] = provider_name
 

lib/streaming_providers/providers/_template_v2/README.md

@@ -673,16 +673,22 @@ Id grammar guidance:
 | backend refused a write | raise `OperationFailedError` |
 | capability is not supported by design | `UnsupportedOperationError` (a `NotImplementedYetError`) |
 
-Names marked above beyond the ones in the shipped base were taken from
-docstrings and imports — **(unverified)**: check `base/errors.py` for the exact
-constructors (the templates assume `Error("message")`).
+**Verified against `base/errors.py` and `base/network/http_manager.py`** (the two
+standing hedges of the earlier migrations are closed):
 
-**Still open after three migrations.** Two standing hedges have never been verified:
-(a) the HTTP manager's behaviour on 4xx/5xx (raise vs. return — HRTi's `api.py` still
-handles both shapes), (b) the `base.errors` constructor kwargs (`url=` on `AuthError`
-is hedged in two places). The hedges work, but each migration copies them. Pin both
-against the real code before the next provider (§17) and replace the hedges with the
-verified behaviour.
+* **Constructors.** Every `ProviderError` subclass in the base takes
+  `Error(message, *, status=None, url=None, code=None)`; `Error("message")` works and
+  `url=` on `AuthError` is fine — delete the `url=` hedges in `api.py` and the
+  templates. `RateLimitError` additionally takes `retry_after=` (seconds, from the
+  `Retry-After` header; `None` if absent). Provider subclasses with their own
+  `__init__` keep working: `retry_after` is a class-level default and pickling/deepcopy
+  restores every attribute.
+* **`TransportError` is a `ServerError`** (connection-level failure, no HTTP status),
+  so "network down → `ServerError`" and "retry only `ServerError`" hold without
+  special-casing.
+* **HTTP manager: status ≥ 400 always raises** `requests.exceptions.HTTPError` with
+  `.response` set — on the `requests` **and** the `curl_cffi` backend. An error
+  response is never *returned* (see Helpers for the one opt-out).
 
 ### Compatibility on purpose
 
@@ -697,14 +703,23 @@ KeyError` / `except RuntimeError` in the operations layer keep working.
   exceptions in `ServerError` and lets typed `ProviderError`s through untouched
   (callers rely on `AuthError`/`RateLimitError` to refresh tokens or back off —
   never flatten them).
-* **HTTP manager on 400/401/403/5xx — until verified, handle both shapes.** What the
-  HTTP manager does for these statuses (return the error response vs. raise an
-  exception carrying `.response`) and whether `response.raise_for_status()` is
-  redundant is **not yet confirmed**. Write code that handles *both* a returned
-  error response and an exception with `.response`; use a single helper
-  `base.utils.transport.response_of(exc)` (to be shipped, §17) that returns the
-  response of either. Map 401/403 to `AuthError` at the one place you call login.
-  Replace this note with the verified behaviour after the first device run.
+* **HTTP manager on 400/401/403/429/5xx — it raises; handle one shape.**
+  `http_manager.get/post/put/delete` raise `requests.exceptions.HTTPError` whose
+  `.response` carries the status, headers and body; `response.raise_for_status()` after
+  the call is redundant. Network failures raise `ProxyError` / `Timeout` / `SSLError` /
+  `ConnectionError` (the `curl_cffi` backend's own exceptions are translated, they do
+  not subclass the `requests` ones). 429 is retried `max_retries` times (`Retry-After`
+  honoured, each wait capped at 30 s); when exhausted the last 429 arrives as a normal
+  `HTTPError`, never `RetryError`. To map to typed errors use
+  `base.errors.error_from_exception(exc)` (`ProviderError` → unchanged, response →
+  `default_error_for_status_heuristic` incl. `retry_after`, no response →
+  `TransportError`) and raise it `from exc`. Two pitfalls: **never test
+  `if exc.response:`** — a `requests.Response` is falsy for every 4xx/5xx; use
+  `is not None`. When you must read an error body to decide (a 400 whose JSON says what
+  went wrong), call with `raise_for_status=False` and inspect the returned response
+  instead of catching the exception. Map 401/403 to `AuthError` at the one place you
+  call login. `RequestConfig.retry_post=False` stops POSTs being replayed after a
+  timeout (default `True` = historical behaviour).
 * **Retry loops.** Retry only `ServerError` / untyped transport errors. Every other
   `ProviderError` (`AuthError`, `RateLimitError`, `GeoBlockError`,
   `EntitlementError`, …) propagates **immediately and unchanged**. Use
@@ -728,11 +743,14 @@ KeyError` / `except RuntimeError` in the operations layer keep working.
   endpoint is classified**. HRTi's EPG was `required=False`; a stale device id made it
   return `[]` — indistinguishable from "no programmes" — for however long nobody
   noticed. Test: a session-state failure on an optional endpoint raises.
-* **`operation=` on `http_manager.post/get`** selects the proxy/timeout scope (`"api"`,
-  `"auth"`, `"manifest"`, `"license"` in the template). v1 providers pass ad-hoc
-  names (`"live_manifest"`, `"token_validate"`) and omit it elsewhere (default).
-  The accepted values are **not verified**: keep v1's strings where v1 passed one;
-  use `"api"` where v1 passed nothing and note the change in §12.5.
+* **`operation=` on `http_manager.post/get`** selects the **proxy** scope only (it does
+  not change timeouts). Known values: `"api"`, `"auth"`, `"manifest"`, `"license"`
+  (`proxy_models.KNOWN_OPERATIONS`). **Any other string is accepted and is proxied
+  whenever the master switch `ProxyScope.all` is on, regardless of the per-operation
+  flags** — v1's ad-hoc names (`"live_manifest"`, `"token_validate"`) therefore keep
+  working but ignore a `manifests=False` / `authentication=False` setting. Where v1
+  passed nothing the default is `"api"`; where a v1 string means manifest/auth/license
+  traffic, prefer the known value and note the change in §12.5.
 
 ### Backend behaviours you must know
 
@@ -1564,7 +1582,9 @@ Known gaps in the base, so you don't hunt for them:
 
 **Tooling still to ship** (the README above already assumes it):
 
-* `base.utils.transport.retry_transient(fn, attempts, sleep)` and `response_of(exc)` (§8).
+* `base.utils.transport.retry_transient(fn, attempts, sleep)` (§8). `response_of(exc)`
+  is no longer needed: use `getattr(exc, "response", None)` with `is not None`, or
+  `base.errors.error_from_exception` (§8).
 * `check_provider(..., sample_ids={"channels": [...], "vod": [...]})`: for each sample
   id exactly one manager must return `True` from `handles_content_id`; verify the
   managers are instances of the ABCs (clear message instead of a late `TypeError`).
@@ -1576,9 +1596,9 @@ Known gaps in the base, so you don't hunt for them:
   matching code, (h) getter/setter pairs for every persisted key on the storage layer
   (§12.1b a), (i) credential/token logging (§12.1b b), (j) leftover `# ADAPT` comments
   (§12.3b).
-* **Pin the two standing (unverified) hedges** (§8): the HTTP manager's 4xx/5xx
-  behaviour and the `base.errors` constructor kwargs; then delete the hedges from
-  `api.py` and the templates.
+* **Delete the two former hedges** (§8, now verified): the "handle both HTTP shapes"
+  code and the `url=` fallbacks on `AuthError` in `api.py` and the templates (HRTi
+  first). Replace them with `error_from_exception`.
 * `optional/session_adapter.py` updated to the rules in §9 (including the recovery
   ladder and the time-window dedup).
 * `tests/conftest.py` with `FakeHttp`, `FakeAuthenticator`, fake clock, `env` (§15).
@@ -1599,9 +1619,9 @@ before it is merged. Two are implemented as patches (below); the rest are propos
   are provider-specific. Add it when a second provider needs the same rules.
 * One shared session adapter with `recover()` (the §9 ladder) and `on_invalidate` (backwards
   compatible with the template adapter's public methods) plus the shared `api_post`
-  helper (401 → one recovery → one retry, both HTTP-manager error shapes) — as a new
+  helper (401 → one recovery → one retry, one HTTP-manager error shape) — as a new
   module; no provider is forced to switch.
-* Document the accepted `operation=` values after reading `HTTPManager` (§8).
+* ~~Document the accepted `operation=` values~~ — done in §8.
 * `migration_pack`: placeholder-body detection (`pass  # ...`), output-inside-tree
   guard, `--bundle` that keeps relative paths, and rows for the actual constructor
   country vs. the declared one.