fix minor network bugs

This commit is contained in:
Nirvana
2026-07-20 11:08:36 +02:00
parent 4ae9782c39
commit df724d33fe
2 changed files with 37 additions and 10 deletions
@@ -128,11 +128,26 @@ class HTTPManager:
"""
Update just the proxy configuration
For plain `requests` sessions the proxy is resolved per-request from
self.config.proxy_config (via get_request_kwargs), so updating the
config alone is sufficient.
For curl_cffi sessions the proxy is baked into the Session object at
construction time (see _setup_session), so simply updating the config
would silently leave the session using the old proxy. In that case we
rebuild the session, preserving cookies across the rebuild.
Args:
proxy_config: New proxy configuration (None to disable proxy)
"""
self.config.proxy_config = proxy_config
if self._using_cffi:
old_cookies = self._session.cookies if self._session else None
self._setup_session()
if old_cookies is not None:
self._session.cookies.update(old_cookies) # type: ignore[arg-type]
def reset_referer(self) -> None:
"""
Reset the tracked referer URL.
@@ -220,12 +235,22 @@ class HTTPManager:
# Merge with any additional kwargs (caller overrides win)
request_kwargs.update(kwargs)
# Inject Referer from last response URL if not already set by caller.
# NOTE: Auto-injecting Referer is disabled by default because WAFs (like Akamai)
# flag API requests that have backend API URLs as the Referer.
# Providers should explicitly set the Referer header in their own header factories if needed.
headers = request_kwargs.setdefault("headers", {})
# Inject Referer from the last response URL, but only when the caller
# opts in via config.inject_referer (mirrors inject_origin below).
# Disabled by default because WAFs (like Akamai) flag API requests
# that carry backend API URLs as the Referer. Providers that want
# this should set inject_referer=True in their RequestConfig, or
# continue setting Referer explicitly in their own header factories.
#
# NOTE: `inject_referer` isn't (yet) a declared field on RequestConfig
# in proxy_models.py — using getattr so this doesn't break if it's
# absent. Add `inject_referer: bool = False` to RequestConfig to make
# this a first-class, documented option.
if getattr(self.config, "inject_referer", False) and self._last_url and "Referer" not in headers:
headers["Referer"] = self._last_url
# Inject Origin derived from target URL only when explicitly requested.
# Origin injection is opt-in because some non-browser APIs reject
# requests that carry an unexpected Origin header.
@@ -1,5 +1,4 @@
# streaming_providers/base/network/proxy_manager.py
import json
import time
from pathlib import Path
from typing import Any, Dict, List, Optional
@@ -436,8 +435,9 @@ class ProxyConfigManager:
"providers": providers_export,
}
with open(export_path, "w", encoding="utf-8") as f:
json.dump(export_data, f, indent=2, ensure_ascii=False)
success = self.vfs.write_json(export_path, export_data)
if not success:
raise IOError(f"VFS write_json failed for {export_path}")
logger.info(f"Exported proxy configurations to {export_path}")
return export_path
@@ -458,8 +458,10 @@ class ProxyConfigManager:
True if successful, False otherwise
"""
try:
with open(import_path, "r", encoding="utf-8") as f:
import_data = json.load(f)
import_data = self.vfs.read_json(import_path)
if import_data is None:
logger.error(f"No data found at {import_path}")
return False
if not merge:
# Clear existing configurations
@@ -575,4 +577,4 @@ class ProxyConfigManager:
# Non-country key
info[cache_key] = self.get_proxy_info(cache_key)
return info
return info