Files
proxy-scraper-checker/proxy_scraper_checker/validators.py
T
monosans 082e12b764 Many changes (see full commit message)
Add ability to set MaxConnections to 0, at which this parameter will be set to the maximum available value for the OS
Add validation for Timeout and SourceTimeout options
Improve existing validators
Improve error and warning output
Misc refactoring
2023-02-19 22:18:30 +03:00

78 lines
2.1 KiB
Python

from __future__ import annotations
import asyncio
import logging
import sys
from typing import Any, Iterable, Optional
from urllib.parse import urlparse
from .folder import Folder
logger = logging.getLogger(__name__)
def timeout(value: float) -> None:
if value <= 0:
msg = "Timeout must be positive"
raise ValueError(msg)
def source_timeout(value: float) -> None:
if value <= 0:
msg = "SourceTimeout must be positive"
raise ValueError(msg)
def max_connections(value: int) -> Optional[int]:
if value < 0:
msg = "MaxConnections must be non-negative"
raise ValueError(msg)
max_supported = _get_supported_max_connections()
if not value:
logger.info("Using %d as MaxConnections value", max_supported or 0)
return max_supported
if not max_supported or value <= max_supported:
return value
logger.warning(
"MaxConnections value is too high. "
+ "Your OS supports a maximum of %d. "
+ "The config value will be ignored and %d will be used.",
max_supported,
max_supported,
)
return max_supported
def _get_supported_max_connections() -> Optional[int]:
if sys.platform == "win32":
if isinstance(
asyncio.get_event_loop_policy(),
asyncio.WindowsSelectorEventLoopPolicy,
):
return 512
return None
import resource
soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
if soft_limit < hard_limit:
resource.setrlimit(resource.RLIMIT_NOFILE, (hard_limit, hard_limit))
return hard_limit
def check_website(value: str) -> None:
parsed_url = urlparse(value)
if not parsed_url.scheme or not parsed_url.netloc:
msg = f"invalid CheckWebsite URL: {value}"
raise ValueError(msg)
def folders(value: Iterable[Folder]) -> None:
if not any(folder for folder in value if folder.is_enabled):
msg = "all folders are disabled in the config"
raise ValueError(msg)
def sources(value: Any) -> None:
if not value:
raise ValueError("proxy sources list is empty")