Support higher MaxConnections & fix Windows errors

Fixes #4, fixes #8, fixes #11, fixes #28, fixes #34, fixes #36, fixes #37
This commit is contained in:
monosans
2023-01-05 18:45:06 +03:00
parent 3670096f26
commit 3abbb7983f
3 changed files with 33 additions and 6 deletions
+5 -3
View File
@@ -2,11 +2,13 @@
; How many seconds to wait for the proxy to make a connection.
; The higher the number, the longer the check will take
; and the more proxies you get.
Timeout = 10
Timeout = 5
; Maximum concurrent connections.
; Don't set higher than 900, please.
MaxConnections = 900
; Windows supports maximum of 512.
; On *nix operating systems, this restriction is much looser.
; The limit on *nix can be seen with the command ulimit -Hn.
MaxConnections = 512
; Set to no to sort proxies alphabetically.
SortBySpeed = yes
+5 -3
View File
@@ -11,8 +11,10 @@ from rich.logging import RichHandler
from .proxy_scraper_checker import ProxyScraperChecker
def install_uvloop() -> None:
if sys.implementation.name == "cpython" and sys.platform in {
def set_event_loop_policy() -> None:
if sys.platform == "win32":
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
elif sys.implementation.name == "cpython" and sys.platform in {
"darwin",
"linux",
}:
@@ -42,7 +44,7 @@ def configure_logging(console: Console) -> None:
def main() -> None:
install_uvloop()
set_event_loop_policy()
console = Console()
configure_logging(console)
@@ -3,6 +3,8 @@ from __future__ import annotations
import asyncio
import logging
import re
import resource
import sys
from configparser import ConfigParser
from pathlib import Path
from random import shuffle
@@ -40,6 +42,25 @@ TProxyScraperChecker = TypeVar(
)
def validate_max_connections(value: int) -> int:
if sys.platform != "win32":
soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
if soft_limit < hard_limit:
resource.setrlimit(
resource.RLIMIT_NOFILE, (hard_limit, hard_limit)
)
elif value > 512 and isinstance(
asyncio.get_event_loop_policy(), asyncio.WindowsSelectorEventLoopPolicy
):
logger.warning(
"MaxConnections value is too high. "
+ "Windows supports a maximum of 512. "
+ "The config value will be ignored and 512 will be used."
)
return 512
return value
class ProxyScraperChecker:
"""HTTP, SOCKS4, SOCKS5 proxies scraper and checker."""
@@ -135,6 +156,8 @@ class ProxyScraperChecker:
self.proxies_count = {proto: 0 for proto in self.sources}
self.cookie_jar = DummyCookieJar()
self.console = console or Console()
max_connections = validate_max_connections(max_connections)
self.sem = asyncio.Semaphore(max_connections)
@classmethod