Support higher MaxConnections & fix Windows errors
Fixes #4, fixes #8, fixes #11, fixes #28, fixes #34, fixes #36, fixes #37
This commit is contained in:
+5
-3
@@ -2,11 +2,13 @@
|
||||
; How many seconds to wait for the proxy to make a connection.
|
||||
; The higher the number, the longer the check will take
|
||||
; and the more proxies you get.
|
||||
Timeout = 10
|
||||
Timeout = 5
|
||||
|
||||
; Maximum concurrent connections.
|
||||
; Don't set higher than 900, please.
|
||||
MaxConnections = 900
|
||||
; Windows supports maximum of 512.
|
||||
; On *nix operating systems, this restriction is much looser.
|
||||
; The limit on *nix can be seen with the command ulimit -Hn.
|
||||
MaxConnections = 512
|
||||
|
||||
; Set to no to sort proxies alphabetically.
|
||||
SortBySpeed = yes
|
||||
|
||||
@@ -11,8 +11,10 @@ from rich.logging import RichHandler
|
||||
from .proxy_scraper_checker import ProxyScraperChecker
|
||||
|
||||
|
||||
def install_uvloop() -> None:
|
||||
if sys.implementation.name == "cpython" and sys.platform in {
|
||||
def set_event_loop_policy() -> None:
|
||||
if sys.platform == "win32":
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
||||
elif sys.implementation.name == "cpython" and sys.platform in {
|
||||
"darwin",
|
||||
"linux",
|
||||
}:
|
||||
@@ -42,7 +44,7 @@ def configure_logging(console: Console) -> None:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
install_uvloop()
|
||||
set_event_loop_policy()
|
||||
|
||||
console = Console()
|
||||
configure_logging(console)
|
||||
|
||||
@@ -3,6 +3,8 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import resource
|
||||
import sys
|
||||
from configparser import ConfigParser
|
||||
from pathlib import Path
|
||||
from random import shuffle
|
||||
@@ -40,6 +42,25 @@ TProxyScraperChecker = TypeVar(
|
||||
)
|
||||
|
||||
|
||||
def validate_max_connections(value: int) -> int:
|
||||
if sys.platform != "win32":
|
||||
soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
|
||||
if soft_limit < hard_limit:
|
||||
resource.setrlimit(
|
||||
resource.RLIMIT_NOFILE, (hard_limit, hard_limit)
|
||||
)
|
||||
elif value > 512 and isinstance(
|
||||
asyncio.get_event_loop_policy(), asyncio.WindowsSelectorEventLoopPolicy
|
||||
):
|
||||
logger.warning(
|
||||
"MaxConnections value is too high. "
|
||||
+ "Windows supports a maximum of 512. "
|
||||
+ "The config value will be ignored and 512 will be used."
|
||||
)
|
||||
return 512
|
||||
return value
|
||||
|
||||
|
||||
class ProxyScraperChecker:
|
||||
"""HTTP, SOCKS4, SOCKS5 proxies scraper and checker."""
|
||||
|
||||
@@ -135,6 +156,8 @@ class ProxyScraperChecker:
|
||||
self.proxies_count = {proto: 0 for proto in self.sources}
|
||||
self.cookie_jar = DummyCookieJar()
|
||||
self.console = console or Console()
|
||||
|
||||
max_connections = validate_max_connections(max_connections)
|
||||
self.sem = asyncio.Semaphore(max_connections)
|
||||
|
||||
@classmethod
|
||||
|
||||
Reference in New Issue
Block a user