From 906a514e101a99efb1d6cf734a408d62ddda42a0 Mon Sep 17 00:00:00 2001 From: monosans Date: Sat, 7 Jan 2023 11:57:57 +0300 Subject: [PATCH] Fix timeouts higher than 60 being ignored --- proxy_scraper_checker/proxy.py | 7 +++---- proxy_scraper_checker/proxy_scraper_checker.py | 10 ++++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/proxy_scraper_checker/proxy.py b/proxy_scraper_checker/proxy.py index 6b65a85..0e6ca32 100644 --- a/proxy_scraper_checker/proxy.py +++ b/proxy_scraper_checker/proxy.py @@ -3,7 +3,7 @@ from __future__ import annotations import asyncio from time import perf_counter -from aiohttp import ClientSession +from aiohttp import ClientSession, ClientTimeout from aiohttp.abc import AbstractCookieJar from aiohttp_socks import ProxyConnector, ProxyType @@ -21,17 +21,16 @@ class Proxy: sem: asyncio.Semaphore, cookie_jar: AbstractCookieJar, proto: str, - timeout: float, + timeout: ClientTimeout, ) -> None: async with sem: start = perf_counter() async with self.get_connector(proto) as connector: async with ClientSession( - connector=connector, cookie_jar=cookie_jar + connector=connector, cookie_jar=cookie_jar, timeout=timeout ) as session: async with session.get( "http://ip-api.com/json/?fields=8217", - timeout=timeout, raise_for_status=True, ) as response: data = await response.json() diff --git a/proxy_scraper_checker/proxy_scraper_checker.py b/proxy_scraper_checker/proxy_scraper_checker.py index 8dc0023..a88041e 100644 --- a/proxy_scraper_checker/proxy_scraper_checker.py +++ b/proxy_scraper_checker/proxy_scraper_checker.py @@ -19,7 +19,7 @@ from typing import ( Union, ) -from aiohttp import ClientSession, DummyCookieJar +from aiohttp import ClientSession, ClientTimeout, DummyCookieJar from rich.console import Console from rich.progress import ( BarColumn, @@ -143,7 +143,7 @@ class ProxyScraperChecker: ) self.sort_by_speed = sort_by_speed - self.timeout = timeout + self.timeout = ClientTimeout(total=timeout, sock_connect=timeout) self.sources = { proto: frozenset(filter(None, sources.splitlines())) for proto, sources in ( @@ -218,7 +218,7 @@ class ProxyScraperChecker: proto: HTTP/SOCKS4/SOCKS5. """ try: - async with session.get(source, timeout=15) as response: + async with session.get(source) as response: status = response.status text = await response.text() except Exception as e: @@ -272,7 +272,9 @@ class ProxyScraperChecker: ) } async with ClientSession( - headers=headers, cookie_jar=self.cookie_jar + headers=headers, + cookie_jar=self.cookie_jar, + timeout=ClientTimeout(total=15), ) as session: coroutines = ( self.fetch_source(