Add raise_for_status=True when scraping sources

This commit is contained in:
monosans
2024-01-23 09:47:43 +03:00
parent 73303cab1b
commit dca437feae
5 changed files with 11 additions and 16 deletions
+6 -7
View File
@@ -7,7 +7,7 @@ import logging
import aiofiles
import aiofiles.os
import aiofiles.ospath
from aiohttp import ClientSession, ClientTimeout
from aiohttp import ClientResponseError, ClientSession, ClientTimeout
from aiohttp_socks import ProxyType
from rich.progress import Progress, TaskID
@@ -41,6 +41,10 @@ async def scrape_one(
async with aiofiles.open(source, "rb") as f:
content = await f.read()
text = bytes_decode(content)
except ClientResponseError as e:
logger.warning(
"%s | HTTP status code %d: %s", source, e.status, e.message
)
except Exception as e:
logger.warning(
"%s | %s.%s: %s",
@@ -54,12 +58,7 @@ async def scrape_one(
try:
proxy = next(proxies)
except StopIteration:
if response and response.status != 200: # noqa: PLR2004
logger.warning(
"%s | HTTP status code %d", source, response.status
)
else:
logger.warning("%s | No proxies found", source)
logger.warning("%s | No proxies found", source)
else:
for proxy in itertools.chain((proxy,), proxies): # noqa: B020
try: