From dca437feae0fea2ac3ade3f2b06520cf24c0bccb Mon Sep 17 00:00:00 2001
From: monosans <hsyqixco@protonmail.com>
Date: Tue, 23 Jan 2024 09:47:43 +0300
Subject: [PATCH] Add raise_for_status=True when scraping sources

---
 proxy_scraper_checker/__main__.py |  1 +
 proxy_scraper_checker/geodb.py    |  4 +---
 proxy_scraper_checker/proxy.py    |  5 ++---
 proxy_scraper_checker/scraper.py  | 13 ++++++-------
 proxy_scraper_checker/settings.py |  4 +---
 5 files changed, 11 insertions(+), 16 deletions(-)

diff --git a/proxy_scraper_checker/__main__.py b/proxy_scraper_checker/__main__.py
index 8a3c449..3306590 100644
--- a/proxy_scraper_checker/__main__.py
+++ b/proxy_scraper_checker/__main__.py
@@ -101,6 +101,7 @@ async def main() -> None:
         connector=TCPConnector(ssl=http.SSL_CONTEXT),
         headers=http.HEADERS,
         cookie_jar=http.get_cookie_jar(),
+        raise_for_status=True,
         fallback_charset_resolver=http.fallback_charset_resolver,
     ) as session:
         settings = await Settings.from_mapping(cfg, session=session)
diff --git a/proxy_scraper_checker/geodb.py b/proxy_scraper_checker/geodb.py
index eedf361..8c8c0e1 100644
--- a/proxy_scraper_checker/geodb.py
+++ b/proxy_scraper_checker/geodb.py
@@ -52,9 +52,7 @@ async def download_geodb(*, progress: Progress, session: ClientSession) -> None:
         else None
     )
 
-    async with session.get(
-        GEODB_URL, headers=headers, raise_for_status=True
-    ) as response:
+    async with session.get(GEODB_URL, headers=headers) as response:
         if response.status == 304:  # noqa: PLR2004
             logger.info(
                 "Latest geolocation database is already cached at %s",
diff --git a/proxy_scraper_checker/proxy.py b/proxy_scraper_checker/proxy.py
index ee40c04..6da2f36 100644
--- a/proxy_scraper_checker/proxy.py
+++ b/proxy_scraper_checker/proxy.py
@@ -52,11 +52,10 @@ class Proxy:
                 connector=connector,
                 headers=HEADERS,
                 cookie_jar=get_cookie_jar(),
+                raise_for_status=True,
                 timeout=settings.timeout,
                 fallback_charset_resolver=fallback_charset_resolver,
-            ) as session, session.get(
-                settings.check_website, raise_for_status=True
-            ) as response:
+            ) as session, session.get(settings.check_website) as response:
                 content = await response.read()
         self.timeout = perf_counter() - start
         if settings.check_website_type == CheckWebsiteType.HTTPBIN_IP:
diff --git a/proxy_scraper_checker/scraper.py b/proxy_scraper_checker/scraper.py
index 7fb59e7..f0a79d5 100644
--- a/proxy_scraper_checker/scraper.py
+++ b/proxy_scraper_checker/scraper.py
@@ -7,7 +7,7 @@ import logging
 import aiofiles
 import aiofiles.os
 import aiofiles.ospath
-from aiohttp import ClientSession, ClientTimeout
+from aiohttp import ClientResponseError, ClientSession, ClientTimeout
 from aiohttp_socks import ProxyType
 from rich.progress import Progress, TaskID
 
@@ -41,6 +41,10 @@ async def scrape_one(
             async with aiofiles.open(source, "rb") as f:
                 content = await f.read()
             text = bytes_decode(content)
+    except ClientResponseError as e:
+        logger.warning(
+            "%s | HTTP status code %d: %s", source, e.status, e.message
+        )
     except Exception as e:
         logger.warning(
             "%s | %s.%s: %s",
@@ -54,12 +58,7 @@ async def scrape_one(
         try:
             proxy = next(proxies)
         except StopIteration:
-            if response and response.status != 200:  # noqa: PLR2004
-                logger.warning(
-                    "%s | HTTP status code %d", source, response.status
-                )
-            else:
-                logger.warning("%s | No proxies found", source)
+            logger.warning("%s | No proxies found", source)
         else:
             for proxy in itertools.chain((proxy,), proxies):  # noqa: B020
                 try:
diff --git a/proxy_scraper_checker/settings.py b/proxy_scraper_checker/settings.py
index 0a33541..ab5d6a4 100644
--- a/proxy_scraper_checker/settings.py
+++ b/proxy_scraper_checker/settings.py
@@ -129,9 +129,7 @@ async def _get_check_website_type_and_real_ip(
     Tuple[Literal[CheckWebsiteType.PLAIN_IP, CheckWebsiteType.HTTPBIN_IP], str],
 ]:
     try:
-        async with session.get(
-            check_website, raise_for_status=True
-        ) as response:
+        async with session.get(check_website) as response:
             content = await response.read()
         text = get_response_text(response=response, content=content)
     except Exception: