diff --git a/proxy_scraper_checker/__main__.py b/proxy_scraper_checker/__main__.py index 11f3a94..364d547 100644 --- a/proxy_scraper_checker/__main__.py +++ b/proxy_scraper_checker/__main__.py @@ -103,7 +103,6 @@ async def main() -> None: connector=TCPConnector(ssl=http.SSL_CONTEXT), headers=http.HEADERS, cookie_jar=http.get_cookie_jar(), - fallback_charset_resolver=http.fallback_charset_resolver, ) as session: settings = await Settings.from_mapping(cfg, session=session) with Progress( diff --git a/proxy_scraper_checker/http.py b/proxy_scraper_checker/http.py index b2818b4..0e1402d 100644 --- a/proxy_scraper_checker/http.py +++ b/proxy_scraper_checker/http.py @@ -5,8 +5,7 @@ from functools import lru_cache from types import MappingProxyType import certifi -import charset_normalizer -from aiohttp import ClientResponse, DummyCookieJar, hdrs +from aiohttp import DummyCookieJar, hdrs SSL_CONTEXT = ssl.create_default_context(cafile=certifi.where()) HEADERS: MappingProxyType[str, str] = MappingProxyType({ @@ -19,7 +18,3 @@ HEADERS: MappingProxyType[str, str] = MappingProxyType({ @lru_cache(None) def get_cookie_jar() -> DummyCookieJar: return DummyCookieJar() - - -def fallback_charset_resolver(r: ClientResponse, b: bytes) -> str: # noqa: ARG001 - return charset_normalizer.from_bytes(b)[0].encoding diff --git a/proxy_scraper_checker/proxy.py b/proxy_scraper_checker/proxy.py index 18569ab..a01f749 100644 --- a/proxy_scraper_checker/proxy.py +++ b/proxy_scraper_checker/proxy.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from io import StringIO from time import perf_counter from typing import Optional @@ -8,14 +9,10 @@ import attrs from aiohttp import ClientSession from aiohttp_socks import ProxyConnector, ProxyType -from .http import ( - HEADERS, - SSL_CONTEXT, - fallback_charset_resolver, - get_cookie_jar, -) +from .http import HEADERS, SSL_CONTEXT, get_cookie_jar from .parsers import parse_ipv4 from .settings import CheckWebsiteType, Settings +from .utils import bytes_decode @attrs.define( @@ -52,17 +49,16 @@ class Proxy: headers=HEADERS, cookie_jar=get_cookie_jar(), timeout=settings.timeout, - fallback_charset_resolver=fallback_charset_resolver, ) as session, session.get( settings.check_website, raise_for_status=True ) as response: - await response.read() + content = await response.read() self.timeout = perf_counter() - start if settings.check_website_type == CheckWebsiteType.HTTPBIN_IP: - r = await response.json(content_type=None) + r = json.loads(bytes_decode(content)) self.exit_ip = r["origin"] elif settings.check_website_type == CheckWebsiteType.PLAIN_IP: - self.exit_ip = parse_ipv4(await response.text()) + self.exit_ip = parse_ipv4(bytes_decode(content)) def as_str(self, *, include_protocol: bool) -> str: with StringIO() as buf: diff --git a/proxy_scraper_checker/scraper.py b/proxy_scraper_checker/scraper.py index f5f798c..599879d 100644 --- a/proxy_scraper_checker/scraper.py +++ b/proxy_scraper_checker/scraper.py @@ -33,13 +33,12 @@ async def scrape_one( try: if is_url(source): async with session.get(source, timeout=timeout) as response: - await response.read() - text = await response.text() + content = await response.read() else: response = None async with aiofiles.open(source, "rb") as f: content = await f.read() - text = bytes_decode(content) + text = bytes_decode(content) except Exception as e: logger.warning( "%s | %s.%s: %s", diff --git a/proxy_scraper_checker/settings.py b/proxy_scraper_checker/settings.py index ef4c918..fd8e4b2 100644 --- a/proxy_scraper_checker/settings.py +++ b/proxy_scraper_checker/settings.py @@ -2,10 +2,10 @@ from __future__ import annotations import asyncio import enum +import json import logging import math import sys -from json import JSONDecodeError from pathlib import Path from typing import ( TYPE_CHECKING, @@ -28,6 +28,7 @@ from . import sort from .null_context import NullContext from .parsers import parse_ipv4 from .typing_compat import Any, Literal, Self +from .utils import bytes_decode if TYPE_CHECKING: from .proxy import Proxy @@ -129,7 +130,8 @@ async def _get_check_website_type_and_real_ip( ]: try: async with session.get(check_website, raise_for_status=True) as r: - await r.read() + content = await r.read() + text = bytes_decode(content) except Exception: logger.exception( "Error when opening check_website without proxy, it will be " @@ -137,15 +139,15 @@ async def _get_check_website_type_and_real_ip( ) return CheckWebsiteType.UNKNOWN, None try: - content = await r.json(content_type=None) - except JSONDecodeError: + js = json.loads(text) + except json.JSONDecodeError: try: - return CheckWebsiteType.PLAIN_IP, parse_ipv4(await r.text()) + return CheckWebsiteType.PLAIN_IP, parse_ipv4(text) except ValueError: pass else: try: - return CheckWebsiteType.HTTPBIN_IP, parse_ipv4(content["origin"]) + return CheckWebsiteType.HTTPBIN_IP, parse_ipv4(js["origin"]) except (KeyError, TypeError, ValueError): pass logger.warning(