Fix encoding issues

proxy_scraper_checker/__main__.py

@@ -103,7 +103,6 @@ async def main() -> None:
         connector=TCPConnector(ssl=http.SSL_CONTEXT),
         headers=http.HEADERS,
         cookie_jar=http.get_cookie_jar(),
-        fallback_charset_resolver=http.fallback_charset_resolver,
     ) as session:
         settings = await Settings.from_mapping(cfg, session=session)
         with Progress(

proxy_scraper_checker/http.py

@@ -5,8 +5,7 @@ from functools import lru_cache
 from types import MappingProxyType
 
 import certifi
-import charset_normalizer
-from aiohttp import ClientResponse, DummyCookieJar, hdrs
+from aiohttp import DummyCookieJar, hdrs
 
 SSL_CONTEXT = ssl.create_default_context(cafile=certifi.where())
 HEADERS: MappingProxyType[str, str] = MappingProxyType({
@@ -19,7 +18,3 @@ HEADERS: MappingProxyType[str, str] = MappingProxyType({
 @lru_cache(None)
 def get_cookie_jar() -> DummyCookieJar:
     return DummyCookieJar()
-
-
-def fallback_charset_resolver(r: ClientResponse, b: bytes) -> str:  # noqa: ARG001
-    return charset_normalizer.from_bytes(b)[0].encoding

proxy_scraper_checker/proxy.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 from io import StringIO
 from time import perf_counter
 from typing import Optional
@@ -8,14 +9,10 @@ import attrs
 from aiohttp import ClientSession
 from aiohttp_socks import ProxyConnector, ProxyType
 
-from .http import (
-    HEADERS,
-    SSL_CONTEXT,
-    fallback_charset_resolver,
-    get_cookie_jar,
-)
+from .http import HEADERS, SSL_CONTEXT, get_cookie_jar
 from .parsers import parse_ipv4
 from .settings import CheckWebsiteType, Settings
+from .utils import bytes_decode
 
 
 @attrs.define(
@@ -52,17 +49,16 @@ class Proxy:
                 headers=HEADERS,
                 cookie_jar=get_cookie_jar(),
                 timeout=settings.timeout,
-                fallback_charset_resolver=fallback_charset_resolver,
             ) as session, session.get(
                 settings.check_website, raise_for_status=True
             ) as response:
-                await response.read()
+                content = await response.read()
         self.timeout = perf_counter() - start
         if settings.check_website_type == CheckWebsiteType.HTTPBIN_IP:
-            r = await response.json(content_type=None)
+            r = json.loads(bytes_decode(content))
             self.exit_ip = r["origin"]
         elif settings.check_website_type == CheckWebsiteType.PLAIN_IP:
-            self.exit_ip = parse_ipv4(await response.text())
+            self.exit_ip = parse_ipv4(bytes_decode(content))
 
     def as_str(self, *, include_protocol: bool) -> str:
         with StringIO() as buf:

proxy_scraper_checker/scraper.py

@@ -33,13 +33,12 @@ async def scrape_one(
     try:
         if is_url(source):
             async with session.get(source, timeout=timeout) as response:
-                await response.read()
-            text = await response.text()
+                content = await response.read()
         else:
             response = None
             async with aiofiles.open(source, "rb") as f:
                 content = await f.read()
-            text = bytes_decode(content)
+        text = bytes_decode(content)
     except Exception as e:
         logger.warning(
             "%s | %s.%s: %s",

proxy_scraper_checker/settings.py

@@ -2,10 +2,10 @@ from __future__ import annotations
 
 import asyncio
 import enum
+import json
 import logging
 import math
 import sys
-from json import JSONDecodeError
 from pathlib import Path
 from typing import (
     TYPE_CHECKING,
@@ -28,6 +28,7 @@ from . import sort
 from .null_context import NullContext
 from .parsers import parse_ipv4
 from .typing_compat import Any, Literal, Self
+from .utils import bytes_decode
 
 if TYPE_CHECKING:
     from .proxy import Proxy
@@ -129,7 +130,8 @@ async def _get_check_website_type_and_real_ip(
 ]:
     try:
         async with session.get(check_website, raise_for_status=True) as r:
-            await r.read()
+            content = await r.read()
+        text = bytes_decode(content)
     except Exception:
         logger.exception(
             "Error when opening check_website without proxy, it will be "
@@ -137,15 +139,15 @@ async def _get_check_website_type_and_real_ip(
         )
         return CheckWebsiteType.UNKNOWN, None
     try:
-        content = await r.json(content_type=None)
-    except JSONDecodeError:
+        js = json.loads(text)
+    except json.JSONDecodeError:
         try:
-            return CheckWebsiteType.PLAIN_IP, parse_ipv4(await r.text())
+            return CheckWebsiteType.PLAIN_IP, parse_ipv4(text)
         except ValueError:
             pass
     else:
         try:
-            return CheckWebsiteType.HTTPBIN_IP, parse_ipv4(content["origin"])
+            return CheckWebsiteType.HTTPBIN_IP, parse_ipv4(js["origin"])
         except (KeyError, TypeError, ValueError):
             pass
     logger.warning(