Files
proxy-scraper-checker/proxy_scraper_checker/utils.py
T

18 lines
403 B
Python

from __future__ import annotations
import os
from urllib.parse import urlparse
import charset_normalizer
IS_DOCKER = os.getenv("IS_DOCKER") == "1"
def is_http_url(value: str, /) -> bool:
parsed_url = urlparse(value)
return bool(parsed_url.scheme in {"http", "https"} and parsed_url.netloc)
def bytes_decode(value: bytes, /) -> str:
return str(charset_normalizer.from_bytes(value)[0])