diff --git a/proxy_scraper_checker/__main__.py b/proxy_scraper_checker/__main__.py index 53d271b..ab94ae7 100644 --- a/proxy_scraper_checker/__main__.py +++ b/proxy_scraper_checker/__main__.py @@ -1,15 +1,17 @@ +# ruff: noqa: E402 from __future__ import annotations +from . import logs + +_console, _logs_listener = logs.configure() + import asyncio import logging import sys from typing import TYPE_CHECKING import aiofiles -import rich.traceback from aiohttp import ClientSession, TCPConnector -from rich.console import Console -from rich.logging import RichHandler from rich.progress import BarColumn, MofNCompleteColumn, Progress, TextColumn from rich.table import Table @@ -36,7 +38,7 @@ if TYPE_CHECKING: T = TypeVar("T") -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) def get_async_run() -> Callable[[Coroutine[Any, Any, T]], T]: @@ -73,27 +75,6 @@ async def read_config(file: str, /) -> dict[str, Any]: return tomllib.loads(utils.bytes_decode(content)) -def configure_logging(*, console: Console, debug: bool) -> None: - rich.traceback.install( - console=console, width=None, extra_lines=0, word_wrap=True - ) - logging.basicConfig( - format="%(message)s", - datefmt=logging.Formatter.default_time_format, - level=logging.DEBUG if debug else logging.INFO, - handlers=( - RichHandler( - console=console, - omit_repeated_times=False, - show_path=False, - rich_tracebacks=True, - tracebacks_extra_lines=0, - ), - ), - force=True, - ) - - def get_summary_table( *, before: Mapping[ProxyType, int], after: Mapping[ProxyType, int] ) -> Table: @@ -113,8 +94,8 @@ def get_summary_table( async def main() -> None: cfg = await read_config("config.toml") - console = Console() - configure_logging(console=console, debug=cfg["debug"]) + if cfg["debug"]: + logging.root.setLevel(logging.DEBUG) should_save = False try: async with ClientSession( @@ -132,7 +113,7 @@ async def main() -> None: TextColumn("[green]{task.fields[col2]}"), BarColumn(), MofNCompleteColumn(), - console=console, + console=_console, transient=True, ) as progress: scrape = scraper.scrape_all( @@ -166,7 +147,7 @@ async def main() -> None: if settings.check_website: storage.remove_unchecked() count_after_checking = storage.get_count() - console.print( + _console.print( get_summary_table( before=count_before_checking, after=count_after_checking ) @@ -176,10 +157,14 @@ async def main() -> None: output.save_proxies, storage=storage, settings=settings ) - logger.info( + _logger.info( "Thank you for using https://github.com/monosans/proxy-scraper-checker" ) if __name__ == "__main__": - get_async_run()(main()) + _logs_listener.start() + try: + get_async_run()(main()) + finally: + _logs_listener.stop() diff --git a/proxy_scraper_checker/checker.py b/proxy_scraper_checker/checker.py index e628c06..3067ec8 100644 --- a/proxy_scraper_checker/checker.py +++ b/proxy_scraper_checker/checker.py @@ -16,7 +16,7 @@ if TYPE_CHECKING: from .settings import Settings from .storage import ProxyStorage -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) async def check_one( @@ -32,9 +32,9 @@ async def check_one( except Exception as e: # Too many open files if isinstance(e, OSError) and e.errno == 24: # noqa: PLR2004 - logger.error("Please, set max_connections to lower value") + _logger.error("Please, set max_connections to lower value") - logger.debug( + _logger.debug( "%s.%s: %s", e.__class__.__module__, e.__class__.__qualname__, e ) storage.remove(proxy) diff --git a/proxy_scraper_checker/fs.py b/proxy_scraper_checker/fs.py index ec556ab..cf83b1b 100644 --- a/proxy_scraper_checker/fs.py +++ b/proxy_scraper_checker/fs.py @@ -8,7 +8,7 @@ import platformdirs if TYPE_CHECKING: from pathlib import Path -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) CACHE_PATH = platformdirs.user_cache_path("proxy_scraper_checker") @@ -20,7 +20,7 @@ def add_permission( new_permissions = current_permissions | permission if current_permissions != new_permissions: path.chmod(new_permissions) - logger.info( + _logger.info( "Changed permissions of %s from %o to %o", path, current_permissions, diff --git a/proxy_scraper_checker/geodb.py b/proxy_scraper_checker/geodb.py index c76ec5f..3eefc98 100644 --- a/proxy_scraper_checker/geodb.py +++ b/proxy_scraper_checker/geodb.py @@ -15,7 +15,7 @@ if TYPE_CHECKING: from aiohttp import ClientResponse, ClientSession from rich.progress import Progress, TaskID -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) GEODB_URL = "https://raw.githubusercontent.com/P3TERX/GeoLite.mmdb/download/GeoLite2-City.mmdb" GEODB_PATH = fs.CACHE_PATH / "geolocation_database.mmdb" @@ -70,7 +70,7 @@ async def download_geodb(*, progress: Progress, session: ClientSession) -> None: async with session.get(GEODB_URL, headers=headers) as response: if response.status == 304: # noqa: PLR2004 - logger.info( + _logger.info( "Latest geolocation database is already cached at %s", GEODB_PATH, ) @@ -87,13 +87,13 @@ async def download_geodb(*, progress: Progress, session: ClientSession) -> None: ) if IS_DOCKER: - logger.info( + _logger.info( "Downloaded geolocation database to proxy_scraper_checker_cache " "Docker volume (%s in container)", GEODB_PATH, ) else: - logger.info("Downloaded geolocation database to %s", GEODB_PATH) + _logger.info("Downloaded geolocation database to %s", GEODB_PATH) if etag := response.headers.get(hdrs.ETAG): await _save_etag(etag) diff --git a/proxy_scraper_checker/logs.py b/proxy_scraper_checker/logs.py new file mode 100644 index 0000000..02f9323 --- /dev/null +++ b/proxy_scraper_checker/logs.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import logging +import logging.handlers +import queue +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + + from rich.console import Console + + +def configure() -> tuple[Console, logging.handlers.QueueListener]: + log_queue: queue.Queue[Any] = queue.Queue() + + logging.root.setLevel(logging.INFO) + logging.root.addHandler(logging.handlers.QueueHandler(log_queue)) + + # Start logging before importing rich for the first time + from rich.console import Console # noqa: PLC0415 + from rich.logging import RichHandler # noqa: PLC0415 + + console = Console() + stream_handler = RichHandler( + console=console, + omit_repeated_times=False, + show_path=False, + log_time_format=logging.Formatter.default_time_format, + ) + + return console, logging.handlers.QueueListener(log_queue, stream_handler) diff --git a/proxy_scraper_checker/output.py b/proxy_scraper_checker/output.py index 6fb9a06..3cdafa9 100644 --- a/proxy_scraper_checker/output.py +++ b/proxy_scraper_checker/output.py @@ -20,7 +20,7 @@ if TYPE_CHECKING: from .settings import Settings from .storage import ProxyStorage -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) def _create_proxy_list_str( @@ -105,11 +105,11 @@ def save_proxies(*, settings: Settings, storage: ProxyStorage) -> None: text, encoding="utf-8" ) if IS_DOCKER: - logger.info( + _logger.info( "Proxies have been saved to ./out (%s in container)", settings.output_path.absolute(), ) else: - logger.info( + _logger.info( "Proxies have been saved to %s", settings.output_path.absolute() ) diff --git a/proxy_scraper_checker/scraper.py b/proxy_scraper_checker/scraper.py index e15bd4b..3264353 100644 --- a/proxy_scraper_checker/scraper.py +++ b/proxy_scraper_checker/scraper.py @@ -21,7 +21,7 @@ if TYPE_CHECKING: from .settings import Settings from .storage import ProxyStorage -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) async def scrape_one( @@ -44,11 +44,11 @@ async def scrape_one( content = await f.read() text = bytes_decode(content) except ClientResponseError as e: - logger.warning( + _logger.warning( "%s | HTTP status code %d: %s", source, e.status, e.message ) except Exception as e: - logger.warning( + _logger.warning( "%s | %s.%s: %s", source, e.__class__.__module__, @@ -60,7 +60,7 @@ async def scrape_one( try: proxy = next(proxies) except StopIteration: - logger.warning("%s | No proxies found", source) + _logger.warning("%s | No proxies found", source) else: for proxy in itertools.chain((proxy,), proxies): # noqa: B020 try: diff --git a/proxy_scraper_checker/settings.py b/proxy_scraper_checker/settings.py index 8152168..e069f40 100644 --- a/proxy_scraper_checker/settings.py +++ b/proxy_scraper_checker/settings.py @@ -32,7 +32,7 @@ if TYPE_CHECKING: from .proxy import Proxy -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) def _get_supported_max_connections() -> int | None: @@ -46,7 +46,7 @@ def _get_supported_max_connections() -> int | None: import resource # type: ignore[unreachable, unused-ignore] # noqa: PLC0415 soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) - logger.debug( + _logger.debug( "max_connections: soft limit = %d, hard limit = %d, infinity = %d", soft_limit, hard_limit, @@ -56,7 +56,7 @@ def _get_supported_max_connections() -> int | None: try: resource.setrlimit(resource.RLIMIT_NOFILE, (hard_limit, hard_limit)) except ValueError as e: - logger.warning("Failed setting max_connections: %s", e) + _logger.warning("Failed setting max_connections: %s", e) else: soft_limit = hard_limit if soft_limit == resource.RLIM_INFINITY: @@ -70,11 +70,11 @@ def _get_max_connections(value: int, /) -> int | None: raise ValueError(msg) max_supported = _get_supported_max_connections() if not value: - logger.info("Using %d as max_connections value", max_supported or 0) + _logger.info("Using %d as max_connections value", max_supported or 0) return max_supported if not max_supported or value <= max_supported: return value - logger.warning( + _logger.warning( "max_connections value is too high for your OS. " "The config value will be ignored and %d will be used.%s", max_supported, @@ -151,7 +151,7 @@ async def _get_check_website_type_and_real_ip( content = await response.read() text = get_response_text(response=response, content=content) except Exception: - logger.exception( + _logger.exception( "Error when opening check_website without proxy, it will be " "impossible to determine anonymity and geolocation of proxies" ) @@ -168,7 +168,7 @@ async def _get_check_website_type_and_real_ip( return CheckWebsiteType.HTTPBIN_IP, parse_ipv4(js["origin"]) except (KeyError, TypeError, ValueError): pass - logger.warning( + _logger.warning( "Check_website is not httpbin and does not return plain ip, so it will" " be impossible to determine the anonymity and geolocation of proxies" ) @@ -249,7 +249,7 @@ class Settings: raise ValueError(msg) if not self.check_website and self.sort_by_speed: - logger.warning( + _logger.warning( "Proxy checking is disabled, so sorting by speed is not" " possible. Alphabetical sorting will be used instead." ) @@ -272,7 +272,7 @@ class Settings: raise ValueError(msg) if parsed_url.scheme == "http": - logger.warning( + _logger.warning( "check_website uses the http protocol. " "It is recommended to use https for correct checking." )