diff --git a/config.toml b/config.toml index 3d2a2af..9b462eb 100644 --- a/config.toml +++ b/config.toml @@ -18,6 +18,8 @@ max_connections = 512 # Example: # check_website = "https://google.com" # check_website = "https://httpbin.smp.io/ip" +# Disable checking: +# check_website = "" check_website = "https://checkip.amazonaws.com" # Set to false to sort proxies alphabetically. diff --git a/proxy_scraper_checker/__main__.py b/proxy_scraper_checker/__main__.py index 9e0e726..616daa7 100644 --- a/proxy_scraper_checker/__main__.py +++ b/proxy_scraper_checker/__main__.py @@ -153,15 +153,17 @@ async def main() -> None: await session.close() count_before_checking = storage.get_count() should_save = True - await checker.check_all( - settings=settings, - storage=storage, - progress=progress, - proxies_count=count_before_checking, - ) + if settings.check_website: + await checker.check_all( + settings=settings, + storage=storage, + progress=progress, + proxies_count=count_before_checking, + ) finally: if should_save: - storage.remove_unchecked() + if settings.check_website: + storage.remove_unchecked() count_after_checking = storage.get_count() console.print( get_summary_table( diff --git a/proxy_scraper_checker/settings.py b/proxy_scraper_checker/settings.py index 19c6a15..de8d74b 100644 --- a/proxy_scraper_checker/settings.py +++ b/proxy_scraper_checker/settings.py @@ -143,6 +143,8 @@ async def _get_check_website_type_and_real_ip( Literal[CheckWebsiteType.PLAIN_IP, CheckWebsiteType.HTTPBIN_IP], str ] ): + if not check_website: + return CheckWebsiteType.UNKNOWN, None try: async with session.get(check_website) as response: content = await response.read() @@ -252,16 +254,20 @@ class Settings: value: str, /, ) -> None: - parsed_url = urlparse(value) - if parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc: - msg = f"invalid check_website: {value}" - raise ValueError(msg) + if value: + parsed_url = urlparse(value) + if ( + parsed_url.scheme not in {"http", "https"} + or not parsed_url.netloc + ): + msg = f"invalid check_website: {value}" + raise ValueError(msg) - if parsed_url.scheme == "http": - logger.warning( - "check_website uses the http protocol. " - "It is recommended to use https for correct checking." - ) + if parsed_url.scheme == "http": + logger.warning( + "check_website uses the http protocol. " + "It is recommended to use https for correct checking." + ) @timeout.validator def _validate_timeout(