diff --git a/proxy_scraper_checker/__main__.py b/proxy_scraper_checker/__main__.py index 9cac2e2..367b0f0 100644 --- a/proxy_scraper_checker/__main__.py +++ b/proxy_scraper_checker/__main__.py @@ -53,8 +53,7 @@ async def main() -> None: console = Console() configure_logging(console, debug=cfg["General"].getboolean("Debug", False)) - psc = ProxyScraperChecker.from_configparser(cfg, console=console) - await psc.run() + await ProxyScraperChecker.from_configparser(cfg, console=console).run() if __name__ == "__main__": diff --git a/proxy_scraper_checker/proxy_scraper_checker.py b/proxy_scraper_checker/proxy_scraper_checker.py index 9a72be7..0ac2768 100644 --- a/proxy_scraper_checker/proxy_scraper_checker.py +++ b/proxy_scraper_checker/proxy_scraper_checker.py @@ -229,8 +229,8 @@ class ProxyScraperChecker: """ try: async with session.get(source) as response: - status = response.status - text = await response.text() + await response.read() + text = await response.text() except asyncio.TimeoutError: logger.warning("%s | Timed out", source) except Exception as e: @@ -253,18 +253,23 @@ class ProxyScraperChecker: ) logger.error(*args) else: - proxies = tuple(self.regex.finditer(text)) - if proxies: - for proxy in proxies: - proxy_obj = Proxy(host=proxy.group(1), port=int(proxy.group(2))) - self.proxies[proto].add(proxy_obj) - else: + proxies = self.regex.finditer(text) + try: + proxy = next(proxies) + except StopIteration: args = ( ("%s | No proxies found", source) - if status == 200 # noqa: PLR2004 - else ("%s | HTTP status code %d", source, status) + if response.status == 200 # noqa: PLR2004 + else ("%s | HTTP status code %d", source, response.status) ) logger.warning(*args) + else: + proxies_set = self.proxies[proto] + proxies_set.add(Proxy(host=proxy.group(1), port=int(proxy.group(2)))) + for proxy in proxies: + proxies_set.add( + Proxy(host=proxy.group(1), port=int(proxy.group(2))) + ) progress.update(task, advance=1) async def check_proxy(