haeaeae/proxy-scraper-checker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Minor optimizations

Browse Source
This commit is contained in:
monosans
2023-04-16 21:11:46 +03:00
parent 031f6f4bbb
commit a6411bc501
2 changed files with 16 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
proxy_scraper_checker/__main__.py
+1 -2
View File
@@ -53,8 +53,7 @@ async def main() -> None:
console = Console()
configure_logging(console, debug=cfg["General"].getboolean("Debug", False))
psc = ProxyScraperChecker.from_configparser(cfg, console=console)
await psc.run()
await ProxyScraperChecker.from_configparser(cfg, console=console).run()
if __name__ == "__main__":
proxy_scraper_checker/proxy_scraper_checker.py
+15 -10
View File
@@ -229,8 +229,8 @@ class ProxyScraperChecker:
"""
try:
async with session.get(source) as response:
status = response.status
text = await response.text()
await response.read()
text = await response.text()
except asyncio.TimeoutError:
logger.warning("%s | Timed out", source)
except Exception as e:
@@ -253,18 +253,23 @@ class ProxyScraperChecker:
)
logger.error(*args)
else:
proxies = tuple(self.regex.finditer(text))
if proxies:
for proxy in proxies:
proxy_obj = Proxy(host=proxy.group(1), port=int(proxy.group(2)))
self.proxies[proto].add(proxy_obj)
else:
proxies = self.regex.finditer(text)
try:
proxy = next(proxies)
except StopIteration:
args = (
("%s | No proxies found", source)
if status == 200 # noqa: PLR2004
else ("%s | HTTP status code %d", source, status)
if response.status == 200 # noqa: PLR2004
else ("%s | HTTP status code %d", source, response.status)
)
logger.warning(*args)
else:
proxies_set = self.proxies[proto]
proxies_set.add(Proxy(host=proxy.group(1), port=int(proxy.group(2))))
for proxy in proxies:
proxies_set.add(
Proxy(host=proxy.group(1), port=int(proxy.group(2)))
)
progress.update(task, advance=1)
async def check_proxy(