Don't shuffle proxies before checking

This commit is contained in:
monosans
2024-01-25 10:30:40 +03:00
parent 3059940212
commit 890c000139
2 changed files with 24 additions and 24 deletions
+11 -12
View File
@@ -2,7 +2,6 @@ from __future__ import annotations
import asyncio
import logging
from random import shuffle
from typing import Mapping
from aiohttp_socks import ProxyType
@@ -53,15 +52,15 @@ async def check_all(
for proto in sort.PROTOCOL_ORDER
if proto in storage.enabled_protocols
}
coroutines = [
check_one(
progress=progress,
proxy=proxy,
settings=settings,
storage=storage,
task=tasks[proxy.protocol],
await asyncio.gather(
*(
check_one(
progress=progress,
proxy=proxy,
settings=settings,
storage=storage,
task=tasks[proxy.protocol],
)
for proxy in storage
)
for proxy in storage
]
shuffle(coroutines)
await asyncio.gather(*coroutines)
)
+13 -12
View File
@@ -94,17 +94,18 @@ async def scrape_all(
for proto, sources in settings.sources.items()
}
timeout = ClientTimeout(total=settings.source_timeout)
coroutines = (
scrape_one(
progress=progress,
proto=proto,
session=session,
source=source,
storage=storage,
task=tasks[proto],
timeout=timeout,
await asyncio.gather(
*(
scrape_one(
progress=progress,
proto=proto,
session=session,
source=source,
storage=storage,
task=tasks[proto],
timeout=timeout,
)
for proto, sources in settings.sources.items()
for source in sources
)
for proto, sources in settings.sources.items()
for source in sources
)
await asyncio.gather(*coroutines)