haeaeae/proxy-scraper-checker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor & improve output

Browse Source
This commit is contained in:
monosans
2023-01-02 00:09:55 +03:00
parent 5582a7860a
commit a8b1db926b
10 changed files with 230 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2 -4
View File
@@ -9,8 +9,6 @@ HTTP, SOCKS4, SOCKS5 proxies scraper and checker.
- Supports determining the geolocation of the proxy exit node.
- Can determine if a proxy is anonymous.
For a version that uses Python's built-in `logging` instead of [rich](https://github.com/willmcgugan/rich), see the [simple-output](https://github.com/monosans/proxy-scraper-checker/tree/simple-output) branch.
You can get proxies obtained using this script in [monosans/proxy-list](https://github.com/monosans/proxy-list).
## Installation and usage
@@ -24,8 +22,8 @@ You can get proxies obtained using this script in [monosans/proxy-list](https://
- On Unix-like OS run `start.sh`
- Manual:
1. `cd` into the unpacked folder
1. Install dependencies with the command `python -m pip install -U -r requirements.txt`
1. Run with the command `python main.py`
1. Install dependencies with the command `python -m pip install -U --no-cache-dir --disable-pip-version-check pip setuptools wheel; python -m pip install -U --no-cache-dir --disable-pip-version-check -r requirements.txt`
1. Run with the command `python -m proxy_scraper_checker`
## Folders description
proxy_scraper_checker/__init__.py
View File
proxy_scraper_checker/__main__.py
+45
View File
@@ -0,0 +1,45 @@
from __future__ import annotations
import asyncio
import logging
import sys
import rich.traceback
from rich.console import Console
from rich.logging import RichHandler
def install_uvloop() -> None:
if sys.implementation.name == "cpython" and sys.platform in {
"darwin",
"linux",
}:
try:
import uvloop
except ImportError:
pass
else:
uvloop.install()
def setup_logging(console: Console) -> None:
rich.traceback.install(console=console)
logging.basicConfig(
level=logging.INFO,
format="%(message)s",
handlers=[
RichHandler(console=console, show_path=False, rich_tracebacks=True)
],
)
if __name__ == "__main__":
install_uvloop()
console = Console()
setup_logging(console)
from .proxy_scraper_checker import ProxyScraperChecker
psc = ProxyScraperChecker.from_ini("config.ini", console=console)
asyncio.run(psc.run())
proxy_scraper_checker/folder.py
+22
View File
@@ -0,0 +1,22 @@
from __future__ import annotations
from pathlib import Path
from shutil import rmtree
class Folder:
__slots__ = ("for_anonymous", "for_geolocation", "path")
def __init__(self, *, path: Path, folder_name: str) -> None:
self.path = path / folder_name
self.for_anonymous = "anon" in folder_name
self.for_geolocation = "geo" in folder_name
def remove(self) -> None:
try:
rmtree(self.path)
except FileNotFoundError:
pass
def create(self) -> None:
self.path.mkdir(parents=True, exist_ok=True)
proxy_scraper_checker/proxy.py
+53
View File
@@ -0,0 +1,53 @@
from __future__ import annotations
import asyncio
from time import perf_counter
from aiohttp import ClientSession
from aiohttp_socks import ProxyConnector
class Proxy:
__slots__ = (
"geolocation",
"ip",
"is_anonymous",
"socket_address",
"timeout",
)
def __init__(self, *, socket_address: str, ip: str) -> None:
"""
Args:
socket_address: ip:port
"""
self.socket_address = socket_address
self.ip = ip
async def check(
self, *, sem: asyncio.Semaphore, proto: str, timeout: float
) -> None:
async with sem:
proxy_url = f"{proto}://{self.socket_address}"
start = perf_counter()
async with ProxyConnector.from_url(proxy_url) as connector:
async with ClientSession(connector=connector) as session:
async with session.get(
"http://ip-api.com/json/?fields=8217",
timeout=timeout,
raise_for_status=True,
) as response:
data = await response.json()
self.timeout = perf_counter() - start
self.is_anonymous = self.ip != data["query"]
self.geolocation = "|{}|{}|{}".format(
data["country"], data["regionName"], data["city"]
)
def __eq__(self, other: object) -> bool:
if not isinstance(other, Proxy):
return NotImplemented
return self.socket_address == other.socket_address
def __hash__(self) -> int:
return hash(self.socket_address)
main.py → proxy_scraper_checker/proxy_scraper_checker.py
Executable → Regular
+91 -151
View File
@@ -1,100 +1,43 @@
#!/usr/bin/env python3
from __future__ import annotations
import asyncio
import logging
import re
import sys
from configparser import ConfigParser
from pathlib import Path
from random import shuffle
from shutil import rmtree
from time import perf_counter
from typing import Callable, Dict, List, Optional, Set, Tuple, Union
from typing import (
Callable,
Dict,
List,
Optional,
Set,
Tuple,
Type,
TypeVar,
Union,
)
from aiohttp import ClientSession
from aiohttp_socks import ProxyConnector
from rich.console import Console
from rich.progress import (
BarColumn,
MofNCompleteColumn,
Progress,
TaskID,
TaskProgressColumn,
TextColumn,
)
from rich.table import Table
from . import sort
from .folder import Folder
from .proxy import Proxy
class Proxy:
__slots__ = (
"geolocation",
"ip",
"is_anonymous",
"socket_address",
"timeout",
)
logger = logging.getLogger(__name__)
def __init__(self, *, socket_address: str, ip: str) -> None:
"""
Args:
socket_address: ip:port
"""
self.socket_address = socket_address
self.ip = ip
async def check(
self, *, sem: asyncio.Semaphore, proto: str, timeout: float
) -> None:
async with sem:
proxy_url = f"{proto}://{self.socket_address}"
start = perf_counter()
async with ProxyConnector.from_url(proxy_url) as connector:
async with ClientSession(connector=connector) as session:
async with session.get(
"http://ip-api.com/json/?fields=8217",
timeout=timeout,
raise_for_status=True,
) as response:
data = await response.json()
self.timeout = perf_counter() - start
self.is_anonymous = self.ip != data["query"]
self.geolocation = "|{}|{}|{}".format(
data["country"], data["regionName"], data["city"]
)
def __eq__(self, other: object) -> bool:
if not isinstance(other, Proxy):
return NotImplemented
return self.socket_address == other.socket_address
def __hash__(self) -> int:
return hash(self.socket_address)
class Folder:
__slots__ = ("for_anonymous", "for_geolocation", "path")
def __init__(self, *, path: Path, folder_name: str) -> None:
self.path = path / folder_name
self.for_anonymous = "anon" in folder_name
self.for_geolocation = "geo" in folder_name
def remove(self) -> None:
try:
rmtree(self.path)
except FileNotFoundError:
pass
def create(self) -> None:
self.path.mkdir(parents=True, exist_ok=True)
def timeout_sort_key(proxy: Proxy) -> float:
return proxy.timeout
def natural_sort_key(proxy: Proxy) -> Tuple[int, ...]:
return tuple(map(int, proxy.socket_address.replace(":", ".").split(".")))
TProxyScraperChecker = TypeVar(
"TProxyScraperChecker", bound="ProxyScraperChecker"
)
class ProxyScraperChecker:
@@ -197,6 +140,45 @@ class ProxyScraperChecker:
self.console = console or Console()
self.sem = asyncio.Semaphore(max_connections)
@classmethod
def from_ini(
cls: Type[TProxyScraperChecker],
file_name: str,
*,
console: Optional[Console] = None,
) -> TProxyScraperChecker:
cfg = ConfigParser(interpolation=None)
cfg.read(file_name, encoding="utf-8")
general = cfg["General"]
folders = cfg["Folders"]
http = cfg["HTTP"]
socks4 = cfg["SOCKS4"]
socks5 = cfg["SOCKS5"]
return cls(
timeout=general.getfloat("Timeout", 10),
max_connections=general.getint("MaxConnections", 900),
sort_by_speed=general.getboolean("SortBySpeed", True),
save_path=general.get("SavePath", ""),
proxies=folders.getboolean("proxies", True),
proxies_anonymous=folders.getboolean("proxies_anonymous", True),
proxies_geolocation=folders.getboolean(
"proxies_geolocation", True
),
proxies_geolocation_anonymous=folders.getboolean(
"proxies_geolocation_anonymous", True
),
http_sources=http.get("Sources")
if http.getboolean("Enabled", True)
else None,
socks4_sources=socks4.get("Sources")
if socks4.getboolean("Enabled", True)
else None,
socks5_sources=socks5.get("Sources")
if socks5.getboolean("Enabled", True)
else None,
console=console,
)
async def fetch_source(
self,
*,
@@ -218,11 +200,11 @@ class ProxyScraperChecker:
status = response.status
text = await response.text()
except Exception as e:
msg = f"{source} | Error"
msg = source
exc_str = str(e)
if exc_str and exc_str != source:
msg += f": {exc_str}"
self.console.print(msg)
logger.error(msg)
else:
proxies = tuple(self.regex.finditer(text))
if proxies:
@@ -235,7 +217,7 @@ class ProxyScraperChecker:
msg = f"{source} | No proxies found"
if status != 200:
msg += f" | Status code {status}"
self.console.print(msg)
logger.warning(msg)
progress.update(task, advance=1)
async def check_proxy(
@@ -247,9 +229,7 @@ class ProxyScraperChecker:
except Exception as e:
# Too many open files
if isinstance(e, OSError) and e.errno == 24:
self.console.print(
"[red]Please, set MAX_CONNECTIONS to lower value."
)
logging.error("Please, set MAX_CONNECTIONS to lower value.")
self.proxies[proto].remove(proxy)
progress.update(task, advance=1)
@@ -306,7 +286,7 @@ class ProxyScraperChecker:
def save_proxies(self) -> None:
"""Delete old proxies and save new ones."""
sorted_proxies = self.sorted_proxies.items()
sorted_proxies = self.get_sorted_proxies().items()
for folder in self.all_folders:
folder.remove()
for folder in self.enabled_folders:
@@ -322,11 +302,35 @@ class ProxyScraperChecker:
file = folder.path / f"{proto}.txt"
file.write_text(text, encoding="utf-8")
async def main(self) -> None:
with self._progress as progress:
async def run(self) -> None:
with self._get_progress_bar() as progress:
await self.fetch_all_sources(progress)
await self.check_all_proxies(progress)
table = self._get_results_table()
self.console.print(table)
self.save_proxies()
logger.info(
"Proxy folders have been created in the %s folder."
+ "\nThank you for using proxy-scraper-checker :)",
self.path.resolve(),
)
def get_sorted_proxies(self) -> Dict[str, List[Proxy]]:
key: Union[
Callable[[Proxy], float], Callable[[Proxy], Tuple[int, ...]]
] = (
sort.timeout_sort_key
if self.sort_by_speed
else sort.natural_sort_key
)
return {
proto: sorted(proxies, key=key)
for proto, proxies in self.proxies.items()
}
def _get_results_table(self) -> Table:
table = Table()
table.add_column("Protocol", style="cyan")
table.add_column("Working", style="magenta")
@@ -338,76 +342,12 @@ class ProxyScraperChecker:
table.add_row(
proto.upper(), f"{working} ({percentage:.1f}%)", str(total)
)
self.console.print(table)
return table
self.save_proxies()
self.console.print(
"[green]Proxy folders have been created in the "
+ f"{self.path.resolve()} folder."
+ "\nThank you for using proxy-scraper-checker :)"
)
@property
def sorted_proxies(self) -> Dict[str, List[Proxy]]:
key: Union[
Callable[[Proxy], float], Callable[[Proxy], Tuple[int, ...]]
] = (timeout_sort_key if self.sort_by_speed else natural_sort_key)
return {
proto: sorted(proxies, key=key)
for proto, proxies in self.proxies.items()
}
@property
def _progress(self) -> Progress:
def _get_progress_bar(self) -> Progress:
return Progress(
TextColumn("[progress.description]{task.description}"),
BarColumn(),
TaskProgressColumn(),
MofNCompleteColumn(),
console=self.console,
)
async def main() -> None:
cfg = ConfigParser(interpolation=None)
cfg.read("config.ini", encoding="utf-8")
general = cfg["General"]
folders = cfg["Folders"]
http = cfg["HTTP"]
socks4 = cfg["SOCKS4"]
socks5 = cfg["SOCKS5"]
await ProxyScraperChecker(
timeout=general.getfloat("Timeout", 10),
max_connections=general.getint("MaxConnections", 900),
sort_by_speed=general.getboolean("SortBySpeed", True),
save_path=general.get("SavePath", ""),
proxies=folders.getboolean("proxies", True),
proxies_anonymous=folders.getboolean("proxies_anonymous", True),
proxies_geolocation=folders.getboolean("proxies_geolocation", True),
proxies_geolocation_anonymous=folders.getboolean(
"proxies_geolocation_anonymous", True
),
http_sources=http.get("Sources")
if http.getboolean("Enabled", True)
else None,
socks4_sources=socks4.get("Sources")
if socks4.getboolean("Enabled", True)
else None,
socks5_sources=socks5.get("Sources")
if socks5.getboolean("Enabled", True)
else None,
).main()
if __name__ == "__main__":
if sys.implementation.name == "cpython" and sys.platform in {
"darwin",
"linux",
}:
try:
import uvloop
except ImportError:
pass
else:
uvloop.install()
asyncio.run(main())
proxy_scraper_checker/sort.py
+13
View File
@@ -0,0 +1,13 @@
from __future__ import annotations
from typing import Tuple
from .proxy import Proxy
def timeout_sort_key(proxy: Proxy) -> float:
return proxy.timeout
def natural_sort_key(proxy: Proxy) -> Tuple[int, ...]:
return tuple(map(int, proxy.socket_address.replace(":", ".").split(".")))
screenshot.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 48 KiB

start.cmd
+3 -2
View File
@@ -1,3 +1,4 @@
python -m pip install -U -r requirements.txt
python main.py
python -m pip install -U --no-cache-dir --disable-pip-version-check pip setuptools wheel
python -m pip install -U --no-cache-dir --disable-pip-version-check -r requirements.txt
python -m proxy_scraper_checker
Pause
start.sh
+1 -2
View File
@@ -1,3 +1,2 @@
#!/bin/sh
python -m pip install -U -r requirements.txt
python main.py
python -m pip install -U --no-cache-dir --disable-pip-version-check pip setuptools wheel && python -m pip install -U --no-cache-dir --disable-pip-version-check -r requirements.txt && python -m proxy_scraper_checker