haeaeae/proxy-scraper-checker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add pre-commit

Browse Source
This commit is contained in:
monosans
2023-02-25 19:31:34 +03:00
parent 8f9569edd9
commit 8d16ec6fff
8 changed files with 169 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.pre-commit-config.yaml
+44
View File
@@ -0,0 +1,44 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
hooks:
- id: check-case-conflict
- id: check-executables-have-shebangs
- id: check-merge-conflict
- id: check-shebang-scripts-are-executable
- id: check-symlinks
- id: check-toml
- id: check-xml
- id: destroyed-symlinks
- id: end-of-file-fixer
- id: mixed-line-ending
args: [--fix=lf]
- id: trailing-whitespace
- repo: https://github.com/monosans/pre-commit-prettier
rev: v2.8.4
hooks:
- id: prettier
- repo: https://github.com/psf/black
rev: 23.1.0
hooks:
- id: black
- repo: https://github.com/charliermarsh/ruff-pre-commit
rev: v0.0.252
hooks:
- id: ruff
args: [--fix]
- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.0.1
hooks:
- id: mypy
args: []
additional_dependencies:
- aiodns>=3.0,<4
- aiohttp-socks>=0.7,<0.9
- aiohttp>=3.8,<4
- rich>=12.3,<14
- uvloop>=0.16,<0.18; implementation_name == 'cpython' and (sys_platform == 'darwin' or sys_platform == 'linux')
proxy_scraper_checker/__main__.py
+4 -5
View File
@@ -15,10 +15,7 @@ from .proxy_scraper_checker import ProxyScraperChecker
def set_event_loop_policy() -> None:
if sys.platform == "win32":
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
elif sys.implementation.name == "cpython" and sys.platform in {
"darwin",
"linux",
}:
elif sys.implementation.name == "cpython" and sys.platform in {"darwin", "linux"}:
try:
import uvloop
except ImportError:
@@ -54,7 +51,9 @@ async def main() -> None:
cfg = get_config("config.ini")
console = Console()
configure_logging(console, debug=cfg["General"].getboolean("Debug", False))
configure_logging(
console, debug=cfg["General"].getboolean("Debug", False) # noqa: FBT003
)
psc = ProxyScraperChecker.from_configparser(cfg, console=console)
await psc.run()
proxy_scraper_checker/constants.py
+1 -3
View File
@@ -1,5 +1,3 @@
from __future__ import annotations
USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/109.0"
)
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/109.0"
proxy_scraper_checker/proxy.py
+1 -3
View File
@@ -41,9 +41,7 @@ class Proxy:
cookie_jar=cookie_jar,
timeout=timeout,
headers=HEADERS,
) as session, session.get(
website, raise_for_status=True
) as response:
) as session, session.get(website, raise_for_status=True) as response:
if website == DEFAULT_CHECK_WEBSITE:
await response.read()
self.timeout = perf_counter() - start
proxy_scraper_checker/proxy_scraper_checker.py
+33 -54
View File
@@ -22,13 +22,7 @@ from typing import (
from aiohttp import ClientSession, ClientTimeout, DummyCookieJar
from aiohttp_socks import ProxyType
from rich.console import Console
from rich.progress import (
BarColumn,
MofNCompleteColumn,
Progress,
TaskID,
TextColumn,
)
from rich.progress import BarColumn, MofNCompleteColumn, Progress, TaskID, TextColumn
from rich.table import Table
from . import sort, validators
@@ -39,9 +33,7 @@ from .proxy import Proxy
logger = logging.getLogger(__name__)
TProxyScraperChecker = TypeVar(
"TProxyScraperChecker", bound="ProxyScraperChecker"
)
TProxyScraperChecker = TypeVar("TProxyScraperChecker", bound="ProxyScraperChecker")
class ProxyScraperChecker:
@@ -121,8 +113,8 @@ class ProxyScraperChecker:
validators.check_website(check_website)
logger.info(
"CheckWebsite is not 'default', "
+ "so it will not be possible to determine "
+ "the anonymity and geolocation of the proxies"
"so it will not be possible to determine "
"the anonymity and geolocation of the proxies"
)
for folder in self.folders:
folder.is_enabled = (
@@ -145,12 +137,12 @@ class ProxyScraperChecker:
self.cookie_jar = DummyCookieJar()
self.regex = re.compile(
r"(?:^|\D)?("
+ r"(?:[1-9]|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])" # 1-255
r"(?:[1-9]|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])" # 1-255
+ r"\.(?:\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])" * 3 # 0-255
+ r"):"
+ (
r"(\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}"
+ r"|65[0-4]\d{2}|655[0-2]\d|6553[0-5])"
r"|65[0-4]\d{2}|655[0-2]\d|6553[0-5])"
) # 0-65535
+ r"(?:\D|$)"
)
@@ -173,31 +165,35 @@ class ProxyScraperChecker:
source_timeout=general.getfloat("SourceTimeout", 15),
max_connections=general.getint("MaxConnections", 512),
check_website=general.get("CheckWebsite", "default"),
sort_by_speed=general.getboolean("SortBySpeed", True),
sort_by_speed=general.getboolean("SortBySpeed", True), # noqa: FBT003
save_path=save_path,
folders=(
Folder(
path=save_path / "proxies",
is_enabled=folders.getboolean("proxies", True),
is_enabled=folders.getboolean("proxies", True), # noqa: FBT003
for_anonymous=False,
for_geolocation=False,
),
Folder(
path=save_path / "proxies_anonymous",
is_enabled=folders.getboolean("proxies_anonymous", True),
is_enabled=folders.getboolean(
"proxies_anonymous", True # noqa: FBT003
),
for_anonymous=True,
for_geolocation=False,
),
Folder(
path=save_path / "proxies_geolocation",
is_enabled=folders.getboolean("proxies_geolocation", True),
is_enabled=folders.getboolean(
"proxies_geolocation", True # noqa: FBT003
),
for_anonymous=False,
for_geolocation=True,
),
Folder(
path=save_path / "proxies_geolocation_anonymous",
is_enabled=folders.getboolean(
"proxies_geolocation_anonymous", True
"proxies_geolocation_anonymous", True # noqa: FBT003
),
for_anonymous=True,
for_geolocation=True,
@@ -206,17 +202,17 @@ class ProxyScraperChecker:
sources={
ProxyType.HTTP: (
http.get("Sources")
if http.getboolean("Enabled", True)
if http.getboolean("Enabled", True) # noqa: FBT003
else None
),
ProxyType.SOCKS4: (
socks4.get("Sources")
if socks4.getboolean("Enabled", True)
if socks4.getboolean("Enabled", True) # noqa: FBT003
else None
),
ProxyType.SOCKS5: (
socks5.get("Sources")
if socks5.getboolean("Enabled", True)
if socks5.getboolean("Enabled", True) # noqa: FBT003
else None
),
},
@@ -264,26 +260,19 @@ class ProxyScraperChecker:
proxies = tuple(self.regex.finditer(text))
if proxies:
for proxy in proxies:
proxy_obj = Proxy(
host=proxy.group(1), port=int(proxy.group(2))
)
proxy_obj = Proxy(host=proxy.group(1), port=int(proxy.group(2)))
self.proxies[proto].add(proxy_obj)
else:
args = (
("%s | No proxies found", source)
if status == 200
if status == 200 # noqa: PLR2004
else ("%s | HTTP status code %d", source, status)
)
logger.warning(*args)
progress.update(task, advance=1)
async def check_proxy(
self,
*,
proxy: Proxy,
proto: ProxyType,
progress: Progress,
task: TaskID,
self, *, proxy: Proxy, proto: ProxyType, progress: Progress, task: TaskID
) -> None:
"""Check if proxy is alive."""
try:
@@ -296,14 +285,11 @@ class ProxyScraperChecker:
)
except Exception as e:
# Too many open files
if isinstance(e, OSError) and e.errno == 24:
if isinstance(e, OSError) and e.errno == 24: # noqa: PLR2004
logger.error("Please, set MaxConnections to lower value.")
logger.debug(
"%s.%s | %s",
e.__class__.__module__,
e.__class__.__qualname__,
e,
"%s.%s | %s", e.__class__.__module__, e.__class__.__qualname__, e
)
self.proxies[proto].remove(proxy)
progress.update(task, advance=1)
@@ -311,8 +297,7 @@ class ProxyScraperChecker:
async def fetch_all_sources(self, progress: Progress) -> None:
tasks = {
proto: progress.add_task(
f"[yellow]Scraper [red]:: [green]{proto.name}",
total=len(sources),
f"[yellow]Scraper [red]:: [green]{proto.name}", total=len(sources)
)
for proto, sources in self.sources.items()
}
@@ -341,8 +326,7 @@ class ProxyScraperChecker:
async def check_all_proxies(self, progress: Progress) -> None:
tasks = {
proto: progress.add_task(
f"[yellow]Checker [red]:: [green]{proto.name}",
total=len(proxies),
f"[yellow]Checker [red]:: [green]{proto.name}", total=len(proxies)
)
for proto, proxies in self.proxies.items()
}
@@ -384,22 +368,19 @@ class ProxyScraperChecker:
self.save_proxies()
logger.info(
"Proxy folders have been created in the %s folder."
+ "\nThank you for using proxy-scraper-checker :)",
(
"Proxy folders have been created in the %s folder."
"\nThank you for using proxy-scraper-checker :)"
),
self.path.resolve(),
)
def get_sorted_proxies(self) -> Dict[ProxyType, List[Proxy]]:
key: Union[
Callable[[Proxy], float], Callable[[Proxy], Tuple[int, ...]]
] = (
sort.timeout_sort_key
if self.sort_by_speed
else sort.natural_sort_key
key: Union[Callable[[Proxy], float], Callable[[Proxy], Tuple[int, ...]]] = (
sort.timeout_sort_key if self.sort_by_speed else sort.natural_sort_key
)
return {
proto: sorted(proxies, key=key)
for proto, proxies in self.proxies.items()
proto: sorted(proxies, key=key) for proto, proxies in self.proxies.items()
}
def _get_results_table(self) -> Table:
@@ -411,9 +392,7 @@ class ProxyScraperChecker:
working = len(proxies)
total = self.proxies_count[proto]
percentage = working / total if total else 0
table.add_row(
proto.name, f"{working} ({percentage:.1%})", str(total)
)
table.add_row(proto.name, f"{working} ({percentage:.1%})", str(total))
return table
def _get_progress_bar(self) -> Progress:
proxy_scraper_checker/validators.py
+8 -6
View File
@@ -34,9 +34,11 @@ def max_connections(value: int) -> Optional[int]:
if not max_supported or value <= max_supported:
return value
logger.warning(
"MaxConnections value is too high. "
+ "Your OS supports a maximum of %d. "
+ "The config value will be ignored and %d will be used.",
(
"MaxConnections value is too high. "
"Your OS supports a maximum of %d. "
"The config value will be ignored and %d will be used."
),
max_supported,
max_supported,
)
@@ -46,8 +48,7 @@ def max_connections(value: int) -> Optional[int]:
def _get_supported_max_connections() -> Optional[int]:
if sys.platform == "win32":
if isinstance(
asyncio.get_event_loop_policy(),
asyncio.WindowsSelectorEventLoopPolicy,
asyncio.get_event_loop_policy(), asyncio.WindowsSelectorEventLoopPolicy
):
return 512
return None
@@ -74,4 +75,5 @@ def folders(value: Iterable[Folder]) -> None:
def sources(value: Any) -> None:
if not value:
raise ValueError("proxy sources list is empty")
msg = "proxy sources list is empty"
raise ValueError(msg)
pyproject.toml
+78
View File
@@ -0,0 +1,78 @@
[tool.black]
target-version = ["py37"]
skip-magic-trailing-comma = true
preview = true
[tool.mypy]
ignore_missing_imports = true
python_version = "3.7"
disallow_subclassing_any = false
disallow_untyped_calls = false
disallow_untyped_decorators = false
warn_unreachable = true
local_partial_types = true
enable_error_code = [
"redundant-self",
"redundant-expr",
"truthy-bool",
"truthy-iterable",
"ignore-without-code",
"unused-awaitable",
]
strict = true
[tool.ruff]
target-version = "py37"
select = ["ALL"]
ignore = [
"ANN",
"B008",
"BLE001",
"C901",
"COM",
"D100",
"D101",
"D102",
"D103",
"D104",
"D105",
"D106",
"D107",
"D203",
"D205",
"D212",
"D213",
"D400",
"D407",
"D415",
"D417",
"DJ008",
"ERA001",
"PD901",
"PLR0911",
"PLR0912",
"PLR0913",
"PLR0915",
"PT012",
"RUF001",
"RUF002",
"RUF003",
"S110",
"S112",
"SIM105",
"TCH001",
"TCH002",
"TCH003",
"TID252",
"TRY400",
]
[tool.ruff.flake8-unused-arguments]
ignore-variadic-names = true
[tool.ruff.isort]
combine-as-imports = true
required-imports = ["from __future__ import annotations"]
[tool.ruff.pyupgrade]
keep-runtime-typing = true
start.cmd
Executable → Regular
View File