From 9afc87f70bb278e53fcf057ed8c415569a92b597 Mon Sep 17 00:00:00 2001 From: monosans Date: Wed, 13 Apr 2022 18:24:56 +0300 Subject: [PATCH] config.py -> config.ini --- CHANGELOG.md | 4 +++ README.md | 2 +- config.ini | 78 +++++++++++++++++++++++++++++++++++++++++++++++++ config.py | 82 ---------------------------------------------------- main.py | 52 ++++++++++++++++++--------------- 5 files changed, 112 insertions(+), 106 deletions(-) create mode 100644 config.ini delete mode 100644 config.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b89794b..c051146 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # YYYY-MM-DD +## 2022-04-13 + +- `config.py` -> `config.ini`. + ## 2022-04-11 - Add `__slots__` attribute to all objects. diff --git a/README.md b/README.md index 9efffce..304953b 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ You can get proxies obtained using this script in [monosans/proxy-list](https:// - Download and unpack [the archive with the program](https://github.com/monosans/proxy-scraper-checker/archive/refs/heads/main.zip). - Install dependencies from `requirements.txt` (`cd` into the unpacked folder and run `python -m pip install -U -r requirements.txt` on the command line). - If you want to improve the performance, you can also install extra dependencies. See [aiohttp documentation](https://docs.aiohttp.org/en/stable/index.html#library-installation). -- Edit `config.py` according to your preference. +- Edit `config.ini` according to your preference. - Run `main.py` (`python main.py` on the command line). ## Folders description diff --git a/config.ini b/config.ini new file mode 100644 index 0000000..c7de997 --- /dev/null +++ b/config.ini @@ -0,0 +1,78 @@ +[General] +; How many seconds to wait for the proxy to make a connection. +; The higher this number, the longer the check will take +; and the more proxies you will receive. +Timeout = 10 + +; Maximum concurrent connections. +; Don't set higher than 900, please. +MaxConnections = 900 + +; Set to no to sort proxies alphabetically. +SortBySpeed = yes + +; Path to the folder where the proxy folders will be saved. +; Leave empty to save the proxies to the current directory. +SavePath = + +[Folders] +; Proxies with any anonymity level. +proxies = yes +; Anonymous proxies. +proxies_anonymous = yes +; Same as proxies, but includes exit-node's geolocation. +; Geolocation format is ip:port|Country|Region|City +proxies_geolocation = yes +; Same as proxies_geolocation, but includes exit-node's geolocation. +proxies_geolocation_anonymous = yes + +[HTTP] +Enabled = yes +Sources = + https://api.proxyscrape.com/v2/?request=getproxies&protocol=http + https://openproxy.space/list/http + https://raw.githubusercontent.com/almroot/proxylist/master/list.txt + https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt + https://raw.githubusercontent.com/hendrikbgr/Free-Proxy-Repo/master/proxy_list.txt + https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-http+https.txt + https://raw.githubusercontent.com/mertguvencli/http-proxy-list/main/proxy-list/data.txt + https://raw.githubusercontent.com/mmpx12/proxy-list/master/http.txt + https://raw.githubusercontent.com/mmpx12/proxy-list/master/https.txt + https://raw.githubusercontent.com/roosterkid/openproxylist/main/HTTPS_RAW.txt + https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/http.txt + https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/https.txt + https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt + https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt + https://raw.githubusercontent.com/User-R3X/proxy-list/main/online/http+s.txt + https://raw.githubusercontent.com/Volodichev/proxy-list/main/http.txt + https://www.proxy-list.download/api/v1/get?type=http + https://www.proxy-list.download/api/v1/get?type=https + +[SOCKS4] +Enabled = yes +Sources = + https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks4 + https://openproxy.space/list/socks4 + https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-socks4.txt + https://raw.githubusercontent.com/mmpx12/proxy-list/master/socks4.txt + https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS4_RAW.txt + https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks4.txt + https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt + https://raw.githubusercontent.com/User-R3X/proxy-list/main/online/socks4.txt + https://www.proxy-list.download/api/v1/get?type=socks4 + +[SOCKS5] +Enabled = yes +Sources = + https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks5 + https://openproxy.space/list/socks5 + https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt + https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-socks5.txt + https://raw.githubusercontent.com/manuGMG/proxy-365/main/SOCKS5.txt + https://raw.githubusercontent.com/mmpx12/proxy-list/master/socks5.txt + https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS5_RAW.txt + https://raw.githubusercontent.com/ryanhaticus/superiorproxy.com/main/proxies.txt + https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks5.txt + https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt + https://raw.githubusercontent.com/User-R3X/proxy-list/main/online/socks5.txt + https://www.proxy-list.download/api/v1/get?type=socks5 diff --git a/config.py b/config.py deleted file mode 100644 index 7a10fc2..0000000 --- a/config.py +++ /dev/null @@ -1,82 +0,0 @@ -# -*- coding: utf-8 -*- - -# How many seconds to wait for the proxy to make a connection. -# The higher this number, the longer the check will take -# and the more proxies you will receive. -TIMEOUT = 10 - -# Maximum concurrent connections. -# Don't set higher than 900, please. -MAX_CONNECTIONS = 900 - -# Set to False to sort proxies alphabetically. -SORT_BY_SPEED = True - -# Path to the folder where the proxy folders will be saved. -# Leave the quotes empty to save the proxies to the current directory. -SAVE_PATH = "" - -# Enable which proxy folders to create. -# Set to False to disable. - -# Proxies with any anonymity level. -PROXIES = True -# Anonymous proxies. -PROXIES_ANONYMOUS = True -# Same as PROXIES, but including exit-node's geolocation. -# Geolocation format is ip:port|Country|Region|City -PROXIES_GEOLOCATION = True -# Same as PROXIES_GEOLOCATION, but including exit-node's geolocation. -PROXIES_GEOLOCATION_ANONYMOUS = True - - -# PROTOCOL - whether to enable checking certain protocol proxies (True or False). -# PROTOCOL_SOURCES - proxy lists URLs. -HTTP = True -HTTP_SOURCES = ( - "https://api.proxyscrape.com/v2/?request=getproxies&protocol=http", - "https://openproxy.space/list/http", - "https://raw.githubusercontent.com/almroot/proxylist/master/list.txt", - "https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt", - "https://raw.githubusercontent.com/hendrikbgr/Free-Proxy-Repo/master/proxy_list.txt", - "https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-http%2Bhttps.txt", - "https://raw.githubusercontent.com/mertguvencli/http-proxy-list/main/proxy-list/data.txt", - "https://raw.githubusercontent.com/mmpx12/proxy-list/master/http.txt", - "https://raw.githubusercontent.com/mmpx12/proxy-list/master/https.txt", - "https://raw.githubusercontent.com/roosterkid/openproxylist/main/HTTPS_RAW.txt", - "https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/http.txt", - "https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/https.txt", - "https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt", - "https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt", - "https://raw.githubusercontent.com/User-R3X/proxy-list/main/online/http%2Bs.txt", - "https://raw.githubusercontent.com/Volodichev/proxy-list/main/http.txt", - "https://www.proxy-list.download/api/v1/get?type=http", - "https://www.proxy-list.download/api/v1/get?type=https", -) -SOCKS4 = True -SOCKS4_SOURCES = ( - "https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks4", - "https://openproxy.space/list/socks4", - "https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-socks4.txt", - "https://raw.githubusercontent.com/mmpx12/proxy-list/master/socks4.txt", - "https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS4_RAW.txt", - "https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks4.txt", - "https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt", - "https://raw.githubusercontent.com/User-R3X/proxy-list/main/online/socks4.txt", - "https://www.proxy-list.download/api/v1/get?type=socks4", -) -SOCKS5 = True -SOCKS5_SOURCES = ( - "https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks5", - "https://openproxy.space/list/socks5", - "https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt", - "https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-socks5.txt", - "https://raw.githubusercontent.com/manuGMG/proxy-365/main/SOCKS5.txt", - "https://raw.githubusercontent.com/mmpx12/proxy-list/master/socks5.txt", - "https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS5_RAW.txt", - "https://raw.githubusercontent.com/ryanhaticus/superiorproxy.com/main/proxies.txt", - "https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks5.txt", - "https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt", - "https://raw.githubusercontent.com/User-R3X/proxy-list/main/online/socks5.txt", - "https://www.proxy-list.download/api/v1/get?type=socks5", -) diff --git a/main.py b/main.py index 5baa305..6eb98d8 100755 --- a/main.py +++ b/main.py @@ -2,11 +2,12 @@ # -*- coding: utf-8 -*- import asyncio import re +from configparser import ConfigParser from pathlib import Path from random import shuffle from shutil import rmtree from time import perf_counter -from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Union +from typing import Callable, Dict, List, Optional, Set, Tuple, Union from aiohttp import ClientSession from aiohttp_socks import ProxyConnector @@ -20,8 +21,6 @@ from rich.progress import ( ) from rich.table import Table -import config - class Proxy: __slots__ = ( @@ -111,9 +110,9 @@ class ProxyScraperChecker: proxies_anonymous: bool, proxies_geolocation: bool, proxies_geolocation_anonymous: bool, - http_sources: Optional[Iterable[str]], - socks4_sources: Optional[Iterable[str]], - socks5_sources: Optional[Iterable[str]], + http_sources: Optional[str], + socks4_sources: Optional[str], + socks5_sources: Optional[str], console: Optional[Console] = None, ) -> None: """HTTP, SOCKS4, SOCKS5 proxies scraper and checker. @@ -154,9 +153,7 @@ class ProxyScraperChecker: self.sort_by_speed = sort_by_speed self.timeout = timeout self.sources = { - proto: (sources,) - if isinstance(sources, str) - else frozenset(sources) + proto: frozenset(filter(None, sources.splitlines())) for proto, sources in ( ("http", http_sources), ("socks4", socks4_sources), @@ -349,23 +346,32 @@ class ProxyScraperChecker: async def main() -> None: + cfg = ConfigParser(interpolation=None) + cfg.read("config.ini", encoding="utf-8") + general = cfg["General"] + folders = cfg["Folders"].getboolean + http = cfg["HTTP"] + socks4 = cfg["SOCKS4"] + socks5 = cfg["SOCKS5"] await ProxyScraperChecker( - timeout=config.TIMEOUT, - max_connections=config.MAX_CONNECTIONS, - sort_by_speed=config.SORT_BY_SPEED, - save_path=config.SAVE_PATH, - proxies=config.PROXIES, - proxies_anonymous=config.PROXIES_ANONYMOUS, - proxies_geolocation=config.PROXIES_GEOLOCATION, - proxies_geolocation_anonymous=config.PROXIES_GEOLOCATION_ANONYMOUS, - http_sources=config.HTTP_SOURCES - if config.HTTP and config.HTTP_SOURCES + timeout=general.getfloat("Timeout", 10), + max_connections=general.getint("MaxConnections"), + sort_by_speed=general.getboolean("SortBySpeed", True), + save_path=general.get("SavePath", ""), + proxies=folders("proxies", True), + proxies_anonymous=folders("proxies_anonymous", True), + proxies_geolocation=folders("proxies_geolocation", True), + proxies_geolocation_anonymous=folders( + "proxies_geolocation_anonymous", True + ), + http_sources=http.get("Sources") + if http.getboolean("Enabled") else None, - socks4_sources=config.SOCKS4_SOURCES - if config.SOCKS4 and config.SOCKS4_SOURCES + socks4_sources=socks4.get("Sources") + if socks4.getboolean("Enabled") else None, - socks5_sources=config.SOCKS5_SOURCES - if config.SOCKS5 and config.SOCKS5_SOURCES + socks5_sources=socks5.get("Sources") + if socks5.getboolean("Enabled") else None, ).main()