config.py -> config.ini
This commit is contained in:
monosans
@@ -1,5 +1,9 @@
|
||||
# YYYY-MM-DD
|
||||
|
||||
## 2022-04-13
|
||||
|
||||
- `config.py` -> `config.ini`.
|
||||
|
||||
## 2022-04-11
|
||||
|
||||
- Add `__slots__` attribute to all objects.
|
||||
|
||||
@@ -19,7 +19,7 @@ You can get proxies obtained using this script in [monosans/proxy-list](https://
|
||||
- Download and unpack [the archive with the program](https://github.com/monosans/proxy-scraper-checker/archive/refs/heads/main.zip).
|
||||
- Install dependencies from `requirements.txt` (`cd` into the unpacked folder and run `python -m pip install -U -r requirements.txt` on the command line).
|
||||
- If you want to improve the performance, you can also install extra dependencies. See [aiohttp documentation](https://docs.aiohttp.org/en/stable/index.html#library-installation).
|
||||
- Edit `config.py` according to your preference.
|
||||
- Edit `config.ini` according to your preference.
|
||||
- Run `main.py` (`python main.py` on the command line).
|
||||
|
||||
## Folders description
|
||||
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
[General]
|
||||
; How many seconds to wait for the proxy to make a connection.
|
||||
; The higher this number, the longer the check will take
|
||||
; and the more proxies you will receive.
|
||||
Timeout = 10
|
||||
|
||||
; Maximum concurrent connections.
|
||||
; Don't set higher than 900, please.
|
||||
MaxConnections = 900
|
||||
|
||||
; Set to no to sort proxies alphabetically.
|
||||
SortBySpeed = yes
|
||||
|
||||
; Path to the folder where the proxy folders will be saved.
|
||||
; Leave empty to save the proxies to the current directory.
|
||||
SavePath =
|
||||
|
||||
[Folders]
|
||||
; Proxies with any anonymity level.
|
||||
proxies = yes
|
||||
; Anonymous proxies.
|
||||
proxies_anonymous = yes
|
||||
; Same as proxies, but includes exit-node's geolocation.
|
||||
; Geolocation format is ip:port|Country|Region|City
|
||||
proxies_geolocation = yes
|
||||
; Same as proxies_geolocation, but includes exit-node's geolocation.
|
||||
proxies_geolocation_anonymous = yes
|
||||
|
||||
[HTTP]
|
||||
Enabled = yes
|
||||
Sources =
|
||||
https://api.proxyscrape.com/v2/?request=getproxies&protocol=http
|
||||
https://openproxy.space/list/http
|
||||
https://raw.githubusercontent.com/almroot/proxylist/master/list.txt
|
||||
https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt
|
||||
https://raw.githubusercontent.com/hendrikbgr/Free-Proxy-Repo/master/proxy_list.txt
|
||||
https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-http+https.txt
|
||||
https://raw.githubusercontent.com/mertguvencli/http-proxy-list/main/proxy-list/data.txt
|
||||
https://raw.githubusercontent.com/mmpx12/proxy-list/master/http.txt
|
||||
https://raw.githubusercontent.com/mmpx12/proxy-list/master/https.txt
|
||||
https://raw.githubusercontent.com/roosterkid/openproxylist/main/HTTPS_RAW.txt
|
||||
https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/http.txt
|
||||
https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/https.txt
|
||||
https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt
|
||||
https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt
|
||||
https://raw.githubusercontent.com/User-R3X/proxy-list/main/online/http+s.txt
|
||||
https://raw.githubusercontent.com/Volodichev/proxy-list/main/http.txt
|
||||
https://www.proxy-list.download/api/v1/get?type=http
|
||||
https://www.proxy-list.download/api/v1/get?type=https
|
||||
|
||||
[SOCKS4]
|
||||
Enabled = yes
|
||||
Sources =
|
||||
https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks4
|
||||
https://openproxy.space/list/socks4
|
||||
https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-socks4.txt
|
||||
https://raw.githubusercontent.com/mmpx12/proxy-list/master/socks4.txt
|
||||
https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS4_RAW.txt
|
||||
https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks4.txt
|
||||
https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt
|
||||
https://raw.githubusercontent.com/User-R3X/proxy-list/main/online/socks4.txt
|
||||
https://www.proxy-list.download/api/v1/get?type=socks4
|
||||
|
||||
[SOCKS5]
|
||||
Enabled = yes
|
||||
Sources =
|
||||
https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks5
|
||||
https://openproxy.space/list/socks5
|
||||
https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt
|
||||
https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-socks5.txt
|
||||
https://raw.githubusercontent.com/manuGMG/proxy-365/main/SOCKS5.txt
|
||||
https://raw.githubusercontent.com/mmpx12/proxy-list/master/socks5.txt
|
||||
https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS5_RAW.txt
|
||||
https://raw.githubusercontent.com/ryanhaticus/superiorproxy.com/main/proxies.txt
|
||||
https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks5.txt
|
||||
https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt
|
||||
https://raw.githubusercontent.com/User-R3X/proxy-list/main/online/socks5.txt
|
||||
https://www.proxy-list.download/api/v1/get?type=socks5
|
||||
@@ -1,82 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# How many seconds to wait for the proxy to make a connection.
|
||||
# The higher this number, the longer the check will take
|
||||
# and the more proxies you will receive.
|
||||
TIMEOUT = 10
|
||||
|
||||
# Maximum concurrent connections.
|
||||
# Don't set higher than 900, please.
|
||||
MAX_CONNECTIONS = 900
|
||||
|
||||
# Set to False to sort proxies alphabetically.
|
||||
SORT_BY_SPEED = True
|
||||
|
||||
# Path to the folder where the proxy folders will be saved.
|
||||
# Leave the quotes empty to save the proxies to the current directory.
|
||||
SAVE_PATH = ""
|
||||
|
||||
# Enable which proxy folders to create.
|
||||
# Set to False to disable.
|
||||
|
||||
# Proxies with any anonymity level.
|
||||
PROXIES = True
|
||||
# Anonymous proxies.
|
||||
PROXIES_ANONYMOUS = True
|
||||
# Same as PROXIES, but including exit-node's geolocation.
|
||||
# Geolocation format is ip:port|Country|Region|City
|
||||
PROXIES_GEOLOCATION = True
|
||||
# Same as PROXIES_GEOLOCATION, but including exit-node's geolocation.
|
||||
PROXIES_GEOLOCATION_ANONYMOUS = True
|
||||
|
||||
|
||||
# PROTOCOL - whether to enable checking certain protocol proxies (True or False).
|
||||
# PROTOCOL_SOURCES - proxy lists URLs.
|
||||
HTTP = True
|
||||
HTTP_SOURCES = (
|
||||
"https://api.proxyscrape.com/v2/?request=getproxies&protocol=http",
|
||||
"https://openproxy.space/list/http",
|
||||
"https://raw.githubusercontent.com/almroot/proxylist/master/list.txt",
|
||||
"https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt",
|
||||
"https://raw.githubusercontent.com/hendrikbgr/Free-Proxy-Repo/master/proxy_list.txt",
|
||||
"https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-http%2Bhttps.txt",
|
||||
"https://raw.githubusercontent.com/mertguvencli/http-proxy-list/main/proxy-list/data.txt",
|
||||
"https://raw.githubusercontent.com/mmpx12/proxy-list/master/http.txt",
|
||||
"https://raw.githubusercontent.com/mmpx12/proxy-list/master/https.txt",
|
||||
"https://raw.githubusercontent.com/roosterkid/openproxylist/main/HTTPS_RAW.txt",
|
||||
"https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/http.txt",
|
||||
"https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/https.txt",
|
||||
"https://raw.githubusercontent.com/sunny9577/proxy-scraper/master/proxies.txt",
|
||||
"https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt",
|
||||
"https://raw.githubusercontent.com/User-R3X/proxy-list/main/online/http%2Bs.txt",
|
||||
"https://raw.githubusercontent.com/Volodichev/proxy-list/main/http.txt",
|
||||
"https://www.proxy-list.download/api/v1/get?type=http",
|
||||
"https://www.proxy-list.download/api/v1/get?type=https",
|
||||
)
|
||||
SOCKS4 = True
|
||||
SOCKS4_SOURCES = (
|
||||
"https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks4",
|
||||
"https://openproxy.space/list/socks4",
|
||||
"https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-socks4.txt",
|
||||
"https://raw.githubusercontent.com/mmpx12/proxy-list/master/socks4.txt",
|
||||
"https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS4_RAW.txt",
|
||||
"https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks4.txt",
|
||||
"https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt",
|
||||
"https://raw.githubusercontent.com/User-R3X/proxy-list/main/online/socks4.txt",
|
||||
"https://www.proxy-list.download/api/v1/get?type=socks4",
|
||||
)
|
||||
SOCKS5 = True
|
||||
SOCKS5_SOURCES = (
|
||||
"https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks5",
|
||||
"https://openproxy.space/list/socks5",
|
||||
"https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt",
|
||||
"https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-socks5.txt",
|
||||
"https://raw.githubusercontent.com/manuGMG/proxy-365/main/SOCKS5.txt",
|
||||
"https://raw.githubusercontent.com/mmpx12/proxy-list/master/socks5.txt",
|
||||
"https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS5_RAW.txt",
|
||||
"https://raw.githubusercontent.com/ryanhaticus/superiorproxy.com/main/proxies.txt",
|
||||
"https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks5.txt",
|
||||
"https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt",
|
||||
"https://raw.githubusercontent.com/User-R3X/proxy-list/main/online/socks5.txt",
|
||||
"https://www.proxy-list.download/api/v1/get?type=socks5",
|
||||
)
|
||||
@@ -2,11 +2,12 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import asyncio
|
||||
import re
|
||||
from configparser import ConfigParser
|
||||
from pathlib import Path
|
||||
from random import shuffle
|
||||
from shutil import rmtree
|
||||
from time import perf_counter
|
||||
from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Union
|
||||
from typing import Callable, Dict, List, Optional, Set, Tuple, Union
|
||||
|
||||
from aiohttp import ClientSession
|
||||
from aiohttp_socks import ProxyConnector
|
||||
@@ -20,8 +21,6 @@ from rich.progress import (
|
||||
)
|
||||
from rich.table import Table
|
||||
|
||||
import config
|
||||
|
||||
|
||||
class Proxy:
|
||||
__slots__ = (
|
||||
@@ -111,9 +110,9 @@ class ProxyScraperChecker:
|
||||
proxies_anonymous: bool,
|
||||
proxies_geolocation: bool,
|
||||
proxies_geolocation_anonymous: bool,
|
||||
http_sources: Optional[Iterable[str]],
|
||||
socks4_sources: Optional[Iterable[str]],
|
||||
socks5_sources: Optional[Iterable[str]],
|
||||
http_sources: Optional[str],
|
||||
socks4_sources: Optional[str],
|
||||
socks5_sources: Optional[str],
|
||||
console: Optional[Console] = None,
|
||||
) -> None:
|
||||
"""HTTP, SOCKS4, SOCKS5 proxies scraper and checker.
|
||||
@@ -154,9 +153,7 @@ class ProxyScraperChecker:
|
||||
self.sort_by_speed = sort_by_speed
|
||||
self.timeout = timeout
|
||||
self.sources = {
|
||||
proto: (sources,)
|
||||
if isinstance(sources, str)
|
||||
else frozenset(sources)
|
||||
proto: frozenset(filter(None, sources.splitlines()))
|
||||
for proto, sources in (
|
||||
("http", http_sources),
|
||||
("socks4", socks4_sources),
|
||||
@@ -349,23 +346,32 @@ class ProxyScraperChecker:
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
cfg = ConfigParser(interpolation=None)
|
||||
cfg.read("config.ini", encoding="utf-8")
|
||||
general = cfg["General"]
|
||||
folders = cfg["Folders"].getboolean
|
||||
http = cfg["HTTP"]
|
||||
socks4 = cfg["SOCKS4"]
|
||||
socks5 = cfg["SOCKS5"]
|
||||
await ProxyScraperChecker(
|
||||
timeout=config.TIMEOUT,
|
||||
max_connections=config.MAX_CONNECTIONS,
|
||||
sort_by_speed=config.SORT_BY_SPEED,
|
||||
save_path=config.SAVE_PATH,
|
||||
proxies=config.PROXIES,
|
||||
proxies_anonymous=config.PROXIES_ANONYMOUS,
|
||||
proxies_geolocation=config.PROXIES_GEOLOCATION,
|
||||
proxies_geolocation_anonymous=config.PROXIES_GEOLOCATION_ANONYMOUS,
|
||||
http_sources=config.HTTP_SOURCES
|
||||
if config.HTTP and config.HTTP_SOURCES
|
||||
timeout=general.getfloat("Timeout", 10),
|
||||
max_connections=general.getint("MaxConnections"),
|
||||
sort_by_speed=general.getboolean("SortBySpeed", True),
|
||||
save_path=general.get("SavePath", ""),
|
||||
proxies=folders("proxies", True),
|
||||
proxies_anonymous=folders("proxies_anonymous", True),
|
||||
proxies_geolocation=folders("proxies_geolocation", True),
|
||||
proxies_geolocation_anonymous=folders(
|
||||
"proxies_geolocation_anonymous", True
|
||||
),
|
||||
http_sources=http.get("Sources")
|
||||
if http.getboolean("Enabled")
|
||||
else None,
|
||||
socks4_sources=config.SOCKS4_SOURCES
|
||||
if config.SOCKS4 and config.SOCKS4_SOURCES
|
||||
socks4_sources=socks4.get("Sources")
|
||||
if socks4.getboolean("Enabled")
|
||||
else None,
|
||||
socks5_sources=config.SOCKS5_SOURCES
|
||||
if config.SOCKS5 and config.SOCKS5_SOURCES
|
||||
socks5_sources=socks5.get("Sources")
|
||||
if socks5.getboolean("Enabled")
|
||||
else None,
|
||||
).main()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user