diff --git a/KC-Scraper.py b/KC-Scraper.py index 48a2dd2..729c351 100644 --- a/KC-Scraper.py +++ b/KC-Scraper.py @@ -13,6 +13,7 @@ except ImportError: proxies = set([]) goodsites = set([]) +sitelist = [] proxycount = 0 threadcount = 0 clearing = "" @@ -21,7 +22,7 @@ color = Colors.StaticMIX((Colors.purple, Colors.blue)) def main(): - global threadcount, clearing + global threadcount, clearing, sitelist, start with open("settings.yaml") as setting: settings = yaml.safe_load(setting.read()) @@ -126,16 +127,18 @@ def main(): print() start = time.time() + with open(config) as sites: sitelist = sites.readlines() + threading.Thread(target=terminalthread).start() while len(sitelist) > 0: if threadcount < threads: threading.Thread(target=scrape, args=[sitelist[0]]).start() threadcount += 1 sitelist.pop(0) - terminal(f"| Remaining sites {len(sitelist)} | active threads {threadcount} | Proxies {proxycount} | Time {time.time()-start:.2f}s") +# terminal(f"| Remaining sites {len(sitelist)} | active threads {threadcount} | Proxies {proxycount} | Time {time.time()-start:.2f}s") while threadcount > 0: try: @@ -179,7 +182,6 @@ def scrape(site: str): #random useragent by (idk dm me if you are the guy) but thank you helped me learn how to use httpx uas=['Mozilla/5.0 (X11; CrOS x86_64 14588.123.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.72 Safari/537.36', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 12.4; rv:101.0) Gecko/20100101 Firefox/101.0', 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36', 'Mozilla/5.0 (Linux; Android 12; SM-G960U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.99 Mobile Safari/537.36'] site = site.replace("\n", "") - proxycount = len(proxies) finished = False try: with httpx.Client(http2=True,headers = {'accept-language': 'en','user-agent':random.choice(uas)},follow_redirects=True) as client: @@ -215,6 +217,9 @@ def scrape(site: str): def terminal(string:str = ""): ctypes.windll.kernel32.SetConsoleTitleW("KC Scraper | github.com/Kuucheen " + string) +def terminalthread(): + while len(sitelist) > 0: + terminal(f"| Remaining sites {len(sitelist)} | active threads {threadcount} | Proxies {proxycount} | Time {time.time()-start:.2f}s") diff --git a/premades/http.txt b/premades/http.txt index d3341a0..682ae55 100644 --- a/premades/http.txt +++ b/premades/http.txt @@ -1,4 +1,3 @@ -https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt https://api.proxyscrape.com/v2/?request=getproxies&protocol=http&timeout=10000&country=all&ssl=all&anonymity=all https://openproxy.space/list/http https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-http.txt @@ -6,3 +5,19 @@ https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/http.txt http://pubproxy.com/api/proxy https://www.juproxy.com/free_api +https://api.openproxylist.xyz/http.txt +https://raw.githubusercontent.com/hyperbeats/proxy-list/main/http.txt +https://raw.githubusercontent.com/jetkai/proxy-list/main/archive/txt/proxies-http.txt +https://raw.githubusercontent.com/jetkai/proxy-list/main/archive/txt/proxies-https.txt +https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-http.txt +https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-https.txt +https://raw.githubusercontent.com/mmpx12/proxy-list/master/http.txt +https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt +https://raw.githubusercontent.com/proxy4parsing/proxy-list/main/http.txt +https://raw.githubusercontent.com/roosterkid/openproxylist/main/https_raw.txt +https://raw.githubusercontent.com/shiftytr/proxy-list/master/http.txt +https://raw.githubusercontent.com/thespeedx/proxy-list/master/http.txt +https://www.proxy-list.download/api/v1/get?type=http +https://www.proxy-list.download/api/v1/get?type=https +https://www.proxyscan.io/download?type=http +https://www.proxyscan.io/download?type=https \ No newline at end of file diff --git a/premades/socks4.txt b/premades/socks4.txt index b31d885..72feca1 100644 --- a/premades/socks4.txt +++ b/premades/socks4.txt @@ -2,4 +2,15 @@ https://openproxy.space/list/socks4 https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-socks4.txt -https://www.socks-proxy.net/ \ No newline at end of file +https://www.socks-proxy.net/ +https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks4&timeout=10000&country=all +https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS4_RAW.txt +https://www.proxy-list.download/api/v1/get?type=socks4 +https://www.proxyscan.io/download?type=socks4 +https://api.openproxylist.xyz/socks4.txt +https://raw.githubusercontent.com/mmpx12/proxy-list/master/socks4.txt +https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks4.txt +https://raw.githubusercontent.com/UptimerBot/proxy-list/main/proxies/socks4.txt +https://raw.githubusercontent.com/rdavydov/proxy-list/main/proxies/socks4.txt +https://raw.githubusercontent.com/saschazesiger/Free-Proxies/master/proxies/socks4.txt +https://raw.githubusercontent.com/HyperBeats/proxy-list/main/socks4.txt \ No newline at end of file diff --git a/premades/socks5.txt b/premades/socks5.txt index 7c97033..ba38b9b 100644 --- a/premades/socks5.txt +++ b/premades/socks5.txt @@ -1,6 +1,16 @@ https://openproxy.space/list/socks5 -https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks5.txt -https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt -https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks5.txt https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt -https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-socks4.txt \ No newline at end of file +https://api.openproxylist.xyz/socks5.txt +https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks5&timeout=10000&country=all +https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt +https://raw.githubusercontent.com/hyperbeats/proxy-list/main/socks5.txt +https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-socks5.txt +https://raw.githubusercontent.com/manugmg/proxy-365/main/socks5.txt +https://raw.githubusercontent.com/mmpx12/proxy-list/master/socks5.txt +https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt +https://raw.githubusercontent.com/roosterkid/openproxylist/main/socks5_raw.txt +https://raw.githubusercontent.com/saschazesiger/free-proxies/master/proxies/socks5.txt +https://raw.githubusercontent.com/shiftytr/proxy-list/master/socks5.txt +https://raw.githubusercontent.com/thespeedx/proxy-list/master/socks5.txt +https://www.proxy-list.download/api/v1/get?type=socks5 +https://www.proxyscan.io/download?type=socks5 \ No newline at end of file