haeaeae/KC-Scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add files via upload

Browse Source
This commit is contained in:
Kuucheen
2023-01-27 22:10:50 +01:00
committed by GitHub
parent 0258c9ecda
commit f69f9ef7d3
7 changed files with 3937 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
KC-Scraper.py
+80 -80
View File
@@ -5,9 +5,7 @@ try:
import httpx, yaml
from pystyle import Colors, Colorate, Center
except ImportError:
os.system('python -m pip install httpx[http2]')
os.system('python -m pip install pyyaml')
os.system('python -m pip install pystyle')
os.system('python -m pip install httpx[http2] pyyaml pystyle')
import httpx, yaml
from pystyle import Colors, Colorate, Center
@@ -27,6 +25,82 @@ color = Colors.StaticMIX((Colors.purple, Colors.blue))
def main():
global threadcount, clearingcnt, clearingproxy, randomUseragent, timeout, sitelist, start
printLogo()
print()
config, clearingcnt, clearingproxy, randomUseragent, threads, timeout = getSettings()
terminal()
start = time.time()
#dispatcher
with open(config) as sites:
sitelist = sites.readlines()
threading.Thread(target=terminalthread).start()
while len(sitelist) > 0:
if threadcount < threads:
threading.Thread(target=scrape, args=[sitelist[0]]).start()
threadcount += 1
sitelist.pop(0)
while threadcount > 0:
terminal(f"| Waiting for threads to finish | active threads {threadcount} | Proxies {proxycount} | Time {time.time()-start:.2f}s")
terminal()
lenproxies = len(proxies)
print(f"\n{white}[{color}^{white}] Removed {color}{proxycount-lenproxies} {white}Duplicates\n")
print(f"{white}[{color}^{white}] Remaining Proxies: {color}{lenproxies}{white}\n")
print(f"{white}[{color}^{white}] Writing {color}Proxies\n")
with open("proxies.txt", "w") as output:
for i in proxies:
output.write(i.replace("\n", "") + "\n")
if clearingcnt == True or clearingproxy == True:
print(f"{white}[{color}^{white}] Removing {color}bad Websites\n")
with open(config, "w") as inp:
for site in goodsites:
inp.write(site + "\n")
terminal()
print(f"{white}[{color}^{white}] Finished in {color}{time.time()-start:.2f}s{white}!\n")
print("You can now close the tab")
input("")
def scrape(site: str):
global proxies, threadcount, proxycount
uas=["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:53.0) Gecko/20100101 Firefox/53.0", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; Trident/5.0)", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0; MDDCJS)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)"]
site = site.replace("\n", "")
try:
with httpx.Client(http2=True,headers = {'accept-language': 'en','user-agent': random.choice(uas) if randomUseragent == True else uas[0]},follow_redirects=True) as client:
r = client.get(site, timeout=timeout).text
except:
print(f"{white}[{Colors.red}!{white}] Failed connecting to {color}{site}")
else:
goodsites.add(site)
r = r.replace("&colon", ":")
locProxies = re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\:\d{1,5}\b", r)
length = len(locProxies)
print(f"{white}[{Colors.green}+{white}] Scraped {color}{length}{white} from {color}{site}")
proxycount += length
proxies = proxies | set(locProxies)
if clearingproxy == True and length == 0:
goodsites.remove(site)
finally:
threadcount -= 1
def getSettings() -> list:
with open("settings.yaml") as setting:
settings = yaml.safe_load(setting.read())
@@ -37,11 +111,6 @@ def main():
threads = settings["threads"]
timeout = settings["timeout"]
terminal()
printLogo()
print()
yes = ["yes", "y", "ye"]
no = ["no", "n", "nah"]
@@ -183,76 +252,8 @@ def main():
time.sleep(3)
main()
exit()
printLogo()
start = time.time()
with open(config) as sites:
sitelist = sites.readlines()
threading.Thread(target=terminalthread).start()
while len(sitelist) > 0:
if threadcount < threads:
threading.Thread(target=scrape, args=[sitelist[0]]).start()
threadcount += 1
sitelist.pop(0)
while threadcount > 0:
terminal(f"| Waiting for threads to finish | active threads {threadcount} | Proxies {proxycount} | Time {time.time()-start:.2f}s")
terminal()
lenproxies = len(proxies)
print(f"\n{white}[{color}^{white}] Removed {color}{proxycount-lenproxies} {white}Duplicates\n")
print(f"{white}[{color}^{white}] Remaining Proxies: {color}{lenproxies}{white}\n")
print(f"{white}[{color}^{white}] Writing {color}Proxies\n")
with open("proxies.txt", "w") as output:
for i in proxies:
output.write(i.replace("\n", "") + "\n")
if clearingcnt == True or clearingproxy == True:
print(f"{white}[{color}^{white}] Removing {color}bad Websites\n")
with open(config, "w") as inp:
for site in goodsites:
inp.write(site + "\n")
terminal()
print(f"{white}[{color}^{white}] Finished in {color}{time.time()-start:.2f}s{white}!\n")
print("You can now close the tab")
input("")
def scrape(site: str):
global proxies, threadcount, proxycount
uas=["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:53.0) Gecko/20100101 Firefox/53.0", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; Trident/5.0)", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0; MDDCJS)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)"]
site = site.replace("\n", "")
try:
with httpx.Client(http2=True,headers = {'accept-language': 'en','user-agent': random.choice(uas) if randomUseragent == True else uas[0]},follow_redirects=True) as client:
r = client.get(site, timeout=timeout).text
except:
print(f"{white}[{Colors.red}!{white}] Failed connecting to {color}{site}")
else:
goodsites.add(site)
r = r.replace("&colon", ":")
locProxies = re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\:\d{1,5}\b", r)
length = len(locProxies)
print(f"{white}[{Colors.green}+{white}] Scraped {color}{length}{white} from {color}{site}")
proxycount += length
proxies = proxies | set(locProxies)
if clearingproxy == True and length == 0:
goodsites.remove(site)
finally:
threadcount -= 1
return config, clearingcnt, clearingproxy, randomUseragent, threads, timeout
def terminal(string:str = ""):
ctypes.windll.kernel32.SetConsoleTitleW("KC Scraper | github.com/Kuucheen " + string)
@@ -277,6 +278,5 @@ def printLogo():
print(Colorate.Diagonal(Colors.DynamicMIX((Colors.dark_gray, Colors.StaticMIX((Colors.purple, Colors.blue)))), Center.XCenter(logo)))
main()
if __name__ == "__main__":
main()
premades/http.txt
+17 -19
View File
@@ -1,23 +1,21 @@
https://api.proxyscrape.com/v2/?request=getproxies&protocol=http&timeout=10000&country=all&ssl=all&anonymity=all
https://www.proxyscan.io/download?type=https
https://openproxy.space/list/http
https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-http.txt
https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt
https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/http.txt
http://pubproxy.com/api/proxy
https://www.juproxy.com/free_api
https://api.openproxylist.xyz/http.txt
https://raw.githubusercontent.com/hyperbeats/proxy-list/main/http.txt
https://raw.githubusercontent.com/jetkai/proxy-list/main/archive/txt/proxies-http.txt
https://raw.githubusercontent.com/jetkai/proxy-list/main/archive/txt/proxies-https.txt
https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-http.txt
https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-https.txt
https://raw.githubusercontent.com/mmpx12/proxy-list/master/http.txt
https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/http.txt
https://raw.githubusercontent.com/proxy4parsing/proxy-list/main/http.txt
https://raw.githubusercontent.com/roosterkid/openproxylist/main/https_raw.txt
https://raw.githubusercontent.com/shiftytr/proxy-list/master/http.txt
https://raw.githubusercontent.com/thespeedx/proxy-list/master/http.txt
https://www.proxy-list.download/api/v1/get?type=http
https://www.proxy-list.download/api/v1/get?type=https
https://www.proxyscan.io/download?type=http
https://www.proxyscan.io/download?type=https
https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/http.txt
https://api.proxyscrape.com/v2/?request=getproxies&protocol=http&timeout=10000&country=all&ssl=all&anonymity=all
https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/https.txt
https://raw.githubusercontent.com/roosterkid/openproxylist/main/HTTPS_RAW.txt
https://raw.githubusercontent.com/proxy4parsing/proxy-list/main/http.txt
https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/http.txt
https://raw.githubusercontent.com/monosans/proxy-list/main/proxies_anonymous/http.txt
https://raw.githubusercontent.com/saschazesiger/Free-Proxies/master/proxies/http.txt
https://www.proxy-list.download/api/v1/get?type=https
https://www.proxy-list.download/api/v1/get?type=http
https://raw.githubusercontent.com/mmpx12/proxy-list/master/http.txt
https://raw.githubusercontent.com/mmpx12/proxy-list/master/https.txt
https://api.openproxylist.xyz/http.txt
http://pubproxy.com/api/proxy
https://raw.githubusercontent.com/HyperBeats/proxy-list/main/http.txt
https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-https.txt
premades/socks4.txt
+12 -13
View File
@@ -1,16 +1,15 @@
https://openproxy.space/list/socks4
https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt
https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt
https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-socks4.txt
https://www.socks-proxy.net/
https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks4&timeout=10000&country=all
https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS4_RAW.txt
https://www.proxy-list.download/api/v1/get?type=socks4
https://www.proxyscan.io/download?type=socks4
https://api.openproxylist.xyz/socks4.txt
https://raw.githubusercontent.com/mmpx12/proxy-list/master/socks4.txt
https://www.proxy-list.download/api/v1/get?type=socks4
https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-socks4.txt
https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/socks4.txt
https://raw.githubusercontent.com/UptimerBot/proxy-list/main/proxies/socks4.txt
https://raw.githubusercontent.com/rdavydov/proxy-list/main/proxies/socks4.txt
https://raw.githubusercontent.com/saschazesiger/Free-Proxies/master/proxies/socks4.txt
https://raw.githubusercontent.com/HyperBeats/proxy-list/main/socks4.txt
https://raw.githubusercontent.com/TheSpeedX/PROXY-List/master/socks4.txt
https://raw.githubusercontent.com/HyperBeats/proxy-list/main/socks4.txt
https://www.proxyscan.io/download?type=socks4
https://www.socks-proxy.net/
https://raw.githubusercontent.com/mmpx12/proxy-list/master/socks4.txt
https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks4.txt
https://openproxy.space/list/socks4
https://raw.githubusercontent.com/monosans/proxy-list/main/proxies_anonymous/socks4.txt
https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS4_RAW.txt
https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks4&timeout=10000&country=all
premades/socks5.txt
+13 -14
View File
@@ -1,16 +1,15 @@
https://openproxy.space/list/socks5
https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt
https://api.openproxylist.xyz/socks5.txt
https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks5&timeout=10000&country=all
https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt
https://raw.githubusercontent.com/hyperbeats/proxy-list/main/socks5.txt
https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-socks5.txt
https://raw.githubusercontent.com/manugmg/proxy-365/main/socks5.txt
https://raw.githubusercontent.com/mmpx12/proxy-list/master/socks5.txt
https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt
https://raw.githubusercontent.com/roosterkid/openproxylist/main/socks5_raw.txt
https://raw.githubusercontent.com/saschazesiger/free-proxies/master/proxies/socks5.txt
https://raw.githubusercontent.com/shiftytr/proxy-list/master/socks5.txt
https://raw.githubusercontent.com/thespeedx/proxy-list/master/socks5.txt
https://api.openproxylist.xyz/socks5.txt
https://www.proxyscan.io/download?type=socks5
https://raw.githubusercontent.com/roosterkid/openproxylist/main/SOCKS5_RAW.txt
https://raw.githubusercontent.com/hookzof/socks5_list/master/proxy.txt
https://www.proxy-list.download/api/v1/get?type=socks5
https://www.proxyscan.io/download?type=socks5
https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/socks5.txt
https://raw.githubusercontent.com/saschazesiger/Free-Proxies/master/proxies/socks5.txt
https://raw.githubusercontent.com/hyperbeats/proxy-list/main/socks5.txt
https://openproxy.space/list/socks5
https://raw.githubusercontent.com/mmpx12/proxy-list/master/socks5.txt
https://api.proxyscrape.com/v2/?request=getproxies&protocol=socks5&timeout=10000&country=all
https://raw.githubusercontent.com/jetkai/proxy-list/main/online-proxies/txt/proxies-socks5.txt
https://raw.githubusercontent.com/monosans/proxy-list/main/proxies_anonymous/socks5.txt
https://raw.githubusercontent.com/shiftytr/proxy-list/master/socks5.txt
proxies.txt
+3813
View File
File diff suppressed because it is too large Load Diff
settings.yaml
+1 -1
View File
@@ -5,7 +5,7 @@
removeProxyless: "?", # Removes websites if no proxies are found | '?' for asking | 'y' or 'n'
randomUseragent: "?", # Decides if random useragent is used | '?' for asking | 'y' for random useragent | 'n' will use the most common useragent
randomUseragent: "n", # Decides if random useragent is used | '?' for asking | 'y' for random useragent | 'n' will use the most common useragent
threads: "?", # '?' for asking | '[Threadcount]'
sites.txt
+1
View File
@@ -0,0 +1 @@
https://www.socks-proxy.net/