haeaeae/KC-Scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add files via upload

Browse Source
This commit is contained in:
Kuucheen
2022-09-02 14:26:03 +02:00
committed by GitHub
parent a910cc6de2
commit 6e59e264dc
2 changed files with 52 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
KC-Scraper.py
+48 -31
View File
@@ -16,19 +16,21 @@ goodsites = set([])
sitelist = []
proxycount = 0
threadcount = 0
clearing = ""
clearingcnt = ""
clearingproxy = ""
white = Colors.light_gray
color = Colors.StaticMIX((Colors.purple, Colors.blue))
def main():
global threadcount, clearing, sitelist, start
global threadcount, clearingcnt, sitelist, start
with open("settings.yaml") as setting:
settings = yaml.safe_load(setting.read())
premades = settings["premades"]
clearing = settings["removewebsites"]
clearingcnt = settings["removewebsites"]
clearingproxy = settings["removeproxyless"]
threads = settings["threads"]
terminal()
@@ -84,40 +86,55 @@ def main():
os.system("cls")
print(Colorate.Diagonal(Colors.DynamicMIX((Colors.dark_gray, Colors.StaticMIX((Colors.purple, Colors.blue)))), Center.XCenter(logo)))
if clearing == "?":
if clearingcnt == "?":
clearing = input(f"\n\n\n{white}[{color}^{white}] {color}Remove not connectable site [y/n] {white}>> {color}")
clearingcnt = input(f"\n\n\n{white}[{color}^{white}] {color}Remove not connectable site [y/n] {white}>> {color}")
if clearing != "y" and clearing != "ye" and clearing != "yes" and clearing != "n" and clearing != "no":
if clearingcnt != "y" and clearingcnt != "ye" and clearingcnt != "yes" and clearingcnt != "n" and clearingcnt != "no":
print(f"{white}[{color}!{white}] {color}No option was choosen returning to home..")
time.sleep(3)
main()
exit()
elif clearing != "y" and clearing != "ye" and clearing != "yes" and clearing != "n" and clearing != "no":
elif clearingcnt != "y" and clearingcnt != "ye" and clearingcnt != "yes" and clearingcnt != "n" and clearingcnt != "no":
print(f"{white}[{Colors.red}!{white}] {Colors.red}Error{white} in settings.json at removewebsites")
input()
exit()
if clearing == "y" or clearing == "ye" or clearing == "yes":
clearing = True
if clearingcnt == "y" or clearingcnt == "ye" or clearingcnt == "yes":
clearingcnt = True
if clearingproxy == "?":
clearingproxy = input(f"\n{white}[{color}^{white}] {color}Remove sites with no proxies [y/n] {white}>> {color}")
if clearingproxy != "y" and clearingproxy != "ye" and clearingproxy != "yes" and clearingproxy != "n" and clearingproxy != "no":
print(f"{white}[{color}!{white}] {color}No option was choosen returning to home..")
time.sleep(3)
main()
exit()
elif clearingproxy != "y" and clearingproxy != "ye" and clearingproxy != "yes" and clearingproxy != "n" and clearingproxy != "no":
print(f"{white}[{Colors.red}!{white}] {Colors.red}Error{white} in settings.json at removeproxyless")
input()
exit()
if clearingproxy == "y" or clearingproxy == "ye" or clearingproxy == "yes":
clearingproxy = True
if threads == "?":
threads = input(f"{white}[{color}^{white}] {color}Threads {white}>> {color}")
try:
threads = int(input(f"\n{white}[{color}^{white}] {color}Threads {white}>> {color}"))
except ValueError:
print(f"{white}[{color}!{white}] {color}Thread needs a number")
time.sleep(3)
main()
exit()
elif threads.isdigit() == False:
print(f"{white}[{Colors.red}!{white}] {Colors.red}Error{white} in settings.json at threads")
input()
exit()
try:
threads = int(threads)
except ValueError:
print(f"{white}[{color}!{white}] {color}Thread needs a number")
time.sleep(3)
main()
exit()
if threads < 1:
print(f"{white}[{color}!{white}] {color}Thread needs a number greater than 0")
time.sleep(3)
@@ -127,7 +144,6 @@ def main():
print()
start = time.time()
with open(config) as sites:
sitelist = sites.readlines()
@@ -138,19 +154,16 @@ def main():
threading.Thread(target=scrape, args=[sitelist[0]]).start()
threadcount += 1
sitelist.pop(0)
# terminal(f"| Remaining sites {len(sitelist)} | active threads {threadcount} | Proxies {proxycount} | Time {time.time()-start:.2f}s")
while threadcount > 0:
try:
terminal(f"| Waiting for threads to finish | active threads {threadcount} | Proxies {proxycount} | Time {time.time()-start:.2f}s")
except KeyboardInterrupt:
threadcount -= 1 #dont use it can cause errors
terminal(f"| Waiting for threads to finish | active threads {threadcount} | Proxies {proxycount} | Time {time.time()-start:.2f}s")
terminal()
print(f"\n{white}[{color}^{white}] Removed {color}Duplicates\n")
lenproxies = len(proxies)
print(f"\n{white}[{color}^{white}] Removed {color}{proxycount-lenproxies} {white}Duplicates\n")
print(f"{white}[{color}^{white}] Remaining Proxies: {color}{lenproxies}{white}\n")
print(f"{white}[{color}^{white}] Writing {color}Proxies\n")
with open("proxies.txt", "w") as output:
@@ -158,7 +171,7 @@ def main():
for i in proxies:
output.write(i.replace("\n", "") + "\n")
if clearing == True:
if clearingcnt == True:
print(f"{white}[{color}^{white}] Removing {color}not reachable Websites\n")
@@ -170,7 +183,6 @@ def main():
terminal()
print(f"{white}[{color}^{white}] Finished in {color}{time.time()-start:.2f}s{white}!\n")
print(f"{white}[{color}^{white}] Scraped {color}{len(proxies)} {white}Proxies\n")
print("You can now close the tab")
input("")
@@ -183,6 +195,7 @@ def scrape(site: str):
uas=['Mozilla/5.0 (X11; CrOS x86_64 14588.123.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.72 Safari/537.36', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 12.4; rv:101.0) Gecko/20100101 Firefox/101.0', 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36', 'Mozilla/5.0 (Linux; Android 12; SM-G960U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.99 Mobile Safari/537.36']
site = site.replace("\n", "")
finished = False
localproxycount = 0
try:
with httpx.Client(http2=True,headers = {'accept-language': 'en','user-agent':random.choice(uas)},follow_redirects=True) as client:
r = client.get(site, timeout=10).text
@@ -190,7 +203,7 @@ def scrape(site: str):
print(f"{white}[{Colors.red}!{white}] Failed connecting to {color}{site}")
else:
print(f"{white}[{Colors.green}+{white}] Scraping {color}{site}")
if clearing == True:
if clearingcnt == True:
goodsites.add(site)
pattern = r'^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+$'
if r.count(":") > 0 and r.count(".") > 2:
@@ -203,6 +216,7 @@ def scrape(site: str):
proxy = f"{r[pos-ip:pos]}{r[pos:pos+port]}"
if re.match(pattern, proxy):
proxies.add(proxy)
localproxycount = 1
proxycount += 1
finished = True
break
@@ -211,7 +225,10 @@ def scrape(site: str):
finished = False
break
r = r.replace(":", "", 1)
if site in goodsites and localproxycount == 0:
print(f"{white}[{Colors.orange}!{white}] No proxies were found on {color}{site}")
if clearingproxy == True:
goodsites.remove(site)
threadcount -= 1
def terminal(string:str = ""):
settings.yaml
+4 -2
View File
@@ -1,7 +1,9 @@
{
premades: "?", # '?' for asking | 'n' for sites.txt | '1' for http/s | '2' for socks4 | '3' for socks5
premades: "n", # '?' for asking | 'n' for sites.txt | '1' for http/s | '2' for socks4 | '3' for socks5
removewebsites: "?", # '?' for asking | 'y' or 'n'
removewebsites: "y", # Removes not connectable websites | '?' for asking | 'y' or 'n'
removeproxyless: "y", # Removes websites if no proxies are found | '?' for asking | 'y' or 'n'
threads: "?" # '?' for asking | '[Threadcount]'
}