diff --git a/KC-Scraper.py b/KC-Scraper.py index 729c351..da2b84a 100644 --- a/KC-Scraper.py +++ b/KC-Scraper.py @@ -16,19 +16,21 @@ goodsites = set([]) sitelist = [] proxycount = 0 threadcount = 0 -clearing = "" +clearingcnt = "" +clearingproxy = "" white = Colors.light_gray color = Colors.StaticMIX((Colors.purple, Colors.blue)) def main(): - global threadcount, clearing, sitelist, start + global threadcount, clearingcnt, sitelist, start with open("settings.yaml") as setting: settings = yaml.safe_load(setting.read()) premades = settings["premades"] - clearing = settings["removewebsites"] + clearingcnt = settings["removewebsites"] + clearingproxy = settings["removeproxyless"] threads = settings["threads"] terminal() @@ -84,40 +86,55 @@ def main(): os.system("cls") print(Colorate.Diagonal(Colors.DynamicMIX((Colors.dark_gray, Colors.StaticMIX((Colors.purple, Colors.blue)))), Center.XCenter(logo))) - if clearing == "?": + if clearingcnt == "?": - clearing = input(f"\n\n\n{white}[{color}^{white}] {color}Remove not connectable site [y/n] {white}>> {color}") + clearingcnt = input(f"\n\n\n{white}[{color}^{white}] {color}Remove not connectable site [y/n] {white}>> {color}") - if clearing != "y" and clearing != "ye" and clearing != "yes" and clearing != "n" and clearing != "no": + if clearingcnt != "y" and clearingcnt != "ye" and clearingcnt != "yes" and clearingcnt != "n" and clearingcnt != "no": print(f"{white}[{color}!{white}] {color}No option was choosen returning to home..") time.sleep(3) main() exit() - elif clearing != "y" and clearing != "ye" and clearing != "yes" and clearing != "n" and clearing != "no": + elif clearingcnt != "y" and clearingcnt != "ye" and clearingcnt != "yes" and clearingcnt != "n" and clearingcnt != "no": print(f"{white}[{Colors.red}!{white}] {Colors.red}Error{white} in settings.json at removewebsites") input() exit() - if clearing == "y" or clearing == "ye" or clearing == "yes": - clearing = True + if clearingcnt == "y" or clearingcnt == "ye" or clearingcnt == "yes": + clearingcnt = True + + if clearingproxy == "?": + clearingproxy = input(f"\n{white}[{color}^{white}] {color}Remove sites with no proxies [y/n] {white}>> {color}") + + if clearingproxy != "y" and clearingproxy != "ye" and clearingproxy != "yes" and clearingproxy != "n" and clearingproxy != "no": + print(f"{white}[{color}!{white}] {color}No option was choosen returning to home..") + time.sleep(3) + main() + exit() + + elif clearingproxy != "y" and clearingproxy != "ye" and clearingproxy != "yes" and clearingproxy != "n" and clearingproxy != "no": + print(f"{white}[{Colors.red}!{white}] {Colors.red}Error{white} in settings.json at removeproxyless") + input() + exit() + + if clearingproxy == "y" or clearingproxy == "ye" or clearingproxy == "yes": + clearingproxy = True if threads == "?": - threads = input(f"{white}[{color}^{white}] {color}Threads {white}>> {color}") + try: + threads = int(input(f"\n{white}[{color}^{white}] {color}Threads {white}>> {color}")) + except ValueError: + print(f"{white}[{color}!{white}] {color}Thread needs a number") + time.sleep(3) + main() + exit() + elif threads.isdigit() == False: print(f"{white}[{Colors.red}!{white}] {Colors.red}Error{white} in settings.json at threads") input() exit() - - try: - threads = int(threads) - except ValueError: - print(f"{white}[{color}!{white}] {color}Thread needs a number") - time.sleep(3) - main() - exit() - if threads < 1: print(f"{white}[{color}!{white}] {color}Thread needs a number greater than 0") time.sleep(3) @@ -127,7 +144,6 @@ def main(): print() start = time.time() - with open(config) as sites: sitelist = sites.readlines() @@ -138,19 +154,16 @@ def main(): threading.Thread(target=scrape, args=[sitelist[0]]).start() threadcount += 1 sitelist.pop(0) -# terminal(f"| Remaining sites {len(sitelist)} | active threads {threadcount} | Proxies {proxycount} | Time {time.time()-start:.2f}s") while threadcount > 0: - try: - terminal(f"| Waiting for threads to finish | active threads {threadcount} | Proxies {proxycount} | Time {time.time()-start:.2f}s") - except KeyboardInterrupt: - threadcount -= 1 #dont use it can cause errors + terminal(f"| Waiting for threads to finish | active threads {threadcount} | Proxies {proxycount} | Time {time.time()-start:.2f}s") terminal() - - print(f"\n{white}[{color}^{white}] Removed {color}Duplicates\n") + lenproxies = len(proxies) + print(f"\n{white}[{color}^{white}] Removed {color}{proxycount-lenproxies} {white}Duplicates\n") + print(f"{white}[{color}^{white}] Remaining Proxies: {color}{lenproxies}{white}\n") print(f"{white}[{color}^{white}] Writing {color}Proxies\n") with open("proxies.txt", "w") as output: @@ -158,7 +171,7 @@ def main(): for i in proxies: output.write(i.replace("\n", "") + "\n") - if clearing == True: + if clearingcnt == True: print(f"{white}[{color}^{white}] Removing {color}not reachable Websites\n") @@ -170,7 +183,6 @@ def main(): terminal() print(f"{white}[{color}^{white}] Finished in {color}{time.time()-start:.2f}s{white}!\n") - print(f"{white}[{color}^{white}] Scraped {color}{len(proxies)} {white}Proxies\n") print("You can now close the tab") input("") @@ -183,6 +195,7 @@ def scrape(site: str): uas=['Mozilla/5.0 (X11; CrOS x86_64 14588.123.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.72 Safari/537.36', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 12.4; rv:101.0) Gecko/20100101 Firefox/101.0', 'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36', 'Mozilla/5.0 (Linux; Android 12; SM-G960U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.99 Mobile Safari/537.36'] site = site.replace("\n", "") finished = False + localproxycount = 0 try: with httpx.Client(http2=True,headers = {'accept-language': 'en','user-agent':random.choice(uas)},follow_redirects=True) as client: r = client.get(site, timeout=10).text @@ -190,7 +203,7 @@ def scrape(site: str): print(f"{white}[{Colors.red}!{white}] Failed connecting to {color}{site}") else: print(f"{white}[{Colors.green}+{white}] Scraping {color}{site}") - if clearing == True: + if clearingcnt == True: goodsites.add(site) pattern = r'^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\:[0-9]+$' if r.count(":") > 0 and r.count(".") > 2: @@ -203,6 +216,7 @@ def scrape(site: str): proxy = f"{r[pos-ip:pos]}{r[pos:pos+port]}" if re.match(pattern, proxy): proxies.add(proxy) + localproxycount = 1 proxycount += 1 finished = True break @@ -211,7 +225,10 @@ def scrape(site: str): finished = False break r = r.replace(":", "", 1) - + if site in goodsites and localproxycount == 0: + print(f"{white}[{Colors.orange}!{white}] No proxies were found on {color}{site}") + if clearingproxy == True: + goodsites.remove(site) threadcount -= 1 def terminal(string:str = ""): diff --git a/settings.yaml b/settings.yaml index dbfecdf..917e0e1 100644 --- a/settings.yaml +++ b/settings.yaml @@ -1,7 +1,9 @@ { - premades: "?", # '?' for asking | 'n' for sites.txt | '1' for http/s | '2' for socks4 | '3' for socks5 + premades: "n", # '?' for asking | 'n' for sites.txt | '1' for http/s | '2' for socks4 | '3' for socks5 - removewebsites: "?", # '?' for asking | 'y' or 'n' + removewebsites: "y", # Removes not connectable websites | '?' for asking | 'y' or 'n' + + removeproxyless: "y", # Removes websites if no proxies are found | '?' for asking | 'y' or 'n' threads: "?" # '?' for asking | '[Threadcount]' } \ No newline at end of file