Add CheckWebsite setting

Fixes #16, fixes #39

proxy_scraper_checker/proxy.py

@@ -7,6 +7,8 @@ from aiohttp import ClientSession, ClientTimeout
 from aiohttp.abc import AbstractCookieJar
 from aiohttp_socks import ProxyConnector, ProxyType
 
+from .constants import USER_AGENT
+
 
 class Proxy:
     __slots__ = ("geolocation", "host", "is_anonymous", "port", "timeout")
@@ -15,30 +17,41 @@ class Proxy:
         self.host = host
         self.port = port
 
+    @property
+    def default_check_website(self) -> str:
+        return "http://ip-api.com/json/?fields=8217"
+
     async def check(
         self,
         *,
+        website: str,
         sem: asyncio.Semaphore,
         cookie_jar: AbstractCookieJar,
         proto: ProxyType,
         timeout: ClientTimeout,
     ) -> None:
+        check_website = (
+            self.default_check_website if website == "default" else website
+        )
         async with sem:
             start = perf_counter()
-            async with self.get_connector(proto) as connector:
-                async with ClientSession(
-                    connector=connector, cookie_jar=cookie_jar, timeout=timeout
-                ) as session:
-                    async with session.get(
-                        "http://ip-api.com/json/?fields=8217",
-                        raise_for_status=True,
-                    ) as response:
-                        data = await response.json()
+            async with self.get_connector(proto) as connector, ClientSession(
+                connector=connector,
+                cookie_jar=cookie_jar,
+                timeout=timeout,
+                headers={"User-Agent": USER_AGENT},
+            ) as session, session.get(
+                check_website, raise_for_status=True
+            ) as response:
+                if website == "default":
+                    await response.read()
         self.timeout = perf_counter() - start
-        self.is_anonymous = self.host != data["query"]
-        self.geolocation = "|{}|{}|{}".format(
-            data["country"], data["regionName"], data["city"]
-        )
+        if website == "default":
+            data = await response.json()
+            self.is_anonymous = self.host != data["query"]
+            self.geolocation = "|{}|{}|{}".format(
+                data["country"], data["regionName"], data["city"]
+            )
 
     def get_connector(self, proto: ProxyType) -> ProxyConnector:
         return ProxyConnector(proxy_type=proto, host=self.host, port=self.port)