chore: pytest

Author: thewebscraping

README.md

@@ -44,6 +44,31 @@ Start using TLS Requests with just a few lines of code:
 200
 ```
 
+Basic automatically rotates:
+
+```pycon
+>>> import tls_requests
+>>> proxy_list = [
+    "http://user1:pass1@proxy.example.com:8080",
+    "http://user2:pass2@proxy.example.com:8081",
+    "socks5://proxy.example.com:8082",
+    "proxy.example.com:8083",  #  (defaults to http)
+    "http://user:pass@proxy.example.com:8084|1.0|US",  #  http://user:pass@host:port|weight|region
+]
+>>> r = tls_requests.get(
+    "https://httpbin.org/get",
+    proxy=proxy,
+    headers=tls_requests.HeaderRotator(),
+    tls_identifier=tls_requests.TLSIdentifierRotator()
+)
+>>> r
+<Response [200 OK]>
+>>> r.status_code
+200
+>>> tls_requests.HeaderRotator(strategy = "round_robin")  # strategy: Literal["round_robin", "random", "weighted"]
+>>> tls_requests.Proxy("http://user1:pass1@proxy.example.com:8080", weight=0.1)  # default weight: 1.0
+```
+
 **Introduction**
 ----------------
 

tests/test_headers.py

@@ -23,18 +23,18 @@ def test_request_headers(httpserver: HTTPServer):
     httpserver.expect_request("/headers").with_post_hook(hook_request_headers).respond_with_data(b"OK")
     response = tls_requests.get(httpserver.url_for("/headers"), headers={"foo": "bar"})
     assert response.status_code == 200
-    assert response.headers.get("foo") == "bar"
+    assert response.request.headers["foo"] == "bar"
 
 
 def test_response_headers(httpserver: HTTPServer):
     httpserver.expect_request("/headers").with_post_hook(hook_response_headers).respond_with_data(b"OK")
     response = tls_requests.get(httpserver.url_for("/headers"))
     assert response.status_code, 200
-    assert response.headers.get("foo") == "bar"
+    assert response.headers["foo"] == "bar"
 
 
 def test_response_case_insensitive_headers(httpserver: HTTPServer):
     httpserver.expect_request("/headers").with_post_hook(hook_response_case_insensitive_headers).respond_with_data(b"OK")
     response = tls_requests.get(httpserver.url_for("/headers"))
     assert response.status_code, 200
-    assert response.headers.get("foo") == "bar"
+    assert response.headers["foo"] == "bar"

tests/test_rotators.py

@@ -183,7 +183,7 @@ async def test_async_strategies(self, strategy, proxy_list_fixture):
 
 class TestProxyRotator:
     def test_mark_result_weighted(self):
-        proxy = Proxy("p1:8080", weight=2.0)
+        proxy = Proxy("proxy.example.com:8080", weight=2.0)
         rotator = ProxyRotator([proxy], strategy="weighted")
 
         initial_weight = proxy.weight
@@ -196,7 +196,7 @@ def test_mark_result_weighted(self):
 
     @pytest.mark.asyncio
     async def test_async_mark_result_weighted(self):
-        proxy = Proxy("p1:8080", weight=2.0)
+        proxy = Proxy("proxy.example.com:8080", weight=2.0)
         rotator = ProxyRotator([proxy], strategy="weighted")
 
         initial_weight = proxy.weight

tls_requests/client.py

@@ -193,12 +193,11 @@ def prepare_auth(
 
     def prepare_headers(self, headers: HeaderTypes = None, user_agent: Optional[str] = None) -> Headers:
         """Prepare Headers. Gets base headers from rotator if available."""
+        if headers is None:
+            return self.headers.copy()
         if isinstance(headers, HeaderRotator):
-            headers_copy = headers.next(user_agent=user_agent)
-        else:
-            headers_copy = self.headers.copy()
-
-        return headers_copy
+            return headers.next(user_agent=user_agent)
+        return Headers(headers)
 
     def prepare_cookies(self, cookies: CookieTypes = None) -> Cookies:
         """Prepare Cookies"""
@@ -371,7 +370,8 @@ def _send(
         self, request: Request, *, history: list = None, start: float = None
     ) -> Response:
         start = start or time.perf_counter()
-        config = self.prepare_config(request, tls_identifier=self.prepare_tls_identifier(self.client_identifier))
+        tls_identifier = self.prepare_tls_identifier(self.client_identifier)
+        config = self.prepare_config(request, tls_identifier=tls_identifier)
         response = Response.from_tls_response(
             self.session.request(config.to_dict()),
             is_byte_response=config.isByteResponse,
@@ -515,6 +515,12 @@ def send(
         self.follow_redirects = follow_redirects
         response = self._send(request, start=time.perf_counter(), history=[])
 
+        if isinstance(self.proxy, ProxyRotator) and response.request.proxy:
+            proxy_success = 200 <= response.status_code < 500 and response.status_code not in [407]
+            self.proxy.mark_result(
+                proxy=response.request.proxy, success=proxy_success, latency=response.elapsed
+            )
+
         if self.hooks.get("response"):
             response_ = self.build_hook_response(response)
             if isinstance(response_, Response):
@@ -757,12 +763,11 @@ class AsyncClient(BaseClient):
 
     async def aprepare_headers(self, headers: HeaderTypes = None, user_agent: Optional[str] = None) -> Headers:
         """Prepare Headers. Gets base headers from rotator if available."""
+        if headers is None:
+            return self.headers.copy()
         if isinstance(headers, HeaderRotator):
-            headers_copy = await headers.anext(user_agent=user_agent)
-        else:
-            headers_copy = self.headers.copy()
-
-        return headers_copy
+            return await headers.anext(user_agent=user_agent)
+        return Headers(headers)
 
     async def aprepare_proxy(self, proxy: ProxyTypes | None) -> Optional[Proxy]:
         if proxy is None:
@@ -1070,6 +1075,12 @@ async def send(
         self.follow_redirects = follow_redirects
         response = await self._send(request, start=time.perf_counter(), history=[])
 
+        if isinstance(self.proxy, ProxyRotator) and response.request.proxy:
+            proxy_success = 200 <= response.status_code < 500 and response.status_code not in [407]
+            await self.proxy.amark_result(
+                proxy=response.request.proxy, success=proxy_success, latency=response.elapsed
+            )
+
         if self.hooks.get("response"):
             response_ = self.build_hook_response(response)
             if isinstance(response_, Response):
@@ -1084,7 +1095,8 @@ async def _send(
         self, request: Request, *, history: list = None, start: float = None
     ) -> Response:
         start = start or time.perf_counter()
-        config = self.prepare_config(request, tls_identifier=await self.aprepare_tls_identifier(self.client_identifier))
+        tls_identifier = await self.aprepare_tls_identifier(self.client_identifier)
+        config = self.prepare_config(request, tls_identifier=tls_identifier)
         response = Response.from_tls_response(
             await self.session.arequest(config.to_dict()),
             is_byte_response=config.isByteResponse,

tls_requests/models/rotators.py

@@ -16,60 +16,21 @@
 
 T = TypeVar("T")
 
-TLS_IDENTIFIER_TEMPLATES = [
-    "chrome_103",
-    "chrome_104",
-    "chrome_105",
-    "chrome_106",
-    "chrome_107",
-    "chrome_108",
-    "chrome_109",
-    "chrome_110",
-    "chrome_111",
-    "chrome_112",
-    "chrome_116_PSK",
-    "chrome_116_PSK_PQ",
-    "chrome_117",
+TLS_IDENTIFIER_TEMPLATES: list[str] = sorted([
     "chrome_120",
     "chrome_124",
-    "safari_15_6_1",
+    "chrome_131",
+    "chrome_133",
+    "firefox_120",
+    "firefox_123",
+    "firefox_132",
+    "firefox_133",
     "safari_16_0",
-    "safari_ios_15_5",
-    "safari_ios_15_6",
     "safari_ios_16_0",
-    "firefox_102",
-    "firefox_104",
-    "firefox_105",
-    "firefox_106",
-    "firefox_108",
-    "firefox_110",
-    "firefox_117",
-    "firefox_120",
-    "opera_89",
-    "opera_90",
-    "opera_91",
-    "okhttp4_android_7",
-    "okhttp4_android_8",
-    "okhttp4_android_9",
-    "okhttp4_android_10",
-    "okhttp4_android_11",
-    "okhttp4_android_12",
-    "okhttp4_android_13",
-    "zalando_ios_mobile",
-    "zalando_android_mobile",
-    "nike_ios_mobile",
-    "nike_android_mobile",
-    "mms_ios",
-    "mms_ios_2",
-    "mms_ios_3",
-    "mesh_ios",
-    "mesh_ios_2",
-    "mesh_android",
-    "mesh_android_2",
-    "confirmed_ios",
-    "confirmed_android",
-    "confirmed_android_2",
-]
+    "safari_ios_17_0",
+    "safari_ios_18_0",
+    "safari_ios_18_5",
+])
 
 USER_AGENTS = [
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
@@ -92,10 +53,9 @@
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0",
     "Mozilla/5.0 (Linux; Android 14; SM-G998B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36",
     "Mozilla/5.0 (Linux; Android 14; SM-S918B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Mobile Safari/537.36",
-    "Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1",
     "Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Mobile/15E148 Safari/604.1",
     "Mozilla/5.0 (Linux; Android 15; SM-S931B Build/AP3A.240905.015.A2; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/127.0.6533.103 Mobile Safari/537.36",
-    "Mozila/5.0 (Linux; Android 14; SM-S928B/DS) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6099.230 Mobile Safari/537.36",
+    "Mozilla/5.0 (Linux; Android 14; SM-S928B/DS) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6099.230 Mobile Safari/537.36",
     "Mozilla/5.0 (Linux; Android 14; SM-F956U) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/80.0.3987.119 Mobile Safari/537.36",
     "Mozilla/5.0 (Linux; Android 13; SM-S911U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Mobile Safari/537.36",
     "Mozilla/5.0 (Linux; Android 13; SM-S901B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Mobile Safari/537.36",
@@ -318,11 +278,9 @@ class ProxyRotator(BaseRotator[Proxy]):
     def rebuild_item(cls, item: Any) -> Optional[Proxy]:
         """Constructs a `Proxy` object from various input types."""
         try:
-            if isinstance(item, Proxy):
-                return item
             if isinstance(item, dict):
                 return Proxy.from_dict(item)
-            if isinstance(item, str):
+            if isinstance(item, (str, Proxy)):
                 return Proxy.from_string(item)
         except Exception:
             return None
@@ -414,7 +372,7 @@ class HeaderRotator(BaseRotator[Headers]):
     def __init__(
         self,
         items: Optional[Iterable[T]] = None,
-        strategy: Literal["round_robin", "random", "weighted"] = "round_robin",
+        strategy: Literal["round_robin", "random", "weighted"] = "random",
     ) -> None:
         super().__init__(items or HEADER_TEMPLATES, strategy)
 
@@ -434,11 +392,9 @@ def rebuild_item(cls, item: HeaderTypes) -> Optional[Headers]:
         try:
             if isinstance(item, Headers):
                 return item
-            if isinstance(item, (dict, list)):
-                return Headers(item)
+            return Headers(item)
         except Exception:
             return None
-        return None
 
     def next(self, user_agent: Optional[str] = None) -> Headers:
         """
@@ -452,12 +408,12 @@ def next(self, user_agent: Optional[str] = None) -> Headers:
         Returns:
             A copy of the next `Headers` object, potentially with a modified User-Agent.
         """
-        base_headers = super().next()
-        headers_copy = base_headers.copy()
-
+        headers = super().next()
+        headers_copy = headers.copy()
+        if not isinstance(headers_copy, Headers):
+            headers_copy = Headers(headers_copy)
         if user_agent:
             headers_copy["User-Agent"] = user_agent
-
         return headers_copy
 
     async def anext(self, user_agent: Optional[str] = None) -> Headers:
@@ -472,8 +428,10 @@ async def anext(self, user_agent: Optional[str] = None) -> Headers:
         Returns:
             A copy of the next `Headers` object, potentially with a modified User-Agent.
         """
-        base_headers = await super().anext()
-        headers_copy = base_headers.copy()
+        headers = await super().anext()
+        headers_copy = headers.copy()
+        if not isinstance(headers_copy, Headers):
+            headers_copy = Headers(headers_copy)
         if user_agent:
             headers_copy["User-Agent"] = user_agent
         return headers_copy

tls_requests/models/urls.py

@@ -455,18 +455,18 @@ def __init__(
         Raises:
             ProxyError: If the URL is invalid or the scheme is not supported.
         """
-        super().__init__(url, **kwargs)
-        self.weight = weight
+        self.weight = weight or 1.0
         self.region = region
         self.latency = latency
         self.success_rate = success_rate
         self.meta = meta or {}
         self.failures: int = 0
         self.last_used: Optional[float] = None
+        super().__init__(url, **kwargs)
 
     def __repr__(self):
         """Returns a secure representation of the proxy with its weight."""
-        return "<%s: %s, weight=%s>" % (self.__class__.__name__, unquote(self._build(True)), self.weight)
+        return "<%s: %s, weight=%s>" % (self.__class__.__name__, unquote(self._build(True)), getattr(self, "weight", "unset"))
 
     def _prepare(self, url: ProxyTypes) -> ParseResult:
         """
@@ -490,6 +490,9 @@ def _prepare(self, url: ProxyTypes) -> ParseResult:
             if isinstance(url, str):
                 url = url.strip()
 
+            if "://" not in str(url):
+                url = f"http://{url}"
+
             parsed = super(Proxy, self)._prepare(url)
             if str(parsed.scheme).lower() not in self.ALLOWED_SCHEMES:
                 raise ProxyError(
@@ -632,13 +635,13 @@ def from_string(cls, raw: str, separator: str = "|") -> "Proxy":
 
         parts = [p.strip() for p in raw.split(separator)]
         url = parts[0]
-        weight = None
+        weight = 1.0
         region = None
         if len(parts) >= 2 and parts[1]:
             try:
                 weight = float(parts[1])
             except Exception:
-                weight = None
+                pass
         if len(parts) >= 3 and parts[2]:
             region = parts[2]
 

tls_requests/settings.py

@@ -9,7 +9,7 @@
 DEFAULT_TLS_DEBUG = False
 DEFAULT_TLS_INSECURE_SKIP_VERIFY = False
 DEFAULT_TLS_HTTP2 = "auto"
-DEFAULT_TLS_IDENTIFIER = "chrome_120"
+DEFAULT_TLS_IDENTIFIER = "chrome_133"
 DEFAULT_HEADERS = {
     "accept": "*/*",
     "connection": "keep-alive",