use yarl underneath ResponseURL and RequestURL

Author: BurnzZ

setup.py

@@ -25,6 +25,7 @@
         'url-matcher',
         'multidict',
         'w3lib >= 1.22.0',
+        'yarl',
     ],
     classifiers=[
         'Development Status :: 2 - Pre-Alpha',

tests/test_page_inputs.py

@@ -5,6 +5,8 @@
 import requests
 
 from web_poet.page_inputs import (
+    ResponseURL,
+    RequestURL,
     HttpRequest,
     HttpResponse,
     HttpRequestBody,
@@ -14,6 +16,22 @@
 )
 
 
+@pytest.mark.parametrize("cls", [ResponseURL, RequestURL])
+def test_url(cls):
+    url_value = "https://example.com/category/product?query=123&id=xyz#frag1"
+
+    url = cls(url_value)
+
+    assert str(url) == url_value
+    assert url.scheme == "https"
+    assert url.host == "example.com"
+    assert url.path == "/category/product"
+    assert url.query_string == "query=123&id=xyz"
+    assert url.fragment == "frag1"
+
+    new_url = cls(url)
+
+
 @pytest.mark.parametrize("body_cls", [HttpRequestBody, HttpResponseBody])
 def test_http_body_hashable(body_cls):
     http_body = body_cls(b"content")

web_poet/.overrides.py.swp

@@ -42,7 +42,7 @@ def base_url(self) -> str:
         # FIXME: move it to HttpResponse
         if self._cached_base_url is None:
             text = self.html[:4096]
-            self._cached_base_url = get_base_url(text, self.url)
+            self._cached_base_url = get_base_url(text, str(self.url))
         return self._cached_base_url
 
     def urljoin(self, url: str) -> str:

web_poet/mixins.py

@@ -1,6 +1,8 @@
 from .meta import Meta
 from .client import HttpClient
 from .http import (
+    ResponseURL,
+    RequestURL,
     HttpRequest,
     HttpResponse,
     HttpRequestHeaders,

web_poet/page_inputs/__init__.py

@@ -10,6 +10,7 @@
     http_content_type_encoding
 )
 
+import yarl
 from web_poet._base import _HttpHeaders
 from web_poet.utils import memoizemethod_noargs
 
@@ -18,12 +19,46 @@
 _AnyStrDict = Dict[AnyStr, Union[AnyStr, List[AnyStr], Tuple[AnyStr, ...]]]
 
 
-class ResponseURL(str):
+class _URL:
+    def __init__(self, url: Union[str, yarl.URL]):
+        self._url = yarl.URL(str(url))
+
+    def __str__(self) -> str:
+        return str(self._url)
+
+    def __repr__(self) -> str:
+        return str(self._url)
+
+    def __eq__(self, other) -> bool:
+        return str(self._url) == str(other)
+
+    @property
+    def scheme(self) -> str:
+        return self._url.scheme
+
+    @property
+    def host(self) -> str:
+        return self._url.host
+
+    @property
+    def path(self) -> str:
+        return self._url.path
+
+    @property
+    def query_string(self) -> str:
+        return self._url.query_string
+
+    @property
+    def fragment(self) -> str:
+        return self._url.fragment
+
+
+class ResponseURL(_URL):
     """ URL of the response """
     pass
 
 
-class RequestURL(str):
+class RequestURL(_URL):
     """ URL of the request """
     pass