Skip to content

Commit

Permalink
Remove HttpRequest.from_form, cover form2request in the HttpRequest r…
Browse files Browse the repository at this point in the history 
…eference
  • Loading branch information
@Gallaecio
Gallaecio committed Jul 15, 2024
1 parent a7f6054 commit 707ce57
Show file tree
Hide file tree
Showing 4 changed files with 5 additions and 246 deletions.
1 change: 0 additions & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
entry_points={"pytest11": ["web-poet = web_poet.testing.pytest"]},
install_requires=[
"attrs >= 21.3.0",
"form2request >= 0.1.1",
"parsel >= 1.8.1",
"url-matcher >= 0.2.0",
"multidict >= 0.5.0",
Expand Down
190 changes: 0 additions & 190 deletions tests/test_page_inputs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import parsel
import pytest
import requests
from parsel import Selector

from web_poet import BrowserResponse, RequestUrl, ResponseUrl
from web_poet.page_inputs import (
Expand Down Expand Up @@ -212,195 +211,6 @@ def test_http_request_init_with_response_url() -> None:
assert str(req.url) == str(resp.url)


def test_http_request_from_form_get() -> None:
url = "https://example.com"
response = HttpResponse(
url,
b"""
<!doctype html>
<title>a</title>
<form id="search-form" accept-charset="utf-8" action="/search" method="GET">
<input type="text" value="" name="query">
<select name="bar">
<option value="code">Barcode</option>
<option selected="selected" value="tender">Bartender</option>
</select>
<input type="submit">
<input type="hidden" name="baz" value="ooka">
</form>
""",
)
form_selector = response.css("#search-form")[0]
request = HttpRequest.from_form(form_selector.root, {"query": "foo"})
assert (
str(request.url) == "https://example.com/search?bar=tender&baz=ooka&query=foo"
)
assert request.method == "GET"
assert request.headers == HttpRequestHeaders()
assert request.body == b""


def test_http_request_from_form_post() -> None:
url = "https://example.com"
response = HttpResponse(
url,
b"""
<!doctype html>
<title>a</title>
<form id="search-form" accept-charset="utf-8" action="/search" method="POST">
<input type="text" value="" name="query">
<select name="bar">
<option value="code">Barcode</option>
<option selected="selected" value="tender">Bartender</option>
</select>
<input type="submit">
<input type="hidden" name="baz" value="ooka">
</form>
""",
)
form_selector = response.css("#search-form")[0]
request = HttpRequest.from_form(form_selector.root, {"query": "foo"})
assert str(request.url) == "https://example.com/search"
assert request.method == "POST"
assert request.headers == HttpRequestHeaders(
{"Content-Type": "application/x-www-form-urlencoded"}
)
assert request.body == b"bar=tender&baz=ooka&query=foo"


def test_http_request_from_form_select_no_selected() -> None:
url = "https://example.com"
response = HttpResponse(
url,
b"""
<!doctype html>
<title>a</title>
<form id="search-form" accept-charset="utf-8" action="/search" method="POST">
<input type="text" value="" name="query">
<select name="bar">
<option value="code">Barcode</option>
<option value="tender">Bartender</option>
</select>
<input type="submit">
<input type="hidden" name="baz" value="ooka">
</form>
""",
)
form_selector = response.css("#search-form")[0]
request = HttpRequest.from_form(form_selector.root)
assert request.body == b"query=&bar=code&baz=ooka"


def test_http_request_from_form_select_no_options() -> None:
url = "https://example.com"
response = HttpResponse(
url,
b"""
<!doctype html>
<title>a</title>
<form id="search-form" accept-charset="utf-8" action="/search" method="POST">
<input type="text" value="" name="query">
<select name="bar"></select>
<input type="submit">
<input type="hidden" name="baz" value="ooka">
</form>
""",
)
form_selector = response.css("#search-form")[0]
request = HttpRequest.from_form(form_selector.root)
assert request.body == b"query=&baz=ooka"


def test_http_request_from_form_no_method() -> None:
url = "https://example.com"
response = HttpResponse(
url,
b"""
<!doctype html>
<title>a</title>
<form id="search-form" accept-charset="utf-8" action="/search">
<input type="text" value="" name="query">
<select name="bar">
<option value="code">Barcode</option>
<option value="tender">Bartender</option>
</select>
<input type="submit">
<input type="hidden" name="baz" value="ooka">
</form>
""",
)
form_selector = response.css("#search-form")[0]
request = HttpRequest.from_form(form_selector.root)
assert request.method == "GET"


def test_http_request_from_form_bad_method() -> None:
url = "https://example.com"
response = HttpResponse(
url,
b"""
<!doctype html>
<title>a</title>
<form id="search-form" accept-charset="utf-8" action="/search" method="PUT">
<input type="text" value="" name="query">
<select name="bar">
<option value="code">Barcode</option>
<option value="tender">Bartender</option>
</select>
<input type="submit">
<input type="hidden" name="baz" value="ooka">
</form>
""",
)
form_selector = response.css("#search-form")[0]
request = HttpRequest.from_form(form_selector.root)
assert request.method == "GET"


def test_http_request_from_form_no_base_url() -> None:
selector = Selector(
text="""
<!doctype html>
<title>a</title>
<form id="search-form" accept-charset="utf-8" action="/search" method="PUT">
<input type="text" value="" name="query">
<select name="bar">
<option value="code">Barcode</option>
<option value="tender">Bartender</option>
</select>
<input type="submit">
<input type="hidden" name="baz" value="ooka">
</form>
""",
)
form_selector = selector.css("#search-form")[0]
with pytest.raises(ValueError):
HttpRequest.from_form(form_selector.root)


def test_http_request_from_form_no_action() -> None:
url = "https://example.com"
response = HttpResponse(
url,
b"""
<!doctype html>
<title>a</title>
<form id="search-form" accept-charset="utf-8" method="POST">
<input type="text" value="" name="query">
<select name="bar">
<option value="code">Barcode</option>
<option value="tender">Bartender</option>
</select>
<input type="submit">
<input type="hidden" name="baz" value="ooka">
</form>
""",
)
form_selector = response.css("#search-form")[0]
request = HttpRequest.from_form(form_selector.root)
assert str(request.url) == url


@pytest.mark.parametrize(
"cls",
(
Expand Down
1 change: 0 additions & 1 deletion tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,6 @@ deps =
zyte-common-items==0.1.0
aiohttp==3.7.0
attrs==21.3.0
form2request==0.1.1
parsel==1.8.1
url-matcher==0.2.0
tldextract==3.0.0
Expand Down
59 changes: 5 additions & 54 deletions web_poet/page_inputs/http.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,10 @@

import json
from hashlib import sha1
from typing import (
TYPE_CHECKING,
Any,
Dict,
Iterable,
List,
Optional,
Tuple,
TypeVar,
Union,
)
from typing import Any, Optional, TypeVar, Union
from urllib.parse import urljoin

import attrs
from form2request import form2request
from w3lib.encoding import (
html_body_declared_encoding,
html_to_unicode,
Expand All @@ -33,17 +22,6 @@
from .url import RequestUrl as _RequestUrl
from .url import ResponseUrl as _ResponseUrl

if TYPE_CHECKING:
# typing.Self requires Python 3.11
from lxml.html import FormElement # nosec
from lxml.html import HtmlElement # nosec
from parsel import Selector, SelectorList
from typing_extensions import Self

FormdataVType = Union[str, Iterable[str]]
FormdataKVType = Tuple[str, FormdataVType]
FormdataType = Optional[Union[Dict[str, FormdataVType], List[FormdataKVType]]]

T_headers = TypeVar("T_headers", bound=_HttpHeaders)


Expand Down Expand Up @@ -146,6 +124,10 @@ def declared_encoding(self) -> Optional[str]:
class HttpRequest:
"""Represents a generic HTTP request used by other functionalities in
**web-poet** like :class:`~.HttpClient`.
.. tip:: To build a request to submit an HTML form, use the
:doc:`form2request library <form2request:index>`, which provides
integration with web-poet.
"""

url: _RequestUrl = attrs.field(converter=_RequestUrl)
Expand All @@ -157,37 +139,6 @@ class HttpRequest:
factory=HttpRequestBody, converter=HttpRequestBody, kw_only=True
)

@classmethod
def from_form(
cls,
form: FormElement | Selector | SelectorList,
data: FormdataType = None,
*,
click: None | bool | HtmlElement = None,
method: None | str = None,
enctype: None | str = None,
**kwargs,
) -> Self:
"""Return an :class:`HttpRequest` to submit an HTML form.
See the :doc:`form2request usage documentation <form2request:usage>`
and :func:`~form2request.form2request` for parameter reference.
"""
request_data = form2request(
form,
data,
click=click,
method=method,
enctype=enctype,
**kwargs,
)
return cls(
url=request_data.url,
method=request_data.method,
headers=request_data.headers,
body=request_data.body,
)

def urljoin(self, url: Union[str, _RequestUrl, _ResponseUrl]) -> _RequestUrl:
"""Return *url* as an absolute URL.
Expand Down

0 comments on commit 707ce57

Please sign in to comment.