diff --git a/setup.py b/setup.py index 8ba4b303..12a15ca4 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,6 @@ python_requires=">=3.7", install_requires=[ "incremental", - "requests >= 2.1.0", "hyperlink >= 21.0.0", "Twisted[tls] >= 22.10.0", # For #11635 "attrs", diff --git a/src/treq/_types.py b/src/treq/_types.py index b758f5e9..673e08cd 100644 --- a/src/treq/_types.py +++ b/src/treq/_types.py @@ -1,6 +1,7 @@ # Copyright (c) The treq Authors. # See LICENSE for details. import io +from .cookies import TreqieJar from http.cookiejar import CookieJar from typing import Any, Dict, Iterable, List, Mapping, Tuple, Union @@ -48,6 +49,7 @@ class _ITreqReactor(IReactorTCP, IReactorTime, IReactorPluggableNameResolver): ] _CookiesType = Union[ + TreqieJar, CookieJar, Mapping[str, str], ] diff --git a/src/treq/api.py b/src/treq/api.py index 19e551f9..6730296f 100644 --- a/src/treq/api.py +++ b/src/treq/api.py @@ -1,9 +1,23 @@ from __future__ import absolute_import, division, print_function +from typing import Callable, Concatenate, ParamSpec, TypeVar + from twisted.web.client import Agent, HTTPConnectionPool +from treq._types import _URLType from treq.client import HTTPClient +P = ParamSpec("P") +R = TypeVar("R") + + +def _like( + method: Callable[Concatenate[HTTPClient, _URLType, P], R] +) -> Callable[ + [Callable[Concatenate[_URLType, P], R]], Callable[Concatenate[_URLType, P], R] +]: + return lambda x: x + def head(url, **kwargs): """ @@ -14,6 +28,7 @@ def head(url, **kwargs): return _client(kwargs).head(url, _stacklevel=4, **kwargs) +@_like(HTTPClient.get) def get(url, headers=None, **kwargs): """ Make a ``GET`` request. diff --git a/src/treq/client.py b/src/treq/client.py index 7c172c69..db17a583 100644 --- a/src/treq/client.py +++ b/src/treq/client.py @@ -1,3 +1,5 @@ +# -*- test-case-name: treq.test.test_client -*- +from __future__ import annotations import io import mimetypes import uuid @@ -7,20 +9,21 @@ from typing import ( Any, Callable, + Concatenate, Iterable, Iterator, List, Mapping, Optional, + ParamSpec, Tuple, + TypeVar, Union, ) from urllib.parse import quote_plus from urllib.parse import urlencode as _urlencode from hyperlink import DecodedURL, EncodedURL -from requests.cookies import merge_cookies -from treq.cookies import scoped_cookie from twisted.internet.defer import Deferred from twisted.internet.interfaces import IProtocol from twisted.python.components import proxyForInterface, registerAdapter @@ -50,8 +53,14 @@ _URLType, ) from treq.auth import add_auth +from treq.cookies import scoped_cookie from treq.response import _Response +from .cookies import TreqieJar + +P = ParamSpec("P") +R = TypeVar("R") + class _Nothing: """Type of the sentinel `_NOTHING`""" @@ -67,7 +76,7 @@ def urlencode(query: _ParamsType, doseq: bool) -> bytes: def _scoped_cookiejar_from_dict( url_object: EncodedURL, cookie_dict: Optional[Mapping[str, str]] -) -> CookieJar: +) -> TreqieJar: """ Create a CookieJar from a dictionary whose cookies are all scoped to the given URL's origin. @@ -75,7 +84,7 @@ def _scoped_cookiejar_from_dict( @note: This does not scope the cookies to any particular path, only the host, port, and scheme of the given URL. """ - cookie_jar = CookieJar() + cookie_jar = TreqieJar() if cookie_dict is None: return cookie_jar for k, v in cookie_dict.items(): @@ -83,6 +92,12 @@ def _scoped_cookiejar_from_dict( return cookie_jar +def _merge_cookies(left: TreqieJar, right: CookieJar) -> TreqieJar: + for cookie in right: + left.set_cookie(cookie) + return left + + class _BodyBufferingProtocol(proxyForInterface(IProtocol)): # type: ignore def __init__(self, original, buffer, finished): self.original = original @@ -130,26 +145,29 @@ def deliverBody(self, protocol): self._waiters.append(protocol) +P2 = ParamSpec("P2") + + +def _like(c: Callable[Concatenate[HTTPClient, str, _URLType, P], R]) -> Callable[ + [Callable[Concatenate[HTTPClient, _URLType, P], R]], + Callable[Concatenate[HTTPClient, _URLType, P], R], +]: + return lambda x: x + + class HTTPClient: def __init__( self, agent: IAgent, - cookiejar: Optional[CookieJar] = None, + cookiejar: Optional[TreqieJar] = None, data_to_body_producer: Callable[[Any], IBodyProducer] = IBodyProducer, ) -> None: self._agent = agent if cookiejar is None: - cookiejar = CookieJar() + cookiejar = TreqieJar() self._cookiejar = cookiejar self._data_to_body_producer = data_to_body_producer - def get(self, url: _URLType, **kwargs: Any) -> "Deferred[_Response]": - """ - See :func:`treq.get()`. - """ - kwargs.setdefault("_stacklevel", 3) - return self.request("GET", url, **kwargs) - def put( self, url: _URLType, data: Optional[_DataType] = None, **kwargs: Any ) -> "Deferred[_Response]": @@ -246,7 +264,7 @@ def request( if not isinstance(cookies, CookieJar): cookies = _scoped_cookiejar_from_dict(parsed_url, cookies) - merge_cookies(self._cookiejar, cookies) + _merge_cookies(self._cookiejar, cookies) wrapped_agent: IAgent = CookieAgent(self._agent, self._cookiejar) if allow_redirects: @@ -283,6 +301,16 @@ def gotResult(result): return d.addCallback(_Response, self._cookiejar) + @_like(request) + def get(self, url: _URLType, **kwargs: Any) -> "Deferred[_Response]": + """ + See :func:`treq.get()`. + """ + kwargs.setdefault("_stacklevel", 3) + return self.request("GET", url, **kwargs) + + reveal_type(get) + def _request_headers( self, headers: Optional[_HeadersType], stacklevel: int ) -> Headers: diff --git a/src/treq/cookies.py b/src/treq/cookies.py index 20abac5f..442e04cd 100644 --- a/src/treq/cookies.py +++ b/src/treq/cookies.py @@ -1,3 +1,4 @@ +# -*- test-case-name: treq.test.test_integration -*- """ Convenience helpers for :mod:`http.cookiejar` """ @@ -8,6 +9,14 @@ from hyperlink import EncodedURL +class TreqieJar(CookieJar): + def __getitem__(self, name: str) -> str: + for cookie in self: + if cookie.name == name and cookie.value is not None: + return cookie.value + raise KeyError(name) + + def scoped_cookie(origin: Union[str, EncodedURL], name: str, value: str) -> Cookie: """ Create a cookie scoped to a given URL's origin. diff --git a/src/treq/response.py b/src/treq/response.py index df6010da..13d5ad64 100644 --- a/src/treq/response.py +++ b/src/treq/response.py @@ -1,5 +1,4 @@ from typing import Any, Callable, List -from requests.cookies import cookiejar_from_dict from http.cookiejar import CookieJar from twisted.internet.defer import Deferred from twisted.python import reflect @@ -16,7 +15,7 @@ class _Response(proxyForInterface(IResponse)): # type: ignore """ original: IResponse - _cookiejar: CookieJar + _cookiejar: TreqieJar def __init__(self, original: IResponse, cookiejar: CookieJar): self.original = original @@ -107,11 +106,7 @@ def cookies(self) -> CookieJar: """ Get a copy of this response's cookies. """ - # NB: This actually returns a RequestsCookieJar, but we type it as a - # regular CookieJar because we want to ditch requests as a dependency. - # Full deprecation deprecation will require a subclass or wrapper that - # warns about the RequestCookieJar extensions. - jar: CookieJar = cookiejar_from_dict({}) + jar = CookieJar() for cookie in self._cookiejar: jar.set_cookie(cookie) diff --git a/src/treq/test/test_cookies.py b/src/treq/test/test_cookies.py index 41946d52..31707544 100644 --- a/src/treq/test/test_cookies.py +++ b/src/treq/test/test_cookies.py @@ -1,18 +1,19 @@ -from http.cookiejar import CookieJar, Cookie +from http.cookiejar import Cookie, CookieJar import attrs -from twisted.internet.testing import StringTransport +from treq._agentspy import RequestRecord, agent_spy +from treq.client import HTTPClient +from treq.cookies import scoped_cookie, search from twisted.internet.interfaces import IProtocol -from twisted.trial.unittest import SynchronousTestCase +from twisted.internet.testing import StringTransport from twisted.python.failure import Failure +from twisted.trial.unittest import SynchronousTestCase from twisted.web.client import ResponseDone from twisted.web.http_headers import Headers from twisted.web.iweb import IClientRequest, IResponse from zope.interface import implementer -from treq._agentspy import agent_spy, RequestRecord -from treq.client import HTTPClient -from treq.cookies import scoped_cookie, search +from ..cookies import TreqieJar @implementer(IClientRequest) @@ -135,7 +136,7 @@ class HTTPClientCookieTests(SynchronousTestCase): def setUp(self) -> None: self.agent, self.requests = agent_spy() - self.cookiejar = CookieJar() + self.cookiejar = TreqieJar() self.client = HTTPClient(self.agent, self.cookiejar) def test_cookies_in_jars(self) -> None: diff --git a/src/treq/test/test_treq_integration.py b/src/treq/test/test_treq_integration.py index 1518fb46..0bef51ce 100644 --- a/src/treq/test/test_treq_integration.py +++ b/src/treq/test/test_treq_integration.py @@ -1,24 +1,25 @@ +from __future__ import annotations from io import BytesIO +from typing import Callable, Concatenate, ParamSpec, TypeVar -from twisted.python.url import URL - -from twisted.trial.unittest import TestCase +import treq +from treq.test.util import DEBUG, skip_on_windows_because_of_199 +from twisted.internet import reactor from twisted.internet.defer import CancelledError, inlineCallbacks +from twisted.internet.ssl import Certificate, trustRootFromCertificates from twisted.internet.task import deferLater -from twisted.internet import reactor from twisted.internet.tcp import Client -from twisted.internet.ssl import Certificate, trustRootFromCertificates - -from twisted.web.client import (Agent, BrowserLikePolicyForHTTPS, - HTTPConnectionPool, ResponseFailed) - -from treq.test.util import DEBUG, skip_on_windows_because_of_199 +from twisted.python.url import URL +from twisted.trial.unittest import TestCase +from twisted.web.client import ( + Agent, + BrowserLikePolicyForHTTPS, + HTTPConnectionPool, + ResponseFailed, +) from .local_httpbin.parent import _HTTPBinProcess -import treq - - skip = skip_on_windows_because_of_199() @@ -26,25 +27,30 @@ def print_response(response): if DEBUG: print() - print('---') + print("---") print(response.code) print(response.headers) print(response.request.headers) text = yield treq.text_content(response) print(text) - print('---') + print("---") -def with_baseurl(method): - def _request(self, url, *args, **kwargs): - return method(self.baseurl + url, - *args, - agent=self.agent, - pool=self.pool, - **kwargs) +P = ParamSpec("P") +R = TypeVar("R") - return _request +def with_baseurl( + method: Callable[Concatenate[str, P], R] +) -> Callable[Concatenate[TreqIntegrationTests, str, P], R]: + def _request( + self: TreqIntegrationTests, url: str, *args: P.args, **kwargs: P.kwargs + ) -> R: + return method( + self.baseurl + url, *args, agent=self.agent, pool=self.pool, **kwargs + ) + + return _request class TreqIntegrationTests(TestCase): get = with_baseurl(treq.get) @@ -58,11 +64,10 @@ class TreqIntegrationTests(TestCase): @inlineCallbacks def setUp(self): - description = yield self._httpbin_process.server_description( - reactor) - self.baseurl = URL(scheme=u"http", - host=description.host, - port=description.port).asText() + description = yield self._httpbin_process.server_description(reactor) + self.baseurl = URL( + scheme="http", host=description.host, port=description.port + ).asText() self.agent = Agent(reactor) self.pool = HTTPConnectionPool(reactor, False) @@ -82,85 +87,83 @@ def _check_fds(_): @inlineCallbacks def assert_data(self, response, expected_data): body = yield treq.json_content(response) - self.assertIn('data', body) - self.assertEqual(body['data'], expected_data) + self.assertIn("data", body) + self.assertEqual(body["data"], expected_data) @inlineCallbacks def assert_sent_header(self, response, header, expected_value): body = yield treq.json_content(response) - self.assertIn(header, body['headers']) - self.assertEqual(body['headers'][header], expected_value) + self.assertIn(header, body["headers"]) + self.assertEqual(body["headers"][header], expected_value) @inlineCallbacks def test_get(self): - response = yield self.get('/get') + response = yield self.get("/get") self.assertEqual(response.code, 200) yield print_response(response) @inlineCallbacks def test_get_headers(self): - response = yield self.get('/get', {b'X-Blah': [b'Foo', b'Bar']}) + response = yield self.get("/get", {b"X-Blah": [b"Foo", b"Bar"]}) self.assertEqual(response.code, 200) - yield self.assert_sent_header(response, 'X-Blah', 'Foo,Bar') + yield self.assert_sent_header(response, "X-Blah", "Foo,Bar") yield print_response(response) @inlineCallbacks def test_get_headers_unicode(self): - response = yield self.get('/get', {u'X-Blah': [u'Foo', b'Bar']}) + response = yield self.get("/get", {"X-Blah": ["Foo", b"Bar"]}) self.assertEqual(response.code, 200) - yield self.assert_sent_header(response, 'X-Blah', 'Foo,Bar') + yield self.assert_sent_header(response, "X-Blah", "Foo,Bar") yield print_response(response) @inlineCallbacks def test_get_302_absolute_redirect(self): - response = yield self.get( - '/redirect-to?url={0}/get'.format(self.baseurl)) + response = yield self.get("/redirect-to?url={0}/get".format(self.baseurl)) self.assertEqual(response.code, 200) yield print_response(response) @inlineCallbacks def test_get_302_relative_redirect(self): - response = yield self.get('/relative-redirect/1') + response = yield self.get("/relative-redirect/1") self.assertEqual(response.code, 200) yield print_response(response) @inlineCallbacks def test_get_302_redirect_disallowed(self): - response = yield self.get('/redirect/1', allow_redirects=False) + response = yield self.get("/redirect/1", allow_redirects=False) self.assertEqual(response.code, 302) yield print_response(response) @inlineCallbacks def test_head(self): - response = yield self.head('/get') + response = yield self.head("/get") body = yield treq.content(response) - self.assertEqual(b'', body) + self.assertEqual(b"", body) yield print_response(response) @inlineCallbacks def test_head_302_absolute_redirect(self): - response = yield self.head( - '/redirect-to?url={0}/get'.format(self.baseurl)) + response = yield self.head("/redirect-to?url={0}/get".format(self.baseurl)) self.assertEqual(response.code, 200) yield print_response(response) @inlineCallbacks def test_head_302_relative_redirect(self): - response = yield self.head('/relative-redirect/1') + response = yield self.head("/relative-redirect/1") self.assertEqual(response.code, 200) yield print_response(response) @inlineCallbacks def test_head_302_redirect_disallowed(self): - response = yield self.head('/redirect/1', allow_redirects=False) + response = yield self.head("/redirect/1", allow_redirects=False) self.assertEqual(response.code, 302) yield print_response(response) @inlineCallbacks def test_post(self): - response = yield self.post('/post', b'Hello!') + response = yield self.post("/post", b"Hello!") self.assertEqual(response.code, 200) - yield self.assert_data(response, 'Hello!') + yield self.assert_data(response, "Hello!") yield print_response(response) @inlineCallbacks @@ -174,70 +177,66 @@ def read(*args, **kwargs): return BytesIO.read(*args, **kwargs) response = yield self.post( - '/post', - data={"a": "b"}, - files={"file1": FileLikeObject(b"file")}) + "/post", data={"a": "b"}, files={"file1": FileLikeObject(b"file")} + ) self.assertEqual(response.code, 200) body = yield treq.json_content(response) - self.assertEqual('b', body['form']['a']) - self.assertEqual('file', body['files']['file1']) + self.assertEqual("b", body["form"]["a"]) + self.assertEqual("file", body["files"]["file1"]) yield print_response(response) @inlineCallbacks def test_post_headers(self): response = yield self.post( - '/post', - b'{msg: "Hello!"}', - headers={'Content-Type': ['application/json']} + "/post", b'{msg: "Hello!"}', headers={"Content-Type": ["application/json"]} ) self.assertEqual(response.code, 200) - yield self.assert_sent_header( - response, 'Content-Type', 'application/json') + yield self.assert_sent_header(response, "Content-Type", "application/json") yield self.assert_data(response, '{msg: "Hello!"}') yield print_response(response) @inlineCallbacks def test_put(self): - response = yield self.put('/put', data=b'Hello!') + response = yield self.put("/put", data=b"Hello!") yield print_response(response) @inlineCallbacks def test_patch(self): - response = yield self.patch('/patch', data=b'Hello!') + response = yield self.patch("/patch", data=b"Hello!") self.assertEqual(response.code, 200) - yield self.assert_data(response, 'Hello!') + yield self.assert_data(response, "Hello!") yield print_response(response) @inlineCallbacks def test_delete(self): - response = yield self.delete('/delete') + response = yield self.delete("/delete") self.assertEqual(response.code, 200) yield print_response(response) @inlineCallbacks def test_gzip(self): - response = yield self.get('/gzip') + response = yield self.get("/gzip") self.assertEqual(response.code, 200) yield print_response(response) json = yield treq.json_content(response) - self.assertTrue(json['gzipped']) + self.assertTrue(json["gzipped"]) @inlineCallbacks def test_basic_auth(self): - response = yield self.get('/basic-auth/treq/treq', - auth=('treq', 'treq')) + response = yield self.get("/basic-auth/treq/treq", auth=("treq", "treq")) self.assertEqual(response.code, 200) yield print_response(response) json = yield treq.json_content(response) - self.assertTrue(json['authenticated']) - self.assertEqual(json['user'], 'treq') + self.assertTrue(json["authenticated"]) + self.assertEqual(json["user"], "treq") @inlineCallbacks def test_failed_basic_auth(self): - response = yield self.get('/basic-auth/treq/treq', - auth=('not-treq', 'not-treq')) + response = yield self.get( + "/basic-auth/treq/treq", auth=("not-treq", "not-treq") + ) self.assertEqual(response.code, 401) yield print_response(response) @@ -246,26 +245,25 @@ def test_timeout(self): """ Verify a timeout fires if a request takes too long. """ - yield self.assertFailure(self.get('/delay/2', timeout=1), - CancelledError, - ResponseFailed) + yield self.assertFailure( + self.get("/delay/2", timeout=1), CancelledError, ResponseFailed + ) @inlineCallbacks def test_cookie(self): - response = yield self.get('/cookies', cookies={'hello': 'there'}) + response = yield self.get("/cookies", cookies={"hello": "there"}) self.assertEqual(response.code, 200) yield print_response(response) json = yield treq.json_content(response) - self.assertEqual(json['cookies']['hello'], 'there') + self.assertEqual(json["cookies"]["hello"], "there") - @inlineCallbacks - def test_set_cookie(self): - response = yield self.get('/cookies/set', - allow_redirects=False, - params={'hello': 'there'}) + async def test_set_cookie(self) -> None: + response = await self.get( + "/cookies/set", allow_redirects=False, params={"hello": "there"} + ) # self.assertEqual(response.code, 200) - yield print_response(response) - self.assertEqual(response.cookies()['hello'], 'there') + await print_response(response) + self.assertEqual(response.cookies()["hello"], "there") class HTTPSTreqIntegrationTests(TreqIntegrationTests): @@ -273,11 +271,10 @@ class HTTPSTreqIntegrationTests(TreqIntegrationTests): @inlineCallbacks def setUp(self): - description = yield self._httpbin_process.server_description( - reactor) - self.baseurl = URL(scheme=u"https", - host=description.host, - port=description.port).asText() + description = yield self._httpbin_process.server_description(reactor) + self.baseurl = URL( + scheme="https", host=description.host, port=description.port + ).asText() root = trustRootFromCertificates( [Certificate.loadPEM(description.cacert)], diff --git a/tox.ini b/tox.ini index 6b70acf2..664343da 100644 --- a/tox.ini +++ b/tox.ini @@ -11,9 +11,9 @@ extras = dev deps = coverage - twisted_lowest: Twisted==22.10.0 - twisted_latest: Twisted - twisted_trunk: https://github.com/twisted/twisted/archive/trunk.zip + twisted_lowest: Twisted[tls]==22.10.0 + twisted_latest: Twisted[tls] + twisted_trunk: Twisted[tls]@https://github.com/twisted/twisted/archive/trunk.zip setenv = # Avoid unnecessary network access when creating virtualenvs for speed. VIRTUALENV_NO_DOWNLOAD=1 @@ -30,7 +30,6 @@ basepython = python3.12 deps = mypy==1.0.1 mypy-zope==0.9.1 - types-requests commands = mypy \ --cache-dir="{toxworkdir}/mypy_cache" \