Merge branch 'master' into drop-py3.6

CHANGES/4942.feature

@@ -0,0 +1,2 @@
+Add predicate to ``AbstractCookieJar.clear``.
+Add ``AbstractCookieJar.clear_domain`` to clean all domain and subdomains cookies only.

CHANGES/5192.bugfix

@@ -0,0 +1 @@
+Add ABNORMAL_CLOSURE and BAD_GATEWAY to WSCloseCode

aiohttp/abc.py

@@ -133,12 +133,19 @@ async def close(self) -> None:
  IterableBase = Iterable
 
 
+ClearCookiePredicate = Callable[["Morsel[str]"], bool]
+
+
 class AbstractCookieJar(Sized, IterableBase):
  """Abstract Cookie Jar."""
 
  @abstractmethod
- def clear(self) -> None:
- """Clear all cookies."""
+ def clear(self, predicate: Optional[ClearCookiePredicate] = None) -> None:
+ """Clear all cookies if no predicate is passed."""
+
+ @abstractmethod
+ def clear_domain(self, domain: str) -> None:
+ """Clear all cookies for domain and all subdomains."""
 
  @abstractmethod
  def update_cookies(self, cookies: LooseCookies, response_url: URL = URL()) -> None:

aiohttp/client_ws.py

@@ -13,6 +13,7 @@
  WS_CLOSED_MESSAGE,
  WS_CLOSING_MESSAGE,
  WebSocketError,
+ WSCloseCode,
  WSMessage,
  WSMsgType,
 )
@@ -109,7 +110,7 @@ def _send_heartbeat(self) -> None:
  def _pong_not_received(self) -> None:
  if not self._closed:
  self._closed = True
- self._close_code = 1006
+ self._close_code = WSCloseCode.ABNORMAL_CLOSURE
  self._exception = asyncio.TimeoutError()
  self._response.close()
 
@@ -171,7 +172,7 @@ async def send_json(
  ) -> None:
  await self.send_str(dumps(data), compress=compress)
 
- async def close(self, *, code: int = 1000, message: bytes = b"") -> bool:
+ async def close(self, *, code: int = WSCloseCode.OK, message: bytes = b"") -> bool:
  # we need to break `receive()` cycle first,
  # `close()` may be called from different task
  if self._waiting is not None and not self._closed:
@@ -184,11 +185,11 @@ async def close(self, *, code: int = 1000, message: bytes = b"") -> bool:
  try:
  await self._writer.close(code, message)
  except asyncio.CancelledError:
- self._close_code = 1006
+ self._close_code = WSCloseCode.ABNORMAL_CLOSURE
  self._response.close()
  raise
  except Exception as exc:
- self._close_code = 1006
+ self._close_code = WSCloseCode.ABNORMAL_CLOSURE
  self._exception = exc
  self._response.close()
  return True
@@ -202,11 +203,11 @@ async def close(self, *, code: int = 1000, message: bytes = b"") -> bool:
  async with async_timeout.timeout(self._timeout.ws_close):
  msg = await self._reader.read()
  except asyncio.CancelledError:
- self._close_code = 1006
+ self._close_code = WSCloseCode.ABNORMAL_CLOSURE
  self._response.close()
  raise
  except Exception as exc:
- self._close_code = 1006
+ self._close_code = WSCloseCode.ABNORMAL_CLOSURE
  self._exception = exc
  self._response.close()
  return True
@@ -242,15 +243,15 @@ async def receive(self, timeout: Optional[float] = None) -> WSMessage:
  self._waiting = None
  set_result(waiter, True)
  except (asyncio.CancelledError, asyncio.TimeoutError):
- self._close_code = 1006
+ self._close_code = WSCloseCode.ABNORMAL_CLOSURE
  raise
  except EofStream:
- self._close_code = 1000
+ self._close_code = WSCloseCode.OK
  await self.close()
  return WSMessage(WSMsgType.CLOSED, None, None)
  except ClientError:
  self._closed = True
- self._close_code = 1006
+ self._close_code = WSCloseCode.ABNORMAL_CLOSURE
  return WS_CLOSED_MESSAGE
  except WebSocketError as exc:
  self._close_code = exc.code
@@ -259,7 +260,7 @@ async def receive(self, timeout: Optional[float] = None) -> WSMessage:
  except Exception as exc:
  self._exception = exc
  self._closing = True
- self._close_code = 1006
+ self._close_code = WSCloseCode.ABNORMAL_CLOSURE
  await self.close()
  return WSMessage(WSMsgType.ERROR, exc, None)
 

aiohttp/cookiejar.py

@@ -22,7 +22,7 @@
 
 from yarl import URL
 
-from .abc import AbstractCookieJar
+from .abc import AbstractCookieJar, ClearCookiePredicate
 from .helpers import is_ip_address, next_whole_second
 from .typedefs import LooseCookies, PathLike
 
@@ -82,11 +82,41 @@ def load(self, file_path: PathLike) -> None:
  with file_path.open(mode="rb") as f:
  self._cookies = pickle.load(f)
 
- def clear(self) -> None:
- self._cookies.clear()
- self._host_only_cookies.clear()
- self._next_expiration = next_whole_second()
- self._expirations.clear()
+ def clear(self, predicate: Optional[ClearCookiePredicate] = None) -> None:
+ if predicate is None:
+ self._next_expiration = next_whole_second()
+ self._cookies.clear()
+ self._host_only_cookies.clear()
+ self._expirations.clear()
+ return
+
+ to_del = []
+ now = datetime.datetime.now(datetime.timezone.utc)
+ for domain, cookie in self._cookies.items():
+ for name, morsel in cookie.items():
+ key = (domain, name)
+ if (
+ key in self._expirations and self._expirations[key] <= now
+ ) or predicate(morsel):
+ to_del.append(key)
+
+ for domain, name in to_del:
+ key = (domain, name)
+ self._host_only_cookies.discard(key)
+ if key in self._expirations:
+ del self._expirations[(domain, name)]
+ self._cookies[domain].pop(name, None)
+
+ next_expiration = min(self._expirations.values(), default=self._max_time)
+ try:
+ self._next_expiration = next_expiration.replace(
+ microsecond=0
+ ) + datetime.timedelta(seconds=1)
+ except OverflowError:
+ self._next_expiration = self._max_time
+
+ def clear_domain(self, domain: str) -> None:
+ self.clear(lambda x: self._is_domain_match(domain, x["domain"]))
 
  def __iter__(self) -> "Iterator[Morsel[str]]":
  self._do_expiration()
@@ -97,31 +127,7 @@ def __len__(self) -> int:
  return sum(1 for i in self)
 
  def _do_expiration(self) -> None:
- now = datetime.datetime.now(datetime.timezone.utc)
- if self._next_expiration > now:
- return
- if not self._expirations:
- return
- next_expiration = self._max_time
- to_del = []
- cookies = self._cookies
- expirations = self._expirations
- for (domain, name), when in expirations.items():
- if when <= now:
- cookies[domain].pop(name, None)
- to_del.append((domain, name))
- self._host_only_cookies.discard((domain, name))
- else:
- next_expiration = min(next_expiration, when)
- for key in to_del:
- del expirations[key]
-
- try:
- self._next_expiration = next_expiration.replace(
- microsecond=0
- ) + datetime.timedelta(seconds=1)
- except OverflowError:
- self._next_expiration = self._max_time
+ self.clear(lambda x: False)
 
  def _expire_cookie(self, when: datetime.datetime, domain: str, name: str) -> None:
  self._next_expiration = min(self._next_expiration, when)
@@ -371,7 +377,10 @@ def __iter__(self) -> "Iterator[Morsel[str]]":
  def __len__(self) -> int:
  return 0
 
- def clear(self) -> None:
+ def clear(self, predicate: Optional[ClearCookiePredicate] = None) -> None:
+ pass
+
+ def clear_domain(self, domain: str) -> None:
  pass
 
  def update_cookies(self, cookies: LooseCookies, response_url: URL = URL()) -> None:

aiohttp/http_websocket.py

@@ -33,13 +33,15 @@ class WSCloseCode(IntEnum):
  GOING_AWAY = 1001
  PROTOCOL_ERROR = 1002
  UNSUPPORTED_DATA = 1003
+ ABNORMAL_CLOSURE = 1006
  INVALID_TEXT = 1007
  POLICY_VIOLATION = 1008
  MESSAGE_TOO_BIG = 1009
  MANDATORY_EXTENSION = 1010
  INTERNAL_ERROR = 1011
  SERVICE_RESTART = 1012
  TRY_AGAIN_LATER = 1013
+ BAD_GATEWAY = 1014
 
 
 ALLOWED_CLOSE_CODES = {int(i) for i in WSCloseCode}

aiohttp/web_ws.py

@@ -19,6 +19,7 @@
  WebSocketError,
  WebSocketReader,
  WebSocketWriter,
+ WSCloseCode,
  WSMessage,
  WSMsgType as WSMsgType,
  ws_ext_gen,
@@ -144,7 +145,7 @@ def _send_heartbeat(self) -> None:
  def _pong_not_received(self) -> None:
  if self._req is not None and self._req.transport is not None:
  self._closed = True
- self._close_code = 1006
+ self._close_code = WSCloseCode.ABNORMAL_CLOSURE
  self._exception = asyncio.TimeoutError()
  self._req.transport.close()
 
@@ -346,7 +347,7 @@ async def write_eof(self) -> None: # type: ignore
  await self.close()
  self._eof_sent = True
 
- async def close(self, *, code: int = 1000, message: bytes = b"") -> bool:
+ async def close(self, *, code: int = WSCloseCode.OK, message: bytes = b"") -> bool:
  if self._writer is None:
  raise RuntimeError("Call .prepare() first")
 
@@ -368,10 +369,10 @@ async def close(self, *, code: int = 1000, message: bytes = b"") -> bool:
  assert writer is not None
  await writer.drain()
  except (asyncio.CancelledError, asyncio.TimeoutError):
- self._close_code = 1006
+ self._close_code = WSCloseCode.ABNORMAL_CLOSURE
  raise
  except Exception as exc:
- self._close_code = 1006
+ self._close_code = WSCloseCode.ABNORMAL_CLOSURE
  self._exception = exc
  return True
 
@@ -384,18 +385,18 @@ async def close(self, *, code: int = 1000, message: bytes = b"") -> bool:
  async with async_timeout.timeout(self._timeout):
  msg = await reader.read()
  except asyncio.CancelledError:
- self._close_code = 1006
+ self._close_code = WSCloseCode.ABNORMAL_CLOSURE
  raise
  except Exception as exc:
- self._close_code = 1006
+ self._close_code = WSCloseCode.ABNORMAL_CLOSURE
  self._exception = exc
  return True
 
  if msg.type == WSMsgType.CLOSE:
  self._close_code = msg.data
  return True
 
- self._close_code = 1006
+ self._close_code = WSCloseCode.ABNORMAL_CLOSURE
  self._exception = asyncio.TimeoutError()
  return True
  else:
@@ -430,10 +431,10 @@ async def receive(self, timeout: Optional[float] = None) -> WSMessage:
  set_result(waiter, True)
  self._waiting = None
  except (asyncio.CancelledError, asyncio.TimeoutError):
- self._close_code = 1006
+ self._close_code = WSCloseCode.ABNORMAL_CLOSURE
  raise
  except EofStream:
- self._close_code = 1000
+ self._close_code = WSCloseCode.OK
  await self.close()
  return WSMessage(WSMsgType.CLOSED, None, None)
  except WebSocketError as exc:
@@ -443,7 +444,7 @@ async def receive(self, timeout: Optional[float] = None) -> WSMessage:
  except Exception as exc:
  self._exception = exc
  self._closing = True
- self._close_code = 1006
+ self._close_code = WSCloseCode.ABNORMAL_CLOSURE
  await self.close()
  return WSMessage(WSMsgType.ERROR, exc, None)
 

docs/abc.rst

@@ -145,6 +145,22 @@ Abstract Cookie Jar
  :return: :class:`http.cookies.SimpleCookie` with filtered
  cookies for given URL.
 
+ .. method:: clear(predicate=None)
+
+ Removes all cookies from the jar if the predicate is ``None``. Otherwise remove only those :class:`~http.cookies.Morsel` that ``predicate(morsel)`` returns ``True``.
+
+ :param predicate: callable that gets :class:`~http.cookies.Morsel` as a parameter and returns ``True`` if this :class:`~http.cookies.Morsel` must be deleted from the jar.
+
+ .. versionadded:: 3.8
+
+ .. method:: clear_domain(domain)
+
+ Remove all cookies from the jar that belongs to the specified domain or its subdomains.
+
+ :param str domain: domain for which cookies must be deleted from the jar.
+
+ .. versionadded:: 3.8
+
 Abstract Abstract Access Logger
 -------------------------------
 

docs/client_reference.rst

@@ -1578,14 +1578,14 @@ manually.
  The method is converted into :term:`coroutine`,
  *compress* parameter added.
 
- .. comethod:: close(*, code=1000, message=b'')
+ .. comethod:: close(*, code=WSCloseCode.OK, message=b'')
 
  A :ref:`coroutine<coroutine>` that initiates closing handshake by sending
  :const:`~aiohttp.WSMsgType.CLOSE` message. It waits for
  close response from server. To add a timeout to `close()` call
  just wrap the call with `asyncio.wait()` or `asyncio.wait_for()`.
 
- :param int code: closing code
+ :param int code: closing code. See also :class:`~aiohttp.WSCloseCode`.
 
  :param message: optional payload of *close* message,
  :class:`str` (converted to *UTF-8* encoded bytes) or :class:`bytes`.
@@ -1839,6 +1839,22 @@ CookieJar
  :param file_path: Path to file from where cookies will be
  imported, :class:`str` or :class:`pathlib.Path` instance.
 
+ .. method:: clear(predicate=None)
+
+ Removes all cookies from the jar if the predicate is ``None``. Otherwise remove only those :class:`~http.cookies.Morsel` that ``predicate(morsel)`` returns ``True``.
+
+ :param predicate: callable that gets :class:`~http.cookies.Morsel` as a parameter and returns ``True`` if this :class:`~http.cookies.Morsel` must be deleted from the jar.
+
+ .. versionadded:: 4.0
+
+ .. method:: clear_domain(domain)
+
+ Remove all cookies from the jar that belongs to the specified domain or its subdomains.
+
+ :param str domain: domain for which cookies must be deleted from the jar.
+
+ .. versionadded:: 4.0
+
 
 .. class:: DummyCookieJar(*, loop=None)
 

docs/web_reference.rst

@@ -1117,14 +1117,14 @@ WebSocketResponse
  The method is converted into :term:`coroutine`,
  *compress* parameter added.
 
- .. comethod:: close(*, code=1000, message=b'')
+ .. comethod:: close(*, code=WSCloseCode.OK, message=b'')
 
  A :ref:`coroutine<coroutine>` that initiates closing
  handshake by sending :const:`~aiohttp.WSMsgType.CLOSE` message.
 
  It is safe to call `close()` from different task.
 
- :param int code: closing code
+ :param int code: closing code. See also :class:`~aiohttp.WSCloseCode`.
 
  :param message: optional payload of *close* message,
  :class:`str` (converted to *UTF-8* encoded bytes)