Force Host header to be first in ClientRequest

CHANGES/1749.feature

@@ -2,3 +2,4 @@ Add type hints to Application and Response
 Add type hints to Exceptions
 Upgrade mypy to 0.630
 Add type hints to payload.py
+Add type hints to tracing.py

CHANGES/3265.bugfix

@@ -0,0 +1 @@
+Ensure Host header is added first to ClientRequest to better replicate browser

CHANGES/3267.feature

@@ -0,0 +1,2 @@
+Don't raise a warning if ``NETRC`` environment variable is not set and ``~/.netrc`` file
+doesn't exist.

CHANGES/3318.removal

@@ -0,0 +1 @@
+Deprecated use of boolean in ``resp.enable_compression()``

CONTRIBUTORS.txt

@@ -1,6 +1,7 @@
 - Contributors -
 ----------------
 A. Jesse Jiryu Davis
+Adam Cooper
 Adam Mills
 Adrián Chaves
 Alec Hanefeld
@@ -69,6 +70,7 @@ Dmitry Doroshev
 Dmitry Lukashin
 Dmitry Shamov
 Dmitry Trofimov
+Dmytro Bohomiakov
 Dmytro Kuznetsov
 Dustin J. Mitchell
 Eduard Iskandarov
@@ -154,6 +156,7 @@ Mun Gwan-gyeong
 Nicolas Braem
 Nikolay Kim
 Nikolay Novik
+Oisin Aylward
 Olaf Conradi
 Pahaz Blinov
 Panagiotis Kolokotronis

aiohttp/client_reqrep.py

@@ -298,12 +298,25 @@ def update_version(self, version):
  def update_headers(self, headers):
  """Update request headers."""
  self.headers = CIMultiDict()
+
+ # add host
+ netloc = self.url.raw_host
+ if helpers.is_ipv6_address(netloc):
+ netloc = '[{}]'.format(netloc)
+ if not self.url.is_default_port():
+ netloc += ':' + str(self.url.port)
+ self.headers[hdrs.HOST] = netloc
+
  if headers:
  if isinstance(headers, (dict, MultiDictProxy, MultiDict)):
  headers = headers.items()
 
  for key, value in headers:
- self.headers.add(key, value)
+ # A special case for Host header
+ if key.lower() == 'host':
+ self.headers[key] = value
+ else:
+ self.headers.add(key, value)
 
  def update_auto_headers(self, skip_auto_headers):
  self.skip_auto_headers = CIMultiDict(
@@ -315,15 +328,6 @@ def update_auto_headers(self, skip_auto_headers):
  if hdr not in used_headers:
  self.headers.add(hdr, val)
 
- # add host
- if hdrs.HOST not in used_headers:
- netloc = self.url.raw_host
- if helpers.is_ipv6_address(netloc):
- netloc = '[{}]'.format(netloc)
- if not self.url.is_default_port():
- netloc += ':' + str(self.url.port)
- self.headers[hdrs.HOST] = netloc
-
  if hdrs.USER_AGENT not in used_headers:
  self.headers[hdrs.USER_AGENT] = SERVER_SOFTWARE
 

aiohttp/helpers.py

@@ -8,6 +8,7 @@
 import inspect
 import netrc
 import os
+import platform
 import re
 import sys
 import time
@@ -155,30 +156,39 @@ def strip_auth_from_url(url: URL) -> Tuple[URL, Optional[BasicAuth]]:
 
 
 def netrc_from_env() -> Optional[netrc.netrc]:
- netrc_obj = None
- netrc_path = os.environ.get('NETRC') # type: Optional[PathLike]
- try:
- if netrc_path is not None:
- netrc_path = Path(netrc_path)
- else:
+ """Attempt to load the netrc file from the path specified by the env-var
+ NETRC or in the default location in the user's home directory.
+
+ Returns None if it couldn't be found or fails to parse.
+ """
+ netrc_env = os.environ.get('NETRC')
+
+ if netrc_env is not None:
+ netrc_path = Path(netrc_env)
+ else:
+ try:
  home_dir = Path.home()
- if os.name == 'nt': # pragma: no cover
- netrc_path = home_dir.joinpath('_netrc')
- else:
- netrc_path = home_dir.joinpath('.netrc')
+ except RuntimeError as e: # pragma: no cover
+ # if pathlib can't resolve home, it may raise a RuntimeError
+ client_logger.warning('Could not resolve home directory when '
+ 'trying to look for .netrc file: %s', e)
+ return None
 
- if netrc_path and netrc_path.is_file():
- try:
- netrc_obj = netrc.netrc(str(netrc_path))
- except (netrc.NetrcParseError, OSError) as e:
- client_logger.warning(".netrc file parses fail: %s", e)
-
- if netrc_obj is None:
- client_logger.warning("could't find .netrc file")
- except RuntimeError as e: # pragma: no cover
- """ handle error raised by pathlib """
- client_logger.warning("could't find .netrc file: %s", e)
- return netrc_obj
+ netrc_path = home_dir / (
+ '_netrc' if platform.system() == 'Windows' else '.netrc')
+
+ try:
+ return netrc.netrc(str(netrc_path))
+ except netrc.NetrcParseError as e:
+ client_logger.warning('Could not parse .netrc file: %s', e)
+ except OSError as e:
+ # we couldn't read the file (doesn't exist, permissions, etc.)
+ if netrc_env or netrc_path.is_file():
+ # only warn if the enviroment wanted us to load it,
+ # or it appears like the default file does actually exist
+ client_logger.warning('Could not read .netrc file: %s', e)
+
+ return None
 
 
 @attr.s(frozen=True, slots=True)