PERF: download in chunks of 256 kB + limit progress bar to 5 refresh/sec

news/12810.feature.rst

@@ -0,0 +1,5 @@
+Improve download performance. Download packages and update the
+progress bar in larger chunks of 256 kB, up from 10 kB.
+Limit the progress bar to 5 refresh per second.
+Improve hash performance. Read package files in larger chunks of 1 MB,
+up from 8192 bytes.

src/pip/_internal/cli/progress_bars.py

@@ -49,7 +49,7 @@ def _rich_progress_bar(
  TimeRemainingColumn(),
  )
 
- progress = Progress(*columns, refresh_per_second=30)
+ progress = Progress(*columns, refresh_per_second=5)
  task_id = progress.add_task(" " * (get_indentation() + 2), total=total)
  with progress:
  for chunk in iterable:

src/pip/_internal/network/download.py

@@ -7,7 +7,7 @@
 import os
 from typing import Iterable, Optional, Tuple
 
-from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
+from pip._vendor.requests.models import Response
 
 from pip._internal.cli.progress_bars import get_download_progress_renderer
 from pip._internal.exceptions import NetworkConnectionError
@@ -56,12 +56,12 @@ def _prepare_download(
  show_progress = False
  elif not total_length:
  show_progress = True
- elif total_length > (40 * 1000):
+ elif total_length > (512 * 1024):
  show_progress = True
  else:
  show_progress = False
 
- chunks = response_chunks(resp, CONTENT_CHUNK_SIZE)
+ chunks = response_chunks(resp)
 
  if not show_progress:
  return chunks

src/pip/_internal/network/utils.py

@@ -1,6 +1,6 @@
 from typing import Dict, Generator
 
-from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
+from pip._vendor.requests.models import Response
 
 from pip._internal.exceptions import NetworkConnectionError
 
@@ -25,6 +25,8 @@
 # possible to make this work.
 HEADERS: Dict[str, str] = {"Accept-Encoding": "identity"}
 
+DOWNLOAD_CHUNK_SIZE = 256 * 1024
+
 
 def raise_for_status(resp: Response) -> None:
  http_error_msg = ""
@@ -55,7 +57,7 @@ def raise_for_status(resp: Response) -> None:
 
 
 def response_chunks(
- response: Response, chunk_size: int = CONTENT_CHUNK_SIZE
+ response: Response, chunk_size: int = DOWNLOAD_CHUNK_SIZE
 ) -> Generator[bytes, None, None]:
  """Given a requests Response, provide the data chunks."""
  try:

src/pip/_internal/utils/misc.py

@@ -1,7 +1,6 @@
 import errno
 import getpass
 import hashlib
-import io
 import logging
 import os
 import posixpath
@@ -70,6 +69,8 @@
 OnExc = Callable[[FunctionType, Path, BaseException], Any]
 OnErr = Callable[[FunctionType, Path, ExcInfo], Any]
 
+FILE_CHUNK_SIZE = 1024 * 1024
+
 
 def get_pip_version() -> str:
  pip_pkg_dir = os.path.join(os.path.dirname(__file__), "..", "..")
@@ -122,9 +123,7 @@ def get_prog() -> str:
 # Retry every half second for up to 3 seconds
 @retry(stop_after_delay=3, wait=0.5)
 def rmtree(
- dir: str,
- ignore_errors: bool = False,
- onexc: Optional[OnExc] = None,
+ dir: str, ignore_errors: bool = False, onexc: Optional[OnExc] = None
 ) -> None:
  if ignore_errors:
  onexc = _onerror_ignore
@@ -313,7 +312,7 @@ def is_installable_dir(path: str) -> bool:
 
 
 def read_chunks(
- file: BinaryIO, size: int = io.DEFAULT_BUFFER_SIZE
+ file: BinaryIO, size: int = FILE_CHUNK_SIZE
 ) -> Generator[bytes, None, None]:
  """Yield pieces of data from a file-like object until EOF."""
  while True:
@@ -643,8 +642,7 @@ def pairwise(iterable: Iterable[Any]) -> Iterator[Tuple[Any, Any]]:
 
 
 def partition(
- pred: Callable[[T], bool],
- iterable: Iterable[T],
+ pred: Callable[[T], bool], iterable: Iterable[T]
 ) -> Tuple[Iterable[T], Iterable[T]]:
  """
  Use a predicate to partition entries into false entries and true entries,