gh-117151: IO performance improvement, increase io.DEFAULT_BUFFER_SIZE to 128k

Doc/library/functions.rst

@@ -1388,10 +1388,11 @@ are always available. They are listed here in alphabetical order.
  :func:`io.TextIOWrapper.reconfigure`. When no *buffering* argument is
  given, the default buffering policy works as follows:
 
- * Binary files are buffered in fixed-size chunks; the size of the buffer is
- chosen using a heuristic trying to determine the underlying device's "block
- size" and falling back on :const:`io.DEFAULT_BUFFER_SIZE`. On many systems,
- the buffer will typically be 4096 or 8192 bytes long.
+ * Binary files are buffered in fixed-size chunks; the size of the buffer
+ is set to ``max(io.DEFAULT_BUFFER_SIZE, st_blksize)`` using a heuristic
+ trying to determine the underlying device's "block size" when available
+ and falling back on :const:`io.DEFAULT_BUFFER_SIZE`.
+ On most systems, the buffer will typically be 131072 bytes long.
 
  * "Interactive" text files (files for which :meth:`~io.IOBase.isatty`
  returns ``True``) use line buffering. Other text files use the policy

Lib/_pyio.py

@@ -23,8 +23,8 @@
  valid_seek_flags.add(os.SEEK_HOLE)
  valid_seek_flags.add(os.SEEK_DATA)
 
-# open() uses st_blksize whenever we can
-DEFAULT_BUFFER_SIZE = 8 * 1024 # bytes
+# open() uses max(st_blksize, io.DEFAULT_BUFFER_SIZE) when st_blksize is available
+DEFAULT_BUFFER_SIZE = 128 * 1024 # bytes
 
 # NOTE: Base classes defined here are registered with the "official" ABCs
 # defined in io.py. We don't use real inheritance though, because we don't want
@@ -123,10 +123,11 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
  the size of a fixed-size chunk buffer. When no buffering argument is
  given, the default buffering policy works as follows:
 
- * Binary files are buffered in fixed-size chunks; the size of the buffer
- is chosen using a heuristic trying to determine the underlying device's
- "block size" and falling back on `io.DEFAULT_BUFFER_SIZE`.
- On many systems, the buffer will typically be 4096 or 8192 bytes long.
+ * Binary files are buffered in fixed-size chunks; the size of the buffer
+ is set to `max(io.DEFAULT_BUFFER_SIZE, st_blksize)` using a heuristic
+ trying to determine the underlying device's "block size" when available
+ and falling back on `io.DEFAULT_BUFFER_SIZE`.
+ On most systems, the buffer will typically be 131072 bytes long.
 
  * "Interactive" text files (files for which isatty() returns True)
  use line buffering. Other text files use the policy described above
@@ -242,7 +243,7 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
  buffering = -1
  line_buffering = True
  if buffering < 0:
- buffering = raw._blksize
+ buffering = max(raw._blksize, DEFAULT_BUFFER_SIZE)
  if buffering < 0:
  raise ValueError("invalid buffering size")
  if buffering == 0:
@@ -1558,15 +1559,18 @@ def __init__(self, file, mode='r', closefd=True, opener=None):
  os.set_inheritable(fd, False)
 
  self._closefd = closefd
- self._stat_atopen = os.fstat(fd)
+ fdfstat = os.fstat(fd)
  try:
- if stat.S_ISDIR(self._stat_atopen.st_mode):
+ if stat.S_ISDIR(fdfstat.st_mode):
  raise IsADirectoryError(errno.EISDIR,
  os.strerror(errno.EISDIR), file)
  except AttributeError:
  # Ignore the AttributeError if stat.S_ISDIR or errno.EISDIR
  # don't exist.
  pass
+ self._blksize = getattr(fdfstat, 'st_blksize', 0)
+ if self._blksize < DEFAULT_BUFFER_SIZE:
+ self._blksize = DEFAULT_BUFFER_SIZE
 
  if _setmode:
  # don't translate newlines (\r\n <=> \n)
@@ -1612,17 +1616,6 @@ def __repr__(self):
  return ('<%s name=%r mode=%r closefd=%r>' %
  (class_name, name, self.mode, self._closefd))
 
- @property
- def _blksize(self):
- if self._stat_atopen is None:
- return DEFAULT_BUFFER_SIZE
-
- blksize = getattr(self._stat_atopen, "st_blksize", 0)
- # WASI sets blsize to 0
- if not blksize:
- return DEFAULT_BUFFER_SIZE
- return blksize
-
  def _checkReadable(self):
  if not self._readable:
  raise UnsupportedOperation('File not open for reading')
@@ -1655,22 +1648,14 @@ def readall(self):
  """
  self._checkClosed()
  self._checkReadable()
- if self._stat_atopen is None or self._stat_atopen.st_size <= 0:
- bufsize = DEFAULT_BUFFER_SIZE
- else:
- # In order to detect end of file, need a read() of at least 1
- # byte which returns size 0. Oversize the buffer by 1 byte so the
- # I/O can be completed with two read() calls (one for all data, one
- # for EOF) without needing to resize the buffer.
- bufsize = self._stat_atopen.st_size + 1
-
- if self._stat_atopen.st_size > 65536:
- try:
- pos = os.lseek(self._fd, 0, SEEK_CUR)
- if self._stat_atopen.st_size >= pos:
- bufsize = self._stat_atopen.st_size - pos + 1
- except OSError:
- pass
+ bufsize = DEFAULT_BUFFER_SIZE
+ try:
+ pos = os.lseek(self._fd, 0, SEEK_CUR)
+ end = os.fstat(self._fd).st_size
+ if end >= pos:
+ bufsize = end - pos + 1
+ except OSError:
+ pass
 
  result = bytearray()
  while True:
@@ -1746,7 +1731,6 @@ def truncate(self, size=None):
  if size is None:
  size = self.tell()
  os.ftruncate(self._fd, size)
- self._stat_atopen = None
  return size
 
  def close(self):

Lib/test/test_file.py

@@ -216,6 +216,22 @@ def testSetBufferSize(self):
  with self.assertWarnsRegex(RuntimeWarning, 'line buffering'):
  self._checkBufferSize(1)
 
+ def testDefaultBufferSize(self):
+ # a larger block size can have preference over the default buffer size,
+ # so we have to verify the block size and skip the test.
+ f = self.open(TESTFN, 'wb')#), buffering=0)
+ blksize = getattr(f, '_blksize', 0)
+ f.write(bytes([0] * 1_000_000))
+ f.close()
+
+ if blksize <= io.DEFAULT_BUFFER_SIZE:
+ # ensure the default buffer size is used.
+ f = self.open(TESTFN, 'rb')
+ data = f.read1()
+ self.assertEqual(len(data), io.DEFAULT_BUFFER_SIZE)
+ else:
+ self.skipTest("device block size greater than io.DEFAULT_BUFFER_SIZE")
+
  def testTruncateOnWindows(self):
  # SF bug <https://bugs.python.org/issue801631>
  # "file.truncate fault on windows"

Lib/test/test_fileio.py

@@ -71,14 +71,15 @@ def testAttributes(self):
  self.assertRaises((AttributeError, TypeError),
  setattr, f, attr, 'oops')
 
- @unittest.skipIf(is_wasi, "WASI does not expose st_blksize.")
  def testBlksize(self):
  # test private _blksize attribute
- blksize = io.DEFAULT_BUFFER_SIZE
+ blksize = 0
  # try to get preferred blksize from stat.st_blksize, if available
  if hasattr(os, 'fstat'):
  fst = os.fstat(self.f.fileno())
  blksize = getattr(fst, 'st_blksize', blksize)
+ if blksize < io.DEFAULT_BUFFER_SIZE:
+ blksize = io.DEFAULT_BUFFER_SIZE
  self.assertEqual(self.f._blksize, blksize)
 
  # verify readinto

Misc/NEWS.d/next/Library/2024-04-30-14-03-09.gh-issue-117151.yt2H8c.rst

@@ -0,0 +1,5 @@
+Increase ``io.DEFAULT_BUFFER_SIZE`` from 8k to 128k and adjust :func:`open` on
+platforms where ``fstat`` provides a ``st_blksize`` field (such as Linux) to use
+``max(io.DEFAULT_BUFFER_SIZE, device block size)`` rather than always using the
+device block size. This should improve I/O performance.
+Patch by Romain Morotti.

Modules/_io/_iomodule.c

@@ -132,9 +132,10 @@ the size of a fixed-size chunk buffer. When no buffering argument is
 given, the default buffering policy works as follows:
 
 * Binary files are buffered in fixed-size chunks; the size of the buffer
- is chosen using a heuristic trying to determine the underlying device's
- "block size" and falling back on `io.DEFAULT_BUFFER_SIZE`.
- On many systems, the buffer will typically be 4096 or 8192 bytes long.
+ is set to `max(io.DEFAULT_BUFFER_SIZE, st_blksize)` using a heuristic
+ trying to determine the underlying device's "block size" when available
+ and falling back on `io.DEFAULT_BUFFER_SIZE`.
+ On most systems, the buffer will typically be 131072 bytes long.
 
 * "Interactive" text files (files for which isatty() returns True)
  use line buffering. Other text files use the policy described above
@@ -200,7 +201,7 @@ static PyObject *
 _io_open_impl(PyObject *module, PyObject *file, const char *mode,
  int buffering, const char *encoding, const char *errors,
  const char *newline, int closefd, PyObject *opener)
-/*[clinic end generated code: output=aefafc4ce2b46dc0 input=cd034e7cdfbf4e78]*/
+/*[clinic end generated code: output=aefafc4ce2b46dc0 input=bac1cd70f431fe9a]*/
 {
  size_t i;
 

Modules/_io/_iomodule.h

@@ -78,7 +78,7 @@ extern Py_ssize_t _PyIO_find_line_ending(
 */
 extern int _PyIO_trap_eintr(void);
 
-#define DEFAULT_BUFFER_SIZE (8 * 1024) /* bytes */
+#define DEFAULT_BUFFER_SIZE (128 * 1024) /* bytes */
 
 /*
  * Offset type for positioning.

Modules/_io/fileio.c

@@ -494,6 +494,10 @@
  goto error;
  }
 #endif /* defined(S_ISDIR) */
+#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE
+ if (fdfstat.st_blksize > DEFAULT_BUFFER_SIZE)
+ self->blksize = fdfstat.st_blksize;
+#endif /* HAVE_STRUCT_STAT_ST_BLKSIZE */
  }
 
 #if defined(MS_WINDOWS) || defined(__CYGWIN__)

Modules/_io/winconsoleio.c

@@ -48,6 +48,30 @@
  of less than one character */
 #define SMALLBUF 4
 
+/* winconsole has to use a different size than the DEFAULT_BUFFER_SIZE.
+
+ issue11395 (2011): there is an unspecified upper bound on how many bytes
+ can be written at once. We cap at 32k - the caller will have to
+ handle partial writes.
+ Since we don't know how many input bytes are being ignored, we
+ have to reduce and recalculate.
+
+ update (2024): looking at historic threads around the issue
+ - the MSDN documentation for WriteConsoleW mentioned a length limitation
+ of 64 KiB at the time. it's not on the page anymore.
+ - that would be 32767 UTF-16 characters and a NULL. encoding may vary.
+ - old threads mention having issues with 26000-27000 characters,
+ it's less than expected so there may have been something more at play.
+ - the python interpreter capped writes to 32766 / sizeof(wchar_t)
+ in this winconsole module in 2011 to work around this bug.
+ - that is 16383 UTF-16 characters and room for a NULL if needed.
+
+ it really looks like a bug in Microsoft Windows, it could have been fixed
+ years ago but it's difficult to tell without Microsoft stepping in.
+*/
+#define WINCONSOLE_MAX_WRITE_SIZE (32766 / sizeof(wchar_t))
+#define WINCONSOLE_DEFAULT_BUFFER_SIZE (8*1024)
+
 char _get_console_type(HANDLE handle) {
  DWORD mode, peek_count;
 
@@ -422,7 +446,7 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj,
  goto error;
  }
 
- self->blksize = DEFAULT_BUFFER_SIZE;
+ self->blksize = WINCONSOLE_DEFAULT_BUFFER_SIZE;
  memset(self->buf, 0, 4);
 
  if (PyObject_SetAttr((PyObject *)self, &_Py_ID(name), nameobj) < 0)
@@ -1023,13 +1047,7 @@ _io__WindowsConsoleIO_write_impl(winconsoleio *self, PyTypeObject *cls,
 
  Py_BEGIN_ALLOW_THREADS
  wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, NULL, 0);
-
- /* issue11395 there is an unspecified upper bound on how many bytes
- can be written at once. We cap at 32k - the caller will have to
- handle partial writes.
- Since we don't know how many input bytes are being ignored, we
- have to reduce and recalculate. */
- while (wlen > 32766 / sizeof(wchar_t)) {
+ while (wlen > WINCONSOLE_MAX_WRITE_SIZE) {
  len /= 2;
  /* Fix for github issues gh-110913 and gh-82052. */
  len = _find_last_utf8_boundary(b->buf, len);