diff --git a/spellcheck.dic b/spellcheck.dic index 5a0745df32d..4b9288118d2 100644 --- a/spellcheck.dic +++ b/spellcheck.dic @@ -1,4 +1,4 @@ -283 +284 & + < @@ -131,6 +131,7 @@ io IOCP iOS IOs +io_uring IP IPv4 IPv6 diff --git a/tokio/src/fs/mod.rs b/tokio/src/fs/mod.rs index f6d9605fe94..c1855c42aeb 100644 --- a/tokio/src/fs/mod.rs +++ b/tokio/src/fs/mod.rs @@ -1,46 +1,218 @@ #![cfg(not(loom))] -//! Asynchronous file and standard stream adaptation. +//! Asynchronous file utilities. //! -//! This module contains utility methods and adapter types for input/output to -//! files or standard streams (`Stdin`, `Stdout`, `Stderr`), and -//! filesystem manipulation, for use within (and only within) a Tokio runtime. +//! This module contains utility methods for working with the file system +//! asynchronously. This includes reading/writing to files, and working with +//! directories. //! -//! Tasks run by *worker* threads should not block, as this could delay -//! servicing reactor events. Portable filesystem operations are blocking, -//! however. This module offers adapters which use a `blocking` annotation -//! to inform the runtime that a blocking operation is required. When -//! necessary, this allows the runtime to convert the current thread from a -//! *worker* to a *backup* thread, where blocking is acceptable. +//! Be aware that most operating systems do not provide asynchronous file system +//! APIs. Because of that, Tokio will use ordinary blocking file operations +//! behind the scenes. This is done using the [`spawn_blocking`] threadpool to +//! run them in the background. //! -//! ## Usage +//! The `tokio::fs` module should only be used for ordinary files. Trying to use +//! it with e.g., a named pipe on Linux can result in surprising behavior, +//! such as hangs during runtime shutdown. For special files, you should use a +//! dedicated type such as [`tokio::net::unix::pipe`] or [`AsyncFd`] instead. //! -//! Where possible, users should prefer the provided asynchronous-specific -//! traits such as [`AsyncRead`], or methods returning a `Future` or `Poll` -//! type. Adaptions also extend to traits like `std::io::Read` where methods -//! return `std::io::Result`. Be warned that these adapted methods may return -//! `std::io::ErrorKind::WouldBlock` if a *worker* thread can not be converted -//! to a *backup* thread immediately. +//! Currently, Tokio will always use [`spawn_blocking`] on all platforms, but it +//! may be changed to use asynchronous file system APIs such as io_uring in the +//! future. //! -//! **Warning**: These adapters may create a large number of temporary tasks, -//! especially when reading large files. When performing a lot of operations -//! in one batch, it may be significantly faster to use [`spawn_blocking`] -//! directly: +//! # Usage +//! +//! The easiest way to use this module is to use the utility functions that +//! operate on entire files: +//! +//! * [`tokio::fs::read`](fn@crate::fs::read) +//! * [`tokio::fs::read_to_string`](fn@crate::fs::read_to_string) +//! * [`tokio::fs::write`](fn@crate::fs::write) +//! +//! The two `read` functions reads the entire file and returns its contents. +//! The `write` function takes the contents of the file and writes those +//! contents to the file. It overwrites the existing file, if any. +//! +//! For example, to read the file: //! //! ``` +//! # async fn dox() -> std::io::Result<()> { +//! let contents = tokio::fs::read_to_string("my_file.txt").await?; +//! +//! println!("File has {} lines.", contents.lines().count()); +//! # Ok(()) +//! # } +//! ``` +//! +//! To overwrite the file: +//! +//! ``` +//! # async fn dox() -> std::io::Result<()> { +//! let contents = "First line.\nSecond line.\nThird line.\n"; +//! +//! tokio::fs::write("my_file.txt", contents.as_bytes()).await?; +//! # Ok(()) +//! # } +//! ``` +//! +//! ## Using `File` +//! +//! The main type for interacting with files is [`File`]. It can be used to read +//! from and write to a given file. This is done using the [`AsyncRead`] and +//! [`AsyncWrite`] traits. This type is generally used when you want to do +//! something more complex than just reading or writing the entire contents in +//! one go. +//! +//! **Note:** It is important to use [`flush`] when writing to a Tokio +//! [`File`]. This is because calls to `write` will return before the write has +//! finished, and [`flush`] will wait for the write to finish. (The write will +//! happen even if you don't flush; it will just happen later.) This is +//! different from [`std::fs::File`], and is due to the fact that `File` uses +//! `spawn_blocking` behind the scenes. +//! +//! For example, to count the number of lines in a file without loading the +//! entire file into memory: +//! +//! ```no_run //! use tokio::fs::File; -//! use std::io::{BufReader, BufRead}; -//! async fn count_lines(file: File) -> Result { -//! let file = file.into_std().await; -//! tokio::task::spawn_blocking(move || { -//! let line_count = BufReader::new(file).lines().count(); -//! Ok(line_count) -//! }).await? +//! use tokio::io::AsyncReadExt; +//! +//! # async fn dox() -> std::io::Result<()> { +//! let mut file = File::open("my_file.txt").await?; +//! +//! let mut chunk = vec![0; 4096]; +//! let mut number_of_lines = 0; +//! loop { +//! let len = file.read(&mut chunk).await?; +//! if len == 0 { +//! // Length of zero means end of file. +//! break; +//! } +//! for &b in &chunk[..len] { +//! if b == b'\n' { +//! number_of_lines += 1; +//! } +//! } //! } +//! +//! println!("File has {} lines.", number_of_lines); +//! # Ok(()) +//! # } +//! ``` +//! +//! For example, to write a file line-by-line: +//! +//! ```no_run +//! use tokio::fs::File; +//! use tokio::io::AsyncWriteExt; +//! +//! # async fn dox() -> std::io::Result<()> { +//! let mut file = File::create("my_file.txt").await?; +//! +//! file.write_all(b"First line.\n").await?; +//! file.write_all(b"Second line.\n").await?; +//! file.write_all(b"Third line.\n").await?; +//! +//! // Remember to call `flush` after writing! +//! file.flush().await?; +//! # Ok(()) +//! # } //! ``` //! +//! ## Tuning your file IO +//! +//! Tokio's file uses [`spawn_blocking`] behind the scenes, and this has serious +//! performance consequences. To get good performance with file IO on Tokio, it +//! is recommended to batch your operations into as few `spawn_blocking` calls +//! as possible. +//! +//! One example of this difference can be seen by comparing the two reading +//! examples above. The first example uses [`tokio::fs::read`], which reads the +//! entire file in a single `spawn_blocking` call, and then returns it. The +//! second example will read the file in chunks using many `spawn_blocking` +//! calls. This means that the second example will most likely be more expensive +//! for large files. (Of course, using chunks may be necessary for very large +//! files that don't fit in memory.) +//! +//! The following examples will show some strategies for this: +//! +//! When creating a file, write the data to a `String` or `Vec` and then +//! write the entire file in a single `spawn_blocking` call with +//! `tokio::fs::write`. +//! +//! ```no_run +//! # async fn dox() -> std::io::Result<()> { +//! let mut contents = String::new(); +//! +//! contents.push_str("First line.\n"); +//! contents.push_str("Second line.\n"); +//! contents.push_str("Third line.\n"); +//! +//! tokio::fs::write("my_file.txt", contents.as_bytes()).await?; +//! # Ok(()) +//! # } +//! ``` +//! +//! Use [`BufReader`] and [`BufWriter`] to buffer many small reads or writes +//! into a few large ones. This example will most likely only perform one +//! `spawn_blocking` call. +//! +//! ```no_run +//! use tokio::fs::File; +//! use tokio::io::{AsyncWriteExt, BufWriter}; +//! +//! # async fn dox() -> std::io::Result<()> { +//! let mut file = BufWriter::new(File::create("my_file.txt").await?); +//! +//! file.write_all(b"First line.\n").await?; +//! file.write_all(b"Second line.\n").await?; +//! file.write_all(b"Third line.\n").await?; +//! +//! // Due to the BufWriter, the actual write and spawn_blocking +//! // call happens when you flush. +//! file.flush().await?; +//! # Ok(()) +//! # } +//! ``` +//! +//! Manually use [`std::fs`] inside [`spawn_blocking`]. +//! +//! ```no_run +//! use std::fs::File; +//! use std::io::{self, Write}; +//! use tokio::task::spawn_blocking; +//! +//! # async fn dox() -> std::io::Result<()> { +//! spawn_blocking(move || { +//! let mut file = File::create("my_file.txt")?; +//! +//! file.write_all(b"First line.\n")?; +//! file.write_all(b"Second line.\n")?; +//! file.write_all(b"Third line.\n")?; +//! +//! // Unlike Tokio's file, the std::fs file does +//! // not need flush. +//! +//! io::Result::Ok(()) +//! }).await.unwrap()?; +//! # Ok(()) +//! # } +//! ``` +//! +//! It's also good to be aware of [`File::set_max_buf_size`], which controls the +//! maximum amount of bytes that Tokio's [`File`] will read or write in a single +//! [`spawn_blocking`] call. The default is two megabytes, but this is subject +//! to change. +//! //! [`spawn_blocking`]: fn@crate::task::spawn_blocking //! [`AsyncRead`]: trait@crate::io::AsyncRead +//! [`AsyncWrite`]: trait@crate::io::AsyncWrite +//! [`BufReader`]: struct@crate::io::BufReader +//! [`BufWriter`]: struct@crate::io::BufWriter +//! [`tokio::net::unix::pipe`]: crate::net::unix::pipe +//! [`AsyncFd`]: crate::io::unix::AsyncFd +//! [`flush`]: crate::io::AsyncWriteExt::flush +//! [`tokio::fs::read`]: fn@crate::fs::read mod canonicalize; pub use self::canonicalize::canonicalize;