From 26301d05f66cfa8f0a508973bb18a914eadc3d27 Mon Sep 17 00:00:00 2001 From: snapdgn Date: Sun, 11 Sep 2022 15:36:18 +0530 Subject: [PATCH 01/16] refactor: declarative macros to rust --- Cargo.lock | 2 +- src/uu/stat/src/stat.rs | 106 +++++++++++++++++++++------------------- 2 files changed, 56 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7d99956597c..3b42eb9dcab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2624,7 +2624,7 @@ dependencies = [ "itertools", "quick-error", "regex", - "time 0.3.14", + "time", "uucore", ] diff --git a/src/uu/stat/src/stat.rs b/src/uu/stat/src/stat.rs index 2daa36daef5..e4658253d39 100644 --- a/src/uu/stat/src/stat.rs +++ b/src/uu/stat/src/stat.rs @@ -9,7 +9,7 @@ extern crate uucore; use clap::builder::ValueParser; use uucore::display::Quotable; -use uucore::error::{FromIo, UResult, USimpleError}; +use uucore::error::{FromIo, UError, UResult, USimpleError}; use uucore::fs::display_permissions; use uucore::fsext::{ pretty_filetype, pretty_fstype, pretty_time, read_fs_list, statfs, BirthTime, FsMeta, @@ -26,57 +26,61 @@ use std::os::unix::prelude::OsStrExt; use std::path::Path; use std::{cmp, fs, iter}; -macro_rules! check_bound { - ($str: ident, $bound:expr, $beg: expr, $end: expr) => { - if $end >= $bound { - return Err(USimpleError::new( - 1, - format!("{}: invalid directive", $str[$beg..$end].quote()), - )); - } - }; +fn check_bound(slice: &str, bound: usize, beg: usize, end: usize) -> Result<(), Box> { + if end >= bound { + return Err(USimpleError::new( + 1, + format!("{}: invalid directive", slice[beg..end].quote()), + )); + } + Ok(()) } -macro_rules! fill_string { - ($str: ident, $c: expr, $cnt: expr) => { - iter::repeat($c) - .take($cnt) - .map(|c| $str.push(c)) - .all(|_| true) - }; + +fn fill_string(strs: &mut String, c: char, cnt: usize) { + iter::repeat(c) + .take(cnt) + .map(|c| strs.push(c)) + .all(|_| true); } -macro_rules! extend_digits { - ($str: expr, $min: expr) => { - if $min > $str.len() { - let mut pad = String::with_capacity($min); - fill_string!(pad, '0', $min - $str.len()); - pad.push_str($str); - pad.into() - } else { - $str.into() - } - }; + +fn extend_digits(strs: &str, min: usize) -> Cow<'_, str> { + if min > strs.len() { + let mut pad = String::with_capacity(min); + fill_string(&mut pad, '0', min - strs.len()); + pad.push_str(strs); + return pad.into(); + } else { + return strs.into(); + } } -macro_rules! pad_and_print { - ($result: ident, $str: ident, $left: expr, $width: expr, $padding: expr) => { - if $str.len() < $width { - if $left { - $result.push_str($str.as_ref()); - fill_string!($result, $padding, $width - $str.len()); - } else { - fill_string!($result, $padding, $width - $str.len()); - $result.push_str($str.as_ref()); - } + +fn pad_and_print>( + result: &mut String, + strs: S, + left: bool, + width: usize, + padding: char, +) { + let strs_ref = strs.as_ref(); + if strs_ref.len() < width { + if left { + result.push_str(strs.as_ref()); + fill_string(result, padding, width - strs_ref.len()); } else { - $result.push_str($str.as_ref()); + fill_string(result, padding, width - strs_ref.len()); + result.push_str(strs.as_ref()); } - print!("{}", $result); - }; + } else { + result.push_str(strs.as_ref()); + } + print!("{}", result); } + macro_rules! print_adjusted { ($str: ident, $left: expr, $width: expr, $padding: expr) => { let field_width = cmp::max($width, $str.len()); let mut result = String::with_capacity(field_width); - pad_and_print!(result, $str, $left, field_width, $padding); + pad_and_print(&mut result, $str, $left, field_width, $padding); }; ($str: ident, $left: expr, $need_prefix: expr, $prefix: expr, $width: expr, $padding: expr) => { let mut field_width = cmp::max($width, $str.len()); @@ -85,7 +89,7 @@ macro_rules! print_adjusted { result.push_str($prefix); field_width -= $prefix.len(); } - pad_and_print!(result, $str, $left, field_width, $padding); + pad_and_print(&mut result, $str, $left, field_width, $padding); }; } @@ -306,7 +310,7 @@ fn print_it(arg: &str, output_type: &OutputType, flag: u8, width: usize, precisi Cow::Borrowed(arg) }; let min_digits = cmp::max(precision, arg.len() as i32) as usize; - let extended: Cow = extend_digits!(arg.as_ref(), min_digits); + let extended: Cow = extend_digits(arg.as_ref(), min_digits); print_adjusted!(extended, left_align, has_sign, prefix, width, padding_char); } OutputType::Unsigned => { @@ -316,12 +320,12 @@ fn print_it(arg: &str, output_type: &OutputType, flag: u8, width: usize, precisi Cow::Borrowed(arg) }; let min_digits = cmp::max(precision, arg.len() as i32) as usize; - let extended: Cow = extend_digits!(arg.as_ref(), min_digits); + let extended: Cow = extend_digits(arg.as_ref(), min_digits); print_adjusted!(extended, left_align, width, padding_char); } OutputType::UnsignedOct => { let min_digits = cmp::max(precision, arg.len() as i32) as usize; - let extended: Cow = extend_digits!(arg, min_digits); + let extended: Cow = extend_digits(arg, min_digits); print_adjusted!( extended, left_align, @@ -333,7 +337,7 @@ fn print_it(arg: &str, output_type: &OutputType, flag: u8, width: usize, precisi } OutputType::UnsignedHex => { let min_digits = cmp::max(precision, arg.len() as i32) as usize; - let extended: Cow = extend_digits!(arg, min_digits); + let extended: Cow = extend_digits(arg, min_digits); print_adjusted!( extended, left_align, @@ -384,7 +388,7 @@ impl Stater { } i += 1; } - check_bound!(format_str, bound, old, i); + check_bound(format_str, bound, old, i)?; let mut width = 0_usize; let mut precision = -1_i32; @@ -394,11 +398,11 @@ impl Stater { width = field_width; j += offset; } - check_bound!(format_str, bound, old, j); + check_bound(format_str, bound, old, j)?; if chars[j] == '.' { j += 1; - check_bound!(format_str, bound, old, j); + check_bound(format_str, bound, old, j)?; match format_str[j..].scan_num::() { Some((value, offset)) => { @@ -409,7 +413,7 @@ impl Stater { } None => precision = 0, } - check_bound!(format_str, bound, old, j); + check_bound(format_str, bound, old, j)?; } i = j; From 39e352e2af915cac8ef79c097447ed8356751411 Mon Sep 17 00:00:00 2001 From: snapdgn Date: Sun, 11 Sep 2022 20:55:37 +0530 Subject: [PATCH 02/16] fix: failing test cases & some refactoring --- src/uu/stat/src/stat.rs | 130 +++++++++++++++++++++------------------- 1 file changed, 68 insertions(+), 62 deletions(-) diff --git a/src/uu/stat/src/stat.rs b/src/uu/stat/src/stat.rs index e4658253d39..b92ad764d65 100644 --- a/src/uu/stat/src/stat.rs +++ b/src/uu/stat/src/stat.rs @@ -9,7 +9,7 @@ extern crate uucore; use clap::builder::ValueParser; use uucore::display::Quotable; -use uucore::error::{FromIo, UError, UResult, USimpleError}; +use uucore::error::{FromIo, UResult, USimpleError}; use uucore::fs::display_permissions; use uucore::fsext::{ pretty_filetype, pretty_fstype, pretty_time, read_fs_list, statfs, BirthTime, FsMeta, @@ -26,7 +26,7 @@ use std::os::unix::prelude::OsStrExt; use std::path::Path; use std::{cmp, fs, iter}; -fn check_bound(slice: &str, bound: usize, beg: usize, end: usize) -> Result<(), Box> { +fn check_bound(slice: &str, bound: usize, beg: usize, end: usize) -> UResult<()> { if end >= bound { return Err(USimpleError::new( 1, @@ -36,61 +36,53 @@ fn check_bound(slice: &str, bound: usize, beg: usize, end: usize) -> Result<(), Ok(()) } -fn fill_string(strs: &mut String, c: char, cnt: usize) { - iter::repeat(c) - .take(cnt) - .map(|c| strs.push(c)) - .all(|_| true); -} - -fn extend_digits(strs: &str, min: usize) -> Cow<'_, str> { - if min > strs.len() { +fn extend_digits(string: &str, min: usize) -> Cow<'_, str> { + if min > string.len() { let mut pad = String::with_capacity(min); - fill_string(&mut pad, '0', min - strs.len()); - pad.push_str(strs); - return pad.into(); + iter::repeat('0') + .take(min - string.len()) + .map(|_| pad.push('0')) + .all(|_| true); + pad.push_str(string); + pad.into() } else { - return strs.into(); + string.into() } } -fn pad_and_print>( - result: &mut String, - strs: S, +enum Padding { + Zero, + Space, +} + +fn pad_and_print(result: &str, left: bool, width: usize, padding: Padding) { + match (left, padding) { + (false, Padding::Zero) => print!("{result:0>width$}"), + (false, Padding::Space) => print!("{result:>width$}"), + (true, Padding::Zero) => print!("{result:0 print!("{result:, + prefix: Option<&str>, width: usize, - padding: char, + padding: Padding, ) { - let strs_ref = strs.as_ref(); - if strs_ref.len() < width { - if left { - result.push_str(strs.as_ref()); - fill_string(result, padding, width - strs_ref.len()); - } else { - fill_string(result, padding, width - strs_ref.len()); - result.push_str(strs.as_ref()); + let mut field_width = cmp::max(width, s.len()); + if let Some(p) = prefix { + if let Some(needprefix) = need_prefix { + if needprefix { + field_width -= p.len(); + } } + pad_and_print(s, left, field_width, padding); } else { - result.push_str(strs.as_ref()); + pad_and_print(s, left, field_width, padding); } - print!("{}", result); -} - -macro_rules! print_adjusted { - ($str: ident, $left: expr, $width: expr, $padding: expr) => { - let field_width = cmp::max($width, $str.len()); - let mut result = String::with_capacity(field_width); - pad_and_print(&mut result, $str, $left, field_width, $padding); - }; - ($str: ident, $left: expr, $need_prefix: expr, $prefix: expr, $width: expr, $padding: expr) => { - let mut field_width = cmp::max($width, $str.len()); - let mut result = String::with_capacity(field_width + $prefix.len()); - if $need_prefix { - result.push_str($prefix); - field_width -= $prefix.len(); - } - pad_and_print(&mut result, $str, $left, field_width, $padding); - }; } static ABOUT: &str = "Display file or file system status."; @@ -271,10 +263,10 @@ fn print_it(arg: &str, output_type: &OutputType, flag: u8, width: usize, precisi } let left_align = has!(flag, F_LEFT); - let padding_char = if has!(flag, F_ZERO) && !left_align && precision == -1 { - '0' + let padding_char: Padding = if has!(flag, F_ZERO) && !left_align && precision == -1 { + Padding::Zero } else { - ' ' + Padding::Space }; let has_sign = has!(flag, F_SIGN) || has!(flag, F_SPACE); @@ -301,7 +293,7 @@ fn print_it(arg: &str, output_type: &OutputType, flag: u8, width: usize, precisi } else { arg }; - print_adjusted!(s, left_align, width, ' '); + print_adjusted(s, left_align, None, None, width, Padding::Space); } OutputType::Integer => { let arg = if has!(flag, F_GROUP) { @@ -311,7 +303,14 @@ fn print_it(arg: &str, output_type: &OutputType, flag: u8, width: usize, precisi }; let min_digits = cmp::max(precision, arg.len() as i32) as usize; let extended: Cow = extend_digits(arg.as_ref(), min_digits); - print_adjusted!(extended, left_align, has_sign, prefix, width, padding_char); + print_adjusted( + extended.as_ref(), + left_align, + Some(has_sign), + Some(prefix), + width, + padding_char, + ); } OutputType::Unsigned => { let arg = if has!(flag, F_GROUP) { @@ -321,30 +320,37 @@ fn print_it(arg: &str, output_type: &OutputType, flag: u8, width: usize, precisi }; let min_digits = cmp::max(precision, arg.len() as i32) as usize; let extended: Cow = extend_digits(arg.as_ref(), min_digits); - print_adjusted!(extended, left_align, width, padding_char); + print_adjusted( + extended.as_ref(), + left_align, + None, + None, + width, + padding_char, + ); } OutputType::UnsignedOct => { let min_digits = cmp::max(precision, arg.len() as i32) as usize; let extended: Cow = extend_digits(arg, min_digits); - print_adjusted!( - extended, + print_adjusted( + extended.as_ref(), left_align, - should_alter, - prefix, + Some(should_alter), + Some(prefix), width, - padding_char + padding_char, ); } OutputType::UnsignedHex => { let min_digits = cmp::max(precision, arg.len() as i32) as usize; let extended: Cow = extend_digits(arg, min_digits); - print_adjusted!( - extended, + print_adjusted( + extended.as_ref(), left_align, - should_alter, - prefix, + Some(should_alter), + Some(prefix), width, - padding_char + padding_char, ); } _ => unreachable!(), From 243546ce967af5aaeeed99880a71d62809eafe97 Mon Sep 17 00:00:00 2001 From: snapdgn Date: Mon, 12 Sep 2022 12:19:56 +0530 Subject: [PATCH 03/16] fix: style/spelling --- src/uu/stat/src/stat.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/uu/stat/src/stat.rs b/src/uu/stat/src/stat.rs index b92ad764d65..883bf66d2c5 100644 --- a/src/uu/stat/src/stat.rs +++ b/src/uu/stat/src/stat.rs @@ -74,8 +74,8 @@ fn print_adjusted( ) { let mut field_width = cmp::max(width, s.len()); if let Some(p) = prefix { - if let Some(needprefix) = need_prefix { - if needprefix { + if let Some(prefix_flag) = need_prefix { + if prefix_flag { field_width -= p.len(); } } From d8226bf658e84fba7a3bd150dd21d221819a5ae3 Mon Sep 17 00:00:00 2001 From: snapdgn Date: Mon, 12 Sep 2022 18:35:30 +0530 Subject: [PATCH 04/16] add: documentation to refactored macros->functions --- src/uu/stat/src/stat.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/uu/stat/src/stat.rs b/src/uu/stat/src/stat.rs index 883bf66d2c5..ed1cba6940d 100644 --- a/src/uu/stat/src/stat.rs +++ b/src/uu/stat/src/stat.rs @@ -26,6 +26,8 @@ use std::os::unix::prelude::OsStrExt; use std::path::Path; use std::{cmp, fs, iter}; +/// checks if the string is within the specified bound +/// fn check_bound(slice: &str, bound: usize, beg: usize, end: usize) -> UResult<()> { if end >= bound { return Err(USimpleError::new( @@ -36,6 +38,9 @@ fn check_bound(slice: &str, bound: usize, beg: usize, end: usize) -> UResult<()> Ok(()) } +/// pads the string with zeroes if supplied min is greater +/// then the length of the string, else returns the original string +/// fn extend_digits(string: &str, min: usize) -> Cow<'_, str> { if min > string.len() { let mut pad = String::with_capacity(min); @@ -55,6 +60,16 @@ enum Padding { Space, } +/// pads the string with zeroes or spaces and prints it +/// +/// # Example +/// ```ignore +/// uu_stat::pad_and_print("1", false, 5, Padding::Zero) == "00001"; +/// ``` +/// currently only supports '0' & ' ' as the padding character +/// because the format specification of print! does not support general +/// fill characters +/// fn pad_and_print(result: &str, left: bool, width: usize, padding: Padding) { match (left, padding) { (false, Padding::Zero) => print!("{result:0>width$}"), @@ -64,6 +79,8 @@ fn pad_and_print(result: &str, left: bool, width: usize, padding: Padding) { }; } +/// prints the adjusted string after padding +/// fn print_adjusted( s: &str, left: bool, From b723be4fe24784eff16da7d4e81976503a121b52 Mon Sep 17 00:00:00 2001 From: snapdgn Date: Mon, 12 Sep 2022 19:56:02 +0530 Subject: [PATCH 05/16] reposition functions --- src/uu/stat/src/stat.rs | 43 ++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/src/uu/stat/src/stat.rs b/src/uu/stat/src/stat.rs index ed1cba6940d..23e66eddd39 100644 --- a/src/uu/stat/src/stat.rs +++ b/src/uu/stat/src/stat.rs @@ -26,6 +26,27 @@ use std::os::unix::prelude::OsStrExt; use std::path::Path; use std::{cmp, fs, iter}; +static ABOUT: &str = "Display file or file system status."; +const USAGE: &str = "{} [OPTION]... FILE..."; + +pub mod options { + pub static DEREFERENCE: &str = "dereference"; + pub static FILE_SYSTEM: &str = "file-system"; + pub static FORMAT: &str = "format"; + pub static PRINTF: &str = "printf"; + pub static TERSE: &str = "terse"; +} + +static ARG_FILES: &str = "files"; + +pub const F_ALTER: u8 = 1; +pub const F_ZERO: u8 = 1 << 1; +pub const F_LEFT: u8 = 1 << 2; +pub const F_SPACE: u8 = 1 << 3; +pub const F_SIGN: u8 = 1 << 4; +// unused at present +pub const F_GROUP: u8 = 1 << 5; + /// checks if the string is within the specified bound /// fn check_bound(slice: &str, bound: usize, beg: usize, end: usize) -> UResult<()> { @@ -101,28 +122,6 @@ fn print_adjusted( pad_and_print(s, left, field_width, padding); } } - -static ABOUT: &str = "Display file or file system status."; -const USAGE: &str = "{} [OPTION]... FILE..."; - -pub mod options { - pub static DEREFERENCE: &str = "dereference"; - pub static FILE_SYSTEM: &str = "file-system"; - pub static FORMAT: &str = "format"; - pub static PRINTF: &str = "printf"; - pub static TERSE: &str = "terse"; -} - -static ARG_FILES: &str = "files"; - -pub const F_ALTER: u8 = 1; -pub const F_ZERO: u8 = 1 << 1; -pub const F_LEFT: u8 = 1 << 2; -pub const F_SPACE: u8 = 1 << 3; -pub const F_SIGN: u8 = 1 << 4; -// unused at present -pub const F_GROUP: u8 = 1 << 5; - #[derive(Debug, PartialEq, Eq)] pub enum OutputType { Str, From 951c51e7407a4aecfb832ac2317d20704f649c32 Mon Sep 17 00:00:00 2001 From: Joining7943 <111500881+Joining7943@users.noreply.github.com> Date: Tue, 13 Sep 2022 22:54:36 +0200 Subject: [PATCH 06/16] tail: large refactoring and cleanup of the tail code base. See also #3905 for details --- src/uu/tail/src/args.rs | 473 +++++++++ src/uu/tail/src/chunks.rs | 14 +- src/uu/tail/src/follow/files.rs | 211 ++++ src/uu/tail/src/follow/mod.rs | 9 + src/uu/tail/src/follow/watch.rs | 595 ++++++++++++ src/uu/tail/src/paths.rs | 279 ++++++ src/uu/tail/src/tail.rs | 1617 ++++--------------------------- src/uu/tail/src/text.rs | 20 + tests/by-util/test_tail.rs | 36 + 9 files changed, 1827 insertions(+), 1427 deletions(-) create mode 100644 src/uu/tail/src/args.rs create mode 100644 src/uu/tail/src/follow/files.rs create mode 100644 src/uu/tail/src/follow/mod.rs create mode 100644 src/uu/tail/src/follow/watch.rs create mode 100644 src/uu/tail/src/paths.rs create mode 100644 src/uu/tail/src/text.rs diff --git a/src/uu/tail/src/args.rs b/src/uu/tail/src/args.rs new file mode 100644 index 00000000000..dfa7f6035a9 --- /dev/null +++ b/src/uu/tail/src/args.rs @@ -0,0 +1,473 @@ +// * This file is part of the uutils coreutils package. +// * +// * For the full copyright and license information, please view the LICENSE +// * file that was distributed with this source code. + +// spell-checker:ignore (ToDO) kqueue Signum + +use crate::paths::Input; +use crate::{parse, platform, Quotable}; +use clap::{Arg, ArgMatches, Command, ValueSource}; +use std::collections::VecDeque; +use std::ffi::OsString; +use std::time::Duration; +use uucore::error::{UResult, USimpleError, UUsageError}; +use uucore::format_usage; +use uucore::parse_size::{parse_size, ParseSizeError}; + +const ABOUT: &str = "\ + Print the last 10 lines of each FILE to standard output.\n\ + With more than one FILE, precede each with a header giving the file name.\n\ + With no FILE, or when FILE is -, read standard input.\n\ + \n\ + Mandatory arguments to long flags are mandatory for short flags too.\ + "; +const USAGE: &str = "{} [FLAG]... [FILE]..."; + +pub mod options { + pub mod verbosity { + pub static QUIET: &str = "quiet"; + pub static VERBOSE: &str = "verbose"; + } + pub static BYTES: &str = "bytes"; + pub static FOLLOW: &str = "follow"; + pub static LINES: &str = "lines"; + pub static PID: &str = "pid"; + pub static SLEEP_INT: &str = "sleep-interval"; + pub static ZERO_TERM: &str = "zero-terminated"; + pub static DISABLE_INOTIFY_TERM: &str = "-disable-inotify"; // NOTE: three hyphens is correct + pub static USE_POLLING: &str = "use-polling"; + pub static RETRY: &str = "retry"; + pub static FOLLOW_RETRY: &str = "F"; + pub static MAX_UNCHANGED_STATS: &str = "max-unchanged-stats"; + pub static ARG_FILES: &str = "files"; + pub static PRESUME_INPUT_PIPE: &str = "-presume-input-pipe"; // NOTE: three hyphens is correct +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Signum { + Negative(u64), + Positive(u64), + PlusZero, + MinusZero, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum FilterMode { + Bytes(Signum), + + /// Mode for lines delimited by delimiter as u8 + Lines(Signum, u8), +} + +impl FilterMode { + fn from(matches: &ArgMatches) -> UResult { + let zero_term = matches.contains_id(options::ZERO_TERM); + let mode = if let Some(arg) = matches.value_of(options::BYTES) { + match parse_num(arg) { + Ok(signum) => Self::Bytes(signum), + Err(e) => { + return Err(UUsageError::new( + 1, + format!("invalid number of bytes: {}", e), + )) + } + } + } else if let Some(arg) = matches.value_of(options::LINES) { + match parse_num(arg) { + Ok(signum) => { + let delimiter = if zero_term { 0 } else { b'\n' }; + Self::Lines(signum, delimiter) + } + Err(e) => { + return Err(UUsageError::new( + 1, + format!("invalid number of lines: {}", e), + )) + } + } + } else if zero_term { + Self::default_zero() + } else { + Self::default() + }; + + Ok(mode) + } + + fn default_zero() -> Self { + Self::Lines(Signum::Negative(10), 0) + } +} + +impl Default for FilterMode { + fn default() -> Self { + Self::Lines(Signum::Negative(10), b'\n') + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum FollowMode { + Descriptor, + Name, +} + +#[derive(Debug, Default)] +pub struct Settings { + pub follow: Option, + pub max_unchanged_stats: u32, + pub mode: FilterMode, + pub pid: platform::Pid, + pub retry: bool, + pub sleep_sec: Duration, + pub use_polling: bool, + pub verbose: bool, + pub presume_input_pipe: bool, + pub inputs: VecDeque, +} + +impl Settings { + pub fn from(matches: &clap::ArgMatches) -> UResult { + let mut settings: Self = Self { + sleep_sec: Duration::from_secs_f32(1.0), + max_unchanged_stats: 5, + ..Default::default() + }; + + settings.follow = if matches.contains_id(options::FOLLOW_RETRY) { + Some(FollowMode::Name) + } else if matches.value_source(options::FOLLOW) != Some(ValueSource::CommandLine) { + None + } else if matches.value_of(options::FOLLOW) == Some("name") { + Some(FollowMode::Name) + } else { + Some(FollowMode::Descriptor) + }; + + settings.retry = + matches.contains_id(options::RETRY) || matches.contains_id(options::FOLLOW_RETRY); + + if settings.retry && settings.follow.is_none() { + show_warning!("--retry ignored; --retry is useful only when following"); + } + + if let Some(s) = matches.value_of(options::SLEEP_INT) { + settings.sleep_sec = match s.parse::() { + Ok(s) => Duration::from_secs_f32(s), + Err(_) => { + return Err(UUsageError::new( + 1, + format!("invalid number of seconds: {}", s.quote()), + )) + } + } + } + + settings.use_polling = matches.contains_id(options::USE_POLLING); + + if let Some(s) = matches.value_of(options::MAX_UNCHANGED_STATS) { + settings.max_unchanged_stats = match s.parse::() { + Ok(s) => s, + Err(_) => { + return Err(UUsageError::new( + 1, + format!( + "invalid maximum number of unchanged stats between opens: {}", + s.quote() + ), + )); + } + } + } + + if let Some(pid_str) = matches.value_of(options::PID) { + match pid_str.parse() { + Ok(pid) => { + // NOTE: on unix platform::Pid is i32, on windows platform::Pid is u32 + #[cfg(unix)] + if pid < 0 { + // NOTE: tail only accepts an unsigned pid + return Err(USimpleError::new( + 1, + format!("invalid PID: {}", pid_str.quote()), + )); + } + settings.pid = pid; + if settings.follow.is_none() { + show_warning!("PID ignored; --pid=PID is useful only when following"); + } + if !platform::supports_pid_checks(settings.pid) { + show_warning!("--pid=PID is not supported on this system"); + settings.pid = 0; + } + } + Err(e) => { + return Err(USimpleError::new( + 1, + format!("invalid PID: {}: {}", pid_str.quote(), e), + )); + } + } + } + + settings.mode = FilterMode::from(matches)?; + + // Mimic GNU's tail for -[nc]0 without -f and exit immediately + if settings.follow.is_none() + && matches!( + settings.mode, + FilterMode::Lines(Signum::MinusZero, _) | FilterMode::Bytes(Signum::MinusZero) + ) + { + std::process::exit(0) + } + + let mut inputs: VecDeque = matches + .get_many::(options::ARG_FILES) + .map(|v| v.map(|string| Input::from(string.clone())).collect()) + .unwrap_or_default(); + + // apply default and add '-' to inputs if none is present + if inputs.is_empty() { + inputs.push_front(Input::default()); + } + + settings.verbose = (matches.contains_id(options::verbosity::VERBOSE) || inputs.len() > 1) + && !matches.contains_id(options::verbosity::QUIET); + + settings.inputs = inputs; + + settings.presume_input_pipe = matches.contains_id(options::PRESUME_INPUT_PIPE); + + Ok(settings) + } +} + +pub fn arg_iterate<'a>( + mut args: impl uucore::Args + 'a, +) -> UResult + 'a>> { + // argv[0] is always present + let first = args.next().unwrap(); + if let Some(second) = args.next() { + if let Some(s) = second.to_str() { + match parse::parse_obsolete(s) { + Some(Ok(iter)) => Ok(Box::new(vec![first].into_iter().chain(iter).chain(args))), + Some(Err(e)) => Err(UUsageError::new( + 1, + match e { + parse::ParseError::Syntax => format!("bad argument format: {}", s.quote()), + parse::ParseError::Overflow => format!( + "invalid argument: {} Value too large for defined datatype", + s.quote() + ), + }, + )), + None => Ok(Box::new(vec![first, second].into_iter().chain(args))), + } + } else { + Err(UUsageError::new(1, "bad argument encoding".to_owned())) + } + } else { + Ok(Box::new(vec![first].into_iter())) + } +} + +fn parse_num(src: &str) -> Result { + let mut size_string = src.trim(); + let mut starting_with = false; + + if let Some(c) = size_string.chars().next() { + if c == '+' || c == '-' { + // tail: '-' is not documented (8.32 man pages) + size_string = &size_string[1..]; + if c == '+' { + starting_with = true; + } + } + } else { + return Err(ParseSizeError::ParseFailure(src.to_string())); + } + + parse_size(size_string).map(|n| match (n, starting_with) { + (0, true) => Signum::PlusZero, + (0, false) => Signum::MinusZero, + (n, true) => Signum::Positive(n), + (n, false) => Signum::Negative(n), + }) +} + +pub fn stdin_is_pipe_or_fifo() -> bool { + #[cfg(unix)] + { + platform::stdin_is_pipe_or_fifo() + } + #[cfg(windows)] + { + winapi_util::file::typ(winapi_util::HandleRef::stdin()) + .map(|t| t.is_disk() || t.is_pipe()) + .unwrap_or(false) + } +} + +pub fn parse_args(args: impl uucore::Args) -> UResult { + let matches = uu_app().try_get_matches_from(arg_iterate(args)?)?; + Settings::from(&matches) +} + +pub fn uu_app<'a>() -> Command<'a> { + #[cfg(target_os = "linux")] + pub static POLLING_HELP: &str = "Disable 'inotify' support and use polling instead"; + #[cfg(all(unix, not(target_os = "linux")))] + pub static POLLING_HELP: &str = "Disable 'kqueue' support and use polling instead"; + #[cfg(target_os = "windows")] + pub static POLLING_HELP: &str = + "Disable 'ReadDirectoryChanges' support and use polling instead"; + + Command::new(uucore::util_name()) + .version(crate_version!()) + .about(ABOUT) + .override_usage(format_usage(USAGE)) + .infer_long_args(true) + .arg( + Arg::new(options::BYTES) + .short('c') + .long(options::BYTES) + .takes_value(true) + .allow_hyphen_values(true) + .overrides_with_all(&[options::BYTES, options::LINES]) + .help("Number of bytes to print"), + ) + .arg( + Arg::new(options::FOLLOW) + .short('f') + .long(options::FOLLOW) + .default_value("descriptor") + .takes_value(true) + .min_values(0) + .max_values(1) + .require_equals(true) + .value_parser(["descriptor", "name"]) + .help("Print the file as it grows"), + ) + .arg( + Arg::new(options::LINES) + .short('n') + .long(options::LINES) + .takes_value(true) + .allow_hyphen_values(true) + .overrides_with_all(&[options::BYTES, options::LINES]) + .help("Number of lines to print"), + ) + .arg( + Arg::new(options::PID) + .long(options::PID) + .takes_value(true) + .help("With -f, terminate after process ID, PID dies"), + ) + .arg( + Arg::new(options::verbosity::QUIET) + .short('q') + .long(options::verbosity::QUIET) + .visible_alias("silent") + .overrides_with_all(&[options::verbosity::QUIET, options::verbosity::VERBOSE]) + .help("Never output headers giving file names"), + ) + .arg( + Arg::new(options::SLEEP_INT) + .short('s') + .takes_value(true) + .long(options::SLEEP_INT) + .help("Number of seconds to sleep between polling the file when running with -f"), + ) + .arg( + Arg::new(options::MAX_UNCHANGED_STATS) + .takes_value(true) + .long(options::MAX_UNCHANGED_STATS) + .help( + "Reopen a FILE which has not changed size after N (default 5) iterations \ + to see if it has been unlinked or renamed (this is the usual case of rotated \ + log files); This option is meaningful only when polling \ + (i.e., with --use-polling) and when --follow=name", + ), + ) + .arg( + Arg::new(options::verbosity::VERBOSE) + .short('v') + .long(options::verbosity::VERBOSE) + .overrides_with_all(&[options::verbosity::QUIET, options::verbosity::VERBOSE]) + .help("Always output headers giving file names"), + ) + .arg( + Arg::new(options::ZERO_TERM) + .short('z') + .long(options::ZERO_TERM) + .help("Line delimiter is NUL, not newline"), + ) + .arg( + Arg::new(options::USE_POLLING) + .alias(options::DISABLE_INOTIFY_TERM) // NOTE: Used by GNU's test suite + .alias("dis") // NOTE: Used by GNU's test suite + .long(options::USE_POLLING) + .help(POLLING_HELP), + ) + .arg( + Arg::new(options::RETRY) + .long(options::RETRY) + .help("Keep trying to open a file if it is inaccessible"), + ) + .arg( + Arg::new(options::FOLLOW_RETRY) + .short('F') + .help("Same as --follow=name --retry") + .overrides_with_all(&[options::RETRY, options::FOLLOW]), + ) + .arg( + Arg::new(options::PRESUME_INPUT_PIPE) + .long(options::PRESUME_INPUT_PIPE) + .alias(options::PRESUME_INPUT_PIPE) + .hide(true), + ) + + .arg( + Arg::new(options::ARG_FILES) + .multiple_occurrences(true) + .takes_value(true) + .min_values(1) + .value_hint(clap::ValueHint::FilePath), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_num_when_sign_is_given() { + let result = parse_num("+0"); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), Signum::PlusZero); + + let result = parse_num("+1"); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), Signum::Positive(1)); + + let result = parse_num("-0"); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), Signum::MinusZero); + + let result = parse_num("-1"); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), Signum::Negative(1)); + } + + #[test] + fn test_parse_num_when_no_sign_is_given() { + let result = parse_num("0"); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), Signum::MinusZero); + + let result = parse_num("1"); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), Signum::Negative(1)); + } +} diff --git a/src/uu/tail/src/chunks.rs b/src/uu/tail/src/chunks.rs index 8fb53c7691e..acfc69a305b 100644 --- a/src/uu/tail/src/chunks.rs +++ b/src/uu/tail/src/chunks.rs @@ -10,7 +10,7 @@ // spell-checker:ignore (ToDO) filehandle BUFSIZ use std::collections::VecDeque; use std::fs::File; -use std::io::{BufReader, Read, Seek, SeekFrom, Write}; +use std::io::{BufRead, Read, Seek, SeekFrom, Write}; use uucore::error::UResult; /// When reading files in reverse in `bounded_tail`, this is the size of each @@ -208,7 +208,7 @@ impl BytesChunk { /// that number of bytes. If EOF is reached (so 0 bytes are read), then returns /// [`UResult`] or else the result with [`Some(bytes)`] where bytes is the number of bytes /// read from the source. - pub fn fill(&mut self, filehandle: &mut BufReader) -> UResult> { + pub fn fill(&mut self, filehandle: &mut impl BufRead) -> UResult> { let num_bytes = filehandle.read(&mut self.buffer)?; self.bytes = num_bytes; if num_bytes == 0 { @@ -283,7 +283,7 @@ impl BytesChunkBuffer { /// let mut chunks = BytesChunkBuffer::new(num_print); /// chunks.fill(&mut reader).unwrap(); /// ``` - pub fn fill(&mut self, reader: &mut BufReader) -> UResult<()> { + pub fn fill(&mut self, reader: &mut impl BufRead) -> UResult<()> { let mut chunk = Box::new(BytesChunk::new()); // fill chunks with all bytes from reader and reuse already instantiated chunks if possible @@ -323,6 +323,10 @@ impl BytesChunkBuffer { } Ok(()) } + + pub fn has_data(&self) -> bool { + !self.chunks.is_empty() + } } /// Works similar to a [`BytesChunk`] but also stores the number of lines encountered in the current @@ -452,7 +456,7 @@ impl LinesChunk { /// that number of bytes. This function works like the [`BytesChunk::fill`] function besides /// that this function also counts and stores the number of lines encountered while reading from /// the `filehandle`. - pub fn fill(&mut self, filehandle: &mut BufReader) -> UResult> { + pub fn fill(&mut self, filehandle: &mut impl BufRead) -> UResult> { match self.chunk.fill(filehandle)? { None => { self.lines = 0; @@ -556,7 +560,7 @@ impl LinesChunkBuffer { /// in sum exactly `self.num_print` lines stored in all chunks. The method returns an iterator /// over these chunks. If there are no chunks, for example because the piped stdin contained no /// lines, or `num_print = 0` then `iterator.next` will return None. - pub fn fill(&mut self, reader: &mut BufReader) -> UResult<()> { + pub fn fill(&mut self, reader: &mut impl BufRead) -> UResult<()> { let mut chunk = Box::new(LinesChunk::new(self.delimiter)); while (chunk.fill(reader)?).is_some() { diff --git a/src/uu/tail/src/follow/files.rs b/src/uu/tail/src/follow/files.rs new file mode 100644 index 00000000000..1be090217c0 --- /dev/null +++ b/src/uu/tail/src/follow/files.rs @@ -0,0 +1,211 @@ +// * This file is part of the uutils coreutils package. +// * +// * For the full copyright and license information, please view the LICENSE +// * file that was distributed with this source code. + +// spell-checker:ignore tailable seekable stdlib (stdlib) + +use crate::args::Settings; +use crate::chunks::BytesChunkBuffer; +use crate::paths::{HeaderPrinter, PathExtTail}; +use crate::text; +use std::collections::hash_map::Keys; +use std::collections::HashMap; +use std::fs::{File, Metadata}; +use std::io::{stdout, BufRead, BufReader, BufWriter}; + +use std::path::{Path, PathBuf}; +use uucore::error::UResult; + +/// Data structure to keep a handle on files to follow. +/// `last` always holds the path/key of the last file that was printed from. +/// The keys of the HashMap can point to an existing file path (normal case), +/// or stdin ("-"), or to a non existing path (--retry). +/// For existing files, all keys in the HashMap are absolute Paths. +pub struct FileHandling { + map: HashMap, + last: Option, + header_printer: HeaderPrinter, +} + +impl FileHandling { + pub fn from(settings: &Settings) -> Self { + Self { + map: HashMap::with_capacity(settings.inputs.len()), + last: None, + header_printer: HeaderPrinter::new(settings.verbose, false), + } + } + + /// Wrapper for HashMap::insert using Path::canonicalize + pub fn insert(&mut self, k: &Path, v: PathData, update_last: bool) { + let k = Self::canonicalize_path(k); + if update_last { + self.last = Some(k.to_owned()); + } + let _ = self.map.insert(k, v); + } + + /// Wrapper for HashMap::remove using Path::canonicalize + pub fn remove(&mut self, k: &Path) -> PathData { + self.map.remove(&Self::canonicalize_path(k)).unwrap() + } + + /// Wrapper for HashMap::get using Path::canonicalize + pub fn get(&self, k: &Path) -> &PathData { + self.map.get(&Self::canonicalize_path(k)).unwrap() + } + + /// Wrapper for HashMap::get_mut using Path::canonicalize + pub fn get_mut(&mut self, k: &Path) -> &mut PathData { + self.map.get_mut(&Self::canonicalize_path(k)).unwrap() + } + + /// Canonicalize `path` if it is not already an absolute path + fn canonicalize_path(path: &Path) -> PathBuf { + if path.is_relative() && !path.is_stdin() { + if let Ok(p) = path.canonicalize() { + return p; + } + } + path.to_owned() + } + + pub fn get_mut_metadata(&mut self, path: &Path) -> Option<&Metadata> { + self.get_mut(path).metadata.as_ref() + } + + pub fn keys(&self) -> Keys { + self.map.keys() + } + + pub fn contains_key(&self, k: &Path) -> bool { + self.map.contains_key(k) + } + + pub fn get_last(&self) -> Option<&PathBuf> { + self.last.as_ref() + } + + /// Return true if there is only stdin remaining + pub fn only_stdin_remaining(&self) -> bool { + self.map.len() == 1 && (self.map.contains_key(Path::new(text::DASH))) + } + + /// Return true if there is at least one "tailable" path (or stdin) remaining + pub fn files_remaining(&self) -> bool { + for path in self.map.keys() { + if path.is_tailable() || path.is_stdin() { + return true; + } + } + false + } + + /// Returns true if there are no files remaining + pub fn no_files_remaining(&self, settings: &Settings) -> bool { + self.map.is_empty() || !self.files_remaining() && !settings.retry + } + + /// Set `reader` to None to indicate that `path` is not an existing file anymore. + pub fn reset_reader(&mut self, path: &Path) { + self.get_mut(path).reader = None; + } + + /// Reopen the file at the monitored `path` + pub fn update_reader(&mut self, path: &Path) -> UResult<()> { + /* + BUG: If it's not necessary to reopen a file, GNU's tail calls seek to offset 0. + However we can't call seek here because `BufRead` does not implement `Seek`. + As a workaround we always reopen the file even though this might not always + be necessary. + */ + self.get_mut(path) + .reader + .replace(Box::new(BufReader::new(File::open(&path)?))); + Ok(()) + } + + /// Reload metadata from `path`, or `metadata` + pub fn update_metadata(&mut self, path: &Path, metadata: Option) { + self.get_mut(path).metadata = if metadata.is_some() { + metadata + } else { + path.metadata().ok() + }; + } + + /// Read new data from `path` and print it to stdout + pub fn tail_file(&mut self, path: &Path, verbose: bool) -> UResult { + let mut chunks = BytesChunkBuffer::new(u64::MAX); + if let Some(reader) = self.get_mut(path).reader.as_mut() { + chunks.fill(reader)?; + } + if chunks.has_data() { + if self.needs_header(path, verbose) { + let display_name = self.get(path).display_name.clone(); + self.header_printer.print(display_name.as_str()); + } + + let stdout = stdout(); + let writer = BufWriter::new(stdout.lock()); + chunks.print(writer)?; + + self.last.replace(path.to_owned()); + self.update_metadata(path, None); + Ok(true) + } else { + Ok(false) + } + } + + /// Decide if printing `path` needs a header based on when it was last printed + pub fn needs_header(&self, path: &Path, verbose: bool) -> bool { + if verbose { + if let Some(ref last) = self.last { + return !last.eq(&path); + } else { + return true; + } + } + false + } +} + +/// Data structure to keep a handle on the BufReader, Metadata +/// and the display_name (header_name) of files that are being followed. +pub struct PathData { + pub reader: Option>, + pub metadata: Option, + pub display_name: String, +} + +impl PathData { + pub fn new( + reader: Option>, + metadata: Option, + display_name: &str, + ) -> Self { + Self { + reader, + metadata, + display_name: display_name.to_owned(), + } + } + pub fn from_other_with_path(data: Self, path: &Path) -> Self { + // Remove old reader + let old_reader = data.reader; + let reader = if old_reader.is_some() { + // Use old reader with the same file descriptor if there is one + old_reader + } else if let Ok(file) = File::open(path) { + // Open new file tail from start + Some(Box::new(BufReader::new(file)) as Box) + } else { + // Probably file was renamed/moved or removed again + None + }; + + Self::new(reader, path.metadata().ok(), data.display_name.as_str()) + } +} diff --git a/src/uu/tail/src/follow/mod.rs b/src/uu/tail/src/follow/mod.rs new file mode 100644 index 00000000000..4bb2798d12a --- /dev/null +++ b/src/uu/tail/src/follow/mod.rs @@ -0,0 +1,9 @@ +// * This file is part of the uutils coreutils package. +// * +// * For the full copyright and license information, please view the LICENSE +// * file that was distributed with this source code. + +mod files; +mod watch; + +pub use watch::{follow, WatcherService}; diff --git a/src/uu/tail/src/follow/watch.rs b/src/uu/tail/src/follow/watch.rs new file mode 100644 index 00000000000..b00d85f44dd --- /dev/null +++ b/src/uu/tail/src/follow/watch.rs @@ -0,0 +1,595 @@ +// * This file is part of the uutils coreutils package. +// * +// * For the full copyright and license information, please view the LICENSE +// * file that was distributed with this source code. + +// spell-checker:ignore (ToDO) tailable untailable stdlib kqueue Uncategorized unwatch + +use crate::args::{FollowMode, Settings}; +use crate::follow::files::{FileHandling, PathData}; +use crate::paths::{Input, InputKind, MetadataExtTail, PathExtTail}; +use crate::{platform, text}; +use notify::{RecommendedWatcher, RecursiveMode, Watcher, WatcherKind}; +use std::collections::VecDeque; +use std::io::BufRead; +use std::path::{Path, PathBuf}; +use std::sync::mpsc; +use std::sync::mpsc::{channel, Receiver}; +use uucore::display::Quotable; +use uucore::error::{set_exit_code, UResult, USimpleError}; + +pub struct WatcherRx { + watcher: Box, + receiver: Receiver>, +} + +impl WatcherRx { + fn new( + watcher: Box, + receiver: Receiver>, + ) -> Self { + Self { watcher, receiver } + } + + /// Wrapper for `notify::Watcher::watch` to also add the parent directory of `path` if necessary. + fn watch_with_parent(&mut self, path: &Path) -> UResult<()> { + let mut path = path.to_owned(); + #[cfg(target_os = "linux")] + if path.is_file() { + /* + NOTE: Using the parent directory instead of the file is a workaround. + This workaround follows the recommendation of the notify crate authors: + > On some platforms, if the `path` is renamed or removed while being watched, behavior may + > be unexpected. See discussions in [#165] and [#166]. If less surprising behavior is wanted + > one may non-recursively watch the _parent_ directory as well and manage related events. + NOTE: Adding both: file and parent results in duplicate/wrong events. + Tested for notify::InotifyWatcher and for notify::PollWatcher. + */ + if let Some(parent) = path.parent() { + if parent.is_dir() { + path = parent.to_owned(); + } else { + path = PathBuf::from("."); + } + } else { + return Err(USimpleError::new( + 1, + format!("cannot watch parent directory of {}", path.display()), + )); + }; + } + if path.is_relative() { + path = path.canonicalize()?; + } + + // for syscalls: 2x "inotify_add_watch" ("filename" and ".") and 1x "inotify_rm_watch" + self.watch(&path, RecursiveMode::NonRecursive)?; + Ok(()) + } + + fn watch(&mut self, path: &Path, mode: RecursiveMode) -> UResult<()> { + self.watcher + .watch(path, mode) + .map_err(|err| USimpleError::new(1, err.to_string())) + } + + fn unwatch(&mut self, path: &Path) -> UResult<()> { + self.watcher + .unwatch(path) + .map_err(|err| USimpleError::new(1, err.to_string())) + } +} + +pub struct WatcherService { + /// Whether --retry was given on the command line + pub retry: bool, + + /// The [`FollowMode`] + pub follow: Option, + + /// Indicates whether to use the fallback `polling` method instead of the + /// platform specific event driven method. Since `use_polling` is subject to + /// change during runtime it is moved out of [`Settings`]. + pub use_polling: bool, + pub watcher_rx: Option, + pub orphans: Vec, + pub files: FileHandling, +} + +impl WatcherService { + pub fn new( + retry: bool, + follow: Option, + use_polling: bool, + files: FileHandling, + ) -> Self { + Self { + retry, + follow, + use_polling, + watcher_rx: None, + orphans: Vec::new(), + files, + } + } + + pub fn from(settings: &Settings) -> Self { + Self::new( + settings.retry, + settings.follow, + settings.use_polling, + FileHandling::from(settings), + ) + } + + pub fn add_path( + &mut self, + path: &Path, + display_name: &str, + reader: Option>, + update_last: bool, + ) -> UResult<()> { + if self.follow.is_some() { + let path = if path.is_relative() { + std::env::current_dir()?.join(path) + } else { + path.to_owned() + }; + let metadata = path.metadata().ok(); + self.files.insert( + &path, + PathData::new(reader, metadata, display_name), + update_last, + ); + } + + Ok(()) + } + + pub fn add_stdin( + &mut self, + display_name: &str, + reader: Option>, + update_last: bool, + ) -> UResult<()> { + if self.follow == Some(FollowMode::Descriptor) { + return self.add_path( + &PathBuf::from(text::DEV_STDIN), + display_name, + reader, + update_last, + ); + } + + Ok(()) + } + + pub fn add_bad_path( + &mut self, + path: &Path, + display_name: &str, + update_last: bool, + ) -> UResult<()> { + if self.retry && self.follow.is_some() { + return self.add_path(path, display_name, None, update_last); + } + + Ok(()) + } + + pub fn start(&mut self, settings: &Settings) -> UResult<()> { + if settings.follow.is_none() { + return Ok(()); + } + + let (tx, rx) = channel(); + + /* + Watcher is implemented per platform using the best implementation available on that + platform. In addition to such event driven implementations, a polling implementation + is also provided that should work on any platform. + Linux / Android: inotify + macOS: FSEvents / kqueue + Windows: ReadDirectoryChangesWatcher + FreeBSD / NetBSD / OpenBSD / DragonflyBSD: kqueue + Fallback: polling every n seconds + + NOTE: + We force the use of kqueue with: features=["macos_kqueue"]. + On macOS only `kqueue` is suitable for our use case because `FSEvents` + waits for file close util it delivers a modify event. See: + https://github.com/notify-rs/notify/issues/240 + */ + + let watcher: Box; + let watcher_config = notify::Config::default() + .with_poll_interval(settings.sleep_sec) + /* + NOTE: By enabling compare_contents, performance will be significantly impacted + as all files will need to be read and hashed at each `poll_interval`. + However, this is necessary to pass: "gnu/tests/tail-2/F-vs-rename.sh" + */ + .with_compare_contents(true); + if self.use_polling || RecommendedWatcher::kind() == WatcherKind::PollWatcher { + self.use_polling = true; // We have to use polling because there's no supported backend + watcher = Box::new(notify::PollWatcher::new(tx, watcher_config).unwrap()); + } else { + let tx_clone = tx.clone(); + match notify::RecommendedWatcher::new(tx, notify::Config::default()) { + Ok(w) => watcher = Box::new(w), + Err(e) if e.to_string().starts_with("Too many open files") => { + /* + NOTE: This ErrorKind is `Uncategorized`, but it is not recommended + to match an error against `Uncategorized` + NOTE: Could be tested with decreasing `max_user_instances`, e.g.: + `sudo sysctl fs.inotify.max_user_instances=64` + */ + show_error!( + "{} cannot be used, reverting to polling: Too many open files", + text::BACKEND + ); + set_exit_code(1); + self.use_polling = true; + watcher = Box::new(notify::PollWatcher::new(tx_clone, watcher_config).unwrap()); + } + Err(e) => return Err(USimpleError::new(1, e.to_string())), + }; + } + + self.watcher_rx = Some(WatcherRx::new(watcher, rx)); + self.init_files(&settings.inputs)?; + + Ok(()) + } + + pub fn follow_descriptor(&self) -> bool { + self.follow == Some(FollowMode::Descriptor) + } + + pub fn follow_name(&self) -> bool { + self.follow == Some(FollowMode::Name) + } + + pub fn follow_descriptor_retry(&self) -> bool { + self.follow_descriptor() && self.retry + } + + pub fn follow_name_retry(&self) -> bool { + self.follow_name() && self.retry + } + + fn init_files(&mut self, inputs: &VecDeque) -> UResult<()> { + if let Some(watcher_rx) = &mut self.watcher_rx { + for input in inputs { + match input.kind() { + InputKind::Stdin => continue, + InputKind::File(path) => { + #[cfg(all(unix, not(target_os = "linux")))] + if !path.is_file() { + continue; + } + let mut path = path.to_owned(); + if path.is_relative() { + path = std::env::current_dir()?.join(path); + } + + if path.is_tailable() { + // Add existing regular files to `Watcher` (InotifyWatcher). + watcher_rx.watch_with_parent(&path)?; + } else if !path.is_orphan() { + // If `path` is not a tailable file, add its parent to `Watcher`. + watcher_rx + .watch(path.parent().unwrap(), RecursiveMode::NonRecursive)?; + } else { + // If there is no parent, add `path` to `orphans`. + self.orphans.push(path); + } + } + } + } + } + Ok(()) + } + + fn handle_event( + &mut self, + event: ¬ify::Event, + settings: &Settings, + ) -> UResult> { + use notify::event::*; + + let event_path = event.paths.first().unwrap(); + let mut paths: Vec = vec![]; + let display_name = self.files.get(event_path).display_name.clone(); + + match event.kind { + EventKind::Modify(ModifyKind::Metadata(MetadataKind::Any | MetadataKind::WriteTime)) + + // | EventKind::Access(AccessKind::Close(AccessMode::Write)) + | EventKind::Create(CreateKind::File | CreateKind::Folder | CreateKind::Any) + | EventKind::Modify(ModifyKind::Data(DataChange::Any)) + | EventKind::Modify(ModifyKind::Name(RenameMode::To)) => { + if let Ok(new_md) = event_path.metadata() { + + let is_tailable = new_md.is_tailable(); + let pd = self.files.get(event_path); + if let Some(old_md) = &pd.metadata { + if is_tailable { + // We resume tracking from the start of the file, + // assuming it has been truncated to 0. This mimics GNU's `tail` + // behavior and is the usual truncation operation for log self.files. + if !old_md.is_tailable() { + show_error!( "{} has become accessible", display_name.quote()); + self.files.update_reader(event_path)?; + } else if pd.reader.is_none() { + show_error!( "{} has appeared; following new file", display_name.quote()); + self.files.update_reader(event_path)?; + } else if event.kind == EventKind::Modify(ModifyKind::Name(RenameMode::To)) + || (self.use_polling + && !old_md.file_id_eq(&new_md)) { + show_error!( "{} has been replaced; following new file", display_name.quote()); + self.files.update_reader(event_path)?; + } else if old_md.got_truncated(&new_md)? { + show_error!("{}: file truncated", display_name); + self.files.update_reader(event_path)?; + } + paths.push(event_path.to_owned()); + } else if !is_tailable && old_md.is_tailable() { + if pd.reader.is_some() { + self.files.reset_reader(event_path); + } else { + show_error!( + "{} has been replaced with an untailable file", + display_name.quote() + ); + } + } + } else if is_tailable { + show_error!( "{} has appeared; following new file", display_name.quote()); + self.files.update_reader(event_path)?; + paths.push(event_path.to_owned()); + } else if settings.retry { + if self.follow_descriptor() { + show_error!( + "{} has been replaced with an untailable file; giving up on this name", + display_name.quote() + ); + let _ = self.watcher_rx.as_mut().unwrap().watcher.unwatch(event_path); + self.files.remove(event_path); + if self.files.no_files_remaining(settings) { + return Err(USimpleError::new(1, text::NO_FILES_REMAINING)); + } + } else { + show_error!( + "{} has been replaced with an untailable file", + display_name.quote() + ); + } + } + self.files.update_metadata(event_path, Some(new_md)); + } + } + EventKind::Remove(RemoveKind::File | RemoveKind::Any) + + // | EventKind::Modify(ModifyKind::Name(RenameMode::Any)) + | EventKind::Modify(ModifyKind::Name(RenameMode::From)) => { + if self.follow_name() { + if settings.retry { + if let Some(old_md) = self.files.get_mut_metadata(event_path) { + if old_md.is_tailable() && self.files.get(event_path).reader.is_some() { + show_error!( + "{} {}: {}", + display_name.quote(), + text::BECOME_INACCESSIBLE, + text::NO_SUCH_FILE + ); + } + } + if event_path.is_orphan() && !self.orphans.contains(event_path) { + show_error!("directory containing watched file was removed"); + show_error!( + "{} cannot be used, reverting to polling", + text::BACKEND + ); + self.orphans.push(event_path.to_owned()); + let _ = self.watcher_rx.as_mut().unwrap().unwatch(event_path); + } + } else { + show_error!("{}: {}", display_name, text::NO_SUCH_FILE); + if !self.files.files_remaining() && self.use_polling { + // NOTE: GNU's tail exits here for `---disable-inotify` + return Err(USimpleError::new(1, text::NO_FILES_REMAINING)); + } + } + self.files.reset_reader(event_path); + } else if self.follow_descriptor_retry() { + // --retry only effective for the initial open + let _ = self.watcher_rx.as_mut().unwrap().unwatch(event_path); + self.files.remove(event_path); + } else if self.use_polling && event.kind == EventKind::Remove(RemoveKind::Any) { + /* + BUG: The watched file was removed. Since we're using Polling, this + could be a rename. We can't tell because `notify::PollWatcher` doesn't + recognize renames properly. + Ideally we want to call seek to offset 0 on the file handle. + But because we only have access to `PathData::reader` as `BufRead`, + we cannot seek to 0 with `BufReader::seek_relative`. + Also because we don't have the new name, we cannot work around this + by simply reopening the file. + */ + } + } + EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => { + /* + NOTE: For `tail -f a`, keep tracking additions to b after `mv a b` + (gnu/tests/tail-2/descriptor-vs-rename.sh) + NOTE: The File/BufReader doesn't need to be updated. + However, we need to update our `files.map`. + This can only be done for inotify, because this EventKind does not + trigger for the PollWatcher. + BUG: As a result, there's a bug if polling is used: + $ tail -f file_a ---disable-inotify + $ mv file_a file_b + $ echo A >> file_b + $ echo A >> file_a + The last append to file_a is printed, however this shouldn't be because + after the "mv" tail should only follow "file_b". + TODO: [2022-05; jhscheer] add test for this bug + */ + + if self.follow_descriptor() { + let new_path = event.paths.last().unwrap(); + paths.push(new_path.to_owned()); + + let new_data = PathData::from_other_with_path(self.files.remove(event_path), new_path); + self.files.insert( + new_path, + new_data, + self.files.get_last().unwrap() == event_path + ); + + // Unwatch old path and watch new path + let _ = self.watcher_rx.as_mut().unwrap().unwatch(event_path); + self.watcher_rx.as_mut().unwrap().watch_with_parent(new_path)?; + } + } + _ => {} + } + Ok(paths) + } +} + +pub fn follow(mut watcher_service: WatcherService, settings: &Settings) -> UResult<()> { + if watcher_service.files.no_files_remaining(settings) + && !watcher_service.files.only_stdin_remaining() + { + return Err(USimpleError::new(1, text::NO_FILES_REMAINING.to_string())); + } + + let mut process = platform::ProcessChecker::new(settings.pid); + + let mut _event_counter = 0; + let mut _timeout_counter = 0; + + // main follow loop + loop { + let mut _read_some = false; + + // If `--pid=p`, tail checks whether process p + // is alive at least every `--sleep-interval=N` seconds + if settings.follow.is_some() && settings.pid != 0 && process.is_dead() { + // p is dead, tail will also terminate + break; + } + + // For `-F` we need to poll if an orphan path becomes available during runtime. + // If a path becomes an orphan during runtime, it will be added to orphans. + // To be able to differentiate between the cases of test_retry8 and test_retry9, + // here paths will not be removed from orphans if the path becomes available. + if watcher_service.follow_name_retry() { + for new_path in &watcher_service.orphans { + if new_path.exists() { + let pd = watcher_service.files.get(new_path); + let md = new_path.metadata().unwrap(); + if md.is_tailable() && pd.reader.is_none() { + show_error!( + "{} has appeared; following new file", + pd.display_name.quote() + ); + watcher_service.files.update_metadata(new_path, Some(md)); + watcher_service.files.update_reader(new_path)?; + _read_some = watcher_service + .files + .tail_file(new_path, settings.verbose)?; + watcher_service + .watcher_rx + .as_mut() + .unwrap() + .watch_with_parent(new_path)?; + } + } + } + } + + // With -f, sleep for approximately N seconds (default 1.0) between iterations; + // We wake up if Notify sends an Event or if we wait more than `sleep_sec`. + let rx_result = watcher_service + .watcher_rx + .as_mut() + .unwrap() + .receiver + .recv_timeout(settings.sleep_sec); + if rx_result.is_ok() { + _event_counter += 1; + _timeout_counter = 0; + } + + let mut paths = vec![]; // Paths worth checking for new content to print + match rx_result { + Ok(Ok(event)) => { + if let Some(event_path) = event.paths.first() { + if watcher_service.files.contains_key(event_path) { + // Handle Event if it is about a path that we are monitoring + paths = watcher_service.handle_event(&event, settings)?; + } + } + } + Ok(Err(notify::Error { + kind: notify::ErrorKind::Io(ref e), + paths, + })) if e.kind() == std::io::ErrorKind::NotFound => { + if let Some(event_path) = paths.first() { + if watcher_service.files.contains_key(event_path) { + let _ = watcher_service + .watcher_rx + .as_mut() + .unwrap() + .watcher + .unwatch(event_path); + } + } + } + Ok(Err(notify::Error { + kind: notify::ErrorKind::MaxFilesWatch, + .. + })) => { + return Err(USimpleError::new( + 1, + format!("{} resources exhausted", text::BACKEND), + )) + } + Ok(Err(e)) => return Err(USimpleError::new(1, format!("NotifyError: {}", e))), + Err(mpsc::RecvTimeoutError::Timeout) => { + _timeout_counter += 1; + } + Err(e) => return Err(USimpleError::new(1, format!("RecvTimeoutError: {}", e))), + } + + if watcher_service.use_polling && settings.follow.is_some() { + // Consider all files to potentially have new content. + // This is a workaround because `Notify::PollWatcher` + // does not recognize the "renaming" of files. + paths = watcher_service.files.keys().cloned().collect::>(); + } + + // main print loop + for path in &paths { + _read_some = watcher_service.files.tail_file(path, settings.verbose)?; + } + + if _timeout_counter == settings.max_unchanged_stats { + /* + TODO: [2021-10; jhscheer] implement timeout_counter for each file. + ‘--max-unchanged-stats=n’ + When tailing a file by name, if there have been n (default n=5) consecutive iterations + for which the file has not changed, then open/fstat the file to determine if that file + name is still associated with the same device/inode-number pair as before. When + following a log file that is rotated, this is approximately the number of seconds + between when tail prints the last pre-rotation lines and when it prints the lines that + have accumulated in the new log file. This option is meaningful only when polling + (i.e., without inotify) and when following by name. + */ + } + } + Ok(()) +} diff --git a/src/uu/tail/src/paths.rs b/src/uu/tail/src/paths.rs new file mode 100644 index 00000000000..0ebb265b2a6 --- /dev/null +++ b/src/uu/tail/src/paths.rs @@ -0,0 +1,279 @@ +// * This file is part of the uutils coreutils package. +// * +// * For the full copyright and license information, please view the LICENSE +// * file that was distributed with this source code. + +// spell-checker:ignore tailable seekable stdlib (stdlib) + +#[cfg(unix)] +use std::os::unix::fs::{FileTypeExt, MetadataExt}; + +use std::collections::VecDeque; +use std::fs::{File, Metadata}; +use std::io::{Seek, SeekFrom}; +use std::path::{Path, PathBuf}; + +use uucore::error::UResult; + +use crate::args::Settings; +use crate::text; + +// * This file is part of the uutils coreutils package. +// * +// * For the full copyright and license information, please view the LICENSE +// * file that was distributed with this source code. + +#[derive(Debug, Clone)] +pub enum InputKind { + File(PathBuf), + Stdin, +} + +#[derive(Debug, Clone)] +pub struct Input { + kind: InputKind, + pub display_name: String, +} + +impl Input { + pub fn from(string: String) -> Self { + let kind = if string == text::DASH { + InputKind::Stdin + } else { + InputKind::File(PathBuf::from(&string)) + }; + + let display_name = match kind { + InputKind::File(_) => string, + InputKind::Stdin => { + if cfg!(unix) { + text::STDIN_HEADER.to_string() + } else { + string + } + } + }; + + Self { kind, display_name } + } + + pub fn kind(&self) -> &InputKind { + &self.kind + } + + pub fn is_stdin(&self) -> bool { + match self.kind { + InputKind::File(_) => false, + InputKind::Stdin => true, + } + } + + pub fn resolve(&self) -> Option { + match &self.kind { + InputKind::File(path) if path != &PathBuf::from(text::DEV_STDIN) => { + path.canonicalize().ok() + } + InputKind::File(_) | InputKind::Stdin => { + if cfg!(unix) { + PathBuf::from(text::DEV_STDIN).canonicalize().ok() + } else { + None + } + } + } + } + + pub fn is_tailable(&self) -> bool { + match &self.kind { + InputKind::File(path) => path_is_tailable(path), + InputKind::Stdin => self.resolve().map_or(false, |path| path_is_tailable(&path)), + } + } +} + +impl Default for Input { + fn default() -> Self { + Self { + kind: InputKind::Stdin, + display_name: String::from(text::STDIN_HEADER), + } + } +} + +#[derive(Debug, Default, Clone, Copy)] +pub struct HeaderPrinter { + verbose: bool, + first_header: bool, +} + +impl HeaderPrinter { + pub fn new(verbose: bool, first_header: bool) -> Self { + Self { + verbose, + first_header, + } + } + + pub fn print_input(&mut self, input: &Input) { + self.print(input.display_name.as_str()); + } + + pub fn print(&mut self, string: &str) { + if self.verbose { + println!( + "{}==> {} <==", + if self.first_header { "" } else { "\n" }, + string, + ); + self.first_header = false; + } + } +} + +#[derive(Debug, Clone)] +pub struct InputService { + pub inputs: VecDeque, + pub presume_input_pipe: bool, + pub header_printer: HeaderPrinter, +} + +impl InputService { + pub fn new(verbose: bool, presume_input_pipe: bool, inputs: VecDeque) -> Self { + Self { + inputs, + presume_input_pipe, + header_printer: HeaderPrinter::new(verbose, true), + } + } + + pub fn from(settings: &Settings) -> Self { + Self::new( + settings.verbose, + settings.presume_input_pipe, + settings.inputs.clone(), + ) + } + + pub fn has_stdin(&mut self) -> bool { + self.inputs.iter().any(|input| input.is_stdin()) + } + + pub fn has_only_stdin(&self) -> bool { + self.inputs.iter().all(|input| input.is_stdin()) + } + + pub fn print_header(&mut self, input: &Input) { + self.header_printer.print_input(input); + } +} + +pub trait FileExtTail { + #[allow(clippy::wrong_self_convention)] + fn is_seekable(&mut self, current_offset: u64) -> bool; +} + +impl FileExtTail for File { + /// Test if File is seekable. + /// Set the current position offset to `current_offset`. + fn is_seekable(&mut self, current_offset: u64) -> bool { + self.seek(SeekFrom::Current(0)).is_ok() + && self.seek(SeekFrom::End(0)).is_ok() + && self.seek(SeekFrom::Start(current_offset)).is_ok() + } +} + +pub trait MetadataExtTail { + fn is_tailable(&self) -> bool; + fn got_truncated(&self, other: &Metadata) -> UResult; + fn get_block_size(&self) -> u64; + fn file_id_eq(&self, other: &Metadata) -> bool; +} + +impl MetadataExtTail for Metadata { + fn is_tailable(&self) -> bool { + let ft = self.file_type(); + #[cfg(unix)] + { + ft.is_file() || ft.is_char_device() || ft.is_fifo() + } + #[cfg(not(unix))] + { + ft.is_file() + } + } + + /// Return true if the file was modified and is now shorter + fn got_truncated(&self, other: &Metadata) -> UResult { + Ok(other.len() < self.len() && other.modified()? != self.modified()?) + } + + fn get_block_size(&self) -> u64 { + #[cfg(unix)] + { + self.blocks() + } + #[cfg(not(unix))] + { + self.len() + } + } + + fn file_id_eq(&self, _other: &Metadata) -> bool { + #[cfg(unix)] + { + self.ino().eq(&_other.ino()) + } + #[cfg(windows)] + { + // use std::os::windows::prelude::*; + // if let Some(self_id) = self.file_index() { + // if let Some(other_id) = other.file_index() { + // + // return self_id.eq(&other_id); + // } + // } + false + } + } +} + +pub trait PathExtTail { + fn is_stdin(&self) -> bool; + fn is_orphan(&self) -> bool; + fn is_tailable(&self) -> bool; +} + +impl PathExtTail for Path { + fn is_stdin(&self) -> bool { + self.eq(Self::new(text::DASH)) + || self.eq(Self::new(text::DEV_STDIN)) + || self.eq(Self::new(text::STDIN_HEADER)) + } + + /// Return true if `path` does not have an existing parent directory + fn is_orphan(&self) -> bool { + !matches!(self.parent(), Some(parent) if parent.is_dir()) + } + + /// Return true if `path` is is a file type that can be tailed + fn is_tailable(&self) -> bool { + path_is_tailable(self) + } +} + +pub fn path_is_tailable(path: &Path) -> bool { + path.is_file() || path.exists() && path.metadata().map_or(false, |meta| meta.is_tailable()) +} + +#[inline] +pub fn stdin_is_bad_fd() -> bool { + // FIXME : Rust's stdlib is reopening fds as /dev/null + // see also: https://github.com/uutils/coreutils/issues/2873 + // (gnu/tests/tail-2/follow-stdin.sh fails because of this) + //#[cfg(unix)] + { + //platform::stdin_is_bad_fd() + } + //#[cfg(not(unix))] + false +} diff --git a/src/uu/tail/src/tail.rs b/src/uu/tail/src/tail.rs index 29176ae5d8e..2a76522d3e2 100644 --- a/src/uu/tail/src/tail.rs +++ b/src/uu/tail/src/tail.rs @@ -7,7 +7,7 @@ // * For the full copyright and license information, please view the LICENSE // * file that was distributed with this source code. -// spell-checker:ignore (ToDO) seekable seek'd tail'ing ringbuffer ringbuf unwatch Uncategorized filehandle +// spell-checker:ignore (ToDO) seekable seek'd tail'ing ringbuffer ringbuf unwatch Uncategorized filehandle Signum // spell-checker:ignore (libs) kqueue // spell-checker:ignore (acronyms) // spell-checker:ignore (env/flags) @@ -23,537 +23,66 @@ extern crate clap; extern crate uucore; extern crate core; +pub mod args; pub mod chunks; +mod follow; mod parse; +mod paths; mod platform; -use crate::files::FileHandling; -use chunks::ReverseChunks; +pub mod text; -use clap::{Arg, Command, ValueSource}; -use notify::{RecommendedWatcher, RecursiveMode, Watcher, WatcherKind}; +use same_file::Handle; use std::cmp::Ordering; -use std::collections::{HashMap, VecDeque}; -use std::ffi::OsString; -use std::fs::{File, Metadata}; +use std::fs::File; use std::io::{self, stdin, stdout, BufRead, BufReader, BufWriter, Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; -use std::sync::mpsc::{self, channel, Receiver}; -use std::time::Duration; -use uucore::display::Quotable; -use uucore::error::{ - get_exit_code, set_exit_code, FromIo, UError, UResult, USimpleError, UUsageError, -}; -use uucore::format_usage; -use uucore::parse_size::{parse_size, ParseSizeError}; - -#[cfg(unix)] -use std::os::unix::fs::MetadataExt; -#[cfg(unix)] -use std::os::unix::prelude::FileTypeExt; - -const ABOUT: &str = "\ - Print the last 10 lines of each FILE to standard output.\n\ - With more than one FILE, precede each with a header giving the file name.\n\ - With no FILE, or when FILE is -, read standard input.\n\ - \n\ - Mandatory arguments to long flags are mandatory for short flags too.\ - "; -const USAGE: &str = "{} [FLAG]... [FILE]..."; - -pub mod text { - pub static DASH: &str = "-"; - pub static DEV_STDIN: &str = "/dev/stdin"; - pub static STDIN_HEADER: &str = "standard input"; - pub static NO_FILES_REMAINING: &str = "no files remaining"; - pub static NO_SUCH_FILE: &str = "No such file or directory"; - pub static BECOME_INACCESSIBLE: &str = "has become inaccessible"; - pub static BAD_FD: &str = "Bad file descriptor"; - #[cfg(target_os = "linux")] - pub static BACKEND: &str = "inotify"; - #[cfg(all(unix, not(target_os = "linux")))] - pub static BACKEND: &str = "kqueue"; - #[cfg(target_os = "windows")] - pub static BACKEND: &str = "ReadDirectoryChanges"; -} - -pub mod options { - pub mod verbosity { - pub static QUIET: &str = "quiet"; - pub static VERBOSE: &str = "verbose"; - } - pub static BYTES: &str = "bytes"; - pub static FOLLOW: &str = "follow"; - pub static LINES: &str = "lines"; - pub static PID: &str = "pid"; - pub static SLEEP_INT: &str = "sleep-interval"; - pub static ZERO_TERM: &str = "zero-terminated"; - pub static DISABLE_INOTIFY_TERM: &str = "-disable-inotify"; // NOTE: three hyphens is correct - pub static USE_POLLING: &str = "use-polling"; - pub static RETRY: &str = "retry"; - pub static FOLLOW_RETRY: &str = "F"; - pub static MAX_UNCHANGED_STATS: &str = "max-unchanged-stats"; - pub static ARG_FILES: &str = "files"; - pub static PRESUME_INPUT_PIPE: &str = "-presume-input-pipe"; // NOTE: three hyphens is correct -} - -#[derive(Debug, PartialEq, Eq)] -enum FilterMode { - Bytes(u64), - Lines(u64, u8), // (number of lines, delimiter) -} - -impl Default for FilterMode { - fn default() -> Self { - Self::Lines(10, b'\n') - } -} - -#[derive(Debug, PartialEq, Eq)] -enum FollowMode { - Descriptor, - Name, -} - -#[derive(Debug, Default)] -pub struct Settings { - beginning: bool, - follow: Option, - max_unchanged_stats: u32, - mode: FilterMode, - paths: VecDeque, - pid: platform::Pid, - retry: bool, - sleep_sec: Duration, - use_polling: bool, - verbose: bool, - stdin_is_pipe_or_fifo: bool, - stdin_offset: u64, - stdin_redirect: PathBuf, -} - -impl Settings { - pub fn from(matches: &clap::ArgMatches) -> UResult { - let mut settings: Self = Self { - sleep_sec: Duration::from_secs_f32(1.0), - max_unchanged_stats: 5, - ..Default::default() - }; - - settings.follow = if matches.contains_id(options::FOLLOW_RETRY) { - Some(FollowMode::Name) - } else if matches.value_source(options::FOLLOW) != Some(ValueSource::CommandLine) { - None - } else if matches.value_of(options::FOLLOW) == Some("name") { - Some(FollowMode::Name) - } else { - Some(FollowMode::Descriptor) - }; - - if let Some(s) = matches.value_of(options::SLEEP_INT) { - settings.sleep_sec = match s.parse::() { - Ok(s) => Duration::from_secs_f32(s), - Err(_) => { - return Err(UUsageError::new( - 1, - format!("invalid number of seconds: {}", s.quote()), - )) - } - } - } - - settings.use_polling = matches.contains_id(options::USE_POLLING); - - if let Some(s) = matches.value_of(options::MAX_UNCHANGED_STATS) { - settings.max_unchanged_stats = match s.parse::() { - Ok(s) => s, - Err(_) => { - // TODO: [2021-10; jhscheer] add test for this - return Err(UUsageError::new( - 1, - format!( - "invalid maximum number of unchanged stats between opens: {}", - s.quote() - ), - )); - } - } - } - - if let Some(pid_str) = matches.value_of(options::PID) { - match pid_str.parse() { - Ok(pid) => { - // NOTE: on unix platform::Pid is i32, on windows platform::Pid is u32 - #[cfg(unix)] - if pid < 0 { - // NOTE: tail only accepts an unsigned pid - return Err(USimpleError::new( - 1, - format!("invalid PID: {}", pid_str.quote()), - )); - } - settings.pid = pid; - if settings.follow.is_none() { - show_warning!("PID ignored; --pid=PID is useful only when following"); - } - if !platform::supports_pid_checks(settings.pid) { - show_warning!("--pid=PID is not supported on this system"); - settings.pid = 0; - } - } - Err(e) => { - return Err(USimpleError::new( - 1, - format!("invalid PID: {}: {}", pid_str.quote(), e), - )); - } - } - } - - let mut starts_with_plus = false; // support for legacy format (+0) - let mode_and_beginning = if let Some(arg) = matches.value_of(options::BYTES) { - starts_with_plus = arg.starts_with('+'); - match parse_num(arg) { - Ok((n, beginning)) => (FilterMode::Bytes(n), beginning), - Err(e) => { - return Err(UUsageError::new( - 1, - format!("invalid number of bytes: {}", e), - )) - } - } - } else if let Some(arg) = matches.value_of(options::LINES) { - starts_with_plus = arg.starts_with('+'); - match parse_num(arg) { - Ok((n, beginning)) => (FilterMode::Lines(n, b'\n'), beginning), - Err(e) => { - return Err(UUsageError::new( - 1, - format!("invalid number of lines: {}", e), - )) - } - } - } else { - (FilterMode::default(), false) - }; - settings.mode = mode_and_beginning.0; - settings.beginning = mode_and_beginning.1; - - // Mimic GNU's tail for -[nc]0 without -f and exit immediately - if settings.follow.is_none() && !starts_with_plus && { - if let FilterMode::Lines(l, _) = settings.mode { - l == 0 - } else { - settings.mode == FilterMode::Bytes(0) - } - } { - std::process::exit(0) - } - - settings.retry = - matches.contains_id(options::RETRY) || matches.contains_id(options::FOLLOW_RETRY); - if settings.retry && settings.follow.is_none() { - show_warning!("--retry ignored; --retry is useful only when following"); - } - - if matches.contains_id(options::ZERO_TERM) { - if let FilterMode::Lines(count, _) = settings.mode { - settings.mode = FilterMode::Lines(count, 0); - } - } - - settings.stdin_is_pipe_or_fifo = matches.contains_id(options::PRESUME_INPUT_PIPE); - - settings.paths = matches - .get_many::(options::ARG_FILES) - .map(|v| v.map(PathBuf::from).collect()) - .unwrap_or_default(); - - settings.verbose = (matches.contains_id(options::verbosity::VERBOSE) - || settings.paths.len() > 1) - && !matches.contains_id(options::verbosity::QUIET); - - Ok(settings) - } - - fn follow_descriptor(&self) -> bool { - self.follow == Some(FollowMode::Descriptor) - } - - fn follow_name(&self) -> bool { - self.follow == Some(FollowMode::Name) - } - - fn follow_descriptor_retry(&self) -> bool { - self.follow_descriptor() && self.retry - } +use uucore::display::Quotable; +use uucore::error::{get_exit_code, set_exit_code, FromIo, UError, UResult, USimpleError}; - fn follow_name_retry(&self) -> bool { - self.follow_name() && self.retry - } -} +pub use args::uu_app; +use args::{parse_args, FilterMode, Settings, Signum}; +use chunks::ReverseChunks; +use follow::WatcherService; +use paths::{FileExtTail, Input, InputKind, InputService, MetadataExtTail}; #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let matches = uu_app().try_get_matches_from(arg_iterate(args)?)?; - let mut settings = Settings::from(&matches)?; - - // skip expensive call to fstat if PRESUME_INPUT_PIPE is selected - if !settings.stdin_is_pipe_or_fifo { - settings.stdin_is_pipe_or_fifo = stdin_is_pipe_or_fifo(); - } - - uu_tail(settings) + let settings = parse_args(args)?; + uu_tail(&settings) } -fn uu_tail(mut settings: Settings) -> UResult<()> { - let dash = PathBuf::from(text::DASH); - +fn uu_tail(settings: &Settings) -> UResult<()> { // Mimic GNU's tail for `tail -F` and exit immediately - if (settings.paths.is_empty() || settings.paths.contains(&dash)) && settings.follow_name() { + let mut input_service = InputService::from(settings); + let mut watcher_service = WatcherService::from(settings); + + if input_service.has_stdin() && watcher_service.follow_name() { return Err(USimpleError::new( 1, format!("cannot follow {} by name", text::DASH.quote()), )); } - settings.stdin_redirect = dash.handle_redirect(); - if cfg!(unix) && settings.stdin_is_pipe_or_fifo { - // Save the current seek position/offset of a stdin redirected file. - // This is needed to pass "gnu/tests/tail-2/start-middle.sh" - use same_file::Handle; - if let Ok(mut stdin_handle) = Handle::stdin() { - if let Ok(offset) = stdin_handle.as_file_mut().seek(SeekFrom::Current(0)) { - settings.stdin_offset = offset; - } - } - } - - // add '-' to paths - if !settings.paths.contains(&dash) && settings.stdin_is_pipe_or_fifo - || settings.paths.is_empty() && !settings.stdin_is_pipe_or_fifo - { - settings.paths.push_front(dash); - } - - // TODO: is there a better way to check for a readable stdin? - let mut buf = [0; 0]; // empty buffer to check if stdin().read().is_err() - let stdin_read_possible = settings.stdin_is_pipe_or_fifo && stdin().read(&mut buf).is_ok(); - - let mut first_header = true; - let mut files = FileHandling::with_capacity(settings.paths.len()); - let mut orphans = Vec::new(); - - let mut watcher_rx = if settings.follow.is_some() { - let (watcher, rx) = start_watcher_thread(&mut settings)?; - Some((watcher, rx)) - } else { - None - }; - - // Iterate user provided `paths` and add them to Watcher. - if let Some((ref mut watcher, _)) = watcher_rx { - for path in &settings.paths { - let mut path = if path.is_stdin() { - settings.stdin_redirect.to_owned() - } else { - path.to_owned() - }; - if path.is_stdin() { - continue; - } - #[cfg(all(unix, not(target_os = "linux")))] - if !path.is_file() { - continue; - } - if path.is_relative() { - path = std::env::current_dir()?.join(&path); - } - - if path.is_tailable() { - // Add existing regular files to `Watcher` (InotifyWatcher). - watcher.watch_with_parent(&path)?; - } else if !path.is_orphan() { - // If `path` is not a tailable file, add its parent to `Watcher`. - watcher - .watch(path.parent().unwrap(), RecursiveMode::NonRecursive) - .unwrap(); - } else { - // If there is no parent, add `path` to `orphans`. - orphans.push(path.to_owned()); - } - } - } - + watcher_service.start(settings)?; // Do an initial tail print of each path's content. // Add `path` and `reader` to `files` map if `--follow` is selected. - for path in &settings.paths { - let display_name = if cfg!(unix) && path.is_stdin() { - PathBuf::from(text::STDIN_HEADER) - } else { - path.to_owned() - }; - let path = if path.is_stdin() { - settings.stdin_redirect.to_owned() - } else { - path.to_owned() - }; - let path_is_tailable = path.is_tailable(); - - if !path.is_stdin() && !path_is_tailable { - if settings.follow_descriptor_retry() { - show_warning!("--retry only effective for the initial open"); - } - - if !path.exists() && !settings.stdin_is_pipe_or_fifo { - set_exit_code(1); - show_error!( - "cannot open {} for reading: {}", - display_name.quote(), - text::NO_SUCH_FILE - ); - } else if path.is_dir() || display_name.is_stdin() && !stdin_read_possible { - if settings.verbose { - files.print_header(&display_name, !first_header); - first_header = false; - } - let err_msg = "Is a directory".to_string(); - - // NOTE: On macOS path.is_dir() can be false for directories - // if it was a redirect, e.g. `$ tail < DIR` - // if !path.is_dir() { - // TODO: match against ErrorKind if unstable - // library feature "io_error_more" becomes stable - // if let Err(e) = stdin().read(&mut buf) { - // if e.kind() != std::io::ErrorKind::IsADirectory { - // err_msg = e.message.to_string(); - // } - // } - // } - - set_exit_code(1); - show_error!("error reading {}: {}", display_name.quote(), err_msg); - if settings.follow.is_some() { - let msg = if !settings.retry { - "; giving up on this name" - } else { - "" - }; - show_error!( - "{}: cannot follow end of this type of file{}", - display_name.display(), - msg - ); - } - if !(settings.follow_name_retry()) { - // skip directory if not retry - continue; - } - } else { - // TODO: [2021-10; jhscheer] how to handle block device or socket? - todo!(); - } - } - - let metadata = path.metadata().ok(); - - if display_name.is_stdin() && !path.is_file() { - if settings.verbose { - files.print_header(&display_name, !first_header); - first_header = false; + for input in &input_service.inputs.clone() { + match input.kind() { + InputKind::File(path) if cfg!(not(unix)) || path != &PathBuf::from(text::DEV_STDIN) => { + tail_file( + settings, + &mut input_service, + input, + path, + &mut watcher_service, + 0, + )?; } - - let mut reader = BufReader::new(stdin()); - if !stdin_is_bad_fd() { - unbounded_tail(&mut reader, &settings)?; - if settings.follow_descriptor() { - // Insert `stdin` into `files.map` - files.insert( - &path, - PathData { - reader: Some(Box::new(reader)), - metadata: None, - display_name, - }, - true, - ); - } - } else { - set_exit_code(1); - show_error!( - "cannot fstat {}: {}", - text::STDIN_HEADER.quote(), - text::BAD_FD - ); - if settings.follow.is_some() { - show_error!( - "error reading {}: {}", - text::STDIN_HEADER.quote(), - text::BAD_FD - ); - } - } - } else if path_is_tailable { - match File::open(&path) { - Ok(mut file) => { - if settings.verbose { - files.print_header(&display_name, !first_header); - first_header = false; - } - - let mut reader; - if file.is_seekable(if display_name.is_stdin() { - settings.stdin_offset - } else { - 0 - }) && metadata.as_ref().unwrap().get_block_size() > 0 - { - bounded_tail(&mut file, &settings); - reader = BufReader::new(file); - } else { - reader = BufReader::new(file); - unbounded_tail(&mut reader, &settings)?; - } - if settings.follow.is_some() { - // Insert existing/file `path` into `files.map` - files.insert( - &path, - PathData { - reader: Some(Box::new(reader)), - metadata, - display_name, - }, - true, - ); - } - } - Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => { - show!(e.map_err_context(|| { - format!("cannot open {} for reading", display_name.quote()) - })); - } - Err(e) => { - return Err(e.map_err_context(|| { - format!("cannot open {} for reading", display_name.quote()) - })); - } + // File points to /dev/stdin here + InputKind::File(_) | InputKind::Stdin => { + tail_stdin(settings, &mut input_service, input, &mut watcher_service)?; } - } else if settings.retry && settings.follow.is_some() { - let path = if path.is_relative() { - std::env::current_dir()?.join(&path) - } else { - path.to_owned() - }; - // Insert non-is_tailable() paths into `files.map` - files.insert( - &path, - PathData { - reader: None, - metadata, - display_name, - }, - false, - ); } } @@ -567,742 +96,163 @@ fn uu_tail(mut settings: Settings) -> UResult<()> { the input file is not a FIFO, pipe, or regular file, it is unspecified whether or not the -f option shall be ignored. */ - if files.no_files_remaining(&settings) { - if !files.only_stdin_remaining() { - show_error!("{}", text::NO_FILES_REMAINING); - } - } else if !(settings.stdin_is_pipe_or_fifo && settings.paths.len() == 1) { - follow(files, &settings, watcher_rx, orphans)?; + + if !input_service.has_only_stdin() { + follow::follow(watcher_service, settings)?; } } - if get_exit_code() > 0 && stdin_is_bad_fd() { + if get_exit_code() > 0 && paths::stdin_is_bad_fd() { show_error!("-: {}", text::BAD_FD); } Ok(()) } -fn arg_iterate<'a>( - mut args: impl uucore::Args + 'a, -) -> Result + 'a>, Box<(dyn UError)>> { - // argv[0] is always present - let first = args.next().unwrap(); - if let Some(second) = args.next() { - if let Some(s) = second.to_str() { - match parse::parse_obsolete(s) { - Some(Ok(iter)) => Ok(Box::new(vec![first].into_iter().chain(iter).chain(args))), - Some(Err(e)) => Err(UUsageError::new( - 1, - match e { - parse::ParseError::Syntax => format!("bad argument format: {}", s.quote()), - parse::ParseError::Overflow => format!( - "invalid argument: {} Value too large for defined datatype", - s.quote() - ), - }, - )), - None => Ok(Box::new(vec![first, second].into_iter().chain(args))), - } - } else { - Err(UUsageError::new(1, "bad argument encoding".to_owned())) - } - } else { - Ok(Box::new(vec![first].into_iter())) - } -} - -pub fn uu_app<'a>() -> Command<'a> { - #[cfg(target_os = "linux")] - pub static POLLING_HELP: &str = "Disable 'inotify' support and use polling instead"; - #[cfg(all(unix, not(target_os = "linux")))] - pub static POLLING_HELP: &str = "Disable 'kqueue' support and use polling instead"; - #[cfg(target_os = "windows")] - pub static POLLING_HELP: &str = - "Disable 'ReadDirectoryChanges' support and use polling instead"; - - Command::new(uucore::util_name()) - .version(crate_version!()) - .about(ABOUT) - .override_usage(format_usage(USAGE)) - .infer_long_args(true) - .arg( - Arg::new(options::BYTES) - .short('c') - .long(options::BYTES) - .takes_value(true) - .allow_hyphen_values(true) - .overrides_with_all(&[options::BYTES, options::LINES]) - .help("Number of bytes to print"), - ) - .arg( - Arg::new(options::FOLLOW) - .short('f') - .long(options::FOLLOW) - .default_value("descriptor") - .takes_value(true) - .min_values(0) - .max_values(1) - .require_equals(true) - .value_parser(["descriptor", "name"]) - .help("Print the file as it grows"), - ) - .arg( - Arg::new(options::LINES) - .short('n') - .long(options::LINES) - .takes_value(true) - .allow_hyphen_values(true) - .overrides_with_all(&[options::BYTES, options::LINES]) - .help("Number of lines to print"), - ) - .arg( - Arg::new(options::PID) - .long(options::PID) - .takes_value(true) - .help("With -f, terminate after process ID, PID dies"), - ) - .arg( - Arg::new(options::verbosity::QUIET) - .short('q') - .long(options::verbosity::QUIET) - .visible_alias("silent") - .overrides_with_all(&[options::verbosity::QUIET, options::verbosity::VERBOSE]) - .help("Never output headers giving file names"), - ) - .arg( - Arg::new(options::SLEEP_INT) - .short('s') - .takes_value(true) - .long(options::SLEEP_INT) - .help("Number of seconds to sleep between polling the file when running with -f"), - ) - .arg( - Arg::new(options::MAX_UNCHANGED_STATS) - .takes_value(true) - .long(options::MAX_UNCHANGED_STATS) - .help( - "Reopen a FILE which has not changed size after N (default 5) iterations \ - to see if it has been unlinked or renamed (this is the usual case of rotated \ - log files); This option is meaningful only when polling \ - (i.e., with --use-polling) and when --follow=name", - ), - ) - .arg( - Arg::new(options::verbosity::VERBOSE) - .short('v') - .long(options::verbosity::VERBOSE) - .overrides_with_all(&[options::verbosity::QUIET, options::verbosity::VERBOSE]) - .help("Always output headers giving file names"), - ) - .arg( - Arg::new(options::ZERO_TERM) - .short('z') - .long(options::ZERO_TERM) - .help("Line delimiter is NUL, not newline"), - ) - .arg( - Arg::new(options::USE_POLLING) - .alias(options::DISABLE_INOTIFY_TERM) // NOTE: Used by GNU's test suite - .alias("dis") // NOTE: Used by GNU's test suite - .long(options::USE_POLLING) - .help(POLLING_HELP), - ) - .arg( - Arg::new(options::RETRY) - .long(options::RETRY) - .help("Keep trying to open a file if it is inaccessible"), - ) - .arg( - Arg::new(options::FOLLOW_RETRY) - .short('F') - .help("Same as --follow=name --retry") - .overrides_with_all(&[options::RETRY, options::FOLLOW]), - ) - .arg( - Arg::new(options::PRESUME_INPUT_PIPE) - .long(options::PRESUME_INPUT_PIPE) - .alias(options::PRESUME_INPUT_PIPE) - .hide(true), - ) - .arg( - Arg::new(options::ARG_FILES) - .multiple_occurrences(true) - .takes_value(true) - .min_values(1) - .value_hint(clap::ValueHint::FilePath), - ) -} - -type WatcherRx = ( - Box<(dyn Watcher)>, - Receiver>, -); - -fn follow( - mut files: FileHandling, +fn tail_file( settings: &Settings, - watcher_rx: Option, - mut orphans: Vec, + input_service: &mut InputService, + input: &Input, + path: &Path, + watcher_service: &mut WatcherService, + offset: u64, ) -> UResult<()> { - let (mut watcher, rx) = watcher_rx.unwrap(); - let mut process = platform::ProcessChecker::new(settings.pid); - - // TODO: [2021-10; jhscheer] - let mut _event_counter = 0; - let mut _timeout_counter = 0; - - // main follow loop - loop { - let mut _read_some = false; - - // If `--pid=p`, tail checks whether process p - // is alive at least every `--sleep-interval=N` seconds - if settings.follow.is_some() && settings.pid != 0 && process.is_dead() { - // p is dead, tail will also terminate - break; - } - - // For `-F` we need to poll if an orphan path becomes available during runtime. - // If a path becomes an orphan during runtime, it will be added to orphans. - // To be able to differentiate between the cases of test_retry8 and test_retry9, - // here paths will not be removed from orphans if the path becomes available. - if settings.follow_name_retry() { - for new_path in &orphans { - if new_path.exists() { - let pd = files.get(new_path); - let md = new_path.metadata().unwrap(); - if md.is_tailable() && pd.reader.is_none() { - show_error!( - "{} has appeared; following new file", - pd.display_name.quote() - ); - files.update_metadata(new_path, Some(md)); - files.update_reader(new_path)?; - _read_some = files.tail_file(new_path, settings.verbose)?; - watcher.watch_with_parent(new_path)?; - } - } - } + if watcher_service.follow_descriptor_retry() { + show_warning!("--retry only effective for the initial open"); + } + + if !path.exists() { + set_exit_code(1); + show_error!( + "cannot open '{}' for reading: {}", + input.display_name, + text::NO_SUCH_FILE + ); + watcher_service.add_bad_path(path, input.display_name.as_str(), false)?; + } else if path.is_dir() { + set_exit_code(1); + + input_service.print_header(input); + let err_msg = "Is a directory".to_string(); + + show_error!("error reading '{}': {}", input.display_name, err_msg); + if settings.follow.is_some() { + let msg = if !settings.retry { + "; giving up on this name" + } else { + "" + }; + show_error!( + "{}: cannot follow end of this type of file{}", + input.display_name, + msg + ); } - - // With -f, sleep for approximately N seconds (default 1.0) between iterations; - // We wake up if Notify sends an Event or if we wait more than `sleep_sec`. - let rx_result = rx.recv_timeout(settings.sleep_sec); - if rx_result.is_ok() { - _event_counter += 1; - _timeout_counter = 0; + if !(watcher_service.follow_name_retry()) { + // skip directory if not retry + return Ok(()); } - - let mut paths = vec![]; // Paths worth checking for new content to print - match rx_result { - Ok(Ok(event)) => { - if let Some(event_path) = event.paths.first() { - if files.contains_key(event_path) { - // Handle Event if it is about a path that we are monitoring - paths = - handle_event(&event, &mut files, settings, &mut watcher, &mut orphans)?; - } - } - } - Ok(Err(notify::Error { - kind: notify::ErrorKind::Io(ref e), - paths, - })) if e.kind() == std::io::ErrorKind::NotFound => { - if let Some(event_path) = paths.first() { - if files.contains_key(event_path) { - let _ = watcher.unwatch(event_path); - } + watcher_service.add_bad_path(path, input.display_name.as_str(), false)?; + } else if input.is_tailable() { + let metadata = path.metadata().ok(); + match File::open(&path) { + Ok(mut file) => { + input_service.print_header(input); + let mut reader; + if file.is_seekable(if input.is_stdin() { offset } else { 0 }) + && metadata.as_ref().unwrap().get_block_size() > 0 + { + bounded_tail(&mut file, settings); + reader = BufReader::new(file); + } else { + reader = BufReader::new(file); + unbounded_tail(&mut reader, settings)?; } + watcher_service.add_path( + path, + input.display_name.as_str(), + Some(Box::new(reader)), + true, + )?; + } + Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => { + watcher_service.add_bad_path(path, input.display_name.as_str(), false)?; + show!(e.map_err_context(|| { + format!("cannot open '{}' for reading", input.display_name) + })); } - Ok(Err(notify::Error { - kind: notify::ErrorKind::MaxFilesWatch, - .. - })) => { - return Err(USimpleError::new( - 1, - format!("{} resources exhausted", text::BACKEND), - )) - } - Ok(Err(e)) => return Err(USimpleError::new(1, format!("NotifyError: {}", e))), - Err(mpsc::RecvTimeoutError::Timeout) => { - _timeout_counter += 1; + Err(e) => { + watcher_service.add_bad_path(path, input.display_name.as_str(), false)?; + return Err(e.map_err_context(|| { + format!("cannot open '{}' for reading", input.display_name) + })); } - Err(e) => return Err(USimpleError::new(1, format!("RecvTimeoutError: {}", e))), - } - - if settings.use_polling && settings.follow.is_some() { - // Consider all files to potentially have new content. - // This is a workaround because `Notify::PollWatcher` - // does not recognize the "renaming" of files. - paths = files.keys().cloned().collect::>(); } - - // main print loop - for path in &paths { - _read_some = files.tail_file(path, settings.verbose)?; - } - - if _timeout_counter == settings.max_unchanged_stats { - /* - TODO: [2021-10; jhscheer] implement timeout_counter for each file. - ‘--max-unchanged-stats=n’ - When tailing a file by name, if there have been n (default n=5) consecutive iterations - for which the file has not changed, then open/fstat the file to determine if that file - name is still associated with the same device/inode-number pair as before. When - following a log file that is rotated, this is approximately the number of seconds - between when tail prints the last pre-rotation lines and when it prints the lines that - have accumulated in the new log file. This option is meaningful only when polling - (i.e., without inotify) and when following by name. - */ - } - } - Ok(()) -} - -fn start_watcher_thread(settings: &mut Settings) -> Result> { - let (tx, rx) = channel(); - - /* - Watcher is implemented per platform using the best implementation available on that - platform. In addition to such event driven implementations, a polling implementation - is also provided that should work on any platform. - Linux / Android: inotify - macOS: FSEvents / kqueue - Windows: ReadDirectoryChangesWatcher - FreeBSD / NetBSD / OpenBSD / DragonflyBSD: kqueue - Fallback: polling every n seconds - - NOTE: - We force the use of kqueue with: features=["macos_kqueue"]. - On macOS only `kqueue` is suitable for our use case because `FSEvents` - waits for file close util it delivers a modify event. See: - https://github.com/notify-rs/notify/issues/240 - */ - - let watcher: Box; - let watcher_config = notify::Config::default() - .with_poll_interval(settings.sleep_sec) - /* - NOTE: By enabling compare_contents, performance will be significantly impacted - as all files will need to be read and hashed at each `poll_interval`. - However, this is necessary to pass: "gnu/tests/tail-2/F-vs-rename.sh" - */ - .with_compare_contents(true); - if settings.use_polling || RecommendedWatcher::kind() == WatcherKind::PollWatcher { - settings.use_polling = true; // We have to use polling because there's no supported backend - watcher = Box::new(notify::PollWatcher::new(tx, watcher_config).unwrap()); } else { - let tx_clone = tx.clone(); - match notify::RecommendedWatcher::new(tx, notify::Config::default()) { - Ok(w) => watcher = Box::new(w), - Err(e) if e.to_string().starts_with("Too many open files") => { - /* - NOTE: This ErrorKind is `Uncategorized`, but it is not recommended - to match an error against `Uncategorized` - NOTE: Could be tested with decreasing `max_user_instances`, e.g.: - `sudo sysctl fs.inotify.max_user_instances=64` - */ - show_error!( - "{} cannot be used, reverting to polling: Too many open files", - text::BACKEND - ); - set_exit_code(1); - settings.use_polling = true; - watcher = Box::new(notify::PollWatcher::new(tx_clone, watcher_config).unwrap()); - } - Err(e) => return Err(USimpleError::new(1, e.to_string())), - }; + watcher_service.add_bad_path(path, input.display_name.as_str(), false)?; } - Ok((watcher, rx)) + + Ok(()) } -fn handle_event( - event: ¬ify::Event, - files: &mut FileHandling, +fn tail_stdin( settings: &Settings, - watcher: &mut Box, - orphans: &mut Vec, -) -> UResult> { - use notify::event::*; - - let event_path = event.paths.first().unwrap(); - let display_name = files.get_display_name(event_path); - let mut paths: Vec = vec![]; - - match event.kind { - EventKind::Modify(ModifyKind::Metadata(MetadataKind::Any | MetadataKind::WriteTime)) - // | EventKind::Access(AccessKind::Close(AccessMode::Write)) - | EventKind::Create(CreateKind::File | CreateKind::Folder | CreateKind::Any) - | EventKind::Modify(ModifyKind::Data(DataChange::Any)) - | EventKind::Modify(ModifyKind::Name(RenameMode::To)) => { - if let Ok(new_md) = event_path.metadata() { - let is_tailable = new_md.is_tailable(); - let pd = files.get(event_path); - if let Some(old_md) = &pd.metadata { - if is_tailable { - // We resume tracking from the start of the file, - // assuming it has been truncated to 0. This mimics GNU's `tail` - // behavior and is the usual truncation operation for log files. - if !old_md.is_tailable() { - show_error!( "{} has become accessible", display_name.quote()); - files.update_reader(event_path)?; - } else if pd.reader.is_none() { - show_error!( "{} has appeared; following new file", display_name.quote()); - files.update_reader(event_path)?; - } else if event.kind == EventKind::Modify(ModifyKind::Name(RenameMode::To)) - || (settings.use_polling - && !old_md.file_id_eq(&new_md)) { - show_error!( "{} has been replaced; following new file", display_name.quote()); - files.update_reader(event_path)?; - } else if old_md.got_truncated(&new_md)? { - show_error!("{}: file truncated", display_name.display()); - files.update_reader(event_path)?; - } - paths.push(event_path.to_owned()); - } else if !is_tailable && old_md.is_tailable() { - if pd.reader.is_some() { - files.reset_reader(event_path); - } else { - show_error!( - "{} has been replaced with an untailable file", - display_name.quote() - ); - } - } - } else if is_tailable { - show_error!( "{} has appeared; following new file", display_name.quote()); - files.update_reader(event_path)?; - paths.push(event_path.to_owned()); - } else if settings.retry { - if settings.follow_descriptor() { - show_error!( - "{} has been replaced with an untailable file; giving up on this name", - display_name.quote() - ); - let _ = watcher.unwatch(event_path); - files.remove(event_path); - if files.no_files_remaining(settings) { - return Err(USimpleError::new(1, text::NO_FILES_REMAINING)); - } - } else { - show_error!( - "{} has been replaced with an untailable file", - display_name.quote() - ); - } - } - files.update_metadata(event_path, Some(new_md)); - } - } - EventKind::Remove(RemoveKind::File | RemoveKind::Any) - // | EventKind::Modify(ModifyKind::Name(RenameMode::Any)) - | EventKind::Modify(ModifyKind::Name(RenameMode::From)) => { - if settings.follow_name() { - if settings.retry { - if let Some(old_md) = files.get_mut_metadata(event_path) { - if old_md.is_tailable() && files.get(event_path).reader.is_some() { - show_error!( - "{} {}: {}", - display_name.quote(), - text::BECOME_INACCESSIBLE, - text::NO_SUCH_FILE - ); - } - } - if event_path.is_orphan() && !orphans.contains(event_path) { - show_error!("directory containing watched file was removed"); - show_error!( - "{} cannot be used, reverting to polling", - text::BACKEND - ); - orphans.push(event_path.to_owned()); - let _ = watcher.unwatch(event_path); - } - } else { - show_error!("{}: {}", display_name.display(), text::NO_SUCH_FILE); - if !files.files_remaining() && settings.use_polling { - // NOTE: GNU's tail exits here for `---disable-inotify` - return Err(USimpleError::new(1, text::NO_FILES_REMAINING)); - } + input_service: &mut InputService, + input: &Input, + watcher_service: &mut WatcherService, +) -> UResult<()> { + match input.resolve() { + // fifo + Some(path) => { + let mut stdin_offset = 0; + if cfg!(unix) { + // Save the current seek position/offset of a stdin redirected file. + // This is needed to pass "gnu/tests/tail-2/start-middle.sh" + if let Ok(mut stdin_handle) = Handle::stdin() { + if let Ok(offset) = stdin_handle.as_file_mut().seek(SeekFrom::Current(0)) { + stdin_offset = offset; } - files.reset_reader(event_path); - } else if settings.follow_descriptor_retry() { - // --retry only effective for the initial open - let _ = watcher.unwatch(event_path); - files.remove(event_path); - } else if settings.use_polling && event.kind == EventKind::Remove(RemoveKind::Any) { - /* - BUG: The watched file was removed. Since we're using Polling, this - could be a rename. We can't tell because `notify::PollWatcher` doesn't - recognize renames properly. - Ideally we want to call seek to offset 0 on the file handle. - But because we only have access to `PathData::reader` as `BufRead`, - we cannot seek to 0 with `BufReader::seek_relative`. - Also because we don't have the new name, we cannot work around this - by simply reopening the file. - */ - } - } - EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => { - /* - NOTE: For `tail -f a`, keep tracking additions to b after `mv a b` - (gnu/tests/tail-2/descriptor-vs-rename.sh) - NOTE: The File/BufReader doesn't need to be updated. - However, we need to update our `files.map`. - This can only be done for inotify, because this EventKind does not - trigger for the PollWatcher. - BUG: As a result, there's a bug if polling is used: - $ tail -f file_a ---disable-inotify - $ mv file_a file_b - $ echo A >> file_b - $ echo A >> file_a - The last append to file_a is printed, however this shouldn't be because - after the "mv" tail should only follow "file_b". - TODO: [2022-05; jhscheer] add test for this bug - */ - - if settings.follow_descriptor() { - let new_path = event.paths.last().unwrap(); - paths.push(new_path.to_owned()); - // Remove old reader - let old_reader = files.remove(event_path).reader; - let reader = if old_reader.is_some() { - // Use old reader with the same file descriptor if there is one - old_reader - } else if let Ok(file) = File::open(&new_path) { - // Open new file tail from start - Some(Box::new(BufReader::new(file)) as Box) - } else { - // Probably file was renamed/moved or removed again - None - }; - // Add new reader but keep old display name - files.insert( - new_path, - PathData { - metadata: new_path.metadata().ok(), - reader, - display_name, // mimic GNU's tail and show old name in header - }, - files.get_last().unwrap() == event_path - ); - // Unwatch old path and watch new path - let _ = watcher.unwatch(event_path); - watcher.watch_with_parent(new_path)?; - } - } - _ => {} - } - Ok(paths) -} - -/// Data structure to keep a handle on the BufReader, Metadata -/// and the display_name (header_name) of files that are being followed. -pub struct PathData { - reader: Option>, - metadata: Option, - display_name: PathBuf, // the path as provided by user input, used for headers -} - -mod files { - use super::*; - use std::collections::hash_map::Keys; - - /// Data structure to keep a handle on files to follow. - /// `last` always holds the path/key of the last file that was printed from. - /// The keys of the HashMap can point to an existing file path (normal case), - /// or stdin ("-"), or to a non existing path (--retry). - /// For existing files, all keys in the HashMap are absolute Paths. - pub struct FileHandling { - map: HashMap, - last: Option, - } - - impl FileHandling { - /// Creates an empty `FileHandling` with the specified capacity - pub fn with_capacity(n: usize) -> Self { - Self { - map: HashMap::with_capacity(n), - last: None, - } - } - - /// Wrapper for HashMap::insert using Path::canonicalize - pub fn insert(&mut self, k: &Path, v: PathData, update_last: bool) { - let k = Self::canonicalize_path(k); - if update_last { - self.last = Some(k.to_owned()); - } - let _ = self.map.insert(k, v); - } - - /// Wrapper for HashMap::remove using Path::canonicalize - pub fn remove(&mut self, k: &Path) -> PathData { - self.map.remove(&Self::canonicalize_path(k)).unwrap() - } - - /// Wrapper for HashMap::get using Path::canonicalize - pub fn get(&self, k: &Path) -> &PathData { - self.map.get(&Self::canonicalize_path(k)).unwrap() - } - - /// Wrapper for HashMap::get_mut using Path::canonicalize - pub fn get_mut(&mut self, k: &Path) -> &mut PathData { - self.map.get_mut(&Self::canonicalize_path(k)).unwrap() - } - - /// Canonicalize `path` if it is not already an absolute path - fn canonicalize_path(path: &Path) -> PathBuf { - if path.is_relative() && !path.is_stdin() { - if let Ok(p) = path.canonicalize() { - return p; } } - path.to_owned() - } - - pub fn get_display_name(&self, path: &Path) -> PathBuf { - self.get(path).display_name.to_owned() - } - - pub fn get_mut_metadata(&mut self, path: &Path) -> Option<&Metadata> { - self.get_mut(path).metadata.as_ref() - } - - pub fn keys(&self) -> Keys { - self.map.keys() - } - - pub fn contains_key(&self, k: &Path) -> bool { - self.map.contains_key(k) - } - - pub fn get_last(&self) -> Option<&PathBuf> { - self.last.as_ref() - } - - /// Return true if there is only stdin remaining - pub fn only_stdin_remaining(&self) -> bool { - self.map.len() == 1 && (self.map.contains_key(Path::new(text::DASH))) - } - - /// Return true if there is at least one "tailable" path (or stdin) remaining - pub fn files_remaining(&self) -> bool { - for path in self.map.keys() { - if path.is_tailable() || path.is_stdin() { - return true; - } - } - false - } - - /// Returns true if there are no files remaining - pub fn no_files_remaining(&self, settings: &Settings) -> bool { - self.map.is_empty() || !self.files_remaining() && !settings.retry - } - - /// Set `reader` to None to indicate that `path` is not an existing file anymore. - pub fn reset_reader(&mut self, path: &Path) { - self.get_mut(path).reader = None; - } - - /// Reopen the file at the monitored `path` - pub fn update_reader(&mut self, path: &Path) -> UResult<()> { - /* - BUG: If it's not necessary to reopen a file, GNU's tail calls seek to offset 0. - However we can't call seek here because `BufRead` does not implement `Seek`. - As a workaround we always reopen the file even though this might not always - be necessary. - */ - self.get_mut(path) - .reader - .replace(Box::new(BufReader::new(File::open(&path)?))); - Ok(()) - } - - /// Reload metadata from `path`, or `metadata` - pub fn update_metadata(&mut self, path: &Path, metadata: Option) { - self.get_mut(path).metadata = if metadata.is_some() { - metadata + tail_file( + settings, + input_service, + input, + &path, + watcher_service, + stdin_offset, + )?; + } + // pipe + None => { + input_service.print_header(input); + if !paths::stdin_is_bad_fd() { + let mut reader = BufReader::new(stdin()); + unbounded_tail(&mut reader, settings)?; + watcher_service.add_stdin( + input.display_name.as_str(), + Some(Box::new(reader)), + true, + )?; } else { - path.metadata().ok() - }; - } - - /// Read `path` from the current seek position forward - pub fn read_file(&mut self, path: &Path, buffer: &mut Vec) -> UResult { - let mut read_some = false; - let pd = self.get_mut(path).reader.as_mut(); - if let Some(reader) = pd { - loop { - match reader.read_until(b'\n', buffer) { - Ok(0) => break, - Ok(_) => { - read_some = true; - } - Err(err) => return Err(USimpleError::new(1, err.to_string())), - } - } - } - Ok(read_some) - } - - /// Print `buffer` to stdout - pub fn print_file(&self, buffer: &[u8]) -> UResult<()> { - let mut stdout = stdout(); - stdout - .write_all(buffer) - .map_err_context(|| String::from("write error"))?; - Ok(()) - } - - /// Read new data from `path` and print it to stdout - pub fn tail_file(&mut self, path: &Path, verbose: bool) -> UResult { - let mut buffer = vec![]; - let read_some = self.read_file(path, &mut buffer)?; - if read_some { - if self.needs_header(path, verbose) { - self.print_header(path, true); - } - self.print_file(&buffer)?; - - self.last.replace(path.to_owned()); - self.update_metadata(path, None); - } - Ok(read_some) - } - - /// Decide if printing `path` needs a header based on when it was last printed - pub fn needs_header(&self, path: &Path, verbose: bool) -> bool { - if verbose { - if let Some(ref last) = self.last { - return !last.eq(&path); - } else { - return true; + set_exit_code(1); + show_error!( + "cannot fstat {}: {}", + text::STDIN_HEADER.quote(), + text::BAD_FD + ); + if settings.follow.is_some() { + show_error!( + "error reading {}: {}", + text::STDIN_HEADER.quote(), + text::BAD_FD + ); } } - false - } - - /// Print header for `path` to stdout - pub fn print_header(&self, path: &Path, needs_newline: bool) { - println!( - "{}==> {} <==", - if needs_newline { "\n" } else { "" }, - self.display_name(path) - ); } + }; - /// Wrapper for `PathData::display_name` - pub fn display_name(&self, path: &Path) -> String { - if let Some(path) = self.map.get(&Self::canonicalize_path(path)) { - path.display_name.display().to_string() - } else { - path.display().to_string() - } - } - } + Ok(()) } /// Find the index after the given number of instances of a given byte. @@ -1430,25 +380,33 @@ fn backwards_thru_file(file: &mut File, num_delimiters: u64, delimiter: u8) { /// being a nice performance win for very large files. fn bounded_tail(file: &mut File, settings: &Settings) { // Find the position in the file to start printing from. - match (&settings.mode, settings.beginning) { - (FilterMode::Lines(count, delimiter), false) => { + // dbg!("bounded"); + // dbg!(&settings.mode); + match &settings.mode { + FilterMode::Lines(Signum::Negative(count), delimiter) => { backwards_thru_file(file, *count, *delimiter); } - (FilterMode::Lines(count, delimiter), true) => { - let i = forwards_thru_file(file, (*count).max(1) - 1, *delimiter).unwrap(); + FilterMode::Lines(Signum::Positive(count), delimiter) if count > &1 => { + let i = forwards_thru_file(file, *count - 1, *delimiter).unwrap(); file.seek(SeekFrom::Start(i as u64)).unwrap(); } - (FilterMode::Bytes(count), false) => { + FilterMode::Lines(Signum::MinusZero, _) => { + return; + } + FilterMode::Bytes(Signum::Negative(count)) => { let len = file.seek(SeekFrom::End(0)).unwrap(); file.seek(SeekFrom::End(-((*count).min(len) as i64))) .unwrap(); } - (FilterMode::Bytes(count), true) => { + FilterMode::Bytes(Signum::Positive(count)) if count > &1 => { // GNU `tail` seems to index bytes and lines starting at 1, not // at 0. It seems to treat `+0` and `+1` as the same thing. - file.seek(SeekFrom::Start(((*count).max(1) - 1) as u64)) - .unwrap(); + file.seek(SeekFrom::Start(*count - 1)).unwrap(); + } + FilterMode::Bytes(Signum::MinusZero) => { + return; } + _ => {} } // Print the target section of the file. @@ -1460,14 +418,19 @@ fn bounded_tail(file: &mut File, settings: &Settings) { fn unbounded_tail(reader: &mut BufReader, settings: &Settings) -> UResult<()> { let stdout = stdout(); let mut writer = BufWriter::new(stdout.lock()); - match (&settings.mode, settings.beginning) { - (FilterMode::Lines(count, sep), false) => { + // dbg!("unbounded"); + // dbg!(&settings.mode); + match &settings.mode { + FilterMode::Lines(Signum::Negative(count), sep) => { let mut chunks = chunks::LinesChunkBuffer::new(*sep, *count); chunks.fill(reader)?; chunks.print(writer)?; } - (FilterMode::Lines(count, sep), true) => { - let mut num_skip = (*count).max(1) - 1; + FilterMode::Lines(Signum::PlusZero | Signum::Positive(1), _) => { + io::copy(reader, &mut writer)?; + } + FilterMode::Lines(Signum::Positive(count), sep) => { + let mut num_skip = *count - 1; let mut chunk = chunks::LinesChunk::new(*sep); while chunk.fill(reader)?.is_some() { let lines = chunk.get_lines() as u64; @@ -1482,13 +445,16 @@ fn unbounded_tail(reader: &mut BufReader, settings: &Settings) -> UR io::copy(reader, &mut writer)?; } } - (FilterMode::Bytes(count), false) => { + FilterMode::Bytes(Signum::Negative(count)) => { let mut chunks = chunks::BytesChunkBuffer::new(*count); chunks.fill(reader)?; chunks.print(writer)?; } - (FilterMode::Bytes(count), true) => { - let mut num_skip = (*count).max(1) - 1; + FilterMode::Bytes(Signum::PlusZero | Signum::Positive(1)) => { + io::copy(reader, &mut writer)?; + } + FilterMode::Bytes(Signum::Positive(count)) => { + let mut num_skip = *count - 1; let mut chunk = chunks::BytesChunk::new(); loop { if let Some(bytes) = chunk.fill(reader)? { @@ -1510,204 +476,11 @@ fn unbounded_tail(reader: &mut BufReader, settings: &Settings) -> UR io::copy(reader, &mut writer)?; } + _ => {} } Ok(()) } -fn parse_num(src: &str) -> Result<(u64, bool), ParseSizeError> { - let mut size_string = src.trim(); - let mut starting_with = false; - - if let Some(c) = size_string.chars().next() { - if c == '+' || c == '-' { - // tail: '-' is not documented (8.32 man pages) - size_string = &size_string[1..]; - if c == '+' { - starting_with = true; - } - } - } else { - return Err(ParseSizeError::ParseFailure(src.to_string())); - } - - parse_size(size_string).map(|n| (n, starting_with)) -} - -pub fn stdin_is_pipe_or_fifo() -> bool { - #[cfg(unix)] - { - platform::stdin_is_pipe_or_fifo() - } - #[cfg(windows)] - { - winapi_util::file::typ(winapi_util::HandleRef::stdin()) - .map(|t| t.is_disk() || t.is_pipe()) - .unwrap_or(false) - } -} -#[inline] -pub fn stdin_is_bad_fd() -> bool { - // FIXME : Rust's stdlib is reopening fds as /dev/null - // see also: https://github.com/uutils/coreutils/issues/2873 - // (gnu/tests/tail-2/follow-stdin.sh fails because of this) - //#[cfg(unix)] - { - //platform::stdin_is_bad_fd() - } - //#[cfg(not(unix))] - false -} - -trait FileExtTail { - #[allow(clippy::wrong_self_convention)] - fn is_seekable(&mut self, current_offset: u64) -> bool; -} - -impl FileExtTail for File { - /// Test if File is seekable. - /// Set the current position offset to `current_offset`. - fn is_seekable(&mut self, current_offset: u64) -> bool { - self.seek(SeekFrom::Current(0)).is_ok() - && self.seek(SeekFrom::End(0)).is_ok() - && self.seek(SeekFrom::Start(current_offset)).is_ok() - } -} - -trait MetadataExtTail { - fn is_tailable(&self) -> bool; - fn got_truncated(&self, other: &Metadata) -> Result>; - fn get_block_size(&self) -> u64; - fn file_id_eq(&self, other: &Metadata) -> bool; -} - -impl MetadataExtTail for Metadata { - fn is_tailable(&self) -> bool { - let ft = self.file_type(); - #[cfg(unix)] - { - ft.is_file() || ft.is_char_device() || ft.is_fifo() - } - #[cfg(not(unix))] - { - ft.is_file() - } - } - - /// Return true if the file was modified and is now shorter - fn got_truncated(&self, other: &Metadata) -> Result> { - Ok(other.len() < self.len() && other.modified()? != self.modified()?) - } - - fn get_block_size(&self) -> u64 { - #[cfg(unix)] - { - self.blocks() - } - #[cfg(not(unix))] - { - self.len() - } - } - - fn file_id_eq(&self, _other: &Metadata) -> bool { - #[cfg(unix)] - { - self.ino().eq(&_other.ino()) - } - #[cfg(windows)] - { - // TODO: `file_index` requires unstable library feature `windows_by_handle` - // use std::os::windows::prelude::*; - // if let Some(self_id) = self.file_index() { - // if let Some(other_id) = other.file_index() { - // // TODO: not sure this is the equivalent of comparing inode numbers - // return self_id.eq(&other_id); - // } - // } - false - } - } -} - -trait PathExtTail { - fn is_stdin(&self) -> bool; - fn is_orphan(&self) -> bool; - fn is_tailable(&self) -> bool; - fn handle_redirect(&self) -> PathBuf; -} - -impl PathExtTail for Path { - fn is_stdin(&self) -> bool { - self.eq(Self::new(text::DASH)) - || self.eq(Self::new(text::DEV_STDIN)) - || self.eq(Self::new(text::STDIN_HEADER)) - } - - /// Return true if `path` does not have an existing parent directory - fn is_orphan(&self) -> bool { - !matches!(self.parent(), Some(parent) if parent.is_dir()) - } - - /// Return true if `path` is is a file type that can be tailed - fn is_tailable(&self) -> bool { - self.is_file() || self.exists() && self.metadata().unwrap().is_tailable() - } - /// Workaround to handle redirects, e.g. `touch f && tail -f - < f` - fn handle_redirect(&self) -> PathBuf { - if cfg!(unix) && self.is_stdin() { - if let Ok(p) = Self::new(text::DEV_STDIN).canonicalize() { - return p; - } else { - return PathBuf::from(text::DEV_STDIN); - } - } - self.into() - } -} - -trait WatcherExtTail { - fn watch_with_parent(&mut self, path: &Path) -> UResult<()>; -} - -impl WatcherExtTail for dyn Watcher { - /// Wrapper for `notify::Watcher::watch` to also add the parent directory of `path` if necessary. - fn watch_with_parent(&mut self, path: &Path) -> UResult<()> { - let mut path = path.to_owned(); - #[cfg(target_os = "linux")] - if path.is_file() { - /* - NOTE: Using the parent directory instead of the file is a workaround. - This workaround follows the recommendation of the notify crate authors: - > On some platforms, if the `path` is renamed or removed while being watched, behavior may - > be unexpected. See discussions in [#165] and [#166]. If less surprising behavior is wanted - > one may non-recursively watch the _parent_ directory as well and manage related events. - NOTE: Adding both: file and parent results in duplicate/wrong events. - Tested for notify::InotifyWatcher and for notify::PollWatcher. - */ - if let Some(parent) = path.parent() { - if parent.is_dir() { - path = parent.to_owned(); - } else { - path = PathBuf::from("."); - } - } else { - // TODO: [2021-10; jhscheer] add test for this - "cannot watch parent directory" - return Err(USimpleError::new( - 1, - format!("cannot watch parent directory of {}", path.display()), - )); - }; - } - if path.is_relative() { - path = path.canonicalize()?; - } - // TODO: [2022-05; jhscheer] "gnu/tests/tail-2/inotify-rotate-resource.sh" is looking - // for syscalls: 2x "inotify_add_watch" ("filename" and ".") and 1x "inotify_rm_watch" - self.watch(&path, RecursiveMode::NonRecursive).unwrap(); - Ok(()) - } -} - #[cfg(test)] mod tests { diff --git a/src/uu/tail/src/text.rs b/src/uu/tail/src/text.rs new file mode 100644 index 00000000000..fba3968dd12 --- /dev/null +++ b/src/uu/tail/src/text.rs @@ -0,0 +1,20 @@ +// * This file is part of the uutils coreutils package. +// * +// * For the full copyright and license information, please view the LICENSE +// * file that was distributed with this source code. + +// spell-checker:ignore (ToDO) kqueue + +pub static DASH: &str = "-"; +pub static DEV_STDIN: &str = "/dev/stdin"; +pub static STDIN_HEADER: &str = "standard input"; +pub static NO_FILES_REMAINING: &str = "no files remaining"; +pub static NO_SUCH_FILE: &str = "No such file or directory"; +pub static BECOME_INACCESSIBLE: &str = "has become inaccessible"; +pub static BAD_FD: &str = "Bad file descriptor"; +#[cfg(target_os = "linux")] +pub static BACKEND: &str = "inotify"; +#[cfg(all(unix, not(target_os = "linux")))] +pub static BACKEND: &str = "kqueue"; +#[cfg(target_os = "windows")] +pub static BACKEND: &str = "ReadDirectoryChanges"; diff --git a/tests/by-util/test_tail.rs b/tests/by-util/test_tail.rs index cbb05ba9da3..8e2e177c8e4 100644 --- a/tests/by-util/test_tail.rs +++ b/tests/by-util/test_tail.rs @@ -7,6 +7,8 @@ // spell-checker:ignore (libs) kqueue // spell-checker:ignore (jargon) tailable untailable +// TODO: add tests for presume_input_pipe + extern crate tail; use crate::common::util::*; @@ -264,6 +266,7 @@ fn test_follow_redirect_stdin_name_retry() { } #[test] +#[cfg(not(target_os = "macos"))] // See test_stdin_redirect_dir_when_target_os_is_macos #[cfg(all(unix, not(any(target_os = "android", target_os = "freebsd"))))] // FIXME: fix this test for Android/FreeBSD fn test_stdin_redirect_dir() { // $ mkdir dir @@ -289,6 +292,39 @@ fn test_stdin_redirect_dir() { .code_is(1); } +// On macOS path.is_dir() can be false for directories if it was a redirect, +// e.g. `$ tail < DIR. The library feature to detect the +// std::io::ErrorKind::IsADirectory isn't stable so we currently show the a wrong +// error message. +// FIXME: If `std::io::ErrorKind::IsADirectory` becomes stable or macos handles +// redirected directories like linux show the correct message like in +// `test_stdin_redirect_dir` +#[test] +#[cfg(target_os = "macos")] +fn test_stdin_redirect_dir_when_target_os_is_macos() { + // $ mkdir dir + // $ tail < dir, $ tail - < dir + // tail: error reading 'standard input': Is a directory + + let ts = TestScenario::new(util_name!()); + let at = &ts.fixtures; + at.mkdir("dir"); + + ts.ucmd() + .set_stdin(std::fs::File::open(at.plus("dir")).unwrap()) + .fails() + .no_stdout() + .stderr_is("tail: cannot open 'standard input' for reading: No such file or directory") + .code_is(1); + ts.ucmd() + .set_stdin(std::fs::File::open(at.plus("dir")).unwrap()) + .arg("-") + .fails() + .no_stdout() + .stderr_is("tail: cannot open 'standard input' for reading: No such file or directory") + .code_is(1); +} + #[test] #[cfg(target_os = "linux")] fn test_follow_stdin_descriptor() { From 61345cbdc97d4820dd564bcb136f1526c4b54e16 Mon Sep 17 00:00:00 2001 From: Jeffrey Finkelstein Date: Sat, 21 May 2022 23:31:12 -0400 Subject: [PATCH 07/16] mktemp: respect TMPDIR environment variable Change `mktemp` so that it respects the value of the `TMPDIR` environment variable if no directory is otherwise specified in its arguments. For example, before this commit $ TMPDIR=. mktemp /tmp/tmp.WDJ66MaS1T After this commit, $ TMPDIR=. mktemp ./tmp.h96VZBhv8P This matches the behavior of GNU `mktemp`. --- src/uu/mktemp/src/mktemp.rs | 21 ++++++++--- tests/by-util/test_mktemp.rs | 72 ++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 6 deletions(-) diff --git a/src/uu/mktemp/src/mktemp.rs b/src/uu/mktemp/src/mktemp.rs index 1131e0f01c0..dfa7fd10303 100644 --- a/src/uu/mktemp/src/mktemp.rs +++ b/src/uu/mktemp/src/mktemp.rs @@ -170,12 +170,21 @@ impl Options { let template = matches.value_of(OPT_TMPDIR).unwrap().to_string(); (tmpdir, template) } else { - let tmpdir = matches.value_of(OPT_TMPDIR).map(String::from); - let template = matches - .value_of(ARG_TEMPLATE) - .unwrap_or(DEFAULT_TEMPLATE) - .to_string(); - (tmpdir, template) + // If no template argument is given, `--tmpdir` is implied. + match matches.value_of(ARG_TEMPLATE) { + None => { + let tmpdir = match matches.value_of(OPT_TMPDIR) { + None => Some(env::temp_dir().display().to_string()), + Some(tmpdir) => Some(tmpdir.to_string()), + }; + let template = DEFAULT_TEMPLATE; + (tmpdir, template.to_string()) + } + Some(template) => { + let tmpdir = matches.value_of(OPT_TMPDIR).map(String::from); + (tmpdir, template.to_string()) + } + } }; Self { directory: matches.contains_id(OPT_DIRECTORY), diff --git a/tests/by-util/test_mktemp.rs b/tests/by-util/test_mktemp.rs index 9e4a2742cd8..8b58672a2d6 100644 --- a/tests/by-util/test_mktemp.rs +++ b/tests/by-util/test_mktemp.rs @@ -5,6 +5,8 @@ use crate::common::util::*; use uucore::display::Quotable; use std::path::PathBuf; +#[cfg(not(windows))] +use std::path::MAIN_SEPARATOR; use tempfile::tempdir; #[cfg(unix)] @@ -39,6 +41,22 @@ macro_rules! assert_matches_template { }}; } +/// Like [`assert_matches_template`] but for the suffix of a string. +#[cfg(windows)] +macro_rules! assert_suffix_matches_template { + ($template:expr, $s:expr) => {{ + let n = ($s).len(); + let m = ($template).len(); + let suffix = &$s[n - m..n]; + assert!( + matches_template($template, suffix), + "\"{}\" does not end with \"{}\"", + $template, + suffix + ); + }}; +} + #[test] fn test_mktemp_mktemp() { let scene = TestScenario::new(util_name!()); @@ -663,3 +681,57 @@ fn test_mktemp_with_posixly_correct() { .args(&["--suffix=b", "aXXXX"]) .succeeds(); } + +/// Test that files are created relative to `TMPDIR` environment variable. +#[test] +fn test_tmpdir_env_var() { + // `TMPDIR=. mktemp` + let (at, mut ucmd) = at_and_ucmd!(); + let result = ucmd.env(TMPDIR, ".").succeeds(); + let filename = result.no_stderr().stdout_str().trim_end(); + #[cfg(not(windows))] + { + let template = format!(".{}tmp.XXXXXXXXXX", MAIN_SEPARATOR); + assert_matches_template!(&template, filename); + } + // On Windows, `env::temp_dir()` seems to give an absolute path + // regardless of the value of `TMPDIR`; see + // * https://github.com/uutils/coreutils/pull/3552#issuecomment-1211804981 + // * https://doc.rust-lang.org/std/env/fn.temp_dir.html + // * https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-gettemppath2w + #[cfg(windows)] + assert_suffix_matches_template!("tmp.XXXXXXXXXX", filename); + assert!(at.file_exists(filename)); + + // FIXME This is not working because --tmpdir is configured to + // require a value. + // + // // `TMPDIR=. mktemp --tmpdir` + // let (at, mut ucmd) = at_and_ucmd!(); + // let result = ucmd.env(TMPDIR, ".").arg("--tmpdir").succeeds(); + // let filename = result.no_stderr().stdout_str().trim_end(); + // let template = format!(".{}tmp.XXXXXXXXXX", MAIN_SEPARATOR); + // assert_matches_template!(&template, filename); + // assert!(at.file_exists(filename)); + + // `TMPDIR=. mktemp --tmpdir XXX` + let (at, mut ucmd) = at_and_ucmd!(); + let result = ucmd.env(TMPDIR, ".").args(&["--tmpdir", "XXX"]).succeeds(); + let filename = result.no_stderr().stdout_str().trim_end(); + #[cfg(not(windows))] + { + let template = format!(".{}XXX", MAIN_SEPARATOR); + assert_matches_template!(&template, filename); + } + #[cfg(windows)] + assert_suffix_matches_template!("XXX", filename); + assert!(at.file_exists(filename)); + + // `TMPDIR=. mktemp XXX` - in this case `TMPDIR` is ignored. + let (at, mut ucmd) = at_and_ucmd!(); + let result = ucmd.env(TMPDIR, ".").arg("XXX").succeeds(); + let filename = result.no_stderr().stdout_str().trim_end(); + let template = "XXX"; + assert_matches_template!(template, filename); + assert!(at.file_exists(filename)); +} From f7601c022e7935f1348080982b47a70349a8ac7a Mon Sep 17 00:00:00 2001 From: snapdgn Date: Wed, 14 Sep 2022 10:51:03 +0530 Subject: [PATCH 08/16] updated documentation --- src/uu/stat/src/stat.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/uu/stat/src/stat.rs b/src/uu/stat/src/stat.rs index 23e66eddd39..658c47f17fa 100644 --- a/src/uu/stat/src/stat.rs +++ b/src/uu/stat/src/stat.rs @@ -47,7 +47,9 @@ pub const F_SIGN: u8 = 1 << 4; // unused at present pub const F_GROUP: u8 = 1 << 5; -/// checks if the string is within the specified bound +/// checks if the string is within the specified bound, +/// if it gets out of bound, error out by printing sub-string from index `beg` to`end`, +/// where `beg` & `end` is the beginning and end index of sub-string, respectively /// fn check_bound(slice: &str, bound: usize, beg: usize, end: usize) -> UResult<()> { if end >= bound { @@ -101,7 +103,11 @@ fn pad_and_print(result: &str, left: bool, width: usize, padding: Padding) { } /// prints the adjusted string after padding -/// +/// `left` flag specifies the type of alignment of the string +/// `width` is the supplied padding width of the string needed +/// `prefix` & `need_prefix` are Optional, which adjusts the `field_width` accordingly, where +/// `field_width` is the max of supplied `width` and size of string +/// `padding`, specifies type of padding, which is '0' or ' ' in this case. fn print_adjusted( s: &str, left: bool, From 18150d45e308cbe661b5789c6ba09015e04a271b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Sep 2022 06:44:06 +0000 Subject: [PATCH 09/16] build(deps): bump thiserror from 1.0.34 to 1.0.35 Bumps [thiserror](https://github.com/dtolnay/thiserror) from 1.0.34 to 1.0.35. - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/1.0.34...1.0.35) --- updated-dependencies: - dependency-name: thiserror dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a5d3c809243..b6479db7e00 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1940,18 +1940,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.34" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c1b05ca9d106ba7d2e31a9dab4a64e7be2cce415321966ea3132c49a656e252" +checksum = "c53f98874615aea268107765aa1ed8f6116782501d18e53d08b471733bea6c85" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.34" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8f2591983642de85c921015f3f070c665a197ed69e417af436115e3a1407487" +checksum = "f8b463991b4eab2d801e724172285ec4195c650e8ec79b149e6c2a8e6dd3f783" dependencies = [ "proc-macro2", "quote", From b2d5fb4300c2df2673dfd53f1e3425f62bb2b2f4 Mon Sep 17 00:00:00 2001 From: Niyaz Nigmatullin Date: Wed, 14 Sep 2022 11:53:28 +0300 Subject: [PATCH 10/16] `cargo +1.59.0 update` --- Cargo.lock | 64 +++++++++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a5d3c809243..10cfc4a6a7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -254,9 +254,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.20" +version = "3.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b71c3ce99b7611011217b366d923f1d0a7e07a92bb2dbf1e84508c673ca3bd" +checksum = "1ed5341b2301a26ab80be5cbdced622e80ed808483c52e45e3310a877d3b37d7" dependencies = [ "atty", "bitflags", @@ -271,9 +271,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "3.2.4" +version = "3.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4179da71abd56c26b54dd0c248cc081c1f43b0a1a7e8448e28e57a29baa993d" +checksum = "3f7a2e0a962c45ce25afce14220bc24f9dade0a1787f185cecf96bfba7847cd8" dependencies = [ "clap", ] @@ -938,9 +938,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "iana-time-zone" -version = "0.1.47" +version = "0.1.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c495f162af0bf17656d0014a0eded5f3cd2f365fdd204548c2869db89359dc7" +checksum = "237a0714f28b1ee39ccec0770ccb544eb02c9ef2c82bb096230eefcffa6468b0" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -997,9 +997,9 @@ checksum = "1ea37f355c05dde75b84bba2d767906ad522e97cd9e2eef2be7a4ab7fb442c06" [[package]] name = "itertools" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" +checksum = "d8bf247779e67a9082a4790b45e71ac7cfd1321331a5c856a74a9faebdab78d0" dependencies = [ "either", ] @@ -1012,9 +1012,9 @@ checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" [[package]] name = "js-sys" -version = "0.3.59" +version = "0.3.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" dependencies = [ "wasm-bindgen", ] @@ -1319,9 +1319,9 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.0.1" +version = "6.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "029d8d0b2f198229de29dca79676f2738ff952edf3fde542eb8bf94d8c21b435" +checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff" [[package]] name = "ouroboros" @@ -1940,18 +1940,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.34" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c1b05ca9d106ba7d2e31a9dab4a64e7be2cce415321966ea3132c49a656e252" +checksum = "c53f98874615aea268107765aa1ed8f6116782501d18e53d08b471733bea6c85" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.34" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8f2591983642de85c921015f3f070c665a197ed69e417af436115e3a1407487" +checksum = "f8b463991b4eab2d801e724172285ec4195c650e8ec79b149e6c2a8e6dd3f783" dependencies = [ "proc-macro2", "quote", @@ -1984,9 +1984,9 @@ checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" [[package]] name = "unicode-ident" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf" +checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd" [[package]] name = "unicode-linebreak" @@ -1999,15 +1999,15 @@ dependencies = [ [[package]] name = "unicode-segmentation" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" +checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" [[package]] name = "unicode-width" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "unicode-xid" @@ -3116,9 +3116,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3126,9 +3126,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" dependencies = [ "bumpalo", "log", @@ -3141,9 +3141,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3151,9 +3151,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" dependencies = [ "proc-macro2", "quote", @@ -3164,9 +3164,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" [[package]] name = "which" From 05b7183112ea2be5c05bd0bbc493f64f6c68d312 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Sep 2022 06:34:05 +0000 Subject: [PATCH 11/16] build(deps): bump digest from 0.10.3 to 0.10.5 Bumps [digest](https://github.com/RustCrypto/traits) from 0.10.3 to 0.10.5. - [Release notes](https://github.com/RustCrypto/traits/releases) - [Commits](https://github.com/RustCrypto/traits/compare/digest-v0.10.3...digest-v0.10.5) --- updated-dependencies: - dependency-name: digest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- src/uu/hashsum/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 10cfc4a6a7f..ab35a5ced48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -675,9 +675,9 @@ checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" [[package]] name = "digest" -version = "0.10.3" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" dependencies = [ "block-buffer", "crypto-common", diff --git a/src/uu/hashsum/Cargo.toml b/src/uu/hashsum/Cargo.toml index 29b59c3c2a2..3d89cb57594 100644 --- a/src/uu/hashsum/Cargo.toml +++ b/src/uu/hashsum/Cargo.toml @@ -15,7 +15,7 @@ edition = "2021" path = "src/hashsum.rs" [dependencies] -digest = "0.10.1" +digest = "0.10.5" clap = { version = "3.2", features = ["wrap_help", "cargo"] } hex = "0.4.3" memchr = "2" From acef46b6295bea61439fbd77b84a5f6a0d7a7b22 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Fri, 16 Sep 2022 10:26:29 +0200 Subject: [PATCH 12/16] add the capability to run several tests at once --- DEVELOPER_INSTRUCTIONS.md | 2 +- README.md | 2 ++ util/run-gnu-test.sh | 18 ++++++++++++++---- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/DEVELOPER_INSTRUCTIONS.md b/DEVELOPER_INSTRUCTIONS.md index c007fba7ec2..d6171bef80d 100644 --- a/DEVELOPER_INSTRUCTIONS.md +++ b/DEVELOPER_INSTRUCTIONS.md @@ -21,7 +21,7 @@ Running GNU tests At the end you should have uutils, gnu and gnulib checked out next to each other. - Run `cd uutils && ./util/build-gnu.sh && cd ..` to get everything ready (this may take a while) -- Finally, you can run tests with `bash uutils/util/run-gnu-test.sh `. Instead of `` insert the test you want to run, e.g. `tests/misc/wc-proc.sh`. +- Finally, you can run tests with `bash uutils/util/run-gnu-test.sh `. Instead of `` insert the tests you want to run, e.g. `tests/misc/wc-proc.sh`. Code Coverage Report Generation diff --git a/README.md b/README.md index 0bf8ca0396d..f04c58ceaba 100644 --- a/README.md +++ b/README.md @@ -369,6 +369,8 @@ $ bash util/build-gnu.sh $ bash util/run-gnu-test.sh # To run a single test: $ bash util/run-gnu-test.sh tests/touch/not-owner.sh # for example +# To run several tests: +$ bash util/run-gnu-test.sh tests/touch/not-owner.sh tests/rm/no-give-up.sh # for example # If this is a perl (.pl) test, to run in debug: $ DEBUG=1 bash util/run-gnu-test.sh tests/misc/sm3sum.pl ``` diff --git a/util/run-gnu-test.sh b/util/run-gnu-test.sh index 9b784699ced..fdd928221a2 100755 --- a/util/run-gnu-test.sh +++ b/util/run-gnu-test.sh @@ -31,16 +31,26 @@ cd "${path_GNU}" && echo "[ pwd:'${PWD}' ]" export RUST_BACKTRACE=1 -if test -n "$1"; then - # if set, run only the test passed - export RUN_TEST="TESTS=$1" +if test $# -ge 1; then + # if set, run only the tests passed + SPECIFIC_TESTS="" + for t in "$@"; do + SPECIFIC_TESTS="$SPECIFIC_TESTS $t" + done + # trim it + SPECIFIC_TESTS=$(echo $SPECIFIC_TESTS| xargs) + echo "Running specific tests: $SPECIFIC_TESTS" fi # * timeout used to kill occasionally errant/"stuck" processes (note: 'release' testing takes ~1 hour; 'debug' testing takes ~2.5 hours) # * `gl_public_submodule_commit=` disables testing for use of a "public" gnulib commit (which will fail when using shallow gnulib checkouts) # * `srcdir=..` specifies the GNU source directory for tests (fixing failing/confused 'tests/factor/tNN.sh' tests and causing no harm to other tests) #shellcheck disable=SC2086 -timeout -sKILL 4h make -j "$(nproc)" check ${RUN_TEST} SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make +if test $# -ge 1; then + timeout -sKILL 4h make -j "$(nproc)" check TESTS="$SPECIFIC_TESTS" SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make +else + timeout -sKILL 4h make -j "$(nproc)" check SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make +fi if test -z "$1" && test -n "$CI"; then sudo make -j "$(nproc)" check-root SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" TEST_SUITE_LOG="tests/test-suite-root.log" || : From 7a6967cdff7b3b5a120b52940749db2c78fdd7b5 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Fri, 16 Sep 2022 09:07:42 +0200 Subject: [PATCH 13/16] GNU test: Generate a few more locales One of the test is skipped with: sort-h-thousands-sep.sh: skipped test: The Swedish locale with blank thousands separator is unavailable. --- .github/workflows/GnuTests.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index 3afaa172301..0e00ca72ca1 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -82,8 +82,13 @@ jobs: ## Some tests fail with 'cannot change locale (en_US.ISO-8859-1): No such file or directory' ## Some others need a French locale sudo locale-gen - sudo locale-gen fr_FR - sudo locale-gen fr_FR.UTF-8 + sudo locale-gen --keep-existing fr_FR + sudo locale-gen --keep-existing fr_FR.UTF-8 + sudo locale-gen --keep-existing sv_SE + sudo locale-gen --keep-existing sv_SE.UTF-8 + sudo locale-gen --keep-existing en_US + sudo locale-gen --keep-existing ru_RU.KOI8-R + sudo update-locale echo "After:" locale -a @@ -257,8 +262,8 @@ jobs: ## Some tests fail with 'cannot change locale (en_US.ISO-8859-1): No such file or directory' ## Some others need a French locale sudo locale-gen - sudo locale-gen fr_FR - sudo locale-gen fr_FR.UTF-8 + sudo locale-gen --keep-existing fr_FR + sudo locale-gen --keep-existing fr_FR.UTF-8 sudo update-locale echo "After:" locale -a From 774f498aa8e8015f7417b727fbba404ff664fdbf Mon Sep 17 00:00:00 2001 From: Niyaz Nigmatullin Date: Sat, 17 Sep 2022 01:08:41 +0300 Subject: [PATCH 14/16] chore(deps): Bump terminal_size + cargo update --- Cargo.lock | 36 ++++++++++++++++++------------------ src/uu/ls/Cargo.toml | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ab35a5ced48..a3ae71adfed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -254,9 +254,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.21" +version = "3.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ed5341b2301a26ab80be5cbdced622e80ed808483c52e45e3310a877d3b37d7" +checksum = "86447ad904c7fb335a790c9d7fe3d0d971dc523b8ccd1561a520de9a85302750" dependencies = [ "atty", "bitflags", @@ -1563,9 +1563,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ "getrandom", ] @@ -1748,9 +1748,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.4" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "006769ba83e921b3085caa8334186b00cf92b4cb1a6cf4632fbccc8eff5c7549" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ "cfg-if", "cpufeatures", @@ -1759,9 +1759,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9db03534dff993187064c4e0c05a5708d2a9728ace9a8959b77bedf415dac5" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if", "cpufeatures", @@ -1770,9 +1770,9 @@ dependencies = [ [[package]] name = "sha3" -version = "0.10.4" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaedf34ed289ea47c2b741bb72e5357a209512d67bcd4bda44359e5bf0470f56" +checksum = "e2904bea16a1ae962b483322a1c7b81d976029203aea1f461e51cd7705db7ba9" dependencies = [ "digest", "keccak", @@ -1918,19 +1918,19 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.1.17" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +checksum = "8440c860cf79def6164e4a0a983bcc2305d82419177a0e0c71930d049e3ac5a1" dependencies = [ - "libc", - "winapi", + "rustix", + "windows-sys", ] [[package]] name = "textwrap" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" +checksum = "949517c0cf1bf4ee812e2e07e08ab448e3ae0d23472aee8a06c985f0c8815b16" dependencies = [ "smawk", "terminal_size", @@ -2011,9 +2011,9 @@ checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "unicode-xid" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "unindent" diff --git a/src/uu/ls/Cargo.toml b/src/uu/ls/Cargo.toml index 9d04ba14052..7bb5039b74b 100644 --- a/src/uu/ls/Cargo.toml +++ b/src/uu/ls/Cargo.toml @@ -20,7 +20,7 @@ clap = { version = "3.2", features = ["wrap_help", "cargo", "env"] } unicode-width = "0.1.8" number_prefix = "0.4" term_grid = "0.1.5" -terminal_size = "0.1" +terminal_size = "0.2.1" glob = "0.3.0" lscolors = { version = "0.12.0", features = ["ansi_term"] } uucore = { version=">=0.0.15", package="uucore", path="../../uucore", features = ["entries", "fs"] } From cfa7ba2ce2076db144fe8bc8cccdb79760d51bfe Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Fri, 16 Sep 2022 09:14:13 +0200 Subject: [PATCH 15/16] gnu: merge the root tests results into the main one --- .github/workflows/GnuTests.yml | 12 +++--- util/analyze-gnu-results.sh | 75 ++++++++++++++++++++++++++++++++++ util/run-gnu-test.sh | 1 + 3 files changed, 82 insertions(+), 6 deletions(-) create mode 100644 util/analyze-gnu-results.sh diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index 3afaa172301..3b35f1871a8 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -34,6 +34,7 @@ jobs: outputs repo_default_branch repo_GNU_ref repo_reference_branch # SUITE_LOG_FILE="${path_GNU_tests}/test-suite.log" + ROOT_SUITE_LOG_FILE="${path_GNU_tests}/test-suite-root.log" TEST_LOGS_GLOB="${path_GNU_tests}/**/*.log" ## note: not usable at bash CLI; [why] double globstar not enabled by default b/c MacOS includes only bash v3 which doesn't have double globstar support TEST_FILESET_PREFIX='test-fileset-IDs.sha1#' TEST_FILESET_SUFFIX='.txt' @@ -108,18 +109,17 @@ jobs: id: summary shell: bash run: | + path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' ## Extract/summarize testing info outputs() { step_id="summary"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo ::set-output name=${var}::${!var}; done; } # SUITE_LOG_FILE='${{ steps.vars.outputs.SUITE_LOG_FILE }}' + ROOT_SUITE_LOG_FILE='${{ steps.vars.outputs.ROOT_SUITE_LOG_FILE }}' + ls -al ${SUITE_LOG_FILE} ${ROOT_SUITE_LOG_FILE} + if test -f "${SUITE_LOG_FILE}" then - TOTAL=$(sed -n "s/.*# TOTAL: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) - PASS=$(sed -n "s/.*# PASS: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) - SKIP=$(sed -n "s/.*# SKIP: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) - FAIL=$(sed -n "s/.*# FAIL: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) - XPASS=$(sed -n "s/.*# XPASS: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) - ERROR=$(sed -n "s/.*# ERROR: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + source ${path_UUTILS}/util/analyze-gnu-results.sh ${SUITE_LOG_FILE} ${ROOT_SUITE_LOG_FILE} if [[ "$TOTAL" -eq 0 || "$TOTAL" -eq 1 ]]; then echo "::error ::Failed to parse test results from '${SUITE_LOG_FILE}'; failing early" exit 1 diff --git a/util/analyze-gnu-results.sh b/util/analyze-gnu-results.sh new file mode 100644 index 00000000000..2bc08a9a4cb --- /dev/null +++ b/util/analyze-gnu-results.sh @@ -0,0 +1,75 @@ +#!/bin/sh +# spell-checker:ignore xpass XPASS testsuite +set -e + +# As we do two builds (with and without root), we need to do some trivial maths +# to present the merge results +# this script will export the values in the term + +if test $# -ne 2; then + echo "syntax:" + echo "$0 testsuite.log root-testsuite.log" +fi + +SUITE_LOG_FILE=$1 +ROOT_SUITE_LOG_FILE=$2 + +if test ! -f "${SUITE_LOG_FILE}"; then + echo "${SUITE_LOG_FILE} has not been found" + exit 1 +fi +if test ! -f "${ROOT_SUITE_LOG_FILE}"; then + echo "${ROOT_SUITE_LOG_FILE} has not been found" + exit 1 +fi + +function get_total { + # Total of tests executed + # They are the normal number of tests as they are skipped in the normal run + NON_ROOT=$(sed -n "s/.*# TOTAL: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + echo $NON_ROOT +} + +function get_pass { + # This is the sum of the two test suites. + # In the normal run, they are SKIP + NON_ROOT=$(sed -n "s/.*# PASS: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + AS_ROOT=$(sed -n "s/.*# PASS: \(.*\)/\1/p" "${ROOT_SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + echo $((NON_ROOT + AS_ROOT)) +} + +function get_skip { + # As some of the tests executed as root as still SKIP (ex: selinux), we + # need to some maths: + # Number of tests skip as user - total test as root + skipped as root + TOTAL_AS_ROOT=$(sed -n "s/.*# TOTAL: \(.*\)/\1/p" "${ROOT_SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + NON_ROOT=$(sed -n "s/.*# SKIP: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + AS_ROOT=$(sed -n "s/.*# SKIP: \(.*\)/\1/p" "${ROOT_SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + echo $((NON_ROOT - TOTAL_AS_ROOT + AS_ROOT)) +} + +function get_fail { + # They used to be SKIP, now they fail (this is a good news) + NON_ROOT=$(sed -n "s/.*# FAIL: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + AS_ROOT=$(sed -n "s/.*# FAIL: \(.*\)/\1/p" "${ROOT_SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + echo $((NON_ROOT + AS_ROOT)) +} + +function get_xpass { + NON_ROOT=$(sed -n "s/.*# XPASS: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + echo $NON_ROOT +} + +function get_error { + # They used to be SKIP, now they error (this is a good news) + NON_ROOT=$(sed -n "s/.*# ERROR: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + AS_ROOT=$(sed -n "s/.*# ERROR:: \(.*\)/\1/p" "${ROOT_SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + echo $((NON_ROOT + AS_ROOT)) +} + +export TOTAL=$(get_total) +export PASS=$(get_pass) +export SKIP=$(get_skip) +export FAIL=$(get_fail) +export XPASS=$(get_xpass) +export ERROR=$(get_error) diff --git a/util/run-gnu-test.sh b/util/run-gnu-test.sh index fdd928221a2..2fbeba501cf 100755 --- a/util/run-gnu-test.sh +++ b/util/run-gnu-test.sh @@ -53,5 +53,6 @@ else fi if test -z "$1" && test -n "$CI"; then + echo "Running check-root to run only root tests" sudo make -j "$(nproc)" check-root SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" TEST_SUITE_LOG="tests/test-suite-root.log" || : fi From 20af659f09f122d1122157aaa29a6fac0fe936a6 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Sat, 17 Sep 2022 11:08:12 +0200 Subject: [PATCH 16/16] Run the GNU root tests in a separate task --- .github/workflows/GnuTests.yml | 8 +++++++- util/run-gnu-test.sh | 25 ++++++++++++++++--------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index 3b35f1871a8..d932538e53e 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -40,7 +40,7 @@ jobs: TEST_FILESET_SUFFIX='.txt' TEST_SUMMARY_FILE='gnu-result.json' TEST_FULL_SUMMARY_FILE='gnu-full-result.json' - outputs SUITE_LOG_FILE TEST_FILESET_PREFIX TEST_FILESET_SUFFIX TEST_LOGS_GLOB TEST_SUMMARY_FILE TEST_FULL_SUMMARY_FILE + outputs SUITE_LOG_FILE ROOT_SUITE_LOG_FILE TEST_FILESET_PREFIX TEST_FILESET_SUFFIX TEST_LOGS_GLOB TEST_SUMMARY_FILE TEST_FULL_SUMMARY_FILE - name: Checkout code (uutil) uses: actions/checkout@v3 with: @@ -100,6 +100,12 @@ jobs: path_GNU='${{ steps.vars.outputs.path_GNU }}' path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' bash "${path_UUTILS}/util/run-gnu-test.sh" + - name: Run GNU root tests + shell: bash + run: | + path_GNU='${{ steps.vars.outputs.path_GNU }}' + path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' + bash "${path_UUTILS}/util/run-gnu-test.sh" run-root - name: Extract testing info into JSON shell: bash run : | diff --git a/util/run-gnu-test.sh b/util/run-gnu-test.sh index 2fbeba501cf..f5c47e6450a 100755 --- a/util/run-gnu-test.sh +++ b/util/run-gnu-test.sh @@ -46,13 +46,20 @@ fi # * `gl_public_submodule_commit=` disables testing for use of a "public" gnulib commit (which will fail when using shallow gnulib checkouts) # * `srcdir=..` specifies the GNU source directory for tests (fixing failing/confused 'tests/factor/tNN.sh' tests and causing no harm to other tests) #shellcheck disable=SC2086 -if test $# -ge 1; then - timeout -sKILL 4h make -j "$(nproc)" check TESTS="$SPECIFIC_TESTS" SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make -else - timeout -sKILL 4h make -j "$(nproc)" check SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make -fi -if test -z "$1" && test -n "$CI"; then - echo "Running check-root to run only root tests" - sudo make -j "$(nproc)" check-root SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" TEST_SUITE_LOG="tests/test-suite-root.log" || : -fi +if test "$1" != "run-root"; then +# run the regular tests + if test $# -ge 1; then + timeout -sKILL 4h make -j "$(nproc)" check TESTS="$SPECIFIC_TESTS" SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make + else + timeout -sKILL 4h make -j "$(nproc)" check SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" || : # Kill after 4 hours in case something gets stuck in make + fi +else +# in case we would like to run tests requiring root + if test -z "$1" -o "$1" == "run-root"; then + if test -n "$CI"; then + echo "Running check-root to run only root tests" + sudo make -j "$(nproc)" check-root SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" TEST_SUITE_LOG="tests/test-suite-root.log" || : + fi + fi +fi \ No newline at end of file