From 7d7d4fb04a6bd053c91cc8cdc469ca734000e60d Mon Sep 17 00:00:00 2001 From: zhitkoff Date: Tue, 22 Aug 2023 13:09:28 -0400 Subject: [PATCH 01/12] build-gnu.sh: `/usr/bin/timeout` should not be hardcoded to /usr/bin location Fixes #5193 --- util/build-gnu.sh | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/util/build-gnu.sh b/util/build-gnu.sh index d852ed66fb6..d02c8c842e6 100755 --- a/util/build-gnu.sh +++ b/util/build-gnu.sh @@ -82,7 +82,16 @@ else ./bootstrap --skip-po ./configure --quiet --disable-gcc-warnings #Add timeout to to protect against hangs - sed -i 's|^"\$@|/usr/bin/timeout 600 "\$@|' build-aux/test-driver + # On MacOS there is no system /usr/bin/timeout + # and trying to add it to /usr/bin (with symlink of copy binary) will fail unless system integrity protection is disabled (not ideal) + # ref: https://support.apple.com/en-us/102149 + # On MacOS the Homebrew coreutils could be installed and then "sudo ln -s /opt/homebrew/bin/timeout /usr/local/bin/timeout" + # Set to /usr/local/timeout instead if /usr/bin/timeout is not found + if [ -x /usr/bin/timeout ] ; then + sed -i 's|^"\$@|/usr/bin/timeout 600 "\$@|' build-aux/test-driver + else + sed -i 's|^"\$@|/usr/local/bin/timeout 600 "\$@|' build-aux/test-driver + fi # Change the PATH in the Makefile to test the uutils coreutils instead of the GNU coreutils sed -i "s/^[[:blank:]]*PATH=.*/ PATH='${UU_BUILD_DIR//\//\\/}\$(PATH_SEPARATOR)'\"\$\$PATH\" \\\/" Makefile sed -i 's| tr | /usr/bin/tr |' tests/init.sh @@ -153,13 +162,28 @@ sed -i 's|touch |/usr/bin/touch |' tests/cp/reflink-perm.sh tests/ls/block-size. sed -i 's|ln -|/usr/bin/ln -|' tests/cp/link-deref.sh sed -i 's|cp |/usr/bin/cp |' tests/mv/hard-2.sh sed -i 's|paste |/usr/bin/paste |' tests/misc/od-endian.sh -sed -i 's|timeout |/usr/bin/timeout |' tests/tail-2/follow-stdin.sh +# On MacOS there is no system /usr/bin/timeout +# and trying to add it to /usr/bin (with symlink of copy binary) will fail unless system integrity protection is disabled (not ideal) +# ref: https://support.apple.com/en-us/102149 +# On MacOS the Homebrew coreutils could be installed and then "sudo ln -s /opt/homebrew/bin/timeout /usr/local/bin/timeout" +# Set to /usr/local/timeout instead if /usr/bin/timeout is not found +if [ -x /usr/bin/timeout ] ; then + sed -i 's|timeout |/usr/bin/timeout |' tests/tail-2/follow-stdin.sh +else + sed -i 's|timeout |/usr/local/bin/timeout |' tests/tail-2/follow-stdin.sh +fi + # Add specific timeout to tests that currently hang to limit time spent waiting -sed -i 's|\(^\s*\)seq \$|\1/usr/bin/timeout 0.1 seq \$|' tests/misc/seq-precision.sh tests/misc/seq-long-double.sh +if [ -x /usr/bin/timeout ] ; then + sed -i 's|\(^\s*\)seq \$|\1/usr/bin/timeout 0.1 seq \$|' tests/misc/seq-precision.sh tests/misc/seq-long-double.sh +else + sed -i 's|\(^\s*\)seq \$|\1/usr/local/bin/timeout 0.1 seq \$|' tests/misc/seq-precision.sh tests/misc/seq-long-double.sh +fi -# Remove dup of /usr/bin/ when executed several times +# Remove dup of /usr/bin/ (and /usr/local/bin) when executed several times grep -rlE '/usr/bin/\s?/usr/bin' init.cfg tests/* | xargs --no-run-if-empty sed -Ei 's|/usr/bin/\s?/usr/bin/|/usr/bin/|g' +grep -rlE '/usr/local/bin/\s?/usr/local/bin' init.cfg tests/* | xargs --no-run-if-empty sed -Ei 's|/usr/local/bin/\s?/usr/local/bin/|/usr/local/bin/|g' #### Adjust tests to make them work with Rust/coreutils # in some cases, what we are doing in rust/coreutils is good (or better) From 75cc6e3fdccedf551e297e54715cc65ed6045c7c Mon Sep 17 00:00:00 2001 From: zhitkoff Date: Tue, 22 Aug 2023 13:22:24 -0400 Subject: [PATCH 02/12] added some TODO(s) for missing/moved locations --- util/build-gnu.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/util/build-gnu.sh b/util/build-gnu.sh index d02c8c842e6..6c608a06077 100755 --- a/util/build-gnu.sh +++ b/util/build-gnu.sh @@ -151,6 +151,9 @@ sed -i -e '/tests\/misc\/seq-precision.sh/ D' \ sed -i '/INT_OFLOW/ D' tests/misc/printf.sh # Use the system coreutils where the test fails due to error in a util that is not the one being tested +# TODO : tests/tail-2/ does not appear to exist +# and have been moved to just tests/tail/ location +# Might need to update the section bvelow to reflect that sed -i 's|stat|/usr/bin/stat|' tests/touch/60-seconds.sh tests/misc/sort-compress-proc.sh sed -i 's|ls -|/usr/bin/ls -|' tests/cp/same-file.sh tests/misc/mknod.sh tests/mv/part-symlink.sh sed -i 's|chmod |/usr/bin/chmod |' tests/du/inacc-dir.sh tests/tail-2/tail-n0f.sh tests/cp/fail-perm.sh tests/mv/i-2.sh tests/misc/shuf.sh @@ -175,6 +178,9 @@ fi # Add specific timeout to tests that currently hang to limit time spent waiting +# TODO : tests/misc/seq-precision.sh tests/misc/seq-long-double.sh do not appear to exist +# and have been moved to tests/seq/ location +# Might need to update the section bvelow to reflect that if [ -x /usr/bin/timeout ] ; then sed -i 's|\(^\s*\)seq \$|\1/usr/bin/timeout 0.1 seq \$|' tests/misc/seq-precision.sh tests/misc/seq-long-double.sh else @@ -205,6 +211,9 @@ sed -i -e "s|rm: cannot remove 'rel': Permission denied|rm: cannot remove 'rel': # overlay-headers.sh test intends to check for inotify events, # however there's a bug because `---dis` is an alias for: `---disable-inotify` +# TODO : tests/tail-2/ does not appear to exist +# and have been moved to just tests/tail/ location +# Might need to update the section bvelow to reflect that sed -i -e "s|---dis ||g" tests/tail-2/overlay-headers.sh test -f "${UU_BUILD_DIR}/getlimits" || cp src/getlimits "${UU_BUILD_DIR}" From c44c3cd71649252ae333dca0c55ab56d9ca52163 Mon Sep 17 00:00:00 2001 From: zhitkoff Date: Tue, 22 Aug 2023 15:28:02 -0400 Subject: [PATCH 03/12] fixed spelling --- util/build-gnu.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/build-gnu.sh b/util/build-gnu.sh index 6c608a06077..4414b7f12ba 100755 --- a/util/build-gnu.sh +++ b/util/build-gnu.sh @@ -153,7 +153,7 @@ sed -i '/INT_OFLOW/ D' tests/misc/printf.sh # Use the system coreutils where the test fails due to error in a util that is not the one being tested # TODO : tests/tail-2/ does not appear to exist # and have been moved to just tests/tail/ location -# Might need to update the section bvelow to reflect that +# Might need to update the section below to reflect that sed -i 's|stat|/usr/bin/stat|' tests/touch/60-seconds.sh tests/misc/sort-compress-proc.sh sed -i 's|ls -|/usr/bin/ls -|' tests/cp/same-file.sh tests/misc/mknod.sh tests/mv/part-symlink.sh sed -i 's|chmod |/usr/bin/chmod |' tests/du/inacc-dir.sh tests/tail-2/tail-n0f.sh tests/cp/fail-perm.sh tests/mv/i-2.sh tests/misc/shuf.sh @@ -180,7 +180,7 @@ fi # Add specific timeout to tests that currently hang to limit time spent waiting # TODO : tests/misc/seq-precision.sh tests/misc/seq-long-double.sh do not appear to exist # and have been moved to tests/seq/ location -# Might need to update the section bvelow to reflect that +# Might need to update the section below to reflect that if [ -x /usr/bin/timeout ] ; then sed -i 's|\(^\s*\)seq \$|\1/usr/bin/timeout 0.1 seq \$|' tests/misc/seq-precision.sh tests/misc/seq-long-double.sh else @@ -213,7 +213,7 @@ sed -i -e "s|rm: cannot remove 'rel': Permission denied|rm: cannot remove 'rel': # however there's a bug because `---dis` is an alias for: `---disable-inotify` # TODO : tests/tail-2/ does not appear to exist # and have been moved to just tests/tail/ location -# Might need to update the section bvelow to reflect that +# Might need to update the section below to reflect that sed -i -e "s|---dis ||g" tests/tail-2/overlay-headers.sh test -f "${UU_BUILD_DIR}/getlimits" || cp src/getlimits "${UU_BUILD_DIR}" From 6468845850a0c847f42fb400ffecbdcf54f8e99c Mon Sep 17 00:00:00 2001 From: zhitkoff Date: Thu, 24 Aug 2023 20:36:52 -0400 Subject: [PATCH 04/12] refactor to check for system timeout once + commented out many moved/deleted test files that make script fail --- util/build-gnu.sh | 71 ++++++++++++++++++++++------------------------- 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/util/build-gnu.sh b/util/build-gnu.sh index 4414b7f12ba..c6991d15c1b 100755 --- a/util/build-gnu.sh +++ b/util/build-gnu.sh @@ -18,6 +18,20 @@ path_GNU="$(readlink -fm -- "${path_GNU:-${path_UUTILS}/../gnu}")" ### +# On MacOS there is no system /usr/bin/timeout +# and trying to add it to /usr/bin (with symlink of copy binary) will fail unless system integrity protection is disabled (not ideal) +# ref: https://support.apple.com/en-us/102149 +# On MacOS the Homebrew coreutils could be installed and then "sudo ln -s /opt/homebrew/bin/timeout /usr/local/bin/timeout" +# Set to /usr/local/bin/timeout instead if /usr/bin/timeout is not found +SYSTEM_TIMEOUT="timeout" +if [ -x /usr/bin/timeout ] ; then + SYSTEM_TIMEOUT="/usr/bin/timeout" +elif [ -x /usr/local/bin/timeout ] ; then + SYSTEM_TIMEOUT="/usr/local/bin/timeout" +fi + +### + if test ! -d "${path_GNU}"; then echo "Could not find GNU coreutils (expected at '${path_GNU}')" echo "Run the following to download into the expected path:" @@ -82,16 +96,7 @@ else ./bootstrap --skip-po ./configure --quiet --disable-gcc-warnings #Add timeout to to protect against hangs - # On MacOS there is no system /usr/bin/timeout - # and trying to add it to /usr/bin (with symlink of copy binary) will fail unless system integrity protection is disabled (not ideal) - # ref: https://support.apple.com/en-us/102149 - # On MacOS the Homebrew coreutils could be installed and then "sudo ln -s /opt/homebrew/bin/timeout /usr/local/bin/timeout" - # Set to /usr/local/timeout instead if /usr/bin/timeout is not found - if [ -x /usr/bin/timeout ] ; then - sed -i 's|^"\$@|/usr/bin/timeout 600 "\$@|' build-aux/test-driver - else - sed -i 's|^"\$@|/usr/local/bin/timeout 600 "\$@|' build-aux/test-driver - fi + sed -i 's|^"\$@|'"${SYSTEM_TIMEOUT}"' 600 "\$@|' build-aux/test-driver # Change the PATH in the Makefile to test the uutils coreutils instead of the GNU coreutils sed -i "s/^[[:blank:]]*PATH=.*/ PATH='${UU_BUILD_DIR//\//\\/}\$(PATH_SEPARATOR)'\"\$\$PATH\" \\\/" Makefile sed -i 's| tr | /usr/bin/tr |' tests/init.sh @@ -148,44 +153,32 @@ sed -i -e '/tests\/misc\/seq-precision.sh/ D' \ Makefile # printf doesn't limit the values used in its arg, so this produced ~2GB of output -sed -i '/INT_OFLOW/ D' tests/misc/printf.sh +# Looks like tests/misc/printf.sh does not exist anymore - comment it out for now +#sed -i '/INT_OFLOW/ D' tests/misc/printf.sh # Use the system coreutils where the test fails due to error in a util that is not the one being tested # TODO : tests/tail-2/ does not appear to exist # and have been moved to just tests/tail/ location # Might need to update the section below to reflect that -sed -i 's|stat|/usr/bin/stat|' tests/touch/60-seconds.sh tests/misc/sort-compress-proc.sh +# Also looks like tests/misc/sort-compress-proc.sh and tests/tail-2/tail-n0f.sh and tests/misc/shuf.sh and many others do not exist anymore or moved - comment it out for now +sed -i 's|stat|/usr/bin/stat|' tests/touch/60-seconds.sh #tests/misc/sort-compress-proc.sh sed -i 's|ls -|/usr/bin/ls -|' tests/cp/same-file.sh tests/misc/mknod.sh tests/mv/part-symlink.sh -sed -i 's|chmod |/usr/bin/chmod |' tests/du/inacc-dir.sh tests/tail-2/tail-n0f.sh tests/cp/fail-perm.sh tests/mv/i-2.sh tests/misc/shuf.sh -sed -i 's|sort |/usr/bin/sort |' tests/ls/hyperlink.sh tests/misc/test-N.sh -sed -i 's|split |/usr/bin/split |' tests/misc/factor-parallel.sh -sed -i 's|id -|/usr/bin/id -|' tests/misc/runcon-no-reorder.sh +sed -i 's|chmod |/usr/bin/chmod |' tests/du/inacc-dir.sh tests/cp/fail-perm.sh tests/mv/i-2.sh #tests/misc/shuf.sh #tests/tail-2/tail-n0f.sh +sed -i 's|sort |/usr/bin/sort |' tests/ls/hyperlink.sh #tests/misc/test-N.sh +#sed -i 's|split |/usr/bin/split |' tests/misc/factor-parallel.sh +#sed -i 's|id -|/usr/bin/id -|' tests/misc/runcon-no-reorder.sh # tests/ls/abmon-align.sh - https://github.com/uutils/coreutils/issues/3505 -sed -i 's|touch |/usr/bin/touch |' tests/cp/reflink-perm.sh tests/ls/block-size.sh tests/mv/update.sh tests/misc/ls-time.sh tests/misc/stat-nanoseconds.sh tests/misc/time-style.sh tests/misc/test-N.sh tests/ls/abmon-align.sh +sed -i 's|touch |/usr/bin/touch |' tests/cp/reflink-perm.sh tests/ls/block-size.sh tests/mv/update.sh tests/misc/time-style.sh tests/ls/abmon-align.sh #tests/misc/ls-time.sh tests/misc/stat-nanoseconds.sh tests/misc/test-N.sh sed -i 's|ln -|/usr/bin/ln -|' tests/cp/link-deref.sh sed -i 's|cp |/usr/bin/cp |' tests/mv/hard-2.sh -sed -i 's|paste |/usr/bin/paste |' tests/misc/od-endian.sh -# On MacOS there is no system /usr/bin/timeout -# and trying to add it to /usr/bin (with symlink of copy binary) will fail unless system integrity protection is disabled (not ideal) -# ref: https://support.apple.com/en-us/102149 -# On MacOS the Homebrew coreutils could be installed and then "sudo ln -s /opt/homebrew/bin/timeout /usr/local/bin/timeout" -# Set to /usr/local/timeout instead if /usr/bin/timeout is not found -if [ -x /usr/bin/timeout ] ; then - sed -i 's|timeout |/usr/bin/timeout |' tests/tail-2/follow-stdin.sh -else - sed -i 's|timeout |/usr/local/bin/timeout |' tests/tail-2/follow-stdin.sh -fi - +#sed -i 's|paste |/usr/bin/paste |' tests/misc/od-endian.sh +#sed -i 's|timeout |'"${SYSTEM_TIMEOUT}"' |' tests/tail-2/follow-stdin.sh # Add specific timeout to tests that currently hang to limit time spent waiting # TODO : tests/misc/seq-precision.sh tests/misc/seq-long-double.sh do not appear to exist # and have been moved to tests/seq/ location -# Might need to update the section below to reflect that -if [ -x /usr/bin/timeout ] ; then - sed -i 's|\(^\s*\)seq \$|\1/usr/bin/timeout 0.1 seq \$|' tests/misc/seq-precision.sh tests/misc/seq-long-double.sh -else - sed -i 's|\(^\s*\)seq \$|\1/usr/local/bin/timeout 0.1 seq \$|' tests/misc/seq-precision.sh tests/misc/seq-long-double.sh -fi +# Might need to update the section below to reflect that, but comment it out for now +#sed -i 's|\(^\s*\)seq \$|\1'"${SYSTEM_TIMEOUT}"' 0.1 seq \$|' tests/misc/seq-precision.sh tests/misc/seq-long-double.sh # Remove dup of /usr/bin/ (and /usr/local/bin) when executed several times grep -rlE '/usr/bin/\s?/usr/bin' init.cfg tests/* | xargs --no-run-if-empty sed -Ei 's|/usr/bin/\s?/usr/bin/|/usr/bin/|g' @@ -214,7 +207,7 @@ sed -i -e "s|rm: cannot remove 'rel': Permission denied|rm: cannot remove 'rel': # TODO : tests/tail-2/ does not appear to exist # and have been moved to just tests/tail/ location # Might need to update the section below to reflect that -sed -i -e "s|---dis ||g" tests/tail-2/overlay-headers.sh +#sed -i -e "s|---dis ||g" tests/tail-2/overlay-headers.sh test -f "${UU_BUILD_DIR}/getlimits" || cp src/getlimits "${UU_BUILD_DIR}" @@ -271,11 +264,13 @@ sed -i -e "s/Try 'mv --help' for more information/For more information, try '--h # GNU doesn't support width > INT_MAX # disable these test cases -sed -i -E "s|^([^#]*2_31.*)$|#\1|g" tests/misc/printf-cov.pl +# TODO: moved or deleted tests/misc/printf-cov.pl +#sed -i -E "s|^([^#]*2_31.*)$|#\1|g" tests/misc/printf-cov.pl sed -i -e "s/du: invalid -t argument/du: invalid --threshold argument/" -e "s/du: option requires an argument/error: a value is required for '--threshold ' but none was supplied/" -e "/Try 'du --help' for more information./d" tests/du/threshold.sh # disable two kind of tests: # "hostid BEFORE --help" doesn't fail for GNU. we fail. we are probably doing better # "hostid BEFORE --help AFTER " same for this -sed -i -e "s/env \$prog \$BEFORE \$opt > out2/env \$prog \$BEFORE \$opt > out2 #/" -e "s/env \$prog \$BEFORE \$opt AFTER > out3/env \$prog \$BEFORE \$opt AFTER > out3 #/" -e "s/compare exp out2/compare exp out2 #/" -e "s/compare exp out3/compare exp out3 #/" tests/misc/help-version-getopt.sh +# TODO moved or deleted tests/misc/help-version-getopt.sh +#sed -i -e "s/env \$prog \$BEFORE \$opt > out2/env \$prog \$BEFORE \$opt > out2 #/" -e "s/env \$prog \$BEFORE \$opt AFTER > out3/env \$prog \$BEFORE \$opt AFTER > out3 #/" -e "s/compare exp out2/compare exp out2 #/" -e "s/compare exp out3/compare exp out3 #/" tests/misc/help-version-getopt.sh From 350b9c3d486449f592b9fd92f349b884926073cb Mon Sep 17 00:00:00 2001 From: zhitkoff Date: Fri, 25 Aug 2023 13:43:48 -0400 Subject: [PATCH 05/12] reverted commented out test files and added more details in TODO(s) --- util/build-gnu.sh | 51 +++++++++++++++++++++-------------------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/util/build-gnu.sh b/util/build-gnu.sh index c6991d15c1b..587cf858096 100755 --- a/util/build-gnu.sh +++ b/util/build-gnu.sh @@ -152,37 +152,37 @@ sed -i -e '/tests\/misc\/invalid-opt.pl/ D' \ sed -i -e '/tests\/misc\/seq-precision.sh/ D' \ Makefile +# TODO: many files and some directories modified with 'sed' in the sections below either no longer exist or have been moved +# TODO: Might need to review and updated sections below +# TODO: As a result this script will fail when executed normally as 'bash util/build-gnu.hs' due to the 'set -e' set at the beginning +# TODO: The rest of the 'sed' commands after first failure in this scenario will not be executed as bash will exit on first error +# TODO: However, the behaviour might be different when running it via GitHub actions (GnuTests) +# TODO: For now, when running in local a workaround would be to comment out the 'set -e' at the beginning of the file + # printf doesn't limit the values used in its arg, so this produced ~2GB of output -# Looks like tests/misc/printf.sh does not exist anymore - comment it out for now -#sed -i '/INT_OFLOW/ D' tests/misc/printf.sh +# TODO: this is the first one to likely fail as tests/misc/printf.sh does not exist/have been moved +sed -i '/INT_OFLOW/ D' tests/misc/printf.sh +# TODO: all commands below might not be executed # Use the system coreutils where the test fails due to error in a util that is not the one being tested -# TODO : tests/tail-2/ does not appear to exist -# and have been moved to just tests/tail/ location -# Might need to update the section below to reflect that -# Also looks like tests/misc/sort-compress-proc.sh and tests/tail-2/tail-n0f.sh and tests/misc/shuf.sh and many others do not exist anymore or moved - comment it out for now -sed -i 's|stat|/usr/bin/stat|' tests/touch/60-seconds.sh #tests/misc/sort-compress-proc.sh +sed -i 's|stat|/usr/bin/stat|' tests/touch/60-seconds.sh tests/misc/sort-compress-proc.sh sed -i 's|ls -|/usr/bin/ls -|' tests/cp/same-file.sh tests/misc/mknod.sh tests/mv/part-symlink.sh -sed -i 's|chmod |/usr/bin/chmod |' tests/du/inacc-dir.sh tests/cp/fail-perm.sh tests/mv/i-2.sh #tests/misc/shuf.sh #tests/tail-2/tail-n0f.sh -sed -i 's|sort |/usr/bin/sort |' tests/ls/hyperlink.sh #tests/misc/test-N.sh -#sed -i 's|split |/usr/bin/split |' tests/misc/factor-parallel.sh -#sed -i 's|id -|/usr/bin/id -|' tests/misc/runcon-no-reorder.sh +sed -i 's|chmod |/usr/bin/chmod |' tests/du/inacc-dir.sh tests/tail-2/tail-n0f.sh tests/cp/fail-perm.sh tests/mv/i-2.sh tests/misc/shuf.sh +sed -i 's|sort |/usr/bin/sort |' tests/ls/hyperlink.sh tests/misc/test-N.sh +sed -i 's|split |/usr/bin/split |' tests/misc/factor-parallel.sh +sed -i 's|id -|/usr/bin/id -|' tests/misc/runcon-no-reorder.sh # tests/ls/abmon-align.sh - https://github.com/uutils/coreutils/issues/3505 -sed -i 's|touch |/usr/bin/touch |' tests/cp/reflink-perm.sh tests/ls/block-size.sh tests/mv/update.sh tests/misc/time-style.sh tests/ls/abmon-align.sh #tests/misc/ls-time.sh tests/misc/stat-nanoseconds.sh tests/misc/test-N.sh +sed -i 's|touch |/usr/bin/touch |' tests/cp/reflink-perm.sh tests/ls/block-size.sh tests/mv/update.sh tests/misc/ls-time.sh tests/misc/stat-nanoseconds.sh tests/misc/time-style.sh tests/misc/test-N.sh tests/ls/abmon-align.sh sed -i 's|ln -|/usr/bin/ln -|' tests/cp/link-deref.sh sed -i 's|cp |/usr/bin/cp |' tests/mv/hard-2.sh -#sed -i 's|paste |/usr/bin/paste |' tests/misc/od-endian.sh -#sed -i 's|timeout |'"${SYSTEM_TIMEOUT}"' |' tests/tail-2/follow-stdin.sh +sed -i 's|paste |/usr/bin/paste |' tests/misc/od-endian.sh +sed -i 's|timeout |/usr/bin/timeout |' tests/tail-2/follow-stdin.sh # Add specific timeout to tests that currently hang to limit time spent waiting -# TODO : tests/misc/seq-precision.sh tests/misc/seq-long-double.sh do not appear to exist -# and have been moved to tests/seq/ location -# Might need to update the section below to reflect that, but comment it out for now -#sed -i 's|\(^\s*\)seq \$|\1'"${SYSTEM_TIMEOUT}"' 0.1 seq \$|' tests/misc/seq-precision.sh tests/misc/seq-long-double.sh +sed -i 's|\(^\s*\)seq \$|\1/usr/bin/timeout 0.1 seq \$|' tests/misc/seq-precision.sh tests/misc/seq-long-double.sh -# Remove dup of /usr/bin/ (and /usr/local/bin) when executed several times +# Remove dup of /usr/bin/ when executed several times grep -rlE '/usr/bin/\s?/usr/bin' init.cfg tests/* | xargs --no-run-if-empty sed -Ei 's|/usr/bin/\s?/usr/bin/|/usr/bin/|g' -grep -rlE '/usr/local/bin/\s?/usr/local/bin' init.cfg tests/* | xargs --no-run-if-empty sed -Ei 's|/usr/local/bin/\s?/usr/local/bin/|/usr/local/bin/|g' #### Adjust tests to make them work with Rust/coreutils # in some cases, what we are doing in rust/coreutils is good (or better) @@ -204,10 +204,7 @@ sed -i -e "s|rm: cannot remove 'rel': Permission denied|rm: cannot remove 'rel': # overlay-headers.sh test intends to check for inotify events, # however there's a bug because `---dis` is an alias for: `---disable-inotify` -# TODO : tests/tail-2/ does not appear to exist -# and have been moved to just tests/tail/ location -# Might need to update the section below to reflect that -#sed -i -e "s|---dis ||g" tests/tail-2/overlay-headers.sh +sed -i -e "s|---dis ||g" tests/tail-2/overlay-headers.sh test -f "${UU_BUILD_DIR}/getlimits" || cp src/getlimits "${UU_BUILD_DIR}" @@ -264,13 +261,11 @@ sed -i -e "s/Try 'mv --help' for more information/For more information, try '--h # GNU doesn't support width > INT_MAX # disable these test cases -# TODO: moved or deleted tests/misc/printf-cov.pl -#sed -i -E "s|^([^#]*2_31.*)$|#\1|g" tests/misc/printf-cov.pl +sed -i -E "s|^([^#]*2_31.*)$|#\1|g" tests/misc/printf-cov.pl sed -i -e "s/du: invalid -t argument/du: invalid --threshold argument/" -e "s/du: option requires an argument/error: a value is required for '--threshold ' but none was supplied/" -e "/Try 'du --help' for more information./d" tests/du/threshold.sh # disable two kind of tests: # "hostid BEFORE --help" doesn't fail for GNU. we fail. we are probably doing better # "hostid BEFORE --help AFTER " same for this -# TODO moved or deleted tests/misc/help-version-getopt.sh -#sed -i -e "s/env \$prog \$BEFORE \$opt > out2/env \$prog \$BEFORE \$opt > out2 #/" -e "s/env \$prog \$BEFORE \$opt AFTER > out3/env \$prog \$BEFORE \$opt AFTER > out3 #/" -e "s/compare exp out2/compare exp out2 #/" -e "s/compare exp out3/compare exp out3 #/" tests/misc/help-version-getopt.sh +sed -i -e "s/env \$prog \$BEFORE \$opt > out2/env \$prog \$BEFORE \$opt > out2 #/" -e "s/env \$prog \$BEFORE \$opt AFTER > out3/env \$prog \$BEFORE \$opt AFTER > out3 #/" -e "s/compare exp out2/compare exp out2 #/" -e "s/compare exp out3/compare exp out3 #/" tests/misc/help-version-getopt.sh From 70dd8eb8dcdfffa7b69a22bc9a3162408d912f84 Mon Sep 17 00:00:00 2001 From: zhitkoff Date: Sat, 26 Aug 2023 13:22:36 -0400 Subject: [PATCH 06/12] split: updates to target correct GNU coreutils release --- util/build-gnu.sh | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/util/build-gnu.sh b/util/build-gnu.sh index 587cf858096..f6be73ff470 100755 --- a/util/build-gnu.sh +++ b/util/build-gnu.sh @@ -32,10 +32,16 @@ fi ### +release_tag_GNU="v9.3" + if test ! -d "${path_GNU}"; then echo "Could not find GNU coreutils (expected at '${path_GNU}')" echo "Run the following to download into the expected path:" echo "git clone --recurse-submodules https://github.com/coreutils/coreutils.git \"${path_GNU}\"" + echo "After downloading GNU coreutils to \"${path_GNU}\" run the following commands to cheout latest release tag" + echo "cd \"${path_GNU}\"" + echo "git fetch --all --tags" + echo "git checkout tags/\"${release_tag_GNU}\"" exit 1 fi @@ -152,17 +158,8 @@ sed -i -e '/tests\/misc\/invalid-opt.pl/ D' \ sed -i -e '/tests\/misc\/seq-precision.sh/ D' \ Makefile -# TODO: many files and some directories modified with 'sed' in the sections below either no longer exist or have been moved -# TODO: Might need to review and updated sections below -# TODO: As a result this script will fail when executed normally as 'bash util/build-gnu.hs' due to the 'set -e' set at the beginning -# TODO: The rest of the 'sed' commands after first failure in this scenario will not be executed as bash will exit on first error -# TODO: However, the behaviour might be different when running it via GitHub actions (GnuTests) -# TODO: For now, when running in local a workaround would be to comment out the 'set -e' at the beginning of the file - # printf doesn't limit the values used in its arg, so this produced ~2GB of output -# TODO: this is the first one to likely fail as tests/misc/printf.sh does not exist/have been moved sed -i '/INT_OFLOW/ D' tests/misc/printf.sh -# TODO: all commands below might not be executed # Use the system coreutils where the test fails due to error in a util that is not the one being tested sed -i 's|stat|/usr/bin/stat|' tests/touch/60-seconds.sh tests/misc/sort-compress-proc.sh From eac08f72c2ef97a7d7206fb3f98fc28a447172be Mon Sep 17 00:00:00 2001 From: zhitkoff Date: Sat, 26 Aug 2023 13:24:08 -0400 Subject: [PATCH 07/12] split: double quotes --- util/build-gnu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/build-gnu.sh b/util/build-gnu.sh index f6be73ff470..74ec2967066 100755 --- a/util/build-gnu.sh +++ b/util/build-gnu.sh @@ -41,7 +41,7 @@ if test ! -d "${path_GNU}"; then echo "After downloading GNU coreutils to \"${path_GNU}\" run the following commands to cheout latest release tag" echo "cd \"${path_GNU}\"" echo "git fetch --all --tags" - echo "git checkout tags/\"${release_tag_GNU}\"" + echo "git checkout tags/${release_tag_GNU}" exit 1 fi From 4a4759c43c7cd10c81e53f9e352cd9f8f2a5fb63 Mon Sep 17 00:00:00 2001 From: zhitkoff Date: Sat, 26 Aug 2023 13:44:34 -0400 Subject: [PATCH 08/12] split: updated to SYSTEM_TIMEOUT in a few more places --- util/build-gnu.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/build-gnu.sh b/util/build-gnu.sh index 74ec2967066..4e196debe5c 100755 --- a/util/build-gnu.sh +++ b/util/build-gnu.sh @@ -173,10 +173,10 @@ sed -i 's|touch |/usr/bin/touch |' tests/cp/reflink-perm.sh tests/ls/block-size. sed -i 's|ln -|/usr/bin/ln -|' tests/cp/link-deref.sh sed -i 's|cp |/usr/bin/cp |' tests/mv/hard-2.sh sed -i 's|paste |/usr/bin/paste |' tests/misc/od-endian.sh -sed -i 's|timeout |/usr/bin/timeout |' tests/tail-2/follow-stdin.sh +sed -i 's|timeout |'"${SYSTEM_TIMEOUT}"' |' tests/tail-2/follow-stdin.sh # Add specific timeout to tests that currently hang to limit time spent waiting -sed -i 's|\(^\s*\)seq \$|\1/usr/bin/timeout 0.1 seq \$|' tests/misc/seq-precision.sh tests/misc/seq-long-double.sh +sed -i 's|\(^\s*\)seq \$|\1'"${SYSTEM_TIMEOUT}"' 0.1 seq \$|' tests/misc/seq-precision.sh tests/misc/seq-long-double.sh # Remove dup of /usr/bin/ when executed several times grep -rlE '/usr/bin/\s?/usr/bin' init.cfg tests/* | xargs --no-run-if-empty sed -Ei 's|/usr/bin/\s?/usr/bin/|/usr/bin/|g' From a384973b1a7e40435e165ab7a65a59ab26da7fd8 Mon Sep 17 00:00:00 2001 From: zhitkoff Date: Sat, 26 Aug 2023 13:51:21 -0400 Subject: [PATCH 09/12] split: remove dup for /usr/local/bin when executed several times --- util/build-gnu.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/util/build-gnu.sh b/util/build-gnu.sh index 4e196debe5c..ce15dd3f8e5 100755 --- a/util/build-gnu.sh +++ b/util/build-gnu.sh @@ -180,6 +180,7 @@ sed -i 's|\(^\s*\)seq \$|\1'"${SYSTEM_TIMEOUT}"' 0.1 seq \$|' tests/misc/seq-pre # Remove dup of /usr/bin/ when executed several times grep -rlE '/usr/bin/\s?/usr/bin' init.cfg tests/* | xargs --no-run-if-empty sed -Ei 's|/usr/bin/\s?/usr/bin/|/usr/bin/|g' +grep -rlE '/usr/local/bin/\s?/usr/local/bin' init.cfg tests/* | xargs --no-run-if-empty sed -Ei 's|/usr/local/bin/\s?/usr/local/bin/|/usr/local/bin/|g' #### Adjust tests to make them work with Rust/coreutils # in some cases, what we are doing in rust/coreutils is good (or better) From 0edba89b55ba51d974c90645079454f128c2ae6b Mon Sep 17 00:00:00 2001 From: zhitkoff Date: Sat, 26 Aug 2023 13:53:36 -0400 Subject: [PATCH 10/12] split: comments --- util/build-gnu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/build-gnu.sh b/util/build-gnu.sh index ce15dd3f8e5..57764ed5076 100755 --- a/util/build-gnu.sh +++ b/util/build-gnu.sh @@ -178,7 +178,7 @@ sed -i 's|timeout |'"${SYSTEM_TIMEOUT}"' |' tests/tail-2/follow-stdin.sh # Add specific timeout to tests that currently hang to limit time spent waiting sed -i 's|\(^\s*\)seq \$|\1'"${SYSTEM_TIMEOUT}"' 0.1 seq \$|' tests/misc/seq-precision.sh tests/misc/seq-long-double.sh -# Remove dup of /usr/bin/ when executed several times +# Remove dup of /usr/bin/ and /usr/local/bin/ when executed several times grep -rlE '/usr/bin/\s?/usr/bin' init.cfg tests/* | xargs --no-run-if-empty sed -Ei 's|/usr/bin/\s?/usr/bin/|/usr/bin/|g' grep -rlE '/usr/local/bin/\s?/usr/local/bin' init.cfg tests/* | xargs --no-run-if-empty sed -Ei 's|/usr/local/bin/\s?/usr/local/bin/|/usr/local/bin/|g' From b2ebe6a1d180fca68b5891ecf542f956e7df8331 Mon Sep 17 00:00:00 2001 From: zhitkoff Date: Wed, 30 Aug 2023 11:15:11 -0400 Subject: [PATCH 11/12] build-gnu.sh: target GNU release v9.4 --- util/build-gnu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/build-gnu.sh b/util/build-gnu.sh index a6d8de29ef6..21a231bc01d 100755 --- a/util/build-gnu.sh +++ b/util/build-gnu.sh @@ -32,7 +32,7 @@ fi ### -release_tag_GNU="v9.3" +release_tag_GNU="v9.4" if test ! -d "${path_GNU}"; then echo "Could not find GNU coreutils (expected at '${path_GNU}')" From 843540d05fafe427ed34d7b8b546357d327d3df5 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Thu, 31 Aug 2023 09:21:59 +0200 Subject: [PATCH 12/12] fix a typo --- util/build-gnu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/build-gnu.sh b/util/build-gnu.sh index 21a231bc01d..157265f61bf 100755 --- a/util/build-gnu.sh +++ b/util/build-gnu.sh @@ -38,7 +38,7 @@ if test ! -d "${path_GNU}"; then echo "Could not find GNU coreutils (expected at '${path_GNU}')" echo "Run the following to download into the expected path:" echo "git clone --recurse-submodules https://github.com/coreutils/coreutils.git \"${path_GNU}\"" - echo "After downloading GNU coreutils to \"${path_GNU}\" run the following commands to cheout latest release tag" + echo "After downloading GNU coreutils to \"${path_GNU}\" run the following commands to checkout latest release tag" echo "cd \"${path_GNU}\"" echo "git fetch --all --tags" echo "git checkout tags/${release_tag_GNU}"