From 785a3f2d777627f39bed44f4ae7a0180d5184109 Mon Sep 17 00:00:00 2001 From: Tavian Barnes Date: Thu, 19 Oct 2023 16:37:47 -0400 Subject: tests: Refactor implementation into separate files --- tests/getopts.sh | 158 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 tests/getopts.sh (limited to 'tests/getopts.sh') diff --git a/tests/getopts.sh b/tests/getopts.sh new file mode 100644 index 0000000..6616a4a --- /dev/null +++ b/tests/getopts.sh @@ -0,0 +1,158 @@ +#!/hint/bash + +# Copyright © Tavian Barnes +# SPDX-License-Identifier: 0BSD + +## Argument parsing + +# Print usage information +usage() { + local pad=$(printf "%*s" ${#0} "") + color <&2 + usage >&2 + exit 1 + ;; + *) + PATTERNS+=("$arg") + ;; + esac + done + + # Try to resolve the path to $BFS before we cd, while also supporting + # --bfs="./bin/bfs -S ids" + read -a BFS <<<"${BFS:-$BIN/bfs}" + BFS[0]=$(_realpath "$(command -v "${BFS[0]}")") + + if ((${#PATTERNS[@]} == 0)); then + PATTERNS=("*") + fi + + TEST_CASES=() + ALL_TESTS=($(cd "$TESTS" && quote {posix,common,bsd,gnu,bfs}/*.sh)) + for TEST in "${ALL_TESTS[@]}"; do + TEST="${TEST%.sh}" + for PATTERN in "${PATTERNS[@]}"; do + if [[ $TEST == $PATTERN ]]; then + TEST_CASES+=("$TEST") + break + fi + done + done + + if ((${#TEST_CASES[@]} == 0)); then + cprintf "${RED}error:${RST} No tests matched" >&2 + cprintf " ${BLD}%s${RST}" "${PATTERNS[@]}" >&2 + cprintf ".\n\n" >&2 + usage >&2 + exit 1 + fi +} -- cgit v1.2.3 From 60366e4583a1d148dd5f8171c9148ebb98890478 Mon Sep 17 00:00:00 2001 From: Tavian Barnes Date: Fri, 20 Oct 2023 15:52:45 -0400 Subject: tests/color: Remove some useless cats --- tests/color.sh | 12 +++------- tests/getopts.sh | 10 ++++---- tests/run.sh | 73 +++++++++++++++++++++++++++++--------------------------- tests/util.sh | 8 +++---- 4 files changed, 50 insertions(+), 53 deletions(-) (limited to 'tests/getopts.sh') diff --git a/tests/color.sh b/tests/color.sh index 0d6ef68..805d2b8 100644 --- a/tests/color.sh +++ b/tests/color.sh @@ -24,20 +24,14 @@ color_fd() { color_fd 1 && COLOR_STDOUT=1 || COLOR_STDOUT=0 color_fd 2 && COLOR_STDERR=1 || COLOR_STDERR=0 -# Save these in case the tests unset PATH -CAT=$(command -v cat) +# Save this in case the tests unset PATH SED=$(command -v sed) # Filter out escape sequences if necessary color() { if color_fd 1; then - "$CAT" + "$@" else - "$SED" $'s/\e\\[[^m]*m//g' + "$@" | "$SED" $'s/\e\\[[^m]*m//g' fi } - -# printf with auto-detected color support -cprintf() { - printf "$@" | color -} diff --git a/tests/getopts.sh b/tests/getopts.sh index 6616a4a..7d3ef4b 100644 --- a/tests/getopts.sh +++ b/tests/getopts.sh @@ -8,7 +8,7 @@ # Print usage information usage() { local pad=$(printf "%*s" ${#0} "") - color <&2 + color printf "${RED}error:${RST} Unrecognized option '%s'.\n\n" "$arg" >&2 usage >&2 exit 1 ;; @@ -149,9 +149,9 @@ parse_args() { done if ((${#TEST_CASES[@]} == 0)); then - cprintf "${RED}error:${RST} No tests matched" >&2 - cprintf " ${BLD}%s${RST}" "${PATTERNS[@]}" >&2 - cprintf ".\n\n" >&2 + color printf "${RED}error:${RST} No tests matched" >&2 + color printf " ${BLD}%s${RST}" "${PATTERNS[@]}" >&2 + color printf ".\n\n" >&2 usage >&2 exit 1 fi diff --git a/tests/run.sh b/tests/run.sh index 70c9cc2..5fcccad 100644 --- a/tests/run.sh +++ b/tests/run.sh @@ -93,7 +93,7 @@ run_tests() { else ((++failed)) ((VERBOSE_ERRORS)) || cat "$TMP/$TEST.err" >&2 - cprintf "${BOL}${RED}%s failed!${RST}\n" "$TEST" + color printf "${BOL}${RED}%s failed!${RST}\n" "$TEST" ((STOP)) && break fi done @@ -101,13 +101,13 @@ run_tests() { printf "${BOL}" if ((passed > 0)); then - cprintf "${GRN}tests passed: %d${RST}\n" "$passed" + color printf "${GRN}tests passed: %d${RST}\n" "$passed" fi if ((skipped > 0)); then - cprintf "${CYN}tests skipped: %s${RST}\n" "$skipped" + color printf "${CYN}tests skipped: %s${RST}\n" "$skipped" fi if ((failed > 0)); then - cprintf "${RED}tests failed: %s${RST}\n" "$failed" + color printf "${RED}tests failed: %s${RST}\n" "$failed" exit 1 fi } @@ -126,7 +126,7 @@ skip() { debug "$file" $line "${CYN}$TEST skipped!${RST}" "$(awk "NR == $line" "$file")" >&3 } elif ((VERBOSE_TESTS)); then - cprintf "${BOL}${CYN}%s skipped!${RST}\n" "$TEST" + color printf "${BOL}${CYN}%s skipped!${RST}\n" "$TEST" fi exit $EX_SKIP @@ -175,39 +175,42 @@ set_acl() { } # Print a bfs invocation for --verbose=commands -bfs_verbose() ( - if ((!VERBOSE_COMMANDS)); then - return +bfs_verbose() { + if ((VERBOSE_COMMANDS)); then + ( + # Close some fds to make room for the pipe, + # even with extremely low ulimit -n + exec >&- 4>&- + exec >&3 3>&- + color bfs_verbose_impl "$@" + ) fi +} - # Free up an fd for the pipe - exec 4>&- - - { - printf "${GRN}%q${RST} " "${BFS[@]}" - - local expr_started= - for arg; do - if [[ $arg == -[A-Z]* ]]; then - printf "${CYN}%q${RST} " "$arg" - elif [[ $arg == [\(!] || $arg == -[ao] || $arg == -and || $arg == -or || $arg == -not ]]; then - expr_started=yes - printf "${RED}%q${RST} " "$arg" - elif [[ $expr_started && $arg == [\),] ]]; then - printf "${RED}%q${RST} " "$arg" - elif [[ $arg == -?* ]]; then - expr_started=yes - printf "${BLU}%q${RST} " "$arg" - elif [ "$expr_started" ]; then - printf "${BLD}%q${RST} " "$arg" - else - printf "${MAG}%q${RST} " "$arg" - fi - done +bfs_verbose_impl() { + printf "${GRN}%q${RST} " "${BFS[@]}" + + local expr_started= + for arg; do + if [[ $arg == -[A-Z]* ]]; then + printf "${CYN}%q${RST} " "$arg" + elif [[ $arg == [\(!] || $arg == -[ao] || $arg == -and || $arg == -or || $arg == -not ]]; then + expr_started=yes + printf "${RED}%q${RST} " "$arg" + elif [[ $expr_started && $arg == [\),] ]]; then + printf "${RED}%q${RST} " "$arg" + elif [[ $arg == -?* ]]; then + expr_started=yes + printf "${BLU}%q${RST} " "$arg" + elif [ "$expr_started" ]; then + printf "${BLD}%q${RST} " "$arg" + else + printf "${MAG}%q${RST} " "$arg" + fi + done - printf '\n' - } | color >&3 -) + printf '\n' +} # Run the bfs we're testing invoke_bfs() { diff --git a/tests/util.sh b/tests/util.sh index efc24b0..31a7b6c 100644 --- a/tests/util.sh +++ b/tests/util.sh @@ -76,14 +76,14 @@ drop_root() { if command -v capsh &>/dev/null; then if capsh --has-p=cap_dac_override &>/dev/null || capsh --has-p=cap_dac_read_search &>/dev/null; then if [ -n "${BFS_TRIED_DROP:-}" ]; then - color >&2 <&2 <&2 <&2 <&2 <&2 < Date: Mon, 23 Oct 2023 10:12:25 -0400 Subject: tests: Run test cases in parallel --- GNUmakefile | 7 ++++++- tests/getopts.sh | 12 +++++++++++ tests/run.sh | 61 ++++++++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 64 insertions(+), 16 deletions(-) (limited to 'tests/getopts.sh') diff --git a/GNUmakefile b/GNUmakefile index 45797e2..62f24fb 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -287,8 +287,13 @@ check: $(CHECKS) $(UNIT_CHECKS): check-%: $(BIN)/tests/% $< +JOBS := $(filter -j%,$(MAKEFLAGS)) +ifndef JOBS + JOBS := -j1 +endif + $(STRATEGY_CHECKS): check-%: $(BIN)/bfs $(TEST_UTILS) - ./tests/tests.sh --bfs="$(BIN)/bfs -S $*" $(TEST_FLAGS) + ./tests/tests.sh $(JOBS) --bfs="$(BIN)/bfs -S $*" $(TEST_FLAGS) # Custom test flags for distcheck DISTCHECK_FLAGS := -s TEST_FLAGS="--sudo --verbose=skipped" diff --git a/tests/getopts.sh b/tests/getopts.sh index 7d3ef4b..d34df4f 100644 --- a/tests/getopts.sh +++ b/tests/getopts.sh @@ -52,6 +52,7 @@ EOF # Parse the command line parse_args() { + JOBS=0 PATTERNS=() SUDO=() STOP=0 @@ -64,6 +65,9 @@ parse_args() { for arg; do case "$arg" in + -j*) + JOBS="${arg#-j}" + ;; --bfs=*) BFS="${arg#*=}" ;; @@ -127,6 +131,14 @@ parse_args() { esac done + if ((JOBS == 0)); then + if command -v nproc &>/dev/null; then + JOBS=$(nproc) + else + JOBS=1 + fi + fi + # Try to resolve the path to $BFS before we cd, while also supporting # --bfs="./bin/bfs -S ids" read -a BFS <<<"${BFS:-$BIN/bfs}" diff --git a/tests/run.sh b/tests/run.sh index 28801f4..b46fde6 100644 --- a/tests/run.sh +++ b/tests/run.sh @@ -42,6 +42,42 @@ run_test() ( source "$@" ) +# Run a test in the background +bg_test() { + if ((VERBOSE_ERRORS)); then + run_test "$1" + else + run_test "$1" 2>"$TMP/TEST.err" + fi + ret=$? + + if ((ret != 0 && ret != EX_SKIP)); then + ((VERBOSE_ERRORS)) || cat "$TMP/$TEST.err" >&2 + color printf "${BOL}${RED}%s failed!${RST}\n" "$TEST" + fi + + return $ret +} + +# Reap a background job +reap() { + wait -n + ret=$? + ((BG--)) + + case $ret in + 0) + ((++passed)) + ;; + $EX_SKIP) + ((++skipped)) + ;; + *) + ((++failed)) + ;; + esac +} + # Run all the tests run_tests() { if ((VERBOSE_TESTS)); then @@ -70,6 +106,8 @@ run_tests() { TEST_FMT="." fi + BG=0 + # Turn off set -e (but turn it back on in run_test) set +e @@ -79,23 +117,16 @@ run_tests() { mkdir -p "$TMP/$TEST" OUT="$TMP/$TEST.out" - if ((VERBOSE_ERRORS)); then - run_test "$TESTS/$TEST.sh" - else - run_test "$TESTS/$TEST.sh" 2>"$TMP/$TEST.err" + if ((BG >= JOBS)); then + reap fi - status=$? + ((++BG)) - if ((status == 0)); then - ((++passed)) - elif ((status == EX_SKIP)); then - ((++skipped)) - else - ((++failed)) - ((VERBOSE_ERRORS)) || cat "$TMP/$TEST.err" >&2 - color printf "${BOL}${RED}%s failed!${RST}\n" "$TEST" - ((STOP)) && break - fi + bg_test "$TESTS/$TEST.sh" & + done + + while ((BG > 0)); do + reap done printf "${BOL}" -- cgit v1.2.3 From 62415c0d1530f271ce9d417fa075ac00b9473f1e Mon Sep 17 00:00:00 2001 From: Tavian Barnes Date: Mon, 23 Oct 2023 13:05:15 -0400 Subject: tests: Document -j --- tests/getopts.sh | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) (limited to 'tests/getopts.sh') diff --git a/tests/getopts.sh b/tests/getopts.sh index d34df4f..ac75140 100644 --- a/tests/getopts.sh +++ b/tests/getopts.sh @@ -5,14 +5,32 @@ ## Argument parsing +if command -v nproc &>/dev/null; then + JOBS=$(nproc) +else + JOBS=1 +fi +PATTERNS=() +SUDO=() +STOP=0 +CLEAN=1 +UPDATE=0 +VERBOSE_COMMANDS=0 +VERBOSE_ERRORS=0 +VERBOSE_SKIPPED=0 +VERBOSE_TESTS=0 + # Print usage information usage() { local pad=$(printf "%*s" ${#0} "") color cat </dev/null; then - JOBS=$(nproc) - else - JOBS=1 - fi - fi - # Try to resolve the path to $BFS before we cd, while also supporting # --bfs="./bin/bfs -S ids" read -a BFS <<<"${BFS:-$BIN/bfs}" -- cgit v1.2.3 From 4d0d84f935159c395ccf3b95d5727a3553003b68 Mon Sep 17 00:00:00 2001 From: Tavian Barnes Date: Thu, 1 Feb 2024 12:11:41 -0500 Subject: tests: Implement jobserver inheritance --- .github/workflows/ci.yml | 12 ++--- GNUmakefile | 7 +-- tests/getopts.sh | 22 ++++++--- tests/run.sh | 119 +++++++++++++++++++++++++++++++++++++---------- tests/tests.mk | 7 +++ tests/util.sh | 12 +++-- 6 files changed, 132 insertions(+), 47 deletions(-) create mode 100644 tests/tests.mk (limited to 'tests/getopts.sh') diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e96526..5ca0a75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: - name: Run tests run: | jobs=$(sysctl -n hw.ncpu) - make -j$jobs distcheck JOBS=-j$jobs + make -j$jobs distcheck freebsd: name: FreeBSD @@ -90,7 +90,7 @@ jobs: run: | chown -R action:action . - sudo -u action gmake -j$(nproc) distcheck JOBS=-j$(nproc) + sudo -u action gmake -j$(nproc) distcheck openbsd: name: OpenBSD @@ -120,7 +120,7 @@ jobs: run: | chown -R action:action . jobs=$(sysctl -n hw.ncpu) - doas -u action gmake -j$jobs check JOBS=-j$jobs TEST_FLAGS="--sudo=doas --verbose=skipped" + doas -u action gmake -j$jobs check TEST_FLAGS="--sudo=doas --verbose=skipped" netbsd: name: NetBSD @@ -153,7 +153,7 @@ jobs: PATH="/sbin:/usr/sbin:$PATH" chown -R action:action . jobs=$(sysctl -n hw.ncpu) - sudo -u action gmake -j$jobs check CC=clang LDFLAGS="-rpath /usr/pkg/lib" JOBS=-j$jobs TEST_FLAGS="--sudo --verbose=skipped" + sudo -u action gmake -j$jobs check CC=clang LDFLAGS="-rpath /usr/pkg/lib" TEST_FLAGS="--sudo --verbose=skipped" dragonflybsd: name: DragonFly BSD @@ -184,7 +184,7 @@ jobs: run: | chown -R action:action . jobs=$(sysctl -n hw.ncpu) - sudo -u action gmake -j$jobs check JOBS=-j$jobs TEST_FLAGS="--sudo --verbose=skipped" + sudo -u action gmake -j$jobs check TEST_FLAGS="--sudo --verbose=skipped" omnios: name: OmniOS @@ -216,4 +216,4 @@ jobs: PATH="/usr/xpg4/bin:$PATH" chown -R action:staff . jobs=$(getconf NPROCESSORS_ONLN) - sudo -u action gmake -j$jobs check LDFLAGS="-Wl,-rpath,/opt/ooce/lib/amd64" JOBS=-j$jobs TEST_FLAGS="--sudo --verbose=skipped" + sudo -u action gmake -j$jobs check LDFLAGS="-Wl,-rpath,/opt/ooce/lib/amd64" TEST_FLAGS="--sudo --verbose=skipped" diff --git a/GNUmakefile b/GNUmakefile index 34c7f37..e01e93a 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -304,13 +304,8 @@ check: $(CHECKS) check-units: $(BIN)/tests/units $< -JOBS := $(filter -j%,$(MAKEFLAGS)) -ifndef JOBS - JOBS := -j1 -endif - $(STRATEGY_CHECKS): check-%: $(BIN)/bfs $(TEST_UTILS) - ./tests/tests.sh $(JOBS) --bfs="$(BIN)/bfs -S $*" $(TEST_FLAGS) + +./tests/tests.sh --make="$(MAKE)" --bfs="$(BIN)/bfs -S $*" $(TEST_FLAGS) # Custom test flags for distcheck DISTCHECK_FLAGS := -s TEST_FLAGS="--sudo --verbose=skipped" diff --git a/tests/getopts.sh b/tests/getopts.sh index ac75140..5214e9f 100644 --- a/tests/getopts.sh +++ b/tests/getopts.sh @@ -10,6 +10,7 @@ if command -v nproc &>/dev/null; then else JOBS=1 fi +MAKE= PATTERNS=() SUDO=() STOP=0 @@ -24,15 +25,19 @@ VERBOSE_TESTS=0 usage() { local pad=$(printf "%*s" ${#0} "") color cat </dev/null)" || : - debug "$file" $line "${RED}error $ret${RST}" "$cmd" >&$DUPERR + debug "$file" $line "${RED}error $ret${RST}" >&$DUPERR break fi done } -# Run a single test -run_test() ( +# Source a test +source_test() ( set -eE trap debug_err ERR + + if ((${#MAKE[@]})); then + # Close the jobserver pipes + exec {READY_PIPE}<&- {DONE_PIPE}>&- + fi + cd "$TMP" source "$@" ) -# Run a test in the background -bg_test() { +# Run a test +run_test() { if ((VERBOSE_ERRORS)); then - run_test "$1" + source_test "$1" else - run_test "$1" 2>"$TMP/$TEST.err" + source_test "$1" 2>"$TMP/$TEST.err" fi ret=$? + if ((${#MAKE[@]})); then + # Write one byte to the done pipe + printf . >&$DONE_PIPE + fi + case $ret in 0) if ((VERBOSE_TESTS)); then @@ -73,28 +83,87 @@ bg_test() { return $ret } -# Wait for a background test to finish -wait_test() { - wait -n - ret=$? +# Count the tests running in the background +BG=0 + +# Run a test in the background +bg_test() { + run_test "$1" & + ((++BG)) +} + +# Reap a finished background test +reap_test() { ((BG--)) - case $ret in + case "$1" in 0) ((++passed)) - return 0 ;; $EX_SKIP) ((++skipped)) - return 0 ;; *) ((++failed)) - return $ret ;; esac } +# Wait for a background test to finish +wait_test() { + local pid + wait -n -ppid + ret=$? + if [ -z "${pid:-}" ]; then + debug "${BASH_SOURCE[0]}" $((LINENO - 3)) "${RED}error $ret${RST}" >&$DUPERR + exit 1 + fi + + reap_test $ret +} + +# Wait until we're ready to run another test +wait_ready() { + if ((${#MAKE[@]})); then + # We'd like to parse the output of jobs -n, but we can't run it in a + # subshell or we won't get the right output + jobs -n >"$TMP/jobs" + while read -r job status ret foo; do + case "$status" in + Done) + reap_test 0 + ;; + Exit) + reap_test $ret + ;; + esac + done <"$TMP/jobs" + + # Read one byte from the ready pipe + read -r -N1 -u$READY_PIPE + elif ((BG >= JOBS)); then + wait_test + fi +} + +# Run make as a co-process to use its job control +comake() { + coproc { + # We can't just use std{in,out}, due to + # https://www.gnu.org/software/make/manual/html_node/Parallel-Input.html + exec {DONE_PIPE}<&0 {READY_PIPE}>&1 + exec "${MAKE[@]}" -s \ + -f "$TESTS/tests.mk" \ + DONE=$DONE_PIPE \ + READY=$READY_PIPE \ + "${TEST_CASES[@]/#/tests/}" \ + /dev/null + } + + # coproc pipes aren't inherited by subshells, so dup them + exec {READY_PIPE}<&${COPROC[0]} {DONE_PIPE}>&${COPROC[1]} +} + # Run all the tests run_tests() { if ((VERBOSE_TESTS)); then @@ -125,17 +194,17 @@ run_tests() { TEST_FMT="." fi - BG=0 + if ((${#MAKE[@]})); then + comake + fi # Turn off set -e (but turn it back on in run_test) set +e for TEST in "${TEST_CASES[@]}"; do - if ((BG >= JOBS)); then - wait_test - if (($? && STOP)); then - break - fi + wait_ready + if (($? && STOP)); then + break fi percent=$((100 * ran / total)) @@ -144,8 +213,8 @@ run_tests() { mkdir -p "$TMP/$TEST" OUT="$TMP/$TEST.out" - bg_test "$TESTS/$TEST.sh" & - ((++BG, ++ran)) + bg_test "$TESTS/$TEST.sh" + ((++ran)) done while ((BG > 0)); do @@ -185,7 +254,7 @@ skip() { caller | { read -r line file printf "${BOL}" - debug "$file" $line "" "$(awk "NR == $line" "$file")" >&$DUPOUT + debug "$file" $line "" >&$DUPOUT } fi diff --git a/tests/tests.mk b/tests/tests.mk new file mode 100644 index 0000000..5bf4f6c --- /dev/null +++ b/tests/tests.mk @@ -0,0 +1,7 @@ +# Copyright © Tavian Barnes +# SPDX-License-Identifier: 0BSD + +# GNU makefile that exposes make's job control to tests.sh + +tests/%: + bash -c 'printf . >&$(READY) && read -r -N1 -u$(DONE)' diff --git a/tests/util.sh b/tests/util.sh index ec24958..7dba9fb 100644 --- a/tests/util.sh +++ b/tests/util.sh @@ -113,9 +113,13 @@ callers() { # Print a message including path, line number, and command debug() { - local file="${1/#*\/tests\//tests/}" - set -- "$file" "${@:2}" - color printf "${BLD}%s:%d:${RST} %s\n %s\n" "$@" + local file="$1" + local line="$2" + local msg="$3" + local cmd="$(awk "NR == $line" "$file" 2>/dev/null)" || : + file="${file/#*\/tests\//tests/}" + + color printf "${BLD}%s:%d:${RST} %s\n %s\n" "$file" "$line" "$msg" "$cmd" } ## Deferred cleanup @@ -163,7 +167,7 @@ pop_defer() { eval "$cmd" || ret=$? if ((ret != 0)); then - debug "$file" $line "${RED}error $ret${RST}" "defer $cmd" >&$DUPERR + debug "$file" $line "${RED}error $ret${RST}" >&$DUPERR fi return $ret -- cgit v1.2.3