From 4f9253e9d549aae04e8ca311140d342d95f28e89 Mon Sep 17 00:00:00 2001 From: Francois Laupretre Date: Mon, 21 May 2018 12:49:44 +0200 Subject: [PATCH 1/4] Add bats_test_succeeds() function --- libexec/bats-exec-test | 32 ++++++++++++++++++++++++++++++++ libexec/bats-preprocess | 14 +++++++++----- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/libexec/bats-exec-test b/libexec/bats-exec-test index 8f3bd510..f2dab896 100755 --- a/libexec/bats-exec-test +++ b/libexec/bats-exec-test @@ -28,6 +28,7 @@ fi BATS_TEST_DIRNAME="$(dirname "$BATS_TEST_FILENAME")" BATS_TEST_NAMES=() +BATS_TEST_RESULTS=() load() { local name="$1" @@ -267,12 +268,43 @@ bats_exit_trap() { exit "$status" } +bats_test_succeeds() { + local name status index i ret + + ret=0 + + for name in $@ + do + index='' + i=0 + while [ $i -lt ${#BATS_TEST_NAMES[@]} ; do + if [ "${BATS_TEST_NAMES[$i]}" = "test_$name" ] ; then + index=$i + break + fi + done + if [ -z $index ] ; then + echo "Error: $name: this test name does not exist" >&2 + else + status=${BATS_TEST_RESULTS[$index]} + if [ -z $status ] ; then + echo "Warning: $name: Cannot check status as test didn't run yet" + else + let ret+=$status + fi + fi + done + + return $ret +} + bats_perform_tests() { echo "1..$#" test_number=1 status=0 for test_name in "$@"; do "$0" $BATS_EXTENDED_SYNTAX "$BATS_TEST_FILENAME" "$test_name" "$test_number" || status=1 + BATS_TEST_RESULTS[$test_number]=$status let test_number+=1 done exit "$status" diff --git a/libexec/bats-preprocess b/libexec/bats-preprocess index 04297ed0..ef6546f6 100755 --- a/libexec/bats-preprocess +++ b/libexec/bats-preprocess @@ -31,15 +31,19 @@ encode_name() { tests=() index=0 -pattern='^ *@test *([^ ].*) *\{ *(.*)$' +pattern='^ *@test(\[([^ ])[^ ]*\])? *([^ ].*) *\{ *(.*)$' while IFS= read -r line; do let index+=1 if [[ "$line" =~ $pattern ]]; then - quoted_name="${BASH_REMATCH[1]}" - body="${BASH_REMATCH[2]}" - name="$(eval echo "$quoted_name")" - encoded_name="$(encode_name "$name")" + quoted_name="${BASH_REMATCH[3]}" + if [ -n "${BASH_REMATCH[2]}" ; then + encoded_name="test_${BASH_REMATCH[2]}" + else + name="$(eval echo "$quoted_name")" + encoded_name="$(encode_name "$name")" + fi + body="${BASH_REMATCH[4]}" tests["${#tests[@]}"]="$encoded_name" echo "${encoded_name}() { bats_test_begin ${quoted_name} ${index}; ${body}" else From ac619feb581aae861a7a40e0f43d1da62e271992 Mon Sep 17 00:00:00 2001 From: Francois Laupretre Date: Mon, 21 May 2018 14:02:05 +0200 Subject: [PATCH 2/4] Improve README.md --- README.md | 71 +++++++++++++++++++++++++++++++++++++++++- libexec/bats-exec-test | 2 +- 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 235bf1ee..5c1f4baf 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,73 @@ Or you can skip conditionally: } ``` +### Explicit test names + +In most cases, tests don't need to be assigned an explicit name, but some features, +like dependencies described below, reference tests by their names. In such cases, +the test(s) you reference must be assigned an explicit name. + +An explicit name is defined using an extension to the base +syntax: replace '@test' by '@test[]'. + +Example: + +```bash +@test[mytest1] "Test description..." { +... +``` + +Notes: + +* Every test may define an explicit name, even if it not referenced anywhere, +* In a given test file, each explicit name must be unique. You cannot assign the +same name to different tests, +* Explicit names must be composed from alphanumeric characters + underscore +('_') only. + +### Test dependencies + +Some tests may have one or more tests as pre-requisites, meaning that tests +are irrelevant if the pre-requisites fail. A common case is test A checking that a +given service is running, while test B and C use this service to retrieve more +information. If service is not running, we know for sure that test B and C will +fail. In order to avoid a bunch of errors when such dependency exists, you can +instruct bash to skip test B and C if A fails. + +In order to implement such a dependency, you will use the 'bats_test_succeeds()' +function. This function returns true (0) if all its argument every test names +succeeded, and false if any of them failed. + +Example: + +```bash +@test[testA] "A pre-requisite check" { + service foo status +} + +@test[testAbis] "Oh, I need this too!" { + service bar status +} + +# I don't want to run this if pre-requisites are unavailable + +@test "Let's check this more in depth..." { + bats_test_succeeds testA testAbis || skip "Pre-requisites are unavailable" + # In-depth check starts here + ... +} +``` + +Notes: + +* You cannot define cross-file dependencies. The test(s) you refer too must +be located in the same test file. +* As tests are executed in the order they appear in the file, you can only depend +on tests that appear in the test file BEFORE/ABOVE the current test. If this is +not the case, a warning message is issued on stderr and the parent test is +supposed to be successful (depending test will run). +* You need to set explicit names for the tests you want to create dependencies on. + ### `setup` and `teardown`: Pre- and post-test hooks You can define special `setup` and `teardown` functions, which run @@ -199,10 +266,12 @@ test case. case. * `$BATS_TEST_NUMBER` is the (1-based) index of the current test case in the test file. +* `BATS_TEST_RESULTS` is an array of results for tests that ran so far. +Only elements from 1 to $BATS_TEST_NUMBER (excluded) are set. Values are +0 if test succeeded, and different from 0 if test failed. * `$BATS_TMPDIR` is the location to a directory that may be used to store temporary files. - ## Installing Bats from source Check out a copy of the Bats repository. Then, either add the Bats diff --git a/libexec/bats-exec-test b/libexec/bats-exec-test index f2dab896..5f9e2344 100755 --- a/libexec/bats-exec-test +++ b/libexec/bats-exec-test @@ -288,7 +288,7 @@ bats_test_succeeds() { else status=${BATS_TEST_RESULTS[$index]} if [ -z $status ] ; then - echo "Warning: $name: Cannot check status as test didn't run yet" + echo "Warning: $name: Cannot check status as test didn't run yet" >&2 else let ret+=$status fi From 3e28a098ad33fdea04cd172008b80d43033ea461 Mon Sep 17 00:00:00 2001 From: Francois Laupretre Date: Mon, 21 May 2018 14:59:00 +0200 Subject: [PATCH 3/4] Make dependencies transitive/inherited --- README.md | 16 +++++++++------- libexec/bats-exec-test | 13 +++++++++---- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 5c1f4baf..f71ec700 100644 --- a/README.md +++ b/README.md @@ -195,14 +195,14 @@ same name to different tests, ### Test dependencies Some tests may have one or more tests as pre-requisites, meaning that tests -are irrelevant if the pre-requisites fail. A common case is test A checking that a -given service is running, while test B and C use this service to retrieve more -information. If service is not running, we know for sure that test B and C will -fail. In order to avoid a bunch of errors when such dependency exists, you can -instruct bash to skip test B and C if A fails. +are irrelevant if the pre-requisites fail. A common case is test A checking that +service 'foo' is running, while test B and C use this service to retrieve more +information. If service 'foo' is not running, we know for sure that tests B and C will +fail. In such cases, readability is much better if we display test A as failed +and tests B and C as skipped. In order to implement such a dependency, you will use the 'bats_test_succeeds()' -function. This function returns true (0) if all its argument every test names +function. This function returns true (0) if all the test names given as arguments succeeded, and false if any of them failed. Example: @@ -227,12 +227,14 @@ Example: Notes: -* You cannot define cross-file dependencies. The test(s) you refer too must +* You cannot define cross-file dependencies. The test(s) you refer to must be located in the same test file. * As tests are executed in the order they appear in the file, you can only depend on tests that appear in the test file BEFORE/ABOVE the current test. If this is not the case, a warning message is issued on stderr and the parent test is supposed to be successful (depending test will run). +* Dependencies are transitive/inherited. If a test depends on a test skipped for any reason, +it will be skipped too, along with its descendants. * You need to set explicit names for the tests you want to create dependencies on. ### `setup` and `teardown`: Pre- and post-test hooks diff --git a/libexec/bats-exec-test b/libexec/bats-exec-test index 5f9e2344..7320621e 100755 --- a/libexec/bats-exec-test +++ b/libexec/bats-exec-test @@ -241,7 +241,7 @@ bats_teardown_trap() { } bats_exit_trap() { - local status + local status # Exit code: 0=OK, 1=error, 2=skipped local skipped trap - err exit @@ -262,6 +262,7 @@ bats_exit_trap() { else echo "ok ${BATS_TEST_NUMBER}${skipped} ${BATS_TEST_DESCRIPTION}" >&3 status=0 + [ -n "$BATS_TEST_SKIPPED" ] && status=2 fi rm -f "$BATS_OUT" @@ -290,7 +291,7 @@ bats_test_succeeds() { if [ -z $status ] ; then echo "Warning: $name: Cannot check status as test didn't run yet" >&2 else - let ret+=$status + [ $status = 0 ] && ret=1 fi fi done @@ -299,15 +300,19 @@ bats_test_succeeds() { } bats_perform_tests() { + local ret status test_number test_name + + ret=0 echo "1..$#" test_number=1 status=0 for test_name in "$@"; do - "$0" $BATS_EXTENDED_SYNTAX "$BATS_TEST_FILENAME" "$test_name" "$test_number" || status=1 + "$0" $BATS_EXTENDED_SYNTAX "$BATS_TEST_FILENAME" "$test_name" "$test_number" || status=$? + [ $status = 1 ] && ret=1 # Skip -> OK BATS_TEST_RESULTS[$test_number]=$status let test_number+=1 done - exit "$status" + exit "$ret" } bats_perform_test() { From ffaf48f3b7c2c0f6892a6c80bc72b59396efa7c6 Mon Sep 17 00:00:00 2001 From: Francois Laupretre Date: Tue, 8 Jan 2019 10:49:27 +0100 Subject: [PATCH 4/4] Update bats-preprocess --- libexec/bats-preprocess | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libexec/bats-preprocess b/libexec/bats-preprocess index ef6546f6..81fae9ce 100755 --- a/libexec/bats-preprocess +++ b/libexec/bats-preprocess @@ -37,7 +37,7 @@ while IFS= read -r line; do let index+=1 if [[ "$line" =~ $pattern ]]; then quoted_name="${BASH_REMATCH[3]}" - if [ -n "${BASH_REMATCH[2]}" ; then + if [ -n "${BASH_REMATCH[2]}" ] ; then encoded_name="test_${BASH_REMATCH[2]}" else name="$(eval echo "$quoted_name")"