Add a --negative-filter option (bats-core#1114)

AdamWill · martin-schulze-vireso · commit 9e6627ebfa24 · 2025-10-11T22:58:13.000+02:00
See https://pagure.io/fedora-qa/os-autoinst-distri-fedora/pull-request/389 . The FAQ claimed "So you could try to invert the regex", but as far as I can tell, there's no practical way to do this with bash regexes, if you want to express "run all tests except foo_in_a" or something like that. You can do it in some regex flavors using negative lookahead matches, but bash's regex flavor does not seem to support that, and you can't just stick a `!` on the front of the regex or anything like that. So, this adds --negative-filter which just does the opposite of --filter. The implementation is intentionally 'dumb' because handling 'what if one or both is not defined?' gets a bit awkward if you try to handle both in one function. Signed-off-by: Adam Williamson <awilliam@redhat.com>
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -14,6 +14,7 @@ The format is based on [Keep a Changelog][kac] and this project adheres to
 ### Added 
 
 * use the [`syntax`](https://docs.docker.com/reference/dockerfile/#syntax) parser directive to declare the Dockerfile syntax version (#1127)
+* Negative test filtering via `--negative-filter` - tests matching the filter are *excluded* (#1114)
 
 ### Fixed
 
diff --git a/docs/source/faq.rst b/docs/source/faq.rst
@@ -18,8 +18,7 @@ If you want to see this output, you need to print it yourself, or use functions
 Can I use `--filter` to exclude files/tests?
 --------------------------------------------
 
-No, not directly. `--filter` uses a regex to match against test names. So you could try to invert the regex.
-The filename won't be part of the strings that are tested, so you cannot filter against files.
+You can use `--negative-filter` to do this.
 
 How can I exclude a single test from a test run?
 ------------------------------------------------
diff --git a/libexec/bats-core/bats b/libexec/bats-core/bats
@@ -47,6 +47,7 @@ HELP_TEXT_HEADER
                               - custom: provide your own via defining bats_format_file_line_reference_custom
                                         with parameters <filename> <line>, store via `printf -v "$output"`
   -f, --filter <regex>      Only run tests that match the regular expression
+  --negative-filter <regex> Only run tests that do not match the regular expression
   --filter-status <status>  Only run tests with the given status in the last completed (no CTRL+C/SIGINT) run.
                             Valid <status> values are:
                               failed - runs tests that failed or were not present in the last run
@@ -190,6 +191,10 @@ while [[ "$#" -ne 0 ]]; do
     shift
     flags+=('-f' "$1")
     ;;
+  --negative-filter)
+    shift
+    flags+=('--negative-filter' "$1")
+    ;;
   -F | --formatter)
     shift
     # allow cat formatter to see extended output but don't advertise to users
diff --git a/libexec/bats-core/bats-exec-suite b/libexec/bats-core/bats-exec-suite
@@ -3,6 +3,7 @@ set -e
 
 count_only_flag=''
 filter=''
+nfilter=''
 num_jobs=${BATS_NUMBER_OF_PARALLEL_JOBS:-1}
 parallel_binary_name=${BATS_PARALLEL_BINARY_NAME:-"parallel"}
 bats_no_parallelize_across_files=${BATS_NO_PARALLELIZE_ACROSS_FILES-}
@@ -61,6 +62,10 @@ while [[ "$#" -ne 0 ]]; do
     shift
     gather_tests_flags+=("--filter-tags" "$1")
     ;;
+  --negative-filter)
+    shift
+    nfilter="$1"
+    ;;
   --dummy-flag) ;;
 
   --trace)
@@ -128,7 +133,7 @@ fi
 
 # create file even if no tests are found so the `read` below won't fail
 touch "$TESTS_LIST_FILE"
-gather_tests_output=$(TESTS_LIST_FILE="$TESTS_LIST_FILE" filter="$filter" filter_status="$filter_status" bats-gather-tests "${gather_tests_flags[@]}" -- "$@")
+gather_tests_output=$(TESTS_LIST_FILE="$TESTS_LIST_FILE" filter="$filter" nfilter="$nfilter" filter_status="$filter_status" bats-gather-tests "${gather_tests_flags[@]}" -- "$@")
 test_count=$(grep -c ^ "$TESTS_LIST_FILE" || true) # don't fail here when there are no tests
 if [[ $gather_tests_output == focus_mode ]]; then
   focus_mode=1
@@ -313,4 +318,4 @@ fi
 set -eET
 bats_run_teardown_suite
 
-bats_exit_suite
+bats_exit_suite
diff --git a/libexec/bats-core/bats-gather-tests b/libexec/bats-core/bats-gather-tests
@@ -85,7 +85,7 @@ bats_test_function() {
   line+=$'\t'
   line+="$*"
   # always execute should_skip_because_of_status for its sideeffects
-  if should_skip_because_of_status || should_skip_because_of_focus || should_skip_because_of_filter_tags || should_skip_because_of_filter; then
+  if should_skip_because_of_status || should_skip_because_of_focus || should_skip_because_of_filter_tags || should_skip_because_of_filter || should_skip_because_of_negative_filter; then
     return 0
   fi
   
@@ -164,6 +164,18 @@ function should_skip_because_of_filter() {
 }
 fi
 
+if [[ -n "${nfilter-}" ]]; then
+function should_skip_because_of_negative_filter() {
+  # shellcheck disable=SC2154 # nfilter should be inherited as env var
+  [[ "$description" =~ $nfilter ]]
+}
+else
+function should_skip_because_of_negative_filter() {
+  # skip this when there is no $nfilter
+  return 1
+}
+fi
+
 function should_skip_because_of_status() {
   # disable this filter if not activated by $filter_status
   return 1
diff --git a/test/suite.bats b/test/suite.bats
@@ -219,6 +219,25 @@ setup() {
   [ "${lines[2]}" = 'ok 2 bar_in_b' ]
 }
 
+@test "use --negative-filter to exclude specific test cases" {
+  reentrant_run bats --negative-filter 'ba[rz]' "${FIXTURE_ROOT}/filter"
+  [ "$status" -eq 0 ]
+  [ "${lines[0]}" = '1..5' ]
+  [ "${lines[1]}" = 'ok 1 foo in a' ]
+  [ "${lines[2]}" = 'ok 2 quux_in_b' ]
+  [ "${lines[3]}" = 'ok 3 quux_in c' ]
+  [ "${lines[4]}" = 'ok 4 xyzzy in c' ]
+  [ "${lines[5]}" = 'ok 5 plugh_in c' ]
+}
+
+@test "--negative-filter together with --filter" {
+  reentrant_run bats -f 'in a' --negative-filter 'baz' "${FIXTURE_ROOT}/filter"
+  [ "$status" -eq 0 ]
+  [ "${lines[0]}" = '1..2' ]
+  [ "${lines[1]}" = 'ok 1 foo in a' ]
+  [ "${lines[2]}" = 'ok 2 --bar in a' ]
+}
+
 @test "skip is handled correctly in setup, test, and teardown" {
   bats "${FIXTURE_ROOT}/skip"
 }
