docker-mailserver · polarathene · Dec 3, 2022 · Nov 3, 2022 · Nov 25, 2022 · Nov 25, 2022
diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml
@@ -14,6 +14,9 @@ jobs:
   run-tests:
     name: 'Test'
     runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        part: [serial, parallel/set1, parallel/set2, parallel/set3]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -45,6 +48,6 @@ jobs:
           cache-from: type=local,src=/tmp/.buildx-cache
 
       - name: 'Run tests'
-        run: make generate-accounts tests
+        run: make generate-accounts tests/${{ matrix.part }}
         env:
           CI: true
diff --git a/Makefile b/Makefile
@@ -1,8 +1,12 @@
-SHELL             := /bin/bash
-.SHELLFLAGS       += -e -u -o pipefail
+SHELL       := /bin/bash
+.SHELLFLAGS += -e -u -o pipefail
 
-export IMAGE_NAME := mailserver-testing:ci
-export NAME       ?= $(IMAGE_NAME)
+PARALLEL_JOBS          ?= 2
+export REPOSITORY_ROOT := $(CURDIR)
+export IMAGE_NAME      ?= mailserver-testing:ci
+export NAME            ?= $(IMAGE_NAME)
+
+.PHONY: ALWAYS_RUN
 
 # -----------------------------------------------
 # --- Generic Targets ---------------------------
@@ -17,7 +21,7 @@ build:
 		--build-arg VCS_REVISION=$(shell cat VERSION) \
 		.
 
-generate-accounts:
+generate-accounts: ALWAYS_RUN
 	@ cp test/config/templates/postfix-accounts.cf test/config/postfix-accounts.cf
 	@ cp test/config/templates/dovecot-masters.cf test/config/dovecot-masters.cf
 
@@ -33,14 +37,29 @@ clean:
 	-@ while read -r LINE; do [[ $${LINE} =~ test/.+ ]] && sudo rm -rf $${LINE}; done < .gitignore
 
 # -----------------------------------------------
-# --- Tests  & Lints ----------------------------
+# --- Tests  ------------------------------------
 # -----------------------------------------------
 
-tests:
-	@ ./test/bats/bin/bats --timing test/*.bats
+tests: ALWAYS_RUN
+# See https://github.com/docker-mailserver/docker-mailserver/pull/2857#issuecomment-1312724303
+# on why `generate-accounts` is run before each set (TODO/FIXME)
+	@ $(MAKE) generate-accounts tests/serial
+	@ $(MAKE) generate-accounts tests/parallel/set1
+	@ $(MAKE) generate-accounts tests/parallel/set2
+	@ $(MAKE) generate-accounts tests/parallel/set3
+
+tests/serial: ALWAYS_RUN
+	@ shopt -s globstar ; ./test/bats/bin/bats --timing --jobs 1 test/$@/**.bats
+
+tests/parallel/set%: ALWAYS_RUN
+	@ shopt -s globstar ; ./test/bats/bin/bats --timing --jobs $(PARALLEL_JOBS) test/$@/**.bats
 
-test/%:
-	@ ./test/bats/bin/bats --timing [email protected]
+test/%: ALWAYS_RUN
+	@ shopt -s globstar nullglob ; ./test/bats/bin/bats --timing test/tests/**/{$*,}.bats
+
+# -----------------------------------------------
+# --- Lints -------------------------------------
+# -----------------------------------------------
 
 lint: eclint hadolint shellcheck
 

diff --git a/docs/content/contributing/general.md b/docs/content/contributing/general.md
@@ -24,6 +24,12 @@ To run the test suite, you will need to
 
     We do not support running linting, tests, etc on macOS at this time. Please use a linux VM.
 
+??? tip "Running a Specific Test"
+
+    To run a specific test, use `make build generate-accounts test/<TEST NAME>`, where `<TEST NAME>` is the file name of the test (_for more precision use a relative path: `test/test/<PATH>`_) excluding the `.bats` suffix.
+
+    To run only the tests in `template.bats`, use `make test/template` (or `make test/parallel/set2/template`).
+
 [Install Docker]: https://docs.docker.com/get-docker/
 
 ## Documentation

diff --git a/test/clamav.bats b/test/clamav.bats
diff --git a/test/default_relay_host.bats b/test/default_relay_host.bats
diff --git a/test/dovecot_inet_protocol.bats b/test/dovecot_inet_protocol.bats
diff --git a/test/helper-functions.bats b/test/helper-functions.bats