docker-mailserver · georglauterbach · Jan 29, 2024 · Jan 25, 2024 · Jan 25, 2024 · Jan 25, 2024
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -37,6 +37,14 @@ The most noteworthy change of this release is the update of the container's base
       - DMS `main.cf` has renamed `postscreen_dnsbl_whitelist_threshold` to `postscreen_dnsbl_allowlist_threshold` as part of this change.
     - `smtpd_relay_restrictions` (relay policy) is now evaluated after `smtpd_recipient_restrictions` (spam policy). Previously it was evaluated before `smtpd_recipient_restrictions`. Mail to be relayed via DMS must now pass through the spam policy first.
     - The TLS fingerprint policy has changed the default from MD5 to SHA256 (_DMS does not modify this Postfix parameter, but may affect any user customizations that do_).
+- **Environment Variables**:
+  - `SA_SPAM_SUBJECT` has been renamed into `SPAM_SUBJECT` to become anti-spam service agnostic. ([3820](https://github.com/docker-mailserver/docker-mailserver/pull/3820))
+    - As this functionality is now handled in Dovecot via a Sieve script instead of the respective anti-spam service during Postfix processing, this feature will only apply to mail stored in Dovecot. If you have relied on this feature in a different context, it will no longer be available.
+    - Rspamd previously handled this functionality via the `rewrite_subject` action which as now been disabled by default in favor of the new approach with `SPAM_SUBJECT`.
+    - `SA_SPAM_SUBJECT` is now deprecated and will log a warning if used. The value is copied as a fallback to `SPAM_SUBJECT`.
+    - The default has changed to not prepend any prefix to the subject unless configured to do so. If you relied on the implicit prefix, you will now need to provide one explicitly.
+    - `undef` was previously supported as an opt-out with `SA_SPAM_SUBJECT`. This is no longer valid, the equivalent opt-out value is now an empty value (_or rather the omission of this ENV being configured_).
+    - The feature to include [`_SCORE_` tag](https://spamassassin.apache.org/full/4.0.x/doc/Mail_SpamAssassin_Conf.html#rewrite_header-subject-from-to-STRING) in your value to be replaced by the associated spam score is no longer available.
 
 ### Updates
 
@@ -45,6 +53,8 @@ The most noteworthy change of this release is the update of the container's base
     - It's only functionality remaining was to opt-out of run-time state consolidation with `ONE_DIR=0` (_when a volume was already mounted to `/var/mail-state`_).
 - **Tests:**
   - Refactored helper methods for sending e-mails with specific `Message-ID` headers and the helpers for retrieving + filtering logs, which together help isolate logs relevant to specific mail when multiple mails have been processed within a single test. ([#3786](https://github.com/docker-mailserver/docker-mailserver/pull/3786))
+- **Rspamd**:
+  - The `rewrite_subject` action, is now disabled by default. It has been replaced with the new `SPAM_SUBJECT` environment variable, which implements the functionality via a Sieve script instead in favor of being anti-spam service agnostic ([3820](https://github.com/docker-mailserver/docker-mailserver/pull/3820))
 
 ### Fixes
 

diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md
@@ -51,7 +51,7 @@ Configures the provisioning source of user accounts (including aliases) for user
 
 !!! tip "OAuth2 Support"
 
-    Presently DMS supports OAuth2 only as an supplementary authentication method. 
+    Presently DMS supports OAuth2 only as an supplementary authentication method.
 
     - A third-party service must provide a valid token for the user which Dovecot validates with the authentication service provider. To enable this feature reference the [OAuth2 configuration example guide][docs::auth::oauth2-config-guide].
     - User accounts must be provisioned to receive mail via one of the supported `ACCOUNT_PROVISIONER` providers.
@@ -354,6 +354,16 @@ Enable to treat received spam as "read" (_avoids notification to MUA client of n
     - `X-Spam: Yes` (_added by Rspamd_)
     - `X-Spam-Flag: YES` (_added by SpamAssassin - requires [`SPAMASSASSIN_SPAM_TO_INBOX=1`](#spamassassin_spam_to_inbox)_)
 
+##### SPAM_SUBJECT
+
+This variable defines a prefix for e-mails tagged with the `X-Spam: Yes` (Rspamd) or `X-Spam-Flag: YES` (SpamAssassin/Amavis) header.
+
+Default: empty (no prefix will be added to e-mails)
+
+??? example "Including trailing white-space"
+
+    Add trailing white-space by quote wrapping the value: `SPAM_SUBJECT='[SPAM] '`
+
 #### Rspamd
 
 ##### ENABLE_RSPAMD
@@ -562,7 +572,7 @@ Changes the interval in which log files are rotated.
 
     - [`MOVE_SPAM_TO_JUNK=1`](#move_spam_to_junk)
     - [`MARK_SPAM_AS_READ=1`](#mark_spam_as_read)
-    - [`SA_SPAM_SUBJECT`](#sa_spam_subject)
+    - [`SPAM_SUBJECT`](#spam_subject)
 
 ##### SA_TAG
 
@@ -602,8 +612,8 @@ When a spam score is high enough, mark mail as spam (_Appends the mail header: `
 
 !!! info "Interaction with other ENV"
 
-    - [`SA_SPAM_SUBJECT`](#sa_spam_subject) modifies the mail subject to better communicate spam mail to the user.
-    - [`MOVE_SPAM_TO_JUNK=1`](#move_spam_to_junk): The mail is still delivered, but to the recipient(s) junk folder instead. This feature reduces the usefulness of `SA_SPAM_SUBJECT`.
+    - [`SPAM_SUBJECT`](#spam_subject) modifies the mail subject to better communicate spam mail to the user.
+    - [`MOVE_SPAM_TO_JUNK=1`](#move_spam_to_junk): The mail is still delivered, but to the recipient(s) junk folder instead. This feature reduces the usefulness of `SPAM_SUBJECT`.
 
 ##### SA_KILL
 
@@ -659,23 +669,6 @@ Controls the spam score threshold for triggering an action on mail that has a hi
 [amavis-docs::actions]: https://www.ijs.si/software/amavisd/amavisd-new-docs.html#actions
 [amavis-docs::quarantine]: https://www.ijs.si/software/amavisd/amavisd-new-docs.html#quarantine
 
-##### SA_SPAM_SUBJECT
-
-Adds a prefix to the subject header when mail is marked as spam (_via [`SA_TAG2`](#sa_tag2)_).
-
-- **`'***SPAM*** '`** => A string value to use as a mail subject prefix.
-- `undef` => Opt-out of modifying the subject for mail marked as spam.
-
-??? example "Including trailing white-space"
-
-    Add trailing white-space by quote wrapping the value: `SA_SPAM_SUBJECT='[SPAM] '`
-
-??? example "Including the associated spam score"
-
-    The [`_SCORE_` tag][sa-docs::score-tag] will be substituted with the SpamAssassin score: `SA_SPAM_SUBJECT=***SPAM(_SCORE_)***`.
-
-[sa-docs::score-tag]: https://spamassassin.apache.org/full/4.0.x/doc/Mail_SpamAssassin_Conf.html#rewrite_header-subject-from-to-STRING
-
 ##### SA_SHORTCIRCUIT_BAYES_SPAM
 
 - **1** => will activate SpamAssassin short circuiting for bayes spam detection.

diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md
@@ -22,8 +22,9 @@ The following environment variables are related to Rspamd:
 5. [`RSPAMD_HFILTER`](../environment.md#rspamd_hfilter)
 6. [`RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE`](../environment.md#rspamd_hfilter_hostname_unknown_score)
 7. [`RSPAMD_LEARN`](../environment.md#rspamd_learn)
-8. [`MOVE_SPAM_TO_JUNK`][docs-spam-to-junk]
-9. [`MARK_SPAM_AS_READ`](../environment.md#mark_spam_as_read)
+8. [`SPAM_SUBJECT`](../environment.md#spam_subject)
+9. [`MOVE_SPAM_TO_JUNK`][docs-spam-to-junk]
+10. [`MARK_SPAM_AS_READ`](../environment.md#mark_spam_as_read)
 
 With these variables, you can enable Rspamd itself, and you can enable / disable certain features related to Rspamd.
 

diff --git a/mailserver.env b/mailserver.env
@@ -130,6 +130,11 @@ ENABLE_IMAP=1
 # **0** => Disabled
 ENABLE_CLAMAV=0
 
+# Add the value as a prefix to the mail subject when spam is detected.
+# NOTE: By default spam is delivered to a junk folder, reducing the value of a subject prefix for spam.
+# NOTE: When not using Docker Compose, other CRI may not support quote-wrapping the value here to preserve any trailing white-space.
+SPAM_SUBJECT=
+
 # Enables Rspamd
 # **0** => Disabled
 #   1   => Enabled
@@ -378,7 +383,7 @@ ENABLE_SPAMASSASSIN=0
 # Note: only has an effect if `ENABLE_SPAMASSASSIN=1`
 ENABLE_SPAMASSASSIN_KAM=0
 
-# deliver spam messages to the inbox (tagged using SA_SPAM_SUBJECT)
+# deliver spam messages to the inbox (tagged using SPAM_SUBJECT)
 SPAMASSASSIN_SPAM_TO_INBOX=1
 
 # spam messages will be moved in the Junk folder (SPAMASSASSIN_SPAM_TO_INBOX=1 required)
@@ -396,12 +401,6 @@ SA_TAG2=6.31
 # triggers spam evasive actions
 SA_KILL=10.0
 
-# add tag to subject if spam detected
-# The value `undef` opts-out of this feature. The value shown below is the default.
-# NOTE: By default spam is delivered to a junk folder, reducing the value of adding a subject prefix.
-# NOTE: If not using Docker Compose, other CRI may require the single quotes removed.
-#SA_SPAM_SUBJECT='***SPAM*** '
-
 # -----------------------------------------------
 # --- Fetchmail Section -------------------------
 # -----------------------------------------------

diff --git a/target/rspamd/local.d/actions.conf b/target/rspamd/local.d/actions.conf
@@ -6,7 +6,11 @@
 # and to be able to explain the impact on the whole system.
 greylist = 4;
 add_header = 6;
-rewrite_subject = 7;
 reject = 11;
 
-subject = "***SPAM*** %s"
+# The value `null` disabled the action. A subject rewrite is handled by `SPAM_SUBJECT`:
+# https://docker-mailserver.github.io/docker-mailserver/latest/config/environment/#spam_subject
+#
+# The reasoning for this can be found in
+# https://github.com/docker-mailserver/docker-mailserver/issues/3804
+rewrite_subject = null;
diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh
@@ -36,6 +36,7 @@ function _register_functions() {
   _register_check_function '_check_improper_restart'
   _register_check_function '_check_hostname'
   _register_check_function '_check_log_level'
+  _register_check_function '_check_spam_prefix'
 
   # ? >> Setup
 
@@ -48,6 +49,7 @@ function _register_functions() {
     _register_setup_function '_setup_dovecot_sieve'
     _register_setup_function '_setup_dovecot_dhparam'
     _register_setup_function '_setup_dovecot_quota'
+    _register_setup_function '_setup_spam_subject'
     _register_setup_function '_setup_spam_to_junk'
     _register_setup_function '_setup_spam_mark_as_read'
   fi

diff --git a/target/scripts/startup/check-stack.sh b/target/scripts/startup/check-stack.sh
@@ -52,3 +52,11 @@ function _check_log_level() {
     LOG_LEVEL="${DEFAULT_LOG_LEVEL}"
   fi
 }
+
+function _check_spam_prefix() {
+  # This check should be independent of ENABLE_POP3 and ENABLE_IMAP
+  if [[ ${MOVE_SPAM_TO_JUNK} -eq 0 ]] \
+  && [[ -z ${SPAM_SUBJECT} ]]; then
+    _log 'warn' "'MOVE_SPAM_TO_JUNK=0' and 'SPAM_SUBJECT' is empty - make sure this is intended: spam e-mails might not be immediately recognizable in this configuration"
+  fi
+}
diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh
@@ -81,13 +81,9 @@ function __setup__security__spamassassin() {
     # shellcheck disable=SC2016
     sed -i -r 's|^\$sa_kill_level_deflt (.*);|\$sa_kill_level_deflt = '"${SA_KILL}"';|g' /etc/amavis/conf.d/20-debian_defaults
 
-    if [[ ${SA_SPAM_SUBJECT} == 'undef' ]]; then
-      # shellcheck disable=SC2016
-      sed -i -r 's|^\$sa_spam_subject_tag (.*);|\$sa_spam_subject_tag = undef;|g' /etc/amavis/conf.d/20-debian_defaults
-    else
-      # shellcheck disable=SC2016
-      sed -i -r 's|^\$sa_spam_subject_tag (.*);|\$sa_spam_subject_tag = '"'${SA_SPAM_SUBJECT}'"';|g' /etc/amavis/conf.d/20-debian_defaults
-    fi
+    # disable rewriting the subject as this is handles by _setup_spam_subject (which uses Dovecot Sieve)
+    # shellcheck disable=SC2016
+    sed -i -r 's|^\$sa_spam_subject_tag (.*);|\$sa_spam_subject_tag = undef;|g' /etc/amavis/conf.d/20-debian_defaults
 
     # activate short circuits when SA BAYES is certain it has spam or ham.
     if [[ ${SA_SHORTCIRCUIT_BAYES_SPAM} -eq 1 ]]; then
@@ -245,6 +241,57 @@ function __setup__security__amavis() {
   fi
 }
 
+# If `SPAM_SUBJECT` is not empty, we create a Sieve script that alters the `Subject`
+# header, in order to prepend a user-defined string.
+function _setup_spam_subject() {
+  if [[ -z ${SPAM_SUBJECT} ]]
+  then
+    _log 'debug' 'Spam subject is not set - no prefix will be added to spam e-mails'
+  else
+    _log 'debug' "Spam subject is set - the prefix '${SPAM_SUBJECT}' will be added to spam e-mails"
+
+    _log 'trace' "Enabling '+editheader' Sieve extension"
+    # check whether sieve_global_extensions is disabled (and enabled it if so)
+    sed -i -E 's|#(sieve_global_extensions.*)|\1|' /etc/dovecot/conf.d/90-sieve.conf
+    # then append the extension
+    sedfile -i -E 's|(sieve_global_extensions.*)|\1 +editheader|' /etc/dovecot/conf.d/90-sieve.conf
+
+    _log 'trace' "Adding global (before) Sieve script for subject rewrite"
+    # This directory contains Sieve scripts that are executed before user-defined Sieve
+    # scripts run.
+    local DOVECOT_SIEVE_GLOBAL_BEFORE_DIR='/usr/lib/dovecot/sieve-global/before'
+    local DOVECOT_SIEVE_FILE='spam_subject'
+    readonly DOVECOT_SIEVE_GLOBAL_BEFORE_DIR DOVECOT_SIEVE_FILE
+
+    mkdir -p "${DOVECOT_SIEVE_GLOBAL_BEFORE_DIR}"
+    # ref: https://superuser.com/a/1502589
+    cat >"${DOVECOT_SIEVE_GLOBAL_BEFORE_DIR}/${DOVECOT_SIEVE_FILE}.sieve" << EOF
+require ["editheader","variables"];
+
+if anyof (header :contains "X-Spam-Flag" "YES",
+          header :contains "X-Spam" "Yes")
+{
+    # Match the entire subject ...
+    if header :matches "Subject" "*" {
+        # ... to get it in a match group that can then be stored in a variable:
+        set "subject" "\${1}";
+    }
+
+    # We can't "replace" a header, but we can delete (all instances of) it and
+    # re-add (a single instance of) it:
+    deleteheader "Subject";
+
+    # Note that the header is added ":last" (so it won't appear before possible
+    # "Received" headers).
+    addheader :last "Subject" "${SPAM_SUBJECT}\${subject}";
+}
+EOF
+
+    sievec "${DOVECOT_SIEVE_GLOBAL_BEFORE_DIR}/${DOVECOT_SIEVE_FILE}.sieve"
+    chown dovecot:root "${DOVECOT_SIEVE_GLOBAL_BEFORE_DIR}/${DOVECOT_SIEVE_FILE}."{sieve,svbin}
+  fi
+}
+
 # We can use Sieve to move spam emails to the "Junk" folder.
 function _setup_spam_to_junk() {
   if [[ ${MOVE_SPAM_TO_JUNK} -eq 1 ]]; then

diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh
@@ -25,6 +25,12 @@ function __environment_variables_backwards_compatibility() {
     _log 'error' "The ENV for which LDAP host to connect to must include the URI scheme ('ldap://', 'ldaps://', 'ldapi://')"
   fi
 
+  if [[ -n ${SA_SPAM_SUBJECT:-} ]]; then
+    _log 'warn' "'SA_SPAM_SUBJECT' has been renamed to 'SPAM_SUBJECT' - this warning will block startup on v15.0.0"
+    _log 'info' "Copying value of 'SA_SPAM_SUBJECT' into 'SPAM_SUBJECT' if 'SPAM_SUBJECT' has not been set explicitly"
+    SPAM_SUBJECT=${SPAM_SUBJECT:-${SA_SPAM_SUBJECT}}
+  fi
+
   # TODO this can be uncommented in a PR handling the HOSTNAME/DOMAINNAME issue
   # TODO see check_for_changes.sh and dns.sh
   # if [[ -n ${OVERRIDE_HOSTNAME:-} ]]
@@ -67,7 +73,7 @@ function __environment_variables_general_setup() {
   VARS[RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE]="${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE:=6}"
   VARS[RSPAMD_LEARN]="${RSPAMD_LEARN:=0}"
   VARS[SA_KILL]=${SA_KILL:="10.0"}
-  VARS[SA_SPAM_SUBJECT]=${SA_SPAM_SUBJECT:="***SPAM*** "}
+  VARS[SPAM_SUBJECT]=${SPAM_SUBJECT:=}
   VARS[SA_TAG]=${SA_TAG:="2.0"}
   VARS[SA_TAG2]=${SA_TAG2:="6.31"}
   VARS[SPAMASSASSIN_SPAM_TO_INBOX]="${SPAMASSASSIN_SPAM_TO_INBOX:=1}"

diff --git a/test/helper/common.bash b/test/helper/common.bash
@@ -417,5 +417,21 @@ function _nc_wrapper() {
   _run_in_container_bash "nc ${NC_PARAMETERS} < /tmp/docker-mailserver-test/${FILE}"
 }
 
+# A simple wrapper for a test that checks whether a file exists.
+#
+# @param ${1} = the path to the file inside the container
+function _file_exists_in_container() {
+  _run_in_container_bash "[[ -f ${1} ]]"
+  assert_success
+}
+
+# A simple wrapper for a test that checks whether a file does not exist.
+#
+# @param ${1} = the path to the file (that should not exists) inside the container
+function _file_does_not_exist_in_container() {
+  _run_in_container_bash "[[ -f ${1} ]]"
+  assert_failure
+}
+
 # ? << Miscellaneous helper functions
 # ! -------------------------------------------------------------------