From 2da3e1b02224b25703ba5cfaf552a8b2f4e68824 Mon Sep 17 00:00:00 2001 From: Casper Date: Sun, 12 Mar 2023 12:10:45 +0100 Subject: [PATCH 001/592] fix: SRS setup (#3158) --- Dockerfile | 2 - target/scripts/build/packages.sh | 10 +++ target/scripts/startup/setup.d/mail_state.sh | 71 ++++++++++++++------ target/scripts/startup/setup.d/postfix.sh | 15 +---- 4 files changed, 61 insertions(+), 37 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2e49b332717..acadbe54c6e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -242,8 +242,6 @@ RUN < add set \\{ type \\; flags interval\\; \\}/' /etc/fail2ban/action.d/nftables.conf } +function _remove_data_after_package_installations +{ + _log 'debug' 'Deleting sensitive files (secrets)' + rm /etc/postsrsd.secret + + _log 'debug' 'Deleting default logwatch cronjob' + rm /etc/cron.daily/00logwatch +} + function _post_installation_steps { _log 'debug' 'Running post-installation steps (cleanup)' @@ -216,4 +225,5 @@ _install_packages _install_dovecot _install_rspamd _install_fail2ban +_remove_data_after_package_installations _post_installation_steps diff --git a/target/scripts/startup/setup.d/mail_state.sh b/target/scripts/startup/setup.d/mail_state.sh index 6a2d3ec6ffa..b96a457c9c9 100644 --- a/target/scripts/startup/setup.d/mail_state.sh +++ b/target/scripts/startup/setup.d/mail_state.sh @@ -4,7 +4,7 @@ # (/var/mail-state) to allow persistence using docker volumes function _setup_save_states { - local STATEDIR FILE FILES + local DEST DESTDIR STATEDIR SERVICEDIR SERVICEDIRS SERVICEFILE SERVICEFILES STATEDIR='/var/mail-state' @@ -13,7 +13,7 @@ function _setup_save_states _log 'debug' "Consolidating all state onto ${STATEDIR}" # Always enabled features: - FILES=( + SERVICEDIRS=( lib/logrotate lib/postfix spool/postfix @@ -21,38 +21,65 @@ function _setup_save_states # Only consolidate state for services that are enabled # Notably avoids copying over 200MB for the ClamAV database - [[ ${ENABLE_AMAVIS} -eq 1 ]] && FILES+=('lib/amavis') - [[ ${ENABLE_CLAMAV} -eq 1 ]] && FILES+=('lib/clamav') - [[ ${ENABLE_FAIL2BAN} -eq 1 ]] && FILES+=('lib/fail2ban') - [[ ${ENABLE_FETCHMAIL} -eq 1 ]] && FILES+=('lib/fetchmail') - [[ ${ENABLE_POSTGREY} -eq 1 ]] && FILES+=('lib/postgrey') - [[ ${ENABLE_RSPAMD} -eq 1 ]] && FILES+=('lib/rspamd') - [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] && FILES+=('lib/redis') - [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] && FILES+=('lib/spamassassin') - [[ ${SMTP_ONLY} -ne 1 ]] && FILES+=('lib/dovecot') - - for FILE in "${FILES[@]}" + [[ ${ENABLE_AMAVIS} -eq 1 ]] && SERVICEDIRS+=('lib/amavis') + [[ ${ENABLE_CLAMAV} -eq 1 ]] && SERVICEDIRS+=('lib/clamav') + [[ ${ENABLE_FAIL2BAN} -eq 1 ]] && SERVICEDIRS+=('lib/fail2ban') + [[ ${ENABLE_FETCHMAIL} -eq 1 ]] && SERVICEDIRS+=('lib/fetchmail') + [[ ${ENABLE_POSTGREY} -eq 1 ]] && SERVICEDIRS+=('lib/postgrey') + [[ ${ENABLE_RSPAMD} -eq 1 ]] && SERVICEDIRS+=('lib/rspamd') + [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] && SERVICEDIRS+=('lib/redis') + [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] && SERVICEDIRS+=('lib/spamassassin') + [[ ${ENABLE_SRS} -eq 1 ]] && SERVICEDIRS+=('lib/postsrsd') + [[ ${SMTP_ONLY} -ne 1 ]] && SERVICEDIRS+=('lib/dovecot') + + # Single service files + [[ ${ENABLE_SRS} -eq 1 ]] && SERVICEFILES+=('/etc/postsrsd.secret') + + for SERVICEFILE in "${SERVICEFILES[@]}"; + do + DEST="${STATEDIR}/${SERVICEFILE}" + DESTDIR="${DEST%/*}" + + mkdir -p "${DESTDIR}" + if [[ -f ${DEST} ]] + then + _log 'trace' "Destination ${DEST} exists, linking ${SERVICEFILE} to it" + # Original content from image no longer relevant, remove it: + rm -f "${SERVICEFILE}" + elif [[ -f "${SERVICEFILE}" ]] + then + _log 'trace' "Moving ${SERVICEFILE} to ${DEST}" + # Empty volume was mounted, or new content from enabling a feature ENV: + mv "${SERVICEFILE}" "${DEST}" + fi + + # Symlink the original file in the container ($SERVICEFILE) to be + # sourced from assocaiated path in /var/mail-state/ ($DEST): + ln -s "${DEST}" "${SERVICEFILE}" + done + + for SERVICEDIR in "${SERVICEDIRS[@]}" do - DEST="${STATEDIR}/${FILE//\//-}" - FILE="/var/${FILE}" + DEST="${STATEDIR}/${SERVICEDIR//\//-}" + SERVICEDIR="/var/${SERVICEDIR}" # If relevant content is found in /var/mail-state (presumably a volume mount), # use it instead. Otherwise copy over any missing directories checked. if [[ -d ${DEST} ]] then - _log 'trace' "Destination ${DEST} exists, linking ${FILE} to it" + _log 'trace' "Destination ${DEST} exists, linking ${SERVICEDIR} to it" # Original content from image no longer relevant, remove it: - rm -rf "${FILE}" - elif [[ -d ${FILE} ]] + rm -rf "${SERVICEDIR}" + elif [[ -d ${SERVICEDIR} ]] then - _log 'trace' "Moving contents of ${FILE} to ${DEST}" + _log 'trace' "Moving contents of ${SERVICEDIR} to ${DEST}" # Empty volume was mounted, or new content from enabling a feature ENV: - mv "${FILE}" "${DEST}" + mv "${SERVICEDIR}" "${DEST}" fi - # Symlink the original path in the container ($FILE) to be + # Symlink the original path in the container ($SERVICEDIR) to be # sourced from assocaiated path in /var/mail-state/ ($DEST): - ln -s "${DEST}" "${FILE}" + ln -s "${DEST}" "${SERVICEDIR}" done # This ensures the user and group of the files from the external mount have their diff --git a/target/scripts/startup/setup.d/postfix.sh b/target/scripts/startup/setup.d/postfix.sh index 4d1bbf6ffc0..12a2f8b6620 100644 --- a/target/scripts/startup/setup.d/postfix.sh +++ b/target/scripts/startup/setup.d/postfix.sh @@ -151,13 +151,11 @@ function _setup_SRS ) } - local POSTSRSD_SECRET_FILE POSTSRSD_STATE_DIR POSTSRSD_STATE_SECRET_FILE + local POSTSRSD_SECRET_FILE sed -i "s/localdomain/${SRS_DOMAINNAME}/g" /etc/default/postsrsd POSTSRSD_SECRET_FILE='/etc/postsrsd.secret' - POSTSRSD_STATE_DIR='/var/mail-state/etc-postsrsd' - POSTSRSD_STATE_SECRET_FILE="${POSTSRSD_STATE_DIR}/postsrsd.secret" if [[ -n ${SRS_SECRET} ]] then @@ -166,16 +164,7 @@ function _setup_SRS echo "${SRS_SECRET}" | tr ',' '\n' >"${POSTSRSD_SECRET_FILE}" ) else - if [[ ${ONE_DIR} -eq 1 ]] - then - if [[ ! -f ${POSTSRSD_STATE_SECRET_FILE} ]] - then - install -d -m 0775 "${POSTSRSD_STATE_DIR}" - __generate_secret "${POSTSRSD_STATE_SECRET_FILE}" - fi - - install -m 0400 "${POSTSRSD_STATE_SECRET_FILE}" "${POSTSRSD_SECRET_FILE}" - elif [[ ! -f ${POSTSRSD_SECRET_FILE} ]] + if [[ ! -f ${POSTSRSD_SECRET_FILE} ]] then __generate_secret "${POSTSRSD_SECRET_FILE}" fi From f19006bd72075e3b51d86b20b290daadee9ec9d5 Mon Sep 17 00:00:00 2001 From: Lin Han Date: Sun, 12 Mar 2023 11:59:43 +0000 Subject: [PATCH 002/592] doc: a ip -> an ip (#3175) Co-authored-by: Casper --- docs/content/config/best-practices/spf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/config/best-practices/spf.md b/docs/content/config/best-practices/spf.md index 308c3fe0888..a64fe425255 100644 --- a/docs/content/config/best-practices/spf.md +++ b/docs/content/config/best-practices/spf.md @@ -34,7 +34,7 @@ In any case, increment the SPF record's TTL to its final value. ## Backup MX, Secondary MX -For whitelisting a IP Address from the SPF test, you can create a config file (see [`policyd-spf.conf`](https://www.linuxcertif.com/man/5/policyd-spf.conf)) and mount that file into `/etc/postfix-policyd-spf-python/policyd-spf.conf`. +For whitelisting an IP Address from the SPF test, you can create a config file (see [`policyd-spf.conf`](https://www.linuxcertif.com/man/5/policyd-spf.conf)) and mount that file into `/etc/postfix-policyd-spf-python/policyd-spf.conf`. **Example:** From 6c97a505be26910c31f64e6d82c7317fcf807ea8 Mon Sep 17 00:00:00 2001 From: Casper Date: Mon, 13 Mar 2023 00:39:03 +0100 Subject: [PATCH 003/592] fix: postsrsd restart loop (#3160) --- target/supervisor/conf.d/supervisor-app.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/supervisor/conf.d/supervisor-app.conf b/target/supervisor/conf.d/supervisor-app.conf index 4c6c10d1d24..d426831f523 100644 --- a/target/supervisor/conf.d/supervisor-app.conf +++ b/target/supervisor/conf.d/supervisor-app.conf @@ -148,7 +148,7 @@ autostart=false autorestart=true stdout_logfile=/var/log/supervisor/%(program_name)s.log stderr_logfile=/var/log/supervisor/%(program_name)s.log -command=/etc/init.d/postsrsd start +command=/bin/bash -c 'env $(grep -vE "^(#.*|\s*)$" /etc/default/postsrsd) postsrsd -e -p /var/run/postsrsd.pid' [program:update-check] startsecs=0 From e890ba46a3e64ea0cc75b1cfd3d56e478655cd4b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Mar 2023 11:45:27 +1300 Subject: [PATCH 004/592] chore(deps): Bump docker/setup-buildx-action from 2.4.1 to 2.5.0 (#3176) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 2.4.1 to 2.5.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v2.4.1...v2.5.0) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index c7857e4838a..290a9e00fcd 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -79,7 +79,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.4.1 + uses: docker/setup-buildx-action@v2.5.0 # NOTE: AMD64 can build within 2 minutes - name: 'Build images' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index ae94fc2dc7e..2c921919550 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -40,7 +40,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.4.1 + uses: docker/setup-buildx-action@v2.5.0 # Try get the cached build layers from a prior `generic_build.yml` job. # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 43c6e8e5be3..fc3a292bcad 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -38,7 +38,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.4.1 + uses: docker/setup-buildx-action@v2.5.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 84022bcb2b5..fd7bae3e62c 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -37,7 +37,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.4.1 + uses: docker/setup-buildx-action@v2.5.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. From e58dd1b95b3552ea850dad6b2ef0b87f8a9bae78 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sat, 18 Mar 2023 23:32:48 +0800 Subject: [PATCH 005/592] Rspamd: more features (#3159) --- Dockerfile | 3 - docs/content/config/environment.md | 102 +++++++++++------- docs/content/config/security/rspamd.md | 7 ++ target/dovecot/90-sieve.conf | 2 +- target/dovecot/sieve/before/60-spam.sieve | 4 - target/scripts/start-mailserver.sh | 2 + target/scripts/startup/setup.d/dovecot.sh | 69 ++++++------ .../scripts/startup/setup.d/security/misc.sh | 26 +++++ .../startup/setup.d/security/rspamd.sh | 55 +++++++++- target/scripts/startup/variables-stack.sh | 1 + test/config/rspamd/postfix-accounts.cf | 2 + test/config/rspamd/user-patches.sh | 13 +++ .../email-templates/rspamd-pass.txt | 12 +++ .../email-templates/rspamd-spam-header.txt | 12 +++ .../email-templates/rspamd-spam.txt | 1 - .../email-templates/rspamd-virus.txt | 2 +- .../rspamd_imap_move_to_inbox.txt | 4 + .../nc_templates/rspamd_imap_move_to_junk.txt | 4 + .../parallel/set1/dovecot/dovecot_sieve.bats | 11 +- .../parallel/set1/spam_virus/rspamd.bats | 96 ++++++++++++++++- 20 files changed, 329 insertions(+), 99 deletions(-) delete mode 100644 target/dovecot/sieve/before/60-spam.sieve create mode 100644 test/config/rspamd/postfix-accounts.cf create mode 100644 test/config/rspamd/user-patches.sh create mode 100644 test/test-files/email-templates/rspamd-pass.txt create mode 100644 test/test-files/email-templates/rspamd-spam-header.txt create mode 100644 test/test-files/nc_templates/rspamd_imap_move_to_inbox.txt create mode 100644 test/test-files/nc_templates/rspamd_imap_move_to_junk.txt diff --git a/Dockerfile b/Dockerfile index acadbe54c6e..b66070de7a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,7 +54,6 @@ EOF # ----------------------------------------------- COPY target/dovecot/*.inc target/dovecot/*.conf /etc/dovecot/conf.d/ -COPY target/dovecot/sieve/ /etc/dovecot/sieve/ COPY target/dovecot/dovecot-purge.cron /etc/cron.d/dovecot-purge.disabled RUN chmod 0 /etc/cron.d/dovecot-purge.disabled WORKDIR /usr/share/dovecot @@ -66,8 +65,6 @@ RUN < disabled -- 1 => enabled - -##### ENABLE_RSPAMD_REDIS - -Explicit control over running a Redis instance within the container. By default, this value will match what is set for [`ENABLE_RSPAMD`](#enable_rspamd). - -The purpose of this setting is to opt-out of starting an internal Redis instance when enabling Rspamd, replacing it with your own external instance. - -??? note "Configuring rspamd for an external Redis instance" - - You will need to [provide configuration][config-rspamd-redis] at `/etc/rspamd/local.d/redis.conf` similar to: - - ``` - servers = "redis.example.test:6379"; - expand_keys = true; - ``` - -[config-rspamd-redis]: https://rspamd.com/doc/configuration/redis.html - -- 0 => Disabled -- 1 => Enabled - ##### ENABLE_AMAVIS Amavis content filter (used for ClamAV & SpamAssassin) @@ -316,6 +285,69 @@ Note: More details at Note: More information at +##### MOVE_SPAM_TO_JUNK + +When enabled, e-mails marked with the + +1. `X-Spam: Yes` header added by Rspamd +2. `X-Spam-Flag: YES` header added by SpamAssassin (requires [`SPAMASSASSIN_SPAM_TO_INBOX=1`](#spamassassin_spam_to_inbox)) + +will be automatically moved to the Junk folder (with the help of a Sieve script). + +- 0 => Spam messages will be delivered in the mailbox. +- **1** => Spam messages will be delivered in the `Junk` folder. + +#### Rspamd + +##### ENABLE_RSPAMD + +Enable or disable Rspamd. + +!!! warning "Current State" + + Rspamd-support is under active development. Be aware that breaking changes can happen at any time. To get more information, see [the detailed documentation page for Rspamd][docs-rspamd]. + +- **0** => disabled +- 1 => enabled + +##### ENABLE_RSPAMD_REDIS + +Explicit control over running a Redis instance within the container. By default, this value will match what is set for [`ENABLE_RSPAMD`](#enable_rspamd). + +The purpose of this setting is to opt-out of starting an internal Redis instance when enabling Rspamd, replacing it with your own external instance. + +??? note "Configuring Rspamd for an external Redis instance" + + You will need to [provide configuration][rspamd-redis-config] at `/etc/rspamd/local.d/redis.conf` similar to: + + ``` + servers = "redis.example.test:6379"; + expand_keys = true; + ``` + +[rspamd-redis-config]: https://rspamd.com/doc/configuration/redis.html + +- 0 => Disabled +- 1 => Enabled + +##### RSPAMD_LEARN + +When enabled, + +1. the "[autolearning][rspamd-autolearn]" feature is turned on; +2. the Bayes classifier will be trained when moving mails from or to the Junk folder (with the help of Sieve scripts). + +!!! attention + + As of now, the spam learning database is global (i.e. available to all users). If one user deliberately trains it with malicious data, then it will ruin your detection rate. + + This feature is suitably only for users who can tell ham from spam and users that can be trusted. + +[rspamd-autolearn]: https://rspamd.com/doc/configuration/statistic.html#autolearning + +- **0** => Disabled +- 1 => Enabled + #### Reports ##### PFLOGSUMM_TRIGGER @@ -418,14 +450,6 @@ Changes the interval in which log files are rotated. - **0** => KAM disabled - 1 => KAM enabled -##### MOVE_SPAM_TO_JUNK - -Spam messages can be moved in the Junk folder. -Note: this setting needs `SPAMASSASSIN_SPAM_TO_INBOX=1` - -- 0 => Spam messages will be delivered in the mailbox. -- **1** => Spam messages will be delivered in the `Junk` folder. - ##### SA_TAG - **2.0** => add spam info headers if at, or above that level diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index e9a95700340..dc81b1fcfa5 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -20,6 +20,13 @@ If you want to have a look at the default configuration files for Rspamd that DM Maintainers noticed only few differences, some of them with a big impact though. For those running Rspamd on ARM64, we recommend [disabling](#with-the-help-of-a-custom-file) the [DKIM signing module][dkim-signing-module] if you don't use it. +The following environment variables are related to Rspamd: + +1. [`ENABLE_RSPAMD`](../environment.md#enable_rspamd) +2. [`ENABLE_RSPAMD_REDIS`](../environment.md#enable_rspamd_redis) +3. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) +4. [`MOVE_SPAM_TO_JUNK`](../environment.md#move_spam_to_junk) + ## The Default Configuration ### Mode of Operation diff --git a/target/dovecot/90-sieve.conf b/target/dovecot/90-sieve.conf index 78fbe47bed3..8b559be23d8 100644 --- a/target/dovecot/90-sieve.conf +++ b/target/dovecot/90-sieve.conf @@ -71,7 +71,7 @@ plugin { # (http://pigeonhole.dovecot.org) for available plugins. # The sieve_extprograms plugin is included in this release. #sieve_plugins = - sieve_plugins = sieve_extprograms + sieve_plugins = sieve_imapsieve sieve_extprograms # The separator that is expected between the :user and :detail # address parts introduced by the subaddress extension. This may diff --git a/target/dovecot/sieve/before/60-spam.sieve b/target/dovecot/sieve/before/60-spam.sieve deleted file mode 100644 index 4fa29fab247..00000000000 --- a/target/dovecot/sieve/before/60-spam.sieve +++ /dev/null @@ -1,4 +0,0 @@ -require "fileinto"; -if header :contains "X-Spam-Flag" "YES" { - fileinto "Junk"; -} diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index c98639b8e8c..83209267c96 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -45,6 +45,7 @@ function _register_functions if [[ ${SMTP_ONLY} -ne 1 ]] then _register_setup_function '_setup_dovecot' + _register_setup_function '_setup_dovecot_sieve' _register_setup_function '_setup_dovecot_dhparam' _register_setup_function '_setup_dovecot_quota' fi @@ -81,6 +82,7 @@ function _register_functions _register_setup_function '_setup_opendmarc' # must come after `_setup_opendkim` _register_setup_function '_setup_security_stack' + _register_setup_function '_setup_spam_to_junk' _register_setup_function '_setup_rspamd' _register_setup_function '_setup_ssl' diff --git a/target/scripts/startup/setup.d/dovecot.sh b/target/scripts/startup/setup.d/dovecot.sh index cf77c9798a7..fbb01871ada 100644 --- a/target/scripts/startup/setup.d/dovecot.sh +++ b/target/scripts/startup/setup.d/dovecot.sh @@ -40,6 +40,20 @@ function _setup_dovecot esac + if [[ ${ENABLE_POP3} -eq 1 ]] + then + _log 'debug' 'Enabling POP3 services' + mv /etc/dovecot/protocols.d/pop3d.protocol.disab /etc/dovecot/protocols.d/pop3d.protocol + fi + + [[ -f /tmp/docker-mailserver/dovecot.cf ]] && cp /tmp/docker-mailserver/dovecot.cf /etc/dovecot/local.conf +} + +function _setup_dovecot_sieve +{ + mkdir -p /usr/lib/dovecot/sieve-{filter,global,pipe} + mkdir -p /usr/lib/dovecot/sieve-global/{before,after} + # enable Managesieve service by setting the symlink # to the configuration file Dovecot will actually find if [[ ${ENABLE_MANAGESIEVE} -eq 1 ]] @@ -48,56 +62,35 @@ function _setup_dovecot mv /etc/dovecot/protocols.d/managesieved.protocol.disab /etc/dovecot/protocols.d/managesieved.protocol fi - # copy pipe and filter programs, if any - rm -f /usr/lib/dovecot/sieve-filter/* - rm -f /usr/lib/dovecot/sieve-pipe/* - [[ -d /tmp/docker-mailserver/sieve-filter ]] && cp /tmp/docker-mailserver/sieve-filter/* /usr/lib/dovecot/sieve-filter/ - [[ -d /tmp/docker-mailserver/sieve-pipe ]] && cp /tmp/docker-mailserver/sieve-pipe/* /usr/lib/dovecot/sieve-pipe/ - - # create global sieve directories - mkdir -p /usr/lib/dovecot/sieve-global/before - mkdir -p /usr/lib/dovecot/sieve-global/after - - if [[ -f /tmp/docker-mailserver/before.dovecot.sieve ]] + if [[ -d /tmp/docker-mailserver/sieve-filter ]] then - cp /tmp/docker-mailserver/before.dovecot.sieve /usr/lib/dovecot/sieve-global/before/50-before.dovecot.sieve - sievec /usr/lib/dovecot/sieve-global/before/50-before.dovecot.sieve - else - rm -f /usr/lib/dovecot/sieve-global/before/50-before.dovecot.sieve /usr/lib/dovecot/sieve-global/before/50-before.dovecot.svbin + cp /tmp/docker-mailserver/sieve-filter/* /usr/lib/dovecot/sieve-filter/ fi - - if [[ -f /tmp/docker-mailserver/after.dovecot.sieve ]] + if [[ -d /tmp/docker-mailserver/sieve-pipe ]] then - cp /tmp/docker-mailserver/after.dovecot.sieve /usr/lib/dovecot/sieve-global/after/50-after.dovecot.sieve - sievec /usr/lib/dovecot/sieve-global/after/50-after.dovecot.sieve - else - rm -f /usr/lib/dovecot/sieve-global/after/50-after.dovecot.sieve /usr/lib/dovecot/sieve-global/after/50-after.dovecot.svbin + cp /tmp/docker-mailserver/sieve-pipe/* /usr/lib/dovecot/sieve-pipe/ fi - # sieve will move spams to .Junk folder when SPAMASSASSIN_SPAM_TO_INBOX=1 and MOVE_SPAM_TO_JUNK=1 - if [[ ${SPAMASSASSIN_SPAM_TO_INBOX} -eq 1 ]] && [[ ${MOVE_SPAM_TO_JUNK} -eq 1 ]] + if [[ -f /tmp/docker-mailserver/before.dovecot.sieve ]] then - _log 'debug' 'Spam messages will be moved to the Junk folder' - cp /etc/dovecot/sieve/before/60-spam.sieve /usr/lib/dovecot/sieve-global/before/ - sievec /usr/lib/dovecot/sieve-global/before/60-spam.sieve - else - rm -f /usr/lib/dovecot/sieve-global/before/60-spam.sieve /usr/lib/dovecot/sieve-global/before/60-spam.svbin + cp \ + /tmp/docker-mailserver/before.dovecot.sieve \ + /usr/lib/dovecot/sieve-global/before/50-before.dovecot.sieve + sievec /usr/lib/dovecot/sieve-global/before/50-before.dovecot.sieve fi - - chown docker:docker -R /usr/lib/dovecot/sieve* - chmod 550 -R /usr/lib/dovecot/sieve* - chmod -f +x /usr/lib/dovecot/sieve-pipe/* - - if [[ ${ENABLE_POP3} -eq 1 ]] + if [[ -f /tmp/docker-mailserver/after.dovecot.sieve ]] then - _log 'debug' 'Enabling POP3 services' - mv /etc/dovecot/protocols.d/pop3d.protocol.disab /etc/dovecot/protocols.d/pop3d.protocol + cp \ + /tmp/docker-mailserver/after.dovecot.sieve \ + /usr/lib/dovecot/sieve-global/after/50-after.dovecot.sieve + sievec /usr/lib/dovecot/sieve-global/after/50-after.dovecot.sieve fi - [[ -f /tmp/docker-mailserver/dovecot.cf ]] && cp /tmp/docker-mailserver/dovecot.cf /etc/dovecot/local.conf + chown dovecot:root -R /usr/lib/dovecot/sieve-* + find /usr/lib/dovecot/sieve-* -type d -exec chmod 755 {} \; + chmod +x /usr/lib/dovecot/sieve-{filter,pipe}/* } - function _setup_dovecot_quota { _log 'debug' 'Setting up Dovecot quota' diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index d441f7d136a..085b632bcee 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -263,3 +263,29 @@ function __setup__security__amavis fi fi } + +# We can use Sieve to move spam emails to the "Junk" folder. +function _setup_spam_to_junk +{ + if [[ ${MOVE_SPAM_TO_JUNK} -eq 1 ]] + then + _log 'debug' 'Spam emails will be moved to the Junk folder' + cat >/usr/lib/dovecot/sieve-global/after/spam_to_junk.sieve << EOF +require ["fileinto","mailbox"]; + +if anyof (header :contains "X-Spam-Flag" "YES", + header :contains "X-Spam" "Yes") { + fileinto "Junk"; +} +EOF + sievec /usr/lib/dovecot/sieve-global/after/spam_to_junk.sieve + chown dovecot:root /usr/lib/dovecot/sieve-global/after/spam_to_junk.{sieve,svbin} + + if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] && [[ ${SPAMASSASSIN_SPAM_TO_INBOX} -eq 0 ]] + then + _log 'warning' "'SPAMASSASSIN_SPAM_TO_INBOX=0' but it is required to be 1 for 'MOVE_SPAM_TO_JUNK=1' to work" + fi + else + _log 'debug' 'Spam emails will not be moved to the Junk folder' + fi +} diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index a476bd4652c..8b478904b88 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -7,10 +7,18 @@ function _setup_rspamd _log 'warn' 'Rspamd integration is work in progress - expect (breaking) changes at any time' _log 'debug' 'Enabling and configuring Rspamd' - __rspamd__preflight_checks + __rspamd__preflight_checks_and_setup __rspamd__adjust_postfix_configuration __rspamd__disable_default_modules __rspamd__handle_modules_configuration + + if [[ ${RSPAMD_LEARN} -eq 1 ]] + then + __rspamd__log 'debug' 'Enabling and configuring learning' + __rspamd__setup_learning + else + __rspamd__log 'debug' 'Learning is disabled' + fi else _log 'debug' 'Rspamd is disabled' fi @@ -28,7 +36,7 @@ function __rspamd__log { _log "${1:-}" "(Rspamd setup) ${2:-}" ; } # # This will also check whether Amavis is enabled and emit a warning as # we discourage users from running Amavis & Rspamd at the same time. -function __rspamd__preflight_checks +function __rspamd__preflight_checks_and_setup { touch /var/lib/rspamd/stats.ucl @@ -225,3 +233,46 @@ function __rspamd__handle_modules_configuration done < <(_get_valid_lines_from_file "${RSPAMD_CUSTOM_COMMANDS_FILE}") fi } + +# This function sets up intelligent learning of Junk, by +# +# 1. enabling auto-learn for the classifier-bayes module +# 2. setting up sieve scripts that detect when a user is moving e-mail +# from or to the "Junk" folder, and learning them as ham or spam. +function __rspamd__setup_learning +{ + __rspamd__log 'debug' 'Setting up intelligent learning of spam and ham' + + local SIEVE_PIPE_BIN_DIR='/usr/lib/dovecot/sieve-pipe' + ln -s "$(type -f -P rspamc)" "${SIEVE_PIPE_BIN_DIR}/rspamc" + + sedfile -i -E 's|(mail_plugins =.*)|\1 imap_sieve|' /etc/dovecot/conf.d/20-imap.conf + sedfile -i -E '/^}/d' /etc/dovecot/conf.d/90-sieve.conf + cat >>/etc/dovecot/conf.d/90-sieve.conf << EOF + + # From elsewhere to Junk folder + imapsieve_mailbox1_name = Junk + imapsieve_mailbox1_causes = COPY + imapsieve_mailbox1_before = file:${SIEVE_PIPE_BIN_DIR}/learn-spam.sieve + + # From Junk folder to elsewhere + imapsieve_mailbox2_name = * + imapsieve_mailbox2_from = Junk + imapsieve_mailbox2_causes = COPY + imapsieve_mailbox2_before = file:${SIEVE_PIPE_BIN_DIR}/learn-ham.sieve +} +EOF + + cat >"${SIEVE_PIPE_BIN_DIR}/learn-spam.sieve" << EOF +require ["vnd.dovecot.pipe", "copy", "imapsieve"]; +pipe :copy "rspamc" ["-h", "127.0.0.1:11334", "learn_spam"]; +EOF + + cat >"${SIEVE_PIPE_BIN_DIR}/learn-ham.sieve" << EOF +require ["vnd.dovecot.pipe", "copy", "imapsieve"]; +pipe :copy "rspamc" ["-h", "127.0.0.1:11334", "learn_ham"]; +EOF + + sievec "${SIEVE_PIPE_BIN_DIR}/learn-spam.sieve" + sievec "${SIEVE_PIPE_BIN_DIR}/learn-ham.sieve" +} diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index 20a94847438..ffdb4528980 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -55,6 +55,7 @@ function __environment_variables_general_setup VARS[POSTGREY_MAX_AGE]="${POSTGREY_MAX_AGE:=35}" VARS[POSTGREY_TEXT]="${POSTGREY_TEXT:=Delayed by Postgrey}" VARS[POSTSCREEN_ACTION]="${POSTSCREEN_ACTION:=enforce}" + VARS[RSPAMD_LEARN]="${RSPAMD_LEARN:=0}" VARS[SA_KILL]=${SA_KILL:="6.31"} VARS[SA_SPAM_SUBJECT]=${SA_SPAM_SUBJECT:="***SPAM*** "} VARS[SA_TAG]=${SA_TAG:="2.0"} diff --git a/test/config/rspamd/postfix-accounts.cf b/test/config/rspamd/postfix-accounts.cf new file mode 100644 index 00000000000..c1b89d3c43f --- /dev/null +++ b/test/config/rspamd/postfix-accounts.cf @@ -0,0 +1,2 @@ +# password is 123 +user1@localhost.localdomain|{SHA512-CRYPT}$6$ARrjj74S83.7c53r$AZXMChav0r03LPEGAJJBmfUt59139rZ1zRIxrPhICSh.Y70Zjq6gClnF/cHDUG95dMoFt4Bkj6N4hvFSZ7L301 diff --git a/test/config/rspamd/user-patches.sh b/test/config/rspamd/user-patches.sh new file mode 100644 index 00000000000..00e6ca97944 --- /dev/null +++ b/test/config/rspamd/user-patches.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +cat >/etc/rspamd/override.d/testmodule_complicated.conf << EOF +complicated { + anOption = someValue; +} +EOF + +echo "enable_test_patterns = true;" >>/etc/rspamd/local.d/options.inc + +echo 'mail_debug = yes' >>/etc/dovecot/dovecot.conf +sed -i -E '/^}/d' /etc/dovecot/conf.d/90-sieve.conf +echo -e 'sieve_trace_debug = yes\n}' >>/etc/dovecot/conf.d/90-sieve.conf diff --git a/test/test-files/email-templates/rspamd-pass.txt b/test/test-files/email-templates/rspamd-pass.txt new file mode 100644 index 00000000000..0f24474040e --- /dev/null +++ b/test/test-files/email-templates/rspamd-pass.txt @@ -0,0 +1,12 @@ +HELO mail.external.tld +MAIL FROM: pass@example.test +RCPT TO: user1@localhost.localdomain +DATA +From: Docker Mail Server +To: Existing Local User +Date: Sat, 22 May 2010 07:43:25 -0400 +Subject: Test Message rspamd-pass.txt +This mail should pass and Rspamd should not mark it. + +. +QUIT diff --git a/test/test-files/email-templates/rspamd-spam-header.txt b/test/test-files/email-templates/rspamd-spam-header.txt new file mode 100644 index 00000000000..7be1a56dcb1 --- /dev/null +++ b/test/test-files/email-templates/rspamd-spam-header.txt @@ -0,0 +1,12 @@ +HELO mail.example.test +MAIL FROM: spam-header@example.test +RCPT TO: user1@localhost.localdomain +DATA +From: Docker Mail Server +To: Existing Local User +Date: Sat, 21 Jan 2023 11:11:11 +0000 +Subject: Test Message rspamd-spam-header.txt +YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X + +. +QUIT diff --git a/test/test-files/email-templates/rspamd-spam.txt b/test/test-files/email-templates/rspamd-spam.txt index 2e6d566a181..88bd719ce73 100644 --- a/test/test-files/email-templates/rspamd-spam.txt +++ b/test/test-files/email-templates/rspamd-spam.txt @@ -6,7 +6,6 @@ From: Docker Mail Server To: Existing Local User Date: Sat, 21 Jan 2023 11:11:11 +0000 Subject: Test Message rspamd-spam.txt -This is a test mail. XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X . diff --git a/test/test-files/email-templates/rspamd-virus.txt b/test/test-files/email-templates/rspamd-virus.txt index cdacf9af43f..c745f2617b2 100644 --- a/test/test-files/email-templates/rspamd-virus.txt +++ b/test/test-files/email-templates/rspamd-virus.txt @@ -2,7 +2,7 @@ HELO mail.example.test MAIL FROM: virus@example.test RCPT TO: user1@localhost.localdomain DATA -From: Docker Mail Server +From: Docker Mail Server To: Existing Local User Date: Sat, 21 Jan 2023 11:11:11 +0000 Subject: Test Message rspamd-virus.txt diff --git a/test/test-files/nc_templates/rspamd_imap_move_to_inbox.txt b/test/test-files/nc_templates/rspamd_imap_move_to_inbox.txt new file mode 100644 index 00000000000..a3f6dc13aa4 --- /dev/null +++ b/test/test-files/nc_templates/rspamd_imap_move_to_inbox.txt @@ -0,0 +1,4 @@ +A LOGIN user1@localhost.localdomain 123 +B SELECT Junk +A UID MOVE 1:1 INBOX +A4 LOGOUT diff --git a/test/test-files/nc_templates/rspamd_imap_move_to_junk.txt b/test/test-files/nc_templates/rspamd_imap_move_to_junk.txt new file mode 100644 index 00000000000..0039d106e9a --- /dev/null +++ b/test/test-files/nc_templates/rspamd_imap_move_to_junk.txt @@ -0,0 +1,4 @@ +A LOGIN user1@localhost.localdomain 123 +B SELECT INBOX +A UID MOVE 1:1 Junk +A4 LOGOUT diff --git a/test/tests/parallel/set1/dovecot/dovecot_sieve.bats b/test/tests/parallel/set1/dovecot/dovecot_sieve.bats index 889890d243e..c4b0112fa07 100644 --- a/test/tests/parallel/set1/dovecot/dovecot_sieve.bats +++ b/test/tests/parallel/set1/dovecot/dovecot_sieve.bats @@ -37,27 +37,24 @@ function teardown_file() { _default_teardown ; } # dovecot-sieve/dovecot.sieve @test "User Sieve - should store mail from 'spam@spam.com' into recipient (user1) mailbox 'INBOX.spam'" { - _run_in_container_bash 'ls -A /var/mail/localhost.localdomain/user1/.INBOX.spam/new' - assert_success - _should_output_number_of_lines 1 + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/.INBOX.spam/new 1 } # dovecot-sieve/before.dovecot.sieve @test "Global Sieve - should have copied mail from 'spam@spam.com' to recipient (user1) inbox" { - _run_in_container grep 'Spambot ' -R /var/mail/localhost.localdomain/user1/new/ + _run_in_container grep -R 'Spambot ' /var/mail/localhost.localdomain/user1/new/ assert_success } # dovecot-sieve/sieve-pipe + dovecot-sieve/user2@otherdomain.tld.dovecot.sieve @test "Sieve Pipe - should pipe mail received for user2 into '/tmp/pipe-test.out'" { - _run_in_container_bash 'ls -A /tmp/pipe-test.out' + _run_in_container_bash '[[ -f /tmp/pipe-test.out ]]' assert_success - _should_output_number_of_lines 1 } # Only test coverage for feature is to check that the service is listening on the expected port: # https://doc.dovecot.org/admin_manual/pigeonhole_managesieve_server/ @test "ENV 'ENABLE_MANAGESIEVE' - should have enabled service on port 4190" { - _run_in_container_bash 'nc -z 0.0.0.0 4190' + _run_in_container nc -z 0.0.0.0 4190 assert_success } diff --git a/test/tests/parallel/set1/spam_virus/rspamd.bats b/test/tests/parallel/set1/spam_virus/rspamd.bats index 6c3855531ce..658a0e6850b 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd.bats @@ -16,8 +16,11 @@ function setup_file() { --env ENABLE_OPENDMARC=0 --env PERMIT_DOCKER=host --env LOG_LEVEL=trace + --env MOVE_SPAM_TO_JUNK=1 + --env RSPAMD_LEARN=1 ) + mv "${TEST_TMP_CONFIG}"/rspamd/* "${TEST_TMP_CONFIG}/" _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' # wait for ClamAV to be fully setup or we will get errors on the log @@ -31,12 +34,15 @@ function setup_file() { # We will send 3 emails: the first one should pass just fine; the second one should # be rejected due to spam; the third one should be rejected due to a virus. - export MAIL_ID1=$(_send_email_and_get_id 'email-templates/existing-user1') + export MAIL_ID1=$(_send_email_and_get_id 'email-templates/rspamd-pass') export MAIL_ID2=$(_send_email_and_get_id 'email-templates/rspamd-spam') export MAIL_ID3=$(_send_email_and_get_id 'email-templates/rspamd-virus') + export MAIL_ID4=$(_send_email_and_get_id 'email-templates/rspamd-spam-header') - # add a nested option to a module - _exec_in_container_bash "echo -e 'complicated {\n anOption = someValue;\n}' >/etc/rspamd/override.d/testmodule_complicated.conf" + for ID in MAIL_ID{1,2,3,4} + do + [[ -n ${!ID} ]] || { echo "${ID} is empty - aborting!" ; return 1 ; } + done } function teardown_file() { _default_teardown ; } @@ -44,6 +50,9 @@ function teardown_file() { _default_teardown ; } @test "Postfix's main.cf was adjusted" { _run_in_container grep -F 'smtpd_milters = $rspamd_milter' /etc/postfix/main.cf assert_success + _run_in_container postconf rspamd_milter + assert_success + assert_output 'rspamd_milter = inet:localhost:11332' } @test 'logs exist and contains proper content' { @@ -62,6 +71,8 @@ function teardown_file() { _default_teardown ; } _print_mail_log_for_id "${MAIL_ID1}" assert_output --partial "stored mail into mailbox 'INBOX'" + + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 } @test 'detects and rejects spam' { @@ -71,6 +82,8 @@ function teardown_file() { _default_teardown ; } _print_mail_log_for_id "${MAIL_ID2}" assert_output --partial 'milter-reject' assert_output --partial '5.7.1 Gtube pattern' + + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 } @test 'detects and rejects virus' { @@ -81,6 +94,8 @@ function teardown_file() { _default_teardown ; } assert_output --partial 'milter-reject' assert_output --partial '5.7.1 ClamAV FOUND VIRUS "Eicar-Signature"' refute_output --partial "stored mail into mailbox 'INBOX'" + + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 } @test 'custom commands work correctly' { @@ -153,3 +168,78 @@ function teardown_file() { _default_teardown ; } _run_in_container grep -F 'OhMy = "PraiseBeLinters !";' "${MODULE_PATH}" assert_success } + +@test 'Check MOVE_SPAM_TO_JUNK works for Rspamd' { + _run_in_container_bash '[[ -f /usr/lib/dovecot/sieve-global/after/spam_to_junk.sieve ]]' + assert_success + _run_in_container_bash '[[ -f /usr/lib/dovecot/sieve-global/after/spam_to_junk.svbin ]]' + assert_success + + _service_log_should_contain_string 'rspamd' 'S \(add header\)' + _service_log_should_contain_string 'rspamd' 'add header "Gtube pattern"' + + _print_mail_log_for_id "${MAIL_ID4}" + assert_output --partial "fileinto action: stored mail into mailbox 'Junk'" + + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/.Junk/new/ 1 +} + +@test 'Check RSPAMD_LEARN works' { + for FILE in learn-{ham,spam}.{sieve,svbin} + do + _run_in_container_bash "[[ -f /usr/lib/dovecot/sieve-pipe/${FILE} ]]" + assert_success + done + + _run_in_container grep 'mail_plugins.*imap_sieve' /etc/dovecot/conf.d/20-imap.conf + local SIEVE_CONFIG_FILE='/etc/dovecot/conf.d/90-sieve.conf' + _run_in_container grep 'sieve_plugins.*sieve_imapsieve' "${SIEVE_CONFIG_FILE}" + _run_in_container grep 'sieve_global_extensions.*\+vnd\.dovecot\.pipe' "${SIEVE_CONFIG_FILE}" + _run_in_container grep -F 'sieve_pipe_bin_dir = /usr/lib/dovecot/sieve-pipe' "${SIEVE_CONFIG_FILE}" + + # Move an email to the "Junk" folder from "INBOX"; the first email we + # sent should pass fine, hence we can now move it + _send_email 'nc_templates/rspamd_imap_move_to_junk' '0.0.0.0 143' + sleep 1 # wait for the transaction to finish + + local MOVE_TO_JUNK_LINES=( + 'imapsieve: mailbox Junk: MOVE event' + 'imapsieve: Matched static mailbox rule [1]' + "sieve: file storage: script: Opened script \`learn-spam'" + 'sieve: file storage: Using Sieve script path: /usr/lib/dovecot/sieve-pipe/learn-spam.sieve' + "sieve: Executing script from \`/usr/lib/dovecot/sieve-pipe/learn-spam.svbin'" + "Finished running script \`/usr/lib/dovecot/sieve-pipe/learn-spam.svbin'" + 'sieve: action pipe: running program: rspamc' + "pipe action: piped message to program \`rspamc'" + "left message in mailbox 'Junk'" + ) + + _run_in_container cat /var/log/mail/mail.log + assert_success + for LINE in "${MOVE_TO_JUNK_LINES[@]}" + do + assert_output --partial "${LINE}" + done + + # Move an email to the "INBOX" folder from "Junk"; there should be two mails + # in the "Junk" folder + _send_email 'nc_templates/rspamd_imap_move_to_inbox' '0.0.0.0 143' + sleep 1 # wait for the transaction to finish + + local MOVE_TO_JUNK_LINES=( + 'imapsieve: Matched static mailbox rule [2]' + "sieve: file storage: script: Opened script \`learn-ham'" + 'sieve: file storage: Using Sieve script path: /usr/lib/dovecot/sieve-pipe/learn-ham.sieve' + "sieve: Executing script from \`/usr/lib/dovecot/sieve-pipe/learn-ham.svbin'" + "Finished running script \`/usr/lib/dovecot/sieve-pipe/learn-ham.svbin'" + "left message in mailbox 'INBOX'" + ) + + _run_in_container cat /var/log/mail/mail.log + assert_success + for LINE in "${MOVE_TO_JUNK_LINES[@]}" + do + assert_output --partial "${LINE}" + done +} From d770c67a2daff475430df4cc30b1f85e4a7799b6 Mon Sep 17 00:00:00 2001 From: Jack Pearson Date: Sun, 19 Mar 2023 15:37:24 -0700 Subject: [PATCH 006/592] ci(docs): Update `latest` symlink via docs-production-deploy workflow (#3183) --- .github/workflows/docs-production-deploy.yml | 30 ++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/docs-production-deploy.yml b/.github/workflows/docs-production-deploy.yml index 0e898952bcd..0694bbe71bc 100644 --- a/.github/workflows/docs-production-deploy.yml +++ b/.github/workflows/docs-production-deploy.yml @@ -105,3 +105,33 @@ jobs: git add versions.json git commit -m "chore: Add ${{ env.DOCS_VERSION }} to version selector list" git push + + update-latest-symlink: + permissions: + contents: write + name: 'update `latest` symlink if neccessary' + runs-on: ubuntu-22.04 + if: startsWith(github.ref, 'refs/tags/') + needs: add-version-to-docs + steps: + - name: 'Checkout the docs deployment branch' + uses: actions/checkout@v3 + with: + ref: gh-pages + + - name: 'Ensure `latest` symlink refers to the substring from tag name' + id: update-latest + run: | + DOCS_VERSION=$(grep -oE 'v[0-9]+\.[0-9]+' <<< "${GITHUB_REF}") + echo "DOCS_VERSION=${DOCS_VERSION}" >> "${GITHUB_ENV}" + ln -sf ${DOCS_VERSION} latest + + - name: 'Push update for `latest` symlink' + # The step will fail if no change was made; ignore that failure + continue-on-error: true + run: | + git config user.name ${{ env.GIT_USER }} + git config user.email ${{ env.GIT_EMAIL }} + git add latest + git commit -m "chore: Update \`latest\` symlink to point to ${{ env.DOCS_VERSION }}" + git push From 1bd25d5e74a5d29392070bca38497783a60bbc35 Mon Sep 17 00:00:00 2001 From: Jack Pearson Date: Mon, 20 Mar 2023 19:56:54 -0700 Subject: [PATCH 007/592] docs: Add FAQ entry for troubleshooting delivery (#3192) * docs: add faq for email deliverability * Apply suggestions from code review --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/faq.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/content/faq.md b/docs/content/faq.md index a750d3f3b0e..78d7bb0aeb3 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -332,6 +332,19 @@ The default bantime is 180 days. This value can be [customized][fail2ban-customi If you got any problems with SPF and/or forwarding mails, give [SRS](https://github.com/roehling/postsrsd/blob/master/README.rst) a try. You enable SRS by setting `ENABLE_SRS=1`. See the variable description for further information. +### Why are my emails not being delivered? + +There are many reasons why email might be rejected, common causes are: + +- Wrong or untrustworthy SSL certificate. +- A TLD (your domain) or IP address with a bad reputation. +- Misconfigured DNS records. + +DMS does not manage those concerns, verify they are not causing your delivery problems before reporting a bug on our issue tracker. Resources that can help you troubleshoot: + +- [mail-tester](https://www.mail-tester.com/) can test your deliverability. +- [helloinbox](https://www.helloinbox.email/) provides a checklist of things to improve your deliverability. + ### SpamAssasin #### How can I manage my custom SpamAssassin rules? From b3249fada776bc1787ced47d35d47211a82bbeca Mon Sep 17 00:00:00 2001 From: Jack Pearson Date: Mon, 20 Mar 2023 22:14:10 -0700 Subject: [PATCH 008/592] docs: move `make build` instruction from paragraph into list (#3193) * docs: move `make build` instruction from paragraph into list * Update docs/content/contributing/tests.md --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/contributing/tests.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/content/contributing/tests.md b/docs/content/contributing/tests.md index 7e24e8f014f..ba63bf74c43 100644 --- a/docs/content/contributing/tests.md +++ b/docs/content/contributing/tests.md @@ -56,12 +56,13 @@ To run the test suite, you will need to: ### Executing Test(s) -We use `make` to run commands. You will first need to build the container image via `make build`. You can then: +We use `make` to run commands. -1. Run all tests: `make clean tests` -2. Run a single test: `make clean generate-accounts test/` -3. Run multiple unrelated tests: `make clean generate-accounts test/,` (just add a `,` and then immediately write the new test name) -4. Run a whole set or all serial tests: `make clean generate-accounts tests/parallel/setX` where `X` is the number of the set or `make clean generate-accounts tests/serial` +1. Run `make build` to create or update the local `mailserver-testing:ci` Docker image (_using the projects `Dockerfile`_) +2. Run all tests: `make clean tests` +3. Run a single test: `make clean generate-accounts test/` +4. Run multiple unrelated tests: `make clean generate-accounts test/,` (just add a `,` and then immediately write the new test name) +5. Run a whole set or all serial tests: `make clean generate-accounts tests/parallel/setX` where `X` is the number of the set or `make clean generate-accounts tests/serial` ??? tip "Setting the Degree of Parallelization for Tests" From e12b032f7700e5968df6cfa10dccb0a504811c8d Mon Sep 17 00:00:00 2001 From: Jack Pearson Date: Wed, 22 Mar 2023 03:43:10 -0700 Subject: [PATCH 009/592] docs: Change `edge` version links to `latest` + fix links intended as relative not absolute (#3190) * docs: change some absolute links to relative links * docs: change most hard-coded links to `edge` to point to `latest` * Apply suggestions from code review * docs: revert 404 page to edge and change canonical link to `latest --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- .github/workflows/docs-production-deploy.yml | 2 +- CHANGELOG.md | 4 ++-- README.md | 18 +++++++++--------- docker-compose.yml | 2 +- docs/content/config/advanced/podman.md | 2 +- .../contributing/issues-and-pull-requests.md | 6 +++--- .../examples/tutorials/basic-installation.md | 4 ++-- docs/content/faq.md | 2 +- docs/content/index.md | 2 +- docs/content/usage.md | 4 ++-- mailserver.env | 4 ++-- 11 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/docs-production-deploy.yml b/.github/workflows/docs-production-deploy.yml index 0694bbe71bc..38219a9633f 100644 --- a/.github/workflows/docs-production-deploy.yml +++ b/.github/workflows/docs-production-deploy.yml @@ -55,7 +55,7 @@ jobs: # Replace the tagged '${DOCS_VERSION}' in the 'canonical' link element of HTML files, # to point to the 'edge' version of docs as the authoritative source: find . -type f -name "*.html" -exec \ - sed -i "s|^\(.* + - Documentation is now versioned related to docker image versions and viewable here: ## `v9.0.1` diff --git a/README.md b/README.md index d318fcd24c5..b549a59300c 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [docker::pulls]: https://img.shields.io/docker/pulls/mailserver/docker-mailserver.svg?style=for-the-badge&logo=docker&logoColor=white [docker::hub]: https://hub.docker.com/r/mailserver/docker-mailserver/ [documentation::badge]: https://img.shields.io/badge/DOCUMENTATION-GH%20PAGES-0078D4?style=for-the-badge&logo=git&logoColor=white -[documentation::web]: https://docker-mailserver.github.io/docker-mailserver/edge/ +[documentation::web]: https://docker-mailserver.github.io/docker-mailserver/latest/ ## :page_with_curl: About @@ -23,18 +23,18 @@ If you have issues, please search through [the documentation][documentation::web ## :link: Links to Useful Resources -1. [FAQ](https://docker-mailserver.github.io/docker-mailserver/edge/faq/) -2. [Usage](https://docker-mailserver.github.io/docker-mailserver/edge/usage/) -3. [Examples](https://docker-mailserver.github.io/docker-mailserver/edge/examples/tutorials/basic-installation/) -4. [Issues and Contributing](https://docker-mailserver.github.io/docker-mailserver/edge/contributing/issues-and-pull-requests/) +1. [FAQ](https://docker-mailserver.github.io/docker-mailserver/latest/faq/) +2. [Usage](https://docker-mailserver.github.io/docker-mailserver/latest/usage/) +3. [Examples](https://docker-mailserver.github.io/docker-mailserver/latest/examples/tutorials/basic-installation/) +4. [Issues and Contributing](https://docker-mailserver.github.io/docker-mailserver/latest/contributing/issues-and-pull-requests/) 5. [Release Notes](./CHANGELOG.md) -6. [Environment Variables](https://docker-mailserver.github.io/docker-mailserver/edge/config/environment/) -7. [Updating](https://docker-mailserver.github.io/docker-mailserver/edge/faq/#how-do-i-update-dms) +6. [Environment Variables](https://docker-mailserver.github.io/docker-mailserver/latest/config/environment/) +7. [Updating](https://docker-mailserver.github.io/docker-mailserver/latest/faq/#how-do-i-update-dms) ## :package: Included Services - [Postfix](http://www.postfix.org) with SMTP or LDAP authentication and support for [extension delimiters](http://www.postfix.org/postconf.5.html#recipient_delimiter) (_mail to `you+extension@example.com` delivered to `you@example.com`_) -- [Dovecot](https://www.dovecot.org) with SASL, IMAP, POP3, LDAP, [basic Sieve support](https://docker-mailserver.github.io/docker-mailserver/edge/config/advanced/mail-sieve) and [quotas](https://docker-mailserver.github.io/docker-mailserver/edge/config/user-management/accounts#notes) +- [Dovecot](https://www.dovecot.org) with SASL, IMAP, POP3, LDAP, [basic Sieve support](https://docker-mailserver.github.io/docker-mailserver/latest/config/advanced/mail-sieve) and [quotas](https://docker-mailserver.github.io/docker-mailserver/latest/config/user-management/accounts#notes) - [Rspamd](https://rspamd.com/) - [Amavis](https://www.amavis.org/) - [SpamAssassin](http://spamassassin.apache.org/) supporting custom rules @@ -45,5 +45,5 @@ If you have issues, please search through [the documentation][documentation::web - [Postscreen](http://www.postfix.org/POSTSCREEN_README.html) - [Postgrey](https://postgrey.schweikert.ch/) - Support for [LetsEncrypt](https://letsencrypt.org/), manual and self-signed certificates -- A [setup script](https://docker-mailserver.github.io/docker-mailserver/edge/config/setup.sh) for easy configuration and maintenance +- A [setup script](https://docker-mailserver.github.io/docker-mailserver/latest/config/setup.sh) for easy configuration and maintenance - SASLauthd with LDAP authentication diff --git a/docker-compose.yml b/docker-compose.yml index ee5f7a86e1d..5bbaaa3e318 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,7 @@ services: domainname: example.com env_file: mailserver.env # More information about the mail-server ports: - # https://docker-mailserver.github.io/docker-mailserver/edge/config/security/understanding-the-ports/ + # https://docker-mailserver.github.io/docker-mailserver/latest/config/security/understanding-the-ports/ # To avoid conflicts with yaml base-60 float, DO NOT remove the quotation marks. ports: - "25:25" # SMTP (explicit TLS => STARTTLS) diff --git a/docs/content/config/advanced/podman.md b/docs/content/config/advanced/podman.md index 599132ccc9b..4e6d9d03d4d 100644 --- a/docs/content/config/advanced/podman.md +++ b/docs/content/config/advanced/podman.md @@ -107,7 +107,7 @@ The `PERMIT_DOCKER` variable in the `mailserver.env` file allows to specify trus #### Use the slip4netns network driver The second workaround is slightly more complicated because the `docker-compose.yml` has to be modified. -As shown in the [fail2ban section](https://docker-mailserver.github.io/docker-mailserver/edge/config/security/fail2ban/#podman-with-slirp4netns-port-driver) the `slirp4netns` network driver has to be enabled. +As shown in the [fail2ban section](../../security/fail2ban/#podman-with-slirp4netns-port-driver) the `slirp4netns` network driver has to be enabled. This network driver enables podman to correctly resolve IP addresses but it is not compatible with user defined networks which might be a problem depending on your setup. diff --git a/docs/content/contributing/issues-and-pull-requests.md b/docs/content/contributing/issues-and-pull-requests.md index 3851ea9d29d..60d44b3ca54 100644 --- a/docs/content/contributing/issues-and-pull-requests.md +++ b/docs/content/contributing/issues-and-pull-requests.md @@ -8,13 +8,13 @@ This project is Open Source. That means that you can contribute on enhancements, !!! attention - **Before opening an issue**, read the [`README`][github-file-readme] carefully, study the [documentation][docs], the Postfix/Dovecot documentation and your search engine you trust. The issue tracker is not meant to be used for unrelated questions! + **Before opening an issue**, read the [`README`][github-file-readme] carefully, study the docs for your version (maybe [latest][docs-latest]), the Postfix/Dovecot documentation and your search engine you trust. The issue tracker is not meant to be used for unrelated questions! When opening an issue, please provide details use case to let the community reproduce your problem. Please start `docker-mailserver` with the environment variable `LOG_LEVEL` set to `debug` or `trace` and paste the output into the issue. !!! attention - **Use the issue templates** to provide the necessary information. Issues which do not use these templates are not worked on and closed. + **Use the issue templates** to provide the necessary information. Issues which do not use these templates are not worked on and closed. By raising issues, I agree to these terms and I understand, that the rules set for the issue tracker will help both maintainers as well as everyone to find a solution. @@ -39,7 +39,7 @@ The development workflow is the following: Pull requests are automatically tested against the CI and will be reviewed when tests pass. When your changes are validated, your branch is merged. CI builds the new `:edge` image immediately and your changes will be includes in the next version release. -[docs]: https://docker-mailserver.github.io/docker-mailserver/edge +[docs-latest]: https://docker-mailserver.github.io/docker-mailserver/latest [github-file-readme]: https://github.com/docker-mailserver/docker-mailserver/blob/master/README.md [docs-environment]: ../config/environment.md [docs-general-tests]: ./general.md#tests diff --git a/docs/content/examples/tutorials/basic-installation.md b/docs/content/examples/tutorials/basic-installation.md index 15c6bd979b7..150d933170c 100644 --- a/docs/content/examples/tutorials/basic-installation.md +++ b/docs/content/examples/tutorials/basic-installation.md @@ -129,9 +129,9 @@ In this setup `docker-mailserver` is not intended to receive email externally, s # Using letsencrypt for SSL/TLS certificates - SSL_TYPE=letsencrypt # Allow sending emails from other docker containers - # Beware creating an Open Relay: https://docker-mailserver.github.io/docker-mailserver/edge/config/environment/#permit_docker + # Beware creating an Open Relay: https://docker-mailserver.github.io/docker-mailserver/latest/config/environment/#permit_docker - PERMIT_DOCKER=network - # You may want to enable this: https://docker-mailserver.github.io/docker-mailserver/edge/config/environment/#spoof_protection + # You may want to enable this: https://docker-mailserver.github.io/docker-mailserver/latest/config/environment/#spoof_protection # See step 8 below, which demonstrates setup with enabled/disabled SPOOF_PROTECTION: - SPOOF_PROTECTION=0 cap_add: diff --git a/docs/content/faq.md b/docs/content/faq.md index 78d7bb0aeb3..abe384b8c56 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -12,7 +12,7 @@ Mails are stored in `/var/mail/${domain}/${username}`. Since `v9.0.0` it is poss ### How are IMAP mailboxes (_aka IMAP Folders_) set up? -`INBOX` is setup by default with the special IMAP folders `Drafts`, `Sent`, `Junk` and `Trash`. You can learn how to modify or add your own folders (_including additional special folders like `Archive`_) by visiting our docs page [_Customizing IMAP Folders_](https://docker-mailserver.github.io/docker-mailserver/edge/examples/use-cases/imap-folders) for more information. +`INBOX` is setup by default with the special IMAP folders `Drafts`, `Sent`, `Junk` and `Trash`. You can learn how to modify or add your own folders (_including additional special folders like `Archive`_) by visiting our docs page [_Customizing IMAP Folders_](../examples/use-cases/imap-folders) for more information. ### How do I update DMS? diff --git a/docs/content/index.md b/docs/content/index.md index e3f86ff823d..b22d5fc5574 100644 --- a/docs/content/index.md +++ b/docs/content/index.md @@ -6,7 +6,7 @@ title: Home !!! info "This Documentation is Versioned" - **Make sure** to select the correct version of this documentation! It should match the version of the image you are using. The default version corresponds to the `:edge` image tag - [the most recent build, not the most recent stable release][docs-tagging]. + **Make sure** to select the correct version of this documentation! It should match the version of the image you are using. The default version corresponds to the `:latest` image tag - [the most recent stable release][docs-tagging]. This documentation provides you not only with the basic setup and configuration of DMS but also with advanced configuration, elaborate usage scenarios, detailed examples, hints and more. diff --git a/docs/content/usage.md b/docs/content/usage.md index 4ece264c348..8361102b284 100644 --- a/docs/content/usage.md +++ b/docs/content/usage.md @@ -42,13 +42,13 @@ chmod a+x ./setup.sh 2. Edit `docker-compose.yml` to your liking - substitute `mail.example.com` according to your FQDN - if you want to use SELinux for the `./docker-data/dms/config/:/tmp/docker-mailserver/` mount, append `-z` or `-Z` -3. Configure the mailserver container to your liking by editing `mailserver.env` ([**Documentation**](https://docker-mailserver.github.io/docker-mailserver/edge/config/environment/)), but keep in mind this `.env` file: +3. Configure the mailserver container to your liking by editing `mailserver.env` ([**Documentation**](../config/environment/)), but keep in mind this `.env` file: - [_only_ basic `VAR=VAL`](https://docs.docker.com/compose/env-file/) is supported (**do not** quote your values) - variable substitution is **not** supported (e.g. :no_entry_sign: `OVERRIDE_HOSTNAME=$HOSTNAME.$DOMAINNAME` :no_entry_sign:) !!! info "Podman Support" - If you're using podman, make sure to read the related [documentation](https://docker-mailserver.github.io/docker-mailserver/edge/config/advanced/podman/) + If you're using podman, make sure to read the related [documentation](../config/advanced/podman/) ## Get up and running diff --git a/mailserver.env b/mailserver.env index 68af83b0420..09a40b8e83c 100644 --- a/mailserver.env +++ b/mailserver.env @@ -3,7 +3,7 @@ # ----------------------------------------------- # DOCUMENTATION FOR THESE VARIABLES IS FOUND UNDER -# https://docker-mailserver.github.io/docker-mailserver/edge/config/environment/ +# https://docker-mailserver.github.io/docker-mailserver/latest/config/environment/ # ----------------------------------------------- # --- General Section --------------------------- @@ -166,7 +166,7 @@ POSTSCREEN_ACTION=enforce # 1 => only launch postfix smtp SMTP_ONLY= -# Please read [the SSL page in the documentation](https://docker-mailserver.github.io/docker-mailserver/edge/config/security/ssl) for more information. +# Please read [the SSL page in the documentation](https://docker-mailserver.github.io/docker-mailserver/latest/config/security/ssl) for more information. # # empty => SSL disabled # letsencrypt => Enables Let's Encrypt certificates From 6fa06f4986f11eb882b8bd0db0338f20389df1dc Mon Sep 17 00:00:00 2001 From: Casper Date: Sun, 26 Mar 2023 14:30:34 +0200 Subject: [PATCH 010/592] Fix: only chmod when there are files (#3203) --- target/scripts/startup/setup.d/dovecot.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/target/scripts/startup/setup.d/dovecot.sh b/target/scripts/startup/setup.d/dovecot.sh index fbb01871ada..2d03b030ae5 100644 --- a/target/scripts/startup/setup.d/dovecot.sh +++ b/target/scripts/startup/setup.d/dovecot.sh @@ -87,8 +87,8 @@ function _setup_dovecot_sieve fi chown dovecot:root -R /usr/lib/dovecot/sieve-* - find /usr/lib/dovecot/sieve-* -type d -exec chmod 755 {} \; - chmod +x /usr/lib/dovecot/sieve-{filter,pipe}/* + find /usr/lib/dovecot/sieve-* -type d -exec chmod 755 {} + + find /usr/lib/dovecot/sieve-{filter,pipe} -type f -exec chmod +x {} + } function _setup_dovecot_quota From f4fe5bf527f019c25ad3b294e9939f1a32188d10 Mon Sep 17 00:00:00 2001 From: Casper Date: Mon, 27 Mar 2023 01:59:43 +0200 Subject: [PATCH 011/592] Update SA_KILL values; follow up to #3058 (#3204) --- docs/content/config/environment.md | 2 +- target/scripts/startup/variables-stack.sh | 2 +- test/tests/parallel/set1/spam_virus/amavis.bats | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 7dbe8f788ea..582c4dad171 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -464,7 +464,7 @@ Note: this SpamAssassin setting needs `ENABLE_SPAMASSASSIN=1` ##### SA_KILL -- **6.31** => triggers spam evasive actions +- **10.0** => triggers spam evasive actions !!! note "This SpamAssassin setting needs `ENABLE_SPAMASSASSIN=1`" diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index ffdb4528980..854121971dd 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -56,7 +56,7 @@ function __environment_variables_general_setup VARS[POSTGREY_TEXT]="${POSTGREY_TEXT:=Delayed by Postgrey}" VARS[POSTSCREEN_ACTION]="${POSTSCREEN_ACTION:=enforce}" VARS[RSPAMD_LEARN]="${RSPAMD_LEARN:=0}" - VARS[SA_KILL]=${SA_KILL:="6.31"} + VARS[SA_KILL]=${SA_KILL:="10.0"} VARS[SA_SPAM_SUBJECT]=${SA_SPAM_SUBJECT:="***SPAM*** "} VARS[SA_TAG]=${SA_TAG:="2.0"} VARS[SA_TAG2]=${SA_TAG2:="6.31"} diff --git a/test/tests/parallel/set1/spam_virus/amavis.bats b/test/tests/parallel/set1/spam_virus/amavis.bats index 875127b1465..ba737806809 100644 --- a/test/tests/parallel/set1/spam_virus/amavis.bats +++ b/test/tests/parallel/set1/spam_virus/amavis.bats @@ -71,7 +71,7 @@ function teardown_file() { _run_in_container grep '\$sa_kill_level_deflt' "${AMAVIS_DEFAULTS_FILE}" assert_success - assert_output --partial '= 6.31' + assert_output --partial '= 10.0' _run_in_container grep '\$sa_spam_subject_tag' "${AMAVIS_DEFAULTS_FILE}" assert_success From 1c231053d0a6efe93cb02112142719c5adc567af Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Mar 2023 01:28:18 +0200 Subject: [PATCH 012/592] chore(deps): Bump actions/stale from 7 to 8 (#3205) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/handle_stalled.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/handle_stalled.yml b/.github/workflows/handle_stalled.yml index 78bdb26b08b..236dcec9428 100644 --- a/.github/workflows/handle_stalled.yml +++ b/.github/workflows/handle_stalled.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Close stale issues - uses: actions/stale@v7 + uses: actions/stale@v8 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 20 From 585a2d64d27d02dd1547af050be58cbc654f82fb Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 31 Mar 2023 12:17:44 +0200 Subject: [PATCH 013/592] config: remove `chroot` for Dovecot & PostSRSd (#3208) * remove PostSRSd chroot * remove chroot for Dovecot A dedicated file for Dovecot's chroot environments is easier to handle and adjust later. --- target/dovecot/10-master.conf | 2 ++ target/dovecot/chroot.inc | 47 +++++++++++++++++++++++++++++++++++ target/postsrsd/postsrsd | 3 --- 3 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 target/dovecot/chroot.inc diff --git a/target/dovecot/10-master.conf b/target/dovecot/10-master.conf index 2c5fa6baf6d..e037367e859 100644 --- a/target/dovecot/10-master.conf +++ b/target/dovecot/10-master.conf @@ -114,3 +114,5 @@ service dict { #group = } } + +!include chroot.inc diff --git a/target/dovecot/chroot.inc b/target/dovecot/chroot.inc new file mode 100644 index 00000000000..dccffa1e335 --- /dev/null +++ b/target/dovecot/chroot.inc @@ -0,0 +1,47 @@ +# This file removes `chroot` environments that +# +# 1. are not strictly needed +# 2. can cause problems +# +# See https://github.com/docker-mailserver/docker-mailserver/pull/3208#pullrequestreview-1366106516 +# and it's related PRs. + +service aggregator { + chroot = +} + +service anvil { + chroot = +} + +service director { + chroot = +} + +service ipc { + chroot = +} + +service old-stats { + chroot = +} + +service imap-login { + chroot = +} + +service managesieve-login { + chroot = +} + +service pop3-login { + chroot = +} + +service submission-login { + chroot = +} + +service imap-urlauth-login { + chroot = +} diff --git a/target/postsrsd/postsrsd b/target/postsrsd/postsrsd index 1ec1822a040..91b648c09f1 100644 --- a/target/postsrsd/postsrsd +++ b/target/postsrsd/postsrsd @@ -36,6 +36,3 @@ SRS_REVERSE_PORT=10002 # This is highly recommended as postsrsd handles untrusted input. # RUN_AS=postsrsd - -# Jail daemon in chroot environment -CHROOT=/var/lib/postsrsd From 78c3200b7ce4048a488d7fbd11f56f0db417e8c9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 1 Apr 2023 16:23:55 +1300 Subject: [PATCH 014/592] docs(CONTRIBUTORS): update contributors (#3210) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 156 ++++++++++++++++++++++++++---------------------- 1 file changed, 85 insertions(+), 71 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 83e766a2e0a..cbd2653b7dc 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -215,15 +215,6 @@ Thanks goes to these wonderful people ✨ Kyle Ondy - - - + + - - + + + + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - +
- - Daniel -
- Daniel Panteleit - -
Michael/ @@ -231,6 +222,8 @@ Thanks goes to these wonderful people ✨ Michael
lukas/ @@ -266,8 +259,6 @@ Thanks goes to these wonderful people ✨ vortex852456
Christian @@ -275,6 +266,8 @@ Thanks goes to these wonderful people ✨ Christian Grasso
Hans-Cees @@ -282,6 +275,13 @@ Thanks goes to these wonderful people ✨ Hans-Cees Speel + + Jack +
+ Jack Pearson + + 		Dashamir @@ -567,6 +567,15 @@ Thanks goes to these wonderful people ✨ Christian Raue + + Daniel +
+ Daniel Panteleit + +
Darren @@ -574,8 +583,6 @@ Thanks goes to these wonderful people ✨ Darren McGrandle
Dominik @@ -611,6 +618,8 @@ Thanks goes to these wonderful people ✨ Guillaume Simon
Ikko @@ -618,8 +627,6 @@ Thanks goes to these wonderful people ✨ Ikko Eltociear Ashimine
James @@ -655,6 +662,8 @@ Thanks goes to these wonderful people ✨ Louis
martinwepner/ @@ -662,8 +671,6 @@ Thanks goes to these wonderful people ✨ martinwepner
Michael @@ -699,6 +706,8 @@ Thanks goes to these wonderful people ✨ Rainer Rillke
Bob @@ -706,8 +715,6 @@ Thanks goes to these wonderful people ✨ Bob Gregor
r-pufky/ @@ -743,6 +750,8 @@ Thanks goes to these wonderful people ✨ j-marz
lokipo/ @@ -750,8 +759,6 @@ Thanks goes to these wonderful people ✨ lokipo
msheakoski/ @@ -787,6 +794,8 @@ Thanks goes to these wonderful people ✨ Thomas Willems
0xflotus/ @@ -794,8 +803,6 @@ Thanks goes to these wonderful people ✨ 0xflotus
Ivan @@ -831,6 +838,8 @@ Thanks goes to these wonderful people ✨ Achim Christ
Adrian @@ -838,8 +847,6 @@ Thanks goes to these wonderful people ✨ Adrian Pistol
Alexander @@ -875,6 +882,8 @@ Thanks goes to these wonderful people ✨ Andrew Cornford
Andrey @@ -882,8 +891,6 @@ Thanks goes to these wonderful people ✨ Andrey Likhodievskiy
Arash @@ -919,6 +926,8 @@ Thanks goes to these wonderful people ✨ Bogdan
Charles @@ -926,8 +935,6 @@ Thanks goes to these wonderful people ✨ Charles Harris
Christian @@ -963,6 +970,8 @@ Thanks goes to these wonderful people ✨ espitall
Daniel @@ -970,8 +979,6 @@ Thanks goes to these wonderful people ✨ Daniel Karski
Daniele @@ -1007,6 +1014,8 @@ Thanks goes to these wonderful people ✨ Dorian Ayllón
Edmond @@ -1014,8 +1023,6 @@ Thanks goes to these wonderful people ✨ Edmond Varga
Eduard @@ -1051,6 +1058,8 @@ Thanks goes to these wonderful people ✨ Huncode
Florian/ @@ -1058,8 +1067,6 @@ Thanks goes to these wonderful people ✨ Florian
Florian @@ -1095,6 +1102,8 @@ Thanks goes to these wonderful people ✨ Gabriel Landais
GiovanH/ @@ -1102,8 +1111,6 @@ Thanks goes to these wonderful people ✨ GiovanH
H4R0/ @@ -1139,6 +1146,8 @@ Thanks goes to these wonderful people ✨ Influencer
jcalfee/ @@ -1146,8 +1155,6 @@ Thanks goes to these wonderful people ✨ jcalfee
JS @@ -1183,6 +1190,8 @@ Thanks goes to these wonderful people ✨ jmccl
Jurek @@ -1190,8 +1199,6 @@ Thanks goes to these wonderful people ✨ Jurek Barth
JOnathan @@ -1227,6 +1234,8 @@ Thanks goes to these wonderful people ✨ Khue Doan
Lars @@ -1234,8 +1243,6 @@ Thanks goes to these wonderful people ✨ Lars Pötter
Leo @@ -1243,6 +1250,13 @@ Thanks goes to these wonderful people ✨ Leo Winter + + Lin +
+ Lin Han + + 		MadsRC/ @@ -1264,6 +1278,8 @@ Thanks goes to these wonderful people ✨ Maximilian Hippler
Michael @@ -1278,8 +1294,6 @@ Thanks goes to these wonderful people ✨ Michael Jensen
Michel @@ -1308,6 +1322,8 @@ Thanks goes to these wonderful people ✨ Moritz Poldrack
Naveen/ @@ -1322,8 +1338,6 @@ Thanks goes to these wonderful people ✨ Nicholas Pepper
Nick @@ -1352,6 +1366,8 @@ Thanks goes to these wonderful people ✨ Orville Q. Song
Ovidiu @@ -1366,8 +1382,6 @@ Thanks goes to these wonderful people ✨ Petar Šegina
Peter @@ -1396,6 +1410,8 @@ Thanks goes to these wonderful people ✨ René Plötz
Roman @@ -1410,8 +1426,6 @@ Thanks goes to these wonderful people ✨ Sam Collins
Scott @@ -1440,6 +1454,8 @@ Thanks goes to these wonderful people ✨ Sergey Nazaryev
Shyim/ @@ -1454,8 +1470,6 @@ Thanks goes to these wonderful people ✨ Simon J Mudd
Simon @@ -1484,6 +1498,8 @@ Thanks goes to these wonderful people ✨ Sven Kauber
Sylvain @@ -1498,8 +1514,6 @@ Thanks goes to these wonderful people ✨ Sylvain Dumont
TechnicLab/ @@ -1528,6 +1542,8 @@ Thanks goes to these wonderful people ✨ Torben Weibert
Toru @@ -1542,8 +1558,6 @@ Thanks goes to these wonderful people ✨ Trangar
Twist235/ @@ -1572,6 +1586,8 @@ Thanks goes to these wonderful people ✨ Vilius
Wim/ @@ -1586,8 +1602,6 @@ Thanks goes to these wonderful people ✨ Y.C.Huang
arcaine2/ @@ -1616,6 +1630,8 @@ Thanks goes to these wonderful people ✨ cternes
dborowy/ @@ -1630,8 +1646,6 @@ Thanks goes to these wonderful people ✨ dimalo
eleith/ @@ -1660,6 +1674,8 @@ Thanks goes to these wonderful people ✨ i-C-o-d-e-r
idaadi/ @@ -1674,8 +1690,6 @@ Thanks goes to these wonderful people ✨ ixeft
jjtt/ @@ -1704,6 +1718,8 @@ Thanks goes to these wonderful people ✨ magnus anderssen
marios88/ @@ -1718,8 +1734,6 @@ Thanks goes to these wonderful people ✨ matrixes
mchamplain/ @@ -1748,6 +1762,8 @@ Thanks goes to these wonderful people ✨ odinis
okami/ @@ -1762,8 +1778,6 @@ Thanks goes to these wonderful people ✨ olaf-mandel
ontheair81/ @@ -1792,6 +1806,8 @@ Thanks goes to these wonderful people ✨ rhyst
schnippl0r/ @@ -1806,8 +1822,6 @@ Thanks goes to these wonderful people ✨ smargold476
sportshead/ @@ -1836,6 +1850,8 @@ Thanks goes to these wonderful people ✨ tamueller
vivacarvajalito/ @@ -1850,8 +1866,6 @@ Thanks goes to these wonderful people ✨ wolkenschieber
worldworm/ From 69031b969dd55da17ce4846f23a0fe2c66db742a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Apr 2023 15:00:45 +0200 Subject: [PATCH 015/592] chore(deps): Bump anchore/scan-action from 3.3.4 to 3.3.5 (#3217) --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index fd7bae3e62c..8da20a7db2f 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v3.3.4 + uses: anchore/scan-action@v3.3.5 id: scan with: image: mailserver-testing:ci From 637d27efc72e11b50da69b262943d3ca29aea14f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Apr 2023 15:22:52 +0200 Subject: [PATCH 016/592] chore(deps): Bump peaceiris/actions-gh-pages from 3.9.2 to 3.9.3 (#3216) --- .github/workflows/docs-production-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-production-deploy.yml b/.github/workflows/docs-production-deploy.yml index 38219a9633f..73d152728bf 100644 --- a/.github/workflows/docs-production-deploy.yml +++ b/.github/workflows/docs-production-deploy.yml @@ -59,7 +59,7 @@ jobs: {} + - name: 'Deploy to Github Pages' - uses: peaceiris/actions-gh-pages@v3.9.2 + uses: peaceiris/actions-gh-pages@v3.9.3 with: github_token: ${{ secrets.GITHUB_TOKEN }} # Build directory contents to publish to the `gh-pages` branch: From e4543da4d5f2e618fa4b3683586b062545fe72b2 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Thu, 6 Apr 2023 19:28:33 +0200 Subject: [PATCH 017/592] GitHub/CI: issue templates improvements (#3225) --- .github/ISSUE_TEMPLATE/bug_report.yml | 34 ++++------ .github/ISSUE_TEMPLATE/feature_request.md | 48 -------------- .github/ISSUE_TEMPLATE/feature_request.yaml | 70 +++++++++++++++++++++ .github/ISSUE_TEMPLATE/question.yml | 26 ++++++++ .github/ISSUE_TEMPLATE/questions.md | 27 -------- 5 files changed, 107 insertions(+), 98 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yaml create mode 100644 .github/ISSUE_TEMPLATE/question.yml delete mode 100644 .github/ISSUE_TEMPLATE/questions.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 3774bf7b377..3944a0bb3cd 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,10 +1,9 @@ name: Bug report description: File a bug report -title: "[BUG] " +title: '[BUG] ' labels: - - kind/bug - - meta/needs triage - - priority/medium + - kind/bug/report + - meta/bug/needs triage body: - type: markdown @@ -14,35 +13,24 @@ body: Thank you for participating in this project and reporting a bug. Docker Mail Server (DMS) is a community-driven project, and each contribution counts. - Please **fill out all the fields and checkboxes of this form** to make it easier for maintainers to understand the problem and to solve it. The maintainers and moderators are volunteers that need you to fill this template with accurate informations in order to help you in the best and quickest way. We will have to label your request with `meta/no template - no support` if your request is sloppy and provides no way to help you correctly. + Please **fill out all the fields and checkboxes of this form** to make it easier for maintainers to understand the problem and to solve it. The maintainers and moderators are volunteers that need you to fill this template with accurate information in order to help you in the best and quickest way. We will have to label your request with `meta/no template - no support` if your request is sloppy and provides no way to help you correctly. Markdown formatting can be used in almost all text fields. The description will tell you if this is not the case for a specific field. - **Make sure** you read through the whole [README](https://github.com/docker-mailserver/docker-mailserver/blob/master/README.md), the [documentation](https://docker-mailserver.github.io/docker-mailserver/edge/) and the [issue tracker](https://github.com/docker-mailserver/docker-mailserver/issues?q=is%3Aissue) before opening a new bug report. + **Make sure** you read through our [documentation](https://docker-mailserver.github.io/docker-mailserver/edge/) and the [issue tracker](https://github.com/docker-mailserver/docker-mailserver/issues?q=is%3Aissue) before opening a new bug report. - Markdown formatting can be used in almost all text fields. The description will tell you if this is not the case for a specific field. - - ## Levels of support - - We provide official support for - - 1. OS/ARCH: Linux on AMD64 (x86_64) and ARM64 (AArch64) - 2. Containerization platform: Docker and Docker Compose (i.e. based on Docker Engine) - - Other configurations are not officially supported, but there are maintainers that may be able to help you. This is the case for - - 1. OS/ARCH: macOS - 2. Containerization platform: Kubernetes (K8s) - - Support for these cases is dependent on specific maintainers, and these cases are marked with `not officially supported`. All other cases are not supported and there are very likely no maintainers that can help you. These cases are marked with `unsupported`. + We provide official support for all options not marked with "not officially supported" or "unsupported". When something is "not officially supported", support for these cases is dependent on specific maintainers. + --- - type: checkboxes - id: miscellaneous-first-checks + id: preliminary-checks attributes: - label: Miscellaneous first checks + label: Preliminary Checks description: Please read these carefully. options: - label: I checked that all ports are open and not blocked by my ISP / hosting provider. required: true - label: I know that SSL errors are likely the result of a wrong setup on the user side and not caused by DMS itself. I'm confident my setup is correct. required: true + - label: I read the [documentation on debugging](https://docker-mailserver.github.io/docker-mailserver/latest/config/debugging/), tried the proposed steps to debug the problem, but was still unable to resolve the issue. + required: true - type: input id: affected-components attributes: diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index ac685461362..00000000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -name: "\U0001F389 Feature request" -about: Suggest an idea for this project -title: '[FR]' -labels: area/enhancement, kind/feature (request), meta/needs triage, priority/low -assignees: '' - ---- - -# Feature Request - -## Context - - - -### Is your Feature Request related to a Problem? - - - -### Describe the Solution you'd like - - - -### Are you going to implement it? - - - -Yes, because I know the probability of someone else doing it is low and I can learn from it. - -No, and I understand that it is highly likely no one will implement it. Furthermore, I understand that this issue will likely become stale and will be closed. - -### What are you going to contribute?? - - - -## Additional context - -### Alternatives you've considered - - - -### Who will that Feature be useful to? - - - -### What have you done already? - - diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml new file mode 100644 index 00000000000..a45f13b4e57 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -0,0 +1,70 @@ +name: Feature Request +description: Suggest an idea for this project +title: '[FR] ' +labels: + - kind/new feature + - meta/needs triage + +body: + - type: markdown + attributes: + value: | + Markdown formatting can be used in almost all text fields. The description will tell you if this is not the case for a specific field. + + --- + - type: textarea + id: context + attributes: + label: Context + validations: + required: true + - type: textarea + id: relations + attributes: + label: Relations + description: Is your request related to a problem? + validations: + required: true + - type: textarea + id: description + attributes: + label: Description + description: Describe the solution you would like + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternatives + description: Which alternatives have you considered? + validations: + required: true + - type: textarea + id: applicable-users + attributes: + label: Applicable Users + description: Who will that feature be useful to? + validations: + required: true + - type: dropdown + id: implementer + attributes: + label: Are you going to implement it? + options: + - Yes, because I know the probability of someone else doing it is low and I can learn from it. + - No, and I understand that it is highly likely no one will implement it. Furthermore, I understand that this issue will likely become stale and will be closed. + validations: + required: true + - type: textarea + id: contribution + attributes: + label: What are you going to contribute? + validations: + required: true + - type: textarea + id: already-done + attributes: + label: Already Done + description: Tell us what you have already done. + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/question.yml b/.github/ISSUE_TEMPLATE/question.yml new file mode 100644 index 00000000000..d13f040c73e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.yml @@ -0,0 +1,26 @@ +name: Other +description: Miscellaneous questions and reports +title: '[OTHER] ' +labels: + - meta/help wanted + +body: + - type: dropdown + id: subject + attributes: + label: Subject + options: + - I would like to contribute to the project + - I would like to configure a not documented mail server use case + - I would like some feedback concerning a use case + - I have questions about TLS/SSL/STARTTLS/OpenSSL + - Other + validations: + required: true + - type: textarea + id: description + attributes: + label: Description + description: Tell us more. Markdown formatting is supported. + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/questions.md b/.github/ISSUE_TEMPLATE/questions.md deleted file mode 100644 index 448ff47f61b..00000000000 --- a/.github/ISSUE_TEMPLATE/questions.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -name: "❓ Question / Other" -about: Ask a question about docker-mailserver -title: '' -labels: kind/question, priority/low, meta/help wanted, meta/needs triage -assignees: '' - ---- - -# Subject - - - -I would like to contribute to the project -I would like to configure a not documented mail server use case -I would like some feedback concerning a use case -I have questions about TLS/SSL/STARTTLS/OpenSSL -Other - -## Description - - - -``` BASH -# CODE GOES HERE - -``` From a9515b49c265620f983e1353eef6ae016db5a79a Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 7 Apr 2023 11:58:51 +0200 Subject: [PATCH 018/592] follow-up to #3225 (#3229) Misc spelling fixes and resolved imprecise statements. Shortened the bug report introduction a bit further and added a statement about being precise to all templates. --- .github/ISSUE_TEMPLATE/bug_report.yml | 15 +++++++++----- .github/ISSUE_TEMPLATE/config.yml | 15 ++++++++------ .github/ISSUE_TEMPLATE/feature_request.yaml | 22 ++++++--------------- .github/ISSUE_TEMPLATE/question.yml | 11 +++++++++-- 4 files changed, 34 insertions(+), 29 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 3944a0bb3cd..723e1f97bd7 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,6 +1,6 @@ -name: Bug report +name: Bug Report description: File a bug report -title: '[BUG] ' +title: 'bug report: ' labels: - kind/bug/report - meta/bug/needs triage @@ -11,13 +11,14 @@ body: value: | # Filing a report - Thank you for participating in this project and reporting a bug. Docker Mail Server (DMS) is a community-driven project, and each contribution counts. + Thank you for participating in this project and reporting a bug. Docker Mail Server (DMS) is a community-driven project, and each contribution counts. Maintainers and moderators are volunteers that need you to fill this template with accurate information in order to help you in the best and quickest way. We will have to label your request with `meta/no template - no support` if your request is sloppy and provides no way to help you correctly. - Please **fill out all the fields and checkboxes of this form** to make it easier for maintainers to understand the problem and to solve it. The maintainers and moderators are volunteers that need you to fill this template with accurate information in order to help you in the best and quickest way. We will have to label your request with `meta/no template - no support` if your request is sloppy and provides no way to help you correctly. Markdown formatting can be used in almost all text fields. The description will tell you if this is not the case for a specific field. + Markdown formatting can be used in almost all text fields. The description will tell you if this is not the case for a specific field. - **Make sure** you read through our [documentation](https://docker-mailserver.github.io/docker-mailserver/edge/) and the [issue tracker](https://github.com/docker-mailserver/docker-mailserver/issues?q=is%3Aissue) before opening a new bug report. + Be as precise as possible, and if in doubt, it's best to add more information that too few. We provide official support for all options not marked with "not officially supported" or "unsupported". When something is "not officially supported", support for these cases is dependent on specific maintainers. + --- - type: checkboxes id: preliminary-checks @@ -29,6 +30,10 @@ body: required: true - label: I know that SSL errors are likely the result of a wrong setup on the user side and not caused by DMS itself. I'm confident my setup is correct. required: true + - label: I searched the issue tracker but was unable to find my issue. + required: true + - label: I read the [extended documentation in general](https://docker-mailserver.github.io/docker-mailserver/latest/) but found nothing to resolve the issue. + required: true - label: I read the [documentation on debugging](https://docker-mailserver.github.io/docker-mailserver/latest/config/debugging/), tried the proposed steps to debug the problem, but was still unable to resolve the issue. required: true - type: input diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index c1efca2e505..d5d80621320 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,9 +1,12 @@ blank_issues_enabled: false contact_links: - - name: Documentation - url: https://docker-mailserver.github.io/docker-mailserver/edge - about: Extended documentaton - visit this first before opening issues - - name: Environment Variables Section - url: https://docker-mailserver.github.io/docker-mailserver/edge/config/environment/ - about: Read this section for information about mail server variables + - name: Documentation | Landing Page + url: https://docker-mailserver.github.io/docker-mailserver/latest + about: Visit this first before opening issues! + - name: Documentation | Environment Variables Page + url: https://docker-mailserver.github.io/docker-mailserver/latest/config/environment/ + about: Read this page for information about mail server variables. + - name: Documentation | Debugging Page + url: https://docker-mailserver.github.io/docker-mailserver/latest/config/debugging/ + about: Read this page for information on how to debug DMS. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml index a45f13b4e57..0ee66a036af 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yaml +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -1,6 +1,6 @@ name: Feature Request description: Suggest an idea for this project -title: '[FR] ' +title: 'feature request: ' labels: - kind/new feature - meta/needs triage @@ -11,25 +11,21 @@ body: value: | Markdown formatting can be used in almost all text fields. The description will tell you if this is not the case for a specific field. + Be as precise as possible, and if in doubt, it's best to add more information that too few. + --- - type: textarea id: context attributes: label: Context - validations: - required: true - - type: textarea - id: relations - attributes: - label: Relations - description: Is your request related to a problem? + description: Tell us how your request is related to DMS, one of its components or another issue / PR. Also **link all conected issues and PRs here**! validations: required: true - type: textarea id: description attributes: label: Description - description: Describe the solution you would like + description: Describe the solution you would like to have implemented. Be as precise as possible! validations: required: true - type: textarea @@ -59,12 +55,6 @@ body: id: contribution attributes: label: What are you going to contribute? - validations: - required: true - - type: textarea - id: already-done - attributes: - label: Already Done - description: Tell us what you have already done. + description: You may also tell us what you have already done. validations: required: true diff --git a/.github/ISSUE_TEMPLATE/question.yml b/.github/ISSUE_TEMPLATE/question.yml index d13f040c73e..7a7435117a4 100644 --- a/.github/ISSUE_TEMPLATE/question.yml +++ b/.github/ISSUE_TEMPLATE/question.yml @@ -1,10 +1,18 @@ name: Other description: Miscellaneous questions and reports -title: '[OTHER] ' +title: 'other: ' labels: - meta/help wanted body: + - type: markdown + attributes: + value: | + Markdown formatting can be used in almost all text fields. The description will tell you if this is not the case for a specific field. + + Be as precise as possible, and if in doubt, it's best to add more information that too few. + + --- - type: dropdown id: subject attributes: @@ -21,6 +29,5 @@ body: id: description attributes: label: Description - description: Tell us more. Markdown formatting is supported. validations: required: true From cf8e5552124eac9222143d4789195f54ad7cbf1e Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sat, 8 Apr 2023 11:54:16 +0200 Subject: [PATCH 019/592] docs: miscellaneous improvements (#3219) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .github/workflows/scripts/docs/build-docs.sh | 2 +- README.md | 2 +- docs/content/config/advanced/auth-ldap.md | 4 +- .../config/advanced/full-text-search.md | 8 +- docs/content/config/advanced/kubernetes.md | 2 +- .../override-defaults/user-patches.md | 4 +- docs/content/config/debugging.md | 48 +++++ docs/content/config/environment.md | 6 +- docs/content/config/security/mail_crypt.md | 8 +- docs/content/config/security/ssl.md | 32 ++- .../security/understanding-the-ports.md | 111 +++++++--- docs/content/config/setup.sh.md | 8 +- .../config/troubleshooting/debugging.md | 65 ------ .../config/user-management/accounts.md | 2 +- .../examples/tutorials/basic-installation.md | 131 ++++++------ .../examples/tutorials/docker-build.md | 3 +- .../tutorials/mailserver-behind-proxy.md | 2 +- .../examples/use-cases/imap-folders.md | 12 +- docs/content/faq.md | 20 +- docs/content/index.md | 14 +- docs/content/introduction.md | 142 ++++++------- docs/content/usage.md | 195 ++++++++++++------ docs/mkdocs.yml | 5 +- 23 files changed, 466 insertions(+), 360 deletions(-) create mode 100644 docs/content/config/debugging.md delete mode 100644 docs/content/config/troubleshooting/debugging.md diff --git a/.github/workflows/scripts/docs/build-docs.sh b/.github/workflows/scripts/docs/build-docs.sh index cf059709032..dd9ef3a533a 100755 --- a/.github/workflows/scripts/docs/build-docs.sh +++ b/.github/workflows/scripts/docs/build-docs.sh @@ -10,7 +10,7 @@ docker run \ --user "$(id -u):$(id -g)" \ --volume "${PWD}:/docs" \ --name "build-docs" \ - squidfunk/mkdocs-material:8.3.9 build --strict + squidfunk/mkdocs-material:9.1.5 build --strict # Remove unnecessary build artifacts: https://github.com/squidfunk/mkdocs-material/issues/2519 # site/ is the build output folder. diff --git a/README.md b/README.md index b549a59300c..1115249fe87 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ If you have issues, please search through [the documentation][documentation::web ## :package: Included Services -- [Postfix](http://www.postfix.org) with SMTP or LDAP authentication and support for [extension delimiters](http://www.postfix.org/postconf.5.html#recipient_delimiter) (_mail to `you+extension@example.com` delivered to `you@example.com`_) +- [Postfix](http://www.postfix.org) with SMTP or LDAP authentication and support for [extension delimiters](https://docker-mailserver.github.io/docker-mailserver/latest/config/user-management/aliases/#address-tags-extension-delimiters-an-alternative-to-aliases) - [Dovecot](https://www.dovecot.org) with SASL, IMAP, POP3, LDAP, [basic Sieve support](https://docker-mailserver.github.io/docker-mailserver/latest/config/advanced/mail-sieve) and [quotas](https://docker-mailserver.github.io/docker-mailserver/latest/config/user-management/accounts#notes) - [Rspamd](https://rspamd.com/) - [Amavis](https://www.amavis.org/) diff --git a/docs/content/config/advanced/auth-ldap.md b/docs/content/config/advanced/auth-ldap.md index 822145cfbfa..723deb7af40 100644 --- a/docs/content/config/advanced/auth-ldap.md +++ b/docs/content/config/advanced/auth-ldap.md @@ -191,7 +191,7 @@ The changes on the configurations necessary to work with Active Directory (**onl ```yaml services: mailserver: - image: docker.io/mailserver/docker-mailserver:latest + image: ghcr.io/docker-mailserver/docker-mailserver:latest container_name: mailserver hostname: mail.example.com @@ -253,7 +253,7 @@ The changes on the configurations necessary to work with Active Directory (**onl ```yaml services: mailserver: - image: docker.io/mailserver/docker-mailserver:latest + image: ghcr.io/docker-mailserver/docker-mailserver:latest container_name: mailserver hostname: mail.example.com diff --git a/docs/content/config/advanced/full-text-search.md b/docs/content/config/advanced/full-text-search.md index efd21d2306b..8c513047106 100644 --- a/docs/content/config/advanced/full-text-search.md +++ b/docs/content/config/advanced/full-text-search.md @@ -61,7 +61,7 @@ While indexing is memory intensive, you can configure the plugin to limit the am version: '3.8' services: mailserver: - image: docker.io/mailserver/docker-mailserver:latest + image: ghcr.io/docker-mailserver/docker-mailserver:latest container_name: mailserver hostname: mail domainname: example.com @@ -134,7 +134,7 @@ While indexing is memory intensive, you can configure the plugin to limit the am ```yaml services: mailserver: - image: docker.io/mailserver/docker-mailserver:latest + image: ghcr.io/docker-mailserver/docker-mailserver:latest volumes: - ./docker-data/dms/cron/fts_xapian:/etc/cron.d/fts_xapian ``` @@ -144,7 +144,7 @@ While indexing is memory intensive, you can configure the plugin to limit the am The [dovecot-solr Plugin](https://wiki2.dovecot.org/Plugins/FTS/Solr) is used in conjunction with [Apache Solr](https://lucene.apache.org/solr/) running in a separate container. This is quite straightforward to setup using the following instructions. -Solr is a mature and fast indexing backend that runs on the JVM. The indexes are relatively compact compared to the size of your total email. +Solr is a mature and fast indexing backend that runs on the JVM. The indexes are relatively compact compared to the size of your total email. However, Solr also requires a fair bit of RAM. While Solr is [highly tuneable](https://solr.apache.org/guide/7_0/query-settings-in-solrconfig.html), it may require a bit of testing to get it right. @@ -162,7 +162,7 @@ However, Solr also requires a fair bit of RAM. While Solr is [highly tuneable](h mailserver: depends_on: - solr - image: docker.io/mailserver/docker-mailserver:latest + image: ghcr.io/docker-mailserver/docker-mailserver:latest ... volumes: ... diff --git a/docs/content/config/advanced/kubernetes.md b/docs/content/config/advanced/kubernetes.md index a84e1064fc7..cc5f51ee4da 100644 --- a/docs/content/config/advanced/kubernetes.md +++ b/docs/content/config/advanced/kubernetes.md @@ -186,7 +186,7 @@ spec: hostname: mail containers: - name: mailserver - image: docker.io/mailserver/docker-mailserver:latest + image: ghcr.io/docker-mailserver/docker-mailserver:latest imagePullPolicy: IfNotPresent securityContext: diff --git a/docs/content/config/advanced/override-defaults/user-patches.md b/docs/content/config/advanced/override-defaults/user-patches.md index b10173368c1..ba3206e3ce5 100644 --- a/docs/content/config/advanced/override-defaults/user-patches.md +++ b/docs/content/config/advanced/override-defaults/user-patches.md @@ -4,7 +4,7 @@ title: 'Custom User Changes & Patches | Scripting' If you'd like to change, patch or alter files or behavior of `docker-mailserver`, you can use a script. -In case you cloned this repository, you can copy the file [`user-patches.sh.dist` (_under `config/`_)][gh-file-userpatches] with `#!sh cp config/user-patches.sh.dist docker-data/dms/config/user-patches.sh` in order to create the `user-patches.sh` script. +In case you cloned this repository, you can copy the file [`user-patches.sh.dist` (_under `config/`_)][github-file-userpatches] with `#!sh cp config/user-patches.sh.dist docker-data/dms/config/user-patches.sh` in order to create the `user-patches.sh` script. If you are managing your directory structure yourself, create a `docker-data/dms/config/` directory and add the `user-patches.sh` file yourself. @@ -40,4 +40,4 @@ And you're done. The user patches script runs right before starting daemons. Tha !!! note Many "patches" can already be done with the Docker Compose-/Stack-file. Adding hostnames to `/etc/hosts` is done with the `#!yaml extra_hosts:` section, `sysctl` commands can be managed with the `#!yaml sysctls:` section, etc. -[gh-file-userpatches]: https://github.com/docker-mailserver/docker-mailserver/blob/master/config-examples/user-patches.sh +[github-file-userpatches]: https://github.com/docker-mailserver/docker-mailserver/blob/master/config-examples/user-patches.sh diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md new file mode 100644 index 00000000000..0392e8fa3b7 --- /dev/null +++ b/docs/content/config/debugging.md @@ -0,0 +1,48 @@ +--- +title: 'Debugging' +hide: + - toc +--- + +This page contains valuable information when it comes to resolving issues you encounter. + +!!! info "Contributions Welcome!" + + Please consider contributing solutions to the [FAQ][docs-faq] :heart: + +## Preliminary Information + +### Mail sent from DMS does not arrive at destination + +Some service providers block outbound traffic on port 25. Common hosting providers known to have this issue: + +- [Azure](https://docs.microsoft.com/en-us/azure/virtual-network/troubleshoot-outbound-smtp-connectivity) +- [AWS EC2](https://aws.amazon.com/premiumsupport/knowledge-center/ec2-port-25-throttle/) +- [Vultr](https://www.vultr.com/docs/what-ports-are-blocked/) + +These links may advise how the provider can unblock the port through additional services offered, or via a support ticket request. + +## Steps for Debugging DMS + +1. **Increase log verbosity**: Very helpful for troubleshooting problems during container startup. Set the environment variable [`LOG_LEVEL`][docs-environment-log-level] to `debug` or `trace`. +2. **Use error logs as a search query**: Try finding an _existing issue_ or _search engine result_ from any errors in your container log output. Often you'll find answers or more insights. If you still need to open an issue, sharing links from your search may help us assist you. The mail server log can be acquired by running `docker log ` (_or `docker logs -f ` if you want to follow the log_). +3. **Understand the basics of mail servers**: Especially for beginners, make sure you read our [Introduction][docs-introduction] and [Usage][docs-usage] articles. +4. **Search the whole FAQ**: Our [FAQ][docs-faq] contains answers for common problems. Make sure you go through the list. +5. **Reduce the scope**: Ensure that you can run a basic setup of DMS first. Then incrementally restore parts of your original configuration until the problem is reproduced again. If you're new to DMS, it is common to find the cause is misunderstanding how to configure a minimal setup. + +### Debug a running container + +To get a shell inside the container run: `docker exec -it bash`. + +If you need more flexibility than `docker logs` offers, within the container `/var/log/mail/mail.log` and `/var/log/supervisor/` are the most useful locations to get relevant DMS logs. Use the `tail` or `cat` commands to view their contents. + +To install additional software: + +- `apt-get update` is needed to update repository metadata. +- `apt-get install ` +- For example if you need a text editor, `nano` is a good package choice for beginners. + +[docs-faq]: ../faq.md +[docs-environment-log-level]: ./environment.md#log_level +[docs-introduction]: ../introduction.md +[docs-usage]: ../usage.md diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 582c4dad171..ad5c86e9090 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -4,7 +4,7 @@ title: Environment Variables !!! info - Values in **bold** are the default values. If an option doesn't work as documented here, check if you are running the latest image. The current `master` branch corresponds to the image `mailserver/docker-mailserver:edge`. + Values in **bold** are the default values. If an option doesn't work as documented here, check if you are running the latest image. The current `master` branch corresponds to the image `ghcr.io/docker-mailserver/docker-mailserver:edge`. #### General @@ -13,10 +13,6 @@ title: Environment Variables - **empty** => uses the `hostname` command to get canonical hostname for `docker-mailserver` to use. - => Specify a fully-qualified domainname to serve mail for. This is used for many of the config features so if you can't set your hostname (_eg: you're in a container platform that doesn't let you_) specify it via this environment variable. It will take priority over `docker run` options: `--hostname` and `--domainname`, or `docker-compose.yml` config equivalents: `hostname:` and `domainname:`. -##### DMS_DEBUG - -**This environment variable was removed in `v11.0.0`!** Use `LOG_LEVEL` instead. - ##### LOG_LEVEL Set the log level for DMS. This is mostly relevant for container startup scripts and change detection event feedback. diff --git a/docs/content/config/security/mail_crypt.md b/docs/content/config/security/mail_crypt.md index 7b641ea7fa4..12f8698b475 100644 --- a/docs/content/config/security/mail_crypt.md +++ b/docs/content/config/security/mail_crypt.md @@ -3,7 +3,7 @@ title: 'Security | mail_crypt (email/storage encryption)' --- !!! info - + The Mail crypt plugin is used to secure email messages stored in a Dovecot system. Messages are encrypted before written to storage and decrypted after reading. Both operations are transparent to the user. In case of unauthorized access to the storage backend, the messages will, without access to the decryption keys, be unreadable to the offending party. @@ -34,11 +34,11 @@ Official Dovecot documentation: https://doc.dovecot.org/configuration_manual/mai 3. You then need to [generate your global EC key](https://doc.dovecot.org/configuration_manual/mail_crypt_plugin/#ec-key). We named them `/certs/ecprivkey.pem` and `/certs/ecpubkey.pem` in step #1. -4. The EC key needs to be available in the container. I prefer to mount a /certs directory into the container: +4. The EC key needs to be available in the container. I prefer to mount a /certs directory into the container: ```yaml services: mailserver: - image: docker.io/mailserver/docker-mailserver:latest + image: ghcr.io/docker-mailserver/docker-mailserver:latest volumes: . . . - ./certs/:/certs @@ -49,7 +49,7 @@ Official Dovecot documentation: https://doc.dovecot.org/configuration_manual/mai ```yaml services: mailserver: - image: docker.io/mailserver/docker-mailserver:latest + image: ghcr.io/docker-mailserver/docker-mailserver:latest volumes: . . . - ./config/dovecot/10-custom.conf:/etc/dovecot/conf.d/10-custom.conf diff --git a/docs/content/config/security/ssl.md b/docs/content/config/security/ssl.md index 3791affe2c1..2588b975123 100644 --- a/docs/content/config/security/ssl.md +++ b/docs/content/config/security/ssl.md @@ -121,7 +121,7 @@ Certbot provisions certificates to `/etc/letsencrypt`. Add a volume to store the !!! example Add these additions to the `mailserver` service in your [`docker-compose.yml`][github-file-compose]: - + ```yaml services: mailserver: @@ -169,7 +169,7 @@ Obtain a Cloudflare API token: 3. Click "Create Token", and choose the `Edit zone DNS` template (_Certbot [requires the `ZONE:DNS:Edit` permission](https://certbot-dns-cloudflare.readthedocs.io/en/stable/#credentials)_). !!! warning "Only include the necessary Zone resource configuration" - + Be sure to configure "Zone Resources" section on this page to `Include -> Specific zone -> `. This restricts the API token to only this zone (domain) which is an important security measure. @@ -264,7 +264,7 @@ After completing the steps above, your certificate should be ready to use. ``` You can manually run this service to renew the cert within 90 days: - + ```sh docker-compose run certbot-cloudflare-renew ``` @@ -274,14 +274,14 @@ After completing the steps above, your certificate should be ready to use. ```log Saving debug log to /var/log/letsencrypt/letsencrypt.log - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Processing /etc/letsencrypt/renewal/mail.example.com.conf - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Account registered. Simulating renewal of an existing certificate for mail.example.com Waiting 10 seconds for DNS changes to propagate - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Congratulations, all simulated renewals succeeded: /etc/letsencrypt/live/mail.example.com/fullchain.pem (success) @@ -640,7 +640,7 @@ This setup only comes with one caveat: The domain has to be configured on anothe version: '3.8' services: mailserver: - image: docker.io/mailserver/docker-mailserver:latest + image: ghcr.io/docker-mailserver/docker-mailserver:latest container_name: mailserver hostname: mail domainname: example.com @@ -696,10 +696,6 @@ Add `SSL_TYPE=self-signed` to your `docker-mailserver` environment variables. Po #### Generating a self-signed certificate -!!! note - - Since `docker-mailserver` v10, support in `setup.sh` for generating a _self-signed SSL certificate_ internally was removed. - One way to generate self-signed certificates is with [Smallstep's `step` CLI](https://smallstep.com/docs/step-cli). This is exactly what [`docker-mailserver` does for creating test certificates][github-file::tls-readme]. For example with the FQDN `mail.example.test`, you can generate the required files by running: @@ -821,15 +817,13 @@ These options in conjunction mean: If you have another source for SSL/TLS certificates you can import them into the server via an external script. The external script can be found here: [external certificate import script][hanscees-renewcerts]. -!!! attention "Only compatible with `docker-mailserver` releases < `v10.2`" - - The script expects `/etc/postfix/ssl/cert` and `/etc/postfix/ssl/key` files to be configured paths for both Postfix and Dovecot to use. - - Since the `docker-mailserver` 10.2 release, certificate files have moved to `/etc/dms/tls/`, and the file name may differ depending on provisioning method. +This is a community contributed script, and in most cases you will have better support via our _Change Detection_ service (_automatic for `SSL_TYPE` of `manual` and `letsencrypt`_) - Unless you're using LDAP which disables the service. - This third-party script also has `fullchain.pem` and `privkey.pem` as hard-coded, thus is incompatible with other filenames. +!!! warning "Script Compatibility" - Additionally it has never supported handling `ALT` fallback certificates (for supporting dual/hybrid, RSA + ECDSA). + - Relies on private filepaths `/etc/dms/tls/cert` and `/etc/dms/tls/key` intended for internal use only. + - Only supports hard-coded `fullchain.key` + `privkey.pem` as your mounted file names. That may not align with your provisioning method. + - No support for `ALT` fallback certificates (_for supporting dual/hybrid, RSA + ECDSA_). The steps to follow are these: @@ -864,7 +858,7 @@ export SITE_URL="mail.example.com" export SITE_IP_URL="192.168.0.72" # can also use `mail.example.com` export SITE_SSL_PORT="993" # imap port dovecot -##works: check if certificate will expire in two weeks +##works: check if certificate will expire in two weeks #2 weeks is 1209600 seconds #3 weeks is 1814400 #12 weeks is 7257600 @@ -921,7 +915,7 @@ if [ "$certcheck_2weeks" = "Certificate will not expire" ]; then echo "Cert seems to be expiring pretty soon, within two weeks: $certcheck_2weeks" echo "we will send an alert email and log as well" logger Certwatch: cert $SITE_URL will expire in two weeks - echo "Certwatch: cert $SITE_URL will expire in two weeks" | mail -s "cert $SITE_URL expires in two weeks " $ALERT_EMAIL_ADDR + echo "Certwatch: cert $SITE_URL will expire in two weeks" | mail -s "cert $SITE_URL expires in two weeks " $ALERT_EMAIL_ADDR fi ``` diff --git a/docs/content/config/security/understanding-the-ports.md b/docs/content/config/security/understanding-the-ports.md index f7d8e364ce7..6a3e91b9473 100644 --- a/docs/content/config/security/understanding-the-ports.md +++ b/docs/content/config/security/understanding-the-ports.md @@ -4,20 +4,31 @@ title: 'Security | Understanding the Ports' ## Quick Reference -Prefer Implicit TLS ports, they're more secure and if you use a Reverse Proxy, should be less hassle (although it's probably wiser to expose these ports directly to `docker-mailserver`). +Prefer ports with Implicit [TLS][wikipedia-tls] ports, they're more secure than ports using Explicit TLS, and if you use a Reverse Proxy should be less hassle. ## Overview of Email Ports -| Protocol | Explicit TLS1 | Implicit TLS | Purpose | -|----------|--------------------------|-----------------|----------------------| -| SMTP | 25 | N/A | Transfer2 | -| ESMTP | 587 | 4653 | Submission | -| POP3 | 110 | 995 | Retrieval | -| IMAP4 | 143 | 993 | Retrieval | +| Protocol | Explicit TLS1 | Implicit TLS | Purpose | Enabled by Default | +|--------------------------|--------------------------|-----------------|----------------------|--------------------| +| [ESMTP][wikipedia-esmtp] | 25 | N/A | Transfer2 | Yes | +| ESMTP | 587 | 4653 | Submission | Yes | +| POP3 | 110 | 995 | Retrieval | No | +| IMAP4 | 143 | 993 | Retrieval | Yes | + +1. A connection _may_ be secured over TLS when both ends support `STARTTLS`. On ports 110, 143 and 587, `docker-mailserver` will reject a connection that cannot be secured. Port 25 is [required][ref-port25-mandatory] to support insecure connections. +2. Receives email, `docker-mailserver` additionally filters for spam and viruses. For submitting email to the server to be sent to third-parties, you should prefer the _submission_ ports (465, 587) - which require authentication. Unless a relay host is configured (eg: SendGrid), outgoing email will leave the server via port 25 (_thus outbound traffic must not be blocked by your provider or firewall_). +3. A _submission_ port since 2018 ([RFC 8314][rfc-8314]). + +??? warning "Beware of outdated advice on port 465" + + There is a common misconception of this port due to it's history detailed by various communities and blogs articles on the topic (_including by popular mail relay services_). + + Port 465 was [briefly assigned the role of SMTPS in 1997][wikipedia-smtps] as an secure alternative to Port 25 between MTA exchanges. Then RFC 2487 (`STARTTLS`) [while still in a draft status in late 1998 had IANA revoke the SMTPS assignment][history-465-revoked]. The [draft history was modified to exclude all mention of port 465 and SMTPS][history-465-politics]. + + In 2018 [RFC 8314][rfc-8314] was published which revives Port 465 as an Implicit TLS alternative to Port 587 for mail submission. It details very clearly that gaining adoption of 465 as the preferred port will take time. IANA reassigned [port 465 as the `submissions` service][iana-services-465]. Any unofficial usage as **SMTPS is legacy and has been for over two decades**. + + Understand that port 587 is more broadly supported due to this history and that lots of software in that time has been built or configured with that port in mind. [`STARTTLS` is known to have various CVEs discovered even in recent years][starttls-vulnerabilities], do not be misled by any advice implying it should be preferred over implicit TLS. Trust in more official sources, such as the [config Postfix has][postfix-upstream-config-mastercf] which acknowledges the `submissions` port (465). -1. A connection *may* be secured over TLS when both ends support `STARTTLS`. On ports 110, 143 and 587, `docker-mailserver` will reject a connection that cannot be secured. Port 25 is [required][ref-port25-mandatory] to support insecure connections. -2. Receives email, `docker-mailserver` additionally filters for spam and viruses. For submitting email to the server to be sent to third-parties, you should prefer the *submission* ports(465, 587) - which require authentication. Unless a relay host is configured(eg SendGrid), outgoing email will leave the server via port 25(thus outbound traffic must not be blocked by your provider or firewall). -3. A *submission* port since 2018 ([RFC 8314][rfc-8314]). Previously a secure variant of port 25. ### What Ports Should I Use? (SMTP) @@ -52,49 +63,101 @@ flowchart LR #### Inbound Traffic (On the left) -- **Port 25:** Think of this like a physical mailbox, it is open to receive email from anyone who wants to. `docker-mailserver` will actively filter email delivered on this port for spam or viruses and refuse mail from known bad sources. While you could also use this port internally to send email outbound without requiring authentication, you really should prefer the *Submission* ports(587, 465). -- **Port 465(*and 587*):** This is the equivalent of a post office box where you would send email to be delivered on your behalf(`docker-mailserver` is that metaphorical post office, aka the MTA). Unlike port 25, these two ports are known as the *Submission* ports and require a valid email account on the server with a password to be able to send email to anyone outside of the server(an MTA you do not control, eg Outlook or Gmail). Prefer port 465 which provides Implicit TLS. +Mail arriving at your server will be processed and stored in a mailbox, or sent outbound to another mail server. + +- **Port 25:** + - Think of this like a physical mailbox, anyone can deliver mail to you here. Typically most mail is delivered to you on this port. + -`docker-mailserver` will actively filter email delivered on this port for spam or viruses, and refuse mail from known bad sources. + - Connections to this port may be secure through STARTTLS, but is not mandatory as [mail is allowed to arrive via an unencrypted connection][ref-port25-mandatory]. + - It is possible for internal clients to submit mail to be sent outbound (_without requiring authentication_), but that is discouraged. Prefer the _submission_ ports. +- **Port 465 and 587:** + - This is the equivalent of a post office box where you would send email to be delivered on your behalf (_`docker-mailserver` is that metaphorical post office, aka the MTA_). + - These two ports are known as the _submission_ ports, they enable mail to be sent outbound to another MTA (eg: Outlook or Gmail) but require authentication via a [mail account][docs-accounts]. + - For inbound traffic, this is relevant when you send mail from your MUA (eg: ThunderBird). It's also used when `docker-mailserver` is configured as a mail relay, or when you have a service sending transactional mail (_eg: order confirmations, password resets, notifications_) through `docker-mailserver`. + - _**Prefer port 465**_ over port 587, as 465 provides Implicit TLS. + +!!! note + + When submitting mail (inbound) to be sent (outbound), this involves two separate connections to negotiate and secure. There may be additional intermediary connections which `docker-mailserver` is not involved in, and thus unable to ensure encrypted transit throughout delivery. #### Outbound Traffic (On the Right) -- **Port 25:** Send the email directly to the given email address MTA as possible. Like your own `docker-mailserver` port 25, this is the standard port for receiving email on, thus email will almost always arrive to the final MTA on this port. Note that, there may be additional MTAs further in the chain, but this would be the public facing one representing that email address. -- **Port 465(*and 587*):** SMTP Relays are a popular choice to hand-off delivery of email through. Services like SendGrid are useful for bulk email(marketing) or when your webhost or ISP are preventing you from using standard ports like port 25 to send out email(which can be abused by spammers). - - `docker-mailserver` can serve as a relay too, but the difference between a DIY relay and a professional service is reputation, which is referenced by MTAs you're delivering to such as Outlook, Gmail or others(perhaps another `docker-mailserver` server!), when deciding if email should be marked as junked or potentially not delivered at all. As a service like SendGrid has a reputation to maintain, relay is restricted to registered users who must authenticate(even on port 25), they do not store email, merely forward it to another MTA which could be delivered on a different port like 25. +Mail being sent from your server is either being relayed through another MTA (eg: SendGrid), or direct to an MTA responsible for an email address (eg: Gmail). + +- **Port 25:** + - As most MTA use port 25 to receive inbound mail, when no authenticated relay is involved this is the outbound port used. + - Outbound traffic on this port is often blocked by service providers (eg: VPS, ISP) to prevent abuse by spammers. If the port cannot be unblocked, you will need to relay outbound mail through a service to send on your behalf. +- **Port 465 and 587:** + - Submission ports for outbound traffic establish trust to forward mail through a third-party relay service. This requires [authenticating to an account on the relay service][docs-relays]. The relay will then deliver the mail through port 25 on your behalf. + - These are the two typical ports used, but smart hosts like SendGrid often document support for additional non-standard ports as alternatives if necessary. + - Usually you'll only use these outbound ports for relaying. It is possible to deliver directly to the relevant MTA for email address, but requires having credentials for each MTA. + +!!! tip + + `docker-mailserver` can function as a relay too, but professional relay services have a trusted reputation (_which increases success of delivery_). + + An MTA with low reputation can affect if mail is treated as junk, or even rejected. + +!!! note + + At best, you can only ensure a secure connection between the MTA you directly connect to. The receiving MTA may relay that mail to another MTA (_and so forth_), each connection may not be enforcing TLS. + ### Explicit vs Implicit TLS #### Explicit TLS (aka Opportunistic TLS) - Opt-in Encryption -Communication on these ports begin in [cleartext][ref-clear-vs-plain], indicating support for `STARTTLS`. If both client and server support `STARTTLS` the connection will be secured over TLS, otherwise the connection is unable to use encryption to secure it. By default, `docker-mailserver` is configured to reject connections that fail to establish a secure connection when authentication is required, rather than allow an insecure connection (_Port 25 will allow receiving unencrypted deliveries which doesn't require authentication_). +Communication on these ports begin in [cleartext][ref-clear-vs-plain]. Upgrading to an encrypted connection must be requested explicitly through the `STARTTLS` protocol **and** successfully negotiated. + +Sometimes a reverse-proxy is involved, but is misconfigured or lacks support for the `STARTTLS` negotiation to succeed. + +!!! note -Support for `STARTTLS` is not always implemented correctly, which can lead to leaking credentials(client sending too early) prior to a TLS connection being established. Third-parties such as some ISPs have also been known to intercept the `STARTTLS` exchange, modifying network traffic to prevent establishing a secure connection. + - By default, `docker-mailserver` is configured to reject connections that fail to establish a secure connection (_when authentication is required_), rather than allow an insecure connection. + - Port 25 does not require authentication. If `STARTTLS` is unsuccessful, mail can be received over an unencrypted connection. You can better secure this port between trusted parties with the addition of MTA-STS, [STARTTLS Policy List][starttls-policy-list], DNSSEC and DANE. + +!!! warning + + `STARTTLS` [continues to have vulnerabilities found][starttls-vulnerabilities] (Nov 2021 article), as per [RFC 8314 (Section 4.1)][rfc-8314-s41] you are encouraged to **prefer Implicit TLS where possible**. + + Support for `STARTTLS` is not always implemented correctly, which can lead to leaking credentials (like a client sending too early) prior to a TLS connection being established. Third-parties such as some ISPs have also been known to intercept the `STARTTLS` exchange, modifying network traffic to prevent establishing a secure connection. -Due to these security concerns, [RFC 8314 (Section 4.1)][rfc-8314-s41] encourages you to **prefer Implicit TLS ports where possible**. #### Implicit TLS - Enforced Encryption -Communication is always encrypted, avoiding the above mentioned issues with Explicit TLS. +Communication on these ports are always encrypted (_enforced, thus implicit_), avoiding the potential risks with `STARTTLS` (Explicit TLS). -You may know of these ports as **SMTPS, POP3S, IMAPS**, which indicate the protocol in combination with a TLS connection. However, Explicit TLS ports provide the same benefit when `STARTTLS` is successfully negotiated; Implicit TLS better communicates the improved security to all three protocols (SMTP/POP3/IMAP over Implicit TLS). +While Explicit TLS can provide the same benefit (_when `STARTTLS` is successfully negotiated_), Implicit TLS more reliably avoids concerns with connection manipulation and compatibility. -Additionally, referring to port 465 as *SMTPS* would be incorrect, as it is a submissions port requiring authentication to proceed via *ESMTP*, whereas ESMTPS has a different meaning(STARTTLS supported). Port 25 may lack Implicit TLS, but can be configured to be more secure between trusted parties via MTA-STS, STARTTLS Policy List, DNSSEC and DANE. ## Security !!! todo + This section should provide any related configuration advice, and probably expand on and link to resources about DANE, DNSSEC, MTA-STS and STARTTLS Policy list, with advice on how to configure/setup these added security layers. !!! todo + A related section or page on ciphers used may be useful, although less important for users to be concerned about. ### TLS connections for a Mail-Server, compared to web browsers -Unlike with HTTP where a web browser client communicates directly with the server providing a website, a secure TLS connection as discussed below is not the equivalent safety that HTTPS provides when the transit of email (receiving or sending) is sent through third-parties, as the secure connection is only between two machines, any additional machines (MTAs) between the MUA and the MDA depends on them establishing secure connections between one another successfully. +Unlike with HTTP where a web browser client communicates directly with the server providing a website, a secure TLS connection as discussed below does not provide the equivalent safety that HTTPS does when the transit of email (receiving or sending) is sent through third-parties, as the secure connection is only between two machines, any additional machines (MTAs) between the MUA and the MDA depends on them establishing secure connections between one another successfully. -Other machines that facilitate a connection that generally aren't taken into account can exist between a client and server, such as those where your connection passes through your ISP provider are capable of compromising a cleartext connection through interception. +Other machines that facilitate a connection that generally aren't taken into account can exist between a client and server, such as those where your connection passes through your ISP provider are capable of compromising a `cleartext` connection through interception. +[docs-accounts]: ../user-management/accounts.md +[docs-relays]: ../advanced/mail-forwarding/relay-hosts.md +[iana-services-465]: https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml?search=465 +[starttls-policy-list]: https://github.com/EFForg/starttls-everywhere#email-security-database-starttls-policy-list +[starttls-vulnerabilities]: https://blog.apnic.net/2021/11/18/vulnerabilities-show-why-starttls-should-be-avoided-if-possible/ [ref-clear-vs-plain]: https://www.denimgroup.com/resources/blog/2007/10/cleartext-vs-pl [ref-port25-mandatory]: https://serverfault.com/questions/623692/is-it-still-wrong-to-require-starttls-on-incoming-smtp-messages [rfc-8314]: https://tools.ietf.org/html/rfc8314 [rfc-8314-s41]: https://tools.ietf.org/html/rfc8314#section-4.1 +[history-465-revoked]: https://web.archive.org/web/20150603202057/http://www.imc.org/ietf-apps-tls/mail-archive/msg00204.html +[history-465-politics]: https://mailing.postfix.users.narkive.com/F3ACwg2F/which-port-to-use-for-ssl-tls#post21 +[postfix-upstream-config-mastercf]: https://github.com/vdukhovni/postfix/blob/62931e5b1f9f1e80d02a496c7fd0062a5aae1d25/postfix/conf/master.cf#L38-L41 +[wikipedia-smtps]: https://en.wikipedia.org/wiki/SMTPS#History +[wikipedia-esmtp]: https://en.wikipedia.org/wiki/Simple_Mail_Transfer_Protocol#Modern_SMTP +[wikipedia-tls]: https://en.wikipedia.org/wiki/Transport_Layer_Security diff --git a/docs/content/config/setup.sh.md b/docs/content/config/setup.sh.md index 181c025d402..b55b7d9d09d 100644 --- a/docs/content/config/setup.sh.md +++ b/docs/content/config/setup.sh.md @@ -13,12 +13,6 @@ wget https://raw.githubusercontent.com/docker-mailserver/docker-mailserver/maste chmod a+x ./setup.sh ``` -!!! warning "`setup.sh` for `docker-mailserver` version `v10.1.x` and below" - - If you're using `docker-mailserver` version `v10.1.x` or below, you will need to get `setup.sh` with a specific version. Substitute `` with the [tagged release version](https://github.com/docker-mailserver/docker-mailserver/tags) that you're using: - - `wget https://raw.githubusercontent.com/docker-mailserver/docker-mailserver//setup.sh`. - ## Usage Run `./setup.sh help` and you'll get ~~all you have ever wanted~~ some usage information: @@ -40,7 +34,7 @@ DESCRIPTION Please note that the script executes most of the commands inside the container itself. If the image was not found, this script will pull the ':latest' tag of - 'mailserver/docker-mailserver'. This tag refers to the latest release, + 'docker.io/mailserver/docker-mailserver'. This tag refers to the latest release, see the tagging convention in the README under https://github.com/docker-mailserver/docker-mailserver/blob/master/README.md diff --git a/docs/content/config/troubleshooting/debugging.md b/docs/content/config/troubleshooting/debugging.md deleted file mode 100644 index 8457cd05306..00000000000 --- a/docs/content/config/troubleshooting/debugging.md +++ /dev/null @@ -1,65 +0,0 @@ ---- -title: 'Troubleshooting | Debugging' ---- - -!!! info "Contributions Welcome!" - Please contribute your solutions to help the community :heart: - -## Enable Verbose Debugging Output - -You may find it useful to set [`LOG_LEVEL`][docs-environment-log-level] environment variable. - -## Invalid Username or Password - -1. Shell into the container: - - ```sh - docker exec -it bash - ``` - -2. Check log files in `/var/log/mail` could not find any mention of incorrect logins here neither in the dovecot logs. - -3. Check the supervisors logs in `/var/log/supervisor`. You can find the logs for startup of fetchmail, postfix and others here - they might indicate problems during startup. - -4. Make sure you set your hostname to `mail` or whatever you specified in your `docker-compose.yml` file or else your FQDN will be wrong. - -## Installation Errors - -During setup, if you get errors trying to edit files inside of the container, you likely need to install `vi`: - -```sh -sudo su -docker exec -it apt-get install -y vim -``` - -## Testing Connection - -I spent HOURS trying to debug "Connection Refused" and "Connection closed by foreign host" errors when trying to use telnet to troubleshoot my connection. I was also trying to connect from my email client (macOS mail) around the same time. Telnet had also worked earlier, so I was extremely confused as to why it suddenly stopped working. I stumbled upon `fail2ban.log` in my container. In short, when trying to get my macOS client working, I exceeded the number of failed login attempts and fail2ban put dovecot and postfix in jail! I got around it by whitelisting my ipaddresses (my ec2 instance and my local computer) - -```sh -sudo su -docker exec -it mailserver bash -cd /var/log -cat fail2ban.log | grep dovecot - -# Whitelist IP addresses: -fail2ban-client set dovecot addignoreip # Server -fail2ban-client set postfix addignoreip -fail2ban-client set dovecot addignoreip # Client -fail2ban-client set postfix addignoreip - -# This will delete the jails entirely - nuclear option -fail2ban-client stop dovecot -fail2ban-client stop postfix -``` - -## Sent email is never received - -Some hosting provides have a stealth block on port 25. Make sure to check with your hosting provider that traffic on port 25 is allowed - -Common hosting providers known to have this issue: - -- [Azure](https://docs.microsoft.com/en-us/azure/virtual-network/troubleshoot-outbound-smtp-connectivity) -- [AWS EC2](https://aws.amazon.com/premiumsupport/knowledge-center/ec2-port-25-throttle/) - -[docs-environment-log-level]: ../environment.md#log_level diff --git a/docs/content/config/user-management/accounts.md b/docs/content/config/user-management/accounts.md index 56bb0224877..cc447627e05 100644 --- a/docs/content/config/user-management/accounts.md +++ b/docs/content/config/user-management/accounts.md @@ -19,7 +19,7 @@ In the example above, we've added 2 mail accounts for 2 different domains. Conse docker run --rm \ -e MAIL_USER=user1@example.com \ -e MAIL_PASS=mypassword \ - -it mailserver/docker-mailserver:latest \ + -it ghcr.io/docker-mailserver/docker-mailserver:latest \ /bin/sh -c 'echo "$MAIL_USER|$(doveadm pw -s SHA512-CRYPT -u $MAIL_USER -p $MAIL_PASS)"' >> docker-data/dms/config/postfix-accounts.cf ``` diff --git a/docs/content/examples/tutorials/basic-installation.md b/docs/content/examples/tutorials/basic-installation.md index 150d933170c..bdb7cfb3db0 100644 --- a/docs/content/examples/tutorials/basic-installation.md +++ b/docs/content/examples/tutorials/basic-installation.md @@ -9,12 +9,13 @@ This example provides you only with a basic example of what a minimal setup coul ``` YAML services: mailserver: - image: docker.io/mailserver/docker-mailserver:latest + image: ghcr.io/docker-mailserver/docker-mailserver:latest container_name: mailserver # Provide the FQDN of your mail server here (Your DNS MX record should point to this value) hostname: mail.example.com ports: - "25:25" + - "465:465" - "587:587" - "993:993" volumes: @@ -24,11 +25,9 @@ services: - ./docker-data/dms/config/:/tmp/docker-mailserver/ - /etc/localtime:/etc/localtime:ro environment: - - ENABLE_SPAMASSASSIN=1 - - SPAMASSASSIN_SPAM_TO_INBOX=1 + - ENABLE_RSPAMD=1 - ENABLE_CLAMAV=1 - ENABLE_FAIL2BAN=1 - - ENABLE_POSTGREY=1 cap_add: - NET_ADMIN # For Fail2Ban to work restart: always @@ -41,12 +40,13 @@ services: ``` YAML services: mailserver: - image: docker.io/mailserver/docker-mailserver:latest + image: ghcr.io/docker-mailserver/docker-mailserver:latest container_name: mailserver # Provide the FQDN of your mail server here (Your DNS MX record should point to this value) hostname: mail.example.com ports: - "25:25" + - "465:465" - "587:587" - "993:993" volumes: @@ -56,11 +56,6 @@ services: - ./docker-data/dms/config/:/tmp/docker-mailserver/ - /etc/localtime:/etc/localtime:ro environment: - - ENABLE_SPAMASSASSIN=1 - - SPAMASSASSIN_SPAM_TO_INBOX=1 - - ENABLE_CLAMAV=1 - - ENABLE_FAIL2BAN=1 - - ENABLE_POSTGREY=1 - ACCOUNT_PROVISIONER=LDAP - LDAP_SERVER_HOST=ldap # your ldap container/IP/ServerName - LDAP_SEARCH_BASE=ou=people,dc=localhost,dc=localdomain @@ -80,30 +75,28 @@ services: - SASLAUTHD_LDAP_SEARCH_BASE=ou=people,dc=localhost,dc=localdomain - SASLAUTHD_LDAP_FILTER=(&(objectClass=PostfixBookMailAccount)(uniqueIdentifier=%U)) - POSTMASTER_ADDRESS=postmaster@localhost.localdomain - cap_add: - - NET_ADMIN # For Fail2Ban to work restart: always ``` -## A Detailed Example +## Using DMS as a local mail relay for containers -!!! note +!!! info - This is a community contributed guide. Please let us know via a Github Issue if you're having any difficulty following the guide so that we can update it. + This was originally a community contributed guide. Please let us know via a Github Issue if you're having any difficulty following the guide so that we can update it. -This guide is focused on only using [SMTP ports (not POP3 and IMAP)][docs-ports] with the intent to send received mail to another MTA service such as _Gmail_. It is not intended to have a MUA client (_eg: Thunderbird_) to retrieve mail directly from `docker-mailserver` via POP3/IMAP. +This guide is focused on only using [SMTP ports (not POP3 and IMAP)][docs-ports] with the intent to relay mail received from another service to an external email address (eg: `user@gmail.com`). It is not intended for mailbox storage of real users. -In this setup `docker-mailserver` is not intended to receive email externally, so no anti-spam or anti-virus software is needed, making the service lighter to run. +In this setup `docker-mailserver` is not intended to receive email from the outside world, so no anti-spam or anti-virus software is needed, making the service lighter to run. -!!! warning "Open Relays" +!!! tip "`setup`" - Adding the docker network's gateway to the list of trusted hosts (_eg: using the `network` or `connected-networks` option_), can create an [**open relay**](https://en.wikipedia.org/wiki/Open_mail_relay). For instance [if IPv6 is enabled on the host machine, but not in Docker][github-issue-1405-comment]. + The `setup` command used below is to be [run inside the container][docs-usage]. -1. If you're running a version of `docker-mailserver` earlier than v10.2, [you'll need to get `setup.sh`][docs-setup-script]. Otherwise you can substitute `./setup.sh ` with `docker exec mailserver setup `. +!!! warning "Open Relays" -2. Pull the docker image: `docker pull docker.io/mailserver/docker-mailserver:latest`. + Adding the docker network's gateway to the list of trusted hosts (_eg: using the `network` or `connected-networks` option_), can create an [**open relay**](https://en.wikipedia.org/wiki/Open_mail_relay). For instance [if IPv6 is enabled on the host machine, but not in Docker][github-issue-1405-comment]. -3. Create the file `docker-compose.yml` with a content like this: +1. Create the file `docker-compose.yml` with a content like this: !!! example @@ -126,48 +119,56 @@ In this setup `docker-mailserver` is not intended to receive email externally, s - /etc/localtime:/etc/localtime:ro environment: - ENABLE_FAIL2BAN=1 - # Using letsencrypt for SSL/TLS certificates + # Using letsencrypt for SSL/TLS certificates: - SSL_TYPE=letsencrypt - # Allow sending emails from other docker containers + # Allow sending emails from other docker containers: # Beware creating an Open Relay: https://docker-mailserver.github.io/docker-mailserver/latest/config/environment/#permit_docker - PERMIT_DOCKER=network # You may want to enable this: https://docker-mailserver.github.io/docker-mailserver/latest/config/environment/#spoof_protection - # See step 8 below, which demonstrates setup with enabled/disabled SPOOF_PROTECTION: + # See step 6 below, which demonstrates setup with enabled/disabled SPOOF_PROTECTION: - SPOOF_PROTECTION=0 cap_add: - NET_ADMIN # For Fail2Ban to work restart: always ``` - - The docs have a detailed page on [Environment Variables][docs-environment] for reference. + The docs have a detailed page on [Environment Variables][docs-environment] for reference. - !!! note "Firewalled ports" + ??? tip "Firewalled ports" - You may need to open ports `25`, `587` and `465` on the firewall. For example, with the firewall `ufw`, run: + If you have a firewall running, you may need to open ports `25`, `587` and `465`. + + For example, with the firewall `ufw`, run: ```sh ufw allow 25 ufw allow 587 ufw allow 465 ``` + + **Caution:** This may [not be sound advice][github-issue-ufw]. + +2. Configure your DNS service to use an MX record for the _hostname_ (eg: `mail`) you configured in the previous step and add the [SPF][docs-spf] TXT record. + + !!! tip "If you manually manage the DNS zone file for the domain" + + It would look something like this: + + ```txt + $ORIGIN example.com + @ IN A 10.11.12.13 + mail IN A 10.11.12.13 + + ; mail-server for example.com + @ IN MX 10 mail.example.com. + + ; Add SPF record + @ IN TXT "v=spf1 mx -all" + ``` + + Then don't forget to change the `SOA` serial number, and to restart the service. -4. Configure your DNS service to use an MX record for the _hostname_ (eg: `mail`) you configured in the previous step and add the [SPF][docs-spf] TXT record. - - If you manually manage the DNS zone file for the domain, it would look something like this: - - ```txt - mail IN A 10.11.12.13 - - ; mail-server for example.com - 3600 IN MX 1 mail.example.com. - - ; Add SPF record - IN TXT "v=spf1 mx ~all" - ``` - - Then don't forget to change the serial number and to restart the service. - -5. [Generate DKIM keys][docs-dkim] for your domain via `./setup.sh config dkim`. +3. [Generate DKIM keys][docs-dkim] for your domain via `setup config dkim`. Copy the content of the file `docker-data/dms/config/opendkim/keys/example.com/mail.txt` and add it to your DNS records as a TXT like SPF was handled above. @@ -179,21 +180,21 @@ In this setup `docker-mailserver` is not intended to receive email externally, s "iqq3bD/BVlwKRp5gH6TEYEmx8EBJUuDxrJhkWRUk2VDl1fqhVBy8A9O7Ah+85nMrlOHIFsTaYo9o6+cDJ6t1i6G1gu+bZD0d3/3bqGLPBQV9LyEL1Rona5V7TJBGg099NQkTz1IwIDAQAB" ) ; ----- DKIM key mail for example.com ``` -6. Get an SSL certificate, [we have a guide for you here][docs-ssl] (_Let's Encrypt_ is a popular service to get free SSL certificates). +4. Get an SSL certificate, [we have a guide for you here][docs-ssl] (_Let's Encrypt_ is a popular service to get free SSL certificates). -7. Start `docker-mailserver` and check the terminal output for any errors: `docker-compose up`. +5. Start `docker-mailserver` and check the terminal output for any errors: `docker-compose up`. -8. Create email accounts and aliases: +6. Create email accounts and aliases: !!! example "With `SPOOF_PROTECTION=0`" ```sh - ./setup.sh email add admin@example.com passwd123 - ./setup.sh email add info@example.com passwd123 - ./setup.sh alias add admin@example.com external-account@gmail.com - ./setup.sh alias add info@example.com external-account@gmail.com - ./setup.sh email list - ./setup.sh alias list + setup email add admin@example.com passwd123 + setup email add info@example.com passwd123 + setup alias add admin@example.com external-account@gmail.com + setup alias add info@example.com external-account@gmail.com + setup email list + setup alias list ``` Aliases make sure that any email that comes to these accounts is forwarded to your third-party email address (`external-account@gmail.com`), where they are retrieved (_eg: via third-party web or mobile app_), instead of connecting directly to `docker-mailserer` with POP3 / IMAP. @@ -201,25 +202,25 @@ In this setup `docker-mailserver` is not intended to receive email externally, s !!! example "With `SPOOF_PROTECTION=1`" ```sh - ./setup.sh email add admin.gmail@example.com passwd123 - ./setup.sh email add info.gmail@example.com passwd123 - ./setup.sh alias add admin@example.com admin.gmail@example.com - ./setup.sh alias add info@example.com info.gmail@example.com - ./setup.sh alias add admin.gmail@example.com external-account@gmail.com - ./setup.sh alias add info.gmail@example.com external-account@gmail.com - ./setup.sh email list - ./setup.sh alias list + setup email add admin.gmail@example.com passwd123 + setup email add info.gmail@example.com passwd123 + setup alias add admin@example.com admin.gmail@example.com + setup alias add info@example.com info.gmail@example.com + setup alias add admin.gmail@example.com external-account@gmail.com + setup alias add info.gmail@example.com external-account@gmail.com + setup email list + setup alias list ``` This extra step is required to avoid the `553 5.7.1 Sender address rejected: not owned by user` error (_the accounts used for submitting mail to Gmail are `admin.gmail@example.com` and `info.gmail@example.com`_) -9. Send some test emails to these addresses and make other tests. Once everything is working well, stop the container with `ctrl+c` and start it again as a daemon: `docker-compose up -d`. +7. Send some test emails to these addresses and make other tests. Once everything is working well, stop the container with `ctrl+c` and start it again as a daemon: `docker-compose up -d`. [docs-ports]: ../../config/security/understanding-the-ports.md -[docs-setup-script]: ../../config/setup.sh.md [docs-environment]: ../../config/environment.md [docs-spf]: ../../config/best-practices/spf.md [docs-dkim]: ../../config/best-practices/dkim.md [docs-ssl]: ../../config/security/ssl.md#lets-encrypt-recommended - +[docs-usage]: ../../usage.md#get-up-and-running +[github-issue-ufw]: https://github.com/docker-mailserver/docker-mailserver/issues/3151 [github-issue-1405-comment]: https://github.com/docker-mailserver/docker-mailserver/issues/1405#issuecomment-590106498 diff --git a/docs/content/examples/tutorials/docker-build.md b/docs/content/examples/tutorials/docker-build.md index 62471579a28..fc6d5c37b23 100644 --- a/docs/content/examples/tutorials/docker-build.md +++ b/docs/content/examples/tutorials/docker-build.md @@ -10,7 +10,7 @@ You'll need to retrieve the git submodules prior to building your own Docker ima ```sh git submodule update --init --recursive -docker build -t mailserver/docker-mailserver . +docker build -t . ``` Or, you can clone and retrieve the submodules in one command: @@ -37,4 +37,3 @@ The `Dockerfile` takes additional, so-called build arguments. These are 2. `VCS_REVISION`: the image revision (default = unknown) When using `make` to build the image, these are filled with proper values. You can build the image without supplying these arguments just fine though. - diff --git a/docs/content/examples/tutorials/mailserver-behind-proxy.md b/docs/content/examples/tutorials/mailserver-behind-proxy.md index 78606b5b5c0..cd5bc02de0f 100644 --- a/docs/content/examples/tutorials/mailserver-behind-proxy.md +++ b/docs/content/examples/tutorials/mailserver-behind-proxy.md @@ -60,7 +60,7 @@ Feel free to add your configuration if you achieved the same goal using differen version: '3.8' services: mailserver: - image: docker.io/mailserver/docker-mailserver:latest + image: ghcr.io/docker-mailserver/docker-mailserver:latest container_name: mailserver hostname: mail domainname: example.com diff --git a/docs/content/examples/use-cases/imap-folders.md b/docs/content/examples/use-cases/imap-folders.md index 1923d6a1b5b..1d1b2539017 100644 --- a/docs/content/examples/use-cases/imap-folders.md +++ b/docs/content/examples/use-cases/imap-folders.md @@ -6,17 +6,17 @@ hide: # Mailboxes (_aka IMAP Folders_) -`INBOX` is setup as the private [`inbox` namespace][dovecot-docs-namespaces]. By default [`target/dovecot/15-mailboxes.conf`][gh-config-dovecot-mailboxes] configures the special IMAP folders `Drafts`, `Sent`, `Junk` and `Trash` to be automatically created and subscribed. They are all assigned to the private [`inbox` namespace][dovecot-docs-namespaces] (_which implicitly provides the `INBOX` folder_). +`INBOX` is setup as the private [`inbox` namespace][dovecot-docs-namespaces]. By default [`target/dovecot/15-mailboxes.conf`][github-config-dovecot-mailboxes] configures the special IMAP folders `Drafts`, `Sent`, `Junk` and `Trash` to be automatically created and subscribed. They are all assigned to the private [`inbox` namespace][dovecot-docs-namespaces] (_which implicitly provides the `INBOX` folder_). These IMAP folders are considered special because they add a [_"SPECIAL-USE"_ attribute][rfc-6154], which is a standardized way to communicate to mail clients that the folder serves a purpose like storing spam/junk mail (`\Junk`) or deleted mail (`\Trash`). This differentiates them from regular mail folders that you may use for organizing. ## Adding a mailbox folder -See [`target/dovecot/15-mailboxes.conf`][gh-config-dovecot-mailboxes] for existing mailbox folders which you can modify or uncomment to enable some other common mailboxes. For more information try the [official Dovecot documentation][dovecot-docs-mailboxes]. +See [`target/dovecot/15-mailboxes.conf`][github-config-dovecot-mailboxes] for existing mailbox folders which you can modify or uncomment to enable some other common mailboxes. For more information try the [official Dovecot documentation][dovecot-docs-mailboxes]. -The `Archive` special IMAP folder may be useful to enable. To do so, make a copy of [`target/dovecot/15-mailboxes.conf`][gh-config-dovecot-mailboxes] and uncomment the `Archive` mailbox definition. Mail clients should understand that this folder is intended for archiving mail due to the [`\Archive` _"SPECIAL-USE"_ attribute][rfc-6154]. +The `Archive` special IMAP folder may be useful to enable. To do so, make a copy of [`target/dovecot/15-mailboxes.conf`][github-config-dovecot-mailboxes] and uncomment the `Archive` mailbox definition. Mail clients should understand that this folder is intended for archiving mail due to the [`\Archive` _"SPECIAL-USE"_ attribute][rfc-6154]. -With the provided [docker-compose.yml][gh-config-dockercompose] example, a volume bind mounts the host directory `docker-data/dms/config/` to the container location `/tmp/docker-mailserver/`. Config file overrides should instead be mounted to a different location as described in [Overriding Configuration for Dovecot][docs-config-overrides-dovecot]: +With the provided [docker-compose.yml][github-config-dockercompose] example, a volume bind mounts the host directory `docker-data/dms/config/` to the container location `/tmp/docker-mailserver/`. Config file overrides should instead be mounted to a different location as described in [Overriding Configuration for Dovecot][docs-config-overrides-dovecot]: ```yaml volumes: @@ -65,8 +65,8 @@ Take care to test localized names work well as well. [docs-config-overrides-dovecot]: ../../config/advanced/override-defaults/dovecot.md#override-configuration -[gh-config-dockercompose]: https://github.com/docker-mailserver/docker-mailserver/blob/master/docker-compose.yml -[gh-config-dovecot-mailboxes]: https://github.com/docker-mailserver/docker-mailserver/blob/master/target/dovecot/15-mailboxes.conf +[github-config-dockercompose]: https://github.com/docker-mailserver/docker-mailserver/blob/master/docker-compose.yml +[github-config-dovecot-mailboxes]: https://github.com/docker-mailserver/docker-mailserver/blob/master/target/dovecot/15-mailboxes.conf [dovecot-docs-namespaces]: https://doc.dovecot.org/configuration_manual/namespace/#namespace-inbox [dovecot-docs-mailboxes]: https://doc.dovecot.org/configuration_manual/namespace/#mailbox-settings [rfc-6154]: https://datatracker.ietf.org/doc/html/rfc6154 diff --git a/docs/content/faq.md b/docs/content/faq.md index abe384b8c56..0734481a455 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -233,6 +233,22 @@ Yes, by adding the environment variable `PERMIT_DOCKER: network`. Adding the Docker network's gateway to the list of trusted hosts, e.g. using the `network` or `connected-networks` option, can create an [**open relay**](https://en.wikipedia.org/wiki/Open_mail_relay), for instance [if IPv6 is enabled on the host machine but not in Docker][github-issue-1405-comment]. +### Connection refused or No response at all + +You see errors like "Connection Refused" and "Connection closed by foreign host", or you cannot connect at all? You may not be able to connect with your mail client (MUA)? Make sure to check Fail2Ban did not ban you (for exceeding the number of tried logins for example)! You can run + +```bash +docker exec setup fail2ban +``` + +and check whether your IP address appears. Use + +```bash +docker exec setup fail2ban unban +``` + +to unban the IP address. + ### How can I authenticate users with `SMTP_ONLY=1`? See [#1247][github-issue-1247] for an example. @@ -429,7 +445,7 @@ The following configuration works nicely: ```yaml services: mailserver: - image: docker.io/mailserver/docker-mailserver:latest + image: ghcr.io/docker-mailserver/docker-mailserver:latest volumes: - ./docker-data/dms/cron/sa-learn:/etc/cron.d/sa-learn ``` @@ -441,7 +457,7 @@ The following configuration works nicely: services: mailserver: - image: docker.io/mailserver/docker-mailserver:latest + image: ghcr.io/docker-mailserver/docker-mailserver:latest # ... configs: - source: my_sa_crontab diff --git a/docs/content/index.md b/docs/content/index.md index b22d5fc5574..10d099ceec7 100644 --- a/docs/content/index.md +++ b/docs/content/index.md @@ -10,7 +10,7 @@ title: Home This documentation provides you not only with the basic setup and configuration of DMS but also with advanced configuration, elaborate usage scenarios, detailed examples, hints and more. -[docs-tagging]: ./usage/#available-images-tags-tagging-convention +[docs-tagging]: ./usage.md#tagging-convention ## About @@ -42,16 +42,18 @@ You might also want to check out: 1. A list of [all configuration options via ENV][docs-environment] 2. A list of [all optional and automatically created configuration files and directories][docs-optionalconfig] +3. How to [debug your mail server][docs-debugging] !!! tip Definitely check out the [FAQ][docs-faq] for more information and tips! Please do not open an issue before you have checked our documentation for answers, including the [FAQ][docs-faq]! -[docs-environment]: ./config/environment/ -[docs-userpatches]: ./faq/#how-to-adjust-settings-with-the-user-patchessh-script -[docs-setupsh]: ./config/setup.sh/ -[docs-optionalconfig]: ./config/advanced/optional-config/ -[docs-faq]: ./faq/ +[docs-environment]: ./config/environment.md +[docs-userpatches]: ./faq.md#how-to-adjust-settings-with-the-user-patchessh-script +[docs-setupsh]: ./config/setup.sh.md +[docs-optionalconfig]: ./config/advanced/optional-config.md +[docs-faq]: ./faq.md +[docs-debugging]: ./config/debugging.md ### Tests diff --git a/docs/content/introduction.md b/docs/content/introduction.md index b0d2f0a5c8f..f136b795575 100644 --- a/docs/content/introduction.md +++ b/docs/content/introduction.md @@ -1,28 +1,28 @@ --- -title: An overview of Mail-Server infrastructure +title: Introduction --- -What is a mail-server, and how does it perform its duty? +# An Overview of Mail Server Infrastructure -Here's an introduction to the field that covers everything you need to know to get started with `docker-mailserver`. +This article answers the question "What is a mail server, and how does it perform its duty?" and it gives the reader an introduction to the field that covers everything you need to know to get started with `docker-mailserver`. -## Anatomy of a Mail-Server +## The Anatomy of a Mail Server -A mail-server is only a part of a [client-server relationship][wikipedia-clientserver] aimed at exchanging information in the form of [emails][wikipedia-email]. Exchanging emails requires using specific means (programs and protocols). +A mail server is only a part of a [client-server relationship][wikipedia-clientserver] aimed at exchanging information in the form of [emails][wikipedia-email]. Exchanging emails requires using specific means (programs and protocols). `docker-mailserver` provides you with the server portion, whereas the client can be anything from a terminal via text-based software (eg. [Mutt][software-mutt]) to a fully-fledged desktop application (eg. [Mozilla Thunderbird][software-thunderbird], [Microsoft Outlook][software-outlook]…), to a web interface, etc. -Unlike the client-side where usually a single program is used to perform retrieval and viewing of emails, the server-side is composed of many specialized components. The mail-server is capable of accepting, forwarding, delivering, storing and overall exchanging messages, but each one of those tasks is actually handled by a specific piece of software. All of these "agents" must be integrated with one another for the exchange to take place. +Unlike the client-side where usually a single program is used to perform retrieval and viewing of emails, the server-side is composed of many specialized components. The mail server is capable of accepting, forwarding, delivering, storing and overall exchanging messages, but each one of those tasks is actually handled by a specific piece of software. All of these "agents" must be integrated with one another for the exchange to take place. -`docker-mailserver` has made informed choices about those components and their (default) configuration. It offers a comprehensive platform to run a fully featured mail-server in no time! +`docker-mailserver` has made informed choices about those components and their (default) configuration. It offers a comprehensive platform to run a fully featured mail server in no time! ## Components The following components are required to create a [complete delivery chain][wikipedia-emailagent]: -- MUA: a [Mail User Agent][wikipedia-mua] is basically any client/program capable of sending emails to a mail-server; while also capable of fetching emails from a mail-server for presenting them to the end users. -- MTA: a [Mail Transfer Agent][wikipedia-mta] is the so-called "mail-server" as seen from the MUA's perspective. It's a piece of software dedicated to accepting submitted emails, then forwarding them-where exactly will depend on an email's final destination. If the receiving MTA is responsible for the FQDN the email is sent to, then an MTA is to forward that email to an MDA (see below). Otherwise, it is to transfer (ie. forward, relay) to another MTA, "closer" to the email's final destination. -- MDA: a [Mail Delivery Agent][wikipedia-mda] is responsible for accepting emails from an MTA and dropping them into their recipients' mailboxes, whichever the form. +- **MUA**: a [Mail User Agent][wikipedia-mua] is basically any client/program capable of sending emails to a mail server; while also capable of fetching emails from a mail server for presenting them to the end users. +- **MTA**: a [Mail Transfer Agent][wikipedia-mta] is the so-called "mail server" as seen from the MUA's perspective. It's a piece of software dedicated to accepting submitted emails, then forwarding them-where exactly will depend on an email's final destination. If the receiving MTA is responsible for the FQDN the email is sent to, then an MTA is to forward that email to an MDA (see below). Otherwise, it is to transfer (ie. forward, relay) to another MTA, "closer" to the email's final destination. +- **MDA**: a [Mail Delivery Agent][wikipedia-mda] is responsible for accepting emails from an MTA and dropping them into their recipients' mailboxes, whichever the form. Here's a schematic view of mail delivery: @@ -49,7 +49,8 @@ Fetching an email: MUA <------------------------------ ┫ MDA ╯ ┃ ┗━━━━━━━┛ ``` -!!! example +??? example "An Example" + Let's say Alice owns a Gmail account, `alice@gmail.com`; and Bob owns an account on a `docker-mailserver`'s instance, `bob@dms.io`. Make sure not to conflate these two very different scenarios: @@ -62,43 +63,41 @@ Fetching an email: MUA <------------------------------ ┫ MDA ╯ ┃ One important thing to note is that MTA and MDA programs may actually handle _multiple_ tasks (which is the case with `docker-mailserver`'s Postfix and Dovecot). -For instance, Postfix is both an SMTP server (accepting emails) and a relaying MTA (transferring, ie. sending emails to other MTA/MDA); Dovecot is both an MDA (delivering emails in mailboxes) and an IMAP server (allowing MUAs to fetch emails from the *mail-server*). On top of that, Postfix may rely on Dovecot's authentication capabilities. +For instance, Postfix is both an SMTP server (accepting emails) and a relaying MTA (transferring, ie. sending emails to other MTA/MDA); Dovecot is both an MDA (delivering emails in mailboxes) and an IMAP server (allowing MUAs to fetch emails from the _mail server_). On top of that, Postfix may rely on Dovecot's authentication capabilities. The exact relationship between all the components and their respective (sometimes shared) responsibilities is beyond the scope of this document. Please explore this wiki & the web to get more insights about `docker-mailserver`'s toolchain. ## About Security & Ports -In the previous section, different components were outlined. Each one of those is responsible for a specific task, it has a specific purpose. +### Introduction -Three main purposes exist when it comes to exchanging emails: +In the previous section, three components were outlined. Each one of those is responsible for a specific task when it comes to exchanging emails: -- _Submission_: for a MUA (client), the act of sending actual email data over the network, toward an MTA (server). -- _Transfer_ (aka. _Relay_): for an MTA, the act of sending actual email data over the network, toward another MTA (server) closer to the final destination (where an MTA will forward data to an MDA). -- _Retrieval_: for a MUA (client), the act of fetching actual email data over the network, from an MDA. +- **Submission**: for a MUA (client), the act of sending actual email data over the network, toward an MTA (server). +- **Transfer** (aka. **Relay**): for an MTA, the act of sending actual email data over the network, toward another MTA (server) closer to the final destination (where an MTA will forward data to an MDA). +- **Retrieval**: for a MUA (client), the act of fetching actual email data over the network, from an MDA. -Postfix handles Submission (and might handle Relay), whereas Dovecot handles Retrieval. They both need to be accessible by MUAs in order to act as servers, therefore they expose public endpoints on specific TCP ports (see. [_Understanding the ports_][docs-understandports] for more details). Those endpoints _may_ be secured, using an encryption scheme and TLS certificates. +Postfix handles **Submission** (_and may handle **Relay**_), whereas Dovecot handles **Retrieval**. They both need to be accessible by MUAs in order to act as servers, therefore they expose public [endpoints on specific TCP ports][docs-understandports]. Those endpoints _may_ be secured, using an encryption scheme and TLS certificates. When it comes to the specifics of email exchange, we have to look at protocols and ports enabled to support all the identified purposes. There are several valid options and they've been evolving overtime. -**Here's `docker-mailserver`'s _default_ configuration:** +### Overview -| Purpose | Protocol | TCP port / encryption | -|----------------|----------|--------------------------------| -| Transfer/Relay | SMTP | 25 (unencrypted) | -| Submission | ESMTP | 587 (encrypted using STARTTLS) | -| Retrieval | IMAP4 | 143 (encrypted using STARTTLS) + 993 (TLS) | -| Retrieval | POP3 | _Not activated_ | +The following picture gives a visualization of the interplay of all components and their [respective ports][docs-understandports]: ```txt - ┏━━━━━━━━━━ Submission ━━━━━━━━━┓┏━━━━━━━━━━━━━ Transfer/Relay ━━━━━━━━━━━┓ - ┌─────────────────────┐ ┌┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┐ -MUA ----- STARTTLS ---> ┤(587) MTA ╮ (25)├ <-- cleartext ---> ┊ Third-party MTA ┊ - ---- cleartext ---> ┤(25) │ | └┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┘ - |┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄| -MUA <---- STARTTLS ---- ┤(143) MDA ╯ | - <-- enforced TLS -- ┤(993) | - └─────────────────────┘ - ┗━━━━━━━━━━ Retrieval ━━━━━━━━━━┛ + ┏━━━━━━━━━━ Submission ━━━━━━━━━━━━┓┏━━━━━━━━━━━━━ Transfer/Relay ━━━━━━━━━━━┓ + + ┌─────────────────────┐ ┌┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┐ +MUA ----- STARTTLS ------> ┤(587) MTA ╮ (25)├ <-- cleartext ---> ┊ Third-party MTA ┊ + ----- implicit TLS --> ┤(465) │ | └┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┘ + ----- cleartext -----> ┤(25) │ | + |┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄| +MUA <---- STARTTLS ------- ┤(143) MDA ╯ | + <---- implicit TLS --- ┤(993) | + └─────────────────────┘ + + ┗━━━━━━━━━━ Retrieval ━━━━━━━━━━━━━┛ ``` If you're new to email infrastructure, both that table and the schema may be confusing. @@ -110,9 +109,7 @@ For a MUA to send an email to an MTA, it needs to establish a connection with th In the case of `docker-mailserver`, the MTA (SMTP server) is Postfix. The MUA (client) may vary, yet its Submission request is performed as [TCP][wikipedia-tcp] packets sent over the _public_ internet. This exchange of information may be secured in order to counter eavesdropping. -#### Two kinds of Submission - -Let's say I own an account on a `docker-mailserver` instance, `me@dms.io`. There are two very different use-cases for Submission: +Now let's say I own an account on a `docker-mailserver` instance, `me@dms.io`. There are two very different use-cases for Submission: 1. I want to send an email to someone 2. Someone wants to send you an email @@ -123,67 +120,69 @@ In the second scenario, a third-party email account owner will be first submitti My MTA will thus have to support two kinds of Submission: -- Outward Submission (self-owned email is submitted directly to the MTA, then is relayed "outside") -- Inward Submission (third-party email has been submitted & relayed, then is accepted "inside" by the MTA) +- Outbound Submission (self-owned email is submitted directly to the MTA, then is relayed "outside") +- Inbound Submission (third-party email has been submitted & relayed, then is accepted "inside" by the MTA) ```txt - ┏━━━━ Outward Submission ━━━━┓ + ┏━━━━ Outbound Submission ━━━━┓ + ┌────────────────────┐ ┌┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┐ Me ---------------> ┤ ├ -----------------> ┊ ┊ │ My MTA │ ┊ Third-party MTA ┊ │ ├ <----------------- ┊ ┊ └────────────────────┘ └┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┘ - ┗━━━━━━━━━━ Inward Submission ━━━━━━━━━━┛ -``` -##### Outward Submission + ┗━━━━━━━━━━ Inbound Submission ━━━━━━━━━━┛ +``` -The best practice as of 2020 when it comes to securing Outward Submission is to use _Implicit TLS connection via ESMTP on port 465_ (see [RFC 8314][rfc-8314]). Let's break it down. +#### Outbound Submission -- Implicit TLS means the server _enforces_ the client into using an encrypted TCP connection, using [TLS][wikipedia-tls]. With this kind of connection, the MUA _has_ to establish a TLS-encrypted connection from the get go (TLS is implied, hence the name "Implicit"). Any client attempting to either submit email in cleartext (unencrypted, not secure), or requesting a cleartext connection to be upgraded to a TLS-encrypted one using `STARTTLS`, is to be denied. Implicit TLS is sometimes called Enforced TLS for that reason. -- [ESMTP][wikipedia-esmtp] is [SMTP][wikipedia-smtp] + extensions. It's the version of the SMTP protocol that a mail-server commonly communicates with today. For the purpose of this documentation, ESMTP and SMTP are synonymous. -- Port 465 is the reserved TCP port for Implicit TLS Submission (since 2018). There is actually a boisterous history to that ports usage, but let's keep it simple. +When it comes to securing Outbound Submission you should prefer to use _Implicit TLS connection via ESMTP on port 465_ (see [RFC 8314][rfc-8314]). Please read our article about [**Understanding the Ports**][docs-understandports] for more details! !!! warning + This Submission setup is sometimes referred to as [SMTPS][wikipedia-smtps]. Long story short: this is incorrect and should be avoided. Although a very satisfactory setup, Implicit TLS on port 465 is somewhat "cutting edge". There exists another well established mail Submission setup that must be supported as well, SMTP+STARTTLS on port 587. It uses Explicit TLS: the client starts with a cleartext connection, then the server informs a TLS-encrypted "upgraded" connection may be established, and the client _may_ eventually decide to establish it prior to the Submission. Basically it's an opportunistic, opt-in TLS upgrade of the connection between the client and the server, at the client's discretion, using a mechanism known as [STARTTLS][wikipedia-starttls] that both ends need to implement. -In many implementations, the mail-server doesn't enforce TLS encryption, for backwards compatibility. Clients are thus free to deny the TLS-upgrade proposal (or [misled by a hacker](https://security.stackexchange.com/questions/168998/what-happens-if-starttls-dropped-in-smtp) about STARTTLS not being available), and the server accepts unencrypted (cleartext) mail exchange, which poses a confidentiality threat and, to some extent, spam issues. [RFC 8314 (section 3.3)][rfc-8314-s33] recommends for a mail-server to support both Implicit and Explicit TLS for Submission, _and_ to enforce TLS-encryption on ports 587 (Explicit TLS) and 465 (Implicit TLS). That's exactly `docker-mailserver`'s default configuration: abiding by RFC 8314, it [enforces a strict (`encrypt`) STARTTLS policy](http://www.postfix.org/postconf.5.html#smtpd_tls_security_level), where a denied TLS upgrade terminates the connection thus (hopefully but at the client's discretion) preventing unencrypted (cleartext) Submission. +In many implementations, the mail server doesn't enforce TLS encryption, for backwards compatibility. Clients are thus free to deny the TLS-upgrade proposal (or [misled by a hacker](https://security.stackexchange.com/questions/168998/what-happens-if-starttls-dropped-in-smtp) about STARTTLS not being available), and the server accepts unencrypted (cleartext) mail exchange, which poses a confidentiality threat and, to some extent, spam issues. [RFC 8314 (section 3.3)][rfc-8314-s33] recommends for a mail server to support both Implicit and Explicit TLS for Submission, _and_ to enforce TLS-encryption on ports 587 (Explicit TLS) and 465 (Implicit TLS). That's exactly `docker-mailserver`'s default configuration: abiding by RFC 8314, it [enforces a strict (`encrypt`) STARTTLS policy](http://www.postfix.org/postconf.5.html#smtpd_tls_security_level), where a denied TLS upgrade terminates the connection thus (hopefully but at the client's discretion) preventing unencrypted (cleartext) Submission. -- **`docker-mailserver`'s default configuration enables and _requires_ Explicit TLS (STARTTLS) on port 587 for Outward Submission.** -- It does not enable Implicit TLS Outward Submission on port 465 by default. One may enable it through simple custom configuration, either as a replacement or (better!) supplementary mean of secure Submission. +- **`docker-mailserver`'s default configuration enables and _requires_ Explicit TLS (STARTTLS) on port 587 for Outbound Submission.** +- It does not enable Implicit TLS Outbound Submission on port 465 by default. One may enable it through simple custom configuration, either as a replacement or (better!) supplementary mean of secure Submission. - It does not support old MUAs (clients) not supporting TLS encryption on ports 587/465 (those should perform Submission on port 25, more details below). One may relax that constraint through advanced custom configuration, for backwards compatibility. -A final Outward Submission setup exists and is akin SMTP+STARTTLS on port 587, but on port 25. That port has historically been reserved specifically for unencrypted (cleartext) mail exchange though, making STARTTLS a bit wrong to use. As is expected by [RFC 5321][rfc-5321], `docker-mailserver` uses port 25 for unencrypted Submission in order to support older clients, but most importantly for unencrypted Transfer/Relay between MTAs. +A final Outbound Submission setup exists and is akin SMTP+STARTTLS on port 587, but on port 25. That port has historically been reserved specifically for unencrypted (cleartext) mail exchange though, making STARTTLS a bit wrong to use. As is expected by [RFC 5321][rfc-5321], `docker-mailserver` uses port 25 for unencrypted Submission in order to support older clients, but most importantly for unencrypted Transfer/Relay between MTAs. -- **`docker-mailserver`'s default configuration also enables unencrypted (cleartext) on port 25 for Outward Submission.** -- It does not enable Explicit TLS (STARTTLS) on port 25 by default. One may enable it through advanced custom configuration, either as a replacement (bad!) or as a supplementary mean of secure Outward Submission. -- One may also secure Outward Submission using advanced encryption scheme, such as DANE/DNSSEC and/or MTA-STS. +- **`docker-mailserver`'s default configuration also enables unencrypted (cleartext) on port 25 for Outbound Submission.** +- It does not enable Explicit TLS (STARTTLS) on port 25 by default. One may enable it through advanced custom configuration, either as a replacement (bad!) or as a supplementary mean of secure Outbound Submission. +- One may also secure Outbound Submission using advanced encryption scheme, such as DANE/DNSSEC and/or MTA-STS. -##### Inward Submission +#### Inbound Submission -Granted it's still very difficult enforcing encryption between MTAs (Transfer/Relay) without risking dropping emails (when relayed by MTAs not supporting TLS-encryption), Inward Submission is to be handled in cleartext on port 25 by default. +Granted it's still very difficult enforcing encryption between MTAs (Transfer/Relay) without risking dropping emails (when relayed by MTAs not supporting TLS-encryption), Inbound Submission is to be handled in cleartext on port 25 by default. -- **`docker-mailserver`'s default configuration enables unencrypted (cleartext) on port 25 for Inward Submission.** -- It does not enable Explicit TLS (STARTTLS) on port 25 by default. One may enable it through advanced custom configuration, either as a replacement (bad!) or as a supplementary mean of secure Inward Submission. -- One may also secure Inward Submission using advanced encryption scheme, such as DANE/DNSSEC and/or MTA-STS. +- **`docker-mailserver`'s default configuration enables unencrypted (cleartext) on port 25 for Inbound Submission.** +- It does not enable Explicit TLS (STARTTLS) on port 25 by default. One may enable it through advanced custom configuration, either as a replacement (bad!) or as a supplementary mean of secure Inbound Submission. +- One may also secure Inbound Submission using advanced encryption scheme, such as DANE/DNSSEC and/or MTA-STS. Overall, `docker-mailserver`'s default configuration for SMTP looks like this: ```txt - ┏━━━━ Outward Submission ━━━━┓ + ┏━━━━ Outbound Submission ━━━━┓ + ┌────────────────────┐ ┌┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┐ Me -- cleartext --> ┤(25) (25)├ --- cleartext ---> ┊ ┊ -Me -- STARTTLS ---> ┤(587) My MTA │ ┊ Third-party MTA ┊ +Me -- TLS ---> ┤(465) My MTA │ ┊ Third-party MTA ┊ +Me -- STARTTLS ---> ┤(587) │ ┊ ┊ │ (25)├ <---cleartext ---- ┊ ┊ └────────────────────┘ └┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┘ - ┗━━━━━━━━━━ Inward Submission ━━━━━━━━━━┛ + + ┗━━━━━━━━━━ Inbound Submission ━━━━━━━━━━┛ ``` ### Retrieval - IMAP -A MUA willing to fetch an email from a mail-server will most likely communicate with its [IMAP][wikipedia-imap] server. As with SMTP described earlier, communication will take place in the form of data packets exchanged over a network that both the client and the server are connected to. The IMAP protocol makes the server capable of handling _Retrieval_. +A MUA willing to fetch an email from a mail server will most likely communicate with its [IMAP][wikipedia-imap] server. As with SMTP described earlier, communication will take place in the form of data packets exchanged over a network that both the client and the server are connected to. The IMAP protocol makes the server capable of handling _Retrieval_. In the case of `docker-mailserver`, the IMAP server is Dovecot. The MUA (client) may vary, yet its Retrieval request is performed as [TCP][wikipedia-tcp] packets sent over the _public_ internet. This exchange of information may be secured in order to counter eavesdropping. @@ -201,31 +200,20 @@ The best practice as of 2020 would be [POP3S][wikipedia-pop3s] on port 995, rath **`docker-mailserver`'s default configuration disables POP3 altogether.** One should expect MUAs to use TLS-encrypted IMAP for Retrieval. -## How does `docker-mailserver` help with setting everything up? - -As a _batteries included_ Docker image, `docker-mailserver` provides you with all the required components and a default configuration, to run a decent and secure mail-server. +## How Does `docker-mailserver` Help With Setting Everything Up? -One may then customize all aspects of its internal components. +As a _batteries included_ container image, `docker-mailserver` provides you with all the required components and a default configuration to run a decent and secure mail server. One may then customize all aspects of its internal components. - Simple customization is supported through [docker-compose configuration][github-file-compose] and the [env-mailserver][github-file-envmailserver] configuration file. - Advanced customization is supported through providing "monkey-patching" configuration files and/or [deriving your own image][github-file-dockerfile] from `docker-mailserver`'s upstream, for a complete control over how things run. -On the subject of security, one might consider `docker-mailserver`'s **default** configuration to _not_ be 100% secure: - -- it enables unencrypted traffic on port 25 -- it enables Explicit TLS (STARTTLS) on port 587, instead of Implicit TLS on port 465 - -We believe `docker-mailserver`'s default configuration to be a good middle ground: it goes slightly beyond "old" (1999) [RFC 2487][rfc-2487]; and with developer friendly configuration settings, it makes it pretty easy to abide by the "newest" (2018) [RFC 8314][rfc-8314]. Eventually, it is up to _you_ deciding exactly what kind of transportation/encryption to use and/or enforce, and to customize your instance accordingly (with looser or stricter security). Be also aware that protocols and ports on your server can only go so far with security; third-party MTAs might relay your emails on insecure connections, man-in-the-middle attacks might still prove effective, etc. Advanced counter-measure such as DANE, MTA-STS and/or full body encryption (eg. PGP) should be considered as well for increased confidentiality, but ideally without compromising backwards compatibility so as to not block emails. -The [README][github-file-readme] is the best starting point in configuring and running your mail-server. You may then explore this wiki to cover additional topics, including but not limited to, security. - [docs-understandports]: ./config/security/understanding-the-ports.md [github-file-compose]: https://github.com/docker-mailserver/docker-mailserver/blob/master/docker-compose.yml [github-file-envmailserver]: https://github.com/docker-mailserver/docker-mailserver/blob/master/mailserver.env [github-file-dockerfile]: https://github.com/docker-mailserver/docker-mailserver/blob/master/Dockerfile -[github-file-readme]: https://github.com/docker-mailserver/docker-mailserver/blob/master/README.md [rfc-2487]: https://tools.ietf.org/html/rfc2487 [rfc-5321]: https://tools.ietf.org/html/rfc5321 [rfc-8314]: https://tools.ietf.org/html/rfc8314 @@ -236,7 +224,6 @@ The [README][github-file-readme] is the best starting point in configuring and r [wikipedia-clientserver]: https://en.wikipedia.org/wiki/Client%E2%80%93server_model [wikipedia-email]: https://en.wikipedia.org/wiki/Email [wikipedia-emailagent]: https://en.wikipedia.org/wiki/Email_agent_(infrastructure) -[wikipedia-esmtp]: https://en.wikipedia.org/wiki/ESMTP [wikipedia-imap]: https://en.wikipedia.org/wiki/IMAP [wikipedia-imaps]: https://en.wikipedia.org/wiki/IMAPS [wikipedia-mda]: https://en.wikipedia.org/wiki/Mail_delivery_agent @@ -248,4 +235,3 @@ The [README][github-file-readme] is the best starting point in configuring and r [wikipedia-smtps]: https://en.wikipedia.org/wiki/SMTPS [wikipedia-starttls]: https://en.wikipedia.org/wiki/Opportunistic_TLS [wikipedia-tcp]: https://en.wikipedia.org/wiki/Transmission_Control_Protocol -[wikipedia-tls]: https://en.wikipedia.org/wiki/Transport_Layer_Security diff --git a/docs/content/usage.md b/docs/content/usage.md index 8361102b284..4d845eaf471 100644 --- a/docs/content/usage.md +++ b/docs/content/usage.md @@ -2,14 +2,74 @@ title: Usage --- -This pages explains how to get started with DMS, basically explaining how you can use it. The procedure uses Docker Compose as a reference. In our examples, [`/docker-data/dms/config/`](../faq/#what-about-the-docker-datadmsmail-state-folder) on the host is mounted to `/tmp/docker-mailserver/` inside the container. +This pages explains how to get started with DMS. The guide uses Docker Compose as a reference. In our examples, a volume mounts the host location [`docker-data/dms/config/`][docs-dms-config-volume] to `/tmp/docker-mailserver/` inside the container. -## Available Images / Tags - Tagging Convention +[docs-dms-config-volume]: ./config/advanced/optional-config.md -[CI/CD](https://github.com/docker-mailserver/docker-mailserver/actions) will automatically build, test and push new images to container registries. Currently, the following registries are supported: +## Preliminary Steps -1. DockerHub ([`docker.io/mailserver/docker-mailserver`](https://hub.docker.com/r/mailserver/docker-mailserver)) -2. GitHub Container Registry ([`ghcr.io/docker-mailserver/docker-mailserver`](https://github.com/docker-mailserver/docker-mailserver/pkgs/container/docker-mailserver)) +Before you can get started with deploying your own mail server, there are some requirements to be met: + +1. You need to have a host that you can manage. +2. You need to own a domain, and you need to able to manage DNS for this domain. + +### Host Setup + +There are a few requirements for a suitable host system: + +1. The host should have a static IP address; otherwise you will need to dynamically update DNS (undesirable due to DNS caching) +2. The host should be able to send/receive on the [necessary ports for mail][docs-ports-overview] +3. You should be able to set a `PTR` record for your host; security-hardened mail servers might otherwise reject your mail server as the IP address of your host does not resolve correctly/at all to the DNS name of your server. + +On the host, you should have a suitable container runtime (like _Docker_ or _Podman_) installed. We assume [_Docker Compose_][docker-compose] is [installed][docker-compose-installation]. + +!!! info "Podman Support" + + If you're using podman, make sure to read the related [documentation][docs-podman]. + +[docs-podman]: ./config/advanced/podman.md +[docs-ports-overview]: ./config/security/understanding-the-ports.md#overview-of-email-ports +[docker-compose]: https://docs.docker.com/compose/ +[docker-compose-installation]: https://docs.docker.com/compose/install/ + +### Minimal DNS Setup + +The DNS setup is a big and essential part of the whole setup. There is a lot of confusion for newcomers and people starting out when setting up DNS. This section provides an example configuration and supplementary explanation. We expect you to be at least a bit familiar with DNS, what it does and what the individual record types are. + +Now let's say you just bought `example.com` and you want to be able to send and receive e-mails for the address `test@example.com`. On the most basic level, you will need to + +1. set an `MX` record for your domain `example.com` - in our example, the MX record contains `mail.example.com` +2. set an `A` record that resolves the name of your mail server - in our example, the A record contains `11.22.33.44` +3. (in a best-case scenario) set a `PTR` record that resolves the IP of your mail server - in our example, the PTR contains `mail.example.com` + +We will later dig into DKIM, DMARC & SPF, but for now, these are the records that suffice in getting you up and running. Here is a short explanation of what the records do: + +- The **MX record** tells everyone which (DNS) name is responsible for e-mails on your domain. + Because you want to keep the option of running another service on the domain name itself, you run your mail server on `mail.example.com`. + This does not imply your e-mails will look like `test@mail.example.com`, the DNS name of your mail server is decoupled of the domain it serves e-mails for. + In theory, you mail server could even serve e-mails for `test@some-other-domain.com`, if the MX record for `some-other-domain.com` points to `mail.example.com`. +- The **A record** tells everyone which IP address the DNS name `mail.example.com` resolves to. +- The **PTR record** is the counterpart of the A record, telling everyone what name the IP address `11.22.33.44` resolves to. + +If you setup everything, it should roughly look like this: + +```console +$ dig @1.1.1.1 +short MX example.com +mail.example.com +$ dig @1.1.1.1 +short A mail.example.com +11.22.33.44 +$ dig @1.1.1.1 +short -x 11.22.33.44 +mail.example.com +``` + +## Deploying the Actual Image + +### Tagging Convention + +To understand which tags you should use, read this section carefully. [Our CI][github-ci] will automatically build, test and push new images to the following container registries: + +1. DockerHub ([`docker.io/mailserver/docker-mailserver`][dockerhub-image]) +2. GitHub Container Registry ([`ghcr.io/docker-mailserver/docker-mailserver`][ghcr-image]) All workflows are using the tagging convention listed below. It is subsequently applied to all images. @@ -18,41 +78,34 @@ All workflows are using the tagging convention listed below. It is subsequently | `push` on `master` | `edge` | | `push` a tag (`v1.2.3`) | `1.2.3`, `1.2`, `1`, `latest` | -## Get the Tools - -!!! note "`setup.sh` Not Required Anymore" +[github-ci]: https://github.com/docker-mailserver/docker-mailserver/actions +[dockerhub-image]: https://hub.docker.com/r/mailserver/docker-mailserver +[ghcr-image]: https://github.com/docker-mailserver/docker-mailserver/pkgs/container/docker-mailserver - Since DMS `v10.2.0`, [`setup.sh` functionality](../faq/#how-to-adjust-settings-with-the-user-patchessh-script) is included within the container image. The external convenience script is no longer required if you prefer using `docker exec setup ` instead. +### Get All Files Issue the following commands to acquire the necessary files: ``` BASH -DMS_GITHUB_URL='https://raw.githubusercontent.com/docker-mailserver/docker-mailserver/master' +DMS_GITHUB_URL="https://github.com/docker-mailserver/docker-mailserver/blob/latest" wget "${DMS_GITHUB_URL}/docker-compose.yml" wget "${DMS_GITHUB_URL}/mailserver.env" - -# Optional -wget "${DMS_GITHUB_URL}/setup.sh" -chmod a+x ./setup.sh ``` -## Create a docker-compose Environment - -1. [Install the latest Docker Compose](https://docs.docker.com/compose/install/) -2. Edit `docker-compose.yml` to your liking - - substitute `mail.example.com` according to your FQDN - - if you want to use SELinux for the `./docker-data/dms/config/:/tmp/docker-mailserver/` mount, append `-z` or `-Z` -3. Configure the mailserver container to your liking by editing `mailserver.env` ([**Documentation**](../config/environment/)), but keep in mind this `.env` file: - - [_only_ basic `VAR=VAL`](https://docs.docker.com/compose/env-file/) is supported (**do not** quote your values) - - variable substitution is **not** supported (e.g. :no_entry_sign: `OVERRIDE_HOSTNAME=$HOSTNAME.$DOMAINNAME` :no_entry_sign:) - -!!! info "Podman Support" +### Configuration Steps - If you're using podman, make sure to read the related [documentation](../config/advanced/podman/) +1. First edit `docker-compose.yml` to your liking + - Substitute `mail.example.com` according to your FQDN. + - If you want to use SELinux for the `./docker-data/dms/config/:/tmp/docker-mailserver/` mount, append `-z` or `-Z`. +2. Then configure the environment specific to the mail server by editing [`mailserver.env`][docs-environment], but keep in mind that: + - only [basic `VAR=VAL`][docker-compose-env-file] is supported + - do not quote your values + - variable substitution is not supported, e.g. `OVERRIDE_HOSTNAME=$HOSTNAME.$DOMAINNAME` does not work -## Get up and running +[docs-environment]: ./config/environment.md +[docker-compose-env-file]: https://docs.docker.com/compose/env-file/ -### First Things First +### Get Up and Running !!! danger "Using the Correct Commands For Stopping and Starting DMS" @@ -60,13 +113,12 @@ chmod a+x ./setup.sh Using `Ctrl+C` **is not supported either**! -You are able to get a full overview of how the configuration works by either running: - -1. `./setup.sh help` which includes the options of `setup.sh`. -2. `docker run --rm docker.io/mailserver/docker-mailserver:latest setup help` which provides you with all the information on configuration provided "inside" the container itself. +For an overview of commands to manage DMS config, run: `docker exec -it setup help`. ??? info "Usage of `setup.sh` when no DMS Container Is Running" + We encourage you to directly use `setup` inside the container (like shown above). If you still want to use `setup.sh`, here's some information about it. + If no DMS container is running, any `./setup.sh` command will check online for the `:latest` image tag (the current _stable_ release), performing a `docker pull ...` if necessary followed by running the command in a temporary container: ```console @@ -78,7 +130,7 @@ You are able to get a full overview of how the configuration works by either run setup - 'docker-mailserver' Administration & Configuration script ... - $ docker run --rm docker.io/mailserver/docker-mailserver:latest setup help + $ docker run --rm ghcr.io/docker-mailserver/docker-mailserver:latest setup help SETUP(1) NAME @@ -86,50 +138,71 @@ You are able to get a full overview of how the configuration works by either run ... ``` -### Starting for the first time +On first start, you will need to add at least one email account (unless you're using LDAP). You have two minutes to do so, otherwise DMS will shutdown and restart. You can add accounts by running `docker exec -ti setup email add user@example.com`. **That's it! It really is that easy**. -On first start, you will need to add at least one email account (unless you're using LDAP). You have two minutes to do so, otherwise DMS will shutdown and restart. You can add accounts with the following two methods: - -1. Use `setup.sh`: `./setup.sh email add ` -2. Run the command directly in the container: `docker exec -ti setup email add ` +## Further Miscellaneous Steps -You can then proceed by creating the postmaster alias and by creating DKIM keys. +### Aliases -``` BASH -docker-compose up -d mailserver +You should add at least one [alias][docs-aliases], the [_postmaster alias_][docs-env-postmaster]. This is a common convention, but not strictly required. -# you may add some more users -# for SELinux, use -Z -./setup.sh [-Z] email add [] +[docs-aliases]: ./config/user-management/aliases.md +[docs-env-postmaster]: ./config/environment.md#postmaster_address -# and configure aliases, DKIM and more -./setup.sh [-Z] alias add postmaster@ +```bash +docker exec -ti setup alias add postmaster@example.com user@example.com ``` -## Further Miscellaneous Steps +### DKIM Keys -### DNS - DKIM +You can (_and you should_) generate DKIM keys. For more information: -You can (and you should) generate DKIM keys by running +- DKIM [with OpenDKIM][docs-dkim-opendkim] (_enabled by default_) +- DKIM [with Rspamd][docs-dkim-rspamd] (_when using `ENABLE_RSPAMD=1`_) -``` BASH -./setup.sh [-Z] config dkim -``` +When keys are generated, you can configure your DNS server by just pasting the content of `config/opendkim/keys/domain.tld/mail.txt` to [set up DKIM][dkim-signing-setup]. See the [documentation][docs-dkim-dns] for more details. -If you want to see detailed usage information, run +!!! note -``` BASH -./setup.sh config dkim help -``` + In case you're using LDAP, the setup looks a bit different as you do not add user accounts directly. Postfix doesn't know your domain(s) and you need to provide it when configuring DKIM: + + ``` BASH + docker exec -ti setup config dkim domain '[,]' + ``` -In case you're using LDAP, the setup looks a bit different as you do not add user accounts directly. Postfix doesn't know your domain(s) and you need to provide it when configuring DKIM: +[dkim-signing-setup]: https://mxtoolbox.com/dmarc/dkim/setup/how-to-setup-dkim +[docs-dkim-dns]: ./config/best-practices/dkim.md#configuration-using-a-web-interface +[docs-dkim-opendkim]: ./config/best-practices/dkim.md#enabling-dkim-signature +[docs-dkim-rspamd]: ./config/security/rspamd.md#dkim-signing -``` BASH -./setup.sh config dkim domain '[,]' +### Advanced DNS Setup + +You will very likely want to configure your DNS with these TXT records: [SPF, DKIM, and DMARC][cloudflare-spf-dkim-dmarc]. + +The following illustrates what a (rather strict) set of records could look like: + +```console +$ dig @1.1.1.1 +short TXT example.com +"v=spf1 mx -all" +$ dig @1.1.1.1 +short TXT dkim-rsa._domainkey.example.com +"v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQ..." +$ dig @1.1.1.1 +short TXT _dmarc.example.com +"v=DMARC1; p=reject; sp=reject; pct=100; adkim=s; aspf=s; fo=1" ``` -When keys are generated, you can configure your DNS server by just pasting the content of `config/opendkim/keys/domain.tld/mail.txt` to [set up DKIM](https://mxtoolbox.com/dmarc/dkim/setup/how-to-setup-dkim). See the [documentation](./config/best-practices/dkim.md) for more details. +[cloudflare-spf-dkim-dmarc]: https://www.cloudflare.com/learning/email-security/dmarc-dkim-spf/ ### Custom User Changes & Patches -If you'd like to change, patch or alter files or behavior of `docker-mailserver`, you can use a script. See [this part of our documentation](./faq.md/#how-to-adjust-settings-with-the-user-patchessh-script) for a detailed explanation. +If you'd like to change, patch or alter files or behavior of `docker-mailserver`, you can use a script. See [this part of our documentation][docs-user-patches] for a detailed explanation. + +[docs-user-patches]: ./faq.md#how-to-adjust-settings-with-the-user-patchessh-script + +## Testing + +Here are some tools you can use to verify your configuration: + +1. [MX Toolbox](https://mxtoolbox.com/SuperTool.aspx) +2. [DMARC Analyzer](https://www.mimecast.com/products/dmarc-analyzer/spf-record-check/) +3. [mail-tester.com](https://www.mail-tester.com/) +4. [multiRBL.valli.org](https://multirbl.valli.org/) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index fb7492e8e3f..13029f65b4d 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -114,7 +114,6 @@ nav: - 'Usage': usage.md - 'Configuration': - 'Environment Variables': config/environment.md - - 'Your Best Friend setup.sh': config/setup.sh.md - 'User Management': - 'Accounts': config/user-management/accounts.md - 'Aliases': config/user-management/aliases.md @@ -129,9 +128,9 @@ nav: - 'Fail2Ban': config/security/fail2ban.md - 'Mail Encryption' : config/security/mail_crypt.md - 'Rspamd' : config/security/rspamd.md - - 'Troubleshooting': - - 'Debugging': config/troubleshooting/debugging.md + - 'Debugging': config/debugging.md - 'Mail Delivery with POP3': config/pop3.md + - 'Your Friend setup.sh': config/setup.sh.md - 'Advanced Configuration': - 'Optional Configuration': config/advanced/optional-config.md - 'Maintenance': From 0c0f55f4e8a1bef063b0ca3c229a12a43bf93248 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 8 Apr 2023 22:01:46 +1200 Subject: [PATCH 020/592] chore: Remove `domainname` field from example compose (#3230) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- docker-compose.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5bbaaa3e318..8834d37ea02 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,10 +2,8 @@ services: mailserver: image: docker.io/mailserver/docker-mailserver:latest container_name: mailserver - # If the FQDN for your mail-server is only two labels (eg: example.com), - # you can assign this entirely to `hostname` and remove `domainname`. - hostname: mail - domainname: example.com + # Provide the FQDN of your mail server here (Your DNS MX record should point to this value) + hostname: mail.example.com env_file: mailserver.env # More information about the mail-server ports: # https://docker-mailserver.github.io/docker-mailserver/latest/config/security/understanding-the-ports/ @@ -24,8 +22,9 @@ services: - /etc/localtime:/etc/localtime:ro restart: always stop_grace_period: 1m - cap_add: - - NET_ADMIN + # Uncomment if using `ENABLE_FAIL2BAN=1`: + # cap_add: + # - NET_ADMIN healthcheck: test: "ss --listening --tcp | grep -P 'LISTEN.+:smtp' || exit 1" timeout: 3s From adb38207ad35101bb01ce21621baa1c342fa205d Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sat, 8 Apr 2023 23:54:35 +0200 Subject: [PATCH 021/592] add a note about TLS to "Usage" page (#3236) --- docs/content/usage.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/content/usage.md b/docs/content/usage.md index 4d845eaf471..fbda6e096eb 100644 --- a/docs/content/usage.md +++ b/docs/content/usage.md @@ -45,9 +45,9 @@ Now let's say you just bought `example.com` and you want to be able to send and We will later dig into DKIM, DMARC & SPF, but for now, these are the records that suffice in getting you up and running. Here is a short explanation of what the records do: - The **MX record** tells everyone which (DNS) name is responsible for e-mails on your domain. - Because you want to keep the option of running another service on the domain name itself, you run your mail server on `mail.example.com`. - This does not imply your e-mails will look like `test@mail.example.com`, the DNS name of your mail server is decoupled of the domain it serves e-mails for. - In theory, you mail server could even serve e-mails for `test@some-other-domain.com`, if the MX record for `some-other-domain.com` points to `mail.example.com`. + Because you want to keep the option of running another service on the domain name itself, you run your mail server on `mail.example.com`. + This does not imply your e-mails will look like `test@mail.example.com`, the DNS name of your mail server is decoupled of the domain it serves e-mails for. + In theory, you mail server could even serve e-mails for `test@some-other-domain.com`, if the MX record for `some-other-domain.com` points to `mail.example.com`. - The **A record** tells everyone which IP address the DNS name `mail.example.com` resolves to. - The **PTR record** is the counterpart of the A record, telling everyone what name the IP address `11.22.33.44` resolves to. @@ -142,6 +142,12 @@ On first start, you will need to add at least one email account (unless you're u ## Further Miscellaneous Steps +### Setting up TLS + +You definitely want to setup TLS. Please refer to [our documentation about TLS][docs-tls]. + +[docs-tls]: ./config/security/ssl.md + ### Aliases You should add at least one [alias][docs-aliases], the [_postmaster alias_][docs-env-postmaster]. This is a common convention, but not strictly required. @@ -165,7 +171,7 @@ When keys are generated, you can configure your DNS server by just pasting the c !!! note In case you're using LDAP, the setup looks a bit different as you do not add user accounts directly. Postfix doesn't know your domain(s) and you need to provide it when configuring DKIM: - + ``` BASH docker exec -ti setup config dkim domain '[,]' ``` From 2b7cab28f73f7cea32104a6ac93415f295894804 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 9 Apr 2023 11:27:00 +0200 Subject: [PATCH 022/592] compress & improve user management docs (#3232) The user management docs are now one page, because the division between accounts and aliases is useless because there simply isn't enough content to justify the split. I improved and updated the text a bit. --- .../config/advanced/optional-config.md | 4 +- docs/content/config/environment.md | 4 +- .../security/understanding-the-ports.md | 16 ++-- docs/content/config/user-management.md | 86 +++++++++++++++++++ .../config/user-management/accounts.md | 47 ---------- .../content/config/user-management/aliases.md | 42 --------- docs/content/usage.md | 2 +- docs/mkdocs.yml | 4 +- 8 files changed, 100 insertions(+), 105 deletions(-) create mode 100644 docs/content/config/user-management.md delete mode 100644 docs/content/config/user-management/accounts.md delete mode 100644 docs/content/config/user-management/aliases.md diff --git a/docs/content/config/advanced/optional-config.md b/docs/content/config/advanced/optional-config.md index 2e5c29e070e..259e05a20cb 100644 --- a/docs/content/config/advanced/optional-config.md +++ b/docs/content/config/advanced/optional-config.md @@ -41,8 +41,8 @@ This is a list of all configuration files and directories which are optional or - **user-patches.sh:** this file will be run after all configuration files are set up, but before the postfix, amavis and other daemons are started. (Docs: [FAQ - How to adjust settings with the `user-patches.sh` script][docs-faq-userpatches]) - **rspamd-commands:** list of simple commands to adjust Rspamd modules in an easy way (Docs: [Rspamd][docs-rspamd-commands]) -[docs-accounts-quota]: ../../config/user-management/accounts.md#notes -[docs-aliases-regex]: ../../config/user-management/aliases.md#configuring-regexp-aliases +[docs-accounts-quota]: ../../config/user-management.md#quotas +[docs-aliases-regex]: ../../config/user-management.md#configuring-regexp-aliases [docs-dkim]: ../../config/best-practices/dkim.md [docs-fail2ban]: ../../config/security/fail2ban.md [docs-faq-spamrules]: ../../faq.md#how-can-i-manage-my-custom-spamassassin-rules diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index ad5c86e9090..68a2fffeac1 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -213,7 +213,7 @@ Set the mailbox size limit for all users. If set to zero, the size will be unlim - **1** => Dovecot quota is enabled - 0 => Dovecot quota is disabled -See [mailbox quota][docs-accounts]. +See [mailbox quota][docs-accounts-quota]. ##### POSTFIX\_MESSAGE\_SIZE\_LIMIT @@ -831,4 +831,4 @@ you to replace both instead of just the envelope sender. [docs-tls-letsencrypt]: ./security/ssl.md#lets-encrypt-recommended [docs-tls-manual]: ./security/ssl.md#bring-your-own-certificates [docs-tls-selfsigned]: ./security/ssl.md#self-signed-certificates -[docs-accounts]: ./user-management/accounts.md#notes +[docs-accounts-quota]: ./user-management.md#quotas diff --git a/docs/content/config/security/understanding-the-ports.md b/docs/content/config/security/understanding-the-ports.md index 6a3e91b9473..4cd6ded5cd1 100644 --- a/docs/content/config/security/understanding-the-ports.md +++ b/docs/content/config/security/understanding-the-ports.md @@ -22,11 +22,11 @@ Prefer ports with Implicit [TLS][wikipedia-tls] ports, they're more secure than ??? warning "Beware of outdated advice on port 465" There is a common misconception of this port due to it's history detailed by various communities and blogs articles on the topic (_including by popular mail relay services_). - + Port 465 was [briefly assigned the role of SMTPS in 1997][wikipedia-smtps] as an secure alternative to Port 25 between MTA exchanges. Then RFC 2487 (`STARTTLS`) [while still in a draft status in late 1998 had IANA revoke the SMTPS assignment][history-465-revoked]. The [draft history was modified to exclude all mention of port 465 and SMTPS][history-465-politics]. - + In 2018 [RFC 8314][rfc-8314] was published which revives Port 465 as an Implicit TLS alternative to Port 587 for mail submission. It details very clearly that gaining adoption of 465 as the preferred port will take time. IANA reassigned [port 465 as the `submissions` service][iana-services-465]. Any unofficial usage as **SMTPS is legacy and has been for over two decades**. - + Understand that port 587 is more broadly supported due to this history and that lots of software in that time has been built or configured with that port in mind. [`STARTTLS` is known to have various CVEs discovered even in recent years][starttls-vulnerabilities], do not be misled by any advice implying it should be preferred over implicit TLS. Trust in more official sources, such as the [config Postfix has][postfix-upstream-config-mastercf] which acknowledges the `submissions` port (465). @@ -82,7 +82,7 @@ Mail arriving at your server will be processed and stored in a mailbox, or sent #### Outbound Traffic (On the Right) -Mail being sent from your server is either being relayed through another MTA (eg: SendGrid), or direct to an MTA responsible for an email address (eg: Gmail). +Mail being sent from your server is either being relayed through another MTA (eg: SendGrid), or direct to an MTA responsible for an email address (eg: Gmail). - **Port 25:** - As most MTA use port 25 to receive inbound mail, when no authenticated relay is involved this is the outbound port used. @@ -95,10 +95,10 @@ Mail being sent from your server is either being relayed through another MTA (eg !!! tip `docker-mailserver` can function as a relay too, but professional relay services have a trusted reputation (_which increases success of delivery_). - + An MTA with low reputation can affect if mail is treated as junk, or even rejected. -!!! note +!!! note At best, you can only ensure a secure connection between the MTA you directly connect to. The receiving MTA may relay that mail to another MTA (_and so forth_), each connection may not be enforcing TLS. @@ -119,7 +119,7 @@ Sometimes a reverse-proxy is involved, but is misconfigured or lacks support for !!! warning `STARTTLS` [continues to have vulnerabilities found][starttls-vulnerabilities] (Nov 2021 article), as per [RFC 8314 (Section 4.1)][rfc-8314-s41] you are encouraged to **prefer Implicit TLS where possible**. - + Support for `STARTTLS` is not always implemented correctly, which can lead to leaking credentials (like a client sending too early) prior to a TLS connection being established. Third-parties such as some ISPs have also been known to intercept the `STARTTLS` exchange, modifying network traffic to prevent establishing a secure connection. @@ -146,7 +146,7 @@ Unlike with HTTP where a web browser client communicates directly with the serve Other machines that facilitate a connection that generally aren't taken into account can exist between a client and server, such as those where your connection passes through your ISP provider are capable of compromising a `cleartext` connection through interception. -[docs-accounts]: ../user-management/accounts.md +[docs-accounts]: ../user-management.md#accounts [docs-relays]: ../advanced/mail-forwarding/relay-hosts.md [iana-services-465]: https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml?search=465 [starttls-policy-list]: https://github.com/EFForg/starttls-everywhere#email-security-database-starttls-policy-list diff --git a/docs/content/config/user-management.md b/docs/content/config/user-management.md new file mode 100644 index 00000000000..a909157e9ff --- /dev/null +++ b/docs/content/config/user-management.md @@ -0,0 +1,86 @@ +# User Management + +## Accounts + +Users (email accounts) are managed in `/tmp/docker-mailserver/postfix-accounts.cf`. The best way to manage accounts is to use the reliable `setup` command inside the container. Just run `docker exec setup help` and have a look at the section about subcommands, specifically the `email` subcommand. + +### Adding a new Account + +#### Via `setup` inside the container + +You can add an account by running `docker exec -ti setup email add `. This method is strongly preferred. + +#### Manually + +!!! warning + + This method is discouraged! + +Alternatively, you may directly add the full email address and its encrypted password, separated by a pipe. To generate a new mail account data, directly from your host, you could for example run the following: + +```sh +docker run --rm -it \ + --env MAIL_USER=user1@example.com \ + --env MAIL_PASS=mypassword \ + ghcr.io/docker-mailserver/docker-mailserver:latest \ + /bin/bash -c \ + 'echo "${MAIL_USER}|$(doveadm pw -s SHA512-CRYPT -u ${MAIL_USER} -p ${MAIL_PASS})" >>docker-data/dms/config/postfix-accounts.cf' +``` + +You will then be asked for a password, and be given back the data for a new account entry, as text. To actually _add_ this new account, just copy all the output text in `docker-data/dms/config/postfix-accounts.cf` file of your running container. + +The result could look like this: + +```cf +user1@example.com|{SHA512-CRYPT}$6$2YpW1nYtPBs2yLYS$z.5PGH1OEzsHHNhl3gJrc3D.YMZkvKw/vp.r5WIiwya6z7P/CQ9GDEJDr2G2V0cAfjDFeAQPUoopsuWPXLk3u1 +``` + +### Quotas + +- `imap-quota` is enabled and allow clients to query their mailbox usage. +- When the mailbox is deleted, the quota directive is deleted as well. +- Dovecot quotas support LDAP, **but it's not implemented** (_PRs are welcome!_). + +## Aliases + +The best way to manage aliases is to use the reliable `setup` script inside the container. Just run `docker exec setup help` and have a look at the section about subcommands, specifically the `alias`-subcommand. + +### About + +You may read [Postfix's documentation on virtual aliases][postfix-docs-alias] first. Aliases are managed in `/tmp/docker-mailserver/postfix-virtual.cf`. An alias is a full email address that will either be: + +- delivered to an existing account registered in `/tmp/docker-mailserver/postfix-accounts.cf` +- redirected to one or more other email addresses + +Alias and target are space separated. An example on a server with `example.com` as its domain: + +```cf +# Alias delivered to an existing account +alias1@example.com user1@example.com + +# Alias forwarded to an external email address +alias2@example.com external-account@gmail.com +``` + +### Configuring RegExp Aliases + +Additional regexp aliases can be configured by placing them into `docker-data/dms/config/postfix-regexp.cf`. The regexp aliases get evaluated after the virtual aliases (container path: `/tmp/docker-mailserver/postfix-virtual.cf`). For example, the following `docker-data/dms/config/postfix-regexp.cf` causes all email sent to "test" users to be delivered to `qa@example.com` instead: + +```cf +/^test[0-9][0-9]*@example.com/ qa@example.com +``` + +### Address Tags (Extension Delimiters) as an alternative to Aliases + +Postfix supports so-called address tags, in the form of plus (+) tags - i.e. `address+tag@example.com` will end up at `address@example.com`. This is configured by default and the (configurable!) separator is set to `+`. For more info, see [Postfix's official documentation][postfix-docs-extension-delimiters]. + +!!! note + + If you do decide to change the configurable separator, you must add the same line to *both* `docker-data/dms/config/postfix-main.cf` and `docker-data/dms/config/dovecot.cf`, because Dovecot is acting as the delivery agent. For example, to switch to `-`, add: + + ```cf + recipient_delimiter = - + ``` + +[postfix-docs-alias]: http://www.postfix.org/VIRTUAL_README.html#virtual_alias +[postfix-docs-extension-delimiters]: http://www.postfix.org/postconf.5.html#recipient_delimiter diff --git a/docs/content/config/user-management/accounts.md b/docs/content/config/user-management/accounts.md deleted file mode 100644 index cc447627e05..00000000000 --- a/docs/content/config/user-management/accounts.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -title: 'User Management | Accounts' -hide: - - toc # Hide Table of Contents for this page ---- - -## Adding a New Account - -Users (email accounts) are managed in `/tmp/docker-mailserver/postfix-accounts.cf`. **_The best way to manage accounts is to use the reliable [`setup.sh`][docs-setupsh] script_**. Or you may directly add the _full_ email address and its encrypted password, separated by a pipe: - -```cf -user1@example.com|{SHA512-CRYPT}$6$2YpW1nYtPBs2yLYS$z.5PGH1OEzsHHNhl3gJrc3D.YMZkvKw/vp.r5WIiwya6z7P/CQ9GDEJDr2G2V0cAfjDFeAQPUoopsuWPXLk3u1 -user2@not-example.com|{SHA512-CRYPT}$6$2YpW1nYtPBs2yLYS$z.5PGH1OEzsHHNhl3gJrc3D.YMZkvKw/vp.r5WIiwya6z7P/CQ9GDEJDr2G2V0cAfjDFeAQPUoopsuWPXLk3u1 -``` - -In the example above, we've added 2 mail accounts for 2 different domains. Consequently, the mail-server will automatically be configured for multi-domains. Therefore, to generate a new mail account data, directly from your docker host, you could for example run the following: - -```sh -docker run --rm \ - -e MAIL_USER=user1@example.com \ - -e MAIL_PASS=mypassword \ - -it ghcr.io/docker-mailserver/docker-mailserver:latest \ - /bin/sh -c 'echo "$MAIL_USER|$(doveadm pw -s SHA512-CRYPT -u $MAIL_USER -p $MAIL_PASS)"' >> docker-data/dms/config/postfix-accounts.cf -``` - -You will then be asked for a password, and be given back the data for a new account entry, as text. To actually _add_ this new account, just copy all the output text in `docker-data/dms/config/postfix-accounts.cf` file of your running container. - -!!! note - - `doveadm pw` command lets you choose between several encryption schemes for the password. - - Use `doveadm pw -l` to get a list of the currently supported encryption schemes. - -!!! note - - Changes to the accounts list require a restart of the container, using `supervisord`. See [#552][github-issue-552]. - ---- - -### Notes - -- `imap-quota` is enabled and allow clients to query their mailbox usage. -- When the mailbox is deleted, the quota directive is deleted as well. -- Dovecot quotas support LDAP, **but it's not implemented** (_PRs are welcome!_). - -[docs-setupsh]: ../setup.sh.md -[github-issue-552]: https://github.com/docker-mailserver/docker-mailserver/issues/552 diff --git a/docs/content/config/user-management/aliases.md b/docs/content/config/user-management/aliases.md deleted file mode 100644 index ed68f9456ac..00000000000 --- a/docs/content/config/user-management/aliases.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: 'User Management | Aliases' ---- - -Please read the [Postfix documentation on virtual aliases](http://www.postfix.org/VIRTUAL_README.html#virtual_alias) first. - -You can use [`setup.sh`][docs-setupsh] instead of creating and editing files manually. Aliases are managed in `/tmp/docker-mailserver/postfix-virtual.cf`. An alias is a _full_ email address that will either be: - -* delivered to an existing account registered in `/tmp/docker-mailserver/postfix-accounts.cf` -* redirected to one or more other email addresses - -Alias and target are space separated. An example on a server with example.com as its domain: - -```cf -# Alias delivered to an existing account -alias1@example.com user1@example.com - -# Alias forwarded to an external email address -alias2@example.com external-account@gmail.com -``` - -## Configuring RegExp Aliases - -Additional regexp aliases can be configured by placing them into `docker-data/dms/config/postfix-regexp.cf`. The regexp aliases get evaluated after the virtual aliases (container path: `/tmp/docker-mailserver/postfix-virtual.cf`). For example, the following `docker-data/dms/config/postfix-regexp.cf` causes all email sent to "test" users to be delivered to `qa@example.com` instead: - -```cf -/^test[0-9][0-9]*@example.com/ qa@example.com -``` - -## Address Tags (Extension Delimiters) an Alternative to Aliases - -Postfix supports so-called address tags, in the form of plus (+) tags - i.e. `address+tag@example.com` will end up at `address@example.com`. This is configured by default and the (configurable !) separator is set to `+`. For more info, see the [official documentation](http://www.postfix.org/postconf.5.html#recipient_delimiter). - -!!! note - - If you do decide to change the configurable separator, you must add the same line to *both* `docker-data/dms/config/postfix-main.cf` and `docker-data/dms/config/dovecot.cf`, because Dovecot is acting as the delivery agent. For example, to switch to `-`, add: - -```cf -recipient_delimiter = - -``` - -[docs-setupsh]: ../setup.sh.md diff --git a/docs/content/usage.md b/docs/content/usage.md index fbda6e096eb..bc34f665902 100644 --- a/docs/content/usage.md +++ b/docs/content/usage.md @@ -152,7 +152,7 @@ You definitely want to setup TLS. Please refer to [our documentation about TLS][ You should add at least one [alias][docs-aliases], the [_postmaster alias_][docs-env-postmaster]. This is a common convention, but not strictly required. -[docs-aliases]: ./config/user-management/aliases.md +[docs-aliases]: ./config/user-management.md#aliases [docs-env-postmaster]: ./config/environment.md#postmaster_address ```bash diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 13029f65b4d..84fb7b7fd78 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -114,9 +114,7 @@ nav: - 'Usage': usage.md - 'Configuration': - 'Environment Variables': config/environment.md - - 'User Management': - - 'Accounts': config/user-management/accounts.md - - 'Aliases': config/user-management/aliases.md + - 'User Management': config/user-management.md - 'Best Practices': - 'DKIM': config/best-practices/dkim.md - 'DMARC': config/best-practices/dmarc.md From fedc3b3ee0e96e12de41e30ea1d8384c83451518 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 9 Apr 2023 11:42:50 +0200 Subject: [PATCH 023/592] docs: update docs that cite compose files (#3234) --- .../config/advanced/full-text-search.md | 4 +- docs/content/config/advanced/ipv6.md | 8 +- docs/content/config/environment.md | 6 +- docs/content/config/security/ssl.md | 85 ++++--------------- .../tutorials/mailserver-behind-proxy.md | 5 +- docs/content/faq.md | 16 +++- 6 files changed, 37 insertions(+), 87 deletions(-) diff --git a/docs/content/config/advanced/full-text-search.md b/docs/content/config/advanced/full-text-search.md index 8c513047106..03379188a13 100644 --- a/docs/content/config/advanced/full-text-search.md +++ b/docs/content/config/advanced/full-text-search.md @@ -58,13 +58,11 @@ While indexing is memory intensive, you can configure the plugin to limit the am 2. Update `docker-compose.yml` to load the previously created dovecot plugin config file: ```yaml - version: '3.8' services: mailserver: image: ghcr.io/docker-mailserver/docker-mailserver:latest container_name: mailserver - hostname: mail - domainname: example.com + hostname: mail.example.com env_file: mailserver.env ports: - "25:25" # SMTP (explicit TLS => STARTTLS) diff --git a/docs/content/config/advanced/ipv6.md b/docs/content/config/advanced/ipv6.md index 56c02ee484a..7b3a8ca4dca 100644 --- a/docs/content/config/advanced/ipv6.md +++ b/docs/content/config/advanced/ipv6.md @@ -12,11 +12,7 @@ This can be solved by supporting IPv6 connections all the way to the `docker-mai ```diff +++ b/serv/docker-compose.yml -@@ -1,4 +1,4 @@ --version: '2' -+version: '2.1' - -@@ -32,6 +32,16 @@ services: +@@ ... @@ services: + ipv6nat: + image: robbertkl/ipv6nat @@ -29,7 +25,7 @@ This can be solved by supporting IPv6 connections all the way to the `docker-mai + - /var/run/docker.sock:/var/run/docker.sock:ro + - /lib/modules:/lib/modules:ro -@@ -306,4 +316,13 @@ networks: +@@ ... @@ networks: + default: + driver: bridge diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 68a2fffeac1..a3307916d13 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -10,8 +10,10 @@ title: Environment Variables ##### OVERRIDE_HOSTNAME -- **empty** => uses the `hostname` command to get canonical hostname for `docker-mailserver` to use. -- => Specify a fully-qualified domainname to serve mail for. This is used for many of the config features so if you can't set your hostname (_eg: you're in a container platform that doesn't let you_) specify it via this environment variable. It will take priority over `docker run` options: `--hostname` and `--domainname`, or `docker-compose.yml` config equivalents: `hostname:` and `domainname:`. +If you can't set your hostname (_eg: you're in a container platform that doesn't let you_) specify it via this environment variable. It will have priority over `docker run --hostname`, or the equivalent `hostname:` field in `docker-compose.yml`. + +- **empty** => Uses the `hostname -f` command to get canonical hostname for `docker-mailserver` to use. +- => Specify an FQDN (fully-qualified domain name) to serve mail for. The hostname is required for `docker-mailserver` to function correctly. ##### LOG_LEVEL diff --git a/docs/content/config/security/ssl.md b/docs/content/config/security/ssl.md index 2588b975123..ff0fe34e0fa 100644 --- a/docs/content/config/security/ssl.md +++ b/docs/content/config/security/ssl.md @@ -27,43 +27,29 @@ After installation, you can test your setup with: An [FQDN](https://en.wikipedia.org/wiki/Fully_qualified_domain_name) (_Fully Qualified Domain Name_) such as `mail.example.com` is required for `docker-mailserver` to function correctly, especially for looking up the correct SSL certificate to use. -Internally, `hostname -f` will be used to retrieve the FQDN as configured in the below examples. +- `mail.example.com` will still use `user@example.com` as the mail address. You do not need a bare domain for that. +- We usually discourage assigning a bare domain (_When your DNS MX record does not point to a subdomain_) to represent `docker-mailserver`. However, an FQDN of [just `example.com` is also supported][docs-faq-baredomain]. +- Internally, `hostname -f` will be used to retrieve the FQDN as configured in the below examples. +- Wildcard certificates (eg: `*.example.com`) are supported for `SSL_TYPE=letsencrypt`. Your configured FQDN below may be `mail.example.com`, and your wildcard certificate provisioned to `/etc/letsencrypt/live/example.com` which will be checked as a fallback FQDN by `docker-mailserver`. -Wildcard certificates (eg: `*.example.com`) are supported for `SSL_TYPE=letsencrypt`. Your configured FQDN below may be `mail.example.com`, and your wildcard certificate provisioned to `/etc/letsencrypt/live/example.com` which will be checked as a fallback FQDN by `docker-mailserver`. +!!! example "Setting the hostname correctly" -!!! example "Docker CLI options `--hostname` and optionally `--domainname`" + Change `mail.example.com` below to your own FQDN. ```sh - docker run --hostname mail --domainname example.com - # `--domainname` is not required: + # CLI: docker run --hostname mail.example.com ``` - -!!! example "`docker-compose.yml` config" - + + or + ```yml - services: - mailserver: - hostname: mail - domainname: example.com - # `domainname` is not required: + # docker-compose.yml services: mailserver: hostname: mail.example.com ``` -!!! example "_Bare domains_ (eg: `example.com`) should only use the hostname option" - - ```sh - docker run --hostname example.com - ``` - - ```yml - services: - mailserver: - hostname: example.com - ``` - ## Provisioning methods ### Let's Encrypt (Recommended) @@ -91,9 +77,7 @@ You don't have to do anything else. Enjoy! ```yaml services: mailserver: - # For the FQDN 'mail.example.com': - hostname: mail - domainname: example.com + hostname: mail.example.com environment: - SSL_TYPE=letsencrypt volumes: @@ -107,7 +91,6 @@ Certbot provisions certificates to `/etc/letsencrypt`. Add a volume to store the 1. Getting a certificate is this simple! (_Referencing: [Certbot docker instructions][certbot::docker] and [`certonly --standalone` mode][certbot::standalone]_): ```sh - # Change `mail.example.com` below to your own FQDN. # Requires access to port 80 from the internet, adjust your firewall if needed. docker run --rm -it \ -v "${PWD}/docker-data/certbot/certs/:/etc/letsencrypt/" \ @@ -125,9 +108,7 @@ Certbot provisions certificates to `/etc/letsencrypt`. Add a volume to store the ```yaml services: mailserver: - # For the FQDN 'mail.example.com': - hostname: mail - domainname: example.com + hostname: mail.example.com environment: - SSL_TYPE=letsencrypt volumes: @@ -367,7 +348,6 @@ The following example is the [basic setup][acme-companion::basic-setup] you need You should have an existing `docker-compose.yml` with a `mailserver` service. Below are the modifications to add for integrating with `nginx-proxy` and `acme-companion` services: ```yaml - version: '3.8' services: # Add the following `environment` and `volumes` to your existing `mailserver` service: mailserver: @@ -395,12 +375,10 @@ The following example is the [basic setup][acme-companion::basic-setup] you need volumes: # `certs/`: Managed by the `acme-companion` container (_read-only_). # `docker.sock`: Required to interact with containers via the Docker API. - # `dhparam`: A named data volume to prevent `nginx-proxy` creating an anonymous volume each time. - ./docker-data/nginx-proxy/html/:/usr/share/nginx/html/ - ./docker-data/nginx-proxy/vhost.d/:/etc/nginx/vhost.d/ - ./docker-data/acme-companion/certs/:/etc/nginx/certs/:ro - /var/run/docker.sock:/tmp/docker.sock:ro - - dhparam:/etc/nginx/dhparam acme-companion: image: nginxproxy/acme-companion @@ -421,12 +399,6 @@ The following example is the [basic setup][acme-companion::basic-setup] you need - ./docker-data/acme-companion/certs/:/etc/nginx/certs/:rw - ./docker-data/acme-companion/acme-state/:/etc/acme.sh/ - /var/run/docker.sock:/var/run/docker.sock:ro - - # Once `nginx-proxy` fixes their Dockerfile, this named data volume can be removed from docs. - # Users can opt for a local bind mount volume like all others if they prefer, but this volume - # is only intended to be temporary. - volumes: - dhparam: ``` !!! tip "Optional ENV vars worth knowing about" @@ -494,8 +466,6 @@ Amongst other things, you can use these to secure your mail-server. DSM locates Navigate to that folder and note the 6 character random folder name of the certificate you'd like to use. Then, add the following to your `docker-compose.yml` declaration file: ```yaml -# Note: If you have an existing setup that was working pre docker-mailserver v10.2, -# '/tmp/dms/custom-certs' below has replaced the previous '/tmp/ssl' container path. volumes: - /usr/syno/etc/certificate/_archive//:/tmp/dms/custom-certs/ environment: @@ -508,16 +478,6 @@ DSM-generated letsencrypt certificates get auto-renewed every three months. ### Caddy -If you are using Caddy to renew your certificates, please note that only RSA certificates work. Read [#1440][github-issue-1440] for details. In short for Caddy v1 the `Caddyfile` should look something like: - -```caddyfile -https://mail.example.com { - tls admin@example.com { - key_type rsa2048 - } -} -``` - For Caddy v2 you can specify the `key_type` in your server's global settings, which would end up looking something like this if you're using a `Caddyfile`: ```caddyfile @@ -533,7 +493,7 @@ For Caddy v2 you can specify the `key_type` in your server's global settings, wh If you are instead using a json config for Caddy v2, you can set it in your site's TLS automation policies: -???+ example "Example Code" +??? example "Caddy v2 JSON example snippet" ```json { @@ -600,7 +560,7 @@ If you are instead using a json config for Caddy v2, you can set it in your site } ``` -The generated certificates can be mounted: +The generated certificates can then be mounted: ```yaml volumes: @@ -608,15 +568,6 @@ volumes: - ${CADDY_DATA_DIR}/certificates/acme-v02.api.letsencrypt.org-directory/mail.example.com/mail.example.com.key:/etc/letsencrypt/live/mail.example.com/privkey.pem ``` -EC certificates fail in the TLS handshake: - -```log -CONNECTED(00000003) -140342221178112:error:14094410:SSL routines:ssl3_read_bytes:sslv3 alert handshake failure:ssl/record/rec_layer_s3.c:1543:SSL alert number 40 -no peer certificate available -No client certificate CA names sent -``` - ### Traefik v2 [Traefik][traefik::github] is an open-source application proxy using the [ACME protocol][ietf::rfc::acme]. [Traefik][traefik::github] can request certificates for domains and subdomains, and it will take care of renewals, challenge negotiations, etc. We strongly recommend to use [Traefik][traefik::github]'s major version 2. @@ -637,13 +588,11 @@ This setup only comes with one caveat: The domain has to be configured on anothe Here is an example setup for [`docker-compose`](https://docs.docker.com/compose/): ```yaml - version: '3.8' services: mailserver: image: ghcr.io/docker-mailserver/docker-mailserver:latest container_name: mailserver - hostname: mail - domainname: example.com + hostname: mail.example.com volumes: - ./docker-data/traefik/acme.json:/etc/letsencrypt/acme.json:ro environment: @@ -927,10 +876,10 @@ Despite this, if you must use non-standard DH parameters or you would like to sw [docs-env::ssl-type]: ../environment.md#ssl_type [docs-optional-config]: ../advanced/optional-config.md +[docs-faq-baredomain]: ../../faq.md#can-i-use-a-nakedbare-domain-ie-no-hostname [github-file-compose]: https://github.com/docker-mailserver/docker-mailserver/blob/master/docker-compose.yml [github-file::tls-readme]: https://github.com/docker-mailserver/docker-mailserver/blob/3b8059f2daca80d967635e04d8d81e9abb755a4d/test/test-files/ssl/example.test/README.md -[github-issue-1440]: https://github.com/docker-mailserver/docker-mailserver/issues/1440 [hanscees-renewcerts]: https://github.com/hanscees/dockerscripts/blob/master/scripts/tomav-renew-certs [traefik::github]: https://github.com/containous/traefik diff --git a/docs/content/examples/tutorials/mailserver-behind-proxy.md b/docs/content/examples/tutorials/mailserver-behind-proxy.md index cd5bc02de0f..208304ae2fc 100644 --- a/docs/content/examples/tutorials/mailserver-behind-proxy.md +++ b/docs/content/examples/tutorials/mailserver-behind-proxy.md @@ -30,7 +30,6 @@ Feel free to add your configuration if you achieved the same goal using differen Truncated configuration of traefik itself: ```yaml - version: '3.8' services: reverse-proxy: image: docker.io/traefik:latest # v2.5 @@ -57,13 +56,11 @@ Feel free to add your configuration if you achieved the same goal using differen Truncated list of necessary labels on the `docker-mailserver` container: ```yaml - version: '3.8' services: mailserver: image: ghcr.io/docker-mailserver/docker-mailserver:latest container_name: mailserver - hostname: mail - domainname: example.com + hostname: mail.example.com restart: always networks: - proxy diff --git a/docs/content/faq.md b/docs/content/faq.md index 0734481a455..7b81bc08820 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -168,7 +168,7 @@ To use a bare domain (_where the host name is `example.com` and the domain is al - From: `mydestination = $myhostname, localhost.$mydomain, localhost` - To: `mydestination = localhost.$mydomain, localhost` -Add the latter line to `docker-data/dms/config/postfix-main.cf`. If that doesn't work, make sure that `OVERRIDE_HOSTNAME` is blank in your `mailserver.env` file (see [#1731](https://github.com/docker-mailserver/docker-mailserver/issues/1731#issuecomment-753968425)). Without these changes there will be warnings in the logs like: +Add the latter line to `docker-data/dms/config/postfix-main.cf`. If that doesn't work, make sure that [`OVERRIDE_HOSTNAME` is blank in your `mailserver.env` file][github-comment-override-hostname]. Without these changes there will be warnings in the logs like: ```log warning: do not list domain example.com in BOTH mydestination and virtual_mailbox_domains @@ -176,7 +176,15 @@ warning: do not list domain example.com in BOTH mydestination and virtual_mailbo Plus of course mail delivery fails. -Also you need to define `hostname: example.com` in your docker-compose.yml and don't sepecify the `domainname:` at all. +Also you need to define `hostname: example.com` in your `docker-compose.yml`. + +!!! tip "You might not want a bare domain" + + We encourage you to consider using a subdomain where possible. + + - There are [benefits][github-comment-baredomain] to preferring a subdomain. + - A bare domain is not required to have `user@example.com`, that is distinct from your hostname which is identified by a DNS MX record. + ### How can I configure a catch-all? @@ -453,8 +461,6 @@ The following configuration works nicely: Or with [Docker Swarm](https://docs.docker.com/engine/swarm/configs/): ```yaml - version: '3.8' - services: mailserver: image: ghcr.io/docker-mailserver/docker-mailserver:latest @@ -519,6 +525,8 @@ $spam_quarantine_to = "amavis\@example.com"; [fail2ban-customize]: ./config/security/fail2ban.md [docs-maintenance]: ./config/advanced/maintenance/update-and-cleanup.md [docs-userpatches]: ./config/advanced/override-defaults/user-patches.md +[github-comment-baredomain]: https://github.com/docker-mailserver/docker-mailserver/issues/3048#issuecomment-1432358353 +[github-comment-override-hostname]: https://github.com/docker-mailserver/docker-mailserver/issues/1731#issuecomment-753968425 [github-issue-95]: https://github.com/docker-mailserver/docker-mailserver/issues/95 [github-issue-97]: https://github.com/docker-mailserver/docker-mailserver/issues/97 [github-issue-1247]: https://github.com/docker-mailserver/docker-mailserver/issues/1247 From dc8a08031fb6599ea7fc488b29a55af84e2483d4 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 10 Apr 2023 10:32:33 +0200 Subject: [PATCH 024/592] release: v12.0.0 (#3146) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Co-authored-by: casperklein --- CHANGELOG.md | 101 ++++++++++++++++++++++++++++++++++++++++++++++++++- VERSION | 2 +- 2 files changed, 101 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df3a6049f6a..3b6fa9b4e41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,109 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v11.3.0...HEAD) +## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v12.0.0...HEAD) > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +## [12.0.0](https://github.com/docker-mailserver/docker-mailserver/compare/v12.0.0...HEAD) + +Notable changes are: + +- Rspamd feature is promoted from preview status +- Services no longer use `chroot` +- Fail2Ban major version upgrade +- ARMv7 platform is no longer suppoted +- TLS 1.2 is the minimum supported protocol +- SMTP authentication on port 25 disabled +- The value of `smtpd_sender_restrictions` for Postfix has replaced the value ([#3127](https://github.com/docker-mailserver/docker-mailserver/pull/3127)): + - In `main.cf` with `$dms_smtpd_sender_restrictions` + - In `master.cf` inbound submissions ports 465 + 587 extend this inherited `smtpd` restriction with `$mua_sender_restrictions` + +### Added + +- **security**: Rspamd support: + - integration into scripts, provisioning of configuration & documentation ([#2902](https://github.com/docker-mailserver/docker-mailserver/pull/2902),[#3016](https://github.com/docker-mailserver/docker-mailserver/pull/3016),[#3039](https://github.com/docker-mailserver/docker-mailserver/pull/3039)) + - easily adjust options & modules ([#3059](https://github.com/docker-mailserver/docker-mailserver/pull/3059)) + - advanced documentation ([#3104](https://github.com/docker-mailserver/docker-mailserver/pull/3104)) + - make disabling Redis possible ([#3132](https://github.com/docker-mailserver/docker-mailserver/pull/3132)) + - persistence for Redis ([#3143](https://github.com/docker-mailserver/docker-mailserver/pull/3143)) + - integrate into `MOVE_SPAM_TO_JUNK` ([#3159](https://github.com/docker-mailserver/docker-mailserver/pull/3159)) + - make it possible to learn from user actions ([#3159](https://github.com/docker-mailserver/docker-mailserver/pull/3159)) +- heavily updated CI & tests: + - added functionality to send mail with a helper function ([#3026](https://github.com/docker-mailserver/docker-mailserver/pull/3026),[#3103](https://github.com/docker-mailserver/docker-mailserver/pull/3103),[#3105](https://github.com/docker-mailserver/docker-mailserver/pull/3105)) + - add a dedicated page for tests with more information ([#3019](https://github.com/docker-mailserver/docker-mailserver/pull/3019)) +- add information to Logwatch's mailer so `Envelope From` is properly set ([#3081](https://github.com/docker-mailserver/docker-mailserver/pull/3081)) +- add vulnerability scanning workflow & security policy ([#3106](https://github.com/docker-mailserver/docker-mailserver/pull/3106)) +- Add tools (ping & dig) to the image ([2989](https://github.com/docker-mailserver/docker-mailserver/pull/2989)) + +### Updates + +- Fail2Ban major version updated to v1.0.2 ([#2959](https://github.com/docker-mailserver/docker-mailserver/pull/2959)) +- heavily updated CI & tests: + - we now run more tests in parallel bringing down overall time to build and test AMD64 to 6 minutes ([#2938](https://github.com/docker-mailserver/docker-mailserver/pull/2938),[#3038](https://github.com/docker-mailserver/docker-mailserver/pull/3038),[#3018](https://github.com/docker-mailserver/docker-mailserver/pull/3018),[#3062](https://github.com/docker-mailserver/docker-mailserver/pull/3062)) + - remove CI ENV & disable fail-fast strategy ([#3065](https://github.com/docker-mailserver/docker-mailserver/pull/3065)) + - streamlined GH Actions runners ([#3025](https://github.com/docker-mailserver/docker-mailserver/pull/3025)) + - updated BATS & helper + minor updates to BATS variables ([#2988](https://github.com/docker-mailserver/docker-mailserver/pull/2988)) + - improved consistency and documentation for test helpers ([#3012](https://github.com/docker-mailserver/docker-mailserver/pull/3012)) +- improve the `clean` recipe (don't require `sudo` anymore) ([#3020](https://github.com/docker-mailserver/docker-mailserver/pull/3020)) +- improve Amavis setup routine ([#3079](https://github.com/docker-mailserver/docker-mailserver/pull/3079)) +- completely refactor README & parts of docs ([#3097](https://github.com/docker-mailserver/docker-mailserver/pull/3097)) +- TLS setup (self-signed) error message now includes `SS_CA_CERT` ([#3168](https://github.com/docker-mailserver/docker-mailserver/pull/3168)) +- Better default value for SA_KILL variable ([#3058](https://github.com/docker-mailserver/docker-mailserver/pull/3058)) + +### Fixed + +- `restrict-access` avoid inserting duplicates ([#3067](https://github.com/docker-mailserver/docker-mailserver/pull/3067)) +- correct the casing for Mime vs. MIME ([#3040](https://github.com/docker-mailserver/docker-mailserver/pull/3040)) +- Dovecot: + - Quota plugin is now properly configured via `mail_plugins` at setup ([#2958](https://github.com/docker-mailserver/docker-mailserver/pull/2958)) + - `quota-status` service (port 65265) now only binds to `127.0.0.1` ([#3057](https://github.com/docker-mailserver/docker-mailserver/pull/3057)) +- OpenDMARC - Change default policy to reject ([#2933](https://github.com/docker-mailserver/docker-mailserver/pull/2933)) +- Change Detection service - Use service `reload` instead of restarting process to minimize downtime ([#2947](https://github.com/docker-mailserver/docker-mailserver/pull/2947)) +- Slightly faster container startup via `postconf` workaround ([#2998](https://github.com/docker-mailserver/docker-mailserver/pull/2998)) +- Better group ownership to `/var/mail-state` + ClamAV in `Dockerfile` ([#3011](https://github.com/docker-mailserver/docker-mailserver/pull/3011)) +- Dropping Postfix `chroot` mode: + - Remove syslog socket created by Debian ([#3134](https://github.com/docker-mailserver/docker-mailserver/pull/3134)) + - Supervisor proxy signals for `postfix start-fg` via PID ([#3118](https://github.com/docker-mailserver/docker-mailserver/pull/3118)) +- Fixed several typos ([#2990](https://github.com/docker-mailserver/docker-mailserver/pull/2990)) ([#2993](https://github.com/docker-mailserver/docker-mailserver/pull/2993)) +- SRS setup fixed ([#3158](https://github.com/docker-mailserver/docker-mailserver/pull/3158)) +- Postsrsd restart loop fixed ([#3160](https://github.com/docker-mailserver/docker-mailserver/pull/3160)) +- Order of DKIM/DMARC milters matters ([#3082](https://github.com/docker-mailserver/docker-mailserver/pull/3082)) +- Make logrotate state persistant ([#3077](https://github.com/docker-mailserver/docker-mailserver/pull/3077)) + +### Changed + +- the Dovecot community repository is now the default ([#2901](https://github.com/docker-mailserver/docker-mailserver/pull/2901)) +- moved SASL authentication socket location ([#3131](https://github.com/docker-mailserver/docker-mailserver/pull/3131)) +- only add Amavis configuration to Postfix when enabled ([#3046](https://github.com/docker-mailserver/docker-mailserver/pull/3046)) +- improve bug report template ([#3080](https://github.com/docker-mailserver/docker-mailserver/pull/3080)) +- remove Postfix DNSBLs ([#3069](https://github.com/docker-mailserver/docker-mailserver/pull/3069)) +- bigger script updates: + - split `setup-stack.sh` ([#3115](https://github.com/docker-mailserver/docker-mailserver/pull/3115)) + - housekeeping & cleanup setup ([#3121](https://github.com/docker-mailserver/docker-mailserver/pull/3121),[#3123](https://github.com/docker-mailserver/docker-mailserver/pull/3123)) + - issue warning in case of improper restart ([#3129](https://github.com/docker-mailserver/docker-mailserver/pull/3129)) + - remove PostSRSD wrapper ([#3128](https://github.com/docker-mailserver/docker-mailserver/pull/3128)) + - miscellaneous small improvements ([#3144](https://github.com/docker-mailserver/docker-mailserver/pull/3144)) +- improve Postfix config for spoof protection ([#3127](https://github.com/docker-mailserver/docker-mailserver/pull/3127)) +- Change Detection service - Remove 10 sec start-up delay ([#3064](https://github.com/docker-mailserver/docker-mailserver/pull/3064)) +- Postfix: + - Stop using `chroot` + remove wrapper script ([#3033](https://github.com/docker-mailserver/docker-mailserver/pull/3033)) + - SMTP Authentication via port 25 disabled ([#3006](https://github.com/docker-mailserver/docker-mailserver/pull/3006)) +- Fail2Ban - Added support packages + remove wrapper script ([#3032](https://github.com/docker-mailserver/docker-mailserver/pull/3032)) +- Replace path with variable in mail_state.sh ([#3153](https://github.com/docker-mailserver/docker-mailserver/pull/3153)) + +### Removed + +- configomat (submodule) ([#3045](https://github.com/docker-mailserver/docker-mailserver/pull/3045)) +- Due to deprecation: + - ARMv7 image support ([#2943](https://github.com/docker-mailserver/docker-mailserver/pull/2943)) + - TLS 1.2 is now the minimum supported protocol ([#2945](https://github.com/docker-mailserver/docker-mailserver/pull/2945)) + - ENV `SASL_PASSWD` ([#2946](https://github.com/docker-mailserver/docker-mailserver/pull/2946)) +- Redundant: + - Makefile `backup` target ([#3000](https://github.com/docker-mailserver/docker-mailserver/pull/3000)) + - ENV `ENABLE_POSTFIX_VIRTUAL_TRANSPORT` ([#3004](https://github.com/docker-mailserver/docker-mailserver/pull/3004)) + - `gamin` package ([#3030](https://github.com/docker-mailserver/docker-mailserver/pull/3030)) + ## [11.3.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v11.3.1) ### Fixed diff --git a/VERSION b/VERSION index 0a47c95bb14..4044f90867d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -11.3.1 +12.0.0 From 1e20e7c332381beedf8b6fa646f1e328248b32b4 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 10 Apr 2023 11:37:25 +0200 Subject: [PATCH 025/592] Image registry and setup update (#3233) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docker-compose.yml | 2 +- docs/content/config/setup.sh.md | 121 ++---------------- .../examples/tutorials/basic-installation.md | 16 +-- docs/content/usage.md | 2 +- docs/mkdocs.yml | 2 +- setup.sh | 2 +- target/bin/setup | 83 ++++++------ .../parallel/set3/scripts/setup_cli.bats | 2 +- 8 files changed, 62 insertions(+), 168 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8834d37ea02..37c42c7c546 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: mailserver: - image: docker.io/mailserver/docker-mailserver:latest + image: ghcr.io/docker-mailserver/docker-mailserver:latest container_name: mailserver # Provide the FQDN of your mail server here (Your DNS MX record should point to this value) hostname: mail.example.com diff --git a/docs/content/config/setup.sh.md b/docs/content/config/setup.sh.md index b55b7d9d09d..d0829bc8078 100644 --- a/docs/content/config/setup.sh.md +++ b/docs/content/config/setup.sh.md @@ -1,123 +1,28 @@ --- -title: Your best friend setup.sh +title: About setup.sh hide: - toc --- -[`setup.sh`][github-file-setupsh] is an administration script that helps with the most common tasks, including initial configuration. It is intended to be run from the host machine, _not_ from inside your running container. +!!! note -The latest version of the script is included in the `docker-mailserver` repository. You may retrieve it at any time by running this command in your console: + `setup.sh` is not required. We encourage you to use `docker exec -ti setup` instead. -```sh -wget https://raw.githubusercontent.com/docker-mailserver/docker-mailserver/master/setup.sh -chmod a+x ./setup.sh -``` - -## Usage - -Run `./setup.sh help` and you'll get ~~all you have ever wanted~~ some usage information: - -```TXT -SETUP(1) - -NAME - setup.sh - docker-mailserver administration script - -SYNOPSIS - ./setup.sh [ OPTIONS... ] COMMAND [ help | ARGUMENTS... ] - - COMMAND := { email | alias | quota | config | relay | debug } SUBCOMMAND - -DESCRIPTION - This is the main administration script that you use for all your interactions with - 'docker-mailserver'. Setup, configuration and much more is done with this script. - - Please note that the script executes most of the commands inside the container itself. - If the image was not found, this script will pull the ':latest' tag of - 'docker.io/mailserver/docker-mailserver'. This tag refers to the latest release, - see the tagging convention in the README under - https://github.com/docker-mailserver/docker-mailserver/blob/master/README.md - - You will be able to see detailed information about the script you're invoking and - its arguments by appending help after your command. Currently, this - does not work with all scripts. - -[SUB]COMMANDS - COMMAND email := - ./setup.sh email add [] - ./setup.sh email update [] - ./setup.sh email del [ OPTIONS... ] [ ... ] - ./setup.sh email restrict [] - ./setup.sh email list - - COMMAND alias := - ./setup.sh alias add - ./setup.sh alias del - ./setup.sh alias list +!!! warning - COMMAND quota := - ./setup.sh quota set [] - ./setup.sh quota del + This script assumes Docker or Podman is used. You will not be able to use `setup.sh` with other container orchestration tools. - COMMAND config := - ./setup.sh config dkim [ ARGUMENTS... ] +[`setup.sh`][github-file-setupsh] is a script that is complimentary to the internal `setup` command in `docker-mailserver`. - COMMAND relay := - ./setup.sh relay add-auth [] - ./setup.sh relay add-domain [] - ./setup.sh relay exclude-domain +It mostly provides the convenience of aliasing `docker exec -ti setup`, inferring the container name of a running `docker-mailserver` instance or running a new instance and bind mounting necessary volumes implicitly. - COMMAND fail2ban = - ./setup.sh fail2ban - ./setup.sh fail2ban ban - ./setup.sh fail2ban unban - - COMMAND debug := - ./setup.sh debug fetchmail - ./setup.sh debug login - ./setup.sh debug show-mail-logs - -EXAMPLES - ./setup.sh email add test@example.com [password] - Add the email account test@example.com. You will be prompted - to input a password afterwards if no password was supplied. - When supplying `[password]`, it should be in plaintext. - - ./setup.sh config dkim keysize 2048 domain 'example.com,not-example.com' - Creates keys of length 2048 but in an LDAP setup where domains are not known to - Postfix by default, so you need to provide them yourself in a comma-separated list. - - ./setup.sh config dkim help - This will provide you with a detailed explanation on how to use the - config dkim command, showing what arguments can be passed and what they do. - -OPTIONS - Config path, container or image adjustments - -i IMAGE_NAME - Provides the name of the 'docker-mailserver' image. The default value is - 'docker.io/mailserver/docker-mailserver:latest' - - -c CONTAINER_NAME - Provides the name of the running container. - - -p PATH - Provides the config folder path to the temporary container - (does not work if a 'docker-mailserver' container already exists). - - SELinux - -z - Allows container access to the bind mount content that is shared among - multiple containers on a SELinux-enabled host. - - -Z - Allows container access to the bind mount content that is private and - unshared with other containers on a SELinux-enabled host. - -EXIT STATUS - Exit status is 0 if the command was successful. If there was an unexpected error, an error - message is shown describing the error. In case of an error, the script will exit with exit - status 1. +It is intended to be run from the host machine, _not_ from inside your running container. The latest version of the script is included in the `docker-mailserver` repository. You may retrieve it at any time by running this command in your console: +```sh +wget https://raw.githubusercontent.com/docker-mailserver/docker-mailserver/master/setup.sh +chmod a+x ./setup.sh ``` +For more information on using the script run: `./setup.sh help`. + [github-file-setupsh]: https://github.com/docker-mailserver/docker-mailserver/blob/master/setup.sh diff --git a/docs/content/examples/tutorials/basic-installation.md b/docs/content/examples/tutorials/basic-installation.md index bdb7cfb3db0..31c6496148c 100644 --- a/docs/content/examples/tutorials/basic-installation.md +++ b/docs/content/examples/tutorials/basic-installation.md @@ -103,7 +103,7 @@ In this setup `docker-mailserver` is not intended to receive email from the outs ```yaml services: mailserver: - image: docker.io/mailserver/docker-mailserver:latest + image: ghcr.io/docker-mailserver/docker-mailserver:latest container_name: mailserver # Provide the FQDN of your mail server here (Your DNS MX record should point to this value) hostname: mail.example.com @@ -137,7 +137,7 @@ In this setup `docker-mailserver` is not intended to receive email from the outs ??? tip "Firewalled ports" If you have a firewall running, you may need to open ports `25`, `587` and `465`. - + For example, with the firewall `ufw`, run: ```sh @@ -145,27 +145,27 @@ In this setup `docker-mailserver` is not intended to receive email from the outs ufw allow 587 ufw allow 465 ``` - + **Caution:** This may [not be sound advice][github-issue-ufw]. 2. Configure your DNS service to use an MX record for the _hostname_ (eg: `mail`) you configured in the previous step and add the [SPF][docs-spf] TXT record. !!! tip "If you manually manage the DNS zone file for the domain" - + It would look something like this: - + ```txt $ORIGIN example.com @ IN A 10.11.12.13 mail IN A 10.11.12.13 - + ; mail-server for example.com @ IN MX 10 mail.example.com. - + ; Add SPF record @ IN TXT "v=spf1 mx -all" ``` - + Then don't forget to change the `SOA` serial number, and to restart the service. 3. [Generate DKIM keys][docs-dkim] for your domain via `setup config dkim`. diff --git a/docs/content/usage.md b/docs/content/usage.md index bc34f665902..5b4f84a3b09 100644 --- a/docs/content/usage.md +++ b/docs/content/usage.md @@ -123,7 +123,7 @@ For an overview of commands to manage DMS config, run: `docker exec -it [] - ${0} email ${CYAN}update${RESET} [] - ${0} email ${CYAN}del${RESET} [ OPTIONS${RED}...${RESET} ] [ ${RED}...${RESET} ] - ${0} email ${CYAN}restrict${RESET} [] - ${0} email ${CYAN}list${RESET} + setup email ${CYAN}add${RESET} [] + setup email ${CYAN}update${RESET} [] + setup email ${CYAN}del${RESET} [ OPTIONS${RED}...${RESET} ] [ ${RED}...${RESET} ] + setup email ${CYAN}restrict${RESET} [] + setup email ${CYAN}list${RESET} ${LBLUE}COMMAND${RESET} alias ${RED}:=${RESET} - ${0} alias ${CYAN}add${RESET} - ${0} alias ${CYAN}del${RESET} - ${0} alias ${CYAN}list${RESET} + setup alias ${CYAN}add${RESET} + setup alias ${CYAN}del${RESET} + setup alias ${CYAN}list${RESET} ${LBLUE}COMMAND${RESET} quota ${RED}:=${RESET} - ${0} quota ${CYAN}set${RESET} [] - ${0} quota ${CYAN}del${RESET} + setup quota ${CYAN}set${RESET} [] + setup quota ${CYAN}del${RESET} ${LBLUE}COMMAND${RESET} dovecot-master ${RED}:=${RESET} - ${0} dovecot-master ${CYAN}add${RESET} [] - ${0} dovecot-master ${CYAN}update${RESET} [] - ${0} dovecot-master ${CYAN}del${RESET} [ OPTIONS${RED}...${RESET} ] [ ${RED}...${RESET} ] - ${0} dovecot-master ${CYAN}list${RESET} + setup dovecot-master ${CYAN}add${RESET} [] + setup dovecot-master ${CYAN}update${RESET} [] + setup dovecot-master ${CYAN}del${RESET} [ OPTIONS${RED}...${RESET} ] [ ${RED}...${RESET} ] + setup dovecot-master ${CYAN}list${RESET} ${LBLUE}COMMAND${RESET} config ${RED}:=${RESET} - ${0} config ${CYAN}dkim${RESET} [ ARGUMENTS${RED}...${RESET} ] + setup config ${CYAN}dkim${RESET} [ ARGUMENTS${RED}...${RESET} ] ${LBLUE}COMMAND${RESET} relay ${RED}:=${RESET} - ${0} relay ${CYAN}add-auth${RESET} [] - ${0} relay ${CYAN}add-domain${RESET} [] - ${0} relay ${CYAN}exclude-domain${RESET} + setup relay ${CYAN}add-auth${RESET} [] + setup relay ${CYAN}add-domain${RESET} [] + setup relay ${CYAN}exclude-domain${RESET} ${LBLUE}COMMAND${RESET} fail2ban ${RED}:=${RESET} - ${0} fail2ban ${RESET} - ${0} fail2ban ${CYAN}ban${RESET} - ${0} fail2ban ${CYAN}unban${RESET} + setup fail2ban ${RESET} + setup fail2ban ${CYAN}ban${RESET} + setup fail2ban ${CYAN}unban${RESET} ${LBLUE}COMMAND${RESET} debug ${RED}:=${RESET} - ${0} debug ${CYAN}fetchmail${RESET} - ${0} debug ${CYAN}login${RESET} - ${0} debug ${CYAN}show-mail-logs${RESET} + setup debug ${CYAN}fetchmail${RESET} + setup debug ${CYAN}login${RESET} + setup debug ${CYAN}show-mail-logs${RESET} ${ORANGE}EXAMPLES${RESET} - ${LWHITE}./setup.sh email add test@example.com${RESET} + ${LWHITE}setup email add test@example.com${RESET} Add the email account ${LWHITE}test@example.com${RESET}. You will be prompted to input a password afterwards since no password was supplied. - ${LWHITE}./setup.sh config dkim keysize 2048 domain 'example.com,not-example.com'${RESET} - Creates keys of length 2048 but in an LDAP setup where domains are not known to - Postfix by default, so you need to provide them yourself in a comma-separated list. + ${LWHITE}setup config dkim keysize 2048 domain 'example.com,not-example.com'${RESET} + Creates keys of length 2048 for the domains in comma-seperated list. + This is necessary when using LDAP as the required domains cannot be inferred. - ${LWHITE}./setup.sh config dkim help${RESET} + ${LWHITE}setup config dkim help${RESET} This will provide you with a detailed explanation on how to use the ${LWHITE} config dkim${RESET} command, showing what arguments can be passed and what they do. " } - function _invalid_command { echo "The command '${*}' is invalid. -Use \`./setup.sh help\` to get an overview of all commands." >&2 +Use \`setup help\` to get an overview of all commands." >&2 exit 2 } diff --git a/test/tests/parallel/set3/scripts/setup_cli.bats b/test/tests/parallel/set3/scripts/setup_cli.bats index 737bf53d8ea..c5d2a2d62b7 100644 --- a/test/tests/parallel/set3/scripts/setup_cli.bats +++ b/test/tests/parallel/set3/scripts/setup_cli.bats @@ -28,7 +28,7 @@ function teardown_file() { _default_teardown ; } @test "show usage when no arguments provided" { run ./setup.sh assert_success - assert_output --partial "This is the main administration script that you use for all your interactions with" + assert_output --partial "This is the main administration command that you use for all your interactions with" } @test "exit with error when wrong arguments provided" { From ff087837bdedf009f46ac1ea683335224e104c0c Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 10 Apr 2023 11:54:52 +0200 Subject: [PATCH 026/592] fix: GH docs update workflow (#3241) --- .github/workflows/docs-production-deploy.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docs-production-deploy.yml b/.github/workflows/docs-production-deploy.yml index 73d152728bf..8c8faa23c3f 100644 --- a/.github/workflows/docs-production-deploy.yml +++ b/.github/workflows/docs-production-deploy.yml @@ -123,12 +123,11 @@ jobs: id: update-latest run: | DOCS_VERSION=$(grep -oE 'v[0-9]+\.[0-9]+' <<< "${GITHUB_REF}") - echo "DOCS_VERSION=${DOCS_VERSION}" >> "${GITHUB_ENV}" - ln -sf ${DOCS_VERSION} latest + echo "DOCS_VERSION=${DOCS_VERSION}" >>"${GITHUB_ENV}" + rm latest + ln -s "${DOCS_VERSION}" latest - name: 'Push update for `latest` symlink' - # The step will fail if no change was made; ignore that failure - continue-on-error: true run: | git config user.name ${{ env.GIT_USER }} git config user.email ${{ env.GIT_EMAIL }} From 34a1fd613f228afa624f6f2f1aa82c97bcd91c95 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 10 Apr 2023 12:08:58 +0200 Subject: [PATCH 027/592] docs: Combine DKIM/DMARC/SPF pages (#3231) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .../config/advanced/optional-config.md | 4 +- .../config/best-practices/autodiscover.md | 2 + docs/content/config/best-practices/dkim.md | 123 ------- .../config/best-practices/dkim_dmarc_spf.md | 346 ++++++++++++++++++ docs/content/config/best-practices/dmarc.md | 36 -- docs/content/config/best-practices/spf.md | 59 --- docs/content/config/security/rspamd.md | 80 +--- .../examples/tutorials/basic-installation.md | 4 +- docs/content/usage.md | 38 +- docs/mkdocs.yml | 6 +- 10 files changed, 363 insertions(+), 335 deletions(-) delete mode 100644 docs/content/config/best-practices/dkim.md create mode 100644 docs/content/config/best-practices/dkim_dmarc_spf.md delete mode 100644 docs/content/config/best-practices/dmarc.md delete mode 100644 docs/content/config/best-practices/spf.md diff --git a/docs/content/config/advanced/optional-config.md b/docs/content/config/advanced/optional-config.md index 259e05a20cb..e9835501ab6 100644 --- a/docs/content/config/advanced/optional-config.md +++ b/docs/content/config/advanced/optional-config.md @@ -4,7 +4,7 @@ hide: - toc # Hide Table of Contents for this page --- -This is a list of all configuration files and directories which are optional or automatically generated in your `docker-data/dms/config/` directory. +This is a list of all configuration files and directories which are optional or automatically generated in your `docker-data/dms/config/` directory. We use this path to reference the local config directory in our docs, which you should attach a volume into the container at `/tmp/docker-mailserver`. ## Directories @@ -43,7 +43,7 @@ This is a list of all configuration files and directories which are optional or [docs-accounts-quota]: ../../config/user-management.md#quotas [docs-aliases-regex]: ../../config/user-management.md#configuring-regexp-aliases -[docs-dkim]: ../../config/best-practices/dkim.md +[docs-dkim]: ../../config/best-practices/dkim_dmarc_spf.md#dkim [docs-fail2ban]: ../../config/security/fail2ban.md [docs-faq-spamrules]: ../../faq.md#how-can-i-manage-my-custom-spamassassin-rules [docs-faq-userpatches]: ../../faq.md#how-to-adjust-settings-with-the-user-patchessh-script diff --git a/docs/content/config/best-practices/autodiscover.md b/docs/content/config/best-practices/autodiscover.md index e082ab774bf..4e3ba8e68d5 100644 --- a/docs/content/config/best-practices/autodiscover.md +++ b/docs/content/config/best-practices/autodiscover.md @@ -4,6 +4,8 @@ hide: - toc # Hide Table of Contents for this page --- +# Auto-Discovery of Services + Email auto-discovery means a client email is able to automagically find out about what ports and security options to use, based on the mail-server URI. It can help simplify the tedious / confusing task of adding own's email account for non-tech savvy users. Email clients will search for auto-discoverable settings and prefill almost everything when a user enters its email address :heart: diff --git a/docs/content/config/best-practices/dkim.md b/docs/content/config/best-practices/dkim.md deleted file mode 100644 index bbf5fff93d8..00000000000 --- a/docs/content/config/best-practices/dkim.md +++ /dev/null @@ -1,123 +0,0 @@ ---- -title: 'Best Practices | DKIM' ---- - -DKIM is a security measure targeting email spoofing. It is greatly recommended one activates it. - -!!! note - See [the Wikipedia page](https://en.wikipedia.org/wiki/DomainKeys_Identified_Mail) for more details on DKIM. - -## Enabling DKIM Signature - -To enable DKIM signature, **you must have created at least one email account**. Once its done, just run the following command to generate the signature: - -```sh -./setup.sh config dkim -``` - -After generating DKIM keys, you should restart `docker-mailserver`. DNS edits may take a few minutes to hours to propagate. - -The script should ideally be run with a volume for _config_ attached (eg: `./docker-data/dms/config/:/tmp/docker-mailserver/`), otherwise by default it will mount `./config/:/tmp/docker-mailserver/`. - -The default keysize when generating the signature is 4096 bits for now. If you need to change it (e.g. your DNS provider limits the size), then provide the size as the first parameter of the command: - -```sh -./setup.sh config dkim keysize -``` - -For LDAP systems that do not have any directly created user account you can run the following command (since `8.0.0`) to generate the signature by additionally providing the desired domain name (if you have multiple domains use the command multiple times or provide a comma-separated list of domains): - -```sh -./setup.sh config dkim keysize domain [,] -``` - -Now the keys are generated, you can configure your DNS server with DKIM signature, simply by adding a TXT record. If you have direct access to your DNS zone file, then it's only a matter of pasting the content of `docker-data/dms/config/opendkim/keys/example.com/mail.txt` in your `example.com.hosts` zone. - -```console -$ dig mail._domainkey.example.com TXT ---- -;; ANSWER SECTION -mail._domainkey. 300 IN TXT "v=DKIM1; k=rsa; p=AZERTYUIOPQSDFGHJKLMWXCVBN/AZERTYUIOPQSDFGHJKLMWXCVBN/AZERTYUIOPQSDFGHJKLMWXCVBN/AZERTYUIOPQSDFGHJKLMWXCVBN/AZERTYUIOPQSDFGHJKLMWXCVBN/AZERTYUIOPQSDFGHJKLMWXCVBN/AZERTYUIOPQSDFGHJKLMWXCVBN/AZERTYUIOPQSDFGHJKLMWXCVBN" -``` - -## Configuration using a Web Interface - -1. Generate a new record of the type `TXT`. -2. Paste `mail._domainkey` the `Name` txt field. -3. In the `Target` or `Value` field fill in `v=DKIM1; k=rsa; p=AZERTYUGHJKLMWX...`. -4. In `TTL` (time to live): Time span in seconds. How long the DNS server should cache the `TXT` record. -5. Save. - -!!! note - Sometimes the key in `docker-data/dms/config/opendkim/keys/example.com/mail.txt` can be on multiple lines. If so then you need to concatenate the values in the TXT record: - -```console -$ dig mail._domainkey.example.com TXT ---- -;; ANSWER SECTION -mail._domainkey. 300 IN TXT "v=DKIM1; k=rsa; " - "p=AZERTYUIOPQSDF..." - "asdfQWERTYUIOPQSDF..." -``` - -The target (or value) field must then have all the parts together: `v=DKIM1; k=rsa; p=AZERTYUIOPQSDF...asdfQWERTYUIOPQSDF...` - -## Verify-Only - -If you want DKIM to only _verify_ incoming emails, the following version of `/etc/opendkim.conf` may be useful (right now there is no easy mechanism for installing it other than forking the repo): - -```conf -# This is a simple config file verifying messages only - -#LogWhy yes -Syslog yes -SyslogSuccess yes - -Socket inet:12301@localhost -PidFile /var/run/opendkim/opendkim.pid - -ReportAddress postmaster@example.com -SendReports yes - -Mode v -``` - -## Switch Off DKIM - -Simply remove the DKIM key by recreating (not just relaunching) the `docker-mailserver` container. - -## Debugging - -- [DKIM-verifer](https://addons.mozilla.org/en-US/thunderbird/addon/dkim-verifier): A add-on for the mail client Thunderbird. -- You can debug your TXT records with the `dig` tool. - -```console -$ dig TXT mail._domainkey.example.com ---- -; <<>> DiG 9.10.3-P4-Debian <<>> TXT mail._domainkey.example.com -;; global options: +cmd -;; Got answer: -;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 39669 -;; flags: qr rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 - -;; OPT PSEUDOSECTION: -; EDNS: version: 0, flags:; udp: 512 -;; QUESTION SECTION: -;mail._domainkey.example.com. IN TXT - -;; ANSWER SECTION: -mail._domainkey.example.com. 3600 IN TXT "v=DKIM1; k=rsa; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCxBSjG6RnWAdU3oOlqsdf2WC0FOUmU8uHVrzxPLW2R3yRBPGLrGO1++yy3tv6kMieWZwEBHVOdefM6uQOQsZ4brahu9lhG8sFLPX4MaKYN/NR6RK4gdjrZu+MYSdfk3THgSbNwIDAQAB" - -;; Query time: 50 msec -;; SERVER: 127.0.1.1#53(127.0.1.1) -;; WHEN: Wed Sep 07 18:22:57 CEST 2016 -;; MSG SIZE rcvd: 310 -``` - ---- - -!!! warning "Key sizes >=4096-bit" - - Keys of 4096 bits could de denied by some mail-servers. According to https://tools.ietf.org/html/rfc6376 keys are preferably between 512 and 2048 bits. See issue [#1854][github-issue-1854]. - -[github-issue-1854]: https://github.com/docker-mailserver/docker-mailserver/issues/1854 diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md new file mode 100644 index 00000000000..5638476230c --- /dev/null +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -0,0 +1,346 @@ +# DKIM, DMARC & SPF + +Cloudflare has written an [article about DKIM, DMARC and SPF][cloudflare-dkim-dmarc-spf] that we highly recommend you to read to get acquainted with the topic. + +!!! note "Rspamd vs Individual validators" + + With v12.0.0, Rspamd was integrated into DMS. It can perform validations for DKIM, DMARC and SPF as part of the `spam-score-calculation` for an email. DMS provides individual alternatives for each validation that can be used instead of deferring to Rspamd: + + - DKIM: `opendkim` is used as a milter (like Rspamd) + - DMARC: `opendmarc` is used as a milter (like Rspamd) + - SPF: `policyd-spf` is used in Postfix's `smtpd_recipient_restrictions` + + In a future release Rspamd will become the default for these validations, with a deprecation notice issued prior to the removal of the above alternatives. + + We encourage everyone to prefer Rspamd via `ENABLE_RSPAMD=1`. + +!!! warning "DNS Caches & Propagation" + + While modern DNS providers are quick, it may take minutes or even hours for new DNS records to become available / propagate. + +## DKIM + +!!! quote "What is DKIM" + + DomainKeys Identified Mail (DKIM) is an email authentication method designed to detect forged sender addresses in email (email spoofing), a technique often used in phishing and email spam. + + [Source][wikipedia-dkim] + +When DKIM is enabled: + +1. Inbound mail will verify any included DKIM signatures +2. Outbound mail is signed (_when you're sending domain has a configured DKIM key_) + +DKIM requires a public/private key pair to enable **signing (_via private key_)** your outgoing mail, while the receiving end must query DNS to **verify (_via public key_)** that the signature is trustworthy. + +### Generating Keys + +You should have: + +- At least one [email account setup][docs-accounts-add] +- Attached a [volume for config][docs-volumes-config] to persist the generated files to local storage + +DKIM is currently supported by either OpenDKIM or Rspamd: + +=== "OpenDKIM" + + OpenDKIM is currently [enabled by default][docs-env-opendkim]. + + The command `docker exec setup config dkim help` details supported config options, along with some examples. + + !!! example "Create a DKIM key" + + Generate the DKIM files with: + + ```sh + docker exec -ti setup config dkim + ``` + + Your new DKIM key(s) and OpenDKIM config files have been added to `/tmp/docker-mailserver/opendkim/`. + + ??? note "LDAP accounts need to specify domains explicitly" + + The command is unable to infer the domains from LDAP user accounts, you must specify them: + + ```sh + setup config dkim domain 'example.com,example.io' + ``` + + ??? tip "Changing the key size" + + The private key presently defaults to RSA-4096. To create an RSA 2048-bit key run: + + ```sh + setup config dkim keysize 2048 + ``` + +=== "Rspamd" + + Opt-in via [`ENABLE_RSPAMD=1`][docs-env-rspamd] (_and disable the default OpenDKIM: `ENABLE_OPENDKIM=0`_). + + Rspamd provides DKIM support through two separate modules: + + 1. [Verifying DKIM signatures from inbound mail][rspamd-docs-dkim-checks] is enabled by default. + 2. [Signing outbound mail with your DKIM key][rspamd-docs-dkim-signing] needs additional setup (key + dns + config). + + !!! example "Create a DKIM key" + + Presently only OpenDKIM is supported with `setup config dkim`. To generate your DKIM key and DNS files you'll need to specify: + + - `-s` The DKIM selector (_eg: `mail`, it can be anything you like_) + - `-d` The sender address domain (_everything after `@` from the email address_) + + See `rspamadm dkim_keygen -h` for an overview of the supported options. + + --- + + 1. Go inside the container with `docker exec -ti bash` + 2. Add `rspamd/dkim/` folder to your config volume and switch to it: `cd /tmp/docker-mailserver/rspamd/dkim` + 3. Run: `rspamadm dkim_keygen -s mail -b 2048 -d example.com -k mail.private > mail.txt` (_change `-d` to your domain-part_) + 4. Presently you must ensure Rspamd can read the `.private` file, run: + -`chgrp _rspamd mail.private` + -`chmod g+r mail.private` + + --- + + !!! bug inline end "DMS config volume support is not ready for Rspamd" + + Presently you'll need to [explicitly mount `rspamd/modules/override.d/`][docs-rspamd-config-dropin] as an additional volume; do not use [`rspamd-modules.conf`][docs-rspamd-config-declarative] for this purpose. + + Create a configuration file for the DKIM signing module at `rspamd/modules/override.d/dkim_signing.conf` and populate it with config as shown in the example below: + + ??? example "DKIM Signing Module Configuration Examples" + + A simple configuration could look like this: + + ```cf + # documentation: https://rspamd.com/doc/modules/dkim_signing.html + + enabled = true; + + sign_authenticated = true; + sign_local = true; + + use_domain = "header"; + use_redis = false; # don't change unless Redis also provides the DKIM keys + use_esld = true; + check_pubkey = true; # you wan't to use this in the beginning + + domain { + example.com { + path = "/tmp/docker-mailserver/rspamd/dkim/mail.private"; + selector = "mail"; + } + } + ``` + + As shown next, you can: + + - You can add more domains into the `domain { ... }` section. + - A domain can also be configured with multiple selectors and keys within a `selectors [ ... ]` array. + + ```cf + # ... + + domain { + example.com { + selectors [ + { + path = "/tmp/docker-mailserver/rspamd/dkim/example.com/rsa.private"; + selector = "dkim-rsa"; + }, + { + path = /tmp/docker-mailserver/rspamd/example.com/ed25519.private"; + selector = "dkim-ed25519"; + } + ] + } + example.org { + selectors [ + { + path = "/tmp/docker-mailserver/rspamd/dkim/example.org/rsa.private"; + selector = "dkim-rsa"; + }, + { + path = "/tmp/docker-mailserver/rspamd/dkim/example.org/ed25519.private"; + selector = "dkim-ed25519"; + } + ] + } + } + ``` + + !!! warning "Support for DKIM keys using Ed25519" + + This modern elliptic curve is supported by Rspamd, but support by third-parties for [verifying Ed25519 DKIM signatures is unreliable][dkim-ed25519-support]. + + If you sign your mail with this key type, you should include RSA as a fallback, like shown in the above example. + + !!! tip "DKIM Signing config: `check_pubkey = true;`" + + This setting will have Rspamd query the DNS record for each DKIM selector, verifying each public key matches the private key configured. + + If there is a mismatch, a warning will be omitted to the Rspamd log (`/var/log/supervisor/rspamd.log`). + +!!! info "Restart required" + + After restarting `docker-mailserver`, outgoing mail will now be signed with your new DKIM key(s) :tada: + + You'll need to repeat this process if you add any new domains. + +!!! warning "RSA Key Sizes >= 4096 Bit" + + Keys of 4096 bits could denied by some mail servers. According to [RFC 6376][rfc-6376] keys are [preferably between 512 and 2048 bits][github-issue-dkimlength]. + +### DNS Record { #dkim-dns } + +When mail signed with your DKIM key is sent from your mail server, the receiver needs to check a DNS `TXT` record to verify the DKIM signature is trustworthy. + +!!! example "Configuring DNS - DKIM record" + + When you generated your key in the previous step, the DNS data was saved into a file `.txt` (default: `mail.txt`). Use this content to update your [DNS via Web Interface][dns::example-webui] or directly edit your [DNS Zone file][dns::wikipedia-zonefile]: + + === "Web Interface" + + Create a new record: + + | Field | Value | + | ----- | ------------------------------------------------------------------------------ | + | Type | `TXT` | + | Name | `._domainkey` (_default: `mail._domainkey`_) | + | TTL | Use the default (_otherwise [3600 seconds is appropriate][dns::digicert-ttl]_) | + | Data | File content within `( ... )` (_formatted as advised below_) | + + === "DNS Zone file" + + `.txt` is already formatted as a snippet for adding to your [DNS Zone file][dns::wikipedia-zonefile]. + + Just copy/paste the file contents into your existing DNS zone. The `TXT` value has been split into separate strings every 255 characters for compatibility. + +??? info "`.txt` - Formatting the `TXT` record value correctly" + + This file was generated for use within a [DNS zone file][dns::wikipedia-zonefile]. DNS `TXT` records values that are longer than 255 characters need to be split into multiple parts. This is why the public key has multiple parts wrapped within double-quotes between `(` and `)`. + + A DNS web-interface may handle this internally instead, while [others may not, but expect the input as a single line][dns::webui-dkim]_). You'll need to manually format the value as described below. + + Your DNS record file (eg: `mail.txt`) should look similar to this: + + ```txt + mail._domainkey IN TXT ( "v=DKIM1; k=rsa; " + "p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqQMMqhb1S52Rg7VFS3EC6JQIMxNDdiBmOKZvY5fiVtD3Z+yd9ZV+V8e4IARVoMXWcJWSR6xkloitzfrRtJRwOYvmrcgugOalkmM0V4Gy/2aXeamuiBuUc4esDQEI3egmtAsHcVY1XCoYfs+9VqoHEq3vdr3UQ8zP/l+FP5UfcaJFCK/ZllqcO2P1GjIDVSHLdPpRHbMP/tU1a9mNZ" + "5QMZBJ/JuJK/s+2bp8gpxKn8rh1akSQjlynlV9NI+7J3CC7CUf3bGvoXIrb37C/lpJehS39KNtcGdaRufKauSfqx/7SxA0zyZC+r13f7ASbMaQFzm+/RRusTqozY/p/MsWx8QIDAQAB" + ) ; + ``` + + Take the content between `( ... )`, and combine all the quote wrapped content and remove the double-quotes including the white-space between them. That is your `TXT` record value, the above example would become this: + + ```txt + v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqQMMqhb1S52Rg7VFS3EC6JQIMxNDdiBmOKZvY5fiVtD3Z+yd9ZV+V8e4IARVoMXWcJWSR6xkloitzfrRtJRwOYvmrcgugOalkmM0V4Gy/2aXeamuiBuUc4esDQEI3egmtAsHcVY1XCoYfs+9VqoHEq3vdr3UQ8zP/l+FP5UfcaJFCK/ZllqcO2P1GjIDVSHLdPpRHbMP/tU1a9mNZ5QMZBJ/JuJK/s+2bp8gpxKn8rh1akSQjlynlV9NI+7J3CC7CUf3bGvoXIrb37C/lpJehS39KNtcGdaRufKauSfqx/7SxA0zyZC+r13f7ASbMaQFzm+/RRusTqozY/p/MsWx8QIDAQAB + ``` + + To test that your new DKIM record is correct, query it with the `dig` command. The `TXT` value response should be a single line split into multiple parts wrapped in double-quotes: + + ```console + $ dig +short TXT dkim-rsa._domainkey.example.com + "v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqQMMqhb1S52Rg7VFS3EC6JQIMxNDdiBmOKZvY5fiVtD3Z+yd9ZV+V8e4IARVoMXWcJWSR6xkloitzfrRtJRwOYvmrcgugOalkmM0V4Gy/2aXeamuiBuUc4esDQEI3egmtAsHcVY1XCoYfs+9VqoHEq3vdr3UQ8zP/l+FP5UfcaJFCK/ZllqcO2P1GjIDVSHLdPpRHbMP/tU1a9mNZ5QMZBJ/JuJK/s+2bp8gpxKn8rh1akSQjlynlV9NI+7J3CC7CUf3bGvoXIrb37C/lpJehS39" "KNtcGdaRufKauSfqx/7SxA0zyZC+r13f7ASbMaQFzm+/RRusTqozY/p/MsWx8QIDAQAB" + ``` + +### Troubleshooting { #dkim-debug } + +[MxToolbox has a DKIM Verifier][mxtoolbox-dkim-verifier] that you can use to check your DKIM DNS record(s). + +When using Rspamd, we recommend you turn on `check_pubkey = true;` in `dkim_signing.conf`. Rspamd will then check whether your private key matches your public key, and you can check possible mismatches by looking at `/var/log/supervisor/rspamd.log`. + +## DMARC + +With DMS, DMARC is pre-configured out of the box. You may disable extra and excessive DMARC checks when using Rspamd via `ENABLE_OPENDMARC=0`. + +The only thing you need to do in order to enable DMARC on a "DNS-level" is to add new `TXT`. In contrast to [DKIM](#dkim), DMARC DNS entries do not require any keys, but merely setting the [configuration values][dmarc-howto-configtags]. You can either handcraft the entry by yourself or use one of available generators (like [this one][dmarc-tool-gca]). + +Typically something like this should be good to start with: + +```txt +_dmarc.example.com. IN TXT "v=DMARC1; p=none; sp=none; fo=0; adkim=4; aspf=r; pct=100; rf=afrf; ri=86400; rua=mailto:dmarc.report@example.com; ruf=mailto:dmarc.report@example.com" +``` + +Or a bit more strict policies (_mind `p=quarantine` and `sp=quarantine`_): + +```txt +_dmarc.example.com. IN TXT "v=DMARC1; p=quarantine; sp=quarantine; fo=0; adkim=r; aspf=r; pct=100; rf=afrf; ri=86400; rua=mailto:dmarc.report@example.com; ruf=mailto:dmarc.report@example.com" +``` + +The DMARC status may not be displayed instantly due to delays in DNS (caches). Dmarcian has [a few tools][dmarcian-tools] you can use to verify your DNS records. + +## SPF + +!!! quote "What is SPF" + + Sender Policy Framework (SPF) is a simple email-validation system designed to detect email spoofing by providing a mechanism to allow receiving mail exchangers to check that incoming mail from a domain comes from a host authorized by that domain's administrators. + + [Source][wikipedia-spf] + +!!! note "Disabling `policyd-spf`?" + + As of now, `policyd-spf` cannot be disabled. This is WIP. + +### Adding an SPF Record + +To add a SPF record in your DNS, insert the following line in your DNS zone: + +```txt +example.com. IN TXT "v=spf1 mx ~all" +``` + +This enables the _Softfail_ mode for SPF. You could first add this SPF record with a very low TTL. _SoftFail_ is a good setting for getting started and testing, as it lets all email through, with spams tagged as such in the mailbox. + +After verification, you _might_ want to change your SPF record to `v=spf1 mx -all` so as to enforce the _HardFail_ policy. See for more details about SPF policies. + +In any case, increment the SPF record's TTL to its final value. + +### Backup MX & Secondary MX for `policyd-spf` + +For whitelisting an IP Address from the SPF test, you can create a config file (see [`policyd-spf.conf`](https://www.linuxcertif.com/man/5/policyd-spf.conf)) and mount that file into `/etc/postfix-policyd-spf-python/policyd-spf.conf`. + +**Example:** Create and edit a `policyd-spf.conf` file at `docker-data/dms/config/postfix-policyd-spf.conf`: + +```conf +debugLevel = 1 +#0(only errors)-4(complete data received) + +skip_addresses = 127.0.0.0/8,::ffff:127.0.0.0/104,::1 + +# Preferably use IP-Addresses for whitelist lookups: +Whitelist = 192.168.0.0/31,192.168.1.0/30 +# Domain_Whitelist = mx1.not-example.com,mx2.not-example.com +``` + +Then add this line to `docker-compose.yml`: + +```yaml +volumes: + - ./docker-data/dms/config/postfix-policyd-spf.conf:/etc/postfix-policyd-spf-python/policyd-spf.conf +``` + + +[docs-accounts-add]: ../user-management.md#adding-a-new-account +[docs-volumes-config]: ../advanced/optional-config.md +[docs-env-opendkim]: ../environment.md#enable_opendkim +[docs-env-rspamd]: ../environment.md#enable_rspamd +[docs-rspamd-config-dropin]: ../security/rspamd.md#manually +[docs-rspamd-config-declarative]: ../security/rspamd.md#with-the-help-of-a-custom-file +[cloudflare-dkim-dmarc-spf]: https://www.cloudflare.com/learning/email-security/dmarc-dkim-spf/ +[rfc-6376]: https://tools.ietf.org/html/rfc6376 +[github-issue-dkimlength]: https://github.com/docker-mailserver/docker-mailserver/issues/1854 +[rspamd-docs-dkim-checks]: https://www.rspamd.com/doc/modules/dkim.html +[rspamd-docs-dkim-signing]: https://www.rspamd.com/doc/modules/dkim_signing.html +[dns::example-webui]: https://www.vultr.com/docs/introduction-to-vultr-dns/ +[dns::digicert-ttl]: https://www.digicert.com/faq/dns/what-is-ttl +[dns::wikipedia-zonefile]: https://en.wikipedia.org/wiki/Zone_file +[dns::webui-dkim]: https://serverfault.com/questions/763815/route-53-doesnt-allow-adding-dkim-keys-because-length-is-too-long +[dkim-ed25519-support]: https://serverfault.com/questions/1023674/is-ed25519-well-supported-for-the-dkim-validation/1074545#1074545 +[mxtoolbox-dkim-verifier]: https://mxtoolbox.com/dkim.aspx +[dmarc-howto-configtags]: https://github.com/internetstandards/toolbox-wiki/blob/master/DMARC-how-to.md#overview-of-dmarc-configuration-tags +[dmarc-tool-gca]: https://dmarcguide.globalcyberalliance.org +[dmarcian-tools]: https://dmarcian.com/dmarc-tools/ +[wikipedia-dkim]: https://en.wikipedia.org/wiki/DomainKeys_Identified_Mail +[wikipedia-spf]: https://en.wikipedia.org/wiki/Sender_Policy_Framework diff --git a/docs/content/config/best-practices/dmarc.md b/docs/content/config/best-practices/dmarc.md deleted file mode 100644 index d8a2710b98e..00000000000 --- a/docs/content/config/best-practices/dmarc.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: 'Best Practices | DMARC' -hide: - - toc # Hide Table of Contents for this page ---- - -More information at [DMARC Guide][dmarc-howto]. - -## Enabling DMARC - -In `docker-mailserver`, DMARC is pre-configured out of the box. The only thing you need to do in order to enable it, is to add new `TXT` entry to your DNS. - -In contrast with [DKIM][docs-dkim], the DMARC DNS entry does not require any keys, but merely setting the [configuration values][dmarc-howto::configtags]. You can either handcraft the entry by yourself or use one of available generators (like [this one][dmarc-tool::gca]). - -Typically something like this should be good to start with (_don't forget to replace `@example.com` to your actual domain_): - -``` -_dmarc.example.com. IN TXT "v=DMARC1; p=none; rua=mailto:dmarc.report@example.com; ruf=mailto:dmarc.report@example.com; sp=none; ri=86400" -``` - -Or a bit more strict policies (_mind `p=quarantine` and `sp=quarantine`_): - -``` -_dmarc IN TXT "v=DMARC1; p=quarantine; rua=mailto:dmarc.report@example.com; ruf=mailto:dmarc.report@example.com; fo=0; adkim=r; aspf=r; pct=100; rf=afrf; ri=86400; sp=quarantine" -``` - -DMARC status is not being displayed instantly in Gmail for instance. If you want to check it directly after DNS entries, you can use some services around the Internet such as from [Global Cyber Alliance][dmarc-tool::gca] or [RedSift][dmarc-tool::redsift]. In other cases, email clients will show "DMARC: PASS" in ~1 day or so. - -Reference: [#1511][github-issue-1511] - -[docs-dkim]: ./dkim.md -[github-issue-1511]: https://github.com/docker-mailserver/docker-mailserver/issues/1511 -[dmarc-howto]: https://github.com/internetstandards/toolbox-wiki/blob/master/DMARC-how-to.md -[dmarc-howto::configtags]: https://github.com/internetstandards/toolbox-wiki/blob/master/DMARC-how-to.md#overview-of-dmarc-configuration-tags -[dmarc-tool::gca]: https://dmarcguide.globalcyberalliance.org -[dmarc-tool::redsift]: https://ondmarc.redsift.com diff --git a/docs/content/config/best-practices/spf.md b/docs/content/config/best-practices/spf.md deleted file mode 100644 index a64fe425255..00000000000 --- a/docs/content/config/best-practices/spf.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -title: 'Best Practices | SPF' -hide: - - toc # Hide Table of Contents for this page ---- - -From [Wikipedia](https://en.wikipedia.org/wiki/Sender_Policy_Framework): - -!!! quote - Sender Policy Framework (SPF) is a simple email-validation system designed to detect email spoofing by providing a mechanism to allow receiving mail exchangers to check that incoming mail from a domain comes from a host authorized by that domain's administrators. The list of authorized sending hosts for a domain is published in the Domain Name System (DNS) records for that domain in the form of a specially formatted TXT record. Email spam and phishing often use forged "from" addresses, so publishing and checking SPF records can be considered anti-spam techniques. - -!!! note - For a more technical review: https://github.com/internetstandards/toolbox-wiki/blob/master/SPF-how-to.md - -## Add a SPF Record - -To add a SPF record in your DNS, insert the following line in your DNS zone: - -```txt -; MX record must be declared for SPF to work -example.com. IN MX 1 mail.example.com. - -; SPF record -example.com. IN TXT "v=spf1 mx ~all" -``` - -This enables the _Softfail_ mode for SPF. You could first add this SPF record with a very low TTL. - -_SoftFail_ is a good setting for getting started and testing, as it lets all email through, with spams tagged as such in the mailbox. - -After verification, you _might_ want to change your SPF record to `v=spf1 mx -all` so as to enforce the _HardFail_ policy. See http://www.open-spf.org/SPF_Record_Syntax for more details about SPF policies. - -In any case, increment the SPF record's TTL to its final value. - -## Backup MX, Secondary MX - -For whitelisting an IP Address from the SPF test, you can create a config file (see [`policyd-spf.conf`](https://www.linuxcertif.com/man/5/policyd-spf.conf)) and mount that file into `/etc/postfix-policyd-spf-python/policyd-spf.conf`. - -**Example:** - -Create and edit a `policyd-spf.conf` file at `docker-data/dms/config/postfix-policyd-spf.conf`: - -```conf -debugLevel = 1 -#0(only errors)-4(complete data received) - -skip_addresses = 127.0.0.0/8,::ffff:127.0.0.0/104,::1 - -# Preferably use IP-Addresses for whitelist lookups: -Whitelist = 192.168.0.0/31,192.168.1.0/30 -# Domain_Whitelist = mx1.not-example.com,mx2.not-example.com -``` - -Then add this line to `docker-compose.yml`: - -```yaml -volumes: - - ./docker-data/dms/config/postfix-policyd-spf.conf:/etc/postfix-policyd-spf-python/policyd-spf.conf -``` diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index dc81b1fcfa5..4dcaf3deacc 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -78,7 +78,7 @@ DMS brings sane default settings for Rspamd. They are located at `/etc/rspamd/lo ### With the Help of a Custom File -DMS provides the ability to do simple adjustments to Rspamd modules with the help of a single file. Just place a file called `rspamd-modules.conf` into the directory `docker-data/dms/config/` (which translates to `/tmp/docker-mailserver/` in the container). If this file is present, DMS will evaluate it. The structure is _very_ simple. Each line in the file looks like this: +DMS provides the ability to do simple adjustments to Rspamd modules with the help of a single file. Just place a file called `rspamd-modules.conf` into the [local config directory `docker-data/dms/config/`][docs-volumes-config]. If this file is present, DMS will evaluate it. The structure is _very_ simple. Each line in the file looks like this: ```txt COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3 @@ -133,80 +133,9 @@ Rspamd is running, but you want or need to adjust it? ### DKIM Signing -By default, DMS offers no option to generate and configure signing e-mails with DKIM. This is because the parsing would be difficult. But don't worry: the process is relatively straightforward nevertheless. The [official Rspamd documentation for the DKIM signing module][dkim-signing-module] is pretty good. Basically, you need to - -1. `exec` into the container -2. Run a command similar to `rspamadm dkim_keygen -s 'woosh' -b 2048 -d example.com -k example.private > example.txt`, adjusted to your needs -3. Make sure to then persists the files `example.private` and `example.txt` (created in step 2) in the container (for example with a Docker bind mount) -4. Create a configuration for the DKIM signing module, i.e. a file called `dkim_signing.conf` that you mount to `/etc/rspamd/local.d/` or `/etc/rspamd/override.d/`. We provide example configurations down below. We recommend mounting this file into the container as well (as described [here](#manually)); do not use [`rspamd-modules.conf`](#with-the-help-of-a-custom-file) for this purpose. - -??? example "DKIM Signing Module Configuration Examples" - - A simple configuration could look like this: - - ```cf - # documentation: https://rspamd.com/doc/modules/dkim_signing.html - - enabled = true; - - sign_authenticated = true; - sign_local = true; - - use_domain = "header"; - use_redis = false; # don't change unless Redis also provides the DKIM keys - use_esld = true; - check_pubkey = true; - - domain { - example.com { - path = "/path/to/example.private"; - selector = "woosh"; - } - } - ``` - - If you have multiple domains and you want to sign with the modern ED25519 elliptic curve but also with RSA (you will likely want to have RSA as a fallback!): - - ```cf - # documentation: https://rspamd.com/doc/modules/dkim_signing.html - - enabled = true; - - sign_authenticated = true; - sign_local = true; - - use_domain = "header"; - use_redis = false; # don't change unless Redis also provides the DKIM keys - use_esld = true; - check_pubkey = true; - - domain { - example.com { - selectors [ - { - path = "/path/to/com.example.rsa.private"; - selector = "dkim-rsa"; - }, - { - path = /path/to/com.example.ed25519.private"; - selector = "dkim-ed25519"; - } - ] - } - example.org { - selectors [ - { - path = "/path/to/org.example.rsa.private"; - selector = "dkim-rsa"; - }, - { - path = "/path/to/org.example.ed25519.private"; - selector = "dkim-ed25519"; - } - ] - } - } - ``` +There is a dedicated [section for setting up DKIM with Rspamd in our documentation][docs-dkim-with-rspamd]. + +[docs-dkim-with-rspamd]: ../best-practices/dkim_dmarc_spf.md#dkim ### _Abusix_ Integration @@ -226,6 +155,7 @@ While _Abusix_ can be integrated into Postfix, Postscreen and a multitude of oth [//]: # (General Links) +[docs-volumes-config]: ../advanced/optional-config.md [homepage]: https://rspamd.com/ [modules]: https://rspamd.com/doc/modules/ [proxy-self-scan-mode]: https://rspamd.com/doc/workers/rspamd_proxy.html#self-scan-mode diff --git a/docs/content/examples/tutorials/basic-installation.md b/docs/content/examples/tutorials/basic-installation.md index 31c6496148c..79e63d6f298 100644 --- a/docs/content/examples/tutorials/basic-installation.md +++ b/docs/content/examples/tutorials/basic-installation.md @@ -218,8 +218,8 @@ In this setup `docker-mailserver` is not intended to receive email from the outs [docs-ports]: ../../config/security/understanding-the-ports.md [docs-environment]: ../../config/environment.md -[docs-spf]: ../../config/best-practices/spf.md -[docs-dkim]: ../../config/best-practices/dkim.md +[docs-spf]: ../../config/best-practices/dkim_dmarc_spf.md#spf +[docs-dkim]: ../../config/best-practices/dkim_dmarc_spf.md#dkim [docs-ssl]: ../../config/security/ssl.md#lets-encrypt-recommended [docs-usage]: ../../usage.md#get-up-and-running [github-issue-ufw]: https://github.com/docker-mailserver/docker-mailserver/issues/3151 diff --git a/docs/content/usage.md b/docs/content/usage.md index 5b4f84a3b09..4d2dfedaeb3 100644 --- a/docs/content/usage.md +++ b/docs/content/usage.md @@ -159,44 +159,12 @@ You should add at least one [alias][docs-aliases], the [_postmaster alias_][docs docker exec -ti setup alias add postmaster@example.com user@example.com ``` -### DKIM Keys +### Advanced DNS Setup - DKIM, DMARC & SPF -You can (_and you should_) generate DKIM keys. For more information: - -- DKIM [with OpenDKIM][docs-dkim-opendkim] (_enabled by default_) -- DKIM [with Rspamd][docs-dkim-rspamd] (_when using `ENABLE_RSPAMD=1`_) - -When keys are generated, you can configure your DNS server by just pasting the content of `config/opendkim/keys/domain.tld/mail.txt` to [set up DKIM][dkim-signing-setup]. See the [documentation][docs-dkim-dns] for more details. - -!!! note - - In case you're using LDAP, the setup looks a bit different as you do not add user accounts directly. Postfix doesn't know your domain(s) and you need to provide it when configuring DKIM: - - ``` BASH - docker exec -ti setup config dkim domain '[,]' - ``` - -[dkim-signing-setup]: https://mxtoolbox.com/dmarc/dkim/setup/how-to-setup-dkim -[docs-dkim-dns]: ./config/best-practices/dkim.md#configuration-using-a-web-interface -[docs-dkim-opendkim]: ./config/best-practices/dkim.md#enabling-dkim-signature -[docs-dkim-rspamd]: ./config/security/rspamd.md#dkim-signing - -### Advanced DNS Setup - -You will very likely want to configure your DNS with these TXT records: [SPF, DKIM, and DMARC][cloudflare-spf-dkim-dmarc]. - -The following illustrates what a (rather strict) set of records could look like: - -```console -$ dig @1.1.1.1 +short TXT example.com -"v=spf1 mx -all" -$ dig @1.1.1.1 +short TXT dkim-rsa._domainkey.example.com -"v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQ..." -$ dig @1.1.1.1 +short TXT _dmarc.example.com -"v=DMARC1; p=reject; sp=reject; pct=100; adkim=s; aspf=s; fo=1" -``` +You will very likely want to configure your DNS with these TXT records: [SPF, DKIM, and DMARC][cloudflare-spf-dkim-dmarc]. We also ship a [dedicated page in our documentation][docs-dkim-dmarc-spf] about the setup of DKIM, DMARC & SPF. [cloudflare-spf-dkim-dmarc]: https://www.cloudflare.com/learning/email-security/dmarc-dkim-spf/ +[docs-dkim-dmarc-spf]: ./config/best-practices/dkim_dmarc_spf.md ### Custom User Changes & Patches diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index b7b5f7ca679..c56e34f1361 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -78,6 +78,8 @@ markdown_extensions: - name: mermaid class: mermaid format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.tabbed: + alternate_style: true - pymdownx.magiclink - pymdownx.inlinehilite - pymdownx.tilde @@ -116,9 +118,7 @@ nav: - 'Environment Variables': config/environment.md - 'User Management': config/user-management.md - 'Best Practices': - - 'DKIM': config/best-practices/dkim.md - - 'DMARC': config/best-practices/dmarc.md - - 'SPF': config/best-practices/spf.md + - 'DKIM, DMARC & SPF': config/best-practices/dkim_dmarc_spf.md - 'Auto-discovery': config/best-practices/autodiscover.md - 'Security': - 'Understanding the Ports': config/security/understanding-the-ports.md From ddcc1dcc5c001122a988c4cf6b783e4c13661727 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 10 Apr 2023 15:36:34 +0200 Subject: [PATCH 028/592] docs: renamings (#3242) --- docs/content/config/advanced/auth-ldap.md | 2 +- .../config/advanced/full-text-search.md | 4 +- docs/content/config/advanced/ipv6.md | 4 +- docs/content/config/advanced/kubernetes.md | 28 +++++----- .../content/config/advanced/mail-fetchmail.md | 2 +- .../advanced/override-defaults/dovecot.md | 6 +-- .../override-defaults/user-patches.md | 2 +- docs/content/config/advanced/podman.md | 4 +- .../config/best-practices/autodiscover.md | 2 +- .../config/best-practices/dkim_dmarc_spf.md | 6 +-- docs/content/config/environment.md | 10 ++-- docs/content/config/security/fail2ban.md | 3 +- docs/content/config/security/ssl.md | 44 ++++++++-------- .../security/understanding-the-ports.md | 19 ++++--- docs/content/config/setup.sh.md | 6 +-- .../contributing/issues-and-pull-requests.md | 2 +- .../examples/tutorials/basic-installation.md | 6 +-- docs/content/examples/tutorials/blog-posts.md | 2 +- .../tutorials/mailserver-behind-proxy.md | 6 +-- ...nly-mailserver-with-ldap-authentication.md | 14 ++--- docs/content/faq.md | 8 +-- docs/content/index.md | 4 +- docs/content/introduction.md | 52 +++++++++---------- docs/content/usage.md | 2 +- 24 files changed, 119 insertions(+), 119 deletions(-) diff --git a/docs/content/config/advanced/auth-ldap.md b/docs/content/config/advanced/auth-ldap.md index 723deb7af40..44d3aec98fe 100644 --- a/docs/content/config/advanced/auth-ldap.md +++ b/docs/content/config/advanced/auth-ldap.md @@ -4,7 +4,7 @@ title: 'Advanced | LDAP Authentication' ## Introduction -Getting started with ldap and `docker-mailserver` we need to take 3 parts in account: +Getting started with ldap and DMS we need to take 3 parts in account: - `postfix` for incoming & outgoing email - `dovecot` for accessing mailboxes diff --git a/docs/content/config/advanced/full-text-search.md b/docs/content/config/advanced/full-text-search.md index 03379188a13..f4f2f760f58 100644 --- a/docs/content/config/advanced/full-text-search.md +++ b/docs/content/config/advanced/full-text-search.md @@ -6,7 +6,7 @@ title: 'Advanced | Full-Text Search' Full-text search allows all messages to be indexed, so that mail clients can quickly and efficiently search messages by their full text content. Dovecot supports a variety of community supported [FTS indexing backends](https://doc.dovecot.org/configuration_manual/fts/). -`docker-mailserver` comes pre-installed with two plugins that can be enabled with a dovecot config file. +DMS comes pre-installed with two plugins that can be enabled with a dovecot config file. Please be aware that indexing consumes memory and takes up additional disk space. @@ -101,7 +101,7 @@ While indexing is memory intensive, you can configure the plugin to limit the am ``` docker-compose exec mailserver doveadm fts optimize -A ``` - Or like the [Spamassassin example][docs-faq-sa-learn-cron] shows, you can instead use `cron` from within `docker-mailserver` to avoid potential errors if the mail-server is not running: + Or like the [Spamassassin example][docs-faq-sa-learn-cron] shows, you can instead use `cron` from within DMS to avoid potential errors if the mail server is not running: ??? example diff --git a/docs/content/config/advanced/ipv6.md b/docs/content/config/advanced/ipv6.md index 7b3a8ca4dca..2853e434153 100644 --- a/docs/content/config/advanced/ipv6.md +++ b/docs/content/config/advanced/ipv6.md @@ -4,9 +4,9 @@ title: 'Advanced | IPv6' ## Background -If your container host supports IPv6, then `docker-mailserver` will automatically accept IPv6 connections by way of the docker host's IPv6. However, incoming mail will fail SPF checks because they will appear to come from the IPv4 gateway that docker is using to proxy the IPv6 connection (`172.20.0.1` is the gateway). +If your container host supports IPv6, then DMS will automatically accept IPv6 connections by way of the docker host's IPv6. However, incoming mail will fail SPF checks because they will appear to come from the IPv4 gateway that docker is using to proxy the IPv6 connection (`172.20.0.1` is the gateway). -This can be solved by supporting IPv6 connections all the way to the `docker-mailserver` container. +This can be solved by supporting IPv6 connections all the way to the DMS container. ## Setup steps diff --git a/docs/content/config/advanced/kubernetes.md b/docs/content/config/advanced/kubernetes.md index cc5f51ee4da..6d7330f4c7d 100644 --- a/docs/content/config/advanced/kubernetes.md +++ b/docs/content/config/advanced/kubernetes.md @@ -4,11 +4,11 @@ title: 'Advanced | Kubernetes' ## Introduction -This article describes how to deploy `docker-mailserver` to Kubernetes. Please note that there is also a [Helm chart] available. +This article describes how to deploy DMS to Kubernetes. Please note that there is also a [Helm chart] available. !!! attention "Requirements" - We assume basic knowledge about Kubernetes from the reader. Moreover, we assume the reader to have a basic understanding of mail servers. Ideally, the reader has deployed `docker-mailserver` before in an easier setup with Docker (Compose). + We assume basic knowledge about Kubernetes from the reader. Moreover, we assume the reader to have a basic understanding of mail servers. Ideally, the reader has deployed DMS before in an easier setup with Docker (Compose). !!! warning "About Support for Kubernetes" @@ -58,7 +58,7 @@ data: SSL_KEY_PATH: /secrets/ssl/rsa/tls.key ``` -We can also make use of user-provided configuration files, e.g. `user-patches.sh`, `postfix-accounts.cf` and more, to adjust `docker-mailserver` to our likings. We encourage you to have a look at [Kustomize][kustomize] for creating `ConfigMap`s from multiple files, but for now, we will provide a simple, hand-written example. This example is absolutely minimal and only goes to show what can be done. +We can also make use of user-provided configuration files, e.g. `user-patches.sh`, `postfix-accounts.cf` and more, to adjust DMS to our likings. We encourage you to have a look at [Kustomize][kustomize] for creating `ConfigMap`s from multiple files, but for now, we will provide a simple, hand-written example. This example is absolutely minimal and only goes to show what can be done. ```yaml --- @@ -149,7 +149,7 @@ spec: ### Deployments -Last but not least, the `Deployment` becomes the most complex component. It instructs Kubernetes how to run the `docker-mailserver` container and how to apply your `ConfigMaps`, persisted storage, etc. Additionally, we can set options to enforce runtime security here. +Last but not least, the `Deployment` becomes the most complex component. It instructs Kubernetes how to run the DMS container and how to apply your `ConfigMaps`, persisted storage, etc. Additionally, we can set options to enforce runtime security here. ```yaml --- @@ -305,7 +305,7 @@ spec: ### Certificates - An Example -In this example, we use [`cert-manager`][cert-manager] to supply RSA certificates. You can also supply RSA certificates as fallback certificates, which `docker-mailserver` supports out of the box with `SSL_ALT_CERT_PATH` and `SSL_ALT_KEY_PATH`, and provide ECDSA as the proper certificates. +In this example, we use [`cert-manager`][cert-manager] to supply RSA certificates. You can also supply RSA certificates as fallback certificates, which DMS supports out of the box with `SSL_ALT_CERT_PATH` and `SSL_ALT_KEY_PATH`, and provide ECDSA as the proper certificates. ```yaml --- @@ -340,15 +340,15 @@ spec: The [TLS docs page][docs-tls] provides guidance when it comes to certificates and transport layer security. Always provide sensitive information vai `Secrets`. -## Exposing your Mail-Server to the Outside World +## Exposing your Mail Server to the Outside World -The more difficult part with Kubernetes is to expose a deployed `docker-mailserver` to the outside world. Kubernetes provides multiple ways for doing that; each has downsides and complexity. The major problem with exposing `docker-mailserver` to outside world in Kubernetes is to [preserve the real client IP][Kubernetes-service-source-ip]. The real client IP is required by `docker-mailserver` for performing IP-based SPF checks and spam checks. If you do not require SPF checks for incoming mails, you may disable them in your [Postfix configuration][docs-postfix] by dropping the line that states: `check_policy_service unix:private/policyd-spf`. +The more difficult part with Kubernetes is to expose a deployed DMS to the outside world. Kubernetes provides multiple ways for doing that; each has downsides and complexity. The major problem with exposing DMS to outside world in Kubernetes is to [preserve the real client IP][Kubernetes-service-source-ip]. The real client IP is required by DMS for performing IP-based SPF checks and spam checks. If you do not require SPF checks for incoming mails, you may disable them in your [Postfix configuration][docs-postfix] by dropping the line that states: `check_policy_service unix:private/policyd-spf`. -The easiest approach was covered above, using `#!yaml externalTrafficPolicy: Local`, which disables the service proxy, but makes the service local as well (which does not scale). This approach only works when you are given the correct (that is, a public and routable) IP address by a load balancer (like MetalLB). In this sense, the approach above is similar to the next example below. We want to provide you with a few alternatives too. **But** we also want to communicate the idea of another simple method: you could use a load-balancer without an external IP and DNAT the network traffic to the mail-server. After all, this does not interfere with SPF checks because it keeps the origin IP address. If no dedicated external IP address is available, you could try the latter approach, if one is available, use the former. +The easiest approach was covered above, using `#!yaml externalTrafficPolicy: Local`, which disables the service proxy, but makes the service local as well (which does not scale). This approach only works when you are given the correct (that is, a public and routable) IP address by a load balancer (like MetalLB). In this sense, the approach above is similar to the next example below. We want to provide you with a few alternatives too. **But** we also want to communicate the idea of another simple method: you could use a load-balancer without an external IP and DNAT the network traffic to the mail server. After all, this does not interfere with SPF checks because it keeps the origin IP address. If no dedicated external IP address is available, you could try the latter approach, if one is available, use the former. ### External IPs Service -The simplest way is to expose `docker-mailserver` as a [Service][Kubernetes-network-service] with [external IPs][Kubernetes-network-external-ip]. This is very similar to the approach taken above. Here, an external IP is given to the service directly by you. With the approach above, you tell your load-balancer to do this. +The simplest way is to expose DMS as a [Service][Kubernetes-network-service] with [external IPs][Kubernetes-network-external-ip]. This is very similar to the approach taken above. Here, an external IP is given to the service directly by you. With the approach above, you tell your load-balancer to do this. ```yaml --- @@ -380,7 +380,7 @@ This approach ### Proxy port to Service -The [proxy pod][Kubernetes-proxy-service] helps to avoid the necessity of specifying external IPs explicitly. This comes at the cost of complexity; you must deploy a proxy pod on each [Node][Kubernetes-nodes] you want to expose `docker-mailserver` on. +The [proxy pod][Kubernetes-proxy-service] helps to avoid the necessity of specifying external IPs explicitly. This comes at the cost of complexity; you must deploy a proxy pod on each [Node][Kubernetes-nodes] you want to expose DMS on. This approach @@ -388,7 +388,7 @@ This approach ### Bind to concrete Node and use host network -One way to preserve the real client IP is to use `hostPort` and `hostNetwork: true`. This comes at the cost of availability; you can reach `docker-mailserver` from the outside world only via IPs of [Node][Kubernetes-nodes] where `docker-mailserver` is deployed. +One way to preserve the real client IP is to use `hostPort` and `hostNetwork: true`. This comes at the cost of availability; you can reach DMS from the outside world only via IPs of [Node][Kubernetes-nodes] where DMS is deployed. ```yaml --- @@ -420,12 +420,12 @@ metadata: With this approach, -- it is not possible to access `docker-mailserver` via other cluster Nodes, only via the Node `docker-mailserver` was deployed at. +- it is not possible to access DMS via other cluster Nodes, only via the Node DMS was deployed at. - every Port within the Container is exposed on the Host side. ### Proxy Port to Service via PROXY Protocol -This way is ideologically the same as [using a proxy pod](#proxy-port-to-service), but instead of a separate proxy pod, you configure your ingress to proxy TCP traffic to the `docker-mailserver` pod using the PROXY protocol, which preserves the real client IP. +This way is ideologically the same as [using a proxy pod](#proxy-port-to-service), but instead of a separate proxy pod, you configure your ingress to proxy TCP traffic to the DMS pod using the PROXY protocol, which preserves the real client IP. #### Configure your Ingress @@ -501,7 +501,7 @@ Then, configure both [Postfix][docs-postfix] and [Dovecot][docs-dovecot] to expe With this approach, -- it is not possible to access `docker-mailserver` via cluster-DNS, as the PROXY protocol is required for incoming connections. +- it is not possible to access DMS via cluster-DNS, as the PROXY protocol is required for incoming connections. [Helm chart]: https://github.com/docker-mailserver/docker-mailserver-helm [kustomize]: https://kustomize.io/ diff --git a/docs/content/config/advanced/mail-fetchmail.md b/docs/content/config/advanced/mail-fetchmail.md index eeb8c9f49de..b364eed096d 100644 --- a/docs/content/config/advanced/mail-fetchmail.md +++ b/docs/content/config/advanced/mail-fetchmail.md @@ -10,7 +10,7 @@ environment: - FETCHMAIL_POLL=300 ``` -Generate a file called `fetchmail.cf` and place it in the `docker-data/dms/config/` folder. Your `docker-mailserver` folder should look like this example: +Generate a file called `fetchmail.cf` and place it in the `docker-data/dms/config/` folder. Your DMS folder should look like this example: ```txt ├── docker-data/dms/config diff --git a/docs/content/config/advanced/override-defaults/dovecot.md b/docs/content/config/advanced/override-defaults/dovecot.md index 87b51ca129f..ab1f3bb5036 100644 --- a/docs/content/config/advanced/override-defaults/dovecot.md +++ b/docs/content/config/advanced/override-defaults/dovecot.md @@ -7,7 +7,7 @@ title: 'Override the Default Configs | Dovecot' The Dovecot default configuration can easily be extended providing a `docker-data/dms/config/dovecot.cf` file. [Dovecot documentation](https://doc.dovecot.org/configuration_manual/) remains the best place to find configuration options. -Your `docker-mailserver` folder should look like this example: +Your DMS folder structure should look like this example: ```txt ├── docker-data/dms/config @@ -26,7 +26,7 @@ mail_max_userip_connections = 100 Another important option is the `default_process_limit` (defaults to `100`). If high-security mode is enabled you'll need to make sure this count is higher than the maximum number of users that can be logged in simultaneously. -This limit is quickly reached if users connect to the `docker-mailserver` with multiple end devices. +This limit is quickly reached if users connect to DMS with multiple end devices. ## Override Configuration @@ -55,7 +55,7 @@ To debug your dovecot configuration you can use: - Or: `docker exec -it mailserver doveconf | grep ` !!! note - [`setup.sh`][github-file-setupsh] is included in the `docker-mailserver` repository. Make sure to use the one matching your image version release. + [`setup.sh`][github-file-setupsh] is included in the DMS repository. Make sure to use the one matching your image version release. The file `docker-data/dms/config/dovecot.cf` is copied internally to `/etc/dovecot/local.conf`. To verify the file content, run: diff --git a/docs/content/config/advanced/override-defaults/user-patches.md b/docs/content/config/advanced/override-defaults/user-patches.md index ba3206e3ce5..597ed6aa52b 100644 --- a/docs/content/config/advanced/override-defaults/user-patches.md +++ b/docs/content/config/advanced/override-defaults/user-patches.md @@ -2,7 +2,7 @@ title: 'Custom User Changes & Patches | Scripting' --- -If you'd like to change, patch or alter files or behavior of `docker-mailserver`, you can use a script. +If you'd like to change, patch or alter files or behavior of DMS, you can use a script. In case you cloned this repository, you can copy the file [`user-patches.sh.dist` (_under `config/`_)][github-file-userpatches] with `#!sh cp config/user-patches.sh.dist docker-data/dms/config/user-patches.sh` in order to create the `user-patches.sh` script. diff --git a/docs/content/config/advanced/podman.md b/docs/content/config/advanced/podman.md index 4e6d9d03d4d..fb149f6e4d7 100644 --- a/docs/content/config/advanced/podman.md +++ b/docs/content/config/advanced/podman.md @@ -8,7 +8,7 @@ Podman is a daemonless container engine for developing, managing, and running OC !!! warning "About Support for Podman" - Please note that Podman **is not** officially supported as `docker-mailserver` is built and verified on top of the _Docker Engine_. This content is entirely community supported. If you find errors, please open an issue and provide a PR. + Please note that Podman **is not** officially supported as DMS is built and verified on top of the _Docker Engine_. This content is entirely community supported. If you find errors, please open an issue and provide a PR. !!! warning "About this Guide" @@ -67,7 +67,7 @@ Also notice that Podman's rootless mode is not about running as a non-root user !!! warning - In order to make rootless `docker-mailserver` work we must modify some settings in the Linux system, it requires some basic linux server knowledge so don't follow this guide if you not sure what this guide is talking about. Podman rootfull mode and Docker are still good and security enough for normal daily usage. + In order to make rootless DMS work we must modify some settings in the Linux system, it requires some basic linux server knowledge so don't follow this guide if you not sure what this guide is talking about. Podman rootfull mode and Docker are still good and security enough for normal daily usage. First, enable `podman.socket` in systemd's userspace with a non-root user. diff --git a/docs/content/config/best-practices/autodiscover.md b/docs/content/config/best-practices/autodiscover.md index 4e3ba8e68d5..351376e4b6e 100644 --- a/docs/content/config/best-practices/autodiscover.md +++ b/docs/content/config/best-practices/autodiscover.md @@ -6,7 +6,7 @@ hide: # Auto-Discovery of Services -Email auto-discovery means a client email is able to automagically find out about what ports and security options to use, based on the mail-server URI. It can help simplify the tedious / confusing task of adding own's email account for non-tech savvy users. +Email auto-discovery means a client email is able to automagically find out about what ports and security options to use, based on the mail server URI. It can help simplify the tedious / confusing task of adding own's email account for non-tech savvy users. Email clients will search for auto-discoverable settings and prefill almost everything when a user enters its email address :heart: diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md index 5638476230c..f8d17072dce 100644 --- a/docs/content/config/best-practices/dkim_dmarc_spf.md +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -11,7 +11,7 @@ Cloudflare has written an [article about DKIM, DMARC and SPF][cloudflare-dkim-dm - SPF: `policyd-spf` is used in Postfix's `smtpd_recipient_restrictions` In a future release Rspamd will become the default for these validations, with a deprecation notice issued prior to the removal of the above alternatives. - + We encourage everyone to prefer Rspamd via `ENABLE_RSPAMD=1`. !!! warning "DNS Caches & Propagation" @@ -184,7 +184,7 @@ DKIM is currently supported by either OpenDKIM or Rspamd: !!! info "Restart required" - After restarting `docker-mailserver`, outgoing mail will now be signed with your new DKIM key(s) :tada: + After restarting DMS, outgoing mail will now be signed with your new DKIM key(s) :tada: You'll need to repeat this process if you add any new domains. @@ -220,7 +220,7 @@ When mail signed with your DKIM key is sent from your mail server, the receiver ??? info "`.txt` - Formatting the `TXT` record value correctly" This file was generated for use within a [DNS zone file][dns::wikipedia-zonefile]. DNS `TXT` records values that are longer than 255 characters need to be split into multiple parts. This is why the public key has multiple parts wrapped within double-quotes between `(` and `)`. - + A DNS web-interface may handle this internally instead, while [others may not, but expect the input as a single line][dns::webui-dkim]_). You'll need to manually format the value as described below. Your DNS record file (eg: `mail.txt`) should look similar to this: diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index a3307916d13..f0b548b2233 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -12,8 +12,8 @@ title: Environment Variables If you can't set your hostname (_eg: you're in a container platform that doesn't let you_) specify it via this environment variable. It will have priority over `docker run --hostname`, or the equivalent `hostname:` field in `docker-compose.yml`. -- **empty** => Uses the `hostname -f` command to get canonical hostname for `docker-mailserver` to use. -- => Specify an FQDN (fully-qualified domain name) to serve mail for. The hostname is required for `docker-mailserver` to function correctly. +- **empty** => Uses the `hostname -f` command to get canonical hostname for DMS to use. +- => Specify an FQDN (fully-qualified domain name) to serve mail for. The hostname is required for DMS to function correctly. ##### LOG_LEVEL @@ -176,7 +176,7 @@ Configures the handling of creating mails with forged sender addresses. ##### ENABLE_SRS -Enables the Sender Rewriting Scheme. SRS is needed if `docker-mailserver` acts as forwarder. See [postsrsd](https://github.com/roehling/postsrsd/blob/master/README.md#sender-rewriting-scheme-crash-course) for further explanation. +Enables the Sender Rewriting Scheme. SRS is needed if DMS acts as forwarder. See [postsrsd](https://github.com/roehling/postsrsd/blob/master/README.md#sender-rewriting-scheme-crash-course) for further explanation. - **0** => Disabled - 1 => Enabled @@ -466,7 +466,7 @@ Note: this SpamAssassin setting needs `ENABLE_SPAMASSASSIN=1` !!! note "This SpamAssassin setting needs `ENABLE_SPAMASSASSIN=1`" - By default, `docker-mailserver` is configured to quarantine spam emails. + By default, DMS is configured to quarantine spam emails. If emails are quarantined, they are compressed and stored in a location dependent on the `ONE_DIR` setting above. To inhibit this behaviour and deliver spam emails, set this to a very high value e.g. `100.0`. @@ -527,7 +527,7 @@ Deprecated. See [`ACCOUNT_PROVISIONER`](#account_provisioner). - **empty** => mail.example.com - => Specify the dns-name/ip-address where the ldap-server is listening, or an URI like `ldaps://mail.example.com` -- NOTE: If you going to use `docker-mailserver` in combination with `docker-compose.yml` you can set the service name here +- NOTE: If you going to use DMS in combination with `docker-compose.yml` you can set the service name here ##### LDAP_SEARCH_BASE diff --git a/docs/content/config/security/fail2ban.md b/docs/content/config/security/fail2ban.md index 2bebf77ace5..954b9bdbff1 100644 --- a/docs/content/config/security/fail2ban.md +++ b/docs/content/config/security/fail2ban.md @@ -29,7 +29,8 @@ Example configuration volume bind: ``` !!! attention - `docker-mailserver` must be launched with the `NET_ADMIN` capability in order to be able to install the nftables rules that actually ban IP addresses. + + DMS must be launched with the `NET_ADMIN` capability in order to be able to install the nftables rules that actually ban IP addresses. Thus either include `--cap-add=NET_ADMIN` in the `docker run` command, or the equivalent in `docker-compose.yml`: diff --git a/docs/content/config/security/ssl.md b/docs/content/config/security/ssl.md index ff0fe34e0fa..2df2056b27c 100644 --- a/docs/content/config/security/ssl.md +++ b/docs/content/config/security/ssl.md @@ -25,12 +25,12 @@ After installation, you can test your setup with: ## The FQDN -An [FQDN](https://en.wikipedia.org/wiki/Fully_qualified_domain_name) (_Fully Qualified Domain Name_) such as `mail.example.com` is required for `docker-mailserver` to function correctly, especially for looking up the correct SSL certificate to use. +An [FQDN](https://en.wikipedia.org/wiki/Fully_qualified_domain_name) (_Fully Qualified Domain Name_) such as `mail.example.com` is required for DMS to function correctly, especially for looking up the correct SSL certificate to use. - `mail.example.com` will still use `user@example.com` as the mail address. You do not need a bare domain for that. -- We usually discourage assigning a bare domain (_When your DNS MX record does not point to a subdomain_) to represent `docker-mailserver`. However, an FQDN of [just `example.com` is also supported][docs-faq-baredomain]. +- We usually discourage assigning a bare domain (_When your DNS MX record does not point to a subdomain_) to represent DMS. However, an FQDN of [just `example.com` is also supported][docs-faq-baredomain]. - Internally, `hostname -f` will be used to retrieve the FQDN as configured in the below examples. -- Wildcard certificates (eg: `*.example.com`) are supported for `SSL_TYPE=letsencrypt`. Your configured FQDN below may be `mail.example.com`, and your wildcard certificate provisioned to `/etc/letsencrypt/live/example.com` which will be checked as a fallback FQDN by `docker-mailserver`. +- Wildcard certificates (eg: `*.example.com`) are supported for `SSL_TYPE=letsencrypt`. Your configured FQDN below may be `mail.example.com`, and your wildcard certificate provisioned to `/etc/letsencrypt/live/example.com` which will be checked as a fallback FQDN by DMS. !!! example "Setting the hostname correctly" @@ -40,9 +40,9 @@ An [FQDN](https://en.wikipedia.org/wiki/Fully_qualified_domain_name) (_Fully Qua # CLI: docker run --hostname mail.example.com ``` - + or - + ```yml # docker-compose.yml services: @@ -54,10 +54,10 @@ An [FQDN](https://en.wikipedia.org/wiki/Fully_qualified_domain_name) (_Fully Qua ### Let's Encrypt (Recommended) -To enable _Let's Encrypt_ for `docker-mailserver`, you have to: +To enable _Let's Encrypt_ for DMS, you have to: 1. Get your certificate using the _Let's Encrypt_ client [Certbot][certbot::github]. -2. For your `docker-mailserver` container: +2. For your DMS container: - Add the environment variable `SSL_TYPE=letsencrypt`. - Mount [your local `letsencrypt` folder][certbot::certs-storage] as a volume to `/etc/letsencrypt`. @@ -68,7 +68,7 @@ You don't have to do anything else. Enjoy! `/etc/letsencrypt/live` stores provisioned certificates in individual folders named by their FQDN. - Make sure that the entire folder is mounted to `docker-mailserver` as there are typically symlinks from `/etc/letsencrypt/live/mail.example.com` to `/etc/letsencrypt/archive`. + Make sure that the entire folder is mounted to DMS as there are typically symlinks from `/etc/letsencrypt/live/mail.example.com` to `/etc/letsencrypt/archive`. !!! example @@ -86,7 +86,7 @@ You don't have to do anything else. Enjoy! #### Example using Docker for _Let's Encrypt_ { data-toc-label='Certbot with Docker' } -Certbot provisions certificates to `/etc/letsencrypt`. Add a volume to store these, so that they can later be accessed by `docker-mailserver` container. You may also want to persist Certbot [logs][certbot::log-rotation], just in case you need to troubleshoot. +Certbot provisions certificates to `/etc/letsencrypt`. Add a volume to store these, so that they can later be accessed by DMS container. You may also want to persist Certbot [logs][certbot::log-rotation], just in case you need to troubleshoot. 1. Getting a certificate is this simple! (_Referencing: [Certbot docker instructions][certbot::docker] and [`certonly --standalone` mode][certbot::standalone]_): @@ -99,7 +99,7 @@ Certbot provisions certificates to `/etc/letsencrypt`. Add a volume to store the certbot/certbot certonly --standalone -d mail.example.com ``` -2. Add a volume for `docker-mailserver` that maps the _local `certbot/certs/` folder_ to the container path `/etc/letsencrypt/`. +2. Add a volume for DMS that maps the _local `certbot/certs/` folder_ to the container path `/etc/letsencrypt/`. !!! example @@ -172,7 +172,7 @@ Obtain a Cloudflare API token: # Set SSL certificate type. - SSL_TYPE=letsencrypt volumes: - # Mount the cert folder generated by Certbot into mail-server: + # Mount the cert folder generated by Certbot: - ./docker-data/certbot/certs/:/etc/letsencrypt/:ro certbot-cloudflare: @@ -280,7 +280,7 @@ After completing the steps above, your certificate should be ready to use. If you are running a web server already, port 80 will be in use which Certbot requires. You could use the [Certbot `--webroot`][certbot::webroot] feature, but it is more common to leverage a _reverse proxy_ that manages the provisioning and renewal of certificates for your services automatically. -In the following example, we show how `docker-mailserver` can be run alongside the docker containers [`nginx-proxy`][nginx-proxy::github] and [`acme-companion`][acme-companion::github] (_Referencing: [`acme-companion` documentation][acme-companion::docs]_): +In the following example, we show how DMS can be run alongside the docker containers [`nginx-proxy`][nginx-proxy::github] and [`acme-companion`][acme-companion::github] (_Referencing: [`acme-companion` documentation][acme-companion::docs]_): 1. Start the _reverse proxy_ (`nginx-proxy`): @@ -314,7 +314,7 @@ In the following example, we show how `docker-mailserver` can be run alongside t 3. Start the rest of your web server containers as usual. -4. Start a _dummy container_ to provision certificates for your FQDN (eg: `mail.example.com`). `acme-companion` will detect the container and generate a _Let's Encrypt_ certificate for your domain, which can be used by `docker-mailserver`: +4. Start a _dummy container_ to provision certificates for your FQDN (eg: `mail.example.com`). `acme-companion` will detect the container and generate a _Let's Encrypt_ certificate for your domain, which can be used by DMS: ```sh docker run --detach \ @@ -341,7 +341,7 @@ In the following example, we show how `docker-mailserver` can be run alongside t #### Example using `nginx-proxy` and `acme-companion` with `docker-compose` { data-toc-label='nginx-proxy with docker-compose' } -The following example is the [basic setup][acme-companion::basic-setup] you need for using `nginx-proxy` and `acme-companion` with `docker-mailserver` (_Referencing: [`acme-companion` documentation][acme-companion::docs]_): +The following example is the [basic setup][acme-companion::basic-setup] you need for using `nginx-proxy` and `acme-companion` with DMS (_Referencing: [`acme-companion` documentation][acme-companion::docs]_): ???+ example "Example: `docker-compose.yml`" @@ -412,7 +412,7 @@ The following example is the [basic setup][acme-companion::basic-setup] you need [`acme-companion` ENV for default settings][acme-companion::env-config] that apply to all containers using `LETSENCRYPT_HOST`: - - `DEFAULT_EMAIL`: An email address that the CA (_eg: Let's Encrypt_) can contact you about expiring certificates, failed renewals, or for account recovery. You may want to use an email address not handled by your mail-server to ensure deliverability in the event your mail-server breaks. + - `DEFAULT_EMAIL`: An email address that the CA (_eg: Let's Encrypt_) can contact you about expiring certificates, failed renewals, or for account recovery. You may want to use an email address not handled by your mail server to ensure deliverability in the event your mail server breaks. - `CERTS_UPDATE_INTERVAL`: If you need to adjust the frequency to check for renewals. 3600 seconds (1 hour) by default. - `DEBUG=1`: Should be helpful when [troubleshooting provisioning issues][acme-companion::troubleshooting] from `acme-companion` logs. - `ACME_CA_URI`: Useful in combination with `CA_BUNDLE` to use a private CA. To change the default _Let's Encrypt_ endpoint to the staging endpoint, use `https://acme-staging-v02.api.letsencrypt.org/directory`. @@ -461,7 +461,7 @@ The following example is the [basic setup][acme-companion::basic-setup] you need Version 6.2 and later of the Synology NAS DSM OS now come with an interface to generate and renew letencrypt certificates. Navigation into your DSM control panel and go to Security, then click on the tab Certificate to generate and manage letsencrypt certificates. -Amongst other things, you can use these to secure your mail-server. DSM locates the generated certificates in a folder below `/usr/syno/etc/certificate/_archive/`. +Amongst other things, you can use these to secure your mail server. DSM locates the generated certificates in a folder below `/usr/syno/etc/certificate/_archive/`. Navigate to that folder and note the 6 character random folder name of the certificate you'd like to use. Then, add the following to your `docker-compose.yml` declaration file: @@ -576,7 +576,7 @@ volumes: Wildcard certificates are supported. If your FQDN is `mail.example.com` and your wildcard certificate is `*.example.com`, add the ENV: `#!bash SSL_DOMAIN=example.com`. -The mail-server will select it's certificate from `acme.json` checking these ENV for a matching FQDN (_in order of priority_): +DMS will select it's certificate from `acme.json` checking these ENV for a matching FQDN (_in order of priority_): 1. `#!bash ${SSL_DOMAIN}` 2. `#!bash ${HOSTNAME}` @@ -639,13 +639,13 @@ This feature requires you to provide the following files into your [`docker-data - `-cert.pem` - `demoCA/cacert.pem` -Where `` is the FQDN you've configured for your `docker-mailserver` container. +Where `` is the FQDN you've configured for your DMS container. -Add `SSL_TYPE=self-signed` to your `docker-mailserver` environment variables. Postfix and Dovecot will be configured to use the provided certificate (_`.pem` files above_) during container startup. +Add `SSL_TYPE=self-signed` to your DMS environment variables. Postfix and Dovecot will be configured to use the provided certificate (_`.pem` files above_) during container startup. #### Generating a self-signed certificate -One way to generate self-signed certificates is with [Smallstep's `step` CLI](https://smallstep.com/docs/step-cli). This is exactly what [`docker-mailserver` does for creating test certificates][github-file::tls-readme]. +One way to generate self-signed certificates is with [Smallstep's `step` CLI](https://smallstep.com/docs/step-cli). This is exactly what [DMS does for creating test certificates][github-file::tls-readme]. For example with the FQDN `mail.example.test`, you can generate the required files by running: @@ -709,7 +709,7 @@ The local and internal paths may be whatever you prefer, so long as both `SSL_CE !!! info - You may have to restart `docker-mailserver` once the certificates change. + You may have to restart DMS once the certificates change. ## Testing a Certificate is Valid @@ -870,7 +870,7 @@ fi ## Custom DH Parameters -By default `docker-mailserver` uses [`ffdhe4096`][ffdhe4096-src] from [IETF RFC 7919][ietf::rfc::ffdhe]. These are standardized pre-defined DH groups and the only available DH groups for TLS 1.3. It is [discouraged to generate your own DH parameters][dh-avoid-selfgenerated] as it is often less secure. +By default DMS uses [`ffdhe4096`][ffdhe4096-src] from [IETF RFC 7919][ietf::rfc::ffdhe]. These are standardized pre-defined DH groups and the only available DH groups for TLS 1.3. It is [discouraged to generate your own DH parameters][dh-avoid-selfgenerated] as it is often less secure. Despite this, if you must use non-standard DH parameters or you would like to swap `ffdhe4096` for a different group (eg `ffdhe2048`); Add your own PEM encoded DH params file via a volume to `/tmp/docker-mailserver/dhparams.pem`. This will replace DH params for both Dovecot and Postfix services during container startup. diff --git a/docs/content/config/security/understanding-the-ports.md b/docs/content/config/security/understanding-the-ports.md index 4cd6ded5cd1..ede8ca3b995 100644 --- a/docs/content/config/security/understanding-the-ports.md +++ b/docs/content/config/security/understanding-the-ports.md @@ -15,8 +15,8 @@ Prefer ports with Implicit [TLS][wikipedia-tls] ports, they're more secure than | POP3 | 110 | 995 | Retrieval | No | | IMAP4 | 143 | 993 | Retrieval | Yes | -1. A connection _may_ be secured over TLS when both ends support `STARTTLS`. On ports 110, 143 and 587, `docker-mailserver` will reject a connection that cannot be secured. Port 25 is [required][ref-port25-mandatory] to support insecure connections. -2. Receives email, `docker-mailserver` additionally filters for spam and viruses. For submitting email to the server to be sent to third-parties, you should prefer the _submission_ ports (465, 587) - which require authentication. Unless a relay host is configured (eg: SendGrid), outgoing email will leave the server via port 25 (_thus outbound traffic must not be blocked by your provider or firewall_). +1. A connection _may_ be secured over TLS when both ends support `STARTTLS`. On ports 110, 143 and 587, DMS will reject a connection that cannot be secured. Port 25 is [required][ref-port25-mandatory] to support insecure connections. +2. Receives email, DMS additionally filters for spam and viruses. For submitting email to the server to be sent to third-parties, you should prefer the _submission_ ports (465, 587) - which require authentication. Unless a relay host is configured (eg: SendGrid), outgoing email will leave the server via port 25 (_thus outbound traffic must not be blocked by your provider or firewall_). 3. A _submission_ port since 2018 ([RFC 8314][rfc-8314]). ??? warning "Beware of outdated advice on port 465" @@ -29,7 +29,6 @@ Prefer ports with Implicit [TLS][wikipedia-tls] ports, they're more secure than Understand that port 587 is more broadly supported due to this history and that lots of software in that time has been built or configured with that port in mind. [`STARTTLS` is known to have various CVEs discovered even in recent years][starttls-vulnerabilities], do not be misled by any advice implying it should be preferred over implicit TLS. Trust in more official sources, such as the [config Postfix has][postfix-upstream-config-mastercf] which acknowledges the `submissions` port (465). - ### What Ports Should I Use? (SMTP) ```mermaid @@ -67,18 +66,18 @@ Mail arriving at your server will be processed and stored in a mailbox, or sent - **Port 25:** - Think of this like a physical mailbox, anyone can deliver mail to you here. Typically most mail is delivered to you on this port. - -`docker-mailserver` will actively filter email delivered on this port for spam or viruses, and refuse mail from known bad sources. + - DMS will actively filter email delivered on this port for spam or viruses, and refuse mail from known bad sources. - Connections to this port may be secure through STARTTLS, but is not mandatory as [mail is allowed to arrive via an unencrypted connection][ref-port25-mandatory]. - It is possible for internal clients to submit mail to be sent outbound (_without requiring authentication_), but that is discouraged. Prefer the _submission_ ports. - **Port 465 and 587:** - - This is the equivalent of a post office box where you would send email to be delivered on your behalf (_`docker-mailserver` is that metaphorical post office, aka the MTA_). + - This is the equivalent of a post office box where you would send email to be delivered on your behalf (_DMS is that metaphorical post office, aka the MTA_). - These two ports are known as the _submission_ ports, they enable mail to be sent outbound to another MTA (eg: Outlook or Gmail) but require authentication via a [mail account][docs-accounts]. - - For inbound traffic, this is relevant when you send mail from your MUA (eg: ThunderBird). It's also used when `docker-mailserver` is configured as a mail relay, or when you have a service sending transactional mail (_eg: order confirmations, password resets, notifications_) through `docker-mailserver`. + - For inbound traffic, this is relevant when you send mail from your MUA (eg: ThunderBird). It's also used when DMS is configured as a mail relay, or when you have a service sending transactional mail (_eg: order confirmations, password resets, notifications_) through DMS. - _**Prefer port 465**_ over port 587, as 465 provides Implicit TLS. !!! note - When submitting mail (inbound) to be sent (outbound), this involves two separate connections to negotiate and secure. There may be additional intermediary connections which `docker-mailserver` is not involved in, and thus unable to ensure encrypted transit throughout delivery. + When submitting mail (inbound) to be sent (outbound), this involves two separate connections to negotiate and secure. There may be additional intermediary connections which DMS is not involved in, and thus unable to ensure encrypted transit throughout delivery. #### Outbound Traffic (On the Right) @@ -94,7 +93,7 @@ Mail being sent from your server is either being relayed through another MTA (eg !!! tip - `docker-mailserver` can function as a relay too, but professional relay services have a trusted reputation (_which increases success of delivery_). + DMS can function as a relay too, but professional relay services have a trusted reputation (_which increases success of delivery_). An MTA with low reputation can affect if mail is treated as junk, or even rejected. @@ -113,7 +112,7 @@ Sometimes a reverse-proxy is involved, but is misconfigured or lacks support for !!! note - - By default, `docker-mailserver` is configured to reject connections that fail to establish a secure connection (_when authentication is required_), rather than allow an insecure connection. + - By default, DMS is configured to reject connections that fail to establish a secure connection (_when authentication is required_), rather than allow an insecure connection. - Port 25 does not require authentication. If `STARTTLS` is unsuccessful, mail can be received over an unencrypted connection. You can better secure this port between trusted parties with the addition of MTA-STS, [STARTTLS Policy List][starttls-policy-list], DNSSEC and DANE. !!! warning @@ -140,7 +139,7 @@ While Explicit TLS can provide the same benefit (_when `STARTTLS` is successfull A related section or page on ciphers used may be useful, although less important for users to be concerned about. -### TLS connections for a Mail-Server, compared to web browsers +### TLS connections for a Mail Server, compared to web browsers Unlike with HTTP where a web browser client communicates directly with the server providing a website, a secure TLS connection as discussed below does not provide the equivalent safety that HTTPS does when the transit of email (receiving or sending) is sent through third-parties, as the secure connection is only between two machines, any additional machines (MTAs) between the MUA and the MDA depends on them establishing secure connections between one another successfully. diff --git a/docs/content/config/setup.sh.md b/docs/content/config/setup.sh.md index d0829bc8078..a1cf86a65c9 100644 --- a/docs/content/config/setup.sh.md +++ b/docs/content/config/setup.sh.md @@ -12,11 +12,11 @@ hide: This script assumes Docker or Podman is used. You will not be able to use `setup.sh` with other container orchestration tools. -[`setup.sh`][github-file-setupsh] is a script that is complimentary to the internal `setup` command in `docker-mailserver`. +[`setup.sh`][github-file-setupsh] is a script that is complimentary to the internal `setup` command in DMS. -It mostly provides the convenience of aliasing `docker exec -ti setup`, inferring the container name of a running `docker-mailserver` instance or running a new instance and bind mounting necessary volumes implicitly. +It mostly provides the convenience of aliasing `docker exec -ti setup`, inferring the container name of a running DMS instance or running a new instance and bind mounting necessary volumes implicitly. -It is intended to be run from the host machine, _not_ from inside your running container. The latest version of the script is included in the `docker-mailserver` repository. You may retrieve it at any time by running this command in your console: +It is intended to be run from the host machine, _not_ from inside your running container. The latest version of the script is included in the DMS repository. You may retrieve it at any time by running this command in your console: ```sh wget https://raw.githubusercontent.com/docker-mailserver/docker-mailserver/master/setup.sh diff --git a/docs/content/contributing/issues-and-pull-requests.md b/docs/content/contributing/issues-and-pull-requests.md index 60d44b3ca54..b96fc23c3ba 100644 --- a/docs/content/contributing/issues-and-pull-requests.md +++ b/docs/content/contributing/issues-and-pull-requests.md @@ -10,7 +10,7 @@ This project is Open Source. That means that you can contribute on enhancements, **Before opening an issue**, read the [`README`][github-file-readme] carefully, study the docs for your version (maybe [latest][docs-latest]), the Postfix/Dovecot documentation and your search engine you trust. The issue tracker is not meant to be used for unrelated questions! -When opening an issue, please provide details use case to let the community reproduce your problem. Please start `docker-mailserver` with the environment variable `LOG_LEVEL` set to `debug` or `trace` and paste the output into the issue. +When opening an issue, please provide details use case to let the community reproduce your problem. Please start DMS with the environment variable `LOG_LEVEL` set to `debug` or `trace` and paste the output into the issue. !!! attention diff --git a/docs/content/examples/tutorials/basic-installation.md b/docs/content/examples/tutorials/basic-installation.md index 79e63d6f298..01b8d5aeead 100644 --- a/docs/content/examples/tutorials/basic-installation.md +++ b/docs/content/examples/tutorials/basic-installation.md @@ -86,7 +86,7 @@ services: This guide is focused on only using [SMTP ports (not POP3 and IMAP)][docs-ports] with the intent to relay mail received from another service to an external email address (eg: `user@gmail.com`). It is not intended for mailbox storage of real users. -In this setup `docker-mailserver` is not intended to receive email from the outside world, so no anti-spam or anti-virus software is needed, making the service lighter to run. +In this setup DMS is not intended to receive email from the outside world, so no anti-spam or anti-virus software is needed, making the service lighter to run. !!! tip "`setup`" @@ -159,7 +159,7 @@ In this setup `docker-mailserver` is not intended to receive email from the outs @ IN A 10.11.12.13 mail IN A 10.11.12.13 - ; mail-server for example.com + ; mail server for example.com @ IN MX 10 mail.example.com. ; Add SPF record @@ -182,7 +182,7 @@ In this setup `docker-mailserver` is not intended to receive email from the outs 4. Get an SSL certificate, [we have a guide for you here][docs-ssl] (_Let's Encrypt_ is a popular service to get free SSL certificates). -5. Start `docker-mailserver` and check the terminal output for any errors: `docker-compose up`. +5. Start DMS and check the terminal output for any errors: `docker-compose up`. 6. Create email accounts and aliases: diff --git a/docs/content/examples/tutorials/blog-posts.md b/docs/content/examples/tutorials/blog-posts.md index 525b778e0fb..7da817a25b6 100644 --- a/docs/content/examples/tutorials/blog-posts.md +++ b/docs/content/examples/tutorials/blog-posts.md @@ -2,7 +2,7 @@ title: 'Tutorials | Blog Posts' --- -This site lists blog entries that write about the project. If you blogged about `docker-mailserver` let us know so we can add it here! +This site lists blog entries that write about the project. If you blogged about DMS let us know so we can add it here! - [Installing docker-mailserver](https://lowtek.ca/roo/2021/installing-docker-mailserver/) by [@andrewlow](https://github.com/andrewlow) - [Self hosted mail-server](https://www.ifthenel.se/self-hosted-mail-server/) by [@matrixes](https://github.com/matrixes) diff --git a/docs/content/examples/tutorials/mailserver-behind-proxy.md b/docs/content/examples/tutorials/mailserver-behind-proxy.md index 208304ae2fc..f9050106597 100644 --- a/docs/content/examples/tutorials/mailserver-behind-proxy.md +++ b/docs/content/examples/tutorials/mailserver-behind-proxy.md @@ -1,8 +1,8 @@ --- -title: 'Tutorials | Mail-Server behind a Proxy' +title: 'Tutorials | Mail Server behind a Proxy' --- -## Using `docker-mailserver` behind a Proxy +## Using DMS behind a Proxy ### Information @@ -53,7 +53,7 @@ Feel free to add your configuration if you achieved the same goal using differen [...] ``` - Truncated list of necessary labels on the `docker-mailserver` container: + Truncated list of necessary labels on the DMS container: ```yaml services: diff --git a/docs/content/examples/use-cases/forward-only-mailserver-with-ldap-authentication.md b/docs/content/examples/use-cases/forward-only-mailserver-with-ldap-authentication.md index acb07fffde9..dc930f13887 100644 --- a/docs/content/examples/use-cases/forward-only-mailserver-with-ldap-authentication.md +++ b/docs/content/examples/use-cases/forward-only-mailserver-with-ldap-authentication.md @@ -1,10 +1,10 @@ --- -title: 'Use Cases | Forward-Only Mail-Server with LDAP' +title: 'Use Cases | Forward-Only Mail Server with LDAP' --- -## Building a Forward-Only Mail-Server +## Building a Forward-Only Mail Server -A **forward-only** mail-server does not have any local mailboxes. Instead, it has only aliases that forward emails to external email accounts (for example to a Gmail account). You can also send email from the localhost (the computer where `docker-mailserver` is installed), using as sender any of the alias addresses. +A **forward-only** mail server does not have any local mailboxes. Instead, it has only aliases that forward emails to external email accounts (for example to a Gmail account). You can also send email from the localhost (the computer where DMS is installed), using as sender any of the alias addresses. The important settings for this setup (on `mailserver.env`) are these: @@ -27,7 +27,7 @@ We can create aliases with `./setup.sh`, like this: ## Authenticating with LDAP -If you want to send emails from outside the mail-server you have to authenticate somehow (with a username and password). One way of doing it is described in [this discussion][github-issue-1247]. However if there are many user accounts, it is better to use authentication with LDAP. The settings for this on `mailserver.env` are: +If you want to send emails from outside the mail server you have to authenticate somehow (with a username and password). One way of doing it is described in [this discussion][github-issue-1247]. However if there are many user accounts, it is better to use authentication with LDAP. The settings for this on `mailserver.env` are: ```env ENABLE_LDAP=1 # with the :edge tag, use ACCOUNT_PROVISIONER @@ -60,7 +60,7 @@ userPassword: {SSHA}abcdefghi123456789 email: external-account@gmail.com ``` -This structure is different from what is expected/assumed from the configuration scripts of `docker-mailserver`, so it doesn't work just by using the `LDAP_QUERY_FILTER_...` settings. Instead, I had to use a custom configuration ([via `user-patches.sh`][docs-userpatches]). I created the script `docker-data/dms/config/user-patches.sh`, with content like this: +This structure is different from what is expected/assumed from the configuration scripts of DMS, so it doesn't work just by using the `LDAP_QUERY_FILTER_...` settings. Instead, I had to use a custom configuration ([via `user-patches.sh`][docs-userpatches]). I created the script `docker-data/dms/config/user-patches.sh`, with content like this: ```bash #!/bin/bash @@ -99,11 +99,11 @@ You see that besides `query_filter`, I had to customize as well `result_attribut !!! note "See also" - For more details about using LDAP see: [LDAP managed mail-server with Postfix and Dovecot for multiple domains](https://www.vennedey.net/resources/2-LDAP-managed-mail-server-with-Postfix-and-Dovecot-for-multiple-domains) + For more details about using LDAP see: [LDAP managed mail server with Postfix and Dovecot for multiple domains](https://www.vennedey.net/resources/2-LDAP-managed-mail-server-with-Postfix-and-Dovecot-for-multiple-domains) !!! note - Another solution that serves as a forward-only mail-server is [this](https://gitlab.com/docker-scripts/postfix). + Another solution that serves as a forward-only mail server is [this](https://gitlab.com/docker-scripts/postfix). [docs-userpatches]: ../../config/advanced/override-defaults/user-patches.md [github-issue-1247]: https://github.com/docker-mailserver/docker-mailserver/issues/1247 diff --git a/docs/content/faq.md b/docs/content/faq.md index 7b81bc08820..d72ea5ceeb8 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -57,7 +57,7 @@ As you'll realistically be deploying to production on a Linux host, if you are o ### How to alter a running DMS instance _without_ relaunching the container? -`docker-mailserver` aggregates multiple "sub-services", such as Postfix, Dovecot, Fail2ban, SpamAssassin, etc. In many cases, one may edit a sub-service's config and reload that very sub-service, without stopping and relaunching the whole mail-server. +DMS aggregates multiple "sub-services", such as Postfix, Dovecot, Fail2ban, SpamAssassin, etc. In many cases, one may edit a sub-service's config and reload that very sub-service, without stopping and relaunching the whole mail server. In order to do so, you'll probably want to push your config updates to your server through a Docker volume (these docs use: `./docker-data/dms/config/:/tmp/docker-mailserver/`), then restart the sub-service to apply your changes, using `supervisorctl`. For instance, after editing fail2ban's config: `supervisorctl restart fail2ban`. @@ -181,7 +181,7 @@ Also you need to define `hostname: example.com` in your `docker-compose.yml`. !!! tip "You might not want a bare domain" We encourage you to consider using a subdomain where possible. - + - There are [benefits][github-comment-baredomain] to preferring a subdomain. - A bare domain is not required to have `user@example.com`, that is distinct from your hostname which is identified by a DNS MX record. @@ -296,7 +296,7 @@ proxy_interfaces = X.X.X.X (your public IP) Suppose you want to change a number of settings that are not listed as variables or add things to the server that are not included? -`docker-mailserver` has a built-in way to do post-install processes. If you place a script called **`user-patches.sh`** in the config directory it will be run after all configuration files are set up, but before the postfix, amavis and other daemons are started. +DMS has a built-in way to do post-install processes. If you place a script called **`user-patches.sh`** in the config directory it will be run after all configuration files are set up, but before the postfix, amavis and other daemons are started. It is common to use a local directory for config added to `docker-mailsever` via a volume mount in your `docker-compose.yml` (eg: `./docker-data/dms/config/:/tmp/docker-mailserver/`). @@ -405,7 +405,7 @@ Put received spams in `.Junk/` imap folder using `SPAMASSASSIN_SPAM_TO_INBOX=1` 0 2 * * * docker exec mailserver sa-learn --spam /var/mail/example.com/username/.Junk --dbpath /var/mail-state/lib-amavis/.spamassassin ``` -With `docker-compose` you can more easily use the internal instance of `cron` within `docker-mailserver`. This is less problematic than the simple solution shown above, because it decouples the learning from the host on which `docker-mailserver` is running, and avoids errors if the mail-server is not running. +With `docker-compose` you can more easily use the internal instance of `cron` within DMS. This is less problematic than the simple solution shown above, because it decouples the learning from the host on which DMS is running, and avoids errors if the mail server is not running. The following configuration works nicely: diff --git a/docs/content/index.md b/docs/content/index.md index 10d099ceec7..51be4fb8585 100644 --- a/docs/content/index.md +++ b/docs/content/index.md @@ -32,11 +32,11 @@ There is also a script - [`setup.sh`][github-file-setupsh] - supplied with this ### Configuration -We have a [dedicated configuration page][docs-environment]. It contains most of the configuration and explanation you need to setup _your_ mail server properly. Be aware that advanced tasks may still require reading through all parts of this documentation; it may also involve inspecting your running container for debugging purposes. After all, a mail-server is a complex arrangement of various programs. +We have a [dedicated configuration page][docs-environment]. It contains most of the configuration and explanation you need to setup _your_ mail server properly. Be aware that advanced tasks may still require reading through all parts of this documentation; it may also involve inspecting your running container for debugging purposes. After all, a mail server is a complex arrangement of various programs. !!! important - If you'd like to change, patch or alter files or behavior of `docker-mailserver`, you can use a script. Just place a script called `user-patches.sh` in your `./docker-data/dms/config/` folder volume (which is mounted to `/tmp/docker-mailserver/` inside the container) and it will be run on container startup. See the ['Modifications via Script' page][docs-userpatches] for additional documentation and an example. + If you'd like to change, patch or alter files or behavior of DMS, you can use a script. Just place a script called `user-patches.sh` in your `./docker-data/dms/config/` folder volume (which is mounted to `/tmp/docker-mailserver/` inside the container) and it will be run on container startup. See the ['Modifications via Script' page][docs-userpatches] for additional documentation and an example. You might also want to check out: diff --git a/docs/content/introduction.md b/docs/content/introduction.md index f136b795575..a92a86d9b62 100644 --- a/docs/content/introduction.md +++ b/docs/content/introduction.md @@ -4,17 +4,17 @@ title: Introduction # An Overview of Mail Server Infrastructure -This article answers the question "What is a mail server, and how does it perform its duty?" and it gives the reader an introduction to the field that covers everything you need to know to get started with `docker-mailserver`. +This article answers the question "What is a mail server, and how does it perform its duty?" and it gives the reader an introduction to the field that covers everything you need to know to get started with DMS. ## The Anatomy of a Mail Server A mail server is only a part of a [client-server relationship][wikipedia-clientserver] aimed at exchanging information in the form of [emails][wikipedia-email]. Exchanging emails requires using specific means (programs and protocols). -`docker-mailserver` provides you with the server portion, whereas the client can be anything from a terminal via text-based software (eg. [Mutt][software-mutt]) to a fully-fledged desktop application (eg. [Mozilla Thunderbird][software-thunderbird], [Microsoft Outlook][software-outlook]…), to a web interface, etc. +DMS provides you with the server portion, whereas the client can be anything from a terminal via text-based software (eg. [Mutt][software-mutt]) to a fully-fledged desktop application (eg. [Mozilla Thunderbird][software-thunderbird], [Microsoft Outlook][software-outlook]…), to a web interface, etc. Unlike the client-side where usually a single program is used to perform retrieval and viewing of emails, the server-side is composed of many specialized components. The mail server is capable of accepting, forwarding, delivering, storing and overall exchanging messages, but each one of those tasks is actually handled by a specific piece of software. All of these "agents" must be integrated with one another for the exchange to take place. -`docker-mailserver` has made informed choices about those components and their (default) configuration. It offers a comprehensive platform to run a fully featured mail server in no time! +DMS has made informed choices about those components and their (default) configuration. It offers a comprehensive platform to run a fully featured mail server in no time! ## Components @@ -33,13 +33,13 @@ Fetching an email: MUA <--------------------------------- MDA There may be other moving parts or sub-divisions (for instance, at several points along the chain, specialized programs may be analyzing, filtering, bouncing, editing… the exchanged emails). -In a nutshell, `docker-mailserver` provides you with the following components: +In a nutshell, DMS provides you with the following components: - A MTA: [Postfix](http://www.postfix.org/) - A MDA: [Dovecot](https://dovecot.org/) - A bunch of additional programs to improve security and emails processing -Here's where `docker-mailserver`'s toochain fits within the delivery chain: +Here's where DMS's toochain fits within the delivery chain: ```txt docker-mailserver is here: @@ -51,21 +51,21 @@ Fetching an email: MUA <------------------------------ ┫ MDA ╯ ┃ ??? example "An Example" - Let's say Alice owns a Gmail account, `alice@gmail.com`; and Bob owns an account on a `docker-mailserver`'s instance, `bob@dms.io`. + Let's say Alice owns a Gmail account, `alice@gmail.com`; and Bob owns an account on a DMS instance, `bob@dms.io`. Make sure not to conflate these two very different scenarios: A) Alice sends an email to `bob@dms.io` => the email is first submitted to MTA `smtp.gmail.com`, then relayed to MTA `smtp.dms.io` where it is then delivered into Bob's mailbox. B) Bob sends an email to `alice@gmail.com` => the email is first submitted to MTA `smtp.dms.io`, then relayed to MTA `smtp.gmail.com` and eventually delivered into Alice's mailbox. - In scenario *A* the email leaves Gmail's premises, that email's *initial* submission is _not_ handled by your `docker-mailserver` instance(MTA); it merely receives the email after it has been relayed by Gmail's MTA. In scenario *B*, the `docker-mailserver` instance(MTA) handles the submission, prior to relaying. + In scenario *A* the email leaves Gmail's premises, that email's *initial* submission is _not_ handled by your DMS instance(MTA); it merely receives the email after it has been relayed by Gmail's MTA. In scenario *B*, the DMS instance(MTA) handles the submission, prior to relaying. - The main takeaway is that when a third-party sends an email to a `docker-mailserver` instance(MTA) (or any MTA for that matter), it does _not_ establish a direct connection with that MTA. Email submission first goes through the sender's MTA, then some relaying between at least two MTAs is required to deliver the email. That will prove very important when it comes to security management. + The main takeaway is that when a third-party sends an email to a DMS instance(MTA) (or any MTA for that matter), it does _not_ establish a direct connection with that MTA. Email submission first goes through the sender's MTA, then some relaying between at least two MTAs is required to deliver the email. That will prove very important when it comes to security management. -One important thing to note is that MTA and MDA programs may actually handle _multiple_ tasks (which is the case with `docker-mailserver`'s Postfix and Dovecot). +One important thing to note is that MTA and MDA programs may actually handle _multiple_ tasks (which is the case with DMS's Postfix and Dovecot). For instance, Postfix is both an SMTP server (accepting emails) and a relaying MTA (transferring, ie. sending emails to other MTA/MDA); Dovecot is both an MDA (delivering emails in mailboxes) and an IMAP server (allowing MUAs to fetch emails from the _mail server_). On top of that, Postfix may rely on Dovecot's authentication capabilities. -The exact relationship between all the components and their respective (sometimes shared) responsibilities is beyond the scope of this document. Please explore this wiki & the web to get more insights about `docker-mailserver`'s toolchain. +The exact relationship between all the components and their respective (sometimes shared) responsibilities is beyond the scope of this document. Please explore this wiki & the web to get more insights about DMS's toolchain. ## About Security & Ports @@ -101,20 +101,20 @@ MUA <---- STARTTLS ------- ┤(143) MDA ╯ | ``` If you're new to email infrastructure, both that table and the schema may be confusing. -Read on to expand your understanding and learn about `docker-mailserver`'s configuration, including how you can customize it. +Read on to expand your understanding and learn about DMS's configuration, including how you can customize it. ### Submission - SMTP For a MUA to send an email to an MTA, it needs to establish a connection with that server, then push data packets over a network that both the MUA (client) and the MTA (server) are connected to. The server implements the [SMTP][wikipedia-smtp] protocol, which makes it capable of handling _Submission_. -In the case of `docker-mailserver`, the MTA (SMTP server) is Postfix. The MUA (client) may vary, yet its Submission request is performed as [TCP][wikipedia-tcp] packets sent over the _public_ internet. This exchange of information may be secured in order to counter eavesdropping. +In the case of DMS, the MTA (SMTP server) is Postfix. The MUA (client) may vary, yet its Submission request is performed as [TCP][wikipedia-tcp] packets sent over the _public_ internet. This exchange of information may be secured in order to counter eavesdropping. -Now let's say I own an account on a `docker-mailserver` instance, `me@dms.io`. There are two very different use-cases for Submission: +Now let's say I own an account on a DMS instance, `me@dms.io`. There are two very different use-cases for Submission: 1. I want to send an email to someone 2. Someone wants to send you an email -In the first scenario, I will be submitting my email directly to my `docker-mailserver` instance/MTA (Postfix), which will then relay the email to its recipient's MTA for final delivery. In this case, Submission is first handled by establishing a direct connection to my own MTA-so at least for this portion of the delivery chain, I'll be able to ensure security/confidentiality. Not so much for what comes next, ie. relaying between MTAs and final delivery. +In the first scenario, I will be submitting my email directly to my DMS instance/MTA (Postfix), which will then relay the email to its recipient's MTA for final delivery. In this case, Submission is first handled by establishing a direct connection to my own MTA-so at least for this portion of the delivery chain, I'll be able to ensure security/confidentiality. Not so much for what comes next, ie. relaying between MTAs and final delivery. In the second scenario, a third-party email account owner will be first submitting an email to some third-party MTA. I have no control over this initial portion of the delivery chain, nor do I have control over the relaying that comes next. My MTA will merely accept a relayed email coming "out of the blue". @@ -145,15 +145,15 @@ When it comes to securing Outbound Submission you should prefer to use _Implicit Although a very satisfactory setup, Implicit TLS on port 465 is somewhat "cutting edge". There exists another well established mail Submission setup that must be supported as well, SMTP+STARTTLS on port 587. It uses Explicit TLS: the client starts with a cleartext connection, then the server informs a TLS-encrypted "upgraded" connection may be established, and the client _may_ eventually decide to establish it prior to the Submission. Basically it's an opportunistic, opt-in TLS upgrade of the connection between the client and the server, at the client's discretion, using a mechanism known as [STARTTLS][wikipedia-starttls] that both ends need to implement. -In many implementations, the mail server doesn't enforce TLS encryption, for backwards compatibility. Clients are thus free to deny the TLS-upgrade proposal (or [misled by a hacker](https://security.stackexchange.com/questions/168998/what-happens-if-starttls-dropped-in-smtp) about STARTTLS not being available), and the server accepts unencrypted (cleartext) mail exchange, which poses a confidentiality threat and, to some extent, spam issues. [RFC 8314 (section 3.3)][rfc-8314-s33] recommends for a mail server to support both Implicit and Explicit TLS for Submission, _and_ to enforce TLS-encryption on ports 587 (Explicit TLS) and 465 (Implicit TLS). That's exactly `docker-mailserver`'s default configuration: abiding by RFC 8314, it [enforces a strict (`encrypt`) STARTTLS policy](http://www.postfix.org/postconf.5.html#smtpd_tls_security_level), where a denied TLS upgrade terminates the connection thus (hopefully but at the client's discretion) preventing unencrypted (cleartext) Submission. +In many implementations, the mail server doesn't enforce TLS encryption, for backwards compatibility. Clients are thus free to deny the TLS-upgrade proposal (or [misled by a hacker](https://security.stackexchange.com/questions/168998/what-happens-if-starttls-dropped-in-smtp) about STARTTLS not being available), and the server accepts unencrypted (cleartext) mail exchange, which poses a confidentiality threat and, to some extent, spam issues. [RFC 8314 (section 3.3)][rfc-8314-s33] recommends for a mail server to support both Implicit and Explicit TLS for Submission, _and_ to enforce TLS-encryption on ports 587 (Explicit TLS) and 465 (Implicit TLS). That's exactly DMS's default configuration: abiding by RFC 8314, it [enforces a strict (`encrypt`) STARTTLS policy](http://www.postfix.org/postconf.5.html#smtpd_tls_security_level), where a denied TLS upgrade terminates the connection thus (hopefully but at the client's discretion) preventing unencrypted (cleartext) Submission. -- **`docker-mailserver`'s default configuration enables and _requires_ Explicit TLS (STARTTLS) on port 587 for Outbound Submission.** +- **DMS's default configuration enables and _requires_ Explicit TLS (STARTTLS) on port 587 for Outbound Submission.** - It does not enable Implicit TLS Outbound Submission on port 465 by default. One may enable it through simple custom configuration, either as a replacement or (better!) supplementary mean of secure Submission. - It does not support old MUAs (clients) not supporting TLS encryption on ports 587/465 (those should perform Submission on port 25, more details below). One may relax that constraint through advanced custom configuration, for backwards compatibility. -A final Outbound Submission setup exists and is akin SMTP+STARTTLS on port 587, but on port 25. That port has historically been reserved specifically for unencrypted (cleartext) mail exchange though, making STARTTLS a bit wrong to use. As is expected by [RFC 5321][rfc-5321], `docker-mailserver` uses port 25 for unencrypted Submission in order to support older clients, but most importantly for unencrypted Transfer/Relay between MTAs. +A final Outbound Submission setup exists and is akin SMTP+STARTTLS on port 587, but on port 25. That port has historically been reserved specifically for unencrypted (cleartext) mail exchange though, making STARTTLS a bit wrong to use. As is expected by [RFC 5321][rfc-5321], DMS uses port 25 for unencrypted Submission in order to support older clients, but most importantly for unencrypted Transfer/Relay between MTAs. -- **`docker-mailserver`'s default configuration also enables unencrypted (cleartext) on port 25 for Outbound Submission.** +- **DMS's default configuration also enables unencrypted (cleartext) on port 25 for Outbound Submission.** - It does not enable Explicit TLS (STARTTLS) on port 25 by default. One may enable it through advanced custom configuration, either as a replacement (bad!) or as a supplementary mean of secure Outbound Submission. - One may also secure Outbound Submission using advanced encryption scheme, such as DANE/DNSSEC and/or MTA-STS. @@ -161,11 +161,11 @@ A final Outbound Submission setup exists and is akin SMTP+STARTTLS on port 587, Granted it's still very difficult enforcing encryption between MTAs (Transfer/Relay) without risking dropping emails (when relayed by MTAs not supporting TLS-encryption), Inbound Submission is to be handled in cleartext on port 25 by default. -- **`docker-mailserver`'s default configuration enables unencrypted (cleartext) on port 25 for Inbound Submission.** +- **DMS's default configuration enables unencrypted (cleartext) on port 25 for Inbound Submission.** - It does not enable Explicit TLS (STARTTLS) on port 25 by default. One may enable it through advanced custom configuration, either as a replacement (bad!) or as a supplementary mean of secure Inbound Submission. - One may also secure Inbound Submission using advanced encryption scheme, such as DANE/DNSSEC and/or MTA-STS. -Overall, `docker-mailserver`'s default configuration for SMTP looks like this: +Overall, DMS's default configuration for SMTP looks like this: ```txt ┏━━━━ Outbound Submission ━━━━┓ @@ -184,13 +184,13 @@ Me -- STARTTLS ---> ┤(587) │ ┊ A MUA willing to fetch an email from a mail server will most likely communicate with its [IMAP][wikipedia-imap] server. As with SMTP described earlier, communication will take place in the form of data packets exchanged over a network that both the client and the server are connected to. The IMAP protocol makes the server capable of handling _Retrieval_. -In the case of `docker-mailserver`, the IMAP server is Dovecot. The MUA (client) may vary, yet its Retrieval request is performed as [TCP][wikipedia-tcp] packets sent over the _public_ internet. This exchange of information may be secured in order to counter eavesdropping. +In the case of DMS, the IMAP server is Dovecot. The MUA (client) may vary, yet its Retrieval request is performed as [TCP][wikipedia-tcp] packets sent over the _public_ internet. This exchange of information may be secured in order to counter eavesdropping. Again, as with SMTP described earlier, the IMAP protocol may be secured with either Implicit TLS (aka. [IMAPS][wikipedia-imaps] / IMAP4S) or Explicit TLS (using STARTTLS). The best practice as of 2020 is to enforce IMAPS on port 993, rather than IMAP+STARTTLS on port 143 (see [RFC 8314][rfc-8314]); yet the latter is usually provided for backwards compatibility. -**`docker-mailserver`'s default configuration enables both Implicit and Explicit TLS for Retrievial, on ports 993 and 143 respectively.** +**DMS's default configuration enables both Implicit and Explicit TLS for Retrievial, on ports 993 and 143 respectively.** ### Retrieval - POP3 @@ -198,14 +198,14 @@ Similarly to IMAP, the older POP3 protocol may be secured with either Implicit o The best practice as of 2020 would be [POP3S][wikipedia-pop3s] on port 995, rather than [POP3][wikipedia-pop3]+STARTTLS on port 110 (see [RFC 8314][rfc-8314]). -**`docker-mailserver`'s default configuration disables POP3 altogether.** One should expect MUAs to use TLS-encrypted IMAP for Retrieval. +**DMS's default configuration disables POP3 altogether.** One should expect MUAs to use TLS-encrypted IMAP for Retrieval. -## How Does `docker-mailserver` Help With Setting Everything Up? +## How Does DMS Help With Setting Everything Up? -As a _batteries included_ container image, `docker-mailserver` provides you with all the required components and a default configuration to run a decent and secure mail server. One may then customize all aspects of its internal components. +As a _batteries included_ container image, DMS provides you with all the required components and a default configuration to run a decent and secure mail server. One may then customize all aspects of its internal components. - Simple customization is supported through [docker-compose configuration][github-file-compose] and the [env-mailserver][github-file-envmailserver] configuration file. -- Advanced customization is supported through providing "monkey-patching" configuration files and/or [deriving your own image][github-file-dockerfile] from `docker-mailserver`'s upstream, for a complete control over how things run. +- Advanced customization is supported through providing "monkey-patching" configuration files and/or [deriving your own image][github-file-dockerfile] from DMS's upstream, for a complete control over how things run. Eventually, it is up to _you_ deciding exactly what kind of transportation/encryption to use and/or enforce, and to customize your instance accordingly (with looser or stricter security). Be also aware that protocols and ports on your server can only go so far with security; third-party MTAs might relay your emails on insecure connections, man-in-the-middle attacks might still prove effective, etc. Advanced counter-measure such as DANE, MTA-STS and/or full body encryption (eg. PGP) should be considered as well for increased confidentiality, but ideally without compromising backwards compatibility so as to not block emails. diff --git a/docs/content/usage.md b/docs/content/usage.md index 4d2dfedaeb3..81a08c716ec 100644 --- a/docs/content/usage.md +++ b/docs/content/usage.md @@ -168,7 +168,7 @@ You will very likely want to configure your DNS with these TXT records: [SPF, DK ### Custom User Changes & Patches -If you'd like to change, patch or alter files or behavior of `docker-mailserver`, you can use a script. See [this part of our documentation][docs-user-patches] for a detailed explanation. +If you'd like to change, patch or alter files or behavior of DMS, you can use a script. See [this part of our documentation][docs-user-patches] for a detailed explanation. [docs-user-patches]: ./faq.md#how-to-adjust-settings-with-the-user-patchessh-script From bbe1d2da3117264c484a87c75813b0d3da71161a Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 10 Apr 2023 16:00:24 +0200 Subject: [PATCH 029/592] docs: add note about Rspamd web interface (#3245) --- docs/content/config/security/rspamd.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 4dcaf3deacc..20871c9551a 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -41,6 +41,14 @@ When Rspamd is enabled, we implicitly also start an instance of Redis in the con Redis uses `/etc/redis/redis.conf` for configuration. We adjust this file when enabling the internal Redis service. If you have an external instance of Redis to use, the internal Redis service can be opt-out via setting the ENV [`ENABLE_RSPAMD_REDIS=0`](../environment.md#enable_rspamd_redis) (_link also details required changes to the DMS rspamd config_). +### Web Interface + +Rspamd provides a [web interface][rspamc-docs-web-interface], which contains statistics and data Rspamd collects. The interface is enabled by default and reachable on port 11334. + +![Rspamd Web Interface](https://rspamd.com/img/webui.png) + +[rspamc-docs-web-interface]: https://rspamd.com/webui/ + ### Modules You can find a list of all Rspamd modules [on their website][modules]. From 9ee33a81b7e78b3f1164ff153b226f00b571eb1b Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 11 Apr 2023 08:52:43 +0200 Subject: [PATCH 030/592] scripts: make `policyd-spf` configurable (#3246) --- docs/content/config/environment.md | 7 +++++++ mailserver.env | 8 ++++++++ target/postfix/master.cf | 3 --- target/scripts/start-mailserver.sh | 1 + .../scripts/startup/setup.d/dmarc_dkim_spf.sh | 18 ++++++++++++++++++ target/scripts/startup/variables-stack.sh | 1 + 6 files changed, 35 insertions(+), 3 deletions(-) diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index f0b548b2233..8c2d108c8aa 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -110,6 +110,13 @@ Enables the OpenDMARC service. - **1** => Enabled - 0 => Disabled +##### ENABLE_POLICYD_SPF + +Enabled `policyd-spf` in Postfix's configuration. You will likely want to set this to `0` in case you're using Rspamd ([`ENABLE_RSPAMD=1`](#enable_rspamd)). + +- 0 => Disabled +- **1** => Enabled + ##### ENABLE_POP3 - **empty** => POP3 service disabled diff --git a/mailserver.env b/mailserver.env index 09a40b8e83c..020d405068b 100644 --- a/mailserver.env +++ b/mailserver.env @@ -105,6 +105,14 @@ ENABLE_OPENDKIM=1 # 0 => Disabled ENABLE_OPENDMARC=1 + +# Enabled `policyd-spf` in Postfix's configuration. You will likely want to set this +# to `0` in case you're using Rspamd (`ENABLE_RSPAMD=1`). +# +# - 0 => Disabled +# - **1** => Enabled +ENABLE_POLICYD_SPF=1 + # 1 => Enables POP3 service # empty => disables POP3 ENABLE_POP3= diff --git a/target/postfix/master.cf b/target/postfix/master.cf index beb3b216c46..3746b6f6bfe 100644 --- a/target/postfix/master.cf +++ b/target/postfix/master.cf @@ -48,9 +48,6 @@ sender-cleanup unix n - n - 0 cleanup -o syslog_name=postfix/sender-cleanup -o header_checks=pcre:/etc/postfix/maps/sender_header_filter.pcre -policyd-spf unix - n n - 0 spawn - user=policyd-spf argv=/usr/bin/policyd-spf - cleanup unix n - n - 0 cleanup qmgr unix n - n 300 1 qmgr tlsmgr unix - - n 1000? 1 tlsmgr diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index 83209267c96..ff550dee527 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -80,6 +80,7 @@ function _register_functions _register_setup_function '_setup_opendkim' _register_setup_function '_setup_opendmarc' # must come after `_setup_opendkim` + _register_setup_function '_setup_policyd_spf' _register_setup_function '_setup_security_stack' _register_setup_function '_setup_spam_to_junk' diff --git a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh index f541eeaa832..c27e5258d4d 100644 --- a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh +++ b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh @@ -84,3 +84,21 @@ function _setup_opendmarc _log 'debug' 'Disabling OpenDMARC' fi } + +# Configures the SPF check inside Postfix's configuration via policyd-spf. When +# using Rspamd, you will likely want to turn that off. +function _setup_policyd_spf +{ + if [[ ${ENABLE_POLICYD_SPF} -eq 1 ]] + then + _log 'debug' 'Configuring policyd-spf' + cat >>/etc/postfix/master.cf < Date: Tue, 11 Apr 2023 09:16:57 +0200 Subject: [PATCH 031/592] Rspamd: add greylisting option & code refactoring (#3206) --- docs/content/config/environment.md | 14 +- docs/content/config/security/rspamd.md | 15 +- mailserver.env | 17 ++ target/rspamd/local.d/greylist.conf | 3 + target/scripts/start-mailserver.sh | 5 +- .../startup/setup.d/security/rspamd.sh | 241 ++++++++++-------- target/scripts/startup/variables-stack.sh | 1 + .../parallel/set1/spam_virus/rspamd.bats | 9 + 8 files changed, 187 insertions(+), 118 deletions(-) create mode 100644 target/rspamd/local.d/greylist.conf diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 8c2d108c8aa..e8168062343 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -310,7 +310,7 @@ Enable or disable Rspamd. !!! warning "Current State" - Rspamd-support is under active development. Be aware that breaking changes can happen at any time. To get more information, see [the detailed documentation page for Rspamd][docs-rspamd]. + Rspamd-support is under active development. Be aware that changes can happen at any time. To get more information, see [the detailed documentation page for Rspamd][docs-rspamd]. - **0** => disabled - 1 => enabled @@ -335,6 +335,16 @@ The purpose of this setting is to opt-out of starting an internal Redis instance - 0 => Disabled - 1 => Enabled +##### RSPAMD_GREYLISTING + +Controls whether the [Rspamd Greylisting module][rspamd-greylisting-module] is enabled. This module can further assist in avoiding spam emails by [greylisting] e-mails with a certain spam score. + +- **0** => Disabled +- 1 => Enabled + +[rspamd-greylisting-module]: https://rspamd.com/doc/modules/greylisting.html +[greylisting]: https://en.wikipedia.org/wiki/Greylisting_(email) + ##### RSPAMD_LEARN When enabled, @@ -342,7 +352,7 @@ When enabled, 1. the "[autolearning][rspamd-autolearn]" feature is turned on; 2. the Bayes classifier will be trained when moving mails from or to the Junk folder (with the help of Sieve scripts). -!!! attention +!!! warning "Attention" As of now, the spam learning database is global (i.e. available to all users). If one user deliberately trains it with malicious data, then it will ruin your detection rate. diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 20871c9551a..5f70fe73894 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -4,7 +4,7 @@ title: 'Security | Rspamd' !!! warning "The current state of Rspamd integration into DMS" - Recent pull requests have stabilized integration of Rspamd to a point that we encourage users to test the feature. We are confident that there are no major bugs in our integration that make using Rspamd infeasible. Please note that there may still be (breaking) changes ahead as integration is still work in progress! + Recent pull requests have stabilized integration of Rspamd to a point that we encourage users to test the feature. We are confident that there are no major bugs in our integration that make using Rspamd infeasible. Please note that there may still be changes ahead as integration is still work in progress! We expect to stabilize this feature with version `v12.1.0`. @@ -16,16 +16,19 @@ If you want to have a look at the default configuration files for Rspamd that DM !!! note "AMD64 vs ARM64" - We are currently doing a best-effort installation of Rspamd for ARM64 (from the Debian backports repository for Debian 11). The current version difference is two minor versions (AMD64 is at version 3.4, ARM64 at 3.2 \[13th Feb 2023\]). + We are currently doing a best-effort installation of Rspamd for ARM64 (from the Debian backports repository for Debian 11). The current version difference as of 1st Apr 2023: AMD64 is at version 3.5 | ARM64 is at version 3.4. - Maintainers noticed only few differences, some of them with a big impact though. For those running Rspamd on ARM64, we recommend [disabling](#with-the-help-of-a-custom-file) the [DKIM signing module][dkim-signing-module] if you don't use it. +## Related Environment Variables The following environment variables are related to Rspamd: 1. [`ENABLE_RSPAMD`](../environment.md#enable_rspamd) 2. [`ENABLE_RSPAMD_REDIS`](../environment.md#enable_rspamd_redis) -3. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) -4. [`MOVE_SPAM_TO_JUNK`](../environment.md#move_spam_to_junk) +3. [`RSPAMD_GREYLISTING`](../environment.md#rspamd_greylisting) +4. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) +5. [`MOVE_SPAM_TO_JUNK`](../environment.md#move_spam_to_junk) + +With these variables, you can enable Rspamd itself and you can enable / disable certain features related to Rspamd. ## The Default Configuration @@ -55,7 +58,7 @@ You can find a list of all Rspamd modules [on their website][modules]. #### Disabled By Default -DMS disables certain modules (clickhouse, elastic, greylist, neural, reputation, spamassassin, url_redirector, metric_exporter) by default. We believe these are not required in a standard setup, and they would otherwise needlessly use system resources. +DMS disables certain modules (clickhouse, elastic, neural, reputation, spamassassin, url_redirector, metric_exporter) by default. We believe these are not required in a standard setup, and they would otherwise needlessly use system resources. #### Anti-Virus (ClamAV) diff --git a/mailserver.env b/mailserver.env index 020d405068b..de8a7653e67 100644 --- a/mailserver.env +++ b/mailserver.env @@ -133,6 +133,23 @@ ENABLE_RSPAMD=0 # 1 => Enabled ENABLE_RSPAMD_REDIS= +# When enabled, +# +# 1. the "[autolearning][rspamd-autolearn]" feature is turned on; +# 2. the Bayes classifier will be trained when moving mails from or to the Junk folder (with the help of Sieve scripts). +# +# **0** => disabled +# 1 => enabled +RSPAMD_LEARN=0 + +# Controls whether the Rspamd Greylisting module is enabled. +# This module can further assist in avoiding spam emails by greylisting +# e-mails with a certain spam score. +# +# **0** => disabled +# 1 => enabled +RSPAMD_GREYLISTING=0 + # Amavis content filter (used for ClamAV & SpamAssassin) # 0 => Disabled # 1 => Enabled diff --git a/target/rspamd/local.d/greylist.conf b/target/rspamd/local.d/greylist.conf new file mode 100644 index 00000000000..4098ee8f370 --- /dev/null +++ b/target/rspamd/local.d/greylist.conf @@ -0,0 +1,3 @@ +# documentation: https://www.rspamd.com/doc/modules/greylisting.html + +enabled = false; diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index ff550dee527..039c8f743d9 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -130,9 +130,10 @@ function _register_functions [[ ${SMTP_ONLY} -ne 1 ]] && _register_start_daemon '_start_daemon_dovecot' [[ ${ENABLE_UPDATE_CHECK} -eq 1 ]] && _register_start_daemon '_start_daemon_update_check' - [[ ${ENABLE_RSPAMD} -eq 1 ]] && _register_start_daemon '_start_daemon_rspamd' + + # The order here matters: Since Rspamd is using Redis, Redis should be started before Rspamd. [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] && _register_start_daemon '_start_daemon_rspamd_redis' - [[ ${ENABLE_UPDATE_CHECK} -eq 1 ]] && _register_start_daemon '_start_daemon_update_check' + [[ ${ENABLE_RSPAMD} -eq 1 ]] && _register_start_daemon '_start_daemon_rspamd' # needs to be started before SASLauthd [[ ${ENABLE_OPENDKIM} -eq 1 ]] && _register_start_daemon '_start_daemon_opendkim' diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 8b478904b88..ba292346bf6 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -1,24 +1,23 @@ #!/bin/bash +# Function called during global setup to handle the complete setup of Rspamd. function _setup_rspamd { if [[ ${ENABLE_RSPAMD} -eq 1 ]] then - _log 'warn' 'Rspamd integration is work in progress - expect (breaking) changes at any time' + _log 'warn' 'Rspamd integration is work in progress - expect changes at any time' _log 'debug' 'Enabling and configuring Rspamd' - __rspamd__preflight_checks_and_setup - __rspamd__adjust_postfix_configuration - __rspamd__disable_default_modules - __rspamd__handle_modules_configuration + __rspamd__run_early_setup_and_checks # must run first + __rspamd__setup_redis + __rspamd__setup_postfix + __rspamd__setup_clamav + __rspamd__setup_default_modules + __rspamd__setup_learning + __rspamd__setup_greylisting + __rspamd__handle_user_modules_adjustments # must run last - if [[ ${RSPAMD_LEARN} -eq 1 ]] - then - __rspamd__log 'debug' 'Enabling and configuring learning' - __rspamd__setup_learning - else - __rspamd__log 'debug' 'Learning is disabled' - fi + _log 'trace' 'Rspamd setup finished' else _log 'debug' 'Rspamd is disabled' fi @@ -31,33 +30,56 @@ function _setup_rspamd # @param ${2} = message function __rspamd__log { _log "${1:-}" "(Rspamd setup) ${2:-}" ; } -# Run miscellaneous checks against the current configuration so we can -# properly handle integration into ClamAV, etc. +# Helper for explicitly enabling or disabling a specific module. # -# This will also check whether Amavis is enabled and emit a warning as -# we discourage users from running Amavis & Rspamd at the same time. -function __rspamd__preflight_checks_and_setup +# @param ${1} = module name +# @param ${2} = `true` when you want to enable the module (default), +# `false` when you want to disable the module [OPTIONAL] +# @param ${3} = whether to use `local` (default) or `override` [OPTIONAL] +function __rspamd__helper__enable_disable_module { - touch /var/lib/rspamd/stats.ucl + local MODULE=${1:?Module name must be provided} + local ENABLE_MODULE=${2:-true} + local LOCAL_OR_OVERRIDE=${3:-local} + local MESSAGE='Enabling' - if [[ ${ENABLE_AMAVIS} -eq 1 ]] || [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] + if [[ ! ${ENABLE_MODULE} =~ ^(true|false)$ ]] then - __rspamd__log 'warn' 'Running Amavis/SA & Rspamd at the same time is discouraged' + __rspamd__log 'warn' "__rspamd__helper__enable_disable_module got non-boolean argument for deciding whether module should be enabled or not" + return 1 fi - if [[ ${ENABLE_CLAMAV} -eq 1 ]] + [[ ${ENABLE_MODULE} == true ]] || MESSAGE='Disabling' + + __rspamd__log 'trace' "${MESSAGE} module '${MODULE}'" + cat >"/etc/rspamd/${LOCAL_OR_OVERRIDE}.d/${MODULE}.conf" << EOF +# documentation: https://rspamd.com/doc/modules/${MODULE}.html + +enabled = ${ENABLE_MODULE}; + +EOF +} + +# Run miscellaneous early setup tasks and checks, such as creating files needed at runtime +# or checking for other anti-spam/anti-virus software. +function __rspamd__run_early_setup_and_checks +{ + mkdir -p /var/lib/rspamd/ + : >/var/lib/rspamd/stats.ucl + + if [[ ${ENABLE_AMAVIS} -eq 1 ]] || [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] then - __rspamd__log 'debug' 'Enabling ClamAV integration' - sedfile -i -E 's|^(enabled).*|\1 = true;|g' /etc/rspamd/local.d/antivirus.conf - # RSpamd uses ClamAV's UNIX socket, and to be able to read it, it must be in the same group - usermod -a -G clamav _rspamd - else - __rspamd__log 'debug' 'Rspamd will not use ClamAV (which has not been enabled)' + __rspamd__log 'warn' 'Running Amavis/SA & Rspamd at the same time is discouraged' fi +} +# Sets up Redis. In case the user does not use a dedicated Redis instance, we +# supply a configuration for our local Redis instance which is started later. +function __rspamd__setup_redis +{ if [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] then - __rspamd__log 'trace' 'Internal Redis is enabled, adding configuration' + __rspamd__log 'debug' 'Internal Redis is enabled, adding configuration' cat >/etc/rspamd/local.d/redis.conf << "EOF" # documentation: https://rspamd.com/doc/configuration/redis.html @@ -69,7 +91,7 @@ EOF # Here we adjust the Redis default configuration that we supply to Redis # when starting it. Note that `/var/lib/redis/` is linked to # `/var/mail-state/redis/` (for persisting it) if `ONE_DIR=1`. - sedfile -i -E \ + sedfile -i -E \ -e 's|^(bind).*|\1 127.0.0.1|g' \ -e 's|^(daemonize).*|\1 no|g' \ -e 's|^(port).*|\1 6379|g' \ @@ -83,56 +105,42 @@ EOF fi } -# Adjust Postfix's configuration files. Append Rspamd at the end of -# `smtpd_milters` in `main.cf`. -function __rspamd__adjust_postfix_configuration +# Adjust Postfix's configuration files. We only need to append Rspamd at the end of +# `smtpd_milters` in `/etc/postfix/main.cf`. +function __rspamd__setup_postfix { - postconf 'rspamd_milter = inet:localhost:11332' + __rspamd__log 'debug' "Adjusting Postfix's configuration" + postconf 'rspamd_milter = inet:localhost:11332' # shellcheck disable=SC2016 sed -i -E 's|^(smtpd_milters =.*)|\1 \$rspamd_milter|g' /etc/postfix/main.cf } -# Helper for explicitly enabling or disabling a specific module. -# -# @param ${1} = module name -# @param ${2} = `true` when you want to enable the module (default), -# `false` when you want to disable the module [OPTIONAL] -# @param ${3} = whether to use `local` (default) or `override` [OPTIONAL] -function __rspamd__enable_disable_module +# If ClamAV is enabled, we will integrate it into Rspamd. +function __rspamd__setup_clamav { - local MODULE=${1:?Module name must be provided} - local ENABLE_MODULE=${2:-true} - local LOCAL_OR_OVERRIDE=${3:-local} - local MESSAGE='Enabling' - - if [[ ! ${ENABLE_MODULE} =~ ^(true|false)$ ]] + if [[ ${ENABLE_CLAMAV} -eq 1 ]] then - __rspamd__log 'warn' "__rspamd__enable_disable_module got non-boolean argument for deciding whether module should be enabled or not" - return 1 + __rspamd__log 'debug' 'Enabling ClamAV integration' + sedfile -i -E 's|^(enabled).*|\1 = true;|g' /etc/rspamd/local.d/antivirus.conf + # Rspamd uses ClamAV's UNIX socket, and to be able to read it, it must be in the same group + usermod -a -G clamav _rspamd + else + __rspamd__log 'debug' 'Rspamd will not use ClamAV (which has not been enabled)' fi - - [[ ${ENABLE_MODULE} == true ]] || MESSAGE='Disabling' - - __rspamd__log 'trace' "${MESSAGE} module '${MODULE}'" - cat >"/etc/rspamd/${LOCAL_OR_OVERRIDE}.d/${MODULE}.conf" << EOF -# documentation: https://rspamd.com/doc/modules/${MODULE}.html - -enabled = ${ENABLE_MODULE}; - -EOF } # Disables certain modules by default. This can be overwritten by the user later. # We disable the modules listed in `DISABLE_MODULES` as we believe these modules # are not commonly used and the average user does not need them. As a consequence, # disabling them saves resources. -function __rspamd__disable_default_modules +function __rspamd__setup_default_modules { + __rspamd__log 'debug' 'Disabling default modules' + local DISABLE_MODULES=( clickhouse elastic - greylist neural reputation spamassassin @@ -142,14 +150,74 @@ function __rspamd__disable_default_modules for MODULE in "${DISABLE_MODULES[@]}" do - __rspamd__enable_disable_module "${MODULE}" 'false' + __rspamd__helper__enable_disable_module "${MODULE}" 'false' done } +# This function sets up intelligent learning of Junk, by +# +# 1. enabling auto-learn for the classifier-bayes module +# 2. setting up sieve scripts that detect when a user is moving e-mail +# from or to the "Junk" folder, and learning them as ham or spam. +function __rspamd__setup_learning +{ + if [[ ${RSPAMD_LEARN} -eq 1 ]] + then + __rspamd__log 'debug' 'Setting up intelligent learning of spam and ham' + + local SIEVE_PIPE_BIN_DIR='/usr/lib/dovecot/sieve-pipe' + ln -s "$(type -f -P rspamc)" "${SIEVE_PIPE_BIN_DIR}/rspamc" + + sedfile -i -E 's|(mail_plugins =.*)|\1 imap_sieve|' /etc/dovecot/conf.d/20-imap.conf + sedfile -i -E '/^}/d' /etc/dovecot/conf.d/90-sieve.conf + cat >>/etc/dovecot/conf.d/90-sieve.conf << EOF + + # From elsewhere to Junk folder + imapsieve_mailbox1_name = Junk + imapsieve_mailbox1_causes = COPY + imapsieve_mailbox1_before = file:${SIEVE_PIPE_BIN_DIR}/learn-spam.sieve + + # From Junk folder to elsewhere + imapsieve_mailbox2_name = * + imapsieve_mailbox2_from = Junk + imapsieve_mailbox2_causes = COPY + imapsieve_mailbox2_before = file:${SIEVE_PIPE_BIN_DIR}/learn-ham.sieve +} +EOF + + cat >"${SIEVE_PIPE_BIN_DIR}/learn-spam.sieve" << EOF +require ["vnd.dovecot.pipe", "copy", "imapsieve"]; +pipe :copy "rspamc" ["-h", "127.0.0.1:11334", "learn_spam"]; +EOF + + cat >"${SIEVE_PIPE_BIN_DIR}/learn-ham.sieve" << EOF +require ["vnd.dovecot.pipe", "copy", "imapsieve"]; +pipe :copy "rspamc" ["-h", "127.0.0.1:11334", "learn_ham"]; +EOF + + sievec "${SIEVE_PIPE_BIN_DIR}/learn-spam.sieve" + sievec "${SIEVE_PIPE_BIN_DIR}/learn-ham.sieve" + else + __rspamd__log 'debug' 'Intelligent learning of spam and ham is disabled' + fi +} + +# Sets up greylisting based on the environment variable RSPAMD_GREYLISTING. +function __rspamd__setup_greylisting +{ + if [[ ${RSPAMD_GREYLISTING} -eq 1 ]] + then + __rspamd__log 'debug' 'Enabling greylisting' + sedfile -i -E "s|(enabled =).*|\1 true;|g" /etc/rspamd/local.d/greylist.conf + else + __rspamd__log 'debug' 'Greylisting is disabled' + fi +} + # Parses `RSPAMD_CUSTOM_COMMANDS_FILE` and executed the directives given by the file. # To get a detailed explanation of the commands and how the file works, visit # https://docker-mailserver.github.io/docker-mailserver/edge/config/security/rspamd/#with-the-help-of-a-custom-file -function __rspamd__handle_modules_configuration +function __rspamd__handle_user_modules_adjustments { # Adds an option with a corresponding value to a module, or, in case the option # is already present, overwrites it. @@ -161,7 +229,7 @@ function __rspamd__handle_modules_configuration # # ## Note # - # While this function is currently bound to the scope of `__rspamd__handle_modules_configuration`, + # While this function is currently bound to the scope of `__rspamd__handle_user_modules_adjustments`, # it is written in a versatile way (taking 4 arguments instead of assuming `ARGUMENT2` / `ARGUMENT3` # are set) so that it may be used elsewhere if needed. function __add_or_replace @@ -196,11 +264,11 @@ function __rspamd__handle_modules_configuration case "${COMMAND}" in ('disable-module') - __rspamd__enable_disable_module "${ARGUMENT1}" 'false' 'override' + __rspamd__helper__enable_disable_module "${ARGUMENT1}" 'false' 'override' ;; ('enable-module') - __rspamd__enable_disable_module "${ARGUMENT1}" 'true' 'override' + __rspamd__helper__enable_disable_module "${ARGUMENT1}" 'true' 'override' ;; ('set-option-for-module') @@ -233,46 +301,3 @@ function __rspamd__handle_modules_configuration done < <(_get_valid_lines_from_file "${RSPAMD_CUSTOM_COMMANDS_FILE}") fi } - -# This function sets up intelligent learning of Junk, by -# -# 1. enabling auto-learn for the classifier-bayes module -# 2. setting up sieve scripts that detect when a user is moving e-mail -# from or to the "Junk" folder, and learning them as ham or spam. -function __rspamd__setup_learning -{ - __rspamd__log 'debug' 'Setting up intelligent learning of spam and ham' - - local SIEVE_PIPE_BIN_DIR='/usr/lib/dovecot/sieve-pipe' - ln -s "$(type -f -P rspamc)" "${SIEVE_PIPE_BIN_DIR}/rspamc" - - sedfile -i -E 's|(mail_plugins =.*)|\1 imap_sieve|' /etc/dovecot/conf.d/20-imap.conf - sedfile -i -E '/^}/d' /etc/dovecot/conf.d/90-sieve.conf - cat >>/etc/dovecot/conf.d/90-sieve.conf << EOF - - # From elsewhere to Junk folder - imapsieve_mailbox1_name = Junk - imapsieve_mailbox1_causes = COPY - imapsieve_mailbox1_before = file:${SIEVE_PIPE_BIN_DIR}/learn-spam.sieve - - # From Junk folder to elsewhere - imapsieve_mailbox2_name = * - imapsieve_mailbox2_from = Junk - imapsieve_mailbox2_causes = COPY - imapsieve_mailbox2_before = file:${SIEVE_PIPE_BIN_DIR}/learn-ham.sieve -} -EOF - - cat >"${SIEVE_PIPE_BIN_DIR}/learn-spam.sieve" << EOF -require ["vnd.dovecot.pipe", "copy", "imapsieve"]; -pipe :copy "rspamc" ["-h", "127.0.0.1:11334", "learn_spam"]; -EOF - - cat >"${SIEVE_PIPE_BIN_DIR}/learn-ham.sieve" << EOF -require ["vnd.dovecot.pipe", "copy", "imapsieve"]; -pipe :copy "rspamc" ["-h", "127.0.0.1:11334", "learn_ham"]; -EOF - - sievec "${SIEVE_PIPE_BIN_DIR}/learn-spam.sieve" - sievec "${SIEVE_PIPE_BIN_DIR}/learn-ham.sieve" -} diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index 4611dba4b89..5bf5a34021f 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -55,6 +55,7 @@ function __environment_variables_general_setup VARS[POSTGREY_MAX_AGE]="${POSTGREY_MAX_AGE:=35}" VARS[POSTGREY_TEXT]="${POSTGREY_TEXT:=Delayed by Postgrey}" VARS[POSTSCREEN_ACTION]="${POSTSCREEN_ACTION:=enforce}" + VARS[RSPAMD_GREYLISTING]="${RSPAMD_GREYLISTING:=0}" VARS[RSPAMD_LEARN]="${RSPAMD_LEARN:=0}" VARS[SA_KILL]=${SA_KILL:="10.0"} VARS[SA_SPAM_SUBJECT]=${SA_SPAM_SUBJECT:="***SPAM*** "} diff --git a/test/tests/parallel/set1/spam_virus/rspamd.bats b/test/tests/parallel/set1/spam_virus/rspamd.bats index 658a0e6850b..b0d0f88b5a0 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd.bats @@ -18,6 +18,7 @@ function setup_file() { --env LOG_LEVEL=trace --env MOVE_SPAM_TO_JUNK=1 --env RSPAMD_LEARN=1 + --env RSPAMD_GREYLISTING=1 ) mv "${TEST_TMP_CONFIG}"/rspamd/* "${TEST_TMP_CONFIG}/" @@ -243,3 +244,11 @@ function teardown_file() { _default_teardown ; } assert_output --partial "${LINE}" done } + +@test 'Check greylisting is enabled' { + _run_in_container grep 'enabled = true;' /etc/rspamd/local.d/greylist.conf + assert_success + _run_in_container rspamadm configdump greylist + assert_success + assert_output --partial 'enabled = true;' +} From 9a284150b2f744acaf4f134ecf78d00494fbf997 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 11 Apr 2023 18:51:23 +0200 Subject: [PATCH 032/592] Rspamd: replace `reject_unknown_client_hostname` with Rspamd `HFILTER_HOSTNAME_UNKNOWN` and make it configurable (#3248) --- docs/content/config/environment.md | 15 ++++++++ docs/content/config/security/rspamd.md | 6 ++-- mailserver.env | 11 ++++++ target/postfix/main.cf | 2 +- target/rspamd/local.d/hfilter_group.conf | 11 ++++++ target/rspamd/local.d/options.inc | 1 + .../startup/setup.d/security/rspamd.sh | 36 ++++++++++++++++++- target/scripts/startup/variables-stack.sh | 2 ++ .../parallel/set1/spam_virus/rspamd.bats | 19 ++++++++-- 9 files changed, 96 insertions(+), 7 deletions(-) create mode 100644 target/rspamd/local.d/hfilter_group.conf diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index e8168062343..e221d13c116 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -363,6 +363,21 @@ When enabled, - **0** => Disabled - 1 => Enabled +##### RSPAMD_HFILTER + +Can be used to enable or disable the [Hfilter group module][rspamd-docs-hfilter-group-module]. This is used by DMS to adjust the `HFILTER_HOSTNAME_UNKNOWN` symbol, increasing it's default weight to act similar to Postfix's `reject_unknown_client_hostname`, without the need to outright reject a message. + +- 0 => Disabled +- **1** => Enabled + +[rspamd-docs-hfilter-group-module]: https://www.rspamd.com/doc/modules/hfilter.html + +##### RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE + +Can be used to control the score when the [`HFILTER_HOSTNAME_UNKNOWN` symbol](#rspamd_hfilter) applies. A higher score is more punishing. Setting it to 15 (the default score for rejecting an e-mail) is equivalent to rejecting the email when the check fails. + +Default: 6 (which corresponds to the `add_header` action) + #### Reports ##### PFLOGSUMM_TRIGGER diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 5f70fe73894..1f40c623a05 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -25,8 +25,10 @@ The following environment variables are related to Rspamd: 1. [`ENABLE_RSPAMD`](../environment.md#enable_rspamd) 2. [`ENABLE_RSPAMD_REDIS`](../environment.md#enable_rspamd_redis) 3. [`RSPAMD_GREYLISTING`](../environment.md#rspamd_greylisting) -4. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) -5. [`MOVE_SPAM_TO_JUNK`](../environment.md#move_spam_to_junk) +4. [`RSPAMD_HFILTER`](../environment.md#rspamd_hfilter) +5. [`RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE`](../environment.md#rspamd_hfilter_hostname_unknown_score) +6. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) +7. [`MOVE_SPAM_TO_JUNK`](../environment.md#move_spam_to_junk) With these variables, you can enable Rspamd itself and you can enable / disable certain features related to Rspamd. diff --git a/mailserver.env b/mailserver.env index de8a7653e67..81356af5bfe 100644 --- a/mailserver.env +++ b/mailserver.env @@ -150,6 +150,17 @@ RSPAMD_LEARN=0 # 1 => enabled RSPAMD_GREYLISTING=0 +# Can be used to enable or diable the Hfilter group module. +# +# - 0 => Disabled +# - **1** => Enabled +RSPAMD_HFILTER=1 + +# Can be used to control the score when the HFILTER_HOSTNAME_UNKNOWN symbol applies. A higher score is more punishing. Setting it to 15 is equivalent to rejecting the email when the check fails. +# +# Default: 6 +RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE=6 + # Amavis content filter (used for ClamAV & SpamAssassin) # 0 => Disabled # 1 => Enabled diff --git a/target/postfix/main.cf b/target/postfix/main.cf index 9d47348f47c..881c3796ddc 100644 --- a/target/postfix/main.cf +++ b/target/postfix/main.cf @@ -54,7 +54,7 @@ smtpd_sender_restrictions = $dms_smtpd_sender_restrictions disable_vrfy_command = yes # Custom defined parameters for DMS: -dms_smtpd_sender_restrictions = permit_sasl_authenticated, permit_mynetworks, reject_unknown_sender_domain, reject_unknown_client_hostname +dms_smtpd_sender_restrictions = permit_sasl_authenticated, permit_mynetworks, reject_unknown_sender_domain # Submission ports 587 and 465 support for SPOOF_PROTECTION=1 mua_sender_restrictions = reject_authenticated_sender_login_mismatch, $dms_smtpd_sender_restrictions diff --git a/target/rspamd/local.d/hfilter_group.conf b/target/rspamd/local.d/hfilter_group.conf new file mode 100644 index 00000000000..7bf73b3efe4 --- /dev/null +++ b/target/rspamd/local.d/hfilter_group.conf @@ -0,0 +1,11 @@ +symbols = { + "HFILTER_HOSTNAME_UNKNOWN" { + # We add 6 because we need a score of exactly 6 for the `add_header` + # action to trigger; that's the least we want to happen. + # + # This is meant as a replacement for `reject_unknown_client_hostname` + # (see https://www.postfix.org/postconf.5.html#reject_unknown_client_hostname) + # which Postfix can employ to directly reject emails. + score = 6; # __TAG__HFILTER_HOSTNAME_UNKNOWN + } +} diff --git a/target/rspamd/local.d/options.inc b/target/rspamd/local.d/options.inc index 6561c2ec43b..f57d4339e68 100644 --- a/target/rspamd/local.d/options.inc +++ b/target/rspamd/local.d/options.inc @@ -1 +1,2 @@ pidfile = false; +soft_reject_on_timeout = true; diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index ba292346bf6..d7feb8024c4 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -15,6 +15,7 @@ function _setup_rspamd __rspamd__setup_default_modules __rspamd__setup_learning __rspamd__setup_greylisting + __rspamd__setup_hfilter_group __rspamd__handle_user_modules_adjustments # must run last _log 'trace' 'Rspamd setup finished' @@ -138,6 +139,9 @@ function __rspamd__setup_default_modules { __rspamd__log 'debug' 'Disabling default modules' + # This array contains all the modules we disable by default. They + # can be re-enabled later (in `__rspamd__handle_user_modules_adjustments`) + # with `rspamd-modules.conf`. local DISABLE_MODULES=( clickhouse elastic @@ -202,7 +206,8 @@ EOF fi } -# Sets up greylisting based on the environment variable RSPAMD_GREYLISTING. +# Sets up greylisting with the greylisting module (see +# https://rspamd.com/doc/modules/greylisting.html). function __rspamd__setup_greylisting { if [[ ${RSPAMD_GREYLISTING} -eq 1 ]] @@ -214,6 +219,35 @@ function __rspamd__setup_greylisting fi } +# This function handles setup of the Hfilter module (see +# https://www.rspamd.com/doc/modules/hfilter.html). This module is mainly +# used for hostname checks, and whether or not a reverse-DNS check +# succeeds. +function __rspamd__setup_hfilter_group +{ + local MODULE_FILE='/etc/rspamd/local.d/hfilter_group.conf' + if [[ ${RSPAMD_HFILTER} -eq 1 ]] + then + __rspamd__log 'debug' 'Hfilter (group) module is enabled' + # Check if we received a number first + if [[ ! ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE} =~ ^[0-9][1-9]*$ ]] + then + __rspamd__log 'warn' "'RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE' is not a number (${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE}) but was expected to be!" + elif [[ ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE} -ne 6 ]] + then + __rspamd__log 'trace' "Adjusting score for 'HFILTER_HOSTNAME_UNKNOWN' in Hfilter group module to ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE}" + sed -i -E \ + "s|(.*score =).*(# __TAG__HFILTER_HOSTNAME_UNKNOWN)|\1 ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE}; \2|g" \ + "${MODULE_FILE}" + else + __rspamd__log 'trace' "Not adjusting score for 'HFILTER_HOSTNAME_UNKNOWN' in Hfilter group module" + fi + else + __rspamd__log 'debug' 'Disabling Hfilter (group) module' + rm -f "${MODULE_FILE}" + fi +} + # Parses `RSPAMD_CUSTOM_COMMANDS_FILE` and executed the directives given by the file. # To get a detailed explanation of the commands and how the file works, visit # https://docker-mailserver.github.io/docker-mailserver/edge/config/security/rspamd/#with-the-help-of-a-custom-file diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index 5bf5a34021f..8919a547e3e 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -56,6 +56,8 @@ function __environment_variables_general_setup VARS[POSTGREY_TEXT]="${POSTGREY_TEXT:=Delayed by Postgrey}" VARS[POSTSCREEN_ACTION]="${POSTSCREEN_ACTION:=enforce}" VARS[RSPAMD_GREYLISTING]="${RSPAMD_GREYLISTING:=0}" + VARS[RSPAMD_HFILTER]="${RSPAMD_HFILTER:=1}" + VARS[RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE]="${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE:=6}" VARS[RSPAMD_LEARN]="${RSPAMD_LEARN:=0}" VARS[SA_KILL]=${SA_KILL:="10.0"} VARS[SA_SPAM_SUBJECT]=${SA_SPAM_SUBJECT:="***SPAM*** "} diff --git a/test/tests/parallel/set1/spam_virus/rspamd.bats b/test/tests/parallel/set1/spam_virus/rspamd.bats index b0d0f88b5a0..ea57e81f473 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd.bats @@ -10,6 +10,8 @@ function setup_file() { # Comment for maintainers about `PERMIT_DOCKER=host`: # https://github.com/docker-mailserver/docker-mailserver/pull/2815/files#r991087509 local CUSTOM_SETUP_ARGUMENTS=( + --env ENABLE_AMAVIS=0 + --env ENABLE_SPAMASSASSIN=0 --env ENABLE_CLAMAV=1 --env ENABLE_RSPAMD=1 --env ENABLE_OPENDKIM=0 @@ -19,6 +21,8 @@ function setup_file() { --env MOVE_SPAM_TO_JUNK=1 --env RSPAMD_LEARN=1 --env RSPAMD_GREYLISTING=1 + --env RSPAMD_HFILTER=1 + --env RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE=7 ) mv "${TEST_TMP_CONFIG}"/rspamd/* "${TEST_TMP_CONFIG}/" @@ -170,7 +174,7 @@ function teardown_file() { _default_teardown ; } assert_success } -@test 'Check MOVE_SPAM_TO_JUNK works for Rspamd' { +@test 'MOVE_SPAM_TO_JUNK works for Rspamd' { _run_in_container_bash '[[ -f /usr/lib/dovecot/sieve-global/after/spam_to_junk.sieve ]]' assert_success _run_in_container_bash '[[ -f /usr/lib/dovecot/sieve-global/after/spam_to_junk.svbin ]]' @@ -186,7 +190,7 @@ function teardown_file() { _default_teardown ; } _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/.Junk/new/ 1 } -@test 'Check RSPAMD_LEARN works' { +@test 'RSPAMD_LEARN works' { for FILE in learn-{ham,spam}.{sieve,svbin} do _run_in_container_bash "[[ -f /usr/lib/dovecot/sieve-pipe/${FILE} ]]" @@ -245,10 +249,19 @@ function teardown_file() { _default_teardown ; } done } -@test 'Check greylisting is enabled' { +@test 'greylisting is enabled' { _run_in_container grep 'enabled = true;' /etc/rspamd/local.d/greylist.conf assert_success _run_in_container rspamadm configdump greylist assert_success assert_output --partial 'enabled = true;' } + +@test 'hfilter group module is configured correctly' { + _run_in_container_bash '[[ -f /etc/rspamd/local.d/hfilter_group.conf ]]' + assert_success + + _run_in_container grep '__TAG__HFILTER_HOSTNAME_UNKNOWN' /etc/rspamd/local.d/hfilter_group.conf + assert_success + assert_output --partial 'score = 7;' +} From de19c6bd36ae2f3f69098fc42c6569fa813622cd Mon Sep 17 00:00:00 2001 From: Andreas Perhab Date: Tue, 11 Apr 2023 20:11:16 +0200 Subject: [PATCH 033/592] tests: fix dovecot: ldap mail delivery works (#3252) When a new version of docker-mailserver is available the account used in this tests also gets the postmaster notification for the new version. The mailbox then may contain 2 mails but only one with 'This is a test mail.'. Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- test/tests/serial/mail_with_ldap.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tests/serial/mail_with_ldap.bats b/test/tests/serial/mail_with_ldap.bats index 79100f01851..75768bbb82f 100644 --- a/test/tests/serial/mail_with_ldap.bats +++ b/test/tests/serial/mail_with_ldap.bats @@ -151,7 +151,7 @@ function teardown_file() { @test "checking dovecot: ldap mail delivery works" { run docker exec mail_with_ldap /bin/sh -c "sendmail -f user@external.tld some.user@${FQDN_LOCALHOST_A} < /tmp/docker-mailserver-test/email-templates/test-email.txt" sleep 10 - run docker exec mail_with_ldap /bin/sh -c "ls -A /var/mail/${FQDN_LOCALHOST_A}/some.user/new | wc -l" + run docker exec mail_with_ldap /bin/sh -c "grep -R 'This is a test mail.' /var/mail/${FQDN_LOCALHOST_A}/some.user/new/ | wc -l" assert_success assert_output 1 } From 1076aac37d3e30e921f3917b7eb35b461904d1c4 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 11 Apr 2023 20:28:43 +0200 Subject: [PATCH 034/592] change F2B configs: made config more aggressive (#3243) --- docs/content/config/security/fail2ban.md | 2 +- target/fail2ban/jail.local | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/content/config/security/fail2ban.md b/docs/content/config/security/fail2ban.md index 954b9bdbff1..fedcc64f63d 100644 --- a/docs/content/config/security/fail2ban.md +++ b/docs/content/config/security/fail2ban.md @@ -4,7 +4,7 @@ hide: - toc # Hide Table of Contents for this page --- -Fail2Ban is installed automatically and bans IP addresses for 3 hours after 3 failed attempts in 10 minutes by default. +Fail2Ban is installed automatically and bans IP addresses for 1 week after 2 failed attempts in a time frame of 1 week by default. ## Configuration files diff --git a/target/fail2ban/jail.local b/target/fail2ban/jail.local index 9611e7e0b66..96f1d4f5981 100644 --- a/target/fail2ban/jail.local +++ b/target/fail2ban/jail.local @@ -1,14 +1,14 @@ [DEFAULT] # "bantime" is the number of seconds that a host is banned. -bantime = 3h +bantime = 1w # A host is banned if it has generated "maxretry" during the last "findtime" # seconds. -findtime = 10m +findtime = 1w # "maxretry" is the number of failures before a host get banned. -maxretry = 3 +maxretry = 2 # "ignoreip" can be a list of IP addresses, CIDR masks or DNS hosts. Fail2ban # will not ban a host which matches an address in this list. Several addresses @@ -25,9 +25,14 @@ enabled = true [postfix] enabled = true +# See https://github.com/fail2ban/fail2ban/blob/27294c4b9ee5d5568a1d5f83af744ea39d5a1acb/config/filter.d/postfix.conf#L58 +# `mode=aggressive` basically combines more filters to match more lines, and hence, apply rules +# more aggressively. The same goes for the `postfix-sasl` jail. +mode = aggressive [postfix-sasl] enabled = true +mode = aggressive # This jail is used for manual bans. # To ban an IP address use: setup.sh fail2ban ban From 03772f612a39fcfe2c4deedde750c77e3f7f0585 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sat, 15 Apr 2023 00:40:42 +0200 Subject: [PATCH 035/592] scripts: get all `policyd-spf` setup in one place (#3263) --- target/postfix/main.cf | 5 +---- target/scripts/startup/setup.d/dmarc_dkim_spf.sh | 8 +++++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/target/postfix/main.cf b/target/postfix/main.cf index 881c3796ddc..4827c47c317 100644 --- a/target/postfix/main.cf +++ b/target/postfix/main.cf @@ -48,7 +48,7 @@ smtpd_helo_required = yes smtpd_delay_reject = yes smtpd_helo_restrictions = permit_mynetworks, reject_invalid_helo_hostname, permit smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination -smtpd_recipient_restrictions = permit_sasl_authenticated, permit_mynetworks, reject_unauth_destination, check_policy_service unix:private/policyd-spf, reject_unauth_pipelining, reject_invalid_helo_hostname, reject_non_fqdn_helo_hostname, reject_unknown_recipient_domain +smtpd_recipient_restrictions = permit_sasl_authenticated, permit_mynetworks, reject_unauth_destination, reject_unauth_pipelining, reject_invalid_helo_hostname, reject_non_fqdn_helo_hostname, reject_unknown_recipient_domain smtpd_client_restrictions = permit_mynetworks, permit_sasl_authenticated, reject_unauth_destination, reject_unauth_pipelining smtpd_sender_restrictions = $dms_smtpd_sender_restrictions disable_vrfy_command = yes @@ -96,9 +96,6 @@ milter_default_action = accept smtpd_milters = non_smtpd_milters = -# SPF policy settings -policyd-spf_time_limit = 3600 - # Header checks for content inspection on receiving header_checks = pcre:/etc/postfix/maps/header_checks.pcre diff --git a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh index c27e5258d4d..bb92e774e27 100644 --- a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh +++ b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh @@ -97,8 +97,14 @@ function _setup_policyd_spf policyd-spf unix - n n - 0 spawn user=policyd-spf argv=/usr/bin/policyd-spf EOF + + sedfile -i -E \ + 's|^(smtpd_recipient_restrictions.*reject_unauth_destination)(.*)|\1, check_policy_service unix:private/policyd-spf\2|' \ + /etc/postfix/main.cf + # SPF policy settings + postconf 'policyd-spf_time_limit = 3600' +EOF else _log 'debug' 'Disabling policyd-spf' - sedfile -i -E 's|check_policy_service unix:private/policyd-spf, ||g' /etc/postfix/main.cf fi } From c8dfb9ac765cd9c2fbacd604e580112ff12daa4e Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 16 Apr 2023 14:09:00 +0200 Subject: [PATCH 036/592] Posfix: add option to re-enable `reject_unknown_client_hostname` after #3248 (#3255) --- docs/content/config/environment.md | 7 ++ mailserver.env | 7 ++ target/scripts/start-mailserver.sh | 16 +-- target/scripts/startup/setup.d/postfix.sh | 145 ++++++++++------------ target/scripts/startup/variables-stack.sh | 3 +- 5 files changed, 87 insertions(+), 91 deletions(-) diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index e221d13c116..bea6b93b475 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -274,6 +274,13 @@ Customize the update check interval. Number + Suffix. Suffix must be 's' for sec This option has been added in November 2019. Using other format than Maildir is considered as experimental in docker-mailserver and should only be used for testing purpose. For more details, please refer to [Dovecot Documentation](https://wiki2.dovecot.org/MailboxFormat). +##### POSTFIX_REJECT_UNKNOWN_CLIENT_HOSTNAME + +If enabled, employs `reject_unknown_client_hostname` to sender restrictions in Postfix's configuration. + +- **0** => Disabled +- 1 => Enabled + ##### POSTFIX_INET_PROTOCOLS - **all** => Listen on all interfaces. diff --git a/mailserver.env b/mailserver.env index 81356af5bfe..7ff2843fc7a 100644 --- a/mailserver.env +++ b/mailserver.env @@ -318,6 +318,13 @@ REPORT_SENDER= # Note: This variable can also determine the interval for Postfix's log summary reports, see [`PFLOGSUMM_TRIGGER`](#pflogsumm_trigger). LOGROTATE_INTERVAL=weekly + +# If enabled, employs `reject_unknown_client_hostname` to sender restrictions in Postfix's configuration. +# +# - **0** => Disabled +# - 1 => Enabled +POSTFIX_REJECT_UNKNOWN_CLIENT_HOSTNAME=0 + # Choose TCP/IP protocols for postfix to use # **all** => All possible protocols. # ipv4 => Use only IPv4 traffic. Most likely you want this behind Docker. diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index 039c8f743d9..ff187f115bd 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -75,7 +75,6 @@ function _register_functions _register_setup_function '_setup_saslauthd' fi - _register_setup_function '_setup_postfix_inet_protocols' _register_setup_function '_setup_dovecot_inet_protocols' _register_setup_function '_setup_opendkim' @@ -91,17 +90,11 @@ function _register_functions _register_setup_function '_setup_mailname' _register_setup_function '_setup_dovecot_hostname' - _register_setup_function '_setup_postfix_hostname' - _register_setup_function '_setup_postfix_smtputf8' - _register_setup_function '_setup_postfix_sasl' - _register_setup_function '_setup_postfix_aliases' - _register_setup_function '_setup_postfix_vhost' - _register_setup_function '_setup_postfix_dhparam' - _register_setup_function '_setup_postfix_sizelimits' + _register_setup_function '_setup_postfix_early' _register_setup_function '_setup_fetchmail' _register_setup_function '_setup_fetchmail_parallel' - # needs to come after _setup_postfix_aliases + # needs to come after _setup_postfix_early _register_setup_function '_setup_spoof_protection' if [[ ${ENABLE_SRS} -eq 1 ]] @@ -110,10 +103,7 @@ function _register_functions _register_start_daemon '_start_daemon_postsrsd' fi - _register_setup_function '_setup_postfix_access_control' - _register_setup_function '_setup_postfix_relay_hosts' - _register_setup_function '_setup_postfix_virtual_transport' - _register_setup_function '_setup_postfix_override_configuration' + _register_setup_function '_setup_postfix_late' _register_setup_function '_setup_logrotate' _register_setup_function '_setup_mail_summary' _register_setup_function '_setup_logwatch' diff --git a/target/scripts/startup/setup.d/postfix.sh b/target/scripts/startup/setup.d/postfix.sh index 12a2f8b6620..bd9feaa64e2 100644 --- a/target/scripts/startup/setup.d/postfix.sh +++ b/target/scripts/startup/setup.d/postfix.sh @@ -1,34 +1,30 @@ #!/bin/bash -function _setup_postfix_sizelimits +# Just a helper to prepend the log messages with `(Postfix setup)` so +# users know exactly where the message originated from. +# +# @param ${1} = log level +# @param ${2} = message +function __postfix__log { _log "${1:-}" "(Postfix setup) ${2:-}" ; } + +function _setup_postfix_early { - _log 'trace' "Configuring Postfix message size limit to '${POSTFIX_MESSAGE_SIZE_LIMIT}'" - postconf "message_size_limit = ${POSTFIX_MESSAGE_SIZE_LIMIT}" - - _log 'trace' "Configuring Postfix mailbox size limit to '${POSTFIX_MAILBOX_SIZE_LIMIT}'" - postconf "mailbox_size_limit = ${POSTFIX_MAILBOX_SIZE_LIMIT}" + _log 'debug' 'Configuring Postfix (early setup)' - _log 'trace' "Configuring Postfix virtual mailbox size limit to '${POSTFIX_MAILBOX_SIZE_LIMIT}'" - postconf "virtual_mailbox_limit = ${POSTFIX_MAILBOX_SIZE_LIMIT}" -} - -function _setup_postfix_access_control -{ - _log 'trace' 'Configuring user access' + __postfix__log 'trace' 'Applying hostname and domainname' + postconf "myhostname = ${HOSTNAME}" + postconf "mydomain = ${DOMAINNAME}" - if [[ -f /tmp/docker-mailserver/postfix-send-access.cf ]] + if [[ ${POSTFIX_INET_PROTOCOLS} != 'all' ]] then - sed -i 's|smtpd_sender_restrictions =|smtpd_sender_restrictions = check_sender_access texthash:/tmp/docker-mailserver/postfix-send-access.cf,|' /etc/postfix/main.cf + __postfix__log 'trace' 'Setting up POSTFIX_INET_PROTOCOLS option' + postconf "inet_protocols = ${POSTFIX_INET_PROTOCOLS}" fi - if [[ -f /tmp/docker-mailserver/postfix-receive-access.cf ]] - then - sed -i 's|smtpd_recipient_restrictions =|smtpd_recipient_restrictions = check_recipient_access texthash:/tmp/docker-mailserver/postfix-receive-access.cf,|' /etc/postfix/main.cf - fi -} + __postfix__log 'trace' "Disabling SMTPUTF8 support" + postconf 'smtputf8_enable = no' -function _setup_postfix_sasl -{ + __postfix__log 'trace' "Configuring SASLauthd" if [[ ${ENABLE_SASLAUTHD} -eq 1 ]] && [[ ! -f /etc/postfix/sasl/smtpd.conf ]] then cat >/etc/postfix/sasl/smtpd.conf << EOF @@ -46,40 +42,65 @@ EOF 's|^ -o smtpd_sasl_auth_enable=.*| -o smtpd_sasl_auth_enable=no|g' \ /etc/postfix/master.cf fi -} -function _setup_postfix_aliases -{ - _log 'debug' 'Setting up Postfix aliases' + __postfix__log 'trace' 'Setting up aliases' _create_aliases -} -function _setup_postfix_vhost -{ - _log 'debug' 'Setting up Postfix vhost' + __postfix__log 'trace' 'Setting up Postfix vhost' _create_postfix_vhost -} -function _setup_postfix_inet_protocols -{ - [[ ${POSTFIX_INET_PROTOCOLS} == 'all' ]] && return 0 + __postfix__log 'trace' 'Setting up DH Parameters' + _setup_dhparam 'Postfix' '/etc/postfix/dhparams.pem' + + __postfix__log 'trace' "Configuring message size limit to '${POSTFIX_MESSAGE_SIZE_LIMIT}'" + postconf "message_size_limit = ${POSTFIX_MESSAGE_SIZE_LIMIT}" - _log 'trace' 'Setting up POSTFIX_INET_PROTOCOLS option' - postconf "inet_protocols = ${POSTFIX_INET_PROTOCOLS}" + __postfix__log 'trace' "Configuring mailbox size limit to '${POSTFIX_MAILBOX_SIZE_LIMIT}'" + postconf "mailbox_size_limit = ${POSTFIX_MAILBOX_SIZE_LIMIT}" + + __postfix__log 'trace' "Configuring virtual mailbox size limit to '${POSTFIX_MAILBOX_SIZE_LIMIT}'" + postconf "virtual_mailbox_limit = ${POSTFIX_MAILBOX_SIZE_LIMIT}" + + if [[ ${POSTFIX_REJECT_UNKNOWN_CLIENT_HOSTNAME} -eq 1 ]] + then + __postfix__log 'trace' 'Enabling reject_unknown_client_hostname to dms_smtpd_sender_restrictions' + sedfile -i -E \ + 's|^(dms_smtpd_sender_restrictions = .*)|\1, reject_unknown_client_hostname|' \ + /etc/postfix/main.cf + fi } -function _setup_postfix_virtual_transport +function _setup_postfix_late { - [[ -z ${POSTFIX_DAGENT} ]] && return 0 + _log 'debug' 'Configuring Postfix (late setup)' - _log 'trace' "Changing Postfix virtual transport to '${POSTFIX_DAGENT}'" - # Default value in main.cf should be 'lmtp:unix:/var/run/dovecot/lmtp' - postconf "virtual_transport = ${POSTFIX_DAGENT}" + __postfix__log 'trace' 'Configuring user access' + if [[ -f /tmp/docker-mailserver/postfix-send-access.cf ]] + then + sed -i 's|(smtpd_sender_restrictions =)|\1 check_sender_access texthash:/tmp/docker-mailserver/postfix-send-access.cf,|' /etc/postfix/main.cf + fi + + if [[ -f /tmp/docker-mailserver/postfix-receive-access.cf ]] + then + sed -i -E 's|(smtpd_recipient_restrictions =)|\1 check_recipient_access texthash:/tmp/docker-mailserver/postfix-receive-access.cf,|' /etc/postfix/main.cf + fi + + __postfix__log 'trace' 'Configuring relay host' + _setup_relayhost + + if [[ -n ${POSTFIX_DAGENT} ]] + then + __postfix__log 'trace' "Changing virtual transport to '${POSTFIX_DAGENT}'" + # Default value in main.cf should be 'lmtp:unix:/var/run/dovecot/lmtp' + postconf "virtual_transport = ${POSTFIX_DAGENT}" + fi + + __postfix__setup_override_configuration } -function _setup_postfix_override_configuration +function __postfix__setup_override_configuration { - _log 'debug' 'Overriding / adjusting Postfix configuration with user-supplied values' + __postfix__log 'debug' 'Overriding / adjusting configuration with user-supplied values' if [[ -f /tmp/docker-mailserver/postfix-main.cf ]] then @@ -91,9 +112,9 @@ function _setup_postfix_override_configuration mv /tmp/postfix-main-new.cf /etc/postfix/main.cf _adjust_mtime_for_postfix_maincf - _log 'trace' "Adjusted '/etc/postfix/main.cf' according to '/tmp/docker-mailserver/postfix-main.cf'" + __postfix__log 'trace' "Adjusted '/etc/postfix/main.cf' according to '/tmp/docker-mailserver/postfix-main.cf'" else - _log 'trace' "No extra Postfix settings loaded because optional '/tmp/docker-mailserver/postfix-main.cf' was not provided" + __postfix__log 'trace' "No extra Postfix settings loaded because optional '/tmp/docker-mailserver/postfix-main.cf' was not provided" fi if [[ -f /tmp/docker-mailserver/postfix-master.cf ]] @@ -105,35 +126,12 @@ function _setup_postfix_override_configuration postconf -P "${LINE}" fi done < /tmp/docker-mailserver/postfix-master.cf - _log 'trace' "Adjusted '/etc/postfix/master.cf' according to '/tmp/docker-mailserver/postfix-master.cf'" + __postfix__log 'trace' "Adjusted '/etc/postfix/master.cf' according to '/tmp/docker-mailserver/postfix-master.cf'" else - _log 'trace' "No extra Postfix settings loaded because optional '/tmp/docker-mailserver/postfix-master.cf' was not provided" + __postfix__log 'trace' "No extra Postfix settings loaded because optional '/tmp/docker-mailserver/postfix-master.cf' was not provided" fi } -function _setup_postfix_relay_hosts -{ - _setup_relayhost -} - -function _setup_postfix_dhparam -{ - _setup_dhparam 'Postfix' '/etc/postfix/dhparams.pem' -} - -function _setup_dnsbl_disable -{ - _log 'debug' 'Disabling postscreen DNS block lists' - postconf 'postscreen_dnsbl_action = ignore' - postconf 'postscreen_dnsbl_sites = ' -} - -function _setup_postfix_smtputf8 -{ - _log 'trace' "Disabling Postfix's smtputf8 support" - postconf 'smtputf8_enable = no' -} - function _setup_SRS { _log 'debug' 'Setting up SRS' @@ -177,10 +175,3 @@ function _setup_SRS /etc/default/postsrsd fi } - -function _setup_postfix_hostname -{ - _log 'debug' 'Applying hostname and domainname to Postfix' - postconf "myhostname = ${HOSTNAME}" - postconf "mydomain = ${DOMAINNAME}" -} diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index 8919a547e3e..0746c55cb03 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -107,10 +107,11 @@ function __environment_variables_general_setup VARS[DOVECOT_MAILBOX_FORMAT]="${DOVECOT_MAILBOX_FORMAT:=maildir}" VARS[DOVECOT_TLS]="${DOVECOT_TLS:=no}" + VARS[POSTFIX_DAGENT]="${POSTFIX_DAGENT:=}" VARS[POSTFIX_INET_PROTOCOLS]="${POSTFIX_INET_PROTOCOLS:=all}" VARS[POSTFIX_MAILBOX_SIZE_LIMIT]="${POSTFIX_MAILBOX_SIZE_LIMIT:=0}" VARS[POSTFIX_MESSAGE_SIZE_LIMIT]="${POSTFIX_MESSAGE_SIZE_LIMIT:=10240000}" # ~10 MB - VARS[POSTFIX_DAGENT]="${POSTFIX_DAGENT:=}" + VARS[POSTFIX_REJECT_UNKNOWN_CLIENT_HOSTNAME]="${POSTFIX_REJECT_UNKNOWN_CLIENT_HOSTNAME:=0}" _log 'trace' 'Setting SRS specific environment variables' From 95c812346d6f2abf0cdd9155efa2b3ac65a3b169 Mon Sep 17 00:00:00 2001 From: Andreas Perhab Date: Sun, 16 Apr 2023 22:17:58 +0200 Subject: [PATCH 037/592] config-examples: update fail2ban config examples with current DMS default values (#3258) Co-authored-by: Casper --- config-examples/fail2ban-fail2ban.cf | 49 ++++++++++++++++++++-------- config-examples/fail2ban-jail.cf | 12 +++++-- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/config-examples/fail2ban-fail2ban.cf b/config-examples/fail2ban-fail2ban.cf index 8ed2833c89a..00e9a25fec5 100644 --- a/config-examples/fail2ban-fail2ban.cf +++ b/config-examples/fail2ban-fail2ban.cf @@ -5,11 +5,11 @@ # Changes: in most of the cases you should not modify this # file, but provide customizations in fail2ban.local file, e.g.: # -# [Definition] +# [DEFAULT] # loglevel = DEBUG # -[Definition] +[DEFAULT] # Option: loglevel # Notes.: Set the log level output. @@ -19,26 +19,26 @@ # NOTICE # INFO # DEBUG -# Values: [ LEVEL ] Default: ERROR +# Values: [ LEVEL ] Default: INFO # -#loglevel = INFO +loglevel = INFO # Option: logtarget -# Notes.: Set the log target. This could be a file, SYSLOG, STDERR or STDOUT. +# Notes.: Set the log target. This could be a file, SYSTEMD-JOURNAL, SYSLOG, STDERR or STDOUT. # Only one log target can be specified. # If you change logtarget from the default value and you are # using logrotate -- also adjust or disable rotation in the # corresponding configuration file # (e.g. /etc/logrotate.d/fail2ban on Debian systems) -# Values: [ STDOUT | STDERR | SYSLOG | SYSOUT | FILE ] Default: STDERR +# Values: [ STDOUT | STDERR | SYSLOG | SYSOUT | SYSTEMD-JOURNAL | FILE ] Default: STDERR # -#logtarget = /var/log/fail2ban.log +logtarget = /var/log/fail2ban.log # Option: syslogsocket # Notes: Set the syslog socket file. Only used when logtarget is SYSLOG # auto uses platform.system() to determine predefined paths # Values: [ auto | FILE ] Default: auto -#syslogsocket = auto +syslogsocket = auto # Option: socket # Notes.: Set the socket file. This is used to communicate with the daemon. Do @@ -46,24 +46,47 @@ # communicate with the server afterwards. # Values: [ FILE ] Default: /var/run/fail2ban/fail2ban.sock # -#socket = /var/run/fail2ban/fail2ban.sock +socket = /var/run/fail2ban/fail2ban.sock # Option: pidfile # Notes.: Set the PID file. This is used to store the process ID of the # fail2ban server. # Values: [ FILE ] Default: /var/run/fail2ban/fail2ban.pid # -#pidfile = /var/run/fail2ban/fail2ban.pid +pidfile = /var/run/fail2ban/fail2ban.pid + +# Option: allowipv6 +# Notes.: Allows IPv6 interface: +# Default: auto +# Values: [ auto yes (on, true, 1) no (off, false, 0) ] Default: auto +#allowipv6 = auto # Options: dbfile # Notes.: Set the file for the fail2ban persistent data to be stored. -# A value of ":memory:" means database is only stored in memory +# A value of ":memory:" means database is only stored in memory # and data is lost when fail2ban is stopped. # A value of "None" disables the database. # Values: [ None :memory: FILE ] Default: /var/lib/fail2ban/fail2ban.sqlite3 -#dbfile = /var/lib/fail2ban/fail2ban.sqlite3 +dbfile = /var/lib/fail2ban/fail2ban.sqlite3 # Options: dbpurgeage # Notes.: Sets age at which bans should be purged from the database # Values: [ SECONDS ] Default: 86400 (24hours) -#dbpurgeage = 1d +dbpurgeage = 1d + +# Options: dbmaxmatches +# Notes.: Number of matches stored in database per ticket (resolvable via +# tags / in actions) +# Values: [ INT ] Default: 10 +dbmaxmatches = 10 + +[Definition] + + +[Thread] + +# Options: stacksize +# Notes.: Specifies the stack size (in KiB) to be used for subsequently created threads, +# and must be 0 or a positive integer value of at least 32. +# Values: [ SIZE ] Default: 0 (use platform or configured default) +#stacksize = 0 diff --git a/config-examples/fail2ban-jail.cf b/config-examples/fail2ban-jail.cf index 9611e7e0b66..73b5bd47cc0 100644 --- a/config-examples/fail2ban-jail.cf +++ b/config-examples/fail2ban-jail.cf @@ -1,14 +1,14 @@ [DEFAULT] # "bantime" is the number of seconds that a host is banned. -bantime = 3h +bantime = 1w # A host is banned if it has generated "maxretry" during the last "findtime" # seconds. -findtime = 10m +findtime = 1w # "maxretry" is the number of failures before a host get banned. -maxretry = 3 +maxretry = 2 # "ignoreip" can be a list of IP addresses, CIDR masks or DNS hosts. Fail2ban # will not ban a host which matches an address in this list. Several addresses @@ -25,9 +25,15 @@ enabled = true [postfix] enabled = true +# See https://github.com/fail2ban/fail2ban/blob/27294c4b9ee5d5568a1d5f83af744ea39d5a1acb/config/filter.d/postfix.conf#L58 +# `mode=aggressive` basically combines more filters to match more lines, and hence, apply rules +# more aggressively. The same goes for the `postfix-sasl` jail. +mode = aggressive + [postfix-sasl] enabled = true +mode = aggressive # This jail is used for manual bans. # To ban an IP address use: setup.sh fail2ban ban From 4b937fda5b5153cb4666f2ff5ef173c66b65ecd9 Mon Sep 17 00:00:00 2001 From: Andreas Perhab Date: Mon, 17 Apr 2023 10:42:35 +0200 Subject: [PATCH 038/592] shellcheck: do not check .git folder (#3267) --- test/linting/lint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/linting/lint.sh b/test/linting/lint.sh index 39c86250109..e30458977d0 100755 --- a/test/linting/lint.sh +++ b/test/linting/lint.sh @@ -51,7 +51,8 @@ function _shellcheck # File paths for shellcheck: F_SH=$(find . -type f -iname '*.sh' \ -not -path './test/bats/*' \ - -not -path './test/test_helper/*' + -not -path './test/test_helper/*' \ + -not -path './.git/*' ) # shellcheck disable=SC2248 F_BIN=$(find 'target/bin' -type f -not -name '*.py') From 3f22cbce01f5197e1ec4f5c380d5bc18d8194c26 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 17 Apr 2023 19:22:50 +0200 Subject: [PATCH 039/592] scripts: disallow alias = account (#3270) --- target/scripts/helpers/database/manage/postfix-accounts.sh | 4 ++++ target/scripts/helpers/database/manage/postfix-virtual.sh | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/target/scripts/helpers/database/manage/postfix-accounts.sh b/target/scripts/helpers/database/manage/postfix-accounts.sh index ad91982f84c..efaf05f34ed 100644 --- a/target/scripts/helpers/database/manage/postfix-accounts.sh +++ b/target/scripts/helpers/database/manage/postfix-accounts.sh @@ -74,6 +74,10 @@ function _arg_expect_mail_account function _account_should_not_exist_yet { __account_already_exists && _exit_with_error "'${MAIL_ACCOUNT}' already exists" + if [[ -f ${DATABASE_VIRTUAL} ]] && grep -q "^${MAIL_ACCOUNT}" "${DATABASE_VIRTUAL}" + then + _exit_with_error "'${MAIL_ACCOUNT}' is already defined as an alias" + fi } # Also used by delmailuser, setquota, delquota diff --git a/target/scripts/helpers/database/manage/postfix-virtual.sh b/target/scripts/helpers/database/manage/postfix-virtual.sh index c4108f90579..c847d630455 100644 --- a/target/scripts/helpers/database/manage/postfix-virtual.sh +++ b/target/scripts/helpers/database/manage/postfix-virtual.sh @@ -25,6 +25,10 @@ function _manage_virtual_aliases case "${ACTION}" in # Associate RECIPIENT to MAIL_ALIAS: ( 'update' ) + if [[ -f ${DATABASE_ACCOUNTS} ]] && grep -q "^${MAIL_ALIAS}" "${DATABASE_ACCOUNTS}" + then + _exit_with_error "'${MAIL_ALIAS}' is already defined as an account" + fi _db_entry_add_or_append "${DATABASE_VIRTUAL}" "${MAIL_ALIAS}" "${RECIPIENT}" ;; From 2f33f44f4a0f877c0115824aa7369f4d0a453b8f Mon Sep 17 00:00:00 2001 From: Andreas Perhab Date: Tue, 18 Apr 2023 11:08:19 +0200 Subject: [PATCH 040/592] postfix.sh: add missing -E for extended regexes in smtpd_sender_restrictions (#3272) --- target/scripts/startup/setup.d/postfix.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/scripts/startup/setup.d/postfix.sh b/target/scripts/startup/setup.d/postfix.sh index bd9feaa64e2..eb830760dce 100644 --- a/target/scripts/startup/setup.d/postfix.sh +++ b/target/scripts/startup/setup.d/postfix.sh @@ -77,7 +77,7 @@ function _setup_postfix_late __postfix__log 'trace' 'Configuring user access' if [[ -f /tmp/docker-mailserver/postfix-send-access.cf ]] then - sed -i 's|(smtpd_sender_restrictions =)|\1 check_sender_access texthash:/tmp/docker-mailserver/postfix-send-access.cf,|' /etc/postfix/main.cf + sed -i -E 's|(smtpd_sender_restrictions =)|\1 check_sender_access texthash:/tmp/docker-mailserver/postfix-send-access.cf,|' /etc/postfix/main.cf fi if [[ -f /tmp/docker-mailserver/postfix-receive-access.cf ]] From a735dddc52f4f77959c0a59cda96970a4826cc39 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 18 Apr 2023 10:07:08 -0500 Subject: [PATCH 041/592] scripts: fix setting `SRS_EXCLUDE_DOMAINS` during startup (#3271) --- target/scripts/startup/setup.d/postfix.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/target/scripts/startup/setup.d/postfix.sh b/target/scripts/startup/setup.d/postfix.sh index eb830760dce..34b35e6edb0 100644 --- a/target/scripts/startup/setup.d/postfix.sh +++ b/target/scripts/startup/setup.d/postfix.sh @@ -170,8 +170,8 @@ function _setup_SRS if [[ -n ${SRS_EXCLUDE_DOMAINS} ]] then - sed -i \ - "s/^#\?(SRS_EXCLUDE_DOMAINS=).*$/\1=${SRS_EXCLUDE_DOMAINS}/g" \ + sedfile -i -E \ + "s|^#?(SRS_EXCLUDE_DOMAINS=).*|\1${SRS_EXCLUDE_DOMAINS}|" \ /etc/default/postsrsd fi } From ea07bcdb4cadc78a98826a868ff84d18cb72a6f6 Mon Sep 17 00:00:00 2001 From: Casper Date: Tue, 18 Apr 2023 23:38:46 +0200 Subject: [PATCH 042/592] scripts: improve shutdown function by making PANIC_STRATEGY obsolete (#3265) --- target/scripts/helpers/accounts.sh | 2 +- target/scripts/helpers/error.sh | 23 ++++++-------------- target/scripts/helpers/utils.sh | 6 ++--- target/scripts/start-mailserver.sh | 4 ++-- target/scripts/startup/check-stack.sh | 2 +- target/scripts/startup/daemons-stack.sh | 2 +- target/scripts/startup/setup-stack.sh | 2 +- target/scripts/startup/setup.d/dovecot.sh | 4 ++-- target/scripts/startup/setup.d/networking.sh | 2 +- 9 files changed, 19 insertions(+), 28 deletions(-) diff --git a/target/scripts/helpers/accounts.sh b/target/scripts/helpers/accounts.sh index fcc6cf9699d..996e6896522 100644 --- a/target/scripts/helpers/accounts.sh +++ b/target/scripts/helpers/accounts.sh @@ -144,7 +144,7 @@ function _create_dovecot_alias_dummy_accounts if [[ -z ${REAL_ACC[1]} ]] then - _dms_panic__misconfigured 'postfix-accounts.cf' 'alias configuration' 'immediate' + _dms_panic__misconfigured 'postfix-accounts.cf' 'alias configuration' fi # test if user has a defined quota diff --git a/target/scripts/helpers/error.sh b/target/scripts/helpers/error.sh index 98e758920bd..e5662b22037 100644 --- a/target/scripts/helpers/error.sh +++ b/target/scripts/helpers/error.sh @@ -24,8 +24,7 @@ function dms_panic { local PANIC_TYPE=${1:-} local PANIC_INFO=${2:-} - local PANIC_SCOPE=${3-} # optional, must not be :- but just - - local PANIC_STRATEGY=${4:-} # optional + local PANIC_SCOPE=${3:-} local SHUTDOWN_MESSAGE @@ -61,9 +60,9 @@ function dms_panic if [[ -n ${PANIC_SCOPE:-} ]] then - _shutdown "${PANIC_SCOPE} | ${SHUTDOWN_MESSAGE}" "${PANIC_STRATEGY}" + _shutdown "${PANIC_SCOPE} | ${SHUTDOWN_MESSAGE}" else - _shutdown "${SHUTDOWN_MESSAGE}" "${PANIC_STRATEGY}" + _shutdown "${SHUTDOWN_MESSAGE}" fi } @@ -77,22 +76,14 @@ function _dms_panic__general { dms_panic 'general' "${1:-}" "${2:-}" # Call this method when you want to panic (i.e. emit an 'ERROR' log, and exit uncleanly). # `dms_panic` methods should be preferred if your failure type is supported. +trap "exit 1" SIGUSR1 +SCRIPT_PID=${$} function _shutdown { _log 'error' "${1:-_shutdown called without message}" _log 'error' 'Shutting down' sleep 1 - kill 1 - - if [[ ${2:-wait} == 'immediate' ]] - then - # In case the user requested an immediate exit, he ensure he is not in a subshell - # call and exiting the whole script is safe. This way, we make the shutdown quicker. - exit 1 - else - # We can simply wait until Supervisord has terminated all processes; this way, - # we do not return from a subshell call and continue as if nothing happened. - sleep 1000 - fi + kill -SIGTERM 1 # Trigger graceful DMS shutdown. + kill -SIGUSR1 "${SCRIPT_PID}" # Stop start-mailserver.sh execution, even when _shutdown() is called from a subshell. } diff --git a/target/scripts/helpers/utils.sh b/target/scripts/helpers/utils.sh index bd4d8e3c984..d65b9e23ee2 100644 --- a/target/scripts/helpers/utils.sh +++ b/target/scripts/helpers/utils.sh @@ -92,13 +92,13 @@ function _replace_by_env_in_file { if [[ -z ${1+set} ]] then - _dms_panic__invalid_value 'first argument unset' 'utils.sh:_replace_by_env_in_file' 'immediate' + _dms_panic__invalid_value 'first argument unset' 'utils.sh:_replace_by_env_in_file' elif [[ -z ${2+set} ]] then - _dms_panic__invalid_value 'second argument unset' 'utils.sh:_replace_by_env_in_file' 'immediate' + _dms_panic__invalid_value 'second argument unset' 'utils.sh:_replace_by_env_in_file' elif [[ ! -f ${2} ]] then - _dms_panic__invalid_value "file '${2}' does not exist" 'utils.sh:_replace_by_env_in_file' 'immediate' + _dms_panic__invalid_value "file '${2}' does not exist" 'utils.sh:_replace_by_env_in_file' fi local ENV_PREFIX=${1} CONFIG_FILE=${2} diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index ff187f115bd..3532311fd1e 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -61,11 +61,11 @@ function _register_functions ;; ( 'OIDC' ) - _dms_panic__fail_init 'OIDC user account provisioning - it is not yet implemented' '' 'immediate' + _dms_panic__fail_init 'OIDC user account provisioning - it is not yet implemented' ;; ( * ) - _dms_panic__invalid_value "'${ACCOUNT_PROVISIONER}' is not a valid value for ACCOUNT_PROVISIONER" '' 'immediate' + _dms_panic__invalid_value "'${ACCOUNT_PROVISIONER}' is not a valid value for ACCOUNT_PROVISIONER" ;; esac diff --git a/target/scripts/startup/check-stack.sh b/target/scripts/startup/check-stack.sh index 8fb7073fc4d..7b7fdfca5cb 100644 --- a/target/scripts/startup/check-stack.sh +++ b/target/scripts/startup/check-stack.sh @@ -38,7 +38,7 @@ function _check_hostname # HOSTNAME should be an FQDN (eg: hostname.domain) if ! grep -q -E '^(\S+[.]\S+)$' <<< "${HOSTNAME}" then - _dms_panic__general 'Setting hostname/domainname is required' '' 'immediate' + _dms_panic__general 'Setting hostname/domainname is required' fi } diff --git a/target/scripts/startup/daemons-stack.sh b/target/scripts/startup/daemons-stack.sh index ac2a85c2d63..b09428c6bc5 100644 --- a/target/scripts/startup/daemons-stack.sh +++ b/target/scripts/startup/daemons-stack.sh @@ -29,7 +29,7 @@ function _default_start_daemon if [[ ${?} -ne 0 ]] then _log 'error' "${RESULT}" - _dms_panic__fail_init "${1}" '' 'immediate' + _dms_panic__fail_init "${1}" fi } diff --git a/target/scripts/startup/setup-stack.sh b/target/scripts/startup/setup-stack.sh index 0cd357277c3..efcbd33ebf5 100644 --- a/target/scripts/startup/setup-stack.sh +++ b/target/scripts/startup/setup-stack.sh @@ -89,7 +89,7 @@ function _setup_apply_fixes_after_configuration _log 'debug' 'Checking /var/mail permissions' if ! _chown_var_mail_if_necessary then - _dms_panic__general 'Failed to fix /var/mail permissions' '' 'immediate' + _dms_panic__general 'Failed to fix /var/mail permissions' fi _log 'debug' 'Removing files and directories from older versions' diff --git a/target/scripts/startup/setup.d/dovecot.sh b/target/scripts/startup/setup.d/dovecot.sh index 2d03b030ae5..60a04a6c933 100644 --- a/target/scripts/startup/setup.d/dovecot.sh +++ b/target/scripts/startup/setup.d/dovecot.sh @@ -175,7 +175,7 @@ function _setup_dovecot_local_user fi done - _dms_panic__fail_init 'accounts provisioning because no accounts were provided - Dovecot could not be started' '' 'immediate' + _dms_panic__fail_init 'accounts provisioning because no accounts were provided - Dovecot could not be started' } __wait_until_an_account_is_added_or_shutdown @@ -199,7 +199,7 @@ function _setup_dovecot_inet_protocols PROTOCOL='[::]' # IPv6 only else # Unknown value, panic. - _dms_panic__invalid_value 'DOVECOT_INET_PROTOCOLS' "${DOVECOT_INET_PROTOCOLS}" 'immediate' + _dms_panic__invalid_value 'DOVECOT_INET_PROTOCOLS' "${DOVECOT_INET_PROTOCOLS}" fi sedfile -i "s|^#listen =.*|listen = ${PROTOCOL}|g" /etc/dovecot/dovecot.conf diff --git a/target/scripts/startup/setup.d/networking.sh b/target/scripts/startup/setup.d/networking.sh index 396db87576f..13751766d1e 100644 --- a/target/scripts/startup/setup.d/networking.sh +++ b/target/scripts/startup/setup.d/networking.sh @@ -22,7 +22,7 @@ function _setup_docker_permit if [[ -z ${CONTAINER_IP} ]] then _log 'error' 'Detecting the container IP address failed' - _dms_panic__misconfigured 'NETWORK_INTERFACE' 'Network Setup [docker_permit]' 'immediate' + _dms_panic__misconfigured 'NETWORK_INTERFACE' 'Network Setup [docker_permit]' fi while read -r IP From e10ca569f1907b044ac5a43f77dff1451219b38f Mon Sep 17 00:00:00 2001 From: Casper Date: Tue, 18 Apr 2023 23:40:21 +0200 Subject: [PATCH 043/592] misc: make Fail2Ban log persistent (#3269) --- Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index b66070de7a9..1af200de8ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -90,7 +90,7 @@ COPY \ RUN < Date: Wed, 19 Apr 2023 11:16:47 +0200 Subject: [PATCH 044/592] ci: update `bug_report.yml` (#3275) --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 723e1f97bd7..77a5bce7e2a 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -3,7 +3,7 @@ description: File a bug report title: 'bug report: ' labels: - kind/bug/report - - meta/bug/needs triage + - meta/needs triage body: - type: markdown From 7371ba225f0a304088ccc7fcb44f7c885e7e43d0 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 19 Apr 2023 11:21:21 +0200 Subject: [PATCH 045/592] ci: simplify `bug_report.yml` (#3276) The extra checks for reading the code of conduct are now in one place; also removed a double-check on searching the docs and the issue tracker. --- .github/ISSUE_TEMPLATE/bug_report.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 77a5bce7e2a..55e8083c2ef 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -36,6 +36,8 @@ body: required: true - label: I read the [documentation on debugging](https://docker-mailserver.github.io/docker-mailserver/latest/config/debugging/), tried the proposed steps to debug the problem, but was still unable to resolve the issue. required: true + - label: I have read this project's [Code of Conduct](https://github.com/docker-mailserver/docker-mailserver/blob/master/CODE_OF_CONDUCT.md) and I agree + required: true - type: input id: affected-components attributes: @@ -153,16 +155,6 @@ body: - label: I am rather experienced with mail servers - label: I am uncomfortable with the CLI - label: I am rather comfortable with the CLI - - type: checkboxes - id: terms-code-of-conduct - attributes: - label: Code of conduct - description: By submitting this issue, you agree to follow [our code of conduct](https://github.com/docker-mailserver/docker-mailserver/blob/master/CODE_OF_CONDUCT.md). - options: - - label: I have read this project's [Code of Conduct](https://github.com/docker-mailserver/docker-mailserver/blob/master/CODE_OF_CONDUCT.md) and I agree - required: true - - label: I have read the [README](https://github.com/docker-mailserver/docker-mailserver/blob/master/README.md) and the [documentation](https://docker-mailserver.github.io/docker-mailserver/edge/) and I searched the [issue tracker](https://github.com/docker-mailserver/docker-mailserver/issues?q=is%3Aissue) but could not find a solution - required: true - type: input id: form-improvements attributes: From 2b330fdc493d983f54a8f8868393d496e1e5a611 Mon Sep 17 00:00:00 2001 From: Andreas Perhab Date: Thu, 20 Apr 2023 09:52:07 +0200 Subject: [PATCH 046/592] scripts: remove superfluous `EOF` in `dmarc_dkim_spf.sh` (#3266) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- target/scripts/startup/setup.d/dmarc_dkim_spf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh index bb92e774e27..2401fce4a07 100644 --- a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh +++ b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh @@ -103,7 +103,7 @@ EOF /etc/postfix/main.cf # SPF policy settings postconf 'policyd-spf_time_limit = 3600' -EOF + else _log 'debug' 'Disabling policyd-spf' fi From 1c9ed6ce3298a02186885a2864fd78eb18b4bd0f Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 21 Apr 2023 12:21:20 +0200 Subject: [PATCH 047/592] docs: improve Rspamd docs (part of its stabilization) (#3257) * revise links in docs * added information about `soft_reject_on_timeout` * added `ENABLE_POLICYD_SPF=0` to basic Rspamd setup docs --- docs/content/config/security/rspamd.md | 46 +++++++++++++++++--------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 1f40c623a05..7dbf819c46d 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -10,7 +10,7 @@ title: 'Security | Rspamd' ## About -Rspamd is a ["fast, free and open-source spam filtering system"][homepage]. DMS integrates Rspamd like any other service. We provide a very simple but easy to maintain setup of Rspamd. +Rspamd is a ["fast, free and open-source spam filtering system"][rspamd-homepage]. DMS integrates Rspamd like any other service. We provide a very simple but easy to maintain setup of Rspamd. If you want to have a look at the default configuration files for Rspamd that DMS packs, navigate to [`target/rspamd/` inside the repository][dms-default-configuration]. Please consult the [section "The Default Configuration"](#the-default-configuration) section down below for a written overview. @@ -18,6 +18,9 @@ If you want to have a look at the default configuration files for Rspamd that DM We are currently doing a best-effort installation of Rspamd for ARM64 (from the Debian backports repository for Debian 11). The current version difference as of 1st Apr 2023: AMD64 is at version 3.5 | ARM64 is at version 3.4. +[rspamd-homepage]: https://rspamd.com/ +[dms-default-configuration]: https://github.com/docker-mailserver/docker-mailserver/tree/master/target/rspamd + ## Related Environment Variables The following environment variables are related to Rspamd: @@ -36,10 +39,12 @@ With these variables, you can enable Rspamd itself and you can enable / disable ### Mode of Operation -The proxy worker operates in [self-scan mode][proxy-self-scan-mode]. This simplifies the setup as we do not require a normal worker. You can easily change this though by [overriding the configuration by DMS](#providing-custom-settings-overriding-settings). +The proxy worker operates in [self-scan mode][rspamd-docs-proxy-self-scan-mode]. This simplifies the setup as we do not require a normal worker. You can easily change this though by [overriding the configuration by DMS](#providing-custom-settings-overriding-settings). DMS does not set a default password for the controller worker. You may want to do that yourself. In setups where you already have an authentication provider in front of the Rspamd webpage, you may want to [set the `secure_ip ` option to `"0.0.0.0/0"` for the controller worker](#with-the-help-of-a-custom-file) to disable password authentication inside Rspamd completely. +[rspamd-docs-proxy-self-scan-mode]: https://rspamd.com/doc/workers/rspamd_proxy.html#self-scan-mode + ### Persistence with Redis When Rspamd is enabled, we implicitly also start an instance of Redis in the container. Redis is configured to persist it's data via RDB snapshots to disk in the directory `/var/lib/redis` (_which is a symbolic link to `/var/mail-state/lib-redis/` when [`ONE_DIR=1`](../environment.md#one_dir) and a volume is mounted to `/var/mail-state/`_). With the volume mount the snapshot will restore the Redis data across container restarts, and provide a way to keep backup. @@ -54,9 +59,25 @@ Rspamd provides a [web interface][rspamc-docs-web-interface], which contains sta [rspamc-docs-web-interface]: https://rspamd.com/webui/ +### DNS + +DMS does not supply custom values for DNS servers to Rspamd. If you need to use custom DNS servers, which could be required when using [DNS-based black/whitelists](#rbls-realtime-blacklists-dnsbls-dns-based-blacklists), you need to adjust [`options.inc`][rspamd-docs-basic-options] yourself. + +!!! tip "Making DNS Servers Configurable" + + If you want to see an environment variable (like `RSPAMD_DNS_SERVERS`) to support custom DNS servers for Rspamd being added to DMS, please raise a feature request issue. + +!!! danger + + While we do not provide values for custom DNS servers by default, we set `soft_reject_on_timeout = true;` by default. This setting will cause a soft reject if a task (presumably a DNS request) timeout takes place. + + This setting is enabled to not allow spam to proceed just because DNS requests did not succeed. It could deny legitimate e-mails to pass though too in case your DNS setup is incorrect or not functioning properly. + ### Modules -You can find a list of all Rspamd modules [on their website][modules]. +You can find a list of all Rspamd modules [on their website][rspamd-docs-modules]. + +[rspamd-docs-modules]: https://rspamd.com/doc/modules/ #### Disabled By Default @@ -76,6 +97,8 @@ The [RBL module](https://rspamd.com/doc/modules/rbl.html) is enabled by default. If you want to use DNSBLs, **try to use your own DNS resolver** and make sure it is set up correctly, i.e. it should be a non-public & **recursive** resolver. Otherwise, you might not be able ([see this Spamhaus post](https://www.spamhaus.org/faq/section/DNSBL%20Usage#365)) to make use of the block lists. +[rbl-vs-dnsbl]: https://forum.eset.com/topic/25277-dnsbl-vs-rbl-mail-security/?do=findComment&comment=119818 + ## Providing Custom Settings & Overriding Settings ### Manually @@ -104,7 +127,7 @@ where `COMMAND` can be: 3. `set-option-for-module`: sets the value for option `ARGUMENT2` to `ARGUMENT3` inside module `ARGUMENT1` 4. `set-option-for-controller`: set the value of option `ARGUMENT1` to `ARGUMENT2` for the controller worker 5. `set-option-for-proxy`: set the value of option `ARGUMENT1` to `ARGUMENT2` for the proxy worker -6. `set-common-option`: set the option `ARGUMENT1` that [defines basic Rspamd behaviour][basic-options] to value `ARGUMENT2` +6. `set-common-option`: set the option `ARGUMENT1` that [defines basic Rspamd behaviour][rspamd-docs-basic-options] to value `ARGUMENT2` 7. `add-line`: this will add the complete line after `ARGUMENT1` (with all characters) to the file `/etc/rspamd/override.d/` !!! example "An Example Is [Shown Down Below](#adjusting-and-extending-the-very-basic-configuration)" @@ -119,6 +142,9 @@ You can also have comments (the line starts with `#`) and blank lines in `rspamd These simple commands are meant to give users the ability to _easily_ alter modules and their options. As a consequence, they are not powerful enough to enable multi-line adjustments. If you need to do something more complex, we advise to do that [manually](#manually)! +[docs-volumes-config]: ../advanced/optional-config.md +[rspamd-docs-basic-options]: https://rspamd.com/doc/configuration/options.html + ## Examples & Advanced Configuration ### A Very Basic Configuration @@ -129,6 +155,7 @@ You want to start using Rspamd? Rspamd is disabled by default, so you need to se ENABLE_RSPAMD=1 ENABLE_OPENDKIM=0 ENABLE_OPENDMARC=0 +ENABLE_POLICYD_SPF=0 ENABLE_AMAVIS=0 ENABLE_SPAMASSASSIN=0 ``` @@ -165,14 +192,3 @@ While _Abusix_ can be integrated into Postfix, Postscreen and a multitude of oth [Abusix]: https://abusix.com/ [abusix-rspamd-integration]: https://docs.abusix.com/abusix-mail-intelligence/gbG8EcJ3x3fSUv8cMZLiwA/getting-started/dmw9dcwSGSNQiLTssFAnBW#rspamd - -[//]: # (General Links) - -[docs-volumes-config]: ../advanced/optional-config.md -[homepage]: https://rspamd.com/ -[modules]: https://rspamd.com/doc/modules/ -[proxy-self-scan-mode]: https://rspamd.com/doc/workers/rspamd_proxy.html#self-scan-mode -[dms-default-configuration]: https://github.com/docker-mailserver/docker-mailserver/tree/master/target/rspamd -[rbl-vs-dnsbl]: https://forum.eset.com/topic/25277-dnsbl-vs-rbl-mail-security/?do=findComment&comment=119818 -[dkim-signing-module]: https://rspamd.com/doc/modules/dkim_signing.html -[basic-options]: https://rspamd.com/doc/configuration/options.html From 88cd244e47411fe7250a6315a19781ffc5a97806 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 23 Apr 2023 12:16:53 +0200 Subject: [PATCH 048/592] scripts: misc improvements (#3281) * corrected typo * corrected indentation --- target/scripts/startup/setup.d/dmarc_dkim_spf.sh | 11 +++++------ target/scripts/startup/setup.d/security/misc.sh | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh index 2401fce4a07..2825898800d 100644 --- a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh +++ b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh @@ -98,12 +98,11 @@ policyd-spf unix - n n - 0 spawn user=policyd-spf argv=/usr/bin/policyd-spf EOF - sedfile -i -E \ - 's|^(smtpd_recipient_restrictions.*reject_unauth_destination)(.*)|\1, check_policy_service unix:private/policyd-spf\2|' \ - /etc/postfix/main.cf - # SPF policy settings - postconf 'policyd-spf_time_limit = 3600' - + # SPF policy settings + postconf 'policyd-spf_time_limit = 3600' + sedfile -i -E \ + 's|^(smtpd_recipient_restrictions.*reject_unauth_destination)(.*)|\1, check_policy_service unix:private/policyd-spf\2|' \ + /etc/postfix/main.cf else _log 'debug' 'Disabling policyd-spf' fi diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index 085b632bcee..b517d513a55 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -52,7 +52,7 @@ function __setup__security__postgrey cp -f /tmp/docker-mailserver/whitelist_recipients /etc/postgrey/whitelist_recipients fi else - _log 'debug' 'Postscreen is disabled' + _log 'debug' 'Postgrey is disabled' fi } From 638975922e124556df0266fc8a470e2aeb7e7298 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 23 Apr 2023 12:22:54 +0200 Subject: [PATCH 049/592] scripts: Rspamd stabilization pt. 1 (#3261) * added checks whether OpenDKIM/OpenDMARC/policyd-spf are enabled * added functions to check if VAR is 0/0 or an int and also added tests. I also adjusted the test file to not run in a container, because there is no need. This also decreases test time, which, in turn, increases maintainers' happiness. * added more checks to Rspamd setup I added the helpers from the previous commit to the Rspamd setup to make the whole setup more robust, and indicate to the user that an ENV variable's value is incorrect. While we did not issues for this in the past, I believe it to be worthwhile for the future. * added canonical directory for users to place files in This dir is canonical with DMS's optional configuration dirs, as it lives in well-known volume mounts. Hence, users will not need to adjust `/etc/rspamd/override.d` manually anymore, or mount a volume to this place. The docs explain this now, but the DKIM page needs a slight update on this too I guess. I will follow-up here. * misc minor improvements * use variables for common directories --- .../config/advanced/optional-config.md | 2 + docs/content/config/security/rspamd.md | 9 ++- target/scripts/helpers/utils.sh | 31 +++++++++ .../startup/setup.d/security/rspamd.sh | 68 ++++++++++++++----- .../parallel/set1/spam_virus/rspamd.bats | 2 + .../set3/scripts/helper_functions.bats | 68 +++++++++++++++---- 6 files changed, 147 insertions(+), 33 deletions(-) diff --git a/docs/content/config/advanced/optional-config.md b/docs/content/config/advanced/optional-config.md index e9835501ab6..45a717f5158 100644 --- a/docs/content/config/advanced/optional-config.md +++ b/docs/content/config/advanced/optional-config.md @@ -12,6 +12,7 @@ This is a list of all configuration files and directories which are optional or - **sieve-pipe:** directory for sieve pipe scripts. (Docs: [Sieve][docs-sieve]) - **opendkim:** DKIM directory. Auto-configurable via [`setup.sh config dkim`][docs-setupsh]. (Docs: [DKIM][docs-dkim]) - **ssl:** SSL Certificate directory if `SSL_TYPE` is set to `self-signed` or `custom`. (Docs: [SSL][docs-ssl]) +- **Rspamd:** Override directory for custom settings when using Rspamd (Docs: [Rspamd][docs-rspamd-override-d]) ## Files @@ -54,5 +55,6 @@ This is a list of all configuration files and directories which are optional or [docs-sieve]: ./mail-sieve.md [docs-setupsh]: ../../config/setup.sh.md [docs-ssl]: ../../config/security/ssl.md +[docs-rspamd-override-d]: ../security/rspamd.md#manually [docs-rspamd-commands]: ../security/rspamd.md#with-the-help-of-a-custom-file [github-commit-setup-stack.sh-L411]: https://github.com/docker-mailserver/docker-mailserver/blob/941e7acdaebe271eaf3d296b36d4d81df4c54b90/target/scripts/startup/setup-stack.sh#L411 diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 7dbf819c46d..b4b48f793f6 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -101,17 +101,20 @@ The [RBL module](https://rspamd.com/doc/modules/rbl.html) is enabled by default. ## Providing Custom Settings & Overriding Settings +DMS brings sane default settings for Rspamd. They are located at `/etc/rspamd/local.d/` inside the container (or `target/rspamd/local.d/` in the repository). + ### Manually -DMS brings sane default settings for Rspamd. They are located at `/etc/rspamd/local.d/` inside the container (or `target/rspamd/local.d/` in the repository). If you want to change these settings and / or provide your own settings, you can +If you want to overwrite the default settings and / or provide your own settings, you can place files at `docker-data/dms/config/rspamd/override.d/` (a directory that is linked to `/etc/rspamd/override.d/`, if it exists) to override Rspamd and DMS default settings. -1. place files at `/etc/rspamd/override.d/` which will override Rspamd settings and DMS settings -2. (re-)place files at `/etc/rspamd/local.d/` to override DMS settings and merge them with Rspamd settings +!!! note "What is [`docker-data/dms/config/`][docs-dms-config-volume]?" !!! warning "Clashing Overrides" Note that when also [using the `rspamd-commands` file](#with-the-help-of-a-custom-file), files in `override.d` may be overwritten in case you adjust them manually and with the help of the file. +[docs-dms-config-volume]: ../../faq.md#what-about-the-docker-datadmsmail-state-folder + ### With the Help of a Custom File DMS provides the ability to do simple adjustments to Rspamd modules with the help of a single file. Just place a file called `rspamd-modules.conf` into the [local config directory `docker-data/dms/config/`][docs-volumes-config]. If this file is present, DMS will evaluate it. The structure is _very_ simple. Each line in the file looks like this: diff --git a/target/scripts/helpers/utils.sh b/target/scripts/helpers/utils.sh index d65b9e23ee2..b9b36f646a6 100644 --- a/target/scripts/helpers/utils.sh +++ b/target/scripts/helpers/utils.sh @@ -114,3 +114,34 @@ function _replace_by_env_in_file sed -i -E "s#^${ESCAPED_KEY}[[:space:]]*=.*#${ESCAPED_KEY} =${ESCAPED_VALUE}#g" "${CONFIG_FILE}" done < <(env | grep "^${ENV_PREFIX}") } + +# Check if an environment variable's value is zero or one. This aids in checking variables +# that act as "booleans" for enabling or disabling a service, configuration option, etc. +# +# This function will log a warning and return with exit code 1 in case the variable's value +# is not zero or one. +# +# @param ${1} = name of the ENV variable to check +function _env_var_expect_zero_or_one +{ + local ENV_VAR_NAME=${1:?ENV var name must be provided to _env_var_expect_zero_or_one} + + [[ ${!ENV_VAR_NAME} =~ ^(0|1)$ ]] && return 0 + _log 'warn' "The value of '${ENV_VAR_NAME}' is not zero or one ('${!ENV_VAR_NAME}'), but was expected to be" + return 1 +} + +# Check if an environment variable's value is an integer. +# +# This function will log a warning and return with exit code 1 in case the variable's value +# is not an integer. +# +# @param ${1} = name of the ENV variable to check +function _env_var_expect_integer +{ + local ENV_VAR_NAME=${1:?ENV var name must be provided to _env_var_expect_integer} + + [[ ${!ENV_VAR_NAME} =~ ^-?[0-9][0-9]*$ ]] && return 0 + _log 'warn' "The value of '${ENV_VAR_NAME}' is not an integer ('${!ENV_VAR_NAME}'), but was expected to be" + return 1 +} diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index d7feb8024c4..bd7f03e074d 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -3,10 +3,11 @@ # Function called during global setup to handle the complete setup of Rspamd. function _setup_rspamd { - if [[ ${ENABLE_RSPAMD} -eq 1 ]] + if _env_var_expect_zero_or_one 'ENABLE_RSPAMD' && [[ ${ENABLE_RSPAMD} -eq 1 ]] then _log 'warn' 'Rspamd integration is work in progress - expect changes at any time' _log 'debug' 'Enabling and configuring Rspamd' + __rspamd__log 'trace' '---------- Setup started ----------' __rspamd__run_early_setup_and_checks # must run first __rspamd__setup_redis @@ -18,7 +19,7 @@ function _setup_rspamd __rspamd__setup_hfilter_group __rspamd__handle_user_modules_adjustments # must run last - _log 'trace' 'Rspamd setup finished' + __rspamd__log 'trace' '---------- Setup finished ----------' else _log 'debug' 'Rspamd is disabled' fi @@ -65,23 +66,61 @@ EOF # or checking for other anti-spam/anti-virus software. function __rspamd__run_early_setup_and_checks { + # Note: Variables not marked with `local` are + # used in other functions as well. + RSPAMD_LOCAL_D='/etc/rspamd/local.d' + RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' + RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' + local RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d/" + mkdir -p /var/lib/rspamd/ : >/var/lib/rspamd/stats.ucl + if [[ -d ${RSPAMD_DMS_OVERRIDE_D} ]] + then + __rspamd__log 'debug' "Found directory '${RSPAMD_DMS_OVERRIDE_D}' - linking it to '${RSPAMD_OVERRIDE_D}'" + if rmdir "${RSPAMD_OVERRIDE_D}" + then + ln -s "${RSPAMD_DMS_OVERRIDE_D}" "${RSPAMD_OVERRIDE_D}" + else + __rspamd__log 'warn' "Could not remove '${RSPAMD_OVERRIDE_D}' - not linking '${RSPAMD_DMS_OVERRIDE_D}'" + fi + fi + if [[ ${ENABLE_AMAVIS} -eq 1 ]] || [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] then __rspamd__log 'warn' 'Running Amavis/SA & Rspamd at the same time is discouraged' fi + + if [[ ${ENABLE_OPENDKIM} -eq 1 ]] + then + __rspamd__log 'warn' 'Running OpenDKIM & Rspamd at the same time is discouraged - we recommend Rspamd for DKIM checks (enabled with Rspamd by default) & signing' + fi + + if [[ ${ENABLE_OPENDMARC} -eq 1 ]] + then + __rspamd__log 'warn' 'Running OpenDMARC & Rspamd at the same time is discouraged - we recommend Rspamd for DMARC checks (enabled with Rspamd by default)' + fi + + if [[ ${ENABLE_POLICYD_SPF} -eq 1 ]] + then + __rspamd__log 'warn' 'Running policyd-spf & Rspamd at the same time is discouraged - we recommend Rspamd for SPF checks (enabled with Rspamd by default)' + fi + + if [[ ${ENABLE_POSTGREY} -eq 1 ]] && [[ ${RSPAMD_GREYLISTING} -eq 1 ]] + then + __rspamd__log 'warn' 'Running Postgrey & Rspamd at the same time is discouraged - we recommend Rspamd for greylisting' + fi } # Sets up Redis. In case the user does not use a dedicated Redis instance, we # supply a configuration for our local Redis instance which is started later. function __rspamd__setup_redis { - if [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] + if _env_var_expect_zero_or_one 'ENABLE_RSPAMD_REDIS' && [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] then __rspamd__log 'debug' 'Internal Redis is enabled, adding configuration' - cat >/etc/rspamd/local.d/redis.conf << "EOF" + cat >"${RSPAMD_LOCAL_D}/redis.conf" << "EOF" # documentation: https://rspamd.com/doc/configuration/redis.html servers = "127.0.0.1:6379"; @@ -120,10 +159,10 @@ function __rspamd__setup_postfix # If ClamAV is enabled, we will integrate it into Rspamd. function __rspamd__setup_clamav { - if [[ ${ENABLE_CLAMAV} -eq 1 ]] + if _env_var_expect_zero_or_one 'ENABLE_CLAMAV' && [[ ${ENABLE_CLAMAV} -eq 1 ]] then __rspamd__log 'debug' 'Enabling ClamAV integration' - sedfile -i -E 's|^(enabled).*|\1 = true;|g' /etc/rspamd/local.d/antivirus.conf + sedfile -i -E 's|^(enabled).*|\1 = true;|g' "${RSPAMD_LOCAL_D}/antivirus.conf" # Rspamd uses ClamAV's UNIX socket, and to be able to read it, it must be in the same group usermod -a -G clamav _rspamd else @@ -165,7 +204,7 @@ function __rspamd__setup_default_modules # from or to the "Junk" folder, and learning them as ham or spam. function __rspamd__setup_learning { - if [[ ${RSPAMD_LEARN} -eq 1 ]] + if _env_var_expect_zero_or_one 'RSPAMD_LEARN' && [[ ${RSPAMD_LEARN} -eq 1 ]] then __rspamd__log 'debug' 'Setting up intelligent learning of spam and ham' @@ -210,10 +249,10 @@ EOF # https://rspamd.com/doc/modules/greylisting.html). function __rspamd__setup_greylisting { - if [[ ${RSPAMD_GREYLISTING} -eq 1 ]] + if _env_var_expect_zero_or_one 'RSPAMD_GREYLISTING' && [[ ${RSPAMD_GREYLISTING} -eq 1 ]] then __rspamd__log 'debug' 'Enabling greylisting' - sedfile -i -E "s|(enabled =).*|\1 true;|g" /etc/rspamd/local.d/greylist.conf + sedfile -i -E "s|(enabled =).*|\1 true;|g" "${RSPAMD_LOCAL_D}/greylist.conf" else __rspamd__log 'debug' 'Greylisting is disabled' fi @@ -225,15 +264,12 @@ function __rspamd__setup_greylisting # succeeds. function __rspamd__setup_hfilter_group { - local MODULE_FILE='/etc/rspamd/local.d/hfilter_group.conf' - if [[ ${RSPAMD_HFILTER} -eq 1 ]] + local MODULE_FILE="${RSPAMD_LOCAL_D}/hfilter_group.conf" + if _env_var_expect_zero_or_one 'RSPAMD_HFILTER' && [[ ${RSPAMD_HFILTER} -eq 1 ]] then __rspamd__log 'debug' 'Hfilter (group) module is enabled' # Check if we received a number first - if [[ ! ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE} =~ ^[0-9][1-9]*$ ]] - then - __rspamd__log 'warn' "'RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE' is not a number (${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE}) but was expected to be!" - elif [[ ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE} -ne 6 ]] + if _env_var_expect_integer 'RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE' && [[ ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE} -ne 6 ]] then __rspamd__log 'trace' "Adjusting score for 'HFILTER_HOSTNAME_UNKNOWN' in Hfilter group module to ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE}" sed -i -E \ @@ -275,7 +311,7 @@ function __rspamd__handle_user_modules_adjustments # remove possible whitespace at the end (e.g., in case ${ARGUMENT3} is empty) VALUE=${VALUE% } - local FILE="/etc/rspamd/override.d/${MODULE_FILE}" + local FILE="${RSPAMD_OVERRIDE_D}/${MODULE_FILE}" [[ -f ${FILE} ]] || touch "${FILE}" if grep -q -E "${OPTION}.*=.*" "${FILE}" diff --git a/test/tests/parallel/set1/spam_virus/rspamd.bats b/test/tests/parallel/set1/spam_virus/rspamd.bats index ea57e81f473..2a4f15f0a43 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd.bats @@ -16,6 +16,8 @@ function setup_file() { --env ENABLE_RSPAMD=1 --env ENABLE_OPENDKIM=0 --env ENABLE_OPENDMARC=0 + --env ENABLE_POLICYD_SPF=0 + --env ENABLE_POSTGREY=0 --env PERMIT_DOCKER=host --env LOG_LEVEL=trace --env MOVE_SPAM_TO_JUNK=1 diff --git a/test/tests/parallel/set3/scripts/helper_functions.bats b/test/tests/parallel/set3/scripts/helper_functions.bats index 1a62414aaf8..95db61d83d4 100644 --- a/test/tests/parallel/set3/scripts/helper_functions.bats +++ b/test/tests/parallel/set3/scripts/helper_functions.bats @@ -1,23 +1,63 @@ -load "${REPOSITORY_ROOT}/test/helper/setup" load "${REPOSITORY_ROOT}/test/helper/common" -BATS_TEST_NAME_PREFIX='[Scripts] (helper functions inside container) ' -CONTAINER_NAME='dms-test_helper_functions' +BATS_TEST_NAME_PREFIX='[Scripts] (helper functions) ' +SOURCE_BASE_PATH="${REPOSITORY_ROOT:?Expected REPOSITORY_ROOT to be set}/target/scripts/helpers" -function setup_file() { - _init_with_defaults - _common_container_setup +@test '(network.sh) _sanitize_ipv4_to_subnet_cidr' { + source "${SOURCE_BASE_PATH}/network.sh" + + run _sanitize_ipv4_to_subnet_cidr '255.255.255.255/0' + assert_output '0.0.0.0/0' + + run _sanitize_ipv4_to_subnet_cidr '192.168.255.14/20' + assert_output '192.168.240.0/20' + + run _sanitize_ipv4_to_subnet_cidr '192.168.255.14/32' + assert_output '192.168.255.14/32' +} + +@test '(utils.sh) _env_var_expect_zero_or_one' { + source "${SOURCE_BASE_PATH}/log.sh" + source "${SOURCE_BASE_PATH}/utils.sh" + + ZERO=0 + ONE=1 + TWO=2 + + run _env_var_expect_zero_or_one ZERO + assert_success + + run _env_var_expect_zero_or_one ONE + assert_success + + run _env_var_expect_zero_or_one TWO + assert_failure + assert_output --partial "The value of 'TWO' is not zero or one ('2'), but was expected to be" + + run _env_var_expect_zero_or_one + assert_failure + assert_output --partial "ENV var name must be provided to _env_var_expect_zero_or_one" } -function teardown_file() { _default_teardown ; } +@test '(utils.sh) _env_var_expect_integer' { + source "${SOURCE_BASE_PATH}/log.sh" + source "${SOURCE_BASE_PATH}/utils.sh" + + INTEGER=1234 + NEGATIVE=-${INTEGER} + NaN=not_an_integer + + run _env_var_expect_integer INTEGER + assert_success -@test "_sanitize_ipv4_to_subnet_cidr" { - _run_in_container_bash "source /usr/local/bin/helpers/index.sh; _sanitize_ipv4_to_subnet_cidr 255.255.255.255/0" - assert_output "0.0.0.0/0" + run _env_var_expect_integer NEGATIVE + assert_success - _run_in_container_bash "source /usr/local/bin/helpers/index.sh; _sanitize_ipv4_to_subnet_cidr 192.168.255.14/20" - assert_output "192.168.240.0/20" + run _env_var_expect_integer NaN + assert_failure + assert_output --partial "The value of 'NaN' is not an integer ('not_an_integer'), but was expected to be" - _run_in_container_bash "source /usr/local/bin/helpers/index.sh; _sanitize_ipv4_to_subnet_cidr 192.168.255.14/32" - assert_output "192.168.255.14/32" + run _env_var_expect_integer + assert_failure + assert_output --partial "ENV var name must be provided to _env_var_expect_integer" } From cd1721334ce4e7a6cd21f7e7f3a84f2a32244db7 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 23 Apr 2023 14:02:56 +0200 Subject: [PATCH 050/592] scripts: Rspamd stabilization pt. 2 (#3282) * move modules adjustment file to new location Because we link `/tmp/docker-mailserver/rspamd/override.d` to `/etc/rspamd/override.d`, I think it makes sense to move the modules adjustment file into `/tmp/docker-mailserver/rspamd/` as well. I write the code in a way that it is backwards compatible for now, so this is NOT a breaking change. * minor improvement to `__rspamd__handle_user_modules_adjustments` The expansion of `ARGUMENT3` is now done in a way that only adds the whitespace in case the variable is set and not null. * move test file structure to respect latest changes Because we're now linking `rspamd/override.d/`, we can simplify the setup a bit. But this requires a change in directory structure. The current Rspamd test will be renamed to `rspamd_full.bats`, because I plan on adding more tests in different files for different feature sets. This is done to make this feature well-tested! * improved and added tests to Rspamd-full FYI: The line ```bats _run_in_container grep 'sieve_global_extensions.*\+vnd\.dovecot\.pipe' "${SIEVE_CONFIG_FILE}" ``` was testing a condition that should actually not be met, but when I started working on this feature, I thought this was the correct configuration. Adding the `assert_success` statements revealed this wrong line. I also added tests to check whether `override.d` is linked correctly. * renamed: `rspamd.bats` => `rspamd_full.bats` * added new tests for incomplete Rspamd feature set We now test that warnings are emitted & features are disabled correctly. * update documentation --- .../config/advanced/optional-config.md | 2 +- docs/content/config/security/rspamd.md | 14 ++-- .../startup/setup.d/security/rspamd.sh | 19 ++++- test/config/rspamd/user-patches.sh | 13 --- .../postfix-accounts.cf | 0 .../rspamd/custom-commands.conf} | 0 .../override.d/testmodule_complicated.conf | 3 + test/config/rspamd_full/user-patches.sh | 16 ++++ .../{rspamd.bats => rspamd_full.bats} | 35 ++++++-- .../set1/spam_virus/rspamd_partly.bats | 84 +++++++++++++++++++ 10 files changed, 155 insertions(+), 31 deletions(-) delete mode 100644 test/config/rspamd/user-patches.sh rename test/config/{rspamd => rspamd_full}/postfix-accounts.cf (100%) rename test/config/{rspamd-modules.conf => rspamd_full/rspamd/custom-commands.conf} (100%) create mode 100644 test/config/rspamd_full/rspamd/override.d/testmodule_complicated.conf create mode 100644 test/config/rspamd_full/user-patches.sh rename test/tests/parallel/set1/spam_virus/{rspamd.bats => rspamd_full.bats} (89%) create mode 100644 test/tests/parallel/set1/spam_virus/rspamd_partly.bats diff --git a/docs/content/config/advanced/optional-config.md b/docs/content/config/advanced/optional-config.md index 45a717f5158..a5266681fec 100644 --- a/docs/content/config/advanced/optional-config.md +++ b/docs/content/config/advanced/optional-config.md @@ -40,7 +40,7 @@ This is a list of all configuration files and directories which are optional or - **dovecot.cf:** replaces `/etc/dovecot/local.conf`. (Docs: [Override Dovecot Defaults][docs-override-dovecot]) - **dovecot-quotas.cf:** list of custom quotas per mailbox. (Docs: [Accounts][docs-accounts-quota]) - **user-patches.sh:** this file will be run after all configuration files are set up, but before the postfix, amavis and other daemons are started. (Docs: [FAQ - How to adjust settings with the `user-patches.sh` script][docs-faq-userpatches]) -- **rspamd-commands:** list of simple commands to adjust Rspamd modules in an easy way (Docs: [Rspamd][docs-rspamd-commands]) +- **rspamd/custom-commands.conf:** list of simple commands to adjust Rspamd modules in an easy way (Docs: [Rspamd][docs-rspamd-commands]) [docs-accounts-quota]: ../../config/user-management.md#quotas [docs-aliases-regex]: ../../config/user-management.md#configuring-regexp-aliases diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index b4b48f793f6..f017ca7f4fd 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -16,7 +16,7 @@ If you want to have a look at the default configuration files for Rspamd that DM !!! note "AMD64 vs ARM64" - We are currently doing a best-effort installation of Rspamd for ARM64 (from the Debian backports repository for Debian 11). The current version difference as of 1st Apr 2023: AMD64 is at version 3.5 | ARM64 is at version 3.4. + We are currently doing a best-effort installation of Rspamd for ARM64 (from the Debian backports repository for Debian 11). The current version difference as of 23rd Apr 2023: AMD64 is at version 3.5 | ARM64 is at version 3.4. [rspamd-homepage]: https://rspamd.com/ [dms-default-configuration]: https://github.com/docker-mailserver/docker-mailserver/tree/master/target/rspamd @@ -117,7 +117,7 @@ If you want to overwrite the default settings and / or provide your own settings ### With the Help of a Custom File -DMS provides the ability to do simple adjustments to Rspamd modules with the help of a single file. Just place a file called `rspamd-modules.conf` into the [local config directory `docker-data/dms/config/`][docs-volumes-config]. If this file is present, DMS will evaluate it. The structure is _very_ simple. Each line in the file looks like this: +DMS provides the ability to do simple adjustments to Rspamd modules with the help of a single file. Just place a file called `custom-commands.conf` into `docker-data/dms/config/rspamd/`. If this file is present, DMS will evaluate it. The structure is _very_ simple. Each line in the file looks like this: ```txt COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3 @@ -139,13 +139,12 @@ where `COMMAND` can be: For command 1 - 3, we append the `.conf` suffix to the module name to get the correct file name automatically. For commands 4 - 6, the file name is fixed (you don't even need to provide it). For command 7, you will need to provide the whole file name (including the suffix) yourself! -You can also have comments (the line starts with `#`) and blank lines in `rspamd-modules.conf` - they are properly handled and not evaluated. +You can also have comments (the line starts with `#`) and blank lines in `custom-commands.conf` - they are properly handled and not evaluated. !!! tip "Adjusting Modules This Way" These simple commands are meant to give users the ability to _easily_ alter modules and their options. As a consequence, they are not powerful enough to enable multi-line adjustments. If you need to do something more complex, we advise to do that [manually](#manually)! -[docs-volumes-config]: ../advanced/optional-config.md [rspamd-docs-basic-options]: https://rspamd.com/doc/configuration/options.html ## Examples & Advanced Configuration @@ -169,10 +168,9 @@ This will enable Rspamd and disable services you don't need when using Rspamd. N Rspamd is running, but you want or need to adjust it? -1. Say you want to be able to easily look at the frontend Rspamd provides on port 11334 (default) without the need to enter a password (maybe because you already provide authorization and authentication). You will need to adjust the controller worker: create a file called `rspamd-modules.conf` and add the line `set-option-for-controller secure_ip "0.0.0.0/0"`. Place the file `rspamd-modules.conf` inside the directory on the host you mount to `/tmp/docker-mailserver/` inside the container (in our documentation, we use `docker-data/dms/config` on the host for this purpose). And you're done! Note: this disables authentication on the website - make sure you know what you're doing! -2. You additionally want to enable the auto-spam-learning for the Bayes module? No problem, just add another line to `rspamd-modules.conf` that looks like this: `set-option-for-module classifier-bayes autolearn true`. -3. But the chartable module gets on your nerves? Just disable it by adding another line: `disable-module chartable -`. +1. Say you want to be able to easily look at the frontend Rspamd provides on port 11334 (default) without the need to enter a password (maybe because you already provide authorization and authentication). You will need to adjust the controller worker: create a file called `custom-commands.conf` and add the line `set-option-for-controller secure_ip "0.0.0.0/0"`. Place the file `custom-commands.conf` inside the directory on the host you mount to `/tmp/docker-mailserver/rspamd/` inside the container (in our documentation, we use `docker-data/dms/config/rspamd/` on the host for this purpose). And you're done! Note: this disables authentication on the website - make sure you know what you're doing! +2. You additionally want to enable the auto-spam-learning for the Bayes module? No problem, just add another line to `custom-commands.conf` that looks like this: `set-option-for-module classifier-bayes autolearn true`. +3. But the chartable module gets on your nerves? Just disable it by adding another line: `disable-module chartable`. ### DKIM Signing diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index bd7f03e074d..cee7c2a900e 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -292,7 +292,7 @@ function __rspamd__handle_user_modules_adjustments # Adds an option with a corresponding value to a module, or, in case the option # is already present, overwrites it. # - # @param ${1} = file name in /etc/rspamd/override.d/ + # @param ${1} = file name in ${RSPAMD_OVERRIDE_D}/ # @param ${2} = module name as it should appear in the log # @patam ${3} = option name in the module # @param ${4} = value of the option @@ -324,10 +324,21 @@ function __rspamd__handle_user_modules_adjustments fi } - local RSPAMD_CUSTOM_COMMANDS_FILE='/tmp/docker-mailserver/rspamd-modules.conf' + local RSPAMD_CUSTOM_COMMANDS_FILE="${RSPAMD_DMS_D}/custom-commands.conf" + local RSPAMD_CUSTOM_COMMANDS_FILE_OLD="${RSPAMD_DMS_D}-modules.conf" + + # We check for usage of the previous location of the commands file. + # This can be removed after the release of v14.0.0. + if [[ -f ${RSPAMD_CUSTOM_COMMANDS_FILE_OLD} ]] + then + __rspamd__log 'warn' "Detected usage of old file location for modules adjustment ('${RSPAMD_CUSTOM_COMMANDS_FILE_OLD}') - please use the new location ('${RSPAMD_CUSTOM_COMMANDS_FILE}')" + __rspamd__log 'warn' "Using old file location now (deprecated) - this will prevent startup in v13.0.0" + RSPAMD_CUSTOM_COMMANDS_FILE=${RSPAMD_CUSTOM_COMMANDS_FILE_OLD} + fi + if [[ -f "${RSPAMD_CUSTOM_COMMANDS_FILE}" ]] then - __rspamd__log 'debug' "Found file 'rspamd-modules.conf' - parsing and applying it" + __rspamd__log 'debug' "Found file '${RSPAMD_CUSTOM_COMMANDS_FILE}' - parsing and applying it" while read -r COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3 do @@ -359,7 +370,7 @@ function __rspamd__handle_user_modules_adjustments ('add-line') __rspamd__log 'trace' "Adding complete line to '${ARGUMENT1}'" - echo "${ARGUMENT2} ${ARGUMENT3:-}" >>"/etc/rspamd/override.d/${ARGUMENT1}" + echo "${ARGUMENT2}${ARGUMENT3+ ${ARGUMENT3}}" >>"${RSPAMD_OVERRIDE_D}/${ARGUMENT1}" ;; (*) diff --git a/test/config/rspamd/user-patches.sh b/test/config/rspamd/user-patches.sh deleted file mode 100644 index 00e6ca97944..00000000000 --- a/test/config/rspamd/user-patches.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -cat >/etc/rspamd/override.d/testmodule_complicated.conf << EOF -complicated { - anOption = someValue; -} -EOF - -echo "enable_test_patterns = true;" >>/etc/rspamd/local.d/options.inc - -echo 'mail_debug = yes' >>/etc/dovecot/dovecot.conf -sed -i -E '/^}/d' /etc/dovecot/conf.d/90-sieve.conf -echo -e 'sieve_trace_debug = yes\n}' >>/etc/dovecot/conf.d/90-sieve.conf diff --git a/test/config/rspamd/postfix-accounts.cf b/test/config/rspamd_full/postfix-accounts.cf similarity index 100% rename from test/config/rspamd/postfix-accounts.cf rename to test/config/rspamd_full/postfix-accounts.cf diff --git a/test/config/rspamd-modules.conf b/test/config/rspamd_full/rspamd/custom-commands.conf similarity index 100% rename from test/config/rspamd-modules.conf rename to test/config/rspamd_full/rspamd/custom-commands.conf diff --git a/test/config/rspamd_full/rspamd/override.d/testmodule_complicated.conf b/test/config/rspamd_full/rspamd/override.d/testmodule_complicated.conf new file mode 100644 index 00000000000..212b4fb3518 --- /dev/null +++ b/test/config/rspamd_full/rspamd/override.d/testmodule_complicated.conf @@ -0,0 +1,3 @@ +complicated { + anOption = someValue; +} diff --git a/test/config/rspamd_full/user-patches.sh b/test/config/rspamd_full/user-patches.sh new file mode 100644 index 00000000000..9f4dc9fb985 --- /dev/null +++ b/test/config/rspamd_full/user-patches.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# Enable GTUBE test patterns so we can properly check whether +# Rspamd is rejecting mail, adding headers, etc. +# +# We do not use `custom-commands.conf` because this a feature +# we are testing too. +echo "enable_test_patterns = true;" >>/etc/rspamd/local.d/options.inc + +# We want Dovecot to be very detailed about what it is doing, +# specificially for Sieve because we need to check whether the +# Sieve scripts are executed so Rspamd is trained when using +# `RSPAMD_LEARN=1`. +echo 'mail_debug = yes' >>/etc/dovecot/dovecot.conf +sed -i -E '/^}/d' /etc/dovecot/conf.d/90-sieve.conf +echo -e '\n sieve_trace_debug = yes\n}' >>/etc/dovecot/conf.d/90-sieve.conf diff --git a/test/tests/parallel/set1/spam_virus/rspamd.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats similarity index 89% rename from test/tests/parallel/set1/spam_virus/rspamd.bats rename to test/tests/parallel/set1/spam_virus/rspamd_full.bats index 2a4f15f0a43..d0c22a8bbfa 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -1,8 +1,10 @@ load "${REPOSITORY_ROOT}/test/helper/setup" load "${REPOSITORY_ROOT}/test/helper/common" -BATS_TEST_NAME_PREFIX='[Rspamd] ' -CONTAINER_NAME='dms-test_rspamd' +# This file tests Rspamd when all of its features are enabled, and +# all other interfering features are disabled. +BATS_TEST_NAME_PREFIX='[Rspamd] (full) ' +CONTAINER_NAME='dms-test_rspamd-full' function setup_file() { _init_with_defaults @@ -27,7 +29,7 @@ function setup_file() { --env RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE=7 ) - mv "${TEST_TMP_CONFIG}"/rspamd/* "${TEST_TMP_CONFIG}/" + cp -r "${TEST_TMP_CONFIG}"/rspamd_full/* "${TEST_TMP_CONFIG}/" _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' # wait for ClamAV to be fully setup or we will get errors on the log @@ -62,7 +64,28 @@ function teardown_file() { _default_teardown ; } assert_output 'rspamd_milter = inet:localhost:11332' } -@test 'logs exist and contains proper content' { +@test "'/etc/rspamd/override.d/' is linked correctly" { + local OVERRIDE_D='/etc/rspamd/override.d' + + _run_in_container_bash "[[ -h ${OVERRIDE_D} ]]" + assert_success + + _run_in_container_bash "[[ -f ${OVERRIDE_D}/testmodule_complicated.conf ]]" + assert_success +} + +@test 'startup log shows all features as properly enabled' { + run docker logs "${CONTAINER_NAME}" + assert_success + assert_line --partial 'Enabling ClamAV integration' + assert_line --partial 'Setting up intelligent learning of spam and ham' + assert_line --partial 'Enabling greylisting' + assert_line --partial 'Hfilter (group) module is enabled' + assert_line --partial "Adjusting score for 'HFILTER_HOSTNAME_UNKNOWN' in Hfilter group module to" + assert_line --partial "Found file '/tmp/docker-mailserver/rspamd/custom-commands.conf' - parsing and applying it" +} + +@test 'service log exist and contains proper content' { _service_log_should_contain_string 'rspamd' 'rspamd .* is loading configuration' _service_log_should_contain_string 'rspamd' 'lua module clickhouse is disabled in the configuration' _service_log_should_contain_string 'rspamd' 'lua module elastic is disabled in the configuration' @@ -200,10 +223,12 @@ function teardown_file() { _default_teardown ; } done _run_in_container grep 'mail_plugins.*imap_sieve' /etc/dovecot/conf.d/20-imap.conf + assert_success local SIEVE_CONFIG_FILE='/etc/dovecot/conf.d/90-sieve.conf' _run_in_container grep 'sieve_plugins.*sieve_imapsieve' "${SIEVE_CONFIG_FILE}" - _run_in_container grep 'sieve_global_extensions.*\+vnd\.dovecot\.pipe' "${SIEVE_CONFIG_FILE}" + assert_success _run_in_container grep -F 'sieve_pipe_bin_dir = /usr/lib/dovecot/sieve-pipe' "${SIEVE_CONFIG_FILE}" + assert_success # Move an email to the "Junk" folder from "INBOX"; the first email we # sent should pass fine, hence we can now move it diff --git a/test/tests/parallel/set1/spam_virus/rspamd_partly.bats b/test/tests/parallel/set1/spam_virus/rspamd_partly.bats new file mode 100644 index 00000000000..29b7bc0e6e3 --- /dev/null +++ b/test/tests/parallel/set1/spam_virus/rspamd_partly.bats @@ -0,0 +1,84 @@ +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" + +# This file tests Rspamd when some of its features are enabled, and +# some other interfering features are enabled. +BATS_TEST_NAME_PREFIX='[Rspamd] (partly) ' +CONTAINER_NAME='dms-test_rspamd-partly' + +function setup_file() { + _init_with_defaults + + # Comment for maintainers about `PERMIT_DOCKER=host`: + # https://github.com/docker-mailserver/docker-mailserver/pull/2815/files#r991087509 + local CUSTOM_SETUP_ARGUMENTS=( + --env ENABLE_AMAVIS=1 + --env ENABLE_SPAMASSASSIN=1 + --env ENABLE_CLAMAV=0 + --env ENABLE_RSPAMD=1 + --env ENABLE_OPENDKIM=1 + --env ENABLE_OPENDMARC=1 + --env ENABLE_POLICYD_SPF=1 + --env ENABLE_POSTGREY=0 + --env PERMIT_DOCKER=host + --env LOG_LEVEL=trace + --env MOVE_SPAM_TO_JUNK=0 + --env RSPAMD_LEARN=0 + --env RSPAMD_GREYLISTING=0 + --env RSPAMD_HFILTER=0 + ) + + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + + _wait_for_service rspamd-redis + _wait_for_service rspamd + _wait_for_service amavis + _wait_for_service postfix + _wait_for_smtp_port_in_container +} + +function teardown_file() { _default_teardown ; } + +@test "log warns about interfering features" { + run docker logs "${CONTAINER_NAME}" + assert_success + for SERVICE in 'Amavis/SA' 'OpenDKIM' 'OpenDMARC' 'policyd-spf' + do + assert_output --regexp ".*WARNING.*Running ${SERVICE} & Rspamd at the same time is discouraged" + done +} + +@test 'log shows all features as properly disabled' { + run docker logs "${CONTAINER_NAME}" + assert_success + assert_line --partial 'Rspamd will not use ClamAV (which has not been enabled)' + assert_line --partial 'Intelligent learning of spam and ham is disabled' + assert_line --partial 'Greylisting is disabled' + assert_line --partial 'Disabling Hfilter (group) module' +} + +@test 'learning is properly disabled' { + for FILE in learn-{ham,spam}.{sieve,svbin} + do + _run_in_container_bash "[[ -f /usr/lib/dovecot/sieve-pipe/${FILE} ]]" + assert_failure + done + + _run_in_container grep 'mail_plugins.*imap_sieve' /etc/dovecot/conf.d/20-imap.conf + assert_failure + local SIEVE_CONFIG_FILE='/etc/dovecot/conf.d/90-sieve.conf' + _run_in_container grep -F 'imapsieve_mailbox1_name = Junk' "${SIEVE_CONFIG_FILE}" + assert_failure + _run_in_container grep -F 'imapsieve_mailbox1_causes = COPY' "${SIEVE_CONFIG_FILE}" + assert_failure +} + +@test 'greylisting is properly disabled' { + _run_in_container grep -F 'enabled = false;' '/etc/rspamd/local.d/greylist.conf' + assert_success +} + +@test 'hfilter group module configuration is deleted' { + _run_in_container_bash '[[ -f /etc/rspamd/local.d/hfilter_group.conf ]]' + assert_failure +} From 449d53fc3f8fd7e2be829dc24ea63d0ce3c86b70 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 23 Apr 2023 15:14:36 +0200 Subject: [PATCH 051/592] docs/scripts: remove WIP warnings for Rspamd (#3283) --- docs/content/config/environment.md | 6 +----- docs/content/config/security/rspamd.md | 6 ------ target/scripts/startup/setup.d/security/rspamd.sh | 5 ++--- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index bea6b93b475..a681ef32985 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -313,11 +313,7 @@ will be automatically moved to the Junk folder (with the help of a Sieve script) ##### ENABLE_RSPAMD -Enable or disable Rspamd. - -!!! warning "Current State" - - Rspamd-support is under active development. Be aware that changes can happen at any time. To get more information, see [the detailed documentation page for Rspamd][docs-rspamd]. +Enable or disable [Rspamd][docs-rspamd]. - **0** => disabled - 1 => enabled diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index f017ca7f4fd..ba3b6d8add6 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -2,12 +2,6 @@ title: 'Security | Rspamd' --- -!!! warning "The current state of Rspamd integration into DMS" - - Recent pull requests have stabilized integration of Rspamd to a point that we encourage users to test the feature. We are confident that there are no major bugs in our integration that make using Rspamd infeasible. Please note that there may still be changes ahead as integration is still work in progress! - - We expect to stabilize this feature with version `v12.1.0`. - ## About Rspamd is a ["fast, free and open-source spam filtering system"][rspamd-homepage]. DMS integrates Rspamd like any other service. We provide a very simple but easy to maintain setup of Rspamd. diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index cee7c2a900e..1daf0188297 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -5,11 +5,10 @@ function _setup_rspamd { if _env_var_expect_zero_or_one 'ENABLE_RSPAMD' && [[ ${ENABLE_RSPAMD} -eq 1 ]] then - _log 'warn' 'Rspamd integration is work in progress - expect changes at any time' _log 'debug' 'Enabling and configuring Rspamd' __rspamd__log 'trace' '---------- Setup started ----------' - __rspamd__run_early_setup_and_checks # must run first + __rspamd__run_early_setup_and_checks # must run first __rspamd__setup_redis __rspamd__setup_postfix __rspamd__setup_clamav @@ -17,7 +16,7 @@ function _setup_rspamd __rspamd__setup_learning __rspamd__setup_greylisting __rspamd__setup_hfilter_group - __rspamd__handle_user_modules_adjustments # must run last + __rspamd__handle_user_modules_adjustments # must run last __rspamd__log 'trace' '---------- Setup finished ----------' else From 7e7497ae5a46e0da7b966b27bd20ad9f56b8298d Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 24 Apr 2023 14:35:19 +0200 Subject: [PATCH 052/592] scripts: apply fixes to helpers when using `set -eE` (#3285) For an upcoming PR, these changes are required, because the script that is using the helpers uses `set -eE`. This leads to situations where errors are not properly handled in our helpers (yet; I plan on changing that in the future). --- target/scripts/helpers/dns.sh | 2 +- target/scripts/helpers/error.sh | 23 +++++++++++++++++++++++ target/scripts/helpers/log.sh | 4 ++-- target/scripts/helpers/utils.sh | 11 ++++++++++- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/target/scripts/helpers/dns.sh b/target/scripts/helpers/dns.sh index 4301fa7d577..0a600959984 100644 --- a/target/scripts/helpers/dns.sh +++ b/target/scripts/helpers/dns.sh @@ -41,7 +41,7 @@ function _obtain_hostname_and_domainname # of this variable or if a more deterministic approach with `cut` should be relied on. if [[ $(_get_label_count "${HOSTNAME}") -gt 2 ]] then - if [[ -n ${OVERRIDE_HOSTNAME} ]] + if [[ -n ${OVERRIDE_HOSTNAME:-} ]] then # Emulates the intended behaviour of `hostname -d`: # Assign the HOSTNAME value minus everything up to and including the first `.` diff --git a/target/scripts/helpers/error.sh b/target/scripts/helpers/error.sh index e5662b22037..d28d5341133 100644 --- a/target/scripts/helpers/error.sh +++ b/target/scripts/helpers/error.sh @@ -87,3 +87,26 @@ function _shutdown kill -SIGTERM 1 # Trigger graceful DMS shutdown. kill -SIGUSR1 "${SCRIPT_PID}" # Stop start-mailserver.sh execution, even when _shutdown() is called from a subshell. } + +# Calling this function sets up a handler for the `ERR` signal, that occurs when +# an error is not properly checked (e.g., in an `if`-clause or in an `&&` block). +# +# This is mostly useful for debugging. It also helps when using something like `set -eE`, +# as it shows where the script aborts. +function _trap_err_signal +{ + trap '__log_unexpected_error "${FUNCNAME[0]:-}" "${BASH_COMMAND:-}" "${LINENO:-}" "${?:-}"' ERR + + # shellcheck disable=SC2317 + function __log_unexpected_error + { + local MESSAGE="Unexpected error occured :: script = ${SCRIPT:-${0}} " + MESSAGE+=" | function = ${1:-none (global)}" + MESSAGE+=" | command = ${2:-?}" + MESSAGE+=" | line = ${3:-?}" + MESSAGE+=" | exit code = ${4:-?}" + + _log 'error' "${MESSAGE}" + return 0 + } +} diff --git a/target/scripts/helpers/log.sh b/target/scripts/helpers/log.sh index 80d8c4b0698..925df2e62d6 100644 --- a/target/scripts/helpers/log.sh +++ b/target/scripts/helpers/log.sh @@ -123,9 +123,9 @@ function _get_log_level_or_default if [[ -n ${LOG_LEVEL+set} ]] then echo "${LOG_LEVEL}" - elif [[ -e /etc/dms-settings ]] + elif [[ -e /etc/dms-settings ]] && grep -q -E "^LOG_LEVEL='[a-z]+'" /etc/dms-settings then - grep "^LOG_LEVEL=" /etc/dms-settings | cut -d "'" -f 2 + grep '^LOG_LEVEL=' /etc/dms-settings | cut -d "'" -f 2 else echo 'info' fi diff --git a/target/scripts/helpers/utils.sh b/target/scripts/helpers/utils.sh index b9b36f646a6..e06ff837476 100644 --- a/target/scripts/helpers/utils.sh +++ b/target/scripts/helpers/utils.sh @@ -16,7 +16,13 @@ function _get_valid_lines_from_file # and it will return its value stored in /etc/dms-settings function _get_dms_env_value { - grep "^${1}=" /etc/dms-settings | cut -d "'" -f 2 + if [[ -f /etc/dms-settings ]] + then + grep "^${1}=" /etc/dms-settings | cut -d "'" -f 2 + else + _log 'warn' "Call to '_get_dms_env_value' but '/etc/dms-settings' is not present" + return 1 + fi } # TODO: `chown -R 5000:5000 /var/mail` has existed since the projects first commit. @@ -33,6 +39,7 @@ function _chown_var_mail_if_necessary _log 'trace' 'Fixing /var/mail permissions' chown -R 5000:5000 /var/mail || return 1 fi + return 0 } function _require_n_parameters_or_print_usage @@ -43,6 +50,7 @@ function _require_n_parameters_or_print_usage [[ ${1:-} == 'help' ]] && { __usage ; exit 0 ; } [[ ${#} -lt ${COUNT} ]] && { __usage ; exit 1 ; } + return 0 } # NOTE: Postfix commands that read `main.cf` will stall execution, @@ -56,6 +64,7 @@ function _adjust_mtime_for_postfix_maincf then touch -d '2 seconds ago' /etc/postfix/main.cf fi + return 0 } function _reload_postfix From 223c76632013aa3e9207279a6c13fe5c3ffd077e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Apr 2023 15:02:26 +0200 Subject: [PATCH 053/592] chore(deps): Bump docker/metadata-action from 4.3.0 to 4.4.0 (#3287) Bumps [docker/metadata-action](https://github.com/docker/metadata-action) from 4.3.0 to 4.4.0. - [Release notes](https://github.com/docker/metadata-action/releases) - [Commits](https://github.com/docker/metadata-action/compare/v4.3.0...v4.4.0) --- updated-dependencies: - dependency-name: docker/metadata-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 2c921919550..a87aee82d12 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -23,7 +23,7 @@ jobs: - name: 'Prepare tags' id: prep - uses: docker/metadata-action@v4.3.0 + uses: docker/metadata-action@v4.4.0 with: images: | ${{ secrets.DOCKER_REPOSITORY }} From f9d55a938414d30c7143b67a83d3ef1f710a6689 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 26 Apr 2023 10:05:42 +0200 Subject: [PATCH 054/592] docs: update F2B docs & bind mount links (#3293) --- docs/content/config/security/fail2ban.md | 140 +++++++++++------------ docs/content/config/security/rspamd.md | 6 +- docs/content/faq.md | 4 + docs/content/usage.md | 2 +- 4 files changed, 74 insertions(+), 78 deletions(-) diff --git a/docs/content/config/security/fail2ban.md b/docs/content/config/security/fail2ban.md index fedcc64f63d..ae91d902754 100644 --- a/docs/content/config/security/fail2ban.md +++ b/docs/content/config/security/fail2ban.md @@ -4,117 +4,109 @@ hide: - toc # Hide Table of Contents for this page --- -Fail2Ban is installed automatically and bans IP addresses for 1 week after 2 failed attempts in a time frame of 1 week by default. +!!! quote "What is Fail2Ban (F2B)?" -## Configuration files + Fail2ban is an intrusion prevention software framework. Written in the Python programming language, it is designed to prevent against brute-force attacks. It is able to run on POSIX systems that have an interface to a packet-control system or firewall installed locally, such as \[NFTables\] or TCP Wrapper. -If you want to change this, you can easily edit our github example file: [`config-examples/fail2ban-jail.cf`][github-file-f2bjail]. + [Source][wikipedia-fail2ban] -You can do the same with the values from `fail2ban.conf`, e.g `dbpurgeage`. In that case you need to edit: [`config-examples/fail2ban-fail2ban.cf`][github-file-f2bconfig]. + [wikipedia-fail2ban]: https://en.wikipedia.org/wiki/Fail2ban -The configuration files need to be located at the root of the `/tmp/docker-mailserver/` volume bind (usually `./docker-data/dms/config/:/tmp/docker-mailserver/`). +## Configuration -This following configuration files from `/tmp/docker-mailserver/` will be copied during container startup. +!!! warning -- `fail2ban-jail.cf` -> `/etc/fail2ban/jail.d/user-jail.local` -- `fail2ban-fail2ban.cf` -> `/etc/fail2ban/fail2ban.local` - -### Docker-compose config - -Example configuration volume bind: - -```yaml - volumes: - - ./docker-data/dms/config/:/tmp/docker-mailserver/ -``` - -!!! attention - - DMS must be launched with the `NET_ADMIN` capability in order to be able to install the nftables rules that actually ban IP addresses. - - Thus either include `--cap-add=NET_ADMIN` in the `docker run` command, or the equivalent in `docker-compose.yml`: + DMS must be launched with the `NET_ADMIN` capability in order to be able to install the NFTables rules that actually ban IP addresses. Thus, either include `--cap-add=NET_ADMIN` in the `docker run` command, or the equivalent in the `compose.yml`: ```yaml cap_add: - NET_ADMIN ``` -## Running fail2ban in a rootless container +!!! bug "Running Fail2Ban on Older Kernels" -[`RootlessKit`][rootless::rootless-kit] is the _fakeroot_ implementation for supporting _rootless mode_ in Docker and Podman. By default RootlessKit uses the [`builtin` port forwarding driver][rootless::port-drivers], which does not propagate source IP addresses. + DMS configures F2B to use NFTables, not IPTables (legacy). We have observed that older systems, for example NAS systems, do not support the modern NFTables rules. You will need to configure F2B to use legacy IPTables again, for example with the [``fail2ban-jail.cf``][github-file-f2bjail], see the [section on configuration further down below](#custom-files). -It is necessary for `fail2ban` to have access to the real source IP addresses in order to correctly identify clients. This is achieved by changing the port forwarding driver to [`slirp4netns`][rootless::slirp4netns], which is slower than `builtin` but does preserve the real source IPs. +### DMS Defaults -### Docker with `slirp4netns` port driver +DMS will automatically ban IP addresses of hosts that have generated 2 failed attempts over the course of the last week. The bans themselves last for one week. -For [rootless mode][rootless::docker] in Docker, create `~/.config/systemd/user/docker.service.d/override.conf` with the following content: +### Custom Files -``` -[Service] -Environment="DOCKERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER=slirp4netns" -``` +!!! question "What is [`docker-data/dms/config/`][docs-dms-config-volume]?" + +This following configuration files inside the `docker-data/dms/config/` volume will be copied inside the container during startup + +1. `fail2ban-jail.cf` is copied to `/etc/fail2ban/jail.d/user-jail.local` + - with this file, you can adjust the configuration of individual jails and their defaults + - the is an example provided [in our repository on GitHub][github-file-f2bjail] +2. `fail2ban-fail2ban.cf` is copied to `/etc/fail2ban/fail2ban.local` + - with this file, you can adjust F2B behavior in general + - the is an example provided [in our repository on GitHub][github-file-f2bconfig] + +[docs-dms-config-volume]: ../../faq.md#what-about-the-docker-datadmsconfig-directory +[github-file-f2bjail]: https://github.com/docker-mailserver/docker-mailserver/blob/master/config-examples/fail2ban-jail.cf +[github-file-f2bconfig]: https://github.com/docker-mailserver/docker-mailserver/blob/master/config-examples/fail2ban-fail2ban.cf -And then restart the daemon: +### Managing Bans -```console -$ systemctl --user daemon-reload -$ systemctl --user restart docker +You can manage F2B with the `setup` script. The usage looks like this: + +```bash +docker exec setup fail2ban [ ] ``` -!!! note +When just running `setup fail2ban`, the script will show all banned IP addresses. - This changes the port driver for all rootless containers managed by Docker. +## Running Inside A Rootless Container - Per container configuration is not supported, if you need that consider Podman instead. +[`RootlessKit`][rootless::rootless-kit] is the _fakeroot_ implementation for supporting _rootless mode_ in Docker and Podman. By default, RootlessKit uses the [`builtin` port forwarding driver][rootless::port-drivers], which does not propagate source IP addresses. -### Podman with `slirp4netns` port driver +It is necessary for F2B to have access to the real source IP addresses in order to correctly identify clients. This is achieved by changing the port forwarding driver to [`slirp4netns`][rootless::slirp4netns], which is slower than the builtin driver but does preserve the real source IPs. -[Rootless Podman][rootless::podman] requires adding the value `slirp4netns:port_handler=slirp4netns` to the `--network` CLI option, or `network_mode` setting in your `docker-compose.yml`. +[rootless::rootless-kit]: https://github.com/rootless-containers/rootlesskit +[rootless::port-drivers]: https://github.com/rootless-containers/rootlesskit/blob/v0.14.5/docs/port.md#port-drivers +[rootless::slirp4netns]: https://github.com/rootless-containers/slirp4netns +=== "Docker" -You must also add the ENV `NETWORK_INTERFACE=tap0`, because Podman uses a [hard-coded interface name][rootless::podman::interface] for `slirp4netns`. + For [rootless mode][rootless::docker] in Docker, create `~/.config/systemd/user/docker.service.d/override.conf` with the following content: + !!! danger inline end -!!! example + This changes the port driver for all rootless containers managed by Docker. Per container configuration is not supported, if you need that consider Podman instead. - ```yaml - services: - mailserver: - network_mode: "slirp4netns:port_handler=slirp4netns" - environment: - - ENABLE_FAIL2BAN=1 - - NETWORK_INTERFACE=tap0 - ... + ```cf + [Service] + Environment="DOCKERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER=slirp4netns" ``` -!!! note - - `slirp4netns` is not compatible with user-defined networks. + And then restart the daemon: -## Manage bans + ```console + $ systemctl --user daemon-reload + $ systemctl --user restart docker + ``` -You can also manage and list the banned IPs with the [`setup.sh`][docs-setupsh] script. + [rootless::docker]: https://docs.docker.com/engine/security/rootless -### List bans +=== "Podman" -```sh -./setup.sh fail2ban -``` + [Rootless Podman][rootless::podman] requires adding the value `slirp4netns:port_handler=slirp4netns` to the `--network` CLI option, or `network_mode` setting in your `compose.yml`: -### Un-ban + !!! example -Here `192.168.1.15` is our banned IP. + ```yaml + services: + mailserver: + network_mode: "slirp4netns:port_handler=slirp4netns" + environment: + - ENABLE_FAIL2BAN=1 + - NETWORK_INTERFACE=tap0 + ... + ``` -```sh -./setup.sh fail2ban unban 192.168.1.15 -``` + You must also add the ENV `NETWORK_INTERFACE=tap0`, because Podman uses a [hard-coded interface name][rootless::podman::interface] for `slirp4netns`. `slirp4netns` is not compatible with user-defined networks! -[docs-setupsh]: ../setup.sh.md -[github-file-f2bjail]: https://github.com/docker-mailserver/docker-mailserver/blob/master/config-examples/fail2ban-jail.cf -[github-file-f2bconfig]: https://github.com/docker-mailserver/docker-mailserver/blob/master/config-examples/fail2ban-fail2ban.cf -[rootless::rootless-kit]: https://github.com/rootless-containers/rootlesskit -[rootless::port-drivers]: https://github.com/rootless-containers/rootlesskit/blob/v0.14.5/docs/port.md#port-drivers -[rootless::slirp4netns]: https://github.com/rootless-containers/slirp4netns -[rootless::docker]: https://docs.docker.com/engine/security/rootless -[rootless::podman]: https://github.com/containers/podman/blob/v3.4.1/docs/source/markdown/podman-run.1.md#--networkmode---net -[rootless::podman::interface]: https://github.com/containers/podman/blob/v3.4.1/libpod/networking_slirp4netns.go#L264 + [rootless::podman]: https://github.com/containers/podman/blob/v3.4.1/docs/source/markdown/podman-run.1.md#--networkmode---net + [rootless::podman::interface]: https://github.com/containers/podman/blob/v3.4.1/libpod/networking_slirp4netns.go#L264 diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index ba3b6d8add6..d61a7c4a6b6 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -99,15 +99,15 @@ DMS brings sane default settings for Rspamd. They are located at `/etc/rspamd/lo ### Manually -If you want to overwrite the default settings and / or provide your own settings, you can place files at `docker-data/dms/config/rspamd/override.d/` (a directory that is linked to `/etc/rspamd/override.d/`, if it exists) to override Rspamd and DMS default settings. +!!! question "What is [`docker-data/dms/config/`][docs-dms-config-volume]?" -!!! note "What is [`docker-data/dms/config/`][docs-dms-config-volume]?" +If you want to overwrite the default settings and / or provide your own settings, you can place files at `docker-data/dms/config/rspamd/override.d/` (a directory that is linked to `/etc/rspamd/override.d/`, if it exists) to override Rspamd and DMS default settings. !!! warning "Clashing Overrides" Note that when also [using the `rspamd-commands` file](#with-the-help-of-a-custom-file), files in `override.d` may be overwritten in case you adjust them manually and with the help of the file. -[docs-dms-config-volume]: ../../faq.md#what-about-the-docker-datadmsmail-state-folder +[docs-dms-config-volume]: ../../faq.md#what-about-the-docker-datadmsconfig-directory ### With the Help of a Custom File diff --git a/docs/content/faq.md b/docs/content/faq.md index d72ea5ceeb8..793dc08d92a 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -10,6 +10,10 @@ None! No database is required. The filesystem is the database. This image is bas Mails are stored in `/var/mail/${domain}/${username}`. Since `v9.0.0` it is possible to add custom `user_attributes` for each accounts to have a different mailbox configuration (See [#1792][github-issue-1792]). +### What About the `docker-data/dms/config/` Directory? + +This documentation and all example configuration files in the GitHub repository use `docker-data/dms/config/` to refer to the directory in the host that is mounted (e.g. via a bind mount) to `/tmp/docker-mailserver/` inside the container,. + ### How are IMAP mailboxes (_aka IMAP Folders_) set up? `INBOX` is setup by default with the special IMAP folders `Drafts`, `Sent`, `Junk` and `Trash`. You can learn how to modify or add your own folders (_including additional special folders like `Archive`_) by visiting our docs page [_Customizing IMAP Folders_](../examples/use-cases/imap-folders) for more information. diff --git a/docs/content/usage.md b/docs/content/usage.md index 81a08c716ec..bf656b52675 100644 --- a/docs/content/usage.md +++ b/docs/content/usage.md @@ -4,7 +4,7 @@ title: Usage This pages explains how to get started with DMS. The guide uses Docker Compose as a reference. In our examples, a volume mounts the host location [`docker-data/dms/config/`][docs-dms-config-volume] to `/tmp/docker-mailserver/` inside the container. -[docs-dms-config-volume]: ./config/advanced/optional-config.md +[docs-dms-config-volume]: ./faq.md#what-about-the-docker-datadmsconfig-directory ## Preliminary Steps From bbc54c7139b4bff7797013a345d2a5325f579a5c Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 26 Apr 2023 10:24:21 +0200 Subject: [PATCH 055/592] docs: update FAQ entries (#3294) * removed FAQ entry about Rancher, see * update FAQ about special directories, see --- .../config/advanced/optional-config.md | 3 +- docs/content/faq.md | 36 ++++++++----------- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/docs/content/config/advanced/optional-config.md b/docs/content/config/advanced/optional-config.md index a5266681fec..94097a8ea4d 100644 --- a/docs/content/config/advanced/optional-config.md +++ b/docs/content/config/advanced/optional-config.md @@ -4,7 +4,7 @@ hide: - toc # Hide Table of Contents for this page --- -This is a list of all configuration files and directories which are optional or automatically generated in your `docker-data/dms/config/` directory. We use this path to reference the local config directory in our docs, which you should attach a volume into the container at `/tmp/docker-mailserver`. +This is a list of all configuration files and directories which are optional or automatically generated in your [`docker-data/dms/config/`][docs-dms-config-volume] directory. ## Directories @@ -42,6 +42,7 @@ This is a list of all configuration files and directories which are optional or - **user-patches.sh:** this file will be run after all configuration files are set up, but before the postfix, amavis and other daemons are started. (Docs: [FAQ - How to adjust settings with the `user-patches.sh` script][docs-faq-userpatches]) - **rspamd/custom-commands.conf:** list of simple commands to adjust Rspamd modules in an easy way (Docs: [Rspamd][docs-rspamd-commands]) +[docs-dms-config-volume]: ../../faq.md#what-about-the-docker-datadmsconfig-directory [docs-accounts-quota]: ../../config/user-management.md#quotas [docs-aliases-regex]: ../../config/user-management.md#configuring-regexp-aliases [docs-dkim]: ../../config/best-practices/dkim_dmarc_spf.md#dkim diff --git a/docs/content/faq.md b/docs/content/faq.md index 793dc08d92a..7c28b0bd3ac 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -10,10 +10,6 @@ None! No database is required. The filesystem is the database. This image is bas Mails are stored in `/var/mail/${domain}/${username}`. Since `v9.0.0` it is possible to add custom `user_attributes` for each accounts to have a different mailbox configuration (See [#1792][github-issue-1792]). -### What About the `docker-data/dms/config/` Directory? - -This documentation and all example configuration files in the GitHub repository use `docker-data/dms/config/` to refer to the directory in the host that is mounted (e.g. via a bind mount) to `/tmp/docker-mailserver/` inside the container,. - ### How are IMAP mailboxes (_aka IMAP Folders_) set up? `INBOX` is setup by default with the special IMAP folders `Drafts`, `Sent`, `Junk` and `Trash`. You can learn how to modify or add your own folders (_including additional special folders like `Archive`_) by visiting our docs page [_Customizing IMAP Folders_](../examples/use-cases/imap-folders) for more information. @@ -130,15 +126,6 @@ docker run --rm -it \ find "${PWD}/docker-data/dms-backups/" -type f -mtime +30 -delete ``` -### What about the `./docker-data/dms/mail-state` folder? - -When you run DMS with the ENV variable `ONE_DIR=1` (default), this folder will: - -- Provide support to persist Fail2Ban blocks, ClamAV signature updates, and the like when the container is restarted or recreated. -- To persist that container state properly this folder should be **volume mounted to `/var/mail-state/` internally**. - -Service data is [relocated to the `mail-state` folder][mail-state-folders] for the following services: Postfix, Dovecot, Fail2Ban, Amavis, PostGrey, ClamAV, SpamAssassin. - ### I Want to Know More About the Ports See [this part of the documentation](../config/security/understanding-the-ports/) for further details and best practice advice, **especially regarding security concerns**. @@ -237,14 +224,6 @@ If everything is OK regarding DNS, please provide [formatted logs](https://guide If we're blind, we won't be able to do anything. -### Can DMS run in a Rancher environment? - -Yes, by adding the environment variable `PERMIT_DOCKER: network`. - -!!! warning - - Adding the Docker network's gateway to the list of trusted hosts, e.g. using the `network` or `connected-networks` option, can create an [**open relay**](https://en.wikipedia.org/wiki/Open_mail_relay), for instance [if IPv6 is enabled on the host machine but not in Docker][github-issue-1405-comment]. - ### Connection refused or No response at all You see errors like "Connection Refused" and "Connection closed by foreign host", or you cannot connect at all? You may not be able to connect with your mail client (MUA)? Make sure to check Fail2Ban did not ban you (for exceeding the number of tried logins for example)! You can run @@ -373,6 +352,20 @@ DMS does not manage those concerns, verify they are not causing your delivery pr - [mail-tester](https://www.mail-tester.com/) can test your deliverability. - [helloinbox](https://www.helloinbox.email/) provides a checklist of things to improve your deliverability. +### Special Directories + +#### What About the `docker-data/dms/config/` Directory? + +This documentation and all example configuration files in the GitHub repository use `docker-data/dms/config/` to refer to the directory in the host that is mounted (e.g. via a bind mount) to `/tmp/docker-mailserver/` inside the container. + +Most configuration files for Postfix, Dovecot, etc. are persisted here. [Optional configuration][docs-optional-configuration] is stored here as well. + +#### What About the `docker-data/dms/mail-state/` Directory? + +This documentation and all example configuration files in the GitHub repository use `docker-data/dms/mail-state/` to refer to the directory in the host that is mounted (e.g. via a bind mount) to `/var/mail-state/` inside the container. + +When you run DMS with the ENV variable `ONE_DIR=1` (default), this directory will provide support to persist Fail2Ban blocks, ClamAV signature updates, and the like when the container is restarted or recreated. Service data is [relocated to the `mail-state` folder][mail-state-folders] for the following services: Postfix, Dovecot, Fail2Ban, Amavis, PostGrey, ClamAV, SpamAssassin, Rspamd & Redis. + ### SpamAssasin #### How can I manage my custom SpamAssassin rules? @@ -539,3 +532,4 @@ $spam_quarantine_to = "amavis\@example.com"; [github-issue-1792]: https://github.com/docker-mailserver/docker-mailserver/pull/1792 [hanscees-userpatches]: https://github.com/hanscees/dockerscripts/blob/master/scripts/tomav-user-patches.sh [mail-state-folders]: https://github.com/docker-mailserver/docker-mailserver/blob/c7e498194546416fb7231cb03254e77e085d18df/target/scripts/startup/misc-stack.sh#L24-L33 +[docs-optional-configuration]: ./config/advanced/optional-config.md From cd7d9b1977e058a0cca7ac6254d690728269eaa1 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 29 Apr 2023 19:03:18 +1200 Subject: [PATCH 056/592] update `contributors.yml` (#2227) --- .github/workflows/contributors.yml | 99 +++++++++++++++++------------- 1 file changed, 56 insertions(+), 43 deletions(-) diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index b4d5204e0fa..654cd557988 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -1,65 +1,78 @@ -name: Update contributors +name: 'Update Contributors List' + on: workflow_dispatch: schedule: - - cron: '0 0 1 * *' + - cron: 0 4 * * 0 -jobs: - delete-old-branch: - runs-on: ubuntu-22.04 - continue-on-error: true - steps: - - name: Delete old contributors-update branch - uses: dawidd6/action-delete-branch@v3 - with: - github_token: ${{secrets.GITHUB_TOKEN}} - branches: contributors-update +permissions: + contents: write + pull-requests: write + statuses: write + +env: + # Assign commit authorship to official Github Actions bot: + GIT_USER: github-actions[bot] + GIT_EMAIL: 41898282+github-actions[bot]@users.noreply.github.com +jobs: add-contributors: + name: 'Add Contributors' runs-on: ubuntu-22.04 - needs: delete-old-branch steps: - - uses: actions/checkout@v3 + - name: 'Checkout' + uses: actions/checkout@v3 - - name: Create contributors-update branch - uses: peterjgrainger/action-create-branch@v2.4.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - branch: 'contributors-update' - - - name: Auto-add contributors + # See https://github.com/marketplace/actions/auto-add-contributors for reference of the action. + # + # This action is not well documented, but it does the job for now. We pin the version in order + # to not have any issues in the future. + - name: 'Update CONTRIBUTORS.md' uses: BobAnkh/add-contributors@v0.2.2 with: - BRANCH: 'contributors-update' - PULL_REQUEST: 'master' - CONTRIBUTOR: '## Contributors' - COLUMN_PER_ROW: '6' ACCESS_TOKEN: ${{secrets.GITHUB_TOKEN}} - IMG_WIDTH: '100' - FONT_SIZE: '14' - PATH: '/CONTRIBUTORS.md' - COMMIT_MESSAGE: 'docs(CONTRIBUTORS): update contributors' - AVATAR_SHAPE: 'round' + COMMIT_MESSAGE: 'docs: update `CONTRIBUTORS.md`' + PATH: /CONTRIBUTORS.md + CONTRIBUTOR: '## Contributors' + COLUMN_PER_ROW: 6 + IMG_WIDTH: 100 + FONT_SIZE: 14 + AVATAR_SHAPE: round + + # See https://github.com/marketplace/actions/create-pull-request for reference of the action. + - name: 'Create Pull Request' + uses: peter-evans/create-pull-request@v5.0.0 + id: create-pr + with: + token: ${{ secrets.GITHUB_TOKEN }} + branch: contributors-update + title: 'docs: update `CONTRIBUTORS.md`' + commit-message: 'docs: update `CONTRIBUTORS.md`' + delete-branch: true + committer: ${{ env.GIT_USER }} <${{ env.GIT_EMAIL }}> + author: ${{ env.GIT_USER }} <${{ env.GIT_EMAIL }}> + signoff: true + body: | + Updated `CONTRIBUTORS.md` via the CI workflow: [`contributors.yml`][workflow]. - # This workflow will not trigger a `pull_request` event without a PAT. - # The lint workflow is not important for this type of PR, skip it and pretend it was successful: - - name: 'Get the latest commit hash from the contributors-update branch' - id: commit-data - run: | - git pull - git checkout contributors-update - echo "head_sha=$(git rev-parse contributors-update)" >>"${GITHUB_OUTPUT}" + [workflow]: https://github.com/docker-mailserver/docker-mailserver/blob/master/.github/workflows/contributors.yml - - name: 'Commit Status: Set Lint status to success (skipped)' + # See https://github.com/marketplace/actions/set-commit-status for reference of the action. + # + # GH Actions are limited when it comes to actions triggering other actions. Hence, + # this whole workflow will not trigger a `pull_request` event without a PAT. The lint + # workflow, which is required due to branch protection, is not important for this type + # of PR, so we skip it and pretend it was successful. + - name: 'Set Status for Linting Actions to Success (Skipped)' uses: myrotvorets/set-commit-status-action@1.1.6 + continue-on-error: true with: token: ${{ secrets.GITHUB_TOKEN }} # Skipped workflows are still assigned a "success" status: status: success # This should be the correct commit SHA on the contributors-update branch: - sha: ${{ steps.commit-data.outputs.head_sha }} + sha: ${{ steps.create-pr.outputs.pull-request-head-sha }} # Name of status check to add/update: - context: 'lint' + context: lint # Optional message/note we can inline to the right of the context name in the UI: - description: "Lint skipped. Not relevant." + description: Lint skipped. Not relevant. From b6261c7387354451968cfde3ddcc411b94adc3e0 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sat, 29 Apr 2023 10:55:54 +0200 Subject: [PATCH 057/592] remove unnecessary `return 0` statements (#3290) See --- target/scripts/helpers/utils.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/target/scripts/helpers/utils.sh b/target/scripts/helpers/utils.sh index e06ff837476..e225b1c8580 100644 --- a/target/scripts/helpers/utils.sh +++ b/target/scripts/helpers/utils.sh @@ -39,7 +39,6 @@ function _chown_var_mail_if_necessary _log 'trace' 'Fixing /var/mail permissions' chown -R 5000:5000 /var/mail || return 1 fi - return 0 } function _require_n_parameters_or_print_usage @@ -64,7 +63,6 @@ function _adjust_mtime_for_postfix_maincf then touch -d '2 seconds ago' /etc/postfix/main.cf fi - return 0 } function _reload_postfix From 2bdbe5d918ab209547e880716a877c76e8321457 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 1 May 2023 15:00:35 +0200 Subject: [PATCH 058/592] F2B: update F2B after discussion in #3256 (#3288) --- config-examples/fail2ban-jail.cf | 11 ++++------- docs/content/config/security/fail2ban.md | 6 +++--- target/fail2ban/jail.local | 10 ++++------ 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/config-examples/fail2ban-jail.cf b/config-examples/fail2ban-jail.cf index 73b5bd47cc0..6866ddf361e 100644 --- a/config-examples/fail2ban-jail.cf +++ b/config-examples/fail2ban-jail.cf @@ -8,7 +8,7 @@ bantime = 1w findtime = 1w # "maxretry" is the number of failures before a host get banned. -maxretry = 2 +maxretry = 6 # "ignoreip" can be a list of IP addresses, CIDR masks or DNS hosts. Fail2ban # will not ban a host which matches an address in this list. Several addresses @@ -25,15 +25,12 @@ enabled = true [postfix] enabled = true -# See https://github.com/fail2ban/fail2ban/blob/27294c4b9ee5d5568a1d5f83af744ea39d5a1acb/config/filter.d/postfix.conf#L58 -# `mode=aggressive` basically combines more filters to match more lines, and hence, apply rules -# more aggressively. The same goes for the `postfix-sasl` jail. -mode = aggressive - +# For a reference on why this mode was chose, see +# https://github.com/docker-mailserver/docker-mailserver/issues/3256#issuecomment-1511188760 +mode = extra [postfix-sasl] enabled = true -mode = aggressive # This jail is used for manual bans. # To ban an IP address use: setup.sh fail2ban ban diff --git a/docs/content/config/security/fail2ban.md b/docs/content/config/security/fail2ban.md index ae91d902754..7a06fbf2248 100644 --- a/docs/content/config/security/fail2ban.md +++ b/docs/content/config/security/fail2ban.md @@ -29,7 +29,7 @@ hide: ### DMS Defaults -DMS will automatically ban IP addresses of hosts that have generated 2 failed attempts over the course of the last week. The bans themselves last for one week. +DMS will automatically ban IP addresses of hosts that have generated 6 failed attempts over the course of the last week. The bans themselves last for one week. The Postfix jail is configured to use `mode = extra` in DMS. ### Custom Files @@ -39,10 +39,10 @@ This following configuration files inside the `docker-data/dms/config/` volume w 1. `fail2ban-jail.cf` is copied to `/etc/fail2ban/jail.d/user-jail.local` - with this file, you can adjust the configuration of individual jails and their defaults - - the is an example provided [in our repository on GitHub][github-file-f2bjail] + - there is an example provided [in our repository on GitHub][github-file-f2bjail] 2. `fail2ban-fail2ban.cf` is copied to `/etc/fail2ban/fail2ban.local` - with this file, you can adjust F2B behavior in general - - the is an example provided [in our repository on GitHub][github-file-f2bconfig] + - there is an example provided [in our repository on GitHub][github-file-f2bconfig] [docs-dms-config-volume]: ../../faq.md#what-about-the-docker-datadmsconfig-directory [github-file-f2bjail]: https://github.com/docker-mailserver/docker-mailserver/blob/master/config-examples/fail2ban-jail.cf diff --git a/target/fail2ban/jail.local b/target/fail2ban/jail.local index 96f1d4f5981..6866ddf361e 100644 --- a/target/fail2ban/jail.local +++ b/target/fail2ban/jail.local @@ -8,7 +8,7 @@ bantime = 1w findtime = 1w # "maxretry" is the number of failures before a host get banned. -maxretry = 2 +maxretry = 6 # "ignoreip" can be a list of IP addresses, CIDR masks or DNS hosts. Fail2ban # will not ban a host which matches an address in this list. Several addresses @@ -25,14 +25,12 @@ enabled = true [postfix] enabled = true -# See https://github.com/fail2ban/fail2ban/blob/27294c4b9ee5d5568a1d5f83af744ea39d5a1acb/config/filter.d/postfix.conf#L58 -# `mode=aggressive` basically combines more filters to match more lines, and hence, apply rules -# more aggressively. The same goes for the `postfix-sasl` jail. -mode = aggressive +# For a reference on why this mode was chose, see +# https://github.com/docker-mailserver/docker-mailserver/issues/3256#issuecomment-1511188760 +mode = extra [postfix-sasl] enabled = true -mode = aggressive # This jail is used for manual bans. # To ban an IP address use: setup.sh fail2ban ban From 869caf35ec304b2bef271d411bb65b0a707df9cc Mon Sep 17 00:00:00 2001 From: georglauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 2 May 2023 20:52:54 +0200 Subject: [PATCH 059/592] ci: fix contributors workflow (again) (#3304) --- .github/workflows/contributors.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index 654cd557988..499fbbbc19f 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -14,6 +14,7 @@ env: # Assign commit authorship to official Github Actions bot: GIT_USER: github-actions[bot] GIT_EMAIL: 41898282+github-actions[bot]@users.noreply.github.com + BRANCH_NAME: contributors-update jobs: add-contributors: @@ -23,6 +24,12 @@ jobs: - name: 'Checkout' uses: actions/checkout@v3 + - name: 'Checkout New Branch and Push It' + run: | + git checkout -b ${{ env.BRANCH_NAME }} + git push --force https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git HEAD:${{ env.BRANCH_NAME }} + git checkout master + # See https://github.com/marketplace/actions/auto-add-contributors for reference of the action. # # This action is not well documented, but it does the job for now. We pin the version in order @@ -31,6 +38,7 @@ jobs: uses: BobAnkh/add-contributors@v0.2.2 with: ACCESS_TOKEN: ${{secrets.GITHUB_TOKEN}} + BRANCH: ${{ env.BRANCH_NAME }} COMMIT_MESSAGE: 'docs: update `CONTRIBUTORS.md`' PATH: /CONTRIBUTORS.md CONTRIBUTOR: '## Contributors' @@ -45,7 +53,8 @@ jobs: id: create-pr with: token: ${{ secrets.GITHUB_TOKEN }} - branch: contributors-update + base: master + branch: ${{ env.BRANCH_NAME }} title: 'docs: update `CONTRIBUTORS.md`' commit-message: 'docs: update `CONTRIBUTORS.md`' delete-branch: true @@ -70,7 +79,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} # Skipped workflows are still assigned a "success" status: status: success - # This should be the correct commit SHA on the contributors-update branch: + # This should be the correct commit SHA on ${{ env.BRANCH_NAME }}: sha: ${{ steps.create-pr.outputs.pull-request-head-sha }} # Name of status check to add/update: context: lint From 423188176f065f0a5f50972d9aa10d3e2b869e1b Mon Sep 17 00:00:00 2001 From: Casper Date: Wed, 3 May 2023 00:13:44 +0200 Subject: [PATCH 060/592] fail2ban: add 'log' command (#3299) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- docs/content/config/security/fail2ban.md | 16 +++++++++++++++- target/bin/fail2ban | 10 +++++++++- target/bin/setup | 1 + 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/content/config/security/fail2ban.md b/docs/content/config/security/fail2ban.md index 7a06fbf2248..b9f8ad3826b 100644 --- a/docs/content/config/security/fail2ban.md +++ b/docs/content/config/security/fail2ban.md @@ -48,6 +48,16 @@ This following configuration files inside the `docker-data/dms/config/` volume w [github-file-f2bjail]: https://github.com/docker-mailserver/docker-mailserver/blob/master/config-examples/fail2ban-jail.cf [github-file-f2bconfig]: https://github.com/docker-mailserver/docker-mailserver/blob/master/config-examples/fail2ban-fail2ban.cf +### Viewing All Bans + +When just running + +```bash +setup fail2ban +``` + +the script will show all banned IP addresses. + ### Managing Bans You can manage F2B with the `setup` script. The usage looks like this: @@ -56,7 +66,11 @@ You can manage F2B with the `setup` script. The usage looks like this: docker exec setup fail2ban [ ] ``` -When just running `setup fail2ban`, the script will show all banned IP addresses. +### Viewing the Log File + +```bash +docker exec setup fail2ban log +``` ## Running Inside A Rootless Container diff --git a/target/bin/fail2ban b/target/bin/fail2ban index 8a01d1b6250..f16fb87de93 100755 --- a/target/bin/fail2ban +++ b/target/bin/fail2ban @@ -3,7 +3,11 @@ # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh -function __usage { echo "Usage: ./setup.sh fail2ban [ ]" ; } +function __usage +{ + echo "Usage: ./setup.sh fail2ban [ ]" + echo " ./setup.sh fail2ban log" +} fail2ban-client ping &>/dev/null || _exit_with_error "Fail2ban not running" @@ -73,6 +77,10 @@ else fi ;; + ( 'log' ) + cat /var/log/mail/fail2ban.log + ;; + ( * ) __usage _exit_with_error "Unknown command '${1}'" diff --git a/target/bin/setup b/target/bin/setup index daa4bf89cb1..6496f74ccb8 100755 --- a/target/bin/setup +++ b/target/bin/setup @@ -60,6 +60,7 @@ ${RED}[${ORANGE}SUB${RED}]${ORANGE}COMMANDS${RESET} setup fail2ban ${RESET} setup fail2ban ${CYAN}ban${RESET} setup fail2ban ${CYAN}unban${RESET} + setup fail2ban ${CYAN}log${RESET} ${LBLUE}COMMAND${RESET} debug ${RED}:=${RESET} setup debug ${CYAN}fetchmail${RESET} From bba72daedf4ecf03a80eea571160371191237863 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 3 May 2023 08:30:49 +0200 Subject: [PATCH 061/592] scripts: add DKIM helper script for Rspamd (#3286) Co-authored-by: Casper --- .../config/best-practices/dkim_dmarc_spf.md | 73 ++--- target/bin/open-dkim | 6 + target/bin/rspamd-dkim | 279 ++++++++++++++++++ .../startup/setup.d/security/rspamd.sh | 2 +- .../parallel/set1/spam_virus/rspamd_dkim.bats | 272 +++++++++++++++++ 5 files changed, 598 insertions(+), 34 deletions(-) create mode 100755 target/bin/rspamd-dkim create mode 100644 test/tests/parallel/set1/spam_virus/rspamd_dkim.bats diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md index f8d17072dce..dfa3b5a2c22 100644 --- a/docs/content/config/best-practices/dkim_dmarc_spf.md +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -40,6 +40,10 @@ You should have: - At least one [email account setup][docs-accounts-add] - Attached a [volume for config][docs-volumes-config] to persist the generated files to local storage +!!! warning "RSA Key Sizes >= 4096 Bit" + + Keys of 4096 bits could be denied by some mail servers. According to [RFC 6376][rfc-6376], keys are [preferably between 512 and 2048 bits][github-issue-dkimlength]. + DKIM is currently supported by either OpenDKIM or Rspamd: === "OpenDKIM" @@ -48,7 +52,7 @@ DKIM is currently supported by either OpenDKIM or Rspamd: The command `docker exec setup config dkim help` details supported config options, along with some examples. - !!! example "Create a DKIM key" + !!! example "Creating a DKIM key" Generate the DKIM files with: @@ -74,6 +78,12 @@ DKIM is currently supported by either OpenDKIM or Rspamd: setup config dkim keysize 2048 ``` + !!! info "Restart required" + + After restarting DMS, outgoing mail will now be signed with your new DKIM key(s) :tada: + + You'll need to repeat this process if you add any new domains. + === "Rspamd" Opt-in via [`ENABLE_RSPAMD=1`][docs-env-rspamd] (_and disable the default OpenDKIM: `ENABLE_OPENDKIM=0`_). @@ -83,31 +93,33 @@ DKIM is currently supported by either OpenDKIM or Rspamd: 1. [Verifying DKIM signatures from inbound mail][rspamd-docs-dkim-checks] is enabled by default. 2. [Signing outbound mail with your DKIM key][rspamd-docs-dkim-signing] needs additional setup (key + dns + config). - !!! example "Create a DKIM key" + !!! example "Creating DKIM Keys" - Presently only OpenDKIM is supported with `setup config dkim`. To generate your DKIM key and DNS files you'll need to specify: + You can simply run - - `-s` The DKIM selector (_eg: `mail`, it can be anything you like_) - - `-d` The sender address domain (_everything after `@` from the email address_) + ```bash + docker exec -ti setup config dkim help + ``` - See `rspamadm dkim_keygen -h` for an overview of the supported options. + which provides you with an overview of what the script can do. Just running - --- + ```bash + docker exec -ti setup config dkim + ``` - 1. Go inside the container with `docker exec -ti bash` - 2. Add `rspamd/dkim/` folder to your config volume and switch to it: `cd /tmp/docker-mailserver/rspamd/dkim` - 3. Run: `rspamadm dkim_keygen -s mail -b 2048 -d example.com -k mail.private > mail.txt` (_change `-d` to your domain-part_) - 4. Presently you must ensure Rspamd can read the `.private` file, run: - -`chgrp _rspamd mail.private` - -`chmod g+r mail.private` + will execute the helper script with default parameters. - --- + !!! info "About the Helper Script" - !!! bug inline end "DMS config volume support is not ready for Rspamd" + The script will persist the keys in `/tmp/docker-mailserver/rspamd/dkim/`. Hence, if you are already using the default volume mounts, the keys are persisted in a volume. The script also restarts Rspamd directly, so changes take effect without restarting DMS. - Presently you'll need to [explicitly mount `rspamd/modules/override.d/`][docs-rspamd-config-dropin] as an additional volume; do not use [`rspamd-modules.conf`][docs-rspamd-config-declarative] for this purpose. + The script provides you with log messages along the way of creating keys. In case you want to read the complete log, use `-v` (verbose) or `-vv` (very verbose). - Create a configuration file for the DKIM signing module at `rspamd/modules/override.d/dkim_signing.conf` and populate it with config as shown in the example below: + --- + + In case you have not already provided a default DKIM signing configuration, the script will create one and write it to `/etc/rspamd/override.d/dkim_signing.conf`. If this file already exist, it will not be overwritten. When you're already using [the `rspamd/override.d/` directory][docs-rspamd-override-d], the file is created inside your volume and therefore persisted correctly. If you are not using `rspamd/override.d/`, you will need to persist the file yourself (otherwise it is lost on container restart). + + An example of what a default configuration file for DKIM signing looks like can be found by expanding the example below. ??? example "DKIM Signing Module Configuration Examples" @@ -124,6 +136,7 @@ DKIM is currently supported by either OpenDKIM or Rspamd: use_domain = "header"; use_redis = false; # don't change unless Redis also provides the DKIM keys use_esld = true; + check_pubkey = true; # you wan't to use this in the beginning domain { @@ -134,7 +147,7 @@ DKIM is currently supported by either OpenDKIM or Rspamd: } ``` - As shown next, you can: + As shown next: - You can add more domains into the `domain { ... }` section. - A domain can also be configured with multiple selectors and keys within a `selectors [ ... ]` array. @@ -170,27 +183,19 @@ DKIM is currently supported by either OpenDKIM or Rspamd: } ``` - !!! warning "Support for DKIM keys using Ed25519" - - This modern elliptic curve is supported by Rspamd, but support by third-parties for [verifying Ed25519 DKIM signatures is unreliable][dkim-ed25519-support]. + ??? warning "Support for DKIM Keys using ED25519" - If you sign your mail with this key type, you should include RSA as a fallback, like shown in the above example. + This modern elliptic curve is supported by Rspamd, but support by third-parties for [verifying Ed25519 DKIM signatures is unreliable][dkim-ed25519-support]. - !!! tip "DKIM Signing config: `check_pubkey = true;`" + If you sign your mail with this key type, you should include RSA as a fallback, like shown in the above example. - This setting will have Rspamd query the DNS record for each DKIM selector, verifying each public key matches the private key configured. + ??? tip "Let Rspamd Check Your Keys" - If there is a mismatch, a warning will be omitted to the Rspamd log (`/var/log/supervisor/rspamd.log`). + When `check_pubkey = true;` is set, Rspamd will query the DNS record for each DKIM selector, verifying each public key matches the private key configured. -!!! info "Restart required" + If there is a mismatch, a warning will be omitted to the Rspamd log `/var/log/supervisor/rspamd.log`. - After restarting DMS, outgoing mail will now be signed with your new DKIM key(s) :tada: - - You'll need to repeat this process if you add any new domains. - -!!! warning "RSA Key Sizes >= 4096 Bit" - - Keys of 4096 bits could denied by some mail servers. According to [RFC 6376][rfc-6376] keys are [preferably between 512 and 2048 bits][github-issue-dkimlength]. + [docs-rspamd-override-d]: ../security/rspamd.md#manually ### DNS Record { #dkim-dns } @@ -211,6 +216,8 @@ When mail signed with your DKIM key is sent from your mail server, the receiver | TTL | Use the default (_otherwise [3600 seconds is appropriate][dns::digicert-ttl]_) | | Data | File content within `( ... )` (_formatted as advised below_) | + When using Rspamd, the helper script has already provided you with the contents (the "Data" field) of the DNS record you need to create - you can just copy-paste this text. + === "DNS Zone file" `.txt` is already formatted as a snippet for adding to your [DNS Zone file][dns::wikipedia-zonefile]. diff --git a/target/bin/open-dkim b/target/bin/open-dkim index 1c461ae063f..b0a6b712d56 100755 --- a/target/bin/open-dkim +++ b/target/bin/open-dkim @@ -3,6 +3,12 @@ # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh +if [[ -f /etc/dms-settings ]] && [[ $(_get_dms_env_value 'ENABLE_RSPAMD') -eq 1 ]] +then + /usr/local/bin/rspamd-dkim "${@}" + exit +fi + KEYSIZE=4096 SELECTOR=mail DOMAINS= diff --git a/target/bin/rspamd-dkim b/target/bin/rspamd-dkim new file mode 100755 index 00000000000..eb555f47145 --- /dev/null +++ b/target/bin/rspamd-dkim @@ -0,0 +1,279 @@ +#!/bin/bash + +# shellcheck source=../scripts/helpers/index.sh +source /usr/local/bin/helpers/index.sh + +_trap_err_signal + +set -u -eE -o pipefail +shopt -s inherit_errexit + +# shellcheck source=/dev/null +source /etc/dms-settings + +function __usage +{ + _log 'trace' 'Showing usage message now' + echo -e "${PURPLE}RSPAMD-DKIM${RED}(${YELLOW}8${RED}) + +${ORANGE}NAME${RESET} + rspamd-dkim - Configure DomainKeys Identified Mail (DKIM) via Rspamd + +${ORANGE}SYNOPSIS${RESET} + setup config dkim [ OPTIONS${RED}...${RESET} ] + +${ORANGE}DESCRIPTION${RESET} + This script aids in creating DKIM signing keys. The keys are created and managed by Rspamd. + OPTIONS can be used to configure a more complex setup. + +${ORANGE}OPTIONS${RESET} + ${BLUE}Generic Program Information${RESET} + -v Enable verbose logging (setting the log level to 'debug'). + -vv Enable very verbose logging (setting the log level to 'trace'). + help Print the usage information. + + ${BLUE}Configuration adjustments${RESET} + keytype Set the type of key you want to use + Possible values: rsa, ed25519 + Default: rsa + keysize Set the size of the keys to be generated + Possible values: 1024, 2048 and 4096 + Default: 2048 + Only applies when using keytype=rsa + selector Set a manual selector for the key + Default: mail + domain Provide the domain for which keys are to be generated + Default: primary domain name of DMS + +${ORANGE}EXAMPLES${RESET} + ${LWHITE}setup config dkim keysize 2048${RESET} + Creates keys of length 2048 bit in a default setup where domains are obtained from + your accounts. + + ${LWHITE}setup config dkim keysize 512 selector 2023-dkim${RESET} + Creates keys of length 512 bit in a default setup where domains are obtained from + your accounts. The DKIM selector used is '2023-dkim'. + + ${LWHITE}setup config dkim keysize 1024 selector 2023-dkim domain whoami.com${RESET} + Creates keys of length 1024 bit in a default setup where domains are obtained from your accounts. + The DKIM selector used is '2023-dkim'. The domain for which DKIM keys are generated is whoami.com. + +${ORANGE}EXIT STATUS${RESET} + Exit status is 0 if command was successful. If wrong arguments are provided or arguments contain + errors, the script will exit early with a non-zero exit status. + +" +} + +function __do_as_rspamd_user +{ + local COMMAND=${1:?Command required when using __do_as_rspamd_user} + _log 'trace' "Running '${*}' as user '_rspamd' now" + shift 1 + su -l '_rspamd' -s "$(command -v "${COMMAND}")" -- "${@}" +} + +function _parse_arguments +{ + KEYTYPE='rsa' + KEYSIZE='2048' + SELECTOR='mail' + DOMAIN=${DOMAINNAME} + + _log 'trace' "Options given to this script: '${*}'" + + while [[ ${#} -gt 0 ]] + do + case "${1}" in + + ( 'keytype' ) + [[ -n ${2:-} ]] || _exit_with_error "No keytype provided after 'keytype' argument" + if [[ ${2} == 'rsa' ]] || [[ ${2} == 'ed25519' ]] + then + KEYTYPE=${2} + _log 'debug' "Keytype set to '${KEYTYPE}'" + else + _exit_with_error "Unknown keytype '${2}'" + fi + ;; + + ( 'keysize' ) + [[ -n ${2:-} ]] || _exit_with_error "No keysize provided after 'keysize' argument" + KEYSIZE=${2} + _log 'debug' "Keysize set to '${KEYSIZE}'" + ;; + + ( 'selector' ) + [[ -n ${2:-} ]] || _exit_with_error "No selector provided after 'selector' argument" + SELECTOR=${2} + _log 'debug' "Selector set to '${SELECTOR}'" + ;; + + ( 'domain' ) + [[ -n ${2:-} ]] || _exit_with_error "No domain provided after 'domain' argument" + DOMAIN=${2} + _log 'debug' "Domain set to '${DOMAIN}'" + ;; + + ( 'help' ) + __usage + exit 0 + ;; + + ( '-vv' ) + # shellcheck disable=SC2034 + LOG_LEVEL='trace' + shift 1 + _log 'trace' 'Enabled trace-logging' + continue + ;; + + ( '-v' ) + # shellcheck disable=SC2034 + LOG_LEVEL='debug' + shift 1 + _log 'debug' 'Enabled debug-logging' + continue + ;; + + ( * ) + __usage + _exit_with_error "Unknown option(s) '${1}' ${2:+"and '${2}'"}" + ;; + + esac + + shift 2 + done + + if [[ ${KEYTYPE} == 'ed25519' ]] && [[ ${KEYSIZE} -ne 2048 ]] + then + _exit_with_error "Chosen keytype does not accept the 'keysize' argument" + fi + + return 0 +} + +function _create_keys +{ + # Note: Variables not marked with `local` are used + # in other functions (after this function was called). + BASE_DIR='/tmp/docker-mailserver/rspamd/dkim' + + if [[ ${KEYTYPE} == 'rsa' ]] + then + local BASE_FILE_NAME="${BASE_DIR}/${KEYTYPE}-${KEYSIZE}-${SELECTOR}-${DOMAIN}" + KEYTYPE_OPTIONS=('-b' "${KEYSIZE}") + _log 'info' "Creating DKIM keys of type '${KEYTYPE}' and lenght '${KEYSIZE}' with selector '${SELECTOR}' for domain '${DOMAIN}'" + else + local BASE_FILE_NAME="${BASE_DIR}/${KEYTYPE}-${SELECTOR}-${DOMAIN}" + KEYTYPE_OPTIONS=('-t' "${KEYTYPE}") + _log 'info' "Creating DKIM keys of type '${KEYTYPE}' with selector '${SELECTOR}' for domain '${DOMAIN}'" + fi + + PUBLIC_KEY_FILE="${BASE_FILE_NAME}.public.txt" + PUBLIC_KEY_DNS_FILE="${BASE_FILE_NAME}.public.dns.txt" + PRIVATE_KEY_FILE="${BASE_FILE_NAME}.private.txt" + + mkdir -p "${BASE_DIR}" + chown _rspamd:_rspamd "${BASE_DIR}" + + # shellcheck disable=SC2310 + if __do_as_rspamd_user rspamadm \ + dkim_keygen \ + -s "${SELECTOR}" \ + -d "${DOMAIN}" \ + "${KEYTYPE_OPTIONS[@]}" \ + -k "${PRIVATE_KEY_FILE}" \ + >"${PUBLIC_KEY_FILE}" + then + _log 'info' 'Successfully created DKIM keys' + _log 'debug' "Public key written to '${PUBLIC_KEY_FILE}'" + _log 'debug' "Private key written to '${PRIVATE_KEY_FILE}'" + else + _exit_with_error 'Creating keys failed' + fi +} + +function _check_permissions +{ + # shellcheck disable=SC2310 + if ! __do_as_rspamd_user ls "${BASE_DIR}" >/dev/null + then + _log 'warn' "The Rspamd user ('_rspamd') seems to be unable to list files in the keys directory ('${BASE_DIR}') - Rspamd may experience permission errors later" + elif ! __do_as_rspamd_user cat "${PRIVATE_KEY_FILE}" >/dev/null + then + _log 'warn' "The Rspamd user ('_rspamd') seems to be unable to read the private key file - Rspamd may experience permission errors later" + else + _log 'debug' 'Permissions on files and directories seem ok' + fi +} + +function _setup_default_signing_conf +{ + local DEFAULT_CONFIG_FILE='/etc/rspamd/override.d/dkim_signing.conf' + if [[ -f ${DEFAULT_CONFIG_FILE} ]] + then + _log 'debug' "'${DEFAULT_CONFIG_FILE}' exists, not supplying a default" + else + _log 'info' "Supplying a default configuration ('${DEFAULT_CONFIG_FILE}')" + cat >"${DEFAULT_CONFIG_FILE}" << EOF +# documentation: https://rspamd.com/doc/modules/dkim_signing.html + +enabled = true; + +sign_authenticated = true; +sign_local = true; + +use_domain = "header"; +use_redis = false; # don't change unless Redis also provides the DKIM keys +use_esld = true; + +check_pubkey = true; # you wan't to use this in the beginning + +domain { + ${DOMAIN} { + path = "${PRIVATE_KEY_FILE}"; + selector = "${SELECTOR}"; + } +} + +EOF + chown _rspamd:_rspamd "${DEFAULT_CONFIG_FILE}" + fi +} + +function _transform_public_key_file_to_dns_record_contents +{ + _log 'trace' 'Transforming DNS zone format to DNS record content now' + : >"${PUBLIC_KEY_DNS_FILE}" + grep -o '".*"' "${PUBLIC_KEY_FILE}" | tr -d '"\n' >>"${PUBLIC_KEY_DNS_FILE}" + echo '' >>"${PUBLIC_KEY_DNS_FILE}" + + if ! _log_level_is '(warn|error)' + then + _log 'info' "Here is the content of the TXT DNS record ${SELECTOR}._domainkey.${DOMAIN} that you need to create:\n" + cat "${PUBLIC_KEY_DNS_FILE}" + printf '\n' + fi +} + +function _final_steps +{ + # We need to restart Rspamd so the changes take effect immediately. + if ! supervisorctl restart rspamd + then + _log 'warn' 'Could not restart Rspamd via Supervisord' + fi + + _log 'trace' 'Finished DKIM key creation' +} + +_obtain_hostname_and_domainname +_require_n_parameters_or_print_usage 0 "${@}" +_parse_arguments "${@}" +_create_keys +_check_permissions +_setup_default_signing_conf +_transform_public_key_file_to_dns_record_contents +_final_steps diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 1daf0188297..e171736a476 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -82,7 +82,7 @@ function __rspamd__run_early_setup_and_checks then ln -s "${RSPAMD_DMS_OVERRIDE_D}" "${RSPAMD_OVERRIDE_D}" else - __rspamd__log 'warn' "Could not remove '${RSPAMD_OVERRIDE_D}' - not linking '${RSPAMD_DMS_OVERRIDE_D}'" + __rspamd__log 'warn' "Could not remove '${RSPAMD_OVERRIDE_D}' (not empty?) - not linking '${RSPAMD_DMS_OVERRIDE_D}'" fi fi diff --git a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats new file mode 100644 index 00000000000..0f615d94a1b --- /dev/null +++ b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats @@ -0,0 +1,272 @@ +load "${REPOSITORY_ROOT}/test/helper/common" +load "${REPOSITORY_ROOT}/test/helper/setup" + +BATS_TEST_NAME_PREFIX='[Rspamd] (DKIM) ' +CONTAINER_NAME='dms-test_rspamd-dkim' + +DOMAIN_NAME='fixed.com' +SIGNING_CONF_FILE='/etc/rspamd/override.d/dkim_signing.conf' + +function setup_file() { + _init_with_defaults + + # Comment for maintainers about `PERMIT_DOCKER=host`: + # https://github.com/docker-mailserver/docker-mailserver/pull/2815/files#r991087509 + local CUSTOM_SETUP_ARGUMENTS=( + --env ENABLE_RSPAMD=1 + --env ENABLE_OPENDKIM=0 + --env ENABLE_OPENDMARC=0 + --env ENABLE_POLICYD_SPF=0 + --env LOG_LEVEL=trace + --env OVERRIDE_HOSTNAME="mail.${DOMAIN_NAME}" + ) + + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + _wait_for_service rspamd-redis + _wait_for_service rspamd +} + +# We want each test to start with a clean state. +function teardown() { + __remove_signing_config_file + _run_in_container rm -rf /tmp/docker-mailserver/rspamd/dkim + assert_success +} + +function teardown_file() { _default_teardown ; } + +@test 'log level is applied correctly' { + _run_in_container setup config dkim -vv help + __log_is_free_of_warnings_and_errors + assert_output --partial 'Enabled trace-logging' + + _run_in_container setup config dkim -v help + __log_is_free_of_warnings_and_errors + assert_output --partial 'Enabled debug-logging' +} + +@test 'help message is properly shown' { + _run_in_container setup config dkim help + __log_is_free_of_warnings_and_errors + assert_output --partial 'Showing usage message now' + assert_output --partial 'rspamd-dkim - Configure DomainKeys Identified Mail (DKIM) via Rspamd' +} + +@test 'default signing config is created if it does not exist and not overwritten' { + # Required pre-condition: no default configuration is present + __remove_signing_config_file + + __create_key + assert_success + __log_is_free_of_warnings_and_errors + assert_output --partial "Supplying a default configuration ('${SIGNING_CONF_FILE}')" + refute_output --partial "'${SIGNING_CONF_FILE}' exists, not supplying a default" + assert_output --partial "Finished DKIM key creation" + _run_in_container_bash "[[ -f ${SIGNING_CONF_FILE} ]]" + assert_success + _exec_in_container_bash "echo "blabla" >${SIGNING_CONF_FILE}" + local INITIAL_SHA512_SUM=$(_exec_in_container sha512sum "${SIGNING_CONF_FILE}") + + __create_key + __log_is_free_of_warnings_and_errors + refute_output --partial "Supplying a default configuration ('${SIGNING_CONF_FILE}')" + assert_output --partial "'${SIGNING_CONF_FILE}' exists, not supplying a default" + assert_output --partial "Finished DKIM key creation" + local SECOND_SHA512_SUM=$(_exec_in_container sha512sum "${SIGNING_CONF_FILE}") + assert_equal "${INITIAL_SHA512_SUM}" "${SECOND_SHA512_SUM}" +} + +@test 'default directories and files are created' { + __create_key + assert_success + + _count_files_in_directory_in_container /tmp/docker-mailserver/rspamd/dkim/ 3 + _run_in_container_bash "[[ -f ${SIGNING_CONF_FILE} ]]" + assert_success + + __check_path_in_signing_config "/tmp/docker-mailserver/rspamd/dkim/rsa-2048-mail-${DOMAIN_NAME}.private.txt" + __check_selector_in_signing_config 'mail' +} + +@test "argument 'domain' is applied correctly" { + for DOMAIN in 'blabla.org' 'someother.com' 'random.de' + do + _run_in_container setup config dkim domain "${DOMAIN}" + assert_success + assert_line --partial "Domain set to '${DOMAIN}'" + + local BASE_FILE_NAME="/tmp/docker-mailserver/rspamd/dkim/rsa-2048-mail-${DOMAIN}" + __check_key_files_are_present "${BASE_FILE_NAME}" + __check_path_in_signing_config "${BASE_FILE_NAME}.private.txt" + __remove_signing_config_file + done +} + +@test "argument 'keytype' is applied correctly" { + _run_in_container setup config dkim keytype foobar + assert_failure + assert_line --partial "Unknown keytype 'foobar'" + + for KEYTYPE in 'rsa' 'ed25519' + do + _run_in_container setup config dkim keytype "${KEYTYPE}" + assert_success + assert_line --partial "Keytype set to '${KEYTYPE}'" + + local BASE_FILE_NAME="/tmp/docker-mailserver/rspamd/dkim/ed25519-mail-${DOMAIN_NAME}" + [[ ${KEYTYPE} == 'rsa' ]] && BASE_FILE_NAME="/tmp/docker-mailserver/rspamd/dkim/rsa-2048-mail-${DOMAIN_NAME}" + __check_key_files_are_present "${BASE_FILE_NAME}" + + _run_in_container grep ".*k=${KEYTYPE};.*" "${BASE_FILE_NAME}.public.txt" + assert_success + _run_in_container grep ".*k=${KEYTYPE};.*" "${BASE_FILE_NAME}.public.dns.txt" + assert_success + __check_path_in_signing_config "${BASE_FILE_NAME}.private.txt" + __remove_signing_config_file + done +} + +@test "argument 'selector' is applied correctly" { + for SELECTOR in 'foo' 'bar' 'baz' + do + __create_key 'rsa' "${SELECTOR}" + assert_success + assert_line --partial "Selector set to '${SELECTOR}'" + + local BASE_FILE_NAME="/tmp/docker-mailserver/rspamd/dkim/rsa-2048-${SELECTOR}-${DOMAIN_NAME}" + __check_key_files_are_present "${BASE_FILE_NAME}" + _run_in_container grep "^${SELECTOR}\._domainkey.*" "${BASE_FILE_NAME}.public.txt" + assert_success + + __check_rsa_keys 2048 "${SELECTOR}-${DOMAIN_NAME}" + __check_path_in_signing_config "${BASE_FILE_NAME}.private.txt" + __check_selector_in_signing_config "${SELECTOR}" + __remove_signing_config_file + done +} + +@test "argument 'keysize' is applied correctly for RSA keys" { + for KEYSIZE in 512 1024 2048 4096 + do + __create_key 'rsa' 'mail' "${DOMAIN_NAME}" "${KEYSIZE}" + assert_success + __log_is_free_of_warnings_and_errors + assert_line --partial "Keysize set to '${KEYSIZE}'" + __check_rsa_keys "${KEYSIZE}" "mail-${DOMAIN_NAME}" + __remove_signing_config_file + done +} + +@test "when 'keytype=ed25519' is set, setting custom 'keysize' is rejected" { + __create_key 'ed25519' 'mail' "${DOMAIN_NAME}" 4096 + assert_failure + assert_line --partial "Chosen keytype does not accept the 'keysize' argument" +} + +@test "setting all arguments to a custom value works" { + local KEYTYPE='ed25519' + local SELECTOR='someselector' + local DOMAIN='dms.org' + + __create_key "${KEYTYPE}" "${SELECTOR}" "${DOMAIN}" + assert_success + __log_is_free_of_warnings_and_errors + + assert_line --partial "Keytype set to '${KEYTYPE}'" + assert_line --partial "Selector set to '${SELECTOR}'" + assert_line --partial "Domain set to '${DOMAIN}'" + + local BASE_FILE_NAME="/tmp/docker-mailserver/rspamd/dkim/${KEYTYPE}-${SELECTOR}-${DOMAIN}" + __check_path_in_signing_config "${BASE_FILE_NAME}.private.txt" + __check_selector_in_signing_config 'someselector' +} + +# Create DKIM keys. +# +# @param ${1} = keytype (default: rsa) +# @param ${2} = selector (default: mail) +# @param ${3} = domain (default: ${DOMAIN}) +# @param ${4} = keysize (default: 2048) +function __create_key() { + local KEYTYPE=${1:-rsa} + local SELECTOR=${2:-mail} + local DOMAIN=${3:-${DOMAIN_NAME}} + local KEYSIZE=${4:-2048} + + _run_in_container setup config dkim \ + keytype "${KEYTYPE}" \ + keysize "${KEYSIZE}" \ + selector "${SELECTOR}" \ + domain "${DOMAIN}" +} + +# Check whether an RSA key is created successfully and correctly +# for a specific key size. +# +# @param ${1} = key size +# @param ${2} = name of the selector and domain name (as one string) +function __check_rsa_keys() { + local KEYSIZE=${1:?Keysize must be supplied to __check_rsa_keys} + local SELECTOR_AND_DOMAIN=${2:?Selector and domain name must be supplied to __check_rsa_keys} + local BASE_FILE_NAME="/tmp/docker-mailserver/rspamd/dkim/rsa-${KEYSIZE}-${SELECTOR_AND_DOMAIN}" + + __check_key_files_are_present "${BASE_FILE_NAME}" + __check_path_in_signing_config "${BASE_FILE_NAME}.private.txt" + + # Check the private key matches the specification + _run_in_container_bash "openssl rsa -in '${BASE_FILE_NAME}.private.txt' -noout -text" + assert_success + assert_line --index 0 "RSA Private-Key: (${KEYSIZE} bit, 2 primes)" + + # Check the public key matches the specification + # + # We utilize the file for the DNS record contents which is already created + # by the Rspamd DKIM helper script. This makes parsing easier here. + local PUBKEY PUBKEY_INFO + PUBKEY=$(_exec_in_container_bash "grep -o 'p=.*' ${BASE_FILE_NAME}.public.dns.txt") + _run_in_container_bash "openssl enc -base64 -d <<< ${PUBKEY#p=} | openssl pkey -inform DER -pubin -noout -text" + assert_success + assert_line --index 0 "RSA Public-Key: (${KEYSIZE} bit)" +} + +# Verify that all DKIM key files are present. +# +# @param ${1} = base file name that all DKIM key files have +function __check_key_files_are_present() { + local BASE_FILE_NAME="${1:?Base file name must be supplied to __check_key_files_are_present}" + for FILE in ${BASE_FILE_NAME}.{public.txt,public.dns.txt,private.txt} + do + _run_in_container_bash "[[ -f ${FILE} ]]" + assert_success + done +} + +# Check whether `path = .*` is set correctly in the signing configuration file. +# +# @param ${1} = file name that `path` should be set to +function __check_path_in_signing_config() { + local BASE_FILE_NAME=${1:?Base file name must be supplied to __check_path_in_signing_config} + _run_in_container grep "[[:space:]]*path = \"${BASE_FILE_NAME}\";" "${SIGNING_CONF_FILE}" + assert_success +} + +# Check whether `selector = .*` is set correctly in the signing configuration file. +# +# @param ${1} = name that `selector` should be set to +function __check_selector_in_signing_config() { + local SELECTOR=${1:?Selector name must be supplied to __check_selector_in_signing_config} + _run_in_container grep "[[:space:]]*selector = \"${SELECTOR}\";" "${SIGNING_CONF_FILE}" + assert_success +} + +# Check whether the script output is free of warnings and errors. +function __log_is_free_of_warnings_and_errors() { + assert_success + refute_output --partial '[ WARN ]' + refute_output --partial '[ ERROR ]' +} + +# Remove the signing configuration file inside the container. +function __remove_signing_config_file() { + _exec_in_container rm -f "${SIGNING_CONF_FILE}" +} From 9577ab5033fceb0b90ae183f3bba3704425570c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 15:07:40 +0200 Subject: [PATCH 062/592] chore(deps): Bump peter-evans/create-pull-request from 5.0.0 to 5.0.1 (#3314) --- .github/workflows/contributors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index 499fbbbc19f..7e49dcfcda3 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -49,7 +49,7 @@ jobs: # See https://github.com/marketplace/actions/create-pull-request for reference of the action. - name: 'Create Pull Request' - uses: peter-evans/create-pull-request@v5.0.0 + uses: peter-evans/create-pull-request@v5.0.1 id: create-pr with: token: ${{ secrets.GITHUB_TOKEN }} From 652bbd831f6d25bf79b619094610011891ecddb7 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 10 May 2023 10:20:46 +0200 Subject: [PATCH 063/592] release: v12.1.0 (#3305) Co-authored-by: Casper Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 143 ++++++++++++++++++++++++++++++++++----------------- VERSION | 2 +- 2 files changed, 97 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b6fa9b4e41..d13329767f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,60 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v12.0.0...HEAD) +## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v12.1.0...HEAD) > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. -## [12.0.0](https://github.com/docker-mailserver/docker-mailserver/compare/v12.0.0...HEAD) +## [v12.1.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v12.1.0) + +### Added + +- Rspamd: + - note about Rspamd's web interface ([#3245](https://github.com/docker-mailserver/docker-mailserver/pull/3245)) + - add greylisting option & code refactoring ([#3206](https://github.com/docker-mailserver/docker-mailserver/pull/3206)) + - added `HFILTER_HOSTNAME_UNKNOWN` and make it configurable ([#3248](https://github.com/docker-mailserver/docker-mailserver/pull/3248)) + - add option to re-enable `reject_unknown_client_hostname` after #3248 ([#3255](https://github.com/docker-mailserver/docker-mailserver/pull/3255)) + - add DKIM helper script ([#3286](https://github.com/docker-mailserver/docker-mailserver/pull/3286)) +- make `policyd-spf` configurable ([#3246](https://github.com/docker-mailserver/docker-mailserver/pull/3246)) +- add 'log' command to setup for Fail2Ban ([#3299](https://github.com/docker-mailserver/docker-mailserver/pull/3299)) +- `setup` command now expects accounts and aliases to be mutually exclusive ([#3270](https://github.com/docker-mailserver/docker-mailserver/pull/3270)) + +### Updated + +- update DKIM/DMARC/SPF docs ([#3231](https://github.com/docker-mailserver/docker-mailserver/pull/3231)) +- Fail2Ban: + - made config more aggressive ([#3243](https://github.com/docker-mailserver/docker-mailserver/pull/3243) & [#3288](https://github.com/docker-mailserver/docker-mailserver/pull/3288)) + - update fail2ban config examples with current DMS default values ([#3258](https://github.com/docker-mailserver/docker-mailserver/pull/3258)) + - make Fail2Ban log persistent ([#3269](https://github.com/docker-mailserver/docker-mailserver/pull/3269)) + - update F2B docs & bind mount links ([#3293](https://github.com/docker-mailserver/docker-mailserver/pull/3293)) +- Rspamd: + - improve Rspamd docs ([#3257](https://github.com/docker-mailserver/docker-mailserver/pull/3257)) + - script stabilization ([#3261](https://github.com/docker-mailserver/docker-mailserver/pull/3261) & [#3282](https://github.com/docker-mailserver/docker-mailserver/pull/3282)) + - remove WIP warnings ([#3283](https://github.com/docker-mailserver/docker-mailserver/pull/3283)) +- improve shutdown function by making PANIC_STRATEGY obsolete ([#3265](https://github.com/docker-mailserver/docker-mailserver/pull/3265)) +- update `bug_report.yml` ([#3275](https://github.com/docker-mailserver/docker-mailserver/pull/3275)) +- simplify `bug_report.yml` ([#3276](https://github.com/docker-mailserver/docker-mailserver/pull/3276)) +- revised the contributor workflow ([#2227](https://github.com/docker-mailserver/docker-mailserver/pull/2227)) + +### Changed + +- default registry changed from DockerHub (`docker.io`) to GHCR (`ghcr.io`) ([#3233](https://github.com/docker-mailserver/docker-mailserver/pull/3233)) +- consistent namings in docs ([#3242](https://github.com/docker-mailserver/docker-mailserver/pull/3242)) +- get all `policyd-spf` setup in one place ([#3263](https://github.com/docker-mailserver/docker-mailserver/pull/3263)) +- miscellaneous script improvements ([#3281](https://github.com/docker-mailserver/docker-mailserver/pull/3281)) +- update FAQ entries ([#3294](https://github.com/docker-mailserver/docker-mailserver/pull/3294)) + +### Fixed + +- GitHub Actions docs update workflow ([#3241](https://github.com/docker-mailserver/docker-mailserver/pull/3241)) +- fix dovecot: ldap mail delivery works ([#3252](https://github.com/docker-mailserver/docker-mailserver/pull/3252)) +- shellcheck: do not check .git folder ([#3267](https://github.com/docker-mailserver/docker-mailserver/pull/3267)) +- add missing -E for extended regexes in `smtpd_sender_restrictions` ([#3272](https://github.com/docker-mailserver/docker-mailserver/pull/3272)) +- fix setting `SRS_EXCLUDE_DOMAINS` during startup ([#3271](https://github.com/docker-mailserver/docker-mailserver/pull/3271)) +- remove superfluous `EOF` in `dmarc_dkim_spf.sh` ([#3266](https://github.com/docker-mailserver/docker-mailserver/pull/3266)) +- apply fixes to helpers when using `set -eE` ([#3285](https://github.com/docker-mailserver/docker-mailserver/pull/3285)) + +## [12.0.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v12.0.0) Notable changes are: @@ -263,49 +312,49 @@ In this release the relay-host support saw [significant internal refactoring](ht ### Merged Pull Requests -- **[improvement]** tests: remove legacy functions / tests by @casperklein in [#2434](https://github.com/docker-mailserver/docker-mailserver/pull/2434) -- **[improvement]** `PERMIT_DOCKER=none` as new default value by @casperklein in [#2424](https://github.com/docker-mailserver/docker-mailserver/pull/2424) -- **[improvement]** Adjust environment variables to more sensible defaults by @georglauterbach in [#2428](https://github.com/docker-mailserver/docker-mailserver/pull/2428) -- **[fix]** macOS linting support by @NorseGaud in [#2448](https://github.com/docker-mailserver/docker-mailserver/pull/2448) -- **[improvement]** Rename config examples directory by @casperklein in [#2438](https://github.com/docker-mailserver/docker-mailserver/pull/2438) -- **[docs]** FAQ - Update naked/bare domain section by @sportshead in [#2446](https://github.com/docker-mailserver/docker-mailserver/pull/2446) -- **[improvement]** Remove obsolete `setup.sh debug inspect` command from usage description by @casperklein in [#2454](https://github.com/docker-mailserver/docker-mailserver/pull/2454) -- **[feature]** Introduce `CLAMAV_MESSAGE_SIZE_LIMIT` env by @casperklein in [#2453](https://github.com/docker-mailserver/docker-mailserver/pull/2453) -- **[fix]** remove SA reload for KAM by @georglauterbach in [#2456](https://github.com/docker-mailserver/docker-mailserver/pull/2456) -- **[docs]** Enhance logrotate description by @casperklein in [#2469](https://github.com/docker-mailserver/docker-mailserver/pull/2469) -- **[improvement]** Remove macOS specific code / support + shellcheck should avoid python, regardless of permissions by @NorseGaud in [#2466](https://github.com/docker-mailserver/docker-mailserver/pull/2466) -- **[docs]** Update fail2ban.md by @casperklein in [#2484](https://github.com/docker-mailserver/docker-mailserver/pull/2484) -- **[fix]** Makefile: Remove backup/restore of obsolete config directory by @casperklein in [#2479](https://github.com/docker-mailserver/docker-mailserver/pull/2479) -- **[improvement]** scripts: small refactorings by @georglauterbach in [#2485](https://github.com/docker-mailserver/docker-mailserver/pull/2485) -- **[fix]** Building on Ubuntu 21.10 failing to install postfix by @NorseGaud in [#2468](https://github.com/docker-mailserver/docker-mailserver/pull/2468) -- **[improvement]** Use FQDN as `REPORT_SENDER` default value by @casperklein in [#2487](https://github.com/docker-mailserver/docker-mailserver/pull/2487) -- **[improvement]** Improve test, get rid of sleep by @casperklein in [#2492](https://github.com/docker-mailserver/docker-mailserver/pull/2492) -- **[feature]** scripts: new log by @georglauterbach in [#2493](https://github.com/docker-mailserver/docker-mailserver/pull/2493) -- **[fix]** Restart supervisord early by @casperklein in [#2494](https://github.com/docker-mailserver/docker-mailserver/pull/2494) -- **[improvement]** scripts: renamed function `_errex` -> `_exit_with_error` by @georglauterbach in [#2497](https://github.com/docker-mailserver/docker-mailserver/pull/2497) -- **[improvement]** Remove invalid URL from SPF message by @casperklein in [#2503](https://github.com/docker-mailserver/docker-mailserver/pull/2503) -- **[improvement]** scripts: refactored scripts located under `target/bin/` by @georglauterbach in [#2500](https://github.com/docker-mailserver/docker-mailserver/pull/2500) -- **[improvement]** scripts: refactoring & miscellaneous small changes by @georglauterbach in [#2499](https://github.com/docker-mailserver/docker-mailserver/pull/2499) -- **[improvement]** scripts: refactored `daemon-stack.sh` by @georglauterbach in [#2496](https://github.com/docker-mailserver/docker-mailserver/pull/2496) -- **[fix]** add compatibility for Bash 4 to setup.sh by @georglauterbach in [#2519](https://github.com/docker-mailserver/docker-mailserver/pull/2519) -- **[fix]** tests: disabled "quota exceeded" test by @georglauterbach in [#2511](https://github.com/docker-mailserver/docker-mailserver/pull/2511) -- **[fix]** typo in setup-stack.sh by @eltociear in [#2521](https://github.com/docker-mailserver/docker-mailserver/pull/2521) -- **[improvement]** scripts: introduce `_log` to `sedfile` by @georglauterbach in [#2507](https://github.com/docker-mailserver/docker-mailserver/pull/2507) -- **[feature]** create `.github/FUNDING.yml` by @georglauterbach in [#2512](https://github.com/docker-mailserver/docker-mailserver/pull/2512) -- **[improvement]** scripts: refactored `check-for-changes.sh` by @georglauterbach in [#2498](https://github.com/docker-mailserver/docker-mailserver/pull/2498) -- **[improvement]** scripts: remove `DMS_DEBUG` by @georglauterbach in [#2523](https://github.com/docker-mailserver/docker-mailserver/pull/2523) -- **[feature]** firewall: replace `iptables` with `nftables` by @georglauterbach in [#2505](https://github.com/docker-mailserver/docker-mailserver/pull/2505) -- **[improvement]** log: adjust level and message(s) slightly for four messages by @georglauterbach in [#2532](https://github.com/docker-mailserver/docker-mailserver/pull/2532) -- **[improvement]** log: introduce proper log level fallback and env getter function by @georglauterbach in [#2506](https://github.com/docker-mailserver/docker-mailserver/pull/2506) -- **[feature]** scripts: added `TZ` environment variable to set timezone by @georglauterbach in [#2530](https://github.com/docker-mailserver/docker-mailserver/pull/2530) -- **[improvement]** setup: added grace period for account creation by @georglauterbach in [#2531](https://github.com/docker-mailserver/docker-mailserver/pull/2531) -- **[improvement]** refactor: letsencrypt implicit location discovery by @polarathene in [#2525](https://github.com/docker-mailserver/docker-mailserver/pull/2525) -- **[improvement]** setup.sh/setup: show usage when no argument is given by @casperklein in [#2540](https://github.com/docker-mailserver/docker-mailserver/pull/2540) -- **[improvement]** Dockerfile: Remove not needed ENVs and add comment by @casperklein in [#2541](https://github.com/docker-mailserver/docker-mailserver/pull/2541) -- **[improvement]** chore: (setup-stack.sh) Fix a small typo by @polarathene in [#2552](https://github.com/docker-mailserver/docker-mailserver/pull/2552) -- **[feature]** Add ban feature to fail2ban script by @casperklein in [#2538](https://github.com/docker-mailserver/docker-mailserver/pull/2538) -- **[fix]** Fix changedetector restart loop by @casperklein in [#2548](https://github.com/docker-mailserver/docker-mailserver/pull/2548) -- **[improvement]** chore: Drop `setup.sh` DATABASE fallback ENV by @polarathene in [#2556](https://github.com/docker-mailserver/docker-mailserver/pull/2556) +- **[improvement]** tests: remove legacy functions / tests [#2434](https://github.com/docker-mailserver/docker-mailserver/pull/2434) +- **[improvement]** `PERMIT_DOCKER=none` as new default value [#2424](https://github.com/docker-mailserver/docker-mailserver/pull/2424) +- **[improvement]** Adjust environment variables to more sensible defaults [#2428](https://github.com/docker-mailserver/docker-mailserver/pull/2428) +- **[fix]** macOS linting support [#2448](https://github.com/docker-mailserver/docker-mailserver/pull/2448) +- **[improvement]** Rename config examples directory [#2438](https://github.com/docker-mailserver/docker-mailserver/pull/2438) +- **[docs]** FAQ - Update naked/bare domain section [#2446](https://github.com/docker-mailserver/docker-mailserver/pull/2446) +- **[improvement]** Remove obsolete `setup.sh debug inspect` command from usage description [#2454](https://github.com/docker-mailserver/docker-mailserver/pull/2454) +- **[feature]** Introduce `CLAMAV_MESSAGE_SIZE_LIMIT` env [#2453](https://github.com/docker-mailserver/docker-mailserver/pull/2453) +- **[fix]** remove SA reload for KAM [#2456](https://github.com/docker-mailserver/docker-mailserver/pull/2456) +- **[docs]** Enhance logrotate description [#2469](https://github.com/docker-mailserver/docker-mailserver/pull/2469) +- **[improvement]** Remove macOS specific code / support + shellcheck should avoid python, regardless of permissions [#2466](https://github.com/docker-mailserver/docker-mailserver/pull/2466) +- **[docs]** Update fail2ban.md [#2484](https://github.com/docker-mailserver/docker-mailserver/pull/2484) +- **[fix]** Makefile: Remove backup/restore of obsolete config directory [#2479](https://github.com/docker-mailserver/docker-mailserver/pull/2479) +- **[improvement]** scripts: small refactorings [#2485](https://github.com/docker-mailserver/docker-mailserver/pull/2485) +- **[fix]** Building on Ubuntu 21.10 failing to install postfix [#2468](https://github.com/docker-mailserver/docker-mailserver/pull/2468) +- **[improvement]** Use FQDN as `REPORT_SENDER` default value [#2487](https://github.com/docker-mailserver/docker-mailserver/pull/2487) +- **[improvement]** Improve test, get rid of sleep [#2492](https://github.com/docker-mailserver/docker-mailserver/pull/2492) +- **[feature]** scripts: new log [#2493](https://github.com/docker-mailserver/docker-mailserver/pull/2493) +- **[fix]** Restart supervisord early [#2494](https://github.com/docker-mailserver/docker-mailserver/pull/2494) +- **[improvement]** scripts: renamed function `_errex` -> `_exit_with_error` [#2497](https://github.com/docker-mailserver/docker-mailserver/pull/2497) +- **[improvement]** Remove invalid URL from SPF message [#2503](https://github.com/docker-mailserver/docker-mailserver/pull/2503) +- **[improvement]** scripts: refactored scripts located under `target/bin/` [#2500](https://github.com/docker-mailserver/docker-mailserver/pull/2500) +- **[improvement]** scripts: refactoring & miscellaneous small changes [#2499](https://github.com/docker-mailserver/docker-mailserver/pull/2499) +- **[improvement]** scripts: refactored `daemon-stack.sh` [#2496](https://github.com/docker-mailserver/docker-mailserver/pull/2496) +- **[fix]** add compatibility for Bash 4 to setup.sh [#2519](https://github.com/docker-mailserver/docker-mailserver/pull/2519) +- **[fix]** tests: disabled "quota exceeded" test [#2511](https://github.com/docker-mailserver/docker-mailserver/pull/2511) +- **[fix]** typo in setup-stack.sh [#2521](https://github.com/docker-mailserver/docker-mailserver/pull/2521) +- **[improvement]** scripts: introduce `_log` to `sedfile` [#2507](https://github.com/docker-mailserver/docker-mailserver/pull/2507) +- **[feature]** create `.github/FUNDING.yml` [#2512](https://github.com/docker-mailserver/docker-mailserver/pull/2512) +- **[improvement]** scripts: refactored `check-for-changes.sh` [#2498](https://github.com/docker-mailserver/docker-mailserver/pull/2498) +- **[improvement]** scripts: remove `DMS_DEBUG` [#2523](https://github.com/docker-mailserver/docker-mailserver/pull/2523) +- **[feature]** firewall: replace `iptables` with `nftables` [#2505](https://github.com/docker-mailserver/docker-mailserver/pull/2505) +- **[improvement]** log: adjust level and message(s) slightly for four messages [#2532](https://github.com/docker-mailserver/docker-mailserver/pull/2532) +- **[improvement]** log: introduce proper log level fallback and env getter function [#2506](https://github.com/docker-mailserver/docker-mailserver/pull/2506) +- **[feature]** scripts: added `TZ` environment variable to set timezone [#2530](https://github.com/docker-mailserver/docker-mailserver/pull/2530) +- **[improvement]** setup: added grace period for account creation [#2531](https://github.com/docker-mailserver/docker-mailserver/pull/2531) +- **[improvement]** refactor: letsencrypt implicit location discovery [#2525](https://github.com/docker-mailserver/docker-mailserver/pull/2525) +- **[improvement]** setup.sh/setup: show usage when no argument is given [#2540](https://github.com/docker-mailserver/docker-mailserver/pull/2540) +- **[improvement]** Dockerfile: Remove not needed ENVs and add comment [#2541](https://github.com/docker-mailserver/docker-mailserver/pull/2541) +- **[improvement]** chore: (setup-stack.sh) Fix a small typo [#2552](https://github.com/docker-mailserver/docker-mailserver/pull/2552) +- **[feature]** Add ban feature to fail2ban script [#2538](https://github.com/docker-mailserver/docker-mailserver/pull/2538) +- **[fix]** Fix changedetector restart loop [#2548](https://github.com/docker-mailserver/docker-mailserver/pull/2548) +- **[improvement]** chore: Drop `setup.sh` DATABASE fallback ENV [#2556](https://github.com/docker-mailserver/docker-mailserver/pull/2556) ## `v10.5.0` @@ -337,9 +386,9 @@ In this release the relay-host support saw [significant internal refactoring](ht - **[fix]** Fixed non-number-argument in `listmailuser` in [#2382](https://github.com/docker-mailserver/docker-mailserver/pull/2382) - **[fix]** docs: Fail2Ban - Fix links for rootless podman in [#2384](https://github.com/docker-mailserver/docker-mailserver/pull/2384) - **[fix]** docs(kubernetes): fix image name in example in [#2385](https://github.com/docker-mailserver/docker-mailserver/pull/2385) -- **[fix]** SSL documentation contains a small bug #2381 by @Twist235 in [#2383](https://github.com/docker-mailserver/docker-mailserver/pull/2383) +- **[fix]** SSL documentation contains a small bug #2381 [#2383](https://github.com/docker-mailserver/docker-mailserver/pull/2383) - **[fix]** get rid of subshell + `exec` in `helper-functions.sh` in [#2401](https://github.com/docker-mailserver/docker-mailserver/pull/2401) -- **[docs]** Rootless Podman security update by @p-fruck in [#2393](https://github.com/docker-mailserver/docker-mailserver/pull/2393) +- **[docs]** Rootless Podman security update [#2393](https://github.com/docker-mailserver/docker-mailserver/pull/2393) - **[fix]** fix: double occurrence of `/etc/postfix/regexp` in [#2397](https://github.com/docker-mailserver/docker-mailserver/pull/2397) - **[improvement]** consistently make 1 the default value for `SPAMASSASSIN_SPAM_TO_INBOX` in [#2361](https://github.com/docker-mailserver/docker-mailserver/pull/2361) - **[docs]** added sieve example for subaddress sorting in [#2410](https://github.com/docker-mailserver/docker-mailserver/pull/2410) diff --git a/VERSION b/VERSION index 4044f90867d..77903b35f3a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -12.0.0 +12.1.0 From c461dabe9e5bb7e263169def89469f7c1c9d9dd5 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 10 May 2023 11:02:44 +0200 Subject: [PATCH 064/592] docs/misc: update to align with Docker Compose v2 (#3295) * rename: `docker-compose.yml` => `compose.yaml` * rename: `docker-compose` => `docker compose` --- .github/ISSUE_TEMPLATE/bug_report.yml | 4 +-- docker-compose.yml => compose.yaml | 0 .../config/advanced/full-text-search.md | 20 ++++++------ docs/content/config/advanced/ipv6.md | 2 +- .../content/config/advanced/mail-fetchmail.md | 4 +-- docs/content/config/advanced/mail-sieve.md | 2 +- .../maintenance/update-and-cleanup.md | 4 +-- .../advanced/override-defaults/dovecot.md | 2 +- docs/content/config/advanced/podman.md | 20 ++++++------ .../config/best-practices/dkim_dmarc_spf.md | 2 +- docs/content/config/environment.md | 8 ++--- docs/content/config/pop3.md | 2 +- docs/content/config/security/fail2ban.md | 4 +-- docs/content/config/security/mail_crypt.md | 4 +-- docs/content/config/security/ssl.md | 32 +++++++++---------- .../examples/tutorials/basic-installation.md | 8 ++--- .../examples/use-cases/imap-folders.md | 6 ++-- docs/content/faq.md | 18 +++++------ docs/content/introduction.md | 5 ++- docs/content/usage.md | 11 ++++--- mailserver.env | 4 +-- target/scripts/helpers/ssl.sh | 4 +-- .../process_check_restart.bats | 2 +- 23 files changed, 84 insertions(+), 84 deletions(-) rename docker-compose.yml => compose.yaml (100%) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 55e8083c2ef..e8d60dabea3 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -125,8 +125,8 @@ body: - type: textarea id: important-environment-variables attributes: - label: docker-compose.yml - description: Show us your `docker-compose.yml` file or your equivalent `docker run` command, if applicable. This filed is formatted as YAML. + label: compose.yaml + description: Show us your `compose.yaml` file or your equivalent `docker run` command, if applicable. This filed is formatted as YAML. render: yml - type: textarea id: relevant-log-output diff --git a/docker-compose.yml b/compose.yaml similarity index 100% rename from docker-compose.yml rename to compose.yaml diff --git a/docs/content/config/advanced/full-text-search.md b/docs/content/config/advanced/full-text-search.md index f4f2f760f58..2e29e49690f 100644 --- a/docs/content/config/advanced/full-text-search.md +++ b/docs/content/config/advanced/full-text-search.md @@ -55,7 +55,7 @@ While indexing is memory intensive, you can configure the plugin to limit the am adjust the settings to tune for your desired memory limits, exclude folders and enable searching text inside of attachments -2. Update `docker-compose.yml` to load the previously created dovecot plugin config file: +2. Update `compose.yaml` to load the previously created dovecot plugin config file: ```yaml services: @@ -86,20 +86,20 @@ While indexing is memory intensive, you can configure the plugin to limit the am 3. Recreate containers: ``` - docker-compose down - docker-compose up -d + docker compose down + docker compose up -d ``` 4. Initialize indexing on all users for all mail: ``` - docker-compose exec mailserver doveadm index -A -q \* + docker compose exec mailserver doveadm index -A -q \* ``` 5. Run the following command in a daily cron job: ``` - docker-compose exec mailserver doveadm fts optimize -A + docker compose exec mailserver doveadm fts optimize -A ``` Or like the [Spamassassin example][docs-faq-sa-learn-cron] shows, you can instead use `cron` from within DMS to avoid potential errors if the mail server is not running: @@ -108,7 +108,7 @@ While indexing is memory intensive, you can configure the plugin to limit the am Create a _system_ cron file: ```sh - # in the docker-compose.yml root directory + # in the compose.yaml root directory mkdir -p ./docker-data/dms/cron # if you didn't have this folder before touch ./docker-data/dms/cron/fts_xapian chown root:root ./docker-data/dms/cron/fts_xapian @@ -127,7 +127,7 @@ While indexing is memory intensive, you can configure the plugin to limit the am 0 4 * * * root doveadm fts optimize -A ``` - Then with `docker-compose.yml`: + Then with `compose.yaml`: ```yaml services: @@ -148,7 +148,7 @@ However, Solr also requires a fair bit of RAM. While Solr is [highly tuneable](h #### Setup -1. `docker-compose.yml`: +1. `compose.yaml`: ```yaml solr: @@ -180,9 +180,9 @@ However, Solr also requires a fair bit of RAM. While Solr is [highly tuneable](h } ``` -3. Recreate containers: `docker-compose down ; docker-compose up -d` +3. Recreate containers: `docker compose down ; docker compose up -d` -4. Flag all user mailbox FTS indexes as invalid, so they are rescanned on demand when they are next searched: `docker-compose exec mailserver doveadm fts rescan -A` +4. Flag all user mailbox FTS indexes as invalid, so they are rescanned on demand when they are next searched: `docker compose exec mailserver doveadm fts rescan -A` #### Further Discussion diff --git a/docs/content/config/advanced/ipv6.md b/docs/content/config/advanced/ipv6.md index 2853e434153..d77f8d7a1ce 100644 --- a/docs/content/config/advanced/ipv6.md +++ b/docs/content/config/advanced/ipv6.md @@ -11,7 +11,7 @@ This can be solved by supporting IPv6 connections all the way to the DMS contain ## Setup steps ```diff -+++ b/serv/docker-compose.yml ++++ b/serv/compose.yaml @@ ... @@ services: + ipv6nat: diff --git a/docs/content/config/advanced/mail-fetchmail.md b/docs/content/config/advanced/mail-fetchmail.md index b364eed096d..59e07a844a2 100644 --- a/docs/content/config/advanced/mail-fetchmail.md +++ b/docs/content/config/advanced/mail-fetchmail.md @@ -2,7 +2,7 @@ title: 'Advanced | Email Gathering with Fetchmail' --- -To enable the [fetchmail][fetchmail-website] service to retrieve e-mails set the environment variable `ENABLE_FETCHMAIL` to `1`. Your `docker-compose.yml` file should look like following snippet: +To enable the [fetchmail][fetchmail-website] service to retrieve e-mails set the environment variable `ENABLE_FETCHMAIL` to `1`. Your `compose.yaml` file should look like following snippet: ```yaml environment: @@ -18,7 +18,7 @@ Generate a file called `fetchmail.cf` and place it in the `docker-data/dms/confi │   ├── fetchmail.cf │   ├── postfix-accounts.cf │   └── postfix-virtual.cf -├── docker-compose.yml +├── compose.yaml └── README.md ``` diff --git a/docs/content/config/advanced/mail-sieve.md b/docs/content/config/advanced/mail-sieve.md index f6577329e6f..2918cdfeb50 100644 --- a/docs/content/config/advanced/mail-sieve.md +++ b/docs/content/config/advanced/mail-sieve.md @@ -85,7 +85,7 @@ The [Manage Sieve](https://doc.dovecot.org/admin_manual/pigeonhole_managesieve_s !!! example ```yaml - # docker-compose.yml + # compose.yaml ports: - "4190:4190" environment: diff --git a/docs/content/config/advanced/maintenance/update-and-cleanup.md b/docs/content/config/advanced/maintenance/update-and-cleanup.md index dcf53c934c7..3e4301f9364 100644 --- a/docs/content/config/advanced/maintenance/update-and-cleanup.md +++ b/docs/content/config/advanced/maintenance/update-and-cleanup.md @@ -8,7 +8,7 @@ Docker images are handy but it can become a hassle to keep them updated. Also wh One could setup a complex action/hook-based workflow using probes, but there is a nice, easy to use docker image that solves this issue and could prove useful: [`watchtower`](https://hub.docker.com/r/containrrr/watchtower). -A docker-compose example: +A Docker Compose example: ```yaml services: @@ -25,7 +25,7 @@ For more details, see the [manual](https://containrrr.github.io/watchtower/) When you are pulling new images in automatically, it would be nice to have them cleaned up as well. There is also a docker image for this: [`spotify/docker-gc`](https://hub.docker.com/r/spotify/docker-gc/). -A docker-compose example: +A Docker Compose example: ```yaml services: diff --git a/docs/content/config/advanced/override-defaults/dovecot.md b/docs/content/config/advanced/override-defaults/dovecot.md index ab1f3bb5036..84d13a04ea2 100644 --- a/docs/content/config/advanced/override-defaults/dovecot.md +++ b/docs/content/config/advanced/override-defaults/dovecot.md @@ -14,7 +14,7 @@ Your DMS folder structure should look like this example: │ ├── dovecot.cf │ ├── postfix-accounts.cf │ └── postfix-virtual.cf -├── docker-compose.yml +├── compose.yaml └── README.md ``` diff --git a/docs/content/config/advanced/podman.md b/docs/content/config/advanced/podman.md index fb149f6e4d7..b32288c1b8b 100644 --- a/docs/content/config/advanced/podman.md +++ b/docs/content/config/advanced/podman.md @@ -23,7 +23,7 @@ Podman is a daemonless container engine for developing, managing, and running OC While using Podman, you can just manage docker-mailserver as what you did with Docker. Your best friend `setup.sh` includes the minimum code in order to support Podman since it's 100% compatible with the Docker CLI. -The installation is basically the same. Podman v3.2 introduced a RESTful API that is 100% compatible with the Docker API, so you can use docker-compose with Podman easily. Install Podman and docker-compose with your package manager first. +The installation is basically the same. Podman v3.2 introduced a RESTful API that is 100% compatible with the Docker API, so you can use Docker Compose with Podman easily. Install Podman and Docker Compose with your package manager first. ```bash sudo dnf install podman docker-compose @@ -39,8 +39,8 @@ This will create a unix socket locate under `/run/podman/podman.sock`, which is ```bash export DOCKER_HOST="unix:///run/podman/podman.sock" -docker-compose up -d mailserver -docker-compose ps +docker compose up -d mailserver +docker compose ps ``` You should see that docker-mailserver is running now. @@ -75,7 +75,7 @@ First, enable `podman.socket` in systemd's userspace with a non-root user. systemctl enable --now --user podman.socket ``` -The socket file should be located at `/var/run/user/$(id -u)/podman/podman.sock`. Then, modify `docker-compose.yml` to make sure all ports are bindings are on non-privileged ports. +The socket file should be located at `/var/run/user/$(id -u)/podman/podman.sock`. Then, modify `compose.yaml` to make sure all ports are bindings are on non-privileged ports. ```yaml services: @@ -88,12 +88,12 @@ services: - "10993:993" # IMAP4 (implicit TLS) ``` -Then, setup your `mailserver.env` file follow the documentation and use docker-compose to start the container. +Then, setup your `mailserver.env` file follow the documentation and use Docker Compose to start the container. ```bash export DOCKER_HOST="unix:///var/run/user/$(id -u)/podman/podman.sock" -docker-compose up -d mailserver -docker-compose ps +docker compose up -d mailserver +docker compose ps ``` ### Security in Rootless Mode @@ -106,12 +106,12 @@ The `PERMIT_DOCKER` variable in the `mailserver.env` file allows to specify trus #### Use the slip4netns network driver -The second workaround is slightly more complicated because the `docker-compose.yml` has to be modified. +The second workaround is slightly more complicated because the `compose.yaml` has to be modified. As shown in the [fail2ban section](../../security/fail2ban/#podman-with-slirp4netns-port-driver) the `slirp4netns` network driver has to be enabled. This network driver enables podman to correctly resolve IP addresses but it is not compatible with user defined networks which might be a problem depending on your setup. -[Rootless Podman][rootless::podman] requires adding the value `slirp4netns:port_handler=slirp4netns` to the `--network` CLI option, or `network_mode` setting in your `docker-compose.yml`. +[Rootless Podman][rootless::podman] requires adding the value `slirp4netns:port_handler=slirp4netns` to the `--network` CLI option, or `network_mode` setting in your `compose.yaml`. You must also add the ENV `NETWORK_INTERFACE=tap0`, because Podman uses a [hard-coded interface name][rootless::podman::interface] for `slirp4netns`. @@ -169,7 +169,7 @@ firewall-cmd --permanent --direct --add-rule nat OUTPUT 0 -p Uses the `hostname -f` command to get canonical hostname for DMS to use. - => Specify an FQDN (fully-qualified domain name) to serve mail for. The hostname is required for DMS to function correctly. @@ -132,7 +132,7 @@ Enabled `policyd-spf` in Postfix's configuration. You will likely want to set th - **0** => fail2ban service disabled - 1 => Enables fail2ban service -If you enable Fail2Ban, don't forget to add the following lines to your `docker-compose.yml`: +If you enable Fail2Ban, don't forget to add the following lines to your `compose.yaml`: ``` BASH cap_add: @@ -458,7 +458,7 @@ Changes the interval in which log files are rotated. The entire log output for the container is still available via `docker logs mailserver` (or your respective container name). If you want to configure external log rotation for that container output as well, : [Docker Logging Drivers](https://docs.docker.com/config/containers/logging/configure/). - By default, the logs are lost when the container is destroyed (eg: re-creating via `docker-compose down && docker-compose up -d`). To keep the logs, mount a volume (to `/var/log/mail/`). + By default, the logs are lost when the container is destroyed (eg: re-creating via `docker compose down && docker compose up -d`). To keep the logs, mount a volume (to `/var/log/mail/`). !!! note @@ -562,7 +562,7 @@ Deprecated. See [`ACCOUNT_PROVISIONER`](#account_provisioner). - **empty** => mail.example.com - => Specify the dns-name/ip-address where the ldap-server is listening, or an URI like `ldaps://mail.example.com` -- NOTE: If you going to use DMS in combination with `docker-compose.yml` you can set the service name here +- NOTE: If you going to use DMS in combination with `compose.yaml` you can set the service name here ##### LDAP_SEARCH_BASE diff --git a/docs/content/config/pop3.md b/docs/content/config/pop3.md index 1acc8e85dc6..0cc9e17f600 100644 --- a/docs/content/config/pop3.md +++ b/docs/content/config/pop3.md @@ -4,7 +4,7 @@ hide: - toc # Hide Table of Contents for this page --- -If you want to use POP3(S), you have to add the ports 110 and/or 995 (TLS secured) and the environment variable `ENABLE_POP3` to your `docker-compose.yml`: +If you want to use POP3(S), you have to add the ports 110 and/or 995 (TLS secured) and the environment variable `ENABLE_POP3` to your `compose.yaml`: ```yaml mailserver: diff --git a/docs/content/config/security/fail2ban.md b/docs/content/config/security/fail2ban.md index b9f8ad3826b..0ea052a1e5c 100644 --- a/docs/content/config/security/fail2ban.md +++ b/docs/content/config/security/fail2ban.md @@ -16,7 +16,7 @@ hide: !!! warning - DMS must be launched with the `NET_ADMIN` capability in order to be able to install the NFTables rules that actually ban IP addresses. Thus, either include `--cap-add=NET_ADMIN` in the `docker run` command, or the equivalent in the `compose.yml`: + DMS must be launched with the `NET_ADMIN` capability in order to be able to install the NFTables rules that actually ban IP addresses. Thus, either include `--cap-add=NET_ADMIN` in the `docker run` command, or the equivalent in the `compose.yaml`: ```yaml cap_add: @@ -106,7 +106,7 @@ It is necessary for F2B to have access to the real source IP addresses in order === "Podman" - [Rootless Podman][rootless::podman] requires adding the value `slirp4netns:port_handler=slirp4netns` to the `--network` CLI option, or `network_mode` setting in your `compose.yml`: + [Rootless Podman][rootless::podman] requires adding the value `slirp4netns:port_handler=slirp4netns` to the `--network` CLI option, or `network_mode` setting in your `compose.yaml`: !!! example diff --git a/docs/content/config/security/mail_crypt.md b/docs/content/config/security/mail_crypt.md index 12f8698b475..463e91f6df4 100644 --- a/docs/content/config/security/mail_crypt.md +++ b/docs/content/config/security/mail_crypt.md @@ -30,7 +30,7 @@ Official Dovecot documentation: https://doc.dovecot.org/configuration_manual/mai } ``` -2. Shutdown your mailserver (`docker-compose down`) +2. Shutdown your mailserver (`docker compose down`) 3. You then need to [generate your global EC key](https://doc.dovecot.org/configuration_manual/mail_crypt_plugin/#ec-key). We named them `/certs/ecprivkey.pem` and `/certs/ecpubkey.pem` in step #1. @@ -45,7 +45,7 @@ Official Dovecot documentation: https://doc.dovecot.org/configuration_manual/mai . . . ``` -5. While you're editing the `docker-compose.yml`, add the configuration file: +5. While you're editing the `compose.yaml`, add the configuration file: ```yaml services: mailserver: diff --git a/docs/content/config/security/ssl.md b/docs/content/config/security/ssl.md index 2df2056b27c..f81470ac11b 100644 --- a/docs/content/config/security/ssl.md +++ b/docs/content/config/security/ssl.md @@ -44,7 +44,7 @@ An [FQDN](https://en.wikipedia.org/wiki/Fully_qualified_domain_name) (_Fully Qua or ```yml - # docker-compose.yml + # compose.yaml services: mailserver: hostname: mail.example.com @@ -72,7 +72,7 @@ You don't have to do anything else. Enjoy! !!! example - Add these additions to the `mailserver` service in your [`docker-compose.yml`][github-file-compose]: + Add these additions to the `mailserver` service in your [`compose.yaml`][github-file-compose]: ```yaml services: @@ -103,7 +103,7 @@ Certbot provisions certificates to `/etc/letsencrypt`. Add a volume to store the !!! example - Add these additions to the `mailserver` service in your [`docker-compose.yml`][github-file-compose]: + Add these additions to the `mailserver` service in your [`compose.yaml`][github-file-compose]: ```yaml services: @@ -163,7 +163,7 @@ Obtain a Cloudflare API token: - As this is sensitive data, you should restrict access to it with `chmod 600` and `chown 0:0`. - Store the file in a folder if you like, such as `docker-data/certbot/secrets/`. -5. Your `docker-compose.yml` should include the following: +5. Your `compose.yaml` should include the following: ```yaml services: @@ -206,7 +206,7 @@ Obtain a Cloudflare API token: 6. Run the service to provision a certificate: ```sh - docker-compose run certbot-cloudflare + docker compose run certbot-cloudflare ``` 7. You should see the following log output: @@ -229,7 +229,7 @@ After completing the steps above, your certificate should be ready to use. We've only demonstrated how to provision a certificate, but it will expire in 90 days and need to be renewed before then. - In the following example, add a new service (`certbot-cloudflare-renew`) into `docker-compose.yml` that will handle certificate renewals: + In the following example, add a new service (`certbot-cloudflare-renew`) into `compose.yaml` that will handle certificate renewals: ```yml services: @@ -247,7 +247,7 @@ After completing the steps above, your certificate should be ready to use. You can manually run this service to renew the cert within 90 days: ```sh - docker-compose run certbot-cloudflare-renew + docker compose run certbot-cloudflare-renew ``` You should see the following output @@ -273,7 +273,7 @@ After completing the steps above, your certificate should be ready to use. (`crontab` example: Checks every day if the certificate should be renewed) ```sh - 0 0 * * * docker-compose -f PATH_TO_YOUR_DOCKER_COMPOSE_YML up certbot-cloudflare-renew + 0 0 * * * docker compose -f PATH_TO_YOUR_DOCKER_COMPOSE_YML up certbot-cloudflare-renew ``` #### Example using `nginx-proxy` and `acme-companion` with Docker { data-toc-label='nginx-proxy with Docker' } @@ -327,7 +327,7 @@ In the following example, we show how DMS can be run alongside the docker contai You may want to add `--env LETSENCRYPT_TEST=true` to the above while testing, to avoid the _Let's Encrypt_ certificate generation rate limits. -5. Make sure your mount path to the `letsencrypt` certificates directory is correct. Edit your `docker-compose.yml` for the `mailserver` service to have volumes added like below: +5. Make sure your mount path to the `letsencrypt` certificates directory is correct. Edit your `compose.yaml` for the `mailserver` service to have volumes added like below: ```yaml volumes: @@ -337,15 +337,15 @@ In the following example, we show how DMS can be run alongside the docker contai - ./docker-data/acme-companion/certs/:/etc/letsencrypt/live/:ro ``` -6. Then from the `docker-compose.yml` project directory, run: `docker-compose up -d mailserver`. +6. Then from the `compose.yaml` project directory, run: `docker compose up -d mailserver`. #### Example using `nginx-proxy` and `acme-companion` with `docker-compose` { data-toc-label='nginx-proxy with docker-compose' } The following example is the [basic setup][acme-companion::basic-setup] you need for using `nginx-proxy` and `acme-companion` with DMS (_Referencing: [`acme-companion` documentation][acme-companion::docs]_): -???+ example "Example: `docker-compose.yml`" +???+ example "Example: `compose.yaml`" - You should have an existing `docker-compose.yml` with a `mailserver` service. Below are the modifications to add for integrating with `nginx-proxy` and `acme-companion` services: + You should have an existing `compose.yaml` with a `mailserver` service. Below are the modifications to add for integrating with `nginx-proxy` and `acme-companion` services: ```yaml services: @@ -385,7 +385,7 @@ The following example is the [basic setup][acme-companion::basic-setup] you need container_name: nginx-proxy-acme restart: always environment: - # Only docker-compose v2 supports: `volumes_from: [nginx-proxy]`, + # When `volumes_from: [nginx-proxy]` is not supported, # reference the _reverse-proxy_ `container_name` here: - NGINX_PROXY_CONTAINER=nginx-proxy volumes: @@ -463,7 +463,7 @@ Version 6.2 and later of the Synology NAS DSM OS now come with an interface to g Amongst other things, you can use these to secure your mail server. DSM locates the generated certificates in a folder below `/usr/syno/etc/certificate/_archive/`. -Navigate to that folder and note the 6 character random folder name of the certificate you'd like to use. Then, add the following to your `docker-compose.yml` declaration file: +Navigate to that folder and note the 6 character random folder name of the certificate you'd like to use. Then, add the following to your `compose.yaml` declaration file: ```yaml volumes: @@ -689,7 +689,7 @@ docker run --rm -it \ ### Bring Your Own Certificates -You can also provide your own certificate files. Add these entries to your `docker-compose.yml`: +You can also provide your own certificate files. Add these entries to your `compose.yaml`: ```yaml volumes: @@ -878,7 +878,7 @@ Despite this, if you must use non-standard DH parameters or you would like to sw [docs-optional-config]: ../advanced/optional-config.md [docs-faq-baredomain]: ../../faq.md#can-i-use-a-nakedbare-domain-ie-no-hostname -[github-file-compose]: https://github.com/docker-mailserver/docker-mailserver/blob/master/docker-compose.yml +[github-file-compose]: https://github.com/docker-mailserver/docker-mailserver/blob/master/compose.yaml [github-file::tls-readme]: https://github.com/docker-mailserver/docker-mailserver/blob/3b8059f2daca80d967635e04d8d81e9abb755a4d/test/test-files/ssl/example.test/README.md [hanscees-renewcerts]: https://github.com/hanscees/dockerscripts/blob/master/scripts/tomav-renew-certs diff --git a/docs/content/examples/tutorials/basic-installation.md b/docs/content/examples/tutorials/basic-installation.md index 01b8d5aeead..2ad4982e770 100644 --- a/docs/content/examples/tutorials/basic-installation.md +++ b/docs/content/examples/tutorials/basic-installation.md @@ -4,7 +4,7 @@ title: 'Tutorials | Basic Installation' ## A Basic Example With Relevant Environmental Variables -This example provides you only with a basic example of what a minimal setup could look like. We **strongly recommend** that you go through the configuration file yourself and adjust everything to your needs. The default [docker-compose.yml](https://github.com/docker-mailserver/docker-mailserver/blob/master/docker-compose.yml) can be used for the purpose out-of-the-box, see the [_Usage_ chapter](../../usage.md). +This example provides you only with a basic example of what a minimal setup could look like. We **strongly recommend** that you go through the configuration file yourself and adjust everything to your needs. The default [compose.yaml](https://github.com/docker-mailserver/docker-mailserver/blob/master/compose.yaml) can be used for the purpose out-of-the-box, see the [_Usage_ chapter](../../usage.md). ``` YAML services: @@ -96,7 +96,7 @@ In this setup DMS is not intended to receive email from the outside world, so no Adding the docker network's gateway to the list of trusted hosts (_eg: using the `network` or `connected-networks` option_), can create an [**open relay**](https://en.wikipedia.org/wiki/Open_mail_relay). For instance [if IPv6 is enabled on the host machine, but not in Docker][github-issue-1405-comment]. -1. Create the file `docker-compose.yml` with a content like this: +1. Create the file `compose.yaml` with a content like this: !!! example @@ -182,7 +182,7 @@ In this setup DMS is not intended to receive email from the outside world, so no 4. Get an SSL certificate, [we have a guide for you here][docs-ssl] (_Let's Encrypt_ is a popular service to get free SSL certificates). -5. Start DMS and check the terminal output for any errors: `docker-compose up`. +5. Start DMS and check the terminal output for any errors: `docker compose up`. 6. Create email accounts and aliases: @@ -214,7 +214,7 @@ In this setup DMS is not intended to receive email from the outside world, so no This extra step is required to avoid the `553 5.7.1 Sender address rejected: not owned by user` error (_the accounts used for submitting mail to Gmail are `admin.gmail@example.com` and `info.gmail@example.com`_) -7. Send some test emails to these addresses and make other tests. Once everything is working well, stop the container with `ctrl+c` and start it again as a daemon: `docker-compose up -d`. +7. Send some test emails to these addresses and make other tests. Once everything is working well, stop the container with `ctrl+c` and start it again as a daemon: `docker compose up -d`. [docs-ports]: ../../config/security/understanding-the-ports.md [docs-environment]: ../../config/environment.md diff --git a/docs/content/examples/use-cases/imap-folders.md b/docs/content/examples/use-cases/imap-folders.md index 1d1b2539017..ad5f3ed3436 100644 --- a/docs/content/examples/use-cases/imap-folders.md +++ b/docs/content/examples/use-cases/imap-folders.md @@ -16,7 +16,7 @@ See [`target/dovecot/15-mailboxes.conf`][github-config-dovecot-mailboxes] for ex The `Archive` special IMAP folder may be useful to enable. To do so, make a copy of [`target/dovecot/15-mailboxes.conf`][github-config-dovecot-mailboxes] and uncomment the `Archive` mailbox definition. Mail clients should understand that this folder is intended for archiving mail due to the [`\Archive` _"SPECIAL-USE"_ attribute][rfc-6154]. -With the provided [docker-compose.yml][github-config-dockercompose] example, a volume bind mounts the host directory `docker-data/dms/config/` to the container location `/tmp/docker-mailserver/`. Config file overrides should instead be mounted to a different location as described in [Overriding Configuration for Dovecot][docs-config-overrides-dovecot]: +With the provided [compose.yaml][github-config-dockercompose] example, a volume bind mounts the host directory `docker-data/dms/config/` to the container location `/tmp/docker-mailserver/`. Config file overrides should instead be mounted to a different location as described in [Overriding Configuration for Dovecot][docs-config-overrides-dovecot]: ```yaml volumes: @@ -60,12 +60,12 @@ Take care to test localized names work well as well. !!! note "Needs citation" This information is provided by the community. - + It presently lacks references to confirm the behaviour. If any information is incorrect please let us know! :smile: [docs-config-overrides-dovecot]: ../../config/advanced/override-defaults/dovecot.md#override-configuration -[github-config-dockercompose]: https://github.com/docker-mailserver/docker-mailserver/blob/master/docker-compose.yml +[github-config-dockercompose]: https://github.com/docker-mailserver/docker-mailserver/blob/master/compose.yaml [github-config-dovecot-mailboxes]: https://github.com/docker-mailserver/docker-mailserver/blob/master/target/dovecot/15-mailboxes.conf [dovecot-docs-namespaces]: https://doc.dovecot.org/configuration_manual/namespace/#namespace-inbox [dovecot-docs-mailboxes]: https://doc.dovecot.org/configuration_manual/namespace/#mailbox-settings diff --git a/docs/content/faq.md b/docs/content/faq.md index 7c28b0bd3ac..dd313631e84 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -21,9 +21,9 @@ Mails are stored in `/var/mail/${domain}/${username}`. Since `v9.0.0` it is poss Then, run the following commands: ``` BASH -docker-compose pull -docker-compose down -docker-compose up -d +docker compose pull +docker compose down +docker compose up -d ``` You should see the new version number on startup, for example: `[ INF ] Welcome to docker-mailserver 11.3.1`. And you're done! Don't forget to have a look at the remaining functions of the `setup.sh` script with `./setup.sh help`. @@ -97,7 +97,7 @@ DMS supports multiple domains out of the box, so you can do this: #### Bind mounts (default) -From the location of your `docker-compose.yml`, create a compressed archive of your `docker-data/dms/config/` and `docker-data/dms/mail-*` folders: +From the location of your `compose.yaml`, create a compressed archive of your `docker-data/dms/config/` and `docker-data/dms/mail-*` folders: ```bash tar --gzip -cf "backup-$(date +%F).tar.gz" ./docker-data/dms @@ -167,7 +167,7 @@ warning: do not list domain example.com in BOTH mydestination and virtual_mailbo Plus of course mail delivery fails. -Also you need to define `hostname: example.com` in your `docker-compose.yml`. +Also you need to define `hostname: example.com` in your `compose.yaml`. !!! tip "You might not want a bare domain" @@ -281,7 +281,7 @@ Suppose you want to change a number of settings that are not listed as variables DMS has a built-in way to do post-install processes. If you place a script called **`user-patches.sh`** in the config directory it will be run after all configuration files are set up, but before the postfix, amavis and other daemons are started. -It is common to use a local directory for config added to `docker-mailsever` via a volume mount in your `docker-compose.yml` (eg: `./docker-data/dms/config/:/tmp/docker-mailserver/`). +It is common to use a local directory for config added to `docker-mailsever` via a volume mount in your `compose.yaml` (eg: `./docker-data/dms/config/:/tmp/docker-mailserver/`). Add or create the script file to your config directory: @@ -376,7 +376,7 @@ Antispam rules are managed in `docker-data/dms/config/spamassassin-rules.cf`. For no subject set `SA_SPAM_SUBJECT=undef`. -For a trailing white-space subject one can define the whole variable with quotes in `docker-compose.yml`: +For a trailing white-space subject one can define the whole variable with quotes in `compose.yaml`: ```yaml environment: @@ -411,7 +411,7 @@ The following configuration works nicely: Create a _system_ cron file: ```sh - # in the docker-compose.yml root directory + # in the compose.yaml root directory mkdir -p ./docker-data/dms/cron touch ./docker-data/dms/cron/sa-learn chown root:root ./docker-data/dms/cron/sa-learn @@ -445,7 +445,7 @@ The following configuration works nicely: 30 3 * * * root sa-learn --ham /var/mail/not-example.com/*/cur* --dbpath /var/mail-state/lib-amavis/.spamassassin > /dev/null ``` - Then with `docker-compose.yml`: + Then with `compose.yaml`: ```yaml services: diff --git a/docs/content/introduction.md b/docs/content/introduction.md index a92a86d9b62..73c7e58c289 100644 --- a/docs/content/introduction.md +++ b/docs/content/introduction.md @@ -204,14 +204,13 @@ The best practice as of 2020 would be [POP3S][wikipedia-pop3s] on port 995, rath As a _batteries included_ container image, DMS provides you with all the required components and a default configuration to run a decent and secure mail server. One may then customize all aspects of its internal components. -- Simple customization is supported through [docker-compose configuration][github-file-compose] and the [env-mailserver][github-file-envmailserver] configuration file. +- Simple customization is supported through [Docker Compose configuration][github-file-compose] and the [env-mailserver][github-file-envmailserver] configuration file. - Advanced customization is supported through providing "monkey-patching" configuration files and/or [deriving your own image][github-file-dockerfile] from DMS's upstream, for a complete control over how things run. - Eventually, it is up to _you_ deciding exactly what kind of transportation/encryption to use and/or enforce, and to customize your instance accordingly (with looser or stricter security). Be also aware that protocols and ports on your server can only go so far with security; third-party MTAs might relay your emails on insecure connections, man-in-the-middle attacks might still prove effective, etc. Advanced counter-measure such as DANE, MTA-STS and/or full body encryption (eg. PGP) should be considered as well for increased confidentiality, but ideally without compromising backwards compatibility so as to not block emails. [docs-understandports]: ./config/security/understanding-the-ports.md -[github-file-compose]: https://github.com/docker-mailserver/docker-mailserver/blob/master/docker-compose.yml +[github-file-compose]: https://github.com/docker-mailserver/docker-mailserver/blob/master/compose.yaml [github-file-envmailserver]: https://github.com/docker-mailserver/docker-mailserver/blob/master/mailserver.env [github-file-dockerfile]: https://github.com/docker-mailserver/docker-mailserver/blob/master/Dockerfile [rfc-2487]: https://tools.ietf.org/html/rfc2487 diff --git a/docs/content/usage.md b/docs/content/usage.md index bf656b52675..0ea8beaf405 100644 --- a/docs/content/usage.md +++ b/docs/content/usage.md @@ -21,16 +21,17 @@ There are a few requirements for a suitable host system: 2. The host should be able to send/receive on the [necessary ports for mail][docs-ports-overview] 3. You should be able to set a `PTR` record for your host; security-hardened mail servers might otherwise reject your mail server as the IP address of your host does not resolve correctly/at all to the DNS name of your server. -On the host, you should have a suitable container runtime (like _Docker_ or _Podman_) installed. We assume [_Docker Compose_][docker-compose] is [installed][docker-compose-installation]. +!!! note "About the Container Runtime" -!!! info "Podman Support" + On the host, you need to have a suitable container runtime (like _Docker_ or _Podman_) installed. We assume [_Docker Compose_][docker-compose] is [installed][docker-compose-installation]. We have aligned file names and configuration conventions with the latest [Docker Compose (currently V2) specification][docker-compose-specification]. If you're using podman, make sure to read the related [documentation][docs-podman]. -[docs-podman]: ./config/advanced/podman.md [docs-ports-overview]: ./config/security/understanding-the-ports.md#overview-of-email-ports [docker-compose]: https://docs.docker.com/compose/ [docker-compose-installation]: https://docs.docker.com/compose/install/ +[docker-compose-specification]: https://docs.docker.com/compose/compose-file/ +[docs-podman]: ./config/advanced/podman.md ### Minimal DNS Setup @@ -88,13 +89,13 @@ Issue the following commands to acquire the necessary files: ``` BASH DMS_GITHUB_URL="https://github.com/docker-mailserver/docker-mailserver/blob/latest" -wget "${DMS_GITHUB_URL}/docker-compose.yml" +wget "${DMS_GITHUB_URL}/compose.yaml" wget "${DMS_GITHUB_URL}/mailserver.env" ``` ### Configuration Steps -1. First edit `docker-compose.yml` to your liking +1. First edit `compose.yaml` to your liking - Substitute `mail.example.com` according to your FQDN. - If you want to use SELinux for the `./docker-data/dms/config/:/tmp/docker-mailserver/` mount, append `-z` or `-Z`. 2. Then configure the environment specific to the mail server by editing [`mailserver.env`][docs-environment], but keep in mind that: diff --git a/mailserver.env b/mailserver.env index 7ff2843fc7a..91ffbd1588b 100644 --- a/mailserver.env +++ b/mailserver.env @@ -178,7 +178,7 @@ AMAVIS_LOGLEVEL=0 # 1 => DNS block lists are enabled ENABLE_DNSBL=0 -# If you enable Fail2Ban, don't forget to add the following lines to your `docker-compose.yml`: +# If you enable Fail2Ban, don't forget to add the following lines to your `compose.yaml`: # cap_add: # - NET_ADMIN # Otherwise, `nftables` won't be able to ban IPs. @@ -397,7 +397,7 @@ ENABLE_LDAP= # yes => LDAP over TLS enabled for Postfix LDAP_START_TLS= -# If you going to use the mailserver in combination with docker-compose you can set the service name here +# If you going to use the mailserver in combination with Docker Compose you can set the service name here # empty => mail.domain.com # Specify the dns-name/ip-address where the ldap-server LDAP_SERVER_HOST= diff --git a/target/scripts/helpers/ssl.sh b/target/scripts/helpers/ssl.sh index 94d5dc59008..f837be00db9 100644 --- a/target/scripts/helpers/ssl.sh +++ b/target/scripts/helpers/ssl.sh @@ -77,13 +77,13 @@ function _setup_ssl # Postfix configuration # NOTE: This operation doesn't replace the line, it appends to the end of the line. # Thus this method should only be used when this line has explicitly been replaced earlier in the script. - # Otherwise without `docker-compose down` first, a `docker-compose up` may + # Otherwise without `docker compose down` first, a `docker compose up` may # persist previous container state and cause a failure in postfix configuration. sedfile -i "s|^smtpd_tls_chain_files =.*|& ${PRIVATE_KEY_ALT} ${CERT_CHAIN_ALT}|" "${POSTFIX_CONFIG_MAIN}" # Dovecot configuration # Conditionally checks for `#`, in the event that internal container state is accidentally persisted, - # can be caused by: `docker-compose up` run again after a `ctrl+c`, without running `docker-compose down` + # can be caused by: `docker compose up` run again after a `ctrl+c`, without running `docker compose down` sedfile -i -r \ -e "s|^#?(ssl_alt_key =).*|\1 <${PRIVATE_KEY_ALT}|" \ -e "s|^#?(ssl_alt_cert =).*|\1 <${CERT_CHAIN_ALT}|" \ diff --git a/test/tests/parallel/set3/container_configuration/process_check_restart.bats b/test/tests/parallel/set3/container_configuration/process_check_restart.bats index 6a743bf87b4..b3d4fc86fa1 100644 --- a/test/tests/parallel/set3/container_configuration/process_check_restart.bats +++ b/test/tests/parallel/set3/container_configuration/process_check_restart.bats @@ -197,7 +197,7 @@ function _check_if_process_is_running() { # The process manager (supervisord) should perform a graceful shutdown: # NOTE: Time limit should never be below these configured values: # - supervisor-app.conf:stopwaitsecs -# - docker-compose.yml:stop_grace_period +# - compose.yaml:stop_grace_period function _should_stop_cleanly() { run docker stop -t 60 "${CONTAINER_NAME}" assert_success From 595ff03804ff68fabf6d43138eb22e7d09a195e9 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 10 May 2023 11:29:51 +0200 Subject: [PATCH 065/592] Postfix: rename "smtps" to "submissions" (#3235) --- docs/content/config/advanced/kubernetes.md | 2 +- docs/content/examples/tutorials/mailserver-behind-proxy.md | 2 +- target/postfix/master.cf | 4 ++-- target/scripts/helpers/ssl.sh | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/content/config/advanced/kubernetes.md b/docs/content/config/advanced/kubernetes.md index 6d7330f4c7d..8846c955f9d 100644 --- a/docs/content/config/advanced/kubernetes.md +++ b/docs/content/config/advanced/kubernetes.md @@ -460,7 +460,7 @@ Then, configure both [Postfix][docs-postfix] and [Dovecot][docs-dovecot] to expe postfix-master.cf: | smtp/inet/postscreen_upstream_proxy_protocol=haproxy submission/inet/smtpd_upstream_proxy_protocol=haproxy - smtps/inet/smtpd_upstream_proxy_protocol=haproxy + submissions/inet/smtpd_upstream_proxy_protocol=haproxy dovecot.cf: | # Assuming your ingress controller is bound to 10.0.0.0/8 haproxy_trusted_networks = 10.0.0.0/8, 127.0.0.0/8 diff --git a/docs/content/examples/tutorials/mailserver-behind-proxy.md b/docs/content/examples/tutorials/mailserver-behind-proxy.md index f9050106597..501d1209cba 100644 --- a/docs/content/examples/tutorials/mailserver-behind-proxy.md +++ b/docs/content/examples/tutorials/mailserver-behind-proxy.md @@ -105,7 +105,7 @@ and to `docker-data/dms/config/postfix-master.cf`: ```cf submission/inet/smtpd_upstream_proxy_protocol=haproxy -smtps/inet/smtpd_upstream_proxy_protocol=haproxy +submissions/inet/smtpd_upstream_proxy_protocol=haproxy ``` Changes for `dovecot` can be applied by adding the following content to `docker-data/dms/config/dovecot.cf`: diff --git a/target/postfix/master.cf b/target/postfix/master.cf index 3746b6f6bfe..6f8877f697e 100644 --- a/target/postfix/master.cf +++ b/target/postfix/master.cf @@ -27,8 +27,8 @@ submission inet n - n - - smtpd -o milter_macro_daemon_name=ORIGINATING -o cleanup_service_name=sender-cleanup -smtps inet n - n - - smtpd - -o syslog_name=postfix/smtps +submissions inet n - n - - smtpd + -o syslog_name=postfix/submissions -o smtpd_tls_wrappermode=yes -o smtpd_sasl_auth_enable=yes -o smtpd_sasl_type=dovecot diff --git a/target/scripts/helpers/ssl.sh b/target/scripts/helpers/ssl.sh index f837be00db9..ae7aea4d18e 100644 --- a/target/scripts/helpers/ssl.sh +++ b/target/scripts/helpers/ssl.sh @@ -342,7 +342,7 @@ function _setup_ssl # | http://www.postfix.org/postconf.5.html#smtpd_tls_auth_only | http://www.postfix.org/TLS_README.html#server_tls_auth # # smtp_tls_wrappermode (default: not applied, 'no') | http://www.postfix.org/postconf.5.html#smtp_tls_wrappermode - # smtpd_tls_wrappermode (default: 'yes' for service port 'smtps') | http://www.postfix.org/postconf.5.html#smtpd_tls_wrappermode + # smtpd_tls_wrappermode (default: 'yes' for service port 'submissions') | http://www.postfix.org/postconf.5.html#smtpd_tls_wrappermode # NOTE: Enabling wrappermode requires a security_level of 'encrypt' or stronger. Port 465 presently does not meet this condition. # # Postfix main.cf (base config): @@ -353,7 +353,7 @@ function _setup_ssl # # Postfix master.cf (per connection overrides): # Disables implicit TLS on port 465 for inbound (smtpd) and outbound (smtp) traffic. Treats it as equivalent to port 25 SMTP with explicit STARTTLS. - # Inbound 465 (aka service port aliases: submissions / smtps) for Postfix to receive over implicit TLS (eg from MUA or functioning as a relay host). + # Inbound 465 (aka service port aliases: submissions) for Postfix to receive over implicit TLS (eg from MUA or functioning as a relay host). # Outbound 465 as alternative to port 587 when sending to another MTA (with authentication), such as a relay service (eg SendGrid). sedfile -i -r \ -e "/smtpd?_tls_security_level/s|=.*|=none|" \ From 272c19c218ab793d5413ff2c7747fb8e8932da5a Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 10 May 2023 11:51:49 +0200 Subject: [PATCH 066/592] docs: update Rspamd docs (small improvement) (#3318) --- docs/content/config/security/rspamd.md | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index d61a7c4a6b6..e837057dc5d 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -156,15 +156,30 @@ ENABLE_AMAVIS=0 ENABLE_SPAMASSASSIN=0 ``` -This will enable Rspamd and disable services you don't need when using Rspamd. Note that with this setup, the default DKIM signing that DMS provides does not work (as it is disabled)! To solve this issue, look at [this subsection](#dkim-signing). +This will enable Rspamd and disable services you don't need when using Rspamd. ### Adjusting and Extending The Very Basic Configuration -Rspamd is running, but you want or need to adjust it? +Rspamd is running, but you want or need to adjust it? First, create a file named `custom-commands.conf` under `docker-data/dms/config/rspamd` (which translates to `/tmp/docker-mailserver/rspamd/` inside the container). Then add you changes: -1. Say you want to be able to easily look at the frontend Rspamd provides on port 11334 (default) without the need to enter a password (maybe because you already provide authorization and authentication). You will need to adjust the controller worker: create a file called `custom-commands.conf` and add the line `set-option-for-controller secure_ip "0.0.0.0/0"`. Place the file `custom-commands.conf` inside the directory on the host you mount to `/tmp/docker-mailserver/rspamd/` inside the container (in our documentation, we use `docker-data/dms/config/rspamd/` on the host for this purpose). And you're done! Note: this disables authentication on the website - make sure you know what you're doing! -2. You additionally want to enable the auto-spam-learning for the Bayes module? No problem, just add another line to `custom-commands.conf` that looks like this: `set-option-for-module classifier-bayes autolearn true`. -3. But the chartable module gets on your nerves? Just disable it by adding another line: `disable-module chartable`. +1. Say you want to be able to easily look at the frontend Rspamd provides on port 11334 (default) without the need to enter a password (maybe because you already provide authorization and authentication). You will need to adjust the controller worker: `set-option-for-controller secure_ip "0.0.0.0/0"`. +2. You additionally want to enable the auto-spam-learning for the Bayes module? No problem: `set-option-for-module classifier-bayes autolearn true`. +3. But the chartable module gets on your nerves? Easy: `disable-module chartable`. + +??? example "What Does the Result Look Like?" + Here is what the file looks like in the end: + + ```bash + # See 1. + # ATTENTION: this disables authentication on the website - make sure you know what you're doing! + set-option-for-controller secure_ip "0.0.0.0/0" + + # See 2. + set-option-for-module classifier-bayes autolearn true + + # See 3. + disable-module chartable + ``` ### DKIM Signing From 793e4026fc7a041b577d0fce8f8c12b8eb61f7ad Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 11 May 2023 11:23:02 +1200 Subject: [PATCH 067/592] chore(main.cf): Add note advising caution changing `mydestination` (#3316) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Co-authored-by: Casper --- target/postfix/main.cf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/target/postfix/main.cf b/target/postfix/main.cf index 4827c47c317..518ad326f98 100644 --- a/target/postfix/main.cf +++ b/target/postfix/main.cf @@ -9,6 +9,9 @@ readme_directory = no # myhostname = alias_maps = hash:/etc/aliases alias_database = hash:/etc/aliases +# Take the following concerns into consideration if adjusting `mydestination`: +# https://github.com/docker-mailserver/docker-mailserver/pull/3264#pullrequestreview-1396816109 +# https://github.com/docker-mailserver/docker-mailserver/pull/3264#issuecomment-1518993555 mydestination = $myhostname, localhost.$mydomain, localhost relayhost = mynetworks = 127.0.0.0/8 [::1]/128 [fe80::]/64 From 45361094e9a2c7dd56940a1b96ad5da8ab3974b5 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Thu, 11 May 2023 11:06:43 +0200 Subject: [PATCH 068/592] fix spelling mistakes (#3324) --- docs/content/config/best-practices/dkim_dmarc_spf.md | 6 ++---- docs/content/config/environment.md | 2 +- mailserver.env | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md index 12b2def13b6..0db4150a81d 100644 --- a/docs/content/config/best-practices/dkim_dmarc_spf.md +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -136,8 +136,7 @@ DKIM is currently supported by either OpenDKIM or Rspamd: use_domain = "header"; use_redis = false; # don't change unless Redis also provides the DKIM keys use_esld = true; - - check_pubkey = true; # you wan't to use this in the beginning + check_pubkey = true; # you want to use this in the beginning domain { example.com { @@ -193,7 +192,7 @@ DKIM is currently supported by either OpenDKIM or Rspamd: When `check_pubkey = true;` is set, Rspamd will query the DNS record for each DKIM selector, verifying each public key matches the private key configured. - If there is a mismatch, a warning will be omitted to the Rspamd log `/var/log/supervisor/rspamd.log`. + If there is a mismatch, a warning will be emitted to the Rspamd log `/var/log/supervisor/rspamd.log`. [docs-rspamd-override-d]: ../security/rspamd.md#manually @@ -328,7 +327,6 @@ volumes: - ./docker-data/dms/config/postfix-policyd-spf.conf:/etc/postfix-policyd-spf-python/policyd-spf.conf ``` - [docs-accounts-add]: ../user-management.md#adding-a-new-account [docs-volumes-config]: ../advanced/optional-config.md [docs-env-opendkim]: ../environment.md#enable_opendkim diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 846f4c117d3..5123fe57a69 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -368,7 +368,7 @@ When enabled, ##### RSPAMD_HFILTER -Can be used to enable or disable the [Hfilter group module][rspamd-docs-hfilter-group-module]. This is used by DMS to adjust the `HFILTER_HOSTNAME_UNKNOWN` symbol, increasing it's default weight to act similar to Postfix's `reject_unknown_client_hostname`, without the need to outright reject a message. +Can be used to enable or disable the [Hfilter group module][rspamd-docs-hfilter-group-module]. This is used by DMS to adjust the `HFILTER_HOSTNAME_UNKNOWN` symbol, increasing its default weight to act similar to Postfix's `reject_unknown_client_hostname`, without the need to outright reject a message. - 0 => Disabled - **1** => Enabled diff --git a/mailserver.env b/mailserver.env index 91ffbd1588b..7c1af20f505 100644 --- a/mailserver.env +++ b/mailserver.env @@ -150,7 +150,7 @@ RSPAMD_LEARN=0 # 1 => enabled RSPAMD_GREYLISTING=0 -# Can be used to enable or diable the Hfilter group module. +# Can be used to enable or disable the Hfilter group module. # # - 0 => Disabled # - **1** => Enabled From 03c0b116c737ac15a6a02b6396ebbe91c4f9c290 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Thu, 11 May 2023 13:33:32 +0200 Subject: [PATCH 069/592] clear up confusion about `override.d` (#3325) See https://github.com/docker-mailserver/docker-mailserver/issues/3323#issuecomment-1543636636 --- docs/content/config/security/rspamd.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index e837057dc5d..44674e9eff0 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -101,12 +101,13 @@ DMS brings sane default settings for Rspamd. They are located at `/etc/rspamd/lo !!! question "What is [`docker-data/dms/config/`][docs-dms-config-volume]?" -If you want to overwrite the default settings and / or provide your own settings, you can place files at `docker-data/dms/config/rspamd/override.d/` (a directory that is linked to `/etc/rspamd/override.d/`, if it exists) to override Rspamd and DMS default settings. +If you want to overwrite the default settings and / or provide your own settings, you can place files at `docker-data/dms/config/rspamd/override.d/` (a directory that is linked to `/etc/rspamd/override.d/`, if it exists) to override Rspamd and DMS default settings. This directory will not do a complete file override, but a [forced override of the specific settings in that file][rspamd-docs-override-dir]. !!! warning "Clashing Overrides" Note that when also [using the `rspamd-commands` file](#with-the-help-of-a-custom-file), files in `override.d` may be overwritten in case you adjust them manually and with the help of the file. +[rspamd-docs-override-dir]: https://www.rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories [docs-dms-config-volume]: ../../faq.md#what-about-the-docker-datadmsconfig-directory ### With the Help of a Custom File From e4274ef113931d7241cec6534494b7fc17de2a6d Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Thu, 11 May 2023 18:08:54 +0200 Subject: [PATCH 070/592] docs: improve Rspamd docs about DKIM signing of multiple domains (#3329) * improve Rspamd docs See #3326 & #3328 * improve warning message See #3328 --- .../config/best-practices/dkim_dmarc_spf.md | 20 ++++++++----------- .../startup/setup.d/security/rspamd.sh | 4 ++-- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md index 0db4150a81d..3fece48b81f 100644 --- a/docs/content/config/best-practices/dkim_dmarc_spf.md +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -109,6 +109,10 @@ DKIM is currently supported by either OpenDKIM or Rspamd: will execute the helper script with default parameters. + ??? warning "Using Multiple Domains" + + Unlike the current script for OpenDKIM, the Rspamd script will **not** create keys for all domains DMS is managing, but only for the one it assumes to be the main domain (derived from DMS' domain name). Moreover, the default `dkim_signing.conf` configuration file that DMS ships will also only contain one domain. If you have multiple domains, you need to run the command `docker exec -ti setup config dkim domain ` multiple times to create all the keys for all domains, and then provide a custom `dkim_signing.conf` (for which an example is shown below). + !!! info "About the Helper Script" The script will persist the keys in `/tmp/docker-mailserver/rspamd/dkim/`. Hence, if you are already using the default volume mounts, the keys are persisted in a volume. The script also restarts Rspamd directly, so changes take effect without restarting DMS. @@ -148,24 +152,16 @@ DKIM is currently supported by either OpenDKIM or Rspamd: As shown next: - - You can add more domains into the `domain { ... }` section. - - A domain can also be configured with multiple selectors and keys within a `selectors [ ... ]` array. + - You can add more domains into the `domain { ... }` section (in the following example: `example.com` and `example.org`). + - A domain can also be configured with multiple selectors and keys within a `selectors [ ... ]` array (in the following example, this is done for `example.org`). ```cf # ... domain { example.com { - selectors [ - { - path = "/tmp/docker-mailserver/rspamd/dkim/example.com/rsa.private"; - selector = "dkim-rsa"; - }, - { - path = /tmp/docker-mailserver/rspamd/example.com/ed25519.private"; - selector = "dkim-ed25519"; - } - ] + path = /tmp/docker-mailserver/rspamd/example.com/ed25519.private"; + selector = "dkim-ed25519"; } example.org { selectors [ diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index e171736a476..4ddb82ca580 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -78,11 +78,11 @@ function __rspamd__run_early_setup_and_checks if [[ -d ${RSPAMD_DMS_OVERRIDE_D} ]] then __rspamd__log 'debug' "Found directory '${RSPAMD_DMS_OVERRIDE_D}' - linking it to '${RSPAMD_OVERRIDE_D}'" - if rmdir "${RSPAMD_OVERRIDE_D}" + if rmdir "${RSPAMD_OVERRIDE_D}" 2>/dev/null then ln -s "${RSPAMD_DMS_OVERRIDE_D}" "${RSPAMD_OVERRIDE_D}" else - __rspamd__log 'warn' "Could not remove '${RSPAMD_OVERRIDE_D}' (not empty?) - not linking '${RSPAMD_DMS_OVERRIDE_D}'" + __rspamd__log 'warn' "Could not remove '${RSPAMD_OVERRIDE_D}' (not empty? not a directory?; did you restart properly?) - not linking '${RSPAMD_DMS_OVERRIDE_D}'" fi fi From 823ef33a92ddf0d016e1ea2ee6b3b66e9d765ba5 Mon Sep 17 00:00:00 2001 From: ghnp5 <57591332+ghnp5@users.noreply.github.com> Date: Thu, 11 May 2023 17:10:51 +0100 Subject: [PATCH 071/592] fix: typo about OpenDMARC (#3330) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- target/scripts/startup/setup.d/dmarc_dkim_spf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh index 2825898800d..a867e1d0cda 100644 --- a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh +++ b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh @@ -51,7 +51,7 @@ function _setup_opendkim fi } -# Set up OpenDKIM +# Set up OpenDMARC # # ## Attention # From 05cd538fa97c22612565cc8a0bc77a6ce65aa311 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 12 May 2023 11:21:08 +0200 Subject: [PATCH 072/592] ci: slim down bug report template (#3317) * slim down bug report template and outsource note to documentation --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/bug_report.yml | 131 +++++++----------- .../contributing/issues-and-pull-requests.md | 12 ++ 2 files changed, 62 insertions(+), 81 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index e8d60dabea3..32063911be8 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -6,20 +6,6 @@ labels: - meta/needs triage body: - - type: markdown - attributes: - value: | - # Filing a report - - Thank you for participating in this project and reporting a bug. Docker Mail Server (DMS) is a community-driven project, and each contribution counts. Maintainers and moderators are volunteers that need you to fill this template with accurate information in order to help you in the best and quickest way. We will have to label your request with `meta/no template - no support` if your request is sloppy and provides no way to help you correctly. - - Markdown formatting can be used in almost all text fields. The description will tell you if this is not the case for a specific field. - - Be as precise as possible, and if in doubt, it's best to add more information that too few. - - We provide official support for all options not marked with "not officially supported" or "unsupported". When something is "not officially supported", support for these cases is dependent on specific maintainers. - - --- - type: checkboxes id: preliminary-checks attributes: @@ -32,60 +18,18 @@ body: required: true - label: I searched the issue tracker but was unable to find my issue. required: true - - label: I read the [extended documentation in general](https://docker-mailserver.github.io/docker-mailserver/latest/) but found nothing to resolve the issue. - required: true - - label: I read the [documentation on debugging](https://docker-mailserver.github.io/docker-mailserver/latest/config/debugging/), tried the proposed steps to debug the problem, but was still unable to resolve the issue. - required: true - - label: I have read this project's [Code of Conduct](https://github.com/docker-mailserver/docker-mailserver/blob/master/CODE_OF_CONDUCT.md) and I agree + - label: | + I read + + - the [extended documentation in general](https://docker-mailserver.github.io/docker-mailserver/latest/) but found nothing to resolve the issue; + - the [documentation on debugging](https://docker-mailserver.github.io/docker-mailserver/latest/config/debugging/), tried the proposed steps to debug the problem, but was still unable to resolve the issue; + - this project's [Code of Conduct](https://github.com/docker-mailserver/docker-mailserver/blob/master/CODE_OF_CONDUCT.md) and I agree; + - the [documentation about filing a bug report](https://docker-mailserver.github.io/docker-mailserver/latest/contributing/issues-and-pull-requests/#filing-a-bug-report). required: true - - type: input - id: affected-components - attributes: - label: Affected Component(s) - description: What is affected by this bug? Please describe it with only a few words here. Detailed description can be given later. - placeholder: No debug output is shown. - validations: - required: true - - type: textarea - id: when-does-it-occur - attributes: - label: What happened and when does this occur? - description: Tell us what happened. Use [fenced code blocks](https://docs.github.com/en/github/writing-on-github/working-with-advanced-formatting/creating-and-highlighting-code-blocks#fenced-code-blocks) when pasting lots of text! - placeholder: Although `LOG_LEVEL=debug` is set, I see no debug output. - validations: - required: true - - type: textarea - id: what-did-you-expect-to-happen - attributes: - label: What did you expect to happen? - description: Tell us what you expected. - placeholder: I expected to see debug messages. - validations: - required: true - - type: textarea - id: how-do-we-replicate-this-issue - attributes: - label: How do we replicate the issue? - description: What did you do and how can we replicate this issue? - value: | - 1. - 2. - 3. - ... - validations: - required: true - - type: input - id: mailserver-version - attributes: - label: DMS version - description: On which version (image tag) did you encounter this bug? - placeholder: v10.1.2 - validations: - required: true - type: dropdown id: operating-system attributes: - label: What operating system is DMS running on? + label: What operating system is the host running? options: - Linux - macOS (not officially supported) @@ -97,16 +41,16 @@ body: id: operating-system-version attributes: label: Which operating system version? - placeholder: e.g. Debian 11 + placeholder: Debian 11 (Bullseye) validations: required: true - type: dropdown id: isa attributes: - label: What instruction set architecture is DMS running on? + label: What computer architecture is the host running? options: - - AMD64 / x86_64 - - ARM64 / AArch64 (ARM v8 and newer) + - AMD64 (x86_64) + - ARM64 (AArch64) - Other (unsupported) validations: required: true @@ -122,11 +66,41 @@ body: - Other (unsupported) validations: required: true + - type: input + id: mailserver-version + attributes: + label: DMS version + description: On which version (image tag) did you encounter this bug? + placeholder: v12.1.0 + validations: + required: true + - type: textarea + id: when-does-it-occur + attributes: + label: What happened and when does this occur? + description: > + Tell us what happened. + **It would also help immensely if you can share precise steps on how to reproduce the issue! ** :heart: + Use [fenced code blocks](https://docs.github.com/en/github/writing-on-github/working-with-advanced-formatting/creating-and-highlighting-code-blocks#fenced-code-blocks) when pasting lots of text! + placeholder: Although `LOG_LEVEL=debug` is set, I see no debug output. + validations: + required: true + - type: textarea + id: what-did-you-expect-to-happen + attributes: + label: What did you expect to happen? + description: Tell us what you expected. + placeholder: I expected to see debug messages. + validations: + required: true - type: textarea - id: important-environment-variables + id: container-configuration-files attributes: - label: compose.yaml - description: Show us your `compose.yaml` file or your equivalent `docker run` command, if applicable. This filed is formatted as YAML. + label: Container configuration files + description: > + Show us the file that you use to run DMS (and possibly other related services). + This is most likely your `compose.yaml` file, but you can also show us your equivalent `docker run` command, if applicable. If you are using Kubernetes, you can also put your manifest files here. + This filed is formatted as YAML. render: yml - type: textarea id: relevant-log-output @@ -134,27 +108,22 @@ body: label: Relevant log output description: Show us relevant log output here. You can enable debug output by setting the environment variable `LOG_LEVEL` to `debug` or `trace`. This field's contents are interpreted as pure text. render: Text - - type: textarea - id: other-relevant-information - attributes: - label: Other relevant information - description: If there is more, you can tell us here. - type: checkboxes id: experience attributes: - label: What level of experience do you have with Docker and mail servers? + label: What level of experience do you have with containers, mail servers, and using a terminal? description: > **You are not obliged to answer this question**. We do encourage answering though as it provides context to better assist you. Less experienced users tend to make common mistakes, which is ok; by letting us know we can spot those more easily. If you are experienced, we can skip basic questions and save time. options: - - label: I am inexperienced with docker - - label: I am rather experienced with docker + - label: I am inexperienced with containers + - label: I am rather experienced with containers - label: I am inexperienced with mail servers - label: I am rather experienced with mail servers - - label: I am uncomfortable with the CLI - - label: I am rather comfortable with the CLI + - label: I am uncomfortable with using a terminal (CLI) + - label: I am rather comfortable with using a terminal (CLI) - type: input id: form-improvements attributes: diff --git a/docs/content/contributing/issues-and-pull-requests.md b/docs/content/contributing/issues-and-pull-requests.md index b96fc23c3ba..0a32baabb9b 100644 --- a/docs/content/contributing/issues-and-pull-requests.md +++ b/docs/content/contributing/issues-and-pull-requests.md @@ -20,6 +20,18 @@ By raising issues, I agree to these terms and I understand, that the rules set f Maintainers take the time to improve on this project and help by solving issues together. It is therefore expected from others to make an effort and **comply with the rules**. +### Filing a Bug Report + +Thank you for participating in this project and reporting a bug. Docker Mail Server (DMS) is a community-driven project, and each contribution counts! + +Maintainers and moderators are volunteers. We greatly appreciate reports that take the time to provide detailed information via the template, enabling us to help you in the best and quickest way. Ignoring the template provided may seem easier, but discourages receiving any support (_via assignment of the label `meta/no template - no support`_). + +Markdown formatting can be used in almost all text fields (_unless stated otherwise in the description_). + +Be as precise as possible, and if in doubt, it's best to add more information that too few. + +When an option is marked with "not officially supported" / "unsupported", then support is dependent on availability from specific maintainers. + ## Pull Requests !!! question "Motivation" From 3340b809724194f895f7d6ce852b5d1c707fba1a Mon Sep 17 00:00:00 2001 From: georglauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 12 May 2023 11:29:36 +0200 Subject: [PATCH 073/592] correct typo --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 32063911be8..8af64886f0f 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -80,7 +80,7 @@ body: label: What happened and when does this occur? description: > Tell us what happened. - **It would also help immensely if you can share precise steps on how to reproduce the issue! ** :heart: + **It would also help immensely if you can share precise steps on how to reproduce the issue!** :heart: Use [fenced code blocks](https://docs.github.com/en/github/writing-on-github/working-with-advanced-formatting/creating-and-highlighting-code-blocks#fenced-code-blocks) when pasting lots of text! placeholder: Although `LOG_LEVEL=debug` is set, I see no debug output. validations: From 78b7f0cbea9c46653535296a7d76d05a0526a0b1 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 12 May 2023 16:04:41 +0200 Subject: [PATCH 074/592] scripts: improve `CLAMAV_MESSAGE_SIZE_LIMIT` usage (#3332) * add sanity check for Clam size & adjusted MaxScanSize The second part is of special importance! See , which explains that the maximum scan size is important as well. We previously just set the maximum file size, which actually is pretty insecure as we silently not scan mile bigger than `MaxScanSize`. This is corrected now. * add SlamAV size configuration to Rspamd --- target/rspamd/local.d/antivirus.conf | 1 + target/scripts/startup/setup.d/security/misc.sh | 14 ++++++++++++-- target/scripts/startup/setup.d/security/rspamd.sh | 8 ++++++++ .../parallel/set1/spam_virus/rspamd_full.bats | 7 +++++++ .../parallel/set1/spam_virus/rspamd_partly.bats | 5 +++++ 5 files changed, 33 insertions(+), 2 deletions(-) diff --git a/target/rspamd/local.d/antivirus.conf b/target/rspamd/local.d/antivirus.conf index 2ce4e96f4c5..78c44051313 100644 --- a/target/rspamd/local.d/antivirus.conf +++ b/target/rspamd/local.d/antivirus.conf @@ -10,4 +10,5 @@ ClamAV { scan_mime_parts = false; symbol = "CLAM_VIRUS"; log_clean = true; + max_size = 25000000; } diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index b517d513a55..be25002a0df 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -186,8 +186,18 @@ function __setup__security__clamav if [[ ${CLAMAV_MESSAGE_SIZE_LIMIT} != '25M' ]] then _log 'trace' "Setting ClamAV message scan size limit to '${CLAMAV_MESSAGE_SIZE_LIMIT}'" - sedfile -i \ - "s/^MaxFileSize.*/MaxFileSize ${CLAMAV_MESSAGE_SIZE_LIMIT}/" \ + + # do a short sanity checks; ClamAV stops scanning at more that 4GB file size + if [[ $(numfmt --from=si "${CLAMAV_MESSAGE_SIZE_LIMIT}") -gt $(numfmt --from=si 4G) ]] + then + _log 'warn' "You set 'CLAMAV_MESSAGE_SIZE_LIMIT' to a value larger than 4 Gigabyte which ClamAV does not support - you should correct your configuration" + fi + + sedfile -i -E \ + "s|^(MaxFileSize).*|\1 ${CLAMAV_MESSAGE_SIZE_LIMIT}|" \ + /etc/clamav/clamd.conf + sedfile -i -E \ + "s|^(MaxScanSize).*|\1 ${CLAMAV_MESSAGE_SIZE_LIMIT}|" \ /etc/clamav/clamd.conf fi else diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 4ddb82ca580..429a8efb69b 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -164,6 +164,14 @@ function __rspamd__setup_clamav sedfile -i -E 's|^(enabled).*|\1 = true;|g' "${RSPAMD_LOCAL_D}/antivirus.conf" # Rspamd uses ClamAV's UNIX socket, and to be able to read it, it must be in the same group usermod -a -G clamav _rspamd + + if [[ ${CLAMAV_MESSAGE_SIZE_LIMIT} != '25M' ]] + then + local SIZE_IN_BYTES + SIZE_IN_BYTES=$(numfmt --from=si "${CLAMAV_MESSAGE_SIZE_LIMIT}") + __rspamd__log 'trace' "Adjusting maximum size for ClamAV to ${SIZE_IN_BYTES} bytes (${CLAMAV_MESSAGE_SIZE_LIMIT})" + sedfile -i -E "s|(.*max_size =).*|\1 ${SIZE_IN_BYTES};|" "${RSPAMD_LOCAL_D}/antivirus.conf" + fi else __rspamd__log 'debug' 'Rspamd will not use ClamAV (which has not been enabled)' fi diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index d0c22a8bbfa..611995c6aa9 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -20,6 +20,7 @@ function setup_file() { --env ENABLE_OPENDMARC=0 --env ENABLE_POLICYD_SPF=0 --env ENABLE_POSTGREY=0 + --env CLAMAV_MESSAGE_SIZE_LIMIT=42M --env PERMIT_DOCKER=host --env LOG_LEVEL=trace --env MOVE_SPAM_TO_JUNK=1 @@ -78,6 +79,7 @@ function teardown_file() { _default_teardown ; } run docker logs "${CONTAINER_NAME}" assert_success assert_line --partial 'Enabling ClamAV integration' + assert_line --partial 'Adjusting maximum size for ClamAV to 42000000 bytes (42M)' assert_line --partial 'Setting up intelligent learning of spam and ham' assert_line --partial 'Enabling greylisting' assert_line --partial 'Hfilter (group) module is enabled' @@ -96,6 +98,11 @@ function teardown_file() { _default_teardown ; } _service_log_should_contain_string 'rspamd' 'lua module metric_exporter is disabled in the configuration' } +@test 'antivirus maximum size was adjusted' { + _run_in_container grep 'max_size = 42000000' /etc/rspamd/local.d/antivirus.conf + assert_success +} + @test 'normal mail passes fine' { _service_log_should_contain_string 'rspamd' 'F \(no action\)' diff --git a/test/tests/parallel/set1/spam_virus/rspamd_partly.bats b/test/tests/parallel/set1/spam_virus/rspamd_partly.bats index 29b7bc0e6e3..bcb2f8d857b 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_partly.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_partly.bats @@ -57,6 +57,11 @@ function teardown_file() { _default_teardown ; } assert_line --partial 'Disabling Hfilter (group) module' } +@test 'antivirus maximum size was not adjusted unnecessarily' { + _run_in_container grep 'max_size = 25000000' /etc/rspamd/local.d/antivirus.conf + assert_success +} + @test 'learning is properly disabled' { for FILE in learn-{ham,spam}.{sieve,svbin} do From 9fd00bd6ad1ea7e1bd969141c4bfb582e83717c4 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sat, 13 May 2023 13:59:16 +0200 Subject: [PATCH 075/592] Rspamd: adjust learning of ham (#3334) * adjust learning of ham See #3333 When moving a mail from the Junk folder to the Trash folder, the mail previously classified as ham due to the wildcard match of `*`. Because the syntax does not allow for negation, we can only change the behavior in a way that mails are learned as ham when they are moved into `INBOX` from `Junk`. This is reasonable though. * adjust tests accordingly * adjust docs accordingly --- docs/content/config/environment.md | 4 +- .../startup/setup.d/security/rspamd.sh | 6 +-- .../rspamd_imap_move_to_inbox.txt | 4 +- .../nc_templates/rspamd_imap_move_to_junk.txt | 4 +- .../parallel/set1/spam_virus/rspamd_full.bats | 40 ++++++++++--------- 5 files changed, 31 insertions(+), 27 deletions(-) diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 5123fe57a69..3d204ccec08 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -353,7 +353,9 @@ Controls whether the [Rspamd Greylisting module][rspamd-greylisting-module] is e When enabled, 1. the "[autolearning][rspamd-autolearn]" feature is turned on; -2. the Bayes classifier will be trained when moving mails from or to the Junk folder (with the help of Sieve scripts). +2. the Bayes classifier will be trained (with the help of Sieve scripts) when moving mails + 1. from anywhere to the `Junk` folder (learning this email as spam); + 2. from the `Junk` folder into the `INBOX` (learning this email as ham). !!! warning "Attention" diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 429a8efb69b..c1c2712a21f 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -222,13 +222,13 @@ function __rspamd__setup_learning sedfile -i -E '/^}/d' /etc/dovecot/conf.d/90-sieve.conf cat >>/etc/dovecot/conf.d/90-sieve.conf << EOF - # From elsewhere to Junk folder + # From anyhwere to Junk imapsieve_mailbox1_name = Junk imapsieve_mailbox1_causes = COPY imapsieve_mailbox1_before = file:${SIEVE_PIPE_BIN_DIR}/learn-spam.sieve - # From Junk folder to elsewhere - imapsieve_mailbox2_name = * + # From Junk to Inbox + imapsieve_mailbox2_name = INBOX imapsieve_mailbox2_from = Junk imapsieve_mailbox2_causes = COPY imapsieve_mailbox2_before = file:${SIEVE_PIPE_BIN_DIR}/learn-ham.sieve diff --git a/test/test-files/nc_templates/rspamd_imap_move_to_inbox.txt b/test/test-files/nc_templates/rspamd_imap_move_to_inbox.txt index a3f6dc13aa4..b9191d627f1 100644 --- a/test/test-files/nc_templates/rspamd_imap_move_to_inbox.txt +++ b/test/test-files/nc_templates/rspamd_imap_move_to_inbox.txt @@ -1,4 +1,4 @@ A LOGIN user1@localhost.localdomain 123 B SELECT Junk -A UID MOVE 1:1 INBOX -A4 LOGOUT +A MOVE 1:1 INBOX +A LOGOUT diff --git a/test/test-files/nc_templates/rspamd_imap_move_to_junk.txt b/test/test-files/nc_templates/rspamd_imap_move_to_junk.txt index 0039d106e9a..1e38081cf47 100644 --- a/test/test-files/nc_templates/rspamd_imap_move_to_junk.txt +++ b/test/test-files/nc_templates/rspamd_imap_move_to_junk.txt @@ -1,4 +1,4 @@ A LOGIN user1@localhost.localdomain 123 B SELECT INBOX -A UID MOVE 1:1 Junk -A4 LOGOUT +A MOVE 1:1 Junk +A LOGOUT diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index 611995c6aa9..eb5de312b5d 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -237,14 +237,8 @@ function teardown_file() { _default_teardown ; } _run_in_container grep -F 'sieve_pipe_bin_dir = /usr/lib/dovecot/sieve-pipe' "${SIEVE_CONFIG_FILE}" assert_success - # Move an email to the "Junk" folder from "INBOX"; the first email we - # sent should pass fine, hence we can now move it - _send_email 'nc_templates/rspamd_imap_move_to_junk' '0.0.0.0 143' - sleep 1 # wait for the transaction to finish - - local MOVE_TO_JUNK_LINES=( + local LEARN_SPAM_LINES=( 'imapsieve: mailbox Junk: MOVE event' - 'imapsieve: Matched static mailbox rule [1]' "sieve: file storage: script: Opened script \`learn-spam'" 'sieve: file storage: Using Sieve script path: /usr/lib/dovecot/sieve-pipe/learn-spam.sieve' "sieve: Executing script from \`/usr/lib/dovecot/sieve-pipe/learn-spam.svbin'" @@ -254,30 +248,38 @@ function teardown_file() { _default_teardown ; } "left message in mailbox 'Junk'" ) + local LEARN_HAM_LINES=( + "sieve: file storage: script: Opened script \`learn-ham'" + 'sieve: file storage: Using Sieve script path: /usr/lib/dovecot/sieve-pipe/learn-ham.sieve' + "sieve: Executing script from \`/usr/lib/dovecot/sieve-pipe/learn-ham.svbin'" + "Finished running script \`/usr/lib/dovecot/sieve-pipe/learn-ham.svbin'" + "left message in mailbox 'INBOX'" + ) + + # Move an email to the "Junk" folder from "INBOX"; the first email we + # sent should pass fine, hence we can now move it. + _send_email 'nc_templates/rspamd_imap_move_to_junk' '0.0.0.0 143' + sleep 1 # wait for the transaction to finish + _run_in_container cat /var/log/mail/mail.log assert_success - for LINE in "${MOVE_TO_JUNK_LINES[@]}" + assert_output --partial 'imapsieve: Matched static mailbox rule [1]' + refute_output --partial 'imapsieve: Matched static mailbox rule [2]' + for LINE in "${LEARN_SPAM_LINES[@]}" do assert_output --partial "${LINE}" done # Move an email to the "INBOX" folder from "Junk"; there should be two mails - # in the "Junk" folder + # in the "Junk" folder, since the second email we sent during setup should + # have landed in the Junk folder already. _send_email 'nc_templates/rspamd_imap_move_to_inbox' '0.0.0.0 143' sleep 1 # wait for the transaction to finish - local MOVE_TO_JUNK_LINES=( - 'imapsieve: Matched static mailbox rule [2]' - "sieve: file storage: script: Opened script \`learn-ham'" - 'sieve: file storage: Using Sieve script path: /usr/lib/dovecot/sieve-pipe/learn-ham.sieve' - "sieve: Executing script from \`/usr/lib/dovecot/sieve-pipe/learn-ham.svbin'" - "Finished running script \`/usr/lib/dovecot/sieve-pipe/learn-ham.svbin'" - "left message in mailbox 'INBOX'" - ) - _run_in_container cat /var/log/mail/mail.log assert_success - for LINE in "${MOVE_TO_JUNK_LINES[@]}" + assert_output --partial 'imapsieve: Matched static mailbox rule [2]' + for LINE in "${LEARN_HAM_LINES[@]}" do assert_output --partial "${LINE}" done From 063b1bf51d991f9c4e87e3f544d976eef4414660 Mon Sep 17 00:00:00 2001 From: Casper Date: Sun, 14 May 2023 23:09:07 +0200 Subject: [PATCH 076/592] docs: Fix URL (#3337) --- docs/content/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/usage.md b/docs/content/usage.md index 0ea8beaf405..110c490d99f 100644 --- a/docs/content/usage.md +++ b/docs/content/usage.md @@ -88,7 +88,7 @@ All workflows are using the tagging convention listed below. It is subsequently Issue the following commands to acquire the necessary files: ``` BASH -DMS_GITHUB_URL="https://github.com/docker-mailserver/docker-mailserver/blob/latest" +DMS_GITHUB_URL="https://raw.githubusercontent.com/docker-mailserver/docker-mailserver/master" wget "${DMS_GITHUB_URL}/compose.yaml" wget "${DMS_GITHUB_URL}/mailserver.env" ``` From 7cc05581d1325c0868fd4d8d291eef3116bfed3f Mon Sep 17 00:00:00 2001 From: Casper Date: Sun, 14 May 2023 23:54:53 +0200 Subject: [PATCH 077/592] docs: Restore missing edit button (#3338) --- docs/mkdocs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index c56e34f1361..b4bef9bfef8 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -35,6 +35,8 @@ theme: - navigation.top - navigation.expand - navigation.instant + - content.action.edit + - content.action.view - content.code.annotate palette: # Light mode From f794d10bb578159765854d38aaebc7191629fd79 Mon Sep 17 00:00:00 2001 From: Casper Date: Mon, 15 May 2023 06:38:52 +0200 Subject: [PATCH 078/592] Update contributing (#3339) --- docs/content/contributing/general.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/content/contributing/general.md b/docs/content/contributing/general.md index d9d2fd58d08..ea88dc85e19 100644 --- a/docs/content/contributing/general.md +++ b/docs/content/contributing/general.md @@ -13,7 +13,9 @@ When refactoring, writing or altering scripts or other files, adhere to these ru ## Documentation -You will need to have Docker installed. Navigate into the `docs/` directory. Then run: +Make sure to select `edge` in the dropdown menu at the top. Navigate to the page you would like to edit and click the edit button in the top right. This allows you to make changes and create a pull-request. + +Alternatively you can make the changes locally. For that you'll need to have Docker installed. Navigate into the `docs/` directory. Then run: ```sh docker run --rm -it -p 8000:8000 -v "${PWD}:/docs" squidfunk/mkdocs-material From a99ae786dba4d65d690fddf42ad7ff28b27fbe21 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 15 May 2023 07:01:13 +0200 Subject: [PATCH 079/592] adjust `antivirus.conf` for Rspamd (#3331) See #3320 --- target/rspamd/local.d/actions.conf | 6 ++++++ target/rspamd/local.d/antivirus.conf | 2 ++ 2 files changed, 8 insertions(+) diff --git a/target/rspamd/local.d/actions.conf b/target/rspamd/local.d/actions.conf index 38632624ec4..b214c33987e 100644 --- a/target/rspamd/local.d/actions.conf +++ b/target/rspamd/local.d/actions.conf @@ -1,3 +1,9 @@ # documentation: https://rspamd.com/doc/configuration/metrics.html#actions +# and https://rspamd.com/doc/configuration/metrics.html + +#greylist = 4; +#add_header = 6; +#rewrite_subject = 7; +#reject = 15; subject = "***SPAM*** %s" diff --git a/target/rspamd/local.d/antivirus.conf b/target/rspamd/local.d/antivirus.conf index 78c44051313..ecd2dae16b2 100644 --- a/target/rspamd/local.d/antivirus.conf +++ b/target/rspamd/local.d/antivirus.conf @@ -11,4 +11,6 @@ ClamAV { symbol = "CLAM_VIRUS"; log_clean = true; max_size = 25000000; + timeout = 10; + retransmits = 2; } From ec330a35a10ac3ad9e4ded0f9d6e4a5df7fce691 Mon Sep 17 00:00:00 2001 From: Andreas Perhab Date: Mon, 15 May 2023 15:46:13 +0200 Subject: [PATCH 080/592] ClamAV: add a warning for the internal message size limit (#3341) --- target/scripts/startup/setup.d/security/misc.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index be25002a0df..3799c2a94db 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -187,10 +187,16 @@ function __setup__security__clamav then _log 'trace' "Setting ClamAV message scan size limit to '${CLAMAV_MESSAGE_SIZE_LIMIT}'" - # do a short sanity checks; ClamAV stops scanning at more that 4GB file size - if [[ $(numfmt --from=si "${CLAMAV_MESSAGE_SIZE_LIMIT}") -gt $(numfmt --from=si 4G) ]] + # do a short sanity check: ClamAV does not support setting a maximum size greater than 4000M (at all) + if [[ $(numfmt --from=si "${CLAMAV_MESSAGE_SIZE_LIMIT}") -gt $(numfmt --from=si 4000M) ]] then - _log 'warn' "You set 'CLAMAV_MESSAGE_SIZE_LIMIT' to a value larger than 4 Gigabyte which ClamAV does not support - you should correct your configuration" + _log 'warn' "You set 'CLAMAV_MESSAGE_SIZE_LIMIT' to a value larger than 4 Gigabyte, but the maximum value is 4000M for this value - you should correct your configuration" + fi + # For more details, see + # https://github.com/docker-mailserver/docker-mailserver/pull/3341 + if [[ $(numfmt --from=si "${CLAMAV_MESSAGE_SIZE_LIMIT}") -ge $(numfmt --from=iec 2G) ]] + then + _log 'warn' "You set 'CLAMAV_MESSAGE_SIZE_LIMIT' to a value larger than 2 GiBiByte but ClamAV does not scan files larger or equal to 2GiBiByte" fi sedfile -i -E \ From a72adc273183c415aa6a7777b7506f2bd3d4f12c Mon Sep 17 00:00:00 2001 From: Casper Date: Mon, 15 May 2023 19:11:36 +0200 Subject: [PATCH 081/592] Fix typos (#3344) --- target/scripts/startup/setup.d/security/misc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index 3799c2a94db..7bd86606c1a 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -196,7 +196,7 @@ function __setup__security__clamav # https://github.com/docker-mailserver/docker-mailserver/pull/3341 if [[ $(numfmt --from=si "${CLAMAV_MESSAGE_SIZE_LIMIT}") -ge $(numfmt --from=iec 2G) ]] then - _log 'warn' "You set 'CLAMAV_MESSAGE_SIZE_LIMIT' to a value larger than 2 GiBiByte but ClamAV does not scan files larger or equal to 2GiBiByte" + _log 'warn' "You set 'CLAMAV_MESSAGE_SIZE_LIMIT' to a value larger than 2 Gibibyte but ClamAV does not scan files larger or equal to 2 Gibibyte" fi sedfile -i -E \ From 7453bc096bca26ac29a65f16fdeaee2ee42bceee Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 15 May 2023 20:10:29 +0200 Subject: [PATCH 082/592] Dovecot: make home dir distinct from mail dir (#3335) * add new home dir for Dovecot I tried changing the mail dir, but this is a _very_ disruptive change, so I took approach 3 on , whereby the home directory is now inside the mail directory. The MDBOX/SDBOX formats are not touched by this change. The change itself could be considered breaking though. * adjust Sieve tests accordingly * Update target/dovecot/10-mail.conf * Update target/dovecot/auth-passwdfile.inc --------- Co-authored-by: Casper --- target/dovecot/10-mail.conf | 1 + target/dovecot/auth-master.inc | 1 - target/dovecot/auth-passwdfile.inc | 2 +- target/scripts/helpers/accounts.sh | 6 +++--- target/scripts/startup/setup.d/dovecot.sh | 7 +++---- test/tests/parallel/set1/dovecot/dovecot_sieve.bats | 4 ++-- 6 files changed, 10 insertions(+), 11 deletions(-) diff --git a/target/dovecot/10-mail.conf b/target/dovecot/10-mail.conf index 055186cdb90..f91fd1b2df2 100644 --- a/target/dovecot/10-mail.conf +++ b/target/dovecot/10-mail.conf @@ -27,6 +27,7 @@ # # # +mail_home = /var/mail/%d/%n/home/ mail_location = maildir:/var/mail/%d/%n # If you need to set multiple mailbox locations or want to change default diff --git a/target/dovecot/auth-master.inc b/target/dovecot/auth-master.inc index b71967cb624..daebb392749 100755 --- a/target/dovecot/auth-master.inc +++ b/target/dovecot/auth-master.inc @@ -6,4 +6,3 @@ passdb { result_success = continue #auth_bind = yes } - diff --git a/target/dovecot/auth-passwdfile.inc b/target/dovecot/auth-passwdfile.inc index d05c9abbe45..6bbf8258d68 100644 --- a/target/dovecot/auth-passwdfile.inc +++ b/target/dovecot/auth-passwdfile.inc @@ -15,5 +15,5 @@ passdb { userdb { driver = passwd-file args = username_format=%u /etc/dovecot/userdb - default_fields = uid=docker gid=docker home=/var/mail/%d/%u + default_fields = uid=docker gid=docker home=/var/mail/%d/%u/home/ } diff --git a/target/scripts/helpers/accounts.sh b/target/scripts/helpers/accounts.sh index 996e6896522..6ea1acbd575 100644 --- a/target/scripts/helpers/accounts.sh +++ b/target/scripts/helpers/accounts.sh @@ -77,7 +77,7 @@ function _create_accounts # Dovecot's userdb has the following format # user:password:uid:gid:(gecos):home:(shell):extra_fields - DOVECOT_USERDB_LINE="${LOGIN}:${PASS}:5000:5000::/var/mail/${DOMAIN}/${USER}::${USER_ATTRIBUTES}" + DOVECOT_USERDB_LINE="${LOGIN}:${PASS}:5000:5000::/var/mail/${DOMAIN}/${USER}/home::${USER_ATTRIBUTES}" if grep -qF "${DOVECOT_USERDB_LINE}" "${DOVECOT_USERDB_FILE}" then _log 'warn' "Login '${LOGIN}' will not be added to '${DOVECOT_USERDB_FILE}' twice" @@ -85,12 +85,12 @@ function _create_accounts echo "${DOVECOT_USERDB_LINE}" >>"${DOVECOT_USERDB_FILE}" fi - mkdir -p "/var/mail/${DOMAIN}/${USER}" + mkdir -p "/var/mail/${DOMAIN}/${USER}/home" # copy user provided sieve file, if present if [[ -e "/tmp/docker-mailserver/${LOGIN}.dovecot.sieve" ]] then - cp "/tmp/docker-mailserver/${LOGIN}.dovecot.sieve" "/var/mail/${DOMAIN}/${USER}/.dovecot.sieve" + cp "/tmp/docker-mailserver/${LOGIN}.dovecot.sieve" "/var/mail/${DOMAIN}/${USER}/home/.dovecot.sieve" fi done < <(_get_valid_lines_from_file "${DATABASE_ACCOUNTS}") diff --git a/target/scripts/startup/setup.d/dovecot.sh b/target/scripts/startup/setup.d/dovecot.sh index 60a04a6c933..be9b71c23d2 100644 --- a/target/scripts/startup/setup.d/dovecot.sh +++ b/target/scripts/startup/setup.d/dovecot.sh @@ -24,10 +24,10 @@ function _setup_dovecot ( 'sdbox' | 'mdbox' ) _log 'trace' "Dovecot ${DOVECOT_MAILBOX_FORMAT} format configured" - sed -i -e \ - "s|^mail_location = .*$|mail_location = ${DOVECOT_MAILBOX_FORMAT}:\/var\/mail\/%d\/%n|g" \ + sedfile -i -E "s|^(mail_home =).*|\1 /var/mail/%d/%n|" /etc/dovecot/conf.d/10-mail.conf + sedfile -i -E \ + "s|^(mail_location =).*|\1 ${DOVECOT_MAILBOX_FORMAT}:/var/mail/%d/%n|" \ /etc/dovecot/conf.d/10-mail.conf - _log 'trace' 'Enabling cron job for dbox purge' mv /etc/cron.d/dovecot-purge.disabled /etc/cron.d/dovecot-purge chmod 644 /etc/cron.d/dovecot-purge @@ -35,7 +35,6 @@ function _setup_dovecot ( * ) _log 'trace' 'Dovecot default format (maildir) configured' - sed -i -e 's|^mail_location = .*$|mail_location = maildir:\/var\/mail\/%d\/%n|g' /etc/dovecot/conf.d/10-mail.conf ;; esac diff --git a/test/tests/parallel/set1/dovecot/dovecot_sieve.bats b/test/tests/parallel/set1/dovecot/dovecot_sieve.bats index c4b0112fa07..c2e9e6c7d69 100644 --- a/test/tests/parallel/set1/dovecot/dovecot_sieve.bats +++ b/test/tests/parallel/set1/dovecot/dovecot_sieve.bats @@ -17,9 +17,9 @@ function setup_file() { --env ENABLE_MANAGESIEVE=1 # Required for mail delivery via nc: --env PERMIT_DOCKER=container - # Mount into mail dir for user1 to treat as a user-sieve: + # Mount into home dir for user1 to treat as a user-sieve: # NOTE: Cannot use ':ro', 'start-mailserver.sh' attempts to 'chown -R' /var/mail: - --volume "${TEST_TMP_CONFIG}/dovecot.sieve:/var/mail/localhost.localdomain/user1/.dovecot.sieve" + --volume "${TEST_TMP_CONFIG}/dovecot.sieve:/var/mail/localhost.localdomain/user1/home/.dovecot.sieve" ) _common_container_setup 'CONTAINER_ARGS_ENV_CUSTOM' From e82f0f2527630f5454be8ca0583c2c1292364c03 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 15 May 2023 20:35:00 +0200 Subject: [PATCH 083/592] ci: fix scheduled build permissions (#3345) --- .github/workflows/scheduled_builds.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/scheduled_builds.yml b/.github/workflows/scheduled_builds.yml index ef198a345fb..efdbb4d4a0c 100644 --- a/.github/workflows/scheduled_builds.yml +++ b/.github/workflows/scheduled_builds.yml @@ -7,6 +7,7 @@ on: permissions: contents: read packages: write + security-events: write jobs: build-images: @@ -21,6 +22,7 @@ jobs: uses: docker-mailserver/docker-mailserver/.github/workflows/generic_vulnerability-scan.yml@master with: cache-key: ${{ needs.build-images.outputs.build-cache-key }} + secrets: inherit publish-images: name: 'Publish Images' From da8d3654b83eb78fdd2b22b1fe2c6b5eaf779869 Mon Sep 17 00:00:00 2001 From: georglauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 15 May 2023 20:36:08 +0200 Subject: [PATCH 084/592] add dispatch to scheduled build workflow --- .github/workflows/scheduled_builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scheduled_builds.yml b/.github/workflows/scheduled_builds.yml index efdbb4d4a0c..f12f84a118f 100644 --- a/.github/workflows/scheduled_builds.yml +++ b/.github/workflows/scheduled_builds.yml @@ -1,6 +1,7 @@ name: 'Deploy :edge on Schedule' on: + workflow_dispatch: schedule: - cron: 0 0 * * 5 @@ -22,7 +23,6 @@ jobs: uses: docker-mailserver/docker-mailserver/.github/workflows/generic_vulnerability-scan.yml@master with: cache-key: ${{ needs.build-images.outputs.build-cache-key }} - secrets: inherit publish-images: name: 'Publish Images' From 1d2df8d4998a7cf06ed2a2b67e7434c7d414b029 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 23 May 2023 11:02:30 +1200 Subject: [PATCH 085/592] fix: DB helper should properly filter entries (#3359) Previously it was assumed the sed operation was applying the sed expressions as a sequence, but it did not seem to filter entries being looked up correctly. Instead any line that matched either sed expression pattern was output (_value without matching key, values split by the delimiter_), then grep would match any of that causing false-positives. Resolved by piping the first sed expression into the next. --- target/scripts/helpers/database/db.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/target/scripts/helpers/database/db.sh b/target/scripts/helpers/database/db.sh index 66476772e81..3299fa21ad0 100644 --- a/target/scripts/helpers/database/db.sh +++ b/target/scripts/helpers/database/db.sh @@ -132,11 +132,11 @@ function __db_list_already_contains_value { # Avoids accidentally matching a substring (case-insensitive acceptable): # 1. Extract the current value of the entry (`\1`), - # 2. If a value list, split into separate lines (`\n`+`g`) at V_DELIMITER, + # 2. Value list support: Split values into separate lines (`\n`+`g`) at V_DELIMITER, # 3. Check each line for an exact match of the target VALUE - sed -e "s/^${KEY_LOOKUP}\(.*\)/\1/" \ - -e "s/${V_DELIMITER}/\n/g" \ - "${DATABASE}" | grep -qi "^${_VALUE_}$" + sed -ne "s/^${KEY_LOOKUP}\+\(.*\)/\1/p" "${DATABASE}" \ + | sed -e "s/${V_DELIMITER}/\n/g" \ + | grep -qi "^${_VALUE_}$" } From abd72b6f10b110f0d97ea1e1d80b4c46a365d2c1 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 23 May 2023 16:33:58 +0200 Subject: [PATCH 086/592] ci: fix ShellCheck linting for BATS tests (#3347) * updated `lint.sh` to lint BATS (again) * fix linting errors --- test/linting/lint.sh | 36 ++++++++++++++----- .../parallel/set1/spam_virus/amavis.bats | 4 +++ .../set1/spam_virus/postgrey_enabled.bats | 2 +- .../parallel/set1/spam_virus/postscreen.bats | 3 +- .../parallel/set1/spam_virus/rspamd_dkim.bats | 2 +- .../parallel/set1/spam_virus/rspamd_full.bats | 2 ++ .../container_configuration/hostname.bats | 1 + .../process_check_restart.bats | 6 ++-- .../set3/scripts/helper_functions.bats | 5 +++ test/tests/serial/test_helper.bats | 2 ++ test/tests/serial/tests.bats | 2 ++ 11 files changed, 51 insertions(+), 14 deletions(-) diff --git a/test/linting/lint.sh b/test/linting/lint.sh index e30458977d0..3f917ec39f6 100755 --- a/test/linting/lint.sh +++ b/test/linting/lint.sh @@ -48,21 +48,24 @@ function _hadolint function _shellcheck { + local F_SH F_BIN F_BATS + # File paths for shellcheck: - F_SH=$(find . -type f -iname '*.sh' \ + readarray -d '' F_SH < <(find . -type f -iname '*.sh' \ -not -path './test/bats/*' \ -not -path './test/test_helper/*' \ - -not -path './.git/*' + -not -path './.git/*' \ + -print0 \ ) # shellcheck disable=SC2248 - F_BIN=$(find 'target/bin' -type f -not -name '*.py') - F_BATS=$(find 'test' -maxdepth 1 -type f -iname '*.bats') + readarray -d '' F_BIN < <(find 'target/bin' -type f -not -name '*.py' -print0) + readarray -d '' F_BATS < <(find 'test/tests/' -type f -iname '*.bats' -print0) # This command is a bit easier to grok as multi-line. # There is a `.shellcheckrc` file, but it's only supports half of the options below, thus kept as CLI: # `SCRIPTDIR` is a special value that represents the path of the script being linted, # all sourced scripts share the same SCRIPTDIR source-path of the original script being linted. - CMD_SHELLCHECK=(shellcheck + local CMD_SHELLCHECK=(shellcheck --external-sources --check-sourced --severity=style @@ -74,7 +77,13 @@ function _shellcheck --exclude=SC2311 --exclude=SC2312 --source-path=SCRIPTDIR - "${F_SH} ${F_BIN} ${F_BATS}" + ) + + local BATS_EXTRA_ARGS=( + --exclude=SC2030 + --exclude=SC2031 + --exclude=SC2034 + --exclude=SC2155 ) # The linter can reference additional source-path values declared in scripts, @@ -87,11 +96,22 @@ function _shellcheck # Otherwise it only applies to the line below it. You can declare multiple source-paths, they don't override the previous. # `source=relative/path/to/file.sh` will check the source value in each source-path as well. # shellcheck disable=SC2068 - if docker run --rm --tty \ + local ERROR=0 + + docker run --rm --tty \ --volume "${REPOSITORY_ROOT}:/ci:ro" \ --workdir "/ci" \ --name dms-test_shellcheck \ - "koalaman/shellcheck-alpine:v${SHELLCHECK_VERSION}" ${CMD_SHELLCHECK[@]} + "koalaman/shellcheck-alpine:v${SHELLCHECK_VERSION}" "${CMD_SHELLCHECK[@]}" "${F_SH[@]}" "${F_BIN[@]}" || ERROR=1 + + docker run --rm --tty \ + --volume "${REPOSITORY_ROOT}:/ci:ro" \ + --workdir "/ci" \ + --name dms-test_shellcheck \ + "koalaman/shellcheck-alpine:v${SHELLCHECK_VERSION}" "${CMD_SHELLCHECK[@]}" \ + "${BATS_EXTRA_ARGS[@]}" "${F_BATS[@]}" || ERROR=1 + + if [[ ${ERROR} -eq 0 ]] then _log 'info' 'ShellCheck succeeded' else diff --git a/test/tests/parallel/set1/spam_virus/amavis.bats b/test/tests/parallel/set1/spam_virus/amavis.bats index ba737806809..5ab8cd9d221 100644 --- a/test/tests/parallel/set1/spam_virus/amavis.bats +++ b/test/tests/parallel/set1/spam_virus/amavis.bats @@ -61,18 +61,22 @@ function teardown_file() { export CONTAINER_NAME=${CONTAINER1_NAME} local AMAVIS_DEFAULTS_FILE='/etc/amavis/conf.d/20-debian_defaults' + # shellcheck disable=SC2016 _run_in_container grep '\$sa_tag_level_deflt' "${AMAVIS_DEFAULTS_FILE}" assert_success assert_output --partial '= 2.0' + # shellcheck disable=SC2016 _run_in_container grep '\$sa_tag2_level_deflt' "${AMAVIS_DEFAULTS_FILE}" assert_success assert_output --partial '= 6.31' + # shellcheck disable=SC2016 _run_in_container grep '\$sa_kill_level_deflt' "${AMAVIS_DEFAULTS_FILE}" assert_success assert_output --partial '= 10.0' + # shellcheck disable=SC2016 _run_in_container grep '\$sa_spam_subject_tag' "${AMAVIS_DEFAULTS_FILE}" assert_success assert_output --partial "= '***SPAM*** ';" diff --git a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats index 3eaaca9de19..d877a1cec58 100644 --- a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats +++ b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats @@ -132,5 +132,5 @@ function _should_have_log_entry() { # `lines` is a special BATS variable updated via `run`: function _should_output_number_of_lines() { - assert_equal "${#lines[@]}" $1 + assert_equal "${#lines[@]}" "${1}" } diff --git a/test/tests/parallel/set1/spam_virus/postscreen.bats b/test/tests/parallel/set1/spam_virus/postscreen.bats index 240ba58a606..a1ddeb2999b 100644 --- a/test/tests/parallel/set1/spam_virus/postscreen.bats +++ b/test/tests/parallel/set1/spam_virus/postscreen.bats @@ -6,7 +6,7 @@ CONTAINER1_NAME='dms-test_postscreen_enforce' CONTAINER2_NAME='dms-test_postscreen_sender' function setup() { - CONTAINER1_IP=$(_get_container_ip ${CONTAINER1_NAME}) + CONTAINER1_IP=$(_get_container_ip "${CONTAINER1_NAME}") } function setup_file() { @@ -70,6 +70,7 @@ function _should_wait_turn_speaking_smtp() { local SMTP_TEMPLATE=$3 local EXPECTED=$4 + # shellcheck disable=SC2016 local UGLY_WORKAROUND='exec 3<>/dev/tcp/'"${TARGET_CONTAINER_IP}"'/25 && \ while IFS= read -r cmd; do \ head -1 <&3; \ diff --git a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats index 0f615d94a1b..b85e8d18a4f 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats @@ -64,7 +64,7 @@ function teardown_file() { _default_teardown ; } assert_output --partial "Finished DKIM key creation" _run_in_container_bash "[[ -f ${SIGNING_CONF_FILE} ]]" assert_success - _exec_in_container_bash "echo "blabla" >${SIGNING_CONF_FILE}" + _exec_in_container_bash "echo 'blabla' >${SIGNING_CONF_FILE}" local INITIAL_SHA512_SUM=$(_exec_in_container sha512sum "${SIGNING_CONF_FILE}") __create_key diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index eb5de312b5d..02664c5fb82 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -58,6 +58,7 @@ function setup_file() { function teardown_file() { _default_teardown ; } @test "Postfix's main.cf was adjusted" { + # shellcheck disable=SC2016 _run_in_container grep -F 'smtpd_milters = $rspamd_milter' /etc/postfix/main.cf assert_success _run_in_container postconf rspamd_milter @@ -171,6 +172,7 @@ function teardown_file() { _default_teardown ; } MODULE_PATH='/etc/rspamd/override.d/testmodule4.something' _run_in_container_bash "[[ -f ${MODULE_PATH} ]]" assert_success + # shellcheck disable=SC2016 _run_in_container grep -F 'some very long line with "weird $charact"ers' "${MODULE_PATH}" assert_success _run_in_container grep -F 'and! ano. ther &line' "${MODULE_PATH}" diff --git a/test/tests/parallel/set3/container_configuration/hostname.bats b/test/tests/parallel/set3/container_configuration/hostname.bats index 98a988513f6..fcb84b28e11 100644 --- a/test/tests/parallel/set3/container_configuration/hostname.bats +++ b/test/tests/parallel/set3/container_configuration/hostname.bats @@ -193,6 +193,7 @@ function _should_be_configured_to_fqdn() { assert_success # Amavis + # shellcheck disable=SC2016 _run_in_container grep '^\$myhostname' /etc/amavis/conf.d/05-node_id assert_output "\$myhostname = \"${EXPECTED_FQDN}\";" assert_success diff --git a/test/tests/parallel/set3/container_configuration/process_check_restart.bats b/test/tests/parallel/set3/container_configuration/process_check_restart.bats index b3d4fc86fa1..2a111af0a87 100644 --- a/test/tests/parallel/set3/container_configuration/process_check_restart.bats +++ b/test/tests/parallel/set3/container_configuration/process_check_restart.bats @@ -179,12 +179,12 @@ function _should_restart_when_killed() { function _check_if_process_is_running() { local PROCESS=${1} local MIN_SECS_RUNNING - [[ -n ${2} ]] && MIN_SECS_RUNNING="--older ${2}" + [[ -n ${2:-} ]] && MIN_SECS_RUNNING=('--older' "${2}") - local IS_RUNNING=$(docker exec "${CONTAINER_NAME}" pgrep --list-full ${MIN_SECS_RUNNING} "${PROCESS}") + local IS_RUNNING=$(docker exec "${CONTAINER_NAME}" pgrep --list-full "${MIN_SECS_RUNNING[@]}" "${PROCESS}") # When no matches are found, nothing is returned. Provide something we can assert on (helpful for debugging): - if [[ ! ${IS_RUNNING} =~ "${PROCESS}" ]] + if [[ ! ${IS_RUNNING} =~ ${PROCESS} ]] then echo "'${PROCESS}' is not running" return 1 diff --git a/test/tests/parallel/set3/scripts/helper_functions.bats b/test/tests/parallel/set3/scripts/helper_functions.bats index 95db61d83d4..5a1fbf74c87 100644 --- a/test/tests/parallel/set3/scripts/helper_functions.bats +++ b/test/tests/parallel/set3/scripts/helper_functions.bats @@ -4,6 +4,7 @@ BATS_TEST_NAME_PREFIX='[Scripts] (helper functions) ' SOURCE_BASE_PATH="${REPOSITORY_ROOT:?Expected REPOSITORY_ROOT to be set}/target/scripts/helpers" @test '(network.sh) _sanitize_ipv4_to_subnet_cidr' { + # shellcheck source=../../../../../target/scripts/helpers/network.sh source "${SOURCE_BASE_PATH}/network.sh" run _sanitize_ipv4_to_subnet_cidr '255.255.255.255/0' @@ -17,7 +18,9 @@ SOURCE_BASE_PATH="${REPOSITORY_ROOT:?Expected REPOSITORY_ROOT to be set}/target/ } @test '(utils.sh) _env_var_expect_zero_or_one' { + # shellcheck source=../../../../../target/scripts/helpers/log.sh source "${SOURCE_BASE_PATH}/log.sh" + # shellcheck source=../../../../../target/scripts/helpers/utils.sh source "${SOURCE_BASE_PATH}/utils.sh" ZERO=0 @@ -40,7 +43,9 @@ SOURCE_BASE_PATH="${REPOSITORY_ROOT:?Expected REPOSITORY_ROOT to be set}/target/ } @test '(utils.sh) _env_var_expect_integer' { + # shellcheck source=../../../../../target/scripts/helpers/log.sh source "${SOURCE_BASE_PATH}/log.sh" + # shellcheck source=../../../../../target/scripts/helpers/utils.sh source "${SOURCE_BASE_PATH}/utils.sh" INTEGER=1234 diff --git a/test/tests/serial/test_helper.bats b/test/tests/serial/test_helper.bats index 9b8f93abaa5..ecca3d85cbf 100644 --- a/test/tests/serial/test_helper.bats +++ b/test/tests/serial/test_helper.bats @@ -1,3 +1,5 @@ +# shellcheck disable=SC2314,SC2317 + load "${REPOSITORY_ROOT}/test/test_helper/common" BATS_TEST_NAME_PREFIX='test helper functions:' diff --git a/test/tests/serial/tests.bats b/test/tests/serial/tests.bats index f28787f8816..20ee0dd1fdb 100644 --- a/test/tests/serial/tests.bats +++ b/test/tests/serial/tests.bats @@ -164,6 +164,7 @@ function teardown_file() { _default_teardown ; } } @test "amavis: old virusmail is wipped by cron" { + # shellcheck disable=SC2016 _exec_in_container_bash 'touch -d "`date --date=2000-01-01`" /var/lib/amavis/virusmails/should-be-deleted' _run_in_container_bash '/usr/local/bin/virus-wiper' assert_success @@ -172,6 +173,7 @@ function teardown_file() { _default_teardown ; } } @test "amavis: recent virusmail is not wipped by cron" { + # shellcheck disable=SC2016 _exec_in_container_bash 'touch -d "`date`" /var/lib/amavis/virusmails/should-not-be-deleted' _run_in_container_bash '/usr/local/bin/virus-wiper' assert_success From 7af7546d88375a67c38d8d78eafe87a00ec9563e Mon Sep 17 00:00:00 2001 From: LucidityCrash <49124523+LucidityCrash@users.noreply.github.com> Date: Tue, 23 May 2023 16:25:08 +0100 Subject: [PATCH 087/592] feature: adding `getmail` as an alternative to `fetchmail` (#2803) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Co-authored-by: Casper --- Dockerfile | 7 +- README.md | 1 + docs/content/config/advanced/mail-getmail.md | 101 +++++++++++++++++++ docs/content/config/environment.md | 12 +++ docs/mkdocs.yml | 3 + mailserver.env | 9 ++ target/bin/debug-getmail | 21 ++++ target/bin/getmail-cron | 9 ++ target/getmail/getmailrc | 7 ++ target/scripts/build/packages.sh | 16 +++ target/scripts/start-mailserver.sh | 2 + target/scripts/startup/setup.d/getmail.sh | 42 ++++++++ target/scripts/startup/setup.d/mail_state.sh | 1 + target/scripts/startup/variables-stack.sh | 2 + test/config/getmail/getmail-user3.cf | 11 ++ test/tests/parallel/set1/getmail.bats | 79 +++++++++++++++ 16 files changed, 320 insertions(+), 3 deletions(-) create mode 100644 docs/content/config/advanced/mail-getmail.md create mode 100644 target/bin/debug-getmail create mode 100644 target/bin/getmail-cron create mode 100644 target/getmail/getmailrc create mode 100644 target/scripts/startup/setup.d/getmail.sh create mode 100644 test/config/getmail/getmail-user3.cf create mode 100644 test/tests/parallel/set1/getmail.bats diff --git a/Dockerfile b/Dockerfile index 1af200de8ef..018c39c251a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -158,15 +158,16 @@ COPY target/opendmarc/opendmarc.conf /etc/opendmarc.conf COPY target/opendmarc/default-opendmarc /etc/default/opendmarc COPY target/opendmarc/ignore.hosts /etc/opendmarc/ignore.hosts -# ----------------------------------------------- -# --- Fetchmail, Postfix & Let'sEncrypt --------- -# ----------------------------------------------- +# -------------------------------------------------- +# --- Fetchmail, Getmail, Postfix & Let'sEncrypt --- +# -------------------------------------------------- # Remove invalid URL from SPF message # https://bugs.launchpad.net/spf-engine/+bug/1896912 RUN echo 'Reason_Message = Message {rejectdefer} due to: {spf}.' >>/etc/postfix-policyd-spf-python/policyd-spf.conf COPY target/fetchmail/fetchmailrc /etc/fetchmailrc_general +COPY target/getmail/getmailrc /etc/getmailrc_general COPY target/postfix/main.cf target/postfix/master.cf /etc/postfix/ # DH parameters for DHE cipher suites, ffdhe4096 is the official standard 4096-bit DH params now part of TLS 1.3 diff --git a/README.md b/README.md index 1115249fe87..b3072cd9528 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ If you have issues, please search through [the documentation][documentation::web - [OpenDKIM](http://www.opendkim.org) & [OpenDMARC](https://github.com/trusteddomainproject/OpenDMARC) - [Fail2ban](https://www.fail2ban.org/wiki/index.php/Main_Page) - [Fetchmail](http://www.fetchmail.info/fetchmail-man.html) +- [Getmail6](https://getmail6.org/documentation.html) - [Postscreen](http://www.postfix.org/POSTSCREEN_README.html) - [Postgrey](https://postgrey.schweikert.ch/) - Support for [LetsEncrypt](https://letsencrypt.org/), manual and self-signed certificates diff --git a/docs/content/config/advanced/mail-getmail.md b/docs/content/config/advanced/mail-getmail.md new file mode 100644 index 00000000000..d08f5d0733d --- /dev/null +++ b/docs/content/config/advanced/mail-getmail.md @@ -0,0 +1,101 @@ +--- +title: 'Advanced | Email Gathering with Getmail' +--- + +To enable the [getmail][getmail-website] service to retrieve e-mails set the environment variable `ENABLE_GETMAIL` to `1`. Your `compose.yaml` file should include the following: + +```yaml +environment: + - ENABLE_GETMAIL=1 + - GETMAIL_POLL=5 +``` + +In your DMS config volume (eg: `docker-data/dms/config/`), create a `getmail-.cf` file for each remote account that you want to retrieve mail and store into a local DMS account. `` should be replaced by you, and is just the rest of the filename (eg: `getmail-example.cf`). The contents of each file should be configuration like documented below. + +The directory structure should similar to this: + +```txt +├── docker-data/dms/config +│   ├── dovecot.cf +│   ├── getmail-example.cf +│   ├── postfix-accounts.cf +│   └── postfix-virtual.cf +├── docker-compose.yml +└── README.md +``` + +## Configuration + +A detailed description of the configuration options can be found in the [online version of the manual page][getmail-docs]. + +### Common Options + +The default options added to each `getmail` config are: + +```getmailrc +[options] +verbose = 0 +read_all = false +delete = false +max_messages_per_session = 500 +received = false +delivered_to = false +``` + +If you want to use a different base config, mount a file to `/etc/getmailrc_general`. This file will replace the default "Common Options" base config above, that all `getmail-.cf` files will extend with their configs when used. + +??? example "IMAP Configuration" + + This example will: + + 1. Connect to the remote IMAP server from Gmail. + 2. Retrieve mail from the gmail account `alice` with password `notsecure`. + 3. Store any mail retrieved from the remote mail-server into DMS for the `user1@example.com` account that DMS manages. + + ```getmailrc + [retriever] + type = SimpleIMAPRetriever + server = imap.gmail.com + username = alice + password = notsecure + [destination] + type = MDA_external + path = /usr/lib/dovecot/deliver + allow_root_commands = true + arguments =("-d","user1@example.com") + ``` + +??? example "POP3 Configuration" + + Just like the IMAP example above, but instead via POP3 protocol if you prefer that over IMAP. + + ```getmailrc + [retriever] + type = SimplePOP3Retriever + server = pop3.gmail.com + username = alice + password = notsecure + [destination] + type = MDA_external + path = /usr/lib/dovecot/deliver + allow_root_commands = true + arguments =("-d","user1@example.com") + ``` + +### Polling Interval + +By default the `getmail` service checks external mail accounts for new mail every 5 minutes. That polling interval is configurable via the `GETMAIL_POLL` ENV variable, with a value in minutes (_default: 5, min: 1, max: 30_): + +```yaml +environment: + - GETMAIL_POLL=1 +``` + +### XOAUTH2 Authentication + +It is possible to utilize the `getmail-gmail-xoauth-tokens` helper to provide authentication using `xoauth2` for [gmail (example 12)][getmail-docs-xoauth-12] or [Microsoft Office 365 (example 13)][getmail-docs-xoauth-13] + +[getmail-website]: https://www.getmail.org +[getmail-docs]: https://getmail6.org/configuration.html +[getmail-docs-xoauth-12]: https://github.com/getmail6/getmail6/blob/1f95606156231f1e074ba62a9baa64f892a92ef8/docs/getmailrc-examples#L286 +[getmail-docs-xoauth-13]: https://github.com/getmail6/getmail6/blob/1f95606156231f1e074ba62a9baa64f892a92ef8/docs/getmailrc-examples#L351 diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 3d204ccec08..64c0b0d0ad5 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -548,6 +548,18 @@ Note: activate this only if you are confident in your bayes database for identif **1** => `/etc/fetchmailrc` is split per poll entry. For every poll entry a separate fetchmail instance is started to allow having multiple imap idle configurations defined. Note: The defaults of your fetchmailrc file need to be at the top of the file. Otherwise it won't be added correctly to all separate `fetchmail` instances. +#### Getmail + +##### ENABLE_GETMAIL + +Enable or disable `getmail`. + +- **0** => Disabled +- 1 => Enabled + +##### GETMAIL_POLL + +- **5** => `getmail` The number of minutes for the interval. Min: 1; Max: 30; Default: 5. #### LDAP diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index b4bef9bfef8..cef4a08226f 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -104,6 +104,8 @@ markdown_extensions: lang: shell-session - name: fetchmailrc lang: txt + - name: getmailrc + lang: txt - name: caddyfile lang: txt @@ -142,6 +144,7 @@ nav: - 'LDAP Authentication': config/advanced/auth-ldap.md - 'Email Filtering with Sieve': config/advanced/mail-sieve.md - 'Email Gathering with Fetchmail': config/advanced/mail-fetchmail.md + - 'Email Gathering with Getmail': config/advanced/mail-getmail.md - 'Email Forwarding': - 'Relay Hosts': config/advanced/mail-forwarding/relay-hosts.md - 'AWS SES': config/advanced/mail-forwarding/aws-ses.md diff --git a/mailserver.env b/mailserver.env index 7c1af20f505..c857aff9377 100644 --- a/mailserver.env +++ b/mailserver.env @@ -382,6 +382,15 @@ ENABLE_FETCHMAIL=0 # The interval to fetch mail in seconds FETCHMAIL_POLL=300 +# Enable or disable `getmail`. +# +# - **0** => Disabled +# - 1 => Enabled +ENABLE_GETMAIL=0 + +# The number of minutes for the interval. Min: 1; Max: 30. +GETMAIL_POLL=5 + # ----------------------------------------------- # --- LDAP Section ------------------------------ # ----------------------------------------------- diff --git a/target/bin/debug-getmail b/target/bin/debug-getmail new file mode 100644 index 00000000000..404855b1cfe --- /dev/null +++ b/target/bin/debug-getmail @@ -0,0 +1,21 @@ +#! /bin/bash + +# shellcheck source=../scripts/helpers/log.sh +source /usr/local/bin/helpers/log.sh +# shellcheck source=../scripts/startup/setup-stack.sh +source /usr/local/bin/setup.d/getmail.sh + +_setup_getmail + +if [[ -d /var/lib/getmail ]] +then + GETMAILDIR=/var/lib/getmail +else + mkdir -p /tmp/docker-mailserver/getmail + GETMAILDIR=/tmp/docker-mailserver/getmail +fi + +for FILE in /etc/getmailrc.d/getmailrc* +do + /usr/local/bin/getmail --getmaildir "${GETMAILDIR}" --rcfile "${FILE}" --dump | tail -n +7 +done diff --git a/target/bin/getmail-cron b/target/bin/getmail-cron new file mode 100644 index 00000000000..04710d1b848 --- /dev/null +++ b/target/bin/getmail-cron @@ -0,0 +1,9 @@ +#! /bin/bash + +for FILE in /etc/getmailrc.d/getmailrc* +do + if ! pgrep -f "${FILE}$" &>/dev/null + then + /usr/local/bin/getmail --getmaildir /var/lib/getmail --rcfile "${FILE}" + fi +done diff --git a/target/getmail/getmailrc b/target/getmail/getmailrc new file mode 100644 index 00000000000..57c2b4a98c9 --- /dev/null +++ b/target/getmail/getmailrc @@ -0,0 +1,7 @@ +[options] +verbose = 0 +read_all = false +delete = false +max_messages_per_session = 500 +received = false +delivered_to = false diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 74dc5ed27b6..5a5dc47c5be 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -201,6 +201,21 @@ function _install_fail2ban sedfile -i -r 's/^_nft_add_set = .+/_nft_add_set = add set \\{ type \\; flags interval\\; \\}/' /etc/fail2ban/action.d/nftables.conf } +# Presently the getmail6 package is v6.14, which is too old. +# v6.18 contains fixes for Google and Microsoft OAuth support. +# using pip to install getmail. +# TODO This can be removed when the base image is updated to Debian 12 (Bookworm) +function _install_getmail +{ + _log 'debug' 'Installing getmail6' + apt-get "${QUIET}" --no-install-recommends install python3-pip + pip3 install --no-cache-dir 'getmail6~=6.18.12' + ln -s /usr/local/bin/getmail /usr/bin/getmail + ln -s /usr/local/bin/getmail-gmail-xoauth-tokens /usr/bin/getmail-gmail-xoauth-tokens + apt-get "${QUIET}" purge python3-pip + apt-get "${QUIET}" autoremove +} + function _remove_data_after_package_installations { _log 'debug' 'Deleting sensitive files (secrets)' @@ -225,5 +240,6 @@ _install_packages _install_dovecot _install_rspamd _install_fail2ban +_install_getmail _remove_data_after_package_installations _post_installation_steps diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index 3532311fd1e..39f0db2b3bb 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -97,6 +97,8 @@ function _register_functions # needs to come after _setup_postfix_early _register_setup_function '_setup_spoof_protection' +_register_setup_function '_setup_getmail' + if [[ ${ENABLE_SRS} -eq 1 ]] then _register_setup_function '_setup_SRS' diff --git a/target/scripts/startup/setup.d/getmail.sh b/target/scripts/startup/setup.d/getmail.sh new file mode 100644 index 00000000000..c0cd4df8019 --- /dev/null +++ b/target/scripts/startup/setup.d/getmail.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +function _setup_getmail +{ + if [[ ${ENABLE_GETMAIL} -eq 1 ]] + then + _log 'trace' 'Preparing Getmail configuration' + + local GETMAILRC ID CONFIGS + + GETMAILRC='/etc/getmailrc.d' + CONFIGS=0 + + mkdir -p "${GETMAILRC}" + + # Generate getmailrc configs, starting with the `/etc/getmailrc_general` base config, + # Add a unique `message_log` config, then append users own config to the end. + for FILE in /tmp/docker-mailserver/getmail-*.cf + do + if [[ -f ${FILE} ]] + then + CONFIGS=1 + ID=$(cut -d '-' -f 3 <<< "${FILE}" | cut -d '.' -f 1) + local GETMAIL_CONFIG="${GETMAILRC}/getmailrc-${ID}" + cat /etc/getmailrc_general >"${GETMAIL_CONFIG}.tmp" + echo -e "message_log = /var/log/mail/getmail-${ID}.log\n" >>"${GETMAIL_CONFIG}.tmp" + cat "${GETMAIL_CONFIG}.tmp" "${FILE}" >"${GETMAIL_CONFIG}" + rm "${GETMAIL_CONFIG}.tmp" + fi + done + + if [[ ${CONFIGS} -eq 1 ]] + then + cat >/etc/cron.d/getmail << EOF +*/${GETMAIL_POLL} * * * * root /usr/local/bin/getmail-cron +EOF + chmod -R 600 "${GETMAILRC}" + fi + else + _log 'debug' 'Getmail is disabled' + fi +} diff --git a/target/scripts/startup/setup.d/mail_state.sh b/target/scripts/startup/setup.d/mail_state.sh index b96a457c9c9..161d06e613d 100644 --- a/target/scripts/startup/setup.d/mail_state.sh +++ b/target/scripts/startup/setup.d/mail_state.sh @@ -25,6 +25,7 @@ function _setup_save_states [[ ${ENABLE_CLAMAV} -eq 1 ]] && SERVICEDIRS+=('lib/clamav') [[ ${ENABLE_FAIL2BAN} -eq 1 ]] && SERVICEDIRS+=('lib/fail2ban') [[ ${ENABLE_FETCHMAIL} -eq 1 ]] && SERVICEDIRS+=('lib/fetchmail') + [[ ${ENABLE_GETMAIL} -eq 1 ]] && SERVICEDIRS+=('lib/getmail') [[ ${ENABLE_POSTGREY} -eq 1 ]] && SERVICEDIRS+=('lib/postgrey') [[ ${ENABLE_RSPAMD} -eq 1 ]] && SERVICEDIRS+=('lib/rspamd') [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] && SERVICEDIRS+=('lib/redis') diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index 0746c55cb03..f5e66a99b65 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -74,6 +74,7 @@ function __environment_variables_general_setup VARS[ENABLE_DNSBL]="${ENABLE_DNSBL:=0}" VARS[ENABLE_FAIL2BAN]="${ENABLE_FAIL2BAN:=0}" VARS[ENABLE_FETCHMAIL]="${ENABLE_FETCHMAIL:=0}" + VARS[ENABLE_GETMAIL]="${ENABLE_GETMAIL:=0}" VARS[ENABLE_MANAGESIEVE]="${ENABLE_MANAGESIEVE:=0}" VARS[ENABLE_OPENDKIM]="${ENABLE_OPENDKIM:=1}" VARS[ENABLE_OPENDMARC]="${ENABLE_OPENDMARC:=1}" @@ -125,6 +126,7 @@ function __environment_variables_general_setup VARS[ACCOUNT_PROVISIONER]="${ACCOUNT_PROVISIONER:=FILE}" VARS[FETCHMAIL_PARALLEL]="${FETCHMAIL_PARALLEL:=0}" VARS[FETCHMAIL_POLL]="${FETCHMAIL_POLL:=300}" + VARS[GETMAIL_POLL]="${GETMAIL_POLL:=5}" VARS[LOG_LEVEL]="${LOG_LEVEL:=info}" VARS[LOGROTATE_INTERVAL]="${LOGROTATE_INTERVAL:=weekly}" VARS[LOGWATCH_INTERVAL]="${LOGWATCH_INTERVAL:=none}" diff --git a/test/config/getmail/getmail-user3.cf b/test/config/getmail/getmail-user3.cf new file mode 100644 index 00000000000..cad73c9827a --- /dev/null +++ b/test/config/getmail/getmail-user3.cf @@ -0,0 +1,11 @@ +[retriever] +type = SimpleIMAPSSLRetriever +server = imap.remote-service.test +username = user3 +password=secret + +[destination] +type = MDA_external +path = /usr/lib/dovecot/deliver +allow_root_commands = true +arguments =("-d","user3@example.test") diff --git a/test/tests/parallel/set1/getmail.bats b/test/tests/parallel/set1/getmail.bats new file mode 100644 index 00000000000..4b5c528a171 --- /dev/null +++ b/test/tests/parallel/set1/getmail.bats @@ -0,0 +1,79 @@ +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" + +BATS_TEST_NAME_PREFIX='[Getmail] ' +CONTAINER_NAME='dms-test_getmail' + +function setup_file() { + + local CUSTOM_SETUP_ARGUMENTS=(--env 'ENABLE_GETMAIL=1') + _init_with_defaults + mv "${TEST_TMP_CONFIG}/getmail/getmail-user3.cf" "${TEST_TMP_CONFIG}/getmail-user3.cf" + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' +} + +function teardown_file() { _default_teardown ; } + +@test 'default configuration exists and is correct' { + _run_in_container cat /etc/getmailrc_general + assert_success + assert_line '[options]' + assert_line 'verbose = 0' + assert_line 'read_all = false' + assert_line 'delete = false' + assert_line 'max_messages_per_session = 500' + assert_line 'received = false' + assert_line 'delivered_to = false' + + _run_in_container stat /usr/local/bin/debug-getmail + assert_success + _run_in_container stat /usr/local/bin/getmail-cron + assert_success +} + +@test 'debug-getmail works as expected' { + _run_in_container cat /etc/getmailrc.d/getmailrc-user3 + assert_success + assert_line '[options]' + assert_line 'verbose = 0' + assert_line 'read_all = false' + assert_line 'delete = false' + assert_line 'max_messages_per_session = 500' + assert_line 'received = false' + assert_line 'delivered_to = false' + assert_line 'message_log = /var/log/mail/getmail-user3.log' + assert_line '[retriever]' + assert_line 'type = SimpleIMAPSSLRetriever' + assert_line 'server = imap.remote-service.test' + assert_line 'username = user3' + assert_line 'password=secret' + assert_line '[destination]' + assert_line 'type = MDA_external' + assert_line 'path = /usr/lib/dovecot/deliver' + assert_line 'allow_root_commands = true' + assert_line 'arguments =("-d","user3@example.test")' + + _run_in_container /usr/local/bin/debug-getmail + assert_success + assert_line --regexp 'retriever:.*SimpleIMAPSSLRetriever\(ca_certs="None", certfile="None", getmaildir="\/tmp\/docker-mailserver\/getmail", imap_on_delete="None", imap_search="None", keyfile="None", mailboxes="\(.*INBOX.*\)", move_on_delete="None", password="\*", password_command="\(\)", port="993", record_mailbox="True", server="imap.remote-service.test", ssl_cert_hostname="None", ssl_ciphers="None", ssl_fingerprints="\(\)", ssl_version="None", timeout="180", use_cram_md5="False", use_kerberos="False", use_peek="True", use_xoauth2="False", username="user3"\)' + assert_line --regexp 'destination:.*MDA_external\(allow_root_commands="True", arguments="\(.*-d.*user3@example.test.*\)", command="deliver", group="None", ignore_stderr="False", path="\/usr\/lib\/dovecot\/deliver", pipe_stdout="True", unixfrom="False", user="None"\)' + assert_line ' delete : False' + assert_line ' delete_after : 0' + assert_line ' delete_bigger_than : 0' + assert_line ' delivered_to : False' + assert_line ' fingerprint : False' + assert_line ' logfile : logfile(filename="/var/log/mail/getmail-user3.log")' + assert_line ' max_bytes_per_session : 0' + assert_line ' max_message_size : 0' + assert_line ' max_messages_per_session : 500' + assert_line ' message_log : /var/log/mail/getmail-user3.log' + assert_line ' message_log_syslog : False' + assert_line ' message_log_verbose : False' + assert_line ' netrc_file : None' + assert_line ' read_all : False' + assert_line ' received : False' + assert_line ' skip_imap_fetch_size : False' + assert_line ' to_oldmail_on_each_mail : False' + assert_line ' use_netrc : False' + assert_line ' verbose : 0' +} From 0e592aa911e7018624d983e111cdcb2e7b2f73da Mon Sep 17 00:00:00 2001 From: Casper Date: Tue, 23 May 2023 19:32:09 +0200 Subject: [PATCH 088/592] SPAM_TO_INBOX=1; add info about SA_KILL (#3360) --- target/scripts/startup/setup.d/security/misc.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index 7bd86606c1a..76fa8f67945 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -126,6 +126,7 @@ function __setup__security__spamassassin if [[ ${SPAMASSASSIN_SPAM_TO_INBOX} -eq 1 ]] then _log 'trace' 'Configuring Spamassassin/Amavis to send SPAM to inbox' + _log 'debug' 'SPAM_TO_INBOX=1 is set. SA_KILL will be ignored.' sed -i "s|\$final_spam_destiny.*=.*$|\$final_spam_destiny = D_PASS;|g" /etc/amavis/conf.d/49-docker-mailserver sed -i "s|\$final_bad_header_destiny.*=.*$|\$final_bad_header_destiny = D_PASS;|g" /etc/amavis/conf.d/49-docker-mailserver From cf74127f786046e297b17e8d5077e12f16b45c8b Mon Sep 17 00:00:00 2001 From: Casper Date: Wed, 24 May 2023 09:06:59 +0200 Subject: [PATCH 089/592] change if style (#3361) --- setup.sh | 45 ++++------- target/bin/debug-getmail | 3 +- target/bin/delmailuser | 6 +- target/bin/fail2ban | 15 ++-- target/bin/getmail-cron | 3 +- target/bin/listmailuser | 9 +-- target/bin/open-dkim | 36 +++------ target/bin/postfix-summary | 3 +- target/bin/restrict-access | 18 ++--- target/bin/rspamd-dkim | 24 ++---- target/bin/sedfile | 9 +-- target/bin/setquota | 6 +- target/bin/setup | 6 +- target/postgrey/postgrey.init | 3 +- target/scripts/build/packages.sh | 12 +-- target/scripts/check-for-changes.sh | 15 ++-- target/scripts/helpers/accounts.sh | 45 ++++------- target/scripts/helpers/aliases.sh | 12 +-- target/scripts/helpers/change-detection.sh | 12 +-- target/scripts/helpers/database/db.sh | 6 +- .../database/manage/postfix-accounts.sh | 6 +- .../database/manage/postfix-virtual.sh | 3 +- target/scripts/helpers/dns.sh | 9 +-- target/scripts/helpers/error.sh | 6 +- target/scripts/helpers/lock.sh | 6 +- target/scripts/helpers/log.sh | 15 ++-- target/scripts/helpers/network.sh | 6 +- target/scripts/helpers/postfix.sh | 9 +-- target/scripts/helpers/relay.sh | 21 ++--- target/scripts/helpers/ssl.sh | 57 +++++-------- target/scripts/helpers/utils.sh | 18 ++--- target/scripts/start-mailserver.sh | 9 +-- target/scripts/startup/check-stack.sh | 6 +- target/scripts/startup/daemons-stack.sh | 6 +- target/scripts/startup/setup-stack.sh | 12 +-- .../scripts/startup/setup.d/dmarc_dkim_spf.sh | 15 ++-- target/scripts/startup/setup.d/dovecot.sh | 45 ++++------- target/scripts/startup/setup.d/fetchmail.sh | 24 ++---- target/scripts/startup/setup.d/getmail.sh | 9 +-- target/scripts/startup/setup.d/ldap.sh | 15 ++-- target/scripts/startup/setup.d/log.sh | 3 +- target/scripts/startup/setup.d/mail_state.sh | 18 ++--- target/scripts/startup/setup.d/networking.sh | 3 +- target/scripts/startup/setup.d/postfix.sh | 39 +++------ target/scripts/startup/setup.d/saslauthd.sh | 3 +- .../scripts/startup/setup.d/security/misc.sh | 81 +++++++------------ .../startup/setup.d/security/rspamd.sh | 57 +++++-------- .../startup/setup.d/security/spoofing.sh | 12 +-- target/scripts/startup/variables-stack.sh | 18 ++--- target/scripts/update-check.sh | 9 +-- test/helper/common.bash | 27 +++---- test/helper/setup.bash | 6 +- test/helper/tls.bash | 9 +-- test/linting/lint.sh | 3 +- test/test_helper/common.bash | 3 +- test/tests/parallel/set1/tls/letsencrypt.bats | 3 +- test/tests/parallel/set2/tls_cipherlists.bats | 9 +-- .../process_check_restart.bats | 3 +- 58 files changed, 297 insertions(+), 594 deletions(-) diff --git a/setup.sh b/setup.sh index 53518df843c..b517d8bbbd5 100755 --- a/setup.sh +++ b/setup.sh @@ -71,11 +71,9 @@ function _show_local_usage function _get_absolute_script_directory { - if dirname "$(readlink -f "${0}")" &>/dev/null - then + if dirname "$(readlink -f "${0}")" &>/dev/null; then DIR=$(dirname "$(readlink -f "${0}")") - elif realpath -e -L "${0}" &>/dev/null - then + elif realpath -e -L "${0}" &>/dev/null; then DIR=$(realpath -e -L "${0}") DIR="${DIR%/setup.sh}" fi @@ -83,8 +81,7 @@ function _get_absolute_script_directory function _set_default_config_path { - if [[ -d "${DIR}/config" ]] - then + if [[ -d "${DIR}/config" ]]; then # legacy path (pre v10.2.0) DEFAULT_CONFIG_PATH="${DIR}/config" else @@ -94,23 +91,19 @@ function _set_default_config_path function _handle_config_path { - if [[ -z ${DESIRED_CONFIG_PATH} ]] - then + if [[ -z ${DESIRED_CONFIG_PATH} ]]; then # no desired config path - if [[ -n ${CONTAINER_NAME} ]] - then + if [[ -n ${CONTAINER_NAME} ]]; then VOLUME=$(${CRI} inspect "${CONTAINER_NAME}" \ --format="{{range .Mounts}}{{ println .Source .Destination}}{{end}}" | \ grep "${DMS_CONFIG}$" 2>/dev/null || :) fi - if [[ -n ${VOLUME} ]] - then + if [[ -n ${VOLUME} ]]; then CONFIG_PATH=$(echo "${VOLUME}" | awk '{print $1}') fi - if [[ -z ${CONFIG_PATH} ]] - then + if [[ -z ${CONFIG_PATH} ]]; then CONFIG_PATH=${DEFAULT_CONFIG_PATH} fi else @@ -121,8 +114,7 @@ function _handle_config_path function _run_in_new_container { # start temporary container with specified image - if ! ${CRI} history -q "${IMAGE_NAME}" &>/dev/null - then + if ! ${CRI} history -q "${IMAGE_NAME}" &>/dev/null; then echo "Image '${IMAGE_NAME}' not found. Pulling ..." ${CRI} pull "${IMAGE_NAME}" fi @@ -151,8 +143,7 @@ function _main ( * ) DESIRED_CONFIG_PATH="${DIR}/${OPTARG}" ;; esac - if [[ ! -d ${DESIRED_CONFIG_PATH} ]] - then + if [[ ! -d ${DESIRED_CONFIG_PATH} ]]; then echo "Specified directory '${DESIRED_CONFIG_PATH}' doesn't exist" >&2 exit 1 fi @@ -169,14 +160,11 @@ function _main done shift $(( OPTIND - 1 )) - if command -v docker &>/dev/null - then + if command -v docker &>/dev/null; then CRI=docker - elif command -v podman &>/dev/null - then + elif command -v podman &>/dev/null; then CRI=podman - if ! ${PODMAN_ROOTLESS} && [[ ${EUID} -ne 0 ]] - then + if ! ${PODMAN_ROOTLESS} && [[ ${EUID} -ne 0 ]]; then read -r -p "You are running Podman in rootless mode. Continue? [Y/n] " [[ -n ${REPLY} ]] && [[ ${REPLY} =~ (n|N) ]] && exit 0 fi @@ -190,13 +178,11 @@ function _main [[ -z ${CONTAINER_NAME} ]] && CONTAINER_NAME=${INFO#*;} [[ -z ${IMAGE_NAME} ]] && IMAGE_NAME=${INFO%;*} - if [[ -z ${IMAGE_NAME} ]] - then + if [[ -z ${IMAGE_NAME} ]]; then IMAGE_NAME=${NAME:-${DEFAULT_IMAGE_NAME}} fi - if test -t 0 - then + if test -t 0; then USE_TTY="-it" else # GitHub Actions will fail (or really anything else @@ -207,8 +193,7 @@ function _main _handle_config_path - if [[ -n ${CONTAINER_NAME} ]] - then + if [[ -n ${CONTAINER_NAME} ]]; then ${CRI} exec "${USE_TTY}" "${CONTAINER_NAME}" setup "${@}" else _run_in_new_container setup "${@}" diff --git a/target/bin/debug-getmail b/target/bin/debug-getmail index 404855b1cfe..fe6527ef074 100644 --- a/target/bin/debug-getmail +++ b/target/bin/debug-getmail @@ -7,8 +7,7 @@ source /usr/local/bin/setup.d/getmail.sh _setup_getmail -if [[ -d /var/lib/getmail ]] -then +if [[ -d /var/lib/getmail ]]; then GETMAILDIR=/var/lib/getmail else mkdir -p /tmp/docker-mailserver/getmail diff --git a/target/bin/delmailuser b/target/bin/delmailuser index 0621e6b2cb1..117a054c9b0 100755 --- a/target/bin/delmailuser +++ b/target/bin/delmailuser @@ -91,14 +91,12 @@ function _parse_options function _maildel_request_if_missing { - if [[ ${MAILDEL} -eq 0 ]] - then + if [[ ${MAILDEL} -eq 0 ]]; then local MAILDEL_CHOSEN read -r -p "Do you want to delete the mailbox as well (removing all mails)? [Y/n] " MAILDEL_CHOSEN # TODO: Why would MAILDEL be set to true if MAILDEL_CHOSEN is empty? - if [[ ${MAILDEL_CHOSEN} =~ (y|Y|yes|Yes) ]] || [[ -z ${MAILDEL_CHOSEN} ]] - then + if [[ ${MAILDEL_CHOSEN} =~ (y|Y|yes|Yes) ]] || [[ -z ${MAILDEL_CHOSEN} ]]; then MAILDEL=1 fi fi diff --git a/target/bin/fail2ban b/target/bin/fail2ban index f16fb87de93..1aa2c1dafd9 100755 --- a/target/bin/fail2ban +++ b/target/bin/fail2ban @@ -19,16 +19,14 @@ do JAILS+=("${LIST}") done -if [[ -z ${1} ]] -then +if [[ -z ${1} ]]; then IPS_BANNED=0 for JAIL in "${JAILS[@]}" do BANNED_IPS=$(fail2ban-client status "${JAIL}" | grep -oP '(?<=Banned IP list:\s).+') - if [[ -n ${BANNED_IPS} ]] - then + if [[ -n ${BANNED_IPS} ]]; then echo "Banned in ${JAIL}: ${BANNED_IPS}" IPS_BANNED=1 fi @@ -43,11 +41,9 @@ else ( 'ban' ) shift - if [[ -n ${1} ]] - then + if [[ -n ${1} ]]; then RESULT=$(fail2ban-client set custom banip "${@}") - if [[ ${RESULT} -gt 0 ]] - then + if [[ ${RESULT} -gt 0 ]]; then echo "Banned custom IP: ${RESULT}" else _log 'error' "Banning '${*}' failed. Already banned?" @@ -61,8 +57,7 @@ else ( 'unban' ) shift - if [[ -n ${1} ]] - then + if [[ -n ${1} ]]; then for JAIL in "${JAILS[@]}" do diff --git a/target/bin/getmail-cron b/target/bin/getmail-cron index 04710d1b848..4eb00dead46 100644 --- a/target/bin/getmail-cron +++ b/target/bin/getmail-cron @@ -2,8 +2,7 @@ for FILE in /etc/getmailrc.d/getmailrc* do - if ! pgrep -f "${FILE}$" &>/dev/null - then + if ! pgrep -f "${FILE}$" &>/dev/null; then /usr/local/bin/getmail --getmaildir /var/lib/getmail --rcfile "${FILE}" fi done diff --git a/target/bin/listmailuser b/target/bin/listmailuser index 5bffee9a071..8f5561d779c 100755 --- a/target/bin/listmailuser +++ b/target/bin/listmailuser @@ -71,12 +71,10 @@ function _quota_show_for function _bytes_to_human_readable_size { # `-` represents a non-applicable value (eg: Like when `SIZE_LIMIT` is not set): - if [[ ${1:-} == '-' ]] - then + if [[ ${1:-} == '-' ]]; then echo '~' # Otherwise a value in KibiBytes (1024 bytes == 1k) is expected (Dovecots internal representation): - elif [[ ${1:-} =~ ^[0-9]+$ ]] - then + elif [[ ${1:-} =~ ^[0-9]+$ ]]; then # kibibytes to bytes, converted to approproate IEC unit (eg: MiB): echo $(( 1024 * ${1} )) | numfmt --to=iec else @@ -105,8 +103,7 @@ function _alias_list_for_account "${DATABASE_VIRTUAL}" ) - if grep --quiet --no-messages "${GREP_OPTIONS[@]}" - then + if grep --quiet --no-messages "${GREP_OPTIONS[@]}"; then grep "${GREP_OPTIONS[@]}" | awk '{print $1;}' | sed ':a;N;$!ba;s/\n/, /g' fi } diff --git a/target/bin/open-dkim b/target/bin/open-dkim index b0a6b712d56..995d73598c9 100755 --- a/target/bin/open-dkim +++ b/target/bin/open-dkim @@ -3,8 +3,7 @@ # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh -if [[ -f /etc/dms-settings ]] && [[ $(_get_dms_env_value 'ENABLE_RSPAMD') -eq 1 ]] -then +if [[ -f /etc/dms-settings ]] && [[ $(_get_dms_env_value 'ENABLE_RSPAMD') -eq 1 ]]; then /usr/local/bin/rspamd-dkim "${@}" exit fi @@ -63,8 +62,7 @@ while [[ ${#} -gt 0 ]] do case "${1}" in ( 'keysize' ) - if [[ -n ${2+set} ]] - then + if [[ -n ${2+set} ]]; then KEYSIZE="${2}" shift shift @@ -74,8 +72,7 @@ do ;; ( 'selector' ) - if [[ -n ${2+set} ]] - then + if [[ -n ${2+set} ]]; then # shellcheck disable=SC2034 SELECTOR="${2}" shift @@ -86,8 +83,7 @@ do ;; ( 'domain' ) - if [[ -n ${2+set} ]] - then + if [[ -n ${2+set} ]]; then DOMAINS="${2}" shift shift @@ -112,8 +108,7 @@ function _generate_domains_config # Generate the default vhost (equivalent to /etc/postfix/vhost), # unless CLI arg DOMAINS provided an alternative list to use instead: - if [[ -z ${DOMAINS} ]] - then + if [[ -z ${DOMAINS} ]]; then _obtain_hostname_and_domainname # uses TMP_VHOST: _vhost_collect_postfix_domains @@ -126,8 +121,7 @@ function _generate_domains_config } _generate_domains_config -if [[ ! -s ${DATABASE_VHOST} ]] -then +if [[ ! -s ${DATABASE_VHOST} ]]; then _log 'warn' 'No entries found, no keys to make' exit 0 fi @@ -136,8 +130,7 @@ while read -r DKIM_DOMAIN do mkdir -p "/tmp/docker-mailserver/opendkim/keys/${DKIM_DOMAIN}" - if [[ ! -f "/tmp/docker-mailserver/opendkim/keys/${DKIM_DOMAIN}/${SELECTOR}.private" ]] - then + if [[ ! -f "/tmp/docker-mailserver/opendkim/keys/${DKIM_DOMAIN}/${SELECTOR}.private" ]]; then _log 'info' "Creating DKIM private key '/tmp/docker-mailserver/opendkim/keys/${DKIM_DOMAIN}/${SELECTOR}.private'" opendkim-genkey \ @@ -150,34 +143,29 @@ do # write to KeyTable if necessary KEYTABLEENTRY="${SELECTOR}._domainkey.${DKIM_DOMAIN} ${DKIM_DOMAIN}:${SELECTOR}:/etc/opendkim/keys/${DKIM_DOMAIN}/${SELECTOR}.private" - if [[ ! -f "/tmp/docker-mailserver/opendkim/KeyTable" ]] - then + if [[ ! -f "/tmp/docker-mailserver/opendkim/KeyTable" ]]; then _log 'debug' 'Creating DKIM KeyTable' echo "${KEYTABLEENTRY}" >/tmp/docker-mailserver/opendkim/KeyTable else - if ! grep -q "${KEYTABLEENTRY}" "/tmp/docker-mailserver/opendkim/KeyTable" - then + if ! grep -q "${KEYTABLEENTRY}" "/tmp/docker-mailserver/opendkim/KeyTable"; then echo "${KEYTABLEENTRY}" >>/tmp/docker-mailserver/opendkim/KeyTable fi fi # write to SigningTable if necessary SIGNINGTABLEENTRY="*@${DKIM_DOMAIN} ${SELECTOR}._domainkey.${DKIM_DOMAIN}" - if [[ ! -f /tmp/docker-mailserver/opendkim/SigningTable ]] - then + if [[ ! -f /tmp/docker-mailserver/opendkim/SigningTable ]]; then _log 'debug' 'Creating DKIM SigningTable' echo "*@${DKIM_DOMAIN} ${SELECTOR}._domainkey.${DKIM_DOMAIN}" >/tmp/docker-mailserver/opendkim/SigningTable else - if ! grep -q "${SIGNINGTABLEENTRY}" /tmp/docker-mailserver/opendkim/SigningTable - then + if ! grep -q "${SIGNINGTABLEENTRY}" /tmp/docker-mailserver/opendkim/SigningTable; then echo "${SIGNINGTABLEENTRY}" >>/tmp/docker-mailserver/opendkim/SigningTable fi fi done < <(_get_valid_lines_from_file "${DATABASE_VHOST}") # create TrustedHosts if missing -if [[ -d /tmp/docker-mailserver/opendkim ]] && [[ ! -f /tmp/docker-mailserver/opendkim/TrustedHosts ]] -then +if [[ -d /tmp/docker-mailserver/opendkim ]] && [[ ! -f /tmp/docker-mailserver/opendkim/TrustedHosts ]]; then _log 'debug' 'Creating DKIM TrustedHosts' echo "127.0.0.1" >/tmp/docker-mailserver/opendkim/TrustedHosts echo "localhost" >>/tmp/docker-mailserver/opendkim/TrustedHosts diff --git a/target/bin/postfix-summary b/target/bin/postfix-summary index 9e20d149742..1136b2a7f06 100755 --- a/target/bin/postfix-summary +++ b/target/bin/postfix-summary @@ -12,8 +12,7 @@ SENDER=${3} # The case that the mail.log.1 file isn't readable shouldn't # actually be possible with logrotate not rotating empty files.. # But you never know! -if [[ -r "/var/log/mail/mail.log.1" ]] -then +if [[ -r "/var/log/mail/mail.log.1" ]]; then BODY=$(/usr/sbin/pflogsumm /var/log/mail/mail.log.1 --problems-first) else BODY="Error: Mail log not readable or not found: /var/log/mail/mail.log.1 diff --git a/target/bin/restrict-access b/target/bin/restrict-access index 4acba2420e3..f368d044793 100755 --- a/target/bin/restrict-access +++ b/target/bin/restrict-access @@ -10,16 +10,14 @@ USER=${3:-} function __usage { _log 'info' "Usage: ${0} []" ; } -if [[ ${DIRECTION} =~ ^(send|receive)$ ]] -then +if [[ ${DIRECTION} =~ ^(send|receive)$ ]]; then DATABASE="/tmp/docker-mailserver/postfix-${DIRECTION}-access.cf" else __usage _exit_with_error "Unknown or missing second parameter '${DIRECTION}' - specify 'send' or 'receive'" fi -if [[ -z ${USER} ]] && [[ ${COMMAND} != list ]] -then +if [[ -z ${USER} ]] && [[ ${COMMAND} != list ]]; then read -r -p 'Provide a username: ' USER [[ -z ${USER} ]] && _exit_with_error 'User must not be empty' fi @@ -27,15 +25,13 @@ fi case "${COMMAND}" in ( 'add' ) - if [[ -f ${DATABASE} ]] && grep -q -F "${USER}" "${DATABASE}" - then + if [[ -f ${DATABASE} ]] && grep -q -F "${USER}" "${DATABASE}"; then _exit_with_error "User '${USER}' already denied to ${DIRECTION} mails" fi echo -e "${USER} \t\t REJECT" >>"${DATABASE}" - if [[ ${DIRECTION} == 'send' ]] - then + if [[ ${DIRECTION} == 'send' ]]; then CHECK='check_sender_access' POSTFIX_OPTION='smtpd_sender_restrictions' else @@ -45,16 +41,14 @@ case "${COMMAND}" in # only adjust Postfix's `main.cf` if we haven't adjusted it before STRING_TO_BE_ADDED="${CHECK} texthash:/tmp/docker-mailserver/postfix-${DIRECTION}-access.cf" - if ! grep -q "${STRING_TO_BE_ADDED}" /etc/postfix/main.cf - then + if ! grep -q "${STRING_TO_BE_ADDED}" /etc/postfix/main.cf; then sed -i -E "s|^(${POSTFIX_OPTION} =)(.*)|\1 ${STRING_TO_BE_ADDED},\2|" /etc/postfix/main.cf _reload_postfix fi ;; ( 'del' ) - if ! sed -i "/^$(_escape "${USER}").*/d" "${DATABASE}" 2>/dev/null - then + if ! sed -i "/^$(_escape "${USER}").*/d" "${DATABASE}" 2>/dev/null; then _exit_with_error "User '${USER}' not found" fi ;; diff --git a/target/bin/rspamd-dkim b/target/bin/rspamd-dkim index eb555f47145..8093dd39b16 100755 --- a/target/bin/rspamd-dkim +++ b/target/bin/rspamd-dkim @@ -88,8 +88,7 @@ function _parse_arguments ( 'keytype' ) [[ -n ${2:-} ]] || _exit_with_error "No keytype provided after 'keytype' argument" - if [[ ${2} == 'rsa' ]] || [[ ${2} == 'ed25519' ]] - then + if [[ ${2} == 'rsa' ]] || [[ ${2} == 'ed25519' ]]; then KEYTYPE=${2} _log 'debug' "Keytype set to '${KEYTYPE}'" else @@ -146,8 +145,7 @@ function _parse_arguments shift 2 done - if [[ ${KEYTYPE} == 'ed25519' ]] && [[ ${KEYSIZE} -ne 2048 ]] - then + if [[ ${KEYTYPE} == 'ed25519' ]] && [[ ${KEYSIZE} -ne 2048 ]]; then _exit_with_error "Chosen keytype does not accept the 'keysize' argument" fi @@ -160,8 +158,7 @@ function _create_keys # in other functions (after this function was called). BASE_DIR='/tmp/docker-mailserver/rspamd/dkim' - if [[ ${KEYTYPE} == 'rsa' ]] - then + if [[ ${KEYTYPE} == 'rsa' ]]; then local BASE_FILE_NAME="${BASE_DIR}/${KEYTYPE}-${KEYSIZE}-${SELECTOR}-${DOMAIN}" KEYTYPE_OPTIONS=('-b' "${KEYSIZE}") _log 'info' "Creating DKIM keys of type '${KEYTYPE}' and lenght '${KEYSIZE}' with selector '${SELECTOR}' for domain '${DOMAIN}'" @@ -198,11 +195,9 @@ function _create_keys function _check_permissions { # shellcheck disable=SC2310 - if ! __do_as_rspamd_user ls "${BASE_DIR}" >/dev/null - then + if ! __do_as_rspamd_user ls "${BASE_DIR}" >/dev/null; then _log 'warn' "The Rspamd user ('_rspamd') seems to be unable to list files in the keys directory ('${BASE_DIR}') - Rspamd may experience permission errors later" - elif ! __do_as_rspamd_user cat "${PRIVATE_KEY_FILE}" >/dev/null - then + elif ! __do_as_rspamd_user cat "${PRIVATE_KEY_FILE}" >/dev/null; then _log 'warn' "The Rspamd user ('_rspamd') seems to be unable to read the private key file - Rspamd may experience permission errors later" else _log 'debug' 'Permissions on files and directories seem ok' @@ -212,8 +207,7 @@ function _check_permissions function _setup_default_signing_conf { local DEFAULT_CONFIG_FILE='/etc/rspamd/override.d/dkim_signing.conf' - if [[ -f ${DEFAULT_CONFIG_FILE} ]] - then + if [[ -f ${DEFAULT_CONFIG_FILE} ]]; then _log 'debug' "'${DEFAULT_CONFIG_FILE}' exists, not supplying a default" else _log 'info' "Supplying a default configuration ('${DEFAULT_CONFIG_FILE}')" @@ -250,8 +244,7 @@ function _transform_public_key_file_to_dns_record_contents grep -o '".*"' "${PUBLIC_KEY_FILE}" | tr -d '"\n' >>"${PUBLIC_KEY_DNS_FILE}" echo '' >>"${PUBLIC_KEY_DNS_FILE}" - if ! _log_level_is '(warn|error)' - then + if ! _log_level_is '(warn|error)'; then _log 'info' "Here is the content of the TXT DNS record ${SELECTOR}._domainkey.${DOMAIN} that you need to create:\n" cat "${PUBLIC_KEY_DNS_FILE}" printf '\n' @@ -261,8 +254,7 @@ function _transform_public_key_file_to_dns_record_contents function _final_steps { # We need to restart Rspamd so the changes take effect immediately. - if ! supervisorctl restart rspamd - then + if ! supervisorctl restart rspamd; then _log 'warn' 'Could not restart Rspamd via Supervisord' fi diff --git a/target/bin/sedfile b/target/bin/sedfile index 106efe17deb..597c46be2e0 100755 --- a/target/bin/sedfile +++ b/target/bin/sedfile @@ -19,16 +19,14 @@ function __usage { echo "Usage: ${0} -i " ; } HASHTOOL='sha1sum' SKIP_ERROR=0 -if [[ ${#} -lt 3 ]] -then +if [[ ${#} -lt 3 ]]; then _log 'error' 'At least three parameters must be given' __usage exit 1 fi >&2 [[ -f /CONTAINER_START ]] && SKIP_ERROR=1 # hide error if container was restarted -if [[ ${1} == '--strict' ]] # show error every time -then +if [[ ${1} == '--strict' ]]; then # show error every time SKIP_ERROR=0 shift fi @@ -41,8 +39,7 @@ sed "${@}" NEW=$(${HASHTOOL} "${FILE}") # fail if file was not modified -if [[ ${OLD} == "${NEW}" ]] && [[ ${SKIP_ERROR} -eq 0 ]] -then +if [[ ${OLD} == "${NEW}" ]] && [[ ${SKIP_ERROR} -eq 0 ]]; then _log 'error' "No difference after call to 'sed' in 'sedfile' (sed ${*})" >&2 exit 1 fi diff --git a/target/bin/setquota b/target/bin/setquota index cfb1dc422bd..9c0bf06e00c 100755 --- a/target/bin/setquota +++ b/target/bin/setquota @@ -56,8 +56,7 @@ function _validate_parameters function _quota_request_if_missing { - if [[ -z ${QUOTA} ]] - then + if [[ -z ${QUOTA} ]]; then read -r -p 'Enter quota (e.g. 10M): ' QUOTA echo [[ -z "${QUOTA}" ]] && _exit_with_error 'Quota must not be empty (use 0 for unlimited quota)' @@ -66,8 +65,7 @@ function _quota_request_if_missing function _quota_unit_is_valid { - if ! grep -qE "^([0-9]+(B|k|M|G|T)|0)\$" <<< "${QUOTA}" - then + if ! grep -qE "^([0-9]+(B|k|M|G|T)|0)\$" <<< "${QUOTA}"; then __usage _exit_with_error 'Invalid quota format. e.g. 302M (B (byte), k (kilobyte), M (megabyte), G (gigabyte) or T (terabyte))' fi diff --git a/target/bin/setup b/target/bin/setup index 6496f74ccb8..49546f70c87 100755 --- a/target/bin/setup +++ b/target/bin/setup @@ -155,8 +155,7 @@ function _main ( show-mail-logs ) cat /var/log/mail/mail.log ;; ( login ) shift 2 - if [[ -z ${1:-} ]] - then + if [[ -z ${1:-} ]]; then /bin/bash else /bin/bash -c "${@}" @@ -171,8 +170,7 @@ function _main esac } -if [[ -z ${1:-} ]] -then +if [[ -z ${1:-} ]]; then _usage else _main "${@}" diff --git a/target/postgrey/postgrey.init b/target/postgrey/postgrey.init index 8486f26fa32..0ac7564b308 100644 --- a/target/postgrey/postgrey.init +++ b/target/postgrey/postgrey.init @@ -42,8 +42,7 @@ SCRIPTNAME="/etc/init.d/${DAEMON_NAME}" POSTGREY_OPTS="--pidfile=${PIDFILE} --daemonize ${POSTGREY_OPTS}" -if [ -z "${POSTGREY_TEXT}" ] -then +if [ -z "${POSTGREY_TEXT}" ]; then POSTGREY_TEXT_OPT="" else POSTGREY_TEXT_OPT="--greylist-text=${POSTGREY_TEXT}" diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 5a5dc47c5be..8258066eed2 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -103,8 +103,7 @@ function _install_dovecot dovecot-pop3d dovecot-sieve dovecot-solr ) - if [[ ${DOVECOT_COMMUNITY_REPO} -eq 1 ]] - then + if [[ ${DOVECOT_COMMUNITY_REPO} -eq 1 ]]; then # The package dovecot-fts-xapian is installed from the debian repository. # Starting with version 1.4.9a-1+deb11u1, a new dependency was added: dovecot-abi-2.3.abiv13 # dovecot-abi-2.3.abiv13 is a virtual package provided by dovecot-core (from the debian repository). @@ -144,8 +143,7 @@ function _install_rspamd # # Not removing it later is fine as you have to explicitly opt into installing a backports package # which is not something you could be doing by accident. - if [[ $(uname --machine) == 'aarch64' ]] - then + if [[ $(uname --machine) == 'aarch64' ]]; then echo '# Official Rspamd PPA does not support aarch64, so we use the Bullseye backports' >"${DEB_FILE}" echo 'deb [arch=arm64] http://deb.debian.org/debian bullseye-backports main' >>"${DEB_FILE}" RSPAMD_PACKAGE_NAME='rspamd/bullseye-backports' @@ -180,14 +178,12 @@ function _install_fail2ban FINGERPRINT=$(LANG=C gpg --verify fail2ban.deb.asc fail2ban.deb |& sed -n 's#Primary key fingerprint: \(.*\)#\1#p') - if [[ -z ${FINGERPRINT} ]] - then + if [[ -z ${FINGERPRINT} ]]; then echo 'ERROR: Invalid GPG signature!' >&2 exit 1 fi - if [[ ${FINGERPRINT} != "${FAIL2BAN_GPG_FINGERPRINT}" ]] - then + if [[ ${FINGERPRINT} != "${FAIL2BAN_GPG_FINGERPRINT}" ]]; then echo "ERROR: Wrong GPG fingerprint!" >&2 exit 1 fi diff --git a/target/scripts/check-for-changes.sh b/target/scripts/check-for-changes.sh index f23a2cb0b2a..cfcc78d9aef 100755 --- a/target/scripts/check-for-changes.sh +++ b/target/scripts/check-for-changes.sh @@ -22,8 +22,7 @@ source /etc/dms-settings _obtain_hostname_and_domainname # verify checksum file exists; must be prepared by start-mailserver.sh -if [[ ! -f ${CHKSUM_FILE} ]] -then +if [[ ! -f ${CHKSUM_FILE} ]]; then _exit_with_error "'${CHKSUM_FILE}' is missing" 0 fi @@ -41,8 +40,7 @@ function _check_for_changes # 0 – files are identical # 1 – files differ # 2 – inaccessible or missing argument - if [[ ${?} -eq 1 ]] - then + if [[ ${?} -eq 1 ]]; then _log_with_date 'info' 'Change detected' _create_lock # Shared config safety lock @@ -85,8 +83,7 @@ function _get_changed_files function _reload_amavis { - if [[ ${CHANGED} =~ ${DMS_DIR}/postfix-accounts.cf ]] || [[ ${CHANGED} =~ ${DMS_DIR}/postfix-virtual.cf ]] - then + if [[ ${CHANGED} =~ ${DMS_DIR}/postfix-accounts.cf ]] || [[ ${CHANGED} =~ ${DMS_DIR}/postfix-virtual.cf ]]; then # /etc/postfix/vhost was updated, amavis must refresh it's config by # reading this file again in case of new domains, otherwise they will be ignored. amavisd-new reload @@ -152,8 +149,7 @@ function _ssl_changes # manual - copy to internal DMS_TLS_PATH (/etc/dms/tls) that Postfix and Dovecot are configured to use. # acme.json - presently uses /etc/letsencrypt/live/ instead of DMS_TLS_PATH, # path may change requiring Postfix/Dovecot config update. - if [[ ${SSL_TYPE} == 'manual' ]] - then + if [[ ${SSL_TYPE} == 'manual' ]]; then # only run the SSL setup again if certificates have really changed. if [[ ${CHANGED} =~ ${SSL_CERT_PATH:-${REGEX_NEVER_MATCH}} ]] \ || [[ ${CHANGED} =~ ${SSL_KEY_PATH:-${REGEX_NEVER_MATCH}} ]] \ @@ -166,8 +162,7 @@ function _ssl_changes # `acme.json` is only relevant to Traefik, and is where it stores the certificates it manages. # When a change is detected it's assumed to be a possible cert renewal that needs to be # extracted for `docker-mailserver` services to adjust to. - elif [[ ${CHANGED} =~ /etc/letsencrypt/acme.json ]] - then + elif [[ ${CHANGED} =~ /etc/letsencrypt/acme.json ]]; then _log_with_date 'debug' "'/etc/letsencrypt/acme.json' has changed - extracting certificates" _setup_ssl diff --git a/target/scripts/helpers/accounts.sh b/target/scripts/helpers/accounts.sh index 6ea1acbd575..397290a343f 100644 --- a/target/scripts/helpers/accounts.sh +++ b/target/scripts/helpers/accounts.sh @@ -19,8 +19,7 @@ function _create_accounts local DATABASE_ACCOUNTS='/tmp/docker-mailserver/postfix-accounts.cf' _create_masters - if [[ -f ${DATABASE_ACCOUNTS} ]] - then + if [[ -f ${DATABASE_ACCOUNTS} ]]; then _log 'trace' "Checking file line endings" sed -i 's|\r||g' "${DATABASE_ACCOUNTS}" @@ -47,19 +46,16 @@ function _create_accounts DOMAIN=$(echo "${LOGIN}" | cut -d @ -f2) # test if user has a defined quota - if [[ -f /tmp/docker-mailserver/dovecot-quotas.cf ]] - then + if [[ -f /tmp/docker-mailserver/dovecot-quotas.cf ]]; then declare -a USER_QUOTA IFS=':' read -r -a USER_QUOTA < <(grep "${USER}@${DOMAIN}:" -i /tmp/docker-mailserver/dovecot-quotas.cf) - if [[ ${#USER_QUOTA[@]} -eq 2 ]] - then + if [[ ${#USER_QUOTA[@]} -eq 2 ]]; then USER_ATTRIBUTES="${USER_ATTRIBUTES:+${USER_ATTRIBUTES} }userdb_quota_rule=*:bytes=${USER_QUOTA[1]}" fi fi - if [[ -z ${USER_ATTRIBUTES} ]] - then + if [[ -z ${USER_ATTRIBUTES} ]]; then _log 'debug' "Creating user '${USER}' for domain '${DOMAIN}'" else _log 'debug' "Creating user '${USER}' for domain '${DOMAIN}' with attributes '${USER_ATTRIBUTES}'" @@ -68,8 +64,7 @@ function _create_accounts local POSTFIX_VMAILBOX_LINE DOVECOT_USERDB_LINE POSTFIX_VMAILBOX_LINE="${LOGIN} ${DOMAIN}/${USER}/" - if grep -qF "${POSTFIX_VMAILBOX_LINE}" /etc/postfix/vmailbox - then + if grep -qF "${POSTFIX_VMAILBOX_LINE}" /etc/postfix/vmailbox; then _log 'warn' "User '${USER}@${DOMAIN}' will not be added to '/etc/postfix/vmailbox' twice" else echo "${POSTFIX_VMAILBOX_LINE}" >>/etc/postfix/vmailbox @@ -78,8 +73,7 @@ function _create_accounts # Dovecot's userdb has the following format # user:password:uid:gid:(gecos):home:(shell):extra_fields DOVECOT_USERDB_LINE="${LOGIN}:${PASS}:5000:5000::/var/mail/${DOMAIN}/${USER}/home::${USER_ATTRIBUTES}" - if grep -qF "${DOVECOT_USERDB_LINE}" "${DOVECOT_USERDB_FILE}" - then + if grep -qF "${DOVECOT_USERDB_LINE}" "${DOVECOT_USERDB_FILE}"; then _log 'warn' "Login '${LOGIN}' will not be added to '${DOVECOT_USERDB_FILE}' twice" else echo "${DOVECOT_USERDB_LINE}" >>"${DOVECOT_USERDB_FILE}" @@ -88,8 +82,7 @@ function _create_accounts mkdir -p "/var/mail/${DOMAIN}/${USER}/home" # copy user provided sieve file, if present - if [[ -e "/tmp/docker-mailserver/${LOGIN}.dovecot.sieve" ]] - then + if [[ -e "/tmp/docker-mailserver/${LOGIN}.dovecot.sieve" ]]; then cp "/tmp/docker-mailserver/${LOGIN}.dovecot.sieve" "/var/mail/${DOMAIN}/${USER}/home/.dovecot.sieve" fi done < <(_get_valid_lines_from_file "${DATABASE_ACCOUNTS}") @@ -109,8 +102,7 @@ function _create_dovecot_alias_dummy_accounts { local DATABASE_VIRTUAL='/tmp/docker-mailserver/postfix-virtual.cf' - if [[ -f ${DATABASE_VIRTUAL} ]] && [[ ${ENABLE_QUOTAS} -eq 1 ]] - then + if [[ -f ${DATABASE_VIRTUAL} ]] && [[ ${ENABLE_QUOTAS} -eq 1 ]]; then # adding aliases to Dovecot's userdb # ${REAL_FQUN} is a user's fully-qualified username local ALIAS REAL_FQUN DOVECOT_USERDB_LINE @@ -129,8 +121,7 @@ function _create_dovecot_alias_dummy_accounts REAL_USERNAME=$(cut -d '@' -f 1 <<< "${REAL_FQUN}") REAL_DOMAINNAME=$(cut -d '@' -f 2 <<< "${REAL_FQUN}") - if ! grep -q "${REAL_FQUN}" "${DATABASE_ACCOUNTS}" - then + if ! grep -q "${REAL_FQUN}" "${DATABASE_ACCOUNTS}"; then _log 'debug' "Alias '${ALIAS}' is non-local (or mapped to a non-existing account) and will not be added to Dovecot's userdb" continue fi @@ -142,24 +133,20 @@ function _create_dovecot_alias_dummy_accounts # ${REAL_ACC[2]} => optional user attributes IFS='|' read -r -a REAL_ACC < <(grep "${REAL_FQUN}" "${DATABASE_ACCOUNTS}") - if [[ -z ${REAL_ACC[1]} ]] - then + if [[ -z ${REAL_ACC[1]} ]]; then _dms_panic__misconfigured 'postfix-accounts.cf' 'alias configuration' fi # test if user has a defined quota - if [[ -f /tmp/docker-mailserver/dovecot-quotas.cf ]] - then + if [[ -f /tmp/docker-mailserver/dovecot-quotas.cf ]]; then IFS=':' read -r -a USER_QUOTA < <(grep "${REAL_FQUN}:" -i /tmp/docker-mailserver/dovecot-quotas.cf) - if [[ ${#USER_QUOTA[@]} -eq 2 ]] - then + if [[ ${#USER_QUOTA[@]} -eq 2 ]]; then REAL_ACC[2]="${REAL_ACC[2]:+${REAL_ACC[2]} }userdb_quota_rule=*:bytes=${USER_QUOTA[1]}" fi fi DOVECOT_USERDB_LINE="${ALIAS}:${REAL_ACC[1]}:5000:5000::/var/mail/${REAL_DOMAINNAME}/${REAL_USERNAME}::${REAL_ACC[2]:-}" - if grep -qi "^${ALIAS}:" "${DOVECOT_USERDB_FILE}" - then + if grep -qi "^${ALIAS}:" "${DOVECOT_USERDB_FILE}"; then _log 'warn' "Alias '${ALIAS}' will not be added to '${DOVECOT_USERDB_FILE}' twice" else echo "${DOVECOT_USERDB_LINE}" >>"${DOVECOT_USERDB_FILE}" @@ -175,8 +162,7 @@ function _create_masters : >"${DOVECOT_MASTERDB_FILE}" local DATABASE_DOVECOT_MASTERS='/tmp/docker-mailserver/dovecot-masters.cf' - if [[ -f ${DATABASE_DOVECOT_MASTERS} ]] - then + if [[ -f ${DATABASE_DOVECOT_MASTERS} ]]; then _log 'trace' "Checking file line endings" sed -i 's|\r||g' "${DATABASE_DOVECOT_MASTERS}" @@ -203,8 +189,7 @@ function _create_masters # Dovecot's masterdb has the following format # user:password DOVECOT_MASTERDB_LINE="${LOGIN}:${PASS}" - if grep -qF "${DOVECOT_MASTERDB_LINE}" "${DOVECOT_MASTERDB_FILE}" - then + if grep -qF "${DOVECOT_MASTERDB_LINE}" "${DOVECOT_MASTERDB_FILE}"; then _log 'warn' "Login '${LOGIN}' will not be added to '${DOVECOT_MASTERDB_FILE}' twice" else echo "${DOVECOT_MASTERDB_LINE}" >>"${DOVECOT_MASTERDB_FILE}" diff --git a/target/scripts/helpers/aliases.sh b/target/scripts/helpers/aliases.sh index 9ded6da3dfe..16b9e599b66 100644 --- a/target/scripts/helpers/aliases.sh +++ b/target/scripts/helpers/aliases.sh @@ -12,11 +12,9 @@ function _handle_postfix_virtual_config local DATABASE_VIRTUAL=/tmp/docker-mailserver/postfix-virtual.cf - if [[ -f ${DATABASE_VIRTUAL} ]] - then + if [[ -f ${DATABASE_VIRTUAL} ]]; then # fixing old virtual user file - if grep -q ",$" "${DATABASE_VIRTUAL}" - then + if grep -q ",$" "${DATABASE_VIRTUAL}"; then sed -i -e "s|, |,|g" -e "s|,$||g" "${DATABASE_VIRTUAL}" fi @@ -30,14 +28,12 @@ function _handle_postfix_regexp_config { : >/etc/postfix/regexp - if [[ -f /tmp/docker-mailserver/postfix-regexp.cf ]] - then + if [[ -f /tmp/docker-mailserver/postfix-regexp.cf ]]; then _log 'trace' "Adding regexp alias file postfix-regexp.cf" cp -f /tmp/docker-mailserver/postfix-regexp.cf /etc/postfix/regexp - if ! grep 'virtual_alias_maps.*pcre:/etc/postfix/regexp' /etc/postfix/main.cf - then + if ! grep 'virtual_alias_maps.*pcre:/etc/postfix/regexp' /etc/postfix/main.cf; then sed -i -E \ 's|virtual_alias_maps(.*)|virtual_alias_maps\1 pcre:/etc/postfix/regexp|g' \ /etc/postfix/main.cf diff --git a/target/scripts/helpers/change-detection.sh b/target/scripts/helpers/change-detection.sh index 412b6322dea..24ea030119d 100644 --- a/target/scripts/helpers/change-detection.sh +++ b/target/scripts/helpers/change-detection.sh @@ -31,8 +31,7 @@ function _monitored_files_checksums # Supported user provided configs: local DMS_DIR=/tmp/docker-mailserver - if [[ -d ${DMS_DIR} ]] - then + if [[ -d ${DMS_DIR} ]]; then STAGING_FILES+=( "${DMS_DIR}/postfix-accounts.cf" "${DMS_DIR}/postfix-virtual.cf" @@ -46,8 +45,7 @@ function _monitored_files_checksums fi # SSL certs: - if [[ ${SSL_TYPE:-} == 'manual' ]] - then + if [[ ${SSL_TYPE:-} == 'manual' ]]; then # When using "manual" as the SSL type, # the following variables may contain the certificate files STAGING_FILES+=( @@ -56,8 +54,7 @@ function _monitored_files_checksums "${SSL_ALT_CERT_PATH:-}" "${SSL_ALT_KEY_PATH:-}" ) - elif [[ ${SSL_TYPE:-} == 'letsencrypt' ]] - then + elif [[ ${SSL_TYPE:-} == 'letsencrypt' ]]; then # React to any cert changes within the following LetsEncrypt locations: STAGING_FILES+=( /etc/letsencrypt/acme.json @@ -74,8 +71,7 @@ function _monitored_files_checksums [[ -f "${FILE}" ]] && CHANGED_FILES+=("${FILE}") done - if [[ -n ${CHANGED_FILES:-} ]] - then + if [[ -n ${CHANGED_FILES:-} ]]; then sha512sum -- "${CHANGED_FILES[@]}" fi } diff --git a/target/scripts/helpers/database/db.sh b/target/scripts/helpers/database/db.sh index 3299fa21ad0..8520f7cb67f 100644 --- a/target/scripts/helpers/database/db.sh +++ b/target/scripts/helpers/database/db.sh @@ -63,8 +63,7 @@ function _db_operation [[ ${DATABASE} == "${DATABASE_VIRTUAL}" ]] && V_DELIMITER=',' # Perform requested operation: - if _db_has_entry_with_key "${KEY}" "${DATABASE}" - then + if _db_has_entry_with_key "${KEY}" "${DATABASE}"; then # Find entry for key and return status code: case "${DB_ACTION}" in ( 'append' ) @@ -79,8 +78,7 @@ function _db_operation ;; ( 'remove' ) - if [[ -z ${VALUE} ]] - then # Remove entry for KEY: + if [[ -z ${VALUE} ]]; then # Remove entry for KEY: sedfile --strict -i "/^${KEY_LOOKUP}/d" "${DATABASE}" else # Remove target VALUE from entry: __db_list_already_contains_value || return 0 diff --git a/target/scripts/helpers/database/manage/postfix-accounts.sh b/target/scripts/helpers/database/manage/postfix-accounts.sh index efaf05f34ed..07d484f483e 100644 --- a/target/scripts/helpers/database/manage/postfix-accounts.sh +++ b/target/scripts/helpers/database/manage/postfix-accounts.sh @@ -74,8 +74,7 @@ function _arg_expect_mail_account function _account_should_not_exist_yet { __account_already_exists && _exit_with_error "'${MAIL_ACCOUNT}' already exists" - if [[ -f ${DATABASE_VIRTUAL} ]] && grep -q "^${MAIL_ACCOUNT}" "${DATABASE_VIRTUAL}" - then + if [[ -f ${DATABASE_VIRTUAL} ]] && grep -q "^${MAIL_ACCOUNT}" "${DATABASE_VIRTUAL}"; then _exit_with_error "'${MAIL_ACCOUNT}' is already defined as an alias" fi } @@ -95,8 +94,7 @@ function __account_already_exists # Also used by addsaslpassword function _password_request_if_missing { - if [[ -z ${PASSWD} ]] - then + if [[ -z ${PASSWD} ]]; then read -r -s -p 'Enter Password: ' PASSWD echo [[ -z ${PASSWD} ]] && _exit_with_error 'Password must not be empty' diff --git a/target/scripts/helpers/database/manage/postfix-virtual.sh b/target/scripts/helpers/database/manage/postfix-virtual.sh index c847d630455..cc40800ed95 100644 --- a/target/scripts/helpers/database/manage/postfix-virtual.sh +++ b/target/scripts/helpers/database/manage/postfix-virtual.sh @@ -25,8 +25,7 @@ function _manage_virtual_aliases case "${ACTION}" in # Associate RECIPIENT to MAIL_ALIAS: ( 'update' ) - if [[ -f ${DATABASE_ACCOUNTS} ]] && grep -q "^${MAIL_ALIAS}" "${DATABASE_ACCOUNTS}" - then + if [[ -f ${DATABASE_ACCOUNTS} ]] && grep -q "^${MAIL_ALIAS}" "${DATABASE_ACCOUNTS}"; then _exit_with_error "'${MAIL_ALIAS}' is already defined as an account" fi _db_entry_add_or_append "${DATABASE_VIRTUAL}" "${MAIL_ALIAS}" "${RECIPIENT}" diff --git a/target/scripts/helpers/dns.sh b/target/scripts/helpers/dns.sh index 0a600959984..72fecec038d 100644 --- a/target/scripts/helpers/dns.sh +++ b/target/scripts/helpers/dns.sh @@ -27,8 +27,7 @@ function _obtain_hostname_and_domainname # If the container is misconfigured.. `hostname -f` (which derives it's return value from `/etc/hosts` or DNS query), # will result in an error that returns an empty value. This warrants a panic. - if [[ -z ${HOSTNAME} ]] - then + if [[ -z ${HOSTNAME} ]]; then _dms_panic__misconfigured 'obtain_hostname' '/etc/hosts' fi @@ -39,10 +38,8 @@ function _obtain_hostname_and_domainname # `hostname -d` was probably not the correct command for this intention either. # Needs further investigation for relevance, and if `/etc/hosts` is important for consumers # of this variable or if a more deterministic approach with `cut` should be relied on. - if [[ $(_get_label_count "${HOSTNAME}") -gt 2 ]] - then - if [[ -n ${OVERRIDE_HOSTNAME:-} ]] - then + if [[ $(_get_label_count "${HOSTNAME}") -gt 2 ]]; then + if [[ -n ${OVERRIDE_HOSTNAME:-} ]]; then # Emulates the intended behaviour of `hostname -d`: # Assign the HOSTNAME value minus everything up to and including the first `.` DOMAINNAME=${HOSTNAME#*.} diff --git a/target/scripts/helpers/error.sh b/target/scripts/helpers/error.sh index d28d5341133..a75cef9c0d6 100644 --- a/target/scripts/helpers/error.sh +++ b/target/scripts/helpers/error.sh @@ -2,8 +2,7 @@ function _exit_with_error { - if [[ -n ${1+set} ]] - then + if [[ -n ${1+set} ]]; then _log 'error' "${1}" else _log 'error' "Call to '_exit_with_error' is missing a message to log" @@ -58,8 +57,7 @@ function dms_panic ;; esac - if [[ -n ${PANIC_SCOPE:-} ]] - then + if [[ -n ${PANIC_SCOPE:-} ]]; then _shutdown "${PANIC_SCOPE} | ${SHUTDOWN_MESSAGE}" else _shutdown "${SHUTDOWN_MESSAGE}" diff --git a/target/scripts/helpers/lock.sh b/target/scripts/helpers/lock.sh index 68bbd537a96..8c708225b1c 100644 --- a/target/scripts/helpers/lock.sh +++ b/target/scripts/helpers/lock.sh @@ -14,8 +14,7 @@ function _create_lock do # Handle stale lock files left behind on crashes # or premature/non-graceful exits of containers while they're making changes - if [[ -n "$(find "${LOCK_FILE}" -mmin +1 2>/dev/null)" ]] - then + if [[ -n "$(find "${LOCK_FILE}" -mmin +1 2>/dev/null)" ]]; then _log 'warn' 'Lock file older than 1 minute - removing stale lock file' rm -f "${LOCK_FILE}" else @@ -33,8 +32,7 @@ function _remove_lock { LOCK_FILE="${LOCK_FILE:-"/tmp/docker-mailserver/${SCRIPT_NAME}.lock"}" [[ -z "${LOCK_ID}" ]] && _exit_with_error "Cannot remove '${LOCK_FILE}' as there is no LOCK_ID set" - if [[ -e "${LOCK_FILE}" ]] && grep -q "${LOCK_ID}" "${LOCK_FILE}" # Ensure we don't delete a lock that's not ours - then + if [[ -e "${LOCK_FILE}" ]] && grep -q "${LOCK_ID}" "${LOCK_FILE}"; then # Ensure we don't delete a lock that's not ours rm -f "${LOCK_FILE}" _log 'trace' "Removed lock '${LOCK_FILE}'" fi diff --git a/target/scripts/helpers/log.sh b/target/scripts/helpers/log.sh index 925df2e62d6..a31d2408793 100644 --- a/target/scripts/helpers/log.sh +++ b/target/scripts/helpers/log.sh @@ -44,14 +44,12 @@ RESET=$(echo -ne '\e[0m') # is missing. Both failures will return with exit code '1'. function _log { - if [[ -z ${1+set} ]] - then + if [[ -z ${1+set} ]]; then _log 'error' "Call to '_log' is missing a valid log level" return 1 fi - if [[ -z ${2+set} ]] - then + if [[ -z ${2+set} ]]; then _log 'error' "Call to '_log' is missing a message to log" return 1 fi @@ -100,8 +98,7 @@ function _log MESSAGE+="${RESET}] ${2}" - if [[ ${1} =~ ^(warn|error)$ ]] - then + if [[ ${1} =~ ^(warn|error)$ ]]; then echo -e "${MESSAGE}" >&2 else echo -e "${MESSAGE}" @@ -120,11 +117,9 @@ function _log_with_date # use the default log level. function _get_log_level_or_default { - if [[ -n ${LOG_LEVEL+set} ]] - then + if [[ -n ${LOG_LEVEL+set} ]]; then echo "${LOG_LEVEL}" - elif [[ -e /etc/dms-settings ]] && grep -q -E "^LOG_LEVEL='[a-z]+'" /etc/dms-settings - then + elif [[ -e /etc/dms-settings ]] && grep -q -E "^LOG_LEVEL='[a-z]+'" /etc/dms-settings; then grep '^LOG_LEVEL=' /etc/dms-settings | cut -d "'" -f 2 else echo 'info' diff --git a/target/scripts/helpers/network.sh b/target/scripts/helpers/network.sh index a1e3665d180..d424cb23031 100644 --- a/target/scripts/helpers/network.sh +++ b/target/scripts/helpers/network.sh @@ -2,11 +2,9 @@ function _mask_ip_digit { - if [[ ${1} -ge 8 ]] - then + if [[ ${1} -ge 8 ]]; then MASK=255 - elif [[ ${1} -le 0 ]] - then + elif [[ ${1} -le 0 ]]; then MASK=0 else VALUES=(0 128 192 224 240 248 252 254 255) diff --git a/target/scripts/helpers/postfix.sh b/target/scripts/helpers/postfix.sh index 09ad14c884f..918b7c41c58 100644 --- a/target/scripts/helpers/postfix.sh +++ b/target/scripts/helpers/postfix.sh @@ -33,8 +33,7 @@ function _create_vhost { : >"${DATABASE_VHOST}" - if [[ -f ${TMP_VHOST} ]] - then + if [[ -f ${TMP_VHOST} ]]; then sort < "${TMP_VHOST}" | uniq >>"${DATABASE_VHOST}" rm "${TMP_VHOST}" fi @@ -48,8 +47,7 @@ function _vhost_collect_postfix_domains local DOMAIN UNAME # getting domains FROM mail accounts - if [[ -f ${DATABASE_ACCOUNTS} ]] - then + if [[ -f ${DATABASE_ACCOUNTS} ]]; then while IFS=$'|' read -r LOGIN _ do DOMAIN=$(echo "${LOGIN}" | cut -d @ -f2) @@ -58,8 +56,7 @@ function _vhost_collect_postfix_domains fi # getting domains FROM mail aliases - if [[ -f ${DATABASE_VIRTUAL} ]] - then + if [[ -f ${DATABASE_VIRTUAL} ]]; then while read -r FROM _ do UNAME=$(echo "${FROM}" | cut -d @ -f1) diff --git a/target/scripts/helpers/relay.sh b/target/scripts/helpers/relay.sh index b930489435e..24c69b0ecb0 100644 --- a/target/scripts/helpers/relay.sh +++ b/target/scripts/helpers/relay.sh @@ -77,8 +77,7 @@ function _relayhost_sasl chmod 0600 /etc/postfix/sasl_passwd local DATABASE_SASL_PASSWD='/tmp/docker-mailserver/postfix-sasl-password.cf' - if [[ -f ${DATABASE_SASL_PASSWD} ]] - then + if [[ -f ${DATABASE_SASL_PASSWD} ]]; then # Add domain-specific auth from config file: _get_valid_lines_from_file "${DATABASE_SASL_PASSWD}" >> /etc/postfix/sasl_passwd @@ -87,8 +86,7 @@ function _relayhost_sasl fi # Add an authenticated relay host defined via ENV config: - if [[ -n ${RELAY_USER} ]] && [[ -n ${RELAY_PASSWORD} ]] - then + if [[ -n ${RELAY_USER} ]] && [[ -n ${RELAY_PASSWORD} ]]; then echo "$(_env_relay_host) ${RELAY_USER}:${RELAY_PASSWORD}" >> /etc/postfix/sasl_passwd fi @@ -122,8 +120,7 @@ function _populate_relayhost_map # This config is mostly compatible with `/etc/postfix/relayhost_map`, but additionally supports # not providing a relay host for a sender domain to opt-out of RELAY_HOST? (2nd half of function) - if [[ -f /tmp/docker-mailserver/postfix-relaymap.cf ]] - then + if [[ -f /tmp/docker-mailserver/postfix-relaymap.cf ]]; then _log 'trace' "Adding relay mappings from postfix-relaymap.cf" # Match two values with some white-space between them (eg: `@example.test [relay.service.test]:465`): @@ -161,8 +158,7 @@ function _populate_relayhost_map # DOMAIN_PART not already present in `/etc/postfix/relayhost_map`, and not listed as a relay opt-out domain in `postfix-relaymap.cf` # `^@${DOMAIN_PART}\b` - To check for existing entry, the `\b` avoids accidental partial matches on similar domain parts. # `^\s*@${DOMAIN_PART}\s*$` - Matches line with only a domain part (eg: @example.test) to avoid including a mapping for those domains to the RELAY_HOST. - if ! grep -q -e "^@${DOMAIN_PART}\b" /etc/postfix/relayhost_map && ! grep -qs -e "^\s*@${DOMAIN_PART}\s*$" /tmp/docker-mailserver/postfix-relaymap.cf - then + if ! grep -q -e "^@${DOMAIN_PART}\b" /etc/postfix/relayhost_map && ! grep -qs -e "^\s*@${DOMAIN_PART}\s*$" /tmp/docker-mailserver/postfix-relaymap.cf; then _log 'trace' "Adding relay mapping for ${DOMAIN_PART}" echo "@${DOMAIN_PART} $(_env_relay_host)" >> /etc/postfix/relayhost_map fi @@ -183,14 +179,12 @@ function _setup_relayhost { _log 'debug' 'Setting up Postfix Relay Hosts' - if [[ -n ${DEFAULT_RELAY_HOST} ]] - then + if [[ -n ${DEFAULT_RELAY_HOST} ]]; then _log 'trace' "Setting default relay host ${DEFAULT_RELAY_HOST} to /etc/postfix/main.cf" postconf "relayhost = ${DEFAULT_RELAY_HOST}" fi - if [[ -n ${RELAY_HOST} ]] - then + if [[ -n ${RELAY_HOST} ]]; then _log 'trace' "Setting up relay hosts (default: ${RELAY_HOST})" _relayhost_sasl @@ -202,8 +196,7 @@ function _setup_relayhost function _rebuild_relayhost { - if [[ -n ${RELAY_HOST} ]] - then + if [[ -n ${RELAY_HOST} ]]; then _relayhost_sasl _populate_relayhost_map fi diff --git a/target/scripts/helpers/ssl.sh b/target/scripts/helpers/ssl.sh index ae7aea4d18e..da2c7756d22 100644 --- a/target/scripts/helpers/ssl.sh +++ b/target/scripts/helpers/ssl.sh @@ -8,8 +8,7 @@ function _setup_dhparam _log 'debug' "Setting up ${DH_SERVICE} dhparam" - if [[ -f ${DH_CUSTOM} ]] - then # use custom supplied dh params (assumes they're probably insecure) + if [[ -f ${DH_CUSTOM} ]]; then # use custom supplied dh params (assumes they're probably insecure) _log 'trace' "${DH_SERVICE} will use custom provided DH paramters" _log 'warn' "Using self-generated dhparams is considered insecure - unless you know what you are doing, please remove '${DH_CUSTOM}'" @@ -39,8 +38,7 @@ function _setup_ssl local DOVECOT_CERT=${1} # If a 2nd param is provided, a separate key and cert was received instead of a fullkeychain - if [[ -n ${2} ]] - then + if [[ -n ${2} ]]; then local PRIVATE_KEY=$1 local CERT_CHAIN=$2 @@ -117,22 +115,18 @@ function _setup_ssl # NOTE: See the `SSL_TYPE=letsencrypt` case below for more details. function _traefik_support { - if [[ -f /etc/letsencrypt/acme.json ]] - then + if [[ -f /etc/letsencrypt/acme.json ]]; then # Variable only intended for troubleshooting via debug output local EXTRACTED_DOMAIN # Conditional handling depends on the success of `_extract_certs_from_acme`, # Failure tries the next fallback FQDN to try extract a certificate from. # Subshell not used in conditional to ensure extraction log output is still captured - if [[ -n ${SSL_DOMAIN} ]] && _extract_certs_from_acme "${SSL_DOMAIN}" - then + if [[ -n ${SSL_DOMAIN} ]] && _extract_certs_from_acme "${SSL_DOMAIN}"; then EXTRACTED_DOMAIN=('SSL_DOMAIN' "${SSL_DOMAIN}") - elif _extract_certs_from_acme "${HOSTNAME}" - then + elif _extract_certs_from_acme "${HOSTNAME}"; then EXTRACTED_DOMAIN=('HOSTNAME' "${HOSTNAME}") - elif _extract_certs_from_acme "${DOMAINNAME}" - then + elif _extract_certs_from_acme "${DOMAINNAME}"; then EXTRACTED_DOMAIN=('DOMAINNAME' "${DOMAINNAME}") else _log 'warn' "letsencrypt (acme.json) failed to identify a certificate to extract" @@ -220,8 +214,7 @@ function _setup_ssl local TMP_KEY_WITH_FULLCHAIN="${TMP_DMS_TLS_PATH}/${COMBINED_PEM_NAME}" local KEY_WITH_FULLCHAIN="${DMS_TLS_PATH}/${COMBINED_PEM_NAME}" - if [[ -f ${TMP_KEY_WITH_FULLCHAIN} ]] - then + if [[ -f ${TMP_KEY_WITH_FULLCHAIN} ]]; then cp "${TMP_KEY_WITH_FULLCHAIN}" "${KEY_WITH_FULLCHAIN}" chmod 600 "${KEY_WITH_FULLCHAIN}" @@ -241,8 +234,7 @@ function _setup_ssl local CERT_CHAIN="${DMS_TLS_PATH}/cert" # Fail early: - if [[ -z ${SSL_KEY_PATH} ]] && [[ -z ${SSL_CERT_PATH} ]] - then + if [[ -z ${SSL_KEY_PATH} ]] && [[ -z ${SSL_CERT_PATH} ]]; then _dms_panic__no_env 'SSL_KEY_PATH or SSL_CERT_PATH' "${SCOPE_SSL_TYPE}" fi @@ -254,8 +246,7 @@ function _setup_ssl _dms_panic__no_file "(ALT) ${SSL_ALT_KEY_PATH} or ${SSL_ALT_CERT_PATH}" "${SCOPE_SSL_TYPE}" fi - if [[ -f ${SSL_KEY_PATH} ]] && [[ -f ${SSL_CERT_PATH} ]] - then + if [[ -f ${SSL_KEY_PATH} ]] && [[ -f ${SSL_CERT_PATH} ]]; then cp "${SSL_KEY_PATH}" "${PRIVATE_KEY}" cp "${SSL_CERT_PATH}" "${CERT_CHAIN}" chmod 600 "${PRIVATE_KEY}" @@ -264,8 +255,7 @@ function _setup_ssl _set_certificate "${PRIVATE_KEY}" "${CERT_CHAIN}" # Support for a fallback certificate, useful for hybrid/dual ECDSA + RSA certs - if [[ -n ${SSL_ALT_KEY_PATH} ]] && [[ -n ${SSL_ALT_CERT_PATH} ]] - then + if [[ -n ${SSL_ALT_KEY_PATH} ]] && [[ -n ${SSL_ALT_CERT_PATH} ]]; then _log 'trace' "Configuring fallback certificates using key ${SSL_ALT_KEY_PATH} and cert ${SSL_ALT_CERT_PATH}" _set_alt_certificate "${SSL_ALT_KEY_PATH}" "${SSL_ALT_CERT_PATH}" @@ -393,14 +383,11 @@ function _find_letsencrypt_domain { local LETSENCRYPT_DOMAIN - if [[ -n ${SSL_DOMAIN} ]] && [[ -e /etc/letsencrypt/live/$(_strip_wildcard_prefix "${SSL_DOMAIN}")/fullchain.pem ]] - then + if [[ -n ${SSL_DOMAIN} ]] && [[ -e /etc/letsencrypt/live/$(_strip_wildcard_prefix "${SSL_DOMAIN}")/fullchain.pem ]]; then LETSENCRYPT_DOMAIN=$(_strip_wildcard_prefix "${SSL_DOMAIN}") - elif [[ -e /etc/letsencrypt/live/${HOSTNAME}/fullchain.pem ]] - then + elif [[ -e /etc/letsencrypt/live/${HOSTNAME}/fullchain.pem ]]; then LETSENCRYPT_DOMAIN=${HOSTNAME} - elif [[ -e /etc/letsencrypt/live/${DOMAINNAME}/fullchain.pem ]] - then + elif [[ -e /etc/letsencrypt/live/${DOMAINNAME}/fullchain.pem ]]; then LETSENCRYPT_DOMAIN=${DOMAINNAME} else _log 'error' "Cannot find a valid DOMAIN for '/etc/letsencrypt/live//', tried: '${SSL_DOMAIN}', '${HOSTNAME}', '${DOMAINNAME}'" @@ -416,16 +403,13 @@ function _find_letsencrypt_key local LETSENCRYPT_KEY local LETSENCRYPT_DOMAIN=${1} - if [[ -z ${LETSENCRYPT_DOMAIN} ]] - then + if [[ -z ${LETSENCRYPT_DOMAIN} ]]; then _dms_panic__misconfigured 'LETSENCRYPT_DOMAIN' '_find_letsencrypt_key' fi - if [[ -e /etc/letsencrypt/live/${LETSENCRYPT_DOMAIN}/privkey.pem ]] - then + if [[ -e /etc/letsencrypt/live/${LETSENCRYPT_DOMAIN}/privkey.pem ]]; then LETSENCRYPT_KEY='privkey' - elif [[ -e /etc/letsencrypt/live/${LETSENCRYPT_DOMAIN}/key.pem ]] - then + elif [[ -e /etc/letsencrypt/live/${LETSENCRYPT_DOMAIN}/key.pem ]]; then LETSENCRYPT_KEY='key' else _log 'error' "Cannot find key file ('privkey.pem' or 'key.pem') in '/etc/letsencrypt/live/${LETSENCRYPT_DOMAIN}/'" @@ -438,8 +422,7 @@ function _find_letsencrypt_key function _extract_certs_from_acme { local CERT_DOMAIN=${1} - if [[ -z ${CERT_DOMAIN} ]] - then + if [[ -z ${CERT_DOMAIN} ]]; then _log 'warn' "_extract_certs_from_acme | CERT_DOMAIN is empty" return 1 fi @@ -448,16 +431,14 @@ function _extract_certs_from_acme KEY=$(acme_extract.py /etc/letsencrypt/acme.json "${CERT_DOMAIN}" --key) CERT=$(acme_extract.py /etc/letsencrypt/acme.json "${CERT_DOMAIN}" --cert) - if [[ -z ${KEY} ]] || [[ -z ${CERT} ]] - then + if [[ -z ${KEY} ]] || [[ -z ${CERT} ]]; then _log 'warn' "_extract_certs_from_acme | Unable to find key and/or cert for '${CERT_DOMAIN}' in '/etc/letsencrypt/acme.json'" return 1 fi # Currently we advise SSL_DOMAIN for wildcard support using a `*.example.com` value, # The filepath however should be `example.com`, avoiding the wildcard part: - if [[ ${SSL_DOMAIN} == "${CERT_DOMAIN}" ]] - then + if [[ ${SSL_DOMAIN} == "${CERT_DOMAIN}" ]]; then CERT_DOMAIN=$(_strip_wildcard_prefix "${SSL_DOMAIN}") fi diff --git a/target/scripts/helpers/utils.sh b/target/scripts/helpers/utils.sh index e225b1c8580..da093deba0d 100644 --- a/target/scripts/helpers/utils.sh +++ b/target/scripts/helpers/utils.sh @@ -16,8 +16,7 @@ function _get_valid_lines_from_file # and it will return its value stored in /etc/dms-settings function _get_dms_env_value { - if [[ -f /etc/dms-settings ]] - then + if [[ -f /etc/dms-settings ]]; then grep "^${1}=" /etc/dms-settings | cut -d "'" -f 2 else _log 'warn' "Call to '_get_dms_env_value' but '/etc/dms-settings' is not present" @@ -34,8 +33,7 @@ function _get_dms_env_value function _chown_var_mail_if_necessary { # fix permissions, but skip this if 3 levels deep the user id is already set - if find /var/mail -maxdepth 3 -a \( \! -user 5000 -o \! -group 5000 \) | read -r - then + if find /var/mail -maxdepth 3 -a \( \! -user 5000 -o \! -group 5000 \) | read -r; then _log 'trace' 'Fixing /var/mail permissions' chown -R 5000:5000 /var/mail || return 1 fi @@ -59,8 +57,7 @@ function _require_n_parameters_or_print_usage # https://github.com/docker-mailserver/docker-mailserver/issues/2985 function _adjust_mtime_for_postfix_maincf { - if [[ $(( $(date '+%s') - $(stat -c '%Y' '/etc/postfix/main.cf') )) -lt 2 ]] - then + if [[ $(( $(date '+%s') - $(stat -c '%Y' '/etc/postfix/main.cf') )) -lt 2 ]]; then touch -d '2 seconds ago' /etc/postfix/main.cf fi } @@ -97,14 +94,11 @@ function _reload_postfix # 2. The second argument is a path to a file that does not exist function _replace_by_env_in_file { - if [[ -z ${1+set} ]] - then + if [[ -z ${1+set} ]]; then _dms_panic__invalid_value 'first argument unset' 'utils.sh:_replace_by_env_in_file' - elif [[ -z ${2+set} ]] - then + elif [[ -z ${2+set} ]]; then _dms_panic__invalid_value 'second argument unset' 'utils.sh:_replace_by_env_in_file' - elif [[ ! -f ${2} ]] - then + elif [[ ! -f ${2} ]]; then _dms_panic__invalid_value "file '${2}' does not exist" 'utils.sh:_replace_by_env_in_file' fi diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index 39f0db2b3bb..2ad4c2dabab 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -42,8 +42,7 @@ function _register_functions _register_setup_function '_setup_logs_general' _register_setup_function '_setup_timezone' - if [[ ${SMTP_ONLY} -ne 1 ]] - then + if [[ ${SMTP_ONLY} -ne 1 ]]; then _register_setup_function '_setup_dovecot' _register_setup_function '_setup_dovecot_sieve' _register_setup_function '_setup_dovecot_dhparam' @@ -69,8 +68,7 @@ function _register_functions ;; esac - if [[ ${ENABLE_SASLAUTHD} -eq 1 ]] - then + if [[ ${ENABLE_SASLAUTHD} -eq 1 ]]; then _environment_variables_saslauthd _register_setup_function '_setup_saslauthd' fi @@ -99,8 +97,7 @@ function _register_functions _register_setup_function '_setup_getmail' - if [[ ${ENABLE_SRS} -eq 1 ]] - then + if [[ ${ENABLE_SRS} -eq 1 ]]; then _register_setup_function '_setup_SRS' _register_start_daemon '_start_daemon_postsrsd' fi diff --git a/target/scripts/startup/check-stack.sh b/target/scripts/startup/check-stack.sh index 7b7fdfca5cb..05b76c45715 100644 --- a/target/scripts/startup/check-stack.sh +++ b/target/scripts/startup/check-stack.sh @@ -21,8 +21,7 @@ function _check_improper_restart { _log 'debug' 'Checking for improper restart' - if [[ -f /CONTAINER_START ]] - then + if [[ -f /CONTAINER_START ]]; then _log 'warn' 'This container was (likely) improperly restarted which can result in undefined behavior' _log 'warn' 'Please destroy the container properly and then start DMS again' fi @@ -36,8 +35,7 @@ function _check_hostname _log 'debug' "Hostname has been set to ${HOSTNAME}" # HOSTNAME should be an FQDN (eg: hostname.domain) - if ! grep -q -E '^(\S+[.]\S+)$' <<< "${HOSTNAME}" - then + if ! grep -q -E '^(\S+[.]\S+)$' <<< "${HOSTNAME}"; then _dms_panic__general 'Setting hostname/domainname is required' fi } diff --git a/target/scripts/startup/daemons-stack.sh b/target/scripts/startup/daemons-stack.sh index b09428c6bc5..ae88dc24937 100644 --- a/target/scripts/startup/daemons-stack.sh +++ b/target/scripts/startup/daemons-stack.sh @@ -26,8 +26,7 @@ function _default_start_daemon RESULT=$(supervisorctl start "${1}" 2>&1) # shellcheck disable=SC2181 - if [[ ${?} -ne 0 ]] - then + if [[ ${?} -ne 0 ]]; then _log 'error' "${RESULT}" _dms_panic__fail_init "${1}" fi @@ -61,8 +60,7 @@ function _start_daemon_postfix function _start_daemon_fetchmail { - if [[ ${FETCHMAIL_PARALLEL} -eq 1 ]] - then + if [[ ${FETCHMAIL_PARALLEL} -eq 1 ]]; then local COUNTER=0 for _ in /etc/fetchmailrc.d/fetchmail-*.rc do diff --git a/target/scripts/startup/setup-stack.sh b/target/scripts/startup/setup-stack.sh index efcbd33ebf5..e31b846411a 100644 --- a/target/scripts/startup/setup-stack.sh +++ b/target/scripts/startup/setup-stack.sh @@ -32,8 +32,7 @@ function _early_supervisor_setup { SUPERVISOR_LOGLEVEL="${SUPERVISOR_LOGLEVEL:-warn}" - if ! grep -q "loglevel = ${SUPERVISOR_LOGLEVEL}" /etc/supervisor/supervisord.conf - then + if ! grep -q "loglevel = ${SUPERVISOR_LOGLEVEL}" /etc/supervisor/supervisord.conf; then case "${SUPERVISOR_LOGLEVEL}" in ( 'critical' | 'error' | 'info' | 'debug' ) sed -i -E \ @@ -64,8 +63,7 @@ function _setup_timezone local ZONEINFO_FILE="/usr/share/zoneinfo/${TZ}" - if [[ ! -e ${ZONEINFO_FILE} ]] - then + if [[ ! -e ${ZONEINFO_FILE} ]]; then _log 'warn' "Cannot find timezone '${TZ}'" return 1 fi @@ -87,8 +85,7 @@ function _setup_apply_fixes_after_configuration touch /dev/shm/supervisor.sock _log 'debug' 'Checking /var/mail permissions' - if ! _chown_var_mail_if_necessary - then + if ! _chown_var_mail_if_necessary; then _dms_panic__general 'Failed to fix /var/mail permissions' fi @@ -100,8 +97,7 @@ function _run_user_patches { local USER_PATCHES='/tmp/docker-mailserver/user-patches.sh' - if [[ -f ${USER_PATCHES} ]] - then + if [[ -f ${USER_PATCHES} ]]; then _log 'debug' 'Applying user patches' /bin/bash "${USER_PATCHES}" else diff --git a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh index a867e1d0cda..85de88cd7fc 100644 --- a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh +++ b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh @@ -8,8 +8,7 @@ # `smtpd_milters` milters options. function _setup_opendkim { - if [[ ${ENABLE_OPENDKIM} -eq 1 ]] - then + if [[ ${ENABLE_OPENDKIM} -eq 1 ]]; then _log 'debug' 'Configuring DKIM' mkdir -p /etc/opendkim/keys/ @@ -24,8 +23,7 @@ function _setup_opendkim /etc/postfix/main.cf # check if any keys are available - if [[ -e /tmp/docker-mailserver/opendkim/KeyTable ]] - then + if [[ -e /tmp/docker-mailserver/opendkim/KeyTable ]]; then cp -a /tmp/docker-mailserver/opendkim/* /etc/opendkim/ _log 'trace' "DKIM keys added for: $(find /etc/opendkim/keys/ -maxdepth 1 -type f -printf '%f ')" chown -R opendkim:opendkim /etc/opendkim/ @@ -35,8 +33,7 @@ function _setup_opendkim fi # setup nameservers parameter from /etc/resolv.conf if not defined - if ! grep -q '^Nameservers' /etc/opendkim.conf - then + if ! grep -q '^Nameservers' /etc/opendkim.conf; then local NAMESERVER_IPS NAMESERVER_IPS=$(grep '^nameserver' /etc/resolv.conf | awk -F " " '{print $2}' | paste -sd ',' -) echo "Nameservers ${NAMESERVER_IPS}" >>/etc/opendkim.conf @@ -59,8 +56,7 @@ function _setup_opendkim # `smtpd_milters` milters options. function _setup_opendmarc { - if [[ ${ENABLE_OPENDMARC} -eq 1 ]] - then + if [[ ${ENABLE_OPENDMARC} -eq 1 ]]; then # TODO When disabling SPF is possible, add a check whether DKIM and SPF is disabled # for DMARC to work, you should have at least one enabled # (see RFC 7489 https://www.rfc-editor.org/rfc/rfc7489#page-24) @@ -89,8 +85,7 @@ function _setup_opendmarc # using Rspamd, you will likely want to turn that off. function _setup_policyd_spf { - if [[ ${ENABLE_POLICYD_SPF} -eq 1 ]] - then + if [[ ${ENABLE_POLICYD_SPF} -eq 1 ]]; then _log 'debug' 'Configuring policyd-spf' cat >>/etc/postfix/master.cf </tmp/docker-mailserver/dovecot-quotas.cf fi @@ -154,8 +143,7 @@ function _setup_dovecot_local_user _log 'debug' 'Setting up Dovecot Local User' - if [[ ! -f /tmp/docker-mailserver/postfix-accounts.cf ]] - then + if [[ ! -f /tmp/docker-mailserver/postfix-accounts.cf ]]; then _log 'trace' "No mail accounts to create - '/tmp/docker-mailserver/postfix-accounts.cf' is missing" fi @@ -165,8 +153,7 @@ function _setup_dovecot_local_user for (( COUNTER = 11 ; COUNTER >= 0 ; COUNTER-- )) do - if [[ $(grep -cE '.+@.+\|' /tmp/docker-mailserver/postfix-accounts.cf 2>/dev/null || printf '%s' '0') -ge 1 ]] - then + if [[ $(grep -cE '.+@.+\|' /tmp/docker-mailserver/postfix-accounts.cf 2>/dev/null || printf '%s' '0') -ge 1 ]]; then return 0 else _log 'warn' "You need at least one mail account to start Dovecot ($(( ( COUNTER + 1 ) * SLEEP_PERIOD ))s left for account creation before shutdown)" @@ -190,11 +177,9 @@ function _setup_dovecot_inet_protocols local PROTOCOL # https://dovecot.org/doc/dovecot-example.conf - if [[ ${DOVECOT_INET_PROTOCOLS} == "ipv4" ]] - then + if [[ ${DOVECOT_INET_PROTOCOLS} == "ipv4" ]]; then PROTOCOL='*' # IPv4 only - elif [[ ${DOVECOT_INET_PROTOCOLS} == "ipv6" ]] - then + elif [[ ${DOVECOT_INET_PROTOCOLS} == "ipv6" ]]; then PROTOCOL='[::]' # IPv6 only else # Unknown value, panic. diff --git a/target/scripts/startup/setup.d/fetchmail.sh b/target/scripts/startup/setup.d/fetchmail.sh index 97b4aa993f8..99b74d84765 100644 --- a/target/scripts/startup/setup.d/fetchmail.sh +++ b/target/scripts/startup/setup.d/fetchmail.sh @@ -2,8 +2,7 @@ function _setup_fetchmail { - if [[ ${ENABLE_FETCHMAIL} -eq 1 ]] - then + if [[ ${ENABLE_FETCHMAIL} -eq 1 ]]; then _log 'trace' 'Enabling and configuring Fetchmail' local CONFIGURATION FETCHMAILRC @@ -11,8 +10,7 @@ function _setup_fetchmail CONFIGURATION='/tmp/docker-mailserver/fetchmail.cf' FETCHMAILRC='/etc/fetchmailrc' - if [[ -f ${CONFIGURATION} ]] - then + if [[ -f ${CONFIGURATION} ]]; then cat /etc/fetchmailrc_general "${CONFIGURATION}" >"${FETCHMAILRC}" else cat /etc/fetchmailrc_general >"${FETCHMAILRC}" @@ -27,8 +25,7 @@ function _setup_fetchmail function _setup_fetchmail_parallel { - if [[ ${FETCHMAIL_PARALLEL} -eq 1 ]] - then + if [[ ${FETCHMAIL_PARALLEL} -eq 1 ]]; then _log 'trace' 'Enabling and configuring Fetchmail parallel' mkdir /etc/fetchmailrc.d/ @@ -44,16 +41,13 @@ function _setup_fetchmail_parallel local FETCHMAILRCD='/etc/fetchmailrc.d' local DEFAULT_FILE="${FETCHMAILRCD}/defaults" - if [[ ! -r ${FETCHMAILRC} ]] - then + if [[ ! -r ${FETCHMAILRC} ]]; then _log 'warn' "File '${FETCHMAILRC}' not found" return 1 fi - if [[ ! -d ${FETCHMAILRCD} ]] - then - if ! mkdir "${FETCHMAILRCD}" - then + if [[ ! -d ${FETCHMAILRCD} ]]; then + if ! mkdir "${FETCHMAILRCD}"; then _log 'warn' "Unable to create folder '${FETCHMAILRCD}'" return 1 fi @@ -62,8 +56,7 @@ function _setup_fetchmail_parallel local COUNTER=0 SERVER=0 while read -r LINE do - if [[ ${LINE} =~ poll ]] - then + if [[ ${LINE} =~ poll ]]; then # If we read "poll" then we reached a new server definition # We need to create a new file with fetchmail defaults from # /etc/fetcmailrc @@ -71,8 +64,7 @@ function _setup_fetchmail_parallel SERVER=1 cat "${DEFAULT_FILE}" >"${FETCHMAILRCD}/fetchmail-${COUNTER}.rc" echo "${LINE}" >>"${FETCHMAILRCD}/fetchmail-${COUNTER}.rc" - elif [[ ${SERVER} -eq 0 ]] - then + elif [[ ${SERVER} -eq 0 ]]; then # We have not yet found "poll". Let's assume we are still reading # the default settings from /etc/fetchmailrc file echo "${LINE}" >>"${DEFAULT_FILE}" diff --git a/target/scripts/startup/setup.d/getmail.sh b/target/scripts/startup/setup.d/getmail.sh index c0cd4df8019..cf4547d48c9 100644 --- a/target/scripts/startup/setup.d/getmail.sh +++ b/target/scripts/startup/setup.d/getmail.sh @@ -2,8 +2,7 @@ function _setup_getmail { - if [[ ${ENABLE_GETMAIL} -eq 1 ]] - then + if [[ ${ENABLE_GETMAIL} -eq 1 ]]; then _log 'trace' 'Preparing Getmail configuration' local GETMAILRC ID CONFIGS @@ -17,8 +16,7 @@ function _setup_getmail # Add a unique `message_log` config, then append users own config to the end. for FILE in /tmp/docker-mailserver/getmail-*.cf do - if [[ -f ${FILE} ]] - then + if [[ -f ${FILE} ]]; then CONFIGS=1 ID=$(cut -d '-' -f 3 <<< "${FILE}" | cut -d '.' -f 1) local GETMAIL_CONFIG="${GETMAILRC}/getmailrc-${ID}" @@ -29,8 +27,7 @@ function _setup_getmail fi done - if [[ ${CONFIGS} -eq 1 ]] - then + if [[ ${CONFIGS} -eq 1 ]]; then cat >/etc/cron.d/getmail << EOF */${GETMAIL_POLL} * * * * root /usr/local/bin/getmail-cron EOF diff --git a/target/scripts/startup/setup.d/ldap.sh b/target/scripts/startup/setup.d/ldap.sh index fce2e309cab..36c60fad81f 100644 --- a/target/scripts/startup/setup.d/ldap.sh +++ b/target/scripts/startup/setup.d/ldap.sh @@ -8,8 +8,7 @@ function _setup_ldap for i in 'users' 'groups' 'aliases' 'domains' do local FPATH="/tmp/docker-mailserver/ldap-${i}.cf" - if [[ -f ${FPATH} ]] - then + if [[ -f ${FPATH} ]]; then cp "${FPATH}" "/etc/postfix/ldap-${i}.cf" fi done @@ -46,8 +45,7 @@ function _setup_ldap # Add protocol to DOVECOT_URIS so that we can use dovecot's "uris" option: # https://doc.dovecot.org/configuration_manual/authentication/ldap/ - if [[ ${DOVECOT_LDAP_MAPPING["DOVECOT_URIS"]} != *'://'* ]] - then + if [[ ${DOVECOT_LDAP_MAPPING["DOVECOT_URIS"]} != *'://'* ]]; then DOVECOT_LDAP_MAPPING['DOVECOT_URIS']="ldap://${DOVECOT_LDAP_MAPPING["DOVECOT_URIS"]}" fi @@ -68,22 +66,19 @@ function _setup_ldap _log 'trace' "Configuring LDAP" - if [[ -f /etc/postfix/ldap-users.cf ]] - then + if [[ -f /etc/postfix/ldap-users.cf ]]; then postconf 'virtual_mailbox_maps = ldap:/etc/postfix/ldap-users.cf' else _log 'warn' "'/etc/postfix/ldap-users.cf' not found" fi - if [[ -f /etc/postfix/ldap-domains.cf ]] - then + if [[ -f /etc/postfix/ldap-domains.cf ]]; then postconf 'virtual_mailbox_domains = /etc/postfix/vhost, ldap:/etc/postfix/ldap-domains.cf' else _log 'warn' "'/etc/postfix/ldap-domains.cf' not found" fi - if [[ -f /etc/postfix/ldap-aliases.cf ]] && [[ -f /etc/postfix/ldap-groups.cf ]] - then + if [[ -f /etc/postfix/ldap-aliases.cf ]] && [[ -f /etc/postfix/ldap-groups.cf ]]; then postconf 'virtual_alias_maps = ldap:/etc/postfix/ldap-aliases.cf, ldap:/etc/postfix/ldap-groups.cf' else _log 'warn' "'/etc/postfix/ldap-aliases.cf' and / or '/etc/postfix/ldap-groups.cf' not found" diff --git a/target/scripts/startup/setup.d/log.sh b/target/scripts/startup/setup.d/log.sh index 95c32d1400c..b87c61301ef 100644 --- a/target/scripts/startup/setup.d/log.sh +++ b/target/scripts/startup/setup.d/log.sh @@ -96,8 +96,7 @@ function _setup_logwatch LOGWATCH_FILE="/etc/cron.${LOGWATCH_INTERVAL}/logwatch" INTERVAL='--range Yesterday' - if [[ ${LOGWATCH_INTERVAL} == 'weekly' ]] - then + if [[ ${LOGWATCH_INTERVAL} == 'weekly' ]]; then INTERVAL="--range 'between -7 days and -1 days'" fi diff --git a/target/scripts/startup/setup.d/mail_state.sh b/target/scripts/startup/setup.d/mail_state.sh index 161d06e613d..aa59ac62c5b 100644 --- a/target/scripts/startup/setup.d/mail_state.sh +++ b/target/scripts/startup/setup.d/mail_state.sh @@ -8,8 +8,7 @@ function _setup_save_states STATEDIR='/var/mail-state' - if [[ ${ONE_DIR} -eq 1 ]] && [[ -d ${STATEDIR} ]] - then + if [[ ${ONE_DIR} -eq 1 ]] && [[ -d ${STATEDIR} ]]; then _log 'debug' "Consolidating all state onto ${STATEDIR}" # Always enabled features: @@ -42,13 +41,11 @@ function _setup_save_states DESTDIR="${DEST%/*}" mkdir -p "${DESTDIR}" - if [[ -f ${DEST} ]] - then + if [[ -f ${DEST} ]]; then _log 'trace' "Destination ${DEST} exists, linking ${SERVICEFILE} to it" # Original content from image no longer relevant, remove it: rm -f "${SERVICEFILE}" - elif [[ -f "${SERVICEFILE}" ]] - then + elif [[ -f "${SERVICEFILE}" ]]; then _log 'trace' "Moving ${SERVICEFILE} to ${DEST}" # Empty volume was mounted, or new content from enabling a feature ENV: mv "${SERVICEFILE}" "${DEST}" @@ -66,13 +63,11 @@ function _setup_save_states # If relevant content is found in /var/mail-state (presumably a volume mount), # use it instead. Otherwise copy over any missing directories checked. - if [[ -d ${DEST} ]] - then + if [[ -d ${DEST} ]]; then _log 'trace' "Destination ${DEST} exists, linking ${SERVICEDIR} to it" # Original content from image no longer relevant, remove it: rm -rf "${SERVICEDIR}" - elif [[ -d ${SERVICEDIR} ]] - then + elif [[ -d ${SERVICEDIR} ]]; then _log 'trace' "Moving contents of ${SERVICEDIR} to ${DEST}" # Empty volume was mounted, or new content from enabling a feature ENV: mv "${SERVICEDIR}" "${DEST}" @@ -117,8 +112,7 @@ function _setup_save_states # Ref: https://github.com/docker-mailserver/docker-mailserver/pull/3149#issuecomment-1454981309 chmod 1730 "${STATEDIR}/spool-postfix/maildrop" chmod 2710 "${STATEDIR}/spool-postfix/public" - elif [[ ${ONE_DIR} -eq 1 ]] - then + elif [[ ${ONE_DIR} -eq 1 ]]; then _log 'warn' "'ONE_DIR=1' but no volume was mounted to '${STATEDIR}'" else _log 'debug' 'Not consolidating state (because it has been disabled)' diff --git a/target/scripts/startup/setup.d/networking.sh b/target/scripts/startup/setup.d/networking.sh index 13751766d1e..d8669ac3894 100644 --- a/target/scripts/startup/setup.d/networking.sh +++ b/target/scripts/startup/setup.d/networking.sh @@ -19,8 +19,7 @@ function _setup_docker_permit grep 'inet ' | sed 's|[^0-9\.\/]*||g' | cut -d '/' -f 1) CONTAINER_NETWORK=$(echo "${CONTAINER_IP}" | cut -d '.' -f1-2).0.0 - if [[ -z ${CONTAINER_IP} ]] - then + if [[ -z ${CONTAINER_IP} ]]; then _log 'error' 'Detecting the container IP address failed' _dms_panic__misconfigured 'NETWORK_INTERFACE' 'Network Setup [docker_permit]' fi diff --git a/target/scripts/startup/setup.d/postfix.sh b/target/scripts/startup/setup.d/postfix.sh index 34b35e6edb0..3ae6741e43c 100644 --- a/target/scripts/startup/setup.d/postfix.sh +++ b/target/scripts/startup/setup.d/postfix.sh @@ -15,8 +15,7 @@ function _setup_postfix_early postconf "myhostname = ${HOSTNAME}" postconf "mydomain = ${DOMAINNAME}" - if [[ ${POSTFIX_INET_PROTOCOLS} != 'all' ]] - then + if [[ ${POSTFIX_INET_PROTOCOLS} != 'all' ]]; then __postfix__log 'trace' 'Setting up POSTFIX_INET_PROTOCOLS option' postconf "inet_protocols = ${POSTFIX_INET_PROTOCOLS}" fi @@ -25,16 +24,14 @@ function _setup_postfix_early postconf 'smtputf8_enable = no' __postfix__log 'trace' "Configuring SASLauthd" - if [[ ${ENABLE_SASLAUTHD} -eq 1 ]] && [[ ! -f /etc/postfix/sasl/smtpd.conf ]] - then + if [[ ${ENABLE_SASLAUTHD} -eq 1 ]] && [[ ! -f /etc/postfix/sasl/smtpd.conf ]]; then cat >/etc/postfix/sasl/smtpd.conf << EOF pwcheck_method: saslauthd mech_list: plain login EOF fi - if [[ ${ENABLE_SASLAUTHD} -eq 0 ]] && [[ ${SMTP_ONLY} -eq 1 ]] - then + if [[ ${ENABLE_SASLAUTHD} -eq 0 ]] && [[ ${SMTP_ONLY} -eq 1 ]]; then sed -i -E \ 's|^smtpd_sasl_auth_enable =.*|smtpd_sasl_auth_enable = no|g' \ /etc/postfix/main.cf @@ -61,8 +58,7 @@ EOF __postfix__log 'trace' "Configuring virtual mailbox size limit to '${POSTFIX_MAILBOX_SIZE_LIMIT}'" postconf "virtual_mailbox_limit = ${POSTFIX_MAILBOX_SIZE_LIMIT}" - if [[ ${POSTFIX_REJECT_UNKNOWN_CLIENT_HOSTNAME} -eq 1 ]] - then + if [[ ${POSTFIX_REJECT_UNKNOWN_CLIENT_HOSTNAME} -eq 1 ]]; then __postfix__log 'trace' 'Enabling reject_unknown_client_hostname to dms_smtpd_sender_restrictions' sedfile -i -E \ 's|^(dms_smtpd_sender_restrictions = .*)|\1, reject_unknown_client_hostname|' \ @@ -75,21 +71,18 @@ function _setup_postfix_late _log 'debug' 'Configuring Postfix (late setup)' __postfix__log 'trace' 'Configuring user access' - if [[ -f /tmp/docker-mailserver/postfix-send-access.cf ]] - then + if [[ -f /tmp/docker-mailserver/postfix-send-access.cf ]]; then sed -i -E 's|(smtpd_sender_restrictions =)|\1 check_sender_access texthash:/tmp/docker-mailserver/postfix-send-access.cf,|' /etc/postfix/main.cf fi - if [[ -f /tmp/docker-mailserver/postfix-receive-access.cf ]] - then + if [[ -f /tmp/docker-mailserver/postfix-receive-access.cf ]]; then sed -i -E 's|(smtpd_recipient_restrictions =)|\1 check_recipient_access texthash:/tmp/docker-mailserver/postfix-receive-access.cf,|' /etc/postfix/main.cf fi __postfix__log 'trace' 'Configuring relay host' _setup_relayhost - if [[ -n ${POSTFIX_DAGENT} ]] - then + if [[ -n ${POSTFIX_DAGENT} ]]; then __postfix__log 'trace' "Changing virtual transport to '${POSTFIX_DAGENT}'" # Default value in main.cf should be 'lmtp:unix:/var/run/dovecot/lmtp' postconf "virtual_transport = ${POSTFIX_DAGENT}" @@ -102,8 +95,7 @@ function __postfix__setup_override_configuration { __postfix__log 'debug' 'Overriding / adjusting configuration with user-supplied values' - if [[ -f /tmp/docker-mailserver/postfix-main.cf ]] - then + if [[ -f /tmp/docker-mailserver/postfix-main.cf ]]; then cat /tmp/docker-mailserver/postfix-main.cf >>/etc/postfix/main.cf _adjust_mtime_for_postfix_maincf @@ -117,12 +109,10 @@ function __postfix__setup_override_configuration __postfix__log 'trace' "No extra Postfix settings loaded because optional '/tmp/docker-mailserver/postfix-main.cf' was not provided" fi - if [[ -f /tmp/docker-mailserver/postfix-master.cf ]] - then + if [[ -f /tmp/docker-mailserver/postfix-master.cf ]]; then while read -r LINE do - if [[ ${LINE} =~ ^[0-9a-z] ]] - then + if [[ ${LINE} =~ ^[0-9a-z] ]]; then postconf -P "${LINE}" fi done < /tmp/docker-mailserver/postfix-master.cf @@ -155,21 +145,18 @@ function _setup_SRS POSTSRSD_SECRET_FILE='/etc/postsrsd.secret' - if [[ -n ${SRS_SECRET} ]] - then + if [[ -n ${SRS_SECRET} ]]; then ( umask 0077 echo "${SRS_SECRET}" | tr ',' '\n' >"${POSTSRSD_SECRET_FILE}" ) else - if [[ ! -f ${POSTSRSD_SECRET_FILE} ]] - then + if [[ ! -f ${POSTSRSD_SECRET_FILE} ]]; then __generate_secret "${POSTSRSD_SECRET_FILE}" fi fi - if [[ -n ${SRS_EXCLUDE_DOMAINS} ]] - then + if [[ -n ${SRS_EXCLUDE_DOMAINS} ]]; then sedfile -i -E \ "s|^#?(SRS_EXCLUDE_DOMAINS=).*|\1${SRS_EXCLUDE_DOMAINS}|" \ /etc/default/postsrsd diff --git a/target/scripts/startup/setup.d/saslauthd.sh b/target/scripts/startup/setup.d/saslauthd.sh index 1a6488d0676..bbe3358cf1a 100644 --- a/target/scripts/startup/setup.d/saslauthd.sh +++ b/target/scripts/startup/setup.d/saslauthd.sh @@ -5,8 +5,7 @@ function _setup_saslauthd { _log 'debug' 'Setting up SASLAUTHD' - if [[ ! -f /etc/saslauthd.conf ]] - then + if [[ ! -f /etc/saslauthd.conf ]]; then _log 'trace' 'Creating /etc/saslauthd.conf' cat > /etc/saslauthd.conf << EOF ldap_servers: ${SASLAUTHD_LDAP_SERVER} diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index 76fa8f67945..863a3afc5ec 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -25,8 +25,7 @@ function _setup_security_stack function __setup__security__postgrey { - if [[ ${ENABLE_POSTGREY} -eq 1 ]] - then + if [[ ${ENABLE_POSTGREY} -eq 1 ]]; then _log 'debug' 'Enabling and configuring Postgrey' sedfile -i -E \ @@ -37,18 +36,15 @@ function __setup__security__postgrey "s|\"--inet=127.0.0.1:10023\"|\"--inet=127.0.0.1:10023 --delay=${POSTGREY_DELAY} --max-age=${POSTGREY_MAX_AGE} --auto-whitelist-clients=${POSTGREY_AUTO_WHITELIST_CLIENTS}\"|" \ /etc/default/postgrey - if ! grep -i 'POSTGREY_TEXT' /etc/default/postgrey - then + if ! grep -i 'POSTGREY_TEXT' /etc/default/postgrey; then printf 'POSTGREY_TEXT=\"%s\"\n\n' "${POSTGREY_TEXT}" >>/etc/default/postgrey fi - if [[ -f /tmp/docker-mailserver/whitelist_clients.local ]] - then + if [[ -f /tmp/docker-mailserver/whitelist_clients.local ]]; then cp -f /tmp/docker-mailserver/whitelist_clients.local /etc/postgrey/whitelist_clients.local fi - if [[ -f /tmp/docker-mailserver/whitelist_recipients ]] - then + if [[ -f /tmp/docker-mailserver/whitelist_recipients ]]; then cp -f /tmp/docker-mailserver/whitelist_recipients /etc/postgrey/whitelist_recipients fi else @@ -64,8 +60,7 @@ function __setup__security__postscreen -e "s|postscreen_greet_action = enforce|postscreen_greet_action = ${POSTSCREEN_ACTION}|" \ -e "s|postscreen_bare_newline_action = enforce|postscreen_bare_newline_action = ${POSTSCREEN_ACTION}|" /etc/postfix/main.cf - if [[ ${ENABLE_DNSBL} -eq 0 ]] - then + if [[ ${ENABLE_DNSBL} -eq 0 ]]; then _log 'debug' 'Disabling Postscreen DNSBLs' postconf 'postscreen_dnsbl_action = ignore' postconf 'postscreen_dnsbl_sites = ' @@ -76,8 +71,7 @@ function __setup__security__postscreen function __setup__security__spamassassin { - if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] - then + if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]]; then _log 'debug' 'Enabling and configuring SpamAssassin' # shellcheck disable=SC2016 @@ -94,8 +88,7 @@ function __setup__security__spamassassin 's|invoke-rc.d spamassassin reload|/etc/init\.d/spamassassin reload|g' \ /etc/cron.daily/spamassassin - if [[ ${SA_SPAM_SUBJECT} == 'undef' ]] - then + if [[ ${SA_SPAM_SUBJECT} == 'undef' ]]; then # shellcheck disable=SC2016 sed -i -r 's|^\$sa_spam_subject_tag (.*);|\$sa_spam_subject_tag = undef;|g' /etc/amavis/conf.d/20-debian_defaults else @@ -104,27 +97,23 @@ function __setup__security__spamassassin fi # activate short circuits when SA BAYES is certain it has spam or ham. - if [[ ${SA_SHORTCIRCUIT_BAYES_SPAM} -eq 1 ]] - then + if [[ ${SA_SHORTCIRCUIT_BAYES_SPAM} -eq 1 ]]; then # automatically activate the Shortcircuit Plugin sed -i -r 's|^# loadplugin Mail::SpamAssassin::Plugin::Shortcircuit|loadplugin Mail::SpamAssassin::Plugin::Shortcircuit|g' /etc/spamassassin/v320.pre sed -i -r 's|^# shortcircuit BAYES_99|shortcircuit BAYES_99|g' /etc/spamassassin/local.cf fi - if [[ ${SA_SHORTCIRCUIT_BAYES_HAM} -eq 1 ]] - then + if [[ ${SA_SHORTCIRCUIT_BAYES_HAM} -eq 1 ]]; then # automatically activate the Shortcircuit Plugin sed -i -r 's|^# loadplugin Mail::SpamAssassin::Plugin::Shortcircuit|loadplugin Mail::SpamAssassin::Plugin::Shortcircuit|g' /etc/spamassassin/v320.pre sed -i -r 's|^# shortcircuit BAYES_00|shortcircuit BAYES_00|g' /etc/spamassassin/local.cf fi - if [[ -e /tmp/docker-mailserver/spamassassin-rules.cf ]] - then + if [[ -e /tmp/docker-mailserver/spamassassin-rules.cf ]]; then cp /tmp/docker-mailserver/spamassassin-rules.cf /etc/spamassassin/ fi - if [[ ${SPAMASSASSIN_SPAM_TO_INBOX} -eq 1 ]] - then + if [[ ${SPAMASSASSIN_SPAM_TO_INBOX} -eq 1 ]]; then _log 'trace' 'Configuring Spamassassin/Amavis to send SPAM to inbox' _log 'debug' 'SPAM_TO_INBOX=1 is set. SA_KILL will be ignored.' @@ -137,8 +126,7 @@ function __setup__security__spamassassin sed -i "s|\$final_bad_header_destiny.*=.*$|\$final_bad_header_destiny = D_BOUNCE;|g" /etc/amavis/conf.d/49-docker-mailserver fi - if [[ ${ENABLE_SPAMASSASSIN_KAM} -eq 1 ]] - then + if [[ ${ENABLE_SPAMASSASSIN_KAM} -eq 1 ]]; then _log 'trace' 'Configuring Spamassassin KAM' local SPAMASSASSIN_KAM_CRON_FILE=/etc/cron.daily/spamassassin_kam @@ -151,8 +139,7 @@ RESULT=$(sa-update --gpgkey 24C063D8 --channel kam.sa-channels.mcgrail.com 2>&1) EXIT_CODE=${?} # see https://spamassassin.apache.org/full/3.1.x/doc/sa-update.html#exit_codes -if [[ ${EXIT_CODE} -ge 4 ]] -then +if [[ ${EXIT_CODE} -ge 4 ]]; then echo -e "Updating SpamAssassin KAM failed:\n${RESULT}\n" >&2 exit 1 fi @@ -172,8 +159,7 @@ EOF function __setup__security__clamav { - if [[ ${ENABLE_CLAMAV} -eq 1 ]] - then + if [[ ${ENABLE_CLAMAV} -eq 1 ]]; then _log 'debug' 'Enabling and configuring ClamAV' local FILE @@ -184,19 +170,16 @@ function __setup__security__clamav chmod 640 "${FILE}" done - if [[ ${CLAMAV_MESSAGE_SIZE_LIMIT} != '25M' ]] - then + if [[ ${CLAMAV_MESSAGE_SIZE_LIMIT} != '25M' ]]; then _log 'trace' "Setting ClamAV message scan size limit to '${CLAMAV_MESSAGE_SIZE_LIMIT}'" # do a short sanity check: ClamAV does not support setting a maximum size greater than 4000M (at all) - if [[ $(numfmt --from=si "${CLAMAV_MESSAGE_SIZE_LIMIT}") -gt $(numfmt --from=si 4000M) ]] - then + if [[ $(numfmt --from=si "${CLAMAV_MESSAGE_SIZE_LIMIT}") -gt $(numfmt --from=si 4000M) ]]; then _log 'warn' "You set 'CLAMAV_MESSAGE_SIZE_LIMIT' to a value larger than 4 Gigabyte, but the maximum value is 4000M for this value - you should correct your configuration" fi # For more details, see # https://github.com/docker-mailserver/docker-mailserver/pull/3341 - if [[ $(numfmt --from=si "${CLAMAV_MESSAGE_SIZE_LIMIT}") -ge $(numfmt --from=iec 2G) ]] - then + if [[ $(numfmt --from=si "${CLAMAV_MESSAGE_SIZE_LIMIT}") -ge $(numfmt --from=iec 2G) ]]; then _log 'warn' "You set 'CLAMAV_MESSAGE_SIZE_LIMIT' to a value larger than 2 Gibibyte but ClamAV does not scan files larger or equal to 2 Gibibyte" fi @@ -216,22 +199,18 @@ function __setup__security__clamav function __setup__security__fail2ban { - if [[ ${ENABLE_FAIL2BAN} -eq 1 ]] - then + if [[ ${ENABLE_FAIL2BAN} -eq 1 ]]; then _log 'debug' 'Enabling and configuring Fail2Ban' - if [[ -e /tmp/docker-mailserver/fail2ban-fail2ban.cf ]] - then + if [[ -e /tmp/docker-mailserver/fail2ban-fail2ban.cf ]]; then cp /tmp/docker-mailserver/fail2ban-fail2ban.cf /etc/fail2ban/fail2ban.local fi - if [[ -e /tmp/docker-mailserver/fail2ban-jail.cf ]] - then + if [[ -e /tmp/docker-mailserver/fail2ban-jail.cf ]]; then cp /tmp/docker-mailserver/fail2ban-jail.cf /etc/fail2ban/jail.d/user-jail.local fi - if [[ ${FAIL2BAN_BLOCKTYPE} != 'reject' ]] - then + if [[ ${FAIL2BAN_BLOCKTYPE} != 'reject' ]]; then echo -e '[Init]\nblocktype = drop' >/etc/fail2ban/action.d/nftables-common.local fi @@ -244,11 +223,9 @@ function __setup__security__fail2ban function __setup__security__amavis { - if [[ ${ENABLE_AMAVIS} -eq 1 ]] - then + if [[ ${ENABLE_AMAVIS} -eq 1 ]]; then _log 'debug' 'Configuring Amavis' - if [[ -f /tmp/docker-mailserver/amavis.cf ]] - then + if [[ -f /tmp/docker-mailserver/amavis.cf ]]; then cp /tmp/docker-mailserver/amavis.cf /etc/amavis/conf.d/50-user fi @@ -269,13 +246,11 @@ function __setup__security__amavis mv /etc/cron.d/amavisd-new /etc/cron.d/amavisd-new.disabled chmod 0 /etc/cron.d/amavisd-new.disabled - if [[ ${ENABLE_CLAMAV} -eq 1 ]] && [[ ${ENABLE_RSPAMD} -eq 0 ]] - then + if [[ ${ENABLE_CLAMAV} -eq 1 ]] && [[ ${ENABLE_RSPAMD} -eq 0 ]]; then _log 'warn' 'ClamAV will not work when Amavis & rspamd are disabled. Enable either Amavis or rspamd to fix it.' fi - if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] - then + if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]]; then _log 'warn' 'Spamassassin will not work when Amavis is disabled. Enable Amavis to fix it.' fi fi @@ -284,8 +259,7 @@ function __setup__security__amavis # We can use Sieve to move spam emails to the "Junk" folder. function _setup_spam_to_junk { - if [[ ${MOVE_SPAM_TO_JUNK} -eq 1 ]] - then + if [[ ${MOVE_SPAM_TO_JUNK} -eq 1 ]]; then _log 'debug' 'Spam emails will be moved to the Junk folder' cat >/usr/lib/dovecot/sieve-global/after/spam_to_junk.sieve << EOF require ["fileinto","mailbox"]; @@ -298,8 +272,7 @@ EOF sievec /usr/lib/dovecot/sieve-global/after/spam_to_junk.sieve chown dovecot:root /usr/lib/dovecot/sieve-global/after/spam_to_junk.{sieve,svbin} - if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] && [[ ${SPAMASSASSIN_SPAM_TO_INBOX} -eq 0 ]] - then + if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] && [[ ${SPAMASSASSIN_SPAM_TO_INBOX} -eq 0 ]]; then _log 'warning' "'SPAMASSASSIN_SPAM_TO_INBOX=0' but it is required to be 1 for 'MOVE_SPAM_TO_JUNK=1' to work" fi else diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index c1c2712a21f..6c0f39034ea 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -3,8 +3,7 @@ # Function called during global setup to handle the complete setup of Rspamd. function _setup_rspamd { - if _env_var_expect_zero_or_one 'ENABLE_RSPAMD' && [[ ${ENABLE_RSPAMD} -eq 1 ]] - then + if _env_var_expect_zero_or_one 'ENABLE_RSPAMD' && [[ ${ENABLE_RSPAMD} -eq 1 ]]; then _log 'debug' 'Enabling and configuring Rspamd' __rspamd__log 'trace' '---------- Setup started ----------' @@ -44,8 +43,7 @@ function __rspamd__helper__enable_disable_module local LOCAL_OR_OVERRIDE=${3:-local} local MESSAGE='Enabling' - if [[ ! ${ENABLE_MODULE} =~ ^(true|false)$ ]] - then + if [[ ! ${ENABLE_MODULE} =~ ^(true|false)$ ]]; then __rspamd__log 'warn' "__rspamd__helper__enable_disable_module got non-boolean argument for deciding whether module should be enabled or not" return 1 fi @@ -75,39 +73,32 @@ function __rspamd__run_early_setup_and_checks mkdir -p /var/lib/rspamd/ : >/var/lib/rspamd/stats.ucl - if [[ -d ${RSPAMD_DMS_OVERRIDE_D} ]] - then + if [[ -d ${RSPAMD_DMS_OVERRIDE_D} ]]; then __rspamd__log 'debug' "Found directory '${RSPAMD_DMS_OVERRIDE_D}' - linking it to '${RSPAMD_OVERRIDE_D}'" - if rmdir "${RSPAMD_OVERRIDE_D}" 2>/dev/null - then + if rmdir "${RSPAMD_OVERRIDE_D}" 2>/dev/null; then ln -s "${RSPAMD_DMS_OVERRIDE_D}" "${RSPAMD_OVERRIDE_D}" else __rspamd__log 'warn' "Could not remove '${RSPAMD_OVERRIDE_D}' (not empty? not a directory?; did you restart properly?) - not linking '${RSPAMD_DMS_OVERRIDE_D}'" fi fi - if [[ ${ENABLE_AMAVIS} -eq 1 ]] || [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] - then + if [[ ${ENABLE_AMAVIS} -eq 1 ]] || [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]]; then __rspamd__log 'warn' 'Running Amavis/SA & Rspamd at the same time is discouraged' fi - if [[ ${ENABLE_OPENDKIM} -eq 1 ]] - then + if [[ ${ENABLE_OPENDKIM} -eq 1 ]]; then __rspamd__log 'warn' 'Running OpenDKIM & Rspamd at the same time is discouraged - we recommend Rspamd for DKIM checks (enabled with Rspamd by default) & signing' fi - if [[ ${ENABLE_OPENDMARC} -eq 1 ]] - then + if [[ ${ENABLE_OPENDMARC} -eq 1 ]]; then __rspamd__log 'warn' 'Running OpenDMARC & Rspamd at the same time is discouraged - we recommend Rspamd for DMARC checks (enabled with Rspamd by default)' fi - if [[ ${ENABLE_POLICYD_SPF} -eq 1 ]] - then + if [[ ${ENABLE_POLICYD_SPF} -eq 1 ]]; then __rspamd__log 'warn' 'Running policyd-spf & Rspamd at the same time is discouraged - we recommend Rspamd for SPF checks (enabled with Rspamd by default)' fi - if [[ ${ENABLE_POSTGREY} -eq 1 ]] && [[ ${RSPAMD_GREYLISTING} -eq 1 ]] - then + if [[ ${ENABLE_POSTGREY} -eq 1 ]] && [[ ${RSPAMD_GREYLISTING} -eq 1 ]]; then __rspamd__log 'warn' 'Running Postgrey & Rspamd at the same time is discouraged - we recommend Rspamd for greylisting' fi } @@ -116,8 +107,7 @@ function __rspamd__run_early_setup_and_checks # supply a configuration for our local Redis instance which is started later. function __rspamd__setup_redis { - if _env_var_expect_zero_or_one 'ENABLE_RSPAMD_REDIS' && [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] - then + if _env_var_expect_zero_or_one 'ENABLE_RSPAMD_REDIS' && [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]]; then __rspamd__log 'debug' 'Internal Redis is enabled, adding configuration' cat >"${RSPAMD_LOCAL_D}/redis.conf" << "EOF" # documentation: https://rspamd.com/doc/configuration/redis.html @@ -158,15 +148,13 @@ function __rspamd__setup_postfix # If ClamAV is enabled, we will integrate it into Rspamd. function __rspamd__setup_clamav { - if _env_var_expect_zero_or_one 'ENABLE_CLAMAV' && [[ ${ENABLE_CLAMAV} -eq 1 ]] - then + if _env_var_expect_zero_or_one 'ENABLE_CLAMAV' && [[ ${ENABLE_CLAMAV} -eq 1 ]]; then __rspamd__log 'debug' 'Enabling ClamAV integration' sedfile -i -E 's|^(enabled).*|\1 = true;|g' "${RSPAMD_LOCAL_D}/antivirus.conf" # Rspamd uses ClamAV's UNIX socket, and to be able to read it, it must be in the same group usermod -a -G clamav _rspamd - if [[ ${CLAMAV_MESSAGE_SIZE_LIMIT} != '25M' ]] - then + if [[ ${CLAMAV_MESSAGE_SIZE_LIMIT} != '25M' ]]; then local SIZE_IN_BYTES SIZE_IN_BYTES=$(numfmt --from=si "${CLAMAV_MESSAGE_SIZE_LIMIT}") __rspamd__log 'trace' "Adjusting maximum size for ClamAV to ${SIZE_IN_BYTES} bytes (${CLAMAV_MESSAGE_SIZE_LIMIT})" @@ -211,8 +199,7 @@ function __rspamd__setup_default_modules # from or to the "Junk" folder, and learning them as ham or spam. function __rspamd__setup_learning { - if _env_var_expect_zero_or_one 'RSPAMD_LEARN' && [[ ${RSPAMD_LEARN} -eq 1 ]] - then + if _env_var_expect_zero_or_one 'RSPAMD_LEARN' && [[ ${RSPAMD_LEARN} -eq 1 ]]; then __rspamd__log 'debug' 'Setting up intelligent learning of spam and ham' local SIEVE_PIPE_BIN_DIR='/usr/lib/dovecot/sieve-pipe' @@ -256,8 +243,7 @@ EOF # https://rspamd.com/doc/modules/greylisting.html). function __rspamd__setup_greylisting { - if _env_var_expect_zero_or_one 'RSPAMD_GREYLISTING' && [[ ${RSPAMD_GREYLISTING} -eq 1 ]] - then + if _env_var_expect_zero_or_one 'RSPAMD_GREYLISTING' && [[ ${RSPAMD_GREYLISTING} -eq 1 ]]; then __rspamd__log 'debug' 'Enabling greylisting' sedfile -i -E "s|(enabled =).*|\1 true;|g" "${RSPAMD_LOCAL_D}/greylist.conf" else @@ -272,12 +258,10 @@ function __rspamd__setup_greylisting function __rspamd__setup_hfilter_group { local MODULE_FILE="${RSPAMD_LOCAL_D}/hfilter_group.conf" - if _env_var_expect_zero_or_one 'RSPAMD_HFILTER' && [[ ${RSPAMD_HFILTER} -eq 1 ]] - then + if _env_var_expect_zero_or_one 'RSPAMD_HFILTER' && [[ ${RSPAMD_HFILTER} -eq 1 ]]; then __rspamd__log 'debug' 'Hfilter (group) module is enabled' # Check if we received a number first - if _env_var_expect_integer 'RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE' && [[ ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE} -ne 6 ]] - then + if _env_var_expect_integer 'RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE' && [[ ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE} -ne 6 ]]; then __rspamd__log 'trace' "Adjusting score for 'HFILTER_HOSTNAME_UNKNOWN' in Hfilter group module to ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE}" sed -i -E \ "s|(.*score =).*(# __TAG__HFILTER_HOSTNAME_UNKNOWN)|\1 ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE}; \2|g" \ @@ -321,8 +305,7 @@ function __rspamd__handle_user_modules_adjustments local FILE="${RSPAMD_OVERRIDE_D}/${MODULE_FILE}" [[ -f ${FILE} ]] || touch "${FILE}" - if grep -q -E "${OPTION}.*=.*" "${FILE}" - then + if grep -q -E "${OPTION}.*=.*" "${FILE}"; then __rspamd__log 'trace' "Overwriting option '${OPTION}' with value '${VALUE}' for ${MODULE_LOG_NAME}" sed -i -E "s|([[:space:]]*${OPTION}).*|\1 = ${VALUE};|g" "${FILE}" else @@ -336,15 +319,13 @@ function __rspamd__handle_user_modules_adjustments # We check for usage of the previous location of the commands file. # This can be removed after the release of v14.0.0. - if [[ -f ${RSPAMD_CUSTOM_COMMANDS_FILE_OLD} ]] - then + if [[ -f ${RSPAMD_CUSTOM_COMMANDS_FILE_OLD} ]]; then __rspamd__log 'warn' "Detected usage of old file location for modules adjustment ('${RSPAMD_CUSTOM_COMMANDS_FILE_OLD}') - please use the new location ('${RSPAMD_CUSTOM_COMMANDS_FILE}')" __rspamd__log 'warn' "Using old file location now (deprecated) - this will prevent startup in v13.0.0" RSPAMD_CUSTOM_COMMANDS_FILE=${RSPAMD_CUSTOM_COMMANDS_FILE_OLD} fi - if [[ -f "${RSPAMD_CUSTOM_COMMANDS_FILE}" ]] - then + if [[ -f "${RSPAMD_CUSTOM_COMMANDS_FILE}" ]]; then __rspamd__log 'debug' "Found file '${RSPAMD_CUSTOM_COMMANDS_FILE}' - parsing and applying it" while read -r COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3 diff --git a/target/scripts/startup/setup.d/security/spoofing.sh b/target/scripts/startup/setup.d/security/spoofing.sh index b6b4f63f11d..4a569ff4b89 100644 --- a/target/scripts/startup/setup.d/security/spoofing.sh +++ b/target/scripts/startup/setup.d/security/spoofing.sh @@ -2,21 +2,17 @@ function _setup_spoof_protection { - if [[ ${SPOOF_PROTECTION} -eq 1 ]] - then + if [[ ${SPOOF_PROTECTION} -eq 1 ]]; then _log 'trace' 'Enabling and configuring spoof protection' - if [[ ${ACCOUNT_PROVISIONER} == 'LDAP' ]] - then - if [[ -z ${LDAP_QUERY_FILTER_SENDERS} ]] - then + if [[ ${ACCOUNT_PROVISIONER} == 'LDAP' ]]; then + if [[ -z ${LDAP_QUERY_FILTER_SENDERS} ]]; then postconf 'smtpd_sender_login_maps = ldap:/etc/postfix/ldap-users.cf ldap:/etc/postfix/ldap-aliases.cf ldap:/etc/postfix/ldap-groups.cf' else postconf 'smtpd_sender_login_maps = ldap:/etc/postfix/ldap-senders.cf' fi else - if [[ -f /etc/postfix/regexp ]] - then + if [[ -f /etc/postfix/regexp ]]; then postconf 'smtpd_sender_login_maps = unionmap:{ texthash:/etc/postfix/virtual, hash:/etc/aliases, pcre:/etc/postfix/maps/sender_login_maps.pcre, pcre:/etc/postfix/regexp }' else postconf 'smtpd_sender_login_maps = texthash:/etc/postfix/virtual, hash:/etc/aliases, pcre:/etc/postfix/maps/sender_login_maps.pcre' diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index f5e66a99b65..f180c7a4e9c 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -15,8 +15,7 @@ function _early_variables_setup # completely with a single version. function __environment_variables_backwards_compatibility { - if [[ ${ENABLE_LDAP:-0} -eq 1 ]] - then + if [[ ${ENABLE_LDAP:-0} -eq 1 ]]; then _log 'warn' "'ENABLE_LDAP=1' is deprecated (and will be removed in v13.0.0) => use 'ACCOUNT_PROVISIONER=LDAP' instead" ACCOUNT_PROVISIONER='LDAP' fi @@ -165,8 +164,7 @@ function _environment_variables_saslauthd # SASL ENV for configuring an LDAP specific # `saslauthd.conf` via `setup-stack.sh:_setup_sasulauthd()` - if [[ ${ACCOUNT_PROVISIONER} == 'LDAP' ]] - then + if [[ ${ACCOUNT_PROVISIONER} == 'LDAP' ]]; then _log 'trace' 'Setting SASLSAUTH-LDAP variables nnow' VARS[SASLAUTHD_LDAP_AUTH_METHOD]="${SASLAUTHD_LDAP_AUTH_METHOD:=bind}" @@ -179,32 +177,28 @@ function _environment_variables_saslauthd VARS[SASLAUTHD_LDAP_START_TLS]="${SASLAUTHD_LDAP_START_TLS:=no}" VARS[SASLAUTHD_LDAP_TLS_CHECK_PEER]="${SASLAUTHD_LDAP_TLS_CHECK_PEER:=no}" - if [[ -z ${SASLAUTHD_LDAP_TLS_CACERT_FILE} ]] - then + if [[ -z ${SASLAUTHD_LDAP_TLS_CACERT_FILE} ]]; then SASLAUTHD_LDAP_TLS_CACERT_FILE='' else SASLAUTHD_LDAP_TLS_CACERT_FILE="ldap_tls_cacert_file: ${SASLAUTHD_LDAP_TLS_CACERT_FILE}" fi VARS[SASLAUTHD_LDAP_TLS_CACERT_FILE]="${SASLAUTHD_LDAP_TLS_CACERT_FILE}" - if [[ -z ${SASLAUTHD_LDAP_TLS_CACERT_DIR} ]] - then + if [[ -z ${SASLAUTHD_LDAP_TLS_CACERT_DIR} ]]; then SASLAUTHD_LDAP_TLS_CACERT_DIR='' else SASLAUTHD_LDAP_TLS_CACERT_DIR="ldap_tls_cacert_dir: ${SASLAUTHD_LDAP_TLS_CACERT_DIR}" fi VARS[SASLAUTHD_LDAP_TLS_CACERT_DIR]="${SASLAUTHD_LDAP_TLS_CACERT_DIR}" - if [[ -z ${SASLAUTHD_LDAP_PASSWORD_ATTR} ]] - then + if [[ -z ${SASLAUTHD_LDAP_PASSWORD_ATTR} ]]; then SASLAUTHD_LDAP_PASSWORD_ATTR='' else SASLAUTHD_LDAP_PASSWORD_ATTR="ldap_password_attr: ${SASLAUTHD_LDAP_PASSWORD_ATTR}" fi VARS[SASLAUTHD_LDAP_PASSWORD_ATTR]="${SASLAUTHD_LDAP_PASSWORD_ATTR}" - if [[ -z ${SASLAUTHD_LDAP_MECH} ]] - then + if [[ -z ${SASLAUTHD_LDAP_MECH} ]]; then SASLAUTHD_LDAP_MECH='' else SASLAUTHD_LDAP_MECH="ldap_mech: ${SASLAUTHD_LDAP_MECH}" diff --git a/target/scripts/update-check.sh b/target/scripts/update-check.sh index bdcc51b2977..a542bdbb1dd 100755 --- a/target/scripts/update-check.sh +++ b/target/scripts/update-check.sh @@ -9,8 +9,7 @@ CHANGELOG_URL='https://github.com/docker-mailserver/docker-mailserver/blob/maste # check for correct syntax # number + suffix. suffix must be 's' for seconds, 'm' for minutes, 'h' for hours or 'd' for days. -if [[ ! ${UPDATE_CHECK_INTERVAL} =~ ^[0-9]+[smhd]{1}$ ]] -then +if [[ ! ${UPDATE_CHECK_INTERVAL} =~ ^[0-9]+[smhd]{1}$ ]]; then _log_with_date 'warn' "Invalid 'UPDATE_CHECK_INTERVAL' value '${UPDATE_CHECK_INTERVAL}'" _log_with_date 'warn' 'Falling back to daily update checks' UPDATE_CHECK_INTERVAL='1d' @@ -22,13 +21,11 @@ do LATEST=$(curl -Lsf "${VERSION_URL}") # did we get a valid response? - if [[ ${LATEST} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] - then + if [[ ${LATEST} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then _log_with_date 'debug' 'Remote version information fetched' # compare versions - if dpkg --compare-versions "${VERSION}" lt "${LATEST}" - then + if dpkg --compare-versions "${VERSION}" lt "${LATEST}"; then # send mail notification to postmaster read -r -d '' MAIL << EOF Hello ${POSTMASTER_ADDRESS}! diff --git a/test/helper/common.bash b/test/helper/common.bash index 6920828b09e..46c0b99733b 100644 --- a/test/helper/common.bash +++ b/test/helper/common.bash @@ -52,12 +52,10 @@ __load_bats_helper # # This function is internal and should not be used in tests. function __handle_container_name() { - if [[ -n ${1:-} ]] && [[ ${1:-} =~ ^dms-test_ ]] - then + if [[ -n ${1:-} ]] && [[ ${1:-} =~ ^dms-test_ ]]; then printf '%s' "${1}" return 0 - elif [[ -n ${CONTAINER_NAME+set} ]] - then + elif [[ -n ${CONTAINER_NAME+set} ]]; then printf '%s' "${CONTAINER_NAME}" return 0 else @@ -169,8 +167,7 @@ function _repeat_in_container_until_success_or_timeout() { function _repeat_until_success_or_timeout() { local FATAL_FAILURE_TEST_COMMAND - if [[ "${1:-}" == "--fatal-test" ]] - then + if [[ "${1:-}" == "--fatal-test" ]]; then FATAL_FAILURE_TEST_COMMAND="${2:?Provided --fatal-test but no command}" shift 2 fi @@ -178,8 +175,7 @@ function _repeat_until_success_or_timeout() { local TIMEOUT=${1:?Timeout duration must be provided} shift 1 - if ! [[ "${TIMEOUT}" =~ ^[0-9]+$ ]] - then + if ! [[ "${TIMEOUT}" =~ ^[0-9]+$ ]]; then echo "First parameter for timeout must be an integer, received \"${TIMEOUT}\"" return 1 fi @@ -188,16 +184,14 @@ function _repeat_until_success_or_timeout() { until "${@}" do - if [[ -n ${FATAL_FAILURE_TEST_COMMAND} ]] && ! eval "${FATAL_FAILURE_TEST_COMMAND}" - then + if [[ -n ${FATAL_FAILURE_TEST_COMMAND} ]] && ! eval "${FATAL_FAILURE_TEST_COMMAND}"; then echo "\`${FATAL_FAILURE_TEST_COMMAND}\` failed, early aborting repeat_until_success of \`${*}\`" >&2 return 1 fi sleep 1 - if [[ $(( SECONDS - STARTTIME )) -gt ${TIMEOUT} ]] - then + if [[ $(( SECONDS - STARTTIME )) -gt ${TIMEOUT} ]]; then echo "Timed out on command: ${*}" >&2 return 1 fi @@ -213,8 +207,7 @@ function _run_until_success_or_timeout() { local TIMEOUT=${1:?Timeout duration must be provided} shift 1 - if [[ ! ${TIMEOUT} =~ ^[0-9]+$ ]] - then + if [[ ! ${TIMEOUT} =~ ^[0-9]+$ ]]; then echo "First parameter for timeout must be an integer, received \"${TIMEOUT}\"" return 1 fi @@ -226,8 +219,7 @@ function _run_until_success_or_timeout() { do sleep 1 - if (( SECONDS - STARTTIME > TIMEOUT )) - then + if (( SECONDS - STARTTIME > TIMEOUT )); then echo "Timed out on command: ${*}" >&2 return 1 fi @@ -270,8 +262,7 @@ function _wait_for_smtp_port_in_container_to_respond() { local COUNT=0 until [[ $(_exec_in_container timeout 10 /bin/bash -c 'echo QUIT | nc localhost 25') == *'221 2.0.0 Bye'* ]] do - if [[ ${COUNT} -eq 20 ]] - then + if [[ ${COUNT} -eq 20 ]]; then echo "Unable to receive a valid response from 'nc localhost 25' within 20 seconds" return 1 fi diff --git a/test/helper/setup.bash b/test/helper/setup.bash index 8fac99038af..ad76c5e2264 100644 --- a/test/helper/setup.bash +++ b/test/helper/setup.bash @@ -17,8 +17,7 @@ # This function is internal and should not be used in tests. function __initialize_variables() { function __check_if_set() { - if [[ ${!1+set} != 'set' ]] - then + if [[ ${!1+set} != 'set' ]]; then echo "ERROR: (helper/setup.sh) '${1:?No variable name given to __check_if_set}' is not set" >&2 exit 1 fi @@ -64,8 +63,7 @@ function _duplicate_config_for_container() { local OUTPUT_FOLDER OUTPUT_FOLDER=$(_print_private_config_path "${2}") - if [[ -z ${OUTPUT_FOLDER} ]] - then + if [[ -z ${OUTPUT_FOLDER} ]]; then echo "'OUTPUT_FOLDER' in '_duplicate_config_for_container' is empty" >&2 return 1 fi diff --git a/test/helper/tls.bash b/test/helper/tls.bash index 529c84c686f..b34c9ddb638 100644 --- a/test/helper/tls.bash +++ b/test/helper/tls.bash @@ -71,14 +71,11 @@ function _generate_openssl_cmd() { local CMD_OPENSSL="timeout 1 openssl s_client -connect ${HOST}:${PORT}" # STARTTLS ports need to add a hint: - if [[ ${PORT} =~ ^(25|587)$ ]] - then + if [[ ${PORT} =~ ^(25|587)$ ]]; then CMD_OPENSSL="${CMD_OPENSSL} -starttls smtp" - elif [[ ${PORT} == 143 ]] - then + elif [[ ${PORT} == 143 ]]; then CMD_OPENSSL="${CMD_OPENSSL} -starttls imap" - elif [[ ${PORT} == 110 ]] - then + elif [[ ${PORT} == 110 ]]; then CMD_OPENSSL="${CMD_OPENSSL} -starttls pop3" fi diff --git a/test/linting/lint.sh b/test/linting/lint.sh index 3f917ec39f6..032691e1b13 100755 --- a/test/linting/lint.sh +++ b/test/linting/lint.sh @@ -111,8 +111,7 @@ function _shellcheck "koalaman/shellcheck-alpine:v${SHELLCHECK_VERSION}" "${CMD_SHELLCHECK[@]}" \ "${BATS_EXTRA_ARGS[@]}" "${F_BATS[@]}" || ERROR=1 - if [[ ${ERROR} -eq 0 ]] - then + if [[ ${ERROR} -eq 0 ]]; then _log 'info' 'ShellCheck succeeded' else _log 'error' 'ShellCheck failed' diff --git a/test/test_helper/common.bash b/test/test_helper/common.bash index e0a4facd539..15287ff687b 100644 --- a/test/test_helper/common.bash +++ b/test/test_helper/common.bash @@ -107,8 +107,7 @@ function wait_for_smtp_port_in_container() { function wait_for_smtp_port_in_container_to_respond() { local COUNT=0 until [[ $(docker exec "${1}" timeout 10 /bin/sh -c "echo QUIT | nc localhost 25") == *"221 2.0.0 Bye"* ]]; do - if [[ $COUNT -eq 20 ]] - then + if [[ $COUNT -eq 20 ]]; then echo "Unable to receive a valid response from 'nc localhost 25' within 20 seconds" return 1 fi diff --git a/test/tests/parallel/set1/tls/letsencrypt.bats b/test/tests/parallel/set1/tls/letsencrypt.bats index 22fc70895f8..91a0599760d 100644 --- a/test/tests/parallel/set1/tls/letsencrypt.bats +++ b/test/tests/parallel/set1/tls/letsencrypt.bats @@ -240,8 +240,7 @@ function _copy_to_letsencrypt_storage() { FQDN_DIR=$(echo "${DEST}" | cut -d '/' -f1) mkdir -p "${TEST_TMP_CONFIG}/letsencrypt/${FQDN_DIR}" - if ! cp "${PWD}/test/test-files/ssl/${SRC}" "${TEST_TMP_CONFIG}/letsencrypt/${DEST}" - then + if ! cp "${PWD}/test/test-files/ssl/${SRC}" "${TEST_TMP_CONFIG}/letsencrypt/${DEST}"; then echo "Could not copy cert file '${SRC}'' to '${DEST}'" >&2 exit 1 fi diff --git a/test/tests/parallel/set2/tls_cipherlists.bats b/test/tests/parallel/set2/tls_cipherlists.bats index 854a65146c8..d9a82eb3cc5 100644 --- a/test/tests/parallel/set2/tls_cipherlists.bats +++ b/test/tests/parallel/set2/tls_cipherlists.bats @@ -76,8 +76,7 @@ function _configure_and_run_dms_container() { local ALT_KEY_TYPE=$3 # Optional parameter export TEST_VARIANT="${TLS_LEVEL}-${KEY_TYPE}" - if [[ -n ${ALT_KEY_TYPE} ]] - then + if [[ -n ${ALT_KEY_TYPE} ]]; then TEST_VARIANT+="-${ALT_KEY_TYPE}" fi @@ -98,8 +97,7 @@ function _configure_and_run_dms_container() { --env SSL_KEY_PATH="/config/ssl/with_ca/ecdsa/key.${KEY_TYPE}.pem" ) - if [[ -n ${ALT_KEY_TYPE} ]] - then + if [[ -n ${ALT_KEY_TYPE} ]]; then CUSTOM_SETUP_ARGUMENTS+=( --env SSL_ALT_CERT_PATH="/config/ssl/with_ca/ecdsa/cert.${ALT_KEY_TYPE}.pem" --env SSL_ALT_KEY_PATH="/config/ssl/with_ca/ecdsa/key.${ALT_KEY_TYPE}.pem" @@ -199,8 +197,7 @@ function compare_cipherlist() { function get_cipherlist() { local TLS_VERSION=$1 - if [[ ${TLS_VERSION} == "TLSv1_3" ]] - then + if [[ ${TLS_VERSION} == "TLSv1_3" ]]; then # TLS v1.3 cipher suites are not user defineable and not unique to the available certificate(s). # They do not support server enforced order either. echo '"TLS_AES_256_GCM_SHA384 TLS_CHACHA20_POLY1305_SHA256 TLS_AES_128_GCM_SHA256"' diff --git a/test/tests/parallel/set3/container_configuration/process_check_restart.bats b/test/tests/parallel/set3/container_configuration/process_check_restart.bats index 2a111af0a87..ef296da913d 100644 --- a/test/tests/parallel/set3/container_configuration/process_check_restart.bats +++ b/test/tests/parallel/set3/container_configuration/process_check_restart.bats @@ -184,8 +184,7 @@ function _check_if_process_is_running() { local IS_RUNNING=$(docker exec "${CONTAINER_NAME}" pgrep --list-full "${MIN_SECS_RUNNING[@]}" "${PROCESS}") # When no matches are found, nothing is returned. Provide something we can assert on (helpful for debugging): - if [[ ! ${IS_RUNNING} =~ ${PROCESS} ]] - then + if [[ ! ${IS_RUNNING} =~ ${PROCESS} ]]; then echo "'${PROCESS}' is not running" return 1 fi From 37ca0f9ba99c6e651d43862373a0e3921fc99661 Mon Sep 17 00:00:00 2001 From: Casper Date: Fri, 26 May 2023 01:01:41 +0200 Subject: [PATCH 090/592] Change 'function' style (#3364) --- setup.sh | 18 ++++------ target/bin/addalias | 6 ++-- target/bin/adddovecotmasteruser | 6 ++-- target/bin/addmailuser | 6 ++-- target/bin/addrelayhost | 12 +++---- target/bin/addsaslpassword | 12 +++---- target/bin/delalias | 6 ++-- target/bin/deldovecotmasteruser | 6 ++-- target/bin/delmailuser | 15 +++----- target/bin/delquota | 9 ++--- target/bin/excluderelaydomain | 9 ++--- target/bin/fail2ban | 3 +- target/bin/listalias | 6 ++-- target/bin/listdovecotmasteruser | 6 ++-- target/bin/listmailuser | 18 ++++------ target/bin/open-dkim | 6 ++-- target/bin/rspamd-dkim | 24 +++++-------- target/bin/setquota | 15 +++----- target/bin/setup | 9 ++--- target/bin/updatedovecotmasteruser | 6 ++-- target/bin/updatemailuser | 6 ++-- target/scripts/build/packages.sh | 27 +++++--------- target/scripts/check-for-changes.sh | 15 +++----- target/scripts/helpers/accounts.sh | 9 ++--- target/scripts/helpers/aliases.sh | 12 +++---- target/scripts/helpers/change-detection.sh | 6 ++-- target/scripts/helpers/database/db.sh | 21 ++++------- .../helpers/database/manage/dovecot-quotas.sh | 3 +- .../database/manage/postfix-accounts.sh | 18 ++++------ .../database/manage/postfix-virtual.sh | 3 +- target/scripts/helpers/dns.sh | 6 ++-- target/scripts/helpers/error.sh | 15 +++----- target/scripts/helpers/index.sh | 3 +- target/scripts/helpers/lock.sh | 6 ++-- target/scripts/helpers/log.sh | 12 +++---- target/scripts/helpers/network.sh | 6 ++-- target/scripts/helpers/postfix.sh | 12 +++---- target/scripts/helpers/relay.sh | 21 ++++------- target/scripts/helpers/ssl.sh | 27 +++++--------- target/scripts/helpers/utils.sh | 30 ++++++---------- target/scripts/start-mailserver.sh | 3 +- target/scripts/startup/check-stack.sh | 15 +++----- target/scripts/startup/daemons-stack.sh | 18 ++++------ target/scripts/startup/setup-stack.sh | 18 ++++------ .../scripts/startup/setup.d/dmarc_dkim_spf.sh | 9 ++--- target/scripts/startup/setup.d/dovecot.sh | 24 +++++-------- target/scripts/startup/setup.d/fetchmail.sh | 9 ++--- target/scripts/startup/setup.d/getmail.sh | 3 +- target/scripts/startup/setup.d/ldap.sh | 3 +- target/scripts/startup/setup.d/log.sh | 12 +++---- target/scripts/startup/setup.d/mail_state.sh | 3 +- target/scripts/startup/setup.d/networking.sh | 12 +++---- target/scripts/startup/setup.d/postfix.sh | 15 +++----- target/scripts/startup/setup.d/saslauthd.sh | 3 +- .../scripts/startup/setup.d/security/misc.sh | 24 +++++-------- .../startup/setup.d/security/rspamd.sh | 36 +++++++------------ .../startup/setup.d/security/spoofing.sh | 3 +- target/scripts/startup/variables-stack.sh | 18 ++++------ test/helper/common.bash | 3 +- test/linting/lint.sh | 12 +++---- 60 files changed, 233 insertions(+), 466 deletions(-) diff --git a/setup.sh b/setup.sh index b517d8bbbd5..7173968649e 100755 --- a/setup.sh +++ b/setup.sh @@ -27,8 +27,7 @@ RESET=$(echo -ne '\e[0m') set -euEo pipefail shopt -s inherit_errexit 2>/dev/null || true -function _show_local_usage -{ +function _show_local_usage() { # shellcheck disable=SC2059 printf '%s' "${ORANGE}OPTIONS${RESET} ${LBLUE}Config path, container or image adjustments${RESET} @@ -69,8 +68,7 @@ function _show_local_usage " } -function _get_absolute_script_directory -{ +function _get_absolute_script_directory() { if dirname "$(readlink -f "${0}")" &>/dev/null; then DIR=$(dirname "$(readlink -f "${0}")") elif realpath -e -L "${0}" &>/dev/null; then @@ -79,8 +77,7 @@ function _get_absolute_script_directory fi } -function _set_default_config_path -{ +function _set_default_config_path() { if [[ -d "${DIR}/config" ]]; then # legacy path (pre v10.2.0) DEFAULT_CONFIG_PATH="${DIR}/config" @@ -89,8 +86,7 @@ function _set_default_config_path fi } -function _handle_config_path -{ +function _handle_config_path() { if [[ -z ${DESIRED_CONFIG_PATH} ]]; then # no desired config path if [[ -n ${CONTAINER_NAME} ]]; then @@ -111,8 +107,7 @@ function _handle_config_path fi } -function _run_in_new_container -{ +function _run_in_new_container() { # start temporary container with specified image if ! ${CRI} history -q "${IMAGE_NAME}" &>/dev/null; then echo "Image '${IMAGE_NAME}' not found. Pulling ..." @@ -124,8 +119,7 @@ function _run_in_new_container "${IMAGE_NAME}" "${@}" } -function _main -{ +function _main() { _get_absolute_script_directory _set_default_config_path diff --git a/target/bin/addalias b/target/bin/addalias index a58bd7f83d6..60f8aa03e9b 100755 --- a/target/bin/addalias +++ b/target/bin/addalias @@ -3,8 +3,7 @@ # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh -function _main -{ +function _main() { _require_n_parameters_or_print_usage 2 "${@}" local MAIL_ALIAS="${1}" @@ -14,8 +13,7 @@ function _main || _exit_with_error "'${MAIL_ALIAS}' is already an alias for recipient: '${RECIPIENT}'" } -function __usage -{ +function __usage() { printf '%s' "${PURPLE}addalias${RED}(${YELLOW}8${RED}) ${ORANGE}USAGE${RESET} diff --git a/target/bin/adddovecotmasteruser b/target/bin/adddovecotmasteruser index 79bc51adbce..41041db2e3b 100755 --- a/target/bin/adddovecotmasteruser +++ b/target/bin/adddovecotmasteruser @@ -3,8 +3,7 @@ # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh -function _main -{ +function _main() { _require_n_parameters_or_print_usage 1 "${@}" local MAIL_ACCOUNT="${1}" @@ -14,8 +13,7 @@ function _main _manage_accounts_dovecotmaster_create "${MAIL_ACCOUNT}" "${PASSWD}" } -function __usage -{ +function __usage() { printf '%s' "${PURPLE}adddovecotmasteruser${RED}(${YELLOW}8${RED}) ${ORANGE}USAGE${RESET} diff --git a/target/bin/addmailuser b/target/bin/addmailuser index 2b6f3556a57..15691c89872 100755 --- a/target/bin/addmailuser +++ b/target/bin/addmailuser @@ -3,8 +3,7 @@ # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh -function _main -{ +function _main() { _require_n_parameters_or_print_usage 1 "${@}" local MAIL_ACCOUNT="${1}" @@ -18,8 +17,7 @@ function _main # where the actual account is created in Dovecot. Expect a delay. } -function __usage -{ +function __usage() { printf '%s' "${PURPLE}addmailuser${RED}(${YELLOW}8${RED}) ${ORANGE}USAGE${RESET} diff --git a/target/bin/addrelayhost b/target/bin/addrelayhost index 5463f15e4a0..b8ac2836bdf 100755 --- a/target/bin/addrelayhost +++ b/target/bin/addrelayhost @@ -3,8 +3,7 @@ # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh -function _main -{ +function _main() { _require_n_parameters_or_print_usage 2 "${@}" local DOMAIN="${1}" @@ -15,8 +14,7 @@ function _main _add_relayhost } -function __usage -{ +function __usage() { printf '%s' "${PURPLE}addrelayhost${RED}(${YELLOW}8${RED}) ${ORANGE}USAGE${RESET} @@ -47,8 +45,7 @@ ${ORANGE}EXIT STATUS${RESET} " } -function _validate_parameters -{ +function _validate_parameters() { [[ -z ${DOMAIN} ]] && { __usage ; _exit_with_error 'No domain specified' ; } [[ -z ${HOST} ]] && { __usage ; _exit_with_error 'No relay host specified' ; } [[ -z ${PORT} ]] && PORT=25 @@ -56,8 +53,7 @@ function _validate_parameters # Config is for sender dependent relay-host mapping, # current support restricts senders to domain scope (port is also enforced). -function _add_relayhost -{ +function _add_relayhost() { local SENDER="@${DOMAIN}" local RELAY_HOST_ENTRY="[${HOST}]:${PORT}" local DATABASE_RELAY='/tmp/docker-mailserver/postfix-relaymap.cf' diff --git a/target/bin/addsaslpassword b/target/bin/addsaslpassword index 27dd67cd5cb..9461c112e22 100755 --- a/target/bin/addsaslpassword +++ b/target/bin/addsaslpassword @@ -3,8 +3,7 @@ # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh -function _main -{ +function _main() { _require_n_parameters_or_print_usage 2 "${@}" local DOMAIN="${1}" @@ -16,8 +15,7 @@ function _main _add_relayhost_credentials } -function __usage -{ +function __usage() { printf '%s' "${PURPLE}addsaslpassword${RED}(${YELLOW}8${RED}) ${ORANGE}USAGE${RESET} @@ -46,8 +44,7 @@ ${ORANGE}EXIT STATUS${RESET} " } -function _validate_parameters -{ +function _validate_parameters() { [[ -z ${DOMAIN} ]] && { __usage ; _exit_with_error 'No domain specified' ; } [[ -z ${RELAY_ACCOUNT} ]] && { __usage ; _exit_with_error 'No relay account specified' ; } _password_request_if_missing @@ -58,8 +55,7 @@ function _validate_parameters # # NOTE: This command does not support providing a relay-host # as the lookup key, it only supports a lookup via sender domain. -function _add_relayhost_credentials -{ +function _add_relayhost_credentials() { local SENDER="@${DOMAIN}" local RELAY_HOST_ENTRY_AUTH="${RELAY_ACCOUNT}:${PASSWD}" local DATABASE_PASSWD='/tmp/docker-mailserver/postfix-sasl-password.cf' diff --git a/target/bin/delalias b/target/bin/delalias index c8b2774553e..c3e575451b9 100755 --- a/target/bin/delalias +++ b/target/bin/delalias @@ -3,8 +3,7 @@ # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh -function _main -{ +function _main() { _require_n_parameters_or_print_usage 2 "${@}" local MAIL_ALIAS="${1}" @@ -13,8 +12,7 @@ function _main _manage_virtual_aliases_delete "${MAIL_ALIAS}" "${RECIPIENT}" } -function __usage -{ +function __usage() { printf '%s' "${PURPLE}delalias${RED}(${YELLOW}8${RED}) ${ORANGE}USAGE${RESET} diff --git a/target/bin/deldovecotmasteruser b/target/bin/deldovecotmasteruser index efd4e7817f3..2ef20276714 100755 --- a/target/bin/deldovecotmasteruser +++ b/target/bin/deldovecotmasteruser @@ -3,8 +3,7 @@ # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh -function _main -{ +function _main() { _require_n_parameters_or_print_usage 1 "${@}" # Actual command to perform: @@ -15,8 +14,7 @@ function _main done } -function __usage -{ +function __usage() { printf '%s' "${PURPLE}deldovecotmasteruser${RED}(${YELLOW}8${RED}) ${ORANGE}USAGE${RESET} diff --git a/target/bin/delmailuser b/target/bin/delmailuser index 117a054c9b0..974a237518c 100755 --- a/target/bin/delmailuser +++ b/target/bin/delmailuser @@ -3,8 +3,7 @@ # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh -function _main -{ +function _main() { _require_n_parameters_or_print_usage 1 "${@}" # Tests expect early exit without error if no DB exists: @@ -37,8 +36,7 @@ function _main done } -function __usage -{ +function __usage() { printf '%s' "${PURPLE}delmailuser${RED}(${YELLOW}8${RED}) ${ORANGE}USAGE${RESET} @@ -71,8 +69,7 @@ ${ORANGE}EXIT STATUS${RESET} " } -function _parse_options -{ +function _parse_options() { while getopts ":yY" OPT do case "${OPT}" in @@ -89,8 +86,7 @@ function _parse_options done } -function _maildel_request_if_missing -{ +function _maildel_request_if_missing() { if [[ ${MAILDEL} -eq 0 ]]; then local MAILDEL_CHOSEN read -r -p "Do you want to delete the mailbox as well (removing all mails)? [Y/n] " MAILDEL_CHOSEN @@ -102,8 +98,7 @@ function _maildel_request_if_missing fi } -function _remove_maildir -{ +function _remove_maildir() { local MAIL_ACCOUNT=${1} local LOCAL_PART="${MAIL_ACCOUNT%@*}" diff --git a/target/bin/delquota b/target/bin/delquota index 1282a827e34..de3759c85e7 100755 --- a/target/bin/delquota +++ b/target/bin/delquota @@ -3,8 +3,7 @@ # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh -function _main -{ +function _main() { _require_n_parameters_or_print_usage 1 "${@}" local MAIL_ACCOUNT="${1}" @@ -13,8 +12,7 @@ function _main _manage_dovecot_quota_delete "${MAIL_ACCOUNT}" } -function __usage -{ +function __usage() { printf '%s' "${PURPLE}delquota${RED}(${YELLOW}8${RED}) ${ORANGE}USAGE${RESET} @@ -37,8 +35,7 @@ ${ORANGE}EXIT STATUS${RESET} " } -function _validate_parameters -{ +function _validate_parameters() { _arg_expect_mail_account _account_should_already_exist } diff --git a/target/bin/excluderelaydomain b/target/bin/excluderelaydomain index d6bdc2a95ea..311ab4330be 100755 --- a/target/bin/excluderelaydomain +++ b/target/bin/excluderelaydomain @@ -3,8 +3,7 @@ # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh -function _main -{ +function _main() { _require_n_parameters_or_print_usage 1 "${@}" local DOMAIN="${1}" @@ -12,8 +11,7 @@ function _main _exclude_domain_from_relayhosts } -function __usage -{ +function __usage() { printf '%s' "${PURPLE}excluderelayhost${RED}(${YELLOW}8${RED}) ${ORANGE}USAGE${RESET} @@ -44,8 +42,7 @@ ${ORANGE}EXIT STATUS${RESET} # Config is for sender dependent relay-host mapping, # excludes appending a sender from the real generated mapping in `helpers/relay.sh`. -function _exclude_domain_from_relayhosts -{ +function _exclude_domain_from_relayhosts() { local SENDER="@${DOMAIN}" local DATABASE_RELAY='/tmp/docker-mailserver/postfix-relaymap.cf' diff --git a/target/bin/fail2ban b/target/bin/fail2ban index 1aa2c1dafd9..71dbd3c046a 100755 --- a/target/bin/fail2ban +++ b/target/bin/fail2ban @@ -3,8 +3,7 @@ # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh -function __usage -{ +function __usage() { echo "Usage: ./setup.sh fail2ban [ ]" echo " ./setup.sh fail2ban log" } diff --git a/target/bin/listalias b/target/bin/listalias index 697794074d9..efb48fd29ee 100755 --- a/target/bin/listalias +++ b/target/bin/listalias @@ -3,14 +3,12 @@ # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh -function _main -{ +function _main() { local DATABASE_VIRTUAL='/tmp/docker-mailserver/postfix-virtual.cf' _list_entries "${DATABASE_VIRTUAL}" } -function _list_entries -{ +function _list_entries() { local DATABASE=${1} _db_should_exist_with_content "${DATABASE}" diff --git a/target/bin/listdovecotmasteruser b/target/bin/listdovecotmasteruser index 1f6f612330f..214befd3e34 100755 --- a/target/bin/listdovecotmasteruser +++ b/target/bin/listdovecotmasteruser @@ -3,14 +3,12 @@ # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh -function _main -{ +function _main() { local DATABASE_DOVECOT_MASTERS='/tmp/docker-mailserver/dovecot-masters.cf' _list_entries "${DATABASE_DOVECOT_MASTERS}" } -function _list_entries -{ +function _list_entries() { local DATABASE=${1} _db_should_exist_with_content "${DATABASE}" diff --git a/target/bin/listmailuser b/target/bin/listmailuser index 8f5561d779c..c1ab9d8e9db 100755 --- a/target/bin/listmailuser +++ b/target/bin/listmailuser @@ -7,16 +7,14 @@ source /usr/local/bin/helpers/index.sh # shellcheck source=/dev/null source /etc/dms-settings 2>/dev/null -function _main -{ +function _main() { local DATABASE_ACCOUNTS='/tmp/docker-mailserver/postfix-accounts.cf' local DATABASE_VIRTUAL='/tmp/docker-mailserver/postfix-virtual.cf' _list_entries "${DATABASE_ACCOUNTS}" } -function _list_entries -{ +function _list_entries() { local DATABASE=${1} _db_should_exist_with_content "${DATABASE}" @@ -29,8 +27,7 @@ function _list_entries done < <(_get_valid_lines_from_file "${DATABASE}") } -function _format_list_item -{ +function _format_list_item() { local LINE=${1} local MAIL_ACCOUNT @@ -49,8 +46,7 @@ function _format_list_item echo "${ACCOUNT_ENTRY}" } -function _quota_show_for -{ +function _quota_show_for() { local MAIL_ACCOUNT=${1} [[ ${ENABLE_QUOTAS} -ne 1 ]] && return 0 @@ -68,8 +64,7 @@ function _quota_show_for echo "( ${CURRENT_SIZE} / ${SIZE_LIMIT} ) [${PERCENT_USED}]" } -function _bytes_to_human_readable_size -{ +function _bytes_to_human_readable_size() { # `-` represents a non-applicable value (eg: Like when `SIZE_LIMIT` is not set): if [[ ${1:-} == '-' ]]; then echo '~' @@ -83,8 +78,7 @@ function _bytes_to_human_readable_size } # Returns a comma delimited list of aliases associated to a recipient (ideally the recipient is a mail account): -function _alias_list_for_account -{ +function _alias_list_for_account() { local GREP_OPTIONS local MAIL_ACCOUNT=${1} diff --git a/target/bin/open-dkim b/target/bin/open-dkim index 995d73598c9..c3e656786a6 100755 --- a/target/bin/open-dkim +++ b/target/bin/open-dkim @@ -12,8 +12,7 @@ KEYSIZE=4096 SELECTOR=mail DOMAINS= -function __usage -{ +function __usage() { printf '%s' "${PURPLE}OPEN-DKIM${RED}(${YELLOW}8${RED}) ${ORANGE}NAME${RESET} @@ -102,8 +101,7 @@ done DATABASE_VHOST='/tmp/vhost.dkim' # Prepare a file with one domain per line: -function _generate_domains_config -{ +function _generate_domains_config() { local TMP_VHOST='/tmp/vhost.dkim.tmp' # Generate the default vhost (equivalent to /etc/postfix/vhost), diff --git a/target/bin/rspamd-dkim b/target/bin/rspamd-dkim index 8093dd39b16..e4250d8ba8b 100755 --- a/target/bin/rspamd-dkim +++ b/target/bin/rspamd-dkim @@ -11,8 +11,7 @@ shopt -s inherit_errexit # shellcheck source=/dev/null source /etc/dms-settings -function __usage -{ +function __usage() { _log 'trace' 'Showing usage message now' echo -e "${PURPLE}RSPAMD-DKIM${RED}(${YELLOW}8${RED}) @@ -65,16 +64,14 @@ ${ORANGE}EXIT STATUS${RESET} " } -function __do_as_rspamd_user -{ +function __do_as_rspamd_user() { local COMMAND=${1:?Command required when using __do_as_rspamd_user} _log 'trace' "Running '${*}' as user '_rspamd' now" shift 1 su -l '_rspamd' -s "$(command -v "${COMMAND}")" -- "${@}" } -function _parse_arguments -{ +function _parse_arguments() { KEYTYPE='rsa' KEYSIZE='2048' SELECTOR='mail' @@ -152,8 +149,7 @@ function _parse_arguments return 0 } -function _create_keys -{ +function _create_keys() { # Note: Variables not marked with `local` are used # in other functions (after this function was called). BASE_DIR='/tmp/docker-mailserver/rspamd/dkim' @@ -192,8 +188,7 @@ function _create_keys fi } -function _check_permissions -{ +function _check_permissions() { # shellcheck disable=SC2310 if ! __do_as_rspamd_user ls "${BASE_DIR}" >/dev/null; then _log 'warn' "The Rspamd user ('_rspamd') seems to be unable to list files in the keys directory ('${BASE_DIR}') - Rspamd may experience permission errors later" @@ -204,8 +199,7 @@ function _check_permissions fi } -function _setup_default_signing_conf -{ +function _setup_default_signing_conf() { local DEFAULT_CONFIG_FILE='/etc/rspamd/override.d/dkim_signing.conf' if [[ -f ${DEFAULT_CONFIG_FILE} ]]; then _log 'debug' "'${DEFAULT_CONFIG_FILE}' exists, not supplying a default" @@ -237,8 +231,7 @@ EOF fi } -function _transform_public_key_file_to_dns_record_contents -{ +function _transform_public_key_file_to_dns_record_contents() { _log 'trace' 'Transforming DNS zone format to DNS record content now' : >"${PUBLIC_KEY_DNS_FILE}" grep -o '".*"' "${PUBLIC_KEY_FILE}" | tr -d '"\n' >>"${PUBLIC_KEY_DNS_FILE}" @@ -251,8 +244,7 @@ function _transform_public_key_file_to_dns_record_contents fi } -function _final_steps -{ +function _final_steps() { # We need to restart Rspamd so the changes take effect immediately. if ! supervisorctl restart rspamd; then _log 'warn' 'Could not restart Rspamd via Supervisord' diff --git a/target/bin/setquota b/target/bin/setquota index 9c0bf06e00c..5b2bba411ac 100755 --- a/target/bin/setquota +++ b/target/bin/setquota @@ -3,8 +3,7 @@ # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh -function _main -{ +function _main() { _require_n_parameters_or_print_usage 1 "${@}" local MAIL_ACCOUNT="${1}" @@ -15,8 +14,7 @@ function _main _manage_dovecot_quota_update "${MAIL_ACCOUNT}" "${QUOTA}" } -function __usage -{ +function __usage() { printf '%s' "${PURPLE}setquota${RED}(${YELLOW}8${RED}) ${ORANGE}USAGE${RESET} @@ -43,8 +41,7 @@ ${ORANGE}EXIT STATUS${RESET} " } -function _validate_parameters -{ +function _validate_parameters() { # MAIL_ACCOUNT _arg_expect_mail_account _account_should_already_exist @@ -54,8 +51,7 @@ function _validate_parameters _quota_unit_is_valid } -function _quota_request_if_missing -{ +function _quota_request_if_missing() { if [[ -z ${QUOTA} ]]; then read -r -p 'Enter quota (e.g. 10M): ' QUOTA echo @@ -63,8 +59,7 @@ function _quota_request_if_missing fi } -function _quota_unit_is_valid -{ +function _quota_unit_is_valid() { if ! grep -qE "^([0-9]+(B|k|M|G|T)|0)\$" <<< "${QUOTA}"; then __usage _exit_with_error 'Invalid quota format. e.g. 302M (B (byte), k (kilobyte), M (megabyte), G (gigabyte) or T (terabyte))' diff --git a/target/bin/setup b/target/bin/setup index 49546f70c87..29782b6f28e 100755 --- a/target/bin/setup +++ b/target/bin/setup @@ -5,8 +5,7 @@ set -euE -o pipefail # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh -function _usage -{ +function _usage() { # shellcheck disable=SC2059 printf '%s' "${PURPLE}SETUP${RED}(${YELLOW}1${RED}) @@ -82,15 +81,13 @@ ${ORANGE}EXAMPLES${RESET} " } -function _invalid_command -{ +function _invalid_command() { echo "The command '${*}' is invalid. Use \`setup help\` to get an overview of all commands." >&2 exit 2 } -function _main -{ +function _main() { case ${1:-} in ( email ) diff --git a/target/bin/updatedovecotmasteruser b/target/bin/updatedovecotmasteruser index bc89f882378..eeff21a591d 100755 --- a/target/bin/updatedovecotmasteruser +++ b/target/bin/updatedovecotmasteruser @@ -3,8 +3,7 @@ # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh -function _main -{ +function _main() { _require_n_parameters_or_print_usage 1 "${@}" local MAIL_ACCOUNT="${1}" @@ -14,8 +13,7 @@ function _main _manage_accounts_dovecotmaster_update "${MAIL_ACCOUNT}" "${PASSWD}" } -function __usage -{ +function __usage() { printf '%s' "${PURPLE}updatedovecotmasteruser${RED}(${YELLOW}8${RED}) ${ORANGE}USAGE${RESET} diff --git a/target/bin/updatemailuser b/target/bin/updatemailuser index 5d2f8012910..ce45533e37b 100755 --- a/target/bin/updatemailuser +++ b/target/bin/updatemailuser @@ -3,8 +3,7 @@ # shellcheck source=../scripts/helpers/index.sh source /usr/local/bin/helpers/index.sh -function _main -{ +function _main() { _require_n_parameters_or_print_usage 1 "${@}" local MAIL_ACCOUNT="${1}" @@ -14,8 +13,7 @@ function _main _manage_accounts_update "${MAIL_ACCOUNT}" "${PASSWD}" } -function __usage -{ +function __usage() { printf '%s' "${PURPLE}updatemailuser${RED}(${YELLOW}8${RED}) ${ORANGE}USAGE${RESET} diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 8258066eed2..9f863899bc5 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -10,8 +10,7 @@ source /usr/local/bin/helpers/log.sh _log_level_is 'trace' && QUIET='-y' || QUIET='-qq' -function _pre_installation_steps -{ +function _pre_installation_steps() { _log 'info' 'Starting package installation' _log 'debug' 'Running pre-installation steps' @@ -25,8 +24,7 @@ function _pre_installation_steps apt-get "${QUIET}" upgrade } -function _install_postfix -{ +function _install_postfix() { _log 'debug' 'Installing Postfix' _log 'warn' 'Applying workaround for Postfix bug (see https://github.com//issues/2023#issuecomment-855326403)' @@ -42,8 +40,7 @@ function _install_postfix rm /etc/rsyslog.d/postfix.conf } -function _install_packages -{ +function _install_packages() { _log 'debug' 'Installing all packages now' declare -a ANTI_VIRUS_SPAM_PACKAGES @@ -93,8 +90,7 @@ function _install_packages "${MAIL_PROGRAMS_PACKAGES[@]}" } -function _install_dovecot -{ +function _install_dovecot() { declare -a DOVECOT_PACKAGES DOVECOT_PACKAGES=( @@ -131,8 +127,7 @@ function _install_dovecot apt-get "${QUIET}" --no-install-recommends install "${DOVECOT_PACKAGES[@]}" } -function _install_rspamd -{ +function _install_rspamd() { _log 'trace' 'Adding Rspamd package signatures' local DEB_FILE='/etc/apt/sources.list.d/rspamd.list' local RSPAMD_PACKAGE_NAME @@ -160,8 +155,7 @@ function _install_rspamd apt-get "${QUIET}" --no-install-recommends install "${RSPAMD_PACKAGE_NAME}" 'redis-server' } -function _install_fail2ban -{ +function _install_fail2ban() { local FAIL2BAN_DEB_URL='https://github.com/fail2ban/fail2ban/releases/download/1.0.2/fail2ban_1.0.2-1.upstream1_all.deb' local FAIL2BAN_DEB_ASC_URL="${FAIL2BAN_DEB_URL}.asc" local FAIL2BAN_GPG_FINGERPRINT='8738 559E 26F6 71DF 9E2C 6D9E 683B F1BE BD0A 882C' @@ -201,8 +195,7 @@ function _install_fail2ban # v6.18 contains fixes for Google and Microsoft OAuth support. # using pip to install getmail. # TODO This can be removed when the base image is updated to Debian 12 (Bookworm) -function _install_getmail -{ +function _install_getmail() { _log 'debug' 'Installing getmail6' apt-get "${QUIET}" --no-install-recommends install python3-pip pip3 install --no-cache-dir 'getmail6~=6.18.12' @@ -212,8 +205,7 @@ function _install_getmail apt-get "${QUIET}" autoremove } -function _remove_data_after_package_installations -{ +function _remove_data_after_package_installations() { _log 'debug' 'Deleting sensitive files (secrets)' rm /etc/postsrsd.secret @@ -221,8 +213,7 @@ function _remove_data_after_package_installations rm /etc/cron.daily/00logwatch } -function _post_installation_steps -{ +function _post_installation_steps() { _log 'debug' 'Running post-installation steps (cleanup)' apt-get "${QUIET}" clean rm -rf /var/lib/apt/lists/* diff --git a/target/scripts/check-for-changes.sh b/target/scripts/check-for-changes.sh index cfcc78d9aef..58f0d96f541 100755 --- a/target/scripts/check-for-changes.sh +++ b/target/scripts/check-for-changes.sh @@ -30,8 +30,7 @@ _log_with_date 'trace' "Using postmaster address '${POSTMASTER_ADDRESS}'" _log_with_date 'debug' "Changedetector is ready" -function _check_for_changes -{ +function _check_for_changes() { # get chksum and check it, no need to lock config yet _monitored_files_checksums >"${CHKSUM_FILE}.new" cmp --silent -- "${CHKSUM_FILE}" "${CHKSUM_FILE}.new" @@ -65,8 +64,7 @@ function _check_for_changes fi } -function _get_changed_files -{ +function _get_changed_files() { local CHKSUM_CURRENT=${1} local CHKSUM_NEW=${2} @@ -81,8 +79,7 @@ function _get_changed_files grep -Fxvf "${CHKSUM_CURRENT}" "${CHKSUM_NEW}" | sed -r 's/^\S+[[:space:]]+//' } -function _reload_amavis -{ +function _reload_amavis() { if [[ ${CHANGED} =~ ${DMS_DIR}/postfix-accounts.cf ]] || [[ ${CHANGED} =~ ${DMS_DIR}/postfix-virtual.cf ]]; then # /etc/postfix/vhost was updated, amavis must refresh it's config by # reading this file again in case of new domains, otherwise they will be ignored. @@ -92,8 +89,7 @@ function _reload_amavis # Also note that changes are performed in place and are not atomic # We should fix that and write to temporary files, stop, swap and start -function _postfix_dovecot_changes -{ +function _postfix_dovecot_changes() { local DMS_DIR=/tmp/docker-mailserver # Regenerate accounts via `helpers/accounts.sh`: @@ -141,8 +137,7 @@ function _postfix_dovecot_changes _chown_var_mail_if_necessary } -function _ssl_changes -{ +function _ssl_changes() { local REGEX_NEVER_MATCH='(?\!)' # _setup_ssl is required for: diff --git a/target/scripts/helpers/accounts.sh b/target/scripts/helpers/accounts.sh index 397290a343f..3637ff4f942 100644 --- a/target/scripts/helpers/accounts.sh +++ b/target/scripts/helpers/accounts.sh @@ -9,8 +9,7 @@ DOVECOT_USERDB_FILE=/etc/dovecot/userdb DOVECOT_MASTERDB_FILE=/etc/dovecot/masterdb -function _create_accounts -{ +function _create_accounts() { : >/etc/postfix/vmailbox : >"${DOVECOT_USERDB_FILE}" @@ -98,8 +97,7 @@ function _create_accounts # # see https://github.com/docker-mailserver/docker-mailserver/pull/2248#issuecomment-953313852 # for more details on this method -function _create_dovecot_alias_dummy_accounts -{ +function _create_dovecot_alias_dummy_accounts() { local DATABASE_VIRTUAL='/tmp/docker-mailserver/postfix-virtual.cf' if [[ -f ${DATABASE_VIRTUAL} ]] && [[ ${ENABLE_QUOTAS} -eq 1 ]]; then @@ -157,8 +155,7 @@ function _create_dovecot_alias_dummy_accounts # Support Dovecot master user: https://doc.dovecot.org/configuration_manual/authentication/master_users/ # Supporting LDAP users requires `auth_bind = yes` in `dovecot-ldap.conf.ext`, see docker-mailserver/docker-mailserver/pull/2535 for details -function _create_masters -{ +function _create_masters() { : >"${DOVECOT_MASTERDB_FILE}" local DATABASE_DOVECOT_MASTERS='/tmp/docker-mailserver/dovecot-masters.cf' diff --git a/target/scripts/helpers/aliases.sh b/target/scripts/helpers/aliases.sh index 16b9e599b66..9690aae5160 100644 --- a/target/scripts/helpers/aliases.sh +++ b/target/scripts/helpers/aliases.sh @@ -6,8 +6,7 @@ # `setup-stack.sh:_setup_ldap` does not seem to configure for `/etc/postfix/virtual however.` # NOTE: `accounts.sh` and `relay.sh:_populate_relayhost_map` also process on `postfix-virtual.cf`. -function _handle_postfix_virtual_config -{ +function _handle_postfix_virtual_config() { : >/etc/postfix/virtual local DATABASE_VIRTUAL=/tmp/docker-mailserver/postfix-virtual.cf @@ -24,8 +23,7 @@ function _handle_postfix_virtual_config fi } -function _handle_postfix_regexp_config -{ +function _handle_postfix_regexp_config() { : >/etc/postfix/regexp if [[ -f /tmp/docker-mailserver/postfix-regexp.cf ]]; then @@ -41,8 +39,7 @@ function _handle_postfix_regexp_config fi } -function _handle_postfix_aliases_config -{ +function _handle_postfix_aliases_config() { _log 'trace' 'Configuring root alias' echo "root: ${POSTMASTER_ADDRESS}" >/etc/aliases @@ -55,8 +52,7 @@ function _handle_postfix_aliases_config } # Other scripts should call this method, rather than the ones above: -function _create_aliases -{ +function _create_aliases() { _handle_postfix_virtual_config _handle_postfix_regexp_config _handle_postfix_aliases_config diff --git a/target/scripts/helpers/change-detection.sh b/target/scripts/helpers/change-detection.sh index 24ea030119d..ddda89b086c 100644 --- a/target/scripts/helpers/change-detection.sh +++ b/target/scripts/helpers/change-detection.sh @@ -12,8 +12,7 @@ CHKSUM_FILE=/tmp/docker-mailserver-config-chksum # Once container startup scripts complete, take a snapshot of # the config state via storing a list of files content hashes. -function _prepare_for_change_detection -{ +function _prepare_for_change_detection() { _log 'debug' 'Setting up configuration checksum file' _log 'trace' "Creating '${CHKSUM_FILE}'" @@ -22,8 +21,7 @@ function _prepare_for_change_detection # Returns a list of changed files, each line is a value pair of: # -function _monitored_files_checksums -{ +function _monitored_files_checksums() { # If a wildcard path pattern (or an empty ENV) would yield an invalid path # or no results, `shopt -s nullglob` prevents it from being added. shopt -s nullglob diff --git a/target/scripts/helpers/database/db.sh b/target/scripts/helpers/database/db.sh index 8520f7cb67f..88717885add 100644 --- a/target/scripts/helpers/database/db.sh +++ b/target/scripts/helpers/database/db.sh @@ -18,8 +18,7 @@ DATABASE_PASSWD="${DMS_CONFIG}/postfix-sasl-password.cf" DATABASE_RELAY="${DMS_CONFIG}/postfix-relaymap.cf" # Individual scripts with convenience methods to manage operations easier: -function _db_import_scripts -{ +function _db_import_scripts() { # This var is stripped by shellcheck from source paths below, # like the shellcheck source-path above, it shouold match this scripts # parent directory, with the rest of the relative path in the source lines: @@ -35,8 +34,7 @@ function _db_entry_add_or_append { _db_operation 'append' "${@}" ; } # Only us function _db_entry_add_or_replace { _db_operation 'replace' "${@}" ; } function _db_entry_remove { _db_operation 'remove' "${@}" ; } -function _db_operation -{ +function _db_operation() { local DB_ACTION=${1} local DATABASE=${2} local KEY=${3} @@ -126,8 +124,7 @@ function _db_operation } # Internal method for: _db_operation -function __db_list_already_contains_value -{ +function __db_list_already_contains_value() { # Avoids accidentally matching a substring (case-insensitive acceptable): # 1. Extract the current value of the entry (`\1`), # 2. Value list support: Split values into separate lines (`\n`+`g`) at V_DELIMITER, @@ -140,8 +137,7 @@ function __db_list_already_contains_value # Internal method for: _db_operation + _db_has_entry_with_key # References global vars `DATABASE_*`: -function __db_get_delimiter_for -{ +function __db_get_delimiter_for() { local DATABASE=${1} case "${DATABASE}" in @@ -171,8 +167,7 @@ function __db_get_delimiter_for # `\` can escape these (`/` exists in postfix-account.cf base64 encoded pw hash), # But otherwise care should be taken with `\`, which should be forbidden for input here? # NOTE: Presently only `.` is escaped with `\` via `_escape`. -function __escape_sed_replacement -{ +function __escape_sed_replacement() { # Matches any `/` or `&`, and escapes them with `\` (`\\\1`): sed 's/\([/&]\)/\\\1/g' <<< "${ENTRY}" } @@ -181,8 +176,7 @@ function __escape_sed_replacement # Validation Methods # -function _db_has_entry_with_key -{ +function _db_has_entry_with_key() { local KEY=${1} local DATABASE=${2} @@ -202,8 +196,7 @@ function _db_has_entry_with_key grep --quiet --no-messages --ignore-case "^${KEY_LOOKUP}" "${DATABASE}" } -function _db_should_exist_with_content -{ +function _db_should_exist_with_content() { local DATABASE=${1} [[ -f ${DATABASE} ]] || _exit_with_error "'${DATABASE}' does not exist" diff --git a/target/scripts/helpers/database/manage/dovecot-quotas.sh b/target/scripts/helpers/database/manage/dovecot-quotas.sh index 802eb78f651..098d17d47e9 100644 --- a/target/scripts/helpers/database/manage/dovecot-quotas.sh +++ b/target/scripts/helpers/database/manage/dovecot-quotas.sh @@ -3,8 +3,7 @@ # Manage DB writes for: DATABASE_QUOTA # Logic to perform for requested operations handled here: -function _manage_dovecot_quota -{ +function _manage_dovecot_quota() { local ACTION=${1} local MAIL_ACCOUNT=${2} # Only for ACTION 'update': diff --git a/target/scripts/helpers/database/manage/postfix-accounts.sh b/target/scripts/helpers/database/manage/postfix-accounts.sh index 07d484f483e..8a2a6133392 100644 --- a/target/scripts/helpers/database/manage/postfix-accounts.sh +++ b/target/scripts/helpers/database/manage/postfix-accounts.sh @@ -5,8 +5,7 @@ # - DATABASE_DOVECOT_MASTERS # Logic to perform for requested operations handled here: -function _manage_accounts -{ +function _manage_accounts() { local ACTION=${1} local DATABASE=${2} local MAIL_ACCOUNT=${3} @@ -60,8 +59,7 @@ function _manage_accounts_dovecotmaster_delete { _manage_accounts 'delete' "${DA # - Calling external method '__usage' as part of error handling. # Also used by setquota, delquota -function _arg_expect_mail_account -{ +function _arg_expect_mail_account() { [[ -z ${MAIL_ACCOUNT} ]] && { __usage ; _exit_with_error 'No account specified' ; } # Dovecot Master accounts are validated (they are not email addresses): @@ -71,8 +69,7 @@ function _arg_expect_mail_account [[ ${MAIL_ACCOUNT} =~ .*\@.* ]] || { __usage ; _exit_with_error "'${MAIL_ACCOUNT}' should include the domain (eg: user@example.com)" ; } } -function _account_should_not_exist_yet -{ +function _account_should_not_exist_yet() { __account_already_exists && _exit_with_error "'${MAIL_ACCOUNT}' already exists" if [[ -f ${DATABASE_VIRTUAL} ]] && grep -q "^${MAIL_ACCOUNT}" "${DATABASE_VIRTUAL}"; then _exit_with_error "'${MAIL_ACCOUNT}' is already defined as an alias" @@ -80,20 +77,17 @@ function _account_should_not_exist_yet } # Also used by delmailuser, setquota, delquota -function _account_should_already_exist -{ +function _account_should_already_exist() { ! __account_already_exists && _exit_with_error "'${MAIL_ACCOUNT}' does not exist" } -function __account_already_exists -{ +function __account_already_exists() { local DATABASE=${DATABASE:-"${DATABASE_ACCOUNTS}"} _db_has_entry_with_key "${MAIL_ACCOUNT}" "${DATABASE}" } # Also used by addsaslpassword -function _password_request_if_missing -{ +function _password_request_if_missing() { if [[ -z ${PASSWD} ]]; then read -r -s -p 'Enter Password: ' PASSWD echo diff --git a/target/scripts/helpers/database/manage/postfix-virtual.sh b/target/scripts/helpers/database/manage/postfix-virtual.sh index cc40800ed95..c2ff5488ae1 100644 --- a/target/scripts/helpers/database/manage/postfix-virtual.sh +++ b/target/scripts/helpers/database/manage/postfix-virtual.sh @@ -11,8 +11,7 @@ # mail to an alias address. # Logic to perform for requested operations handled here: -function _manage_virtual_aliases -{ +function _manage_virtual_aliases() { local ACTION=${1} local MAIL_ALIAS=${2} local RECIPIENT=${3} diff --git a/target/scripts/helpers/dns.sh b/target/scripts/helpers/dns.sh index 72fecec038d..688ac50880a 100644 --- a/target/scripts/helpers/dns.sh +++ b/target/scripts/helpers/dns.sh @@ -2,15 +2,13 @@ # Outputs the DNS label count (delimited by `.`) for the given input string. # Useful for determining an FQDN like `mail.example.com` (3), vs `example.com` (2). -function _get_label_count -{ +function _get_label_count() { awk -F '.' '{ print NF }' <<< "${1}" } # Sets HOSTNAME and DOMAINNAME globals used throughout the scripts, # and any subprocesses called that intereact with it. -function _obtain_hostname_and_domainname -{ +function _obtain_hostname_and_domainname() { # Normally this value would match the output of `hostname` which mirrors `/proc/sys/kernel/hostname`, # However for legacy reasons, the system ENV `HOSTNAME` was replaced here with `hostname -f` instead. # diff --git a/target/scripts/helpers/error.sh b/target/scripts/helpers/error.sh index a75cef9c0d6..3af736df6df 100644 --- a/target/scripts/helpers/error.sh +++ b/target/scripts/helpers/error.sh @@ -1,7 +1,6 @@ #!/bin/bash -function _exit_with_error -{ +function _exit_with_error() { if [[ -n ${1+set} ]]; then _log 'error' "${1}" else @@ -19,8 +18,7 @@ function _exit_with_error # PANIC_TYPE => (Internal value for matching). You should use the convenience methods below based on your panic type. # PANIC_INFO => Provide your own message string to insert into the error message for that PANIC_TYPE. # PANIC_SCOPE => Optionally provide a string for debugging to better identify/locate the source of the panic. -function dms_panic -{ +function dms_panic() { local PANIC_TYPE=${1:-} local PANIC_INFO=${2:-} local PANIC_SCOPE=${3:-} @@ -76,8 +74,7 @@ function _dms_panic__general { dms_panic 'general' "${1:-}" "${2:-}" # `dms_panic` methods should be preferred if your failure type is supported. trap "exit 1" SIGUSR1 SCRIPT_PID=${$} -function _shutdown -{ +function _shutdown() { _log 'error' "${1:-_shutdown called without message}" _log 'error' 'Shutting down' @@ -91,13 +88,11 @@ function _shutdown # # This is mostly useful for debugging. It also helps when using something like `set -eE`, # as it shows where the script aborts. -function _trap_err_signal -{ +function _trap_err_signal() { trap '__log_unexpected_error "${FUNCNAME[0]:-}" "${BASH_COMMAND:-}" "${LINENO:-}" "${?:-}"' ERR # shellcheck disable=SC2317 - function __log_unexpected_error - { + function __log_unexpected_error() { local MESSAGE="Unexpected error occured :: script = ${SCRIPT:-${0}} " MESSAGE+=" | function = ${1:-none (global)}" MESSAGE+=" | command = ${2:-?}" diff --git a/target/scripts/helpers/index.sh b/target/scripts/helpers/index.sh index 1e919513de9..8d87673986f 100644 --- a/target/scripts/helpers/index.sh +++ b/target/scripts/helpers/index.sh @@ -3,8 +3,7 @@ # shellcheck source-path=target/scripts/helpers # This file serves as a single import for all helpers -function _import_scripts -{ +function _import_scripts() { local PATH_TO_SCRIPTS='/usr/local/bin/helpers' source "${PATH_TO_SCRIPTS}/accounts.sh" diff --git a/target/scripts/helpers/lock.sh b/target/scripts/helpers/lock.sh index 8c708225b1c..7156c3ec4f2 100644 --- a/target/scripts/helpers/lock.sh +++ b/target/scripts/helpers/lock.sh @@ -7,8 +7,7 @@ SCRIPT_NAME=$(basename "$0") # prevent removal by other instances of docker-mailserver LOCK_ID=$(uuid) -function _create_lock -{ +function _create_lock() { LOCK_FILE="/tmp/docker-mailserver/${SCRIPT_NAME}.lock" while [[ -e "${LOCK_FILE}" ]] do @@ -28,8 +27,7 @@ function _create_lock echo "${LOCK_ID}" >"${LOCK_FILE}" } -function _remove_lock -{ +function _remove_lock() { LOCK_FILE="${LOCK_FILE:-"/tmp/docker-mailserver/${SCRIPT_NAME}.lock"}" [[ -z "${LOCK_ID}" ]] && _exit_with_error "Cannot remove '${LOCK_FILE}' as there is no LOCK_ID set" if [[ -e "${LOCK_FILE}" ]] && grep -q "${LOCK_ID}" "${LOCK_FILE}"; then # Ensure we don't delete a lock that's not ours diff --git a/target/scripts/helpers/log.sh b/target/scripts/helpers/log.sh index a31d2408793..d771a3f563d 100644 --- a/target/scripts/helpers/log.sh +++ b/target/scripts/helpers/log.sh @@ -42,8 +42,7 @@ RESET=$(echo -ne '\e[0m') # If the first argument is not set or invalid, an error # message is logged. Likewise when the second argument # is missing. Both failures will return with exit code '1'. -function _log -{ +function _log() { if [[ -z ${1+set} ]]; then _log 'error' "Call to '_log' is missing a valid log level" return 1 @@ -106,8 +105,7 @@ function _log } # Like `_log` but adds a timestamp in front of the message. -function _log_with_date -{ +function _log_with_date() { _log "${1}" "$(date '+%Y-%m-%d %H:%M:%S') ${2}" } @@ -115,8 +113,7 @@ function _log_with_date # it is set. Otherwise, try to query the common environment # variables file. If this does not yield a value either, # use the default log level. -function _get_log_level_or_default -{ +function _get_log_level_or_default() { if [[ -n ${LOG_LEVEL+set} ]]; then echo "${LOG_LEVEL}" elif [[ -e /etc/dms-settings ]] && grep -q -E "^LOG_LEVEL='[a-z]+'" /etc/dms-settings; then @@ -128,7 +125,6 @@ function _get_log_level_or_default # This function checks whether the log level is the one # provided as the first argument. -function _log_level_is -{ +function _log_level_is() { [[ $(_get_log_level_or_default) =~ ^${1}$ ]] } diff --git a/target/scripts/helpers/network.sh b/target/scripts/helpers/network.sh index d424cb23031..afbe5a983a3 100644 --- a/target/scripts/helpers/network.sh +++ b/target/scripts/helpers/network.sh @@ -1,7 +1,6 @@ #!/bin/bash -function _mask_ip_digit -{ +function _mask_ip_digit() { if [[ ${1} -ge 8 ]]; then MASK=255 elif [[ ${1} -le 0 ]]; then @@ -21,8 +20,7 @@ function _mask_ip_digit # like 1.2.3.4/16 to subnet with cidr suffix # like 1.2.0.0/16. # Assumes correct IP and subnet are provided. -function _sanitize_ipv4_to_subnet_cidr -{ +function _sanitize_ipv4_to_subnet_cidr() { local DIGIT_PREFIX_LENGTH="${1#*/}" declare -a MASKED_DIGITS DIGITS diff --git a/target/scripts/helpers/postfix.sh b/target/scripts/helpers/postfix.sh index 918b7c41c58..47e57733480 100644 --- a/target/scripts/helpers/postfix.sh +++ b/target/scripts/helpers/postfix.sh @@ -17,8 +17,7 @@ # Should not be a concern for most types used by `docker-mailserver`: texthash, ldap, pcre, tcp, unionmap, unix. # The only other type in use by `docker-mailserver` is the hash type for /etc/aliases, which `postalias` handles. -function _create_postfix_vhost -{ +function _create_postfix_vhost() { # `main.cf` configures `virtual_mailbox_domains = /etc/postfix/vhost` # NOTE: Amavis also consumes this file. local DATABASE_VHOST='/etc/postfix/vhost' @@ -29,8 +28,7 @@ function _create_postfix_vhost } # Filter unique values into a proper DATABASE_VHOST config: -function _create_vhost -{ +function _create_vhost() { : >"${DATABASE_VHOST}" if [[ -f ${TMP_VHOST} ]]; then @@ -40,8 +38,7 @@ function _create_vhost } # Collects domains from configs (DATABASE_) into TMP_VHOST -function _vhost_collect_postfix_domains -{ +function _vhost_collect_postfix_domains() { local DATABASE_ACCOUNTS='/tmp/docker-mailserver/postfix-accounts.cf' local DATABASE_VIRTUAL='/tmp/docker-mailserver/postfix-virtual.cf' local DOMAIN UNAME @@ -75,8 +72,7 @@ function _vhost_collect_postfix_domains # - `main.cf:mydestination` setting removes `$mydestination` as an LDAP bugfix. # - `main.cf:virtual_mailbox_domains` uses `/etc/postfix/vhost`, but may # conditionally include a 2nd table (ldap:/etc/postfix/ldap-domains.cf). -function _vhost_ldap_support -{ +function _vhost_ldap_support() { [[ ${ACCOUNT_PROVISIONER} == 'LDAP' ]] && echo "${DOMAINNAME}" >>"${TMP_VHOST}" } diff --git a/target/scripts/helpers/relay.sh b/target/scripts/helpers/relay.sh index 24c69b0ecb0..4a1d8641469 100644 --- a/target/scripts/helpers/relay.sh +++ b/target/scripts/helpers/relay.sh @@ -53,15 +53,13 @@ # That shouldn't be a breaking change, as long as the mapping is maintained correctly. # TODO: RELAY_HOST should consider dropping `[]` and require the user to include that? # Future refactor for _populate_relayhost_map may warrant dropping these two ENV in favor of DEFAULT_RELAY_HOST? -function _env_relay_host -{ +function _env_relay_host() { echo "[${RELAY_HOST}]:${RELAY_PORT:-25}" } # Responsible for `postfix-sasl-password.cf` support: # `/etc/postfix/sasl_passwd` example at end of file. -function _relayhost_sasl -{ +function _relayhost_sasl() { if [[ ! -f /tmp/docker-mailserver/postfix-sasl-password.cf ]] \ && [[ -z ${RELAY_USER} || -z ${RELAY_PASSWORD} ]] then @@ -108,8 +106,7 @@ function _relayhost_sasl # to a separate transport (which can drop the `relayhost` setting) would be more appropriate. # TODO: With `sender_dependent_default_transport_maps`, we can extract out the excluded domains and route them through a separate transport. # while deprecating that support in favor of a transport config, similar to what is offered currently via sasl_passwd and relayhost_map. -function _populate_relayhost_map -{ +function _populate_relayhost_map() { # Create the relayhost_map config file: : >/etc/postfix/relayhost_map chown root:root /etc/postfix/relayhost_map @@ -142,8 +139,7 @@ function _populate_relayhost_map # map to a different relay-host, or use a separate transport (needs feature support added). # Args: - function _list_domain_parts - { + function _list_domain_parts() { [[ -f $2 ]] && sed -n -r "/${MATCH_VALID}/ ${1}" "${2}" } # Matches and outputs (capture group via `/\1/p`) the domain part (value of address after `@`) in the config file. @@ -167,16 +163,14 @@ function _populate_relayhost_map postconf 'sender_dependent_relayhost_maps = texthash:/etc/postfix/relayhost_map' } -function _relayhost_configure_postfix -{ +function _relayhost_configure_postfix() { postconf \ 'smtp_sasl_auth_enable = yes' \ 'smtp_sasl_security_options = noanonymous' \ 'smtp_tls_security_level = encrypt' } -function _setup_relayhost -{ +function _setup_relayhost() { _log 'debug' 'Setting up Postfix Relay Hosts' if [[ -n ${DEFAULT_RELAY_HOST} ]]; then @@ -194,8 +188,7 @@ function _setup_relayhost fi } -function _rebuild_relayhost -{ +function _rebuild_relayhost() { if [[ -n ${RELAY_HOST} ]]; then _relayhost_sasl _populate_relayhost_map diff --git a/target/scripts/helpers/ssl.sh b/target/scripts/helpers/ssl.sh index da2c7756d22..6a7610ad641 100644 --- a/target/scripts/helpers/ssl.sh +++ b/target/scripts/helpers/ssl.sh @@ -1,7 +1,6 @@ #!/bin/bash -function _setup_dhparam -{ +function _setup_dhparam() { local DH_SERVICE=$1 local DH_DEST=$2 local DH_CUSTOM='/tmp/docker-mailserver/dhparams.pem' @@ -18,8 +17,7 @@ function _setup_dhparam fi } -function _setup_ssl -{ +function _setup_ssl() { _log 'debug' 'Setting up SSL' local POSTFIX_CONFIG_MAIN='/etc/postfix/main.cf' @@ -31,8 +29,7 @@ function _setup_ssl mkdir -p "${DMS_TLS_PATH}" # Primary certificate to serve for TLS - function _set_certificate - { + function _set_certificate() { local POSTFIX_KEY_WITH_FULLCHAIN=${1} local DOVECOT_KEY=${1} local DOVECOT_CERT=${1} @@ -60,8 +57,7 @@ function _setup_ssl } # Enables supporting two certificate types such as ECDSA with an RSA fallback - function _set_alt_certificate - { + function _set_alt_certificate() { local COPY_KEY_FROM_PATH=$1 local COPY_CERT_FROM_PATH=$2 local PRIVATE_KEY_ALT="${DMS_TLS_PATH}/fallback_key" @@ -88,8 +84,7 @@ function _setup_ssl "${DOVECOT_CONFIG_SSL}" } - function _apply_tls_level - { + function _apply_tls_level() { local TLS_CIPHERS_ALLOW=$1 local TLS_PROTOCOL_IGNORE=$2 local TLS_PROTOCOL_MINIMUM=$3 @@ -113,8 +108,7 @@ function _setup_ssl # Extracts files `key.pem` and `fullchain.pem`. # `_extract_certs_from_acme` is located in `helpers/ssl.sh` # NOTE: See the `SSL_TYPE=letsencrypt` case below for more details. - function _traefik_support - { + function _traefik_support() { if [[ -f /etc/letsencrypt/acme.json ]]; then # Variable only intended for troubleshooting via debug output local EXTRACTED_DOMAIN @@ -379,8 +373,7 @@ function _setup_ssl # Identify a valid letsencrypt FQDN folder to use. -function _find_letsencrypt_domain -{ +function _find_letsencrypt_domain() { local LETSENCRYPT_DOMAIN if [[ -n ${SSL_DOMAIN} ]] && [[ -e /etc/letsencrypt/live/$(_strip_wildcard_prefix "${SSL_DOMAIN}")/fullchain.pem ]]; then @@ -398,8 +391,7 @@ function _find_letsencrypt_domain } # Verify the FQDN folder also includes a valid private key (`privkey.pem` for Certbot, `key.pem` for extraction by Traefik) -function _find_letsencrypt_key -{ +function _find_letsencrypt_key() { local LETSENCRYPT_KEY local LETSENCRYPT_DOMAIN=${1} @@ -419,8 +411,7 @@ function _find_letsencrypt_key echo "${LETSENCRYPT_KEY}" } -function _extract_certs_from_acme -{ +function _extract_certs_from_acme() { local CERT_DOMAIN=${1} if [[ -z ${CERT_DOMAIN} ]]; then _log 'warn' "_extract_certs_from_acme | CERT_DOMAIN is empty" diff --git a/target/scripts/helpers/utils.sh b/target/scripts/helpers/utils.sh index da093deba0d..59e0f63b39d 100644 --- a/target/scripts/helpers/utils.sh +++ b/target/scripts/helpers/utils.sh @@ -1,21 +1,18 @@ #!/bin/bash -function _escape -{ +function _escape() { echo "${1//./\\.}" } # Returns input after filtering out lines that are: # empty, white-space, comments (`#` as the first non-whitespace character) -function _get_valid_lines_from_file -{ +function _get_valid_lines_from_file() { grep --extended-regexp --invert-match "^\s*$|^\s*#" "${1}" || true } # Provide the name of an environment variable to this function # and it will return its value stored in /etc/dms-settings -function _get_dms_env_value -{ +function _get_dms_env_value() { if [[ -f /etc/dms-settings ]]; then grep "^${1}=" /etc/dms-settings | cut -d "'" -f 2 else @@ -30,8 +27,7 @@ function _get_dms_env_value # # `helpers/accounts.sh:_create_accounts` (mkdir, cp) appears to be the only writer to # /var/mail folders (used during startup and change detection handling). -function _chown_var_mail_if_necessary -{ +function _chown_var_mail_if_necessary() { # fix permissions, but skip this if 3 levels deep the user id is already set if find /var/mail -maxdepth 3 -a \( \! -user 5000 -o \! -group 5000 \) | read -r; then _log 'trace' 'Fixing /var/mail permissions' @@ -39,8 +35,7 @@ function _chown_var_mail_if_necessary fi } -function _require_n_parameters_or_print_usage -{ +function _require_n_parameters_or_print_usage() { local COUNT COUNT=${1} shift @@ -55,15 +50,13 @@ function _require_n_parameters_or_print_usage # After we modify the config explicitly, we can safely assume (reasonably) # that the write stream has completed, and it is safe to read the config. # https://github.com/docker-mailserver/docker-mailserver/issues/2985 -function _adjust_mtime_for_postfix_maincf -{ +function _adjust_mtime_for_postfix_maincf() { if [[ $(( $(date '+%s') - $(stat -c '%Y' '/etc/postfix/main.cf') )) -lt 2 ]]; then touch -d '2 seconds ago' /etc/postfix/main.cf fi } -function _reload_postfix -{ +function _reload_postfix() { _adjust_mtime_for_postfix_maincf postfix reload } @@ -92,8 +85,7 @@ function _reload_postfix # # 1. No first and second argument is supplied # 2. The second argument is a path to a file that does not exist -function _replace_by_env_in_file -{ +function _replace_by_env_in_file() { if [[ -z ${1+set} ]]; then _dms_panic__invalid_value 'first argument unset' 'utils.sh:_replace_by_env_in_file' elif [[ -z ${2+set} ]]; then @@ -123,8 +115,7 @@ function _replace_by_env_in_file # is not zero or one. # # @param ${1} = name of the ENV variable to check -function _env_var_expect_zero_or_one -{ +function _env_var_expect_zero_or_one() { local ENV_VAR_NAME=${1:?ENV var name must be provided to _env_var_expect_zero_or_one} [[ ${!ENV_VAR_NAME} =~ ^(0|1)$ ]] && return 0 @@ -138,8 +129,7 @@ function _env_var_expect_zero_or_one # is not an integer. # # @param ${1} = name of the ENV variable to check -function _env_var_expect_integer -{ +function _env_var_expect_integer() { local ENV_VAR_NAME=${1:?ENV var name must be provided to _env_var_expect_integer} [[ ${!ENV_VAR_NAME} =~ ^-?[0-9][0-9]*$ ]] && return 0 diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index 2ad4c2dabab..6a22e12d3c9 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -27,8 +27,7 @@ source /usr/local/bin/daemons-stack.sh # ? >> Registering functions # ------------------------------------------------------------ -function _register_functions -{ +function _register_functions() { _log 'debug' 'Registering functions' # ? >> Checks diff --git a/target/scripts/startup/check-stack.sh b/target/scripts/startup/check-stack.sh index 05b76c45715..acefa55d9be 100644 --- a/target/scripts/startup/check-stack.sh +++ b/target/scripts/startup/check-stack.sh @@ -2,14 +2,12 @@ declare -a FUNCS_CHECK -function _register_check_function -{ +function _register_check_function() { FUNCS_CHECK+=("${1}") _log 'trace' "${1}() registered" } -function _check -{ +function _check() { _log 'info' 'Checking configuration' for FUNC in "${FUNCS_CHECK[@]}" do @@ -17,8 +15,7 @@ function _check done } -function _check_improper_restart -{ +function _check_improper_restart() { _log 'debug' 'Checking for improper restart' if [[ -f /CONTAINER_START ]]; then @@ -27,8 +24,7 @@ function _check_improper_restart fi } -function _check_hostname -{ +function _check_hostname() { _log 'debug' 'Checking that hostname/domainname is provided or overridden' _log 'debug' "Domain has been set to ${DOMAINNAME}" @@ -40,8 +36,7 @@ function _check_hostname fi } -function _check_log_level -{ +function _check_log_level() { if [[ ${LOG_LEVEL} == 'trace' ]] \ || [[ ${LOG_LEVEL} == 'debug' ]] \ || [[ ${LOG_LEVEL} == 'info' ]] \ diff --git a/target/scripts/startup/daemons-stack.sh b/target/scripts/startup/daemons-stack.sh index ae88dc24937..d8d6f808c4e 100644 --- a/target/scripts/startup/daemons-stack.sh +++ b/target/scripts/startup/daemons-stack.sh @@ -2,14 +2,12 @@ declare -a DAEMONS_START -function _register_start_daemon -{ +function _register_start_daemon() { DAEMONS_START+=("${1}") _log 'trace' "${1}() registered" } -function _start_daemons -{ +function _start_daemons() { _log 'info' 'Starting daemons' for FUNCTION in "${DAEMONS_START[@]}" @@ -18,8 +16,7 @@ function _start_daemons done } -function _default_start_daemon -{ +function _default_start_daemon() { _log 'debug' "Starting ${1:?}" local RESULT @@ -47,19 +44,16 @@ function _start_daemon_rspamd_redis { _default_start_daemon 'rspamd-redis' ; function _start_daemon_rsyslog { _default_start_daemon 'rsyslog' ; } function _start_daemon_update_check { _default_start_daemon 'update-check' ; } -function _start_daemon_saslauthd -{ +function _start_daemon_saslauthd() { _default_start_daemon "saslauthd_${SASLAUTHD_MECHANISMS}" } -function _start_daemon_postfix -{ +function _start_daemon_postfix() { _adjust_mtime_for_postfix_maincf _default_start_daemon 'postfix' } -function _start_daemon_fetchmail -{ +function _start_daemon_fetchmail() { if [[ ${FETCHMAIL_PARALLEL} -eq 1 ]]; then local COUNTER=0 for _ in /etc/fetchmailrc.d/fetchmail-*.rc diff --git a/target/scripts/startup/setup-stack.sh b/target/scripts/startup/setup-stack.sh index e31b846411a..a83ae5c62eb 100644 --- a/target/scripts/startup/setup-stack.sh +++ b/target/scripts/startup/setup-stack.sh @@ -2,14 +2,12 @@ declare -a FUNCS_SETUP -function _register_setup_function -{ +function _register_setup_function() { FUNCS_SETUP+=("${1}") _log 'trace' "${1}() registered" } -function _setup -{ +function _setup() { # Requires `shopt -s globstar` because of `**` which in # turn is required as we're decending through directories for FILE in /usr/local/bin/setup.d/**/*.sh @@ -28,8 +26,7 @@ function _setup _prepare_for_change_detection } -function _early_supervisor_setup -{ +function _early_supervisor_setup() { SUPERVISOR_LOGLEVEL="${SUPERVISOR_LOGLEVEL:-warn}" if ! grep -q "loglevel = ${SUPERVISOR_LOGLEVEL}" /etc/supervisor/supervisord.conf; then @@ -56,8 +53,7 @@ function _early_supervisor_setup return 0 } -function _setup_timezone -{ +function _setup_timezone() { [[ -n ${TZ} ]] || return 0 _log 'debug' "Setting timezone to '${TZ}'" @@ -78,8 +74,7 @@ function _setup_timezone fi } -function _setup_apply_fixes_after_configuration -{ +function _setup_apply_fixes_after_configuration() { _log 'trace' 'Removing leftover PID files from a stop/start' find /var/run/ -not -name 'supervisord.pid' -name '*.pid' -delete touch /dev/shm/supervisor.sock @@ -93,8 +88,7 @@ function _setup_apply_fixes_after_configuration rm -rf /var/mail-state/spool-postfix/{dev,etc,lib,pid,usr,private/auth} } -function _run_user_patches -{ +function _run_user_patches() { local USER_PATCHES='/tmp/docker-mailserver/user-patches.sh' if [[ -f ${USER_PATCHES} ]]; then diff --git a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh index 85de88cd7fc..20b88ea5cc8 100644 --- a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh +++ b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh @@ -6,8 +6,7 @@ # # The OpenDKIM milter must come before the OpenDMARC milter in Postfix's # `smtpd_milters` milters options. -function _setup_opendkim -{ +function _setup_opendkim() { if [[ ${ENABLE_OPENDKIM} -eq 1 ]]; then _log 'debug' 'Configuring DKIM' @@ -54,8 +53,7 @@ function _setup_opendkim # # The OpenDMARC milter must come after the OpenDKIM milter in Postfix's # `smtpd_milters` milters options. -function _setup_opendmarc -{ +function _setup_opendmarc() { if [[ ${ENABLE_OPENDMARC} -eq 1 ]]; then # TODO When disabling SPF is possible, add a check whether DKIM and SPF is disabled # for DMARC to work, you should have at least one enabled @@ -83,8 +81,7 @@ function _setup_opendmarc # Configures the SPF check inside Postfix's configuration via policyd-spf. When # using Rspamd, you will likely want to turn that off. -function _setup_policyd_spf -{ +function _setup_policyd_spf() { if [[ ${ENABLE_POLICYD_SPF} -eq 1 ]]; then _log 'debug' 'Configuring policyd-spf' cat >>/etc/postfix/master.cf <= 0 ; COUNTER-- )) @@ -169,8 +164,7 @@ function _setup_dovecot_local_user _create_accounts } -function _setup_dovecot_inet_protocols -{ +function _setup_dovecot_inet_protocols() { [[ ${DOVECOT_INET_PROTOCOLS} == 'all' ]] && return 0 _log 'trace' 'Setting up DOVECOT_INET_PROTOCOLS option' @@ -189,13 +183,11 @@ function _setup_dovecot_inet_protocols sedfile -i "s|^#listen =.*|listen = ${PROTOCOL}|g" /etc/dovecot/dovecot.conf } -function _setup_dovecot_dhparam -{ +function _setup_dovecot_dhparam() { _setup_dhparam 'Dovecot' '/etc/dovecot/dh.pem' } -function _setup_dovecot_hostname -{ +function _setup_dovecot_hostname() { _log 'debug' 'Applying hostname to Dovecot' sed -i "s|^#hostname =.*$|hostname = '${HOSTNAME}'|g" /etc/dovecot/conf.d/15-lda.conf } diff --git a/target/scripts/startup/setup.d/fetchmail.sh b/target/scripts/startup/setup.d/fetchmail.sh index 99b74d84765..1d5b9570f8a 100644 --- a/target/scripts/startup/setup.d/fetchmail.sh +++ b/target/scripts/startup/setup.d/fetchmail.sh @@ -1,7 +1,6 @@ #!/bin/bash -function _setup_fetchmail -{ +function _setup_fetchmail() { if [[ ${ENABLE_FETCHMAIL} -eq 1 ]]; then _log 'trace' 'Enabling and configuring Fetchmail' @@ -23,8 +22,7 @@ function _setup_fetchmail fi } -function _setup_fetchmail_parallel -{ +function _setup_fetchmail_parallel() { if [[ ${FETCHMAIL_PARALLEL} -eq 1 ]]; then _log 'trace' 'Enabling and configuring Fetchmail parallel' mkdir /etc/fetchmailrc.d/ @@ -35,8 +33,7 @@ function _setup_fetchmail_parallel # # The sole purpose for this is to work around what is known # as the Fetchmail IMAP idle issue. - function _fetchmailrc_split - { + function _fetchmailrc_split() { local FETCHMAILRC='/etc/fetchmailrc' local FETCHMAILRCD='/etc/fetchmailrc.d' local DEFAULT_FILE="${FETCHMAILRCD}/defaults" diff --git a/target/scripts/startup/setup.d/getmail.sh b/target/scripts/startup/setup.d/getmail.sh index cf4547d48c9..ede7561ee9b 100644 --- a/target/scripts/startup/setup.d/getmail.sh +++ b/target/scripts/startup/setup.d/getmail.sh @@ -1,7 +1,6 @@ #!/bin/bash -function _setup_getmail -{ +function _setup_getmail() { if [[ ${ENABLE_GETMAIL} -eq 1 ]]; then _log 'trace' 'Preparing Getmail configuration' diff --git a/target/scripts/startup/setup.d/ldap.sh b/target/scripts/startup/setup.d/ldap.sh index 36c60fad81f..349fd63f5da 100644 --- a/target/scripts/startup/setup.d/ldap.sh +++ b/target/scripts/startup/setup.d/ldap.sh @@ -1,7 +1,6 @@ #!/bin/bash -function _setup_ldap -{ +function _setup_ldap() { _log 'debug' 'Setting up LDAP' _log 'trace' 'Checking for custom configs' diff --git a/target/scripts/startup/setup.d/log.sh b/target/scripts/startup/setup.d/log.sh index b87c61301ef..eef76a3f4bf 100644 --- a/target/scripts/startup/setup.d/log.sh +++ b/target/scripts/startup/setup.d/log.sh @@ -1,7 +1,6 @@ #!/bin/bash -function _setup_logs_general -{ +function _setup_logs_general() { _log 'debug' 'Setting up general log files' # File/folder permissions are fine when using docker volumes, but may be wrong @@ -11,8 +10,7 @@ function _setup_logs_general chown syslog:root /var/log/mail } -function _setup_logrotate -{ +function _setup_logrotate() { _log 'debug' 'Setting up logrotate' LOGROTATE='/var/log/mail/mail.log\n{\n compress\n copytruncate\n delaycompress\n' @@ -42,8 +40,7 @@ function _setup_logrotate echo -e "${LOGROTATE}}" >/etc/logrotate.d/maillog } -function _setup_mail_summary -{ +function _setup_mail_summary() { local ENABLED_MESSAGE ENABLED_MESSAGE="Enabling Postfix log summary reports with recipient '${PFLOGSUMM_RECIPIENT}'" @@ -80,8 +77,7 @@ EOF esac } -function _setup_logwatch -{ +function _setup_logwatch() { echo 'LogFile = /var/log/mail/freshclam.log' >>/etc/logwatch/conf/logfiles/clam-update.conf echo "MailFrom = ${LOGWATCH_SENDER}" >>/etc/logwatch/conf/logwatch.conf echo "Mailer = \"sendmail -t -f ${LOGWATCH_SENDER}\"" >>/etc/logwatch/conf/logwatch.conf diff --git a/target/scripts/startup/setup.d/mail_state.sh b/target/scripts/startup/setup.d/mail_state.sh index aa59ac62c5b..c3f565645d6 100644 --- a/target/scripts/startup/setup.d/mail_state.sh +++ b/target/scripts/startup/setup.d/mail_state.sh @@ -2,8 +2,7 @@ # Consolidate all states into a single directory # (/var/mail-state) to allow persistence using docker volumes -function _setup_save_states -{ +function _setup_save_states() { local DEST DESTDIR STATEDIR SERVICEDIR SERVICEDIRS SERVICEFILE SERVICEFILES STATEDIR='/var/mail-state' diff --git a/target/scripts/startup/setup.d/networking.sh b/target/scripts/startup/setup.d/networking.sh index d8669ac3894..8bf019e9a62 100644 --- a/target/scripts/startup/setup.d/networking.sh +++ b/target/scripts/startup/setup.d/networking.sh @@ -1,13 +1,11 @@ #!/bin/bash -function _setup_mailname -{ +function _setup_mailname() { _log 'debug' "Setting up mailname and creating '/etc/mailname'" echo "${DOMAINNAME}" >/etc/mailname } -function _setup_docker_permit -{ +function _setup_docker_permit() { _log 'debug' 'Setting up PERMIT_DOCKER option' local CONTAINER_IP CONTAINER_NETWORK @@ -29,14 +27,12 @@ function _setup_docker_permit CONTAINER_NETWORKS+=("${IP}") done < <(ip -o -4 addr show type veth | grep -E -o '[0-9\.]+/[0-9]+') - function __clear_postfix_mynetworks - { + function __clear_postfix_mynetworks() { _log 'trace' "Clearing Postfix's 'mynetworks'" postconf "mynetworks =" } - function __add_to_postfix_mynetworks - { + function __add_to_postfix_mynetworks() { local NETWORK_TYPE=$1 local NETWORK=$2 diff --git a/target/scripts/startup/setup.d/postfix.sh b/target/scripts/startup/setup.d/postfix.sh index 3ae6741e43c..5a57ee7c98c 100644 --- a/target/scripts/startup/setup.d/postfix.sh +++ b/target/scripts/startup/setup.d/postfix.sh @@ -7,8 +7,7 @@ # @param ${2} = message function __postfix__log { _log "${1:-}" "(Postfix setup) ${2:-}" ; } -function _setup_postfix_early -{ +function _setup_postfix_early() { _log 'debug' 'Configuring Postfix (early setup)' __postfix__log 'trace' 'Applying hostname and domainname' @@ -66,8 +65,7 @@ EOF fi } -function _setup_postfix_late -{ +function _setup_postfix_late() { _log 'debug' 'Configuring Postfix (late setup)' __postfix__log 'trace' 'Configuring user access' @@ -91,8 +89,7 @@ function _setup_postfix_late __postfix__setup_override_configuration } -function __postfix__setup_override_configuration -{ +function __postfix__setup_override_configuration() { __postfix__log 'debug' 'Overriding / adjusting configuration with user-supplied values' if [[ -f /tmp/docker-mailserver/postfix-main.cf ]]; then @@ -122,8 +119,7 @@ function __postfix__setup_override_configuration fi } -function _setup_SRS -{ +function _setup_SRS() { _log 'debug' 'Setting up SRS' postconf 'sender_canonical_maps = tcp:localhost:10001' @@ -131,8 +127,7 @@ function _setup_SRS postconf 'recipient_canonical_maps = tcp:localhost:10002' postconf 'recipient_canonical_classes = envelope_recipient,header_recipient' - function __generate_secret - { + function __generate_secret() { ( umask 0077 dd if=/dev/urandom bs=24 count=1 2>/dev/null | base64 -w0 >"${1}" diff --git a/target/scripts/startup/setup.d/saslauthd.sh b/target/scripts/startup/setup.d/saslauthd.sh index bbe3358cf1a..12f00726a04 100644 --- a/target/scripts/startup/setup.d/saslauthd.sh +++ b/target/scripts/startup/setup.d/saslauthd.sh @@ -1,8 +1,7 @@ #!/bin/bash -function _setup_saslauthd -{ +function _setup_saslauthd() { _log 'debug' 'Setting up SASLAUTHD' if [[ ! -f /etc/saslauthd.conf ]]; then diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index 863a3afc5ec..fe09c8a4a28 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -1,7 +1,6 @@ #!/bin/bash -function _setup_security_stack -{ +function _setup_security_stack() { _log 'debug' 'Setting up Security Stack' __setup__security__postgrey @@ -23,8 +22,7 @@ function _setup_security_stack __setup__security__amavis } -function __setup__security__postgrey -{ +function __setup__security__postgrey() { if [[ ${ENABLE_POSTGREY} -eq 1 ]]; then _log 'debug' 'Enabling and configuring Postgrey' @@ -52,8 +50,7 @@ function __setup__security__postgrey fi } -function __setup__security__postscreen -{ +function __setup__security__postscreen() { _log 'debug' 'Configuring Postscreen' sed -i \ -e "s|postscreen_dnsbl_action = enforce|postscreen_dnsbl_action = ${POSTSCREEN_ACTION}|" \ @@ -69,8 +66,7 @@ function __setup__security__postscreen fi } -function __setup__security__spamassassin -{ +function __setup__security__spamassassin() { if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]]; then _log 'debug' 'Enabling and configuring SpamAssassin' @@ -157,8 +153,7 @@ EOF fi } -function __setup__security__clamav -{ +function __setup__security__clamav() { if [[ ${ENABLE_CLAMAV} -eq 1 ]]; then _log 'debug' 'Enabling and configuring ClamAV' @@ -197,8 +192,7 @@ function __setup__security__clamav fi } -function __setup__security__fail2ban -{ +function __setup__security__fail2ban() { if [[ ${ENABLE_FAIL2BAN} -eq 1 ]]; then _log 'debug' 'Enabling and configuring Fail2Ban' @@ -221,8 +215,7 @@ function __setup__security__fail2ban fi } -function __setup__security__amavis -{ +function __setup__security__amavis() { if [[ ${ENABLE_AMAVIS} -eq 1 ]]; then _log 'debug' 'Configuring Amavis' if [[ -f /tmp/docker-mailserver/amavis.cf ]]; then @@ -257,8 +250,7 @@ function __setup__security__amavis } # We can use Sieve to move spam emails to the "Junk" folder. -function _setup_spam_to_junk -{ +function _setup_spam_to_junk() { if [[ ${MOVE_SPAM_TO_JUNK} -eq 1 ]]; then _log 'debug' 'Spam emails will be moved to the Junk folder' cat >/usr/lib/dovecot/sieve-global/after/spam_to_junk.sieve << EOF diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 6c0f39034ea..e56433b05fc 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -1,8 +1,7 @@ #!/bin/bash # Function called during global setup to handle the complete setup of Rspamd. -function _setup_rspamd -{ +function _setup_rspamd() { if _env_var_expect_zero_or_one 'ENABLE_RSPAMD' && [[ ${ENABLE_RSPAMD} -eq 1 ]]; then _log 'debug' 'Enabling and configuring Rspamd' __rspamd__log 'trace' '---------- Setup started ----------' @@ -36,8 +35,7 @@ function __rspamd__log { _log "${1:-}" "(Rspamd setup) ${2:-}" ; } # @param ${2} = `true` when you want to enable the module (default), # `false` when you want to disable the module [OPTIONAL] # @param ${3} = whether to use `local` (default) or `override` [OPTIONAL] -function __rspamd__helper__enable_disable_module -{ +function __rspamd__helper__enable_disable_module() { local MODULE=${1:?Module name must be provided} local ENABLE_MODULE=${2:-true} local LOCAL_OR_OVERRIDE=${3:-local} @@ -61,8 +59,7 @@ EOF # Run miscellaneous early setup tasks and checks, such as creating files needed at runtime # or checking for other anti-spam/anti-virus software. -function __rspamd__run_early_setup_and_checks -{ +function __rspamd__run_early_setup_and_checks() { # Note: Variables not marked with `local` are # used in other functions as well. RSPAMD_LOCAL_D='/etc/rspamd/local.d' @@ -105,8 +102,7 @@ function __rspamd__run_early_setup_and_checks # Sets up Redis. In case the user does not use a dedicated Redis instance, we # supply a configuration for our local Redis instance which is started later. -function __rspamd__setup_redis -{ +function __rspamd__setup_redis() { if _env_var_expect_zero_or_one 'ENABLE_RSPAMD_REDIS' && [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]]; then __rspamd__log 'debug' 'Internal Redis is enabled, adding configuration' cat >"${RSPAMD_LOCAL_D}/redis.conf" << "EOF" @@ -136,8 +132,7 @@ EOF # Adjust Postfix's configuration files. We only need to append Rspamd at the end of # `smtpd_milters` in `/etc/postfix/main.cf`. -function __rspamd__setup_postfix -{ +function __rspamd__setup_postfix() { __rspamd__log 'debug' "Adjusting Postfix's configuration" postconf 'rspamd_milter = inet:localhost:11332' @@ -146,8 +141,7 @@ function __rspamd__setup_postfix } # If ClamAV is enabled, we will integrate it into Rspamd. -function __rspamd__setup_clamav -{ +function __rspamd__setup_clamav() { if _env_var_expect_zero_or_one 'ENABLE_CLAMAV' && [[ ${ENABLE_CLAMAV} -eq 1 ]]; then __rspamd__log 'debug' 'Enabling ClamAV integration' sedfile -i -E 's|^(enabled).*|\1 = true;|g' "${RSPAMD_LOCAL_D}/antivirus.conf" @@ -169,8 +163,7 @@ function __rspamd__setup_clamav # We disable the modules listed in `DISABLE_MODULES` as we believe these modules # are not commonly used and the average user does not need them. As a consequence, # disabling them saves resources. -function __rspamd__setup_default_modules -{ +function __rspamd__setup_default_modules() { __rspamd__log 'debug' 'Disabling default modules' # This array contains all the modules we disable by default. They @@ -197,8 +190,7 @@ function __rspamd__setup_default_modules # 1. enabling auto-learn for the classifier-bayes module # 2. setting up sieve scripts that detect when a user is moving e-mail # from or to the "Junk" folder, and learning them as ham or spam. -function __rspamd__setup_learning -{ +function __rspamd__setup_learning() { if _env_var_expect_zero_or_one 'RSPAMD_LEARN' && [[ ${RSPAMD_LEARN} -eq 1 ]]; then __rspamd__log 'debug' 'Setting up intelligent learning of spam and ham' @@ -241,8 +233,7 @@ EOF # Sets up greylisting with the greylisting module (see # https://rspamd.com/doc/modules/greylisting.html). -function __rspamd__setup_greylisting -{ +function __rspamd__setup_greylisting() { if _env_var_expect_zero_or_one 'RSPAMD_GREYLISTING' && [[ ${RSPAMD_GREYLISTING} -eq 1 ]]; then __rspamd__log 'debug' 'Enabling greylisting' sedfile -i -E "s|(enabled =).*|\1 true;|g" "${RSPAMD_LOCAL_D}/greylist.conf" @@ -255,8 +246,7 @@ function __rspamd__setup_greylisting # https://www.rspamd.com/doc/modules/hfilter.html). This module is mainly # used for hostname checks, and whether or not a reverse-DNS check # succeeds. -function __rspamd__setup_hfilter_group -{ +function __rspamd__setup_hfilter_group() { local MODULE_FILE="${RSPAMD_LOCAL_D}/hfilter_group.conf" if _env_var_expect_zero_or_one 'RSPAMD_HFILTER' && [[ ${RSPAMD_HFILTER} -eq 1 ]]; then __rspamd__log 'debug' 'Hfilter (group) module is enabled' @@ -278,8 +268,7 @@ function __rspamd__setup_hfilter_group # Parses `RSPAMD_CUSTOM_COMMANDS_FILE` and executed the directives given by the file. # To get a detailed explanation of the commands and how the file works, visit # https://docker-mailserver.github.io/docker-mailserver/edge/config/security/rspamd/#with-the-help-of-a-custom-file -function __rspamd__handle_user_modules_adjustments -{ +function __rspamd__handle_user_modules_adjustments() { # Adds an option with a corresponding value to a module, or, in case the option # is already present, overwrites it. # @@ -293,8 +282,7 @@ function __rspamd__handle_user_modules_adjustments # While this function is currently bound to the scope of `__rspamd__handle_user_modules_adjustments`, # it is written in a versatile way (taking 4 arguments instead of assuming `ARGUMENT2` / `ARGUMENT3` # are set) so that it may be used elsewhere if needed. - function __add_or_replace - { + function __add_or_replace() { local MODULE_FILE=${1:?Module file name must be provided} local MODULE_LOG_NAME=${2:?Module log name must be provided} local OPTION=${3:?Option name must be provided} diff --git a/target/scripts/startup/setup.d/security/spoofing.sh b/target/scripts/startup/setup.d/security/spoofing.sh index 4a569ff4b89..7c38821d50b 100644 --- a/target/scripts/startup/setup.d/security/spoofing.sh +++ b/target/scripts/startup/setup.d/security/spoofing.sh @@ -1,7 +1,6 @@ #!/bin/bash -function _setup_spoof_protection -{ +function _setup_spoof_protection() { if [[ ${SPOOF_PROTECTION} -eq 1 ]]; then _log 'trace' 'Enabling and configuring spoof protection' diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index f180c7a4e9c..e59119abc85 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -3,8 +3,7 @@ # shellcheck disable=SC2034 declare -A VARS -function _early_variables_setup -{ +function _early_variables_setup() { _obtain_hostname_and_domainname __environment_variables_backwards_compatibility __environment_variables_general_setup @@ -13,8 +12,7 @@ function _early_variables_setup # This function handles variables that are deprecated. This allows a # smooth transition period, without the need of removing a variable # completely with a single version. -function __environment_variables_backwards_compatibility -{ +function __environment_variables_backwards_compatibility() { if [[ ${ENABLE_LDAP:-0} -eq 1 ]]; then _log 'warn' "'ENABLE_LDAP=1' is deprecated (and will be removed in v13.0.0) => use 'ACCOUNT_PROVISIONER=LDAP' instead" ACCOUNT_PROVISIONER='LDAP' @@ -32,8 +30,7 @@ function __environment_variables_backwards_compatibility # This function sets almost all environment variables. This involves setting # a default if no value was provided and writing the variable and its value # to the VARS map. -function __environment_variables_general_setup -{ +function __environment_variables_general_setup() { _log 'debug' 'Handling general environment variable setup' # these variables must be defined first @@ -143,8 +140,7 @@ function __environment_variables_general_setup } # This function handles environment variables related to LDAP. -function _environment_variables_ldap -{ +function _environment_variables_ldap() { _log 'debug' 'Setting LDAP-related environment variables now' VARS[LDAP_BIND_DN]="${LDAP_BIND_DN:=}" @@ -156,8 +152,7 @@ function _environment_variables_ldap # This function handles environment variables related to SASLAUTHD # and, if activated, variables related to SASLAUTHD and LDAP. -function _environment_variables_saslauthd -{ +function _environment_variables_saslauthd() { _log 'debug' 'Setting SASLAUTHD-related environment variables now' VARS[SASLAUTHD_MECHANISMS]="${SASLAUTHD_MECHANISMS:=pam}" @@ -210,8 +205,7 @@ function _environment_variables_saslauthd # This function Writes the contents of the `VARS` map (associative array) # to locations where they can be sourced from (e.g. `/etc/dms-settings`) # or where they can be used by Bash directly (e.g. `/root/.bashrc`). -function _environment_variables_export -{ +function _environment_variables_export() { _log 'debug' "Exporting environment variables now (creating '/etc/dms-settings')" : >/root/.bashrc # make DMS variables available in login shells and their subprocesses diff --git a/test/helper/common.bash b/test/helper/common.bash index 46c0b99733b..34b5f97ca05 100644 --- a/test/helper/common.bash +++ b/test/helper/common.bash @@ -396,8 +396,7 @@ function _container_is_running() { # # @param ${1} = directory # @param ${2} = number of files that should be in ${1} -function _count_files_in_directory_in_container() -{ +function _count_files_in_directory_in_container() { local DIRECTORY=${1:?No directory provided} local NUMBER_OF_LINES=${2:?No line count provided} diff --git a/test/linting/lint.sh b/test/linting/lint.sh index 032691e1b13..3bcb04cd242 100755 --- a/test/linting/lint.sh +++ b/test/linting/lint.sh @@ -16,8 +16,7 @@ SHELLCHECK_VERSION='0.9.0' # shellcheck source=./../../target/scripts/helpers/log.sh source "${REPOSITORY_ROOT}/target/scripts/helpers/log.sh" -function _eclint -{ +function _eclint() { if docker run --rm --tty \ --volume "${REPOSITORY_ROOT}:/ci:ro" \ --workdir "/ci" \ @@ -31,8 +30,7 @@ function _eclint fi } -function _hadolint -{ +function _hadolint() { if docker run --rm --tty \ --volume "${REPOSITORY_ROOT}:/ci:ro" \ --workdir "/ci" \ @@ -46,8 +44,7 @@ function _hadolint fi } -function _shellcheck -{ +function _shellcheck() { local F_SH F_BIN F_BATS # File paths for shellcheck: @@ -119,8 +116,7 @@ function _shellcheck fi } -function _main -{ +function _main() { case "${1:-}" in ( 'eclint' ) _eclint ;; ( 'hadolint' ) _hadolint ;; From c2d0b748b24814db1eb2b827b36d17f5d95bd31c Mon Sep 17 00:00:00 2001 From: Casper Date: Fri, 26 May 2023 01:39:39 +0200 Subject: [PATCH 091/592] Change 'while' style (#3365) --- setup.sh | 3 +-- target/bin/delmailuser | 3 +-- target/bin/listalias | 3 +-- target/bin/listdovecotmasteruser | 3 +-- target/bin/listmailuser | 3 +-- target/bin/open-dkim | 6 ++---- target/bin/rspamd-dkim | 3 +-- target/scripts/check-for-changes.sh | 3 +-- target/scripts/helpers/accounts.sh | 9 +++------ target/scripts/helpers/lock.sh | 3 +-- target/scripts/helpers/postfix.sh | 6 ++---- target/scripts/helpers/relay.sh | 3 +-- target/scripts/helpers/utils.sh | 3 +-- target/scripts/startup/setup.d/fetchmail.sh | 3 +-- target/scripts/startup/setup.d/networking.sh | 3 +-- target/scripts/startup/setup.d/postfix.sh | 3 +-- target/scripts/startup/setup.d/security/rspamd.sh | 3 +-- target/scripts/update-check.sh | 3 +-- 18 files changed, 22 insertions(+), 44 deletions(-) diff --git a/setup.sh b/setup.sh index 7173968649e..6a89a436dde 100755 --- a/setup.sh +++ b/setup.sh @@ -124,8 +124,7 @@ function _main() { _set_default_config_path local OPTIND - while getopts ":c:i:p:zZR" OPT - do + while getopts ":c:i:p:zZR" OPT; do case ${OPT} in ( i ) IMAGE_NAME="${OPTARG}" ;; ( z | Z ) USE_SELINUX=":${OPT}" ;; diff --git a/target/bin/delmailuser b/target/bin/delmailuser index 974a237518c..d796f2a77b9 100755 --- a/target/bin/delmailuser +++ b/target/bin/delmailuser @@ -70,8 +70,7 @@ ${ORANGE}EXIT STATUS${RESET} } function _parse_options() { - while getopts ":yY" OPT - do + while getopts ":yY" OPT; do case "${OPT}" in ( 'y' | 'Y' ) MAILDEL=1 diff --git a/target/bin/listalias b/target/bin/listalias index efb48fd29ee..7081ea48f83 100755 --- a/target/bin/listalias +++ b/target/bin/listalias @@ -12,8 +12,7 @@ function _list_entries() { local DATABASE=${1} _db_should_exist_with_content "${DATABASE}" - while read -r LINE - do + while read -r LINE; do echo -e "* ${LINE}\n" done < <(_get_valid_lines_from_file "${DATABASE}") } diff --git a/target/bin/listdovecotmasteruser b/target/bin/listdovecotmasteruser index 214befd3e34..193eaa9604d 100755 --- a/target/bin/listdovecotmasteruser +++ b/target/bin/listdovecotmasteruser @@ -13,8 +13,7 @@ function _list_entries() { _db_should_exist_with_content "${DATABASE}" local MASTER_ACCOUNT - while read -r LINE - do + while read -r LINE; do MASTER_ACCOUNT=$(echo "${LINE}" | cut -d'|' -f1) echo -e "* ${MASTER_ACCOUNT}\n" diff --git a/target/bin/listmailuser b/target/bin/listmailuser index c1ab9d8e9db..1253c448eef 100755 --- a/target/bin/listmailuser +++ b/target/bin/listmailuser @@ -19,8 +19,7 @@ function _list_entries() { _db_should_exist_with_content "${DATABASE}" local ENTRY_TO_DISPLAY - while read -r LINE - do + while read -r LINE; do ENTRY_TO_DISPLAY=$(_format_list_item "${LINE}") echo -e "* ${ENTRY_TO_DISPLAY}\n" diff --git a/target/bin/open-dkim b/target/bin/open-dkim index c3e656786a6..50b18f6c96a 100755 --- a/target/bin/open-dkim +++ b/target/bin/open-dkim @@ -57,8 +57,7 @@ ${ORANGE}EXIT STATUS${RESET} _require_n_parameters_or_print_usage 0 "${@}" -while [[ ${#} -gt 0 ]] -do +while [[ ${#} -gt 0 ]]; do case "${1}" in ( 'keysize' ) if [[ -n ${2+set} ]]; then @@ -124,8 +123,7 @@ if [[ ! -s ${DATABASE_VHOST} ]]; then exit 0 fi -while read -r DKIM_DOMAIN -do +while read -r DKIM_DOMAIN; do mkdir -p "/tmp/docker-mailserver/opendkim/keys/${DKIM_DOMAIN}" if [[ ! -f "/tmp/docker-mailserver/opendkim/keys/${DKIM_DOMAIN}/${SELECTOR}.private" ]]; then diff --git a/target/bin/rspamd-dkim b/target/bin/rspamd-dkim index e4250d8ba8b..7aa767731a2 100755 --- a/target/bin/rspamd-dkim +++ b/target/bin/rspamd-dkim @@ -79,8 +79,7 @@ function _parse_arguments() { _log 'trace' "Options given to this script: '${*}'" - while [[ ${#} -gt 0 ]] - do + while [[ ${#} -gt 0 ]]; do case "${1}" in ( 'keytype' ) diff --git a/target/scripts/check-for-changes.sh b/target/scripts/check-for-changes.sh index 58f0d96f541..a8cdeb6870d 100755 --- a/target/scripts/check-for-changes.sh +++ b/target/scripts/check-for-changes.sh @@ -174,8 +174,7 @@ function _ssl_changes() { # They presently have no special handling other than to trigger a change that will restart Postfix/Dovecot. } -while true -do +while true; do _check_for_changes sleep 2 done diff --git a/target/scripts/helpers/accounts.sh b/target/scripts/helpers/accounts.sh index 3637ff4f942..82b626c1511 100644 --- a/target/scripts/helpers/accounts.sh +++ b/target/scripts/helpers/accounts.sh @@ -38,8 +38,7 @@ function _create_accounts() { # creating users ; 'pass' is encrypted # comments and empty lines are ignored local LOGIN PASS USER_ATTRIBUTES - while IFS=$'|' read -r LOGIN PASS USER_ATTRIBUTES - do + while IFS=$'|' read -r LOGIN PASS USER_ATTRIBUTES; do # Setting variables for better readability USER=$(echo "${LOGIN}" | cut -d @ -f1) DOMAIN=$(echo "${LOGIN}" | cut -d @ -f2) @@ -104,8 +103,7 @@ function _create_dovecot_alias_dummy_accounts() { # adding aliases to Dovecot's userdb # ${REAL_FQUN} is a user's fully-qualified username local ALIAS REAL_FQUN DOVECOT_USERDB_LINE - while read -r ALIAS REAL_FQUN - do + while read -r ALIAS REAL_FQUN; do # alias is assumed to not be a proper e-mail # these aliases do not need to be added to Dovecot's userdb [[ ! ${ALIAS} == *@* ]] && continue @@ -177,8 +175,7 @@ function _create_masters() { # creating users ; 'pass' is encrypted # comments and empty lines are ignored local LOGIN PASS - while IFS=$'|' read -r LOGIN PASS - do + while IFS=$'|' read -r LOGIN PASS; do _log 'debug' "Creating master user '${LOGIN}'" local DOVECOT_MASTERDB_LINE diff --git a/target/scripts/helpers/lock.sh b/target/scripts/helpers/lock.sh index 7156c3ec4f2..9aa5520088e 100644 --- a/target/scripts/helpers/lock.sh +++ b/target/scripts/helpers/lock.sh @@ -9,8 +9,7 @@ LOCK_ID=$(uuid) function _create_lock() { LOCK_FILE="/tmp/docker-mailserver/${SCRIPT_NAME}.lock" - while [[ -e "${LOCK_FILE}" ]] - do + while [[ -e "${LOCK_FILE}" ]]; do # Handle stale lock files left behind on crashes # or premature/non-graceful exits of containers while they're making changes if [[ -n "$(find "${LOCK_FILE}" -mmin +1 2>/dev/null)" ]]; then diff --git a/target/scripts/helpers/postfix.sh b/target/scripts/helpers/postfix.sh index 47e57733480..32897d0e878 100644 --- a/target/scripts/helpers/postfix.sh +++ b/target/scripts/helpers/postfix.sh @@ -45,8 +45,7 @@ function _vhost_collect_postfix_domains() { # getting domains FROM mail accounts if [[ -f ${DATABASE_ACCOUNTS} ]]; then - while IFS=$'|' read -r LOGIN _ - do + while IFS=$'|' read -r LOGIN _; do DOMAIN=$(echo "${LOGIN}" | cut -d @ -f2) echo "${DOMAIN}" >>"${TMP_VHOST}" done < <(_get_valid_lines_from_file "${DATABASE_ACCOUNTS}") @@ -54,8 +53,7 @@ function _vhost_collect_postfix_domains() { # getting domains FROM mail aliases if [[ -f ${DATABASE_VIRTUAL} ]]; then - while read -r FROM _ - do + while read -r FROM _; do UNAME=$(echo "${FROM}" | cut -d @ -f1) DOMAIN=$(echo "${FROM}" | cut -d @ -f2) diff --git a/target/scripts/helpers/relay.sh b/target/scripts/helpers/relay.sh index 4a1d8641469..c2713651bf6 100644 --- a/target/scripts/helpers/relay.sh +++ b/target/scripts/helpers/relay.sh @@ -149,8 +149,7 @@ function _populate_relayhost_map() { { _list_domain_parts "${PRINT_DOMAIN_PART_ACCOUNTS}" /tmp/docker-mailserver/postfix-accounts.cf _list_domain_parts "${PRINT_DOMAIN_PART_VIRTUAL}" /tmp/docker-mailserver/postfix-virtual.cf - } | sort -u | while read -r DOMAIN_PART - do + } | sort -u | while read -r DOMAIN_PART; do # DOMAIN_PART not already present in `/etc/postfix/relayhost_map`, and not listed as a relay opt-out domain in `postfix-relaymap.cf` # `^@${DOMAIN_PART}\b` - To check for existing entry, the `\b` avoids accidental partial matches on similar domain parts. # `^\s*@${DOMAIN_PART}\s*$` - Matches line with only a domain part (eg: @example.test) to avoid including a mapping for those domains to the RELAY_HOST. diff --git a/target/scripts/helpers/utils.sh b/target/scripts/helpers/utils.sh index 59e0f63b39d..0c5a0321290 100644 --- a/target/scripts/helpers/utils.sh +++ b/target/scripts/helpers/utils.sh @@ -97,8 +97,7 @@ function _replace_by_env_in_file() { local ENV_PREFIX=${1} CONFIG_FILE=${2} local ESCAPED_VALUE ESCAPED_KEY - while IFS='=' read -r KEY VALUE - do + while IFS='=' read -r KEY VALUE; do KEY=${KEY#"${ENV_PREFIX}"} # strip prefix ESCAPED_KEY=$(sed -E 's#([\=\&\|\$\.\*\/\[\\^]|\])#\\\1#g' <<< "${KEY,,}") ESCAPED_VALUE=$(sed -E 's#([\=\&\|\$\.\*\/\[\\^]|\])#\\\1#g' <<< "${VALUE}") diff --git a/target/scripts/startup/setup.d/fetchmail.sh b/target/scripts/startup/setup.d/fetchmail.sh index 1d5b9570f8a..d019f7a8116 100644 --- a/target/scripts/startup/setup.d/fetchmail.sh +++ b/target/scripts/startup/setup.d/fetchmail.sh @@ -51,8 +51,7 @@ function _setup_fetchmail_parallel() { fi local COUNTER=0 SERVER=0 - while read -r LINE - do + while read -r LINE; do if [[ ${LINE} =~ poll ]]; then # If we read "poll" then we reached a new server definition # We need to create a new file with fetchmail defaults from diff --git a/target/scripts/startup/setup.d/networking.sh b/target/scripts/startup/setup.d/networking.sh index 8bf019e9a62..fd9185200cd 100644 --- a/target/scripts/startup/setup.d/networking.sh +++ b/target/scripts/startup/setup.d/networking.sh @@ -22,8 +22,7 @@ function _setup_docker_permit() { _dms_panic__misconfigured 'NETWORK_INTERFACE' 'Network Setup [docker_permit]' fi - while read -r IP - do + while read -r IP; do CONTAINER_NETWORKS+=("${IP}") done < <(ip -o -4 addr show type veth | grep -E -o '[0-9\.]+/[0-9]+') diff --git a/target/scripts/startup/setup.d/postfix.sh b/target/scripts/startup/setup.d/postfix.sh index 5a57ee7c98c..0d7cb1ae186 100644 --- a/target/scripts/startup/setup.d/postfix.sh +++ b/target/scripts/startup/setup.d/postfix.sh @@ -107,8 +107,7 @@ function __postfix__setup_override_configuration() { fi if [[ -f /tmp/docker-mailserver/postfix-master.cf ]]; then - while read -r LINE - do + while read -r LINE; do if [[ ${LINE} =~ ^[0-9a-z] ]]; then postconf -P "${LINE}" fi diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index e56433b05fc..d6051177c96 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -316,8 +316,7 @@ function __rspamd__handle_user_modules_adjustments() { if [[ -f "${RSPAMD_CUSTOM_COMMANDS_FILE}" ]]; then __rspamd__log 'debug' "Found file '${RSPAMD_CUSTOM_COMMANDS_FILE}' - parsing and applying it" - while read -r COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3 - do + while read -r COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3; do case "${COMMAND}" in ('disable-module') diff --git a/target/scripts/update-check.sh b/target/scripts/update-check.sh index a542bdbb1dd..9010371f241 100755 --- a/target/scripts/update-check.sh +++ b/target/scripts/update-check.sh @@ -15,8 +15,7 @@ if [[ ! ${UPDATE_CHECK_INTERVAL} =~ ^[0-9]+[smhd]{1}$ ]]; then UPDATE_CHECK_INTERVAL='1d' fi -while true -do +while true; do # get remote version information LATEST=$(curl -Lsf "${VERSION_URL}") From 8512dba8adeef3f73fa4d9ffbb2e309c2df64d4e Mon Sep 17 00:00:00 2001 From: Casper Date: Fri, 26 May 2023 07:42:03 +0200 Subject: [PATCH 092/592] Change 'until' style (#3366) --- test/helper/common.bash | 9 +++------ test/test_helper/common.bash | 6 ++---- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/test/helper/common.bash b/test/helper/common.bash index 34b5f97ca05..cc0eda3d020 100644 --- a/test/helper/common.bash +++ b/test/helper/common.bash @@ -182,8 +182,7 @@ function _repeat_until_success_or_timeout() { local STARTTIME=${SECONDS} - until "${@}" - do + until "${@}"; do if [[ -n ${FATAL_FAILURE_TEST_COMMAND} ]] && ! eval "${FATAL_FAILURE_TEST_COMMAND}"; then echo "\`${FATAL_FAILURE_TEST_COMMAND}\` failed, early aborting repeat_until_success of \`${*}\`" >&2 return 1 @@ -215,8 +214,7 @@ function _run_until_success_or_timeout() { local STARTTIME=${SECONDS} # shellcheck disable=SC2154 - until run "${@}" && [[ ${status} -eq 0 ]] - do + until run "${@}" && [[ ${status} -eq 0 ]]; do sleep 1 if (( SECONDS - STARTTIME > TIMEOUT )); then @@ -260,8 +258,7 @@ function _wait_for_smtp_port_in_container_to_respond() { local CONTAINER_NAME=$(__handle_container_name "${1:-}") local COUNT=0 - until [[ $(_exec_in_container timeout 10 /bin/bash -c 'echo QUIT | nc localhost 25') == *'221 2.0.0 Bye'* ]] - do + until [[ $(_exec_in_container timeout 10 /bin/bash -c 'echo QUIT | nc localhost 25') == *'221 2.0.0 Bye'* ]]; do if [[ ${COUNT} -eq 20 ]]; then echo "Unable to receive a valid response from 'nc localhost 25' within 20 seconds" return 1 diff --git a/test/test_helper/common.bash b/test/test_helper/common.bash index 15287ff687b..979b3c08d52 100644 --- a/test/test_helper/common.bash +++ b/test/test_helper/common.bash @@ -37,8 +37,7 @@ function repeat_until_success_or_timeout { local STARTTIME=${SECONDS} shift 1 - until "${@}" - do + until "${@}"; do if [[ -n ${FATAL_FAILURE_TEST_COMMAND} ]] && ! eval "${FATAL_FAILURE_TEST_COMMAND}"; then echo "\`${FATAL_FAILURE_TEST_COMMAND}\` failed, early aborting repeat_until_success of \`${*}\`" >&2 return 1 @@ -66,8 +65,7 @@ function run_until_success_or_timeout { local STARTTIME=${SECONDS} shift 1 - until run "${@}" && [[ $status -eq 0 ]] - do + until run "${@}" && [[ $status -eq 0 ]]; do sleep 1 if (( SECONDS - STARTTIME > TIMEOUT )); then From 8bfe8424fc72d499543618e2606ef29d4d30a4d4 Mon Sep 17 00:00:00 2001 From: Casper Date: Fri, 26 May 2023 14:00:40 +0200 Subject: [PATCH 093/592] Change 'for' style (#3368) --- target/bin/debug-getmail | 3 +-- target/bin/deldovecotmasteruser | 3 +-- target/bin/delmailuser | 3 +-- target/bin/fail2ban | 9 +++------ target/bin/getmail-cron | 3 +-- target/scripts/helpers/change-detection.sh | 3 +-- target/scripts/helpers/network.sh | 3 +-- target/scripts/startup/check-stack.sh | 3 +-- target/scripts/startup/daemons-stack.sh | 6 ++---- target/scripts/startup/setup-stack.sh | 6 ++---- target/scripts/startup/setup.d/dovecot.sh | 3 +-- target/scripts/startup/setup.d/fetchmail.sh | 3 +-- target/scripts/startup/setup.d/getmail.sh | 3 +-- target/scripts/startup/setup.d/ldap.sh | 9 +++------ target/scripts/startup/setup.d/mail_state.sh | 6 ++---- target/scripts/startup/setup.d/networking.sh | 3 +-- target/scripts/startup/setup.d/security/misc.sh | 3 +-- target/scripts/startup/setup.d/security/rspamd.sh | 3 +-- target/scripts/startup/variables-stack.sh | 3 +-- test/helper/setup.bash | 3 +-- test/helper/tls.bash | 3 +-- test/tests/parallel/set1/spam_virus/fail2ban.bats | 3 +-- .../parallel/set1/spam_virus/rspamd_dkim.bats | 15 +++++---------- .../parallel/set1/spam_virus/rspamd_full.bats | 12 ++++-------- .../parallel/set1/spam_virus/rspamd_partly.bats | 6 ++---- .../process_check_restart.bats | 9 +++------ 26 files changed, 43 insertions(+), 86 deletions(-) diff --git a/target/bin/debug-getmail b/target/bin/debug-getmail index fe6527ef074..c007ebd2bb7 100644 --- a/target/bin/debug-getmail +++ b/target/bin/debug-getmail @@ -14,7 +14,6 @@ else GETMAILDIR=/tmp/docker-mailserver/getmail fi -for FILE in /etc/getmailrc.d/getmailrc* -do +for FILE in /etc/getmailrc.d/getmailrc*; do /usr/local/bin/getmail --getmaildir "${GETMAILDIR}" --rcfile "${FILE}" --dump | tail -n +7 done diff --git a/target/bin/deldovecotmasteruser b/target/bin/deldovecotmasteruser index 2ef20276714..68e5531bbef 100755 --- a/target/bin/deldovecotmasteruser +++ b/target/bin/deldovecotmasteruser @@ -7,8 +7,7 @@ function _main() { _require_n_parameters_or_print_usage 1 "${@}" # Actual command to perform: - for MAIL_ACCOUNT in "${@}" - do + for MAIL_ACCOUNT in "${@}"; do _manage_accounts_dovecotmaster_delete "${MAIL_ACCOUNT}" \ || _exit_with_error "'${MAIL_ACCOUNT}' could not be deleted" done diff --git a/target/bin/delmailuser b/target/bin/delmailuser index d796f2a77b9..c4b68ff3905 100755 --- a/target/bin/delmailuser +++ b/target/bin/delmailuser @@ -16,8 +16,7 @@ function _main() { _create_lock # Actual command to perform: - for MAIL_ACCOUNT in "${@}" - do + for MAIL_ACCOUNT in "${@}"; do _account_should_already_exist [[ ${MAILDEL} -eq 1 ]] && _remove_maildir "${MAIL_ACCOUNT}" diff --git a/target/bin/fail2ban b/target/bin/fail2ban index 71dbd3c046a..2ae9deafdef 100755 --- a/target/bin/fail2ban +++ b/target/bin/fail2ban @@ -13,16 +13,14 @@ fail2ban-client ping &>/dev/null || _exit_with_error "Fail2ban not running" unset JAILS declare -a JAILS -for LIST in $(fail2ban-client status | grep "Jail list" | cut -f2- | sed 's/,/ /g') -do +for LIST in $(fail2ban-client status | grep "Jail list" | cut -f2- | sed 's/,/ /g'); do JAILS+=("${LIST}") done if [[ -z ${1} ]]; then IPS_BANNED=0 - for JAIL in "${JAILS[@]}" - do + for JAIL in "${JAILS[@]}"; do BANNED_IPS=$(fail2ban-client status "${JAIL}" | grep -oP '(?<=Banned IP list:\s).+') if [[ -n ${BANNED_IPS} ]]; then @@ -58,8 +56,7 @@ else shift if [[ -n ${1} ]]; then - for JAIL in "${JAILS[@]}" - do + for JAIL in "${JAILS[@]}"; do RESULT=$(fail2ban-client set "${JAIL}" unbanip "${@}" 2>&1) [[ ${RESULT} != *"is not banned"* ]] && [[ ${RESULT} != *"NOK"* ]] && echo "Unbanned IP from ${JAIL}: ${RESULT}" diff --git a/target/bin/getmail-cron b/target/bin/getmail-cron index 4eb00dead46..8f31e3b1a47 100644 --- a/target/bin/getmail-cron +++ b/target/bin/getmail-cron @@ -1,7 +1,6 @@ #! /bin/bash -for FILE in /etc/getmailrc.d/getmailrc* -do +for FILE in /etc/getmailrc.d/getmailrc*; do if ! pgrep -f "${FILE}$" &>/dev/null; then /usr/local/bin/getmail --getmaildir /var/lib/getmail --rcfile "${FILE}" fi diff --git a/target/scripts/helpers/change-detection.sh b/target/scripts/helpers/change-detection.sh index ddda89b086c..e396bfe2b66 100644 --- a/target/scripts/helpers/change-detection.sh +++ b/target/scripts/helpers/change-detection.sh @@ -64,8 +64,7 @@ function _monitored_files_checksums() { # If the file actually exists, add to CHANGED_FILES # and generate a content hash entry: - for FILE in "${STAGING_FILES[@]}" - do + for FILE in "${STAGING_FILES[@]}"; do [[ -f "${FILE}" ]] && CHANGED_FILES+=("${FILE}") done diff --git a/target/scripts/helpers/network.sh b/target/scripts/helpers/network.sh index afbe5a983a3..a710df31370 100644 --- a/target/scripts/helpers/network.sh +++ b/target/scripts/helpers/network.sh @@ -26,8 +26,7 @@ function _sanitize_ipv4_to_subnet_cidr() { declare -a MASKED_DIGITS DIGITS IFS='.' ; read -r -a DIGITS < <(echo "${1%%/*}") ; unset IFS - for ((i = 0 ; i < 4 ; i++)) - do + for ((i = 0 ; i < 4 ; i++)); do MASKED_DIGITS[i]=$(_mask_ip_digit "${DIGIT_PREFIX_LENGTH}" "${DIGITS[i]}") DIGIT_PREFIX_LENGTH=$((DIGIT_PREFIX_LENGTH - 8)) done diff --git a/target/scripts/startup/check-stack.sh b/target/scripts/startup/check-stack.sh index acefa55d9be..61f21d1ce2c 100644 --- a/target/scripts/startup/check-stack.sh +++ b/target/scripts/startup/check-stack.sh @@ -9,8 +9,7 @@ function _register_check_function() { function _check() { _log 'info' 'Checking configuration' - for FUNC in "${FUNCS_CHECK[@]}" - do + for FUNC in "${FUNCS_CHECK[@]}"; do ${FUNC} done } diff --git a/target/scripts/startup/daemons-stack.sh b/target/scripts/startup/daemons-stack.sh index d8d6f808c4e..5476fc9fec9 100644 --- a/target/scripts/startup/daemons-stack.sh +++ b/target/scripts/startup/daemons-stack.sh @@ -10,8 +10,7 @@ function _register_start_daemon() { function _start_daemons() { _log 'info' 'Starting daemons' - for FUNCTION in "${DAEMONS_START[@]}" - do + for FUNCTION in "${DAEMONS_START[@]}"; do ${FUNCTION} done } @@ -56,8 +55,7 @@ function _start_daemon_postfix() { function _start_daemon_fetchmail() { if [[ ${FETCHMAIL_PARALLEL} -eq 1 ]]; then local COUNTER=0 - for _ in /etc/fetchmailrc.d/fetchmail-*.rc - do + for _ in /etc/fetchmailrc.d/fetchmail-*.rc; do COUNTER=$(( COUNTER + 1 )) _default_start_daemon "fetchmail-${COUNTER}" done diff --git a/target/scripts/startup/setup-stack.sh b/target/scripts/startup/setup-stack.sh index a83ae5c62eb..060dadb2947 100644 --- a/target/scripts/startup/setup-stack.sh +++ b/target/scripts/startup/setup-stack.sh @@ -10,15 +10,13 @@ function _register_setup_function() { function _setup() { # Requires `shopt -s globstar` because of `**` which in # turn is required as we're decending through directories - for FILE in /usr/local/bin/setup.d/**/*.sh - do + for FILE in /usr/local/bin/setup.d/**/*.sh; do # shellcheck source=/dev/null source "${FILE}" done _log 'info' 'Configuring mail server' - for FUNC in "${FUNCS_SETUP[@]}" - do + for FUNC in "${FUNCS_SETUP[@]}"; do ${FUNC} done diff --git a/target/scripts/startup/setup.d/dovecot.sh b/target/scripts/startup/setup.d/dovecot.sh index ebe413e6b66..3eeda286c37 100644 --- a/target/scripts/startup/setup.d/dovecot.sh +++ b/target/scripts/startup/setup.d/dovecot.sh @@ -146,8 +146,7 @@ function _setup_dovecot_local_user() { function __wait_until_an_account_is_added_or_shutdown() { local SLEEP_PERIOD='10' - for (( COUNTER = 11 ; COUNTER >= 0 ; COUNTER-- )) - do + for (( COUNTER = 11 ; COUNTER >= 0 ; COUNTER-- )); do if [[ $(grep -cE '.+@.+\|' /tmp/docker-mailserver/postfix-accounts.cf 2>/dev/null || printf '%s' '0') -ge 1 ]]; then return 0 else diff --git a/target/scripts/startup/setup.d/fetchmail.sh b/target/scripts/startup/setup.d/fetchmail.sh index d019f7a8116..7ea698ea11e 100644 --- a/target/scripts/startup/setup.d/fetchmail.sh +++ b/target/scripts/startup/setup.d/fetchmail.sh @@ -76,8 +76,7 @@ function _setup_fetchmail_parallel() { _fetchmailrc_split local COUNTER=0 - for RC in /etc/fetchmailrc.d/fetchmail-*.rc - do + for RC in /etc/fetchmailrc.d/fetchmail-*.rc; do COUNTER=$(( COUNTER + 1 )) cat >"/etc/supervisor/conf.d/fetchmail-${COUNTER}.conf" << EOF [program:fetchmail-${COUNTER}] diff --git a/target/scripts/startup/setup.d/getmail.sh b/target/scripts/startup/setup.d/getmail.sh index ede7561ee9b..a6c304c6b96 100644 --- a/target/scripts/startup/setup.d/getmail.sh +++ b/target/scripts/startup/setup.d/getmail.sh @@ -13,8 +13,7 @@ function _setup_getmail() { # Generate getmailrc configs, starting with the `/etc/getmailrc_general` base config, # Add a unique `message_log` config, then append users own config to the end. - for FILE in /tmp/docker-mailserver/getmail-*.cf - do + for FILE in /tmp/docker-mailserver/getmail-*.cf; do if [[ -f ${FILE} ]]; then CONFIGS=1 ID=$(cut -d '-' -f 3 <<< "${FILE}" | cut -d '.' -f 1) diff --git a/target/scripts/startup/setup.d/ldap.sh b/target/scripts/startup/setup.d/ldap.sh index 349fd63f5da..ceaca4eb4bc 100644 --- a/target/scripts/startup/setup.d/ldap.sh +++ b/target/scripts/startup/setup.d/ldap.sh @@ -4,8 +4,7 @@ function _setup_ldap() { _log 'debug' 'Setting up LDAP' _log 'trace' 'Checking for custom configs' - for i in 'users' 'groups' 'aliases' 'domains' - do + for i in 'users' 'groups' 'aliases' 'domains'; do local FPATH="/tmp/docker-mailserver/ldap-${i}.cf" if [[ -f ${FPATH} ]]; then cp "${FPATH}" "/etc/postfix/ldap-${i}.cf" @@ -23,8 +22,7 @@ function _setup_ldap() { /etc/postfix/maps/sender_login_maps.ldap ) - for FILE in "${FILES[@]}" - do + for FILE in "${FILES[@]}"; do [[ ${FILE} =~ ldap-user ]] && export LDAP_QUERY_FILTER="${LDAP_QUERY_FILTER_USER}" [[ ${FILE} =~ ldap-group ]] && export LDAP_QUERY_FILTER="${LDAP_QUERY_FILTER_GROUP}" [[ ${FILE} =~ ldap-aliases ]] && export LDAP_QUERY_FILTER="${LDAP_QUERY_FILTER_ALIAS}" @@ -51,8 +49,7 @@ function _setup_ldap() { # Default DOVECOT_PASS_FILTER to the same value as DOVECOT_USER_FILTER DOVECOT_LDAP_MAPPING['DOVECOT_PASS_FILTER']="${DOVECOT_PASS_FILTER:="${DOVECOT_USER_FILTER}"}" - for VAR in "${!DOVECOT_LDAP_MAPPING[@]}" - do + for VAR in "${!DOVECOT_LDAP_MAPPING[@]}"; do export "${VAR}=${DOVECOT_LDAP_MAPPING[${VAR}]}" done diff --git a/target/scripts/startup/setup.d/mail_state.sh b/target/scripts/startup/setup.d/mail_state.sh index c3f565645d6..ffc3179106c 100644 --- a/target/scripts/startup/setup.d/mail_state.sh +++ b/target/scripts/startup/setup.d/mail_state.sh @@ -34,8 +34,7 @@ function _setup_save_states() { # Single service files [[ ${ENABLE_SRS} -eq 1 ]] && SERVICEFILES+=('/etc/postsrsd.secret') - for SERVICEFILE in "${SERVICEFILES[@]}"; - do + for SERVICEFILE in "${SERVICEFILES[@]}"; do DEST="${STATEDIR}/${SERVICEFILE}" DESTDIR="${DEST%/*}" @@ -55,8 +54,7 @@ function _setup_save_states() { ln -s "${DEST}" "${SERVICEFILE}" done - for SERVICEDIR in "${SERVICEDIRS[@]}" - do + for SERVICEDIR in "${SERVICEDIRS[@]}"; do DEST="${STATEDIR}/${SERVICEDIR//\//-}" SERVICEDIR="/var/${SERVICEDIR}" diff --git a/target/scripts/startup/setup.d/networking.sh b/target/scripts/startup/setup.d/networking.sh index fd9185200cd..35c33db85e9 100644 --- a/target/scripts/startup/setup.d/networking.sh +++ b/target/scripts/startup/setup.d/networking.sh @@ -48,8 +48,7 @@ function _setup_docker_permit() { ;; ( 'connected-networks' ) - for CONTAINER_NETWORK in "${CONTAINER_NETWORKS[@]}" - do + for CONTAINER_NETWORK in "${CONTAINER_NETWORKS[@]}"; do CONTAINER_NETWORK=$(_sanitize_ipv4_to_subnet_cidr "${CONTAINER_NETWORK}") __add_to_postfix_mynetworks 'Docker Network' "${CONTAINER_NETWORK}" done diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index fe09c8a4a28..26eb0998ce6 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -158,8 +158,7 @@ function __setup__security__clamav() { _log 'debug' 'Enabling and configuring ClamAV' local FILE - for FILE in /var/log/mail/{clamav,freshclam}.log - do + for FILE in /var/log/mail/{clamav,freshclam}.log; do touch "${FILE}" chown clamav:adm "${FILE}" chmod 640 "${FILE}" diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index d6051177c96..400e1a825ba 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -179,8 +179,7 @@ function __rspamd__setup_default_modules() { metric_exporter ) - for MODULE in "${DISABLE_MODULES[@]}" - do + for MODULE in "${DISABLE_MODULES[@]}"; do __rspamd__helper__enable_disable_module "${MODULE}" 'false' done } diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index e59119abc85..c2a52d5c46a 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -212,8 +212,7 @@ function _environment_variables_export() { : >/etc/dms-settings # this file can be sourced by other scripts local VAR - for VAR in "${!VARS[@]}" - do + for VAR in "${!VARS[@]}"; do echo "export ${VAR}='${VARS[${VAR}]}'" >>/root/.bashrc echo "${VAR}='${VARS[${VAR}]}'" >>/etc/dms-settings done diff --git a/test/helper/setup.bash b/test/helper/setup.bash index ad76c5e2264..65e2999f130 100644 --- a/test/helper/setup.bash +++ b/test/helper/setup.bash @@ -29,8 +29,7 @@ function __initialize_variables() { 'CONTAINER_NAME' ) - for VARIABLE in "${REQUIRED_VARIABLES_FOR_TESTS[@]}" - do + for VARIABLE in "${REQUIRED_VARIABLES_FOR_TESTS[@]}"; do __check_if_set "${VARIABLE}" done diff --git a/test/helper/tls.bash b/test/helper/tls.bash index b34c9ddb638..3e4507c6e80 100644 --- a/test/helper/tls.bash +++ b/test/helper/tls.bash @@ -29,8 +29,7 @@ function _should_succesfully_negotiate_tls() { assert_success local PORTS=(25 587 465 143 993) - for PORT in "${PORTS[@]}" - do + for PORT in "${PORTS[@]}"; do _negotiate_tls "${FQDN}" "${PORT}" done } diff --git a/test/tests/parallel/set1/spam_virus/fail2ban.bats b/test/tests/parallel/set1/spam_virus/fail2ban.bats index 8ce843fc953..9ae3075898b 100644 --- a/test/tests/parallel/set1/spam_virus/fail2ban.bats +++ b/test/tests/parallel/set1/spam_virus/fail2ban.bats @@ -49,8 +49,7 @@ function teardown_file() { } @test "fail2ban-jail.cf overrides" { - for FILTER in 'dovecot' 'postfix' 'postfix-sasl' - do + for FILTER in 'dovecot' 'postfix' 'postfix-sasl'; do _run_in_container fail2ban-client get "${FILTER}" bantime assert_output 1234 diff --git a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats index b85e8d18a4f..97c98b6aab8 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats @@ -89,8 +89,7 @@ function teardown_file() { _default_teardown ; } } @test "argument 'domain' is applied correctly" { - for DOMAIN in 'blabla.org' 'someother.com' 'random.de' - do + for DOMAIN in 'blabla.org' 'someother.com' 'random.de'; do _run_in_container setup config dkim domain "${DOMAIN}" assert_success assert_line --partial "Domain set to '${DOMAIN}'" @@ -107,8 +106,7 @@ function teardown_file() { _default_teardown ; } assert_failure assert_line --partial "Unknown keytype 'foobar'" - for KEYTYPE in 'rsa' 'ed25519' - do + for KEYTYPE in 'rsa' 'ed25519'; do _run_in_container setup config dkim keytype "${KEYTYPE}" assert_success assert_line --partial "Keytype set to '${KEYTYPE}'" @@ -127,8 +125,7 @@ function teardown_file() { _default_teardown ; } } @test "argument 'selector' is applied correctly" { - for SELECTOR in 'foo' 'bar' 'baz' - do + for SELECTOR in 'foo' 'bar' 'baz'; do __create_key 'rsa' "${SELECTOR}" assert_success assert_line --partial "Selector set to '${SELECTOR}'" @@ -146,8 +143,7 @@ function teardown_file() { _default_teardown ; } } @test "argument 'keysize' is applied correctly for RSA keys" { - for KEYSIZE in 512 1024 2048 4096 - do + for KEYSIZE in 512 1024 2048 4096; do __create_key 'rsa' 'mail' "${DOMAIN_NAME}" "${KEYSIZE}" assert_success __log_is_free_of_warnings_and_errors @@ -234,8 +230,7 @@ function __check_rsa_keys() { # @param ${1} = base file name that all DKIM key files have function __check_key_files_are_present() { local BASE_FILE_NAME="${1:?Base file name must be supplied to __check_key_files_are_present}" - for FILE in ${BASE_FILE_NAME}.{public.txt,public.dns.txt,private.txt} - do + for FILE in ${BASE_FILE_NAME}.{public.txt,public.dns.txt,private.txt}; do _run_in_container_bash "[[ -f ${FILE} ]]" assert_success done diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index 02664c5fb82..536ce43dd0f 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -49,8 +49,7 @@ function setup_file() { export MAIL_ID3=$(_send_email_and_get_id 'email-templates/rspamd-virus') export MAIL_ID4=$(_send_email_and_get_id 'email-templates/rspamd-spam-header') - for ID in MAIL_ID{1,2,3,4} - do + for ID in MAIL_ID{1,2,3,4}; do [[ -n ${!ID} ]] || { echo "${ID} is empty - aborting!" ; return 1 ; } done } @@ -225,8 +224,7 @@ function teardown_file() { _default_teardown ; } } @test 'RSPAMD_LEARN works' { - for FILE in learn-{ham,spam}.{sieve,svbin} - do + for FILE in learn-{ham,spam}.{sieve,svbin}; do _run_in_container_bash "[[ -f /usr/lib/dovecot/sieve-pipe/${FILE} ]]" assert_success done @@ -267,8 +265,7 @@ function teardown_file() { _default_teardown ; } assert_success assert_output --partial 'imapsieve: Matched static mailbox rule [1]' refute_output --partial 'imapsieve: Matched static mailbox rule [2]' - for LINE in "${LEARN_SPAM_LINES[@]}" - do + for LINE in "${LEARN_SPAM_LINES[@]}"; do assert_output --partial "${LINE}" done @@ -281,8 +278,7 @@ function teardown_file() { _default_teardown ; } _run_in_container cat /var/log/mail/mail.log assert_success assert_output --partial 'imapsieve: Matched static mailbox rule [2]' - for LINE in "${LEARN_HAM_LINES[@]}" - do + for LINE in "${LEARN_HAM_LINES[@]}"; do assert_output --partial "${LINE}" done } diff --git a/test/tests/parallel/set1/spam_virus/rspamd_partly.bats b/test/tests/parallel/set1/spam_virus/rspamd_partly.bats index bcb2f8d857b..0c7983f86fa 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_partly.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_partly.bats @@ -42,8 +42,7 @@ function teardown_file() { _default_teardown ; } @test "log warns about interfering features" { run docker logs "${CONTAINER_NAME}" assert_success - for SERVICE in 'Amavis/SA' 'OpenDKIM' 'OpenDMARC' 'policyd-spf' - do + for SERVICE in 'Amavis/SA' 'OpenDKIM' 'OpenDMARC' 'policyd-spf'; do assert_output --regexp ".*WARNING.*Running ${SERVICE} & Rspamd at the same time is discouraged" done } @@ -63,8 +62,7 @@ function teardown_file() { _default_teardown ; } } @test 'learning is properly disabled' { - for FILE in learn-{ham,spam}.{sieve,svbin} - do + for FILE in learn-{ham,spam}.{sieve,svbin}; do _run_in_container_bash "[[ -f /usr/lib/dovecot/sieve-pipe/${FILE} ]]" assert_failure done diff --git a/test/tests/parallel/set3/container_configuration/process_check_restart.bats b/test/tests/parallel/set3/container_configuration/process_check_restart.bats index ef296da913d..2c932bd26c4 100644 --- a/test/tests/parallel/set3/container_configuration/process_check_restart.bats +++ b/test/tests/parallel/set3/container_configuration/process_check_restart.bats @@ -72,16 +72,14 @@ ENV_PROCESS_LIST=( # Required for Postfix (when launched by wrapper script which is slow to start) _wait_for_smtp_port_in_container - for PROCESS in "${CORE_PROCESS_LIST[@]}" - do + for PROCESS in "${CORE_PROCESS_LIST[@]}"; do run _check_if_process_is_running "${PROCESS}" assert_success assert_output --partial "${PROCESS}" refute_output --partial "is not running" done - for PROCESS in "${ENV_PROCESS_LIST[@]}" clamd - do + for PROCESS in "${ENV_PROCESS_LIST[@]}" clamd; do run _check_if_process_is_running "${PROCESS}" assert_failure assert_output --partial "'${PROCESS}' is not running" @@ -116,8 +114,7 @@ ENV_PROCESS_LIST=( "${ENV_PROCESS_LIST[@]}" ) - for PROCESS in "${ENABLED_PROCESS_LIST[@]}" - do + for PROCESS in "${ENABLED_PROCESS_LIST[@]}"; do _should_restart_when_killed "${PROCESS}" done From 69ae4ff31924e2817119e856b3b06e66ca120dac Mon Sep 17 00:00:00 2001 From: Arun Date: Fri, 26 May 2023 17:54:07 +0530 Subject: [PATCH 094/592] Update dkim_dmarc_spf.md (#3367) --- docs/content/config/best-practices/dkim_dmarc_spf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md index 3fece48b81f..8ab00f48f69 100644 --- a/docs/content/config/best-practices/dkim_dmarc_spf.md +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -262,7 +262,7 @@ The only thing you need to do in order to enable DMARC on a "DNS-level" is to ad Typically something like this should be good to start with: ```txt -_dmarc.example.com. IN TXT "v=DMARC1; p=none; sp=none; fo=0; adkim=4; aspf=r; pct=100; rf=afrf; ri=86400; rua=mailto:dmarc.report@example.com; ruf=mailto:dmarc.report@example.com" +_dmarc.example.com. IN TXT "v=DMARC1; p=none; sp=none; fo=0; adkim=r; aspf=r; pct=100; rf=afrf; ri=86400; rua=mailto:dmarc.report@example.com; ruf=mailto:dmarc.report@example.com" ``` Or a bit more strict policies (_mind `p=quarantine` and `sp=quarantine`_): From 3d6260adf854a22ed7d5752218514368c76499b2 Mon Sep 17 00:00:00 2001 From: Casper Date: Sat, 27 May 2023 22:12:24 +0200 Subject: [PATCH 095/592] Add BASH syntax check to linter (#3369) --- Makefile | 5 ++++- test/linting/lint.sh | 32 +++++++++++++++++++++++++------- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 2050da5639e..5732cc07ed6 100644 --- a/Makefile +++ b/Makefile @@ -61,11 +61,14 @@ test/%: ALWAYS_RUN # --- Lints ------------------------------------- # ----------------------------------------------- -lint: ALWAYS_RUN eclint hadolint shellcheck +lint: ALWAYS_RUN eclint hadolint bashcheck shellcheck hadolint: ALWAYS_RUN @ ./test/linting/lint.sh hadolint +bashcheck: ALWAYS_RUN + @ ./test/linting/lint.sh bashcheck + shellcheck: ALWAYS_RUN @ ./test/linting/lint.sh shellcheck diff --git a/test/linting/lint.sh b/test/linting/lint.sh index 3bcb04cd242..a056b02c515 100755 --- a/test/linting/lint.sh +++ b/test/linting/lint.sh @@ -44,20 +44,37 @@ function _hadolint() { fi } -function _shellcheck() { - local F_SH F_BIN F_BATS - - # File paths for shellcheck: +# Create three arrays (F_SH, F_BIN, F_BATS) containing our BASH scripts +function _getBashScripts() { readarray -d '' F_SH < <(find . -type f -iname '*.sh' \ -not -path './test/bats/*' \ -not -path './test/test_helper/*' \ -not -path './.git/*' \ -print0 \ ) + # shellcheck disable=SC2248 readarray -d '' F_BIN < <(find 'target/bin' -type f -not -name '*.py' -print0) readarray -d '' F_BATS < <(find 'test/tests/' -type f -iname '*.bats' -print0) +} + +# Check BASH files for correct syntax +function _bashcheck() { + local ERROR=0 SCRIPT + # .bats files are excluded from the test below: Due to their custom syntax ( @test ), .bats files are not standard bash + for SCRIPT in "${F_SH[@]}" "${F_BIN[@]}"; do + bash -n "${SCRIPT}" || ERROR=1 + done + + if [[ ${ERROR} -eq 0 ]]; then + _log 'info' 'BASH syntax check succeeded' + else + _log 'error' 'BASH syntax check failed' + return 1 + fi +} +function _shellcheck() { # This command is a bit easier to grok as multi-line. # There is a `.shellcheckrc` file, but it's only supports half of the options below, thus kept as CLI: # `SCRIPTDIR` is a special value that represents the path of the script being linted, @@ -118,9 +135,10 @@ function _shellcheck() { function _main() { case "${1:-}" in - ( 'eclint' ) _eclint ;; - ( 'hadolint' ) _hadolint ;; - ( 'shellcheck' ) _shellcheck ;; + ( 'eclint' ) _eclint ;; + ( 'hadolint' ) _hadolint ;; + ( 'bashcheck' ) _getBashScripts; _bashcheck ;; + ( 'shellcheck' ) _getBashScripts; _shellcheck ;; ( * ) _log 'error' "'${1:-}' is not a command nor an option" return 3 From 5c504a57411b6e50cf58b3fa220c86c079e38fc9 Mon Sep 17 00:00:00 2001 From: Casper Date: Sun, 28 May 2023 22:48:11 +0200 Subject: [PATCH 096/592] Bump hadolint/eclint version (#3371) --- Dockerfile | 1 + test/linting/lint.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 018c39c251a..abd68a659da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -123,6 +123,7 @@ RUN < Date: Mon, 29 May 2023 16:52:34 +0200 Subject: [PATCH 097/592] chore(deps): Bump myrotvorets/set-commit-status-action from 1.1.6 to 1.1.7 (#3377) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/contributors.yml | 2 +- .github/workflows/docs-preview-deploy.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index 7e49dcfcda3..abf1c03c4df 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -73,7 +73,7 @@ jobs: # workflow, which is required due to branch protection, is not important for this type # of PR, so we skip it and pretend it was successful. - name: 'Set Status for Linting Actions to Success (Skipped)' - uses: myrotvorets/set-commit-status-action@1.1.6 + uses: myrotvorets/set-commit-status-action@v1.1.7 continue-on-error: true with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index bd57a819fd4..18b0bab94f3 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -46,7 +46,7 @@ jobs: # but presently does not work correctly via split workflow. It is useful in a split workflow as the 1st stage # no longer indicates if the entire workflow/deployment was successful. - name: 'Commit Status: Set Workflow Status as Pending' - uses: myrotvorets/set-commit-status-action@1.1.6 + uses: myrotvorets/set-commit-status-action@v1.1.7 with: token: ${{ secrets.GITHUB_TOKEN }} status: pending @@ -106,7 +106,7 @@ jobs: Built with commit: ${{ env.PR_HEADSHA }} - name: 'Commit Status: Update deployment status' - uses: myrotvorets/set-commit-status-action@1.1.6 + uses: myrotvorets/set-commit-status-action@v1.1.7 # Always run this step regardless of job failing early: if: ${{ always() }} env: From 68265b744d6ca4450026f47165aed645b0ede231 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 29 May 2023 18:34:58 +0200 Subject: [PATCH 098/592] add note about DMS FQDN (#3372) --- docs/content/usage.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/content/usage.md b/docs/content/usage.md index 110c490d99f..3bd106086e9 100644 --- a/docs/content/usage.md +++ b/docs/content/usage.md @@ -52,6 +52,17 @@ We will later dig into DKIM, DMARC & SPF, but for now, these are the records tha - The **A record** tells everyone which IP address the DNS name `mail.example.com` resolves to. - The **PTR record** is the counterpart of the A record, telling everyone what name the IP address `11.22.33.44` resolves to. +!!! note "About The Mail Server's Fully Qualified Domain Name" + + The mail server's fully qualified domain name (FQDN) in our example above is `mail.example.com`. Please note though that this is more of a convention, and not due to technical restrictions. One could also run the mail server + + 1. on `foo.example.com`: you would just need to change your `MX` record; + 2. on `example.com` directly: you would need to change your `MX` record and probably [read our docs on bare domain setups][docs-faq-bare-domain], as these setups are called "bare domain" setups. + + The FQDN is what is relevant for TLS certificates, it has no (inherent/technical) relation to the email addresses and accounts DMS manages. That is to say: even though DMS runs on `mail.example.com`, or `foo.example.com`, or `example.com`, there is nothing that prevents it from managing mail for `barbaz.org` - `barbaz.org` will just need to set its `MX` record to `mail.example.com` (or `foo.example.com` or `example.com`). + + [docs-faq-bare-domain]: ./faq.md#can-i-use-a-nakedbare-domain-ie-no-hostname + If you setup everything, it should roughly look like this: ```console From 6a4fac61f896ee37e32a969c0c71b791a04a1c92 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 29 May 2023 19:07:45 +0200 Subject: [PATCH 099/592] misc: remaining v13 todos (#3370) --- Dockerfile | 5 ++- docs/content/config/advanced/mail-sieve.md | 3 +- docs/content/faq.md | 39 +++++++++++++--------- target/scripts/start-mailserver.sh | 3 +- 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index abd68a659da..3196d30316e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ EOF COPY target/scripts/build/* /build/ COPY target/scripts/helpers/log.sh /usr/local/bin/helpers/log.sh -RUN /bin/bash /build/packages.sh +RUN /bin/bash /build/packages.sh && rm -r /build # ----------------------------------------------- # --- ClamAV & FeshClam ------------------------- @@ -41,6 +41,9 @@ RUN /bin/bash /build/packages.sh COPY --link --chown=200 --from=docker.io/clamav/clamav:latest /var/lib/clamav /var/lib/clamav RUN </etc/cron.d/clamav-freshclam chmod 644 /etc/clamav/freshclam.conf sedfile -i 's/Foreground false/Foreground true/g' /etc/clamav/clamd.conf diff --git a/docs/content/config/advanced/mail-sieve.md b/docs/content/config/advanced/mail-sieve.md index 2918cdfeb50..627e12edc7f 100644 --- a/docs/content/config/advanced/mail-sieve.md +++ b/docs/content/config/advanced/mail-sieve.md @@ -92,9 +92,10 @@ The [Manage Sieve](https://doc.dovecot.org/admin_manual/pigeonhole_managesieve_s - ENABLE_MANAGESIEVE=1 ``` -All user defined sieve scripts that are managed by ManageSieve are stored in the user's home folder in `/var/mail/example.com/user1/sieve`. Just one sieve script might be active for a user and is sym-linked to `/var/mail/example.com/user1/.dovecot.sieve` automatically. +All user defined sieve scripts that are managed by ManageSieve are stored in the user's home folder in `/var/mail/example.com/user1/home/sieve`. Just one Sieve script might be active for a user and is sym-linked to `/var/mail/example.com/user1/home/.dovecot.sieve` automatically. !!! note + ManageSieve makes sure to not overwrite an existing `.dovecot.sieve` file. If a user activates a new sieve script the old one is backuped and moved to the `sieve` folder. The extension is known to work with the following ManageSieve clients: diff --git a/docs/content/faq.md b/docs/content/faq.md index dd313631e84..64728d0919d 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -176,7 +176,6 @@ Also you need to define `hostname: example.com` in your `compose.yaml`. - There are [benefits][github-comment-baredomain] to preferring a subdomain. - A bare domain is not required to have `user@example.com`, that is distinct from your hostname which is identified by a DNS MX record. - ### How can I configure a catch-all? Considering you want to redirect all incoming e-mails for the domain `example.com` to `user1@example.com`, add the following line to `docker-data/dms/config/postfix-virtual.cf`: @@ -250,23 +249,32 @@ See [#1247][github-issue-1247] for an example. ### Common Errors +#### Creating an alias or account with an address for `hostname` + +Normally you will assign DMS a `hostname` such as `mail.example.com`. If you instead use a bare domain (_such as `example.com`_) or add an alias / account with the same value as your `hostname`, this can cause a conflict for mail addressed to `@hostname` as Postfix gets confused where to deliver the mail (_`hostname` is configured for only system accounts via the Postfix `main.cf` setting `mydestination`_). + +When this conflict is detected you'll find logs similar to this: + ```log -warning: connect to Milter service inet:localhost:8893: Connection refused -# DMARC not running -# => /etc/init.d/opendmarc restart - -warning: connect to Milter service inet:localhost:8891: Connection refused -# DKIM not running -# => /etc/init.d/opendkim restart - -mail amavis[1459]: (01459-01) (!)connect to /var/run/clamav/clamd.ctl failed, attempt #1: Can't connect to a UNIX socket /var/run/clamav/clamd.ctl: No such file or directory -mail amavis[1459]: (01459-01) (!)ClamAV-clamd: All attempts (1) failed connecting to /var/run/clamav/clamd.ctl, retrying (2) -mail amavis[1459]: (01459-01) (!)ClamAV-clamscan av-scanner FAILED: /usr/bin/clamscan KILLED, signal 9 (0009) at (eval 100) line 905. -mail amavis[1459]: (01459-01) (!!)AV: ALL VIRUS SCANNERS FAILED -# Clamav is not running (not started or because you don't have enough memory) -# => check requirements and/or start Clamav +warning: do not list domain mail.example.com in BOTH mydestination and virtual_mailbox_domains +... +NOQUEUE: reject: RCPT from HOST[IP]: 550 5.1.1 : Recipient address rejected: User unknown in local recipient table; ... +``` + +Opt-out of mail being directed to services by excluding `$myhostname` as a destination with a [`postfix-main.cf`][docs-override-postfix] override config: + +```cf +mydestination = localhost.$mydomain, localhost ``` +!!! tip + + You may want to configure a `postmaster` alias via `setup alias add` to receive system notifications. + +!!! warning + + Internal mail destined for `root`, `amavis` or other accounts will now no longer be received without an alias or account created for them. + ### How to use DMS behind a proxy [Using `user-patches.sh`][docs-userpatches], update the container file `/etc/postfix/main.cf` to include: @@ -521,6 +529,7 @@ $spam_quarantine_to = "amavis\@example.com"; [fail2ban-customize]: ./config/security/fail2ban.md [docs-maintenance]: ./config/advanced/maintenance/update-and-cleanup.md +[docs-override-postfix]: ./config/advanced/override-defaults/postfix.md [docs-userpatches]: ./config/advanced/override-defaults/user-patches.md [github-comment-baredomain]: https://github.com/docker-mailserver/docker-mailserver/issues/3048#issuecomment-1432358353 [github-comment-override-hostname]: https://github.com/docker-mailserver/docker-mailserver/issues/1731#issuecomment-753968425 diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index 6a22e12d3c9..562320d9248 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -1,6 +1,7 @@ #!/bin/bash -shopt -s globstar +set -o pipefail +shopt -s globstar inherit_errexit # ------------------------------------------------------------ # ? >> Sourcing helpers & stacks From 86e18d04ddf6b39f33a36e2fd5d8b388265656de Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 31 May 2023 01:22:42 +1200 Subject: [PATCH 100/592] chore: Revise Dockerfile comment on COPY bug (#3378) --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3196d30316e..a9095e9bc9d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,8 +41,8 @@ RUN /bin/bash /build/packages.sh && rm -r /build COPY --link --chown=200 --from=docker.io/clamav/clamav:latest /var/lib/clamav /var/lib/clamav RUN </etc/cron.d/clamav-freshclam chmod 644 /etc/clamav/freshclam.conf From e68062282a159e556e5a5f14c85298fb2aaf5fd4 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 1 Jun 2023 12:57:05 +1200 Subject: [PATCH 101/592] ci: Simplify GH bug report template (#3381) Simplify the bug report form further by dropping / merging form sections. Change Overview: - Minor revisions and formatting changes (_multi-line pipe operator, emoji, fix typos, etc_). - Collapsed OS + Arch into single input field (_not much benefit from the two additional dropdown items_). - Description/reproduction and expectation sections revised (_expectation intent is typically inferred by the issue description, while detailed reproduction steps can belong a separate optional section_). - Removed platform dropdown (_Windows and macOS are mentioned in description as unsupported_). - Removed experience checkboxes (_context doesn't really change responses_). - Removed the orchestrator dropdown (_we don't seem to use this information, it's just noise_) - Relocate the DMS version + OS/Arch sections to come after the Reproduction steps. --- .github/ISSUE_TEMPLATE/bug_report.yml | 120 ++++++++------------------ 1 file changed, 37 insertions(+), 83 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 8af64886f0f..1e2ee1f805f 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,5 +1,5 @@ name: Bug Report -description: File a bug report +description: Submit a bug report to help us improve title: 'bug report: ' labels: - kind/bug/report @@ -9,7 +9,7 @@ body: - type: checkboxes id: preliminary-checks attributes: - label: Preliminary Checks + label: 📝 Preliminary Checks description: Please read these carefully. options: - label: I checked that all ports are open and not blocked by my ISP / hosting provider. @@ -19,111 +19,65 @@ body: - label: I searched the issue tracker but was unable to find my issue. required: true - label: | - I read - - - the [extended documentation in general](https://docker-mailserver.github.io/docker-mailserver/latest/) but found nothing to resolve the issue; - - the [documentation on debugging](https://docker-mailserver.github.io/docker-mailserver/latest/config/debugging/), tried the proposed steps to debug the problem, but was still unable to resolve the issue; - - this project's [Code of Conduct](https://github.com/docker-mailserver/docker-mailserver/blob/master/CODE_OF_CONDUCT.md) and I agree; - - the [documentation about filing a bug report](https://docker-mailserver.github.io/docker-mailserver/latest/contributing/issues-and-pull-requests/#filing-a-bug-report). + I have read: + - The [extended documentation in general](https://docker-mailserver.github.io/docker-mailserver/latest/) but found nothing to resolve the issue; + - The [documentation on debugging](https://docker-mailserver.github.io/docker-mailserver/latest/config/debugging/), tried the proposed steps to debug the problem, but was still unable to resolve the issue; + - This project's [Code of Conduct](https://github.com/docker-mailserver/docker-mailserver/blob/master/CODE_OF_CONDUCT.md) and I agree; + - The [documentation about filing a bug report](https://docker-mailserver.github.io/docker-mailserver/latest/contributing/issues-and-pull-requests/#filing-a-bug-report). required: true - - type: dropdown - id: operating-system - attributes: - label: What operating system is the host running? - options: - - Linux - - macOS (not officially supported) - - Windows (unsupported) - - Other (unsupported) - validations: - required: true - - type: input - id: operating-system-version + - type: textarea + id: what-happened attributes: - label: Which operating system version? - placeholder: Debian 11 (Bullseye) + label: 👀 What Happened? + description: How did this differ from your expectations? + placeholder: Although `LOG_LEVEL=debug` is set, the logs are missing debug output. validations: required: true - - type: dropdown - id: isa - attributes: - label: What computer architecture is the host running? - options: - - AMD64 (x86_64) - - ARM64 (AArch64) - - Other (unsupported) - validations: - required: true - - type: dropdown - id: container-orchestrator + - type: textarea + id: steps-to-reproduce attributes: - label: What container orchestration tool are you using? - options: - - Docker - - Docker Compose - - Podman (not officially supported) - - Kubernetes (not officially supported) - - Other (unsupported) - validations: - required: true + label: 👟 Reproduction Steps + description: | + How did you trigger this bug? Please walk us through it step by step. + Please use [fenced code blocks](https://docs.github.com/en/github/writing-on-github/working-with-advanced-formatting/creating-and-highlighting-code-blocks#fenced-code-blocks) when pasting lots of text! + placeholder: The easier it is for us to reproduce your issue, the sooner we can help resolve it 😉 - type: input id: mailserver-version attributes: - label: DMS version + label: 🐋 DMS Version description: On which version (image tag) did you encounter this bug? placeholder: v12.1.0 validations: required: true - - type: textarea - id: when-does-it-occur - attributes: - label: What happened and when does this occur? - description: > - Tell us what happened. - **It would also help immensely if you can share precise steps on how to reproduce the issue!** :heart: - Use [fenced code blocks](https://docs.github.com/en/github/writing-on-github/working-with-advanced-formatting/creating-and-highlighting-code-blocks#fenced-code-blocks) when pasting lots of text! - placeholder: Although `LOG_LEVEL=debug` is set, I see no debug output. - validations: - required: true - - type: textarea - id: what-did-you-expect-to-happen + - type: input + id: operating-system attributes: - label: What did you expect to happen? - description: Tell us what you expected. - placeholder: I expected to see debug messages. + label: 💻 Operating System and Architecture + description: | + Which OS is your docker host running on? + **NOTE:** Windows and macOS have limited support. + placeholder: Debian 11 (Bullseye) x86_64, Fedora 38 ARM64 validations: required: true - type: textarea id: container-configuration-files attributes: - label: Container configuration files - description: > + label: ⚙️ Container configuration files + description: | Show us the file that you use to run DMS (and possibly other related services). - This is most likely your `compose.yaml` file, but you can also show us your equivalent `docker run` command, if applicable. If you are using Kubernetes, you can also put your manifest files here. - This filed is formatted as YAML. + - This field is formatted as YAML. + - This is most likely your `compose.yaml` file, but you can also show us your equivalent `docker run` command, if applicable. + - If you are using Kubernetes, you can also put your manifest files here. render: yml - type: textarea id: relevant-log-output attributes: - label: Relevant log output - description: Show us relevant log output here. You can enable debug output by setting the environment variable `LOG_LEVEL` to `debug` or `trace`. This field's contents are interpreted as pure text. + label: 📜 Relevant log output + description: | + Show us relevant log output here. + - This field expects only plain text (_rendered as a fenced code block_). + - You can enable debug output by setting the environment variable `LOG_LEVEL` to `debug` or `trace`. render: Text - - type: checkboxes - id: experience - attributes: - label: What level of experience do you have with containers, mail servers, and using a terminal? - description: > - **You are not obliged to answer this question**. - We do encourage answering though as it provides context to better assist you. - Less experienced users tend to make common mistakes, which is ok; by letting us know we can spot those more easily. If you are experienced, we can skip basic questions and save time. - - options: - - label: I am inexperienced with containers - - label: I am rather experienced with containers - - label: I am inexperienced with mail servers - - label: I am rather experienced with mail servers - - label: I am uncomfortable with using a terminal (CLI) - - label: I am rather comfortable with using a terminal (CLI) - type: input id: form-improvements attributes: From efed9d8012e8ce0b91a8f3c5eda7e959e0c39890 Mon Sep 17 00:00:00 2001 From: Thomas Butter Date: Thu, 1 Jun 2023 10:50:31 +0200 Subject: [PATCH 102/592] Dovecot: compile `fts_xapian` from source to match Dovecot ABI (#3373) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Co-authored-by: Casper --- Dockerfile | 49 ++++++++++++++++++++++++++------ target/scripts/build/compile.sh | 30 +++++++++++++++++++ target/scripts/build/packages.sh | 19 +++---------- 3 files changed, 74 insertions(+), 24 deletions(-) create mode 100644 target/scripts/build/compile.sh diff --git a/Dockerfile b/Dockerfile index a9095e9bc9d..b5c5ba395bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,18 @@ # syntax=docker.io/docker/dockerfile:1 -# This Dockerfile provides two stages: stage-base and stage-final +# This Dockerfile provides four stages: stage-base, stage-compile, stage-main and stage-final # This is in preparation for more granular stages (eg ClamAV and Fail2Ban split into their own) -# -# Base stage provides all packages, config, and adds scripts -# - -FROM docker.io/debian:11-slim AS stage-base - ARG DEBIAN_FRONTEND=noninteractive ARG DOVECOT_COMMUNITY_REPO=1 ARG LOG_LEVEL=trace +FROM docker.io/debian:11-slim AS stage-base + +ARG DEBIAN_FRONTEND +ARG DOVECOT_COMMUNITY_REPO +ARG LOG_LEVEL + SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] # ----------------------------------------------- @@ -25,11 +25,37 @@ RUN <&1 +} + +_compile_dovecot_fts_xapian diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 9f863899bc5..40e7365ee78 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -94,26 +94,12 @@ function _install_dovecot() { declare -a DOVECOT_PACKAGES DOVECOT_PACKAGES=( - dovecot-core dovecot-fts-xapian dovecot-imapd + dovecot-core dovecot-imapd dovecot-ldap dovecot-lmtpd dovecot-managesieved dovecot-pop3d dovecot-sieve dovecot-solr ) if [[ ${DOVECOT_COMMUNITY_REPO} -eq 1 ]]; then - # The package dovecot-fts-xapian is installed from the debian repository. - # Starting with version 1.4.9a-1+deb11u1, a new dependency was added: dovecot-abi-2.3.abiv13 - # dovecot-abi-2.3.abiv13 is a virtual package provided by dovecot-core (from the debian repository). - # However dovecot-core from the community repository provides dovecot-abi-2.3.abiv19. - _log 'trace' "Create and install dummy package 'dovecot-abi-2.3.abiv13' to satisfy 'dovecot-fts-xapian' dependency" - apt-get "${QUIET}" --no-install-recommends install checkinstall - echo -e 'install:\n\t@true' > Makefile - echo 'Dummy package to satisfy dovecot-fts-xapian dependency' > description-pak - checkinstall -y --install=yes --pkgname="dovecot-abi-2.3.abiv13" --pkgversion=1 --maintainer=Nobody --pkggroup=mail - # Cleanup - rm description-pak dovecot-abi-2.3.abiv13*.deb Makefile - apt-get "${QUIET}" purge checkinstall - apt-get "${QUIET}" autoremove - _log 'trace' 'Using Dovecot community repository' curl https://repo.dovecot.org/DOVECOT-REPO-GPG | gpg --import gpg --export ED409DA1 > /etc/apt/trusted.gpg.d/dovecot.gpg @@ -125,6 +111,9 @@ function _install_dovecot() { _log 'debug' 'Installing Dovecot' apt-get "${QUIET}" --no-install-recommends install "${DOVECOT_PACKAGES[@]}" + + # dependency for fts_xapian + apt-get "${QUIET}" --no-install-recommends install libxapian30 } function _install_rspamd() { From e0c7cd475b7cd0fe0d619b029cfd5bbf38e5c9ee Mon Sep 17 00:00:00 2001 From: Casper Date: Sun, 11 Jun 2023 22:59:26 +0200 Subject: [PATCH 103/592] Don't register _setup_spam_to_junk() when SMTP_ONLY=1 (#3385) --- target/scripts/start-mailserver.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index 562320d9248..a002c6c3fa9 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -47,6 +47,7 @@ function _register_functions() { _register_setup_function '_setup_dovecot_sieve' _register_setup_function '_setup_dovecot_dhparam' _register_setup_function '_setup_dovecot_quota' + _register_setup_function '_setup_spam_to_junk' fi case "${ACCOUNT_PROVISIONER}" in @@ -80,7 +81,6 @@ function _register_functions() { _register_setup_function '_setup_policyd_spf' _register_setup_function '_setup_security_stack' - _register_setup_function '_setup_spam_to_junk' _register_setup_function '_setup_rspamd' _register_setup_function '_setup_ssl' @@ -95,7 +95,7 @@ function _register_functions() { # needs to come after _setup_postfix_early _register_setup_function '_setup_spoof_protection' -_register_setup_function '_setup_getmail' + _register_setup_function '_setup_getmail' if [[ ${ENABLE_SRS} -eq 1 ]]; then _register_setup_function '_setup_SRS' From 7bf772e2d62761b1c5483f58be0fc21319b7cbed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 19:10:10 +0200 Subject: [PATCH 104/592] chore(deps): Bump docker/build-push-action from 4.0.0 to 4.1.0 (#3390) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 290a9e00fcd..aae7caf9409 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v4.0.0 + uses: docker/build-push-action@v4.1.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index a87aee82d12..1a9821ce50a 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -72,7 +72,7 @@ jobs: run: echo "version=$(>"${GITHUB_OUTPUT}" - name: 'Build and publish images' - uses: docker/build-push-action@v4.0.0 + uses: docker/build-push-action@v4.1.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index fc3a292bcad..ec24b1b6097 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v4.0.0 + uses: docker/build-push-action@v4.1.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 8da20a7db2f..8938931b05e 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v4.0.0 + uses: docker/build-push-action@v4.1.0 with: context: . tags: mailserver-testing:ci From 8e87a4d8457b6b838febe01cf416a43a46722a08 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 17:12:00 +0000 Subject: [PATCH 105/592] chore(deps): Bump docker/setup-buildx-action from 2.5.0 to 2.6.0 (#3388) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index aae7caf9409..284e08dd38b 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -79,7 +79,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.5.0 + uses: docker/setup-buildx-action@v2.6.0 # NOTE: AMD64 can build within 2 minutes - name: 'Build images' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 1a9821ce50a..98ff75b1a9c 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -40,7 +40,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.5.0 + uses: docker/setup-buildx-action@v2.6.0 # Try get the cached build layers from a prior `generic_build.yml` job. # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index ec24b1b6097..14666f8230c 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -38,7 +38,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.5.0 + uses: docker/setup-buildx-action@v2.6.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 8938931b05e..f8a8270241d 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -37,7 +37,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.5.0 + uses: docker/setup-buildx-action@v2.6.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. From 7b1a712c9168d7b90b5c4e2670a7fc3947986b56 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 17:13:51 +0000 Subject: [PATCH 106/592] chore(deps): Bump docker/metadata-action from 4.4.0 to 4.5.0 (#3387) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 98ff75b1a9c..bb243a6e25d 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -23,7 +23,7 @@ jobs: - name: 'Prepare tags' id: prep - uses: docker/metadata-action@v4.4.0 + uses: docker/metadata-action@v4.5.0 with: images: | ${{ secrets.DOCKER_REPOSITORY }} From 8fbc58cf5d2779ea375a73b7465ed14332b53cbb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 19:19:35 +0200 Subject: [PATCH 107/592] chore(deps): Bump docker/setup-qemu-action from 2.1.0 to 2.2.0 (#3389) --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 284e08dd38b..9400a2741bb 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -74,7 +74,7 @@ jobs: cache-buildx- - name: 'Set up QEMU' - uses: docker/setup-qemu-action@v2.1.0 + uses: docker/setup-qemu-action@v2.2.0 with: platforms: arm64 diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index bb243a6e25d..c77020ffe69 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -35,7 +35,7 @@ jobs: type=semver,pattern={{major}}.{{minor}}.{{patch}} - name: 'Set up QEMU' - uses: docker/setup-qemu-action@v2.1.0 + uses: docker/setup-qemu-action@v2.2.0 with: platforms: arm64 From 7a5dfb71c24eb5c7cd310b987e5fd323e4fc3f25 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 23:03:45 +0200 Subject: [PATCH 108/592] chore(deps): Bump docker/metadata-action from 4.5.0 to 4.6.0 (#3401) --- .github/workflows/generic_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index c77020ffe69..325754d85ea 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -23,7 +23,7 @@ jobs: - name: 'Prepare tags' id: prep - uses: docker/metadata-action@v4.5.0 + uses: docker/metadata-action@v4.6.0 with: images: | ${{ secrets.DOCKER_REPOSITORY }} From 59bcab6127c945b250ebf86276df7113ffc26bd3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 23:14:09 +0200 Subject: [PATCH 109/592] chore(deps): Bump docker/build-push-action from 4.1.0 to 4.1.1 (#3400) --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 9400a2741bb..50ddd187d2f 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v4.1.0 + uses: docker/build-push-action@v4.1.1 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 325754d85ea..57a0483f6c6 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -72,7 +72,7 @@ jobs: run: echo "version=$(>"${GITHUB_OUTPUT}" - name: 'Build and publish images' - uses: docker/build-push-action@v4.1.0 + uses: docker/build-push-action@v4.1.1 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 14666f8230c..e5d5f7e5e36 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v4.1.0 + uses: docker/build-push-action@v4.1.1 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index f8a8270241d..0bfccb077ba 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v4.1.0 + uses: docker/build-push-action@v4.1.1 with: context: . tags: mailserver-testing:ci From e380cc30657c33a2dc24ed528c5ffbd7784ec810 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 23:21:13 +0200 Subject: [PATCH 110/592] chore(deps): Bump docker/setup-buildx-action from 2.6.0 to 2.7.0 (#3398) --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 50ddd187d2f..4405c581687 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -79,7 +79,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.6.0 + uses: docker/setup-buildx-action@v2.7.0 # NOTE: AMD64 can build within 2 minutes - name: 'Build images' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 57a0483f6c6..80e5d73080c 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -40,7 +40,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.6.0 + uses: docker/setup-buildx-action@v2.7.0 # Try get the cached build layers from a prior `generic_build.yml` job. # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index e5d5f7e5e36..457dd399625 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -38,7 +38,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.6.0 + uses: docker/setup-buildx-action@v2.7.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 0bfccb077ba..60d04342ea1 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -37,7 +37,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.6.0 + uses: docker/setup-buildx-action@v2.7.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. From 4dae83b2566e1b67346a4d3c5e054d366cdab62b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 20 Jun 2023 09:43:57 +0200 Subject: [PATCH 111/592] chore(deps): Bump peter-evans/create-pull-request from 5.0.1 to 5.0.2 (#3399) Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 5.0.1 to 5.0.2. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/v5.0.1...v5.0.2) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/contributors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index abf1c03c4df..6e51495a748 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -49,7 +49,7 @@ jobs: # See https://github.com/marketplace/actions/create-pull-request for reference of the action. - name: 'Create Pull Request' - uses: peter-evans/create-pull-request@v5.0.1 + uses: peter-evans/create-pull-request@v5.0.2 id: create-pr with: token: ${{ secrets.GITHUB_TOKEN }} From 2b400a9269ed2f573fac99364ffacc1c4553a7fb Mon Sep 17 00:00:00 2001 From: Claude Brisson Date: Tue, 20 Jun 2023 14:37:31 +0300 Subject: [PATCH 112/592] Fix sieve setup (#3397) --- target/scripts/startup/setup.d/security/misc.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index 26eb0998ce6..b1fa48606c7 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -252,6 +252,7 @@ function __setup__security__amavis() { function _setup_spam_to_junk() { if [[ ${MOVE_SPAM_TO_JUNK} -eq 1 ]]; then _log 'debug' 'Spam emails will be moved to the Junk folder' + mkdir -p /usr/lib/dovecot/sieve-global/after/ cat >/usr/lib/dovecot/sieve-global/after/spam_to_junk.sieve << EOF require ["fileinto","mailbox"]; From 68c6f247a608daa3e0799f227cbfd2cf3376ff5b Mon Sep 17 00:00:00 2001 From: wligtenberg Date: Tue, 20 Jun 2023 21:44:54 +0200 Subject: [PATCH 113/592] Fix issue with concatenating $dmarc_milter and $dkim_milter in main.cf (#3380) * Fix issue with concatenating $dmarc_milter and $dkim_milter in main.cf Upon each start the `smtpd_milters` and `non_smtpd_milters` would be extended with the following: ``` smtpd_milters = $dmarc_milter $dkim_milter non_smtpd_milters = $dkim_milter ``` In my case they became long enough that mail delivery stopped. I think this was because of the extra headers that are added by these steps. (which in turn would cause the mail to be dropped) * fix sed to work when the variables are there and when they are not. --------- Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- target/scripts/startup/setup.d/dmarc_dkim_spf.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh index 20b88ea5cc8..85c63d963d3 100644 --- a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh +++ b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh @@ -17,8 +17,8 @@ function _setup_opendkim() { postconf 'dkim_milter = inet:localhost:8891' # shellcheck disable=SC2016 sed -i -E \ - -e 's|^(smtpd_milters =.*)|\1 \$dkim_milter|g' \ - -e 's|^(non_smtpd_milters =.*)|\1 \$dkim_milter|g' \ + -e '/\$dkim_milter/! s|^(smtpd_milters =.*)|\1 \$dkim_milter|g' \ + -e '/\$dkim_milter/! s|^(non_smtpd_milters =.*)|\1 \$dkim_milter|g' \ /etc/postfix/main.cf # check if any keys are available @@ -64,7 +64,7 @@ function _setup_opendmarc() { postconf 'dmarc_milter = inet:localhost:8893' # Make sure to append the OpenDMARC milter _after_ the OpenDKIM milter! # shellcheck disable=SC2016 - sed -i -E 's|^(smtpd_milters =.*)|\1 \$dmarc_milter|g' /etc/postfix/main.cf + sed -i -E '/\$dmarc_milter/! s|^(smtpd_milters =.*)|\1 \$dmarc_milter|g' /etc/postfix/main.cf sed -i \ -e "s|^AuthservID.*$|AuthservID ${HOSTNAME}|g" \ From a276589e40a96602cf503ada5b6d5e38dffbdeb3 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 22 Jun 2023 19:17:41 +1200 Subject: [PATCH 114/592] docs: Add compatibility section to debugging page (#3404) docs: Add compatibility section to debugging page ci: Adjust bug report template Reduce some text + compress the preliminary checks down to single check item. --- .github/ISSUE_TEMPLATE/bug_report.yml | 21 +++--------- docs/content/config/debugging.md | 47 ++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 1e2ee1f805f..be54660b866 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -10,20 +10,10 @@ body: id: preliminary-checks attributes: label: 📝 Preliminary Checks - description: Please read these carefully. + description: | + By submitting this issue, you agree to our [Code of Conduct](https://github.com/docker-mailserver/docker-mailserver/blob/master/CODE_OF_CONDUCT.md). options: - - label: I checked that all ports are open and not blocked by my ISP / hosting provider. - required: true - - label: I know that SSL errors are likely the result of a wrong setup on the user side and not caused by DMS itself. I'm confident my setup is correct. - required: true - - label: I searched the issue tracker but was unable to find my issue. - required: true - - label: | - I have read: - - The [extended documentation in general](https://docker-mailserver.github.io/docker-mailserver/latest/) but found nothing to resolve the issue; - - The [documentation on debugging](https://docker-mailserver.github.io/docker-mailserver/latest/config/debugging/), tried the proposed steps to debug the problem, but was still unable to resolve the issue; - - This project's [Code of Conduct](https://github.com/docker-mailserver/docker-mailserver/blob/master/CODE_OF_CONDUCT.md) and I agree; - - The [documentation about filing a bug report](https://docker-mailserver.github.io/docker-mailserver/latest/contributing/issues-and-pull-requests/#filing-a-bug-report). + - label: I tried searching for an existing issue and followed the [debugging docs](https://docker-mailserver.github.io/docker-mailserver/latest/config/debugging/) advice, but still need assistance. required: true - type: textarea id: what-happened @@ -64,10 +54,9 @@ body: attributes: label: ⚙️ Container configuration files description: | - Show us the file that you use to run DMS (and possibly other related services). + Show us the `compose.yaml` file or command that you used to run DMS (and possibly other related services). - This field is formatted as YAML. - - This is most likely your `compose.yaml` file, but you can also show us your equivalent `docker run` command, if applicable. - - If you are using Kubernetes, you can also put your manifest files here. + - If you are using Kubernetes, you can alternatively share your manifest files here. render: yml - type: textarea id: relevant-log-output diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index 0392e8fa3b7..48ef270ce13 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -10,7 +10,10 @@ This page contains valuable information when it comes to resolving issues you en Please consider contributing solutions to the [FAQ][docs-faq] :heart: -## Preliminary Information +## Preliminary Checks + +- Check that all published DMS ports are actually open and not blocked by your ISP / hosting provider. +- SSL errors are likely the result of a wrong setup on the user side and not caused by DMS itself. ### Mail sent from DMS does not arrive at destination @@ -25,7 +28,7 @@ These links may advise how the provider can unblock the port through additional ## Steps for Debugging DMS 1. **Increase log verbosity**: Very helpful for troubleshooting problems during container startup. Set the environment variable [`LOG_LEVEL`][docs-environment-log-level] to `debug` or `trace`. -2. **Use error logs as a search query**: Try finding an _existing issue_ or _search engine result_ from any errors in your container log output. Often you'll find answers or more insights. If you still need to open an issue, sharing links from your search may help us assist you. The mail server log can be acquired by running `docker log ` (_or `docker logs -f ` if you want to follow the log_). +2. **Use error logs as a search query**: Try [finding an _existing issue_][gh-issues] or _search engine result_ from any errors in your container log output. Often you'll find answers or more insights. If you still need to open an issue, sharing links from your search may help us assist you. The mail server log can be acquired by running `docker log ` (_or `docker logs -f ` if you want to follow the log_). 3. **Understand the basics of mail servers**: Especially for beginners, make sure you read our [Introduction][docs-introduction] and [Usage][docs-usage] articles. 4. **Search the whole FAQ**: Our [FAQ][docs-faq] contains answers for common problems. Make sure you go through the list. 5. **Reduce the scope**: Ensure that you can run a basic setup of DMS first. Then incrementally restore parts of your original configuration until the problem is reproduced again. If you're new to DMS, it is common to find the cause is misunderstanding how to configure a minimal setup. @@ -38,11 +41,45 @@ If you need more flexibility than `docker logs` offers, within the container `/v To install additional software: -- `apt-get update` is needed to update repository metadata. -- `apt-get install ` -- For example if you need a text editor, `nano` is a good package choice for beginners. +1. `apt-get update` to update repository metadata. +2. `apt-get install ` + +For example a text editor you can use in the terminal: `apt-get install nano` + +## Compatibility + +It's possible that the issue you're experiencing is due to a compatibility conflict. + +This could be from outdated software updates, or running a system that isn't able to provide you newer software and kernels. You may want to verify if you can reproduce the issue on a system that is not affected by these concerns. + +### Network + +- **`userland-proxy`:** Prior to Docker `v23`, [changing the `userland-proxy` setting did not reliably remove NAT rules][network::docker-userlandproxy]. +- **UFW / firewalld:** Some users expect only their firewall frontend to manage the firewall rules, but these will be bypassed when Docker publishes a container port as there is no integration between the two. +- **`iptables` / `nftables`:** + - Docker [only manages the NAT rules via `iptables`][network::docker-nftables], relying on compatibility shims for supporting the successor `nftables`. Internally DMS expects `nftables` support on the host kernel for services like Fail2Ban to function correctly. + - [Kernels older than 5.2 may affect management of NAT rules via `nftables`][network::kernel-nftables]. Other software outside of DMS may also manipulate these rules, such as firewall frontends. +- **IPv6:** + - Requires [additional configuration][docs-ipv6] to prevent or properly support IPv6 connections (eg: Preservering the Client IP). + - Support in 2023 is still considered experimental. You are advised to use at least Docker Engine `v23` (2023Q1). + - Various networking bug fixes have been addressed since the intitial IPv6 support arrived in Docker Engine `v20.10.0` (2020Q4). + +### System + +- **Kernel:** Some systems provide [kernels with modifications (_replacing defaults and backporting patches_)][network::kernels-modified] to support running legacy software or kernels, complicating compatibility. This can be commonly experienced with products like NAS. +- **CGroups v2:** Hosts running older kernels (prior to 5.2) and systemd (prior to v244) are not likely to leverage cgroup v2, or have not defaulted to the cgroup v2 `unified` hierarchy. Not meeting this baseline may influence the behaviour of your DMS container, even with the latest Docker Engine installed. +- **Rootless containers** have additional constraints that vary by container runtime (_Docker, Podman, etc - which already have subtle differences_). + - This can introduce differences such as for container networking which may further impact support for IPv6 and preserving the client IP (Remote address). + - cgroup v2 is required for supporting rootless containers. + +[network::docker-userlandproxy]: https://github.com/moby/moby/issues/44721 +[network::docker-nftables]: https://github.com/moby/moby/issues/26824 +[network::kernels-modified]: https://github.com/docker-mailserver/docker-mailserver/pull/2662#issuecomment-1168435970 +[network::kernel-nftables]: https://unix.stackexchange.com/questions/596493/can-nftables-and-iptables-ip6tables-rules-be-applied-at-the-same-time-if-so-wh/596497#596497 [docs-faq]: ../faq.md [docs-environment-log-level]: ./environment.md#log_level +[docs-ipv6]: ./advanced/ipv6.md [docs-introduction]: ../introduction.md [docs-usage]: ../usage.md +[gh-issues]: https://github.com/docker-mailserver/docker-mailserver/issues From 32c3ecd00e8741c826e7e85827eb3c7c4973ab9b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jun 2023 20:01:26 +0200 Subject: [PATCH 115/592] chore(deps): Bump anchore/scan-action from 3.3.5 to 3.3.6 (#3406) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 60d04342ea1..c9c3ad8e3c5 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v3.3.5 + uses: anchore/scan-action@v3.3.6 id: scan with: image: mailserver-testing:ci From a2247bf655c5bdcf2f8cdd08e7ecf3f4fd7c6a26 Mon Sep 17 00:00:00 2001 From: Felix N Date: Wed, 28 Jun 2023 22:42:57 +0200 Subject: [PATCH 116/592] fix spelling issues in rspamd-dkim (#3411) Co-authored-by: Felix Nieuwenhuizen --- target/bin/rspamd-dkim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/target/bin/rspamd-dkim b/target/bin/rspamd-dkim index 7aa767731a2..2992de5e51f 100755 --- a/target/bin/rspamd-dkim +++ b/target/bin/rspamd-dkim @@ -156,7 +156,7 @@ function _create_keys() { if [[ ${KEYTYPE} == 'rsa' ]]; then local BASE_FILE_NAME="${BASE_DIR}/${KEYTYPE}-${KEYSIZE}-${SELECTOR}-${DOMAIN}" KEYTYPE_OPTIONS=('-b' "${KEYSIZE}") - _log 'info' "Creating DKIM keys of type '${KEYTYPE}' and lenght '${KEYSIZE}' with selector '${SELECTOR}' for domain '${DOMAIN}'" + _log 'info' "Creating DKIM keys of type '${KEYTYPE}' and length '${KEYSIZE}' with selector '${SELECTOR}' for domain '${DOMAIN}'" else local BASE_FILE_NAME="${BASE_DIR}/${KEYTYPE}-${SELECTOR}-${DOMAIN}" KEYTYPE_OPTIONS=('-t' "${KEYTYPE}") @@ -216,7 +216,7 @@ use_domain = "header"; use_redis = false; # don't change unless Redis also provides the DKIM keys use_esld = true; -check_pubkey = true; # you wan't to use this in the beginning +check_pubkey = true; # you want to use this in the beginning domain { ${DOMAIN} { From 9f5d662da7c692fffcf3af78dabff5a732c0bdca Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 3 Jul 2023 01:33:14 +0200 Subject: [PATCH 117/592] docs: Rewrite of IPv6 page (#3244) Much better docs for IPv6 support. Third-party container no longer required, Docker has `ip6tables` feature now. --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/config/advanced/ipv6.md | 173 +++++++++++++++++++++------ docs/content/config/debugging.md | 20 ++-- 2 files changed, 150 insertions(+), 43 deletions(-) diff --git a/docs/content/config/advanced/ipv6.md b/docs/content/config/advanced/ipv6.md index d77f8d7a1ce..2afd1d7d912 100644 --- a/docs/content/config/advanced/ipv6.md +++ b/docs/content/config/advanced/ipv6.md @@ -2,43 +2,144 @@ title: 'Advanced | IPv6' --- -## Background - -If your container host supports IPv6, then DMS will automatically accept IPv6 connections by way of the docker host's IPv6. However, incoming mail will fail SPF checks because they will appear to come from the IPv4 gateway that docker is using to proxy the IPv6 connection (`172.20.0.1` is the gateway). - -This can be solved by supporting IPv6 connections all the way to the DMS container. - -## Setup steps - -```diff -+++ b/serv/compose.yaml -@@ ... @@ services: - -+ ipv6nat: -+ image: robbertkl/ipv6nat -+ restart: always -+ network_mode: "host" -+ cap_add: -+ - NET_ADMIN -+ - SYS_MODULE -+ volumes: -+ - /var/run/docker.sock:/var/run/docker.sock:ro -+ - /lib/modules:/lib/modules:ro - -@@ ... @@ networks: - -+ default: -+ driver: bridge -+ enable_ipv6: true -+ ipam: -+ driver: default -+ config: -+ - subnet: fd00:0123:4567::/48 -+ gateway: fd00:0123:4567::1 +!!! bug "Ample Opportunities for Issues" + + Numerous bug reports have been raised in the past about IPv6. Please make sure your setup around DMS is correct when using IPv6! + +## IPv6 networking problems with Docker defaults + +### What can go wrong? + +If your host system supports IPv6 and an `AAAA` DNS record exists to direct IPv6 traffic to DMS, you may experience issues when an IPv6 connection is made: + +- The original client IP is replaced with the gateway IP of a docker network. +- Connections fail or hang. + +The impact of losing the real IP of the client connection can negatively affect DMS: + +- Users unable to login (_Fail2Ban action triggered by repeated login failures all seen as from the same internal Gateway IP_) +- Mail inbound to DMS is rejected (_[SPF verification failure][gh-issue-1438-spf], IP mismatch_) +- Delivery failures from [sender reputation][sender-score] being reduced (_due to [bouncing inbound mail][gh-issue-3057-bounce] from rejected IPv6 clients_) +- Some services may be configured to trust connecting clients within the containers subnet, which includes the Gateway IP. This can risk bypassing or relaxing security measures, such as exposing an [open relay][wikipedia-openrelay]. + +### Why does this happen? + +When the host network receives a connection to a containers published port, it is routed to the containers internal network managed by Docker (_typically a bridge network_). + +By default, the Docker daemon only assigns IPv4 addresses to containers, thus it will only accept IPv4 connections (_unless a `docker-proxy` process is listening, which the default daemon setting `userland-proxy: true` enables_). With the daemon setting `userland-proxy: true` (default), IPv6 connections from the host can also be accepted and routed to containers (_even when they only have IPv4 addresses assigned_). `userland-proxy: false` will require the container to have atleast an IPv6 address assigned. + +This can be problematic for IPv6 host connections when internally the container is no longer aware of the original client IPv6 address, as it has been proxied through the IPv4 or IPv6 gateway address of it's connected network (_eg: `172.17.0.1` - Docker allocates networks from a set of [default subnets][docker-subnets]_). + +This can be fixed by enabling a Docker network to assign IPv6 addresses to containers, along with some additional configuration. Alternatively you could configure the opposite to prevent IPv6 connections being made. + +## Prevent IPv6 connections + +- Avoiding an `AAAA` DNS record for your DMS FQDN would prevent resolving an IPv6 address to connect to. +- You can also use `userland-proxy: false`, which will fail to establish a remote connection to DMS (_provided no IPv6 address was assigned_). + +!!! tip "With UFW or Firewalld" + + When one of these firewall frontends are active, remote clients should fail to connect instead of being masqueraded as the docker network gateway IP. Keep in mind that this only affects remote clients, it does not affect local IPv6 connections originating within the same host. + +## Enable proper IPv6 support + +You can enable IPv6 support in Docker for container networks, however [compatibility concerns][docs-compat] may affect your success. + +The [official Docker documentation on enabling IPv6][docker-docs-enable-ipv6] has been improving and is a good resource to reference. + +Enable `ip6tables` support so that Docker will manage IPv6 networking rules as well. This will allow for IPv6 NAT to work like the existing IPv4 NAT already does for your containers, avoiding the above issue with external connections having their IP address seen as the container network gateway IP (_provided an IPv6 address is also assigned to the container_). + +!!! example "Configure the following in `/etc/docker/daemon.json`" + + ```json + { + "ip6tables": true, + "experimental" : true, + "userland-proxy": true + } + ``` + + - `experimental: true` is currently required for `ip6tables: true` to work. + - `userland-proxy` setting [can potentially affect connection behaviour][gh-pull-3244-proxy] for local connections. + + Now restart the daemon if it's running: `systemctl restart docker`. + +Next, configure a network for your container with any of these: + +- [User-defined networks via `docker network create` or `compose.yaml`][docker-docs-ipv6-create-custom] +- [Default docker bridge][docker-docs-ipv6-create-default] (_docker CLI only, not helpful for `compose.yaml`_) +- [Default network for a `compose.yaml`][ipv6-config-example] (_`/etc/docker/daemon.json` settings for default bridge do not apply, instead override the generated `default` network_) + +!!! danger "Do not use `2001:db8:1::/64` for your private subnet" + + The `2001:db8` address prefix is [reserved for documentation][wikipedia-ipv6-reserved]. Avoid using a subnet with this prefix. + +!!! example "User-defined IPv6 ULA subnet" + + - Either of these should work well. You can use a smaller subnet size like `/112` if you prefer. + - The network will also include an IPv4 subnet assigned implicitly. + + ```bash + # CLI + docker network create --ipv6 --subnet fd00:cafe:face:feed::/64 dms-ipv6 + ``` + + ```yaml + # compose.yaml + networks: + # Overrides the `default` compose generated network, avoids needing to attach to each service: + default: + enable_ipv6: true + subnet: fd00:cafe:face:feed::/64 + ``` + +### Configuring an IPv6 subnet + +If you've [configured IPv6 address pools in `/etc/docker/daemon.json`][docker-docs-ipv6-supernets], you do not need to specify a subnet explicitly. Otherwise if you're unsure what value to provide, here's a quick guide (_Tip: Prefer IPv6 ULA, it's the least hassle_): + +- `fd00:cafe:face:feed::/64` is an example of a [IPv6 ULA subnet][wikipedia-ipv6-ula]. ULA addresses are akin to the [private IPv4 subnets][wikipedia-ipv4-private] you may already be familiar with. You can use that example, or choose your own ULA address. This is a good choice for getting Docker containers to their have networks support IPv6 via NAT like they already do by default with IPv4. +- IPv6 without NAT, using public address space like your server is assigned belongs to an [IPv6 GUA subnet][wikipedia-ipv6-gua]. + - Typically these will be a `/64` block assigned to your host, but this varies by provider. + - These addresses do not need to publish ports of a container to another IP to be publicly reached (_thus `ip6tables: true` is not required_), you will want a firewall configured to manage which ports are accessible instead as no NAT is involved. Note that this may not be desired if the container should also be reachable via the host IPv4 public address. + - You may want to subdivide the `/64` into smaller subnets for Docker to use only portions of the `/64`. This can reduce some routing features, and [require additional setup / management via a NDP Proxy][gh-pull-3244-gua] for your public interface to know of IPv6 assignments managed by Docker and accept external traffic. + +### Verify remote IP is correct + +With Docker CLI or Docker Compose, run a `traefik/whoami` container with your IPv6 docker network and port 80 published. You can then send a curl request (or via address in the browser) from another host (as your remote client) with an IPv6 network, the `RemoteAddr` value returned should match your client IPv6 address. + +```bash +docker run --rm -d --network dms-ipv6 -p 80:80 traefik/whoami +# On a different host, replace `2001:db8::1` with your DMS host IPv6 address +curl --max-time 5 http://[2001:db8::1]:80 ``` -## Further Discussion +!!! info "IPv6 ULA address priority" + + DNS lookups that have records for both IPv4 and IPv6 addresses (_eg: `localhost`_) may prefer IPv4 over IPv6 (ULA) for private addresses, whereas for public addresses IPv6 has priority. This shouldn't be anything to worry about, but can come across as a surprise when testing your IPv6 setup on the same host instead of from a remote client. + + The preference can be controlled with [`/etc/gai.conf`][networking-gai], and appears was configured this way based on [the assumption that IPv6 ULA would never be used with NAT][networking-gai-blog]. It should only affect the destination resolved for outgoing connections, which for IPv6 ULA should only really affect connections between your containers / host. In future [IPv6 ULA may also be prioritized][networking-gai-rfc]. + +[docker-subnets]: https://straz.to/2021-09-08-docker-address-pools/#what-are-the-default-address-pools-when-no-configuration-is-given-vanilla-pools +[sender-score]: https://senderscore.org/assess/get-your-score/ +[gh-issue-1438-spf]: https://github.com/docker-mailserver/docker-mailserver/issues/1438 +[gh-issue-3057-bounce]: https://github.com/docker-mailserver/docker-mailserver/pull/3057#issuecomment-1416700046 +[wikipedia-openrelay]: https://en.wikipedia.org/wiki/Open_mail_relay + +[docs-compat]: ../debugging.md#compatibility + +[gh-pull-3244-proxy]: https://github.com/docker-mailserver/docker-mailserver/pull/3244#issuecomment-1603436809 +[docker-docs-enable-ipv6]: https://docs.docker.com/config/daemon/ipv6/ +[docker-docs-ipv6-create-custom]: https://docs.docker.com/config/daemon/ipv6/#create-an-ipv6-network +[docker-docs-ipv6-create-default]: https://docs.docker.com/config/daemon/ipv6/#use-ipv6-for-the-default-bridge-network +[docker-docs-ipv6-supernets]: https://docs.docker.com/config/daemon/ipv6/#dynamic-ipv6-subnet-allocation -See [#1438][github-issue-1438] +[ipv6-config-example]: https://github.com/nginx-proxy/nginx-proxy/issues/133#issuecomment-1368745843 +[wikipedia-ipv6-reserved]: https://en.wikipedia.org/wiki/IPv6_address#Documentation +[wikipedia-ipv4-private]: https://en.wikipedia.org/wiki/Private_network#Private_IPv4_addresses +[wikipedia-ipv6-ula]: https://en.wikipedia.org/wiki/Unique_local_address +[wikipedia-ipv6-gua]: https://en.wikipedia.org/wiki/IPv6#Global_addressing +[gh-pull-3244-gua]: https://github.com/docker-mailserver/docker-mailserver/pull/3244#issuecomment-1528984894 -[github-issue-1438]: https://github.com/docker-mailserver/docker-mailserver/issues/1438 +[networking-gai]: https://linux.die.net/man/5/gai.conf +[networking-gai-blog]: https://thomas-leister.de/en/lxd-prefer-ipv6-outgoing/ +[networking-gai-rfc]:https://datatracker.ietf.org/doc/html/draft-ietf-v6ops-ula diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index 48ef270ce13..725fb4d57f1 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -50,28 +50,30 @@ For example a text editor you can use in the terminal: `apt-get install nano` It's possible that the issue you're experiencing is due to a compatibility conflict. -This could be from outdated software updates, or running a system that isn't able to provide you newer software and kernels. You may want to verify if you can reproduce the issue on a system that is not affected by these concerns. +This could be from outdated software, or running a system that isn't able to provide you newer software and kernels. You may want to verify if you can reproduce the issue on a system that is not affected by these concerns. ### Network +- Misconfigured network connections can cause the client IP address to be proxied through a docker network gateway IP, or a [service that acts on behalf of connecting clients for logins][gh-discuss-roundcube-fail2ban] where the connections client IP appears to be only from that service (eg: Container IP) instead. This can relay the wrong information to other services (eg: monitoring like Fail2Ban, SPF verification) causing unexpected failures. - **`userland-proxy`:** Prior to Docker `v23`, [changing the `userland-proxy` setting did not reliably remove NAT rules][network::docker-userlandproxy]. -- **UFW / firewalld:** Some users expect only their firewall frontend to manage the firewall rules, but these will be bypassed when Docker publishes a container port as there is no integration between the two. +- **UFW / firewalld:** Some users expect only their firewall frontend to manage the firewall rules, but these will be bypassed when Docker publishes a container port (_as there is no integration between the two_). - **`iptables` / `nftables`:** - Docker [only manages the NAT rules via `iptables`][network::docker-nftables], relying on compatibility shims for supporting the successor `nftables`. Internally DMS expects `nftables` support on the host kernel for services like Fail2Ban to function correctly. - [Kernels older than 5.2 may affect management of NAT rules via `nftables`][network::kernel-nftables]. Other software outside of DMS may also manipulate these rules, such as firewall frontends. - **IPv6:** - - Requires [additional configuration][docs-ipv6] to prevent or properly support IPv6 connections (eg: Preservering the Client IP). + - Requires [additional configuration][docs-ipv6] to prevent or properly support IPv6 connections (eg: Preserving the Client IP). - Support in 2023 is still considered experimental. You are advised to use at least Docker Engine `v23` (2023Q1). - Various networking bug fixes have been addressed since the intitial IPv6 support arrived in Docker Engine `v20.10.0` (2020Q4). ### System - **Kernel:** Some systems provide [kernels with modifications (_replacing defaults and backporting patches_)][network::kernels-modified] to support running legacy software or kernels, complicating compatibility. This can be commonly experienced with products like NAS. -- **CGroups v2:** Hosts running older kernels (prior to 5.2) and systemd (prior to v244) are not likely to leverage cgroup v2, or have not defaulted to the cgroup v2 `unified` hierarchy. Not meeting this baseline may influence the behaviour of your DMS container, even with the latest Docker Engine installed. -- **Rootless containers** have additional constraints that vary by container runtime (_Docker, Podman, etc - which already have subtle differences_). - - This can introduce differences such as for container networking which may further impact support for IPv6 and preserving the client IP (Remote address). +- **CGroups v2:** Hosts running older kernels (prior to 5.2) and systemd (prior to v244) are not likely to leverage cgroup v2, or have not defaulted to the cgroup v2 `unified` hierarchy. Not meeting this baseline may influence the behaviour of your DMS container, even with the latest Docker Engine installed. +- **Container runtime:** Docker and Podman for example have subtle differences. DMS docs are primarily focused on Docker, but we try to document known issues where relevant. +- **Rootless containers:** Introduces additional differences in behaviour or requirements: - cgroup v2 is required for supporting rootless containers. - + - Differences such as for container networking which may further affect support for IPv6 and preserving the client IP (Remote address). Example with Docker rootless are [binding a port to a specific interface][docker-rootless-interface] and the choice of [port forwarding driver][docs-rootless-portdriver]. + [network::docker-userlandproxy]: https://github.com/moby/moby/issues/44721 [network::docker-nftables]: https://github.com/moby/moby/issues/26824 [network::kernels-modified]: https://github.com/docker-mailserver/docker-mailserver/pull/2662#issuecomment-1168435970 @@ -83,3 +85,7 @@ This could be from outdated software updates, or running a system that isn't abl [docs-introduction]: ../introduction.md [docs-usage]: ../usage.md [gh-issues]: https://github.com/docker-mailserver/docker-mailserver/issues +[gh-discuss-roundcube-fail2ban]: https://github.com/orgs/docker-mailserver/discussions/3273#discussioncomment-5654603 + +[docker-rootless-interface]: https://github.com/moby/moby/issues/45742 +[docs-rootless-portdriver]: ./security/fail2ban.md#running-inside-a-rootless-container From ee7c4b1ede534fcc565c2c8c4b1f085203859257 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jul 2023 14:54:14 +0200 Subject: [PATCH 118/592] chore(deps): Bump docker/setup-buildx-action from 2.7.0 to 2.8.0 (#3414) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 2.7.0 to 2.8.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v2.7.0...v2.8.0) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 4405c581687..80b89e39868 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -79,7 +79,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.7.0 + uses: docker/setup-buildx-action@v2.8.0 # NOTE: AMD64 can build within 2 minutes - name: 'Build images' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 80e5d73080c..1375f5bca85 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -40,7 +40,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.7.0 + uses: docker/setup-buildx-action@v2.8.0 # Try get the cached build layers from a prior `generic_build.yml` job. # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 457dd399625..ae1e09b8c1e 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -38,7 +38,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.7.0 + uses: docker/setup-buildx-action@v2.8.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index c9c3ad8e3c5..9dbdf12ff56 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -37,7 +37,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.7.0 + uses: docker/setup-buildx-action@v2.8.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. From 18f8d2573bcd8241fb77c7b1431af300c2216e4f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jul 2023 19:25:14 +0200 Subject: [PATCH 119/592] chore(deps): Bump docker/setup-buildx-action from 2.8.0 to 2.9.0 (#3421) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 80b89e39868..1326e728000 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -79,7 +79,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.8.0 + uses: docker/setup-buildx-action@v2.9.0 # NOTE: AMD64 can build within 2 minutes - name: 'Build images' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 1375f5bca85..21dc1f98060 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -40,7 +40,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.8.0 + uses: docker/setup-buildx-action@v2.9.0 # Try get the cached build layers from a prior `generic_build.yml` job. # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index ae1e09b8c1e..3c74d16433f 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -38,7 +38,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.8.0 + uses: docker/setup-buildx-action@v2.9.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 9dbdf12ff56..9b71b880273 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -37,7 +37,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.8.0 + uses: docker/setup-buildx-action@v2.9.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. From 7d5c2736cec0cc6db198da57e94a8ca589570d9f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jul 2023 21:48:19 +0200 Subject: [PATCH 120/592] chore(deps): Bump docker/setup-buildx-action from 2.9.0 to 2.9.1 (#3430) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 1326e728000..7dbe05173af 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -79,7 +79,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.9.0 + uses: docker/setup-buildx-action@v2.9.1 # NOTE: AMD64 can build within 2 minutes - name: 'Build images' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 21dc1f98060..ea837364fec 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -40,7 +40,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.9.0 + uses: docker/setup-buildx-action@v2.9.1 # Try get the cached build layers from a prior `generic_build.yml` job. # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 3c74d16433f..7c9f6f056dc 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -38,7 +38,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.9.0 + uses: docker/setup-buildx-action@v2.9.1 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 9b71b880273..934c4d487f1 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -37,7 +37,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.9.0 + uses: docker/setup-buildx-action@v2.9.1 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. From 5ef048bfaeac38269bf28c9cd0a6754f5f2e4576 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 21 Jul 2023 08:45:33 +1200 Subject: [PATCH 121/592] chore: Discourage `latest` in bug report version field (#3435) --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index be54660b866..c12439ce5f5 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -36,7 +36,7 @@ body: attributes: label: 🐋 DMS Version description: On which version (image tag) did you encounter this bug? - placeholder: v12.1.0 + placeholder: v12.1.0 (do not put "latest") validations: required: true - type: input From a0fde8b83ff6084e4b79d2d327dc00a185d48f5a Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 21 Jul 2023 09:05:19 +1200 Subject: [PATCH 122/592] docs: IPv6 config examples with content tabs (#3436) For added clarity, a user requested we document the example config snippets instead of only linking external references to them. Revised section and adjusted to presenting via the content tabs feature. --- docs/content/config/advanced/ipv6.md | 108 +++++++++++++++++++++------ 1 file changed, 86 insertions(+), 22 deletions(-) diff --git a/docs/content/config/advanced/ipv6.md b/docs/content/config/advanced/ipv6.md index 2afd1d7d912..4a45d168535 100644 --- a/docs/content/config/advanced/ipv6.md +++ b/docs/content/config/advanced/ipv6.md @@ -64,34 +64,96 @@ Enable `ip6tables` support so that Docker will manage IPv6 networking rules as w Now restart the daemon if it's running: `systemctl restart docker`. -Next, configure a network for your container with any of these: +Next, configure a network with an IPv6 subnet for your container with any of these examples: -- [User-defined networks via `docker network create` or `compose.yaml`][docker-docs-ipv6-create-custom] -- [Default docker bridge][docker-docs-ipv6-create-default] (_docker CLI only, not helpful for `compose.yaml`_) -- [Default network for a `compose.yaml`][ipv6-config-example] (_`/etc/docker/daemon.json` settings for default bridge do not apply, instead override the generated `default` network_) +???+ example "Create an IPv6 ULA subnet" -!!! danger "Do not use `2001:db8:1::/64` for your private subnet" + ??? info "About these examples" - The `2001:db8` address prefix is [reserved for documentation][wikipedia-ipv6-reserved]. Avoid using a subnet with this prefix. + These examples are focused on a [IPv6 ULA subnet][wikipedia-ipv6-ula] which is suitable for most users as described in the next section. -!!! example "User-defined IPv6 ULA subnet" + - You may prefer a subnet size smaller than `/64` (eg: `/112`, which still provides over 65k IPv6 addresses), especially if instead configuring for an IPv6 GUA subnet. + - The network will also implicitly be assigned an IPv4 subnet (_from the Docker daemon config `default-address-pools`_). - - Either of these should work well. You can use a smaller subnet size like `/112` if you prefer. - - The network will also include an IPv4 subnet assigned implicitly. + === "User-defined Network" - ```bash - # CLI - docker network create --ipv6 --subnet fd00:cafe:face:feed::/64 dms-ipv6 - ``` + The preferred approach is with [user-defined networks][docker-docs-ipv6-create-custom] via `compose.yaml` (recommended) or CLI with `docker network create`: - ```yaml - # compose.yaml - networks: - # Overrides the `default` compose generated network, avoids needing to attach to each service: - default: - enable_ipv6: true - subnet: fd00:cafe:face:feed::/64 - ``` + === "Compose" + + Create the network in `compose.yaml` and attach a service to it: + + ```yaml title="compose.yaml" + services: + mailserver: + networks: + - dms-ipv6 + + networks: + dms-ipv6: + enable_ipv6: true + subnet: fd00:cafe:face:feed::/64 + ``` + + ??? tip "Override the implicit `default` network" + + You can optionally avoid the service assignment by [overriding the `default` user-defined network that Docker Compose generates](docker-docs-network-compose-default). Just replace `dms-ipv6` with `default`. + + The Docker Compose `default` bridge is not affected by settings for the default `bridge` (aka `docker0`) in `/etc/docker/daemon.json`. + + ??? tip "Using the network outside of this `compose.yaml`" + + To reference this network externally (_from other compose files or `docker run`_), assign the [networks `name` key in `compose.yaml`][docker-docs-network-external]. + + === "CLI" + + Create the network via a CLI command (_which can then be used with `docker run --network dms-ipv6`_): + + ```bash + docker network create --ipv6 --subnet fd00:cafe:face:feed::/64 dms-ipv6 + ``` + + Optionally reference it from one or more `compose.yaml` files: + + ```yaml title="compose.yaml" + services: + mailserver: + networks: + - dms-ipv6 + + networks: + dms-ipv6: + external: true + ``` + + === "Default Bridge (daemon)" + + !!! warning "This approach is discouraged" + + The [`bridge` network is considered legacy][docker-docs-network-bridge-legacy]. + + Add these two extra IPv6 settings to your daemon config. They only apply to the [default `bridge` docker network][docker-docs-ipv6-create-default] aka `docker0` (_which containers are attached to by default when using `docker run`_). + + ```json title="/etc/docker/daemon.json" + { + "ipv6": true, + "fixed-cidr-v6": "fd00:cafe:face:feed::/64", + } + ``` + + Compose projects can also use this network via `network_mode`: + + ```yaml title="compose.yaml" + services: + mailserver: + network_mode: bridge + ``` + +!!! danger "Do not use `2001:db8:1::/64` for your private subnet" + + The `2001:db8` address prefix is [reserved for documentation][wikipedia-ipv6-reserved]. Avoid creating a subnet with this prefix. + + Presently this is used in examples for Dockers IPv6 docs as a placeholder, while mixed in with private IPv4 addresses which can be misleading. ### Configuring an IPv6 subnet @@ -132,8 +194,10 @@ curl --max-time 5 http://[2001:db8::1]:80 [docker-docs-ipv6-create-custom]: https://docs.docker.com/config/daemon/ipv6/#create-an-ipv6-network [docker-docs-ipv6-create-default]: https://docs.docker.com/config/daemon/ipv6/#use-ipv6-for-the-default-bridge-network [docker-docs-ipv6-supernets]: https://docs.docker.com/config/daemon/ipv6/#dynamic-ipv6-subnet-allocation +[docker-docs-network-external]: https://docs.docker.com/compose/compose-file/06-networks/#name +[docker-docs-network-compose-default]: https://docs.docker.com/compose/networking/#configure-the-default-network +[docker-docs-network-bridge-legacy]: https://docs.docker.com/network/drivers/bridge/#use-the-default-bridge-network -[ipv6-config-example]: https://github.com/nginx-proxy/nginx-proxy/issues/133#issuecomment-1368745843 [wikipedia-ipv6-reserved]: https://en.wikipedia.org/wiki/IPv6_address#Documentation [wikipedia-ipv4-private]: https://en.wikipedia.org/wiki/Private_network#Private_IPv4_addresses [wikipedia-ipv6-ula]: https://en.wikipedia.org/wiki/Unique_local_address From 59f483f1575c4062098af3d2ecd0686f1dc205cb Mon Sep 17 00:00:00 2001 From: rriski Date: Thu, 27 Jul 2023 03:24:36 +0300 Subject: [PATCH 123/592] docs: Fix typos (#3443) Various typos fixed in docs, in addition to a config and ENV template. --- config-examples/dovecot.cf | 2 +- docs/content/config/advanced/kubernetes.md | 2 +- docs/content/config/advanced/mail-fetchmail.md | 2 +- docs/content/config/advanced/mail-forwarding/aws-ses.md | 2 +- docs/content/config/advanced/mail-sieve.md | 2 +- docs/content/config/debugging.md | 2 +- docs/content/config/environment.md | 2 +- docs/content/config/security/ssl.md | 4 ++-- docs/content/faq.md | 2 +- docs/content/introduction.md | 2 +- mailserver.env | 6 +++--- 11 files changed, 14 insertions(+), 14 deletions(-) diff --git a/config-examples/dovecot.cf b/config-examples/dovecot.cf index 0222fc909f1..d9010b7ac09 100644 --- a/config-examples/dovecot.cf +++ b/config-examples/dovecot.cf @@ -1,4 +1,4 @@ # File for additional dovecot configurations. -# For more informations read https://doc.dovecot.org/configuration_manual/quick_configuration/ +# For more information read https://doc.dovecot.org/configuration_manual/quick_configuration/ #mail_max_userip_connections = 50 diff --git a/docs/content/config/advanced/kubernetes.md b/docs/content/config/advanced/kubernetes.md index 8846c955f9d..93cc0884557 100644 --- a/docs/content/config/advanced/kubernetes.md +++ b/docs/content/config/advanced/kubernetes.md @@ -163,7 +163,7 @@ metadata: ignore-check.kube-linter.io/run-as-non-root: >- 'mailserver' needs to run as root ignore-check.kube-linter.io/privileged-ports: >- - 'mailserver' needs privilegdes ports + 'mailserver' needs privileged ports ignore-check.kube-linter.io/no-read-only-root-fs: >- There are too many files written to make The root FS read-only diff --git a/docs/content/config/advanced/mail-fetchmail.md b/docs/content/config/advanced/mail-fetchmail.md index 59e07a844a2..254196a191e 100644 --- a/docs/content/config/advanced/mail-fetchmail.md +++ b/docs/content/config/advanced/mail-fetchmail.md @@ -75,7 +75,7 @@ To debug your `fetchmail.cf` configuration run this command: ./setup.sh debug fetchmail ``` -For more informations about the configuration script `setup.sh` [read the corresponding docs][docs-setup]. +For more information about the configuration script `setup.sh` [read the corresponding docs][docs-setup]. Here a sample output of `./setup.sh debug fetchmail`: diff --git a/docs/content/config/advanced/mail-forwarding/aws-ses.md b/docs/content/config/advanced/mail-forwarding/aws-ses.md index 891295b450b..2a4dab170d9 100644 --- a/docs/content/config/advanced/mail-forwarding/aws-ses.md +++ b/docs/content/config/advanced/mail-forwarding/aws-ses.md @@ -2,7 +2,7 @@ title: 'Mail Forwarding | AWS SES' --- -[Amazon SES (Simple Email Service)](https://aws.amazon.com/ses/) is intended to provide a simple way for cloud based applications to send email and receive email. For the purposes of this project only sending email via SES is supported. Older versions of docker-mailserver used `AWS_SES_HOST` and `AWS_SES_USERPASS` to configure sending, this has changed and the setup is mananged through [Configure Relay Hosts][docs-relay]. +[Amazon SES (Simple Email Service)](https://aws.amazon.com/ses/) is intended to provide a simple way for cloud based applications to send email and receive email. For the purposes of this project only sending email via SES is supported. Older versions of docker-mailserver used `AWS_SES_HOST` and `AWS_SES_USERPASS` to configure sending, this has changed and the setup is managed through [Configure Relay Hosts][docs-relay]. You will need to create some [Amazon SES SMTP credentials](https://docs.aws.amazon.com/ses/latest/DeveloperGuide/smtp-credentials.html). The SMTP credentials you create will be used to populate the `RELAY_USER` and `RELAY_PASSWORD` environment variables. diff --git a/docs/content/config/advanced/mail-sieve.md b/docs/content/config/advanced/mail-sieve.md index 627e12edc7f..d050bf9fc5e 100644 --- a/docs/content/config/advanced/mail-sieve.md +++ b/docs/content/config/advanced/mail-sieve.md @@ -96,7 +96,7 @@ All user defined sieve scripts that are managed by ManageSieve are stored in the !!! note - ManageSieve makes sure to not overwrite an existing `.dovecot.sieve` file. If a user activates a new sieve script the old one is backuped and moved to the `sieve` folder. + ManageSieve makes sure to not overwrite an existing `.dovecot.sieve` file. If a user activates a new sieve script the old one is backed up and moved to the `sieve` folder. The extension is known to work with the following ManageSieve clients: diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index 725fb4d57f1..a17fd5ffce9 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -63,7 +63,7 @@ This could be from outdated software, or running a system that isn't able to pro - **IPv6:** - Requires [additional configuration][docs-ipv6] to prevent or properly support IPv6 connections (eg: Preserving the Client IP). - Support in 2023 is still considered experimental. You are advised to use at least Docker Engine `v23` (2023Q1). - - Various networking bug fixes have been addressed since the intitial IPv6 support arrived in Docker Engine `v20.10.0` (2020Q4). + - Various networking bug fixes have been addressed since the initial IPv6 support arrived in Docker Engine `v20.10.0` (2020Q4). ### System diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 64c0b0d0ad5..ab9171ac629 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -645,7 +645,7 @@ The following variables overwrite the default values for ```/etc/dovecot/dovecot ##### DOVECOT_DNPASS - **empty** => same as `LDAP_BIND_PW` -- => Password for LDAP dn sepecifified in `DOVECOT_DN`. +- => Password for LDAP dn specified in `DOVECOT_DN`. ##### DOVECOT_URIS diff --git a/docs/content/config/security/ssl.md b/docs/content/config/security/ssl.md index f81470ac11b..4618fda1bb2 100644 --- a/docs/content/config/security/ssl.md +++ b/docs/content/config/security/ssl.md @@ -220,7 +220,7 @@ Obtain a Cloudflare API token: This certificate expires on YYYY-MM-DD. These files will be updated when the certificate renews. NEXT STEPS: - - The certificate will need to be renewed before it expires. Certbot can automatically renew the certificate in background, but you may need to take steps to enable that functionality. See https://certbot.org/renewal structions. + - The certificate will need to be renewed before it expires. Certbot can automatically renew the certificate in background, but you may need to take steps to enable that functionality. See https://certbot.org/renewal instructions. ``` After completing the steps above, your certificate should be ready to use. @@ -835,7 +835,7 @@ You can of course run the script by cron once a week or something. In that way y ```sh # This script is run inside docker-mailserver via 'docker exec ...', using the 'mail' command to send alerts. ## code below will alert if certificate expires in less than two weeks -## please adjust varables! +## please adjust variables! ## make sure the 'mail -s' command works! Test! export SITE_URL="mail.example.com" diff --git a/docs/content/faq.md b/docs/content/faq.md index 64728d0919d..58a327b949c 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -493,7 +493,7 @@ SA_TAG2=3.75 SA_KILL=100000.0 ``` -- The very negative vaue in `SA_TAG` makes sure, that all emails have the SpamAssassin headers included. +- The very negative value in `SA_TAG` makes sure, that all emails have the SpamAssassin headers included. - `SA_TAG2` is the actual threshold to set the YES/NO flag for spam detection. - `SA_KILL` needs to be very high, to make sure nothing is bounced at all (`SA_KILL` superseeds `SPAMASSASSIN_SPAM_TO_INBOX`) diff --git a/docs/content/introduction.md b/docs/content/introduction.md index 73c7e58c289..bbfd6ef41ed 100644 --- a/docs/content/introduction.md +++ b/docs/content/introduction.md @@ -39,7 +39,7 @@ In a nutshell, DMS provides you with the following components: - A MDA: [Dovecot](https://dovecot.org/) - A bunch of additional programs to improve security and emails processing -Here's where DMS's toochain fits within the delivery chain: +Here's where DMS's toolchain fits within the delivery chain: ```txt docker-mailserver is here: diff --git a/mailserver.env b/mailserver.env index c857aff9377..038e23b1e56 100644 --- a/mailserver.env +++ b/mailserver.env @@ -398,8 +398,8 @@ GETMAIL_POLL=5 # A second container for the ldap service is necessary (i.e. https://github.com/osixia/docker-openldap) # with the :edge tag, use ACCOUNT_PROVISIONER=LDAP -# empty => LDAP authentification is disabled -# 1 => LDAP authentification is enabled +# empty => LDAP authentication is disabled +# 1 => LDAP authentication is enabled ENABLE_LDAP= # empty => no @@ -499,7 +499,7 @@ SASLAUTHD_MECH_OPTIONS= SASLAUTHD_LDAP_SERVER= # empty => Use value of LDAP_BIND_DN -# specify an object with priviliges to search the directory tree +# specify an object with privileges to search the directory tree # e.g. active directory: SASLAUTHD_LDAP_BIND_DN=cn=Administrator,cn=Users,dc=mydomain,dc=net # e.g. openldap: SASLAUTHD_LDAP_BIND_DN=cn=admin,dc=mydomain,dc=net SASLAUTHD_LDAP_BIND_DN= From f53a40d2aed38d5143c69365e71444084b353a2b Mon Sep 17 00:00:00 2001 From: rmlhuk Date: Fri, 28 Jul 2023 00:07:26 +0100 Subject: [PATCH 124/592] docs(page:usage): Add `internet.nl` to the testing tools section (#3445) Adding `internet.nl` mail tester, this testing services gives users in-depth analysis of their mail server, connectivity, DKIM/SPF/DMARC records and DNS. --- docs/content/usage.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/usage.md b/docs/content/usage.md index 3bd106086e9..0550c24e936 100644 --- a/docs/content/usage.md +++ b/docs/content/usage.md @@ -192,3 +192,4 @@ Here are some tools you can use to verify your configuration: 2. [DMARC Analyzer](https://www.mimecast.com/products/dmarc-analyzer/spf-record-check/) 3. [mail-tester.com](https://www.mail-tester.com/) 4. [multiRBL.valli.org](https://multirbl.valli.org/) +5. [internet.nl](https://internet.nl/test-mail/) From da984e5696f124373c4098c30a49e4c5a979278d Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 28 Jul 2023 13:39:23 +0200 Subject: [PATCH 125/592] see https://github.com/docker-mailserver/docker-mailserver/issues/3433#issuecomment-1646532264 (#3439) --- target/bin/rspamd-dkim | 5 +++-- target/rspamd/local.d/options.inc | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/target/bin/rspamd-dkim b/target/bin/rspamd-dkim index 2992de5e51f..8943094db98 100755 --- a/target/bin/rspamd-dkim +++ b/target/bin/rspamd-dkim @@ -210,10 +210,11 @@ function _setup_default_signing_conf() { enabled = true; sign_authenticated = true; -sign_local = true; +sign_local = false; +try_fallback = false; use_domain = "header"; -use_redis = false; # don't change unless Redis also provides the DKIM keys +use_redis = false; # don't change unless Redis also provides the DKIM keys use_esld = true; check_pubkey = true; # you want to use this in the beginning diff --git a/target/rspamd/local.d/options.inc b/target/rspamd/local.d/options.inc index f57d4339e68..9f2a6f19846 100644 --- a/target/rspamd/local.d/options.inc +++ b/target/rspamd/local.d/options.inc @@ -1,2 +1,3 @@ pidfile = false; soft_reject_on_timeout = true; +local_networks = "127.0.0.1/8, 10.0.0.0/8, 172.16.0.0/12"; From 85603193a2f4aae0a72c929446470edafd91d865 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nils=20H=C3=B6ll?= <33981934+nilshoell@users.noreply.github.com> Date: Wed, 2 Aug 2023 02:09:01 +0200 Subject: [PATCH 126/592] feat(setup): Add `fail2ban` sub-command `status ` (#3455) * Added status command to fail2ban setup script * Switched to `printf` for command output Co-authored-by: Casper * Update docs/content/config/security/fail2ban.md Co-authored-by: Casper --------- Co-authored-by: Casper --- docs/content/config/security/fail2ban.md | 6 ++++++ target/bin/fail2ban | 7 +++++++ target/bin/setup | 1 + 3 files changed, 14 insertions(+) diff --git a/docs/content/config/security/fail2ban.md b/docs/content/config/security/fail2ban.md index 0ea052a1e5c..04e9a6814a4 100644 --- a/docs/content/config/security/fail2ban.md +++ b/docs/content/config/security/fail2ban.md @@ -58,6 +58,12 @@ setup fail2ban the script will show all banned IP addresses. +To get a more detailed `status` view, run + +```bash +setup fail2ban status +``` + ### Managing Bans You can manage F2B with the `setup` script. The usage looks like this: diff --git a/target/bin/fail2ban b/target/bin/fail2ban index 2ae9deafdef..8a76fcda1f5 100755 --- a/target/bin/fail2ban +++ b/target/bin/fail2ban @@ -6,6 +6,7 @@ source /usr/local/bin/helpers/index.sh function __usage() { echo "Usage: ./setup.sh fail2ban [ ]" echo " ./setup.sh fail2ban log" + echo " ./setup.sh fail2ban status" } fail2ban-client ping &>/dev/null || _exit_with_error "Fail2ban not running" @@ -72,6 +73,12 @@ else cat /var/log/mail/fail2ban.log ;; + ( 'status' ) + for JAIL in "${JAILS[@]}"; do + printf '%s\n\n' "$(fail2ban-client status "${JAIL}" 2>&1)" + done + ;; + ( * ) __usage _exit_with_error "Unknown command '${1}'" diff --git a/target/bin/setup b/target/bin/setup index 29782b6f28e..ac0d63282c9 100755 --- a/target/bin/setup +++ b/target/bin/setup @@ -60,6 +60,7 @@ ${RED}[${ORANGE}SUB${RED}]${ORANGE}COMMANDS${RESET} setup fail2ban ${CYAN}ban${RESET} setup fail2ban ${CYAN}unban${RESET} setup fail2ban ${CYAN}log${RESET} + setup fail2ban ${CYAN}status${RESET} ${LBLUE}COMMAND${RESET} debug ${RED}:=${RESET} setup debug ${CYAN}fetchmail${RESET} From b001f5a14056fa4df5bf50466247f3f4e4d8000e Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 4 Aug 2023 13:45:35 +0200 Subject: [PATCH 127/592] Rspamd: local network addition and user name mismatch (#3453) --- target/bin/rspamd-dkim | 1 + target/rspamd/local.d/options.inc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/target/bin/rspamd-dkim b/target/bin/rspamd-dkim index 8943094db98..7627a863625 100755 --- a/target/bin/rspamd-dkim +++ b/target/bin/rspamd-dkim @@ -216,6 +216,7 @@ try_fallback = false; use_domain = "header"; use_redis = false; # don't change unless Redis also provides the DKIM keys use_esld = true; +allow_username_mismatch = true; check_pubkey = true; # you want to use this in the beginning diff --git a/target/rspamd/local.d/options.inc b/target/rspamd/local.d/options.inc index 9f2a6f19846..8755cfade77 100644 --- a/target/rspamd/local.d/options.inc +++ b/target/rspamd/local.d/options.inc @@ -1,3 +1,3 @@ pidfile = false; soft_reject_on_timeout = true; -local_networks = "127.0.0.1/8, 10.0.0.0/8, 172.16.0.0/12"; +local_networks = "127.0.0.1/8, 10.0.0.0/8, 172.16.0.0/12 192.168.0.0/16"; From f28fce9cc432f1f447bd963d9e54e44bcf2c27dd Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 8 Aug 2023 10:43:21 +0200 Subject: [PATCH 128/592] rspamd: disable checks for authenticated users (#3440) Co-authored-by: Casper Co-authored-by: William Desportes --- docs/content/config/environment.md | 9 +++++++++ docs/content/config/security/rspamd.md | 11 ++++++----- mailserver.env | 7 +++++++ target/rspamd/local.d/settings.conf | 12 ++++++++++++ .../scripts/startup/setup.d/security/rspamd.sh | 18 +++++++++++++++++- target/scripts/startup/variables-stack.sh | 1 + .../parallel/set1/spam_virus/rspamd_full.bats | 17 +++++++++++++++-- .../set1/spam_virus/rspamd_partly.bats | 10 ++++++++++ 8 files changed, 77 insertions(+), 8 deletions(-) create mode 100644 target/rspamd/local.d/settings.conf diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index ab9171ac629..b81cef1261d 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -338,6 +338,15 @@ The purpose of this setting is to opt-out of starting an internal Redis instance - 0 => Disabled - 1 => Enabled +##### RSPAMD_CHECK_AUTHENTICATED + +This settings controls whether checks should be performed on emails coming from authenticated users (i.e. most likely outgoing emails). The default value is `0` in order to align better with SpamAssassin. **We recommend** reading through [the Rspamd documentation on scanning outbound emails][rspamd-scanning-outbound] though to decide for yourself whether you need and want this feature. + +- **0** => No checks will be performed for authenticated users +- 1 => All default checks will be performed for authenticated users + +[rspamd-scanning-outbound]: https://rspamd.com/doc/tutorials/scanning_outbound.html + ##### RSPAMD_GREYLISTING Controls whether the [Rspamd Greylisting module][rspamd-greylisting-module] is enabled. This module can further assist in avoiding spam emails by [greylisting] e-mails with a certain spam score. diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 44674e9eff0..d1d0987ebf8 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -21,11 +21,12 @@ The following environment variables are related to Rspamd: 1. [`ENABLE_RSPAMD`](../environment.md#enable_rspamd) 2. [`ENABLE_RSPAMD_REDIS`](../environment.md#enable_rspamd_redis) -3. [`RSPAMD_GREYLISTING`](../environment.md#rspamd_greylisting) -4. [`RSPAMD_HFILTER`](../environment.md#rspamd_hfilter) -5. [`RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE`](../environment.md#rspamd_hfilter_hostname_unknown_score) -6. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) -7. [`MOVE_SPAM_TO_JUNK`](../environment.md#move_spam_to_junk) +3. [`RSPAMD_CHECK_AUTHENTICATED`](../environment.md#rspamd_check_authenticated) +4. [`RSPAMD_GREYLISTING`](../environment.md#rspamd_greylisting) +5. [`RSPAMD_HFILTER`](../environment.md#rspamd_hfilter) +6. [`RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE`](../environment.md#rspamd_hfilter_hostname_unknown_score) +7. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) +8. [`MOVE_SPAM_TO_JUNK`](../environment.md#move_spam_to_junk) With these variables, you can enable Rspamd itself and you can enable / disable certain features related to Rspamd. diff --git a/mailserver.env b/mailserver.env index 038e23b1e56..cb040b9f06f 100644 --- a/mailserver.env +++ b/mailserver.env @@ -142,6 +142,13 @@ ENABLE_RSPAMD_REDIS= # 1 => enabled RSPAMD_LEARN=0 +# This settings controls whether checks should be performed on emails coming +# from authenticated users (i.e. most likely outgoing emails). The default value +# is `0` in order to align better with SpamAssassin. We recommend reading +# through https://rspamd.com/doc/tutorials/scanning_outbound.html though to +# decide for yourself whether you need and want this feature. +RSPAMD_CHECK_AUTHENTICATED=0 + # Controls whether the Rspamd Greylisting module is enabled. # This module can further assist in avoiding spam emails by greylisting # e-mails with a certain spam score. diff --git a/target/rspamd/local.d/settings.conf b/target/rspamd/local.d/settings.conf new file mode 100644 index 00000000000..4f635e749cd --- /dev/null +++ b/target/rspamd/local.d/settings.conf @@ -0,0 +1,12 @@ +# documentation: https://rspamd.com/doc/configuration/settings.html + +# DMS::SED_TAG::1::START +# Disable all checks for authenticated users +authenticated { + priority = high; + authenticated = yes; + apply { + groups_enabled = []; + } +} +# DMS::SED_TAG::1::END diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 400e1a825ba..4ece646bfc6 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -14,6 +14,7 @@ function _setup_rspamd() { __rspamd__setup_learning __rspamd__setup_greylisting __rspamd__setup_hfilter_group + __rspamd__setup_check_authenticated __rspamd__handle_user_modules_adjustments # must run last __rspamd__log 'trace' '---------- Setup finished ----------' @@ -250,7 +251,8 @@ function __rspamd__setup_hfilter_group() { if _env_var_expect_zero_or_one 'RSPAMD_HFILTER' && [[ ${RSPAMD_HFILTER} -eq 1 ]]; then __rspamd__log 'debug' 'Hfilter (group) module is enabled' # Check if we received a number first - if _env_var_expect_integer 'RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE' && [[ ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE} -ne 6 ]]; then + if _env_var_expect_integer 'RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE' \ + && [[ ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE} -ne 6 ]]; then __rspamd__log 'trace' "Adjusting score for 'HFILTER_HOSTNAME_UNKNOWN' in Hfilter group module to ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE}" sed -i -E \ "s|(.*score =).*(# __TAG__HFILTER_HOSTNAME_UNKNOWN)|\1 ${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE}; \2|g" \ @@ -264,6 +266,20 @@ function __rspamd__setup_hfilter_group() { fi } +function __rspamd__setup_check_authenticated() { + local MODULE_FILE="${RSPAMD_LOCAL_D}/settings.conf" + if _env_var_expect_zero_or_one 'RSPAMD_CHECK_AUTHENTICATED' \ + && [[ ${RSPAMD_CHECK_AUTHENTICATED} -eq 0 ]] + then + __rspamd__log 'debug' 'Content checks for authenticated users are disabled' + else + __rspamd__log 'debug' 'Enabling content checks for authenticated users' + sed -i -E \ + '/DMS::SED_TAG::1::START/{:a;N;/DMS::SED_TAG::1::END/!ba};/authenticated/d' \ + "${MODULE_FILE}" + fi +} + # Parses `RSPAMD_CUSTOM_COMMANDS_FILE` and executed the directives given by the file. # To get a detailed explanation of the commands and how the file works, visit # https://docker-mailserver.github.io/docker-mailserver/edge/config/security/rspamd/#with-the-help-of-a-custom-file diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index c2a52d5c46a..d6c5453cd5f 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -51,6 +51,7 @@ function __environment_variables_general_setup() { VARS[POSTGREY_MAX_AGE]="${POSTGREY_MAX_AGE:=35}" VARS[POSTGREY_TEXT]="${POSTGREY_TEXT:=Delayed by Postgrey}" VARS[POSTSCREEN_ACTION]="${POSTSCREEN_ACTION:=enforce}" + VARS[RSPAMD_CHECK_AUTHENTICATED]="${RSPAMD_CHECK_AUTHENTICATED:=0}" VARS[RSPAMD_GREYLISTING]="${RSPAMD_GREYLISTING:=0}" VARS[RSPAMD_HFILTER]="${RSPAMD_HFILTER:=1}" VARS[RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE]="${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE:=6}" diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index 536ce43dd0f..3fbf59d2b04 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -25,6 +25,7 @@ function setup_file() { --env LOG_LEVEL=trace --env MOVE_SPAM_TO_JUNK=1 --env RSPAMD_LEARN=1 + --env RSPAMD_CHECK_AUTHENTICATED=0 --env RSPAMD_GREYLISTING=1 --env RSPAMD_HFILTER=1 --env RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE=7 @@ -292,10 +293,22 @@ function teardown_file() { _default_teardown ; } } @test 'hfilter group module is configured correctly' { - _run_in_container_bash '[[ -f /etc/rspamd/local.d/hfilter_group.conf ]]' + local MODULE_FILE='/etc/rspamd/local.d/hfilter_group.conf' + _run_in_container_bash "[[ -f ${MODULE_FILE} ]]" assert_success - _run_in_container grep '__TAG__HFILTER_HOSTNAME_UNKNOWN' /etc/rspamd/local.d/hfilter_group.conf + _run_in_container grep '__TAG__HFILTER_HOSTNAME_UNKNOWN' "${MODULE_FILE}" assert_success assert_output --partial 'score = 7;' } + +@test 'checks on authenticated users are disabled' { + local MODULE_FILE='/etc/rspamd/local.d/settings.conf' + _run_in_container_bash "[[ -f ${MODULE_FILE} ]]" + assert_success + + _run_in_container grep -E -A 6 'authenticated \{' "${MODULE_FILE}" + assert_success + assert_output --partial 'authenticated = yes;' + assert_output --partial 'groups_enabled = [];' +} diff --git a/test/tests/parallel/set1/spam_virus/rspamd_partly.bats b/test/tests/parallel/set1/spam_virus/rspamd_partly.bats index 0c7983f86fa..9fc8af31570 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_partly.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_partly.bats @@ -24,6 +24,7 @@ function setup_file() { --env LOG_LEVEL=trace --env MOVE_SPAM_TO_JUNK=0 --env RSPAMD_LEARN=0 + --env RSPAMD_CHECK_AUTHENTICATED=1 --env RSPAMD_GREYLISTING=0 --env RSPAMD_HFILTER=0 ) @@ -85,3 +86,12 @@ function teardown_file() { _default_teardown ; } _run_in_container_bash '[[ -f /etc/rspamd/local.d/hfilter_group.conf ]]' assert_failure } + +@test 'checks on authenticated users are enabled' { + local MODULE_FILE='/etc/rspamd/local.d/settings.conf' + _run_in_container_bash "[[ -f ${MODULE_FILE} ]]" + assert_success + + _run_in_container grep -E 'authenticated \{' "${MODULE_FILE}" + assert_failure +} From 8f971713360c4620046c7f223fe66df9ea800253 Mon Sep 17 00:00:00 2001 From: Casper Date: Mon, 14 Aug 2023 01:58:54 +0200 Subject: [PATCH 129/592] compose.yaml: Add comment about disabled authentication on port 25 (#3464) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index 37c42c7c546..e9c49596234 100644 --- a/compose.yaml +++ b/compose.yaml @@ -9,7 +9,7 @@ services: # https://docker-mailserver.github.io/docker-mailserver/latest/config/security/understanding-the-ports/ # To avoid conflicts with yaml base-60 float, DO NOT remove the quotation marks. ports: - - "25:25" # SMTP (explicit TLS => STARTTLS) + - "25:25" # SMTP (explicit TLS => STARTTLS, Authentication is DISABLED => use port 465/587 instead) - "143:143" # IMAP4 (explicit TLS => STARTTLS) - "465:465" # ESMTP (implicit TLS) - "587:587" # ESMTP (explicit TLS => STARTTLS) From 5bada0a83b0875b36934e1de856b724979c83346 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 17 Aug 2023 14:33:34 +1200 Subject: [PATCH 130/592] tests: Refactor LDAP tests to current conventions (#3483) * tests: Switch to setup helper and conventions * tests: Adapt run command to new conventions - We have two helper methods with implicit `CONTAINER_NAME` reference, which is a bit more DRY and improves readability. - `wc -l` + `assert_output 1` converted to use helper `_should_output_number_of_lines 1` - `DOMAIN` var changed from `my-domain.com` to local testing domain `example.test`. * tests: Refactor `setup_file()` - Test wide ENV defined at the top - OpenLDAP build and run logic grouped together. Added notes, network alias and tty not required. - Extracted out special LDAP Postfix/Dovecot ENV into separate array. LDAP specific provisioning / auth ENV also included, with comments + linebreak to better group related ENV. - Likewise additional ENV to support test cases has been extracted to a separate array with additional comments for context. - Those two arrays are expanded back into the main `CUSTOM_SETUP_ARGUMENTS` that configure hostname and network for the DMS container. * tests: Refactor the LDAP account table query testcase - Covers 3 accounts to test from LDAP. - 2 are the same query against users/aliases/groups tables in Postfix, only differing by account queried (and expected as returned result). - 1 separate test to ensure a difference in config is supported correctly. - Extracted repeated test logic into a helper method. - Added additional context in comments about the creation source of these LDAP accounts and their related Postfix config / interaction. Direct reference to special case PR (since `git blame` will be less useful). * tests: Use iteration for `grep` setting checks More DRY approach. With a bit more helpful failure context via `assert_output` (_and only grepping the key_). Simpler to grok what's being covered. * tests: DRY test email delivery A bit more verbose with the new helper method. `test-email.txt` template is only used by the LDAP test, as is the `sendmail` command. Helper will take two args to support the testcases, but at a later date should be refactored to be consistent with the `_send_email()` helper (_which presently uses `nc` that is required for plain-text mail/auth, otherwise we'd have used `openssl`, bigger refactor required_). * tests: Slight revisions and relocating testcases - Dovecot quota plugin testcase revised to check files exist instead of rely on `ls` failure. - Moved Postfix quota plugin testcase into prior dovecot testcase for quota plugin check. Better error output by only querying the `smtpd_recipient_restrictions` setting (_which should be the only one configured for the service_). - Moved the saslauthd and pflogsumm testcases (_no changes beyond revised comments_) above the `ATTENTION` comment, and one testcase below the comment that belonged to that group. * tests: Simplify openldap `docker build` command - `--no-cache` was creating a new image on the Docker host each time the test is run. Shouldn't be any need to build without cache. - No need to use `pushd` + `popd`, can just provide the path context directly, and the `./Dockerfile` is an implicit default thus `-f` not required either. Additionally removed the old `checking` prefix from testcase names. * tests: Move LDAP specific config into `test/config/ldap/` - No changes to any of these config files, just better isolation as not relevant to any other tests. - Section heading in `setup_file()` added to distinguish the remainder of the function is dedicated to the DMS container setup. - Comment providing some context about the `mv` to maintainers, this should be done after defaults are initialized but before starting up the container. * chore: Appease the lint gods * Apply suggestions from code review --- .../ldap}/docker-openldap/Dockerfile | 0 .../bootstrap/ldif/01_mail-tree.ldif | 0 .../bootstrap/ldif/02_user-email.ldif | 0 .../03_user-email-other-primary-domain.ldif | 0 .../ldif/04_user-email-different-uid.ldif | 0 .../bootstrap/schema/mmc/postfix-book.schema | 0 .../{ => ldap/overrides}/ldap-aliases.cf | 0 .../{ => ldap/overrides}/ldap-groups.cf | 0 .../config/{ => ldap/overrides}/ldap-users.cf | 0 test/tests/serial/mail_with_ldap.bats | 418 ++++++++++-------- 10 files changed, 243 insertions(+), 175 deletions(-) rename test/{ => config/ldap}/docker-openldap/Dockerfile (100%) rename test/{ => config/ldap}/docker-openldap/bootstrap/ldif/01_mail-tree.ldif (100%) rename test/{ => config/ldap}/docker-openldap/bootstrap/ldif/02_user-email.ldif (100%) rename test/{ => config/ldap}/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif (100%) rename test/{ => config/ldap}/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif (100%) rename test/{ => config/ldap}/docker-openldap/bootstrap/schema/mmc/postfix-book.schema (100%) rename test/config/{ => ldap/overrides}/ldap-aliases.cf (100%) rename test/config/{ => ldap/overrides}/ldap-groups.cf (100%) rename test/config/{ => ldap/overrides}/ldap-users.cf (100%) diff --git a/test/docker-openldap/Dockerfile b/test/config/ldap/docker-openldap/Dockerfile similarity index 100% rename from test/docker-openldap/Dockerfile rename to test/config/ldap/docker-openldap/Dockerfile diff --git a/test/docker-openldap/bootstrap/ldif/01_mail-tree.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif similarity index 100% rename from test/docker-openldap/bootstrap/ldif/01_mail-tree.ldif rename to test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif diff --git a/test/docker-openldap/bootstrap/ldif/02_user-email.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif similarity index 100% rename from test/docker-openldap/bootstrap/ldif/02_user-email.ldif rename to test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif diff --git a/test/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif similarity index 100% rename from test/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif rename to test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif diff --git a/test/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif similarity index 100% rename from test/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif rename to test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif diff --git a/test/docker-openldap/bootstrap/schema/mmc/postfix-book.schema b/test/config/ldap/docker-openldap/bootstrap/schema/mmc/postfix-book.schema similarity index 100% rename from test/docker-openldap/bootstrap/schema/mmc/postfix-book.schema rename to test/config/ldap/docker-openldap/bootstrap/schema/mmc/postfix-book.schema diff --git a/test/config/ldap-aliases.cf b/test/config/ldap/overrides/ldap-aliases.cf similarity index 100% rename from test/config/ldap-aliases.cf rename to test/config/ldap/overrides/ldap-aliases.cf diff --git a/test/config/ldap-groups.cf b/test/config/ldap/overrides/ldap-groups.cf similarity index 100% rename from test/config/ldap-groups.cf rename to test/config/ldap/overrides/ldap-groups.cf diff --git a/test/config/ldap-users.cf b/test/config/ldap/overrides/ldap-users.cf similarity index 100% rename from test/config/ldap-users.cf rename to test/config/ldap/overrides/ldap-users.cf diff --git a/test/tests/serial/mail_with_ldap.bats b/test/tests/serial/mail_with_ldap.bats index 75768bbb82f..57251aa1356 100644 --- a/test/tests/serial/mail_with_ldap.bats +++ b/test/tests/serial/mail_with_ldap.bats @@ -1,245 +1,313 @@ -load "${REPOSITORY_ROOT}/test/test_helper/common" +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" -function setup_file() { - pushd test/docker-openldap/ || return 1 - docker build -f Dockerfile -t ldap --no-cache . - popd || return 1 +BATS_TEST_NAME_PREFIX='[LDAP] ' +CONTAINER1_NAME='dms-test_ldap' +CONTAINER2_NAME='dms-test_ldap_provider' - export DOMAIN='my-domain.com' +function setup_file() { + export DMS_TEST_NETWORK='test-network-ldap' + export DOMAIN='example.test' export FQDN_MAIL="mail.${DOMAIN}" export FQDN_LDAP="ldap.${DOMAIN}" + # LDAP is provisioned with two domains (via `.ldif` files) unrelated to the FQDN of DMS: export FQDN_LOCALHOST_A='localhost.localdomain' export FQDN_LOCALHOST_B='localhost.otherdomain' - export DMS_TEST_NETWORK='test-network-ldap' + # Link the test containers to separate network: # NOTE: If the network already exists, test will fail to start. docker network create "${DMS_TEST_NETWORK}" - docker run -d --name ldap_for_mail \ + # Setup local openldap service: + # NOTE: Building via Dockerfile is required? Image won't accept read-only if it needs to adjust permissions for bootstrap files. + # TODO: Upstream image is no longer maintained, may want to migrate? + docker build -t dms-openldap test/config/ldap/docker-openldap/ + + docker run -d --name "${CONTAINER2_NAME}" \ --env LDAP_DOMAIN="${FQDN_LOCALHOST_A}" \ - --network "${DMS_TEST_NETWORK}" \ - --network-alias 'ldap' \ --hostname "${FQDN_LDAP}" \ - --tty \ - ldap # Image name - - # _setup_ldap uses _replace_by_env_in_file with ENV vars like DOVECOT_TLS with a prefix (eg. DOVECOT_ or LDAP_) - local PRIVATE_CONFIG - PRIVATE_CONFIG=$(duplicate_config_for_container .) - docker run -d --name mail_with_ldap \ - -v "${PRIVATE_CONFIG}:/tmp/docker-mailserver" \ - -v "$(pwd)/test/test-files:/tmp/docker-mailserver-test:ro" \ - -e DOVECOT_PASS_FILTER="(&(objectClass=PostfixBookMailAccount)(uniqueIdentifier=%n))" \ - -e DOVECOT_TLS=no \ - -e DOVECOT_USER_FILTER="(&(objectClass=PostfixBookMailAccount)(uniqueIdentifier=%n))" \ - -e ACCOUNT_PROVISIONER=LDAP \ - -e PFLOGSUMM_TRIGGER=logrotate \ - -e ENABLE_SASLAUTHD=1 \ - -e LDAP_BIND_DN=cn=admin,dc=localhost,dc=localdomain \ - -e LDAP_BIND_PW=admin \ - -e LDAP_QUERY_FILTER_ALIAS="(|(&(mailAlias=%s)(objectClass=PostfixBookMailForward))(&(mailAlias=%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)))" \ - -e LDAP_QUERY_FILTER_DOMAIN="(|(&(mail=*@%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE))(&(mailGroupMember=*@%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE))(&(mailalias=*@%s)(objectClass=PostfixBookMailForward)))" \ - -e LDAP_QUERY_FILTER_GROUP="(&(mailGroupMember=%s)(mailEnabled=TRUE))" \ - -e LDAP_QUERY_FILTER_SENDERS="(|(&(mail=%s)(mailEnabled=TRUE))(&(mailGroupMember=%s)(mailEnabled=TRUE))(|(&(mailAlias=%s)(objectClass=PostfixBookMailForward))(&(mailAlias=%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)))(uniqueIdentifier=some.user.id))" \ - -e LDAP_QUERY_FILTER_USER="(&(mail=%s)(mailEnabled=TRUE))" \ - -e LDAP_START_TLS=no \ - -e LDAP_SEARCH_BASE=ou=people,dc=localhost,dc=localdomain \ - -e LDAP_SERVER_HOST=ldap \ - -e PERMIT_DOCKER=container \ - -e POSTMASTER_ADDRESS="postmaster@${FQDN_LOCALHOST_A}" \ - -e REPORT_RECIPIENT=1 \ - -e SASLAUTHD_MECHANISMS=ldap \ - -e SPOOF_PROTECTION=1 \ - -e SSL_TYPE='snakeoil' \ --network "${DMS_TEST_NETWORK}" \ - --hostname "${FQDN_MAIL}" \ - --tty \ - "${NAME}" # Image name - - wait_for_smtp_port_in_container mail_with_ldap + dms-openldap + + # + # Setup DMS container + # + + local ENV_LDAP_CONFIG=( + # Configure for LDAP account provisioner and alternative to Dovecot SASL: + --env ACCOUNT_PROVISIONER=LDAP + --env ENABLE_SASLAUTHD=1 + --env SASLAUTHD_MECHANISMS=ldap + + # ENV to configure LDAP configs for Dovecot + Postfix: + # NOTE: `scripts/startup/setup.d/ldap.sh:_setup_ldap()` uses `_replace_by_env_in_file()` to configure settings (stripping `DOVECOT_` / `LDAP_` prefixes): + # Dovecot: + --env DOVECOT_PASS_FILTER='(&(objectClass=PostfixBookMailAccount)(uniqueIdentifier=%n))' + --env DOVECOT_TLS=no + --env DOVECOT_USER_FILTER='(&(objectClass=PostfixBookMailAccount)(uniqueIdentifier=%n))' + # Postfix: + --env LDAP_BIND_DN='cn=admin,dc=localhost,dc=localdomain' + --env LDAP_BIND_PW='admin' + --env LDAP_QUERY_FILTER_ALIAS='(|(&(mailAlias=%s)(objectClass=PostfixBookMailForward))(&(mailAlias=%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)))' + --env LDAP_QUERY_FILTER_DOMAIN='(|(&(mail=*@%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE))(&(mailGroupMember=*@%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE))(&(mailalias=*@%s)(objectClass=PostfixBookMailForward)))' + --env LDAP_QUERY_FILTER_GROUP='(&(mailGroupMember=%s)(mailEnabled=TRUE))' + --env LDAP_QUERY_FILTER_SENDERS='(|(&(mail=%s)(mailEnabled=TRUE))(&(mailGroupMember=%s)(mailEnabled=TRUE))(|(&(mailAlias=%s)(objectClass=PostfixBookMailForward))(&(mailAlias=%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)))(uniqueIdentifier=some.user.id))' + --env LDAP_QUERY_FILTER_USER='(&(mail=%s)(mailEnabled=TRUE))' + --env LDAP_SEARCH_BASE='ou=people,dc=localhost,dc=localdomain' + --env LDAP_SERVER_HOST="${FQDN_LDAP}" + --env LDAP_START_TLS=no + ) + + # Extra ENV needed to support specific testcases: + local ENV_SUPPORT=( + --env PERMIT_DOCKER=container # Required for attempting SMTP auth on port 25 via nc + # Required for openssl commands to be successul: + # NOTE: snakeoil cert is created (for `docker-mailserver.invalid`) via Debian post-install script for Postfix package. + # TODO: Use proper TLS cert + --env SSL_TYPE='snakeoil' + + # TODO; All below are questionable value to LDAP tests? + --env POSTMASTER_ADDRESS="postmaster@${FQDN_LOCALHOST_A}" # TODO: Only required because LDAP accounts use unrelated domain part. FQDN_LOCALHOST_A / ldif files can be adjusted to FQDN_MAIL + --env PFLOGSUMM_TRIGGER=logrotate + --env REPORT_RECIPIENT=1 # TODO: Invalid value, should be a recipient address (if not default postmaster), remove? + --env SPOOF_PROTECTION=1 + ) + + local CUSTOM_SETUP_ARGUMENTS=( + --hostname "${FQDN_MAIL}" + --network "${DMS_TEST_NETWORK}" + + "${ENV_LDAP_CONFIG[@]}" + "${ENV_SUPPORT[@]}" + ) + + # Set default implicit container fallback for helpers: + export CONTAINER_NAME=${CONTAINER1_NAME} + + _init_with_defaults + # NOTE: `test/config/` has now been duplicated, can move test specific files to host-side `/tmp/docker-mailserver`: + mv "${TEST_TMP_CONFIG}/ldap/overrides/"*.cf "${TEST_TMP_CONFIG}/" + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + _wait_for_smtp_port_in_container } function teardown_file() { - docker rm -f ldap_for_mail mail_with_ldap + docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}" docker network rm "${DMS_TEST_NETWORK}" } # postfix -@test "checking postfix: ldap lookup works correctly" { - run docker exec mail_with_ldap /bin/sh -c "postmap -q some.user@${FQDN_LOCALHOST_A} ldap:/etc/postfix/ldap-users.cf" - assert_success - assert_output "some.user@${FQDN_LOCALHOST_A}" - run docker exec mail_with_ldap /bin/sh -c "postmap -q postmaster@${FQDN_LOCALHOST_A} ldap:/etc/postfix/ldap-aliases.cf" - assert_success - assert_output "some.user@${FQDN_LOCALHOST_A}" - run docker exec mail_with_ldap /bin/sh -c "postmap -q employees@${FQDN_LOCALHOST_A} ldap:/etc/postfix/ldap-groups.cf" - assert_success - assert_output "some.user@${FQDN_LOCALHOST_A}" - - # Test of the user part of the domain is not the same as the uniqueIdentifier part in the ldap - run docker exec mail_with_ldap /bin/sh -c "postmap -q some.user.email@${FQDN_LOCALHOST_A} ldap:/etc/postfix/ldap-users.cf" - assert_success - assert_output "some.user.email@${FQDN_LOCALHOST_A}" +# NOTE: Each of the 3 user accounts tested below are defined in separate LDIF config files, +# Those are bundled into the locally built OpenLDAP Dockerfile. +@test "postfix: ldap lookup works correctly" { + _should_exist_in_ldap_tables "some.user@${FQDN_LOCALHOST_A}" # Test email receiving from a other domain then the primary domain of the mailserver - run docker exec mail_with_ldap /bin/sh -c "postmap -q some.other.user@${FQDN_LOCALHOST_B} ldap:/etc/postfix/ldap-users.cf" - assert_success - assert_output "some.other.user@${FQDN_LOCALHOST_B}" - run docker exec mail_with_ldap /bin/sh -c "postmap -q postmaster@${FQDN_LOCALHOST_B} ldap:/etc/postfix/ldap-aliases.cf" - assert_success - assert_output "some.other.user@${FQDN_LOCALHOST_B}" - run docker exec mail_with_ldap /bin/sh -c "postmap -q employees@${FQDN_LOCALHOST_B} ldap:/etc/postfix/ldap-groups.cf" + _should_exist_in_ldap_tables "some.other.user@${FQDN_LOCALHOST_B}" + + # Should not require `uniqueIdentifier` to match the local part of `mail` (`.ldif` defined settings): + # REF: https://github.com/docker-mailserver/docker-mailserver/pull/642#issuecomment-313916384 + # NOTE: This account has no `mailAlias` or `mailGroupMember` defined in it's `.ldif`. + local MAIL_ACCOUNT="some.user.email@${FQDN_LOCALHOST_A}" + _run_in_container postmap -q "${MAIL_ACCOUNT}" ldap:/etc/postfix/ldap-users.cf assert_success - assert_output "some.other.user@${FQDN_LOCALHOST_B}" + assert_output "${MAIL_ACCOUNT}" } -@test "checking postfix: ldap custom config files copied" { - run docker exec mail_with_ldap /bin/sh -c "grep '# Testconfig for ldap integration' /etc/postfix/ldap-users.cf" - assert_success +# Custom LDAP config files support: +# TODO: Compare to provided configs and if they're just including a test comment, +# could just copy the config and append without carrying a separate test config? +@test "postfix: ldap custom config files copied" { + local LDAP_CONFIGS_POSTFIX=( + /etc/postfix/ldap-users.cf + /etc/postfix/ldap-groups.cf + /etc/postfix/ldap-aliases.cf + ) + + for LDAP_CONFIG in "${LDAP_CONFIGS_POSTFIX[@]}"; do + _run_in_container grep '# Testconfig for ldap integration' "${LDAP_CONFIG}" + assert_success + done +} - run docker exec mail_with_ldap /bin/sh -c "grep '# Testconfig for ldap integration' /etc/postfix/ldap-groups.cf" - assert_success +@test "postfix: ldap config overwrites success" { + local LDAP_SETTINGS_POSTFIX=( + "server_host = ${FQDN_LDAP}" + 'start_tls = no' + 'search_base = ou=people,dc=localhost,dc=localdomain' + 'bind_dn = cn=admin,dc=localhost,dc=localdomain' + ) + + for LDAP_SETTING in "${LDAP_SETTINGS_POSTFIX[@]}"; do + # "${LDAP_SETTING%=*}" is to match only the key portion of the var (helpful for assert_output error messages) + # NOTE: `start_tls = no` is a default setting, but the white-space differs when ENV `LDAP_START_TLS` is not set explicitly. + _run_in_container grep "${LDAP_SETTING%=*}" /etc/postfix/ldap-users.cf + assert_output "${LDAP_SETTING}" + assert_success + + _run_in_container grep "${LDAP_SETTING%=*}" /etc/postfix/ldap-groups.cf + assert_output "${LDAP_SETTING}" + assert_success + + _run_in_container grep "${LDAP_SETTING%=*}" /etc/postfix/ldap-aliases.cf + assert_output "${LDAP_SETTING}" + assert_success + done +} - run docker exec mail_with_ldap /bin/sh -c "grep '# Testconfig for ldap integration' /etc/postfix/ldap-aliases.cf" +# dovecot +@test "dovecot: ldap imap connection and authentication works" { + _run_in_container_bash 'nc -w 1 0.0.0.0 143 < /tmp/docker-mailserver-test/auth/imap-ldap-auth.txt' assert_success } -@test "checking postfix: ldap config overwrites success" { - run docker exec mail_with_ldap /bin/sh -c "grep 'server_host = ldap' /etc/postfix/ldap-users.cf" - assert_success +@test "dovecot: ldap mail delivery works" { + _should_successfully_deliver_mail_to "some.user@${FQDN_LOCALHOST_A}" "/var/mail/${FQDN_LOCALHOST_A}/some.user/new/" - run docker exec mail_with_ldap /bin/sh -c "grep 'start_tls = no' /etc/postfix/ldap-users.cf" - assert_success + # Should support delivering to a local recipient with a different domain (and disjoint mail location): + # NOTE: Mail is delivered to location defined in `.ldif` (an account config setting, either `mailHomeDirectory` or `mailStorageDirectory`). + # `some.other.user` has been configured to use a mailbox domain different from it's address domain part, hence the difference here: + _should_successfully_deliver_mail_to "some.other.user@${FQDN_LOCALHOST_B}" "/var/mail/${FQDN_LOCALHOST_A}/some.other.user/new/" +} - run docker exec mail_with_ldap /bin/sh -c "grep 'search_base = ou=people,dc=localhost,dc=localdomain' /etc/postfix/ldap-users.cf" - assert_success +@test "dovecot: ldap config overwrites success" { + local LDAP_SETTINGS_DOVECOT=( + "uris = ldap://${FQDN_LDAP}" + 'tls = no' + 'base = ou=people,dc=localhost,dc=localdomain' + 'dn = cn=admin,dc=localhost,dc=localdomain' + ) + + for LDAP_SETTING in "${LDAP_SETTINGS_DOVECOT[@]}"; do + _run_in_container grep "${LDAP_SETTING%=*}" /etc/dovecot/dovecot-ldap.conf.ext + assert_output "${LDAP_SETTING}" + assert_success + done +} - run docker exec mail_with_ldap /bin/sh -c "grep 'bind_dn = cn=admin,dc=localhost,dc=localdomain' /etc/postfix/ldap-users.cf" +# Requires ENV `POSTMASTER_ADDRESS` +# NOTE: Not important to LDAP feature tests? +@test "dovecot: postmaster address" { + _run_in_container grep "postmaster_address = postmaster@${FQDN_LOCALHOST_A}" /etc/dovecot/conf.d/15-lda.conf assert_success +} - run docker exec mail_with_ldap /bin/sh -c "grep 'server_host = ldap' /etc/postfix/ldap-groups.cf" - assert_success +# NOTE: `target/scripts/startup/setup.d/dovecot.sh` should prevent enabling the quotas feature when using LDAP: +@test "dovecot: quota plugin is disabled" { + # Dovecot configs have not enabled the quota plugins: + _run_in_container grep "\$mail_plugins quota" /etc/dovecot/conf.d/10-mail.conf + assert_failure + _run_in_container grep "\$mail_plugins imap_quota" /etc/dovecot/conf.d/20-imap.conf + assert_failure - run docker exec mail_with_ldap /bin/sh -c "grep 'start_tls = no' /etc/postfix/ldap-groups.cf" + # Dovecot Quota config only present with disabled extension: + _run_in_container_bash '[[ -f /etc/dovecot/conf.d/90-quota.conf ]]' + assert_failure + _run_in_container_bash '[[ -f /etc/dovecot/conf.d/90-quota.conf.disab ]]' assert_success - run docker exec mail_with_ldap /bin/sh -c "grep 'search_base = ou=people,dc=localhost,dc=localdomain' /etc/postfix/ldap-groups.cf" - assert_success + # Postfix quotas policy service not configured in `main.cf`: + _run_in_container postconf smtpd_recipient_restrictions + refute_output --partial 'check_policy_service inet:localhost:65265' +} - run docker exec mail_with_ldap /bin/sh -c "grep 'bind_dn = cn=admin,dc=localhost,dc=localdomain' /etc/postfix/ldap-groups.cf" +@test "saslauthd: sasl ldap authentication works" { + _run_in_container testsaslauthd -u some.user -p secret assert_success +} - run docker exec mail_with_ldap /bin/sh -c "grep 'server_host = ldap' /etc/postfix/ldap-aliases.cf" +# Requires ENV `PFLOGSUMM_TRIGGER=logrotate` +@test "pflogsumm delivery" { + # Verify default sender is `mailserver-report` when ENV `PFLOGSUMM_SENDER` + `REPORT_SENDER` are unset: + # NOTE: Mail is sent from Postfix (configured hostname used as domain part) + _run_in_container grep "mailserver-report@${FQDN_MAIL}" /etc/logrotate.d/maillog assert_success - run docker exec mail_with_ldap /bin/sh -c "grep 'start_tls = no' /etc/postfix/ldap-aliases.cf" + # When `LOGROTATE_INTERVAL` is unset, the default should be configured as `weekly`: + _run_in_container grep 'weekly' /etc/logrotate.d/maillog assert_success +} - run docker exec mail_with_ldap /bin/sh -c "grep 'search_base = ou=people,dc=localhost,dc=localdomain' /etc/postfix/ldap-aliases.cf" - assert_success +# ATTENTION: Remaining tests must come after "dovecot: ldap mail delivery works" since the below tests would affect the expected count (by delivering extra mail), +# Thus not friendly for running testcases in this file in parallel - run docker exec mail_with_ldap /bin/sh -c "grep 'bind_dn = cn=admin,dc=localhost,dc=localdomain' /etc/postfix/ldap-aliases.cf" - assert_success -} +# Requires ENV `SPOOF_PROTECTION=1` for the expected assert_output +@test "spoofing (with LDAP): rejects sender forging" { + _wait_for_smtp_port_in_container_to_respond dms-test_ldap -# dovecot -@test "checking dovecot: ldap imap connection and authentication works" { - run docker exec mail_with_ldap /bin/sh -c "nc -w 1 0.0.0.0 143 < /tmp/docker-mailserver-test/auth/imap-ldap-auth.txt" - assert_success + _run_in_container_bash 'openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/ldap-smtp-auth-spoofed.txt' + assert_output --partial 'Sender address rejected: not owned by user' } -@test "checking dovecot: ldap mail delivery works" { - run docker exec mail_with_ldap /bin/sh -c "sendmail -f user@external.tld some.user@${FQDN_LOCALHOST_A} < /tmp/docker-mailserver-test/email-templates/test-email.txt" - sleep 10 - run docker exec mail_with_ldap /bin/sh -c "grep -R 'This is a test mail.' /var/mail/${FQDN_LOCALHOST_A}/some.user/new/ | wc -l" - assert_success - assert_output 1 +@test "spoofing (with LDAP): accepts sending as alias" { + _run_in_container_bash 'openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/ldap-smtp-auth-spoofed-alias.txt' + assert_output --partial 'End data with' } -@test "checking dovecot: ldap mail delivery works for a different domain then the mailserver" { - run docker exec mail_with_ldap /bin/sh -c "sendmail -f user@external.tld some.other.user@${FQDN_LOCALHOST_B} < /tmp/docker-mailserver-test/email-templates/test-email.txt" - sleep 10 - run docker exec mail_with_ldap /bin/sh -c "ls -A /var/mail/${FQDN_LOCALHOST_A}/some.other.user/new | wc -l" - assert_success - assert_output 1 -} +@test "spoofing (with LDAP): uses senders filter" { + # skip introduced with #3006, changing port 25 to 465 + # Template used has invalid AUTH: https://github.com/docker-mailserver/docker-mailserver/pull/3006#discussion_r1073321432 + skip 'TODO: This test seems to have been broken from the start (?)' -@test "checking dovecot: ldap config overwrites success" { - run docker exec mail_with_ldap /bin/sh -c "grep 'uris = ldap://ldap' /etc/dovecot/dovecot-ldap.conf.ext" - assert_success - run docker exec mail_with_ldap /bin/sh -c "grep 'tls = no' /etc/dovecot/dovecot-ldap.conf.ext" - assert_success - run docker exec mail_with_ldap /bin/sh -c "grep 'base = ou=people,dc=localhost,dc=localdomain' /etc/dovecot/dovecot-ldap.conf.ext" - assert_success - run docker exec mail_with_ldap /bin/sh -c "grep 'dn = cn=admin,dc=localhost,dc=localdomain' /etc/dovecot/dovecot-ldap.conf.ext" - assert_success + _run_in_container_bash 'openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt' + assert_output --partial 'Sender address rejected: not owned by user' } -@test "checking dovecot: postmaster address" { - run docker exec mail_with_ldap /bin/sh -c "grep 'postmaster_address = postmaster@${FQDN_LOCALHOST_A}' /etc/dovecot/conf.d/15-lda.conf" - assert_success -} +@test "saslauthd: ldap smtp authentication" { + # Requires ENV `PERMIT_DOCKER=container` + _send_email 'auth/sasl-ldap-smtp-auth' '-w 5 0.0.0.0 25' + assert_output --partial 'Error: authentication not enabled' -@test "checking dovecot: quota plugin is disabled" { - run docker exec mail_with_ldap /bin/sh -c "grep '\$mail_plugins quota' /etc/dovecot/conf.d/10-mail.conf" - assert_failure - run docker exec mail_with_ldap /bin/sh -c "grep '\$mail_plugins imap_quota' /etc/dovecot/conf.d/20-imap.conf" - assert_failure - run docker exec mail_with_ldap ls /etc/dovecot/conf.d/90-quota.conf - assert_failure - run docker exec mail_with_ldap ls /etc/dovecot/conf.d/90-quota.conf.disab - assert_success -} + _run_in_container_bash 'openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/sasl-ldap-smtp-auth.txt' + assert_output --partial 'Authentication successful' -@test "checking postfix: dovecot quota absent in postconf" { - run docker exec mail_with_ldap /bin/bash -c "postconf | grep 'check_policy_service inet:localhost:65265'" - assert_failure + _run_in_container_bash 'openssl s_client -quiet -starttls smtp -connect 0.0.0.0:587 < /tmp/docker-mailserver-test/auth/sasl-ldap-smtp-auth.txt' + assert_output --partial 'Authentication successful' } -@test "checking spoofing (with LDAP): rejects sender forging" { - wait_for_smtp_port_in_container_to_respond mail_with_ldap - run docker exec mail_with_ldap /bin/sh -c "openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/ldap-smtp-auth-spoofed.txt | grep 'Sender address rejected: not owned by user'" - assert_success -} +# +# Test helper methods: +# -# ATTENTION: these tests must come after "checking dovecot: ldap mail delivery works" since they will deliver an email which skews the count in said test, leading to failure -@test "checking spoofing: accepts sending as alias (with LDAP)" { - run docker exec mail_with_ldap /bin/sh -c "openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/ldap-smtp-auth-spoofed-alias.txt | grep 'End data with'" - assert_success -} -@test "checking spoofing: uses senders filter" { - # skip introduced with #3006, changing port 25 to 465 - skip 'TODO: This test seems to have been broken from the start (?)' - - run docker exec mail_with_ldap /bin/sh -c "openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt | grep 'Sender address rejected: not owned by user'" - assert_success -} +function _should_exist_in_ldap_tables() { + local MAIL_ACCOUNT=${1:?Mail account is required} + local DOMAIN_PART="${MAIL_ACCOUNT#*@}" -# saslauthd -@test "checking saslauthd: sasl ldap authentication works" { - run docker exec mail_with_ldap bash -c "testsaslauthd -u some.user -p secret" - assert_success -} + # Each LDAP config file sets `query_filter` to lookup a key in LDAP (values defined in `.ldif` test files) + # `mail` (ldap-users), `mailAlias` (ldap-aliases), `mailGroupMember` (ldap-groups) + # `postmap` is queried with the mail account address, and the LDAP service should respond with + # `result_attribute` which is the LDAP `mail` value (should match what we'r'e quering `postmap` with) -@test "checking saslauthd: ldap smtp authentication" { - run docker exec mail_with_ldap /bin/sh -c "nc -w 5 0.0.0.0 25 < /tmp/docker-mailserver-test/auth/sasl-ldap-smtp-auth.txt | grep 'Error: authentication not enabled'" + _run_in_container postmap -q "${MAIL_ACCOUNT}" ldap:/etc/postfix/ldap-users.cf assert_success - run docker exec mail_with_ldap /bin/sh -c "openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/sasl-ldap-smtp-auth.txt | grep 'Authentication successful'" + assert_output "${MAIL_ACCOUNT}" + + # Check which account has the `postmaster` virtual alias: + _run_in_container postmap -q "postmaster@${DOMAIN_PART}" ldap:/etc/postfix/ldap-aliases.cf assert_success - run docker exec mail_with_ldap /bin/sh -c "openssl s_client -quiet -starttls smtp -connect 0.0.0.0:587 < /tmp/docker-mailserver-test/auth/sasl-ldap-smtp-auth.txt | grep 'Authentication successful'" + assert_output "${MAIL_ACCOUNT}" + + _run_in_container postmap -q "employees@${DOMAIN_PART}" ldap:/etc/postfix/ldap-groups.cf assert_success + assert_output "${MAIL_ACCOUNT}" } -# -# Pflogsumm delivery check -# +# NOTE: `test-email.txt` is only used for these two LDAP tests with `sendmail` command. +# The file excludes sender/recipient addresses, thus not usable with `_send_email()` helper (`nc` command)? +# TODO: Could probably adapt? +function _should_successfully_deliver_mail_to() { + local SENDER_ADDRESS='user@external.tld' + local RECIPIENT_ADDRESS=${1:?Recipient address is required} + local MAIL_STORAGE_RECIPIENT=${2:?Recipient storage location is required} + local MAIL_TEMPLATE='/tmp/docker-mailserver-test/email-templates/test-email.txt' -@test "checking pflogsum delivery" { - # checking default sender is correctly set when env variable not defined - run docker exec mail_with_ldap grep "mailserver-report@${FQDN_MAIL}" /etc/logrotate.d/maillog - assert_success + _run_in_container_bash "sendmail -f ${SENDER_ADDRESS} ${RECIPIENT_ADDRESS} < ${MAIL_TEMPLATE}" + _wait_for_empty_mail_queue_in_container - # checking default logrotation setup - run docker exec mail_with_ldap grep "weekly" /etc/logrotate.d/maillog + _run_in_container grep -R 'This is a test mail.' "${MAIL_STORAGE_RECIPIENT}" assert_success + _should_output_number_of_lines 1 + + # NOTE: Prevents compatibility for running testcases in parallel (for same container) when the count could become racey: + _count_files_in_directory_in_container "${MAIL_STORAGE_RECIPIENT}" 1 } From bb2038e8c65eaeaef20c61694fa68041de0a8d25 Mon Sep 17 00:00:00 2001 From: H4R0 Date: Mon, 21 Aug 2023 00:32:26 +0200 Subject: [PATCH 131/592] feat: Allow marking spam as read via a sieve filter (ENV `MARK_SPAM_AS_READ=1`) (#3489) * add MARK_SPAM_AS_READ environment variable * review changes Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> * update unit test --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 4 +++ docs/content/config/environment.md | 12 +++++++++ docs/content/config/security/rspamd.md | 1 + mailserver.env | 3 +++ target/scripts/start-mailserver.sh | 1 + .../scripts/startup/setup.d/security/misc.sh | 25 +++++++++++++++++++ target/scripts/startup/variables-stack.sh | 1 + .../set1/spam_virus/spam_junk_folder.bats | 23 +++++++++++++++++ 8 files changed, 70 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d13329767f0..028bffe403f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Added + +- New environment variable `MARK_SPAM_AS_READ`. When set to `1`, marks incoming junk as "read" to avoid unwanted notification of junk as new mail ([#3489](https://github.com/docker-mailserver/docker-mailserver/pull/3489)) + ## [v12.1.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v12.1.0) ### Added diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index b81cef1261d..bea02ecb358 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -309,6 +309,18 @@ will be automatically moved to the Junk folder (with the help of a Sieve script) - 0 => Spam messages will be delivered in the mailbox. - **1** => Spam messages will be delivered in the `Junk` folder. +##### MARK_SPAM_AS_READ + +Enable to treat received spam as "read" (_avoids notification to MUA client of new mail_). + +Mail is received as spam when it has been marked with either header: + +1. `X-Spam: Yes` (_by Rspamd_) +2. `X-Spam-Flag: YES` (_by SpamAssassin - requires [`SPAMASSASSIN_SPAM_TO_INBOX=1`](#spamassassin_spam_to_inbox)_) + +- **0** => disabled +- 1 => Spam messages will be marked as read + #### Rspamd ##### ENABLE_RSPAMD diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index d1d0987ebf8..7876b998816 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -27,6 +27,7 @@ The following environment variables are related to Rspamd: 6. [`RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE`](../environment.md#rspamd_hfilter_hostname_unknown_score) 7. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) 8. [`MOVE_SPAM_TO_JUNK`](../environment.md#move_spam_to_junk) +9. [`MARK_SPAM_AS_READ`](../environment.md#mark_spam_as_read) With these variables, you can enable Rspamd itself and you can enable / disable certain features related to Rspamd. diff --git a/mailserver.env b/mailserver.env index cb040b9f06f..6f270af4e84 100644 --- a/mailserver.env +++ b/mailserver.env @@ -368,6 +368,9 @@ ENABLE_SPAMASSASSIN_KAM=0 # spam messages will be moved in the Junk folder (SPAMASSASSIN_SPAM_TO_INBOX=1 required) MOVE_SPAM_TO_JUNK=1 +# spam messages wil be marked as read +MARK_SPAM_AS_READ=0 + # add spam info headers if at, or above that level: SA_TAG=2.0 diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index a002c6c3fa9..e41ae71e75f 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -48,6 +48,7 @@ function _register_functions() { _register_setup_function '_setup_dovecot_dhparam' _register_setup_function '_setup_dovecot_quota' _register_setup_function '_setup_spam_to_junk' + _register_setup_function '_setup_spam_mark_as_read' fi case "${ACCOUNT_PROVISIONER}" in diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index b1fa48606c7..78c1e60adf2 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -271,3 +271,28 @@ EOF _log 'debug' 'Spam emails will not be moved to the Junk folder' fi } + +function _setup_spam_mark_as_read() { + if [[ ${MARK_SPAM_AS_READ} -eq 1 ]]; then + _log 'debug' 'Spam emails will be marked as read' + mkdir -p /usr/lib/dovecot/sieve-global/after/ + + # Header support: `X-Spam-Flag` (SpamAssassin), `X-Spam` (Rspamd) + cat >/usr/lib/dovecot/sieve-global/after/spam_mark_as_read.sieve << EOF +require ["mailbox","imap4flags"]; + +if anyof (header :contains "X-Spam-Flag" "YES", + header :contains "X-Spam" "Yes") { + setflag "\\\\Seen"; +} +EOF + sievec /usr/lib/dovecot/sieve-global/after/spam_mark_as_read.sieve + chown dovecot:root /usr/lib/dovecot/sieve-global/after/spam_mark_as_read.{sieve,svbin} + + if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] && [[ ${SPAMASSASSIN_SPAM_TO_INBOX} -eq 0 ]]; then + _log 'warning' "'SPAMASSASSIN_SPAM_TO_INBOX=0' but it is required to be 1 for 'MARK_SPAM_AS_READ=1' to work" + fi + else + _log 'debug' 'Spam emails will not be marked as read' + fi +} diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index d6c5453cd5f..a0d61242255 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -46,6 +46,7 @@ function __environment_variables_general_setup() { VARS[CLAMAV_MESSAGE_SIZE_LIMIT]="${CLAMAV_MESSAGE_SIZE_LIMIT:=25M}" VARS[FAIL2BAN_BLOCKTYPE]="${FAIL2BAN_BLOCKTYPE:=drop}" VARS[MOVE_SPAM_TO_JUNK]="${MOVE_SPAM_TO_JUNK:=1}" + VARS[MARK_SPAM_AS_READ]="${MARK_SPAM_AS_READ:=0}" VARS[POSTGREY_AUTO_WHITELIST_CLIENTS]="${POSTGREY_AUTO_WHITELIST_CLIENTS:=5}" VARS[POSTGREY_DELAY]="${POSTGREY_DELAY:=300}" VARS[POSTGREY_MAX_AGE]="${POSTGREY_MAX_AGE:=35}" diff --git a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats index 237218e8b05..94a9b9c47e7 100644 --- a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats +++ b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats @@ -8,6 +8,7 @@ BATS_TEST_NAME_PREFIX='[Spam - Amavis] ENV SPAMASSASSIN_SPAM_TO_INBOX ' CONTAINER1_NAME='dms-test_spam-amavis_bounced' CONTAINER2_NAME='dms-test_spam-amavis_env-move-spam-to-junk-0' CONTAINER3_NAME='dms-test_spam-amavis_env-move-spam-to-junk-1' +CONTAINER4_NAME='dms-test_spam-amavis_env-mark-spam-as-read-1' function teardown() { _default_teardown ; } @@ -69,6 +70,28 @@ function teardown() { _default_teardown ; } _should_receive_spam_at '/var/mail/localhost.localdomain/user1/.Junk/new/' } +# NOTE: Same as test for `CONTAINER3_NAME`, only differing by ENV `MARK_SPAM_AS_READ=1` + `_should_receive_spam_at` location +@test "(enabled + MARK_SPAM_AS_READ=1) should mark spam message as read" { + export CONTAINER_NAME=${CONTAINER4_NAME} + + local CUSTOM_SETUP_ARGUMENTS=( + --env ENABLE_AMAVIS=1 + --env ENABLE_SPAMASSASSIN=1 + --env SA_SPAM_SUBJECT="SPAM: " + --env SPAMASSASSIN_SPAM_TO_INBOX=1 + --env MARK_SPAM_AS_READ=1 + --env PERMIT_DOCKER=container + ) + _init_with_defaults + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + + _should_send_spam_message + _should_be_received_by_amavis 'Passed SPAM {RelayedTaggedInbound,Quarantined}' + + # Should move delivered spam to Junk folder as read (`cur` instead of `new`) + _should_receive_spam_at '/var/mail/localhost.localdomain/user1/.Junk/cur/' +} + function _should_send_spam_message() { _wait_for_smtp_port_in_container _wait_for_tcp_port_in_container 10024 # port 10024 is for Amavis From 0dc862156f7cd748628643c6a3e964d37f7dfb50 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Aug 2023 10:30:10 +1200 Subject: [PATCH 132/592] chore(deps): Bump nwtgck/actions-netlify from 2.0 to 2.1 (#3495) Bumps [nwtgck/actions-netlify](https://github.com/nwtgck/actions-netlify) from 2.0 to 2.1. - [Release notes](https://github.com/nwtgck/actions-netlify/releases) - [Changelog](https://github.com/nwtgck/actions-netlify/blob/develop/CHANGELOG.md) - [Commits](https://github.com/nwtgck/actions-netlify/compare/v2.0...v2.1) --- updated-dependencies: - dependency-name: nwtgck/actions-netlify dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs-preview-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index 18b0bab94f3..278cf60c347 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -56,7 +56,7 @@ jobs: context: 'Deploy Preview (pull_request => workflow_run)' - name: 'Send preview build to Netlify' - uses: nwtgck/actions-netlify@v2.0 + uses: nwtgck/actions-netlify@v2.1 id: preview timeout-minutes: 1 env: From 758fd9c913cee012dd25b8215b8b0b16f12743ca Mon Sep 17 00:00:00 2001 From: HeySora Date: Tue, 22 Aug 2023 07:49:15 +0200 Subject: [PATCH 133/592] Docs: Drop mention of port 25 support for authenticated submission (#3496) * FAQ: Remove outdated port 25 for mail client use --- docs/content/faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/faq.md b/docs/content/faq.md index 58a327b949c..bb2fbe6382b 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -143,7 +143,7 @@ imap port: 143 or 993 with STARTTLS/SSL (recommended) imap path prefix: INBOX # SMTP -smtp port: 25 or 587/465 with STARTTLS/SSL (recommended) +smtp port: 587 or 465 with STARTTLS/SSL (recommended) username: password: ``` From cf9eb8278ad56d6a06a7e4370b9a0d7855ce9999 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 22 Aug 2023 10:03:41 +0200 Subject: [PATCH 134/592] scripts: add wrapper to update Postfix configuration safely (#3484) The new function can 1. update/append 2. update/prepend 3. initialize if non-existent options in `/etc/postfix/main.cf` in a safe and secure manner. When the container is improperly restarted, the option is not applied twice. --- Co-authored-by: Casper Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- target/scripts/helpers/aliases.sh | 7 +--- target/scripts/helpers/postfix.sh | 37 +++++++++++++++++++ target/scripts/helpers/relay.sh | 2 +- target/scripts/helpers/utils.sh | 9 +++++ .../scripts/startup/setup.d/dmarc_dkim_spf.sh | 9 ++--- .../startup/setup.d/security/rspamd.sh | 7 ++-- 6 files changed, 55 insertions(+), 16 deletions(-) diff --git a/target/scripts/helpers/aliases.sh b/target/scripts/helpers/aliases.sh index 9690aae5160..0890d994103 100644 --- a/target/scripts/helpers/aliases.sh +++ b/target/scripts/helpers/aliases.sh @@ -30,12 +30,7 @@ function _handle_postfix_regexp_config() { _log 'trace' "Adding regexp alias file postfix-regexp.cf" cp -f /tmp/docker-mailserver/postfix-regexp.cf /etc/postfix/regexp - - if ! grep 'virtual_alias_maps.*pcre:/etc/postfix/regexp' /etc/postfix/main.cf; then - sed -i -E \ - 's|virtual_alias_maps(.*)|virtual_alias_maps\1 pcre:/etc/postfix/regexp|g' \ - /etc/postfix/main.cf - fi + _add_to_or_update_postfix_main 'virtual_alias_maps' 'pcre:/etc/postfix/regexp' fi } diff --git a/target/scripts/helpers/postfix.sh b/target/scripts/helpers/postfix.sh index 32897d0e878..b376dce9eac 100644 --- a/target/scripts/helpers/postfix.sh +++ b/target/scripts/helpers/postfix.sh @@ -91,3 +91,40 @@ function _vhost_ldap_support() { # # /etc/aliases is handled by `alias.sh` and uses `postalias` to update the Postfix alias database. No need for `postmap`. # http://www.postfix.org/postalias.1.html + +# Add an key with an value to Postfix's main configuration file +# or update an existing key. An already existing key can be updated +# by either appending to the existing value (default) or by prepending. +# +# @param ${1} = key name in Postfix's main configuration file +# @param ${2} = new value (appended or prepended) +# @param ${3} = "append" (default) or "prepend" [OPTIONAL] +function _add_to_or_update_postfix_main() { + local KEY=${1:?Key name is required} + local NEW_VALUE=${2:?New value is required} + local ACTION=${3:-append} + + # If entry does not exist, add it - otherwise update with ACTION: + if ! grep -q -E "^${KEY}" /etc/postfix/main.cf; then + postconf "${KEY} = ${NEW_VALUE}" + else + KEY=$(_escape_for_sed "${KEY}") + NEW_VALUE=$(_escape_for_sed "${NEW_VALUE}") + local SED_STRING + + case "${ACTION}" in + ('append') + SED_STRING="/${NEW_VALUE}/! s|^(${KEY} *=.*)|\1 ${NEW_VALUE}|g" + ;; + ('prepend') + SED_STRING="/${NEW_VALUE}/! s|^(${KEY}) *= *(.*)|\1 = ${NEW_VALUE} \2|g" + ;; + (*) + _log 'error' "Action '${3}' in _add_to_or_update_postfix_main is unknown" + return 1 + ;; + esac + + sed -i -E "${SED_STRING}" /etc/postfix/main.cf + fi +} diff --git a/target/scripts/helpers/relay.sh b/target/scripts/helpers/relay.sh index c2713651bf6..de29dddc7a6 100644 --- a/target/scripts/helpers/relay.sh +++ b/target/scripts/helpers/relay.sh @@ -173,7 +173,7 @@ function _setup_relayhost() { _log 'debug' 'Setting up Postfix Relay Hosts' if [[ -n ${DEFAULT_RELAY_HOST} ]]; then - _log 'trace' "Setting default relay host ${DEFAULT_RELAY_HOST} to /etc/postfix/main.cf" + _log 'trace' "Setting default relay host ${DEFAULT_RELAY_HOST}" postconf "relayhost = ${DEFAULT_RELAY_HOST}" fi diff --git a/target/scripts/helpers/utils.sh b/target/scripts/helpers/utils.sh index 0c5a0321290..02d29cc9e18 100644 --- a/target/scripts/helpers/utils.sh +++ b/target/scripts/helpers/utils.sh @@ -4,6 +4,15 @@ function _escape() { echo "${1//./\\.}" } +# Replaces a string so that it can be used inside +# `sed` safely. +# +# @param ${1} = string to escape +# @output = prints the escaped string +function _escape_for_sed() { + sed -E 's/[]\/$*.^[]/\\&/g' <<< "${1:?String to escape for sed is required}" +} + # Returns input after filtering out lines that are: # empty, white-space, comments (`#` as the first non-whitespace character) function _get_valid_lines_from_file() { diff --git a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh index 85c63d963d3..c0d731f2fcb 100644 --- a/target/scripts/startup/setup.d/dmarc_dkim_spf.sh +++ b/target/scripts/startup/setup.d/dmarc_dkim_spf.sh @@ -16,10 +16,9 @@ function _setup_opendkim() { _log 'trace' "Adding OpenDKIM to Postfix's milters" postconf 'dkim_milter = inet:localhost:8891' # shellcheck disable=SC2016 - sed -i -E \ - -e '/\$dkim_milter/! s|^(smtpd_milters =.*)|\1 \$dkim_milter|g' \ - -e '/\$dkim_milter/! s|^(non_smtpd_milters =.*)|\1 \$dkim_milter|g' \ - /etc/postfix/main.cf + _add_to_or_update_postfix_main 'smtpd_milters' '$dkim_milter' + # shellcheck disable=SC2016 + _add_to_or_update_postfix_main 'non_smtpd_milters' '$dkim_milter' # check if any keys are available if [[ -e /tmp/docker-mailserver/opendkim/KeyTable ]]; then @@ -64,7 +63,7 @@ function _setup_opendmarc() { postconf 'dmarc_milter = inet:localhost:8893' # Make sure to append the OpenDMARC milter _after_ the OpenDKIM milter! # shellcheck disable=SC2016 - sed -i -E '/\$dmarc_milter/! s|^(smtpd_milters =.*)|\1 \$dmarc_milter|g' /etc/postfix/main.cf + _add_to_or_update_postfix_main 'smtpd_milters' '$dmarc_milter' sed -i \ -e "s|^AuthservID.*$|AuthservID ${HOSTNAME}|g" \ diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 4ece646bfc6..d2389a07dd6 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -138,7 +138,7 @@ function __rspamd__setup_postfix() { postconf 'rspamd_milter = inet:localhost:11332' # shellcheck disable=SC2016 - sed -i -E 's|^(smtpd_milters =.*)|\1 \$rspamd_milter|g' /etc/postfix/main.cf + _add_to_or_update_postfix_main 'smtpd_milters' '$rspamd_milter' } # If ClamAV is enabled, we will integrate it into Rspamd. @@ -180,6 +180,7 @@ function __rspamd__setup_default_modules() { metric_exporter ) + local MODULE for MODULE in "${DISABLE_MODULES[@]}"; do __rspamd__helper__enable_disable_module "${MODULE}" 'false' done @@ -289,7 +290,7 @@ function __rspamd__handle_user_modules_adjustments() { # # @param ${1} = file name in ${RSPAMD_OVERRIDE_D}/ # @param ${2} = module name as it should appear in the log - # @patam ${3} = option name in the module + # @param ${3} = option name in the module # @param ${4} = value of the option # # ## Note @@ -333,7 +334,6 @@ function __rspamd__handle_user_modules_adjustments() { while read -r COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3; do case "${COMMAND}" in - ('disable-module') __rspamd__helper__enable_disable_module "${ARGUMENT1}" 'false' 'override' ;; @@ -367,7 +367,6 @@ function __rspamd__handle_user_modules_adjustments() { __rspamd__log 'warn' "Command '${COMMAND}' is invalid" continue ;; - esac done < <(_get_valid_lines_from_file "${RSPAMD_CUSTOM_COMMANDS_FILE}") fi From 39ae101266af7511d85fbb5e9e4b19324390f5ce Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 22 Aug 2023 21:38:25 +1200 Subject: [PATCH 135/592] tests: Change OpenLDAP image to `bitnami/openldap` (#3494) **TL;DR:** - New image is actively maintained vs existing one that is over 5 years old. - Slight improvement to LDAP tree config via `.ldif` files. - No more `Dockerfile` required to build, we can just rely on `docker run`. `osixia/openldap` has not seen any activity since Feb 2021, while our `Dockerfile` was fixed to v1.1.6` (Feb 2018). Startup time for this new image is around 5 seconds? (_The LDAP test uses a standard 20 second timeout check to wait until the server is ready before continuing with starting the DMS image_). This commit migrates to `bitnami/openldap` which required modifying the `01_mail-tree.ldif` to also include adding the root object to start successfully. This image is actively maintained and one of the most popular OpenLDAP images on DockerHub. The user account `.ldif` files have minimal changes: - Lines moved around for better organization - Additional comments for context - Removal of inherited `objectClass` attributes (`person`, `top`) from the `orgnizationalPerson` class. Attribute `sn` changed to long form `surname` and values corrected with `givenName`. `changetype: add` was also not necessary. Additionally the image does not support the `.schema` format, they must be converted to `.ldif` which has been done for `postfix-book.schema`. See PR for more details. --- test/config/ldap/docker-openldap/Dockerfile | 5 -- .../bootstrap/ldif/01_mail-tree.ldif | 10 ++- .../bootstrap/ldif/02_user-email.ldif | 25 +++---- .../03_user-email-other-primary-domain.ldif | 26 ++++--- .../ldif/04_user-email-different-uid.ldif | 26 ++++--- .../bootstrap/schema/mmc/postfix-book.schema | 70 ------------------- .../bootstrap/schemas/postfix-book.ldif | 14 ++++ test/tests/serial/mail_with_ldap.bats | 17 +++-- 8 files changed, 67 insertions(+), 126 deletions(-) delete mode 100644 test/config/ldap/docker-openldap/Dockerfile delete mode 100644 test/config/ldap/docker-openldap/bootstrap/schema/mmc/postfix-book.schema create mode 100644 test/config/ldap/docker-openldap/bootstrap/schemas/postfix-book.ldif diff --git a/test/config/ldap/docker-openldap/Dockerfile b/test/config/ldap/docker-openldap/Dockerfile deleted file mode 100644 index 934c498fd8c..00000000000 --- a/test/config/ldap/docker-openldap/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM osixia/openldap:1.1.6 -LABEL maintainer="Dennis Stumm " - -COPY bootstrap /container/service/slapd/assets/config/bootstrap -RUN rm /container/service/slapd/assets/config/bootstrap/schema/mmc/mail.schema diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif index 940fef243fb..5e3d8b0a8e1 100644 --- a/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif +++ b/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif @@ -1,5 +1,11 @@ +# The root object, all entries will branch off this one: +dn: dc=localhost,dc=localdomain +objectClass: dcObject +objectClass: organization +dc: localhost +o: DMS Test + +# User accounts will belong to this subtree: dn: ou=people,dc=localhost,dc=localdomain -changetype: add objectClass: organizationalUnit -objectClass: top ou: people diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif index 993a4e70d2b..cd656bf82d8 100644 --- a/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif +++ b/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif @@ -1,25 +1,22 @@ -# -------------------------------------------------------------------- -# Create mail accounts -# -------------------------------------------------------------------- -# Some User +# NOTE: A standard user account to test against dn: uniqueIdentifier=some.user,ou=people,dc=localhost,dc=localdomain -changetype: add objectClass: organizationalPerson -objectClass: person -objectClass: top objectClass: PostfixBookMailAccount objectClass: extensibleObject cn: Some User -givenName: User +givenName: Some +surname: User +uniqueIdentifier: some.user +# Password is: secret +userPassword: {SSHA}eLtqGpid+hkSVhxvsdTPztv4uapRofGx mail: some.user@localhost.localdomain +# postfix-book.schema: mailAlias: postmaster@localhost.localdomain mailGroupMember: employees@localhost.localdomain -mailEnabled: TRUE -mailGidNumber: 5000 mailHomeDirectory: /var/mail/localhost.localdomain/some.user/ -mailQuota: 10240 mailStorageDirectory: maildir:/var/mail/localhost.localdomain/some.user/ +# postfix-book.schema generic options: +mailEnabled: TRUE mailUidNumber: 5000 -sn: Some -uniqueIdentifier: some.user -userPassword: {SSHA}eLtqGpid+hkSVhxvsdTPztv4uapRofGx +mailGidNumber: 5000 +mailQuota: 10240 diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif index f949349cc0c..d27cec7e9d2 100644 --- a/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif +++ b/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif @@ -1,25 +1,23 @@ -# -------------------------------------------------------------------- -# Create mail accounts -# -------------------------------------------------------------------- -# Some User +# NOTE: This user differs via the domain-part of their mail address +# They also have their mail directory attributes using the primary domain, not their domain-part dn: uniqueIdentifier=some.other.user,ou=people,dc=localhost,dc=localdomain -changetype: add objectClass: organizationalPerson -objectClass: person -objectClass: top objectClass: PostfixBookMailAccount objectClass: extensibleObject cn: Some Other User -givenName: Other User +givenName: Some +surname: Other User +uniqueIdentifier: some.other.user +# Password is: secret +userPassword: {SSHA}eLtqGpid+hkSVhxvsdTPztv4uapRofGx mail: some.other.user@localhost.otherdomain +# postfix-book.schema: mailAlias: postmaster@localhost.otherdomain mailGroupMember: employees@localhost.otherdomain -mailEnabled: TRUE -mailGidNumber: 5000 mailHomeDirectory: /var/mail/localhost.localdomain/some.other.user/ -mailQuota: 10240 mailStorageDirectory: maildir:/var/mail/localhost.localdomain/some.other.user/ +# postfix-book.schema generic options: +mailEnabled: TRUE mailUidNumber: 5000 -sn: Some -uniqueIdentifier: some.other.user -userPassword: {SSHA}eLtqGpid+hkSVhxvsdTPztv4uapRofGx +mailGidNumber: 5000 +mailQuota: 10240 diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif index b991993faa9..be49fd9f7ea 100644 --- a/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif +++ b/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif @@ -1,23 +1,21 @@ -# -------------------------------------------------------------------- -# Create mail accounts -# -------------------------------------------------------------------- -# Some User +# NOTE: This user differs by local-part of mail address not matching their uniqueIdentifier attribute +# They also do not have any alias or groups configured dn: uniqueIdentifier=some.user.id,ou=people,dc=localhost,dc=localdomain -changetype: add objectClass: organizationalPerson -objectClass: person -objectClass: top objectClass: PostfixBookMailAccount objectClass: extensibleObject cn: Some User -givenName: User +givenName: Some +surname: User +uniqueIdentifier: some.user.id +# Password is: secret +userPassword: {SSHA}eLtqGpid+hkSVhxvsdTPztv4uapRofGx mail: some.user.email@localhost.localdomain -mailEnabled: TRUE -mailGidNumber: 5000 +# postfix-book.schema: mailHomeDirectory: /var/mail/localhost.localdomain/some.user.id/ -mailQuota: 10240 mailStorageDirectory: maildir:/var/mail/localhost.localdomain/some.user.id/ +# postfix-book.schema generic options: +mailEnabled: TRUE mailUidNumber: 5000 -sn: Some -uniqueIdentifier: some.user.id -userPassword: {SSHA}eLtqGpid+hkSVhxvsdTPztv4uapRofGx +mailGidNumber: 5000 +mailQuota: 10240 diff --git a/test/config/ldap/docker-openldap/bootstrap/schema/mmc/postfix-book.schema b/test/config/ldap/docker-openldap/bootstrap/schema/mmc/postfix-book.schema deleted file mode 100644 index 9f0d7e53479..00000000000 --- a/test/config/ldap/docker-openldap/bootstrap/schema/mmc/postfix-book.schema +++ /dev/null @@ -1,70 +0,0 @@ -# $Id$ -# -# State of Mind -# Private Enterprise Number: 29426 -# -# OID prefix: 1.3.6.1.4.1.29426 -# -# Attributes: 1.3.6.1.4.1.29426.1.10.x -# - - -attributetype ( 1.3.6.1.4.1.29426.1.10.1 NAME 'mailHomeDirectory' - DESC 'The absolute path to the mail user home directory' - EQUALITY caseExactIA5Match - SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) - -attributetype ( 1.3.6.1.4.1.29426.1.10.2 NAME 'mailAlias' - DESC 'RFC822 Mailbox - mail alias' - EQUALITY caseIgnoreIA5Match - SUBSTR caseIgnoreIA5SubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} ) - -attributetype ( 1.3.6.1.4.1.29426.1.10.3 NAME 'mailUidNumber' - DESC 'UID required to access the mailbox' - EQUALITY integerMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE ) - -attributetype ( 1.3.6.1.4.1.29426.1.10.4 NAME 'mailGidNumber' - DESC 'GID required to access the mailbox' - EQUALITY integerMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE ) - -attributetype ( 1.3.6.1.4.1.29426.1.10.5 NAME 'mailEnabled' - DESC 'TRUE to enable, FALSE to disable account' - EQUALITY booleanMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) - -attributetype ( 1.3.6.1.4.1.29426.1.10.6 NAME 'mailGroupMember' - DESC 'Name of a mail distribution list' - EQUALITY caseExactIA5Match - SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) - -attributetype ( 1.3.6.1.4.1.29426.1.10.7 NAME 'mailQuota' - DESC 'Mail quota limit in kilobytes' - EQUALITY caseExactIA5Match - SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) - -attributetype ( 1.3.6.1.4.1.29426.1.10.8 NAME 'mailStorageDirectory' - DESC 'The absolute path to the mail users mailbox' - EQUALITY caseExactIA5Match - SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) - - -# -# Objects: 1.3.6.1.4.1.29426.1.2.2.x -# - -objectclass ( 1.3.6.1.4.1.29426.1.2.2.1 NAME 'PostfixBookMailAccount' - SUP top AUXILIARY - DESC 'Mail account used in Postfix Book' - MUST ( mail ) - MAY ( mailHomeDirectory $ mailAlias $ mailGroupMember - $ mailUidNumber $ mailGidNumber $ mailEnabled - $ mailQuota $mailStorageDirectory ) ) - -objectclass ( 1.3.6.1.4.1.29426.1.2.2.2 NAME 'PostfixBookMailForward' - SUP top AUXILIARY - DESC 'Mail forward used in Postfix Book' - MUST ( mail $ mailAlias )) - diff --git a/test/config/ldap/docker-openldap/bootstrap/schemas/postfix-book.ldif b/test/config/ldap/docker-openldap/bootstrap/schemas/postfix-book.ldif new file mode 100644 index 00000000000..9ea787d505f --- /dev/null +++ b/test/config/ldap/docker-openldap/bootstrap/schemas/postfix-book.ldif @@ -0,0 +1,14 @@ +dn: cn=postfix-book,cn=schema,cn=config +objectClass: olcSchemaConfig +cn: postfix-book +olcAttributeTypes: {0}( 1.3.6.1.4.1.29426.1.10.1 NAME 'mailHomeDirectory' DESC 'The absolute path to the mail user home directory' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) +olcAttributeTypes: {1}( 1.3.6.1.4.1.29426.1.10.2 NAME 'mailAlias' DESC 'RFC822 Mailbox - mail alias' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} ) +olcAttributeTypes: {2}( 1.3.6.1.4.1.29426.1.10.3 NAME 'mailUidNumber' DESC 'UID required to access the mailbox' EQUALITY integerMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE ) +olcAttributeTypes: {3}( 1.3.6.1.4.1.29426.1.10.4 NAME 'mailGidNumber' DESC 'GID required to access the mailbox' EQUALITY integerMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE ) +olcAttributeTypes: {4}( 1.3.6.1.4.1.29426.1.10.5 NAME 'mailEnabled' DESC 'TRUE to enable, FALSE to disable account' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) +olcAttributeTypes: {5}( 1.3.6.1.4.1.29426.1.10.6 NAME 'mailGroupMember' DESC 'Name of a mail distribution list' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) +olcAttributeTypes: {6}( 1.3.6.1.4.1.29426.1.10.7 NAME 'mailQuota' DESC 'Mail quota limit in kilobytes' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) +olcAttributeTypes: {7}( 1.3.6.1.4.1.29426.1.10.8 NAME 'mailStorageDirectory' DESC 'The absolute path to the mail users mailbox' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) +# PostfixBook object classes: +olcObjectClasses: {0}( 1.3.6.1.4.1.29426.1.2.2.1 NAME 'PostfixBookMailAccount' DESC 'Mail account used in Postfix Book' SUP top AUXILIARY MUST mail MAY ( mailHomeDirectory $ mailAlias $ mailGroupMember $ mailUidNumber $ mailGidNumber $ mailEnabled $ mailQuota $ mailStorageDirectory ) ) +olcObjectClasses: {1}( 1.3.6.1.4.1.29426.1.2.2.2 NAME 'PostfixBookMailForward' DESC 'Mail forward used in Postfix Book' SUP top AUXILIARY MUST ( mail $ mailAlias ) ) diff --git a/test/tests/serial/mail_with_ldap.bats b/test/tests/serial/mail_with_ldap.bats index 57251aa1356..baeb61e2f44 100644 --- a/test/tests/serial/mail_with_ldap.bats +++ b/test/tests/serial/mail_with_ldap.bats @@ -19,15 +19,18 @@ function setup_file() { docker network create "${DMS_TEST_NETWORK}" # Setup local openldap service: - # NOTE: Building via Dockerfile is required? Image won't accept read-only if it needs to adjust permissions for bootstrap files. - # TODO: Upstream image is no longer maintained, may want to migrate? - docker build -t dms-openldap test/config/ldap/docker-openldap/ - - docker run -d --name "${CONTAINER2_NAME}" \ - --env LDAP_DOMAIN="${FQDN_LOCALHOST_A}" \ + docker run --rm -d --name "${CONTAINER2_NAME}" \ + --env LDAP_ADMIN_PASSWORD=admin \ + --env LDAP_ROOT='dc=localhost,dc=localdomain' \ + --env LDAP_PORT_NUMBER=389 \ + --env LDAP_SKIP_DEFAULT_TREE=yes \ + --volume './test/config/ldap/docker-openldap/bootstrap/ldif/:/ldifs/:ro' \ + --volume './test/config/ldap/docker-openldap/bootstrap/schemas/:/schemas/:ro' \ --hostname "${FQDN_LDAP}" \ --network "${DMS_TEST_NETWORK}" \ - dms-openldap + bitnami/openldap:latest + + _run_until_success_or_timeout 20 sh -c "docker logs ${CONTAINER2_NAME} 2>&1 | grep 'LDAP setup finished'" # # Setup DMS container From af09db6648b2bbb13b7364f39b9cf76d482cdfb2 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 24 Aug 2023 02:56:24 +1200 Subject: [PATCH 136/592] ci: `question.yml` - Clarify that the issue tracker is not for personal support (#3498) * ci: Revise `question.yml` to better clarify the issue tracker is not for support queries Users have been making low effort reports (_bypassing the dedicated form_) through this alternative that is intended for addressing other concerns related to the project - not troubleshooting user problems. When a user does not want to put the effort in of a full bug report (_and following our debug docs tips that it refers them to_), they should be using the Github Discussions page which provides the same free-form input, but should not require attention of project devs (contributors / maintainers). --- The markdown rendered field above the "Description" input field didn't seem too relevant for this template. I've opted for a markdown comment (so it won't render if kept) into the input field with hopes that'll be more visible to the readers attention. * chore: Fix typo --- .github/ISSUE_TEMPLATE/question.yml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/question.yml b/.github/ISSUE_TEMPLATE/question.yml index 7a7435117a4..e94062e0aa0 100644 --- a/.github/ISSUE_TEMPLATE/question.yml +++ b/.github/ISSUE_TEMPLATE/question.yml @@ -1,18 +1,10 @@ name: Other -description: Miscellaneous questions and reports +description: Miscellaneous questions and reports for the project (not support) title: 'other: ' labels: - meta/help wanted body: - - type: markdown - attributes: - value: | - Markdown formatting can be used in almost all text fields. The description will tell you if this is not the case for a specific field. - - Be as precise as possible, and if in doubt, it's best to add more information that too few. - - --- - type: dropdown id: subject attributes: @@ -21,8 +13,7 @@ body: - I would like to contribute to the project - I would like to configure a not documented mail server use case - I would like some feedback concerning a use case - - I have questions about TLS/SSL/STARTTLS/OpenSSL - - Other + - Something else that requires developers attention validations: required: true - type: textarea @@ -31,3 +22,9 @@ body: label: Description validations: required: true + value: | + From c8a0bfd36195281b629916d64edf0a2c1fbd89d1 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 25 Aug 2023 00:29:08 +1200 Subject: [PATCH 137/592] ci: Fix `question.yml` template - `value` should be an attribute (#3502) The recent change to this template was invalid, as `value` should have been nested under the `attributes` object. --- .github/ISSUE_TEMPLATE/question.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/question.yml b/.github/ISSUE_TEMPLATE/question.yml index e94062e0aa0..b58d0370c34 100644 --- a/.github/ISSUE_TEMPLATE/question.yml +++ b/.github/ISSUE_TEMPLATE/question.yml @@ -18,13 +18,13 @@ body: required: true - type: textarea id: description - attributes: - label: Description validations: required: true - value: | - + attributes: + label: Description + value: | + From f89cbac21c26ac306ff649bfef96605b44dba86f Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sun, 27 Aug 2023 23:44:53 +1200 Subject: [PATCH 138/592] tests: TLS cipher suites - Update `testssl.sh` tag to `3.2` (#3504) --- test/tests/parallel/set2/tls_cipherlists.bats | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/tests/parallel/set2/tls_cipherlists.bats b/test/tests/parallel/set2/tls_cipherlists.bats index d9a82eb3cc5..2b9511b9c78 100644 --- a/test/tests/parallel/set2/tls_cipherlists.bats +++ b/test/tests/parallel/set2/tls_cipherlists.bats @@ -25,7 +25,7 @@ function setup_file() { # Pull `testssl.sh` image in advance to avoid it interfering with the `run` captured output. # Only interferes (potential test failure) with `assert_output` not `assert_success`? - docker pull drwetter/testssl.sh:3.1dev + docker pull drwetter/testssl.sh:3.2 # Only used in `_should_support_expected_cipherlists()` to set a storage location for `testssl.sh` JSON output: # `${BATS_TMPDIR}` maps to `/tmp`: https://bats-core.readthedocs.io/en/v1.8.2/writing-tests.html#special-variables @@ -166,7 +166,7 @@ function _collect_cipherlists() { --volume "${TLS_CONFIG_VOLUME}" \ --volume "${RESULTS_PATH}:/output" \ --workdir "/output" \ - drwetter/testssl.sh:3.1dev "${TESTSSL_CMD[@]}" + drwetter/testssl.sh:3.2 "${TESTSSL_CMD[@]}" assert_success } From 43a122fe18abec5d7607e9659a890780930bb1ce Mon Sep 17 00:00:00 2001 From: Casper Date: Sun, 27 Aug 2023 23:40:24 +0200 Subject: [PATCH 139/592] scripts: add wrapper to update Postfix configuration safely (follow up) (#3503) --- target/scripts/helpers/postfix.sh | 26 +++++++++++++++----------- target/scripts/helpers/utils.sh | 1 + 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/target/scripts/helpers/postfix.sh b/target/scripts/helpers/postfix.sh index b376dce9eac..5fa4fa83313 100644 --- a/target/scripts/helpers/postfix.sh +++ b/target/scripts/helpers/postfix.sh @@ -92,39 +92,43 @@ function _vhost_ldap_support() { # /etc/aliases is handled by `alias.sh` and uses `postalias` to update the Postfix alias database. No need for `postmap`. # http://www.postfix.org/postalias.1.html -# Add an key with an value to Postfix's main configuration file +# Add a key with a value to Postfix's main configuration file # or update an existing key. An already existing key can be updated # by either appending to the existing value (default) or by prepending. # # @param ${1} = key name in Postfix's main configuration file # @param ${2} = new value (appended or prepended) -# @param ${3} = "append" (default) or "prepend" [OPTIONAL] +# @param ${3} = action "append" (default) or "prepend" [OPTIONAL] function _add_to_or_update_postfix_main() { local KEY=${1:?Key name is required} local NEW_VALUE=${2:?New value is required} local ACTION=${3:-append} + local CURRENT_VALUE - # If entry does not exist, add it - otherwise update with ACTION: - if ! grep -q -E "^${KEY}" /etc/postfix/main.cf; then + # Get current value from /etc/postfix/main.cf + _adjust_mtime_for_postfix_maincf + CURRENT_VALUE=$(postconf -h "${KEY}" 2>/dev/null) + + # If key does not exist or value is empty, add it - otherwise update with ACTION: + if [[ -z ${CURRENT_VALUE} ]]; then postconf "${KEY} = ${NEW_VALUE}" else - KEY=$(_escape_for_sed "${KEY}") - NEW_VALUE=$(_escape_for_sed "${NEW_VALUE}") - local SED_STRING + # If $NEW_VALUE is already present --> nothing to do, skip. + if [[ " ${CURRENT_VALUE} " == *" ${NEW_VALUE} "* ]]; then + return 0 + fi case "${ACTION}" in ('append') - SED_STRING="/${NEW_VALUE}/! s|^(${KEY} *=.*)|\1 ${NEW_VALUE}|g" + postconf "${KEY} = ${CURRENT_VALUE} ${NEW_VALUE}" ;; ('prepend') - SED_STRING="/${NEW_VALUE}/! s|^(${KEY}) *= *(.*)|\1 = ${NEW_VALUE} \2|g" + postconf "${KEY} = ${NEW_VALUE} ${CURRENT_VALUE}" ;; (*) _log 'error' "Action '${3}' in _add_to_or_update_postfix_main is unknown" return 1 ;; esac - - sed -i -E "${SED_STRING}" /etc/postfix/main.cf fi } diff --git a/target/scripts/helpers/utils.sh b/target/scripts/helpers/utils.sh index 02d29cc9e18..c74fd7e8fd9 100644 --- a/target/scripts/helpers/utils.sh +++ b/target/scripts/helpers/utils.sh @@ -4,6 +4,7 @@ function _escape() { echo "${1//./\\.}" } +# TODO: Not in use currently. Maybe in the future: https://github.com/docker-mailserver/docker-mailserver/pull/3484/files#r1299410851 # Replaces a string so that it can be used inside # `sed` safely. # From 855d9acb5306aa79c8c5e4164a222d388f1cef05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Aug 2023 19:33:05 +0200 Subject: [PATCH 140/592] chore(deps): Bump docker/setup-buildx-action from 2.9.1 to 2.10.0 (#3511) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 7dbe05173af..a86b0ed6044 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -79,7 +79,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.9.1 + uses: docker/setup-buildx-action@v2.10.0 # NOTE: AMD64 can build within 2 minutes - name: 'Build images' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index ea837364fec..598c12a3d9c 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -40,7 +40,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.9.1 + uses: docker/setup-buildx-action@v2.10.0 # Try get the cached build layers from a prior `generic_build.yml` job. # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 7c9f6f056dc..5b460ccecdb 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -38,7 +38,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.9.1 + uses: docker/setup-buildx-action@v2.10.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 934c4d487f1..69e8cea3c21 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -37,7 +37,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.9.1 + uses: docker/setup-buildx-action@v2.10.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. From e9f04cf8a7d31efbc8ae9a4a35603409f637f3b4 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 29 Aug 2023 09:40:02 +1200 Subject: [PATCH 141/592] chore: Change `setup config dkim` default key size to `2048` (`open-dkim`) (#3508) * chore: Adjust default DKIM size (`open-dkim`) from 4096-bit to 2048-bit 4096-bit is excessive in size for DKIM key. 2048-bit is plenty. * chore: Additional revisions to `open-dkim` command help output - The examples use `keysize 2048`, but as that's the new default it makes sense to change that. - Other help text was also revised. - Last example for domains did not need to demonstrate the other options. Changed example domains to more appropriate values. * docs: Revise DKIM docs Primarily for the change in default key size, but does revise some text to better communicate to the user. - While the referenced RFC advises 512-bit to 2048-bit key size, we now explicitly discourage `512-bit` as it's not secure. `1024-bit` is still likely safe for most, but `2048-bit` is a good default for those not rotating their keys. - Adjusted the domains example to match the new `setup config dkim domain` domains example. - Tip for changing default key size changed to "info" with added clarity of lowering security or increasing it (excessively). - Rspamd section is minor formatting changes, with the exception of clarifying the "main domain" for the mail accounts is assumed as the DMS FQDN with any subdomain (like `mail.`) stripped away. This is not great, but a legacy issue that needs to be addressed in future. - `docs-rspamd-override-d` ref removed, and usage replaced with equivalent ref `docs-rspamd-config-dropin`, while `docs-rspamd-config-declarative` ref was not in use and also removed. - Revised the `.txt` DNS formatting info section to better communicate with the reader. Additionally it had mixed usage of default `mail` and custom `dkim-rsa` selectors (_file content and output_). * docs: Sync DKIM commands help messages and update DKIM docs for LDAP - Adopt the help options format style from the `rspamd-dkim` into `open-dkim` command. And convert `./setup.sh` to `setup`. `selector` option has been implemented. for a while now. - Update `rspamd-dkim` examples help output to align with `open-dkim` command examples. - Give both DKIM command tools a consistent description. The two tools differ in support for the `domain` option (_implicit domain sourcing for default account provisioner, and support for multiple domains as input_). - DKIM docs for LDAP domain support revised to better communicate when explicit domain config is necessary. * tests: Adjust test-cases for `setup config dkim` change `rspamd_dkim.bats`: - Update assert for command help output. - Don't bother creating a DKIM key at 512-bit size. `setup_cli.bats`: - Update assert for command help output of the `setup config dkim` (OpenDKIM) command. * docs: Update DKIM section for large keys to newer RFC The linked discussion from 2021 does mention this updated RFC over the original. That removes outdated advice about `512-bit` key length support. The discussion link is still kept to reference a comment for the reader to better understand the security strength of 2048-bit RSA keys and why larger keys are not worthwhile, especially for DKIM. * docs: Extract out common DKIM generation command from content tabs Should be fine to be DRY here, not specific to `open-dkim` or `rspamd` generation/support. Previously rspamd lacked support of an equivalent command in DMS. * docs: DKIM refactoring - Shifted out the info admonition on key size advice out of the content tabs as it's now generic information. - Indented the 4096-bit warning into this, which is less of a concern as the default for our DKIM generation tools is consistently 2048-bit now. - Reworked the LDAP and Rspamd multi-domain advice. To avoid causing a bad diff, these sections haven't been moved/merged yet. * docs: Revise DKIM docs Advice for managing domains individually with LDAP and Rspamd extracted out of the content tabs. Default domain behaviour explained with extra info about OpenDKIM + FILE provisioner sourcing extra domains implicitly. --- .../config/best-practices/dkim_dmarc_spf.md | 115 ++++++++++-------- target/bin/open-dkim | 45 +++---- target/bin/rspamd-dkim | 31 +++-- .../parallel/set1/spam_virus/rspamd_dkim.bats | 4 +- .../parallel/set3/scripts/setup_cli.bats | 2 +- test/tests/serial/open_dkim.bats | 8 +- 6 files changed, 109 insertions(+), 96 deletions(-) diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md index 8ab00f48f69..eda986008bb 100644 --- a/docs/content/config/best-practices/dkim_dmarc_spf.md +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -35,83 +35,95 @@ DKIM requires a public/private key pair to enable **signing (_via private key_)* ### Generating Keys +You'll need to repeat this process if you add any new domains. + You should have: - At least one [email account setup][docs-accounts-add] - Attached a [volume for config][docs-volumes-config] to persist the generated files to local storage -!!! warning "RSA Key Sizes >= 4096 Bit" +!!! example "Creating DKIM Keys" - Keys of 4096 bits could be denied by some mail servers. According to [RFC 6376][rfc-6376], keys are [preferably between 512 and 2048 bits][github-issue-dkimlength]. + DKIM keys can be generated with good defaults by running: -DKIM is currently supported by either OpenDKIM or Rspamd: + ```bash + docker exec -it setup config dkim + ``` -=== "OpenDKIM" + If you need to generate your keys with different settings, check the `help` output for supported config options and examples: - OpenDKIM is currently [enabled by default][docs-env-opendkim]. + ```bash + docker exec -it setup config dkim help + ``` - The command `docker exec setup config dkim help` details supported config options, along with some examples. + As described by the help output, you may need to use the `domain` option explicitly when you're using LDAP or Rspamd. - !!! example "Creating a DKIM key" +??? info "Changing the key size" - Generate the DKIM files with: + The keypair generated for using with DKIM presently defaults to RSA-2048. This is a good size but you can lower the security to `1024-bit`, or increase it to `4096-bit` (_discouraged as that is excessive_). + + To generate a key with different size (_for RSA 1024-bit_) run: - ```sh - docker exec -ti setup config dkim - ``` + ```sh + setup config dkim keysize 1024 + ``` - Your new DKIM key(s) and OpenDKIM config files have been added to `/tmp/docker-mailserver/opendkim/`. + !!! warning "RSA Key Sizes >= 4096 Bit" - ??? note "LDAP accounts need to specify domains explicitly" + According to [RFC 8301][rfc-8301], keys are preferably between 1024 and 2048 bits. Keys of size 4096-bit or larger may not be compatible to all systems your mail is intended for. - The command is unable to infer the domains from LDAP user accounts, you must specify them: + You [should not need a key length beyond 2048-bit][github-issue-dkimlength]. If 2048-bit does not meet your security needs, you may want to instead consider adopting key rotation or switching from RSA to ECC keys for DKIM. - ```sh - setup config dkim domain 'example.com,example.io' - ``` +??? note "You may need to specify mail domains explicitly" - ??? tip "Changing the key size" + Required when using LDAP and Rspamd. - The private key presently defaults to RSA-4096. To create an RSA 2048-bit key run: + `setup config dkim` will generate DKIM keys for what is assumed as the primary mail domain (_derived from the FQDN assigned to DMS, minus any subdomain_). - ```sh - setup config dkim keysize 2048 - ``` + When the DMS FQDN is `mail.example.com` or `example.com`, by default this command will generate DKIM keys for `example.com` as the primary domain for your users mail accounts (eg: `hello@example.com`). - !!! info "Restart required" + The DKIM generation does not have support to query LDAP for additionanl mail domains it should know about. If the primary mail domain is not sufficient, then you must explicitly specify any extra domains via the `domain` option: - After restarting DMS, outgoing mail will now be signed with your new DKIM key(s) :tada: + ```sh + # ENABLE_OPENDKIM=1 (default): + setup config dkim domain 'example.com,another-example.com' - You'll need to repeat this process if you add any new domains. + # ENABLE_RSPAMD=1 + ENABLE_OPENDKIM=0: + setup config dkim domain example.com + setup config dkim domain another-example.com + ``` -=== "Rspamd" + !!! info "OpenDKIM with `ACCOUNT_PROVISIONER=FILE`" - Opt-in via [`ENABLE_RSPAMD=1`][docs-env-rspamd] (_and disable the default OpenDKIM: `ENABLE_OPENDKIM=0`_). + When DMS uses this configuration, it will by default also detect mail domains (_from accounts added via `setup email add`_), generating additional DKIM keys. - Rspamd provides DKIM support through two separate modules: +DKIM is currently supported by either OpenDKIM or Rspamd: - 1. [Verifying DKIM signatures from inbound mail][rspamd-docs-dkim-checks] is enabled by default. - 2. [Signing outbound mail with your DKIM key][rspamd-docs-dkim-signing] needs additional setup (key + dns + config). +=== "OpenDKIM" - !!! example "Creating DKIM Keys" + OpenDKIM is currently [enabled by default][docs-env-opendkim]. - You can simply run + After running `setup config dkim`, your new DKIM key files (_and OpenDKIM config_) have been added to `/tmp/docker-mailserver/opendkim/`. - ```bash - docker exec -ti setup config dkim help - ``` + !!! info "Restart required" - which provides you with an overview of what the script can do. Just running + After restarting DMS, outgoing mail will now be signed with your new DKIM key(s) :tada: - ```bash - docker exec -ti setup config dkim - ``` +=== "Rspamd" - will execute the helper script with default parameters. + Requires opt-in via [`ENABLE_RSPAMD=1`][docs-env-rspamd] (_and disable the default OpenDKIM: `ENABLE_OPENDKIM=0`_). + + Rspamd provides DKIM support through two separate modules: + + 1. [Verifying DKIM signatures from inbound mail][rspamd-docs-dkim-checks] is enabled by default. + 2. [Signing outbound mail with your DKIM key][rspamd-docs-dkim-signing] needs additional setup (key + dns + config). ??? warning "Using Multiple Domains" - Unlike the current script for OpenDKIM, the Rspamd script will **not** create keys for all domains DMS is managing, but only for the one it assumes to be the main domain (derived from DMS' domain name). Moreover, the default `dkim_signing.conf` configuration file that DMS ships will also only contain one domain. If you have multiple domains, you need to run the command `docker exec -ti setup config dkim domain ` multiple times to create all the keys for all domains, and then provide a custom `dkim_signing.conf` (for which an example is shown below). + If you have multiple domains, you need to: + + - Create a key wth `docker exec -it setup config dkim domain ` for each domain DMS should sign outgoing mail for. + - Provide a custom `dkim_signing.conf` (for which an example is shown below), as the default config only supports one domain. !!! info "About the Helper Script" @@ -121,7 +133,9 @@ DKIM is currently supported by either OpenDKIM or Rspamd: --- - In case you have not already provided a default DKIM signing configuration, the script will create one and write it to `/etc/rspamd/override.d/dkim_signing.conf`. If this file already exist, it will not be overwritten. When you're already using [the `rspamd/override.d/` directory][docs-rspamd-override-d], the file is created inside your volume and therefore persisted correctly. If you are not using `rspamd/override.d/`, you will need to persist the file yourself (otherwise it is lost on container restart). + In case you have not already provided a default DKIM signing configuration, the script will create one and write it to `/etc/rspamd/override.d/dkim_signing.conf`. If this file already exists, it will not be overwritten. + + When you're already using [the `rspamd/override.d/` directory][docs-rspamd-config-dropin], the file is created inside your volume and therefore persisted correctly. If you are not using `rspamd/override.d/`, you will need to persist the file yourself (otherwise it is lost on container restart). An example of what a default configuration file for DKIM signing looks like can be found by expanding the example below. @@ -190,8 +204,6 @@ DKIM is currently supported by either OpenDKIM or Rspamd: If there is a mismatch, a warning will be emitted to the Rspamd log `/var/log/supervisor/rspamd.log`. - [docs-rspamd-override-d]: ../security/rspamd.md#manually - ### DNS Record { #dkim-dns } When mail signed with your DKIM key is sent from your mail server, the receiver needs to check a DNS `TXT` record to verify the DKIM signature is trustworthy. @@ -221,11 +233,13 @@ When mail signed with your DKIM key is sent from your mail server, the receiver ??? info "`.txt` - Formatting the `TXT` record value correctly" - This file was generated for use within a [DNS zone file][dns::wikipedia-zonefile]. DNS `TXT` records values that are longer than 255 characters need to be split into multiple parts. This is why the public key has multiple parts wrapped within double-quotes between `(` and `)`. + This file was generated for use within a [DNS zone file][dns::wikipedia-zonefile]. The file name uses the DKIM selector it was generated with (default DKIM selector is `mail`, which creates `mail.txt`_). + + For your DNS setup, DKIM support needs to create a `TXT` record to store the public key for mail clients to use. `TXT` records with values that are longer than 255 characters need to be split into multiple parts. This is why the generated `.txt` file (_containing your public key for use with DKIM_) has multiple value parts wrapped within double-quotes between `(` and `)`. - A DNS web-interface may handle this internally instead, while [others may not, but expect the input as a single line][dns::webui-dkim]_). You'll need to manually format the value as described below. + A DNS web-interface may handle this separation internally instead, and [could expect the value provided all as a single line][dns::webui-dkim] instead of split. When that is required, you'll need to manually format the value as described below. - Your DNS record file (eg: `mail.txt`) should look similar to this: + Your generated DNS record file (`.txt`) should look similar to this: ```txt mail._domainkey IN TXT ( "v=DKIM1; k=rsa; " @@ -243,7 +257,7 @@ When mail signed with your DKIM key is sent from your mail server, the receiver To test that your new DKIM record is correct, query it with the `dig` command. The `TXT` value response should be a single line split into multiple parts wrapped in double-quotes: ```console - $ dig +short TXT dkim-rsa._domainkey.example.com + $ dig +short TXT mail._domainkey.example.com "v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqQMMqhb1S52Rg7VFS3EC6JQIMxNDdiBmOKZvY5fiVtD3Z+yd9ZV+V8e4IARVoMXWcJWSR6xkloitzfrRtJRwOYvmrcgugOalkmM0V4Gy/2aXeamuiBuUc4esDQEI3egmtAsHcVY1XCoYfs+9VqoHEq3vdr3UQ8zP/l+FP5UfcaJFCK/ZllqcO2P1GjIDVSHLdPpRHbMP/tU1a9mNZ5QMZBJ/JuJK/s+2bp8gpxKn8rh1akSQjlynlV9NI+7J3CC7CUf3bGvoXIrb37C/lpJehS39" "KNtcGdaRufKauSfqx/7SxA0zyZC+r13f7ASbMaQFzm+/RRusTqozY/p/MsWx8QIDAQAB" ``` @@ -328,10 +342,9 @@ volumes: [docs-env-opendkim]: ../environment.md#enable_opendkim [docs-env-rspamd]: ../environment.md#enable_rspamd [docs-rspamd-config-dropin]: ../security/rspamd.md#manually -[docs-rspamd-config-declarative]: ../security/rspamd.md#with-the-help-of-a-custom-file [cloudflare-dkim-dmarc-spf]: https://www.cloudflare.com/learning/email-security/dmarc-dkim-spf/ -[rfc-6376]: https://tools.ietf.org/html/rfc6376 -[github-issue-dkimlength]: https://github.com/docker-mailserver/docker-mailserver/issues/1854 +[rfc-8301]: https://datatracker.ietf.org/doc/html/rfc8301#section-3.2 +[github-issue-dkimlength]: https://github.com/docker-mailserver/docker-mailserver/issues/1854#issuecomment-806280929 [rspamd-docs-dkim-checks]: https://www.rspamd.com/doc/modules/dkim.html [rspamd-docs-dkim-signing]: https://www.rspamd.com/doc/modules/dkim_signing.html [dns::example-webui]: https://www.vultr.com/docs/introduction-to-vultr-dns/ diff --git a/target/bin/open-dkim b/target/bin/open-dkim index 50b18f6c96a..0036ff8daa3 100755 --- a/target/bin/open-dkim +++ b/target/bin/open-dkim @@ -8,7 +8,7 @@ if [[ -f /etc/dms-settings ]] && [[ $(_get_dms_env_value 'ENABLE_RSPAMD') -eq 1 exit fi -KEYSIZE=4096 +KEYSIZE=2048 SELECTOR=mail DOMAINS= @@ -16,37 +16,40 @@ function __usage() { printf '%s' "${PURPLE}OPEN-DKIM${RED}(${YELLOW}8${RED}) ${ORANGE}NAME${RESET} - open-dkim - configure DomainKeys Identified Mail (DKIM) + open-dkim - Configure DKIM (DomainKeys Identified Mail) ${ORANGE}SYNOPSIS${RESET} - ./setup.sh config dkim [ OPTIONS${RED}...${RESET} ] + setup config dkim [ OPTIONS${RED}...${RESET} ] ${ORANGE}DESCRIPTION${RESET} - Configures DKIM keys. OPTIONS can be used to configure a more complex setup. - LDAP setups require these options. + Creates DKIM keys and configures them within DMS for OpenDKIM. + OPTIONS can be used when your requirements are not met by the defaults. + When not using 'ACCOUNT_PROVISIONER=FILE' (default), you may need to explicitly + use the 'domain' option to generate DKIM keys for your mail account domains. ${ORANGE}OPTIONS${RESET} ${BLUE}Generic Program Information${RESET} - help Print the usage information. + help Print the usage information. ${BLUE}Configuration adjustments${RESET} - keysize Set the size of the keys to be generated. Possible are 1024, 2048 and 4096 (default). - selector Set a manual selector (default is 'mail') for the key. (${LCYAN}ATTENTION${RESET}: NOT IMPLEMENTED YET!) - domain Provide the domain(s) for which keys are to be generated. + keysize Set the size of the keys to be generated. + Possible values: 1024, 2048 and 4096 + Default: 2048 + selector Set a manual selector for the key. + Default: mail + domain Provide the domain(s) for which to generate keys for. + Default: The FQDN assigned to DMS, excluding any subdomain. + 'ACCOUNT_PROVISIONER=FILE' also sources domains from mail accounts. ${ORANGE}EXAMPLES${RESET} - ${LWHITE}./setup.sh config dkim keysize 2048${RESET} - Creates keys of length 2048 bit in a default setup where domains are obtained from - your accounts. - - ${LWHITE}./setup.sh config dkim keysize 2048 selector 2021-dkim${RESET} - Creates keys of length 2048 bit in a default setup where domains are obtained from - your accounts. The DKIM selector used is '2021-dkim'. - - ${LWHITE}./setup.sh config dkim keysize 2048 selector 2021-dkim domain 'whoami.com,whoareyou.org'${RESET} - Appropriate for an LDAP setup. Creates keys of length 2048 bit in a default setup - where domains are obtained from your accounts. The DKIM selector used is '2021-dkim'. - The domains for which DKIM keys are generated are 'whoami.com' and 'whoareyou.org'. + ${LWHITE}setup config dkim keysize 4096${RESET} + Creates keys with their length increased to a size of 4096-bit. + + ${LWHITE}setup config dkim keysize 1024 selector 2023-dkim${RESET} + Creates 1024-bit sized keys, and changes the DKIM selector to '2023-dkim'. + + ${LWHITE}setup config dkim domain 'example.com,another-example.com'${RESET} + Only generates DKIM keys for the specified domains: 'example.com' and 'another-example.com'. ${ORANGE}EXIT STATUS${RESET} Exit status is 0 if command was successful. If wrong arguments are provided or arguments contain diff --git a/target/bin/rspamd-dkim b/target/bin/rspamd-dkim index 7627a863625..de7129de2f2 100755 --- a/target/bin/rspamd-dkim +++ b/target/bin/rspamd-dkim @@ -16,14 +16,14 @@ function __usage() { echo -e "${PURPLE}RSPAMD-DKIM${RED}(${YELLOW}8${RED}) ${ORANGE}NAME${RESET} - rspamd-dkim - Configure DomainKeys Identified Mail (DKIM) via Rspamd + rspamd-dkim - Configure DKIM (DomainKeys Identified Mail) ${ORANGE}SYNOPSIS${RESET} setup config dkim [ OPTIONS${RED}...${RESET} ] ${ORANGE}DESCRIPTION${RESET} - This script aids in creating DKIM signing keys. The keys are created and managed by Rspamd. - OPTIONS can be used to configure a more complex setup. + Creates DKIM keys and configures them within DMS for Rspamd. + OPTIONS can be used when your requirements are not met by the defaults. ${ORANGE}OPTIONS${RESET} ${BLUE}Generic Program Information${RESET} @@ -32,30 +32,27 @@ ${ORANGE}OPTIONS${RESET} help Print the usage information. ${BLUE}Configuration adjustments${RESET} - keytype Set the type of key you want to use + keytype Set the type of key you want to use. Possible values: rsa, ed25519 Default: rsa - keysize Set the size of the keys to be generated + keysize Set the size of the keys to be generated. Possible values: 1024, 2048 and 4096 Default: 2048 Only applies when using keytype=rsa - selector Set a manual selector for the key + selector Set a manual selector for the key. Default: mail - domain Provide the domain for which keys are to be generated - Default: primary domain name of DMS + domain Provide the domain for which to generate keys for. + Default: The FQDN assigned to DMS, excluding any subdomain. ${ORANGE}EXAMPLES${RESET} - ${LWHITE}setup config dkim keysize 2048${RESET} - Creates keys of length 2048 bit in a default setup where domains are obtained from - your accounts. + ${LWHITE}setup config dkim keysize 4096${RESET} + Creates keys with their length increased to a size of 4096-bit. - ${LWHITE}setup config dkim keysize 512 selector 2023-dkim${RESET} - Creates keys of length 512 bit in a default setup where domains are obtained from - your accounts. The DKIM selector used is '2023-dkim'. + ${LWHITE}setup config dkim keysize 1024 selector 2023-dkim${RESET} + Creates 1024-bit sized keys, and changes the DKIM selector to '2023-dkim'. - ${LWHITE}setup config dkim keysize 1024 selector 2023-dkim domain whoami.com${RESET} - Creates keys of length 1024 bit in a default setup where domains are obtained from your accounts. - The DKIM selector used is '2023-dkim'. The domain for which DKIM keys are generated is whoami.com. + ${LWHITE}setup config dkim domain example.com${RESET} + Generate the DKIM key for a different domain (example.com). ${ORANGE}EXIT STATUS${RESET} Exit status is 0 if command was successful. If wrong arguments are provided or arguments contain diff --git a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats index 97c98b6aab8..cd880e52575 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats @@ -49,7 +49,7 @@ function teardown_file() { _default_teardown ; } _run_in_container setup config dkim help __log_is_free_of_warnings_and_errors assert_output --partial 'Showing usage message now' - assert_output --partial 'rspamd-dkim - Configure DomainKeys Identified Mail (DKIM) via Rspamd' + assert_output --partial 'rspamd-dkim - Configure DKIM (DomainKeys Identified Mail)' } @test 'default signing config is created if it does not exist and not overwritten' { @@ -143,7 +143,7 @@ function teardown_file() { _default_teardown ; } } @test "argument 'keysize' is applied correctly for RSA keys" { - for KEYSIZE in 512 1024 2048 4096; do + for KEYSIZE in 1024 2048 4096; do __create_key 'rsa' 'mail' "${DOMAIN_NAME}" "${KEYSIZE}" assert_success __log_is_free_of_warnings_and_errors diff --git a/test/tests/parallel/set3/scripts/setup_cli.bats b/test/tests/parallel/set3/scripts/setup_cli.bats index c5d2a2d62b7..dca613586a7 100644 --- a/test/tests/parallel/set3/scripts/setup_cli.bats +++ b/test/tests/parallel/set3/scripts/setup_cli.bats @@ -237,7 +237,7 @@ function teardown_file() { _default_teardown ; } @test "config dkim (help correctly displayed)" { run ./setup.sh -c "${CONTAINER_NAME}" config dkim help assert_success - assert_line --index 3 --partial " open-dkim - configure DomainKeys Identified Mail (DKIM)" + assert_line --index 3 --partial "open-dkim - Configure DKIM (DomainKeys Identified Mail)" } # debug diff --git a/test/tests/serial/open_dkim.bats b/test/tests/serial/open_dkim.bats index a71cb3e7025..b4950d1f1c7 100644 --- a/test/tests/serial/open_dkim.bats +++ b/test/tests/serial/open_dkim.bats @@ -113,14 +113,14 @@ function teardown() { _default_teardown ; } __init_container_without_waiting '/tmp/docker-mailserver' # generate first key (with a custom selector) - __should_generate_dkim_key 4 '2048' 'domain1.tld' 'mailer' + __should_generate_dkim_key 4 '1024' 'domain1.tld' 'mailer' __assert_outputs_common_dkim_logs # generate two additional keys different to the previous one - __should_generate_dkim_key 2 '2048' 'domain2.tld,domain3.tld' + __should_generate_dkim_key 2 '1024' 'domain2.tld,domain3.tld' __assert_logged_dkim_creation 'domain2.tld' __assert_logged_dkim_creation 'domain3.tld' # generate an additional key whilst providing already existing domains - __should_generate_dkim_key 1 '2048' 'domain3.tld,domain4.tld' + __should_generate_dkim_key 1 '1024' 'domain3.tld,domain4.tld' __assert_logged_dkim_creation 'domain4.tld' __should_have_tables_trustedhosts_for_domain @@ -197,7 +197,7 @@ function __should_support_creating_key_of_size() { __assert_logged_dkim_creation 'localhost.localdomain' __assert_logged_dkim_creation 'otherdomain.tld' - __should_have_expected_files "${EXPECTED_KEYSIZE:-4096}" + __should_have_expected_files "${EXPECTED_KEYSIZE:-2048}" _run_in_container rm -r /tmp/docker-mailserver/opendkim } From 351ef2afa171070c97ac3a0281acfebf987d7304 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 29 Aug 2023 10:16:08 +1200 Subject: [PATCH 142/592] tests: LDAP - Improvements to LDIF (#3506) - The `uniqueIdentifier` attribute is not appropriate and was relying on `objectClass: extensibleObject` as a workaround to allow it. A more appropriate attribute to use instead is `userID` (_short name: `uid`_). - Removing `extensibleObject` now requires switching the user accounts to use `inetOrgPerson` class (_which inherits from `organizationalPerson`_). which allows the attributes `givenName`, `userID` and `mail` (_also provided via the `PostfixBookMailAccount` class_). - The LDAP root object now uses `dc` attributes for `example.test` instead of `localhost.localdomain`. This has nothing to do with DMS or LDAP containers networking config, nor the users mail addresses. - Users are now grouped under the organizational unit of `users` instead of `people`. Purely a naming change out of preference, no functional difference. The LDAP test ENV has been updated to accommodate the above changes. An additional ENV override was required for SASLAuthd to switch an attribute set for `ldap_filter` in `/etc/saslauthd.conf` from the implicit default of `uniqueIdentifier` (_that we set during startup as an ENV default for fallback_) to the `userID` attribute. --- .../bootstrap/ldif/01_mail-tree.ldif | 15 +++++++---- .../bootstrap/ldif/02_user-email.ldif | 7 +++-- .../03_user-email-other-primary-domain.ldif | 7 +++-- .../ldif/04_user-email-different-uid.ldif | 7 +++-- test/tests/serial/mail_with_ldap.bats | 27 ++++++++++--------- 5 files changed, 34 insertions(+), 29 deletions(-) diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif index 5e3d8b0a8e1..a6587f89b5c 100644 --- a/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif +++ b/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif @@ -1,11 +1,16 @@ -# The root object, all entries will branch off this one: -dn: dc=localhost,dc=localdomain +# The root object of the tree, all entries will branch off this one: +dn: dc=example,dc=test +# DN is formed from `example.test` DNS labels: +# NOTE: This is just a common convention (not dependent on hostname or any external config) objectClass: dcObject +# Must reference left most component: +dc: example +# It's required to use an `objectClass` that implements a "Structural Class": objectClass: organization -dc: localhost +# Value is purely descriptive, not important to tests: o: DMS Test # User accounts will belong to this subtree: -dn: ou=people,dc=localhost,dc=localdomain +dn: ou=users,dc=example,dc=test objectClass: organizationalUnit -ou: people +ou: users diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif index cd656bf82d8..67696a81119 100644 --- a/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif +++ b/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif @@ -1,12 +1,11 @@ # NOTE: A standard user account to test against -dn: uniqueIdentifier=some.user,ou=people,dc=localhost,dc=localdomain -objectClass: organizationalPerson +dn: uid=some.user,ou=users,dc=example,dc=test +objectClass: inetOrgPerson objectClass: PostfixBookMailAccount -objectClass: extensibleObject cn: Some User givenName: Some surname: User -uniqueIdentifier: some.user +userID: some.user # Password is: secret userPassword: {SSHA}eLtqGpid+hkSVhxvsdTPztv4uapRofGx mail: some.user@localhost.localdomain diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif index d27cec7e9d2..66a84343035 100644 --- a/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif +++ b/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif @@ -1,13 +1,12 @@ # NOTE: This user differs via the domain-part of their mail address # They also have their mail directory attributes using the primary domain, not their domain-part -dn: uniqueIdentifier=some.other.user,ou=people,dc=localhost,dc=localdomain -objectClass: organizationalPerson +dn: uid=some.other.user,ou=users,dc=example,dc=test +objectClass: inetOrgPerson objectClass: PostfixBookMailAccount -objectClass: extensibleObject cn: Some Other User givenName: Some surname: Other User -uniqueIdentifier: some.other.user +userID: some.other.user # Password is: secret userPassword: {SSHA}eLtqGpid+hkSVhxvsdTPztv4uapRofGx mail: some.other.user@localhost.otherdomain diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif b/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif index be49fd9f7ea..e405c7b202e 100644 --- a/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif +++ b/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif @@ -1,13 +1,12 @@ # NOTE: This user differs by local-part of mail address not matching their uniqueIdentifier attribute # They also do not have any alias or groups configured -dn: uniqueIdentifier=some.user.id,ou=people,dc=localhost,dc=localdomain -objectClass: organizationalPerson +dn: uid=some.user.id,ou=users,dc=example,dc=test +objectClass: inetOrgPerson objectClass: PostfixBookMailAccount -objectClass: extensibleObject cn: Some User givenName: Some surname: User -uniqueIdentifier: some.user.id +userID: some.user.id # Password is: secret userPassword: {SSHA}eLtqGpid+hkSVhxvsdTPztv4uapRofGx mail: some.user.email@localhost.localdomain diff --git a/test/tests/serial/mail_with_ldap.bats b/test/tests/serial/mail_with_ldap.bats index baeb61e2f44..03a9052b534 100644 --- a/test/tests/serial/mail_with_ldap.bats +++ b/test/tests/serial/mail_with_ldap.bats @@ -21,7 +21,7 @@ function setup_file() { # Setup local openldap service: docker run --rm -d --name "${CONTAINER2_NAME}" \ --env LDAP_ADMIN_PASSWORD=admin \ - --env LDAP_ROOT='dc=localhost,dc=localdomain' \ + --env LDAP_ROOT='dc=example,dc=test' \ --env LDAP_PORT_NUMBER=389 \ --env LDAP_SKIP_DEFAULT_TREE=yes \ --volume './test/config/ldap/docker-openldap/bootstrap/ldif/:/ldifs/:ro' \ @@ -37,26 +37,29 @@ function setup_file() { # local ENV_LDAP_CONFIG=( - # Configure for LDAP account provisioner and alternative to Dovecot SASL: --env ACCOUNT_PROVISIONER=LDAP + + # Postfix SASL auth provider (SASLAuthd instead of default Dovecot provider): --env ENABLE_SASLAUTHD=1 --env SASLAUTHD_MECHANISMS=ldap + --env SASLAUTHD_LDAP_FILTER='(&(userID=%U)(mailEnabled=TRUE))' # ENV to configure LDAP configs for Dovecot + Postfix: # NOTE: `scripts/startup/setup.d/ldap.sh:_setup_ldap()` uses `_replace_by_env_in_file()` to configure settings (stripping `DOVECOT_` / `LDAP_` prefixes): # Dovecot: - --env DOVECOT_PASS_FILTER='(&(objectClass=PostfixBookMailAccount)(uniqueIdentifier=%n))' + --env DOVECOT_PASS_FILTER='(&(objectClass=PostfixBookMailAccount)(userID=%n))' --env DOVECOT_TLS=no - --env DOVECOT_USER_FILTER='(&(objectClass=PostfixBookMailAccount)(uniqueIdentifier=%n))' + --env DOVECOT_USER_FILTER='(&(objectClass=PostfixBookMailAccount)(userID=%n))' + # Postfix: - --env LDAP_BIND_DN='cn=admin,dc=localhost,dc=localdomain' + --env LDAP_BIND_DN='cn=admin,dc=example,dc=test' --env LDAP_BIND_PW='admin' --env LDAP_QUERY_FILTER_ALIAS='(|(&(mailAlias=%s)(objectClass=PostfixBookMailForward))(&(mailAlias=%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)))' --env LDAP_QUERY_FILTER_DOMAIN='(|(&(mail=*@%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE))(&(mailGroupMember=*@%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE))(&(mailalias=*@%s)(objectClass=PostfixBookMailForward)))' --env LDAP_QUERY_FILTER_GROUP='(&(mailGroupMember=%s)(mailEnabled=TRUE))' - --env LDAP_QUERY_FILTER_SENDERS='(|(&(mail=%s)(mailEnabled=TRUE))(&(mailGroupMember=%s)(mailEnabled=TRUE))(|(&(mailAlias=%s)(objectClass=PostfixBookMailForward))(&(mailAlias=%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)))(uniqueIdentifier=some.user.id))' + --env LDAP_QUERY_FILTER_SENDERS='(|(&(mail=%s)(mailEnabled=TRUE))(&(mailGroupMember=%s)(mailEnabled=TRUE))(|(&(mailAlias=%s)(objectClass=PostfixBookMailForward))(&(mailAlias=%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)))(userID=some.user.id))' --env LDAP_QUERY_FILTER_USER='(&(mail=%s)(mailEnabled=TRUE))' - --env LDAP_SEARCH_BASE='ou=people,dc=localhost,dc=localdomain' + --env LDAP_SEARCH_BASE='ou=users,dc=example,dc=test' --env LDAP_SERVER_HOST="${FQDN_LDAP}" --env LDAP_START_TLS=no ) @@ -108,7 +111,7 @@ function teardown_file() { # Test email receiving from a other domain then the primary domain of the mailserver _should_exist_in_ldap_tables "some.other.user@${FQDN_LOCALHOST_B}" - # Should not require `uniqueIdentifier` to match the local part of `mail` (`.ldif` defined settings): + # Should not require `userID` / `uid` to match the local part of `mail` (`.ldif` defined settings): # REF: https://github.com/docker-mailserver/docker-mailserver/pull/642#issuecomment-313916384 # NOTE: This account has no `mailAlias` or `mailGroupMember` defined in it's `.ldif`. local MAIL_ACCOUNT="some.user.email@${FQDN_LOCALHOST_A}" @@ -137,8 +140,8 @@ function teardown_file() { local LDAP_SETTINGS_POSTFIX=( "server_host = ${FQDN_LDAP}" 'start_tls = no' - 'search_base = ou=people,dc=localhost,dc=localdomain' - 'bind_dn = cn=admin,dc=localhost,dc=localdomain' + 'search_base = ou=users,dc=example,dc=test' + 'bind_dn = cn=admin,dc=example,dc=test' ) for LDAP_SETTING in "${LDAP_SETTINGS_POSTFIX[@]}"; do @@ -177,8 +180,8 @@ function teardown_file() { local LDAP_SETTINGS_DOVECOT=( "uris = ldap://${FQDN_LDAP}" 'tls = no' - 'base = ou=people,dc=localhost,dc=localdomain' - 'dn = cn=admin,dc=localhost,dc=localdomain' + 'base = ou=users,dc=example,dc=test' + 'dn = cn=admin,dc=example,dc=test' ) for LDAP_SETTING in "${LDAP_SETTINGS_DOVECOT[@]}"; do From 9446fa9b9ad135001daae153ec95e8afcb1a2ca5 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 29 Aug 2023 10:19:03 +1200 Subject: [PATCH 143/592] chore: Adapt `ENABLE_LDAP=1` to `ACCOUNT_PROVISIONER=LDAP` (#3507) - Deprecation startup script check is kept for `ENABLE_LDAP=1` but adjusted to emit an error instead. It can be dropped in a future release. Just a precaution for those who mistakenly update (_possibly via automation_) without checking the release notes, an error log is somewhat helpful, although it could alternatively panic? - Docs updated to remove the `ENABLE_LDAP=1` usage - ENV docs updated to reference a maintained LDAP image. - Changelog includes the breaking change, and slight revision to prior release mention of deprecation. --- CHANGELOG.md | 14 +++++++++----- docs/content/config/advanced/auth-ldap.md | 3 --- docs/content/config/environment.md | 4 +--- ...ard-only-mailserver-with-ldap-authentication.md | 1 - mailserver.env | 7 +------ target/scripts/startup/variables-stack.sh | 3 +-- 6 files changed, 12 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 028bffe403f..8b22fec56b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Breaking + +- The environment variable `ENABLE_LDAP=1` has been changed to `ACCOUNT_PROVISIONER=LDAP`. + ### Added - New environment variable `MARK_SPAM_AS_READ`. When set to `1`, marks incoming junk as "read" to avoid unwanted notification of junk as new mail ([#3489](https://github.com/docker-mailserver/docker-mailserver/pull/3489)) @@ -210,7 +214,11 @@ Notable changes are: ### Summary -This release features a lot of small and medium-sized changes, many related to how the image is build and tested during CI. The build now requires Docker Buildkit as the ClamAV Signatures are added via `COPY --link ...` during build-time. Moreover, the build is now multi-stage. `ENABLE_LDAP` is now deprecated. +This release features a lot of small and medium-sized changes, many related to how the image is build and tested during CI. The build now multi-stage based and requires Docker Buildkit, as the ClamAV Signatures are added via `COPY --link ...` during build-time. + +### Deprecated + +- The environment variable `ENABLE_LDAP` is deprecated and will be removed in [13.0.0]. Use `ACCOUNT_PROVISIONER=LDAP` now. ### Added @@ -237,10 +245,6 @@ This release features a lot of small and medium-sized changes, many related to h - **build**: adjust build arguments - **build**: enhance build process -### Deprecated - -- The environment variable `ENABLE_LDAP` is deprecated and will be removed in [13.0.0]. Use `ACCOUNT_PROVISIONER=LDAP` now. - ### Removed - **configuration**: remove unnecessary configuration files diff --git a/docs/content/config/advanced/auth-ldap.md b/docs/content/config/advanced/auth-ldap.md index 44d3aec98fe..c275fd6ce22 100644 --- a/docs/content/config/advanced/auth-ldap.md +++ b/docs/content/config/advanced/auth-ldap.md @@ -34,7 +34,6 @@ Those variables contain the LDAP lookup filters for postfix, using `%s` as the p A really simple `LDAP_QUERY_FILTER` configuration, using only the _user filter_ and allowing only `admin@*` to spoof any sender addresses. ```yaml - - ENABLE_LDAP=1 # with the :edge tag, use ACCOUNT_PROVISIONER - LDAP_START_TLS=yes - ACCOUNT_PROVISIONER=LDAP - LDAP_SERVER_HOST=ldap.example.org @@ -215,7 +214,6 @@ The changes on the configurations necessary to work with Active Directory (**onl - ENABLE_POSTGREY=1 # >>> Postfix LDAP Integration - - ENABLE_LDAP=1 # with the :edge tag, use ACCOUNT_PROVISIONER - ACCOUNT_PROVISIONER=LDAP - LDAP_SERVER_HOST=ldap.example.org - LDAP_BIND_DN=cn=admin,ou=users,dc=example,dc=org @@ -284,7 +282,6 @@ The changes on the configurations necessary to work with Active Directory (**onl # <<< SASL Authentication # >>> Postfix Ldap Integration - - ENABLE_LDAP=1 # with the :edge tag, use ACCOUNT_PROVISIONER - ACCOUNT_PROVISIONER=LDAP - LDAP_SERVER_HOST= - LDAP_SEARCH_BASE=dc=mydomain,dc=loc diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index bea02ecb358..3e40bd89753 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -49,7 +49,7 @@ User provisioning via OIDC is planned for the future, see [this tracking issue]( - OIDC => use OIDC authentication (**not yet implemented**) - FILE => use local files (this is used as the default) -A second container for the ldap service is necessary (e.g. [docker-openldap](https://github.com/osixia/docker-openldap)) +A second container for the ldap service is necessary (e.g. [`bitnami/openldap`](https://hub.docker.com/r/bitnami/openldap/)). ##### PERMIT_DOCKER @@ -584,9 +584,7 @@ Enable or disable `getmail`. #### LDAP -##### ENABLE_LDAP -Deprecated. See [`ACCOUNT_PROVISIONER`](#account_provisioner). ##### LDAP_START_TLS diff --git a/docs/content/examples/use-cases/forward-only-mailserver-with-ldap-authentication.md b/docs/content/examples/use-cases/forward-only-mailserver-with-ldap-authentication.md index dc930f13887..85a5726a2a6 100644 --- a/docs/content/examples/use-cases/forward-only-mailserver-with-ldap-authentication.md +++ b/docs/content/examples/use-cases/forward-only-mailserver-with-ldap-authentication.md @@ -30,7 +30,6 @@ We can create aliases with `./setup.sh`, like this: If you want to send emails from outside the mail server you have to authenticate somehow (with a username and password). One way of doing it is described in [this discussion][github-issue-1247]. However if there are many user accounts, it is better to use authentication with LDAP. The settings for this on `mailserver.env` are: ```env -ENABLE_LDAP=1 # with the :edge tag, use ACCOUNT_PROVISIONER ACCOUNT_PROVISIONER=LDAP LDAP_START_TLS=yes LDAP_SERVER_HOST=ldap.example.org diff --git a/mailserver.env b/mailserver.env index 6f270af4e84..8e7530068a2 100644 --- a/mailserver.env +++ b/mailserver.env @@ -405,12 +405,7 @@ GETMAIL_POLL=5 # --- LDAP Section ------------------------------ # ----------------------------------------------- -# A second container for the ldap service is necessary (i.e. https://github.com/osixia/docker-openldap) - -# with the :edge tag, use ACCOUNT_PROVISIONER=LDAP -# empty => LDAP authentication is disabled -# 1 => LDAP authentication is enabled -ENABLE_LDAP= +# A second container for the ldap service is necessary (i.e. https://hub.docker.com/r/bitnami/openldap/) # empty => no # yes => LDAP over TLS enabled for Postfix diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index a0d61242255..cf0992200b1 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -14,8 +14,7 @@ function _early_variables_setup() { # completely with a single version. function __environment_variables_backwards_compatibility() { if [[ ${ENABLE_LDAP:-0} -eq 1 ]]; then - _log 'warn' "'ENABLE_LDAP=1' is deprecated (and will be removed in v13.0.0) => use 'ACCOUNT_PROVISIONER=LDAP' instead" - ACCOUNT_PROVISIONER='LDAP' + _log 'error' "'ENABLE_LDAP=1' has been changed to 'ACCOUNT_PROVISIONER=LDAP' since DMS v13" fi # TODO this can be uncommented in a PR handling the HOSTNAME/DOMAINNAME issue From 19b72aead3e8015b3b855ed0678352ff596ab74c Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 29 Aug 2023 23:33:39 +1200 Subject: [PATCH 144/592] docs: Update docs builder image (#3516) - Bump to release `9.2.x` - Image now has `MAJOR.MINOR` tag support to pull latest `PATCH` versions. --- .github/workflows/scripts/docs/build-docs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scripts/docs/build-docs.sh b/.github/workflows/scripts/docs/build-docs.sh index dd9ef3a533a..aff66ab5812 100755 --- a/.github/workflows/scripts/docs/build-docs.sh +++ b/.github/workflows/scripts/docs/build-docs.sh @@ -10,7 +10,7 @@ docker run \ --user "$(id -u):$(id -g)" \ --volume "${PWD}:/docs" \ --name "build-docs" \ - squidfunk/mkdocs-material:9.1.5 build --strict + squidfunk/mkdocs-material:9.2 build --strict # Remove unnecessary build artifacts: https://github.com/squidfunk/mkdocs-material/issues/2519 # site/ is the build output folder. From e025e4c6962d2f0f6147461f383853025e1a6986 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 29 Aug 2023 23:52:06 +1200 Subject: [PATCH 145/592] tests: Revise LDAP config + setup (#3514) * chore: Use white-space in query filters to improve readability * tests: LDAP ENV query filters documented - These filters remain roughly the same as they were before. The conditions are the same, but restructured to make the complimentary constraints more separated from the actual target attribtues. - The DOMAIN query additionally includes the `mailAlias` from `PostfixBookMailAccount` class as well. - Heavy inline documentation breaking down how the filters work for any maintainer to reference. This will likely be migrated after revision into our user docs for LDAP. Some quirks have also been noted with advice for future changes. * tests: LDAP - Support test-case specific containers A bit more complicated than other test files due to the larger ENV config array that most containers may need to share. Example introduced with the test-case checking custom config file support. * tests: Adjust LDAP test configs - Paths for `.ldif` files used with volumes shortened - Postfix LDAP `.cf` files adjusted to conventions used in LDAP tests. --- .../ldif => openldap/ldifs}/01_mail-tree.ldif | 0 .../ldifs}/02_user-email.ldif | 0 .../03_user-email-other-primary-domain.ldif | 0 .../ldifs}/04_user-email-different-uid.ldif | 0 .../schemas/postfix-book.ldif | 0 test/config/ldap/overrides/ldap-aliases.cf | 6 +- test/config/ldap/overrides/ldap-groups.cf | 18 +-- test/config/ldap/overrides/ldap-users.cf | 6 +- test/tests/serial/mail_with_ldap.bats | 127 +++++++++++++++--- 9 files changed, 120 insertions(+), 37 deletions(-) rename test/config/ldap/{docker-openldap/bootstrap/ldif => openldap/ldifs}/01_mail-tree.ldif (100%) rename test/config/ldap/{docker-openldap/bootstrap/ldif => openldap/ldifs}/02_user-email.ldif (100%) rename test/config/ldap/{docker-openldap/bootstrap/ldif => openldap/ldifs}/03_user-email-other-primary-domain.ldif (100%) rename test/config/ldap/{docker-openldap/bootstrap/ldif => openldap/ldifs}/04_user-email-different-uid.ldif (100%) rename test/config/ldap/{docker-openldap/bootstrap => openldap}/schemas/postfix-book.ldif (100%) diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif b/test/config/ldap/openldap/ldifs/01_mail-tree.ldif similarity index 100% rename from test/config/ldap/docker-openldap/bootstrap/ldif/01_mail-tree.ldif rename to test/config/ldap/openldap/ldifs/01_mail-tree.ldif diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif b/test/config/ldap/openldap/ldifs/02_user-email.ldif similarity index 100% rename from test/config/ldap/docker-openldap/bootstrap/ldif/02_user-email.ldif rename to test/config/ldap/openldap/ldifs/02_user-email.ldif diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif b/test/config/ldap/openldap/ldifs/03_user-email-other-primary-domain.ldif similarity index 100% rename from test/config/ldap/docker-openldap/bootstrap/ldif/03_user-email-other-primary-domain.ldif rename to test/config/ldap/openldap/ldifs/03_user-email-other-primary-domain.ldif diff --git a/test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif b/test/config/ldap/openldap/ldifs/04_user-email-different-uid.ldif similarity index 100% rename from test/config/ldap/docker-openldap/bootstrap/ldif/04_user-email-different-uid.ldif rename to test/config/ldap/openldap/ldifs/04_user-email-different-uid.ldif diff --git a/test/config/ldap/docker-openldap/bootstrap/schemas/postfix-book.ldif b/test/config/ldap/openldap/schemas/postfix-book.ldif similarity index 100% rename from test/config/ldap/docker-openldap/bootstrap/schemas/postfix-book.ldif rename to test/config/ldap/openldap/schemas/postfix-book.ldif diff --git a/test/config/ldap/overrides/ldap-aliases.cf b/test/config/ldap/overrides/ldap-aliases.cf index a4579393cae..2436aa41aa7 100644 --- a/test/config/ldap/overrides/ldap-aliases.cf +++ b/test/config/ldap/overrides/ldap-aliases.cf @@ -1,10 +1,10 @@ # Testconfig for ldap integration bind = yes -bind_dn = cn=admin,dc=domain,dc=com +bind_dn = cn=admin,dc=example,dc=test bind_pw = admin query_filter = (&(mailAlias=%s)(mailEnabled=TRUE)) result_attribute = mail -search_base = ou=people,dc=domain,dc=com -server_host = mail.domain.com +search_base = ou=users,dc=example,dc=test +server_host = mail.example.test start_tls = no version = 3 diff --git a/test/config/ldap/overrides/ldap-groups.cf b/test/config/ldap/overrides/ldap-groups.cf index 6712db9eb32..b0b8da211ef 100644 --- a/test/config/ldap/overrides/ldap-groups.cf +++ b/test/config/ldap/overrides/ldap-groups.cf @@ -1,10 +1,10 @@ # Testconfig for ldap integration -bind = yes -bind_dn = cn=admin,dc=domain,dc=com -bind_pw = admin -query_filter = (&(mailGroupMember=%s)(mailEnabled=TRUE)) -result_attribute = mail -search_base = ou=people,dc=domain,dc=com -server_host = mail.domain.com -start_tls = no -version = 3 +bind = yes +bind_dn = cn=admin,dc=example,dc=test +bind_pw = admin +query_filter = (&(mailGroupMember=%s)(mailEnabled=TRUE)) +result_attribute = mail +search_base = ou=users,dc=example,dc=test +server_host = mail.example.test +start_tls = no +version = 3 diff --git a/test/config/ldap/overrides/ldap-users.cf b/test/config/ldap/overrides/ldap-users.cf index 92cd2ed5048..b32db9fd70c 100644 --- a/test/config/ldap/overrides/ldap-users.cf +++ b/test/config/ldap/overrides/ldap-users.cf @@ -1,10 +1,10 @@ # Testconfig for ldap integration bind = yes -bind_dn = cn=admin,dc=domain,dc=com +bind_dn = cn=admin,dc=example,dc=test bind_pw = admin query_filter = (&(mail=%s)(mailEnabled=TRUE)) result_attribute = mail -search_base = ou=people,dc=domain,dc=com -server_host = mail.domain.com +search_base = ou=users,dc=example,dc=test +server_host = mail.example.test start_tls = no version = 3 diff --git a/test/tests/serial/mail_with_ldap.bats b/test/tests/serial/mail_with_ldap.bats index 03a9052b534..c3c60b92026 100644 --- a/test/tests/serial/mail_with_ldap.bats +++ b/test/tests/serial/mail_with_ldap.bats @@ -4,12 +4,14 @@ load "${REPOSITORY_ROOT}/test/helper/common" BATS_TEST_NAME_PREFIX='[LDAP] ' CONTAINER1_NAME='dms-test_ldap' CONTAINER2_NAME='dms-test_ldap_provider' +# Single test-case specific containers: +CONTAINER3_NAME='dms-test_ldap_custom-config' function setup_file() { export DMS_TEST_NETWORK='test-network-ldap' - export DOMAIN='example.test' - export FQDN_MAIL="mail.${DOMAIN}" - export FQDN_LDAP="ldap.${DOMAIN}" + export DMS_DOMAIN='example.test' + export FQDN_MAIL="mail.${DMS_DOMAIN}" + export FQDN_LDAP="ldap.${DMS_DOMAIN}" # LDAP is provisioned with two domains (via `.ldif` files) unrelated to the FQDN of DMS: export FQDN_LOCALHOST_A='localhost.localdomain' export FQDN_LOCALHOST_B='localhost.otherdomain' @@ -24,8 +26,8 @@ function setup_file() { --env LDAP_ROOT='dc=example,dc=test' \ --env LDAP_PORT_NUMBER=389 \ --env LDAP_SKIP_DEFAULT_TREE=yes \ - --volume './test/config/ldap/docker-openldap/bootstrap/ldif/:/ldifs/:ro' \ - --volume './test/config/ldap/docker-openldap/bootstrap/schemas/:/schemas/:ro' \ + --volume "${REPOSITORY_ROOT}/test/config/ldap/openldap/ldifs/:/ldifs/:ro" \ + --volume "${REPOSITORY_ROOT}/test/config/ldap/openldap/schemas/:/schemas/:ro" \ --hostname "${FQDN_LDAP}" \ --network "${DMS_TEST_NETWORK}" \ bitnami/openldap:latest @@ -36,35 +38,86 @@ function setup_file() { # Setup DMS container # + # LDAP filter queries explained. + # NOTE: All LDAP configs for Postfix (with the exception of `ldap-senders.cf`), return the `mail` attribute value of matched results. + # This is through the config key `result_attribute`, which the ENV substitution feature can only replace across all configs, not selectively like `query_filter`. + # NOTE: The queries below rely specifically upon attributes and classes defined by the schema `postfix-book.ldif`. These are not compatible with all LDAP setups. + + # `mailAlias`` is supported by both classes provided from the schema `postfix-book.ldif`, but `mailEnabled` is only available to `PostfixBookMailAccount` class: + local QUERY_ALIAS='(&(mailAlias=%s) (| (objectClass=PostfixBookMailForward) (&(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)) ))' + + # Postfix does domain lookups with the domain of the recipient to check if DMS manages the mail domain. + # For this lookup `%s` only represents the domain, not a full email address. Hence the match pattern using a wildcard prefix `*@`. + # For a breakdown, see QUERY_SENDERS comment. + # NOTE: Although `result_attribute = mail` will return each accounts full email address, Postfix will only compare to domain-part. + local QUERY_DOMAIN='(| (& (|(mail=*@%s) (mailAlias=*@%s) (mailGroupMember=*@%s)) (&(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)) ) (&(mailAlias=*@%s)(objectClass=PostfixBookMailForward)) )' + + # Simple queries for a single attribute that additionally requires `mailEnabled=TRUE` from the `PostfixBookMailAccount` class: + # NOTE: `mail` attribute is not unique to `PostfixBookMailAccount`. The `mailEnabled` attribute is to further control valid mail accounts. + # TODO: For tests, since `mailEnabled` is not relevant (always configured as TRUE currently), + # a simpler query like `mail=%s` or `mailGroupMember=%s` would be sufficient. The additional constraints could be covered in our docs instead. + local QUERY_GROUP='(&(mailGroupMember=%s) (&(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)) )' + local QUERY_USER='(&(mail=%s) (&(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)) )' + + # Given the sender address `%s` from Postfix, query LDAP for accounts that meet the search filter, + # the `result_attribute` is `mail` + `uid` (`userID`) attributes for login names that are authorized to use that sender address. + # One of the result values returned must match the login name of the user trying to send mail with that sender address. + # + # Filter: Search for accounts that meet any of the following conditions: + # - Has any attribute (`mail`, `mailAlias`, `mailGroupMember`) with a value of `%s`, AND is of class `PostfixBookMailAccount` with `mailEnabled=TRUE` attribute set. + # - Has attribute `mailAlias` with value of `%s` AND is of class PostfixBookMailForward + # - Has the attribute `userID` with value `some.user.id` + local QUERY_SENDERS='(| (& (|(mail=%s) (mailAlias=%s) (mailGroupMember=%s)) (&(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)) ) (&(mailAlias=%s)(objectClass=PostfixBookMailForward)) (userID=some.user.id))' + + # NOTE: The remaining queries below have been left as they were instead of rewritten, `mailEnabled` has no associated class requirement, nor does the class requirement ensure `mailEnabled=TRUE`. + + # When using SASLAuthd for login auth (only valid to Postfix in DMS), + # Postfix will pass on the login username and SASLAuthd will use it's backend to do a lookup. + # The current default is `uniqueIdentifier=%u`, but `%u` is inaccurate currently with LDAP backend + # as SASLAuthd will ignore the domain-part received (if any) and forward only the local-part as the query. + # TODO: Fix this by having supervisor start the service with `-r` option like it does `rimap` backend. + # NOTE: `%u` (full login with realm/domain-part) is presently equivalent to `%U` (local-part) only, + # As the `userID` is not a mail address, we ensure any domain-part is ignored, as a login name is not + # required to match the mail accounts actual `mail` attribute (nor the local-part), they are distinct. + # TODO: Docs should better cover this difference, as it does confuse some users of DMS (and past contributors to our tests..). + local SASLAUTHD_QUERY='(&(userID=%U)(mailEnabled=TRUE))' + + # Dovecot is configured to lookup a user account by their login name (`userID` in this case, but it could be any attribute like `mail`). + # Dovecot syntax token `%n` is the local-part of the full email address supplied as the login name. There must be a unique match on `userID` (which there will be as each account is configured via LDIF to use it in their DN) + # NOTE: We already have a constraint on the LDAP tree to search (`LDAP_SEARCH_BASE`), if all objects in that subtree use `PostfixBookMailAccount` class then there is no benefit in the extra constraint. + # TODO: For tests, that additional constraint is meaningless. We can detail it in our docs instead and just use `userID=%n`. + local DOVECOT_QUERY_PASS='(&(userID=%n)(objectClass=PostfixBookMailAccount))' + local DOVECOT_QUERY_USER='(&(userID=%n)(objectClass=PostfixBookMailAccount))' + local ENV_LDAP_CONFIG=( --env ACCOUNT_PROVISIONER=LDAP # Postfix SASL auth provider (SASLAuthd instead of default Dovecot provider): --env ENABLE_SASLAUTHD=1 --env SASLAUTHD_MECHANISMS=ldap - --env SASLAUTHD_LDAP_FILTER='(&(userID=%U)(mailEnabled=TRUE))' + --env SASLAUTHD_LDAP_FILTER="${SASLAUTHD_QUERY}" # ENV to configure LDAP configs for Dovecot + Postfix: # NOTE: `scripts/startup/setup.d/ldap.sh:_setup_ldap()` uses `_replace_by_env_in_file()` to configure settings (stripping `DOVECOT_` / `LDAP_` prefixes): # Dovecot: - --env DOVECOT_PASS_FILTER='(&(objectClass=PostfixBookMailAccount)(userID=%n))' + --env DOVECOT_PASS_FILTER="${DOVECOT_QUERY_PASS}" --env DOVECOT_TLS=no - --env DOVECOT_USER_FILTER='(&(objectClass=PostfixBookMailAccount)(userID=%n))' + --env DOVECOT_USER_FILTER="${DOVECOT_QUERY_USER}" # Postfix: --env LDAP_BIND_DN='cn=admin,dc=example,dc=test' --env LDAP_BIND_PW='admin' - --env LDAP_QUERY_FILTER_ALIAS='(|(&(mailAlias=%s)(objectClass=PostfixBookMailForward))(&(mailAlias=%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)))' - --env LDAP_QUERY_FILTER_DOMAIN='(|(&(mail=*@%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE))(&(mailGroupMember=*@%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE))(&(mailalias=*@%s)(objectClass=PostfixBookMailForward)))' - --env LDAP_QUERY_FILTER_GROUP='(&(mailGroupMember=%s)(mailEnabled=TRUE))' - --env LDAP_QUERY_FILTER_SENDERS='(|(&(mail=%s)(mailEnabled=TRUE))(&(mailGroupMember=%s)(mailEnabled=TRUE))(|(&(mailAlias=%s)(objectClass=PostfixBookMailForward))(&(mailAlias=%s)(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)))(userID=some.user.id))' - --env LDAP_QUERY_FILTER_USER='(&(mail=%s)(mailEnabled=TRUE))' + --env LDAP_QUERY_FILTER_ALIAS="${QUERY_ALIAS}" + --env LDAP_QUERY_FILTER_DOMAIN="${QUERY_DOMAIN}" + --env LDAP_QUERY_FILTER_GROUP="${QUERY_GROUP}" + --env LDAP_QUERY_FILTER_SENDERS="${QUERY_SENDERS}" + --env LDAP_QUERY_FILTER_USER="${QUERY_USER}" --env LDAP_SEARCH_BASE='ou=users,dc=example,dc=test' --env LDAP_SERVER_HOST="${FQDN_LDAP}" --env LDAP_START_TLS=no ) - # Extra ENV needed to support specific testcases: + # Extra ENV needed to support specific test-cases: local ENV_SUPPORT=( --env PERMIT_DOCKER=container # Required for attempting SMTP auth on port 25 via nc # Required for openssl commands to be successul: @@ -79,22 +132,40 @@ function setup_file() { --env SPOOF_PROTECTION=1 ) + export CONTAINER_NAME=${CONTAINER1_NAME} local CUSTOM_SETUP_ARGUMENTS=( - --hostname "${FQDN_MAIL}" - --network "${DMS_TEST_NETWORK}" - "${ENV_LDAP_CONFIG[@]}" "${ENV_SUPPORT[@]}" + + --hostname "${FQDN_MAIL}" + --network "${DMS_TEST_NETWORK}" ) - # Set default implicit container fallback for helpers: - export CONTAINER_NAME=${CONTAINER1_NAME} + _init_with_defaults + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + _wait_for_smtp_port_in_container + # Single test-case containers below cannot be defined in a test-case when expanding arrays + # defined in `setup()`. Those arrays would need to be hoisted up to the top of the file vars. + # Alternatively for ENV overrides, separate `.env` files could be used. Better options + # are available once switching to `compose.yaml` in tests. + + export CONTAINER_NAME=${CONTAINER3_NAME} + local CUSTOM_SETUP_ARGUMENTS=( + "${ENV_LDAP_CONFIG[@]}" + + # `hostname` should be unique when connecting containers via network: + --hostname "custom-config.${DMS_DOMAIN}" + --network "${DMS_TEST_NETWORK}" + ) _init_with_defaults # NOTE: `test/config/` has now been duplicated, can move test specific files to host-side `/tmp/docker-mailserver`: mv "${TEST_TMP_CONFIG}/ldap/overrides/"*.cf "${TEST_TMP_CONFIG}/" _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' - _wait_for_smtp_port_in_container + + + # Set default implicit container fallback for helpers: + export CONTAINER_NAME=${CONTAINER1_NAME} } function teardown_file() { @@ -102,6 +173,17 @@ function teardown_file() { docker network rm "${DMS_TEST_NETWORK}" } +# Could optionally call `_default_teardown` in test-cases that have specific containers. +# This will otherwise handle it implicitly which is helpful when the test-case hits a failure, +# As failure will bail early missing teardown, which then prevents network cleanup. This way is safer: +function teardown() { + if [[ ${CONTAINER_NAME} != "${CONTAINER1_NAME}" ]] \ + && [[ ${CONTAINER_NAME} != "${CONTAINER2_NAME}" ]] + then + _default_teardown + fi +} + # postfix # NOTE: Each of the 3 user accounts tested below are defined in separate LDIF config files, # Those are bundled into the locally built OpenLDAP Dockerfile. @@ -121,9 +203,10 @@ function teardown_file() { } # Custom LDAP config files support: -# TODO: Compare to provided configs and if they're just including a test comment, -# could just copy the config and append without carrying a separate test config? @test "postfix: ldap custom config files copied" { + # Use the test-case specific container from `setup()` (change only applies to test-case): + export CONTAINER_NAME=${CONTAINER3_NAME} + local LDAP_CONFIGS_POSTFIX=( /etc/postfix/ldap-users.cf /etc/postfix/ldap-groups.cf From 3d5f6aeec40ac1c719becdb031fe306f33bcfcc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Pl=C3=B6tz?= Date: Tue, 29 Aug 2023 23:40:54 +0200 Subject: [PATCH 146/592] docs: Add documentation for iOS mail push support (#3513) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add documentation for iOS mail push support --------- Signed-off-by: René Plötz --- .../use-cases/ios-mail-push-support.md | 232 ++++++++++++++++++ docs/mkdocs.yml | 1 + 2 files changed, 233 insertions(+) create mode 100644 docs/content/examples/use-cases/ios-mail-push-support.md diff --git a/docs/content/examples/use-cases/ios-mail-push-support.md b/docs/content/examples/use-cases/ios-mail-push-support.md new file mode 100644 index 00000000000..d0eacd183cd --- /dev/null +++ b/docs/content/examples/use-cases/ios-mail-push-support.md @@ -0,0 +1,232 @@ +--- +title: 'Advanced | iOS Mail Push Support' +--- + +## Introduction + +iOS Mail currently does not support the IMAP idle extension. Therefore users can only either check manually or configure intervals for fetching mails in their mail account preferences when using the default configuration. + +To support mail push Dovecot needs to advertise the `XAPPLEPUSHSERVICE` IMAP extension as well as sending the actual push notifications to the Apple Push Notification service (APNs) which will forward them to the device. + +This can be done with two components: + +- A Dovecot plugin (`dovecot-xaps-plugin`) which is triggered whenever a mail is created or moved from/to a mail folder. +- A daemon service (`dovecot-xaps-daemon`) that manages both the device registrations as well as sending notifications to the APNs. + +## Prerequisites + +- An Apple developer account to create the required Apple Push Notification service certificate. +- Knowledge creating Docker images, using the command-line, and creating shell scripts. + +## Limitations + +- You need to maintain a custom `docker-mailserver` image. +- Push support is limited to the INBOX folder. Changes to other folders will not be pushed to the device regardless of the configuration settings. +- You currently cannot use the same account UUID on multiple devices. This means that if you use the same backup on multiple devices (e.g. old phone / new phone) only one of them will get the notification. Use different backups or recreate the mail account. + +## Privacy concerns + +- The service does not send any part of the actual message to Apple. +- The information sent contains the device UUID to notify and the (on-device) account UUID which was generated by the iOS mail application when creating the account. +- Upon receiving the notification, the iOS mail application will connect to the IMAP server given by the provided account UUID and fetch the mail to notify the user. +- Apple therefore does not know the mail address for which the mail was received, only that a specific account on a specific device should be notified that a new mail or that a mail was moved to the INBOX folder. + +## Installation + +Both components will be built using Docker and included into a custom `docker-mailserver` image. Afterwards the required configuration is added to `docker-data/dms/config`. The registration data is stored in `/var/mail-state/lib-xapsd`. + +1. Create a Dockerfile to build a `docker-mailserver` image that includes the [`dovecot-xaps-plugin`](https://github.com/freswa/dovecot-xaps-plugin) as well as the [`dovecot-xaps-daemon`](https://github.com/freswa/dovecot-xaps-daemon). This is required to ensure that the Dovecot plugin is built against the same Dovecot version. The `:edge` tag is used here, but you might want to use a released version instead. + + ```dockerfile + FROM mailserver/docker-mailserver:edge AS dovecot-plugin-xaps + WORKDIR /tmp/dovecot-xaps-plugin + RUN <= 1.20 causes this issue: https://github.com/freswa/dovecot-xaps-daemon/issues/24#issuecomment-1483876081 + # Note that the underlying issue are non-standard-compliant Apple http servers which might get fixed at some point + FROM golang:1.19-alpine AS dovecot-xaps-daemon + ENV GOPROXY=https://proxy.golang.org,direct + ENV CGO_ENABLED=0 + WORKDIR /go/dovecot-xaps-daemon + RUN < Date: Fri, 1 Sep 2023 13:54:38 +0200 Subject: [PATCH 147/592] tests: add tests for helper function `_add_to_or_update_postfix_main()` (#3505) --- .../set3/scripts/postconf-helper.bats | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 test/tests/parallel/set3/scripts/postconf-helper.bats diff --git a/test/tests/parallel/set3/scripts/postconf-helper.bats b/test/tests/parallel/set3/scripts/postconf-helper.bats new file mode 100644 index 00000000000..d77d50ac2db --- /dev/null +++ b/test/tests/parallel/set3/scripts/postconf-helper.bats @@ -0,0 +1,48 @@ +load "${REPOSITORY_ROOT}/test/helper/common" +load "${REPOSITORY_ROOT}/test/helper/setup" + +BATS_TEST_NAME_PREFIX='[Scripts] (helper functions) (postfix - _add_to_or_update_postfix_main) ' +CONTAINER_NAME='dms-test_postconf-helper' +# Various tests for the helper function `_add_to_or_update_postfix_main()` +function setup_file() { + _init_with_defaults + _common_container_setup + # Begin tests without 'relayhost' defined in 'main.cf' + _run_in_container postconf -X relayhost + assert_success +} +function teardown_file() { _default_teardown ; } +# Add or modify in Postfix config `main.cf` a parameter key with the provided value. +# When the key already exists, the new value is appended (default), or prepended (explicitly requested). +# NOTE: This test-case helper is hard-coded for testing with the 'relayhost' parameter. +# +# @param ${1} = new value (appended or prepended) +# @param ${2} = action "append" (default) or "prepend" [OPTIONAL] +function _modify_postfix_main_config() { + _run_in_container_bash "source /usr/local/bin/helpers/{postfix,utils}.sh && _add_to_or_update_postfix_main relayhost '${1}' '${2}'" + _run_in_container grep '^relayhost' '/etc/postfix/main.cf' +} +@test "check if initial value is empty" { + _run_in_container postconf -h 'relayhost' + assert_output '' +} +@test "add single value" { + _modify_postfix_main_config 'single-value-test' + assert_output 'relayhost = single-value-test' +} +@test "prepend value" { + _modify_postfix_main_config 'prepend-test' 'prepend' + assert_output 'relayhost = prepend-test single-value-test' +} +@test "append value (explicit)" { + _modify_postfix_main_config 'append-test-explicit' 'append' + assert_output 'relayhost = prepend-test single-value-test append-test-explicit' +} +@test "append value (implicit)" { + _modify_postfix_main_config 'append-test-implicit' + assert_output 'relayhost = prepend-test single-value-test append-test-explicit append-test-implicit' +} +@test "try to append already existing value" { + _modify_postfix_main_config 'append-test-implicit' + assert_output 'relayhost = prepend-test single-value-test append-test-explicit append-test-implicit' +} From ed84dca1471fa6d599f1672c874ab3585a2b0074 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 2 Sep 2023 22:07:02 +1200 Subject: [PATCH 148/592] chore: LDAP config improvements (#3522) * chore: Drop management of `SASLAUTHD_*` ENV - `variables-stack.sh` does not need to manage all these extra ENV or store them. They're not used anywhere else. - `saslauthd.sh` is the only consumer of these ENV which are effectively direct key/value mappings, with some defaults provided / inherited. Instead of trying to conditionally support key/value pairs when ENV is set, we could instead use `sed` to delete lines with empty values. * chore: Drop fallbacks + update configs to match docs - Drop deprecated support: - `DOVECOT_HOSTS` is an ENV deprecated since v10. - Fallback for missing URI scheme introduced for Dovecot and SASLAuthd in v10. - Adding error log message when no LDAP URI scheme is detected for the supported ENV (when set). - Docs updated for ENV to reflect the mandatory requirement. `mailserver.env` partially synced equivalent sections. - Provided base LDAP configs (for overriding) likewise updated from `domain.com` to `example.com`. - LDAP test updated for required `ldap://` URI scheme. Common ENV shared across LDAP configs hoisted out of the Postfix group. * chore: Remove unset lines in generated `saslauthd.conf` --- docs/content/config/environment.md | 11 ++-- mailserver.env | 8 +-- target/dovecot/dovecot-ldap.conf.ext | 6 +-- target/postfix/ldap-aliases.cf | 6 +-- target/postfix/ldap-domains.cf | 6 +-- target/postfix/ldap-groups.cf | 6 +-- target/postfix/ldap-senders.cf | 6 +-- target/postfix/ldap-users.cf | 6 +-- target/scripts/startup/setup.d/ldap.sh | 8 +-- target/scripts/startup/setup.d/saslauthd.sh | 39 +++++++------- target/scripts/startup/variables-stack.sh | 56 ++++----------------- test/tests/serial/mail_with_ldap.bats | 19 ++++--- 12 files changed, 69 insertions(+), 108 deletions(-) diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 3e40bd89753..e5aba88941d 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -594,8 +594,8 @@ Enable or disable `getmail`. ##### LDAP_SERVER_HOST - **empty** => mail.example.com -- => Specify the dns-name/ip-address where the ldap-server is listening, or an URI like `ldaps://mail.example.com` -- NOTE: If you going to use DMS in combination with `compose.yaml` you can set the service name here +- => Specify the `` / `` where the LDAP server is reachable via a URI like: `ldaps://mail.example.com`. +- Note: You must include the desired URI scheme (`ldap://`, `ldaps://`, `ldapi://`). ##### LDAP_SEARCH_BASE @@ -669,9 +669,8 @@ The following variables overwrite the default values for ```/etc/dovecot/dovecot ##### DOVECOT_URIS - **empty** => same as `LDAP_SERVER_HOST` -- => Specify a space separated list of LDAP uris. -- Note: If the protocol is missing, `ldap://` will be used. -- Note: This deprecates `DOVECOT_HOSTS` (as it didn't allow to use LDAPS), which is currently still supported for backwards compatibility. +- => Specify a space separated list of LDAP URIs. +- Note: You must include the desired URI scheme (`ldap://`, `ldaps://`, `ldapi://`). ##### DOVECOT_LDAP_VERSION @@ -764,7 +763,7 @@ Note: This postgrey setting needs `ENABLE_POSTGREY=1` ##### SASLAUTHD_LDAP_SERVER - **empty** => same as `LDAP_SERVER_HOST` -- Note: since version 10.0.0, you can specify a protocol here (like ldaps://); this deprecates SASLAUTHD_LDAP_SSL. +- Note: You must include the desired URI scheme (`ldap://`, `ldaps://`, `ldapi://`). ##### SASLAUTHD_LDAP_START_TLS diff --git a/mailserver.env b/mailserver.env index 8e7530068a2..632f6edff9a 100644 --- a/mailserver.env +++ b/mailserver.env @@ -411,9 +411,9 @@ GETMAIL_POLL=5 # yes => LDAP over TLS enabled for Postfix LDAP_START_TLS= -# If you going to use the mailserver in combination with Docker Compose you can set the service name here -# empty => mail.domain.com -# Specify the dns-name/ip-address where the ldap-server +# empty => mail.example.com +# Specify the `` / `` where the LDAP server is reachable via a URI like: `ldaps://mail.example.com`. +# Note: You must include the desired URI scheme (`ldap://`, `ldaps://`, `ldapi://`). LDAP_SERVER_HOST= # empty => ou=people,dc=domain,dc=com @@ -500,7 +500,7 @@ SASLAUTHD_MECHANISMS= SASLAUTHD_MECH_OPTIONS= # empty => Use value of LDAP_SERVER_HOST -# Note: since version 10.0.0, you can specify a protocol here (like ldaps://); this deprecates SASLAUTHD_LDAP_SSL. +# Note: You must include the desired URI scheme (`ldap://`, `ldaps://`, `ldapi://`). SASLAUTHD_LDAP_SERVER= # empty => Use value of LDAP_BIND_DN diff --git a/target/dovecot/dovecot-ldap.conf.ext b/target/dovecot/dovecot-ldap.conf.ext index edf04b9e4f9..6631110639a 100644 --- a/target/dovecot/dovecot-ldap.conf.ext +++ b/target/dovecot/dovecot-ldap.conf.ext @@ -1,8 +1,8 @@ -base = ou=people,dc=domain,dc=com +base = ou=people,dc=example,dc=com default_pass_scheme = SSHA -dn = cn=admin,dc=domain,dc=com +dn = cn=admin,dc=example,dc=com dnpass = admin -uris = ldap://mail.domain.com +uris = ldap://mail.example.com tls = no ldap_version = 3 pass_attrs = uniqueIdentifier=user,userPassword=password diff --git a/target/postfix/ldap-aliases.cf b/target/postfix/ldap-aliases.cf index 73bfe7229d9..5c8d443a2ce 100644 --- a/target/postfix/ldap-aliases.cf +++ b/target/postfix/ldap-aliases.cf @@ -1,9 +1,9 @@ bind = yes -bind_dn = cn=admin,dc=domain,dc=com +bind_dn = cn=admin,dc=example,dc=com bind_pw = admin query_filter = (&(mailAlias=%s)(mailEnabled=TRUE)) result_attribute = mail -search_base = ou=people,dc=domain,dc=com -server_host = mail.domain.com +search_base = ou=people,dc=example,dc=com +server_host = mail.example.com start_tls = no version = 3 diff --git a/target/postfix/ldap-domains.cf b/target/postfix/ldap-domains.cf index 5edd2441c0d..c118ebf633b 100644 --- a/target/postfix/ldap-domains.cf +++ b/target/postfix/ldap-domains.cf @@ -1,9 +1,9 @@ bind = yes -bind_dn = cn=admin,dc=domain,dc=com +bind_dn = cn=admin,dc=example,dc=com bind_pw = admin query_filter = (&(|(mail=*@%s)(mailalias=*@%s))(mailEnabled=TRUE)) result_attribute = mail -search_base = ou=people,dc=domain,dc=com -server_host = mail.domain.com +search_base = ou=people,dc=example,dc=com +server_host = mail.example.com start_tls = no version = 3 diff --git a/target/postfix/ldap-groups.cf b/target/postfix/ldap-groups.cf index 914e31a172e..dc7fa14f285 100644 --- a/target/postfix/ldap-groups.cf +++ b/target/postfix/ldap-groups.cf @@ -1,9 +1,9 @@ bind = yes -bind_dn = cn=admin,dc=domain,dc=com +bind_dn = cn=admin,dc=example,dc=com bind_pw = admin query_filter = (&(mailGroupMember=%s)(mailEnabled=TRUE)) result_attribute = mail -search_base = ou=people,dc=domain,dc=com -server_host = mail.domain.com +search_base = ou=people,dc=example,dc=com +server_host = mail.example.com start_tls = no version = 3 diff --git a/target/postfix/ldap-senders.cf b/target/postfix/ldap-senders.cf index 5f1ed6f56a8..88a6cd8701c 100644 --- a/target/postfix/ldap-senders.cf +++ b/target/postfix/ldap-senders.cf @@ -1,9 +1,9 @@ bind = yes -bind_dn = cn=admin,dc=domain,dc=com +bind_dn = cn=admin,dc=example,dc=com bind_pw = admin query_filter = (mail=%s) result_attribute = mail, uid -search_base = ou=people,dc=domain,dc=com -server_host = mail.domain.com +search_base = ou=people,dc=example,dc=com +server_host = mail.example.com start_tls = no version = 3 diff --git a/target/postfix/ldap-users.cf b/target/postfix/ldap-users.cf index a7b29cb6f40..943801f4614 100644 --- a/target/postfix/ldap-users.cf +++ b/target/postfix/ldap-users.cf @@ -1,9 +1,9 @@ bind = yes -bind_dn = cn=admin,dc=domain,dc=com +bind_dn = cn=admin,dc=example,dc=com bind_pw = admin query_filter = (&(mail=%s)(mailEnabled=TRUE)) result_attribute = mail -search_base = ou=people,dc=domain,dc=com -server_host = mail.domain.com +search_base = ou=people,dc=example,dc=com +server_host = mail.example.com start_tls = no version = 3 diff --git a/target/scripts/startup/setup.d/ldap.sh b/target/scripts/startup/setup.d/ldap.sh index ceaca4eb4bc..1451ec32d1a 100644 --- a/target/scripts/startup/setup.d/ldap.sh +++ b/target/scripts/startup/setup.d/ldap.sh @@ -38,13 +38,7 @@ function _setup_ldap() { DOVECOT_LDAP_MAPPING['DOVECOT_BASE']="${DOVECOT_BASE:="${LDAP_SEARCH_BASE}"}" DOVECOT_LDAP_MAPPING['DOVECOT_DN']="${DOVECOT_DN:="${LDAP_BIND_DN}"}" DOVECOT_LDAP_MAPPING['DOVECOT_DNPASS']="${DOVECOT_DNPASS:="${LDAP_BIND_PW}"}" - DOVECOT_LDAP_MAPPING['DOVECOT_URIS']="${DOVECOT_URIS:="${DOVECOT_HOSTS:="${LDAP_SERVER_HOST}"}"}" - - # Add protocol to DOVECOT_URIS so that we can use dovecot's "uris" option: - # https://doc.dovecot.org/configuration_manual/authentication/ldap/ - if [[ ${DOVECOT_LDAP_MAPPING["DOVECOT_URIS"]} != *'://'* ]]; then - DOVECOT_LDAP_MAPPING['DOVECOT_URIS']="ldap://${DOVECOT_LDAP_MAPPING["DOVECOT_URIS"]}" - fi + DOVECOT_LDAP_MAPPING['DOVECOT_URIS']="${DOVECOT_URIS:="${LDAP_SERVER_HOST}"}" # Default DOVECOT_PASS_FILTER to the same value as DOVECOT_USER_FILTER DOVECOT_LDAP_MAPPING['DOVECOT_PASS_FILTER']="${DOVECOT_PASS_FILTER:="${DOVECOT_USER_FILTER}"}" diff --git a/target/scripts/startup/setup.d/saslauthd.sh b/target/scripts/startup/setup.d/saslauthd.sh index 12f00726a04..eb33a24329c 100644 --- a/target/scripts/startup/setup.d/saslauthd.sh +++ b/target/scripts/startup/setup.d/saslauthd.sh @@ -1,29 +1,29 @@ #!/bin/bash - function _setup_saslauthd() { _log 'debug' 'Setting up SASLAUTHD' - if [[ ! -f /etc/saslauthd.conf ]]; then + # NOTE: It's unlikely this file would already exist, + # Unlike Dovecot/Postfix LDAP support, this file has no ENV replacement + # nor does it copy from the DMS config volume to this internal location. + if [[ ${ACCOUNT_PROVISIONER} == 'LDAP' ]] \ + && [[ ! -f /etc/saslauthd.conf ]]; then _log 'trace' 'Creating /etc/saslauthd.conf' - cat > /etc/saslauthd.conf << EOF -ldap_servers: ${SASLAUTHD_LDAP_SERVER} - -ldap_auth_method: ${SASLAUTHD_LDAP_AUTH_METHOD} -ldap_bind_dn: ${SASLAUTHD_LDAP_BIND_DN} -ldap_bind_pw: ${SASLAUTHD_LDAP_PASSWORD} - -ldap_search_base: ${SASLAUTHD_LDAP_SEARCH_BASE} -ldap_filter: ${SASLAUTHD_LDAP_FILTER} - -ldap_start_tls: ${SASLAUTHD_LDAP_START_TLS} -ldap_tls_check_peer: ${SASLAUTHD_LDAP_TLS_CHECK_PEER} - -${SASLAUTHD_LDAP_TLS_CACERT_FILE} -${SASLAUTHD_LDAP_TLS_CACERT_DIR} -${SASLAUTHD_LDAP_PASSWORD_ATTR} -${SASLAUTHD_LDAP_MECH} + # Create a config based on ENV + sed '/^.*: $/d'> /etc/saslauthd.conf << EOF +ldap_servers: ${SASLAUTHD_LDAP_SERVER:=${LDAP_SERVER_HOST}} +ldap_auth_method: ${SASLAUTHD_LDAP_AUTH_METHOD:=bind} +ldap_bind_dn: ${SASLAUTHD_LDAP_BIND_DN:=${LDAP_BIND_DN}} +ldap_bind_pw: ${SASLAUTHD_LDAP_PASSWORD:=${LDAP_BIND_PW}} +ldap_search_base: ${SASLAUTHD_LDAP_SEARCH_BASE:=${LDAP_SEARCH_BASE}} +ldap_filter: ${SASLAUTHD_LDAP_FILTER:=(&(uniqueIdentifier=%u)(mailEnabled=TRUE))} +ldap_start_tls: ${SASLAUTHD_LDAP_START_TLS:=no} +ldap_tls_check_peer: ${SASLAUTHD_LDAP_TLS_CHECK_PEER:=no} +ldap_tls_cacert_file: ${SASLAUTHD_LDAP_TLS_CACERT_FILE} +ldap_tls_cacert_dir: ${SASLAUTHD_LDAP_TLS_CACERT_DIR} +ldap_password_attr: ${SASLAUTHD_LDAP_PASSWORD_ATTR} +ldap_mech: ${SASLAUTHD_LDAP_MECH} ldap_referrals: yes log_level: 10 EOF @@ -42,4 +42,3 @@ EOF gpasswd -a postfix sasl >/dev/null } - diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index cf0992200b1..b18a61dcdc0 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -17,6 +17,14 @@ function __environment_variables_backwards_compatibility() { _log 'error' "'ENABLE_LDAP=1' has been changed to 'ACCOUNT_PROVISIONER=LDAP' since DMS v13" fi + # Dovecot and SASLAuthd have applied an 'ldap://' fallback for compatibility since v10 (June 2021) + # This was silently applied, but users should be explicit: + if [[ ${LDAP_SERVER_HOST:-'://'} != *'://'* ]] \ + || [[ ${DOVECOT_URIS:-'://'} != *'://'* ]] \ + || [[ ${SASLAUTHD_LDAP_SERVER:-'://'} != *'://'* ]]; then + _log 'error' "The ENV for which LDAP host to connect to must include the URI scheme ('ldap://', 'ldaps://', 'ldapi://')" + fi + # TODO this can be uncommented in a PR handling the HOSTNAME/DOMAINNAME issue # TODO see check_for_changes.sh and dns.sh # if [[ -n ${OVERRIDE_HOSTNAME:-} ]] @@ -141,6 +149,7 @@ function __environment_variables_general_setup() { } # This function handles environment variables related to LDAP. +# NOTE: SASLAuthd and Dovecot LDAP support inherit these common ENV. function _environment_variables_ldap() { _log 'debug' 'Setting LDAP-related environment variables now' @@ -152,55 +161,12 @@ function _environment_variables_ldap() { } # This function handles environment variables related to SASLAUTHD -# and, if activated, variables related to SASLAUTHD and LDAP. +# LDAP specific ENV handled in: `startup/setup.d/saslauthd.sh:_setup_saslauthd()` function _environment_variables_saslauthd() { _log 'debug' 'Setting SASLAUTHD-related environment variables now' + # Only used by the supervisor service command (upstream default: `/etc/default/saslauthd`) VARS[SASLAUTHD_MECHANISMS]="${SASLAUTHD_MECHANISMS:=pam}" - - # SASL ENV for configuring an LDAP specific - # `saslauthd.conf` via `setup-stack.sh:_setup_sasulauthd()` - if [[ ${ACCOUNT_PROVISIONER} == 'LDAP' ]]; then - _log 'trace' 'Setting SASLSAUTH-LDAP variables nnow' - - VARS[SASLAUTHD_LDAP_AUTH_METHOD]="${SASLAUTHD_LDAP_AUTH_METHOD:=bind}" - VARS[SASLAUTHD_LDAP_BIND_DN]="${SASLAUTHD_LDAP_BIND_DN:=${LDAP_BIND_DN}}" - VARS[SASLAUTHD_LDAP_FILTER]="${SASLAUTHD_LDAP_FILTER:=(&(uniqueIdentifier=%u)(mailEnabled=TRUE))}" - VARS[SASLAUTHD_LDAP_PASSWORD]="${SASLAUTHD_LDAP_PASSWORD:=${LDAP_BIND_PW}}" - VARS[SASLAUTHD_LDAP_SEARCH_BASE]="${SASLAUTHD_LDAP_SEARCH_BASE:=${LDAP_SEARCH_BASE}}" - VARS[SASLAUTHD_LDAP_SERVER]="${SASLAUTHD_LDAP_SERVER:=${LDAP_SERVER_HOST}}" - [[ ${SASLAUTHD_LDAP_SERVER} != *'://'* ]] && SASLAUTHD_LDAP_SERVER="ldap://${SASLAUTHD_LDAP_SERVER}" - VARS[SASLAUTHD_LDAP_START_TLS]="${SASLAUTHD_LDAP_START_TLS:=no}" - VARS[SASLAUTHD_LDAP_TLS_CHECK_PEER]="${SASLAUTHD_LDAP_TLS_CHECK_PEER:=no}" - - if [[ -z ${SASLAUTHD_LDAP_TLS_CACERT_FILE} ]]; then - SASLAUTHD_LDAP_TLS_CACERT_FILE='' - else - SASLAUTHD_LDAP_TLS_CACERT_FILE="ldap_tls_cacert_file: ${SASLAUTHD_LDAP_TLS_CACERT_FILE}" - fi - VARS[SASLAUTHD_LDAP_TLS_CACERT_FILE]="${SASLAUTHD_LDAP_TLS_CACERT_FILE}" - - if [[ -z ${SASLAUTHD_LDAP_TLS_CACERT_DIR} ]]; then - SASLAUTHD_LDAP_TLS_CACERT_DIR='' - else - SASLAUTHD_LDAP_TLS_CACERT_DIR="ldap_tls_cacert_dir: ${SASLAUTHD_LDAP_TLS_CACERT_DIR}" - fi - VARS[SASLAUTHD_LDAP_TLS_CACERT_DIR]="${SASLAUTHD_LDAP_TLS_CACERT_DIR}" - - if [[ -z ${SASLAUTHD_LDAP_PASSWORD_ATTR} ]]; then - SASLAUTHD_LDAP_PASSWORD_ATTR='' - else - SASLAUTHD_LDAP_PASSWORD_ATTR="ldap_password_attr: ${SASLAUTHD_LDAP_PASSWORD_ATTR}" - fi - VARS[SASLAUTHD_LDAP_PASSWORD_ATTR]="${SASLAUTHD_LDAP_PASSWORD_ATTR}" - - if [[ -z ${SASLAUTHD_LDAP_MECH} ]]; then - SASLAUTHD_LDAP_MECH='' - else - SASLAUTHD_LDAP_MECH="ldap_mech: ${SASLAUTHD_LDAP_MECH}" - fi - VARS[SASLAUTHD_LDAP_MECH]="${SASLAUTHD_LDAP_MECH}" - fi } # This function Writes the contents of the `VARS` map (associative array) diff --git a/test/tests/serial/mail_with_ldap.bats b/test/tests/serial/mail_with_ldap.bats index c3c60b92026..1b73643edd7 100644 --- a/test/tests/serial/mail_with_ldap.bats +++ b/test/tests/serial/mail_with_ldap.bats @@ -92,29 +92,32 @@ function setup_file() { local ENV_LDAP_CONFIG=( --env ACCOUNT_PROVISIONER=LDAP + # Common LDAP ENV: + # NOTE: `scripts/startup/setup.d/ldap.sh:_setup_ldap()` uses `_replace_by_env_in_file()` to configure settings (stripping `DOVECOT_` / `LDAP_` prefixes): + --env LDAP_SERVER_HOST="ldap://${FQDN_LDAP}" + --env LDAP_SEARCH_BASE='ou=users,dc=example,dc=test' + --env LDAP_START_TLS=no + # Credentials needed for read access to LDAP_SEARCH_BASE: + --env LDAP_BIND_DN='cn=admin,dc=example,dc=test' + --env LDAP_BIND_PW='admin' + # Postfix SASL auth provider (SASLAuthd instead of default Dovecot provider): --env ENABLE_SASLAUTHD=1 --env SASLAUTHD_MECHANISMS=ldap --env SASLAUTHD_LDAP_FILTER="${SASLAUTHD_QUERY}" # ENV to configure LDAP configs for Dovecot + Postfix: - # NOTE: `scripts/startup/setup.d/ldap.sh:_setup_ldap()` uses `_replace_by_env_in_file()` to configure settings (stripping `DOVECOT_` / `LDAP_` prefixes): # Dovecot: --env DOVECOT_PASS_FILTER="${DOVECOT_QUERY_PASS}" - --env DOVECOT_TLS=no --env DOVECOT_USER_FILTER="${DOVECOT_QUERY_USER}" + --env DOVECOT_TLS=no # Postfix: - --env LDAP_BIND_DN='cn=admin,dc=example,dc=test' - --env LDAP_BIND_PW='admin' --env LDAP_QUERY_FILTER_ALIAS="${QUERY_ALIAS}" --env LDAP_QUERY_FILTER_DOMAIN="${QUERY_DOMAIN}" --env LDAP_QUERY_FILTER_GROUP="${QUERY_GROUP}" --env LDAP_QUERY_FILTER_SENDERS="${QUERY_SENDERS}" --env LDAP_QUERY_FILTER_USER="${QUERY_USER}" - --env LDAP_SEARCH_BASE='ou=users,dc=example,dc=test' - --env LDAP_SERVER_HOST="${FQDN_LDAP}" - --env LDAP_START_TLS=no ) # Extra ENV needed to support specific test-cases: @@ -221,7 +224,7 @@ function teardown() { @test "postfix: ldap config overwrites success" { local LDAP_SETTINGS_POSTFIX=( - "server_host = ${FQDN_LDAP}" + "server_host = ldap://${FQDN_LDAP}" 'start_tls = no' 'search_base = ou=users,dc=example,dc=test' 'bind_dn = cn=admin,dc=example,dc=test' From c5420530b7e21d4c0f77dbb910f9afd051f031f9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 17:07:38 +0200 Subject: [PATCH 149/592] chore(deps): Bump actions/checkout from 3 to 4 (#3525) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/contributors.yml | 2 +- .github/workflows/docs-preview-prepare.yml | 2 +- .github/workflows/docs-production-deploy.yml | 8 ++++---- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- .github/workflows/linting.yml | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index 6e51495a748..00b581c484f 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: 'Checkout' - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: 'Checkout New Branch and Push It' run: | diff --git a/.github/workflows/docs-preview-prepare.yml b/.github/workflows/docs-preview-prepare.yml index a509ac778ff..befd4008d82 100644 --- a/.github/workflows/docs-preview-prepare.yml +++ b/.github/workflows/docs-preview-prepare.yml @@ -29,7 +29,7 @@ jobs: NETLIFY_SITE_PREFIX: pullrequest-${{ github.event.pull_request.number }} NETLIFY_SITE_NAME: dms-doc-previews steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: 'Build with mkdocs-material via Docker' working-directory: docs diff --git a/.github/workflows/docs-production-deploy.yml b/.github/workflows/docs-production-deploy.yml index 8c8faa23c3f..cb8bdbed226 100644 --- a/.github/workflows/docs-production-deploy.yml +++ b/.github/workflows/docs-production-deploy.yml @@ -28,7 +28,7 @@ jobs: name: 'Deploy Docs' runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: 'Check if deploy is for a `v.` tag version instead of `edge`' if: startsWith(github.ref, 'refs/tags/') @@ -79,10 +79,10 @@ jobs: needs: deploy steps: - name: 'Checkout the tagged commit (shallow clone)' - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: 'Checkout the docs deployment branch to a subdirectory' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: gh-pages path: gh-pages @@ -115,7 +115,7 @@ jobs: needs: add-version-to-docs steps: - name: 'Checkout the docs deployment branch' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: gh-pages diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index a86b0ed6044..d1e837a9134 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -28,7 +28,7 @@ jobs: build-cache-key: ${{ steps.derive-image-cache-key.outputs.digest }} steps: - name: 'Checkout' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 598c12a3d9c..b3a64ef0891 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: 'Checkout' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 5b460ccecdb..bc2adb6c3eb 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -20,7 +20,7 @@ jobs: fail-fast: false steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: # Required to retrieve bats (core + extras): submodules: recursive diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 69e8cea3c21..b5ed5e9d2e6 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: 'Checkout' - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Get the cached build layers from the build job: # This should always be a cache-hit, thus `restore-keys` fallback is not used. diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index e9339a671b3..bdde2d0fa46 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Hadolint run: make hadolint From 20241691b832a1d4115f4fd19cdd05751e558c8f Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Thu, 7 Sep 2023 18:35:08 -0400 Subject: [PATCH 150/592] docs: Fix IPv6 example for Compose (#3531) The subnet must be specified as part of `ipam.configs`. This was unfortunately slightly incorrect due to a mistake in the official Docker docs being propagated, which has since been fixed upstream. Refer to the official Compose Spec for more details: * https://docs.docker.com/compose/compose-file/06-networks/#ipam --- docs/content/config/advanced/ipv6.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/content/config/advanced/ipv6.md b/docs/content/config/advanced/ipv6.md index 4a45d168535..69ed069bd1e 100644 --- a/docs/content/config/advanced/ipv6.md +++ b/docs/content/config/advanced/ipv6.md @@ -92,7 +92,9 @@ Next, configure a network with an IPv6 subnet for your container with any of the networks: dms-ipv6: enable_ipv6: true - subnet: fd00:cafe:face:feed::/64 + ipam: + config: + - subnet: fd00:cafe:face:feed::/64 ``` ??? tip "Override the implicit `default` network" From ad8b618b461236fe504cdbb0d0ddf0bff0283840 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 9 Sep 2023 09:35:57 +1200 Subject: [PATCH 151/592] fix: Ensure files are committed with `eol=lf` via `.gitattributes` (#3527) * chore: Use `.yml` extension Both of these files support the `.yml` extension. Normalize on that. * fix: Add `.gitattributes` to ensure `LF` line-endings are committed Avoids accidentally committing files with `CRLF` when they're created on Windows. Or worse, if some editors don't detect `LF` and would introduce mixed line-endings with `CRLF`. Shouldn't be a problem in practice as we already have a linting check to catch this via CI during PRs. This file is complimentary, in that it should automate that concern away. --- .gitattributes | 160 ++++++++++++++++++ ...ature_request.yaml => feature_request.yml} | 0 .../linting/{.hadolint.yaml => .hadolint.yml} | 0 test/linting/lint.sh | 2 +- 4 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 .gitattributes rename .github/ISSUE_TEMPLATE/{feature_request.yaml => feature_request.yml} (100%) rename test/linting/{.hadolint.yaml => .hadolint.yml} (100%) diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000..d3dba13d237 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,160 @@ +# Normalize line endings of all non-binary files to LF upon check-in (`git add` / `git commit`): +* text=auto + +################################################# +### General ################################### +################################################# + +## GENERIC +### CI + docs/mkdocs.yml +*.yml text +### Documentation (Project, Tests, Docs site) +*.md text +### TLS certs (test/test-files/) + DHE params (target/shared/) +*.pem text +*.pem.sha512sum text + +################################################# +### Project ################################### +################################################# + +## BUILD: +.dockerignore text +Dockerfile text +Makefile +VERSION + +## EXAMPLE (RUNTIME): +*.env text +*.yaml text + +## PROJECT +.all-contributorsrc text export-ignore +.editorconfig text export-ignore +.gitattributes text export-ignore +.gitignore text export-ignore +.gitkeep text export-ignore +.gitmodules text export-ignore +LICENSE text + +## SOURCE CODE +*.sh text eol=lf +### acme.json extractor (target/bin/) +*.py text eol=lf +### Only contain scripts (glob for extensionless) +target/bin/** text eol=lf + +################################################# +### Config #################################### +################################################# + +## CONFIG +### Contains all text files (glob for extensionless) +target/amavis/** text +target/fetchmail/** text +target/getmail/** text +target/opendkim/** text +target/opendmarc/** text +target/postgrey/** text +target/postsrsd/** text +### Generic target/ + test/config/ +*.cf text +*.conf text +### Dovecot +*.ext text +*.sieve text +### Dovecot + Rspamd +*.inc text +### Fail2Ban + Postgrey (test/config/) +*.local text +### Postfix +*.pcre text + +################################################# +### Tests ##################################### +################################################# + +## BATS +*.bash text +*.bats text + +## CONFIG (test/config/) +### OpenLDAP image +*.ldif text +### OpenDKIM +*.private text +KeyTable text +SigningTable text +TrustedHosts text +### Postgrey +whitelist_recipients text + +## MISC +### test/config/ + test/test-files/ +*.txt text +### test/linting/ (.ecrc.json) + test/test-files/ (*.acme.json): +*.json text + +################################################# +### Documentation Website ##################### +################################################# + +## DOCUMENTATION +### docs/content/assets/ +*.css text +*.png binary +*.svg text -diff +*.woff binary +### docs/overrides/ +*.html text +*.ico binary +*.webp binary + +################################################# +### Info # ##################################### +################################################# + +### WHAT IS THIS FILE? +# `.gitattributes` - Pattern-based overrides (Project specific) +# Documentation: https://git-scm.com/docs/gitattributes +# +# Travels with the project and can override the defaults from `.gitconfig`. +# This helps to enforce consistent line endings (CRLF / LF) where needed via +# patterns (_when the git client supports `.gitattributes`_). + +# `.gitconfig` - Global Git defaults (Dev environment) +# Documentation: https://git-scm.com/docs/git-config +# +# Git settings `core.autocrlf` and `core.eol` can vary across dev environments. +# Those defaults can introduce subtle bugs due to incompatible line endings. + + +### WHY SHOULD I CARE? +# The desired result is to ensure the repo contains normalized LF line endings, +# notably avoiding unhelpful noise in diffs or issues incurred from mixed line +# endings. Storing as LF ensures no surprises for line endings during checkout. +# Additionally for checkout to the local working directory, line endings can be +# forced to CRLF or LF per file where appropriate, which ensures the files have +# compatible line endings where software expects a specific kind. +# +# Examples: +# Diffs with nothing visual changed. Line endings appear invisible. +# Tests that compare text from two sources where only line endings differ fail. +# /bin/sh with a shebang fails to run a binary at the given path due to a CRLF. + + +### ATTRIBUTES +# `text` normalizes the line endings of a file to LF upon commit (CRLF -> LF). +# `text=auto` sets `text` if Git doesn't consider the file as binary data. + +# `eol` sets an explicit line ending to write files to the working directory. +# `core.eol` is used for any files not explicitly set with an `eol` attr value. +# `core.eol` uses the native line endings for your platform by default. +# `core.autocrlf` (if set to `true` or `input`) overrides the `core.eol` value. + +# `binary` is an alias for `-text -diff`. The file won't be normalized (-text). +# `-diff` indicates to avoid creating a diff. Useful when diffs are unlikely +# to be meaningful, such as generated content (SVG, Source Maps, Lockfiles). + +# `export-ignore` excludes matched files and directories during `git archive`, +# which services like Github use to create releases with archived source files. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yml similarity index 100% rename from .github/ISSUE_TEMPLATE/feature_request.yaml rename to .github/ISSUE_TEMPLATE/feature_request.yml diff --git a/test/linting/.hadolint.yaml b/test/linting/.hadolint.yml similarity index 100% rename from test/linting/.hadolint.yaml rename to test/linting/.hadolint.yml diff --git a/test/linting/lint.sh b/test/linting/lint.sh index c5718447b33..88103ddd098 100755 --- a/test/linting/lint.sh +++ b/test/linting/lint.sh @@ -35,7 +35,7 @@ function _hadolint() { --volume "${REPOSITORY_ROOT}:/ci:ro" \ --workdir "/ci" \ --name dms-test_hadolint \ - "hadolint/hadolint:v${HADOLINT_VERSION}-alpine" hadolint --config "/ci/test/linting/.hadolint.yaml" Dockerfile + "hadolint/hadolint:v${HADOLINT_VERSION}-alpine" hadolint --config "/ci/test/linting/.hadolint.yml" Dockerfile then _log 'info' 'Hadolint succeeded' else From fc3229f8d205f5d5a7e74a8fbe95f4005940db38 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 18:58:36 +0200 Subject: [PATCH 152/592] chore(deps): Bump docker/build-push-action from 4.1.1 to 4.2.1 (#3533) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index d1e837a9134..23c710a6cee 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v4.1.1 + uses: docker/build-push-action@v4.2.1 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index b3a64ef0891..3498555304a 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -72,7 +72,7 @@ jobs: run: echo "version=$(>"${GITHUB_OUTPUT}" - name: 'Build and publish images' - uses: docker/build-push-action@v4.1.1 + uses: docker/build-push-action@v4.2.1 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index bc2adb6c3eb..48d2cbacf58 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v4.1.1 + uses: docker/build-push-action@v4.2.1 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index b5ed5e9d2e6..746674315ff 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v4.1.1 + uses: docker/build-push-action@v4.2.1 with: context: . tags: mailserver-testing:ci From 8329fa19cc8b38a3776b1f4a5f68f9bf7fb6f5c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 10:50:55 +1200 Subject: [PATCH 153/592] chore(deps): Bump myrotvorets/set-commit-status-action (#3534) Bumps [myrotvorets/set-commit-status-action](https://github.com/myrotvorets/set-commit-status-action) from 1.1.7 to 2.0.0. - [Release notes](https://github.com/myrotvorets/set-commit-status-action/releases) - [Commits](https://github.com/myrotvorets/set-commit-status-action/compare/v1.1.7...v2.0.0) --- updated-dependencies: - dependency-name: myrotvorets/set-commit-status-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/contributors.yml | 2 +- .github/workflows/docs-preview-deploy.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index 00b581c484f..9028a60b6d0 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -73,7 +73,7 @@ jobs: # workflow, which is required due to branch protection, is not important for this type # of PR, so we skip it and pretend it was successful. - name: 'Set Status for Linting Actions to Success (Skipped)' - uses: myrotvorets/set-commit-status-action@v1.1.7 + uses: myrotvorets/set-commit-status-action@v2.0.0 continue-on-error: true with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index 278cf60c347..cac2ac64bc0 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -46,7 +46,7 @@ jobs: # but presently does not work correctly via split workflow. It is useful in a split workflow as the 1st stage # no longer indicates if the entire workflow/deployment was successful. - name: 'Commit Status: Set Workflow Status as Pending' - uses: myrotvorets/set-commit-status-action@v1.1.7 + uses: myrotvorets/set-commit-status-action@v2.0.0 with: token: ${{ secrets.GITHUB_TOKEN }} status: pending @@ -106,7 +106,7 @@ jobs: Built with commit: ${{ env.PR_HEADSHA }} - name: 'Commit Status: Update deployment status' - uses: myrotvorets/set-commit-status-action@v1.1.7 + uses: myrotvorets/set-commit-status-action@v2.0.0 # Always run this step regardless of job failing early: if: ${{ always() }} env: From 86edaf9a8a3ad8d5b2bba3d9a6ad94b675ca7d1a Mon Sep 17 00:00:00 2001 From: Lucas Bartholemy Date: Wed, 13 Sep 2023 10:42:52 +0200 Subject: [PATCH 154/592] fix: DKIM key generation broken when Rspamd & OpenDKIM are enabled (#3535) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- target/bin/open-dkim | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/target/bin/open-dkim b/target/bin/open-dkim index 0036ff8daa3..11b9897554b 100755 --- a/target/bin/open-dkim +++ b/target/bin/open-dkim @@ -4,8 +4,12 @@ source /usr/local/bin/helpers/index.sh if [[ -f /etc/dms-settings ]] && [[ $(_get_dms_env_value 'ENABLE_RSPAMD') -eq 1 ]]; then - /usr/local/bin/rspamd-dkim "${@}" - exit + if [[ $(_get_dms_env_value 'ENABLE_OPENDKIM') -eq 1 ]]; then + log 'error' "You enabled Rspamd and OpenDKIM - OpenDKIM will be implicitly used for DKIM keys" + else + /usr/local/bin/rspamd-dkim "${@}" + exit + fi fi KEYSIZE=2048 From 62f4544dd2ebe1ff68f3e59bb0e9bca7a6905ed4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 14:53:39 +1200 Subject: [PATCH 155/592] chore(deps): Bump docker/setup-buildx-action from 2.10.0 to 3.0.0 (#3540) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 2.10.0 to 3.0.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v2.10.0...v3.0.0) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 23c710a6cee..bc2f62ad2a0 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -79,7 +79,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.10.0 + uses: docker/setup-buildx-action@v3.0.0 # NOTE: AMD64 can build within 2 minutes - name: 'Build images' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 3498555304a..218c677932c 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -40,7 +40,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.10.0 + uses: docker/setup-buildx-action@v3.0.0 # Try get the cached build layers from a prior `generic_build.yml` job. # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 48d2cbacf58..19994a40f70 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -38,7 +38,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.10.0 + uses: docker/setup-buildx-action@v3.0.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 746674315ff..d2a2709e30d 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -37,7 +37,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v2.10.0 + uses: docker/setup-buildx-action@v3.0.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. From c425cdddc56880637c8b1d4d193d75fdf8556d17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 02:55:20 +0000 Subject: [PATCH 156/592] chore(deps): Bump docker/build-push-action from 4.2.1 to 5.0.0 (#3541) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 4.2.1 to 5.0.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v4.2.1...v5.0.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index bc2f62ad2a0..dc46b681821 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v4.2.1 + uses: docker/build-push-action@v5.0.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 218c677932c..b1830c66144 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -72,7 +72,7 @@ jobs: run: echo "version=$(>"${GITHUB_OUTPUT}" - name: 'Build and publish images' - uses: docker/build-push-action@v4.2.1 + uses: docker/build-push-action@v5.0.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 19994a40f70..3c6e12883cc 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v4.2.1 + uses: docker/build-push-action@v5.0.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index d2a2709e30d..ff8c8c1af9c 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v4.2.1 + uses: docker/build-push-action@v5.0.0 with: context: . tags: mailserver-testing:ci From af65189a82672f986a026c92039dbb68a0922273 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 02:57:36 +0000 Subject: [PATCH 157/592] chore(deps): Bump docker/setup-qemu-action from 2.2.0 to 3.0.0 (#3542) Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 2.2.0 to 3.0.0. - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/v2.2.0...v3.0.0) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index dc46b681821..c7dc6e3fad9 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -74,7 +74,7 @@ jobs: cache-buildx- - name: 'Set up QEMU' - uses: docker/setup-qemu-action@v2.2.0 + uses: docker/setup-qemu-action@v3.0.0 with: platforms: arm64 diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index b1830c66144..4e32d190f4a 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -35,7 +35,7 @@ jobs: type=semver,pattern={{major}}.{{minor}}.{{patch}} - name: 'Set up QEMU' - uses: docker/setup-qemu-action@v2.2.0 + uses: docker/setup-qemu-action@v3.0.0 with: platforms: arm64 From 285266a6aab88a210efbc02e2b67b068ee7d75b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 02:59:39 +0000 Subject: [PATCH 158/592] chore(deps): Bump docker/metadata-action from 4.6.0 to 5.0.0 (#3544) Bumps [docker/metadata-action](https://github.com/docker/metadata-action) from 4.6.0 to 5.0.0. - [Release notes](https://github.com/docker/metadata-action/releases) - [Upgrade guide](https://github.com/docker/metadata-action/blob/master/UPGRADE.md) - [Commits](https://github.com/docker/metadata-action/compare/v4.6.0...v5.0.0) --- updated-dependencies: - dependency-name: docker/metadata-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 4e32d190f4a..1b84a6da26b 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -23,7 +23,7 @@ jobs: - name: 'Prepare tags' id: prep - uses: docker/metadata-action@v4.6.0 + uses: docker/metadata-action@v5.0.0 with: images: | ${{ secrets.DOCKER_REPOSITORY }} From 539a7bc3bb92d237f2f3b190b6a33bacfa71e2bf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 03:01:59 +0000 Subject: [PATCH 159/592] chore(deps): Bump docker/login-action from 2 to 3 (#3543) Bumps [docker/login-action](https://github.com/docker/login-action) from 2 to 3. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 1b84a6da26b..5d68cd73d83 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -54,13 +54,13 @@ jobs: cache-buildx- - name: 'Login to DockerHub' - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: 'Login to GitHub Container Registry' - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} From 8c0cfa0836bcf20e78a2d5d1ae02270004ee0aed Mon Sep 17 00:00:00 2001 From: jpduyx Date: Tue, 19 Sep 2023 07:21:33 +0300 Subject: [PATCH 160/592] docs: Revise `update-and-cleanup.md` (#3539) * Update update-and-cleanup.md spotify dockergc is UNMAINTAINED, they advice to consider using the `docker system prune` command instead. "This repository has been archived by the owner on Feb 2, 2021. It is now read-only." https://github.com/spotify/docker-gc * Revise `update-and-cleanup.md` Merges the image update + cleanup sections. --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .../maintenance/update-and-cleanup.md | 55 ++++++++++--------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/docs/content/config/advanced/maintenance/update-and-cleanup.md b/docs/content/config/advanced/maintenance/update-and-cleanup.md index 3e4301f9364..b972c4a5446 100644 --- a/docs/content/config/advanced/maintenance/update-and-cleanup.md +++ b/docs/content/config/advanced/maintenance/update-and-cleanup.md @@ -2,40 +2,41 @@ title: 'Maintenance | Update and Cleanup' --- -## Automatic Update +[`containrrr/watchtower`][watchtower-dockerhub] is a service that monitors Docker images for updates, automatically applying them to running containers. -Docker images are handy but it can become a hassle to keep them updated. Also when a repository is automated you want to get these images when they get out. +!!! example "Automatic image updates + cleanup" -One could setup a complex action/hook-based workflow using probes, but there is a nice, easy to use docker image that solves this issue and could prove useful: [`watchtower`](https://hub.docker.com/r/containrrr/watchtower). + ```yaml title="compose.yaml" + services: + watchtower: + image: containrrr/watchtower:latest + # Automatic cleanup (removes older image pulls from wasting disk space): + environment: + - WATCHTOWER_CLEANUP=true + volumes: + - /var/run/docker.sock:/var/run/docker.sock + ``` -A Docker Compose example: +!!! tip "Updating only specific containers" -```yaml -services: - watchtower: - restart: always - image: containrrr/watchtower:latest - volumes: - - /var/run/docker.sock:/var/run/docker.sock -``` + The default `watchtower` service will check every 24 hours for any new image updates to pull, **not only the images** defined within your `compose.yaml`. -For more details, see the [manual](https://containrrr.github.io/watchtower/) + The images to update can be restricted with a custom command that provides a list of containers names and other config options. Configuration is detailed in the [`watchtower` docs][watchtower-docs]. -## Automatic Cleanup +!!! info "Manual cleanup" -When you are pulling new images in automatically, it would be nice to have them cleaned up as well. There is also a docker image for this: [`spotify/docker-gc`](https://hub.docker.com/r/spotify/docker-gc/). + `watchtower` also supports running on-demand with `docker run` or `compose.yaml` via the `--run-once` option. + + You can also directly invoke cleanup of Docker storage with: -A Docker Compose example: + - [`docker image prune --all`][docker-docs-prune-image] + - [`docker system prune --all`][docker-docs-prune-system] (_also removes unused containers, networks, build cache_). + - Avoid the `--all` option to only remove ["dangling" content][docker-prune-dangling] (_eg: Orphaned images_). -```yaml -services: - docker-gc: - restart: always - image: spotify/docker-gc:latest - volumes: - - /var/run/docker.sock:/var/run/docker.sock -``` +[watchtower-dockerhub]: https://hub.docker.com/r/containrrr/watchtower +[watchtower-cleanup]: https://containrrr.github.io/watchtower/arguments/#cleanup +[watchtower-docs]: https://containrrr.dev/watchtower/ -For more details, see the [manual](https://github.com/spotify/docker-gc/blob/master/README.md) - -Or you can just use the [`--cleanup`](https://containrrr.github.io/watchtower/arguments/#cleanup) option provided by `containrrr/watchtower`. +[docker-docs-prune-image]: https://docs.docker.com/engine/reference/commandline/image_prune/ +[docker-docs-prune-system]: https://docs.docker.com/engine/reference/commandline/system_prune/ +[docker-prune-dangling]: https://stackoverflow.com/questions/45142528/what-is-a-dangling-image-and-what-is-an-unused-image/60756668#60756668 From a9d6e329cd20b8e3cc68f31a12726dc56b1793e8 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 29 Sep 2023 01:37:15 +1300 Subject: [PATCH 161/592] tests(fix): `process_check_restart.bats` - Run `pgrep` within the actual container (#3553) This was missed during original review. On a linux host, processes running within a container have been visible via commands like `pgrep`. This is does not appear to be the case with WSL2 + Docker Desktop (Windows), resulting in test failure. The command should have been run from within the container regardless. --- .../set3/container_configuration/process_check_restart.bats | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/tests/parallel/set3/container_configuration/process_check_restart.bats b/test/tests/parallel/set3/container_configuration/process_check_restart.bats index 2c932bd26c4..b559d21d7f7 100644 --- a/test/tests/parallel/set3/container_configuration/process_check_restart.bats +++ b/test/tests/parallel/set3/container_configuration/process_check_restart.bats @@ -120,9 +120,9 @@ ENV_PROCESS_LIST=( # By this point the fetchmail processes have been verified to exist and restart, # For FETCHMAIL_PARALLEL=1 coverage, match full commandline for COUNTER values: - pgrep --full 'fetchmail-1.rc' + _run_in_container pgrep --full 'fetchmail-1.rc' assert_success - pgrep --full 'fetchmail-2.rc' + _run_in_container pgrep --full 'fetchmail-2.rc' assert_success _should_stop_cleanly From 89cb6d85b90f6070d53c15e455ebee916c899c7a Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 29 Sep 2023 10:17:57 +1300 Subject: [PATCH 162/592] tests(fix): `lmtp_ip.bats` improve partial failure output (#3552) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of exit status of `124` (_signifies timeout_), it should fail with `1` (failure) like the others. Handled via using `_run_in_container_bash()` (_`timeout` failure `124` does not propagate and is treated as `1` instead_). In this case we are waiting on the status of the mail being sent, the pattern provided to `grep` is too specific and results in a timeout. Instead since we only expect the one log entry, match any status and assert the expected pattern afterwards. This provides a more helpful failure output that informs us that mail was at least processed by Postfix, but the sent status is not what we expected. ### Before ``` ✗ [ENV] (POSTFIX_DAGENT) delivers mail to existing account [60327] (from function `assert_success' in file test/test_helper/bats-assert/src/assert_success.bash, line 42, in test file test/tests/parallel/set3/mta/lmtp_ip.bats, line 47) `assert_success' failed -- command failed -- status : 124 output : -- ``` ### After ``` ✗ [ENV] (POSTFIX_DAGENT) delivers mail to existing account [1425] (from function `assert_output' in file test/test_helper/bats-assert/src/assert_output.bash, line 178, in test file test/tests/parallel/set3/mta/lmtp_ip.bats, line 48) `assert_output --regexp "${MATCH_LOG_LINE}=sent .* Saved)"' failed -- regular expression does not match output -- regexp : postfix/lmtp.* status=sent .* Saved) output : Sep 28 04:12:52 mail postfix/lmtp[721]: 23701B575: to=, relay=127.0.0.1[127.0.0.1]:24, delay=0.08, delays=0.07/0/0.01/0, dsn=4.2.0, status=deferred (host 127.0.0.1[127.0.0.1] said: 451 4.2.0 Internal error occurred. Refer to server log for more information. [2023-09-28 04:12:52] (in reply to end of DATA command)) -- ``` The expected pattern is logged as `assert_success` confirms a valid match for the log line of interest was found, and we have the mismatched value to debug the failure against. --- test/tests/parallel/set3/mta/lmtp_ip.bats | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/tests/parallel/set3/mta/lmtp_ip.bats b/test/tests/parallel/set3/mta/lmtp_ip.bats index 861f7f60969..8d35c062750 100644 --- a/test/tests/parallel/set3/mta/lmtp_ip.bats +++ b/test/tests/parallel/set3/mta/lmtp_ip.bats @@ -42,7 +42,9 @@ function teardown_file() { _default_teardown ; } # Verify delivery was successful, log line should look similar to: # postfix/lmtp[1274]: 0EA424ABE7D9: to=, relay=127.0.0.1[127.0.0.1]:24, delay=0.13, delays=0.07/0.01/0.01/0.05, dsn=2.0.0, status=sent (250 2.0.0 ixPpB+Zvv2P7BAAAUi6ngw Saved) - local MATCH_LOG_LINE='postfix/lmtp.* status=sent .* Saved)' - run timeout 60 docker exec "${CONTAINER_NAME}" bash -c "tail -F /var/log/mail/mail.log | grep --max-count 1 '${MATCH_LOG_LINE}'" + local MATCH_LOG_LINE='postfix/lmtp.* status' + _run_in_container_bash "timeout 60 tail -F /var/log/mail/mail.log | grep --max-count 1 '${MATCH_LOG_LINE}'" assert_success + # Assertion of full pattern here (instead of via grep) is a bit more helpful for debugging partial failures: + assert_output --regexp "${MATCH_LOG_LINE}=sent .* Saved)" } From bd96c1161e9f9b32bb0b3aaec4b930980a56945f Mon Sep 17 00:00:00 2001 From: Vincent Ducamps Date: Sat, 30 Sep 2023 13:20:03 +0200 Subject: [PATCH 163/592] feat: Allow changing the Dovecot vmail UID/GID via ENV (#3550) Some deployment scenarios are not compatible with `5000:5000` static vmail user with `/var/mail`. This feature allows adjusting the defaults to a UID / GID that is compatible. Signed-off-by: vincent Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/config/environment.md | 12 ++++ mailserver.env | 6 ++ target/scripts/helpers/accounts.sh | 4 +- target/scripts/helpers/utils.sh | 4 +- target/scripts/start-mailserver.sh | 1 + target/scripts/startup/setup.d/vmail-id.sh | 12 ++++ target/scripts/startup/variables-stack.sh | 2 + test/tests/serial/vmail-id.bats | 75 ++++++++++++++++++++++ 8 files changed, 112 insertions(+), 4 deletions(-) create mode 100644 target/scripts/startup/setup.d/vmail-id.sh create mode 100644 test/tests/serial/vmail-id.bats diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index e5aba88941d..678ec9654d8 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -33,6 +33,18 @@ Here you can adjust the [log-level for Supervisor](http://supervisord.org/loggin The log-level will show everything in its class and above. +##### DMS_VMAIL_UID + +Default: 5000 + +The User ID assigned to the static vmail user for `/var/mail` (_Mail storage managed by Dovecot_). + +##### DMS_VMAIL_GID + +Default: 5000 + +The Group ID assigned to the static vmail group for `/var/mail` (_Mail storage managed by Dovecot_). + ##### ONE_DIR - 0 => state in default directories. diff --git a/mailserver.env b/mailserver.env index 632f6edff9a..1a57cecab62 100644 --- a/mailserver.env +++ b/mailserver.env @@ -34,6 +34,12 @@ SUPERVISOR_LOGLEVEL= # 1 => consolidate all states into a single directory (`/var/mail-state`) to allow persistence using docker volumes ONE_DIR=1 +# Support for deployment where these defaults are not compatible (eg: some NAS appliances): +# /var/mail vmail User ID (default: 5000) +DMS_VMAIL_UID= +# /var/mail vmail Group ID (default: 5000) +DMS_VMAIL_GID= + # **empty** => use FILE # LDAP => use LDAP authentication # OIDC => use OIDC authentication (not yet implemented) diff --git a/target/scripts/helpers/accounts.sh b/target/scripts/helpers/accounts.sh index 82b626c1511..7499a2ec8e5 100644 --- a/target/scripts/helpers/accounts.sh +++ b/target/scripts/helpers/accounts.sh @@ -70,7 +70,7 @@ function _create_accounts() { # Dovecot's userdb has the following format # user:password:uid:gid:(gecos):home:(shell):extra_fields - DOVECOT_USERDB_LINE="${LOGIN}:${PASS}:5000:5000::/var/mail/${DOMAIN}/${USER}/home::${USER_ATTRIBUTES}" + DOVECOT_USERDB_LINE="${LOGIN}:${PASS}:${DMS_VMAIL_UID}:${DMS_VMAIL_GID}::/var/mail/${DOMAIN}/${USER}/home::${USER_ATTRIBUTES}" if grep -qF "${DOVECOT_USERDB_LINE}" "${DOVECOT_USERDB_FILE}"; then _log 'warn' "Login '${LOGIN}' will not be added to '${DOVECOT_USERDB_FILE}' twice" else @@ -141,7 +141,7 @@ function _create_dovecot_alias_dummy_accounts() { fi fi - DOVECOT_USERDB_LINE="${ALIAS}:${REAL_ACC[1]}:5000:5000::/var/mail/${REAL_DOMAINNAME}/${REAL_USERNAME}::${REAL_ACC[2]:-}" + DOVECOT_USERDB_LINE="${ALIAS}:${REAL_ACC[1]}:${DMS_VMAIL_UID}:${DMS_VMAIL_GID}::/var/mail/${REAL_DOMAINNAME}/${REAL_USERNAME}::${REAL_ACC[2]:-}" if grep -qi "^${ALIAS}:" "${DOVECOT_USERDB_FILE}"; then _log 'warn' "Alias '${ALIAS}' will not be added to '${DOVECOT_USERDB_FILE}' twice" else diff --git a/target/scripts/helpers/utils.sh b/target/scripts/helpers/utils.sh index c74fd7e8fd9..e44f0aff577 100644 --- a/target/scripts/helpers/utils.sh +++ b/target/scripts/helpers/utils.sh @@ -39,9 +39,9 @@ function _get_dms_env_value() { # /var/mail folders (used during startup and change detection handling). function _chown_var_mail_if_necessary() { # fix permissions, but skip this if 3 levels deep the user id is already set - if find /var/mail -maxdepth 3 -a \( \! -user 5000 -o \! -group 5000 \) | read -r; then + if find /var/mail -maxdepth 3 -a \( \! -user "${DMS_VMAIL_UID}" -o \! -group "${DMS_VMAIL_GID}" \) | read -r; then _log 'trace' 'Fixing /var/mail permissions' - chown -R 5000:5000 /var/mail || return 1 + chown -R "${DMS_VMAIL_UID}:${DMS_VMAIL_GID}" /var/mail || return 1 fi } diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index e41ae71e75f..f0f385f3f6c 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -39,6 +39,7 @@ function _register_functions() { # ? >> Setup + _register_setup_function '_setup_vmail_id' _register_setup_function '_setup_logs_general' _register_setup_function '_setup_timezone' diff --git a/target/scripts/startup/setup.d/vmail-id.sh b/target/scripts/startup/setup.d/vmail-id.sh new file mode 100644 index 00000000000..2b692169afd --- /dev/null +++ b/target/scripts/startup/setup.d/vmail-id.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +function _setup_vmail_id() { + if [[ "${DMS_VMAIL_UID}" != "5000" ]]; then + _log 'debug' "Setting 'docker' UID to ${DMS_VMAIL_UID}" + usermod --uid "${DMS_VMAIL_UID}" docker + fi + if [[ "${DMS_VMAIL_GID}" != "5000" ]]; then + _log 'debug' "Setting 'docker' GID to ${DMS_VMAIL_GID}" + groupmod --gid "${DMS_VMAIL_GID}" docker + fi +} diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index b18a61dcdc0..3b575f506ea 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -46,6 +46,8 @@ function __environment_variables_general_setup() { VARS[POSTMASTER_ADDRESS]="${POSTMASTER_ADDRESS:=postmaster@${DOMAINNAME}}" VARS[REPORT_RECIPIENT]="${REPORT_RECIPIENT:=${POSTMASTER_ADDRESS}}" VARS[REPORT_SENDER]="${REPORT_SENDER:=mailserver-report@${HOSTNAME}}" + VARS[DMS_VMAIL_UID]="${DMS_VMAIL_UID:=5000}" + VARS[DMS_VMAIL_GID]="${DMS_VMAIL_GID:=5000}" _log 'trace' 'Setting anti-spam & anti-virus environment variables' diff --git a/test/tests/serial/vmail-id.bats b/test/tests/serial/vmail-id.bats new file mode 100644 index 00000000000..b44670b2081 --- /dev/null +++ b/test/tests/serial/vmail-id.bats @@ -0,0 +1,75 @@ +load "${REPOSITORY_ROOT}/test/helper/common" +load "${REPOSITORY_ROOT}/test/helper/setup" + +BATS_TEST_NAME_PREFIX='[ENV] (DMS_VMAIL_UID + DMS_VMAIL_GID) ' +CONTAINER_NAME='dms-test_env-change-vmail-id' + +function setup_file() { + _init_with_defaults + + local CUSTOM_SETUP_ARGUMENTS=( + --env PERMIT_DOCKER=container + --env DMS_VMAIL_UID=9042 + --env DMS_VMAIL_GID=9042 + ) + + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + _wait_for_smtp_port_in_container +} + +function teardown_file() { _default_teardown ; } + +@test 'should successfully deliver mail' { + _send_email 'email-templates/existing-user1' + _wait_for_empty_mail_queue_in_container + + # Should be successfully sent (received) by Postfix: + _run_in_container grep 'to=' /var/log/mail/mail.log + assert_success + assert_output --partial 'status=sent' + _should_output_number_of_lines 1 + + # Verify successful delivery via Dovecot to `/var/mail` account by searching for the subject: + _repeat_in_container_until_success_or_timeout 20 "${CONTAINER_NAME}" grep -R \ + 'Subject: Test Message existing-user1.txt' \ + '/var/mail/localhost.localdomain/user1/new/' + assert_success + _should_output_number_of_lines 1 +} + +# TODO: Migrate to test/helper/common.bash +# This test case is shared with tests.bats, but provides context on errors + some minor edits +# TODO: Could improve in future with keywords from https://github.com/docker-mailserver/docker-mailserver/pull/3550#issuecomment-1738509088 +# Potentially via a helper that allows an optional fixed number of errors to be present if they were intentional +@test '/var/log/mail/mail.log is error free' { + # Postfix: https://serverfault.com/questions/934703/postfix-451-4-3-0-temporary-lookup-failure + _run_in_container grep 'non-null host address bits in' /var/log/mail/mail.log + assert_failure + + # Postfix delivery failure: https://github.com/docker-mailserver/docker-mailserver/issues/230 + _run_in_container grep 'mail system configuration error' /var/log/mail/mail.log + assert_failure + + # Unknown error source: https://github.com/docker-mailserver/docker-mailserver/pull/85 + _run_in_container grep -i ': error:' /var/log/mail/mail.log + assert_failure + + # Unknown error source: https://github.com/docker-mailserver/docker-mailserver/pull/320 + _run_in_container grep -i 'not writable' /var/log/mail/mail.log + assert_failure + _run_in_container grep -i 'permission denied' /var/log/mail/mail.log + assert_failure + + # Amavis: https://forum.howtoforge.com/threads/postfix-smtp-error-caused-by-clamav-cant-connect-to-a-unix-socket-var-run-clamav-clamd-ctl.81002/ + _run_in_container grep -i '(!)connect' /var/log/mail/mail.log + assert_failure + + # Postfix: https://github.com/docker-mailserver/docker-mailserver/pull/2597 + _run_in_container grep -i 'using backwards-compatible default setting' /var/log/mail/mail.log + assert_failure + + # Postgrey: https://github.com/docker-mailserver/docker-mailserver/pull/612#discussion_r117635774 + _run_in_container grep -i 'connect to 127.0.0.1:10023: Connection refused' /var/log/mail/mail.log + assert_failure +} + From aae42fae9bd0190c7b4326e3855d8492ecd61dfa Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 4 Oct 2023 23:53:32 +1300 Subject: [PATCH 164/592] ci(fix): Normalize for `.gitattributes` + improve `eclint` coverage (#3566) --- .editorconfig | 15 +- .github/pull_request_template.md | 6 +- CHANGELOG.md | 7 +- Dockerfile | 2 - .../advanced/dovecot-master-accounts.md | 2 +- docs/content/config/advanced/ipv6.md | 2 +- docs/content/config/advanced/mail-sieve.md | 12 +- docs/content/config/security/ssl.md | 35 ++-- docs/content/contributing/tests.md | 40 ++--- docs/content/faq.md | 5 +- docs/content/introduction.md | 40 ++--- target/amavis/conf.d/60-dms_default_config | 2 +- target/bin/open-dkim | 4 +- target/postgrey/postgrey.init | 154 ------------------ .../keys/localhost.localdomain/mail.txt | 2 +- .../keys/otherdomain.tld/mail.txt | 2 +- test/config/fetchmail/fetchmail.cf | 16 +- .../ldap/openldap/schemas/postfix-book.ldif | 28 ++-- test/config/override-configs/postfix-main.cf | 2 +- .../config/override-configs/postfix-master.cf | 2 +- test/config/postfix-regexp.cf | 3 +- test/config/postfix-virtual.cf | 2 +- test/config/templates/dovecot-masters.cf | 2 +- test/config/templates/postfix-accounts.cf | 8 +- test/linting/.ecrc.json | 26 +-- .../parallel/set1/spam_virus/amavis.bats | 2 +- test/tests/serial/mail_with_ldap.bats | 2 +- 27 files changed, 122 insertions(+), 301 deletions(-) delete mode 100644 target/postgrey/postgrey.init diff --git a/.editorconfig b/.editorconfig index fd68e2f2b88..f5f1e31766a 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,6 +8,7 @@ root = true [*] charset = utf-8 end_of_line = lf +indent_size = 2 indent_style = space insert_final_newline = true trim_trailing_whitespace = true @@ -16,21 +17,9 @@ trim_trailing_whitespace = true # --- Specific ---------------------------------- # ----------------------------------------------- -[*.{yaml,yml,sh,bats}] -indent_size = 2 - -[Makefile] +[{Makefile,.gitmodules}] indent_style = tab indent_size = 4 [*.md] trim_trailing_whitespace = false - -# ----------------------------------------------- -# --- Git Submodules ---------------------------- -# ----------------------------------------------- - -[{test/bats/**,test/test_helper/**}] -indent_style = none -indent_size = none -end_of_line = none diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index bcdf248a1ef..26dc801e45d 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,7 +1,9 @@ # Description - + Fixes # diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b22fec56b8..8ce7b6fb7a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -311,8 +311,8 @@ In this release the relay-host support saw [significant internal refactoring](ht 1. **Many** minor improvements were made (cleanup & refactoring). Please refer to the section below to get an overview over all improvements. Moreover, there was a lot of cleanup in the scripts and in the tests. The documentation was adjusted accordingly. 2. New environment variables were added: - 1. [`CLAMAV_MESSAGE_SIZE_LIMIT`](https://docker-mailserver.github.io/docker-mailserver/v11.0/config/environment/#clamav_message_size_limit) - 2. [`TZ`](https://docker-mailserver.github.io/docker-mailserver/v11.0/config/environment/#tz) + 1. [`CLAMAV_MESSAGE_SIZE_LIMIT`](https://docker-mailserver.github.io/docker-mailserver/v11.0/config/environment/#clamav_message_size_limit) + 2. [`TZ`](https://docker-mailserver.github.io/docker-mailserver/v11.0/config/environment/#tz) 3. SpamAssassin KAM was added with [`ENABLE_SPAMASSASSIN_KAM`](https://docker-mailserver.github.io/docker-mailserver/v11.0/config/environment/#enable_spamassassin_kam). 4. The `fail2ban` command was reworked and can now ban IP addresses as well. 5. There were a few small fixes, especially when it comes to bugs in scripts and service restart loops (no functionality changes, only fixes of existing functionality). When building an image from the Dockerfile - Installation of Postfix on modern Linux distributions should now always succeed. @@ -368,8 +368,7 @@ In this release the relay-host support saw [significant internal refactoring](ht ### Critical Changes -1. This release fixes a critical issue for LDAP users, installing a needed package on Debian 11 - on build-time. Moreover, a race-condition was eliminated ([#2341](https://github.com/docker-mailserver/docker-mailserver/pull/2341)). +1. This release fixes a critical issue for LDAP users, installing a needed package on Debian 11 on build-time. Moreover, a race-condition was eliminated ([#2341](https://github.com/docker-mailserver/docker-mailserver/pull/2341)). 2. A resource leak in `check-for-changes.sh` was fixed ([#2401](https://github.com/docker-mailserver/docker-mailserver/pull/2401)) ### Other Minor Changes diff --git a/Dockerfile b/Dockerfile index b5c5ba395bd..5e12689da60 100644 --- a/Dockerfile +++ b/Dockerfile @@ -134,9 +134,7 @@ EOF COPY target/postsrsd/postsrsd /etc/default/postsrsd COPY target/postgrey/postgrey /etc/default/postgrey -COPY target/postgrey/postgrey.init /etc/init.d/postgrey RUN <*` -Password: `` \ No newline at end of file +Password: `` diff --git a/docs/content/config/advanced/ipv6.md b/docs/content/config/advanced/ipv6.md index 69ed069bd1e..f035523a3f2 100644 --- a/docs/content/config/advanced/ipv6.md +++ b/docs/content/config/advanced/ipv6.md @@ -132,7 +132,7 @@ Next, configure a network with an IPv6 subnet for your container with any of the !!! warning "This approach is discouraged" - The [`bridge` network is considered legacy][docker-docs-network-bridge-legacy]. + The [`bridge` network is considered legacy][docker-docs-network-bridge-legacy]. Add these two extra IPv6 settings to your daemon config. They only apply to the [default `bridge` docker network][docker-docs-ipv6-create-default] aka `docker0` (_which containers are attached to by default when using `docker run`_). diff --git a/docs/content/config/advanced/mail-sieve.md b/docs/content/config/advanced/mail-sieve.md index d050bf9fc5e..fe540efb9b2 100644 --- a/docs/content/config/advanced/mail-sieve.md +++ b/docs/content/config/advanced/mail-sieve.md @@ -69,12 +69,12 @@ It is possible to sort subaddresses such as `user+mailing-lists@example.com` int require ["envelope", "fileinto", "mailbox", "subaddress", "variables"]; if envelope :detail :matches "to" "*" { - set :lower :upperfirst "tag" "${1}"; - if mailboxexists "INBOX.${1}" { - fileinto "INBOX.${1}"; - } else { - fileinto :create "INBOX.${tag}"; - } + set :lower :upperfirst "tag" "${1}"; + if mailboxexists "INBOX.${1}" { + fileinto "INBOX.${1}"; + } else { + fileinto :create "INBOX.${tag}"; + } } ``` diff --git a/docs/content/config/security/ssl.md b/docs/content/config/security/ssl.md index 4618fda1bb2..c0c615cc14a 100644 --- a/docs/content/config/security/ssl.md +++ b/docs/content/config/security/ssl.md @@ -161,8 +161,9 @@ Obtain a Cloudflare API token: dns_cloudflare_api_token = YOUR_CLOUDFLARE_TOKEN_HERE ``` - - As this is sensitive data, you should restrict access to it with `chmod 600` and `chown 0:0`. - - Store the file in a folder if you like, such as `docker-data/certbot/secrets/`. + - As this is sensitive data, you should restrict access to it with `chmod 600` and `chown 0:0`. + - Store the file in a folder if you like, such as `docker-data/certbot/secrets/`. + 5. Your `compose.yaml` should include the following: ```yaml @@ -594,7 +595,7 @@ This setup only comes with one caveat: The domain has to be configured on anothe container_name: mailserver hostname: mail.example.com volumes: - - ./docker-data/traefik/acme.json:/etc/letsencrypt/acme.json:ro + - ./docker-data/traefik/acme.json:/etc/letsencrypt/acme.json:ro environment: SSL_TYPE: letsencrypt SSL_DOMAIN: mail.example.com @@ -605,26 +606,26 @@ This setup only comes with one caveat: The domain has to be configured on anothe image: docker.io/traefik:latest #v2.5 container_name: docker-traefik ports: - - "80:80" - - "443:443" + - "80:80" + - "443:443" command: - - --providers.docker - - --entrypoints.http.address=:80 - - --entrypoints.http.http.redirections.entryPoint.to=https - - --entrypoints.http.http.redirections.entryPoint.scheme=https - - --entrypoints.https.address=:443 - - --entrypoints.https.http.tls.certResolver=letsencrypt - - --certificatesresolvers.letsencrypt.acme.email=admin@example.com - - --certificatesresolvers.letsencrypt.acme.storage=/acme.json - - --certificatesresolvers.letsencrypt.acme.httpchallenge.entrypoint=http + - --providers.docker + - --entrypoints.http.address=:80 + - --entrypoints.http.http.redirections.entryPoint.to=https + - --entrypoints.http.http.redirections.entryPoint.scheme=https + - --entrypoints.https.address=:443 + - --entrypoints.https.http.tls.certResolver=letsencrypt + - --certificatesresolvers.letsencrypt.acme.email=admin@example.com + - --certificatesresolvers.letsencrypt.acme.storage=/acme.json + - --certificatesresolvers.letsencrypt.acme.httpchallenge.entrypoint=http volumes: - - ./docker-data/traefik/acme.json:/acme.json - - /var/run/docker.sock:/var/run/docker.sock:ro + - ./docker-data/traefik/acme.json:/acme.json + - /var/run/docker.sock:/var/run/docker.sock:ro whoami: image: docker.io/traefik/whoami:latest labels: - - "traefik.http.routers.whoami.rule=Host(`mail.example.com`)" + - "traefik.http.routers.whoami.rule=Host(`mail.example.com`)" ``` ### Self-Signed Certificates diff --git a/docs/content/contributing/tests.md b/docs/content/contributing/tests.md index ba63bf74c43..6f6495299f3 100644 --- a/docs/content/contributing/tests.md +++ b/docs/content/contributing/tests.md @@ -85,10 +85,10 @@ In this example, you've made a change to the Rspamd feature support (_or adjuste ```console $ make clean generate-accounts test/rspamd rspamd.bats - ✓ [Rspamd] Postfix's main.cf was adjusted [12] - ✓ [Rspamd] normal mail passes fine [44] - ✓ [Rspamd] detects and rejects spam [122] - ✓ [Rspamd] detects and rejects virus [189] + ✓ [Rspamd] Postfix's main.cf was adjusted [12] + ✓ [Rspamd] normal mail passes fine [44] + ✓ [Rspamd] detects and rejects spam [122] + ✓ [Rspamd] detects and rejects virus [189] ``` As your feature work progresses your change for Rspamd also affects ClamAV. As your change now spans more than just the Rspamd test file, you could run multiple test files serially: @@ -96,16 +96,17 @@ As your feature work progresses your change for Rspamd also affects ClamAV. As y ```console $ make clean generate-accounts test/rspamd,clamav rspamd.bats - ✓ [Rspamd] Postfix's main.cf was adjusted [12] - ✓ [Rspamd] normal mail passes fine [44] - ✓ [Rspamd] detects and rejects spam [122] - ✓ [Rspamd] detects and rejects virus [189] + ✓ [Rspamd] Postfix's main.cf was adjusted [12] + ✓ [Rspamd] normal mail passes fine [44] + ✓ [Rspamd] detects and rejects spam [122] + ✓ [Rspamd] detects and rejects virus [189] + clamav.bats - ✓ [ClamAV] log files exist at /var/log/mail directory [68] - ✓ [ClamAV] should be identified by Amavis [67] - ✓ [ClamAV] freshclam cron is enabled [76] - ✓ [ClamAV] env CLAMAV_MESSAGE_SIZE_LIMIT is set correctly [63] - ✓ [ClamAV] rejects virus [60] + ✓ [ClamAV] log files exist at /var/log/mail directory [68] + ✓ [ClamAV] should be identified by Amavis [67] + ✓ [ClamAV] freshclam cron is enabled [76] + ✓ [ClamAV] env CLAMAV_MESSAGE_SIZE_LIMIT is set correctly [63] + ✓ [ClamAV] rejects virus [60] ``` You're almost finished with your change before submitting it as a PR. It's a good idea to run the full parallel set those individual tests belong to (_especially if you've modified any tests_): @@ -113,13 +114,15 @@ You're almost finished with your change before submitting it as a PR. It's a goo ```console $ make clean generate-accounts tests/parallel/set1 default_relay_host.bats - ✓ [Relay] (ENV) 'DEFAULT_RELAY_HOST' should configure 'main.cf:relayhost' [88] + ✓ [Relay] (ENV) 'DEFAULT_RELAY_HOST' should configure 'main.cf:relayhost' [88] + spam_virus/amavis.bats - ✓ [Amavis] SpamAssassin integration should be active [1165] + ✓ [Amavis] SpamAssassin integration should be active [1165] + spam_virus/clamav.bats - ✓ [ClamAV] log files exist at /var/log/mail directory [73] - ✓ [ClamAV] should be identified by Amavis [67] - ✓ [ClamAV] freshclam cron is enabled [76] + ✓ [ClamAV] log files exist at /var/log/mail directory [73] + ✓ [ClamAV] should be identified by Amavis [67] + ✓ [ClamAV] freshclam cron is enabled [76] ... ``` @@ -127,7 +130,6 @@ Even better, before opening a PR run the full test suite: ```console $ make clean tests -... ``` [BATS]: https://github.com/bats-core/bats-core diff --git a/docs/content/faq.md b/docs/content/faq.md index bb2fbe6382b..4da64b60081 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -511,8 +511,9 @@ require ["comparator-i;ascii-numeric","relational","fileinto"]; if header :contains "X-Spam-Flag" "YES" { fileinto "Junk"; } elsif allof ( - not header :matches "x-spam-score" "-*", - header :value "ge" :comparator "i;ascii-numeric" "x-spam-score" "3.75" ) { + not header :matches "x-spam-score" "-*", + header :value "ge" :comparator "i;ascii-numeric" "x-spam-score" "3.75" +) { fileinto "Junk"; } ``` diff --git a/docs/content/introduction.md b/docs/content/introduction.md index bbfd6ef41ed..3be9ed9de32 100644 --- a/docs/content/introduction.md +++ b/docs/content/introduction.md @@ -43,10 +43,10 @@ Here's where DMS's toolchain fits within the delivery chain: ```txt docker-mailserver is here: - ┏━━━━━━━┓ -Sending an email: MUA ---> MTA ---> (MTA relays) ---> ┫ MTA ╮ ┃ -Fetching an email: MUA <------------------------------ ┫ MDA ╯ ┃ - ┗━━━━━━━┛ + ┏━━━━━━━┓ +Sending an email: MUA ---> MTA ---> (MTA relays) ---> ┫ MTA ╮ ┃ +Fetching an email: MUA <------------------------------ ┫ MDA ╯ ┃ + ┗━━━━━━━┛ ``` ??? example "An Example" @@ -86,18 +86,18 @@ When it comes to the specifics of email exchange, we have to look at protocols a The following picture gives a visualization of the interplay of all components and their [respective ports][docs-understandports]: ```txt - ┏━━━━━━━━━━ Submission ━━━━━━━━━━━━┓┏━━━━━━━━━━━━━ Transfer/Relay ━━━━━━━━━━━┓ - - ┌─────────────────────┐ ┌┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┐ -MUA ----- STARTTLS ------> ┤(587) MTA ╮ (25)├ <-- cleartext ---> ┊ Third-party MTA ┊ - ----- implicit TLS --> ┤(465) │ | └┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┘ - ----- cleartext -----> ┤(25) │ | - |┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄| -MUA <---- STARTTLS ------- ┤(143) MDA ╯ | - <---- implicit TLS --- ┤(993) | - └─────────────────────┘ - - ┗━━━━━━━━━━ Retrieval ━━━━━━━━━━━━━┛ + ┏━━━━━━━━━━ Submission ━━━━━━━━━━━━━┓┏━━━━━━━━━━━━━ Transfer/Relay ━━━━━━━━━━━┓ + + ┌─────────────────────┐ ┌┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┐ +MUA ----- STARTTLS -------> ┤(587) MTA ╮ (25)├ <-- cleartext ---> ┊ Third-party MTA ┊ + ----- implicit TLS ---> ┤(465) │ | └┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┘ + ----- cleartext ------> ┤(25) │ | + |┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄| +MUA <---- STARTTLS -------- ┤(143) MDA ╯ | + <---- implicit TLS ---- ┤(993) | + └─────────────────────┘ + + ┗━━━━━━━━━━ Retrieval ━━━━━━━━━━━━━━┛ ``` If you're new to email infrastructure, both that table and the schema may be confusing. @@ -124,7 +124,7 @@ My MTA will thus have to support two kinds of Submission: - Inbound Submission (third-party email has been submitted & relayed, then is accepted "inside" by the MTA) ```txt - ┏━━━━ Outbound Submission ━━━━┓ + ┏━━━ Outbound Submission ━━━┓ ┌────────────────────┐ ┌┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┐ Me ---------------> ┤ ├ -----------------> ┊ ┊ @@ -132,7 +132,7 @@ Me ---------------> ┤ ├ -----------------> ┊ │ ├ <----------------- ┊ ┊ └────────────────────┘ └┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┘ - ┗━━━━━━━━━━ Inbound Submission ━━━━━━━━━━┛ + ┗━━━━━━━━━━ Inbound Submission ━━━━━━━━━━┛ ``` #### Outbound Submission @@ -168,7 +168,7 @@ Granted it's still very difficult enforcing encryption between MTAs (Transfer/Re Overall, DMS's default configuration for SMTP looks like this: ```txt - ┏━━━━ Outbound Submission ━━━━┓ + ┏━━━ Outbound Submission ━━━┓ ┌────────────────────┐ ┌┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┐ Me -- cleartext --> ┤(25) (25)├ --- cleartext ---> ┊ ┊ @@ -177,7 +177,7 @@ Me -- STARTTLS ---> ┤(587) │ ┊ │ (25)├ <---cleartext ---- ┊ ┊ └────────────────────┘ └┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┄┘ - ┗━━━━━━━━━━ Inbound Submission ━━━━━━━━━━┛ + ┗━━━━━━━━━━ Inbound Submission ━━━━━━━━━━┛ ``` ### Retrieval - IMAP diff --git a/target/amavis/conf.d/60-dms_default_config b/target/amavis/conf.d/60-dms_default_config index e5b71e2f68e..5a22b3ada57 100644 --- a/target/amavis/conf.d/60-dms_default_config +++ b/target/amavis/conf.d/60-dms_default_config @@ -1,7 +1,7 @@ use strict; @local_domains_maps = ( - read_hash('/etc/postfix/vhost') + read_hash('/etc/postfix/vhost') ); 1; # ensure a defined return diff --git a/target/bin/open-dkim b/target/bin/open-dkim index 11b9897554b..86fbfb8181a 100755 --- a/target/bin/open-dkim +++ b/target/bin/open-dkim @@ -42,8 +42,8 @@ ${ORANGE}OPTIONS${RESET} selector Set a manual selector for the key. Default: mail domain Provide the domain(s) for which to generate keys for. - Default: The FQDN assigned to DMS, excluding any subdomain. - 'ACCOUNT_PROVISIONER=FILE' also sources domains from mail accounts. + Default: The FQDN assigned to DMS, excluding any subdomain. + 'ACCOUNT_PROVISIONER=FILE' also sources domains from mail accounts. ${ORANGE}EXAMPLES${RESET} ${LWHITE}setup config dkim keysize 4096${RESET} diff --git a/target/postgrey/postgrey.init b/target/postgrey/postgrey.init deleted file mode 100644 index 0ac7564b308..00000000000 --- a/target/postgrey/postgrey.init +++ /dev/null @@ -1,154 +0,0 @@ -#! /bin/sh - -# postgrey start/stop the postgrey greylisting deamon for postfix -# (priority should be smaller than that of postfix) -# -# Author: (c)2004-2006 Adrian von Bidder -# Based on Debian sarge's 'skeleton' example -# Distribute and/or modify at will. -# -# Version: $Id: postgrey.init 1436 2006-12-07 07:15:03Z avbidder $ -# altered by Georg Lauterbach as aendeavor 2020-11.05 14:02:00Z - -### BEGIN INIT INFO -# Provides: postgrey -# Required-Start: $syslog $local_fs $remote_fs -# Required-Stop: $syslog $local_fs $remote_fs -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Start/stop the postgrey daemon -### END INIT INFO - -set -e - -PATH='/sbin:/bin:/usr/sbin:/usr/bin' -DAEMON='/usr/sbin/postgrey' -DAEMON_NAME='postgrey' -DESC='postfix greylisting daemon' -DAEMON_USER='postgrey' - -PIDFILE="/var/run/${DAEMON_NAME}/${DAEMON_NAME}.pid" -SCRIPTNAME="/etc/init.d/${DAEMON_NAME}" - -# gracefully exit if the package has been removed. -[ -x "${DAEMON}" ] || exit 0 - -# shellcheck source=/dev/null -. /lib/lsb/init-functions - -# Read config file if it is present. -# shellcheck source=/dev/null -[ -r "/etc/default/${DAEMON_NAME}" ] && . "/etc/default/${DAEMON_NAME}" - -POSTGREY_OPTS="--pidfile=${PIDFILE} --daemonize ${POSTGREY_OPTS}" - -if [ -z "${POSTGREY_TEXT}" ]; then - POSTGREY_TEXT_OPT="" -else - POSTGREY_TEXT_OPT="--greylist-text=${POSTGREY_TEXT}" -fi - -ret=0 - -do_start() -{ - # Return - # 0 if daemon has been started - # 1 if daemon was already running - # 2 if daemon could not be started - start-stop-daemon --start --quiet --pidfile \ - "${PIDFILE}" --exec "${DAEMON}" --test >/dev/null || return 1 - - start-stop-daemon --start --quiet --pidfile \ - "${PIDFILE}" --exec "${DAEMON}" -- "${POSTGREY_OPTS}" \ - "${POSTGREY_TEXT_OPT}" || return 2 -} - -do_stop() -{ - # Return - # 0 if daemon has been stopped - # 1 if daemon was already stopped - # 2 if daemon could not be stopped - # other if a failure occurred - start-stop-daemon --user "${DAEMON_USER}" --stop --quiet \ - --retry=TERM/30/KILL/5 --pidfile "${PIDFILE}" - - RETVAL="$?" - [ "${RETVAL}" -eq 2 ] && return 2 - - # Wait for children to finish too if this is a daemon that forks - # and if the daemon is only ever run from this initscript. - # If the above conditions are not satisfied then add some other code - # that waits for the process to drop all resources that could be - # needed by services started subsequently. A last resort is to - # sleep for some time. - start-stop-daemon --user "${DAEMON_USER}" --stop --quiet \ - --oknodo --retry=0/30/KILL/5 --exec "${DAEMON}" - [ "$?" -eq 2 ] && return 2 - - # Many daemons don't delete their pidfiles when they exit. - rm -f "${PIDFILE}" - return "${RETVAL}" -} - -do_reload() -{ - # - # If the daemon can reload its configuration without - # restarting (for example, when it is sent a SIGHUP), - # then implement that here. - # - start-stop-daemon --stop --signal 1 --quiet --pidfile "${PIDFILE}" - return 0 -} - -case "${1}" in - start ) - [ "${VERBOSE}" != no ] && log_daemon_msg "Starting ${DESC}" "${DAEMON_NAME}" - do_start - - case "${?}" in - 0|1) [ "${VERBOSE}" != no ] && log_end_msg 0 ;; - 2) [ "${VERBOSE}" != no ] && log_end_msg 1 ;; - esac - ;; - - stop ) - [ "${VERBOSE}" != no ] && log_daemon_msg "Stopping ${DESC}" "${DAEMON_NAME}" - do_stop - - case "${?}" in - 0|1) [ "${VERBOSE}" != no ] && log_end_msg 0 ;; - 2) [ "${VERBOSE}" != no ] && log_end_msg 1 ;; - esac - ;; - - reload|force-reload) - [ "${VERBOSE}" != no ] && log_daemon_msg "Reloading ${DESC}" "${DAEMON_NAME}" - do_reload - - case "${?}" in - 0|1) [ "${VERBOSE}" != no ] && log_end_msg 0 ;; - 2) [ "${VERBOSE}" != no ] && log_end_msg 1 ;; - esac - ;; - - restart ) - do_stop - do_start - ;; - - status ) - status_of_proc -p "${PIDFILE}" "${DAEMON}" "${DAEMON_NAME}" 2>/dev/null - ret=${?} - ;; - - * ) - echo "Usage: ${SCRIPTNAME} {start|stop|restart|reload|force-reload|status}" >&2 - exit 1 - ;; -esac - -exit ${ret} - diff --git a/test/config/example-opendkim/keys/localhost.localdomain/mail.txt b/test/config/example-opendkim/keys/localhost.localdomain/mail.txt index ccc08dc0195..e9c8cd1a173 100644 --- a/test/config/example-opendkim/keys/localhost.localdomain/mail.txt +++ b/test/config/example-opendkim/keys/localhost.localdomain/mail.txt @@ -1,2 +1,2 @@ mail._domainkey IN TXT ( "v=DKIM1; k=rsa; " - "p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCzUJyyhq+TeT1wlIth5Z0yr7Ohd62n4rL5X3vRJO4EDyOEicJ73cjuaU4JLTYhbqmbNalOyXE9btS9I55Gv3RyomVBD1JpVTKdjVBUQug2L/ggw2dtt1FAn99svQWMs1XxmxiTR+sCEVkgKMmLSkCJuDCIfY/Bc9nlcng9+juB8wIDAQAB" ) ; ----- DKIM key mail for localhost.localdomain + "p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCzUJyyhq+TeT1wlIth5Z0yr7Ohd62n4rL5X3vRJO4EDyOEicJ73cjuaU4JLTYhbqmbNalOyXE9btS9I55Gv3RyomVBD1JpVTKdjVBUQug2L/ggw2dtt1FAn99svQWMs1XxmxiTR+sCEVkgKMmLSkCJuDCIfY/Bc9nlcng9+juB8wIDAQAB" ) ; ----- DKIM key mail for localhost.localdomain diff --git a/test/config/example-opendkim/keys/otherdomain.tld/mail.txt b/test/config/example-opendkim/keys/otherdomain.tld/mail.txt index d132a31c1cd..9d1079f4eb3 100644 --- a/test/config/example-opendkim/keys/otherdomain.tld/mail.txt +++ b/test/config/example-opendkim/keys/otherdomain.tld/mail.txt @@ -1,2 +1,2 @@ mail._domainkey IN TXT ( "v=DKIM1; k=rsa; " - "p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCurRsOh4NyTOqDnpPlPLGlQDuoQl32Gdkfzw7BBRKDcelIZBmQf0uhXKSZVKe5Q596w/3ESJ9WOlB03SISnHy8lq/ZJ1+vhSZQfHvp0cHQl4BgNzktRCARdPY+5nVerF8aUSsT3bG2O+2r09AY4okLCVfkiwg6Nz2Eo7j4Z7mqNwIDAQAB" ) ; ----- DKIM key mail for otherdomain.tld + "p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCurRsOh4NyTOqDnpPlPLGlQDuoQl32Gdkfzw7BBRKDcelIZBmQf0uhXKSZVKe5Q596w/3ESJ9WOlB03SISnHy8lq/ZJ1+vhSZQfHvp0cHQl4BgNzktRCARdPY+5nVerF8aUSsT3bG2O+2r09AY4okLCVfkiwg6Nz2Eo7j4Z7mqNwIDAQAB" ) ; ----- DKIM key mail for otherdomain.tld diff --git a/test/config/fetchmail/fetchmail.cf b/test/config/fetchmail/fetchmail.cf index aead698cd13..11168505046 100644 --- a/test/config/fetchmail/fetchmail.cf +++ b/test/config/fetchmail/fetchmail.cf @@ -1,11 +1,11 @@ poll pop3.third-party.test. with proto POP3 - user 'remote_username' there with - password 'secret' - is 'local_username' here - options keep ssl + user 'remote_username' there with + password 'secret' + is 'local_username' here + options keep ssl poll imap.remote-service.test. with proto IMAP - user 'user3' there with - password 'secret' - is 'user3@example.test' here - options keep ssl + user 'user3' there with + password 'secret' + is 'user3@example.test' here + options keep ssl diff --git a/test/config/ldap/openldap/schemas/postfix-book.ldif b/test/config/ldap/openldap/schemas/postfix-book.ldif index 9ea787d505f..543dc61a051 100644 --- a/test/config/ldap/openldap/schemas/postfix-book.ldif +++ b/test/config/ldap/openldap/schemas/postfix-book.ldif @@ -1,14 +1,14 @@ -dn: cn=postfix-book,cn=schema,cn=config -objectClass: olcSchemaConfig -cn: postfix-book -olcAttributeTypes: {0}( 1.3.6.1.4.1.29426.1.10.1 NAME 'mailHomeDirectory' DESC 'The absolute path to the mail user home directory' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) -olcAttributeTypes: {1}( 1.3.6.1.4.1.29426.1.10.2 NAME 'mailAlias' DESC 'RFC822 Mailbox - mail alias' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} ) -olcAttributeTypes: {2}( 1.3.6.1.4.1.29426.1.10.3 NAME 'mailUidNumber' DESC 'UID required to access the mailbox' EQUALITY integerMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE ) -olcAttributeTypes: {3}( 1.3.6.1.4.1.29426.1.10.4 NAME 'mailGidNumber' DESC 'GID required to access the mailbox' EQUALITY integerMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE ) -olcAttributeTypes: {4}( 1.3.6.1.4.1.29426.1.10.5 NAME 'mailEnabled' DESC 'TRUE to enable, FALSE to disable account' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) -olcAttributeTypes: {5}( 1.3.6.1.4.1.29426.1.10.6 NAME 'mailGroupMember' DESC 'Name of a mail distribution list' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) -olcAttributeTypes: {6}( 1.3.6.1.4.1.29426.1.10.7 NAME 'mailQuota' DESC 'Mail quota limit in kilobytes' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) -olcAttributeTypes: {7}( 1.3.6.1.4.1.29426.1.10.8 NAME 'mailStorageDirectory' DESC 'The absolute path to the mail users mailbox' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) -# PostfixBook object classes: -olcObjectClasses: {0}( 1.3.6.1.4.1.29426.1.2.2.1 NAME 'PostfixBookMailAccount' DESC 'Mail account used in Postfix Book' SUP top AUXILIARY MUST mail MAY ( mailHomeDirectory $ mailAlias $ mailGroupMember $ mailUidNumber $ mailGidNumber $ mailEnabled $ mailQuota $ mailStorageDirectory ) ) -olcObjectClasses: {1}( 1.3.6.1.4.1.29426.1.2.2.2 NAME 'PostfixBookMailForward' DESC 'Mail forward used in Postfix Book' SUP top AUXILIARY MUST ( mail $ mailAlias ) ) +dn: cn=postfix-book,cn=schema,cn=config +objectClass: olcSchemaConfig +cn: postfix-book +olcAttributeTypes: {0}( 1.3.6.1.4.1.29426.1.10.1 NAME 'mailHomeDirectory' DESC 'The absolute path to the mail user home directory' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) +olcAttributeTypes: {1}( 1.3.6.1.4.1.29426.1.10.2 NAME 'mailAlias' DESC 'RFC822 Mailbox - mail alias' EQUALITY caseIgnoreIA5Match SUBSTR caseIgnoreIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26{256} ) +olcAttributeTypes: {2}( 1.3.6.1.4.1.29426.1.10.3 NAME 'mailUidNumber' DESC 'UID required to access the mailbox' EQUALITY integerMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE ) +olcAttributeTypes: {3}( 1.3.6.1.4.1.29426.1.10.4 NAME 'mailGidNumber' DESC 'GID required to access the mailbox' EQUALITY integerMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.27 SINGLE-VALUE ) +olcAttributeTypes: {4}( 1.3.6.1.4.1.29426.1.10.5 NAME 'mailEnabled' DESC 'TRUE to enable, FALSE to disable account' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) +olcAttributeTypes: {5}( 1.3.6.1.4.1.29426.1.10.6 NAME 'mailGroupMember' DESC 'Name of a mail distribution list' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) +olcAttributeTypes: {6}( 1.3.6.1.4.1.29426.1.10.7 NAME 'mailQuota' DESC 'Mail quota limit in kilobytes' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) +olcAttributeTypes: {7}( 1.3.6.1.4.1.29426.1.10.8 NAME 'mailStorageDirectory' DESC 'The absolute path to the mail users mailbox' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 SINGLE-VALUE ) +# PostfixBook object classes: +olcObjectClasses: {0}( 1.3.6.1.4.1.29426.1.2.2.1 NAME 'PostfixBookMailAccount' DESC 'Mail account used in Postfix Book' SUP top AUXILIARY MUST mail MAY ( mailHomeDirectory $ mailAlias $ mailGroupMember $ mailUidNumber $ mailGidNumber $ mailEnabled $ mailQuota $ mailStorageDirectory ) ) +olcObjectClasses: {1}( 1.3.6.1.4.1.29426.1.2.2.2 NAME 'PostfixBookMailForward' DESC 'Mail forward used in Postfix Book' SUP top AUXILIARY MUST ( mail $ mailAlias ) ) diff --git a/test/config/override-configs/postfix-main.cf b/test/config/override-configs/postfix-main.cf index 77c2d787c32..ce07bd56eb6 100644 --- a/test/config/override-configs/postfix-main.cf +++ b/test/config/override-configs/postfix-main.cf @@ -1,4 +1,4 @@ max_idle = 600s # this is a comment - # this is also a comment + # this is also a comment readme_directory = /tmp diff --git a/test/config/override-configs/postfix-master.cf b/test/config/override-configs/postfix-master.cf index 4ebc7d21fd7..516fea81b9d 100644 --- a/test/config/override-configs/postfix-master.cf +++ b/test/config/override-configs/postfix-master.cf @@ -1,3 +1,3 @@ submission/inet/smtpd_sasl_security_options=noanonymous # this is a test comment, please don't delete me :'( - # this is also a test comment, :O + # this is also a test comment, :O diff --git a/test/config/postfix-regexp.cf b/test/config/postfix-regexp.cf index 6bbb6fa71ce..01d448a809c 100644 --- a/test/config/postfix-regexp.cf +++ b/test/config/postfix-regexp.cf @@ -1,6 +1,5 @@ /^test[0-9][0-9]*@localhost.localdomain/ user1@localhost.localdomain # this is a test comment, please don't delete me :'( - # this is also a test comment, :O + # this is also a test comment, :O /^bounce.*@.*/ external1@otherdomain.tld /^postmaster@/ user1@localhost.localdomain - diff --git a/test/config/postfix-virtual.cf b/test/config/postfix-virtual.cf index 67a58cff733..4dec6bbbc56 100644 --- a/test/config/postfix-virtual.cf +++ b/test/config/postfix-virtual.cf @@ -1,5 +1,5 @@ alias1@localhost.localdomain user1@localhost.localdomain # this is a test comment, please don't delete me :'( - # this is also a test comment, :O + # this is also a test comment, :O alias2@localhost.localdomain external1@otherdomain.tld @localdomain2.com user1@localhost.localdomain diff --git a/test/config/templates/dovecot-masters.cf b/test/config/templates/dovecot-masters.cf index e519ec75ec8..8d3f8977af2 100644 --- a/test/config/templates/dovecot-masters.cf +++ b/test/config/templates/dovecot-masters.cf @@ -1 +1 @@ -masterusername|{SHA512-CRYPT}$6$IOybywiyl1nuDno0$gRW625qH7ThmbRaByNVpuAGgDOkMd7tc3yuVmwVRuk7IXgiN8KDwcqtMcU0LyvS5RGAskbplavjPpCmFjbKEt1 +masterusername|{SHA512-CRYPT}$6$IOybywiyl1nuDno0$gRW625qH7ThmbRaByNVpuAGgDOkMd7tc3yuVmwVRuk7IXgiN8KDwcqtMcU0LyvS5RGAskbplavjPpCmFjbKEt1 diff --git a/test/config/templates/postfix-accounts.cf b/test/config/templates/postfix-accounts.cf index 03f6ef0b46b..9d538ad35a8 100644 --- a/test/config/templates/postfix-accounts.cf +++ b/test/config/templates/postfix-accounts.cf @@ -1,5 +1,5 @@ -user1@localhost.localdomain|{SHA512-CRYPT}$6$DBEbjh4I9P7aROk8$XosqE.YI2Z4bUkWD1/bedrSNpw79nsO60yiAKk04jARhPVX5VD/SaVM5HWFDQyzftESVDjbVdhzn/d4TJxFwg0 -user2@otherdomain.tld|{SHA512-CRYPT}$6$PQRkR3RRzpYP4WET$NKLJk3PkwTRRSxryqFhQloBR7qSAYjoQH/IbD1ZQKX2UJJ3jmdbOMQPfMRGXBZv3JGhDUPmAiWzoJL6/NJN5d/ -user3@localhost.localdomain|{SHA512-CRYPT}$6$lZwv0IoijHyEjDtM$vGsAS7KM5O5Q1NdWjard1LbJyGiHcqHhKAXBKDIMudjB/CuVvOvXKVy2yKeeRvKxVtkCdYac738VQPL.kpSVB.|userdb_mail=mbox:~/mail:INBOX=~/inbox +user1@localhost.localdomain|{SHA512-CRYPT}$6$DBEbjh4I9P7aROk8$XosqE.YI2Z4bUkWD1/bedrSNpw79nsO60yiAKk04jARhPVX5VD/SaVM5HWFDQyzftESVDjbVdhzn/d4TJxFwg0 +user2@otherdomain.tld|{SHA512-CRYPT}$6$PQRkR3RRzpYP4WET$NKLJk3PkwTRRSxryqFhQloBR7qSAYjoQH/IbD1ZQKX2UJJ3jmdbOMQPfMRGXBZv3JGhDUPmAiWzoJL6/NJN5d/ +user3@localhost.localdomain|{SHA512-CRYPT}$6$lZwv0IoijHyEjDtM$vGsAS7KM5O5Q1NdWjard1LbJyGiHcqHhKAXBKDIMudjB/CuVvOvXKVy2yKeeRvKxVtkCdYac738VQPL.kpSVB.|userdb_mail=mbox:~/mail:INBOX=~/inbox # this is a test comment, please don't delete me :'( - # this is also a test comment, :O + # this is also a test comment, :O diff --git a/test/linting/.ecrc.json b/test/linting/.ecrc.json index d9abd2f6438..ba92bf7777a 100644 --- a/test/linting/.ecrc.json +++ b/test/linting/.ecrc.json @@ -1,25 +1,9 @@ { - "Verbose": false, - "Debug": false, "IgnoreDefaults": false, - "SpacesAftertabs": true, - "NoColor": false, "Exclude": [ - "^test/", - "\\.git.*", - "\\.cf$", - "\\.conf$", - "\\.init$", - "\\.md$" - ], - "AllowedContentTypes": [], - "PassedFiles": [], - "Disable": { - "EndOfLine": false, - "Indentation": false, - "InsertFinalNewline": false, - "TrimTrailingWhitespace": false, - "IndentSize": false, - "MaxLineLength": false - } + "^test/bats/", + "^test/test_helper/bats-(assert|support)", + "^test/test-files/", + "\\.git/" + ] } diff --git a/test/tests/parallel/set1/spam_virus/amavis.bats b/test/tests/parallel/set1/spam_virus/amavis.bats index 5ab8cd9d221..bc920234490 100644 --- a/test/tests/parallel/set1/spam_virus/amavis.bats +++ b/test/tests/parallel/set1/spam_virus/amavis.bats @@ -19,7 +19,7 @@ function setup_file() { CONTAINER_NAME=${CONTAINER2_NAME} _init_with_defaults - local CUSTOM_SETUP_ARGUMENTS=( + local CUSTOM_SETUP_ARGUMENTS=( --env ENABLE_AMAVIS=0 --env ENABLE_SPAMASSASSIN=0 ) diff --git a/test/tests/serial/mail_with_ldap.bats b/test/tests/serial/mail_with_ldap.bats index 1b73643edd7..b7b3884b767 100644 --- a/test/tests/serial/mail_with_ldap.bats +++ b/test/tests/serial/mail_with_ldap.bats @@ -50,7 +50,7 @@ function setup_file() { # For this lookup `%s` only represents the domain, not a full email address. Hence the match pattern using a wildcard prefix `*@`. # For a breakdown, see QUERY_SENDERS comment. # NOTE: Although `result_attribute = mail` will return each accounts full email address, Postfix will only compare to domain-part. - local QUERY_DOMAIN='(| (& (|(mail=*@%s) (mailAlias=*@%s) (mailGroupMember=*@%s)) (&(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)) ) (&(mailAlias=*@%s)(objectClass=PostfixBookMailForward)) )' + local QUERY_DOMAIN='(| (& (|(mail=*@%s) (mailAlias=*@%s) (mailGroupMember=*@%s)) (&(objectClass=PostfixBookMailAccount)(mailEnabled=TRUE)) ) (&(mailAlias=*@%s)(objectClass=PostfixBookMailForward)) )' # Simple queries for a single attribute that additionally requires `mailEnabled=TRUE` from the `PostfixBookMailAccount` class: # NOTE: `mail` attribute is not unique to `PostfixBookMailAccount`. The `mailEnabled` attribute is to further control valid mail accounts. From 82c38f242648290ab109de8e3994ea7564120deb Mon Sep 17 00:00:00 2001 From: Vincent Ducamps Date: Sun, 8 Oct 2023 00:25:57 +0200 Subject: [PATCH 165/592] docs: TLS - Include `passthrough=true` on implicit ports for Traefik example (#3568) --- docs/content/examples/tutorials/mailserver-behind-proxy.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/content/examples/tutorials/mailserver-behind-proxy.md b/docs/content/examples/tutorials/mailserver-behind-proxy.md index 501d1209cba..7bdef5d604e 100644 --- a/docs/content/examples/tutorials/mailserver-behind-proxy.md +++ b/docs/content/examples/tutorials/mailserver-behind-proxy.md @@ -72,14 +72,15 @@ Feel free to add your configuration if you achieved the same goal using differen - "traefik.tcp.services.smtp.loadbalancer.server.port=25" - "traefik.tcp.services.smtp.loadbalancer.proxyProtocol.version=1" - "traefik.tcp.routers.smtp-ssl.rule=HostSNI(`*`)" - - "traefik.tcp.routers.smtp-ssl.tls=false" - "traefik.tcp.routers.smtp-ssl.entrypoints=smtp-ssl" + - "traefik.tcp.routers.smtp-ssl.tls.passthrough=true" - "traefik.tcp.routers.smtp-ssl.service=smtp-ssl" - "traefik.tcp.services.smtp-ssl.loadbalancer.server.port=465" - "traefik.tcp.services.smtp-ssl.loadbalancer.proxyProtocol.version=1" - "traefik.tcp.routers.imap-ssl.rule=HostSNI(`*`)" - "traefik.tcp.routers.imap-ssl.entrypoints=imap-ssl" - "traefik.tcp.routers.imap-ssl.service=imap-ssl" + - "traefik.tcp.routers.imap-ssl.tls.passthrough=true" - "traefik.tcp.services.imap-ssl.loadbalancer.server.port=10993" - "traefik.tcp.services.imap-ssl.loadbalancer.proxyProtocol.version=2" - "traefik.tcp.routers.sieve.rule=HostSNI(`*`)" From 894978ddd751f0d09e9f68e1a3de13926e17d432 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sat, 14 Oct 2023 17:14:10 +0200 Subject: [PATCH 166/592] refactor: `logrotate` setup + rspamd log path + tests log helper fallback path (#3576) * simplify `_setup_logrotate` * adjust Rspamd's log file and improve it's management * add information to docs about Rspamd log * update log query helper to allow another file location * bail in case `LOGROTATE_INTERVAL` is invalid --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/config/security/rspamd.md | 4 ++ target/scripts/startup/setup.d/log.sh | 41 ++++++++----------- .../startup/setup.d/security/rspamd.sh | 15 +++++++ target/supervisor/conf.d/supervisor-app.conf | 4 +- test/helper/common.bash | 5 ++- 5 files changed, 41 insertions(+), 28 deletions(-) diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 7876b998816..94be2db9ffe 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -69,6 +69,10 @@ DMS does not supply custom values for DNS servers to Rspamd. If you need to use This setting is enabled to not allow spam to proceed just because DNS requests did not succeed. It could deny legitimate e-mails to pass though too in case your DNS setup is incorrect or not functioning properly. +### Logs + +You can find the Rspamd logs at `/var/log/mail/rspamd.log`, and the corresponding logs for [Redis](#persistence-with-redis), if it is enabled, at `/var/log/supervisor/rspamd-redis.log`. We recommend inspecting these logs (with `docker exec -it cat /var/log/mail/rspamd.log`) in case Rspamd does not work as expected. + ### Modules You can find a list of all Rspamd modules [on their website][rspamd-docs-modules]. diff --git a/target/scripts/startup/setup.d/log.sh b/target/scripts/startup/setup.d/log.sh index eef76a3f4bf..cf282966485 100644 --- a/target/scripts/startup/setup.d/log.sh +++ b/target/scripts/startup/setup.d/log.sh @@ -13,31 +13,22 @@ function _setup_logs_general() { function _setup_logrotate() { _log 'debug' 'Setting up logrotate' - LOGROTATE='/var/log/mail/mail.log\n{\n compress\n copytruncate\n delaycompress\n' - - case "${LOGROTATE_INTERVAL}" in - ( 'daily' ) - _log 'trace' 'Setting postfix logrotate interval to daily' - LOGROTATE="${LOGROTATE} rotate 4\n daily\n" - ;; - - ( 'weekly' ) - _log 'trace' 'Setting postfix logrotate interval to weekly' - LOGROTATE="${LOGROTATE} rotate 4\n weekly\n" - ;; - - ( 'monthly' ) - _log 'trace' 'Setting postfix logrotate interval to monthly' - LOGROTATE="${LOGROTATE} rotate 4\n monthly\n" - ;; - - ( * ) - _log 'warn' 'LOGROTATE_INTERVAL not found in _setup_logrotate' - ;; - - esac - - echo -e "${LOGROTATE}}" >/etc/logrotate.d/maillog + if [[ ${LOGROTATE_INTERVAL} =~ ^(daily|weekly|monthly)$ ]]; then + _log 'trace' "Logrotate interval set to ${LOGROTATE_INTERVAL}" + else + _dms_panic__invalid_value 'LOGROTATE_INTERVAL' 'Setup -> Logrotate' + fi + + cat >/etc/logrotate.d/maillog << EOF +/var/log/mail/mail.log +{ + compress + copytruncate + delaycompress + rotate 4 + ${LOGROTATE_INTERVAL} +} +EOF } function _setup_mail_summary() { diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index d2389a07dd6..3e943285793 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -7,6 +7,7 @@ function _setup_rspamd() { __rspamd__log 'trace' '---------- Setup started ----------' __rspamd__run_early_setup_and_checks # must run first + __rspamd__setup_logfile __rspamd__setup_redis __rspamd__setup_postfix __rspamd__setup_clamav @@ -101,6 +102,20 @@ function __rspamd__run_early_setup_and_checks() { fi } +# Keep in sync with `target/scripts/startup/setup.d/log.sh:_setup_logrotate()` +function __rspamd__setup_logfile() { + cat >/etc/logrotate.d/rspamd << EOF +/var/log/mail/rspamd.log +{ + compress + copytruncate + delaycompress + rotate 4 + ${LOGROTATE_INTERVAL} +} +EOF +} + # Sets up Redis. In case the user does not use a dedicated Redis instance, we # supply a configuration for our local Redis instance which is started later. function __rspamd__setup_redis() { diff --git a/target/supervisor/conf.d/supervisor-app.conf b/target/supervisor/conf.d/supervisor-app.conf index d426831f523..2dd8b917337 100644 --- a/target/supervisor/conf.d/supervisor-app.conf +++ b/target/supervisor/conf.d/supervisor-app.conf @@ -101,8 +101,8 @@ startsecs=0 stopwaitsecs=55 autostart=false autorestart=true -stdout_logfile=/var/log/supervisor/%(program_name)s.log -stderr_logfile=/var/log/supervisor/%(program_name)s.log +stdout_logfile=/var/log/mail/%(program_name)s.log +stderr_logfile=/var/log/mail/%(program_name)s.log command=/usr/bin/rspamd --no-fork --user=_rspamd --group=_rspamd [program:rspamd-redis] diff --git a/test/helper/common.bash b/test/helper/common.bash index cc0eda3d020..8fb7854e146 100644 --- a/test/helper/common.bash +++ b/test/helper/common.bash @@ -429,8 +429,11 @@ function _filter_service_log() { local SERVICE=${1:?Service name must be provided} local STRING=${2:?String to match must be provided} local CONTAINER_NAME=$(__handle_container_name "${3:-}") + local FILE="/var/log/supervisor/${SERVICE}.log" - _run_in_container grep -E "${STRING}" "/var/log/supervisor/${SERVICE}.log" + # Fallback to alternative log location: + [[ -f ${FILE} ]] || FILE="/var/log/mail/${SERVICE}.log" + _run_in_container grep -E "${STRING}" "${FILE}" } # Like `_filter_service_log` but asserts that the string was found. From 128e6b4d1f3873522806e067a0163cee637b8510 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 16 Oct 2023 09:51:48 +0200 Subject: [PATCH 167/592] chore: Add debug group (`packages.sh`) + more resilient rspamd setup (#3578) --- target/scripts/build/packages.sh | 26 ++++++++++--------- target/scripts/helpers/utils.sh | 14 +++++++--- .../startup/setup.d/security/rspamd.sh | 22 ++++++++++++---- .../set3/scripts/helper_functions.bats | 6 ++++- 4 files changed, 47 insertions(+), 21 deletions(-) diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 40e7365ee78..e9b2d479d75 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -43,10 +43,6 @@ function _install_postfix() { function _install_packages() { _log 'debug' 'Installing all packages now' - declare -a ANTI_VIRUS_SPAM_PACKAGES - declare -a CODECS_PACKAGES MISCELLANEOUS_PACKAGES - declare -a POSTFIX_PACKAGES MAIL_PROGRAMS_PACKAGES - ANTI_VIRUS_SPAM_PACKAGES=( amavisd-new clamav clamav-daemon pyzor razor spamassassin @@ -62,14 +58,13 @@ function _install_packages() { ) MISCELLANEOUS_PACKAGES=( - apt-transport-https bind9-dnsutils binutils bsd-mailx + apt-transport-https binutils bsd-mailx ca-certificates curl dbconfig-no-thanks - dumb-init ed gnupg iproute2 iputils-ping - libdate-manip-perl libldap-common - libmail-spf-perl libnet-dns-perl - locales logwatch netcat-openbsd - nftables rsyslog supervisor - uuid whois + dumb-init gnupg iproute2 libdate-manip-perl + libldap-common libmail-spf-perl + libnet-dns-perl locales logwatch + netcat-openbsd nftables rsyslog + supervisor uuid whois ) POSTFIX_PACKAGES=( @@ -82,12 +77,19 @@ function _install_packages() { opendmarc libsasl2-modules sasl2-bin ) + # `bind9-dnsutils` provides the `dig` command + # `iputils-ping` provides the `ping` command + DEBUG_PACKAGES=( + bind9-dnsutils iputils-ping less nano + ) + apt-get "${QUIET}" --no-install-recommends install \ "${ANTI_VIRUS_SPAM_PACKAGES[@]}" \ "${CODECS_PACKAGES[@]}" \ "${MISCELLANEOUS_PACKAGES[@]}" \ "${POSTFIX_PACKAGES[@]}" \ - "${MAIL_PROGRAMS_PACKAGES[@]}" + "${MAIL_PROGRAMS_PACKAGES[@]}" \ + "${DEBUG_PACKAGES[@]}" } function _install_dovecot() { diff --git a/target/scripts/helpers/utils.sh b/target/scripts/helpers/utils.sh index e44f0aff577..f7095bf3690 100644 --- a/target/scripts/helpers/utils.sh +++ b/target/scripts/helpers/utils.sh @@ -127,9 +127,17 @@ function _replace_by_env_in_file() { function _env_var_expect_zero_or_one() { local ENV_VAR_NAME=${1:?ENV var name must be provided to _env_var_expect_zero_or_one} - [[ ${!ENV_VAR_NAME} =~ ^(0|1)$ ]] && return 0 - _log 'warn' "The value of '${ENV_VAR_NAME}' is not zero or one ('${!ENV_VAR_NAME}'), but was expected to be" - return 1 + if [[ ! -v ${ENV_VAR_NAME} ]]; then + _log 'warn' "'${ENV_VAR_NAME}' is not set, but was expected to be" + return 1 + fi + + if [[ ! ${!ENV_VAR_NAME} =~ ^(0|1)$ ]]; then + _log 'warn' "The value of '${ENV_VAR_NAME}' (= '${!ENV_VAR_NAME}') is not 0 or 1, but was expected to be" + return 1 + fi + + return 0 } # Check if an environment variable's value is an integer. diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 3e943285793..4199b07747f 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -43,6 +43,8 @@ function __rspamd__helper__enable_disable_module() { local LOCAL_OR_OVERRIDE=${3:-local} local MESSAGE='Enabling' + readonly MODULE ENABLE_MODULE LOCAL_OR_OVERRIDE + if [[ ! ${ENABLE_MODULE} =~ ^(true|false)$ ]]; then __rspamd__log 'warn' "__rspamd__helper__enable_disable_module got non-boolean argument for deciding whether module should be enabled or not" return 1 @@ -64,10 +66,12 @@ EOF function __rspamd__run_early_setup_and_checks() { # Note: Variables not marked with `local` are # used in other functions as well. - RSPAMD_LOCAL_D='/etc/rspamd/local.d' - RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' - RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' + readonly RSPAMD_LOCAL_D='/etc/rspamd/local.d' + readonly RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' + readonly RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' + local RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d/" + readonly RSPAMD_DMS_OVERRIDE_D mkdir -p /var/lib/rspamd/ : >/var/lib/rspamd/stats.ucl @@ -77,7 +81,7 @@ function __rspamd__run_early_setup_and_checks() { if rmdir "${RSPAMD_OVERRIDE_D}" 2>/dev/null; then ln -s "${RSPAMD_DMS_OVERRIDE_D}" "${RSPAMD_OVERRIDE_D}" else - __rspamd__log 'warn' "Could not remove '${RSPAMD_OVERRIDE_D}' (not empty? not a directory?; did you restart properly?) - not linking '${RSPAMD_DMS_OVERRIDE_D}'" + __rspamd__log 'warn' "Could not remove '${RSPAMD_OVERRIDE_D}' (not empty?; not a directory?; did you restart properly?) - not linking '${RSPAMD_DMS_OVERRIDE_D}'" fi fi @@ -195,6 +199,7 @@ function __rspamd__setup_default_modules() { metric_exporter ) + readonly -a DISABLE_MODULES local MODULE for MODULE in "${DISABLE_MODULES[@]}"; do __rspamd__helper__enable_disable_module "${MODULE}" 'false' @@ -211,6 +216,7 @@ function __rspamd__setup_learning() { __rspamd__log 'debug' 'Setting up intelligent learning of spam and ham' local SIEVE_PIPE_BIN_DIR='/usr/lib/dovecot/sieve-pipe' + readonly SIEVE_PIPE_BIN_DIR ln -s "$(type -f -P rspamc)" "${SIEVE_PIPE_BIN_DIR}/rspamc" sedfile -i -E 's|(mail_plugins =.*)|\1 imap_sieve|' /etc/dovecot/conf.d/20-imap.conf @@ -264,6 +270,7 @@ function __rspamd__setup_greylisting() { # succeeds. function __rspamd__setup_hfilter_group() { local MODULE_FILE="${RSPAMD_LOCAL_D}/hfilter_group.conf" + readonly MODULE_FILE if _env_var_expect_zero_or_one 'RSPAMD_HFILTER' && [[ ${RSPAMD_HFILTER} -eq 1 ]]; then __rspamd__log 'debug' 'Hfilter (group) module is enabled' # Check if we received a number first @@ -284,6 +291,7 @@ function __rspamd__setup_hfilter_group() { function __rspamd__setup_check_authenticated() { local MODULE_FILE="${RSPAMD_LOCAL_D}/settings.conf" + readonly MODULE_FILE if _env_var_expect_zero_or_one 'RSPAMD_CHECK_AUTHENTICATED' \ && [[ ${RSPAMD_CHECK_AUTHENTICATED} -eq 0 ]] then @@ -320,8 +328,10 @@ function __rspamd__handle_user_modules_adjustments() { local VALUE=${4:?Value belonging to an option must be provided} # remove possible whitespace at the end (e.g., in case ${ARGUMENT3} is empty) VALUE=${VALUE% } - local FILE="${RSPAMD_OVERRIDE_D}/${MODULE_FILE}" + + readonly MODULE_FILE MODULE_LOG_NAME OPTION VALUE FILE + [[ -f ${FILE} ]] || touch "${FILE}" if grep -q -E "${OPTION}.*=.*" "${FILE}"; then @@ -335,6 +345,7 @@ function __rspamd__handle_user_modules_adjustments() { local RSPAMD_CUSTOM_COMMANDS_FILE="${RSPAMD_DMS_D}/custom-commands.conf" local RSPAMD_CUSTOM_COMMANDS_FILE_OLD="${RSPAMD_DMS_D}-modules.conf" + readonly RSPAMD_CUSTOM_COMMANDS_FILE RSPAMD_CUSTOM_COMMANDS_FILE_OLD # We check for usage of the previous location of the commands file. # This can be removed after the release of v14.0.0. @@ -347,6 +358,7 @@ function __rspamd__handle_user_modules_adjustments() { if [[ -f "${RSPAMD_CUSTOM_COMMANDS_FILE}" ]]; then __rspamd__log 'debug' "Found file '${RSPAMD_CUSTOM_COMMANDS_FILE}' - parsing and applying it" + local COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3 while read -r COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3; do case "${COMMAND}" in ('disable-module') diff --git a/test/tests/parallel/set3/scripts/helper_functions.bats b/test/tests/parallel/set3/scripts/helper_functions.bats index 5a1fbf74c87..332de448e2a 100644 --- a/test/tests/parallel/set3/scripts/helper_functions.bats +++ b/test/tests/parallel/set3/scripts/helper_functions.bats @@ -35,7 +35,11 @@ SOURCE_BASE_PATH="${REPOSITORY_ROOT:?Expected REPOSITORY_ROOT to be set}/target/ run _env_var_expect_zero_or_one TWO assert_failure - assert_output --partial "The value of 'TWO' is not zero or one ('2'), but was expected to be" + assert_output --partial "The value of 'TWO' (= '2') is not 0 or 1, but was expected to be" + + run _env_var_expect_zero_or_one UNSET + assert_failure + assert_output --partial "'UNSET' is not set, but was expected to be" run _env_var_expect_zero_or_one assert_failure From 4a58e2eb8342203d43f78b3a67fc5bbd1f3cd980 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 17 Oct 2023 02:20:50 +1300 Subject: [PATCH 168/592] docs: Revise `watchtower` page (#3583) - Better clarify images are only updated to newer image updates to the same tag of the running container(s). - Slight revisions to existing content. --- .../maintenance/update-and-cleanup.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/content/config/advanced/maintenance/update-and-cleanup.md b/docs/content/config/advanced/maintenance/update-and-cleanup.md index b972c4a5446..42f039fca2a 100644 --- a/docs/content/config/advanced/maintenance/update-and-cleanup.md +++ b/docs/content/config/advanced/maintenance/update-and-cleanup.md @@ -6,6 +6,8 @@ title: 'Maintenance | Update and Cleanup' !!! example "Automatic image updates + cleanup" + Run a `watchtower` container with access to `docker.sock`, enabling the service to manage Docker: + ```yaml title="compose.yaml" services: watchtower: @@ -17,26 +19,35 @@ title: 'Maintenance | Update and Cleanup' - /var/run/docker.sock:/var/run/docker.sock ``` +!!! tip "The image tag used for a container is monitored for updates (eg: `:latest`, `:edge`, `:13`)" + + The automatic update support is **only for updates to that specific image tag**. + + - Your container will not update to a new major version tag (_unless using `:latest`_). + - Omit the minor or patch portion of the semver tag to receive updates for the omitted portion (_eg: `13` will represent the latest minor + patch release of `v13`_). + !!! tip "Updating only specific containers" - The default `watchtower` service will check every 24 hours for any new image updates to pull, **not only the images** defined within your `compose.yaml`. + By default the `watchtower` service will check every 24 hours for new image updates to pull, based on currently running containers (_**not restricted** to only those running within your `compose.yaml`_). - The images to update can be restricted with a custom command that provides a list of containers names and other config options. Configuration is detailed in the [`watchtower` docs][watchtower-docs]. + Images eligible for updates can configured with a [custom `command`][docker-docs-compose-command] that provides a list of container names, or via other supported options (eg: labels). This configuration is detailed in the [`watchtower` docs][watchtower-docs]. !!! info "Manual cleanup" `watchtower` also supports running on-demand with `docker run` or `compose.yaml` via the `--run-once` option. - You can also directly invoke cleanup of Docker storage with: + You can alternatively invoke cleanup of Docker storage directly with: - [`docker image prune --all`][docker-docs-prune-image] - [`docker system prune --all`][docker-docs-prune-system] (_also removes unused containers, networks, build cache_). - - Avoid the `--all` option to only remove ["dangling" content][docker-prune-dangling] (_eg: Orphaned images_). + + If you omit the `--all` option, this will instead only remove ["dangling" content][docker-prune-dangling] (_eg: Orphaned images_). [watchtower-dockerhub]: https://hub.docker.com/r/containrrr/watchtower [watchtower-cleanup]: https://containrrr.github.io/watchtower/arguments/#cleanup [watchtower-docs]: https://containrrr.dev/watchtower/ +[docker-docs-compose-command]: https://docs.docker.com/compose/compose-file/05-services/#command [docker-docs-prune-image]: https://docs.docker.com/engine/reference/commandline/image_prune/ [docker-docs-prune-system]: https://docs.docker.com/engine/reference/commandline/system_prune/ [docker-prune-dangling]: https://stackoverflow.com/questions/45142528/what-is-a-dangling-image-and-what-is-an-unused-image/60756668#60756668 From f6e556e33f86591ee189d318c27fcc97a59c9e10 Mon Sep 17 00:00:00 2001 From: Andreas Perhab Date: Tue, 17 Oct 2023 10:24:01 +0200 Subject: [PATCH 169/592] docs: fix path to rspamd.log (#3585) --- docs/content/config/best-practices/dkim_dmarc_spf.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md index eda986008bb..da496aa5d25 100644 --- a/docs/content/config/best-practices/dkim_dmarc_spf.md +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -202,7 +202,7 @@ DKIM is currently supported by either OpenDKIM or Rspamd: When `check_pubkey = true;` is set, Rspamd will query the DNS record for each DKIM selector, verifying each public key matches the private key configured. - If there is a mismatch, a warning will be emitted to the Rspamd log `/var/log/supervisor/rspamd.log`. + If there is a mismatch, a warning will be emitted to the Rspamd log `/var/log/mail/rspamd.log`. ### DNS Record { #dkim-dns } @@ -265,7 +265,7 @@ When mail signed with your DKIM key is sent from your mail server, the receiver [MxToolbox has a DKIM Verifier][mxtoolbox-dkim-verifier] that you can use to check your DKIM DNS record(s). -When using Rspamd, we recommend you turn on `check_pubkey = true;` in `dkim_signing.conf`. Rspamd will then check whether your private key matches your public key, and you can check possible mismatches by looking at `/var/log/supervisor/rspamd.log`. +When using Rspamd, we recommend you turn on `check_pubkey = true;` in `dkim_signing.conf`. Rspamd will then check whether your private key matches your public key, and you can check possible mismatches by looking at `/var/log/mail/rspamd.log`. ## DMARC From 811a7698459ebd40cbd520740c2074f4ad7402a6 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 18 Oct 2023 10:47:43 +1300 Subject: [PATCH 170/592] ci: Update `eclint` to `2.7.2` (#3584) This release contains a bugfix for charset detection, resolving a linting bug affecting the test-files directory. --- test/linting/.ecrc.json | 1 - test/linting/lint.sh | 2 +- .../email-templates/send-privacy-email.txt | 2 +- test/test-files/ssl/example.test/README.md | 62 +++++++++---------- 4 files changed, 33 insertions(+), 34 deletions(-) diff --git a/test/linting/.ecrc.json b/test/linting/.ecrc.json index ba92bf7777a..374615fec3a 100644 --- a/test/linting/.ecrc.json +++ b/test/linting/.ecrc.json @@ -3,7 +3,6 @@ "Exclude": [ "^test/bats/", "^test/test_helper/bats-(assert|support)", - "^test/test-files/", "\\.git/" ] } diff --git a/test/linting/lint.sh b/test/linting/lint.sh index 88103ddd098..7c281e8f416 100755 --- a/test/linting/lint.sh +++ b/test/linting/lint.sh @@ -10,7 +10,7 @@ shopt -s inherit_errexit REPOSITORY_ROOT=$(realpath "$(dirname "$(readlink -f "${0}")")"/../../) LOG_LEVEL=${LOG_LEVEL:-debug} HADOLINT_VERSION='2.12.0' -ECLINT_VERSION='2.7.0' +ECLINT_VERSION='2.7.2' SHELLCHECK_VERSION='0.9.0' # shellcheck source=./../../target/scripts/helpers/log.sh diff --git a/test/test-files/email-templates/send-privacy-email.txt b/test/test-files/email-templates/send-privacy-email.txt index 2273a46cc4f..0c51ec5b4be 100644 --- a/test/test-files/email-templates/send-privacy-email.txt +++ b/test/test-files/email-templates/send-privacy-email.txt @@ -7,7 +7,7 @@ data From: Some User To: Some User User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:52.0) - Gecko/20100101 Thunderbird/52.2.1 + Gecko/20100101 Thunderbird/52.2.1 Subject: Test ESMTP Auth LOGIN and remove privacy This is a test mail. diff --git a/test/test-files/ssl/example.test/README.md b/test/test-files/ssl/example.test/README.md index 274f633867e..e1f8cdaf46a 100644 --- a/test/test-files/ssl/example.test/README.md +++ b/test/test-files/ssl/example.test/README.md @@ -71,21 +71,21 @@ Certificate: X509v3 Subject Alternative Name: DNS:example.test, DNS:mail.example.test Signature Algorithm: SHA256-RSA - 50:47:7b:59:26:9d:8d:f7:e4:dc:03:94:b0:35:e4:03:b7:94: - 16:7e:b6:79:c5:bb:e7:61:db:ca:e6:22:cc:c8:a0:9f:9d:b0: - 7c:12:43:ec:a7:f3:fe:ad:0a:44:69:69:7f:c7:31:f7:3f:e8: - 98:a7:37:43:bd:fb:5b:c6:85:85:91:dc:29:23:cb:6b:a9:aa: - f0:f0:62:79:ce:43:8c:5f:28:49:ee:a1:d4:16:67:6b:59:c3: - 15:65:e3:d3:3b:35:da:59:35:33:2a:5e:8a:59:ff:14:b9:51: - a5:8e:0b:7c:1b:a1:b1:f4:89:1a:3f:2f:d7:b1:8d:23:0a:7a: - 79:e1:c2:03:b5:2f:ee:34:16:a9:67:27:b6:10:67:5d:f4:1d: - d6:b3:e0:ab:80:3d:59:fc:bc:4b:1a:55:fb:36:75:ff:e3:88: - 73:e3:16:4d:2b:17:7b:2a:21:a3:18:14:04:19:b3:b8:11:39: - 55:3f:ce:21:b7:d3:5d:8d:78:d5:3a:e0:b2:17:41:ad:3c:8e: - a5:a2:ba:eb:3d:b6:9e:2c:ef:7d:d5:cc:71:cb:07:54:21:42: - 81:79:45:2b:93:74:93:a1:c9:f1:5e:5e:11:3d:ac:df:55:98: - 37:44:d2:55:a5:15:a9:33:79:6e:fe:49:6d:e5:7b:a0:1c:12: - c5:1b:4d:33 + 50:47:7b:59:26:9d:8d:f7:e4:dc:03:94:b0:35:e4:03:b7:94: + 16:7e:b6:79:c5:bb:e7:61:db:ca:e6:22:cc:c8:a0:9f:9d:b0: + 7c:12:43:ec:a7:f3:fe:ad:0a:44:69:69:7f:c7:31:f7:3f:e8: + 98:a7:37:43:bd:fb:5b:c6:85:85:91:dc:29:23:cb:6b:a9:aa: + f0:f0:62:79:ce:43:8c:5f:28:49:ee:a1:d4:16:67:6b:59:c3: + 15:65:e3:d3:3b:35:da:59:35:33:2a:5e:8a:59:ff:14:b9:51: + a5:8e:0b:7c:1b:a1:b1:f4:89:1a:3f:2f:d7:b1:8d:23:0a:7a: + 79:e1:c2:03:b5:2f:ee:34:16:a9:67:27:b6:10:67:5d:f4:1d: + d6:b3:e0:ab:80:3d:59:fc:bc:4b:1a:55:fb:36:75:ff:e3:88: + 73:e3:16:4d:2b:17:7b:2a:21:a3:18:14:04:19:b3:b8:11:39: + 55:3f:ce:21:b7:d3:5d:8d:78:d5:3a:e0:b2:17:41:ad:3c:8e: + a5:a2:ba:eb:3d:b6:9e:2c:ef:7d:d5:cc:71:cb:07:54:21:42: + 81:79:45:2b:93:74:93:a1:c9:f1:5e:5e:11:3d:ac:df:55:98: + 37:44:d2:55:a5:15:a9:33:79:6e:fe:49:6d:e5:7b:a0:1c:12: + c5:1b:4d:33 ``` @@ -139,10 +139,10 @@ Certificate: X509v3 Subject Alternative Name: DNS:example.test, DNS:mail.example.test Signature Algorithm: ECDSA-SHA256 - 30:46:02:21:00:f8:72:3d:90:7e:db:9e:7a:4f:6d:80:fb:fa: - dc:42:43:e2:dc:8f:6a:ec:18:c5:af:e1:ea:03:fd:66:78:a2: - 01:02:21:00:f7:86:58:81:17:f5:74:5b:14:c8:0f:93:e2:bb: - b8:e9:90:47:c0:f7:b1:60:82:d9:b4:1a:fc:fa:66:fa:48:5c + 30:46:02:21:00:f8:72:3d:90:7e:db:9e:7a:4f:6d:80:fb:fa: + dc:42:43:e2:dc:8f:6a:ec:18:c5:af:e1:ea:03:fd:66:78:a2: + 01:02:21:00:f7:86:58:81:17:f5:74:5b:14:c8:0f:93:e2:bb: + b8:e9:90:47:c0:f7:b1:60:82:d9:b4:1a:fc:fa:66:fa:48:5c ``` @@ -224,10 +224,10 @@ Certificate: X509v3 Subject Alternative Name: DNS:mail.example.test Signature Algorithm: ECDSA-SHA256 - 30:46:02:21:00:ad:08:7b:f0:82:41:2e:0e:cd:2b:f7:95:fd: - ee:73:d9:93:8d:74:7c:ef:29:4d:d5:da:33:04:f0:b6:b1:6b: - 13:02:21:00:d7:f1:95:db:be:18:b8:db:77:b9:57:07:e6:b9: - 5a:3d:00:34:d3:f5:eb:18:67:9b:ba:bf:88:62:72:e9:c9:99 + 30:46:02:21:00:ad:08:7b:f0:82:41:2e:0e:cd:2b:f7:95:fd: + ee:73:d9:93:8d:74:7c:ef:29:4d:d5:da:33:04:f0:b6:b1:6b: + 13:02:21:00:d7:f1:95:db:be:18:b8:db:77:b9:57:07:e6:b9: + 5a:3d:00:34:d3:f5:eb:18:67:9b:ba:bf:88:62:72:e9:c9:99 ``` @@ -268,10 +268,10 @@ Certificate: X509v3 Subject Key Identifier: DE:90:B3:B9:4D:C1:B3:EE:77:00:88:8B:69:EC:71:C4:30:F9:F6:7F Signature Algorithm: ECDSA-SHA256 - 30:44:02:20:3f:3b:90:e7:ca:82:70:8e:3f:2e:72:2a:b9:27: - 46:ac:e9:e2:4a:db:56:02:bc:a2:b2:99:e4:8d:10:7a:d5:73: - 02:20:72:25:64:b6:1c:aa:a6:c3:14:e1:66:35:bf:a1:db:90: - ea:49:59:f9:44:e8:63:de:a8:c0:bb:9b:21:08:59:87 + 30:44:02:20:3f:3b:90:e7:ca:82:70:8e:3f:2e:72:2a:b9:27: + 46:ac:e9:e2:4a:db:56:02:bc:a2:b2:99:e4:8d:10:7a:d5:73: + 02:20:72:25:64:b6:1c:aa:a6:c3:14:e1:66:35:bf:a1:db:90: + ea:49:59:f9:44:e8:63:de:a8:c0:bb:9b:21:08:59:87 ``` @@ -337,10 +337,10 @@ Certificate: X509v3 Subject Alternative Name: DNS:*.example.test Signature Algorithm: ECDSA-SHA256 - 30:46:02:21:00:f2:50:c0:b5:c9:24:e5:e9:36:a6:7b:35:5d: - 38:a7:7d:81:af:02:fc:9d:fd:79:f4:2d:4c:8a:04:55:44:a8: - 3a:02:21:00:b1:2d:d2:25:18:2d:35:19:20:97:78:f1:d5:18: - 9f:11:d5:97:a9:dc:64:95:2a:6c:9d:4e:78:69:c1:92:23:23 + 30:46:02:21:00:f2:50:c0:b5:c9:24:e5:e9:36:a6:7b:35:5d: + 38:a7:7d:81:af:02:fc:9d:fd:79:f4:2d:4c:8a:04:55:44:a8: + 3a:02:21:00:b1:2d:d2:25:18:2d:35:19:20:97:78:f1:d5:18: + 9f:11:d5:97:a9:dc:64:95:2a:6c:9d:4e:78:69:c1:92:23:23 ``` From eacc379cf12661a1795b382a667da39650154334 Mon Sep 17 00:00:00 2001 From: allddd <117767298+allddd@users.noreply.github.com> Date: Sun, 22 Oct 2023 15:16:41 +0200 Subject: [PATCH 171/592] feat: Postfix permit DSN (Delivery Status Notification) only on authenticated ports (465 + 587) (#3572) * add POSTFIX_DSN * add tests for POSTFIX_DSN * Revert "add POSTFIX_DSN" This reverts commit d5bd0e911737eb8b9efa9566caac6e95c6570af2. * discard DSN requests on unauthenticated ports * make tests work with overrides instead of ENV * Apply suggestions from code review * fix test inconsistencies --------- Co-authored-by: allddd Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 11 +++ target/postfix/main.cf | 1 + target/postfix/master.cf | 2 + test/config/dsn/postfix-main.cf | 1 + test/config/dsn/postfix-master.cf | 2 + .../email-templates/dsn-authenticated.txt | 14 +++ .../email-templates/dsn-unauthenticated.txt | 12 +++ test/tests/parallel/set3/mta/dsn.bats | 95 +++++++++++++++++++ 8 files changed, 138 insertions(+) create mode 100644 test/config/dsn/postfix-main.cf create mode 100644 test/config/dsn/postfix-master.cf create mode 100644 test/test-files/email-templates/dsn-authenticated.txt create mode 100644 test/test-files/email-templates/dsn-unauthenticated.txt create mode 100644 test/tests/parallel/set3/mta/dsn.bats diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ce7b6fb7a4..a2e15c77f51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,17 @@ All notable changes to this project will be documented in this file. The format ### Breaking - The environment variable `ENABLE_LDAP=1` has been changed to `ACCOUNT_PROVISIONER=LDAP`. +- Postfix now defaults to supporting DSNs (_[Delivery Status Notifications](https://github.com/docker-mailserver/docker-mailserver/pull/3572#issuecomment-1751880574)_) only for authenticated users. This is a security measure to reduce spammer abuse of your DMS instance as a backscatter source. + - If you need to modify this change, please let us know by opening an issue / discussion. + - You can [opt-out (_enable DSNs_) via the `postfix-main.cf` override support](https://docker-mailserver.github.io/docker-mailserver/v12.1/config/advanced/override-defaults/postfix/) using the contents: `smtpd_discard_ehlo_keywords =`. + - Likewise for authenticated users, the submission(s) ports (465 + 587) are configured internally via `master.cf` to keep DSNs enabled (_since authentication protects from abuse_). + + If necessary, DSNs for authenticated users can be disabled via the `postfix-master.cf` override with the following contents: + + ``` + submission/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn + submissions/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn + ``` ### Added diff --git a/target/postfix/main.cf b/target/postfix/main.cf index 518ad326f98..405dc0fbbfe 100644 --- a/target/postfix/main.cf +++ b/target/postfix/main.cf @@ -54,6 +54,7 @@ smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_una smtpd_recipient_restrictions = permit_sasl_authenticated, permit_mynetworks, reject_unauth_destination, reject_unauth_pipelining, reject_invalid_helo_hostname, reject_non_fqdn_helo_hostname, reject_unknown_recipient_domain smtpd_client_restrictions = permit_mynetworks, permit_sasl_authenticated, reject_unauth_destination, reject_unauth_pipelining smtpd_sender_restrictions = $dms_smtpd_sender_restrictions +smtpd_discard_ehlo_keywords = silent-discard, dsn disable_vrfy_command = yes # Custom defined parameters for DMS: diff --git a/target/postfix/master.cf b/target/postfix/master.cf index 6f8877f697e..e5b955a48dc 100644 --- a/target/postfix/master.cf +++ b/target/postfix/master.cf @@ -24,6 +24,7 @@ submission inet n - n - - smtpd -o smtpd_client_restrictions=permit_sasl_authenticated,reject -o smtpd_relay_restrictions=permit_sasl_authenticated,reject -o smtpd_sender_restrictions=$mua_sender_restrictions + -o smtpd_discard_ehlo_keywords= -o milter_macro_daemon_name=ORIGINATING -o cleanup_service_name=sender-cleanup @@ -37,6 +38,7 @@ submissions inet n - n - - smtpd -o smtpd_client_restrictions=permit_sasl_authenticated,reject -o smtpd_relay_restrictions=permit_sasl_authenticated,reject -o smtpd_sender_restrictions=$mua_sender_restrictions + -o smtpd_discard_ehlo_keywords= -o milter_macro_daemon_name=ORIGINATING -o cleanup_service_name=sender-cleanup diff --git a/test/config/dsn/postfix-main.cf b/test/config/dsn/postfix-main.cf new file mode 100644 index 00000000000..1cb0db1e598 --- /dev/null +++ b/test/config/dsn/postfix-main.cf @@ -0,0 +1 @@ +smtpd_discard_ehlo_keywords = diff --git a/test/config/dsn/postfix-master.cf b/test/config/dsn/postfix-master.cf new file mode 100644 index 00000000000..bb6aad153f8 --- /dev/null +++ b/test/config/dsn/postfix-master.cf @@ -0,0 +1,2 @@ +submission/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn +submissions/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn diff --git a/test/test-files/email-templates/dsn-authenticated.txt b/test/test-files/email-templates/dsn-authenticated.txt new file mode 100644 index 00000000000..c187bd67b60 --- /dev/null +++ b/test/test-files/email-templates/dsn-authenticated.txt @@ -0,0 +1,14 @@ +EHLO mail +AUTH LOGIN dXNlcjFAbG9jYWxob3N0LmxvY2FsZG9tYWlu +bXlwYXNzd29yZA== +MAIL FROM: user1@localhost.localdomain +RCPT TO: user1@localhost.localdomain NOTIFY=success,failure +DATA +From: Existing Local User +To: Existing Local User +Date: Sat, 22 May 2010 07:43:25 -0400 +Subject: Test Message +This is a test mail. + +. +QUIT diff --git a/test/test-files/email-templates/dsn-unauthenticated.txt b/test/test-files/email-templates/dsn-unauthenticated.txt new file mode 100644 index 00000000000..8232ea68ece --- /dev/null +++ b/test/test-files/email-templates/dsn-unauthenticated.txt @@ -0,0 +1,12 @@ +HELO mail.external.tld +MAIL FROM: user@external.tld +RCPT TO: user1@localhost.localdomain NOTIFY=success,failure +DATA +From: Docker Mail Server +To: Existing Local User +Date: Sat, 22 May 2010 07:43:25 -0400 +Subject: Test Message +This is a test mail. + +. +QUIT diff --git a/test/tests/parallel/set3/mta/dsn.bats b/test/tests/parallel/set3/mta/dsn.bats new file mode 100644 index 00000000000..dcbb79b6603 --- /dev/null +++ b/test/tests/parallel/set3/mta/dsn.bats @@ -0,0 +1,95 @@ +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" + +BATS_TEST_NAME_PREFIX='[DSN] ' +CONTAINER1_NAME='dms-test_dsn_send_always' +CONTAINER2_NAME='dms-test_dsn_send_auth' +CONTAINER3_NAME='dms-test_dsn_send_none' +# A similar line is added to the log when a DSN (Delivery Status Notification) is sent: +# +# postfix/bounce[1023]: C943BA6B46: sender delivery status notification: DBF86A6B4CO +# +LOG_DSN='delivery status notification' + +function setup_file() { + local CUSTOM_SETUP_ARGUMENTS=( + # Required only for delivery via nc (_send_email) + --env PERMIT_DOCKER=container + ) + + export CONTAINER_NAME=${CONTAINER1_NAME} + _init_with_defaults + # Unset `smtpd_discard_ehlo_keywords` to allow DSNs by default on any `smtpd` service: + cp "${TEST_TMP_CONFIG}/dsn/postfix-main.cf" "${TEST_TMP_CONFIG}/postfix-main.cf" + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + _wait_for_service postfix + _wait_for_smtp_port_in_container + + export CONTAINER_NAME=${CONTAINER2_NAME} + _init_with_defaults + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + _wait_for_service postfix + _wait_for_smtp_port_in_container + + export CONTAINER_NAME=${CONTAINER3_NAME} + _init_with_defaults + # Mirror default main.cf (disable DSN on ports 465 + 587 too): + cp "${TEST_TMP_CONFIG}/dsn/postfix-master.cf" "${TEST_TMP_CONFIG}/postfix-master.cf" + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + _wait_for_service postfix + _wait_for_smtp_port_in_container +} + +function teardown_file() { + docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}" "${CONTAINER3_NAME}" +} + +@test "should always send a DSN when requested" { + export CONTAINER_NAME=${CONTAINER1_NAME} + + _send_email 'email-templates/dsn-unauthenticated' + _send_email 'email-templates/dsn-authenticated' '0.0.0.0 465' + _send_email 'email-templates/dsn-authenticated' '0.0.0.0 587' + _wait_for_empty_mail_queue_in_container + + _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log + _should_output_number_of_lines 3 +} + +# Defaults test case +@test "should only send a DSN when requested from ports 465/587" { + export CONTAINER_NAME=${CONTAINER2_NAME} + + _send_email 'email-templates/dsn-unauthenticated' + _wait_for_empty_mail_queue_in_container + + # DSN requests can now only be made on ports 465 and 587, + # so grep should not find anything. + # + # Although external requests are discarded, anyone who has requested a DSN + # will still receive it, but it will come from the sending mail server, not this one. + _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log + assert_failure + + # These ports are excluded via master.cf. + _send_email 'email-templates/dsn-authenticated' '0.0.0.0 465' + _send_email 'email-templates/dsn-authenticated' '0.0.0.0 587' + _wait_for_empty_mail_queue_in_container + + _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log + _should_output_number_of_lines 2 +} + +@test "should never send a DSN" { + export CONTAINER_NAME=${CONTAINER3_NAME} + + _send_email 'email-templates/dsn-unauthenticated' + _send_email 'email-templates/dsn-authenticated' '0.0.0.0 465' + _send_email 'email-templates/dsn-authenticated' '0.0.0.0 587' + _wait_for_empty_mail_queue_in_container + + # DSN requests are rejected regardless of origin. + # This is usually a bad idea, as you won't get them either. + _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log + assert_failure +} From eb7b1882e16b1c2076f2d5b77b3696fe17b27453 Mon Sep 17 00:00:00 2001 From: georglauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 22 Oct 2023 17:42:01 +0200 Subject: [PATCH 172/592] update `contributors.yml` action --- .github/workflows/contributors.yml | 75 +- CONTRIBUTORS.md | 1885 +--------------------------- 2 files changed, 15 insertions(+), 1945 deletions(-) diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index 9028a60b6d0..e784ae0e282 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -10,12 +10,6 @@ permissions: pull-requests: write statuses: write -env: - # Assign commit authorship to official Github Actions bot: - GIT_USER: github-actions[bot] - GIT_EMAIL: 41898282+github-actions[bot]@users.noreply.github.com - BRANCH_NAME: contributors-update - jobs: add-contributors: name: 'Add Contributors' @@ -24,64 +18,15 @@ jobs: - name: 'Checkout' uses: actions/checkout@v4 - - name: 'Checkout New Branch and Push It' - run: | - git checkout -b ${{ env.BRANCH_NAME }} - git push --force https://${GITHUB_ACTOR}:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git HEAD:${{ env.BRANCH_NAME }} - git checkout master - - # See https://github.com/marketplace/actions/auto-add-contributors for reference of the action. - # - # This action is not well documented, but it does the job for now. We pin the version in order - # to not have any issues in the future. - name: 'Update CONTRIBUTORS.md' - uses: BobAnkh/add-contributors@v0.2.2 - with: - ACCESS_TOKEN: ${{secrets.GITHUB_TOKEN}} - BRANCH: ${{ env.BRANCH_NAME }} - COMMIT_MESSAGE: 'docs: update `CONTRIBUTORS.md`' - PATH: /CONTRIBUTORS.md - CONTRIBUTOR: '## Contributors' - COLUMN_PER_ROW: 6 - IMG_WIDTH: 100 - FONT_SIZE: 14 - AVATAR_SHAPE: round - - # See https://github.com/marketplace/actions/create-pull-request for reference of the action. - - name: 'Create Pull Request' - uses: peter-evans/create-pull-request@v5.0.2 - id: create-pr - with: - token: ${{ secrets.GITHUB_TOKEN }} - base: master - branch: ${{ env.BRANCH_NAME }} - title: 'docs: update `CONTRIBUTORS.md`' - commit-message: 'docs: update `CONTRIBUTORS.md`' - delete-branch: true - committer: ${{ env.GIT_USER }} <${{ env.GIT_EMAIL }}> - author: ${{ env.GIT_USER }} <${{ env.GIT_EMAIL }}> - signoff: true - body: | - Updated `CONTRIBUTORS.md` via the CI workflow: [`contributors.yml`][workflow]. - - [workflow]: https://github.com/docker-mailserver/docker-mailserver/blob/master/.github/workflows/contributors.yml - - # See https://github.com/marketplace/actions/set-commit-status for reference of the action. - # - # GH Actions are limited when it comes to actions triggering other actions. Hence, - # this whole workflow will not trigger a `pull_request` event without a PAT. The lint - # workflow, which is required due to branch protection, is not important for this type - # of PR, so we skip it and pretend it was successful. - - name: 'Set Status for Linting Actions to Success (Skipped)' - uses: myrotvorets/set-commit-status-action@v2.0.0 - continue-on-error: true + uses: akhilmhdh/contributors-readme-action@v2.3.6 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - token: ${{ secrets.GITHUB_TOKEN }} - # Skipped workflows are still assigned a "success" status: - status: success - # This should be the correct commit SHA on ${{ env.BRANCH_NAME }}: - sha: ${{ steps.create-pr.outputs.pull-request-head-sha }} - # Name of status check to add/update: - context: lint - # Optional message/note we can inline to the right of the context name in the UI: - description: Lint skipped. Not relevant. + readme_path: CONTRIBUTORS.md + collaborators: all + commit_message: 'docs: updated `CONTRIBUTORS.md`' + committer_username: github-actions[bot] + committer_email: 41898282+github-actions[bot]@users.noreply.github.com + pr_title_on_protected: 'docs: update `CONTRIBUTORS.md' + auto_detect_branch_protection: true diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index cbd2653b7dc..08412f247f4 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1,1880 +1,9 @@ -Thanks goes to these wonderful people ✨ +# Contributors -## Contributors +Thanks goes to these wonderful people ✨ -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - Thomas -
- Thomas VIAL - - 		- - Georg -
- Georg Lauterbach - - 		- - Casper/ -
- Casper - - 		- - Erik -
- Erik Wramner - - 		- - Brennan -
- Brennan Kinney - - 		- - Jean-Denis -
- Jean-Denis Vauguet - -
- - Martin -
- Martin Schulze - - 		- - Frederic -
- Frederic Werner - - 		- - Josef -
- Josef Friedrich - - 		- - Johan -
- Johan Smits - - 		- - youtous/ -
- youtous - - 		- - 17Halbe/ -
- 17Halbe - -
- - Nathan -
- Nathan Pierce - - 		- - Thorsten -
- Thorsten von Eicken - - 		- - Germain -
- Germain Masse - - 		- - 00angus/ -
- 00angus - - 		- - Paul -
- Paul Steinlechner - - 		- - Dominik -
- Dominik Winter - -
- - Paul -
- Paul Adams - - 		- - Felix -
- Felix Bartels - - 		- - Sebastian -
- Sebastian Wiesendahl - - 		- - Steve -
- Steve Johnson - - 		- - André -
- André Stein - - 		- - William -
- William Desportes - -
- - omarc1492/ -
- omarc1492 - - 		- - Christian -
- Christian Glahn - - 		- - Marek -
- Marek Walczak - - 		- - Kai -
- Kai Ren - - 		- - Kyle -
- Kyle Ondy - - 		- - Michael/ -
- Michael - -
- - lukas/ -
- lukas - - 		- - Sascha -
- Sascha Scandella - - 		- - Lukáš -
- Lukáš Vasek - - 		- - Andreas -
- Andreas Perhab - - 		- - vortex852456/ -
- vortex852456 - - 		- - Christian -
- Christian Grasso - -
- - Hans-Cees -
- Hans-Cees Speel - - 		- - Jack -
- Jack Pearson - - 		- - Dashamir -
- Dashamir Hoxha - - 		- - GAVARD -
- GAVARD Ewann - - 		- - Jack -
- Jack Twilley - - 		- - Luke -
- Luke Cyca - -
- - Oleg -
- Oleg Kainov - - 		- - Robert -
- Robert Dolca - - 		- - Thomas -
- Thomas Kilian - - 		- - Tobias -
- Tobias Rittig - - 		- - akmet/ -
- akmet - - 		- - Arne -
- Arne Kepp - -
- - Dennis -
- Dennis Stumm - - 		- - Moritz -
- Moritz Marquardt - - 		- - pyy/ -
- pyy - - 		- - Anne/ -
- Anne - - 		- - Birkenstab/ -
- Birkenstab - - 		- - Brandon -
- Brandon Schmitt - -
- - Cédric -
- Cédric Laubacher - - 		- - GrupoCITEC/ -
- GrupoCITEC - - 		- - Jairo -
- Jairo Llopis - - 		- - James/ -
- James - - 		- - Jarrod -
- Jarrod Smith - - 		- - Patrizio -
- Patrizio Bekerle - -
- - Rubytastic2/ -
- Rubytastic2 - - 		- - Semir -
- Semir Patel - - 		- - Wolfgang -
- Wolfgang Ocker - - 		- - Zehir/ -
- Zehir - - 		- - guardiande/ -
- guardiande - - 		- - kamuri/ -
- kamuri - -
- - davidszp/ -
- davidszp - - 		- - Andreas -
- Andreas Gerstmayr - - 		- - Marko -
- Marko J - - 		- - Michael -
- Michael Schmoock - - 		- - VanVan/ -
- VanVan - - 		- - Alexander -
- Alexander Elbracht - -
- - Amin -
- Amin Vakil - - 		- - Andrew -
- Andrew Low - - 		- - Ask -
- Ask Bjørn Hansen - - 		- - Ben/ -
- Ben - - 		- - Christian -
- Christian Raue - - 		- - Daniel -
- Daniel Panteleit - -
- - Darren -
- Darren McGrandle - - 		- - Dominik -
- Dominik Bruhn - - 		- - DuncanvR/ -
- DuncanvR - - 		- - Emanuele -
- Emanuele Mazzotta - - 		- - FL42/ -
- FL42 - - 		- - Guillaume -
- Guillaume Simon - -
- - Ikko -
- Ikko Eltociear Ashimine - - 		- - James -
- James Fryer - - 		- - Millaguie/ -
- Millaguie - - 		- - Jeremy -
- Jeremy Shipman - - 		- - Jonas -
- Jonas Kalderstam - - 		- - Louis/ -
- Louis - -
- - martinwepner/ -
- martinwepner - - 		- - Michael -
- Michael Als - - 		- - Morgan -
- Morgan Kesler - - 		- - Pablo -
- Pablo Castorino - - 		- - Philipp -
- Philipp Fruck - - 		- - Rainer -
- Rainer Rillke - -
- - Bob -
- Bob Gregor - - 		- - r-pufky/ -
- r-pufky - - 		- - andymel/ -
- andymel - - 		- - bigpigeon/ -
- bigpigeon - - 		- - engelant/ -
- engelant - - 		- - j-marz/ -
- j-marz - -
- - lokipo/ -
- lokipo - - 		- - msheakoski/ -
- msheakoski - - 		- - Felix/ -
- Felix - - 		- - Leon -
- Leon Busch-George - - 		- - Marius -
- Marius Panneck - - 		- - Thomas -
- Thomas Willems - -
- - 0xflotus/ -
- 0xflotus - - 		- - Ivan -
- Ivan Fokeev - - 		- - 20th/ -
- 20th - - 		- - 2b/ -
- 2b - - 		- - Max:/ -
- Max: - - 		- - Achim -
- Achim Christ - -
- - Adrian -
- Adrian Pistol - - 		- - Alexander -
- Alexander Kachkaev - - 		- - Alexander -
- Alexander Neu - - 		- - Bedniakov -
- Bedniakov Aleksei - - 		- - Andreas -
- Andreas Egli - - 		- - Andrew -
- Andrew Cornford - -
- - Andrey -
- Andrey Likhodievskiy - - 		- - Arash -
- Arash Fatahzade - - 		- - Arthur -
- Arthur Outhenin-Chalandre - - 		- - Astro/ -
- Astro - - 		- - Benedict -
- Benedict Endemann - - 		- - Bogdan/ -
- Bogdan - -
- - Charles -
- Charles Harris - - 		- - Christian -
- Christian Musa - - 		- - Christoph/ -
- Christoph - - 		- - Claus -
- Claus Beerta - - 		- - Damian -
- Damian Moore - - 		- - espitall/ -
- espitall - -
- - Daniel -
- Daniel Karski - - 		- - Daniele -
- Daniele Bellavista - - 		- - Daniël -
- Daniël van den Berg - - 		- - Dingoz/ -
- Dingoz - - 		- - Dmitry -
- Dmitry R. - - 		- - Dorian -
- Dorian Ayllón - -
- - Edmond -
- Edmond Varga - - 		- - Eduard -
- Eduard Knysh - - 		- - Elisei -
- Elisei Roca - - 		- - Erick -
- Erick Calder - - 		- - Erik -
- Erik Brakkee - - 		- - Huncode/ -
- Huncode - -
- - Florian/ -
- Florian - - 		- - Florian -
- Florian Roks - - 		- - Franz -
- Franz Keferböck - - 		- - Frugan/ -
- Frugan - - 		- - Gabriel -
- Gabriel Euzet - - 		- - Gabriel -
- Gabriel Landais - -
- - GiovanH/ -
- GiovanH - - 		- - H4R0/ -
- H4R0 - - 		- - Harry -
- Harry Youd - - 		- - Hugues -
- Hugues Granger - - 		- - Ian -
- Ian Andrews - - 		- - Influencer/ -
- Influencer - -
- - jcalfee/ -
- jcalfee - - 		- - JS -
- JS Légaré - - 		- - Jeidnx/ -
- Jeidnx - - 		- - JiLleON/ -
- JiLleON - - 		- - Jiří -
- Jiří Kozlovský - - 		- - jmccl/ -
- jmccl - -
- - Jurek -
- Jurek Barth - - 		- - JOnathan -
- JOnathan duMonT - - 		- - Kaan/ -
- Kaan - - 		- - Karthik -
- Karthik K - - 		- - KCrawley/ -
- KCrawley - - 		- - Khue -
- Khue Doan - -
- - Lars -
- Lars Pötter - - 		- - Leo -
- Leo Winter - - 		- - Lin -
- Lin Han - - 		- - MadsRC/ -
- MadsRC - - 		- - Mathieu -
- Mathieu Brunot - - 		- - Maximilian -
- Maximilian Hippler - -
- - Michael -
- Michael G. - - 		- - Michael -
- Michael Jensen - - 		- - Michel -
- Michel Albert - - 		- - Mohammed -
- Mohammed Chotia - - 		- - Mohammed -
- Mohammed Noureldin - - 		- - Moritz -
- Moritz Poldrack - -
- - Naveen/ -
- Naveen - - 		- - Nicholas -
- Nicholas Pepper - - 		- - Nick -
- Nick Pappas - - 		- - Nils -
- Nils Knappmeier - - 		- - Olivier -
- Olivier Picquenot - - 		- - Orville -
- Orville Q. Song - -
- - Ovidiu -
- Ovidiu Predescu - - 		- - Petar -
- Petar Šegina - - 		- - Peter -
- Peter Hartmann - - 		- - Pierre-Yves -
- Pierre-Yves Rofes - - 		- - Remo -
- Remo E - - 		- - René -
- René Plötz - -
- - Roman -
- Roman Seyffarth - - 		- - Sam -
- Sam Collins - - 		- - Scott -
- Scott Weldon - - 		- - Sebastian -
- Sebastian Straub - - 		- - Serge -
- Serge van den Boom - - 		- - Sergey -
- Sergey Nazaryev - -
- - Shyim/ -
- Shyim - - 		- - Simon -
- Simon J Mudd - - 		- - Simon -
- Simon Schröter - - 		- - Stephan/ -
- Stephan - - 		- - Stig -
- Stig Otnes Kolstad - - 		- - Sven -
- Sven Kauber - -
- - Sylvain -
- Sylvain Benner - - 		- - Sylvain -
- Sylvain Dumont - - 		- - TechnicLab/ -
- TechnicLab - - 		- - Thomas -
- Thomas Schmit - - 		- - Tin/ -
- Tin - - 		- - Torben -
- Torben Weibert - -
- - Toru -
- Toru Hisai - - 		- - Trangar/ -
- Trangar - - 		- - Twist235/ -
- Twist235 - - 		- - Vasiliy -
- Vasiliy Gokoyev - - 		- - Victoria -
- Victoria Brekenfeld - - 		- - Vilius/ -
- Vilius - -
- - Wim/ -
- Wim - - 		- - Y.C.Huang/ -
- Y.C.Huang - - 		- - arcaine2/ -
- arcaine2 - - 		- - awb99/ -
- awb99 - - 		- - brainkiller/ -
- brainkiller - - 		- - cternes/ -
- cternes - -
- - dborowy/ -
- dborowy - - 		- - dimalo/ -
- dimalo - - 		- - eleith/ -
- eleith - - 		- - helmutundarnold/ -
- helmutundarnold - - 		- - hnws/ -
- hnws - - 		- - i-C-o-d-e-r/ -
- i-C-o-d-e-r - -
- - idaadi/ -
- idaadi - - 		- - ixeft/ -
- ixeft - - 		- - jjtt/ -
- jjtt - - 		- - jose -
- jose nazario - - 		- - landergate/ -
- landergate - - 		- - magnus -
- magnus anderssen - -
- - marios88/ -
- marios88 - - 		- - matrixes/ -
- matrixes - - 		- - mchamplain/ -
- mchamplain - - 		- - Jason -
- Jason Miller - - 		- - mplx/ -
- mplx - - 		- - odinis/ -
- odinis - -
- - okami/ -
- okami - - 		- - olaf-mandel/ -
- olaf-mandel - - 		- - ontheair81/ -
- ontheair81 - - 		- - pravynandas/ -
- pravynandas - - 		- - presocratics/ -
- presocratics - - 		- - rhyst/ -
- rhyst - -
- - schnippl0r/ -
- schnippl0r - - 		- - smargold476/ -
- smargold476 - - 		- - sportshead/ -
- sportshead - - 		- - squash/ -
- squash - - 		- - strarsis/ -
- strarsis - - 		- - tamueller/ -
- tamueller - -
- - vivacarvajalito/ -
- vivacarvajalito - - 		- - wolkenschieber/ -
- wolkenschieber - - 		- - worldworm/ -
- worldworm - -
+ + ## Further Contributors @@ -1898,8 +27,4 @@ Also thanks goes to these wonderful people, that have contributed in various oth This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! -____ - -Note: - -We started using [all-contributors](https://github.com/all-contributors/all-contributors) in July 2021. We will add contributors with their future PRs or Issues. Code contributions are added automatically. If you are [one of the 200+](https://github.com/docker-mailserver/docker-mailserver/graphs/contributors) that contributed to the project in the past and would like to see your name here too, please reach out! +Note: We started using [all-contributors](https://github.com/all-contributors/all-contributors) in July 2021. We will add contributors with their future PRs or Issues. Code contributions are added automatically. If you are [one of the 200+](https://github.com/docker-mailserver/docker-mailserver/graphs/contributors) that contributed to the project in the past and would like to see your name here too, please reach out! From d988d8a8d1be8bd07c271eaa9911f15f5cd033c7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 22 Oct 2023 18:11:16 +0200 Subject: [PATCH 173/592] docs: updated `CONTRIBUTORS.md` (#3596) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 1937 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1937 insertions(+) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 08412f247f4..538f3740018 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -3,6 +3,1943 @@ Thanks goes to these wonderful people ✨ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + casperklein +
+ Casper + + 		+ + fbartels +
+ Felix Bartels + + 		+ + NorseGaud +
+ Nathan Pierce + + 		+ + williamdes +
+ William Desportes + + 		+ + wernerfred +
+ Frederic Werner + + 		+ + georglauterbach +
+ Georg Lauterbach + +
+ + tomav +
+ Thomas VIAL + + 		+ + erik-wramner +
+ Erik Wramner + + 		+ + polarathene +
+ Brennan Kinney + + 		+ + chikamichi +
+ Jean-Denis Vauguet + + 		+ + martin-schulze-vireso +
+ Martin Schulze + + 		+ + Josef-Friedrich +
+ Josef Friedrich + +
+ + johansmitsnl +
+ Johan Smits + + 		+ + youtous +
+ Null + + 		+ + 17Halbe +
+ Null + + 		+ + tve +
+ Thorsten Von Eicken + + 		+ + gmasse +
+ Germain Masse + + 		+ + 00angus +
+ Null + +
+ + alinmear +
+ Paul Steinlechner + + 		+ + ap-wtioit +
+ Andreas Perhab + + 		+ + dominikwinter +
+ Dominik Winter + + 		+ + crazystick +
+ Paul Adams + + 		+ + swiesend +
+ Sebastian Wiesendahl + + 		+ + svenyonson +
+ Steve Johnson + +
+ + stonemaster +
+ André Stein + + 		+ + omarc1492 +
+ Null + + 		+ + phish108 +
+ Christian Glahn + + 		+ + mwlczk +
+ Marek Walczak + + 		+ + tyranron +
+ Kai Ren + + 		+ + KyleOndy +
+ Kyle Ondy + +
+ + MichaelSp +
+ Michael + + 		+ + mindrunner +
+ Lukas + + 		+ + m-a-v +
+ Sascha Scandella + + 		+ + bilak +
+ Lukáš Vasek + + 		+ + vortex852456 +
+ Null + + 		+ + chris54721 +
+ Christian Grasso + +
+ + hanscees +
+ Hans-Cees Speel + + 		+ + jrpear +
+ Jack Pearson + + 		+ + dashohoxha +
+ Dashamir Hoxha + + 		+ + egavard +
+ GAVARD Ewann + + 		+ + mathuin +
+ Jack Twilley + + 		+ + jamebus +
+ James + +
+ + lukecyca +
+ Luke Cyca + + 		+ + okainov +
+ Oleg Kainov + + 		+ + robertdolca +
+ Robert Dolca + + 		+ + kiliant +
+ Thomas Kilian + + 		+ + diiigle +
+ Tobias Rittig + + 		+ + akmet +
+ Akmet + +
+ + arneke +
+ Arne Kepp + + 		+ + dennis95stumm +
+ Dennis Stumm + + 		+ + moqmar +
+ Moritz Marquardt + + 		+ + pyy +
+ Null + + 		+ + voordev +
+ Anne + + 		+ + Birkenstab +
+ Null + +
+ + BrandonSchmitt +
+ Brandon Schmitt + + 		+ + Starbix +
+ Cédric Laubacher + + 		+ + citec +
+ GrupoCITEC + + 		+ + yajo +
+ Jairo Llopis + + 		+ + MakerMatrix +
+ Jarrod Smith + + 		+ + pbek +
+ Patrizio Bekerle + +
+ + Rubytastic2 +
+ Null + + 		+ + analogue +
+ Semir Patel + + 		+ + weo +
+ Wolfgang Ocker + + 		+ + Zehir +
+ Zehir + + 		+ + guardiande +
+ Null + + 		+ + kamuri +
+ Null + +
+ + davidszp +
+ Null + + 		+ + andreasgerstmayr +
+ Andreas Gerstmayr + + 		+ + mjung +
+ Marko J + + 		+ + m-schmoock +
+ Michael Schmoock + + 		+ + VanVan +
+ Null + + 		+ + elbracht +
+ Alexander Elbracht + +
+ + aminvakil +
+ Amin Vakil + + 		+ + andrewlow +
+ Andrew Low + + 		+ + abh +
+ Ask Bjørn Hansen + + 		+ + ubenmackin +
+ Ben + + 		+ + craue +
+ Christian Raue + + 		+ + danielpanteleit +
+ Daniel Panteleit + +
+ + dmcgrandle +
+ Darren McGrandle + + 		+ + theomega +
+ Dominik Bruhn + + 		+ + DuncanvR +
+ Null + + 		+ + emazzotta +
+ Emanuele Mazzotta + + 		+ + fl42 +
+ FL42 + + 		+ + ipernet +
+ Guillaume Simon + +
+ + H4R0 +
+ Null + + 		+ + eltociear +
+ Ikko Eltociear Ashimine + + 		+ + jamesfryer +
+ James Fryer + + 		+ + millaguie +
+ Millaguie + + 		+ + jedateach +
+ Jeremy Shipman + + 		+ + spacecowboy +
+ Jonas Kalderstam + +
+ + artonge +
+ Louis + + 		+ + martinwepner +
+ Null + + 		+ + nueaf +
+ Michael Als + + 		+ + keslerm +
+ Morgan Kesler + + 		+ + castorinop +
+ Pablo Castorino + + 		+ + p-fruck +
+ Philipp Fruck + +
+ + Rillke +
+ Rainer Rillke + + 		+ + reneploetz +
+ René Plötz + + 		+ + bobbravo2 +
+ Bob Gregor + + 		+ + r-pufky +
+ Robert Pufky + + 		+ + vincentDcmps +
+ Vincent Ducamps + + 		+ + andymel123 +
+ Andymel + +
+ + bigpigeon +
+ Bigpigeon + + 		+ + engelant +
+ Null + + 		+ + j-marz +
+ Null + + 		+ + lokipo +
+ Null + + 		+ + msheakoski +
+ Null + + 		+ + GoliathLabs +
+ Felix + +
+ + yogo1212 +
+ Leon Busch-George + + 		+ + mpanneck +
+ Marius Panneck + + 		+ + willtho89 +
+ Thomas Willems + + 		+ + tbutter +
+ Thomas Butter + + 		+ + 0xflotus +
+ 0xflotus + + 		+ + ifokeev +
+ Johan Fokeev + +
+ + 20th +
+ Null + + 		+ + 2b +
+ Null + + 		+ + askz +
+ Max: + + 		+ + acch +
+ Achim Christ + + 		+ + vifino +
+ Adrian Pistol + + 		+ + kachkaev +
+ Alexander Kachkaev + +
+ + alexanderneu +
+ Alexander Neu + + 		+ + ch3sh1r +
+ Bedniakov Aleksei + + 		+ + eglia +
+ Andreas Egli + + 		+ + groupmsl +
+ Andrew Cornford + + 		+ + green-anger +
+ Andrey Likhodievskiy + + 		+ + iRhonin +
+ Arash Fatahzade + +
+ + MrFreezeex +
+ Arthur Outhenin-Chalandre + + 		+ + arunvc +
+ Arun + + 		+ + astrocket +
+ Astro + + 		+ + baxerus +
+ Benedict Endemann + + 		+ + spock +
+ Bogdan + + 		+ + erdos4d +
+ Charles Harris + +
+ + crash7 +
+ Christian Musa + + 		+ + auchri +
+ Christoph + + 		+ + arkanovicz +
+ Claude Brisson + + 		+ + CBeerta +
+ Claus Beerta + + 		+ + damianmoore +
+ Damian Moore + + 		+ + espitall +
+ Null + +
+ + dkarski +
+ Daniel Karski + + 		+ + dbellavista +
+ Daniele Bellavista + + 		+ + danielvandenberg95 +
+ Daniël Van Den Berg + + 		+ + mlatorre31 +
+ Dingoz + + 		+ + mazzz1y +
+ Dmitry R. + + 		+ + aydodo +
+ Dorian Ayllón + +
+ + vedtam +
+ Edmond Varga + + 		+ + edvorg +
+ Eduard Knyshov + + 		+ + eliroca +
+ Elisei Roca + + 		+ + ekkis +
+ Erick Calder + + 		+ + ErikEngerd +
+ Erik Brakkee + + 		+ + huncode +
+ Huncode + +
+ + felixn +
+ Felix N + + 		+ + flole +
+ Florian + + 		+ + froks +
+ Florian Roks + + 		+ + fkefer +
+ Franz Keferböck + + 		+ + frugan-it +
+ Frugan + + 		+ + Marsu31 +
+ Gabriel Euzet + +
+ + glandais +
+ Gabriel Landais + + 		+ + GiovanH +
+ GiovanH + + 		+ + harryyoud +
+ Harry Youd + + 		+ + HeySora +
+ HeySora + + 		+ + sirgantrithon +
+ Ian Andrews + + 		+ + Influencer +
+ Influencer + +
+ + jcalfee +
+ Null + + 		+ + init-js +
+ JS Légaré + + 		+ + Jeidnx +
+ Jeidnx + + 		+ + JiLleON +
+ Null + + 		+ + jirislav +
+ Jiří Kozlovský + + 		+ + jmccl +
+ Null + +
+ + jurekbarth +
+ Jurek Barth + + 		+ + JOduMonT +
+ JOnathan DuMonT + + 		+ + Kaan88 +
+ Kaan + + 		+ + akkumar +
+ Karthik K + + 		+ + KCrawley +
+ Null + + 		+ + khuedoan +
+ Khue Doan + +
+ + JustAnother1 +
+ Lars Pötter + + 		+ + LeoWinterDE +
+ Leo Winter + + 		+ + linhandev +
+ Lin Han + + 		+ + luke- +
+ Lucas Bartholemy + + 		+ + LucidityCrash +
+ Null + + 		+ + MadsRC +
+ MadsRC + +
+ + madmath03 +
+ Mathieu Brunot + + 		+ + maxemann96 +
+ Maximilian Hippler + + 		+ + dragetd +
+ Michael G. + + 		+ + michaeljensen +
+ Michael Jensen + + 		+ + exhuma +
+ Michel Albert + + 		+ + milas +
+ Milas Bowman + +
+ + mcchots +
+ Mohammed Chotia + + 		+ + MohammedNoureldin +
+ Mohammed Noureldin + + 		+ + mpldr +
+ Moritz Poldrack + + 		+ + naveensrinivasan +
+ Naveen + + 		+ + neuralp +
+ Nicholas Pepper + + 		+ + radicand +
+ Nick Pappas + +
+ + nilshoell +
+ Nils Höll + + 		+ + nknapp +
+ Nils Knappmeier + + 		+ + pcqnt +
+ Olivier Picquenot + + 		+ + OrvilleQ +
+ Orville Q. Song + + 		+ + ovidiucp +
+ Ovidiu Predescu + + 		+ + mrPjer +
+ Petar Šegina + +
+ + peter-hartmann +
+ Peter Hartmann + + 		+ + piwai +
+ Pierre-Yves Rofes + + 		+ + remoe +
+ Remo E + + 		+ + romansey +
+ Roman Seyffarth + + 		+ + MightySCollins +
+ Sam Collins + + 		+ + 501st-alpha1 +
+ Scott Weldon + +
+ + klamann +
+ Sebastian Straub + + 		+ + svdb0 +
+ Serge Van Den Boom + + 		+ + 3ap +
+ Sergey Nazaryev + + 		+ + shyim +
+ Shyim + + 		+ + sjmudd +
+ Simon J Mudd + + 		+ + simonsystem +
+ Simon Schröter + +
+ + stephan-devop +
+ Stephan + + 		+ + stigok +
+ Stig Otnes Kolstad + + 		+ + 5ven +
+ Sven Kauber + + 		+ + syl20bnr +
+ Sylvain Benner + + 		+ + sylvaindumont +
+ Sylvain Dumont + + 		+ + TechnicLab +
+ Null + +
+ + thomasschmit +
+ Thomas Schmit + + 		+ + Thiritin +
+ Tin + + 		+ + tweibert +
+ Torben Weibert + + 		+ + torus +
+ Toru Hisai + + 		+ + VictorKoenders +
+ Trangar + + 		+ + Twist235 +
+ Null + +
+ + k3it +
+ Vasiliy Gokoyev + + 		+ + Drakulix +
+ Victoria Brekenfeld + + 		+ + vilisas +
+ Vilius + + 		+ + 42wim +
+ Wim + + 		+ + ShiriNmi1520 +
+ Y.C.Huang + + 		+ + allddd +
+ Allddd + +
+ + arcaine2 +
+ Null + + 		+ + awb99 +
+ Awb99 + + 		+ + brainkiller +
+ Null + + 		+ + cternes +
+ Null + + 		+ + dborowy +
+ Null + + 		+ + dimalo +
+ Null + +
+ + eleith +
+ Eleith + + 		+ + ghnp5 +
+ Null + + 		+ + helmutundarnold +
+ Null + + 		+ + hnws +
+ Null + + 		+ + i-C-o-d-e-r +
+ Null + + 		+ + idaadi +
+ Null + +
+ + ixeft +
+ Null + + 		+ + jjtt +
+ Null + + 		+ + paralax +
+ Jose Nazario + + 		+ + jpduyx +
+ Null + + 		+ + landergate +
+ Null + + 		+ + callmemagnus +
+ Magnus Anderssen + +
+ + marios88 +
+ Null + + 		+ + matrixes +
+ Null + + 		+ + mchamplain +
+ Mchamplain + + 		+ + millerjason +
+ Jason Miller + + 		+ + mplx +
+ Null + + 		+ + odinis +
+ Null + +
+ + okamidash +
+ Okami + + 		+ + olaf-mandel +
+ Null + + 		+ + ontheair81 +
+ Null + + 		+ + pravynandas +
+ Null + + 		+ + presocratics +
+ Null + + 		+ + rhyst +
+ Null + +
+ + rmlhuk +
+ Null + + 		+ + rriski +
+ Null + + 		+ + schnippl0r +
+ Null + + 		+ + smargold476 +
+ Null + + 		+ + sportshead +
+ Null + + 		+ + squash +
+ Null + +
+ + strarsis +
+ Null + + 		+ + tamueller +
+ Null + + 		+ + vivacarvajalito +
+ Null + + 		+ + wligtenberg +
+ Null + + 		+ + wolkenschieber +
+ Null + + 		+ + worldworm +
+ Null + +
## Further Contributors From cb62ce20e6f49d211a8f264b11feaed65582a338 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 24 Oct 2023 10:31:22 +0200 Subject: [PATCH 174/592] bugfix: change Rspamd DKIM default config location (#3597) Instead of using `etc/rspamd/override.d/dkim_signing.conf`, we will now be using `/tmp/docker-mailserver/rspamd/override.d/dkim_signing.conf`. The new location is persisted (and linked again during startup) and hence better suited. --- target/bin/rspamd-dkim | 55 ++++++++++++++----- .../startup/setup.d/security/rspamd.sh | 3 +- .../parallel/set1/spam_virus/rspamd_dkim.bats | 6 +- 3 files changed, 47 insertions(+), 17 deletions(-) diff --git a/target/bin/rspamd-dkim b/target/bin/rspamd-dkim index de7129de2f2..576c55ab063 100755 --- a/target/bin/rspamd-dkim +++ b/target/bin/rspamd-dkim @@ -138,24 +138,55 @@ function _parse_arguments() { shift 2 done + return 0 +} + +function _preflight_checks() { if [[ ${KEYTYPE} == 'ed25519' ]] && [[ ${KEYSIZE} -ne 2048 ]]; then _exit_with_error "Chosen keytype does not accept the 'keysize' argument" fi - return 0 -} + if [[ ! -d /tmp/docker-mailserver ]]; then + _log 'warn' "The directory '/tmp/docker-mailserver' does not seem to be mounted by a volume - the Rspamd (DKIM) configuration will not be persisted" + fi -function _create_keys() { # Note: Variables not marked with `local` are used # in other functions (after this function was called). - BASE_DIR='/tmp/docker-mailserver/rspamd/dkim' + # Also keep in sync with: target/scripts/startup/setup.d/security/rspamd.sh:__rspamd__run_early_setup_and_checks + local RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' + local RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' + readonly RSPAMD_DMS_DKIM_D="${RSPAMD_DMS_D}/dkim" + readonly RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d" + + mkdir -p "${RSPAMD_DMS_DKIM_D}" "${RSPAMD_DMS_OVERRIDE_D}" + chown _rspamd:_rspamd "${RSPAMD_DMS_DKIM_D}" + + # Mimmick target/scripts/startup/setup.d/security/rspamd.sh:__rspamd__run_early_setup_and_checks where + # ${RSPAMD_OVERRIDE_D} is linked to ${RSPAMD_DMS_OVERRIDE_D}, but not if + # + # 1. ${RSPAMD_OVERRIDE_D} has already been linked to ${RSPAMD_DMS_OVERRIDE_D} + # 2. ${RSPAMD_OVERRIDE_D} has contents already + # + # If 1. is true, then we're good since DMS' default setup linked the directory already and we will save + # a persisted location in every case. If 1. is false, 2. should be false as well since by default, + # ${RSPAMD_OVERRIDE_D} has no contents - we're good as well. What should logically never happen is + # that 1. is false but 2. is true; this case is caught nevertheless and a warning is emitted. + if [[ ! -h "${RSPAMD_OVERRIDE_D}" ]]; then + if rmdir "${RSPAMD_OVERRIDE_D}" &>/dev/null; then + ln -s "${RSPAMD_DMS_OVERRIDE_D}" "${RSPAMD_OVERRIDE_D}" + else + _log 'warn' "Could not link '${RSPAMD_OVERRIDE_D}' to '${RSPAMD_DMS_OVERRIDE_D}' (as '${RSPAMD_OVERRIDE_D}' does not appear to be empty, which is unexpected) - you will need to restart DMS for changes to take effect" + fi + fi +} +function _create_keys() { if [[ ${KEYTYPE} == 'rsa' ]]; then - local BASE_FILE_NAME="${BASE_DIR}/${KEYTYPE}-${KEYSIZE}-${SELECTOR}-${DOMAIN}" + local BASE_FILE_NAME="${RSPAMD_DMS_DKIM_D}/${KEYTYPE}-${KEYSIZE}-${SELECTOR}-${DOMAIN}" KEYTYPE_OPTIONS=('-b' "${KEYSIZE}") _log 'info' "Creating DKIM keys of type '${KEYTYPE}' and length '${KEYSIZE}' with selector '${SELECTOR}' for domain '${DOMAIN}'" else - local BASE_FILE_NAME="${BASE_DIR}/${KEYTYPE}-${SELECTOR}-${DOMAIN}" + local BASE_FILE_NAME="${RSPAMD_DMS_DKIM_D}/${KEYTYPE}-${SELECTOR}-${DOMAIN}" KEYTYPE_OPTIONS=('-t' "${KEYTYPE}") _log 'info' "Creating DKIM keys of type '${KEYTYPE}' with selector '${SELECTOR}' for domain '${DOMAIN}'" fi @@ -164,9 +195,6 @@ function _create_keys() { PUBLIC_KEY_DNS_FILE="${BASE_FILE_NAME}.public.dns.txt" PRIVATE_KEY_FILE="${BASE_FILE_NAME}.private.txt" - mkdir -p "${BASE_DIR}" - chown _rspamd:_rspamd "${BASE_DIR}" - # shellcheck disable=SC2310 if __do_as_rspamd_user rspamadm \ dkim_keygen \ @@ -186,8 +214,8 @@ function _create_keys() { function _check_permissions() { # shellcheck disable=SC2310 - if ! __do_as_rspamd_user ls "${BASE_DIR}" >/dev/null; then - _log 'warn' "The Rspamd user ('_rspamd') seems to be unable to list files in the keys directory ('${BASE_DIR}') - Rspamd may experience permission errors later" + if ! __do_as_rspamd_user ls "${RSPAMD_DMS_DKIM_D}" >/dev/null; then + _log 'warn' "The Rspamd user ('_rspamd') seems to be unable to list files in the keys directory ('${RSPAMD_DMS_DKIM_D}') - Rspamd may experience permission errors later" elif ! __do_as_rspamd_user cat "${PRIVATE_KEY_FILE}" >/dev/null; then _log 'warn' "The Rspamd user ('_rspamd') seems to be unable to read the private key file - Rspamd may experience permission errors later" else @@ -196,11 +224,11 @@ function _check_permissions() { } function _setup_default_signing_conf() { - local DEFAULT_CONFIG_FILE='/etc/rspamd/override.d/dkim_signing.conf' + local DEFAULT_CONFIG_FILE="${RSPAMD_DMS_OVERRIDE_D}/dkim_signing.conf" if [[ -f ${DEFAULT_CONFIG_FILE} ]]; then _log 'debug' "'${DEFAULT_CONFIG_FILE}' exists, not supplying a default" else - _log 'info' "Supplying a default configuration ('${DEFAULT_CONFIG_FILE}')" + _log 'info' "Supplying a default configuration (to '${DEFAULT_CONFIG_FILE}')" cat >"${DEFAULT_CONFIG_FILE}" << EOF # documentation: https://rspamd.com/doc/modules/dkim_signing.html @@ -254,6 +282,7 @@ function _final_steps() { _obtain_hostname_and_domainname _require_n_parameters_or_print_usage 0 "${@}" _parse_arguments "${@}" +_preflight_checks _create_keys _check_permissions _setup_default_signing_conf diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 4199b07747f..38b83444354 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -70,7 +70,7 @@ function __rspamd__run_early_setup_and_checks() { readonly RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' readonly RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' - local RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d/" + local RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d" readonly RSPAMD_DMS_OVERRIDE_D mkdir -p /var/lib/rspamd/ @@ -82,6 +82,7 @@ function __rspamd__run_early_setup_and_checks() { ln -s "${RSPAMD_DMS_OVERRIDE_D}" "${RSPAMD_OVERRIDE_D}" else __rspamd__log 'warn' "Could not remove '${RSPAMD_OVERRIDE_D}' (not empty?; not a directory?; did you restart properly?) - not linking '${RSPAMD_DMS_OVERRIDE_D}'" + __rspamd__log 'warn' "Note that using '${RSPAMD_DMS_OVERRIDE_D}' and placing files manually in '${RSPAMD_OVERRIDE_D}' is not supported" fi fi diff --git a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats index cd880e52575..76c301eabf7 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats @@ -4,8 +4,8 @@ load "${REPOSITORY_ROOT}/test/helper/setup" BATS_TEST_NAME_PREFIX='[Rspamd] (DKIM) ' CONTAINER_NAME='dms-test_rspamd-dkim' -DOMAIN_NAME='fixed.com' -SIGNING_CONF_FILE='/etc/rspamd/override.d/dkim_signing.conf' +DOMAIN_NAME='example.test' +SIGNING_CONF_FILE='/tmp/docker-mailserver/rspamd/override.d/dkim_signing.conf' function setup_file() { _init_with_defaults @@ -59,7 +59,7 @@ function teardown_file() { _default_teardown ; } __create_key assert_success __log_is_free_of_warnings_and_errors - assert_output --partial "Supplying a default configuration ('${SIGNING_CONF_FILE}')" + assert_output --partial "Supplying a default configuration (to '${SIGNING_CONF_FILE}')" refute_output --partial "'${SIGNING_CONF_FILE}' exists, not supplying a default" assert_output --partial "Finished DKIM key creation" _run_in_container_bash "[[ -f ${SIGNING_CONF_FILE} ]]" From 097dc6c9a4b48d5955c4d9c40c191bfb4e0ca29d Mon Sep 17 00:00:00 2001 From: Joerg Sonnenberger Date: Thu, 26 Oct 2023 02:22:36 +0200 Subject: [PATCH 175/592] docs(bin/setup): Add an example for an alias with multiple recipients (#3600) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- target/bin/addalias | 3 +++ 1 file changed, 3 insertions(+) diff --git a/target/bin/addalias b/target/bin/addalias index 60f8aa03e9b..9bcb6cec245 100755 --- a/target/bin/addalias +++ b/target/bin/addalias @@ -33,6 +33,9 @@ ${ORANGE}EXAMPLES${RESET} ${LWHITE}./setup.sh alias add alias@example.com recipient@example.com${RESET} Add the alias 'alias@example.com' for the mail account 'recipient@example.com'. + ${LWHITE}./setup.sh alias add alias@example.com 'recipient@example.com, another-recipient@example.com'${RESET} + Multiple recipients are separated by comma. + ${ORANGE}EXIT STATUS${RESET} Exit status is 0 if command was successful. If wrong arguments are provided or arguments contain errors, the script will exit early with exit status 1. From 5efd2497864ab9bcf185b7901503786054b4a791 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 29 Oct 2023 17:04:07 +0100 Subject: [PATCH 176/592] docs: updated `CONTRIBUTORS.md` (#3606) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 110 ++++++++++++++++++++++++++---------------------- 1 file changed, 59 insertions(+), 51 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 538f3740018..615cc80e10b 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1201,14 +1201,21 @@ Thanks goes to these wonderful people ✨ Jiří Kozlovský 		+ + jsonn +
+ Joerg Sonnenberger + +
jmccl
Null -
jurekbarth @@ -1243,15 +1250,15 @@ Thanks goes to these wonderful people ✨
Null -
khuedoan
Khue Doan -
JustAnother1 @@ -1286,15 +1293,15 @@ Thanks goes to these wonderful people ✨
Null -
MadsRC
MadsRC -
madmath03 @@ -1329,15 +1336,15 @@ Thanks goes to these wonderful people ✨
Michel Albert -
milas
Milas Bowman -
mcchots @@ -1372,15 +1379,15 @@ Thanks goes to these wonderful people ✨
Nicholas Pepper -
radicand
Nick Pappas -
nilshoell @@ -1415,15 +1422,15 @@ Thanks goes to these wonderful people ✨
Ovidiu Predescu -
mrPjer
Petar Šegina -
peter-hartmann @@ -1458,15 +1465,15 @@ Thanks goes to these wonderful people ✨
Sam Collins -
501st-alpha1
Scott Weldon -
klamann @@ -1501,15 +1508,15 @@ Thanks goes to these wonderful people ✨
Simon J Mudd -
simonsystem
Simon Schröter -
stephan-devop @@ -1544,15 +1551,15 @@ Thanks goes to these wonderful people ✨
Sylvain Dumont -
TechnicLab
Null -
thomasschmit @@ -1587,15 +1594,15 @@ Thanks goes to these wonderful people ✨
Trangar -
Twist235
Null -
k3it @@ -1630,15 +1637,15 @@ Thanks goes to these wonderful people ✨
Y.C.Huang -
allddd
Allddd -
arcaine2 @@ -1673,15 +1680,15 @@ Thanks goes to these wonderful people ✨
Null -
dimalo
Null -
eleith @@ -1716,15 +1723,15 @@ Thanks goes to these wonderful people ✨
Null -
idaadi
Null -
ixeft @@ -1759,15 +1766,15 @@ Thanks goes to these wonderful people ✨
Null -
callmemagnus
Magnus Anderssen -
marios88 @@ -1802,15 +1809,15 @@ Thanks goes to these wonderful people ✨
Null -
odinis
Null -
okamidash @@ -1845,15 +1852,15 @@ Thanks goes to these wonderful people ✨
Null -
rhyst
Null -
rmlhuk @@ -1888,15 +1895,15 @@ Thanks goes to these wonderful people ✨
Null -
squash
Null -
strarsis @@ -1931,7 +1938,8 @@ Thanks goes to these wonderful people ✨
Null -
worldworm From f674232f7116acd1d5b2758c52ab5f5827d4d928 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 30 Oct 2023 10:20:37 +0100 Subject: [PATCH 177/592] misc: final Rspamd adjustments for v13 (#3599) * outsource Rspamd ENVs into explicit helper This will allow us to uniformly source the helper and get the values from everywhere consistently. This is more than desirable since we will be using these values not only for the Rspamd setup, but also for DKIM management and during change-detection. * integrate Rspamd into changedetection We outsource one more function to reside in the helper script for Rspamd so that we can call this function from the Rspamd setup and from the changedetection functionality too. * realize deprecation of old commands file for Rspamd THIS IS A BREAKING CHANGE! This change realizes the log message: "Using old file location now (deprecated) - this will prevent startup in v13.0.0" Startup will now fail. * added '--force' option to Rspamd DKIM script * use new helper to get ENVs for Rspamd in DKIM script * remove the need for linking directories This was unnecessary, as explained in https://github.com/docker-mailserver/docker-mailserver/pull/3597#discussion_r1369413599 * Apply suggestions from code review review by @polarathene * apply more review feedback from @polarathene - - * update documentation --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 4 +- docs/content/config/security/rspamd.md | 4 +- target/bin/rspamd-dkim | 63 ++++----- target/scripts/check-for-changes.sh | 32 +++++ target/scripts/helpers/change-detection.sh | 6 + target/scripts/helpers/index.sh | 1 + target/scripts/helpers/rspamd.sh | 105 +++++++++++++++ .../startup/setup.d/security/rspamd.sh | 123 ++---------------- .../parallel/set1/spam_virus/rspamd_dkim.bats | 20 ++- .../parallel/set1/spam_virus/rspamd_full.bats | 5 +- 10 files changed, 206 insertions(+), 157 deletions(-) create mode 100644 target/scripts/helpers/rspamd.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index a2e15c77f51..0b875e2b39e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,11 +16,13 @@ All notable changes to this project will be documented in this file. The format If necessary, DSNs for authenticated users can be disabled via the `postfix-master.cf` override with the following contents: - ``` + ```cf submission/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn submissions/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn ``` +- using the old path for the Rspamd custom commands file (`/tmp/docker-mailserver/rspamd-modules.conf`), which was deprecated, will now prevent startup; use `/tmp/docker-mailserver/rspamd/custom-commands.conf` instead + ### Added - New environment variable `MARK_SPAM_AS_READ`. When set to `1`, marks incoming junk as "read" to avoid unwanted notification of junk as new mail ([#3489](https://github.com/docker-mailserver/docker-mailserver/pull/3489)) diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 94be2db9ffe..5867b1e8b45 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -107,11 +107,11 @@ DMS brings sane default settings for Rspamd. They are located at `/etc/rspamd/lo !!! question "What is [`docker-data/dms/config/`][docs-dms-config-volume]?" -If you want to overwrite the default settings and / or provide your own settings, you can place files at `docker-data/dms/config/rspamd/override.d/` (a directory that is linked to `/etc/rspamd/override.d/`, if it exists) to override Rspamd and DMS default settings. This directory will not do a complete file override, but a [forced override of the specific settings in that file][rspamd-docs-override-dir]. +If you want to overwrite the default settings and / or provide your own settings, you can place files at `docker-data/dms/config/rspamd/override.d/`. Files from this directory are copied to `/etc/rspamd/override.d/` during startup. These files [forcibly override][rspamd-docs-override-dir] Rspamd and DMS default settings. !!! warning "Clashing Overrides" - Note that when also [using the `rspamd-commands` file](#with-the-help-of-a-custom-file), files in `override.d` may be overwritten in case you adjust them manually and with the help of the file. + Note that when also [using the `custom-commands.conf` file](#with-the-help-of-a-custom-file), files in `override.d` may be overwritten in case you adjust them manually and with the help of the file. [rspamd-docs-override-dir]: https://www.rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories [docs-dms-config-volume]: ../../faq.md#what-about-the-docker-datadmsconfig-directory diff --git a/target/bin/rspamd-dkim b/target/bin/rspamd-dkim index 576c55ab063..357c2a9cab6 100755 --- a/target/bin/rspamd-dkim +++ b/target/bin/rspamd-dkim @@ -27,9 +27,10 @@ ${ORANGE}DESCRIPTION${RESET} ${ORANGE}OPTIONS${RESET} ${BLUE}Generic Program Information${RESET} - -v Enable verbose logging (setting the log level to 'debug'). - -vv Enable very verbose logging (setting the log level to 'trace'). - help Print the usage information. + -f | --force Overwrite existing files if there are any + -v Enable verbose logging (setting the log level to 'debug'). + -vv Enable very verbose logging (setting the log level to 'trace'). + help Print the usage information. ${BLUE}Configuration adjustments${RESET} keytype Set the type of key you want to use. @@ -69,6 +70,7 @@ function __do_as_rspamd_user() { } function _parse_arguments() { + FORCE=0 KEYTYPE='rsa' KEYSIZE='2048' SELECTOR='mail' @@ -112,6 +114,12 @@ function _parse_arguments() { exit 0 ;; + ( '-f' | '--force' ) + FORCE=1 + shift 1 + continue + ;; + ( '-vv' ) # shellcheck disable=SC2034 LOG_LEVEL='trace' @@ -132,7 +140,6 @@ function _parse_arguments() { __usage _exit_with_error "Unknown option(s) '${1}' ${2:+"and '${2}'"}" ;; - esac shift 2 @@ -150,34 +157,10 @@ function _preflight_checks() { _log 'warn' "The directory '/tmp/docker-mailserver' does not seem to be mounted by a volume - the Rspamd (DKIM) configuration will not be persisted" fi - # Note: Variables not marked with `local` are used - # in other functions (after this function was called). - # Also keep in sync with: target/scripts/startup/setup.d/security/rspamd.sh:__rspamd__run_early_setup_and_checks - local RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' - local RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' - readonly RSPAMD_DMS_DKIM_D="${RSPAMD_DMS_D}/dkim" - readonly RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d" + _rspamd_get_envs mkdir -p "${RSPAMD_DMS_DKIM_D}" "${RSPAMD_DMS_OVERRIDE_D}" chown _rspamd:_rspamd "${RSPAMD_DMS_DKIM_D}" - - # Mimmick target/scripts/startup/setup.d/security/rspamd.sh:__rspamd__run_early_setup_and_checks where - # ${RSPAMD_OVERRIDE_D} is linked to ${RSPAMD_DMS_OVERRIDE_D}, but not if - # - # 1. ${RSPAMD_OVERRIDE_D} has already been linked to ${RSPAMD_DMS_OVERRIDE_D} - # 2. ${RSPAMD_OVERRIDE_D} has contents already - # - # If 1. is true, then we're good since DMS' default setup linked the directory already and we will save - # a persisted location in every case. If 1. is false, 2. should be false as well since by default, - # ${RSPAMD_OVERRIDE_D} has no contents - we're good as well. What should logically never happen is - # that 1. is false but 2. is true; this case is caught nevertheless and a warning is emitted. - if [[ ! -h "${RSPAMD_OVERRIDE_D}" ]]; then - if rmdir "${RSPAMD_OVERRIDE_D}" &>/dev/null; then - ln -s "${RSPAMD_DMS_OVERRIDE_D}" "${RSPAMD_OVERRIDE_D}" - else - _log 'warn' "Could not link '${RSPAMD_OVERRIDE_D}' to '${RSPAMD_DMS_OVERRIDE_D}' (as '${RSPAMD_OVERRIDE_D}' does not appear to be empty, which is unexpected) - you will need to restart DMS for changes to take effect" - fi - fi } function _create_keys() { @@ -195,6 +178,16 @@ function _create_keys() { PUBLIC_KEY_DNS_FILE="${BASE_FILE_NAME}.public.dns.txt" PRIVATE_KEY_FILE="${BASE_FILE_NAME}.private.txt" + if [[ -f ${PUBLIC_KEY_FILE} ]] || [[ -f ${PUBLIC_KEY_DNS_FILE} ]] || [[ -f ${PRIVATE_KEY_FILE} ]]; then + if [[ ${FORCE} -eq 0 ]]; then + _log 'error' "Not overwriting existing files (use '--force' to overwrite existing files)" + exit 1 + else + _log 'info' "Overwriting existing files as the '--force' option was supplied" + rm "${PUBLIC_KEY_FILE}" "${PUBLIC_KEY_DNS_FILE}" "${PRIVATE_KEY_FILE}" + fi + fi + # shellcheck disable=SC2310 if __do_as_rspamd_user rspamadm \ dkim_keygen \ @@ -226,7 +219,7 @@ function _check_permissions() { function _setup_default_signing_conf() { local DEFAULT_CONFIG_FILE="${RSPAMD_DMS_OVERRIDE_D}/dkim_signing.conf" if [[ -f ${DEFAULT_CONFIG_FILE} ]]; then - _log 'debug' "'${DEFAULT_CONFIG_FILE}' exists, not supplying a default" + _log 'info' "'${DEFAULT_CONFIG_FILE}' exists, not supplying a default ('--force' does not overwrite this file, manual adjustment required)" else _log 'info' "Supplying a default configuration (to '${DEFAULT_CONFIG_FILE}')" cat >"${DEFAULT_CONFIG_FILE}" << EOF @@ -253,7 +246,15 @@ domain { } EOF - chown _rspamd:_rspamd "${DEFAULT_CONFIG_FILE}" + + # We copy here immediately in order to not rely on the changedetector - this way, users + # can immediately use the new keys. The file should not already exist in ${RSPAMD_OVERRIDE_D} + # since it would have been copied already. + cp "${DEFAULT_CONFIG_FILE}" "${RSPAMD_OVERRIDE_D}/dkim_signing.conf" + chown _rspamd:_rspamd "${DEFAULT_CONFIG_FILE}" "${RSPAMD_OVERRIDE_D}/dkim_signing.conf" + + _log 'debug' 'Restarting Rspamd as initial DKIM configuration was suppplied' + supervisorctl restart rspamd fi } diff --git a/target/scripts/check-for-changes.sh b/target/scripts/check-for-changes.sh index a8cdeb6870d..66417c09f32 100755 --- a/target/scripts/check-for-changes.sh +++ b/target/scripts/check-for-changes.sh @@ -21,6 +21,10 @@ source /etc/dms-settings # usage with DMS_HOSTNAME, which should remove the need to call this: _obtain_hostname_and_domainname +# This is a helper to properly set all Rspamd-related environment variables +# correctly and in one place. +_rspamd_get_envs + # verify checksum file exists; must be prepared by start-mailserver.sh if [[ ! -f ${CHKSUM_FILE} ]]; then _exit_with_error "'${CHKSUM_FILE}' is missing" 0 @@ -49,6 +53,7 @@ function _check_for_changes() { # Handle any changes _ssl_changes _postfix_dovecot_changes + _rspamd_changes _log_with_date 'debug' 'Reloading services due to detected changes' @@ -174,6 +179,33 @@ function _ssl_changes() { # They presently have no special handling other than to trigger a change that will restart Postfix/Dovecot. } +function _rspamd_changes() { + # RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' + if [[ ${CHANGED} =~ ${RSPAMD_DMS_D}/.* ]]; then + + # "${RSPAMD_DMS_D}/override.d" + if [[ ${CHANGED} =~ ${RSPAMD_DMS_OVERRIDE_D}/.* ]]; then + _log_with_date 'trace' 'Rspamd - Copying configuration overrides' + rm "${RSPAMD_OVERRIDE_D}"/* + cp "${RSPAMD_DMS_OVERRIDE_D}"/* "${RSPAMD_OVERRIDE_D}" + fi + + # "${RSPAMD_DMS_D}/custom-commands.conf" + if [[ ${CHANGED} =~ ${RSPAMD_DMS_CUSTOM_COMMANDS_F} ]]; then + _log_with_date 'trace' 'Rspamd - Generating new configuration from custom commands' + _rspamd_handle_user_modules_adjustments + fi + + # "${RSPAMD_DMS_D}/dkim" + if [[ ${CHANGED} =~ ${RSPAMD_DMS_DKIM_D} ]]; then + _log_with_date 'trace' 'Rspamd - DKIM files updated' + fi + + _log_with_date 'debug' 'Rspamd configuration has changed - restarting service' + supervisorctl restart rspamd + fi +} + while true; do _check_for_changes sleep 2 diff --git a/target/scripts/helpers/change-detection.sh b/target/scripts/helpers/change-detection.sh index e396bfe2b66..08f6906c260 100644 --- a/target/scripts/helpers/change-detection.sh +++ b/target/scripts/helpers/change-detection.sh @@ -40,6 +40,12 @@ function _monitored_files_checksums() { "${DMS_DIR}/dovecot-quotas.cf" "${DMS_DIR}/dovecot-masters.cf" ) + + # Check whether Rspamd is used and if so, monitor it's changes as well + if [[ ${ENABLE_RSPAMD} -eq 1 ]] && [[ -d ${RSPAMD_DMS_D} ]]; then + readarray -d '' STAGING_FILES_RSPAMD < <(find "${RSPAMD_DMS_D}" -type f -name "*.sh" -print0) + STAGING_FILES+=("${STAGING_FILES_RSPAMD[@]}") + fi fi # SSL certs: diff --git a/target/scripts/helpers/index.sh b/target/scripts/helpers/index.sh index 8d87673986f..0d77c9c4740 100644 --- a/target/scripts/helpers/index.sh +++ b/target/scripts/helpers/index.sh @@ -16,6 +16,7 @@ function _import_scripts() { source "${PATH_TO_SCRIPTS}/network.sh" source "${PATH_TO_SCRIPTS}/postfix.sh" source "${PATH_TO_SCRIPTS}/relay.sh" + source "${PATH_TO_SCRIPTS}/rspamd.sh" source "${PATH_TO_SCRIPTS}/ssl.sh" source "${PATH_TO_SCRIPTS}/utils.sh" diff --git a/target/scripts/helpers/rspamd.sh b/target/scripts/helpers/rspamd.sh new file mode 100644 index 00000000000..868e3d3aafe --- /dev/null +++ b/target/scripts/helpers/rspamd.sh @@ -0,0 +1,105 @@ +#! /bin/bash + +# shellcheck disable=SC2034 # VAR appears unused. + +function _rspamd_get_envs() { + readonly RSPAMD_LOCAL_D='/etc/rspamd/local.d' + readonly RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' + + readonly RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' + readonly RSPAMD_DMS_DKIM_D="${RSPAMD_DMS_D}/dkim" + readonly RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d" + + readonly RSPAMD_DMS_CUSTOM_COMMANDS_F="${RSPAMD_DMS_D}/custom-commands.conf" +} + +# Parses `RSPAMD_DMS_CUSTOM_COMMANDS_F` and executed the directives given by the file. +# To get a detailed explanation of the commands and how the file works, visit +# https://docker-mailserver.github.io/docker-mailserver/latest/config/security/rspamd/#with-the-help-of-a-custom-file +function _rspamd_handle_user_modules_adjustments() { + # Adds an option with a corresponding value to a module, or, in case the option + # is already present, overwrites it. + # + # @param ${1} = file name in ${RSPAMD_OVERRIDE_D}/ + # @param ${2} = module name as it should appear in the log + # @param ${3} = option name in the module + # @param ${4} = value of the option + # + # ## Note + # + # While this function is currently bound to the scope of `_rspamd_handle_user_modules_adjustments`, + # it is written in a versatile way (taking 4 arguments instead of assuming `ARGUMENT2` / `ARGUMENT3` + # are set) so that it may be used elsewhere if needed. + function __add_or_replace() { + local MODULE_FILE=${1:?Module file name must be provided} + local MODULE_LOG_NAME=${2:?Module log name must be provided} + local OPTION=${3:?Option name must be provided} + local VALUE=${4:?Value belonging to an option must be provided} + # remove possible whitespace at the end (e.g., in case ${ARGUMENT3} is empty) + VALUE=${VALUE% } + local FILE="${RSPAMD_OVERRIDE_D}/${MODULE_FILE}" + + readonly MODULE_FILE MODULE_LOG_NAME OPTION VALUE FILE + + [[ -f ${FILE} ]] || touch "${FILE}" + + if grep -q -E "${OPTION}.*=.*" "${FILE}"; then + __rspamd__log 'trace' "Overwriting option '${OPTION}' with value '${VALUE}' for ${MODULE_LOG_NAME}" + sed -i -E "s|([[:space:]]*${OPTION}).*|\1 = ${VALUE};|g" "${FILE}" + else + __rspamd__log 'trace' "Setting option '${OPTION}' for ${MODULE_LOG_NAME} to '${VALUE}'" + echo "${OPTION} = ${VALUE};" >>"${FILE}" + fi + } + + # We check for usage of the previous location of the commands file. + # TODO This can be removed after the release of v14.0.0. + local RSPAMD_DMS_CUSTOM_COMMANDS_F_OLD="${RSPAMD_DMS_D}-modules.conf" + readonly RSPAMD_DMS_CUSTOM_COMMANDS_F_OLD + if [[ -f ${RSPAMD_DMS_CUSTOM_COMMANDS_F_OLD} ]]; then + _dms_panic__general "Old custom command file location '${RSPAMD_DMS_CUSTOM_COMMANDS_F_OLD}' is deprecated (use '${RSPAMD_DMS_CUSTOM_COMMANDS_F}' now)" 'Rspamd setup' + fi + + if [[ -f "${RSPAMD_DMS_CUSTOM_COMMANDS_F}" ]]; then + __rspamd__log 'debug' "Found file '${RSPAMD_DMS_CUSTOM_COMMANDS_F}' - parsing and applying it" + + local COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3 + while read -r COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3; do + case "${COMMAND}" in + ('disable-module') + __rspamd__helper__enable_disable_module "${ARGUMENT1}" 'false' 'override' + ;; + + ('enable-module') + __rspamd__helper__enable_disable_module "${ARGUMENT1}" 'true' 'override' + ;; + + ('set-option-for-module') + __add_or_replace "${ARGUMENT1}.conf" "module '${ARGUMENT1}'" "${ARGUMENT2}" "${ARGUMENT3}" + ;; + + ('set-option-for-controller') + __add_or_replace 'worker-controller.inc' 'controller worker' "${ARGUMENT1}" "${ARGUMENT2} ${ARGUMENT3}" + ;; + + ('set-option-for-proxy') + __add_or_replace 'worker-proxy.inc' 'proxy worker' "${ARGUMENT1}" "${ARGUMENT2} ${ARGUMENT3}" + ;; + + ('set-common-option') + __add_or_replace 'options.inc' 'common options' "${ARGUMENT1}" "${ARGUMENT2} ${ARGUMENT3}" + ;; + + ('add-line') + __rspamd__log 'trace' "Adding complete line to '${ARGUMENT1}'" + echo "${ARGUMENT2}${ARGUMENT3+ ${ARGUMENT3}}" >>"${RSPAMD_OVERRIDE_D}/${ARGUMENT1}" + ;; + + (*) + __rspamd__log 'warn' "Command '${COMMAND}' is invalid" + continue + ;; + esac + done < <(_get_valid_lines_from_file "${RSPAMD_DMS_CUSTOM_COMMANDS_F}") + fi +} diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 38b83444354..19ce75dca46 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -1,12 +1,17 @@ #!/bin/bash -# Function called during global setup to handle the complete setup of Rspamd. +# This file is executed during startup of DMS. Hence, the `index.sh` helper has already +# been sourced, and thus, all helper functions from `rspamd.sh` are available. + +# Function called during global setup to handle the complete setup of Rspamd. Functions +# with a single `_` prefix are sourced from the `rspamd.sh` helper. function _setup_rspamd() { if _env_var_expect_zero_or_one 'ENABLE_RSPAMD' && [[ ${ENABLE_RSPAMD} -eq 1 ]]; then _log 'debug' 'Enabling and configuring Rspamd' __rspamd__log 'trace' '---------- Setup started ----------' - __rspamd__run_early_setup_and_checks # must run first + _rspamd_get_envs # must run first + __rspamd__run_early_setup_and_checks # must run second __rspamd__setup_logfile __rspamd__setup_redis __rspamd__setup_postfix @@ -16,7 +21,7 @@ function _setup_rspamd() { __rspamd__setup_greylisting __rspamd__setup_hfilter_group __rspamd__setup_check_authenticated - __rspamd__handle_user_modules_adjustments # must run last + _rspamd_handle_user_modules_adjustments # must run last __rspamd__log 'trace' '---------- Setup finished ----------' else @@ -64,26 +69,11 @@ EOF # Run miscellaneous early setup tasks and checks, such as creating files needed at runtime # or checking for other anti-spam/anti-virus software. function __rspamd__run_early_setup_and_checks() { - # Note: Variables not marked with `local` are - # used in other functions as well. - readonly RSPAMD_LOCAL_D='/etc/rspamd/local.d' - readonly RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' - readonly RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' - - local RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d" - readonly RSPAMD_DMS_OVERRIDE_D - mkdir -p /var/lib/rspamd/ : >/var/lib/rspamd/stats.ucl if [[ -d ${RSPAMD_DMS_OVERRIDE_D} ]]; then - __rspamd__log 'debug' "Found directory '${RSPAMD_DMS_OVERRIDE_D}' - linking it to '${RSPAMD_OVERRIDE_D}'" - if rmdir "${RSPAMD_OVERRIDE_D}" 2>/dev/null; then - ln -s "${RSPAMD_DMS_OVERRIDE_D}" "${RSPAMD_OVERRIDE_D}" - else - __rspamd__log 'warn' "Could not remove '${RSPAMD_OVERRIDE_D}' (not empty?; not a directory?; did you restart properly?) - not linking '${RSPAMD_DMS_OVERRIDE_D}'" - __rspamd__log 'warn' "Note that using '${RSPAMD_DMS_OVERRIDE_D}' and placing files manually in '${RSPAMD_OVERRIDE_D}' is not supported" - fi + cp "${RSPAMD_DMS_OVERRIDE_D}"/* "${RSPAMD_OVERRIDE_D}" fi if [[ ${ENABLE_AMAVIS} -eq 1 ]] || [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]]; then @@ -304,98 +294,3 @@ function __rspamd__setup_check_authenticated() { "${MODULE_FILE}" fi } - -# Parses `RSPAMD_CUSTOM_COMMANDS_FILE` and executed the directives given by the file. -# To get a detailed explanation of the commands and how the file works, visit -# https://docker-mailserver.github.io/docker-mailserver/edge/config/security/rspamd/#with-the-help-of-a-custom-file -function __rspamd__handle_user_modules_adjustments() { - # Adds an option with a corresponding value to a module, or, in case the option - # is already present, overwrites it. - # - # @param ${1} = file name in ${RSPAMD_OVERRIDE_D}/ - # @param ${2} = module name as it should appear in the log - # @param ${3} = option name in the module - # @param ${4} = value of the option - # - # ## Note - # - # While this function is currently bound to the scope of `__rspamd__handle_user_modules_adjustments`, - # it is written in a versatile way (taking 4 arguments instead of assuming `ARGUMENT2` / `ARGUMENT3` - # are set) so that it may be used elsewhere if needed. - function __add_or_replace() { - local MODULE_FILE=${1:?Module file name must be provided} - local MODULE_LOG_NAME=${2:?Module log name must be provided} - local OPTION=${3:?Option name must be provided} - local VALUE=${4:?Value belonging to an option must be provided} - # remove possible whitespace at the end (e.g., in case ${ARGUMENT3} is empty) - VALUE=${VALUE% } - local FILE="${RSPAMD_OVERRIDE_D}/${MODULE_FILE}" - - readonly MODULE_FILE MODULE_LOG_NAME OPTION VALUE FILE - - [[ -f ${FILE} ]] || touch "${FILE}" - - if grep -q -E "${OPTION}.*=.*" "${FILE}"; then - __rspamd__log 'trace' "Overwriting option '${OPTION}' with value '${VALUE}' for ${MODULE_LOG_NAME}" - sed -i -E "s|([[:space:]]*${OPTION}).*|\1 = ${VALUE};|g" "${FILE}" - else - __rspamd__log 'trace' "Setting option '${OPTION}' for ${MODULE_LOG_NAME} to '${VALUE}'" - echo "${OPTION} = ${VALUE};" >>"${FILE}" - fi - } - - local RSPAMD_CUSTOM_COMMANDS_FILE="${RSPAMD_DMS_D}/custom-commands.conf" - local RSPAMD_CUSTOM_COMMANDS_FILE_OLD="${RSPAMD_DMS_D}-modules.conf" - readonly RSPAMD_CUSTOM_COMMANDS_FILE RSPAMD_CUSTOM_COMMANDS_FILE_OLD - - # We check for usage of the previous location of the commands file. - # This can be removed after the release of v14.0.0. - if [[ -f ${RSPAMD_CUSTOM_COMMANDS_FILE_OLD} ]]; then - __rspamd__log 'warn' "Detected usage of old file location for modules adjustment ('${RSPAMD_CUSTOM_COMMANDS_FILE_OLD}') - please use the new location ('${RSPAMD_CUSTOM_COMMANDS_FILE}')" - __rspamd__log 'warn' "Using old file location now (deprecated) - this will prevent startup in v13.0.0" - RSPAMD_CUSTOM_COMMANDS_FILE=${RSPAMD_CUSTOM_COMMANDS_FILE_OLD} - fi - - if [[ -f "${RSPAMD_CUSTOM_COMMANDS_FILE}" ]]; then - __rspamd__log 'debug' "Found file '${RSPAMD_CUSTOM_COMMANDS_FILE}' - parsing and applying it" - - local COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3 - while read -r COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3; do - case "${COMMAND}" in - ('disable-module') - __rspamd__helper__enable_disable_module "${ARGUMENT1}" 'false' 'override' - ;; - - ('enable-module') - __rspamd__helper__enable_disable_module "${ARGUMENT1}" 'true' 'override' - ;; - - ('set-option-for-module') - __add_or_replace "${ARGUMENT1}.conf" "module '${ARGUMENT1}'" "${ARGUMENT2}" "${ARGUMENT3}" - ;; - - ('set-option-for-controller') - __add_or_replace 'worker-controller.inc' 'controller worker' "${ARGUMENT1}" "${ARGUMENT2} ${ARGUMENT3}" - ;; - - ('set-option-for-proxy') - __add_or_replace 'worker-proxy.inc' 'proxy worker' "${ARGUMENT1}" "${ARGUMENT2} ${ARGUMENT3}" - ;; - - ('set-common-option') - __add_or_replace 'options.inc' 'common options' "${ARGUMENT1}" "${ARGUMENT2} ${ARGUMENT3}" - ;; - - ('add-line') - __rspamd__log 'trace' "Adding complete line to '${ARGUMENT1}'" - echo "${ARGUMENT2}${ARGUMENT3+ ${ARGUMENT3}}" >>"${RSPAMD_OVERRIDE_D}/${ARGUMENT1}" - ;; - - (*) - __rspamd__log 'warn' "Command '${COMMAND}' is invalid" - continue - ;; - esac - done < <(_get_valid_lines_from_file "${RSPAMD_CUSTOM_COMMANDS_FILE}") - fi -} diff --git a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats index 76c301eabf7..215b334deb6 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats @@ -68,8 +68,14 @@ function teardown_file() { _default_teardown ; } local INITIAL_SHA512_SUM=$(_exec_in_container sha512sum "${SIGNING_CONF_FILE}") __create_key + assert_failure + assert_output --partial "Not overwriting existing files (use '--force' to overwrite existing files)" + + # the same as before, but with the '--force' option + __create_key 'rsa' 'mail' "${DOMAIN_NAME}" '2048' '--force' __log_is_free_of_warnings_and_errors refute_output --partial "Supplying a default configuration ('${SIGNING_CONF_FILE}')" + assert_output --partial "Overwriting existing files as the '--force' option was supplied" assert_output --partial "'${SIGNING_CONF_FILE}' exists, not supplying a default" assert_output --partial "Finished DKIM key creation" local SECOND_SHA512_SUM=$(_exec_in_container sha512sum "${SIGNING_CONF_FILE}") @@ -188,11 +194,15 @@ function __create_key() { local SELECTOR=${2:-mail} local DOMAIN=${3:-${DOMAIN_NAME}} local KEYSIZE=${4:-2048} - - _run_in_container setup config dkim \ - keytype "${KEYTYPE}" \ - keysize "${KEYSIZE}" \ - selector "${SELECTOR}" \ + local FORCE=${5:-} + + # Not quoting is intended here as we would othewise provide + # the argument "''" (empty string), which would cause errors + # shellcheck disable=SC2086 + _run_in_container setup config dkim ${FORCE} \ + keytype "${KEYTYPE}" \ + keysize "${KEYSIZE}" \ + selector "${SELECTOR}" \ domain "${DOMAIN}" } diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index 3fbf59d2b04..09d42d46ed1 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -66,12 +66,9 @@ function teardown_file() { _default_teardown ; } assert_output 'rspamd_milter = inet:localhost:11332' } -@test "'/etc/rspamd/override.d/' is linked correctly" { +@test "contents of '/etc/rspamd/override.d/' are copied" { local OVERRIDE_D='/etc/rspamd/override.d' - _run_in_container_bash "[[ -h ${OVERRIDE_D} ]]" - assert_success - _run_in_container_bash "[[ -f ${OVERRIDE_D}/testmodule_complicated.conf ]]" assert_success } From 8c0777b6691fbfc091c9c67d9eb525a45b2b5127 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 8 Nov 2023 10:49:36 +0100 Subject: [PATCH 178/592] docs: improve docs about how to work with logs (#3626) * improvide docs about how to work with logs Most importantly, 1. I added information on the recently adopted `less` / `nano` 2. I added information about `/var/log/mail/` * fix typos * Apply suggestions from code review * Update docs/content/config/debugging.md --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/config/debugging.md | 25 ++++++++++++++++--------- docs/content/config/security/rspamd.md | 2 +- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index a17fd5ffce9..24e415661db 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -29,22 +29,29 @@ These links may advise how the provider can unblock the port through additional 1. **Increase log verbosity**: Very helpful for troubleshooting problems during container startup. Set the environment variable [`LOG_LEVEL`][docs-environment-log-level] to `debug` or `trace`. 2. **Use error logs as a search query**: Try [finding an _existing issue_][gh-issues] or _search engine result_ from any errors in your container log output. Often you'll find answers or more insights. If you still need to open an issue, sharing links from your search may help us assist you. The mail server log can be acquired by running `docker log ` (_or `docker logs -f ` if you want to follow the log_). -3. **Understand the basics of mail servers**: Especially for beginners, make sure you read our [Introduction][docs-introduction] and [Usage][docs-usage] articles. -4. **Search the whole FAQ**: Our [FAQ][docs-faq] contains answers for common problems. Make sure you go through the list. -5. **Reduce the scope**: Ensure that you can run a basic setup of DMS first. Then incrementally restore parts of your original configuration until the problem is reproduced again. If you're new to DMS, it is common to find the cause is misunderstanding how to configure a minimal setup. +3. **Inspect the logs of the service that is failing**: We provide a dedicated paragraph on this topic [further down below](#logs). +4. **Understand the basics of mail servers**: Especially for beginners, make sure you read our [Introduction][docs-introduction] and [Usage][docs-usage] articles. +5. **Search the whole FAQ**: Our [FAQ][docs-faq] contains answers for common problems. Make sure you go through the list. +6. **Reduce the scope**: Ensure that you can run a basic setup of DMS first. Then incrementally restore parts of your original configuration until the problem is reproduced again. If you're new to DMS, it is common to find the cause is misunderstanding how to configure a minimal setup. ### Debug a running container -To get a shell inside the container run: `docker exec -it bash`. +#### General -If you need more flexibility than `docker logs` offers, within the container `/var/log/mail/mail.log` and `/var/log/supervisor/` are the most useful locations to get relevant DMS logs. Use the `tail` or `cat` commands to view their contents. - -To install additional software: +To get a shell inside the container run: `docker exec -it bash`. To install additional software, run: 1. `apt-get update` to update repository metadata. -2. `apt-get install ` +2. `apt-get install ` to install a package, e.g., `apt-get install neovim` if you want to use NeoVim instead of `nano` (which is shipped by default). + +#### Logs + +If you need more flexibility than what the `docker logs` command offers, then the most useful locations to get relevant DMS logs within the container are: + +- `/var/log/mail/mail.log` +- `/var/log/mail/mail/.log` +- `/var/log/supervisor/.log` -For example a text editor you can use in the terminal: `apt-get install nano` +You may use `nano` (a text editor) to edit files, while `less` (a file viewer) and `tail`/`cat` are useful tools to inspect the contents of logs. ## Compatibility diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 5867b1e8b45..442e5e78df8 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -71,7 +71,7 @@ DMS does not supply custom values for DNS servers to Rspamd. If you need to use ### Logs -You can find the Rspamd logs at `/var/log/mail/rspamd.log`, and the corresponding logs for [Redis](#persistence-with-redis), if it is enabled, at `/var/log/supervisor/rspamd-redis.log`. We recommend inspecting these logs (with `docker exec -it cat /var/log/mail/rspamd.log`) in case Rspamd does not work as expected. +You can find the Rspamd logs at `/var/log/mail/rspamd.log`, and the corresponding logs for [Redis](#persistence-with-redis), if it is enabled, at `/var/log/supervisor/rspamd-redis.log`. We recommend inspecting these logs (with `docker exec -it less /var/log/mail/rspamd.log`) in case Rspamd does not work as expected. ### Modules From 460f2d5be5b9ee52b3b30ace3a365a3854f5181a Mon Sep 17 00:00:00 2001 From: Joerg Sonnenberger Date: Wed, 8 Nov 2023 17:57:16 +0100 Subject: [PATCH 179/592] docs: correct Rspamd directory name (#3629) --- docs/content/config/advanced/optional-config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/config/advanced/optional-config.md b/docs/content/config/advanced/optional-config.md index 94097a8ea4d..8a43e4db537 100644 --- a/docs/content/config/advanced/optional-config.md +++ b/docs/content/config/advanced/optional-config.md @@ -12,7 +12,7 @@ This is a list of all configuration files and directories which are optional or - **sieve-pipe:** directory for sieve pipe scripts. (Docs: [Sieve][docs-sieve]) - **opendkim:** DKIM directory. Auto-configurable via [`setup.sh config dkim`][docs-setupsh]. (Docs: [DKIM][docs-dkim]) - **ssl:** SSL Certificate directory if `SSL_TYPE` is set to `self-signed` or `custom`. (Docs: [SSL][docs-ssl]) -- **Rspamd:** Override directory for custom settings when using Rspamd (Docs: [Rspamd][docs-rspamd-override-d]) +- **rspamd:** Override directory for custom settings when using Rspamd (Docs: [Rspamd][docs-rspamd-override-d]) ## Files From 290355cf5abc23fc0c5d95d79bd88f1084e7a2df Mon Sep 17 00:00:00 2001 From: Zepmann Date: Wed, 8 Nov 2023 22:18:17 +0100 Subject: [PATCH 180/592] docs: Add Dovecot Lua auth guide + required package (#3579) * Dovecot: add deb package dovecot-lua to support Lua scripting * Adding documentation for Lua authentication * Updated documentation and made a better distinction between Dovecot packages for officially supported features and for community supported features. --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/examples/use-cases/auth-lua.md | 162 ++++++++++++++++++++ docs/mkdocs.yml | 1 + target/scripts/build/packages.sh | 11 +- 3 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 docs/content/examples/use-cases/auth-lua.md diff --git a/docs/content/examples/use-cases/auth-lua.md b/docs/content/examples/use-cases/auth-lua.md new file mode 100644 index 00000000000..8258688589e --- /dev/null +++ b/docs/content/examples/use-cases/auth-lua.md @@ -0,0 +1,162 @@ +--- +title: 'Examples | Use Cases | Lua Authentication' +--- + +## Introduction + +Dovecot has the ability to let users create their own custom user provisioning and authentication providers in [Lua](https://en.wikipedia.org/wiki/Lua_(programming_language)#Syntax). This allows any data source that can be approached from Lua to be used for authentication, including web servers. It is possible to do more with Dovecot and Lua, but other use cases fall outside of the scope of this documentation page. + +!!! warning "Community contributed guide" + Dovecot authentication via Lua scripting is not officially supported in DMS. No assistance will be provided should you encounter any issues. + + DMS provides the required packages to support this guide. Note that these packages will be removed should they introduce any future maintenance burden. + + The example in this guide relies on the current way in which DMS works with Dovecot configuration files. Changes to this to accommodate new authentication methods such as OpenID Connect will likely break this example in the future. This guide is updated on a best-effort base. + +Dovecot's Lua support can be used for user provisioning (userdb functionality) and/or password verification (passdb functionality). Consider using other userdb and passdb options before considering Lua, since Lua does require the use of additional (unsupported) program code that might require maintenance when updating DMS. + +Each implementation of Lua-based authentication is custom. Therefore it is impossible to write documentation that covers every scenario. Instead, this page describes a single example scenario. If that scenario is followed, you will learn vital aspects that are necessary to kickstart your own Lua development: + +- How to override Dovecot's default configuration to disable parts that conflict with your scenario. +- How to make Dovecot use your Lua script. +- How to add your own Lua script and any libraries it uses. +- How to debug your Lua script. + +## The example scenario + +This scenario starts with [DMS being configured to use LDAP][docs::auth-ldap] for mailbox identification, user authorization and user authentication. In this scenario, [Nextcloud](https://nextcloud.com/) is also a service that uses the same LDAP server for user identification, authorization and authentication. + +The goal of this scenario is to have Dovecot not authenticate the user against LDAP, but against Nextcloud using an [application password](https://docs.nextcloud.com/server/latest/user_manual/en/session_management.html#managing-devices). The idea behind this is that a compromised mailbox password does not compromise the user's account entirely. To make this work, Nextcloud is configured to [deny the use of account passwords by clients](https://docs.nextcloud.com/server/latest/admin_manual/configuration_server/config_sample_php_parameters.html#token-auth-enforced) and to [disable account password reset through mail verification](https://docs.nextcloud.com/server/latest/admin_manual/configuration_server/config_sample_php_parameters.html#lost-password-link). + +If the application password is configured correctly, an adversary can only use it to access the user's mailbox on DMS, and CalDAV and CardDAV data on Nextcloud. File access through WebDAV can be disabled for the application password used to access mail. Having CalDAV and CardDAV compromised by the same password is a minor setback. If an adversary gets access to a Nextcloud application password through a device of the user, it is likely that the adversary also gets access to the user's calendars and contact lists anyway (locally or through the same account settings used for mail and CalDAV/CardDAV synchronization). The user's stored files in Nextcloud, the LDAP account password and any other services that rely on it would still be protected. A bonus is that a user is able to revoke and renew the mailbox password in Nextcloud for whatever reason, through a friendly user interface with all the security measures with which the Nextcloud instance is configured (e.g. verification of the current account password). + +A drawback of this method is that any (compromised) Nextcloud application password can be used to access the user's mailbox. This introduces a risk that a Nextcloud application password used for something else (e.g. WebDAV file access) is compromised and used to access the user's mailbox. Discussion of that risk and possible mitigations fall outside of the scope of this scenario. + +To answer the questions asked earlier for this specific scenario: + +1. Do I want to use Lua to identify mailboxes and verify that users are are authorized to use mail services? **No. Provisioning is done through LDAP.** +1. Do I want to use Lua to verify passwords that users authenticate with for IMAP/POP3/SMTP in their mail clients? **Yes. Password authentication is done through Lua against Nextcloud.** +1. If the answer is 'yes' to question 1 or 2: are there other methods that better facilitate my use case instead of custom scripts which rely on me being a developer and not just a user? **No. Only HTTP can be used to authenticate against Nextcloud, which is not supported natively by Dovecot or DMS.** + +While it is possible to extend the authentication methods which Nextcloud can facilitate with [Nextcloud apps](https://apps.nextcloud.com/), there is currently a mismatch between what DMS supports and what Nextcloud applications can provide. This might change in the future. For now, Lua will be used to bridge the gap between DMS and Nextcloud for authentication only (Dovecot passdb), while LDAP will still be used to identify mailboxes and verify authorization (Dovecot userdb). + +## Modify Dovecot's configuration + +???+ example "Add to DMS volumes in `compose.yaml`" + + ```yaml + # All new volumes are marked :ro to configure them as read-only, since their contents are not changed from inside the container + volumes: + # Configuration override to disable LDAP authentication + - ./docker-data/dms/config/dovecot/auth-ldap.conf.ext:/etc/dovecot/conf.d/auth-ldap.conf.ext:ro + # Configuration addition to enable Lua authentication + - ./docker-data/dms/config/dovecot/auth-lua-httpbasic.conf:/etc/dovecot/conf.d/auth-lua-httpbasic.conf:ro + # Directory containing Lua scripts + - ./docker-data/dms/config/dovecot/lua/:/etc/dovecot/lua/:ro + ``` + +Create a directory for Lua scripts: +```bash +mkdir -p ./docker-data/dms/config/dovecot/lua +``` + +Create configuration file `./docker-data/dms/config/dovecot/auth-ldap.conf.ext` for LDAP user provisioning: +``` +userdb { + driver = ldap + args = /etc/dovecot/dovecot-ldap.conf.ext +} +``` + +Create configuration file `./docker-data/dms/config/dovecot/auth-lua-httpbasic.conf` for Lua user authentication: +``` +passdb { + driver = lua + args = file=/etc/dovecot/lua/auth-httpbasic.lua blocking=yes +} +``` + +That is all for configuring Dovecot. + +## Create the Lua script + +Create Lua file `./docker-data/dms/config/dovecot/lua/auth-httpbasic.lua` with contents: + +```lua +local http_url = "https://nextcloud.example.com/remote.php/dav/" +local http_method = "PROPFIND" +local http_status_ok = 207 +local http_status_failure = 401 +local http_header_forwarded_for = "X-Forwarded-For" + +package.path = package.path .. ";/etc/dovecot/lua/?.lua" +local base64 = require("base64") + +local http_client = dovecot.http.client { + timeout = 1000; + max_attempts = 1; + debug = false; +} + +function script_init() + return 0 +end + +function script_deinit() +end + +function auth_passdb_lookup(req) + local auth_request = http_client:request { + url = http_url; + method = http_method; + } + auth_request:add_header("Authorization", "Basic " .. (base64.encode(req.user .. ":" .. req.password))) + auth_request:add_header(http_header_forwarded_for, req.remote_ip) + local auth_response = auth_request:submit() + local resp_status = auth_response:status() + local reason = auth_response:reason() + + local returnStatus = dovecot.auth.PASSDB_RESULT_INTERNAL_FAILURE + local returnDesc = http_method .. " - " .. http_url .. " - " .. resp_status .. " " .. reason + if resp_status == http_status_ok + then + returnStatus = dovecot.auth.PASSDB_RESULT_OK + returnDesc = "nopassword=y" + elseif resp_status == http_status_failure + then + returnStatus = dovecot.auth.PASSDB_RESULT_PASSWORD_MISMATCH + returnDesc = "" + end + return returnStatus, returnDesc +end +``` + +Replace the hostname in the URL to the actual hostname of Nextcloud. + +Dovecot [provides an HTTP client for use in Lua](https://doc.dovecot.org/admin_manual/lua/#dovecot.http.client). Aside of that, Lua by itself is pretty barebones. It chooses library compactness over included functionality. You can see that in that a separate library is referenced to add support for Base64 encoding, which is required for [HTTP basic access authentication](https://en.wikipedia.org/wiki/Basic_access_authentication). This library (also a Lua script) is not included. It must be downloaded and stored in the same directory: + +```bash +cd ./docker-data/dms/config/dovecot/lua +curl -JLO https://raw.githubusercontent.com/iskolbin/lbase64/master/base64.lua +``` + +Only use native (pure Lua) libraries as dependencies if possible, such as `base64.lua` from the example. This ensures maximum compatibility. Performance is less of an issue since Lua scripts written for Dovecot probably won't be long or complex, and there won't be a lot of data processing by Lua itself. + +## Debugging a Lua script + +To see which Lua version is used by Dovecot if you plan to do something that is version dependent, run: + +```bash +docker exec CONTAINER_NAME strings /usr/lib/dovecot/libdovecot-lua.so|grep '^LUA_' +``` + +While Dovecot logs the status of authentication attempts for any passdb backend, Dovecot will also log Lua scripting errors and messages sent to Dovecot's [Lua API log functions](https://doc.dovecot.org/admin_manual/lua/#dovecot.i_debug). The combined DMS log (including that of Dovecot) can be viewed using `docker logs CONTAINER_NAME`. If the log is too noisy (_due to other processes in the container also logging to it_), `docker exec CONTAINER_NAME cat /var/log/mail/mail.log` can be used to view the log of Dovecot and Postfix specifically. + +If working with HTTP in Lua, setting `debug = true;` when initiating `dovecot.http.client` will create debug log messages for every HTTP request and response. + +Note that Lua runs compiled bytecode, and that scripts will be compiled when they are initially started. Once compiled, the bytecode is cached and changes in the Lua script will not be processed automatically. Dovecot will reload its configuration and clear its cached Lua bytecode when running `docker exec CONTAINER_NAME dovecot reload`. A (changed) Lua script will be compiled to bytecode the next time it is executed after running the Dovecot reload command. + +[docs::auth-ldap]: ../../config/advanced/auth-ldap.md +[docs::dovecot-override-configuration]: ../../config/advanced/override-defaults/dovecot.md#override-configuration +[docs::dovecot-add-configuration]: ../../config/advanced/override-defaults/dovecot.md#add-configuration +[docs::faq-alter-running-dms-instance-without-container-relaunch]: ../../faq.md#how-to-alter-a-running-dms-instance-without-relaunching-the-container diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index ba920774f14..fdc2bc5950a 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -163,6 +163,7 @@ nav: - 'Forward-Only Mail-Server with LDAP': examples/use-cases/forward-only-mailserver-with-ldap-authentication.md - 'Customize IMAP Folders': examples/use-cases/imap-folders.md - 'iOS Mail Push Support': examples/use-cases/ios-mail-push-support.md + - 'Lua Authentication': examples/use-cases/auth-lua.md - 'FAQ' : faq.md - 'Contributing': - 'General Information': contributing/general.md diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index e9b2d479d75..a025c3b4e08 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -95,13 +95,19 @@ function _install_packages() { function _install_dovecot() { declare -a DOVECOT_PACKAGES + # Dovecot packages for officially supported features. DOVECOT_PACKAGES=( dovecot-core dovecot-imapd dovecot-ldap dovecot-lmtpd dovecot-managesieved dovecot-pop3d dovecot-sieve dovecot-solr ) - if [[ ${DOVECOT_COMMUNITY_REPO} -eq 1 ]]; then + # Dovecot packages for community supported features. + DOVECOT_PACKAGES+=(dovecot-auth-lua) + + # Dovecot's deb community repository only provides x86_64 packages, so do not include it + # when building for another architecture. + if [[ ${DOVECOT_COMMUNITY_REPO} -eq 1 ]] && [[ "$(uname --machine)" == "x86_64" ]]; then _log 'trace' 'Using Dovecot community repository' curl https://repo.dovecot.org/DOVECOT-REPO-GPG | gpg --import gpg --export ED409DA1 > /etc/apt/trusted.gpg.d/dovecot.gpg @@ -109,6 +115,9 @@ function _install_dovecot() { _log 'trace' 'Updating Dovecot package signatures' apt-get "${QUIET}" update + + # Additional community package needed for Lua support if the Dovecot community repository is used. + DOVECOT_PACKAGES+=(dovecot-lua) fi _log 'debug' 'Installing Dovecot' From 0703e014925824a07ba8da8239e7cb2468405b03 Mon Sep 17 00:00:00 2001 From: Joerg Sonnenberger Date: Fri, 10 Nov 2023 01:03:21 +0100 Subject: [PATCH 181/592] docs: Clarify default for ENV `FETCHMAIL_PARALLEL` (#3603) - Make this easier to find when browsing the example environment file. - Adjust ENV documentation to properly mark the actual default value. --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/config/environment.md | 6 ++++-- mailserver.env | 4 ++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 678ec9654d8..284549f13b7 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -577,8 +577,10 @@ Note: activate this only if you are confident in your bayes database for identif ##### FETCHMAIL_PARALLEL - **0** => `fetchmail` runs with a single config file `/etc/fetchmailrc` - **1** => `/etc/fetchmailrc` is split per poll entry. For every poll entry a separate fetchmail instance is started to allow having multiple imap idle configurations defined. +- **0** => `fetchmail` runs with a single config file `/etc/fetchmailrc` +- 1 => `/etc/fetchmailrc` is split per poll entry. For every poll entry a separate fetchmail instance is started to [allow having multiple imap idle connections per server][fetchmail-imap-workaround] (_when poll entries reference the same IMAP server_). + +[fetchmail-imap-workaround]: https://otremba.net/wiki/Fetchmail_(Debian)#Immediate_Download_via_IMAP_IDLE Note: The defaults of your fetchmailrc file need to be at the top of the file. Otherwise it won't be added correctly to all separate `fetchmail` instances. #### Getmail diff --git a/mailserver.env b/mailserver.env index 1a57cecab62..957a632e128 100644 --- a/mailserver.env +++ b/mailserver.env @@ -397,6 +397,10 @@ ENABLE_FETCHMAIL=0 # The interval to fetch mail in seconds FETCHMAIL_POLL=300 +# Use multiple fetchmail instances (1 per poll entry in fetchmail.cf) +# Supports multiple IMAP IDLE connections when a server is used across multiple poll entries +# https://otremba.net/wiki/Fetchmail_(Debian)#Immediate_Download_via_IMAP_IDLE +FETCHMAIL_PARALLEL=0 # Enable or disable `getmail`. # From 26214491efb2ec07242c24a9191633f0d72998d9 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 10 Nov 2023 19:57:17 +0100 Subject: [PATCH 182/592] fix: Drop special bits from Postfix `maildrop/` and `public/` directory permissions (#3625) * update K8s deployment Because `allowPrivilegeEscalation` controls SUID/SGID, we require it when postdrop is invoked. * correct permissions for maildrop/public The reason our permissions previously worked out as that in setups where SUID/SGID worked, the binaries used to place files in these directories already have SGID set; the current set of permissions makes less sense (as explained in this comment: https://github.com/docker-mailserver/docker-mailserver/issues/3619#issuecomment-1793816412) Since the binaries used to place files inside these directories alredy have SUID/SGID set, we do not require these bits (or the sticky bit) to be set on the directories. * Apply suggestions from code review --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/config/advanced/kubernetes.md | 5 ++++- target/scripts/startup/setup.d/mail_state.sh | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/content/config/advanced/kubernetes.md b/docs/content/config/advanced/kubernetes.md index 93cc0884557..8a47bffcace 100644 --- a/docs/content/config/advanced/kubernetes.md +++ b/docs/content/config/advanced/kubernetes.md @@ -190,7 +190,10 @@ spec: imagePullPolicy: IfNotPresent securityContext: - allowPrivilegeEscalation: false + # Required to support SGID via `postdrop` executable + # in `/var/mail-state` for Postfix (maildrop + public dirs): + # https://github.com/docker-mailserver/docker-mailserver/pull/3625 + allowPrivilegeEscalation: true readOnlyRootFilesystem: false runAsUser: 0 runAsGroup: 0 diff --git a/target/scripts/startup/setup.d/mail_state.sh b/target/scripts/startup/setup.d/mail_state.sh index ffc3179106c..73c2515be30 100644 --- a/target/scripts/startup/setup.d/mail_state.sh +++ b/target/scripts/startup/setup.d/mail_state.sh @@ -105,10 +105,10 @@ function _setup_save_states() { # These two require the postdrop(103) group: chgrp -R postdrop "${STATEDIR}"/spool-postfix/{maildrop,public} - # After changing the group, special bits (set-gid, sticky) may be stripped, restore them: - # Ref: https://github.com/docker-mailserver/docker-mailserver/pull/3149#issuecomment-1454981309 - chmod 1730 "${STATEDIR}/spool-postfix/maildrop" - chmod 2710 "${STATEDIR}/spool-postfix/public" + # These permissions rely on the `postdrop` binary having the SGID bit set. + # Ref: https://github.com/docker-mailserver/docker-mailserver/pull/3625 + chmod 730 "${STATEDIR}/spool-postfix/maildrop" + chmod 710 "${STATEDIR}/spool-postfix/public" elif [[ ${ONE_DIR} -eq 1 ]]; then _log 'warn' "'ONE_DIR=1' but no volume was mounted to '${STATEDIR}'" else From 5f2fb72c9ccfc1f24b581ac6f7f80837ed434c66 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 13 Nov 2023 12:34:46 +0100 Subject: [PATCH 183/592] Rspamd: add check for DKIM private key files' permissions (#3627) * added check for Rspamd DKIM on startup The newly added function `__rspamd__check_dkim_permissions` performs a check on DKIM private key files. This is useful to prevent issues like #3621 in the future. The function is deliberately kept simple and may not catch every single misconfiguration in terms of permissions and ownership, but it should be quite accurate. Please note that the Rspamd setup does NOT change at all, and the checks will not abort the setup in case they fail. A simple warning is emmited. * add more documentation to Rspamd functions * Apply suggestions from code review * improve `__do_as_rspamd_user` * rework check similar to review suggestion see https://github.com/docker-mailserver/docker-mailserver/pull/3627#discussion_r1388697547 --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- target/bin/rspamd-dkim | 7 ---- target/scripts/helpers/rspamd.sh | 12 ++++++ .../startup/setup.d/security/rspamd.sh | 41 +++++++++++++++++++ 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/target/bin/rspamd-dkim b/target/bin/rspamd-dkim index 357c2a9cab6..6dfcc1a0803 100755 --- a/target/bin/rspamd-dkim +++ b/target/bin/rspamd-dkim @@ -62,13 +62,6 @@ ${ORANGE}EXIT STATUS${RESET} " } -function __do_as_rspamd_user() { - local COMMAND=${1:?Command required when using __do_as_rspamd_user} - _log 'trace' "Running '${*}' as user '_rspamd' now" - shift 1 - su -l '_rspamd' -s "$(command -v "${COMMAND}")" -- "${@}" -} - function _parse_arguments() { FORCE=0 KEYTYPE='rsa' diff --git a/target/scripts/helpers/rspamd.sh b/target/scripts/helpers/rspamd.sh index 868e3d3aafe..2f4dcc46137 100644 --- a/target/scripts/helpers/rspamd.sh +++ b/target/scripts/helpers/rspamd.sh @@ -2,6 +2,18 @@ # shellcheck disable=SC2034 # VAR appears unused. +# Perform a specific command as the Rspamd user (`_rspamd`). This is useful +# in case you want to have correct permissions on newly created files or if +# you want to check whether Rspamd can perform a specific action. +function __do_as_rspamd_user() { + _log 'trace' "Running '${*}' as user '_rspamd'" + su _rspamd -s /bin/bash -c "${*}" +} + +# Calling this function brings common Rspamd-related environment variables +# into the current context. The environment variables are `readonly`, i.e. +# they cannot be modified. Use this function when you require common directory +# names, file names, etc. function _rspamd_get_envs() { readonly RSPAMD_LOCAL_D='/etc/rspamd/local.d' readonly RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 19ce75dca46..239397e5925 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -23,6 +23,9 @@ function _setup_rspamd() { __rspamd__setup_check_authenticated _rspamd_handle_user_modules_adjustments # must run last + # only performing checks, no further setup handled from here onwards + __rspamd__check_dkim_permissions + __rspamd__log 'trace' '---------- Setup finished ----------' else _log 'debug' 'Rspamd is disabled' @@ -280,6 +283,12 @@ function __rspamd__setup_hfilter_group() { fi } +# If 'RSPAMD_CHECK_AUTHENTICATED' is enabled, then content checks for all users, i.e. +# also for authenticated users, are performed. +# +# The default that DMS ships does not check authenticated users. In case the checks are +# enabled, this function will remove the part of the Rspamd configuration that disables +# checks for authenticated users. function __rspamd__setup_check_authenticated() { local MODULE_FILE="${RSPAMD_LOCAL_D}/settings.conf" readonly MODULE_FILE @@ -294,3 +303,35 @@ function __rspamd__setup_check_authenticated() { "${MODULE_FILE}" fi } + +# This function performs a simple check: go through DKIM configuration files, acquire +# all private key file locations and check whether they exist and whether they can be +# accessed by Rspamd. +function __rspamd__check_dkim_permissions() { + local DKIM_CONF_FILES DKIM_KEY_FILES + [[ -f ${RSPAMD_LOCAL_D}/dkim_signing.conf ]] && DKIM_CONF_FILES+=("${RSPAMD_LOCAL_D}/dkim_signing.conf") + [[ -f ${RSPAMD_OVERRIDE_D}/dkim_signing.conf ]] && DKIM_CONF_FILES+=("${RSPAMD_OVERRIDE_D}/dkim_signing.conf") + + # Here, we populate DKIM_KEY_FILES which we later iterate over. DKIM_KEY_FILES + # contains all keys files configured by the user. + local FILE + for FILE in "${DKIM_CONF_FILES[@]}"; do + readarray -t DKIM_KEY_FILES_TMP < <(grep -o -E 'path = .*' "${FILE}" | cut -d '=' -f 2 | tr -d ' ";') + DKIM_KEY_FILES+=("${DKIM_KEY_FILES_TMP[@]}") + done + + for FILE in "${DKIM_KEY_FILES[@]}"; do + if [[ -f ${FILE} ]]; then + __rspamd__log 'trace' "Checking DKIM file '${FILE}'" + # See https://serverfault.com/a/829314 for an explanation on `-exec false {} +` + # We additionally resolve symbolic links to check the permissions of the actual files + if find "$(realpath -eL "${FILE}")" -user _rspamd -or -group _rspamd -or -perm -o=r -exec false {} +; then + __rspamd__log 'warn' "Rspamd DKIM private key file '${FILE}' does not appear to have correct permissions/ownership for Rspamd to use it" + else + __rspamd__log 'trace' "DKIM file '${FILE}' permissions and ownership appear correct" + fi + else + __rspamd__log 'warn' "Rspamd DKIM private key file '${FILE}' is configured for usage, but does not appear to exist" + fi + done +} From d2efedf91c8622c6db09bd3fd524b1643e92fe1b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 13:23:21 +0000 Subject: [PATCH 184/592] docs: updated `CONTRIBUTORS.md` (#3637) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 173 +++++++++++++++++++++++++----------------------- 1 file changed, 90 insertions(+), 83 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 615cc80e10b..3fcdd463fa9 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -427,14 +427,21 @@ Thanks goes to these wonderful people ✨ Jarrod Smith + + jsonn +
+ Joerg Sonnenberger + +
pbek
Patrizio Bekerle -
Rubytastic2 @@ -469,15 +476,15 @@ Thanks goes to these wonderful people ✨
Null -
kamuri
Null -
davidszp @@ -512,15 +519,15 @@ Thanks goes to these wonderful people ✨
Null -
elbracht
Alexander Elbracht -
aminvakil @@ -555,15 +562,15 @@ Thanks goes to these wonderful people ✨
Christian Raue -
danielpanteleit
Daniel Panteleit -
dmcgrandle @@ -598,15 +605,15 @@ Thanks goes to these wonderful people ✨
FL42 -
ipernet
Guillaume Simon -
H4R0 @@ -641,15 +648,15 @@ Thanks goes to these wonderful people ✨
Jeremy Shipman -
spacecowboy
Jonas Kalderstam -
artonge @@ -684,15 +691,15 @@ Thanks goes to these wonderful people ✨
Pablo Castorino -
p-fruck
Philipp Fruck -
Rillke @@ -727,15 +734,15 @@ Thanks goes to these wonderful people ✨
Vincent Ducamps -
andymel123
Andymel -
bigpigeon @@ -770,15 +777,15 @@ Thanks goes to these wonderful people ✨
Null -
GoliathLabs
Felix -
yogo1212 @@ -813,15 +820,15 @@ Thanks goes to these wonderful people ✨
0xflotus -
ifokeev
Johan Fokeev -
20th @@ -856,15 +863,15 @@ Thanks goes to these wonderful people ✨
Adrian Pistol -
kachkaev
Alexander Kachkaev -
alexanderneu @@ -899,15 +906,15 @@ Thanks goes to these wonderful people ✨
Andrey Likhodievskiy -
iRhonin
Arash Fatahzade -
MrFreezeex @@ -942,15 +949,15 @@ Thanks goes to these wonderful people ✨
Bogdan -
erdos4d
Charles Harris -
crash7 @@ -985,15 +992,15 @@ Thanks goes to these wonderful people ✨
Damian Moore -
espitall
Null -
dkarski @@ -1028,15 +1035,15 @@ Thanks goes to these wonderful people ✨
Dmitry R. -
aydodo
Dorian Ayllón -
vedtam @@ -1071,15 +1078,15 @@ Thanks goes to these wonderful people ✨
Erik Brakkee -
huncode
Huncode -
felixn @@ -1109,20 +1116,20 @@ Thanks goes to these wonderful people ✨ - - frugan-it + + frugan-dev
Frugan -
Marsu31
Gabriel Euzet -
glandais @@ -1157,15 +1164,15 @@ Thanks goes to these wonderful people ✨
Ian Andrews -
Influencer
Influencer -
jcalfee @@ -1200,13 +1207,6 @@ Thanks goes to these wonderful people ✨
Jiří Kozlovský - 		- - jsonn -
- Joerg Sonnenberger -
@@ -1639,6 +1639,13 @@ Thanks goes to these wonderful people ✨
+ + Zepmann +
+ Null + + 		allddd @@ -1673,15 +1680,15 @@ Thanks goes to these wonderful people ✨
Null -
dborowy
Null -
dimalo @@ -1716,15 +1723,15 @@ Thanks goes to these wonderful people ✨
Null -
i-C-o-d-e-r
Null -
idaadi @@ -1759,15 +1766,15 @@ Thanks goes to these wonderful people ✨
Null -
landergate
Null -
callmemagnus @@ -1802,15 +1809,15 @@ Thanks goes to these wonderful people ✨
Jason Miller -
mplx
Null -
odinis @@ -1845,15 +1852,15 @@ Thanks goes to these wonderful people ✨
Null -
presocratics
Null -
rhyst @@ -1888,15 +1895,15 @@ Thanks goes to these wonderful people ✨
Null -
sportshead
Null -
squash @@ -1931,15 +1938,15 @@ Thanks goes to these wonderful people ✨
Null -
wolkenschieber
Null -
worldworm From f5a7e9d119ff944de8cfbc2fa70abdb2b8b79bc0 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 14 Nov 2023 16:15:39 +0100 Subject: [PATCH 185/592] contributors: fix spelling & remove dedicated AllContributors section (#3638) With the latest `contributors.yml` workflow, everyone is included in the list of contributors. Hence, we do not need the extra section anymore. --- .github/workflows/contributors.yml | 3 ++- CONTRIBUTORS.md | 24 ------------------------ 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index e784ae0e282..6f8476f274b 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -25,8 +25,9 @@ jobs: with: readme_path: CONTRIBUTORS.md collaborators: all + use_username: true commit_message: 'docs: updated `CONTRIBUTORS.md`' committer_username: github-actions[bot] committer_email: 41898282+github-actions[bot]@users.noreply.github.com - pr_title_on_protected: 'docs: update `CONTRIBUTORS.md' + pr_title_on_protected: 'docs: update `CONTRIBUTORS.md`' auto_detect_branch_protection: true diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 3fcdd463fa9..b7b7ce3538c 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1956,27 +1956,3 @@ Thanks goes to these wonderful people ✨
- -## Further Contributors - -Also thanks goes to these wonderful people, that have contributed in various other ways than code lines ✨ - -[Emoji Key ✨ (and Contribution Types)](https://allcontributors.org/docs/en/emoji-key) - - - - - - - - -

matrixes
📝
- - - - - - -This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! - -Note: We started using [all-contributors](https://github.com/all-contributors/all-contributors) in July 2021. We will add contributors with their future PRs or Issues. Code contributions are added automatically. If you are [one of the 200+](https://github.com/docker-mailserver/docker-mailserver/graphs/contributors) that contributed to the project in the past and would like to see your name here too, please reach out! From 218b56b123b6b74d6ac604db37ba21ae00090f2b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 16:16:52 +0100 Subject: [PATCH 186/592] docs: updated `CONTRIBUTORS.md` (#3639) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 530 ++++++++++++++++++++++++------------------------ 1 file changed, 265 insertions(+), 265 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index b7b7ce3538c..893b9572733 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -9,42 +9,42 @@ Thanks goes to these wonderful people ✨ casperklein
- Casper + casperklein fbartels
- Felix Bartels + fbartels NorseGaud
- Nathan Pierce + NorseGaud williamdes
- William Desportes + williamdes wernerfred
- Frederic Werner + wernerfred georglauterbach
- Georg Lauterbach + georglauterbach @@ -52,42 +52,42 @@ Thanks goes to these wonderful people ✨ tomav
- Thomas VIAL + tomav erik-wramner
- Erik Wramner + erik-wramner polarathene
- Brennan Kinney + polarathene chikamichi
- Jean-Denis Vauguet + chikamichi martin-schulze-vireso
- Martin Schulze + martin-schulze-vireso Josef-Friedrich
- Josef Friedrich + Josef-Friedrich @@ -95,42 +95,42 @@ Thanks goes to these wonderful people ✨ johansmitsnl
- Johan Smits + johansmitsnl youtous
- Null + youtous 17Halbe
- Null + 17Halbe tve
- Thorsten Von Eicken + tve gmasse
- Germain Masse + gmasse 00angus
- Null + 00angus @@ -138,42 +138,42 @@ Thanks goes to these wonderful people ✨ alinmear
- Paul Steinlechner + alinmear ap-wtioit
- Andreas Perhab + ap-wtioit dominikwinter
- Dominik Winter + dominikwinter crazystick
- Paul Adams + crazystick swiesend
- Sebastian Wiesendahl + swiesend svenyonson
- Steve Johnson + svenyonson @@ -181,42 +181,42 @@ Thanks goes to these wonderful people ✨ stonemaster
- André Stein + stonemaster omarc1492
- Null + omarc1492 phish108
- Christian Glahn + phish108 mwlczk
- Marek Walczak + mwlczk tyranron
- Kai Ren + tyranron KyleOndy
- Kyle Ondy + KyleOndy @@ -224,42 +224,42 @@ Thanks goes to these wonderful people ✨ MichaelSp
- Michael + MichaelSp mindrunner
- Lukas + mindrunner m-a-v
- Sascha Scandella + m-a-v bilak
- Lukáš Vasek + bilak vortex852456
- Null + vortex852456 chris54721
- Christian Grasso + chris54721 @@ -267,42 +267,42 @@ Thanks goes to these wonderful people ✨ hanscees
- Hans-Cees Speel + hanscees jrpear
- Jack Pearson + jrpear dashohoxha
- Dashamir Hoxha + dashohoxha egavard
- GAVARD Ewann + egavard mathuin
- Jack Twilley + mathuin jamebus
- James + jamebus @@ -310,42 +310,42 @@ Thanks goes to these wonderful people ✨ lukecyca
- Luke Cyca + lukecyca okainov
- Oleg Kainov + okainov robertdolca
- Robert Dolca + robertdolca kiliant
- Thomas Kilian + kiliant diiigle
- Tobias Rittig + diiigle akmet
- Akmet + akmet @@ -353,42 +353,42 @@ Thanks goes to these wonderful people ✨ arneke
- Arne Kepp + arneke dennis95stumm
- Dennis Stumm + dennis95stumm moqmar
- Moritz Marquardt + moqmar pyy
- Null + pyy voordev
- Anne + voordev Birkenstab
- Null + Birkenstab @@ -396,42 +396,42 @@ Thanks goes to these wonderful people ✨ BrandonSchmitt
- Brandon Schmitt + BrandonSchmitt Starbix
- Cédric Laubacher + Starbix citec
- GrupoCITEC + citec yajo
- Jairo Llopis + yajo MakerMatrix
- Jarrod Smith + MakerMatrix jsonn
- Joerg Sonnenberger + jsonn @@ -439,28 +439,28 @@ Thanks goes to these wonderful people ✨ pbek
- Patrizio Bekerle + pbek Rubytastic2
- Null + Rubytastic2 analogue
- Semir Patel + analogue weo
- Wolfgang Ocker + weo @@ -474,7 +474,7 @@ Thanks goes to these wonderful people ✨ guardiande
- Null + guardiande @@ -482,42 +482,42 @@ Thanks goes to these wonderful people ✨ kamuri
- Null + kamuri davidszp
- Null + davidszp andreasgerstmayr
- Andreas Gerstmayr + andreasgerstmayr mjung
- Marko J + mjung m-schmoock
- Michael Schmoock + m-schmoock VanVan
- Null + VanVan @@ -525,42 +525,42 @@ Thanks goes to these wonderful people ✨ elbracht
- Alexander Elbracht + elbracht aminvakil
- Amin Vakil + aminvakil andrewlow
- Andrew Low + andrewlow abh
- Ask Bjørn Hansen + abh ubenmackin
- Ben + ubenmackin craue
- Christian Raue + craue @@ -568,42 +568,42 @@ Thanks goes to these wonderful people ✨ danielpanteleit
- Daniel Panteleit + danielpanteleit dmcgrandle
- Darren McGrandle + dmcgrandle theomega
- Dominik Bruhn + theomega DuncanvR
- Null + DuncanvR emazzotta
- Emanuele Mazzotta + emazzotta fl42
- FL42 + fl42 @@ -611,42 +611,42 @@ Thanks goes to these wonderful people ✨ ipernet
- Guillaume Simon + ipernet H4R0
- Null + H4R0 eltociear
- Ikko Eltociear Ashimine + eltociear jamesfryer
- James Fryer + jamesfryer millaguie
- Millaguie + millaguie jedateach
- Jeremy Shipman + jedateach @@ -654,42 +654,42 @@ Thanks goes to these wonderful people ✨ spacecowboy
- Jonas Kalderstam + spacecowboy artonge
- Louis + artonge martinwepner
- Null + martinwepner nueaf
- Michael Als + nueaf keslerm
- Morgan Kesler + keslerm castorinop
- Pablo Castorino + castorinop @@ -697,42 +697,42 @@ Thanks goes to these wonderful people ✨ p-fruck
- Philipp Fruck + p-fruck Rillke
- Rainer Rillke + Rillke reneploetz
- René Plötz + reneploetz bobbravo2
- Bob Gregor + bobbravo2 r-pufky
- Robert Pufky + r-pufky vincentDcmps
- Vincent Ducamps + vincentDcmps @@ -740,42 +740,42 @@ Thanks goes to these wonderful people ✨ andymel123
- Andymel + andymel123 bigpigeon
- Bigpigeon + bigpigeon engelant
- Null + engelant j-marz
- Null + j-marz lokipo
- Null + lokipo msheakoski
- Null + msheakoski @@ -783,35 +783,35 @@ Thanks goes to these wonderful people ✨ GoliathLabs
- Felix + GoliathLabs yogo1212
- Leon Busch-George + yogo1212 mpanneck
- Marius Panneck + mpanneck willtho89
- Thomas Willems + willtho89 tbutter
- Thomas Butter + tbutter @@ -826,42 +826,42 @@ Thanks goes to these wonderful people ✨ ifokeev
- Johan Fokeev + ifokeev 20th
- Null + 20th 2b
- Null + 2b askz
- Max: + askz acch
- Achim Christ + acch vifino
- Adrian Pistol + vifino @@ -869,42 +869,42 @@ Thanks goes to these wonderful people ✨ kachkaev
- Alexander Kachkaev + kachkaev alexanderneu
- Alexander Neu + alexanderneu ch3sh1r
- Bedniakov Aleksei + ch3sh1r eglia
- Andreas Egli + eglia groupmsl
- Andrew Cornford + groupmsl green-anger
- Andrey Likhodievskiy + green-anger @@ -912,42 +912,42 @@ Thanks goes to these wonderful people ✨ iRhonin
- Arash Fatahzade + iRhonin MrFreezeex
- Arthur Outhenin-Chalandre + MrFreezeex arunvc
- Arun + arunvc astrocket
- Astro + astrocket baxerus
- Benedict Endemann + baxerus spock
- Bogdan + spock @@ -955,42 +955,42 @@ Thanks goes to these wonderful people ✨ erdos4d
- Charles Harris + erdos4d crash7
- Christian Musa + crash7 auchri
- Christoph + auchri arkanovicz
- Claude Brisson + arkanovicz CBeerta
- Claus Beerta + CBeerta damianmoore
- Damian Moore + damianmoore @@ -998,42 +998,42 @@ Thanks goes to these wonderful people ✨ espitall
- Null + espitall dkarski
- Daniel Karski + dkarski dbellavista
- Daniele Bellavista + dbellavista danielvandenberg95
- Daniël Van Den Berg + danielvandenberg95 mlatorre31
- Dingoz + mlatorre31 mazzz1y
- Dmitry R. + mazzz1y @@ -1041,42 +1041,42 @@ Thanks goes to these wonderful people ✨ aydodo
- Dorian Ayllón + aydodo vedtam
- Edmond Varga + vedtam edvorg
- Eduard Knyshov + edvorg eliroca
- Elisei Roca + eliroca ekkis
- Erick Calder + ekkis ErikEngerd
- Erik Brakkee + ErikEngerd @@ -1084,42 +1084,42 @@ Thanks goes to these wonderful people ✨ huncode
- Huncode + huncode felixn
- Felix N + felixn flole
- Florian + flole froks
- Florian Roks + froks fkefer
- Franz Keferböck + fkefer frugan-dev
- Frugan + frugan-dev @@ -1127,14 +1127,14 @@ Thanks goes to these wonderful people ✨ Marsu31
- Gabriel Euzet + Marsu31 glandais
- Gabriel Landais + glandais @@ -1148,7 +1148,7 @@ Thanks goes to these wonderful people ✨ harryyoud
- Harry Youd + harryyoud @@ -1162,7 +1162,7 @@ Thanks goes to these wonderful people ✨ sirgantrithon
- Ian Andrews + sirgantrithon @@ -1177,14 +1177,14 @@ Thanks goes to these wonderful people ✨ jcalfee
- Null + jcalfee init-js
- JS Légaré + init-js @@ -1198,14 +1198,14 @@ Thanks goes to these wonderful people ✨ JiLleON
- Null + JiLleON jirislav
- Jiří Kozlovský + jirislav @@ -1213,42 +1213,42 @@ Thanks goes to these wonderful people ✨ jmccl
- Null + jmccl jurekbarth
- Jurek Barth + jurekbarth JOduMonT
- JOnathan DuMonT + JOduMonT Kaan88
- Kaan + Kaan88 akkumar
- Karthik K + akkumar KCrawley
- Null + KCrawley @@ -1256,42 +1256,42 @@ Thanks goes to these wonderful people ✨ khuedoan
- Khue Doan + khuedoan JustAnother1
- Lars Pötter + JustAnother1 LeoWinterDE
- Leo Winter + LeoWinterDE linhandev
- Lin Han + linhandev luke-
- Lucas Bartholemy + luke- LucidityCrash
- Null + LucidityCrash @@ -1306,35 +1306,35 @@ Thanks goes to these wonderful people ✨ madmath03
- Mathieu Brunot + madmath03 maxemann96
- Maximilian Hippler + maxemann96 dragetd
- Michael G. + dragetd michaeljensen
- Michael Jensen + michaeljensen exhuma
- Michel Albert + exhuma @@ -1342,42 +1342,42 @@ Thanks goes to these wonderful people ✨ milas
- Milas Bowman + milas mcchots
- Mohammed Chotia + mcchots MohammedNoureldin
- Mohammed Noureldin + MohammedNoureldin mpldr
- Moritz Poldrack + mpldr naveensrinivasan
- Naveen + naveensrinivasan neuralp
- Nicholas Pepper + neuralp @@ -1385,42 +1385,42 @@ Thanks goes to these wonderful people ✨ radicand
- Nick Pappas + radicand nilshoell
- Nils Höll + nilshoell nknapp
- Nils Knappmeier + nknapp pcqnt
- Olivier Picquenot + pcqnt OrvilleQ
- Orville Q. Song + OrvilleQ ovidiucp
- Ovidiu Predescu + ovidiucp @@ -1428,42 +1428,42 @@ Thanks goes to these wonderful people ✨ mrPjer
- Petar Šegina + mrPjer peter-hartmann
- Peter Hartmann + peter-hartmann piwai
- Pierre-Yves Rofes + piwai remoe
- Remo E + remoe romansey
- Roman Seyffarth + romansey MightySCollins
- Sam Collins + MightySCollins @@ -1471,42 +1471,42 @@ Thanks goes to these wonderful people ✨ 501st-alpha1
- Scott Weldon + 501st-alpha1 klamann
- Sebastian Straub + klamann svdb0
- Serge Van Den Boom + svdb0 3ap
- Sergey Nazaryev + 3ap shyim
- Shyim + shyim sjmudd
- Simon J Mudd + sjmudd @@ -1514,42 +1514,42 @@ Thanks goes to these wonderful people ✨ simonsystem
- Simon Schröter + simonsystem stephan-devop
- Stephan + stephan-devop stigok
- Stig Otnes Kolstad + stigok 5ven
- Sven Kauber + 5ven syl20bnr
- Sylvain Benner + syl20bnr sylvaindumont
- Sylvain Dumont + sylvaindumont @@ -1557,42 +1557,42 @@ Thanks goes to these wonderful people ✨ TechnicLab
- Null + TechnicLab thomasschmit
- Thomas Schmit + thomasschmit Thiritin
- Tin + Thiritin tweibert
- Torben Weibert + tweibert torus
- Toru Hisai + torus VictorKoenders
- Trangar + VictorKoenders @@ -1600,42 +1600,42 @@ Thanks goes to these wonderful people ✨ Twist235
- Null + Twist235 k3it
- Vasiliy Gokoyev + k3it Drakulix
- Victoria Brekenfeld + Drakulix vilisas
- Vilius + vilisas 42wim
- Wim + 42wim ShiriNmi1520
- Y.C.Huang + ShiriNmi1520 @@ -1643,42 +1643,42 @@ Thanks goes to these wonderful people ✨ Zepmann
- Null + Zepmann allddd
- Allddd + allddd arcaine2
- Null + arcaine2 awb99
- Awb99 + awb99 brainkiller
- Null + brainkiller cternes
- Null + cternes @@ -1686,42 +1686,42 @@ Thanks goes to these wonderful people ✨ dborowy
- Null + dborowy dimalo
- Null + dimalo eleith
- Eleith + eleith ghnp5
- Null + ghnp5 helmutundarnold
- Null + helmutundarnold hnws
- Null + hnws @@ -1729,42 +1729,42 @@ Thanks goes to these wonderful people ✨ i-C-o-d-e-r
- Null + i-C-o-d-e-r idaadi
- Null + idaadi ixeft
- Null + ixeft jjtt
- Null + jjtt paralax
- Jose Nazario + paralax jpduyx
- Null + jpduyx @@ -1772,42 +1772,42 @@ Thanks goes to these wonderful people ✨ landergate
- Null + landergate callmemagnus
- Magnus Anderssen + callmemagnus marios88
- Null + marios88 matrixes
- Null + matrixes mchamplain
- Mchamplain + mchamplain millerjason
- Jason Miller + millerjason @@ -1815,42 +1815,42 @@ Thanks goes to these wonderful people ✨ mplx
- Null + mplx odinis
- Null + odinis okamidash
- Okami + okamidash olaf-mandel
- Null + olaf-mandel ontheair81
- Null + ontheair81 pravynandas
- Null + pravynandas @@ -1858,42 +1858,42 @@ Thanks goes to these wonderful people ✨ presocratics
- Null + presocratics rhyst
- Null + rhyst rmlhuk
- Null + rmlhuk rriski
- Null + rriski schnippl0r
- Null + schnippl0r smargold476
- Null + smargold476 @@ -1901,42 +1901,42 @@ Thanks goes to these wonderful people ✨ sportshead
- Null + sportshead squash
- Null + squash strarsis
- Null + strarsis tamueller
- Null + tamueller vivacarvajalito
- Null + vivacarvajalito wligtenberg
- Null + wligtenberg @@ -1944,14 +1944,14 @@ Thanks goes to these wonderful people ✨ wolkenschieber
- Null + wolkenschieber worldworm
- Null + worldworm From d8ebf591f9ff320821c5779521fed1505f1b2916 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 14 Nov 2023 22:34:56 +0100 Subject: [PATCH 187/592] docs: correct path for logs (#3640) --- docs/content/config/debugging.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index 24e415661db..9888a9e9bb3 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -47,8 +47,7 @@ To get a shell inside the container run: `docker exec -it bash` If you need more flexibility than what the `docker logs` command offers, then the most useful locations to get relevant DMS logs within the container are: -- `/var/log/mail/mail.log` -- `/var/log/mail/mail/.log` +- `/var/log/mail/.log` - `/var/log/supervisor/.log` You may use `nano` (a text editor) to edit files, while `less` (a file viewer) and `tail`/`cat` are useful tools to inspect the contents of logs. From 042bd872bf8500d748f60468534fbb701ed09896 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:03:32 +0100 Subject: [PATCH 188/592] chore(deps): Bump docker/build-push-action from 5.0.0 to 5.1.0 (#3645) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index c7dc6e3fad9..0f375d4570a 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v5.0.0 + uses: docker/build-push-action@v5.1.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 5d68cd73d83..0ed2fd3e3d7 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -72,7 +72,7 @@ jobs: run: echo "version=$(>"${GITHUB_OUTPUT}" - name: 'Build and publish images' - uses: docker/build-push-action@v5.0.0 + uses: docker/build-push-action@v5.1.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 3c6e12883cc..5b8bac62825 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v5.0.0 + uses: docker/build-push-action@v5.1.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index ff8c8c1af9c..b39cced8616 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v5.0.0 + uses: docker/build-push-action@v5.1.0 with: context: . tags: mailserver-testing:ci From 020542a66cadeaaab69027587e52433814e7612b Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 23 Nov 2023 14:05:51 +1300 Subject: [PATCH 189/592] docs: Debugging - Add macOS suggestion to use `gRPC FUSE` file sharing implementation (#3652) --- docs/content/config/debugging.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index 9888a9e9bb3..a01dbe6a576 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -73,6 +73,7 @@ This could be from outdated software, or running a system that isn't able to pro ### System +- **macOS:** DMS has limited support for macOS. Often an issue encountered is due to permissions related to the `volumes` config in `compose.yaml`. You may have luck [trying `gRPC FUSE`][gh-macos-support] as the file sharing implementation; [`VirtioFS` is the successor][docker-macos-virtiofs] but presently appears incompatible with DMS. - **Kernel:** Some systems provide [kernels with modifications (_replacing defaults and backporting patches_)][network::kernels-modified] to support running legacy software or kernels, complicating compatibility. This can be commonly experienced with products like NAS. - **CGroups v2:** Hosts running older kernels (prior to 5.2) and systemd (prior to v244) are not likely to leverage cgroup v2, or have not defaulted to the cgroup v2 `unified` hierarchy. Not meeting this baseline may influence the behaviour of your DMS container, even with the latest Docker Engine installed. - **Container runtime:** Docker and Podman for example have subtle differences. DMS docs are primarily focused on Docker, but we try to document known issues where relevant. @@ -85,13 +86,16 @@ This could be from outdated software, or running a system that isn't able to pro [network::kernels-modified]: https://github.com/docker-mailserver/docker-mailserver/pull/2662#issuecomment-1168435970 [network::kernel-nftables]: https://unix.stackexchange.com/questions/596493/can-nftables-and-iptables-ip6tables-rules-be-applied-at-the-same-time-if-so-wh/596497#596497 -[docs-faq]: ../faq.md [docs-environment-log-level]: ./environment.md#log_level +[docs-faq]: ../faq.md [docs-ipv6]: ./advanced/ipv6.md [docs-introduction]: ../introduction.md +[docs-rootless-portdriver]: ./security/fail2ban.md#running-inside-a-rootless-container [docs-usage]: ../usage.md + [gh-issues]: https://github.com/docker-mailserver/docker-mailserver/issues +[gh-macos-support]: https://github.com/docker-mailserver/docker-mailserver/issues/3648#issuecomment-1822774080 [gh-discuss-roundcube-fail2ban]: https://github.com/orgs/docker-mailserver/discussions/3273#discussioncomment-5654603 [docker-rootless-interface]: https://github.com/moby/moby/issues/45742 -[docs-rootless-portdriver]: ./security/fail2ban.md#running-inside-a-rootless-container +[docker-macos-virtiofs]: https://www.docker.com/blog/speed-boost-achievement-unlocked-on-docker-desktop-4-6-for-mac/ From 2a716cf4a4e0eec9ec12d99fcc6e9c5e625a63f8 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 23 Nov 2023 22:47:11 +1300 Subject: [PATCH 190/592] docs: Dovecot Sieve - Adjust to new path for user home folder (#3650) --- docs/content/config/advanced/mail-sieve.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/config/advanced/mail-sieve.md b/docs/content/config/advanced/mail-sieve.md index fe540efb9b2..b3cef249c24 100644 --- a/docs/content/config/advanced/mail-sieve.md +++ b/docs/content/config/advanced/mail-sieve.md @@ -11,9 +11,9 @@ There are global and user specific filters which are filtering the incoming emai Global filters are applied to EVERY incoming mail for EVERY email address. To specify a global Sieve filter provide a `docker-data/dms/config/before.dovecot.sieve` or a `docker-data/dms/config/after.dovecot.sieve` file with your filter rules. -If any filter in this filtering chain discards an incoming mail, the delivery process will stop as well and the mail will not reach any following filters(e.g. global-before stops an incoming spam mail: The mail will get discarded and a user-specific filter won't get applied.) +If any filter in this filtering chain discards an incoming mail, the delivery process will stop as well and the mail will not reach any following filters (e.g. global-before stops an incoming spam mail: The mail will get discarded and a user-specific filter won't get applied.) -To specify a user-defined Sieve filter place a `.dovecot.sieve` file into a virtual user's mail folder e.g. `/var/mail/example.com/user1/.dovecot.sieve`. If this file exists dovecot will apply the filtering rules. +To specify a user-defined Sieve filter place a `.dovecot.sieve` file into a virtual user's mail folder (e.g. `/var/mail/example.com/user1/home/.dovecot.sieve`). If this file exists dovecot will apply the filtering rules. It's even possible to install a user provided Sieve filter at startup during users setup: simply include a Sieve file in the `docker-data/dms/config/` path for each user login that needs a filter. The file name provided should be in the form `.dovecot.sieve`, so for example for `user1@example.com` you should provide a Sieve file named `docker-data/dms/config/user1@example.com.dovecot.sieve`. From 7d1fcb75d76db7a2d30441afa3074681273a45c8 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 25 Nov 2023 22:57:34 +1300 Subject: [PATCH 191/592] docs: Debugging - Correctly starting DMS (#3654) --- docs/content/config/debugging.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index a01dbe6a576..d1e29376c07 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -14,6 +14,27 @@ This page contains valuable information when it comes to resolving issues you en - Check that all published DMS ports are actually open and not blocked by your ISP / hosting provider. - SSL errors are likely the result of a wrong setup on the user side and not caused by DMS itself. +- Ensure that you have correctly started DMS. Many problems related to configuration are due to this. + +!!! danger "Correctly starting DMS" + + Use the [`--force-recreate`][docker-docs::force-recreate] option to avoid configuration mishaps: `docker compose up --force-recreate` + + Alternatively, always use `docker compose down` to stop DMS. **Do not** rely on `CTRL + C`, `docker compose stop`, or `docker compose restart`. + + --- + + DMS setup scripts are run when a container starts, but may fail to work properly if you do the following: + + - Stopping a container with commands like: `docker stop` or `docker compose up` stopped via `CTRL + C` instead of `docker compose down`. + - Restarting a container. + + Volumes persist data across container instances, however the same container instance will keep internal changes not stored in a volume until the container is removed. + + Due to this, DMS setup scripts may modify configuration it has already modified in the past. + + - This is brittle as some changes are naive by assuming they are applied to the original configs from the image. + - Volumes in `compose.yaml` are expected to persist any important data. Thus it should be safe to throwaway the container created each time, avoiding this config problem. ### Mail sent from DMS does not arrive at destination @@ -99,3 +120,4 @@ This could be from outdated software, or running a system that isn't able to pro [docker-rootless-interface]: https://github.com/moby/moby/issues/45742 [docker-macos-virtiofs]: https://www.docker.com/blog/speed-boost-achievement-unlocked-on-docker-desktop-4-6-for-mac/ +[docker-docs::force-recreate]: https://docs.docker.com/compose/reference/up/ From cedd360ebd8d66b9bc7ef1ab986abc9c8ffeebc4 Mon Sep 17 00:00:00 2001 From: Jean-Kevin KPADEY <9912558+mivek@users.noreply.github.com> Date: Sat, 25 Nov 2023 11:02:42 +0100 Subject: [PATCH 192/592] docs: add a new example explaining how to integrate crowdsec with (#3651) --- docs/content/examples/tutorials/crowdsec.md | 74 +++++++++++++++++++++ docs/mkdocs.yml | 1 + 2 files changed, 75 insertions(+) create mode 100644 docs/content/examples/tutorials/crowdsec.md diff --git a/docs/content/examples/tutorials/crowdsec.md b/docs/content/examples/tutorials/crowdsec.md new file mode 100644 index 00000000000..2e8efe06351 --- /dev/null +++ b/docs/content/examples/tutorials/crowdsec.md @@ -0,0 +1,74 @@ +--- +title: 'Tutorials | Crowdsec' +--- + +!!! quote "What is Crowdsec?" + + Crowdsec is an open source software that detects and blocks attackers using log analysis. + It has access to a global community-wide IP reputation database. + + [Source](https://www.crowdsec.net) + +## Installation + +Crowdsec supports multiple [installation methods][crowdsec-installation-docs], however this page will use the docker installation. + + +### Docker mailserver + +In your `compose.yaml` for the DMS service, add a bind mount volume for `/var/log/mail`. This is to share the DMS logs to a separate crowdsec container. + +!!! example + ```yaml + services: + mailserver: + - /docker-data/dms/mail-logs/:/var/log/mail/ + ``` + +### Crowdsec + +The crowdsec container should also bind mount the same host path for the DMS logs that was added in the DMS example above. + +```yaml +services: + image: crowdsecurity/crowdsec + restart: unless-stopped + ports: + - "8080:8080" + - "6060:6060" + volumes: + - /docker-data/dms/mail-logs/:/var/log/dms:ro + - ./acquis.d:/etc/crowdsec/acquis.d + - crowdsec-db:/var/lib/crowdsec/data/ + environment: + # These collection contains parsers and scenarios for postfix and dovecot + COLLECTIONS: crowdsecurity/postfix crowdsecurity/dovecot + TZ: Europe/Paris +volumes: + crowdsec-db: +``` + +## Configuration + +Configure crowdsec to read and parse DMS logs file. + +!!! example + + Create the file `dms.yml` in `./acquis.d/` + + ```yaml + --- + source: file + filenames: + - /var/log/dms/mail.log + labels: + type: syslog + ``` + +!!! warning Bouncers + + Crowdsec on its own is just a detection software, the remediation is done by components called bouncers. + This page does not explain how to install or configure a bouncer. It can be found in [crowdsec documentation][crowdsec-bouncer-docs]. + +[crowdsec-installation-docs]: https://doc.crowdsec.net/docs/getting_started/install_crowdsec +[crowdsec-bouncer-docs]: https://doc.crowdsec.net/docs/bouncers/intro diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index fdc2bc5950a..bd41a798c4e 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -157,6 +157,7 @@ nav: - 'Tutorials': - 'Basic Installation': examples/tutorials/basic-installation.md - 'Mailserver behind Proxy': examples/tutorials/mailserver-behind-proxy.md + - 'Crowdsec': examples/tutorials/crowdsec.md - 'Building your own Docker image': examples/tutorials/docker-build.md - 'Blog Posts': examples/tutorials/blog-posts.md - 'Use Cases': From ba814f421302e3545cfee342f508f01d0ae98ce0 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sun, 26 Nov 2023 01:34:00 +1300 Subject: [PATCH 193/592] ci: Linter EC should use `/check` as the mount path (#3655) --- test/linting/lint.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/linting/lint.sh b/test/linting/lint.sh index 7c281e8f416..30cab39fc23 100755 --- a/test/linting/lint.sh +++ b/test/linting/lint.sh @@ -17,11 +17,14 @@ SHELLCHECK_VERSION='0.9.0' source "${REPOSITORY_ROOT}/target/scripts/helpers/log.sh" function _eclint() { + # `/check` is used instead of `/ci` as the mount path due to: + # https://github.com/editorconfig-checker/editorconfig-checker/issues/268#issuecomment-1826200253 + # `.ecrc.json` continues to explicitly ignores the `.git/` path to avoid any potential confusion if docker run --rm --tty \ - --volume "${REPOSITORY_ROOT}:/ci:ro" \ - --workdir "/ci" \ + --volume "${REPOSITORY_ROOT}:/check:ro" \ + --workdir "/check" \ --name dms-test_eclint \ - "mstruebing/editorconfig-checker:${ECLINT_VERSION}" ec -config "/ci/test/linting/.ecrc.json" + "mstruebing/editorconfig-checker:${ECLINT_VERSION}" ec -config "/check/test/linting/.ecrc.json" then _log 'info' 'ECLint succeeded' else From b663e10841d7d95f25d51efb7696c0c5b588d55f Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 26 Nov 2023 02:00:56 +0100 Subject: [PATCH 194/592] release: v13.0.0 (#3641) * adjust PR template I am tired of writing `CHANGELOG.md` myself --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Co-authored-by: Casper --- .github/pull_request_template.md | 1 + CHANGELOG.md | 131 +++++++++++++++++++++++++++---- VERSION | 2 +- 3 files changed, 118 insertions(+), 16 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 26dc801e45d..040d67dd751 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -26,3 +26,4 @@ Fixes # - [ ] I have made corresponding changes to the documentation (README.md or the documentation under `docs/`) - [ ] If necessary I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes +- [ ] **I have added information about changes made in this PR to `CHANGELOG.md`** diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b875e2b39e..71c9de3f44f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,30 +2,131 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v12.1.0...HEAD) +## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v13.0.0...HEAD) > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +## [v13.0.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.0.0) + ### Breaking -- The environment variable `ENABLE_LDAP=1` has been changed to `ACCOUNT_PROVISIONER=LDAP`. -- Postfix now defaults to supporting DSNs (_[Delivery Status Notifications](https://github.com/docker-mailserver/docker-mailserver/pull/3572#issuecomment-1751880574)_) only for authenticated users. This is a security measure to reduce spammer abuse of your DMS instance as a backscatter source. - - If you need to modify this change, please let us know by opening an issue / discussion. - - You can [opt-out (_enable DSNs_) via the `postfix-main.cf` override support](https://docker-mailserver.github.io/docker-mailserver/v12.1/config/advanced/override-defaults/postfix/) using the contents: `smtpd_discard_ehlo_keywords =`. - - Likewise for authenticated users, the submission(s) ports (465 + 587) are configured internally via `master.cf` to keep DSNs enabled (_since authentication protects from abuse_). +- **LDAP:** + - ENV `LDAP_SERVER_HOST`, `DOVECOT_URIS`, and `SASLAUTHD_LDAP_SERVER` will now log an error if the LDAP URI scheme is missing. Previously there was an implicit fallback to `ldap://` ([#3522](https://github.com/docker-mailserver/docker-mailserver/pull/3522)) + - `ENABLE_LDAP=1` is no longer supported, please use `ACCOUNT_PROVISIONER=LDAP` ([#3507](https://github.com/docker-mailserver/docker-mailserver/pull/3507)) +- **Rspamd:** + - The deprecated path for the Rspamd custom commands file (`/tmp/docker-mailserver/rspamd-modules.conf`) now prevents successful startup. The correct path is `/tmp/docker-mailserver/rspamd/custom-commands.conf`. +- **Dovecot:** + - Dovecot mail storage per account in `/var/mail` previously shared the same path for the accounts home directory ([#3335](https://github.com/docker-mailserver/docker-mailserver/pull/3335)) + - The home directory now is a subdirectory `home/`. This change better supports sieve scripts. + - **NOTE:** The change has not yet been implemented for `ACCOUNT_PROVISIONER=LDAP`. +- **Postfix:** + - `/etc/postfix/master.cf` has renamed the "smtps" service to "submissions" ([#3235](https://github.com/docker-mailserver/docker-mailserver/pull/3235)) + - This is the modern `/etc/services` name for port 465, aligning with the similar "submission" port 587. + - Postfix now defaults to supporting DSNs (_[Delivery Status Notifications](https://github.com/docker-mailserver/docker-mailserver/pull/3572#issuecomment-1751880574)_) only for authenticated users (_via ports 465 + 587_). This is a security measure to reduce spammer abuse of your DMS instance as a backscatter source. ([#3572](https://github.com/docker-mailserver/docker-mailserver/pull/3572)) + - If you need to modify this change, please let us know by opening an issue / discussion. + - You can [opt out (_enable DSNs_) via the `postfix-main.cf` override support](https://docker-mailserver.github.io/docker-mailserver/v12.1/config/advanced/override-defaults/postfix/) using the contents: `smtpd_discard_ehlo_keywords =`. + - Likewise for authenticated users, the submission(s) ports (465 + 587) are configured internally via `master.cf` to keep DSNs enabled (_since authentication protects from abuse_). + + If necessary, DSNs for authenticated users can be disabled via the `postfix-master.cf` override with the following contents: + + ```cf + submission/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn + submissions/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn + ``` + +### Added - If necessary, DSNs for authenticated users can be disabled via the `postfix-master.cf` override with the following contents: +- **Features:** + - `getmail` as an alternative to `fetchmail` ([#2803](https://github.com/docker-mailserver/docker-mailserver/pull/2803)) + - `setup` CLI - `setup fail2ban` gained a new `status ` subcommand ([#3455](https://github.com/docker-mailserver/docker-mailserver/pull/3455)) +- **Environment Variables:** + - `MARK_SPAM_AS_READ`. When set to `1`, marks incoming spam as "read" to avoid unwanted "new mail" notifications for junk mail ([#3489](https://github.com/docker-mailserver/docker-mailserver/pull/3489)) + - `DMS_VMAIL_UID` and `DMS_VMAIL_GID` allow changing the default ID values (`5000:5000`) for the Dovecot vmail user and group ([#3550](https://github.com/docker-mailserver/docker-mailserver/pull/3550)) + - `RSPAMD_CHECK_AUTHENTICATED` allows authenticated users to avoid additional security checks by Rspamd ([#3440](https://github.com/docker-mailserver/docker-mailserver/pull/3440)) +- **Documentation:** + - Use-case examples / tutorials: + - iOS mail push support ([#3513](https://github.com/docker-mailserver/docker-mailserver/pull/3513)) + - Guide for setting up Dovecot Authentication via Lua ([#3579](https://github.com/docker-mailserver/docker-mailserver/pull/3579)) + - Guide for integrating with the Crowdsec service ([#3651](https://github.com/docker-mailserver/docker-mailserver/pull/3651)) + - Debugging page: + - New compatibility section ([#3404](https://github.com/docker-mailserver/docker-mailserver/pull/3404)) + - Now advises how to (re)start DMS correctly ([#3654](https://github.com/docker-mailserver/docker-mailserver/pull/3654)) + - Better communicate distinction between DMS FQDN and DMS mail accounts ([#3372](https://github.com/docker-mailserver/docker-mailserver/pull/3372)) + - Traefik example now includes `passthrough=true` on implicit ports ([#3568](https://github.com/docker-mailserver/docker-mailserver/pull/3568)) + - Rspamd docs have received a variety of revisions ([#3318](https://github.com/docker-mailserver/docker-mailserver/pull/3318), [#3325](https://github.com/docker-mailserver/docker-mailserver/pull/3325), [#3329](https://github.com/docker-mailserver/docker-mailserver/pull/3329)) + - IPv6 config examples with content tabs ([#3436](https://github.com/docker-mailserver/docker-mailserver/pull/3436)) + - Mention [internet.nl](https://internet.nl/test-mail/) as another testing service ([#3445](https://github.com/docker-mailserver/docker-mailserver/pull/3445)) + - `setup alias add ...` CLI help message now includes an example for aliasing to multiple recipients ([#3600](https://github.com/docker-mailserver/docker-mailserver/pull/3600)) + - `SPAMASSASSIN_SPAM_TO_INBOX=1`, now emits a debug log to raise awareness that `SA_KILL` will be ignored ([#3360](https://github.com/docker-mailserver/docker-mailserver/pull/3360)) + - `CLAMAV_MESSAGE_SIZE_LIMIT` now logs a warning when the value exceeds what ClamAV is capable of supporting (4GiB max scan size [#3332](https://github.com/docker-mailserver/docker-mailserver/pull/3332), 2GiB max file size [#3341](https://github.com/docker-mailserver/docker-mailserver/pull/3341)) + - Added note to caution against changing `mydestination` in Postfix's `main.cf` ([#3316](https://github.com/docker-mailserver/docker-mailserver/pull/3316)) +- **Internal:** + - Added a wrapper to update Postfix configuration safely ([#3484](https://github.com/docker-mailserver/docker-mailserver/pull/3484), [#3503](https://github.com/docker-mailserver/docker-mailserver/pull/3503)) + - Add debug group to `packages.sh` ([#3578](https://github.com/docker-mailserver/docker-mailserver/pull/3578)) +- **Tests:** + - Additional linting check for BASH syntax ([#3369](https://github.com/docker-mailserver/docker-mailserver/pull/3369)) - ```cf - submission/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn - submissions/inet/smtpd_discard_ehlo_keywords=silent-discard,dsn - ``` +### Updates -- using the old path for the Rspamd custom commands file (`/tmp/docker-mailserver/rspamd-modules.conf`), which was deprecated, will now prevent startup; use `/tmp/docker-mailserver/rspamd/custom-commands.conf` instead +- **Misc:** + - Changed `setup config dkim` default key size to `2048` (`open-dkim`) ([#3508](https://github.com/docker-mailserver/docker-mailserver/pull/3508)) +- **Postfix:** + - Dropped special bits from `maildrop/` and `public/` directory permissions ([#3625](https://github.com/docker-mailserver/docker-mailserver/pull/3625)) +- **Rspamd:** + - Adjusted learning of ham ([#3334](https://github.com/docker-mailserver/docker-mailserver/pull/3334)) + - Adjusted `antivirus.conf` ([#3331](https://github.com/docker-mailserver/docker-mailserver/pull/3331)) + - `logrotate` setup + Rspamd log path + tests log helper fallback path ([#3576](https://github.com/docker-mailserver/docker-mailserver/pull/3576)) + - Setup during container startup is now more resilient ([#3578](https://github.com/docker-mailserver/docker-mailserver/pull/3578)) + - Changed DKIM default config location ([#3597](https://github.com/docker-mailserver/docker-mailserver/pull/3597)) + - Removed the symlink for the `override.d/` directory in favor of using `cp`, integrated into the changedetector service, , added a `--force` option for the Rspamd DKIM management, and provided a dedicated helper script for common ENV variables ([#3599](https://github.com/docker-mailserver/docker-mailserver/pull/3599)) + - Required permissions are now verified for DKIM private key files ([#3627](https://github.com/docker-mailserver/docker-mailserver/pull/3627)) +- **Documentation:** + - Documentation aligned to Compose v2 conventions, `docker-compose` command changed to `docker compose`, `docker-compose.yaml` to `compose.yaml` ([#3295](https://github.com/docker-mailserver/docker-mailserver/pull/3295)) + - Restored missing edit button ([#3338](https://github.com/docker-mailserver/docker-mailserver/pull/3338)) + - Complete rewrite of the IPv6 page ([#3244](https://github.com/docker-mailserver/docker-mailserver/pull/3244), [#3531](https://github.com/docker-mailserver/docker-mailserver/pull/3531)) + - Complete rewrite of the "Update and Cleanup" maintenance page ([#3539](https://github.com/docker-mailserver/docker-mailserver/pull/3539), [#3583](https://github.com/docker-mailserver/docker-mailserver/pull/3583)) + - Improved debugging page advice on working with logs ([#3626](https://github.com/docker-mailserver/docker-mailserver/pull/3626), [#3640](https://github.com/docker-mailserver/docker-mailserver/pull/3640)) + - Clarified the default for ENV `FETCHMAIL_PARALLEL` ([#3603](https://github.com/docker-mailserver/docker-mailserver/pull/3603)) + - Removed port 25 from FAQ entry for mail client ports supporting authenticated submission ([#3496](https://github.com/docker-mailserver/docker-mailserver/pull/3496)) + - Updated home path in docs for Dovecot Sieve ([#3370](https://github.com/docker-mailserver/docker-mailserver/pull/3370), [#3650](https://github.com/docker-mailserver/docker-mailserver/pull/3650)) + - Fixed path to `rspamd.log` ([#3585](https://github.com/docker-mailserver/docker-mailserver/pull/3585)) + - "Optional Config" page now uses consistent lowercase convention for directory names ([#3629](https://github.com/docker-mailserver/docker-mailserver/pull/3629)) + - `CONTRIBUTORS.md`: Removed redundant "All Contributors" section ([#3638](https://github.com/docker-mailserver/docker-mailserver/pull/3638)) +- **Internal:** + - LDAP config improvements (Removed implicit `ldap://` LDAP URI scheme fallback) ([#3522](https://github.com/docker-mailserver/docker-mailserver/pull/3522)) + - Changed style conventions for internal scripts ([#3361](https://github.com/docker-mailserver/docker-mailserver/pull/3361), [#3364](https://github.com/docker-mailserver/docker-mailserver/pull/3364), [#3365](https://github.com/docker-mailserver/docker-mailserver/pull/3365), [#3366](https://github.com/docker-mailserver/docker-mailserver/pull/3366), [#3368](https://github.com/docker-mailserver/docker-mailserver/pull/3368), [#3464](https://github.com/docker-mailserver/docker-mailserver/pull/3464)) +- **CI / Automation:** + - `.gitattributes` now ensures files are committed with `eol=lf` ([#3527](https://github.com/docker-mailserver/docker-mailserver/pull/3527)) + - Revised the GitHub issue bug report template ([#3317](https://github.com/docker-mailserver/docker-mailserver/pull/3317), [#3381](https://github.com/docker-mailserver/docker-mailserver/pull/3381), [#3435](https://github.com/docker-mailserver/docker-mailserver/pull/3435)) + - Clarified that the issue tracker is not for personal support ([#3498](https://github.com/docker-mailserver/docker-mailserver/pull/3498), [#3502](https://github.com/docker-mailserver/docker-mailserver/pull/3502)) + - Bumped versions of miscellaneous software (also shoutout to @dependabot) ([#3371](https://github.com/docker-mailserver/docker-mailserver/pull/3371), [#3584](https://github.com/docker-mailserver/docker-mailserver/pull/3584), [#3504](https://github.com/docker-mailserver/docker-mailserver/pull/3504), [#3516](https://github.com/docker-mailserver/docker-mailserver/pull/3516)) +- **Tests:** + - Refactored LDAP tests to current conventions ([#3483](https://github.com/docker-mailserver/docker-mailserver/pull/3483)) + - Changed OpenLDAP image to `bitnami/openldap` ([#3494](https://github.com/docker-mailserver/docker-mailserver/pull/3494)) + - Revised LDAP config + setup ([#3514](https://github.com/docker-mailserver/docker-mailserver/pull/3514)) + - Added tests for the helper function `_add_to_or_update_postfix_main()` ([#3505](https://github.com/docker-mailserver/docker-mailserver/pull/3505)) + - EditorConfig Checker lint now uses a mount path to `/check` instead of `/ci` ([#3655](https://github.com/docker-mailserver/docker-mailserver/pull/3655)) -### Added +### Fixed -- New environment variable `MARK_SPAM_AS_READ`. When set to `1`, marks incoming junk as "read" to avoid unwanted notification of junk as new mail ([#3489](https://github.com/docker-mailserver/docker-mailserver/pull/3489)) +- **Security:** + - Fixed issue with concatenating `$dmarc_milter` and `$dkim_milter` in `main.cf` ([#3380](https://github.com/docker-mailserver/docker-mailserver/pull/3380)) + - Fixed Rspamd DKIM signing for inbound emails ([#3439](https://github.com/docker-mailserver/docker-mailserver/pull/3439), [#3453](https://github.com/docker-mailserver/docker-mailserver/pull/3453)) + - OpenDKIM key generation is no longer broken when Rspamd is also enabled ([#3535](https://github.com/docker-mailserver/docker-mailserver/pull/3535)) +- **Internal:** + - The "database" files (_for managing users and aliases_) now correctly filters within lookup query ([#3359](https://github.com/docker-mailserver/docker-mailserver/pull/3359)) + - `_setup_spam_to_junk()` no longer registered when `SMTP_ONLY=1` ([#3385](https://github.com/docker-mailserver/docker-mailserver/pull/3385)) + - Dovecot `fts_xapian` is now compiled from source to match the Dovecot package ABI ([#3373](https://github.com/docker-mailserver/docker-mailserver/pull/3373)) +- **CI:** + - Scheduled build now have the correct permissions to run successfully ([#3345](https://github.com/docker-mailserver/docker-mailserver/pull/3345)) +- **Documentation:** + - Miscellaneous spelling and wording improvements ([#3324](https://github.com/docker-mailserver/docker-mailserver/pull/3324), [#3330](https://github.com/docker-mailserver/docker-mailserver/pull/3330), [#3337](https://github.com/docker-mailserver/docker-mailserver/pull/3337), [#3339](https://github.com/docker-mailserver/docker-mailserver/pull/3339), [#3344](https://github.com/docker-mailserver/docker-mailserver/pull/3344), [#3367](https://github.com/docker-mailserver/docker-mailserver/pull/3367), [#3411](https://github.com/docker-mailserver/docker-mailserver/pull/3411), [#3443](https://github.com/docker-mailserver/docker-mailserver/pull/3443)) +- **Tests:** + - Run `pgrep` within the actual container ([#3553](https://github.com/docker-mailserver/docker-mailserver/pull/3553)) + - `lmtp_ip.bats` improved partial failure output ([#3552](https://github.com/docker-mailserver/docker-mailserver/pull/3552)) + - Improvements to LDIF test data ([#3506](https://github.com/docker-mailserver/docker-mailserver/pull/3506)) + - Normalized for `.gitattributes` + improved `eclint` coverage ([#3566](https://github.com/docker-mailserver/docker-mailserver/pull/3566)) + - Fixed ShellCheck linting for BATS tests ([#3347](https://github.com/docker-mailserver/docker-mailserver/pull/3347)) ## [v12.1.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v12.1.0) @@ -38,7 +139,7 @@ All notable changes to this project will be documented in this file. The format - add option to re-enable `reject_unknown_client_hostname` after #3248 ([#3255](https://github.com/docker-mailserver/docker-mailserver/pull/3255)) - add DKIM helper script ([#3286](https://github.com/docker-mailserver/docker-mailserver/pull/3286)) - make `policyd-spf` configurable ([#3246](https://github.com/docker-mailserver/docker-mailserver/pull/3246)) -- add 'log' command to setup for Fail2Ban ([#3299](https://github.com/docker-mailserver/docker-mailserver/pull/3299)) +- add 'log' command to set up for Fail2Ban ([#3299](https://github.com/docker-mailserver/docker-mailserver/pull/3299)) - `setup` command now expects accounts and aliases to be mutually exclusive ([#3270](https://github.com/docker-mailserver/docker-mailserver/pull/3270)) ### Updated diff --git a/VERSION b/VERSION index 77903b35f3a..02161ca86e5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -12.1.0 +13.0.0 From 2c602299136fcac8a8fd02ada86039687e6ae18c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 26 Nov 2023 20:15:14 +0100 Subject: [PATCH 195/592] docs: updated `CONTRIBUTORS.md` (#3656) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 115 +++++++++++++++++++++++++----------------------- 1 file changed, 61 insertions(+), 54 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 893b9572733..46e0523f084 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1180,6 +1180,13 @@ Thanks goes to these wonderful people ✨ jcalfee + + + mivek +
+ mivek + + init-js @@ -1200,15 +1207,15 @@ Thanks goes to these wonderful people ✨
JiLleON - + + jirislav
jirislav - - + jmccl @@ -1243,15 +1250,15 @@ Thanks goes to these wonderful people ✨
akkumar - + + KCrawley
KCrawley - - + khuedoan @@ -1286,15 +1293,15 @@ Thanks goes to these wonderful people ✨
luke- - + + LucidityCrash
LucidityCrash - - + MadsRC @@ -1329,15 +1336,15 @@ Thanks goes to these wonderful people ✨
michaeljensen - + + exhuma
exhuma - - + milas @@ -1372,15 +1379,15 @@ Thanks goes to these wonderful people ✨
naveensrinivasan - + + neuralp
neuralp - - + radicand @@ -1415,15 +1422,15 @@ Thanks goes to these wonderful people ✨
OrvilleQ - + + ovidiucp
ovidiucp - - + mrPjer @@ -1458,15 +1465,15 @@ Thanks goes to these wonderful people ✨
romansey - + + MightySCollins
MightySCollins - - + 501st-alpha1 @@ -1501,15 +1508,15 @@ Thanks goes to these wonderful people ✨
shyim - + + sjmudd
sjmudd - - + simonsystem @@ -1544,15 +1551,15 @@ Thanks goes to these wonderful people ✨
syl20bnr - + + sylvaindumont
sylvaindumont - - + TechnicLab @@ -1587,15 +1594,15 @@ Thanks goes to these wonderful people ✨
torus - + + VictorKoenders
VictorKoenders - - + Twist235 @@ -1630,15 +1637,15 @@ Thanks goes to these wonderful people ✨
42wim - + + ShiriNmi1520
ShiriNmi1520 - - + Zepmann @@ -1673,15 +1680,15 @@ Thanks goes to these wonderful people ✨
brainkiller - + + cternes
cternes - - + dborowy @@ -1716,15 +1723,15 @@ Thanks goes to these wonderful people ✨
helmutundarnold - + + hnws
hnws - - + i-C-o-d-e-r @@ -1759,15 +1766,15 @@ Thanks goes to these wonderful people ✨
paralax - + + jpduyx
jpduyx - - + landergate @@ -1802,15 +1809,15 @@ Thanks goes to these wonderful people ✨
mchamplain - + + millerjason
millerjason - - + mplx @@ -1845,15 +1852,15 @@ Thanks goes to these wonderful people ✨
ontheair81 - + + pravynandas
pravynandas - - + presocratics @@ -1888,15 +1895,15 @@ Thanks goes to these wonderful people ✨
schnippl0r - + + smargold476
smargold476 - - + sportshead @@ -1931,15 +1938,15 @@ Thanks goes to these wonderful people ✨
vivacarvajalito - + + wligtenberg
wligtenberg - - + wolkenschieber From 68a43eb4970f2ab7680ffa470a1d26e37fa375f0 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 26 Nov 2023 21:44:47 +0100 Subject: [PATCH 196/592] ci: push `:edge` when `VERSION` is updated (#3662) Previously, we did not run the workflow on push on `master` when a release happened because the push on master is guarded by a check on which files were changed. With this change, I added `VERSION` to the list of files to consider when updating `:edge`. --- .github/workflows/default_on_push.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/default_on_push.yml b/.github/workflows/default_on_push.yml index a598398920a..209219843df 100644 --- a/.github/workflows/default_on_push.yml +++ b/.github/workflows/default_on_push.yml @@ -11,6 +11,7 @@ on: - .gitmodules - Dockerfile - setup.sh + - VERSION # also update :edge when a release happens tags: - '*.*.*' From b037288e5ad20b11fab4f375eaf0fe51f2783cd9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 14:22:17 +0100 Subject: [PATCH 197/592] chore(deps): Bump anchore/scan-action from 3.3.6 to 3.3.7 (#3667) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index b39cced8616..cad2ac41088 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v3.3.6 + uses: anchore/scan-action@v3.3.7 id: scan with: image: mailserver-testing:ci From a11951e39801dac78628a5ad15d2bc15d4f24e7e Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 28 Nov 2023 10:33:29 +0100 Subject: [PATCH 198/592] hotfix: solve #3665 (#3669) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 14 +++++++++++++- docs/content/config/environment.md | 4 ++++ mailserver.env | 2 ++ target/rspamd/local.d/settings.conf | 2 +- target/scripts/startup/setup.d/security/rspamd.sh | 2 +- .../parallel/set1/spam_virus/rspamd_full.bats | 2 +- 6 files changed, 22 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71c9de3f44f..a8544b808be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +## [v13.0.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.0.1) + +This patch release fixes two bugs that Rspamd users encounter on `v13.0.0`. Big thanks to the those that helped to identify these issues! + +### Fixed + +- **Rspamd:** + - The check for correct permission on the private key when signing e-mails with DKIM was flawed. The result was that a false warning was emitted ([#3669](https://github.com/docker-mailserver/docker-mailserver/pull/3669)) + - When [`RSPAMD_CHECK_AUTHENTICATED=0`][docs::env-rspamd-check-auth], DKIM signing for outbound e-mail was disabled, which is undesirable ([#3669](https://github.com/docker-mailserver/docker-mailserver/pull/3669)). **Make sure to check the documentation of [`RSPAMD_CHECK_AUTHENTICATED`][docs::env-rspamd-check-auth]**! + +[docs::env-rspamd-check-auth]: https://docker-mailserver.github.io/docker-mailserver/v13.0/config/environment/#rspamd_check_authenticated + ## [v13.0.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.0.0) ### Breaking @@ -78,7 +90,7 @@ All notable changes to this project will be documented in this file. The format - `logrotate` setup + Rspamd log path + tests log helper fallback path ([#3576](https://github.com/docker-mailserver/docker-mailserver/pull/3576)) - Setup during container startup is now more resilient ([#3578](https://github.com/docker-mailserver/docker-mailserver/pull/3578)) - Changed DKIM default config location ([#3597](https://github.com/docker-mailserver/docker-mailserver/pull/3597)) - - Removed the symlink for the `override.d/` directory in favor of using `cp`, integrated into the changedetector service, , added a `--force` option for the Rspamd DKIM management, and provided a dedicated helper script for common ENV variables ([#3599](https://github.com/docker-mailserver/docker-mailserver/pull/3599)) + - Removed the symlink for the `override.d/` directory in favor of using `cp`, integrated into the changedetector service, added a `--force` option for the Rspamd DKIM management, and provided a dedicated helper script for common ENV variables ([#3599](https://github.com/docker-mailserver/docker-mailserver/pull/3599)) - Required permissions are now verified for DKIM private key files ([#3627](https://github.com/docker-mailserver/docker-mailserver/pull/3627)) - **Documentation:** - Documentation aligned to Compose v2 conventions, `docker-compose` command changed to `docker compose`, `docker-compose.yaml` to `compose.yaml` ([#3295](https://github.com/docker-mailserver/docker-mailserver/pull/3295)) diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 284549f13b7..b8e257cca33 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -366,6 +366,10 @@ The purpose of this setting is to opt-out of starting an internal Redis instance This settings controls whether checks should be performed on emails coming from authenticated users (i.e. most likely outgoing emails). The default value is `0` in order to align better with SpamAssassin. **We recommend** reading through [the Rspamd documentation on scanning outbound emails][rspamd-scanning-outbound] though to decide for yourself whether you need and want this feature. +!!! note "Not all checks and actions are disabled" + + DKIM signing of e-mails will still happen. + - **0** => No checks will be performed for authenticated users - 1 => All default checks will be performed for authenticated users diff --git a/mailserver.env b/mailserver.env index 957a632e128..6878622499a 100644 --- a/mailserver.env +++ b/mailserver.env @@ -153,6 +153,8 @@ RSPAMD_LEARN=0 # is `0` in order to align better with SpamAssassin. We recommend reading # through https://rspamd.com/doc/tutorials/scanning_outbound.html though to # decide for yourself whether you need and want this feature. +# +# Note that DKIM signing of e-mails will still happen. RSPAMD_CHECK_AUTHENTICATED=0 # Controls whether the Rspamd Greylisting module is enabled. diff --git a/target/rspamd/local.d/settings.conf b/target/rspamd/local.d/settings.conf index 4f635e749cd..10c4de88d03 100644 --- a/target/rspamd/local.d/settings.conf +++ b/target/rspamd/local.d/settings.conf @@ -6,7 +6,7 @@ authenticated { priority = high; authenticated = yes; apply { - groups_enabled = []; + groups_enabled = [dkim]; } } # DMS::SED_TAG::1::END diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 239397e5925..86786932393 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -325,7 +325,7 @@ function __rspamd__check_dkim_permissions() { __rspamd__log 'trace' "Checking DKIM file '${FILE}'" # See https://serverfault.com/a/829314 for an explanation on `-exec false {} +` # We additionally resolve symbolic links to check the permissions of the actual files - if find "$(realpath -eL "${FILE}")" -user _rspamd -or -group _rspamd -or -perm -o=r -exec false {} +; then + if find "$(realpath -eL "${FILE}")" \( -user _rspamd -or -group _rspamd -or -perm -o=r \) -exec false {} +; then __rspamd__log 'warn' "Rspamd DKIM private key file '${FILE}' does not appear to have correct permissions/ownership for Rspamd to use it" else __rspamd__log 'trace' "DKIM file '${FILE}' permissions and ownership appear correct" diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index 09d42d46ed1..ba8a23f59c8 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -307,5 +307,5 @@ function teardown_file() { _default_teardown ; } _run_in_container grep -E -A 6 'authenticated \{' "${MODULE_FILE}" assert_success assert_output --partial 'authenticated = yes;' - assert_output --partial 'groups_enabled = [];' + assert_output --partial 'groups_enabled = [dkim];' } From 19e96b5131ba935a0e54e554c3f3a0e6fc66f3b4 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 30 Nov 2023 10:21:26 +1300 Subject: [PATCH 199/592] fix: `update-check.sh` should query GH Releases (#3666) * fix: Source `VERSION` from image ENV Now CI builds triggered from tagged releases will always have the correct version. No need for manually updating a separate file. * fix: Query latest GH release tag Compare to the remote GH release tag published, rather than contents of a `VERSION` file. `VERSION` file remains in source for now as prior releases still rely on it for an update notification. * chore: Switch from `yq` to `jaq` - Can more easily express a string subslice. - Lighter weight: 9.3M vs 1.7M. - Drawback, no YAML input/output support. If `yq` is preferred, the `v` prefix could be removed via BASH easily enough. * chore: Add entry to `CHANGELOG.md` * ci: `VERSION` has no relevance to `:edge` * docs: Update build guide + simplify `make build` --------- Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- .github/workflows/default_on_push.yml | 1 - .github/workflows/generic_publish.yml | 7 +---- CHANGELOG.md | 2 ++ Dockerfile | 5 ++-- Makefile | 6 +---- .../examples/tutorials/docker-build.md | 27 ++++++++++++------- target/scripts/build/packages.sh | 6 +++++ target/scripts/start-mailserver.sh | 2 +- target/scripts/update-check.sh | 7 ++--- 9 files changed, 35 insertions(+), 28 deletions(-) diff --git a/.github/workflows/default_on_push.yml b/.github/workflows/default_on_push.yml index 209219843df..a598398920a 100644 --- a/.github/workflows/default_on_push.yml +++ b/.github/workflows/default_on_push.yml @@ -11,7 +11,6 @@ on: - .gitmodules - Dockerfile - setup.sh - - VERSION # also update :edge when a release happens tags: - '*.*.*' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 0ed2fd3e3d7..6df534ef94d 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -66,18 +66,13 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: 'Acquire the image version' - id: get-version - shell: bash - run: echo "version=$(>"${GITHUB_OUTPUT}" - - name: 'Build and publish images' uses: docker/build-push-action@v5.1.0 with: context: . build-args: | + DMS_RELEASE=${{ github.ref_type == 'tag' && github.ref_name || 'edge' }} VCS_REVISION=${{ github.sha }} - VCS_VERSION=${{ steps.get-version.outputs.version }} platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.prep.outputs.tags }} diff --git a/CHANGELOG.md b/CHANGELOG.md index a8544b808be..67aa3ec0eec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ This patch release fixes two bugs that Rspamd users encounter on `v13.0.0`. Big ### Fixed +- **Internal:** + - The update check service now queries the latest GH release for a version tag instead of a `VERSION` file from the repo. - **Rspamd:** - The check for correct permission on the private key when signing e-mails with DKIM was flawed. The result was that a false warning was emitted ([#3669](https://github.com/docker-mailserver/docker-mailserver/pull/3669)) - When [`RSPAMD_CHECK_AUTHENTICATED=0`][docs::env-rspamd-check-auth], DKIM signing for outbound e-mail was disabled, which is undesirable ([#3669](https://github.com/docker-mailserver/docker-mailserver/pull/3669)). **Make sure to check the documentation of [`RSPAMD_CHECK_AUTHENTICATED`][docs::env-rspamd-check-auth]**! diff --git a/Dockerfile b/Dockerfile index 5e12689da60..0f19521add4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -295,8 +295,8 @@ COPY target/scripts/startup/setup.d /usr/local/bin/setup.d # FROM stage-main AS stage-final +ARG DMS_RELEASE=edge ARG VCS_REVISION=unknown -ARG VCS_VERSION=edge WORKDIR / EXPOSE 25 587 143 465 993 110 995 4190 @@ -327,4 +327,5 @@ LABEL org.opencontainers.image.source="https://github.com/docker-mailserver/dock # ARG invalidates cache when it is used by a layer (implicitly affects RUN) # Thus to maximize cache, keep these lines last: LABEL org.opencontainers.image.revision=${VCS_REVISION} -LABEL org.opencontainers.image.version=${VCS_VERSION} +LABEL org.opencontainers.image.version=${DMS_RELEASE} +ENV DMS_RELEASE=${DMS_RELEASE} diff --git a/Makefile b/Makefile index 5732cc07ed6..0962c11ae4c 100644 --- a/Makefile +++ b/Makefile @@ -18,11 +18,7 @@ BATS_PARALLEL_JOBS ?= 2 all: lint build generate-accounts tests clean build: ALWAYS_RUN - @ DOCKER_BUILDKIT=1 docker build \ - --tag $(IMAGE_NAME) \ - --build-arg VCS_VERSION=$(shell git rev-parse --short HEAD) \ - --build-arg VCS_REVISION=$(shell cat VERSION) \ - . + @ docker build --tag $(IMAGE_NAME) . generate-accounts: ALWAYS_RUN @ cp test/config/templates/postfix-accounts.cf test/config/postfix-accounts.cf diff --git a/docs/content/examples/tutorials/docker-build.md b/docs/content/examples/tutorials/docker-build.md index fc6d5c37b23..538da822029 100644 --- a/docs/content/examples/tutorials/docker-build.md +++ b/docs/content/examples/tutorials/docker-build.md @@ -10,7 +10,7 @@ You'll need to retrieve the git submodules prior to building your own Docker ima ```sh git submodule update --init --recursive -docker build -t . +docker build --tag . ``` Or, you can clone and retrieve the submodules in one command: @@ -21,19 +21,26 @@ git clone --recurse-submodules https://github.com/docker-mailserver/docker-mails ### About Docker -#### Version +#### Minimum supported version -We make use of build-features that require a recent version of Docker. Depending on your distribution, please have a look at [the official installation documentation for Docker](https://docs.docker.com/engine/install/) to get the latest version. Otherwise, you may encounter issues, for example with the `--link` flag for a [`#!dockerfile COPY`](https://docs.docker.com/engine/reference/builder/#copy) command. +We make use of build features that require a recent version of Docker. v23.0 or newer is advised, but earlier releases may work. -#### Environment +- To get the latest version for your distribution, please have a look at [the official installation documentation for Docker](https://docs.docker.com/engine/install/). +- If you are using a version of Docker prior to v23.0, you will need to enable BuildKit via the ENV [`DOCKER_BUILDKIT=1`](https://docs.docker.com/build/buildkit/#getting-started). -If you are not using `make` to build the image, note that you will need to provide `DOCKER_BUILDKIT=1` to the `docker build` command for the build to succeed. +#### Build Arguments (Optional) -#### Build Arguments +The `Dockerfile` includes several build [`ARG`][docker-docs::builder-arg] instructions that can be configured: -The `Dockerfile` takes additional, so-called build arguments. These are +- `DOVECOT_COMMUNITY_REPO`: Install Dovecot from the community repo instead of from Debian (default = 1) +- `DMS_RELEASE`: The image version (default = edge) +- `VCS_REVISION`: The git commit hash used for the build (default = unknown) -1. `VCS_VERSION`: the image version (default = edge) -2. `VCS_REVISION`: the image revision (default = unknown) +!!! note -When using `make` to build the image, these are filled with proper values. You can build the image without supplying these arguments just fine though. + - `DMS_RELEASE` (_when not `edge`_) will be used to check for updates from our GH releases page at runtime due to the default feature [`ENABLE_UPDATE_CHECK=1`][docs::env-update-check]. + - Both `DMS_RELEASE` and `VCS_REVISION` are also used with `opencontainers` metadata [`LABEL`][docker-docs::builder-label] instructions. + +[docs::env-update-check]: https://docker-mailserver.github.io/docker-mailserver/latest/config/environment/#enable_update_check +[docker-docs::builder-arg]: https://docs.docker.com/engine/reference/builder/#using-arg-variables +[docker-docs::builder-label]: https://docs.docker.com/engine/reference/builder/#label diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index a025c3b4e08..97ebae04b9c 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -205,6 +205,11 @@ function _install_getmail() { apt-get "${QUIET}" autoremove } +function _install_utils() { + _log 'debug' 'Installing utils sourced from Github' + curl -sL https://github.com/01mf02/jaq/releases/latest/download/jaq-v1.2.0-x86_64-unknown-linux-musl -o /usr/bin/jaq && chmod +x /usr/bin/jaq +} + function _remove_data_after_package_installations() { _log 'debug' 'Deleting sensitive files (secrets)' rm /etc/postsrsd.secret @@ -228,5 +233,6 @@ _install_dovecot _install_rspamd _install_fail2ban _install_getmail +_install_utils _remove_data_after_package_installations _post_installation_steps diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index f0f385f3f6c..cc6c2244e95 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -120,7 +120,7 @@ function _register_functions() { [[ ${SMTP_ONLY} -ne 1 ]] && _register_start_daemon '_start_daemon_dovecot' - [[ ${ENABLE_UPDATE_CHECK} -eq 1 ]] && _register_start_daemon '_start_daemon_update_check' + [[ ${ENABLE_UPDATE_CHECK} -eq 1 ]] && [[ ${DMS_RELEASE} != 'edge' ]] && _register_start_daemon '_start_daemon_update_check' # The order here matters: Since Rspamd is using Redis, Redis should be started before Rspamd. [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] && _register_start_daemon '_start_daemon_rspamd_redis' diff --git a/target/scripts/update-check.sh b/target/scripts/update-check.sh index 9010371f241..c30594f4fe1 100755 --- a/target/scripts/update-check.sh +++ b/target/scripts/update-check.sh @@ -3,8 +3,8 @@ # shellcheck source=./helpers/log.sh source /usr/local/bin/helpers/log.sh -VERSION=$( Date: Thu, 30 Nov 2023 14:47:31 +1300 Subject: [PATCH 200/592] fix: Logging - Welcome should use `DMS_RELEASE` ENV (#3676) --- CHANGELOG.md | 10 ++++++++-- VERSION | 2 +- target/scripts/start-mailserver.sh | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67aa3ec0eec..f7fdb98122e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,14 +6,20 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Fixed + +- **Internal**: + - The container startup welcome log message now references `DMS_RELEASE` ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) + - `VERSION` was incremented for prior releases to be notified of the v13.0.1 patch release ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) + ## [v13.0.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.0.1) -This patch release fixes two bugs that Rspamd users encounter on `v13.0.0`. Big thanks to the those that helped to identify these issues! +This patch release fixes two bugs that Rspamd users encountered with the `v13.0.0` release. Big thanks to the those that helped to identify these issues! ❤️ ### Fixed - **Internal:** - - The update check service now queries the latest GH release for a version tag instead of a `VERSION` file from the repo. + - The update check service now queries the latest GH release for a version tag (_instead of from a `VERSION` file at the GH repo_). This should provide more reliable update notifications ([#3666](https://github.com/docker-mailserver/docker-mailserver/pull/3666)) - **Rspamd:** - The check for correct permission on the private key when signing e-mails with DKIM was flawed. The result was that a false warning was emitted ([#3669](https://github.com/docker-mailserver/docker-mailserver/pull/3669)) - When [`RSPAMD_CHECK_AUTHENTICATED=0`][docs::env-rspamd-check-auth], DKIM signing for outbound e-mail was disabled, which is undesirable ([#3669](https://github.com/docker-mailserver/docker-mailserver/pull/3669)). **Make sure to check the documentation of [`RSPAMD_CHECK_AUTHENTICATED`][docs::env-rspamd-check-auth]**! diff --git a/VERSION b/VERSION index 02161ca86e5..5cb7d8566ff 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -13.0.0 +13.0.1 diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index cc6c2244e95..fa8214e11dc 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -153,7 +153,7 @@ function _register_functions() { _early_supervisor_setup _early_variables_setup -_log 'info' "Welcome to docker-mailserver $( Date: Sun, 3 Dec 2023 22:28:40 +0100 Subject: [PATCH 201/592] ci: add `run-local-instance` target to `Makefile` (#3663) --- CHANGELOG.md | 4 ++++ Makefile | 16 ++++++++++++++++ docs/content/contributing/tests.md | 4 ++++ 3 files changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7fdb98122e..e6faa74aa3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Added + +- command (`run-local-instance`) to test a version of DMS that was built locally to test changes + ### Fixed - **Internal**: diff --git a/Makefile b/Makefile index 0962c11ae4c..37267d2c464 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,22 @@ clean: ALWAYS_RUN -@ while read -r LINE; do [[ $${LINE} =~ test/.+ ]] && FILES+=("/mnt$${LINE#test}"); done < .gitignore ; \ docker run --rm -v "$(REPOSITORY_ROOT)/test/:/mnt" alpine ash -c "rm -rf $${FILES[@]}" +run-local-instance: ALWAYS_RUN + bash -c 'sleep 8 ; ./setup.sh email add postmaster@example.test 123' & + docker run --rm --interactive --tty --name dms-test_example \ + --env OVERRIDE_HOSTNAME=mail.example.test \ + --env POSTFIX_INET_PROTOCOLS=ipv4 \ + --env DOVECOT_INET_PROTOCOLS=ipv4 \ + --env ENABLE_CLAMAV=0 \ + --env ENABLE_AMAVIS=0 \ + --env ENABLE_RSPAMD=0 \ + --env ENABLE_OPENDKIM=0 \ + --env ENABLE_OPENDMARC=0 \ + --env ENABLE_POLICYD_SPF=0 \ + --env ENABLE_SPAMASSASSIN=0 \ + --env LOG_LEVEL=trace \ + $(IMAGE_NAME) + # ----------------------------------------------- # --- Tests ------------------------------------ # ----------------------------------------------- diff --git a/docs/content/contributing/tests.md b/docs/content/contributing/tests.md index 6f6495299f3..8816a228bb0 100644 --- a/docs/content/contributing/tests.md +++ b/docs/content/contributing/tests.md @@ -78,6 +78,10 @@ We use `make` to run commands. When writing tests, ensure that parallel set tests still pass when run in parallel. You need to account for other tests running in parallel that may interfere with your own tests logic. +!!! tip + + You may use `make run-local-instance` to run a version of the image built locally to test and edit your changes in a running DMS instance. + ### An Example In this example, you've made a change to the Rspamd feature support (_or adjusted it's tests_). First verify no regressions have been introduced by running it's specific test file: From 01689ab788022097e79d80979303e206a5d48f24 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:22:43 +1300 Subject: [PATCH 202/592] docs: Troubleshooting - Bare domain misconfiguration (#3680) --- CHANGELOG.md | 8 +++++++- docs/content/config/debugging.md | 11 +++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6faa74aa3e..8ecfbe554ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,13 @@ All notable changes to this project will be documented in this file. The format ### Added -- command (`run-local-instance`) to test a version of DMS that was built locally to test changes +- **Tests:** + - You can now use `make run-local-instance` to run a DMS image that was built locally to test changes ([#3663](https://github.com/docker-mailserver/docker-mailserver/pull/3663)) + +### Updates + +- **Documentation:** + - Raise awareness in the troubleshooting page for a common misconfiguration when deviating from our advice by using a bare domain ([#3680](https://github.com/docker-mailserver/docker-mailserver/pull/3680)) ### Fixed diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index d1e29376c07..9c3bebb565a 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -46,6 +46,15 @@ Some service providers block outbound traffic on port 25. Common hosting provide These links may advise how the provider can unblock the port through additional services offered, or via a support ticket request. +### Mail sent to DMS does not get delivered to user + +Common logs related to this are: + +- `warning: do not list domain domain.fr in BOTH mydestination and virtual_mailbox_domains` +- `Recipient address rejected: User unknown in local recipient table` + +If your logs look like this, you likely have [assigned the same FQDN to the DMS `hostname` and your mail accounts][gh-issues::dms-fqdn-misconfigured] which is not supported by default. You can either adjust your DMS `hostname` or follow [this FAQ advice][docs::faq-bare-domain] + ## Steps for Debugging DMS 1. **Increase log verbosity**: Very helpful for troubleshooting problems during container startup. Set the environment variable [`LOG_LEVEL`][docs-environment-log-level] to `debug` or `trace`. @@ -109,12 +118,14 @@ This could be from outdated software, or running a system that isn't able to pro [docs-environment-log-level]: ./environment.md#log_level [docs-faq]: ../faq.md +[docs::faq-bare-domain]: ../faq.md#can-i-use-a-nakedbare-domain-ie-no-hostname [docs-ipv6]: ./advanced/ipv6.md [docs-introduction]: ../introduction.md [docs-rootless-portdriver]: ./security/fail2ban.md#running-inside-a-rootless-container [docs-usage]: ../usage.md [gh-issues]: https://github.com/docker-mailserver/docker-mailserver/issues +[gh-issues::dms-fqdn-misconfigured]: https://github.com/docker-mailserver/docker-mailserver/issues/3679#issuecomment-1837609043 [gh-macos-support]: https://github.com/docker-mailserver/docker-mailserver/issues/3648#issuecomment-1822774080 [gh-discuss-roundcube-fail2ban]: https://github.com/orgs/docker-mailserver/discussions/3273#discussioncomment-5654603 From 7ce745a82dfcb92c1527cc21f4bb4d78d412a292 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 21:32:49 +0100 Subject: [PATCH 203/592] chore(deps): Bump docker/metadata-action from 5.0.0 to 5.3.0 (#3683) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 6df534ef94d..36b44770856 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -23,7 +23,7 @@ jobs: - name: 'Prepare tags' id: prep - uses: docker/metadata-action@v5.0.0 + uses: docker/metadata-action@v5.3.0 with: images: | ${{ secrets.DOCKER_REPOSITORY }} From 1ff8d57ea1bd2d318240be71d7da760784ea9969 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 20:34:20 +0000 Subject: [PATCH 204/592] chore(deps): Bump anchore/scan-action from 3.3.7 to 3.3.8 (#3682) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index cad2ac41088..9524555792e 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v3.3.7 + uses: anchore/scan-action@v3.3.8 id: scan with: image: mailserver-testing:ci From c75975d59e7cb4d8b25b67792c4b74b60b35545d Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 5 Dec 2023 17:16:39 +1300 Subject: [PATCH 205/592] chore: Postfix should integrate Dovecot at runtime (#3681) * chore: Better establish startup scope * chore: Configure `main.cf` for Dovecot at runtime --- CHANGELOG.md | 2 ++ target/postfix/main.cf | 6 ++-- target/scripts/helpers/aliases.sh | 1 + target/scripts/start-mailserver.sh | 12 ++++---- target/scripts/startup/setup.d/postfix.sh | 30 +++++++++++++++---- .../startup/setup.d/security/spoofing.sh | 3 ++ 6 files changed, 40 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ecfbe554ee..5d539271d5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ All notable changes to this project will be documented in this file. The format - **Documentation:** - Raise awareness in the troubleshooting page for a common misconfiguration when deviating from our advice by using a bare domain ([#3680](https://github.com/docker-mailserver/docker-mailserver/pull/3680)) +- **Internal:** + - Postfix configures `virtual_mailbox_maps` and `virtual_transport` during startup instead of using defaults (configured for Dovecot) via our `main.cf` ([#3681](https://github.com/docker-mailserver/docker-mailserver/pull/3681)) ### Fixed diff --git a/target/postfix/main.cf b/target/postfix/main.cf index 405dc0fbbfe..8c329c943d5 100644 --- a/target/postfix/main.cf +++ b/target/postfix/main.cf @@ -88,10 +88,10 @@ smtpd_sasl_security_options = noanonymous smtpd_sasl_local_domain = $mydomain broken_sasl_auth_clients = yes -# Mail directory -virtual_transport = lmtp:unix:/var/run/dovecot/lmtp +# Postfix lookup tables for verifying valid users and managed mail domains: +# Populated during startup in: scripts/helpers/postfix.sh virtual_mailbox_domains = /etc/postfix/vhost -virtual_mailbox_maps = texthash:/etc/postfix/vmailbox +# Populated during startup in: scripts/helpers/aliases.sh virtual_alias_maps = texthash:/etc/postfix/virtual # Milters used by DKIM diff --git a/target/scripts/helpers/aliases.sh b/target/scripts/helpers/aliases.sh index 0890d994103..04a56da3061 100644 --- a/target/scripts/helpers/aliases.sh +++ b/target/scripts/helpers/aliases.sh @@ -23,6 +23,7 @@ function _handle_postfix_virtual_config() { fi } +# TODO: Investigate why this file is always created, nothing seems to append only the cp below? function _handle_postfix_regexp_config() { : >/etc/postfix/regexp diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index fa8214e11dc..aadac2b5ad7 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -91,20 +91,22 @@ function _register_functions() { _register_setup_function '_setup_dovecot_hostname' _register_setup_function '_setup_postfix_early' - _register_setup_function '_setup_fetchmail' - _register_setup_function '_setup_fetchmail_parallel' - # needs to come after _setup_postfix_early + # Dependent upon _setup_postfix_early first calling _create_aliases + # Due to conditional check for /etc/postfix/regexp _register_setup_function '_setup_spoof_protection' - _register_setup_function '_setup_getmail' + _register_setup_function '_setup_postfix_late' if [[ ${ENABLE_SRS} -eq 1 ]]; then _register_setup_function '_setup_SRS' _register_start_daemon '_start_daemon_postsrsd' fi - _register_setup_function '_setup_postfix_late' + _register_setup_function '_setup_fetchmail' + _register_setup_function '_setup_fetchmail_parallel' + _register_setup_function '_setup_getmail' + _register_setup_function '_setup_logrotate' _register_setup_function '_setup_mail_summary' _register_setup_function '_setup_logwatch' diff --git a/target/scripts/startup/setup.d/postfix.sh b/target/scripts/startup/setup.d/postfix.sh index 0d7cb1ae186..126a195cf98 100644 --- a/target/scripts/startup/setup.d/postfix.sh +++ b/target/scripts/startup/setup.d/postfix.sh @@ -30,18 +30,25 @@ mech_list: plain login EOF fi + # User has explicitly requested to disable SASL auth: + # TODO: Additive config by feature would be better. Should only enable SASL auth + # on submission(s) services in master.cf when SASLAuthd or Dovecot is enabled. if [[ ${ENABLE_SASLAUTHD} -eq 0 ]] && [[ ${SMTP_ONLY} -eq 1 ]]; then + # Default for services (eg: Port 25); NOTE: This has since become the default: sed -i -E \ 's|^smtpd_sasl_auth_enable =.*|smtpd_sasl_auth_enable = no|g' \ /etc/postfix/main.cf + # Submission services that are explicitly enabled by default: sed -i -E \ 's|^ -o smtpd_sasl_auth_enable=.*| -o smtpd_sasl_auth_enable=no|g' \ /etc/postfix/master.cf fi + # scripts/helpers/aliases.sh:_create_aliases() __postfix__log 'trace' 'Setting up aliases' _create_aliases + # scripts/helpers/postfix.sh:_create_postfix_vhost() __postfix__log 'trace' 'Setting up Postfix vhost' _create_postfix_vhost @@ -63,6 +70,23 @@ EOF 's|^(dms_smtpd_sender_restrictions = .*)|\1, reject_unknown_client_hostname|' \ /etc/postfix/main.cf fi + + # Dovecot feature integration + # TODO: Alias SMTP_ONLY=0 to DOVECOT_ENABLED=1? + if [[ ${SMTP_ONLY} -ne 1 ]]; then + __postfix__log 'trace' 'Configuring Postfix with Dovecot integration' + + # /etc/postfix/vmailbox is created by: scripts/helpers/accounts.sh:_create_accounts() + # This file config is for Postfix to verify a mail account exists before accepting + # mail arriving and delivering it to Dovecot over LMTP. + postconf 'virtual_mailbox_maps = texthash:/etc/postfix/vmailbox' + postconf 'virtual_transport = lmtp:unix:/var/run/dovecot/lmtp' + fi + + if [[ -n ${POSTFIX_DAGENT} ]]; then + __postfix__log 'trace' "Changing virtual transport to '${POSTFIX_DAGENT}'" + postconf "virtual_transport = ${POSTFIX_DAGENT}" + fi } function _setup_postfix_late() { @@ -80,12 +104,6 @@ function _setup_postfix_late() { __postfix__log 'trace' 'Configuring relay host' _setup_relayhost - if [[ -n ${POSTFIX_DAGENT} ]]; then - __postfix__log 'trace' "Changing virtual transport to '${POSTFIX_DAGENT}'" - # Default value in main.cf should be 'lmtp:unix:/var/run/dovecot/lmtp' - postconf "virtual_transport = ${POSTFIX_DAGENT}" - fi - __postfix__setup_override_configuration } diff --git a/target/scripts/startup/setup.d/security/spoofing.sh b/target/scripts/startup/setup.d/security/spoofing.sh index 7c38821d50b..ffefb2797b1 100644 --- a/target/scripts/startup/setup.d/security/spoofing.sh +++ b/target/scripts/startup/setup.d/security/spoofing.sh @@ -11,6 +11,9 @@ function _setup_spoof_protection() { postconf 'smtpd_sender_login_maps = ldap:/etc/postfix/ldap-senders.cf' fi else + # NOTE: This file is always created at startup, it potentially has content added. + # TODO: From section: "SPOOF_PROTECTION=1 handling for smtpd_sender_login_maps" + # https://github.com/docker-mailserver/docker-mailserver/issues/2819#issue-1402114383 if [[ -f /etc/postfix/regexp ]]; then postconf 'smtpd_sender_login_maps = unionmap:{ texthash:/etc/postfix/virtual, hash:/etc/aliases, pcre:/etc/postfix/maps/sender_login_maps.pcre, pcre:/etc/postfix/regexp }' else From 908d38047ce4be6dd1e8ca5021d0bce86625a3c3 Mon Sep 17 00:00:00 2001 From: Casper Date: Tue, 5 Dec 2023 21:42:30 +0100 Subject: [PATCH 206/592] scripts: add warning when update-check is enabled, but no stable release image is used (#3684) --- CHANGELOG.md | 1 + target/scripts/start-mailserver.sh | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d539271d5c..18ef5057056 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ All notable changes to this project will be documented in this file. The format - **Tests:** - You can now use `make run-local-instance` to run a DMS image that was built locally to test changes ([#3663](https://github.com/docker-mailserver/docker-mailserver/pull/3663)) +- Log a warning when update-check is enabled, but no stable release image is used ([#3684](https://github.com/docker-mailserver/docker-mailserver/pull/3684)) ### Updates diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index aadac2b5ad7..2129b74aa70 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -122,7 +122,13 @@ function _register_functions() { [[ ${SMTP_ONLY} -ne 1 ]] && _register_start_daemon '_start_daemon_dovecot' - [[ ${ENABLE_UPDATE_CHECK} -eq 1 ]] && [[ ${DMS_RELEASE} != 'edge' ]] && _register_start_daemon '_start_daemon_update_check' + if [[ ${ENABLE_UPDATE_CHECK} -eq 1 ]]; then + if [[ ${DMS_RELEASE} != 'edge' ]]; then + _register_start_daemon '_start_daemon_update_check' + else + _log 'warn' "ENABLE_UPDATE_CHECK=1 is configured, but image is not a stable release. Update-Check is disabled." + fi + fi # The order here matters: Since Rspamd is using Redis, Redis should be started before Rspamd. [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] && _register_start_daemon '_start_daemon_rspamd_redis' From 77917f5cc64b444db2a76def92161186db66670b Mon Sep 17 00:00:00 2001 From: Peter Adam Date: Thu, 7 Dec 2023 23:45:02 +0100 Subject: [PATCH 207/592] scripts: Install arm64 rspamd from official repository (#3686) * scripts: Install rspamd from official repository instead of debian backports on arm64 architecture * Remove unnecessary deb-src repository for rspamd * Remove note about ARM64 rspamd version, update CHANGELOG.md --------- Co-authored-by: Peter Adam --- CHANGELOG.md | 2 ++ docs/content/config/security/rspamd.md | 4 ---- target/scripts/build/packages.sh | 25 +++++-------------------- 3 files changed, 7 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18ef5057056..10d2bb256f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ All notable changes to this project will be documented in this file. The format - **Internal**: - The container startup welcome log message now references `DMS_RELEASE` ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) - `VERSION` was incremented for prior releases to be notified of the v13.0.1 patch release ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) +- **Rspamd:** + - Switch to official arm64 packages to avoid segfaults ([#3686](https://github.com/docker-mailserver/docker-mailserver/pull/3686)) ## [v13.0.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.0.1) diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 442e5e78df8..fe9bd5ea669 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -8,10 +8,6 @@ Rspamd is a ["fast, free and open-source spam filtering system"][rspamd-homepage If you want to have a look at the default configuration files for Rspamd that DMS packs, navigate to [`target/rspamd/` inside the repository][dms-default-configuration]. Please consult the [section "The Default Configuration"](#the-default-configuration) section down below for a written overview. -!!! note "AMD64 vs ARM64" - - We are currently doing a best-effort installation of Rspamd for ARM64 (from the Debian backports repository for Debian 11). The current version difference as of 23rd Apr 2023: AMD64 is at version 3.5 | ARM64 is at version 3.4. - [rspamd-homepage]: https://rspamd.com/ [dms-default-configuration]: https://github.com/docker-mailserver/docker-mailserver/tree/master/target/rspamd diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 97ebae04b9c..ec468d41834 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -130,29 +130,14 @@ function _install_dovecot() { function _install_rspamd() { _log 'trace' 'Adding Rspamd package signatures' local DEB_FILE='/etc/apt/sources.list.d/rspamd.list' - local RSPAMD_PACKAGE_NAME - - # We try getting the most recent version of Rspamd for aarch64 (from an official source, which - # is the backports repository). The version for aarch64 is 3.2; the most recent version for amd64 - # that we get with the official PPA is 3.4. - # - # Not removing it later is fine as you have to explicitly opt into installing a backports package - # which is not something you could be doing by accident. - if [[ $(uname --machine) == 'aarch64' ]]; then - echo '# Official Rspamd PPA does not support aarch64, so we use the Bullseye backports' >"${DEB_FILE}" - echo 'deb [arch=arm64] http://deb.debian.org/debian bullseye-backports main' >>"${DEB_FILE}" - RSPAMD_PACKAGE_NAME='rspamd/bullseye-backports' - else - curl -sSfL https://rspamd.com/apt-stable/gpg.key | gpg --dearmor >/etc/apt/trusted.gpg.d/rspamd.gpg - local URL='[arch=amd64 signed-by=/etc/apt/trusted.gpg.d/rspamd.gpg] http://rspamd.com/apt-stable/ bullseye main' - echo "deb ${URL}" >"${DEB_FILE}" - echo "deb-src ${URL}" >>"${DEB_FILE}" - RSPAMD_PACKAGE_NAME='rspamd' - fi + + curl -sSfL https://rspamd.com/apt-stable/gpg.key | gpg --dearmor >/etc/apt/trusted.gpg.d/rspamd.gpg + local URL='[signed-by=/etc/apt/trusted.gpg.d/rspamd.gpg] http://rspamd.com/apt-stable/ bullseye main' + echo "deb ${URL}" >"${DEB_FILE}" _log 'debug' 'Installing Rspamd' apt-get "${QUIET}" update - apt-get "${QUIET}" --no-install-recommends install "${RSPAMD_PACKAGE_NAME}" 'redis-server' + apt-get "${QUIET}" --no-install-recommends install 'rspamd' 'redis-server' } function _install_fail2ban() { From d3b4e94d0681b98ea6a7a955051c397cf19c0b88 Mon Sep 17 00:00:00 2001 From: Casper Date: Fri, 8 Dec 2023 01:20:17 +0100 Subject: [PATCH 208/592] update-check: fix 'read' exit status (#3688) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- CHANGELOG.md | 1 + target/scripts/update-check.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10d2bb256f2..3553804fa64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ All notable changes to this project will be documented in this file. The format - **Internal**: - The container startup welcome log message now references `DMS_RELEASE` ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) - `VERSION` was incremented for prior releases to be notified of the v13.0.1 patch release ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) + - Update-check: fix 'read' exit status ([#3688](https://github.com/docker-mailserver/docker-mailserver/pull/3688)) - **Rspamd:** - Switch to official arm64 packages to avoid segfaults ([#3686](https://github.com/docker-mailserver/docker-mailserver/pull/3686)) diff --git a/target/scripts/update-check.sh b/target/scripts/update-check.sh index c30594f4fe1..257fc37d380 100755 --- a/target/scripts/update-check.sh +++ b/target/scripts/update-check.sh @@ -27,7 +27,7 @@ while true; do # compare versions if dpkg --compare-versions "${VERSION}" lt "${LATEST}"; then # send mail notification to postmaster - read -r -d '' MAIL << EOF + read -r -d '#' MAIL << EOF Hello ${POSTMASTER_ADDRESS}! There is a docker-mailserver update available on your host: $(hostname -f) @@ -35,7 +35,7 @@ There is a docker-mailserver update available on your host: $(hostname -f) Current version: ${VERSION} Latest version: ${LATEST} -Changelog: ${CHANGELOG_URL} +Changelog: ${CHANGELOG_URL}#END EOF _log_with_date 'info' "Update available [ ${VERSION} --> ${LATEST} ]" From bc9172fa74f7307b64b57e5c05ed740a3109ef86 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 10 Dec 2023 21:08:19 +0100 Subject: [PATCH 209/592] docs: updated `CONTRIBUTORS.md` (#3691) --- CONTRIBUTORS.md | 79 +++++++++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 36 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 46e0523f084..860f5df192f 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1438,6 +1438,13 @@ Thanks goes to these wonderful people ✨ mrPjer + + + p3dda +
+ p3dda + + peter-hartmann @@ -1458,15 +1465,15 @@ Thanks goes to these wonderful people ✨
remoe - + + romansey
romansey - - + MightySCollins @@ -1501,15 +1508,15 @@ Thanks goes to these wonderful people ✨
3ap - + + shyim
shyim - - + sjmudd @@ -1544,15 +1551,15 @@ Thanks goes to these wonderful people ✨
5ven - + + syl20bnr
syl20bnr - - + sylvaindumont @@ -1587,15 +1594,15 @@ Thanks goes to these wonderful people ✨
tweibert - + + torus
torus - - + VictorKoenders @@ -1630,15 +1637,15 @@ Thanks goes to these wonderful people ✨
vilisas - + + 42wim
42wim - - + ShiriNmi1520 @@ -1673,15 +1680,15 @@ Thanks goes to these wonderful people ✨
awb99 - + + brainkiller
brainkiller - - + cternes @@ -1716,15 +1723,15 @@ Thanks goes to these wonderful people ✨
ghnp5 - + + helmutundarnold
helmutundarnold - - + hnws @@ -1759,15 +1766,15 @@ Thanks goes to these wonderful people ✨
jjtt - + + paralax
paralax - - + jpduyx @@ -1802,15 +1809,15 @@ Thanks goes to these wonderful people ✨
matrixes - + + mchamplain
mchamplain - - + millerjason @@ -1845,15 +1852,15 @@ Thanks goes to these wonderful people ✨
olaf-mandel - + + ontheair81
ontheair81 - - + pravynandas @@ -1888,15 +1895,15 @@ Thanks goes to these wonderful people ✨
rriski - + + schnippl0r
schnippl0r - - + smargold476 @@ -1931,15 +1938,15 @@ Thanks goes to these wonderful people ✨
tamueller - + + vivacarvajalito
vivacarvajalito - - + wligtenberg From 2f5dfed7261be9338cce74b4e5ed64763b7ef038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Pl=C3=B6tz?= Date: Sun, 10 Dec 2023 22:22:31 +0100 Subject: [PATCH 210/592] fix: Only set `virtual_mailbox_maps` to `texthash` when using the `FILE` account provisioner (#3693) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: René Plötz --- target/scripts/startup/setup.d/postfix.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/target/scripts/startup/setup.d/postfix.sh b/target/scripts/startup/setup.d/postfix.sh index 126a195cf98..58998376a79 100644 --- a/target/scripts/startup/setup.d/postfix.sh +++ b/target/scripts/startup/setup.d/postfix.sh @@ -79,7 +79,9 @@ EOF # /etc/postfix/vmailbox is created by: scripts/helpers/accounts.sh:_create_accounts() # This file config is for Postfix to verify a mail account exists before accepting # mail arriving and delivering it to Dovecot over LMTP. - postconf 'virtual_mailbox_maps = texthash:/etc/postfix/vmailbox' + if [[ ${ACCOUNT_PROVISIONER} == 'FILE' ]]; then + postconf 'virtual_mailbox_maps = texthash:/etc/postfix/vmailbox' + fi postconf 'virtual_transport = lmtp:unix:/var/run/dovecot/lmtp' fi From bbed3f66081a838275c54ac32de5d3ceac4f49b1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 21:36:30 +0100 Subject: [PATCH 211/592] chore(deps): Bump actions/stale from 8 to 9 (#3696) --- .github/workflows/handle_stalled.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/handle_stalled.yml b/.github/workflows/handle_stalled.yml index 236dcec9428..672286fe1f2 100644 --- a/.github/workflows/handle_stalled.yml +++ b/.github/workflows/handle_stalled.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Close stale issues - uses: actions/stale@v8 + uses: actions/stale@v9 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 20 From 98a4c13ca9c8340c38a2d9ae6aef5607d695a9d5 Mon Sep 17 00:00:00 2001 From: Casper Date: Mon, 18 Dec 2023 12:26:28 +0100 Subject: [PATCH 212/592] Add ENV `ENABLE_IMAP` (#3703) --- CHANGELOG.md | 5 ++- docs/content/config/environment.md | 7 ++++- mailserver.env | 10 ++++-- target/scripts/startup/setup.d/dovecot.sh | 38 ++++++++++++++--------- target/scripts/startup/variables-stack.sh | 1 + 5 files changed, 43 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3553804fa64..339c92e0ede 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,12 @@ All notable changes to this project will be documented in this file. The format ### Added +- **Dovecot:** + - ENV `ENABLE_IMAP` ([#3703](https://github.com/docker-mailserver/docker-mailserver/pull/3703)) - **Tests:** - You can now use `make run-local-instance` to run a DMS image that was built locally to test changes ([#3663](https://github.com/docker-mailserver/docker-mailserver/pull/3663)) -- Log a warning when update-check is enabled, but no stable release image is used ([#3684](https://github.com/docker-mailserver/docker-mailserver/pull/3684)) +- **Internal**: + - Log a warning when update-check is enabled, but no stable release image is used ([#3684](https://github.com/docker-mailserver/docker-mailserver/pull/3684)) ### Updates diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index b8e257cca33..5ef3b968afe 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -131,9 +131,14 @@ Enabled `policyd-spf` in Postfix's configuration. You will likely want to set th ##### ENABLE_POP3 -- **empty** => POP3 service disabled +- **0** => POP3 service disabled - 1 => Enables POP3 service +##### ENABLE_IMAP + +- 0 => Disabled +- **1** => Enabled + ##### ENABLE_CLAMAV - **0** => ClamAV is disabled diff --git a/mailserver.env b/mailserver.env index 6878622499a..48b537bcf32 100644 --- a/mailserver.env +++ b/mailserver.env @@ -119,10 +119,16 @@ ENABLE_OPENDMARC=1 # - **1** => Enabled ENABLE_POLICYD_SPF=1 -# 1 => Enables POP3 service -# empty => disables POP3 +# Enables POP3 service +# - **0** => Disabled +# - 1 => Enabled ENABLE_POP3= +# Enables IMAP service +# - 0 => Disabled +# - **1** => Enabled +ENABLE_IMAP=1 + # Enables ClamAV, and anti-virus scanner. # 1 => Enabled # **0** => Disabled diff --git a/target/scripts/startup/setup.d/dovecot.sh b/target/scripts/startup/setup.d/dovecot.sh index 3eeda286c37..e46aca21d91 100644 --- a/target/scripts/startup/setup.d/dovecot.sh +++ b/target/scripts/startup/setup.d/dovecot.sh @@ -6,12 +6,10 @@ function _setup_dovecot() { cp -a /usr/share/dovecot/protocols.d /etc/dovecot/ # disable pop3 (it will be eventually enabled later in the script, if requested) mv /etc/dovecot/protocols.d/pop3d.protocol /etc/dovecot/protocols.d/pop3d.protocol.disab + # disable imap (it will be eventually enabled later in the script, if requested) + mv /etc/dovecot/protocols.d/imapd.protocol /etc/dovecot/protocols.d/imapd.protocol.disab mv /etc/dovecot/protocols.d/managesieved.protocol /etc/dovecot/protocols.d/managesieved.protocol.disab - sed -i -e 's|#ssl = yes|ssl = yes|g' /etc/dovecot/conf.d/10-master.conf - sed -i -e 's|#port = 993|port = 993|g' /etc/dovecot/conf.d/10-master.conf - sed -i -e 's|#port = 995|port = 995|g' /etc/dovecot/conf.d/10-master.conf - sed -i -e 's|#ssl = yes|ssl = required|g' /etc/dovecot/conf.d/10-ssl.conf - sed -i 's|^postmaster_address = .*$|postmaster_address = '"${POSTMASTER_ADDRESS}"'|g' /etc/dovecot/conf.d/15-lda.conf + sedfile -i 's|^postmaster_address = .*$|postmaster_address = '"${POSTMASTER_ADDRESS}"'|g' /etc/dovecot/conf.d/15-lda.conf if ! grep -q -E '^stats_writer_socket_path=' /etc/dovecot/dovecot.conf; then printf '\n%s\n' 'stats_writer_socket_path=' >>/etc/dovecot/dovecot.conf @@ -37,9 +35,21 @@ function _setup_dovecot() { esac + if [[ ${ENABLE_POP3} -eq 1 || ${ENABLE_IMAP} -eq 1 ]]; then + sedfile -i -e 's|#ssl = yes|ssl = yes|g' /etc/dovecot/conf.d/10-master.conf + sedfile -i -e 's|#ssl = yes|ssl = required|g' /etc/dovecot/conf.d/10-ssl.conf + fi + if [[ ${ENABLE_POP3} -eq 1 ]]; then _log 'debug' 'Enabling POP3 services' mv /etc/dovecot/protocols.d/pop3d.protocol.disab /etc/dovecot/protocols.d/pop3d.protocol + sedfile -i -e 's|#port = 995|port = 995|g' /etc/dovecot/conf.d/10-master.conf + fi + + if [[ ${ENABLE_IMAP} -eq 1 ]]; then + _log 'debug' 'Enabling IMAP services' + mv /etc/dovecot/protocols.d/imapd.protocol.disab /etc/dovecot/protocols.d/imapd.protocol + sedfile -i -e 's|#port = 993|port = 993|g' /etc/dovecot/conf.d/10-master.conf fi [[ -f /tmp/docker-mailserver/dovecot.cf ]] && cp /tmp/docker-mailserver/dovecot.cf /etc/dovecot/local.conf @@ -89,23 +99,23 @@ function _setup_dovecot_quota() { # disable dovecot quota in docevot confs if [[ -f /etc/dovecot/conf.d/90-quota.conf ]]; then mv /etc/dovecot/conf.d/90-quota.conf /etc/dovecot/conf.d/90-quota.conf.disab - sed -i \ + sedfile -i \ "s|mail_plugins = \$mail_plugins quota|mail_plugins = \$mail_plugins|g" \ /etc/dovecot/conf.d/10-mail.conf - sed -i \ + sedfile -i \ "s|mail_plugins = \$mail_plugins imap_quota|mail_plugins = \$mail_plugins|g" \ /etc/dovecot/conf.d/20-imap.conf fi # disable quota policy check in postfix - sed -i "s|check_policy_service inet:localhost:65265||g" /etc/postfix/main.cf + sedfile -i "s|check_policy_service inet:localhost:65265||g" /etc/postfix/main.cf else if [[ -f /etc/dovecot/conf.d/90-quota.conf.disab ]]; then mv /etc/dovecot/conf.d/90-quota.conf.disab /etc/dovecot/conf.d/90-quota.conf - sed -i \ + sedfile -i \ "s|mail_plugins = \$mail_plugins|mail_plugins = \$mail_plugins quota|g" \ /etc/dovecot/conf.d/10-mail.conf - sed -i \ + sedfile -i \ "s|mail_plugins = \$mail_plugins|mail_plugins = \$mail_plugins imap_quota|g" \ /etc/dovecot/conf.d/20-imap.conf fi @@ -113,11 +123,11 @@ function _setup_dovecot_quota() { local MESSAGE_SIZE_LIMIT_MB=$((POSTFIX_MESSAGE_SIZE_LIMIT / 1000000)) local MAILBOX_LIMIT_MB=$((POSTFIX_MAILBOX_SIZE_LIMIT / 1000000)) - sed -i \ + sedfile -i \ "s|quota_max_mail_size =.*|quota_max_mail_size = ${MESSAGE_SIZE_LIMIT_MB}$([[ ${MESSAGE_SIZE_LIMIT_MB} -eq 0 ]] && echo "" || echo "M")|g" \ /etc/dovecot/conf.d/90-quota.conf - sed -i \ + sedfile -i \ "s|quota_rule = \*:storage=.*|quota_rule = *:storage=${MAILBOX_LIMIT_MB}$([[ ${MAILBOX_LIMIT_MB} -eq 0 ]] && echo "" || echo "M")|g" \ /etc/dovecot/conf.d/90-quota.conf @@ -127,7 +137,7 @@ function _setup_dovecot_quota() { fi # enable quota policy check in postfix - sed -i -E \ + sedfile -i -E \ "s|(reject_unknown_recipient_domain)|\1, check_policy_service inet:localhost:65265|g" \ /etc/postfix/main.cf fi @@ -188,5 +198,5 @@ function _setup_dovecot_dhparam() { function _setup_dovecot_hostname() { _log 'debug' 'Applying hostname to Dovecot' - sed -i "s|^#hostname =.*$|hostname = '${HOSTNAME}'|g" /etc/dovecot/conf.d/15-lda.conf + sedfile -i "s|^#hostname =.*$|hostname = '${HOSTNAME}'|g" /etc/dovecot/conf.d/15-lda.conf } diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index 3b575f506ea..2660ce89761 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -87,6 +87,7 @@ function __environment_variables_general_setup() { VARS[ENABLE_OPENDMARC]="${ENABLE_OPENDMARC:=1}" VARS[ENABLE_POLICYD_SPF]="${ENABLE_POLICYD_SPF:=1}" VARS[ENABLE_POP3]="${ENABLE_POP3:=0}" + VARS[ENABLE_IMAP]="${ENABLE_IMAP:=1}" VARS[ENABLE_POSTGREY]="${ENABLE_POSTGREY:=0}" VARS[ENABLE_QUOTAS]="${ENABLE_QUOTAS:=1}" VARS[ENABLE_RSPAMD]="${ENABLE_RSPAMD:=0}" From 083e46408413c11e71f7e9ae299ae89874d8c35c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 14:05:37 +0100 Subject: [PATCH 213/592] chore(deps): Bump github/codeql-action from 2 to 3 (#3709) --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 9524555792e..08b4fd29fc4 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -62,6 +62,6 @@ jobs: fail-build: false - name: 'Upload vulnerability report' - uses: github/codeql-action/upload-sarif@v2 + uses: github/codeql-action/upload-sarif@v3 with: sarif_file: ${{ steps.scan.outputs.sarif }} From b9f7ff86dea6709f785be0bf900c2845f9e321ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 15:50:56 +0100 Subject: [PATCH 214/592] chore(deps): Bump docker/metadata-action from 5.3.0 to 5.4.0 (#3710) Bumps [docker/metadata-action](https://github.com/docker/metadata-action) from 5.3.0 to 5.4.0. - [Release notes](https://github.com/docker/metadata-action/releases) - [Commits](https://github.com/docker/metadata-action/compare/v5.3.0...v5.4.0) --- updated-dependencies: - dependency-name: docker/metadata-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 36b44770856..b0a32b93b12 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -23,7 +23,7 @@ jobs: - name: 'Prepare tags' id: prep - uses: docker/metadata-action@v5.3.0 + uses: docker/metadata-action@v5.4.0 with: images: | ${{ secrets.DOCKER_REPOSITORY }} From 5015dc80b16f61e29cb9b496ceb6c6210de635a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 00:16:38 +0100 Subject: [PATCH 215/592] chore(deps): Bump actions/upload-artifact from 3 to 4 (#3708) --- .github/workflows/docs-preview-prepare.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-preview-prepare.yml b/.github/workflows/docs-preview-prepare.yml index befd4008d82..cee5562abfd 100644 --- a/.github/workflows/docs-preview-prepare.yml +++ b/.github/workflows/docs-preview-prepare.yml @@ -73,7 +73,7 @@ jobs: tar --zstd -cf artifact.tar.zst pr.env ${{ env.BUILD_DIR }} - name: 'Upload artifact for workflow transfer' - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: preview-build path: artifact.tar.zst From ee87291225f55feb4a4c96b2383c55827db54736 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 01:29:15 +0100 Subject: [PATCH 216/592] chore(deps): Bump dawidd6/action-download-artifact from 2 to 3 (#3707) --- .github/workflows/docs-preview-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index cac2ac64bc0..93819efaeb4 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -25,7 +25,7 @@ jobs: # The official Github Action for downloading artifacts does not support multi-workflow - name: 'Download build artifact' - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v3 with: github_token: ${{ secrets.GITHUB_TOKEN }} run_id: ${{ github.event.workflow_run.id }} From 5908d9f060b1c74d7f168a578d11869ff15f7ad4 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 19 Dec 2023 14:33:38 +1300 Subject: [PATCH 217/592] tests(refactor): Dovecot quotas (#3068) * chore: Extract out Dovecot Quota test cases into new test file Test cases are just cut + paste, no logic changed there yet. * chore: Rename test case descriptions * chore: Use `setup ...` methods instead of direct calls * chore: Adjust `_run_in_container_bash` to `_run_in_container` Plus some additional bug fixes in the disabled test case * tests(refactor): Revise ENV test cases for max mailbox and message sizes * tests(refactor): Revise ENV test cases for mailbox and message limits v2 Removes the extra variables and filtering in favour of explicit values instead of matching for comparison. - Easier at a glance to know what is actually expected. - Additionally reworks the quota limit checks in other test cases. Using a different formatter for `doveadm` is easier to match the desired value (`Limit`). * chore: Sync improvement from `tests.bats` master --- NOTE: This PR has been merged to avoid additional maintenance burden without losing the improvements. It was not considered complete, but remaining tasks were not documented in the PR. --- docs/content/config/environment.md | 8 +- mailserver.env | 4 +- target/bin/setquota | 6 +- .../parallel/set1/dovecot/dovecot_quotas.bats | 246 ++++++++++++++++++ test/tests/serial/tests.bats | 193 -------------- 5 files changed, 257 insertions(+), 200 deletions(-) create mode 100644 test/tests/parallel/set1/dovecot/dovecot_quotas.bats diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 5ef3b968afe..71807c87401 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -228,9 +228,9 @@ Provide any valid URI. Examples: - `lmtps:inet::` (secure lmtp with starttls) - `lmtp::2003` (use kopano as mailstore) -##### POSTFIX\_MAILBOX\_SIZE\_LIMIT +##### POSTFIX_MAILBOX_SIZE_LIMIT -Set the mailbox size limit for all users. If set to zero, the size will be unlimited (default). +Set the mailbox size limit for all users. If set to zero, the size will be unlimited (default). Size is in bytes. - **empty** => 0 (no limit) @@ -241,9 +241,9 @@ Set the mailbox size limit for all users. If set to zero, the size will be unlim See [mailbox quota][docs-accounts-quota]. -##### POSTFIX\_MESSAGE\_SIZE\_LIMIT +##### POSTFIX_MESSAGE_SIZE_LIMIT -Set the message size limit for all users. If set to zero, the size will be unlimited (not recommended!) +Set the message size limit for all users. If set to zero, the size will be unlimited (not recommended!). Size is in bytes. - **empty** => 10240000 (~10 MB) diff --git a/mailserver.env b/mailserver.env index 48b537bcf32..e84157a2331 100644 --- a/mailserver.env +++ b/mailserver.env @@ -254,7 +254,7 @@ VIRUSMAILS_DELETE_DELAY= # `lmtp::2003` (use kopano as mailstore) POSTFIX_DAGENT= -# Set the mailbox size limit for all users. If set to zero, the size will be unlimited (default). +# Set the mailbox size limit for all users. If set to zero, the size will be unlimited (default). Size is in bytes. # # empty => 0 POSTFIX_MAILBOX_SIZE_LIMIT= @@ -264,7 +264,7 @@ POSTFIX_MAILBOX_SIZE_LIMIT= # 1 => Dovecot quota is enabled ENABLE_QUOTAS=1 -# Set the message size limit for all users. If set to zero, the size will be unlimited (not recommended!) +# Set the message size limit for all users. If set to zero, the size will be unlimited (not recommended!). Size is in bytes. # # empty => 10240000 (~10 MB) POSTFIX_MESSAGE_SIZE_LIMIT= diff --git a/target/bin/setquota b/target/bin/setquota index 5b2bba411ac..039421b24e8 100755 --- a/target/bin/setquota +++ b/target/bin/setquota @@ -59,10 +59,14 @@ function _quota_request_if_missing() { fi } + +# Dovecot docs incorrectly refer to these units with names for SI types (base 10), +# But then mentions they're actually treated as IEC type (base 2): +# https://doc.dovecot.org/settings/types/#size function _quota_unit_is_valid() { if ! grep -qE "^([0-9]+(B|k|M|G|T)|0)\$" <<< "${QUOTA}"; then __usage - _exit_with_error 'Invalid quota format. e.g. 302M (B (byte), k (kilobyte), M (megabyte), G (gigabyte) or T (terabyte))' + _exit_with_error 'Invalid quota format. e.g. 302M (B (byte), k (kibibyte), M (mebibyte), G (gibibyte) or T (tebibyte))' fi } diff --git a/test/tests/parallel/set1/dovecot/dovecot_quotas.bats b/test/tests/parallel/set1/dovecot/dovecot_quotas.bats new file mode 100644 index 00000000000..2c176235dc7 --- /dev/null +++ b/test/tests/parallel/set1/dovecot/dovecot_quotas.bats @@ -0,0 +1,246 @@ +load "${REPOSITORY_ROOT}/test/helper/common" +load "${REPOSITORY_ROOT}/test/helper/setup" + +# upstream default: 10 240 000 +# https://www.postfix.org/postconf.5.html#message_size_limit +# > The maximal size in bytes of a message, including envelope information. +# > The value cannot exceed LONG_MAX (typically, a 32-bit or 64-bit signed integer). +# > Note: Be careful when making changes. Excessively small values will result in the loss of non-delivery notifications, when a bounce message size exceeds the local or remote MTA's message size limit. + +# upstream default: 51 200 000 +# https://www.postfix.org/postconf.5.html#mailbox_size_limit +# > The maximal size of any local(8) individual mailbox or maildir file, or zero (no limit). +# > In fact, this limits the size of any file that is written to upon local delivery, including files written by external commands that are executed by the local(8) delivery agent. +# > The value cannot exceed LONG_MAX (typically, a 32-bit or 64-bit signed integer). +# > This limit must not be smaller than the message size limit. + +# upstream default: 51 200 000 +# https://www.postfix.org/postconf.5.html#virtual_mailbox_limit +# > The maximal size in bytes of an individual virtual(8) mailbox or maildir file, or zero (no limit). +# > This parameter is specific to the virtual(8) delivery agent. +# > It does not apply when mail is delivered with a different mail delivery program. + +BATS_TEST_NAME_PREFIX='[Dovecot Quotas] ' +CONTAINER_NAME='dms-test_dovecot-quotas' + +function setup_file() { + _init_with_defaults + + local CONTAINER_ARGS_ENV_CUSTOM=( + --env ENABLE_QUOTAS=1 + --env POSTFIX_MAILBOX_SIZE_LIMIT=4096000 + --env POSTFIX_MESSAGE_SIZE_LIMIT=2048000 + --env PERMIT_DOCKER=container + ) + _common_container_setup 'CONTAINER_ARGS_ENV_CUSTOM' +} + +function teardown_file() { _default_teardown ; } + +@test 'should only support setting quota for a valid account' { + # Prepare + _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' + + # Actual tests + _run_in_container setup quota set quota_user 50M + assert_failure + + _run_in_container setup quota set username@fulldomain 50M + assert_failure + + _run_in_container setup quota set quota_user@domain.tld 50M + assert_success + + # Cleanup + _run_in_container setup email del -y quota_user@domain.tld + assert_success +} + +@test 'should only allow valid units as quota size' { + # Prepare + _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' + + # Actual tests + _run_in_container setup quota set quota_user@domain.tld 26GIGOTS + assert_failure + _run_in_container setup quota set quota_user@domain.tld 123 + assert_failure + _run_in_container setup quota set quota_user@domain.tld M + assert_failure + _run_in_container setup quota set quota_user@domain.tld -60M + assert_failure + + + _run_in_container setup quota set quota_user@domain.tld 10B + assert_success + _run_in_container setup quota set quota_user@domain.tld 10k + assert_success + _run_in_container setup quota set quota_user@domain.tld 10M + assert_success + _run_in_container setup quota set quota_user@domain.tld 10G + assert_success + _run_in_container setup quota set quota_user@domain.tld 10T + assert_success + + # Cleanup + _run_in_container setup email del -y quota_user@domain.tld + assert_success +} + +@test 'should only support removing quota from a valid account' { + # Prepare + _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' + + # Actual tests + _run_in_container setup quota del uota_user@domain.tld + assert_failure + _run_in_container setup quota del quota_user + assert_failure + _run_in_container setup quota del dontknowyou@domain.tld + assert_failure + + _run_in_container setup quota set quota_user@domain.tld 10T + assert_success + _run_in_container setup quota del quota_user@domain.tld + assert_success + _run_in_container grep -i 'quota_user@domain.tld' /tmp/docker-mailserver/dovecot-quotas.cf + assert_failure + + # Cleanup + _run_in_container setup email del -y quota_user@domain.tld + assert_success +} + +@test 'should not error when there is no quota to remove for an account' { + # Prepare + _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' + + # Actual tests + _run_in_container grep -i 'quota_user@domain.tld' /tmp/docker-mailserver/dovecot-quotas.cf + assert_failure + + _run_in_container setup quota del quota_user@domain.tld + assert_success + _run_in_container setup quota del quota_user@domain.tld + assert_success + + # Cleanup + _run_in_container setup email del -y quota_user@domain.tld + assert_success +} + +@test 'should have configured Postfix to use the Dovecot quota-status service' { + _run_in_container postconf + assert_success + assert_output --partial 'check_policy_service inet:localhost:65265' +} + +@test '(ENV POSTFIX_MAILBOX_SIZE_LIMIT) should be configured for both Postfix and Dovecot' { + _run_in_container postconf -h mailbox_size_limit + assert_output 4096000 + + # Dovecot mailbox is sized by `virtual_mailbox_size` from Postfix: + _run_in_container postconf -h virtual_mailbox_limit + assert_output 4096000 + + # Quota support: + _run_in_container doveconf -h plugin/quota_rule + # Global default storage limit quota for each mailbox 4 MiB: + assert_output '*:storage=4M' + + # Sizes are equivalent - Bytes to MiB (rounded): + run numfmt --to=iec --format '%.0f' 4096000 + assert_output '4M' +} + +@test '(ENV POSTFIX_MESSAGE_SIZE_LIMIT) should be configured for both Postfix and Dovecot' { + _run_in_container postconf -h message_size_limit + assert_output 2048000 + + _run_in_container doveconf -h plugin/quota_max_mail_size + assert_output '2M' + + # Sizes are equivalent - Bytes to MiB (rounded): + run numfmt --to=iec --format '%.0f' 2048000 + assert_output '2M' +} + +@test 'Deleting an mailbox account should also remove that account from dovecot-quotas.cf' { + _add_mail_account_then_wait_until_ready 'quserremoved@domain.tld' + + _run_in_container setup quota set quserremoved@domain.tld 12M + assert_success + + _run_in_container cat '/tmp/docker-mailserver/dovecot-quotas.cf' + assert_success + assert_output 'quserremoved@domain.tld:12M' + + _run_in_container setup email del -y quserremoved@domain.tld + assert_success + + _run_in_container cat /tmp/docker-mailserver/dovecot-quotas.cf + assert_success + refute_output --partial 'quserremoved@domain.tld:12M' +} + +@test 'Dovecot should acknowledge quota configured for accounts' { + # sed -nE 's/.*STORAGE.*Limit=([0-9]+).*/\1/p' | numfmt --from-unit=1024 --to=iec --format '%.0f' + local CMD_GET_QUOTA="doveadm -f flow quota get -u 'user1@localhost.localdomain'" + + # 4M == 4096 kiB (numfmt --to-unit=1024 --from=iec 4M) + _run_in_container_bash "${CMD_GET_QUOTA}" + assert_line --partial 'Type=STORAGE Value=0 Limit=4096' + + # Setting a new limit for the user: + _run_in_container setup quota set 'user1@localhost.localdomain' 50M + assert_success + # 50M (50 * 1024^2) == 51200 kiB (numfmt --to-unit=1024 --from=iec 52428800) + run _repeat_until_success_or_timeout 20 _exec_in_container_bash "${CMD_GET_QUOTA} | grep -o 'Type=STORAGE Value=0 Limit=51200'" + assert_success + + # Deleting quota resets it to default global quota limit (`plugin/quota_rule`): + _run_in_container setup quota del 'user1@localhost.localdomain' + assert_success + run _repeat_until_success_or_timeout 20 _exec_in_container_bash "${CMD_GET_QUOTA} | grep -o 'Type=STORAGE Value=0 Limit=4096'" + assert_success +} + +@test 'should receive a warning mail from Dovecot when quota is exceeded' { + # skip 'disabled as it fails randomly: https://github.com/docker-mailserver/docker-mailserver/pull/2511' + + # Prepare + _add_mail_account_then_wait_until_ready 'quotauser@otherdomain.tld' + + # Actual tests + _run_in_container setup quota set quotauser@otherdomain.tld 10k + assert_success + + # wait until quota has been updated + run _repeat_until_success_or_timeout 20 _exec_in_container_bash "doveadm -f flow quota get -u 'quotauser@otherdomain.tld' | grep -o 'Type=STORAGE Value=0 Limit=10'" + assert_success + + # dovecot and postfix has been restarted + _wait_for_service postfix + _wait_for_service dovecot + sleep 10 + + # send some big emails + _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' + _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' + _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' + # check for quota warn message existence + run _repeat_until_success_or_timeout 20 _exec_in_container grep -R 'Subject: quota warning' /var/mail/otherdomain.tld/quotauser/new/ + assert_success + + run _repeat_until_success_or_timeout 20 sh -c "docker logs ${CONTAINER_NAME} | grep 'Quota exceeded (mailbox for user is full)'" + assert_success + + # ensure only the first big message and the warn message are present (other messages are rejected: mailbox is full) + _run_in_container sh -c 'ls /var/mail/otherdomain.tld/quotauser/new/ | wc -l' + assert_success + assert_output "2" + + # Cleanup + _run_in_container setup email del -y quotauser@otherdomain.tld + assert_success +} diff --git a/test/tests/serial/tests.bats b/test/tests/serial/tests.bats index 20ee0dd1fdb..094454f0040 100644 --- a/test/tests/serial/tests.bats +++ b/test/tests/serial/tests.bats @@ -17,7 +17,6 @@ function setup_file() { local CONTAINER_ARGS_ENV_CUSTOM=( --env ENABLE_AMAVIS=1 --env AMAVIS_LOGLEVEL=2 - --env ENABLE_QUOTAS=1 --env ENABLE_SRS=1 --env PERMIT_DOCKER=host --env PFLOGSUMM_TRIGGER=logrotate @@ -244,198 +243,6 @@ zip EOF } -@test "quota: setquota user must be existing" { - _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' - - _run_in_container_bash "setquota quota_user 50M" - assert_failure - _run_in_container_bash "setquota quota_user@domain.tld 50M" - assert_success - - _run_in_container_bash "setquota username@fulldomain 50M" - assert_failure - - _run_in_container_bash "delmailuser -y quota_user@domain.tld" - assert_success -} - -@test "quota: setquota must be well formatted" { - _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' - - _run_in_container_bash "setquota quota_user@domain.tld 26GIGOTS" - assert_failure - _run_in_container_bash "setquota quota_user@domain.tld 123" - assert_failure - _run_in_container_bash "setquota quota_user@domain.tld M" - assert_failure - _run_in_container_bash "setquota quota_user@domain.tld -60M" - assert_failure - - - _run_in_container_bash "setquota quota_user@domain.tld 10B" - assert_success - _run_in_container_bash "setquota quota_user@domain.tld 10k" - assert_success - _run_in_container_bash "setquota quota_user@domain.tld 10M" - assert_success - _run_in_container_bash "setquota quota_user@domain.tld 10G" - assert_success - _run_in_container_bash "setquota quota_user@domain.tld 10T" - assert_success - - - _run_in_container_bash "delmailuser -y quota_user@domain.tld" - assert_success -} - -@test "quota: delquota user must be existing" { - _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' - - _run_in_container_bash "delquota uota_user@domain.tld" - assert_failure - _run_in_container_bash "delquota quota_user" - assert_failure - _run_in_container_bash "delquota dontknowyou@domain.tld" - assert_failure - - _run_in_container_bash "setquota quota_user@domain.tld 10T" - assert_success - _run_in_container_bash "delquota quota_user@domain.tld" - assert_success - _run_in_container_bash "grep -i 'quota_user@domain.tld' /tmp/docker-mailserver/dovecot-quotas.cf" - assert_failure - - _run_in_container_bash "delmailuser -y quota_user@domain.tld" - assert_success -} - -@test "quota: delquota allow when no quota for existing user" { - _add_mail_account_then_wait_until_ready 'quota_user@domain.tld' - - _run_in_container_bash "grep -i 'quota_user@domain.tld' /tmp/docker-mailserver/dovecot-quotas.cf" - assert_failure - - _run_in_container_bash "delquota quota_user@domain.tld" - assert_success - _run_in_container_bash "delquota quota_user@domain.tld" - assert_success - - _run_in_container_bash "delmailuser -y quota_user@domain.tld" - assert_success -} - -@test "quota: dovecot quota present in postconf" { - _run_in_container_bash "postconf | grep 'check_policy_service inet:localhost:65265'" - assert_success -} - - -@test "quota: dovecot mailbox max size must be equal to postfix mailbox max size" { - postfix_mailbox_size=$(_exec_in_container_bash "postconf | grep -Po '(?<=mailbox_size_limit = )[0-9]+'") - run echo "${postfix_mailbox_size}" - refute_output "" - - # dovecot relies on virtual_mailbox_size by default - postfix_virtual_mailbox_size=$(_exec_in_container_bash "postconf | grep -Po '(?<=virtual_mailbox_limit = )[0-9]+'") - assert_equal "${postfix_virtual_mailbox_size}" "${postfix_mailbox_size}" - - postfix_mailbox_size_mb=$(( postfix_mailbox_size / 1000000)) - - dovecot_mailbox_size_mb=$(_exec_in_container_bash "doveconf | grep -oP '(?<=quota_rule \= \*\:storage=)[0-9]+'") - run echo "${dovecot_mailbox_size_mb}" - refute_output "" - - assert_equal "${postfix_mailbox_size_mb}" "${dovecot_mailbox_size_mb}" -} - - -@test "quota: dovecot message max size must be equal to postfix messsage max size" { - postfix_message_size=$(_exec_in_container_bash "postconf | grep -Po '(?<=message_size_limit = )[0-9]+'") - run echo "${postfix_message_size}" - refute_output "" - - postfix_message_size_mb=$(( postfix_message_size / 1000000)) - - dovecot_message_size_mb=$(_exec_in_container_bash "doveconf | grep -oP '(?<=quota_max_mail_size = )[0-9]+'") - run echo "${dovecot_message_size_mb}" - refute_output "" - - assert_equal "${postfix_message_size_mb}" "${dovecot_message_size_mb}" -} - -@test "quota: quota directive is removed when mailbox is removed" { - _add_mail_account_then_wait_until_ready 'quserremoved@domain.tld' - - _run_in_container_bash "setquota quserremoved@domain.tld 12M" - assert_success - - _run_in_container_bash 'cat /tmp/docker-mailserver/dovecot-quotas.cf | grep -E "^quserremoved@domain.tld\:12M\$" | wc -l | grep 1' - assert_success - - _run_in_container_bash "delmailuser -y quserremoved@domain.tld" - assert_success - - _run_in_container_bash 'cat /tmp/docker-mailserver/dovecot-quotas.cf | grep -E "^quserremoved@domain.tld\:12M\$"' - assert_failure -} - -@test "quota: dovecot applies user quota" { - _run_in_container_bash "doveadm quota get -u 'user1@localhost.localdomain' | grep 'User quota STORAGE'" - assert_output --partial "- 0" - - _run_in_container_bash "setquota user1@localhost.localdomain 50M" - assert_success - - # wait until quota has been updated - run _repeat_until_success_or_timeout 20 _exec_in_container_bash 'doveadm quota get -u user1@localhost.localdomain | grep -oP "(User quota STORAGE\s+[0-9]+\s+)51200(.*)"' - assert_success - - _run_in_container_bash "delquota user1@localhost.localdomain" - assert_success - - # wait until quota has been updated - run _repeat_until_success_or_timeout 20 _exec_in_container_bash 'doveadm quota get -u user1@localhost.localdomain | grep -oP "(User quota STORAGE\s+[0-9]+\s+)-(.*)"' - assert_success -} - -@test "quota: warn message received when quota exceeded" { - skip 'disabled as it fails randomly: https://github.com/docker-mailserver/docker-mailserver/pull/2511' - - # create user - _add_mail_account_then_wait_until_ready 'quotauser@otherdomain.tld' - _run_in_container_bash 'setquota quotauser@otherdomain.tld 10k' - assert_success - - # wait until quota has been updated - run _repeat_until_success_or_timeout 20 _exec_in_container_bash 'doveadm quota get -u quotauser@otherdomain.tld | grep -oP \"(User quota STORAGE\s+[0-9]+\s+)10(.*)\"' - assert_success - - # dovecot and postfix has been restarted - _wait_for_service postfix - _wait_for_service dovecot - sleep 10 - - # send some big emails - _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' - _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' - _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' - - # check for quota warn message existence - run _repeat_until_success_or_timeout 20 _exec_in_container_bash 'grep \"Subject: quota warning\" /var/mail/otherdomain.tld/quotauser/new/ -R' - assert_success - - run _repeat_until_success_or_timeout 20 sh -c "docker logs mail | grep 'Quota exceeded (mailbox for user is full)'" - assert_success - - # ensure only the first big message and the warn message are present (other messages are rejected: mailbox is full) - _run_in_container sh -c 'ls /var/mail/otherdomain.tld/quotauser/new/ | wc -l' - assert_success - assert_output "2" - - _run_in_container_bash "delmailuser -y quotauser@otherdomain.tld" - assert_success -} - # # PERMIT_DOCKER mynetworks # From ca2c53dde714a5c4d64aebb6724cfb6d87352a54 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 19 Dec 2023 14:41:07 +1300 Subject: [PATCH 218/592] ci: Avoiding linting `CONTRIBUTORS.yml` (#3705) The file is managed by the `contributors.yml` workflow, no need for linting to be triggered on PRs for that change. This should ideally skip the required check status for the lint workflow which cannot trigger implicitly for automated PRs. If this doesn't work the change should be reverted. --- .github/workflows/linting.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index bdde2d0fa46..1e93377f759 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -2,6 +2,9 @@ name: Lint on: pull_request: + paths-ignore: + # Managed by workflow: contributors.yml + - CONTRIBUTORS.md push: branches: [ master ] From 03052a65b87542ce714d1e58dca579907b534d39 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 19 Dec 2023 18:59:59 +1300 Subject: [PATCH 219/592] ci: Allow lint workflow to be manually triggered (#3714) * ci: Allow lint workflow to be manually triggered Without this a different event must occur to trigger the workflow, which is inconvenient for automated PRs. --- .github/workflows/linting.yml | 6 +++--- CHANGELOG.md | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 1e93377f759..d419e5ec8b1 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -1,10 +1,10 @@ name: Lint on: + # A workflow that creates a PR will not trigger this workflow, + # Providing a manual trigger as a workaround + workflow_dispatch: pull_request: - paths-ignore: - # Managed by workflow: contributors.yml - - CONTRIBUTORS.md push: branches: [ master ] diff --git a/CHANGELOG.md b/CHANGELOG.md index 339c92e0ede..8b1331f1b5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,8 @@ All notable changes to this project will be documented in this file. The format - Update-check: fix 'read' exit status ([#3688](https://github.com/docker-mailserver/docker-mailserver/pull/3688)) - **Rspamd:** - Switch to official arm64 packages to avoid segfaults ([#3686](https://github.com/docker-mailserver/docker-mailserver/pull/3686)) +- **CI / Automation:** + - The lint workflow can now be manually triggered by maintainers ([#3714]https://github.com/docker-mailserver/docker-mailserver/pull/3714) ## [v13.0.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.0.1) From 6a56c7e74936488626db973f1c586d888894fc82 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 06:01:25 +0000 Subject: [PATCH 220/592] docs: update `CONTRIBUTORS.md` (#3704) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 50 ++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 860f5df192f..4103ad5f549 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -442,6 +442,13 @@ Thanks goes to these wonderful people ✨ pbek + + + reneploetz +
+ reneploetz + + Rubytastic2 @@ -469,15 +476,15 @@ Thanks goes to these wonderful people ✨
Zehir - + + guardiande
guardiande - - + kamuri @@ -512,15 +519,15 @@ Thanks goes to these wonderful people ✨
m-schmoock - + + VanVan
VanVan - - + elbracht @@ -555,15 +562,15 @@ Thanks goes to these wonderful people ✨
ubenmackin - + + craue
craue - - + danielpanteleit @@ -598,15 +605,15 @@ Thanks goes to these wonderful people ✨
emazzotta - + + fl42
fl42 - - + ipernet @@ -641,15 +648,15 @@ Thanks goes to these wonderful people ✨
millaguie - + + jedateach
jedateach - - + spacecowboy @@ -684,15 +691,15 @@ Thanks goes to these wonderful people ✨
keslerm - + + castorinop
castorinop - - + p-fruck @@ -707,13 +714,6 @@ Thanks goes to these wonderful people ✨ Rillke - - - reneploetz -
- reneploetz - - bobbravo2 From 226ec847a4875e80c4ceff4c486f71a88176b897 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 19 Dec 2023 21:35:16 +1300 Subject: [PATCH 221/592] ci: Remove `VERSION` from `Dockerfile` (#3711) * ci: Remove `VERSION` from `Dockerfile` This line was meant to be dropped with the switch to using the `DMS_RELEASE` ENV. --- CHANGELOG.md | 1 + Dockerfile | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b1331f1b5d..37a7a123675 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ All notable changes to this project will be documented in this file. The format - **Internal**: - The container startup welcome log message now references `DMS_RELEASE` ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) - `VERSION` was incremented for prior releases to be notified of the v13.0.1 patch release ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) + - `VERSION` is no longer included in the image ([#3711](https://github.com/docker-mailserver/docker-mailserver/pull/3711)) - Update-check: fix 'read' exit status ([#3688](https://github.com/docker-mailserver/docker-mailserver/pull/3688)) - **Rspamd:** - Switch to official arm64 packages to avoid segfaults ([#3686](https://github.com/docker-mailserver/docker-mailserver/pull/3686)) diff --git a/Dockerfile b/Dockerfile index 0f19521add4..4d0e3568868 100644 --- a/Dockerfile +++ b/Dockerfile @@ -277,8 +277,6 @@ RUN < Date: Wed, 20 Dec 2023 01:43:32 +0100 Subject: [PATCH 222/592] Remove sed statement (#3715) --- CHANGELOG.md | 1 + target/scripts/startup/setup.d/dovecot.sh | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37a7a123675..3fc60c05eec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ All notable changes to this project will be documented in this file. The format - `VERSION` was incremented for prior releases to be notified of the v13.0.1 patch release ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) - `VERSION` is no longer included in the image ([#3711](https://github.com/docker-mailserver/docker-mailserver/pull/3711)) - Update-check: fix 'read' exit status ([#3688](https://github.com/docker-mailserver/docker-mailserver/pull/3688)) + - `ENABLE_QUOTAS=0` no longer tries to remove non-existent config ([#3715](https://github.com/docker-mailserver/docker-mailserver/pull/3715)) - **Rspamd:** - Switch to official arm64 packages to avoid segfaults ([#3686](https://github.com/docker-mailserver/docker-mailserver/pull/3686)) - **CI / Automation:** diff --git a/target/scripts/startup/setup.d/dovecot.sh b/target/scripts/startup/setup.d/dovecot.sh index e46aca21d91..8e7dcfe7c56 100644 --- a/target/scripts/startup/setup.d/dovecot.sh +++ b/target/scripts/startup/setup.d/dovecot.sh @@ -106,9 +106,6 @@ function _setup_dovecot_quota() { "s|mail_plugins = \$mail_plugins imap_quota|mail_plugins = \$mail_plugins|g" \ /etc/dovecot/conf.d/20-imap.conf fi - - # disable quota policy check in postfix - sedfile -i "s|check_policy_service inet:localhost:65265||g" /etc/postfix/main.cf else if [[ -f /etc/dovecot/conf.d/90-quota.conf.disab ]]; then mv /etc/dovecot/conf.d/90-quota.conf.disab /etc/dovecot/conf.d/90-quota.conf From 72517d3f824859cb15a3ccc653ad8cc4bb1a4c32 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 26 Dec 2023 09:53:32 +1300 Subject: [PATCH 223/592] docs: Debugging - Delivery failure from service downtime (#3718) * docs: Debugging - Delivery failure from service downtime Services may be temporarily down, such as when restarted when certificates are updated due to the `check-for-changes.sh` service. This is another known source of intermittent delivery failures. --- CHANGELOG.md | 3 ++- docs/content/config/debugging.md | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fc60c05eec..7a2605088a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,8 @@ All notable changes to this project will be documented in this file. The format ### Updates - **Documentation:** - - Raise awareness in the troubleshooting page for a common misconfiguration when deviating from our advice by using a bare domain ([#3680](https://github.com/docker-mailserver/docker-mailserver/pull/3680)) + - Debugging - Raise awareness in the troubleshooting page for a common misconfiguration when deviating from our advice by using a bare domain ([#3680](https://github.com/docker-mailserver/docker-mailserver/pull/3680)) + - Debugging - Raise awareness of temporary downtime during certificate renewal that can cause a failure to deliver local mail ([#3718](https://github.com/docker-mailserver/docker-mailserver/pull/3718)) - **Internal:** - Postfix configures `virtual_mailbox_maps` and `virtual_transport` during startup instead of using defaults (configured for Dovecot) via our `main.cf` ([#3681](https://github.com/docker-mailserver/docker-mailserver/pull/3681)) diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index 9c3bebb565a..d58430e1f2d 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -55,6 +55,8 @@ Common logs related to this are: If your logs look like this, you likely have [assigned the same FQDN to the DMS `hostname` and your mail accounts][gh-issues::dms-fqdn-misconfigured] which is not supported by default. You can either adjust your DMS `hostname` or follow [this FAQ advice][docs::faq-bare-domain] +It is also possible that [DMS services are temporarily unavailable][gh-issues::dms-services-unavailable] when configuration changes are detected, producing the 2nd error. Certificate updates may be a less obvious trigger. + ## Steps for Debugging DMS 1. **Increase log verbosity**: Very helpful for troubleshooting problems during container startup. Set the environment variable [`LOG_LEVEL`][docs-environment-log-level] to `debug` or `trace`. @@ -126,6 +128,7 @@ This could be from outdated software, or running a system that isn't able to pro [gh-issues]: https://github.com/docker-mailserver/docker-mailserver/issues [gh-issues::dms-fqdn-misconfigured]: https://github.com/docker-mailserver/docker-mailserver/issues/3679#issuecomment-1837609043 +[gh-issues::dms-services-unavailable]: https://github.com/docker-mailserver/docker-mailserver/issues/3679#issuecomment-1848083358 [gh-macos-support]: https://github.com/docker-mailserver/docker-mailserver/issues/3648#issuecomment-1822774080 [gh-discuss-roundcube-fail2ban]: https://github.com/orgs/docker-mailserver/discussions/3273#discussioncomment-5654603 From 8392e3c1a8db660e6fd458b1bf426ab4736104c4 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 29 Dec 2023 13:58:54 +0100 Subject: [PATCH 224/592] release: v13.1.0 (#3720) Co-authored-by: Casper Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 4 ++++ VERSION | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a2605088a0..32aaf78da3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +## [v13.1.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.1.0) + ### Added - **Dovecot:** @@ -22,6 +24,8 @@ All notable changes to this project will be documented in this file. The format - Debugging - Raise awareness of temporary downtime during certificate renewal that can cause a failure to deliver local mail ([#3718](https://github.com/docker-mailserver/docker-mailserver/pull/3718)) - **Internal:** - Postfix configures `virtual_mailbox_maps` and `virtual_transport` during startup instead of using defaults (configured for Dovecot) via our `main.cf` ([#3681](https://github.com/docker-mailserver/docker-mailserver/pull/3681)) +- **Rspamd:** + - Upgraded to version `3.7.5`. This was previously inconsistent between our AMD64 (`3.5`) and ARM64 (`3.4`) images ([#3686](https://github.com/docker-mailserver/docker-mailserver/pull/3686)) ### Fixed diff --git a/VERSION b/VERSION index 5cb7d8566ff..e6ba351366d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -13.0.1 +13.1.0 From 0889b0ff063a37b482113a684d934e4bd728a33c Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 30 Dec 2023 09:59:09 +1300 Subject: [PATCH 225/592] fix: `supervisor-app.conf` - Correct the log location for `postgrey` (#3724) * fix: `supervisor-app.conf` - Correct `postgrey` log location Looks like this should have been like every other service and reference a log file(s) based on program name in the supervisor log directory. * tests: Adjust log location for `postgrey_enabled.bats` --- CHANGELOG.md | 1 + target/supervisor/conf.d/supervisor-app.conf | 4 ++-- test/tests/parallel/set1/spam_virus/postgrey_enabled.bats | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32aaf78da3c..f37fded40c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ All notable changes to this project will be documented in this file. The format - `VERSION` is no longer included in the image ([#3711](https://github.com/docker-mailserver/docker-mailserver/pull/3711)) - Update-check: fix 'read' exit status ([#3688](https://github.com/docker-mailserver/docker-mailserver/pull/3688)) - `ENABLE_QUOTAS=0` no longer tries to remove non-existent config ([#3715](https://github.com/docker-mailserver/docker-mailserver/pull/3715)) + - The `postgrey` service now writes logs to the supervisor directory like all other services. Previously this was `/var/log/mail/mail.log` ([#3724](https://github.com/docker-mailserver/docker-mailserver/pull/3724)) - **Rspamd:** - Switch to official arm64 packages to avoid segfaults ([#3686](https://github.com/docker-mailserver/docker-mailserver/pull/3686)) - **CI / Automation:** diff --git a/target/supervisor/conf.d/supervisor-app.conf b/target/supervisor/conf.d/supervisor-app.conf index 2dd8b917337..431357d8c52 100644 --- a/target/supervisor/conf.d/supervisor-app.conf +++ b/target/supervisor/conf.d/supervisor-app.conf @@ -83,8 +83,8 @@ startsecs=0 stopwaitsecs=55 autostart=false autorestart=true -stdout_logfile=/var/log/mail/mail.log -stderr_logfile=/var/log/mail/mail.log +stdout_logfile=/var/log/supervisor/%(program_name)s.log +stderr_logfile=/var/log/supervisor/%(program_name)s.log command=/usr/sbin/postgrey --inet=127.0.0.1:10023 --syslog-facility=mail --delay="%(ENV_POSTGREY_DELAY)s" --max-age="%(ENV_POSTGREY_MAX_AGE)s" --auto-whitelist-clients="%(ENV_POSTGREY_AUTO_WHITELIST_CLIENTS)s" --greylist-text="%(ENV_POSTGREY_TEXT)s" [program:amavis] diff --git a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats index d877a1cec58..e32210cae7f 100644 --- a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats +++ b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats @@ -123,7 +123,7 @@ function _should_have_log_entry() { # Allow some extra time for logs to update to avoids a false-positive failure: _run_until_success_or_timeout 10 _exec_in_container grep \ "${ACTION}, ${REASON}," \ - /var/log/mail/mail.log + /var/log/supervisor/postgrey.log # Log entry matched should be for the expected triplet: assert_output --partial "${TRIPLET}" From 9e81517fe36d95597d3c8890f998bd7f9ea29aa7 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 3 Jan 2024 01:17:54 +0100 Subject: [PATCH 226/592] tests: Use `swaks` instead of `nc` for sending mail (#3732) See associated `CHANGELOG.md` entry for details. --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .gitattributes | 6 +- CHANGELOG.md | 7 ++ target/scripts/build/packages.sh | 2 +- .../auth/added-imap-auth.txt | 0 .../auth/added-pop3-auth.txt | 0 test/{test-files => files}/auth/imap-auth.txt | 0 .../auth/imap-ldap-auth.txt | 0 test/{test-files => files}/auth/pop3-auth.txt | 0 .../emails/amavis/spam.txt} | 9 +- .../emails/amavis/virus.txt} | 9 +- .../auth/added-smtp-auth-spoofed-alias.txt | 9 -- .../emails}/auth/added-smtp-auth-spoofed.txt | 9 -- .../auth/ldap-smtp-auth-spoofed-alias.txt | 10 --- ...h-spoofed-sender-with-filter-exception.txt | 10 --- .../emails}/auth/ldap-smtp-auth-spoofed.txt | 10 --- .../emails/existing/added.txt} | 7 -- .../emails/existing/alias-external.txt} | 7 -- .../emails/existing/alias-local.txt} | 7 -- .../existing/alias-recipient-delimiter.txt} | 7 -- .../emails/existing/catchall-local.txt} | 7 -- .../existing/regexp-alias-external.txt} | 7 -- .../emails/existing/regexp-alias-local.txt} | 7 -- .../existing/user-and-cc-local-alias.txt} | 7 -- .../emails/existing/user1.txt} | 7 -- .../emails/nc_raw/dsn/authenticated.txt} | 0 .../emails/nc_raw/dsn/unauthenticated.txt} | 0 .../emails/nc_raw}/postscreen.txt | 0 .../emails/nc_raw}/smtp-only.txt | 0 .../emails}/non-existing-user.txt | 7 -- .../emails}/postgrey.txt | 7 -- test/files/emails/postscreen.txt | 5 ++ .../emails/privacy.txt} | 9 -- .../emails}/quota-exceeded.txt | 7 -- .../emails/rspamd/pass.txt} | 9 +- .../emails/rspamd/spam-header.txt} | 7 -- .../emails/rspamd/spam.txt} | 7 -- .../emails/rspamd/virus.txt} | 7 -- .../emails/sendmail}/root-email.txt | 0 .../emails/sieve/pipe.txt} | 7 -- .../emails/sieve/spam-folder.txt} | 7 -- .../emails}/test-email.txt | 0 .../nc}/imap_special_use_folders.txt | 0 .../nc}/postgrey_whitelist.txt | 0 .../nc}/postgrey_whitelist_recipients.txt | 0 .../nc}/rspamd_imap_move_to_inbox.txt | 0 .../nc}/rspamd_imap_move_to_junk.txt | 0 .../ssl/custom-dhe-params.pem | 0 .../ssl/example.test/README.md | 0 .../ssl/example.test/cert.ecdsa.pem | 0 .../ssl/example.test/cert.rsa.pem | 0 .../ssl/example.test/key.ecdsa.pem | 0 .../ssl/example.test/key.rsa.pem | 0 .../ssl/example.test/testssl.txt | 0 .../ssl/example.test/traefik.md | 0 .../with_ca/ecdsa/ca-cert.ecdsa.pem | 0 .../with_ca/ecdsa/ca-key.ecdsa.pem | 0 .../example.test/with_ca/ecdsa/cert.ecdsa.pem | 0 .../example.test/with_ca/ecdsa/cert.rsa.pem | 0 .../with_ca/ecdsa/ecdsa.acme.json | 0 .../example.test/with_ca/ecdsa/key.ecdsa.pem | 0 .../example.test/with_ca/ecdsa/key.rsa.pem | 0 .../example.test/with_ca/ecdsa/rsa.acme.json | 0 .../with_ca/ecdsa/wildcard/cert.ecdsa.pem | 0 .../with_ca/ecdsa/wildcard/ecdsa.acme.json | 0 .../with_ca/ecdsa/wildcard/key.ecdsa.pem | 0 .../example.test/with_ca/rsa/ca-cert.rsa.pem | 0 .../example.test/with_ca/rsa/ca-key.rsa.pem | 0 .../example.test/with_ca/rsa/cert.ecdsa.pem | 0 .../ssl/example.test/with_ca/rsa/cert.rsa.pem | 0 .../example.test/with_ca/rsa/ecdsa.acme.json | 0 .../example.test/with_ca/rsa/key.ecdsa.pem | 0 .../ssl/example.test/with_ca/rsa/key.rsa.pem | 0 .../example.test/with_ca/rsa/rsa.acme.json | 0 .../with_ca/rsa/wildcard/cert.rsa.pem | 0 .../with_ca/rsa/wildcard/key.rsa.pem | 0 .../with_ca/rsa/wildcard/rsa.acme.json | 0 test/helper/common.bash | 14 +++ test/helper/sending.bash | 59 +++++++++---- test/helper/setup.bash | 2 +- .../auth/added-smtp-auth-login-wrong.txt | 4 - .../test-files/auth/added-smtp-auth-login.txt | 4 - .../auth/added-smtp-auth-plain-wrong.txt | 3 - .../test-files/auth/added-smtp-auth-plain.txt | 3 - test/test-files/auth/sasl-ldap-smtp-auth.txt | 5 -- .../test-files/auth/smtp-auth-login-wrong.txt | 4 - test/test-files/auth/smtp-auth-login.txt | 4 - .../test-files/auth/smtp-auth-plain-wrong.txt | 3 - test/test-files/auth/smtp-auth-plain.txt | 3 - .../email-templates/existing-user2.txt | 12 --- .../email-templates/existing-user3.txt | 12 --- test/test-files/email-templates/smtp-ehlo.txt | 2 - .../parallel/set1/dovecot/dovecot_quotas.bats | 9 +- .../parallel/set1/dovecot/dovecot_sieve.bats | 6 +- .../set1/dovecot/mailbox_format_dbox.bats | 6 +- .../set1/dovecot/special_use_folders.bats | 5 +- .../parallel/set1/spam_virus/clamav.bats | 13 +-- .../disabled_clamav_spamassassin.bats | 4 +- .../parallel/set1/spam_virus/fail2ban.bats | 13 ++- .../set1/spam_virus/postgrey_enabled.bats | 37 +++----- .../parallel/set1/spam_virus/postscreen.bats | 55 +++++------- .../parallel/set1/spam_virus/rspamd_full.bats | 12 +-- .../set1/spam_virus/spam_junk_folder.bats | 2 +- test/tests/parallel/set1/tls/dhparams.bats | 2 +- test/tests/parallel/set1/tls/letsencrypt.bats | 4 +- test/tests/parallel/set1/tls/manual.bats | 6 +- test/tests/parallel/set2/tls_cipherlists.bats | 2 +- .../container_configuration/hostname.bats | 2 +- test/tests/parallel/set3/mta/dsn.bats | 20 +++-- test/tests/parallel/set3/mta/lmtp_ip.bats | 2 +- test/tests/parallel/set3/mta/privacy.bats | 6 +- .../parallel/set3/mta/smtp_delivery.bats | 88 +++++++++++-------- test/tests/parallel/set3/mta/smtponly.bats | 11 ++- test/tests/serial/mail_pop3.bats | 6 +- test/tests/serial/mail_with_imap.bats | 28 ++++-- test/tests/serial/mail_with_ldap.bats | 59 ++++++++++--- test/tests/serial/permit_docker.bats | 10 +-- test/tests/serial/test_helper.bats | 8 +- test/tests/serial/tests.bats | 31 ++++++- test/tests/serial/vmail-id.bats | 2 +- 119 files changed, 353 insertions(+), 453 deletions(-) rename test/{test-files => files}/auth/added-imap-auth.txt (100%) rename test/{test-files => files}/auth/added-pop3-auth.txt (100%) rename test/{test-files => files}/auth/imap-auth.txt (100%) rename test/{test-files => files}/auth/imap-ldap-auth.txt (100%) rename test/{test-files => files}/auth/pop3-auth.txt (100%) rename test/{test-files/email-templates/amavis-spam.txt => files/emails/amavis/spam.txt} (63%) rename test/{test-files/email-templates/amavis-virus.txt => files/emails/amavis/virus.txt} (83%) rename test/{test-files => files/emails}/auth/added-smtp-auth-spoofed-alias.txt (52%) rename test/{test-files => files/emails}/auth/added-smtp-auth-spoofed.txt (53%) rename test/{test-files => files/emails}/auth/ldap-smtp-auth-spoofed-alias.txt (57%) rename test/{test-files => files/emails}/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt (58%) rename test/{test-files => files/emails}/auth/ldap-smtp-auth-spoofed.txt (53%) rename test/{test-files/email-templates/existing-added.txt => files/emails/existing/added.txt} (67%) rename test/{test-files/email-templates/existing-alias-external.txt => files/emails/existing/alias-external.txt} (68%) rename test/{test-files/email-templates/existing-alias-local.txt => files/emails/existing/alias-local.txt} (68%) rename test/{test-files/email-templates/existing-alias-recipient-delimiter.txt => files/emails/existing/alias-recipient-delimiter.txt} (70%) rename test/{test-files/email-templates/existing-catchall-local.txt => files/emails/existing/catchall-local.txt} (68%) rename test/{test-files/email-templates/existing-regexp-alias-external.txt => files/emails/existing/regexp-alias-external.txt} (68%) rename test/{test-files/email-templates/existing-regexp-alias-local.txt => files/emails/existing/regexp-alias-local.txt} (68%) rename test/{test-files/email-templates/existing-user-and-cc-local-alias.txt => files/emails/existing/user-and-cc-local-alias.txt} (73%) rename test/{test-files/email-templates/existing-user1.txt => files/emails/existing/user1.txt} (67%) rename test/{test-files/email-templates/dsn-authenticated.txt => files/emails/nc_raw/dsn/authenticated.txt} (100%) rename test/{test-files/email-templates/dsn-unauthenticated.txt => files/emails/nc_raw/dsn/unauthenticated.txt} (100%) rename test/{test-files/email-templates => files/emails/nc_raw}/postscreen.txt (100%) rename test/{test-files/email-templates => files/emails/nc_raw}/smtp-only.txt (100%) rename test/{test-files/email-templates => files/emails}/non-existing-user.txt (67%) rename test/{test-files/email-templates => files/emails}/postgrey.txt (66%) create mode 100644 test/files/emails/postscreen.txt rename test/{test-files/email-templates/send-privacy-email.txt => files/emails/privacy.txt} (61%) rename test/{test-files/email-templates => files/emails}/quota-exceeded.txt (98%) rename test/{test-files/email-templates/rspamd-pass.txt => files/emails/rspamd/pass.txt} (57%) rename test/{test-files/email-templates/rspamd-spam-header.txt => files/emails/rspamd/spam-header.txt} (70%) rename test/{test-files/email-templates/rspamd-spam.txt => files/emails/rspamd/spam.txt} (70%) rename test/{test-files/email-templates/rspamd-virus.txt => files/emails/rspamd/virus.txt} (70%) rename test/{test-files/email-templates => files/emails/sendmail}/root-email.txt (100%) rename test/{test-files/email-templates/sieve-pipe.txt => files/emails/sieve/pipe.txt} (67%) rename test/{test-files/email-templates/sieve-spam-folder.txt => files/emails/sieve/spam-folder.txt} (64%) rename test/{test-files/email-templates => files/emails}/test-email.txt (100%) rename test/{test-files/nc_templates => files/nc}/imap_special_use_folders.txt (100%) rename test/{test-files/nc_templates => files/nc}/postgrey_whitelist.txt (100%) rename test/{test-files/nc_templates => files/nc}/postgrey_whitelist_recipients.txt (100%) rename test/{test-files/nc_templates => files/nc}/rspamd_imap_move_to_inbox.txt (100%) rename test/{test-files/nc_templates => files/nc}/rspamd_imap_move_to_junk.txt (100%) rename test/{test-files => files}/ssl/custom-dhe-params.pem (100%) rename test/{test-files => files}/ssl/example.test/README.md (100%) rename test/{test-files => files}/ssl/example.test/cert.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/cert.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/key.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/key.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/testssl.txt (100%) rename test/{test-files => files}/ssl/example.test/traefik.md (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/ca-cert.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/ca-key.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/cert.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/cert.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/ecdsa.acme.json (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/key.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/key.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/rsa.acme.json (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/wildcard/cert.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/wildcard/ecdsa.acme.json (100%) rename test/{test-files => files}/ssl/example.test/with_ca/ecdsa/wildcard/key.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/ca-cert.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/ca-key.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/cert.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/cert.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/ecdsa.acme.json (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/key.ecdsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/key.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/rsa.acme.json (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/wildcard/cert.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/wildcard/key.rsa.pem (100%) rename test/{test-files => files}/ssl/example.test/with_ca/rsa/wildcard/rsa.acme.json (100%) delete mode 100644 test/test-files/auth/added-smtp-auth-login-wrong.txt delete mode 100644 test/test-files/auth/added-smtp-auth-login.txt delete mode 100644 test/test-files/auth/added-smtp-auth-plain-wrong.txt delete mode 100644 test/test-files/auth/added-smtp-auth-plain.txt delete mode 100644 test/test-files/auth/sasl-ldap-smtp-auth.txt delete mode 100644 test/test-files/auth/smtp-auth-login-wrong.txt delete mode 100644 test/test-files/auth/smtp-auth-login.txt delete mode 100644 test/test-files/auth/smtp-auth-plain-wrong.txt delete mode 100644 test/test-files/auth/smtp-auth-plain.txt delete mode 100644 test/test-files/email-templates/existing-user2.txt delete mode 100644 test/test-files/email-templates/existing-user3.txt delete mode 100644 test/test-files/email-templates/smtp-ehlo.txt diff --git a/.gitattributes b/.gitattributes index d3dba13d237..869c153e500 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10,7 +10,7 @@ *.yml text ### Documentation (Project, Tests, Docs site) *.md text -### TLS certs (test/test-files/) + DHE params (target/shared/) +### TLS certs (test/files/) + DHE params (target/shared/) *.pem text *.pem.sha512sum text @@ -90,9 +90,9 @@ TrustedHosts text whitelist_recipients text ## MISC -### test/config/ + test/test-files/ +### test/config/ + test/files/ *.txt text -### test/linting/ (.ecrc.json) + test/test-files/ (*.acme.json): +### test/linting/ (.ecrc.json) + test/files/ (*.acme.json): *.json text ################################################# diff --git a/CHANGELOG.md b/CHANGELOG.md index f37fded40c4..eeeb843d6e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Updates + +- The test suite now uses `swaks` instead of `nc`, which has multiple benefits ([#3732](https://github.com/docker-mailserver/docker-mailserver/pull/3732)): + - `swaks` handles pipelining correctly, hence we can now use `reject_unauth_pipelining` in Postfix's configuration. + - `swaks` provides better CLI options that make many files superflous. + - `swaks` can also replace `openssl s_client` and handles authentication on submission ports better. + ## [v13.1.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.1.0) ### Added diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index ec468d41834..566d5441dff 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -80,7 +80,7 @@ function _install_packages() { # `bind9-dnsutils` provides the `dig` command # `iputils-ping` provides the `ping` command DEBUG_PACKAGES=( - bind9-dnsutils iputils-ping less nano + bind9-dnsutils iputils-ping less nano swaks ) apt-get "${QUIET}" --no-install-recommends install \ diff --git a/test/test-files/auth/added-imap-auth.txt b/test/files/auth/added-imap-auth.txt similarity index 100% rename from test/test-files/auth/added-imap-auth.txt rename to test/files/auth/added-imap-auth.txt diff --git a/test/test-files/auth/added-pop3-auth.txt b/test/files/auth/added-pop3-auth.txt similarity index 100% rename from test/test-files/auth/added-pop3-auth.txt rename to test/files/auth/added-pop3-auth.txt diff --git a/test/test-files/auth/imap-auth.txt b/test/files/auth/imap-auth.txt similarity index 100% rename from test/test-files/auth/imap-auth.txt rename to test/files/auth/imap-auth.txt diff --git a/test/test-files/auth/imap-ldap-auth.txt b/test/files/auth/imap-ldap-auth.txt similarity index 100% rename from test/test-files/auth/imap-ldap-auth.txt rename to test/files/auth/imap-ldap-auth.txt diff --git a/test/test-files/auth/pop3-auth.txt b/test/files/auth/pop3-auth.txt similarity index 100% rename from test/test-files/auth/pop3-auth.txt rename to test/files/auth/pop3-auth.txt diff --git a/test/test-files/email-templates/amavis-spam.txt b/test/files/emails/amavis/spam.txt similarity index 63% rename from test/test-files/email-templates/amavis-spam.txt rename to test/files/emails/amavis/spam.txt index 66be1df3d32..e8d26138668 100644 --- a/test/test-files/email-templates/amavis-spam.txt +++ b/test/files/emails/amavis/spam.txt @@ -1,13 +1,6 @@ -HELO mail.external.tld -MAIL FROM: spam@external.tld -RCPT TO: user1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message amavis-spam.txt +Subject: Test Message amavis/spam.txt This is a test mail. XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X - -. -QUIT diff --git a/test/test-files/email-templates/amavis-virus.txt b/test/files/emails/amavis/virus.txt similarity index 83% rename from test/test-files/email-templates/amavis-virus.txt rename to test/files/emails/amavis/virus.txt index 1343a07ca8f..2c47dcad23b 100644 --- a/test/test-files/email-templates/amavis-virus.txt +++ b/test/files/emails/amavis/virus.txt @@ -1,11 +1,7 @@ -HELO mail.external.tld -MAIL FROM: virus@external.tld -RCPT TO: user1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message amavis-virus.txt +Subject: Test Message amavis/virus.txt Content-type: multipart/mixed; boundary="emailboundary" MIME-version: 1.0 @@ -27,6 +23,3 @@ ACAA/4EAAAAAZWljYXIuY29tUEsFBgAAAAABAAEANwAAAGsAAAAAAA== --emailboundary-- - -. -QUIT diff --git a/test/test-files/auth/added-smtp-auth-spoofed-alias.txt b/test/files/emails/auth/added-smtp-auth-spoofed-alias.txt similarity index 52% rename from test/test-files/auth/added-smtp-auth-spoofed-alias.txt rename to test/files/emails/auth/added-smtp-auth-spoofed-alias.txt index 4814518376d..eeb68ac801c 100644 --- a/test/test-files/auth/added-smtp-auth-spoofed-alias.txt +++ b/test/files/emails/auth/added-smtp-auth-spoofed-alias.txt @@ -1,14 +1,5 @@ -EHLO mail -AUTH LOGIN dXNlcjFAbG9jYWxob3N0LmxvY2FsZG9tYWlu -bXlwYXNzd29yZA== -MAIL FROM: alias1@localhost.localdomain -RCPT TO: user1@localhost.localdomain -DATA From: user1_alias To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message This is a test mail. - -. -QUIT diff --git a/test/test-files/auth/added-smtp-auth-spoofed.txt b/test/files/emails/auth/added-smtp-auth-spoofed.txt similarity index 53% rename from test/test-files/auth/added-smtp-auth-spoofed.txt rename to test/files/emails/auth/added-smtp-auth-spoofed.txt index 279b6c0eb3b..fd96d40132a 100644 --- a/test/test-files/auth/added-smtp-auth-spoofed.txt +++ b/test/files/emails/auth/added-smtp-auth-spoofed.txt @@ -1,14 +1,5 @@ -EHLO mail -AUTH LOGIN YWRkZWRAbG9jYWxob3N0LmxvY2FsZG9tYWlu -bXlwYXNzd29yZA== -MAIL FROM: user2@localhost.localdomain -RCPT TO: user1@localhost.localdomain -DATA From: Not_My_Business To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message This is a test mail. - -. -QUIT diff --git a/test/test-files/auth/ldap-smtp-auth-spoofed-alias.txt b/test/files/emails/auth/ldap-smtp-auth-spoofed-alias.txt similarity index 57% rename from test/test-files/auth/ldap-smtp-auth-spoofed-alias.txt rename to test/files/emails/auth/ldap-smtp-auth-spoofed-alias.txt index 007b0f99b32..7453675ce59 100644 --- a/test/test-files/auth/ldap-smtp-auth-spoofed-alias.txt +++ b/test/files/emails/auth/ldap-smtp-auth-spoofed-alias.txt @@ -1,15 +1,5 @@ -EHLO mail -AUTH LOGIN -c29tZS51c2VyQGxvY2FsaG9zdC5sb2NhbGRvbWFpbg== -c2VjcmV0 -MAIL FROM: postmaster@localhost.localdomain -RCPT TO: some.user@localhost.localdomain -DATA From: alias_address To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message This is a test mail from ldap-smtp-auth-spoofed-alias.txt - -. -QUIT diff --git a/test/test-files/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt b/test/files/emails/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt similarity index 58% rename from test/test-files/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt rename to test/files/emails/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt index bc0447afb74..3b500bf6727 100644 --- a/test/test-files/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt +++ b/test/files/emails/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt @@ -1,15 +1,5 @@ -EHLO mail -AUTH LOGIN -c29tZS51c2VyLmVtYWlsQGxvY2FsaG9zdC5sb2NhbGRvbWFpbgo= -c2VjcmV0 -MAIL FROM: randomspoofedaddress@localhost.localdomain -RCPT TO: some.user@localhost.localdomain -DATA From: spoofed_address To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message This is a test mail from ldap-smtp-auth-spoofed-sender-with-filter-exception.txt - -. -QUIT diff --git a/test/test-files/auth/ldap-smtp-auth-spoofed.txt b/test/files/emails/auth/ldap-smtp-auth-spoofed.txt similarity index 53% rename from test/test-files/auth/ldap-smtp-auth-spoofed.txt rename to test/files/emails/auth/ldap-smtp-auth-spoofed.txt index cc0b164dc66..83193e17e73 100644 --- a/test/test-files/auth/ldap-smtp-auth-spoofed.txt +++ b/test/files/emails/auth/ldap-smtp-auth-spoofed.txt @@ -1,15 +1,5 @@ -EHLO mail -AUTH LOGIN -c29tZS51c2VyQGxvY2FsaG9zdC5sb2NhbGRvbWFpbg== -c2VjcmV0 -MAIL FROM: ldap@localhost.localdomain -RCPT TO: user1@localhost.localdomain -DATA From: forged_address To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-added.txt b/test/files/emails/existing/added.txt similarity index 67% rename from test/test-files/email-templates/existing-added.txt rename to test/files/emails/existing/added.txt index 320fa4d2a10..827b681f0e6 100644 --- a/test/test-files/email-templates/existing-added.txt +++ b/test/files/emails/existing/added.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: added@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message existing-added.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-alias-external.txt b/test/files/emails/existing/alias-external.txt similarity index 68% rename from test/test-files/email-templates/existing-alias-external.txt rename to test/files/emails/existing/alias-external.txt index 61b1df3c5c8..03f1af6c618 100644 --- a/test/test-files/email-templates/existing-alias-external.txt +++ b/test/files/emails/existing/alias-external.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: alias1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message existing-alias-external.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-alias-local.txt b/test/files/emails/existing/alias-local.txt similarity index 68% rename from test/test-files/email-templates/existing-alias-local.txt rename to test/files/emails/existing/alias-local.txt index c1bbc890c53..9b481a98550 100644 --- a/test/test-files/email-templates/existing-alias-local.txt +++ b/test/files/emails/existing/alias-local.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: alias2@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local Alias Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message existing-alias-local.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-alias-recipient-delimiter.txt b/test/files/emails/existing/alias-recipient-delimiter.txt similarity index 70% rename from test/test-files/email-templates/existing-alias-recipient-delimiter.txt rename to test/files/emails/existing/alias-recipient-delimiter.txt index 47b0139769f..07cb8d40f1a 100644 --- a/test/test-files/email-templates/existing-alias-recipient-delimiter.txt +++ b/test/files/emails/existing/alias-recipient-delimiter.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: alias1~test@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local Alias With Delimiter Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message existing-alias-recipient-delimiter.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-catchall-local.txt b/test/files/emails/existing/catchall-local.txt similarity index 68% rename from test/test-files/email-templates/existing-catchall-local.txt rename to test/files/emails/existing/catchall-local.txt index c80db170988..ab3e1988b52 100644 --- a/test/test-files/email-templates/existing-catchall-local.txt +++ b/test/files/emails/existing/catchall-local.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: wildcard@localdomain2.com -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message existing-catchall-local.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-regexp-alias-external.txt b/test/files/emails/existing/regexp-alias-external.txt similarity index 68% rename from test/test-files/email-templates/existing-regexp-alias-external.txt rename to test/files/emails/existing/regexp-alias-external.txt index 0e214db4a2b..b50ac90f1b3 100644 --- a/test/test-files/email-templates/existing-regexp-alias-external.txt +++ b/test/files/emails/existing/regexp-alias-external.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: bounce-always@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message existing-regexp-alias-external.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-regexp-alias-local.txt b/test/files/emails/existing/regexp-alias-local.txt similarity index 68% rename from test/test-files/email-templates/existing-regexp-alias-local.txt rename to test/files/emails/existing/regexp-alias-local.txt index 6af46e92e7a..e45b7c6c79a 100644 --- a/test/test-files/email-templates/existing-regexp-alias-local.txt +++ b/test/files/emails/existing/regexp-alias-local.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: test123@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message existing-regexp-alias-local.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-user-and-cc-local-alias.txt b/test/files/emails/existing/user-and-cc-local-alias.txt similarity index 73% rename from test/test-files/email-templates/existing-user-and-cc-local-alias.txt rename to test/files/emails/existing/user-and-cc-local-alias.txt index 5fcb333b45f..37814f91013 100644 --- a/test/test-files/email-templates/existing-user-and-cc-local-alias.txt +++ b/test/files/emails/existing/user-and-cc-local-alias.txt @@ -1,13 +1,6 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: user1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Cc: Existing Local Alias Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message existing-user-and-cc-local-alias.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-user1.txt b/test/files/emails/existing/user1.txt similarity index 67% rename from test/test-files/email-templates/existing-user1.txt rename to test/files/emails/existing/user1.txt index 5ab0333f1d0..23d49dc92de 100644 --- a/test/test-files/email-templates/existing-user1.txt +++ b/test/files/emails/existing/user1.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: user1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message existing-user1.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/dsn-authenticated.txt b/test/files/emails/nc_raw/dsn/authenticated.txt similarity index 100% rename from test/test-files/email-templates/dsn-authenticated.txt rename to test/files/emails/nc_raw/dsn/authenticated.txt diff --git a/test/test-files/email-templates/dsn-unauthenticated.txt b/test/files/emails/nc_raw/dsn/unauthenticated.txt similarity index 100% rename from test/test-files/email-templates/dsn-unauthenticated.txt rename to test/files/emails/nc_raw/dsn/unauthenticated.txt diff --git a/test/test-files/email-templates/postscreen.txt b/test/files/emails/nc_raw/postscreen.txt similarity index 100% rename from test/test-files/email-templates/postscreen.txt rename to test/files/emails/nc_raw/postscreen.txt diff --git a/test/test-files/email-templates/smtp-only.txt b/test/files/emails/nc_raw/smtp-only.txt similarity index 100% rename from test/test-files/email-templates/smtp-only.txt rename to test/files/emails/nc_raw/smtp-only.txt diff --git a/test/test-files/email-templates/non-existing-user.txt b/test/files/emails/non-existing-user.txt similarity index 67% rename from test/test-files/email-templates/non-existing-user.txt rename to test/files/emails/non-existing-user.txt index 406f67555fc..3d92470e9f9 100644 --- a/test/test-files/email-templates/non-existing-user.txt +++ b/test/files/emails/non-existing-user.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: nouser@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message non-existing-user.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/postgrey.txt b/test/files/emails/postgrey.txt similarity index 66% rename from test/test-files/email-templates/postgrey.txt rename to test/files/emails/postgrey.txt index 33a3b153723..cdfe8f937bf 100644 --- a/test/test-files/email-templates/postgrey.txt +++ b/test/files/emails/postgrey.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: user1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Postgrey Test Message This is a test mail. - -. -QUIT diff --git a/test/files/emails/postscreen.txt b/test/files/emails/postscreen.txt new file mode 100644 index 00000000000..732ac897abd --- /dev/null +++ b/test/files/emails/postscreen.txt @@ -0,0 +1,5 @@ +From: Docker Mail Server +To: Existing Local User +Date: Sat, 22 May 2010 07:43:25 -0400 +Subject: Test Message postscreen.txt +This is a test mail for postscreen. diff --git a/test/test-files/email-templates/send-privacy-email.txt b/test/files/emails/privacy.txt similarity index 61% rename from test/test-files/email-templates/send-privacy-email.txt rename to test/files/emails/privacy.txt index 0c51ec5b4be..1d3a1b96cef 100644 --- a/test/test-files/email-templates/send-privacy-email.txt +++ b/test/files/emails/privacy.txt @@ -1,15 +1,6 @@ -EHLO mail -AUTH LOGIN dXNlcjFAbG9jYWxob3N0LmxvY2FsZG9tYWlu -bXlwYXNzd29yZA== -mail from: -rcpt to: -data From: Some User To: Some User User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:52.0) Gecko/20100101 Thunderbird/52.2.1 Subject: Test ESMTP Auth LOGIN and remove privacy This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/quota-exceeded.txt b/test/files/emails/quota-exceeded.txt similarity index 98% rename from test/test-files/email-templates/quota-exceeded.txt rename to test/files/emails/quota-exceeded.txt index 71d221a15f5..c5281637bab 100644 --- a/test/test-files/email-templates/quota-exceeded.txt +++ b/test/files/emails/quota-exceeded.txt @@ -1,7 +1,3 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: quotauser@otherdomain.tld -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 @@ -20,6 +16,3 @@ Et voluptatum nobis ut odio voluptatem et quibusdam fugit ut libero sapiente vel Sit sint obcaecati et reiciendis tenetur aut dolorum culpa. Ab veritatis maxime qui necessitatibus facilis eum voluptate asperiores non totam omnis. Nam modi officia in reiciendis odit sit rerum laudantium est rerum voluptatem ut fugit cupiditate! Sit atque sint aut delectus omnis ut asperiores enim quo reprehenderit quae! In quasi nemo ut error totam ut quia harum ut commodi tenetur? Non quod dolorum eum explicabo labore vel asperiores quas est perferendis nulla eum nemo tenetur. Ut libero blanditiis ex voluptatibus repudiandae ab reiciendis nemo id debitis impedit hic quia incidunt sed quam excepturi ut magnam odit. Qui dolor deleniti aut sunt voluptas aut blanditiis distinctio nam omnis deleniti hic omnis rerum eum magni voluptatem. Nam labore facere eum molestiae dolorum ea consectetur praesentium ut cupiditate iste ad magnam aut neque maiores! Et excepturi ducimus ut nemo voluptas eum voluptas nihil hic perferendis quos vel quasi nesciunt est praesentium dolore hic quia quis. Et maxime ducimus ea cupiditate voluptatem ad quia dolores! Sed quos quaerat vel aperiam minus non sapiente quia ut ratione dolore eum officiis rerum. Non dolor vitae qui facilis dignissimos aut voluptate odit et ullam consequuntur. Et laudantium perspiciatis sit nisi temporibus a temporibus itaque ut iure dolor a voluptatum mollitia eos officia nobis et quibusdam voluptas. Amet eligendi eos nulla corporis et blanditiis nihil vel eveniet veritatis et sunt perferendis id molestiae eius! Quo harum quod aut nemo autem ut adipisci sint sed quia sunt. Aut voluptas error ut quae perferendis eos adipisci internos. Nam rerum fugiat aut minima nostrum quo repellendus quas exercitationem tenetur. Et molestiae architecto id quibusdam reprehenderit et magnam aliquam! Quo tempora veritatis At dolorem sint ex nulla blanditiis At voluptas laudantium est molestiae exercitationem et sequi voluptates aut ipsa atque. Et animi ipsum aut atque recusandae ea nemo ullam non quisquam quos sit libero sint vel libero delectus. Eos labore quidem a velit obcaecati nam explicabo consequatur eos maxime blanditiis? Et ipsam molestiae non quia explicabo ex galisum repudiandae et tempora veniam. Sed optio repellendus ut consequatur temporibus et harum quas hic ipsa officia? Aut dolores ipsum sit nulla dignissimos id quia perferendis aut dolores dolor et quibusdam porro aut Quis consequatur. - -. -QUIT diff --git a/test/test-files/email-templates/rspamd-pass.txt b/test/files/emails/rspamd/pass.txt similarity index 57% rename from test/test-files/email-templates/rspamd-pass.txt rename to test/files/emails/rspamd/pass.txt index 0f24474040e..ce9286b1460 100644 --- a/test/test-files/email-templates/rspamd-pass.txt +++ b/test/files/emails/rspamd/pass.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: pass@example.test -RCPT TO: user1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message rspamd-pass.txt +Subject: Test Message rspamd/pass.txt This mail should pass and Rspamd should not mark it. - -. -QUIT diff --git a/test/test-files/email-templates/rspamd-spam-header.txt b/test/files/emails/rspamd/spam-header.txt similarity index 70% rename from test/test-files/email-templates/rspamd-spam-header.txt rename to test/files/emails/rspamd/spam-header.txt index 7be1a56dcb1..8722e42f0c9 100644 --- a/test/test-files/email-templates/rspamd-spam-header.txt +++ b/test/files/emails/rspamd/spam-header.txt @@ -1,12 +1,5 @@ -HELO mail.example.test -MAIL FROM: spam-header@example.test -RCPT TO: user1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 21 Jan 2023 11:11:11 +0000 Subject: Test Message rspamd-spam-header.txt YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X - -. -QUIT diff --git a/test/test-files/email-templates/rspamd-spam.txt b/test/files/emails/rspamd/spam.txt similarity index 70% rename from test/test-files/email-templates/rspamd-spam.txt rename to test/files/emails/rspamd/spam.txt index 88bd719ce73..c561e779bb0 100644 --- a/test/test-files/email-templates/rspamd-spam.txt +++ b/test/files/emails/rspamd/spam.txt @@ -1,12 +1,5 @@ -HELO mail.example.test -MAIL FROM: spam@example.test -RCPT TO: user1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 21 Jan 2023 11:11:11 +0000 Subject: Test Message rspamd-spam.txt XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X - -. -QUIT diff --git a/test/test-files/email-templates/rspamd-virus.txt b/test/files/emails/rspamd/virus.txt similarity index 70% rename from test/test-files/email-templates/rspamd-virus.txt rename to test/files/emails/rspamd/virus.txt index c745f2617b2..cb18927d5d3 100644 --- a/test/test-files/email-templates/rspamd-virus.txt +++ b/test/files/emails/rspamd/virus.txt @@ -1,12 +1,5 @@ -HELO mail.example.test -MAIL FROM: virus@example.test -RCPT TO: user1@localhost.localdomain -DATA From: Docker Mail Server To: Existing Local User Date: Sat, 21 Jan 2023 11:11:11 +0000 Subject: Test Message rspamd-virus.txt X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H* - -. -QUIT diff --git a/test/test-files/email-templates/root-email.txt b/test/files/emails/sendmail/root-email.txt similarity index 100% rename from test/test-files/email-templates/root-email.txt rename to test/files/emails/sendmail/root-email.txt diff --git a/test/test-files/email-templates/sieve-pipe.txt b/test/files/emails/sieve/pipe.txt similarity index 67% rename from test/test-files/email-templates/sieve-pipe.txt rename to test/files/emails/sieve/pipe.txt index f13dba8709b..4e8cfb39b33 100644 --- a/test/test-files/email-templates/sieve-pipe.txt +++ b/test/files/emails/sieve/pipe.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: user2@otherdomain.tld -DATA From: Sieve-pipe-test To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Sieve pipe test message This is a test mail to sieve pipe. - -. -QUIT diff --git a/test/test-files/email-templates/sieve-spam-folder.txt b/test/files/emails/sieve/spam-folder.txt similarity index 64% rename from test/test-files/email-templates/sieve-spam-folder.txt rename to test/files/emails/sieve/spam-folder.txt index 8e802817f35..7ffd09a7dc6 100644 --- a/test/test-files/email-templates/sieve-spam-folder.txt +++ b/test/files/emails/sieve/spam-folder.txt @@ -1,12 +1,5 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: user1@localhost.localdomain -DATA From: Spambot To: Existing Local User Date: Sat, 22 May 2010 07:43:25 -0400 Subject: Test Message sieve-spam-folder.txt This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/test-email.txt b/test/files/emails/test-email.txt similarity index 100% rename from test/test-files/email-templates/test-email.txt rename to test/files/emails/test-email.txt diff --git a/test/test-files/nc_templates/imap_special_use_folders.txt b/test/files/nc/imap_special_use_folders.txt similarity index 100% rename from test/test-files/nc_templates/imap_special_use_folders.txt rename to test/files/nc/imap_special_use_folders.txt diff --git a/test/test-files/nc_templates/postgrey_whitelist.txt b/test/files/nc/postgrey_whitelist.txt similarity index 100% rename from test/test-files/nc_templates/postgrey_whitelist.txt rename to test/files/nc/postgrey_whitelist.txt diff --git a/test/test-files/nc_templates/postgrey_whitelist_recipients.txt b/test/files/nc/postgrey_whitelist_recipients.txt similarity index 100% rename from test/test-files/nc_templates/postgrey_whitelist_recipients.txt rename to test/files/nc/postgrey_whitelist_recipients.txt diff --git a/test/test-files/nc_templates/rspamd_imap_move_to_inbox.txt b/test/files/nc/rspamd_imap_move_to_inbox.txt similarity index 100% rename from test/test-files/nc_templates/rspamd_imap_move_to_inbox.txt rename to test/files/nc/rspamd_imap_move_to_inbox.txt diff --git a/test/test-files/nc_templates/rspamd_imap_move_to_junk.txt b/test/files/nc/rspamd_imap_move_to_junk.txt similarity index 100% rename from test/test-files/nc_templates/rspamd_imap_move_to_junk.txt rename to test/files/nc/rspamd_imap_move_to_junk.txt diff --git a/test/test-files/ssl/custom-dhe-params.pem b/test/files/ssl/custom-dhe-params.pem similarity index 100% rename from test/test-files/ssl/custom-dhe-params.pem rename to test/files/ssl/custom-dhe-params.pem diff --git a/test/test-files/ssl/example.test/README.md b/test/files/ssl/example.test/README.md similarity index 100% rename from test/test-files/ssl/example.test/README.md rename to test/files/ssl/example.test/README.md diff --git a/test/test-files/ssl/example.test/cert.ecdsa.pem b/test/files/ssl/example.test/cert.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/cert.ecdsa.pem rename to test/files/ssl/example.test/cert.ecdsa.pem diff --git a/test/test-files/ssl/example.test/cert.rsa.pem b/test/files/ssl/example.test/cert.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/cert.rsa.pem rename to test/files/ssl/example.test/cert.rsa.pem diff --git a/test/test-files/ssl/example.test/key.ecdsa.pem b/test/files/ssl/example.test/key.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/key.ecdsa.pem rename to test/files/ssl/example.test/key.ecdsa.pem diff --git a/test/test-files/ssl/example.test/key.rsa.pem b/test/files/ssl/example.test/key.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/key.rsa.pem rename to test/files/ssl/example.test/key.rsa.pem diff --git a/test/test-files/ssl/example.test/testssl.txt b/test/files/ssl/example.test/testssl.txt similarity index 100% rename from test/test-files/ssl/example.test/testssl.txt rename to test/files/ssl/example.test/testssl.txt diff --git a/test/test-files/ssl/example.test/traefik.md b/test/files/ssl/example.test/traefik.md similarity index 100% rename from test/test-files/ssl/example.test/traefik.md rename to test/files/ssl/example.test/traefik.md diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/ca-cert.ecdsa.pem b/test/files/ssl/example.test/with_ca/ecdsa/ca-cert.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/ca-cert.ecdsa.pem rename to test/files/ssl/example.test/with_ca/ecdsa/ca-cert.ecdsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/ca-key.ecdsa.pem b/test/files/ssl/example.test/with_ca/ecdsa/ca-key.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/ca-key.ecdsa.pem rename to test/files/ssl/example.test/with_ca/ecdsa/ca-key.ecdsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/cert.ecdsa.pem b/test/files/ssl/example.test/with_ca/ecdsa/cert.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/cert.ecdsa.pem rename to test/files/ssl/example.test/with_ca/ecdsa/cert.ecdsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/cert.rsa.pem b/test/files/ssl/example.test/with_ca/ecdsa/cert.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/cert.rsa.pem rename to test/files/ssl/example.test/with_ca/ecdsa/cert.rsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/ecdsa.acme.json b/test/files/ssl/example.test/with_ca/ecdsa/ecdsa.acme.json similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/ecdsa.acme.json rename to test/files/ssl/example.test/with_ca/ecdsa/ecdsa.acme.json diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/key.ecdsa.pem b/test/files/ssl/example.test/with_ca/ecdsa/key.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/key.ecdsa.pem rename to test/files/ssl/example.test/with_ca/ecdsa/key.ecdsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/key.rsa.pem b/test/files/ssl/example.test/with_ca/ecdsa/key.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/key.rsa.pem rename to test/files/ssl/example.test/with_ca/ecdsa/key.rsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/rsa.acme.json b/test/files/ssl/example.test/with_ca/ecdsa/rsa.acme.json similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/rsa.acme.json rename to test/files/ssl/example.test/with_ca/ecdsa/rsa.acme.json diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/wildcard/cert.ecdsa.pem b/test/files/ssl/example.test/with_ca/ecdsa/wildcard/cert.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/wildcard/cert.ecdsa.pem rename to test/files/ssl/example.test/with_ca/ecdsa/wildcard/cert.ecdsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/wildcard/ecdsa.acme.json b/test/files/ssl/example.test/with_ca/ecdsa/wildcard/ecdsa.acme.json similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/wildcard/ecdsa.acme.json rename to test/files/ssl/example.test/with_ca/ecdsa/wildcard/ecdsa.acme.json diff --git a/test/test-files/ssl/example.test/with_ca/ecdsa/wildcard/key.ecdsa.pem b/test/files/ssl/example.test/with_ca/ecdsa/wildcard/key.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/ecdsa/wildcard/key.ecdsa.pem rename to test/files/ssl/example.test/with_ca/ecdsa/wildcard/key.ecdsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/rsa/ca-cert.rsa.pem b/test/files/ssl/example.test/with_ca/rsa/ca-cert.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/ca-cert.rsa.pem rename to test/files/ssl/example.test/with_ca/rsa/ca-cert.rsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/rsa/ca-key.rsa.pem b/test/files/ssl/example.test/with_ca/rsa/ca-key.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/ca-key.rsa.pem rename to test/files/ssl/example.test/with_ca/rsa/ca-key.rsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/rsa/cert.ecdsa.pem b/test/files/ssl/example.test/with_ca/rsa/cert.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/cert.ecdsa.pem rename to test/files/ssl/example.test/with_ca/rsa/cert.ecdsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/rsa/cert.rsa.pem b/test/files/ssl/example.test/with_ca/rsa/cert.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/cert.rsa.pem rename to test/files/ssl/example.test/with_ca/rsa/cert.rsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/rsa/ecdsa.acme.json b/test/files/ssl/example.test/with_ca/rsa/ecdsa.acme.json similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/ecdsa.acme.json rename to test/files/ssl/example.test/with_ca/rsa/ecdsa.acme.json diff --git a/test/test-files/ssl/example.test/with_ca/rsa/key.ecdsa.pem b/test/files/ssl/example.test/with_ca/rsa/key.ecdsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/key.ecdsa.pem rename to test/files/ssl/example.test/with_ca/rsa/key.ecdsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/rsa/key.rsa.pem b/test/files/ssl/example.test/with_ca/rsa/key.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/key.rsa.pem rename to test/files/ssl/example.test/with_ca/rsa/key.rsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/rsa/rsa.acme.json b/test/files/ssl/example.test/with_ca/rsa/rsa.acme.json similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/rsa.acme.json rename to test/files/ssl/example.test/with_ca/rsa/rsa.acme.json diff --git a/test/test-files/ssl/example.test/with_ca/rsa/wildcard/cert.rsa.pem b/test/files/ssl/example.test/with_ca/rsa/wildcard/cert.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/wildcard/cert.rsa.pem rename to test/files/ssl/example.test/with_ca/rsa/wildcard/cert.rsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/rsa/wildcard/key.rsa.pem b/test/files/ssl/example.test/with_ca/rsa/wildcard/key.rsa.pem similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/wildcard/key.rsa.pem rename to test/files/ssl/example.test/with_ca/rsa/wildcard/key.rsa.pem diff --git a/test/test-files/ssl/example.test/with_ca/rsa/wildcard/rsa.acme.json b/test/files/ssl/example.test/with_ca/rsa/wildcard/rsa.acme.json similarity index 100% rename from test/test-files/ssl/example.test/with_ca/rsa/wildcard/rsa.acme.json rename to test/files/ssl/example.test/with_ca/rsa/wildcard/rsa.acme.json diff --git a/test/helper/common.bash b/test/helper/common.bash index 8fb7854e146..ab21ef60a15 100644 --- a/test/helper/common.bash +++ b/test/helper/common.bash @@ -469,5 +469,19 @@ function _print_mail_log_for_id() { _run_in_container grep -F "${MAIL_ID}" /var/log/mail.log } +# A simple wrapper for netcat (`nc`). This is useful when sending +# "raw" e-mails or doing IMAP-related work. +# +# @param ${1} = the file that is given to `nc` +# @param ${1} = custom parameters for `nc` [OPTIONAL] (default: 0.0.0.0 25) +function _nc_wrapper() { + local FILE=${1:?Must provide name of template file} + local NC_PARAMETERS=${2:-0.0.0.0 25} + + [[ -v CONTAINER_NAME ]] || return 1 + + _run_in_container_bash "nc ${NC_PARAMETERS} < /tmp/docker-mailserver-test/${FILE}.txt" +} + # ? << Miscellaneous helper functions # ! ------------------------------------------------------------------- diff --git a/test/helper/sending.bash b/test/helper/sending.bash index 631617a172d..48012178c89 100644 --- a/test/helper/sending.bash +++ b/test/helper/sending.bash @@ -8,11 +8,12 @@ # ! ATTENTION: This file requires helper functions from `common.sh`! # Sends a mail from localhost (127.0.0.1) to a container. To send -# a custom email, create a file at `test/test-files/`, +# a custom email, create a file at `test/files/`, # and provide `` as an argument to this function. # -# @param ${1} = template file (path) name -# @param ${2} = parameters for `nc` [OPTIONAL] (default: `0.0.0.0 25`) +# Parameters include all options that one can supply to `swaks` +# itself. The `--data` parameter expects a relative path from `emails/` +# where the contents will be implicitly provided to `swaks` via STDIN. # # ## Attention # @@ -23,17 +24,42 @@ # send the email but it will not make sure the mail queue is empty after the mail # has been sent. function _send_email() { - local TEMPLATE_FILE=${1:?Must provide name of template file} - local NC_PARAMETERS=${2:-0.0.0.0 25} + [[ -v CONTAINER_NAME ]] || return 1 - assert_not_equal "${NC_PARAMETERS}" '' - assert_not_equal "${CONTAINER_NAME:-}" '' + # Parameter defaults common to our testing needs: + local EHLO='mail.external.tld' + local FROM='user@external.tld' + local TO='user1@localhost.localdomain' + local SERVER='0.0.0.0' + local PORT=25 + # Extra options for `swaks` that aren't covered by the default options above: + local ADDITIONAL_SWAKS_OPTIONS=() + # Specifically for handling `--data` option below: + local FINAL_SWAKS_OPTIONS=() - _run_in_container_bash "nc ${NC_PARAMETERS} < /tmp/docker-mailserver-test/${TEMPLATE_FILE}.txt" - assert_success + while [[ ${#} -gt 0 ]]; do + case "${1}" in + ( '--ehlo' ) EHLO=${2:?--ehlo given but no argument} ; shift 2 ;; + ( '--from' ) FROM=${2:?--from given but no argument} ; shift 2 ;; + ( '--to' ) TO=${2:?--to given but no argument} ; shift 2 ;; + ( '--server' ) SERVER=${2:?--server given but no argument} ; shift 2 ;; + ( '--port' ) PORT=${2:?--port given but no argument} ; shift 2 ;; + ( '--data' ) + local TEMPLATE_FILE="/tmp/docker-mailserver-test/emails/${2:?--data given but no argument provided}.txt" + FINAL_SWAKS_OPTIONS+=('--data') + FINAL_SWAKS_OPTIONS+=('-') + FINAL_SWAKS_OPTIONS+=('<') + FINAL_SWAKS_OPTIONS+=("${TEMPLATE_FILE}") + shift 2 + ;; + ( * ) ADDITIONAL_SWAKS_OPTIONS+=("${1}") ; shift 1 ;; + esac + done + + _run_in_container_bash "swaks --server ${SERVER} --port ${PORT} --ehlo ${EHLO} --from ${FROM} --to ${TO} ${ADDITIONAL_SWAKS_OPTIONS[*]} ${FINAL_SWAKS_OPTIONS[*]}" } -# Like `_send_mail` with two major differences: +# Like `_send_email` with two major differences: # # 1. this function waits for the mail to be processed; there is no asynchronicity # because filtering the logs in a synchronous way is easier and safer! @@ -42,8 +68,7 @@ function _send_email() { # No. 2 is especially useful in case you send more than one email in a single # test file and need to assert certain log entries for each mail individually. # -# @param ${1} = template file (path) name -# @param ${2} = parameters for `nc` [OPTIONAL] (default: `0.0.0.0 25`) +# This function takes the same arguments as `_send_mail`. # # ## Attention # @@ -57,17 +82,13 @@ function _send_email() { # chosen. Sending more than one mail at any given point in time with this function # is UNDEFINED BEHAVIOR! function _send_email_and_get_id() { - local TEMPLATE_FILE=${1:?Must provide name of template file} - local NC_PARAMETERS=${2:-0.0.0.0 25} - local MAIL_ID - - assert_not_equal "${NC_PARAMETERS}" '' - assert_not_equal "${CONTAINER_NAME:-}" '' + [[ -v CONTAINER_NAME ]] || return 1 _wait_for_empty_mail_queue_in_container - _send_email "${TEMPLATE_FILE}" + _send_email "${@}" _wait_for_empty_mail_queue_in_container + local MAIL_ID # The unique ID Postfix (and other services) use may be different in length # on different systems (e.g. amd64 (11) vs aarch64 (10)). Hence, we use a # range to safely capture it. diff --git a/test/helper/setup.bash b/test/helper/setup.bash index 65e2999f130..0dd57bd6b80 100644 --- a/test/helper/setup.bash +++ b/test/helper/setup.bash @@ -98,7 +98,7 @@ function _init_with_defaults() { # Common complimentary test files, read-only safe to share across containers: export TEST_FILES_CONTAINER_PATH='/tmp/docker-mailserver-test' - export TEST_FILES_VOLUME="${REPOSITORY_ROOT}/test/test-files:${TEST_FILES_CONTAINER_PATH}:ro" + export TEST_FILES_VOLUME="${REPOSITORY_ROOT}/test/files:${TEST_FILES_CONTAINER_PATH}:ro" # The config volume cannot be read-only as some data needs to be written at container startup # diff --git a/test/test-files/auth/added-smtp-auth-login-wrong.txt b/test/test-files/auth/added-smtp-auth-login-wrong.txt deleted file mode 100644 index a75856f11e5..00000000000 --- a/test/test-files/auth/added-smtp-auth-login-wrong.txt +++ /dev/null @@ -1,4 +0,0 @@ -EHLO mail -AUTH LOGIN YWRkZWRAbG9jYWxob3N0LmxvY2FsZG9tYWlu -Bn3JKisq4HQ2RO== -QUIT diff --git a/test/test-files/auth/added-smtp-auth-login.txt b/test/test-files/auth/added-smtp-auth-login.txt deleted file mode 100644 index 5276b7f4181..00000000000 --- a/test/test-files/auth/added-smtp-auth-login.txt +++ /dev/null @@ -1,4 +0,0 @@ -EHLO mail -AUTH LOGIN YWRkZWRAbG9jYWxob3N0LmxvY2FsZG9tYWlu -bXlwYXNzd29yZA== -QUIT diff --git a/test/test-files/auth/added-smtp-auth-plain-wrong.txt b/test/test-files/auth/added-smtp-auth-plain-wrong.txt deleted file mode 100644 index 6ce5a3833ef..00000000000 --- a/test/test-files/auth/added-smtp-auth-plain-wrong.txt +++ /dev/null @@ -1,3 +0,0 @@ -EHLO mail -AUTH PLAIN YWRkZWRAbG9jYWxob3N0LmxvY2FsZG9tYWluAGFkZGVkQGxvY2FsaG9zdC5sb2NhbGRvbWFpbgBCQURQQVNTV09SRA== -QUIT diff --git a/test/test-files/auth/added-smtp-auth-plain.txt b/test/test-files/auth/added-smtp-auth-plain.txt deleted file mode 100644 index ed48d77df8a..00000000000 --- a/test/test-files/auth/added-smtp-auth-plain.txt +++ /dev/null @@ -1,3 +0,0 @@ -EHLO mail -AUTH PLAIN YWRkZWRAbG9jYWxob3N0LmxvY2FsZG9tYWluAGFkZGVkQGxvY2FsaG9zdC5sb2NhbGRvbWFpbgBteXBhc3N3b3Jk -QUIT diff --git a/test/test-files/auth/sasl-ldap-smtp-auth.txt b/test/test-files/auth/sasl-ldap-smtp-auth.txt deleted file mode 100644 index df4d7db4064..00000000000 --- a/test/test-files/auth/sasl-ldap-smtp-auth.txt +++ /dev/null @@ -1,5 +0,0 @@ -EHLO mail -AUTH LOGIN -c29tZS51c2VyQGxvY2FsaG9zdC5sb2NhbGRvbWFpbg== -c2VjcmV0 -QUIT diff --git a/test/test-files/auth/smtp-auth-login-wrong.txt b/test/test-files/auth/smtp-auth-login-wrong.txt deleted file mode 100644 index 39b4f01c619..00000000000 --- a/test/test-files/auth/smtp-auth-login-wrong.txt +++ /dev/null @@ -1,4 +0,0 @@ -EHLO mail -AUTH LOGIN dXNlcjFAbG9jYWxob3N0LmxvY2FsZG9tYWlu -Bn3JKisq4HQ2RO== -QUIT diff --git a/test/test-files/auth/smtp-auth-login.txt b/test/test-files/auth/smtp-auth-login.txt deleted file mode 100644 index 50ff99f3104..00000000000 --- a/test/test-files/auth/smtp-auth-login.txt +++ /dev/null @@ -1,4 +0,0 @@ -EHLO mail -AUTH LOGIN dXNlcjFAbG9jYWxob3N0LmxvY2FsZG9tYWlu -bXlwYXNzd29yZA== -QUIT diff --git a/test/test-files/auth/smtp-auth-plain-wrong.txt b/test/test-files/auth/smtp-auth-plain-wrong.txt deleted file mode 100644 index d8d8ad2a355..00000000000 --- a/test/test-files/auth/smtp-auth-plain-wrong.txt +++ /dev/null @@ -1,3 +0,0 @@ -EHLO mail -AUTH PLAIN WRONGPASSWORD -QUIT diff --git a/test/test-files/auth/smtp-auth-plain.txt b/test/test-files/auth/smtp-auth-plain.txt deleted file mode 100644 index 2e60fdc3e20..00000000000 --- a/test/test-files/auth/smtp-auth-plain.txt +++ /dev/null @@ -1,3 +0,0 @@ -EHLO mail -AUTH PLAIN dXNlcjFAbG9jYWxob3N0LmxvY2FsZG9tYWluAHVzZXIxQGxvY2FsaG9zdC5sb2NhbGRvbWFpbgBteXBhc3N3b3Jk -QUIT diff --git a/test/test-files/email-templates/existing-user2.txt b/test/test-files/email-templates/existing-user2.txt deleted file mode 100644 index 63554f27ab3..00000000000 --- a/test/test-files/email-templates/existing-user2.txt +++ /dev/null @@ -1,12 +0,0 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: user2@otherdomain.tld -DATA -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-user2.txt -This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/existing-user3.txt b/test/test-files/email-templates/existing-user3.txt deleted file mode 100644 index facd5328b66..00000000000 --- a/test/test-files/email-templates/existing-user3.txt +++ /dev/null @@ -1,12 +0,0 @@ -HELO mail.external.tld -MAIL FROM: user@external.tld -RCPT TO: user3@localhost.localdomain -DATA -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:33 -0400 -Subject: Test Message existing-user1.txt -This is a test mail. - -. -QUIT diff --git a/test/test-files/email-templates/smtp-ehlo.txt b/test/test-files/email-templates/smtp-ehlo.txt deleted file mode 100644 index 05524efd1f6..00000000000 --- a/test/test-files/email-templates/smtp-ehlo.txt +++ /dev/null @@ -1,2 +0,0 @@ -EHLO mail.localhost -QUIT diff --git a/test/tests/parallel/set1/dovecot/dovecot_quotas.bats b/test/tests/parallel/set1/dovecot/dovecot_quotas.bats index 2c176235dc7..81cf9bc15cf 100644 --- a/test/tests/parallel/set1/dovecot/dovecot_quotas.bats +++ b/test/tests/parallel/set1/dovecot/dovecot_quotas.bats @@ -225,9 +225,12 @@ function teardown_file() { _default_teardown ; } sleep 10 # send some big emails - _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' - _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' - _send_email 'email-templates/quota-exceeded' '0.0.0.0 25' + _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded' + assert_success + _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded' + assert_success + _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded' + assert_success # check for quota warn message existence run _repeat_until_success_or_timeout 20 _exec_in_container grep -R 'Subject: quota warning' /var/mail/otherdomain.tld/quotauser/new/ assert_success diff --git a/test/tests/parallel/set1/dovecot/dovecot_sieve.bats b/test/tests/parallel/set1/dovecot/dovecot_sieve.bats index c2e9e6c7d69..e3e076a5267 100644 --- a/test/tests/parallel/set1/dovecot/dovecot_sieve.bats +++ b/test/tests/parallel/set1/dovecot/dovecot_sieve.bats @@ -26,9 +26,11 @@ function setup_file() { _wait_for_smtp_port_in_container # Single mail sent from 'spam@spam.com' that is handled by User (relocate) and Global (copy) sieves for user1: - _send_email 'email-templates/sieve-spam-folder' + _send_email --data 'sieve/spam-folder' + assert_success # Mail for user2 triggers the sieve-pipe: - _send_email 'email-templates/sieve-pipe' + _send_email --to 'user2@otherdomain.tld' --data 'sieve/pipe' + assert_success _wait_for_empty_mail_queue_in_container } diff --git a/test/tests/parallel/set1/dovecot/mailbox_format_dbox.bats b/test/tests/parallel/set1/dovecot/mailbox_format_dbox.bats index 8ce03d9aee9..033a5bdec6b 100644 --- a/test/tests/parallel/set1/dovecot/mailbox_format_dbox.bats +++ b/test/tests/parallel/set1/dovecot/mailbox_format_dbox.bats @@ -26,7 +26,8 @@ function teardown() { _default_teardown ; } _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' _wait_for_smtp_port_in_container - _send_email 'email-templates/existing-user1' + _send_email --data 'existing/user1' + assert_success _wait_for_empty_mail_queue_in_container # Mail received should be stored as `u.1` (one file per message) @@ -47,7 +48,8 @@ function teardown() { _default_teardown ; } _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' _wait_for_smtp_port_in_container - _send_email 'email-templates/existing-user1' + _send_email --data 'existing/user1' + assert_success _wait_for_empty_mail_queue_in_container # Mail received should be stored in `m.1` (1 or more messages) diff --git a/test/tests/parallel/set1/dovecot/special_use_folders.bats b/test/tests/parallel/set1/dovecot/special_use_folders.bats index e70899a050c..fe1f554e5a2 100644 --- a/test/tests/parallel/set1/dovecot/special_use_folders.bats +++ b/test/tests/parallel/set1/dovecot/special_use_folders.bats @@ -14,7 +14,8 @@ function setup_file() { function teardown_file() { _default_teardown ; } @test 'normal delivery works' { - _send_email 'email-templates/existing-user1' + _send_email --data 'existing/user1' + assert_success _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new 1 } @@ -26,7 +27,7 @@ function teardown_file() { _default_teardown ; } } @test "(IMAP) special-use folders should be created when necessary" { - _send_email 'nc_templates/imap_special_use_folders' '-w 8 0.0.0.0 143' + _nc_wrapper 'nc/imap_special_use_folders' '-w 8 0.0.0.0 143' assert_output --partial 'Drafts' assert_output --partial 'Junk' assert_output --partial 'Trash' diff --git a/test/tests/parallel/set1/spam_virus/clamav.bats b/test/tests/parallel/set1/spam_virus/clamav.bats index 31608ef86aa..9232f90fa74 100644 --- a/test/tests/parallel/set1/spam_virus/clamav.bats +++ b/test/tests/parallel/set1/spam_virus/clamav.bats @@ -25,34 +25,35 @@ function setup_file() { _wait_for_service postfix _wait_for_smtp_port_in_container - _send_email 'email-templates/amavis-virus' + _send_email --from 'virus@external.tld' --data 'amavis/virus' + assert_success _wait_for_empty_mail_queue_in_container } function teardown_file() { _default_teardown ; } -@test "log files exist at /var/log/mail directory" { +@test 'log files exist at /var/log/mail directory' { _run_in_container_bash "ls -1 /var/log/mail/ | grep -E 'clamav|freshclam|mail.log' | wc -l" assert_success assert_output 3 } -@test "should be identified by Amavis" { +@test 'should be identified by Amavis' { _run_in_container grep -i 'Found secondary av scanner ClamAV-clamscan' /var/log/mail/mail.log assert_success } -@test "freshclam cron is enabled" { +@test 'freshclam cron is enabled' { _run_in_container_bash "grep '/usr/bin/freshclam' -r /etc/cron.d" assert_success } -@test "env CLAMAV_MESSAGE_SIZE_LIMIT is set correctly" { +@test 'env CLAMAV_MESSAGE_SIZE_LIMIT is set correctly' { _run_in_container grep -q '^MaxFileSize 30M$' /etc/clamav/clamd.conf assert_success } -@test "rejects virus" { +@test 'rejects virus' { _run_in_container_bash "grep 'Blocked INFECTED' /var/log/mail/mail.log | grep ' -> '" assert_success } diff --git a/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats b/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats index 8402422c308..f2474cc07a1 100644 --- a/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats +++ b/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats @@ -12,12 +12,14 @@ function setup_file() { --env ENABLE_CLAMAV=0 --env ENABLE_SPAMASSASSIN=0 --env AMAVIS_LOGLEVEL=2 + --env PERMIT_DOCKER=container ) _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' _wait_for_smtp_port_in_container - _send_email 'email-templates/existing-user1' + _send_email --data 'existing/user1' + assert_success _wait_for_empty_mail_queue_in_container } diff --git a/test/tests/parallel/set1/spam_virus/fail2ban.bats b/test/tests/parallel/set1/spam_virus/fail2ban.bats index 9ae3075898b..8a03ba04acd 100644 --- a/test/tests/parallel/set1/spam_virus/fail2ban.bats +++ b/test/tests/parallel/set1/spam_virus/fail2ban.bats @@ -73,8 +73,17 @@ function teardown_file() { @test "ban ip on multiple failed login" { CONTAINER1_IP=$(_get_container_ip "${CONTAINER1_NAME}") # Trigger a ban by failing to login twice: - CONTAINER_NAME=${CONTAINER2_NAME} _send_email 'auth/smtp-auth-login-wrong' "${CONTAINER1_IP} 465" - CONTAINER_NAME=${CONTAINER2_NAME} _send_email 'auth/smtp-auth-login-wrong' "${CONTAINER1_IP} 465" + for _ in {1..2}; do + CONTAINER_NAME=${CONTAINER2_NAME} _send_email \ + --server "${CONTAINER1_IP}" \ + --port 465 \ + --auth PLAIN \ + --auth-user user1@localhost.localdomain \ + --auth-password wrongpassword + assert_failure + assert_output --partial 'authentication failed' + assert_output --partial 'No authentication type succeeded' + done # Checking that CONTAINER2_IP is banned in "${CONTAINER1_NAME}" CONTAINER2_IP=$(_get_container_ip "${CONTAINER2_NAME}") diff --git a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats index e32210cae7f..316e33500da 100644 --- a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats +++ b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats @@ -51,17 +51,15 @@ function teardown_file() { _default_teardown ; } _reload_postfix # Send test mail (it should fail to deliver): - _send_test_mail '/tmp/docker-mailserver-test/email-templates/postgrey.txt' '25' + _send_email --from 'user@external.tld' --port 25 --data 'postgrey' + assert_failure + assert_output --partial 'Recipient address rejected: Delayed by Postgrey' # Confirm mail was greylisted: _should_have_log_entry \ 'action=greylist' \ 'reason=new' \ 'client_address=127.0.0.1/32, sender=user@external.tld, recipient=user1@localhost.localdomain' - - _repeat_until_success_or_timeout 10 _run_in_container grep \ - 'Recipient address rejected: Delayed by Postgrey' \ - /var/log/mail/mail.log } # NOTE: This test case depends on the previous one @@ -69,7 +67,8 @@ function teardown_file() { _default_teardown ; } # Wait until `$POSTGREY_DELAY` seconds pass before trying again: sleep 3 # Retry delivering test mail (it should be trusted this time): - _send_test_mail '/tmp/docker-mailserver-test/email-templates/postgrey.txt' '25' + _send_email --from 'user@external.tld' --port 25 --data 'postgrey' + assert_success # Confirm postgrey permitted delivery (triplet is now trusted): _should_have_log_entry \ @@ -78,8 +77,9 @@ function teardown_file() { _default_teardown ; } 'client_address=127.0.0.1/32, sender=user@external.tld, recipient=user1@localhost.localdomain' } - -# NOTE: These two whitelist tests use `test-files/nc_templates/` instead of `test-files/email-templates`. +# NOTE: These two whitelist tests use `files/nc/` instead of `files/emails`. +# `nc` option `-w 0` terminates the connection after sending the template, it does not wait for a response. +# This is required for port 10023, otherwise the connection never drops. # - This allows to bypass the SMTP protocol on port 25, and send data directly to Postgrey instead. # - Appears to be a workaround due to `client_name=localhost` when sent from Postfix. # - Could send over port 25 if whitelisting `localhost`, @@ -87,7 +87,7 @@ function teardown_file() { _default_teardown ; } # - It'd also cause the earlier greylist test to fail. # - TODO: Actually confirm whitelist feature works correctly as these test cases are using a workaround: @test "should whitelist sender 'user@whitelist.tld'" { - _send_test_mail '/tmp/docker-mailserver-test/nc_templates/postgrey_whitelist.txt' '10023' + _nc_wrapper 'nc/postgrey_whitelist' '-w 0 0.0.0.0 10023' _should_have_log_entry \ 'action=pass' \ @@ -96,7 +96,7 @@ function teardown_file() { _default_teardown ; } } @test "should whitelist recipient 'user2@otherdomain.tld'" { - _send_test_mail '/tmp/docker-mailserver-test/nc_templates/postgrey_whitelist_recipients.txt' '10023' + _nc_wrapper 'nc/postgrey_whitelist_recipients' '-w 0 0.0.0.0 10023' _should_have_log_entry \ 'action=pass' \ @@ -104,21 +104,10 @@ function teardown_file() { _default_teardown ; } 'client_address=127.0.0.1/32, sender=test@nonwhitelist.tld, recipient=user2@otherdomain.tld' } -function _send_test_mail() { - local MAIL_TEMPLATE=$1 - local PORT=${2:-25} - - # `-w 0` terminates the connection after sending the template, it does not wait for a response. - # This is required for port 10023, otherwise the connection never drops. - # It could increase the number of seconds to wait for port 25 to allow for asserting a response, - # but that would enforce the delay in tests for port 10023. - _run_in_container_bash "nc -w 0 0.0.0.0 ${PORT} < ${MAIL_TEMPLATE}" -} - function _should_have_log_entry() { - local ACTION=$1 - local REASON=$2 - local TRIPLET=$3 + local ACTION=${1} + local REASON=${2} + local TRIPLET=${3} # Allow some extra time for logs to update to avoids a false-positive failure: _run_until_success_or_timeout 10 _exec_in_container grep \ diff --git a/test/tests/parallel/set1/spam_virus/postscreen.bats b/test/tests/parallel/set1/spam_virus/postscreen.bats index a1ddeb2999b..377b2479357 100644 --- a/test/tests/parallel/set1/spam_virus/postscreen.bats +++ b/test/tests/parallel/set1/spam_virus/postscreen.bats @@ -37,46 +37,35 @@ function teardown_file() { docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}" } +# `POSTSCREEN_ACTION=enforce` (DMS default) should reject delivery with a 550 SMTP reply +# A legitimate mail client should speak SMTP by waiting it's turn, which postscreen defaults enforce (only on port 25) +# https://www.postfix.org/postconf.5.html#postscreen_greet_wait +# +# Use `nc` to send all SMTP commands at once instead (emulate a misbehaving client that should be rejected) +# NOTE: Postscreen only runs on port 25, avoid implicit ports in test methods @test 'should fail send when talking out of turn' { - CONTAINER_NAME=${CONTAINER2_NAME} _send_email 'email-templates/postscreen' "${CONTAINER1_IP} 25" + CONTAINER_NAME=${CONTAINER2_NAME} _nc_wrapper 'emails/nc_raw/postscreen' "${CONTAINER1_IP} 25" + # Expected postscreen log entry: assert_output --partial 'Protocol error' - # Expected postscreen log entry: - _run_in_container cat /var/log/mail/mail.log + _run_in_container cat /var/log/mail.log assert_output --partial 'COMMAND PIPELINING' + assert_output --partial 'DATA without valid RCPT' } @test "should successfully pass postscreen and get postfix greeting message (respecting postscreen_greet_wait time)" { - # NOTE: Sometimes fails on first attempt (trying too soon?), - # Instead of a `run` + asserting partial, Using repeat + internal grep match: - _repeat_until_success_or_timeout 10 _should_wait_turn_speaking_smtp \ - "${CONTAINER2_NAME}" \ - "${CONTAINER1_IP}" \ - '/tmp/docker-mailserver-test/email-templates/postscreen.txt' \ - '220 mail.example.test ESMTP' - - # Expected postscreen log entry: - _run_in_container cat /var/log/mail/mail.log - assert_output --partial 'PASS NEW' -} + # Configure `send_email()` to send from the mail client container (CONTAINER2_NAME) via ENV override, + # mail is sent to the DMS server container (CONTAINER1_NAME) via `--server` parameter: + CONTAINER_NAME=${CONTAINER2_NAME} _send_email --server "${CONTAINER1_IP}" --port 25 --data 'postscreen' + # NOTE: Cannot assert_success due to sender address not being resolvable. + # TODO: Uncomment when proper resolution of domain names is possible: + # assert_success -# When postscreen is active, it prevents the usual method of piping a file through nc: -# (Won't work: CONTAINER_NAME=${CLIENT_CONTAINER_NAME} _send_email "${SMTP_TEMPLATE}" "${TARGET_CONTAINER_IP} 25") -# The below workaround respects `postscreen_greet_wait` time (default 6 sec), talking to the mail-server in turn: -# https://www.postfix.org/postconf.5.html#postscreen_greet_wait -function _should_wait_turn_speaking_smtp() { - local CLIENT_CONTAINER_NAME=$1 - local TARGET_CONTAINER_IP=$2 - local SMTP_TEMPLATE=$3 - local EXPECTED=$4 + # TODO: Prefer this approach when `_send_email_and_get_id()` can support separate client and server containers: + # local MAIL_ID=$(_send_email_and_get_id --port 25 --data 'postscreen') + # _print_mail_log_for_id "${MAIL_ID}" + # assert_output --partial "stored mail into mailbox 'INBOX'" - # shellcheck disable=SC2016 - local UGLY_WORKAROUND='exec 3<>/dev/tcp/'"${TARGET_CONTAINER_IP}"'/25 && \ - while IFS= read -r cmd; do \ - head -1 <&3; \ - [[ ${cmd} == "EHLO"* ]] && sleep 6; \ - echo ${cmd} >&3; \ - done < '"${SMTP_TEMPLATE}" - - docker exec "${CLIENT_CONTAINER_NAME}" bash -c "${UGLY_WORKAROUND}" | grep "${EXPECTED}" + _run_in_container cat /var/log/mail.log + assert_output --partial 'PASS NEW' } diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index ba8a23f59c8..2e610d72618 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -45,10 +45,10 @@ function setup_file() { # We will send 3 emails: the first one should pass just fine; the second one should # be rejected due to spam; the third one should be rejected due to a virus. - export MAIL_ID1=$(_send_email_and_get_id 'email-templates/rspamd-pass') - export MAIL_ID2=$(_send_email_and_get_id 'email-templates/rspamd-spam') - export MAIL_ID3=$(_send_email_and_get_id 'email-templates/rspamd-virus') - export MAIL_ID4=$(_send_email_and_get_id 'email-templates/rspamd-spam-header') + export MAIL_ID1=$(_send_email_and_get_id --from 'rspamd-pass@example.test' --data 'rspamd/pass') + export MAIL_ID2=$(_send_email_and_get_id --from 'rspamd-spam@example.test' --data 'rspamd/spam') + export MAIL_ID3=$(_send_email_and_get_id --from 'rspamd-virus@example.test' --data 'rspamd/virus') + export MAIL_ID4=$(_send_email_and_get_id --from 'rspamd-spam-header@example.test' --data 'rspamd/spam-header') for ID in MAIL_ID{1,2,3,4}; do [[ -n ${!ID} ]] || { echo "${ID} is empty - aborting!" ; return 1 ; } @@ -256,7 +256,7 @@ function teardown_file() { _default_teardown ; } # Move an email to the "Junk" folder from "INBOX"; the first email we # sent should pass fine, hence we can now move it. - _send_email 'nc_templates/rspamd_imap_move_to_junk' '0.0.0.0 143' + _nc_wrapper 'nc/rspamd_imap_move_to_junk' '0.0.0.0 143' sleep 1 # wait for the transaction to finish _run_in_container cat /var/log/mail/mail.log @@ -270,7 +270,7 @@ function teardown_file() { _default_teardown ; } # Move an email to the "INBOX" folder from "Junk"; there should be two mails # in the "Junk" folder, since the second email we sent during setup should # have landed in the Junk folder already. - _send_email 'nc_templates/rspamd_imap_move_to_inbox' '0.0.0.0 143' + _nc_wrapper 'nc/rspamd_imap_move_to_inbox' '0.0.0.0 143' sleep 1 # wait for the transaction to finish _run_in_container cat /var/log/mail/mail.log diff --git a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats index 94a9b9c47e7..fea23b0bdc7 100644 --- a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats +++ b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats @@ -95,7 +95,7 @@ function teardown() { _default_teardown ; } function _should_send_spam_message() { _wait_for_smtp_port_in_container _wait_for_tcp_port_in_container 10024 # port 10024 is for Amavis - _send_email 'email-templates/amavis-spam' + _send_email --from 'spam@external.tld' --data 'amavis/spam' } function _should_be_received_by_amavis() { diff --git a/test/tests/parallel/set1/tls/dhparams.bats b/test/tests/parallel/set1/tls/dhparams.bats index 3157034c254..8b3047d1737 100644 --- a/test/tests/parallel/set1/tls/dhparams.bats +++ b/test/tests/parallel/set1/tls/dhparams.bats @@ -38,7 +38,7 @@ function teardown() { _default_teardown ; } # - A warning is raised about usage of potentially insecure parameters. @test "Custom" { export CONTAINER_NAME=${CONTAINER2_NAME} - local DH_PARAMS_CUSTOM='test/test-files/ssl/custom-dhe-params.pem' + local DH_PARAMS_CUSTOM='test/files/ssl/custom-dhe-params.pem' local DH_CHECKSUM_CUSTOM=$(sha512sum "${DH_PARAMS_CUSTOM}" | awk '{print $1}') _init_with_defaults diff --git a/test/tests/parallel/set1/tls/letsencrypt.bats b/test/tests/parallel/set1/tls/letsencrypt.bats index 91a0599760d..bcdb1758939 100644 --- a/test/tests/parallel/set1/tls/letsencrypt.bats +++ b/test/tests/parallel/set1/tls/letsencrypt.bats @@ -88,7 +88,7 @@ function _initial_setup() { # All of these certs support both FQDNs (`mail.example.test` and `example.test`), # Except for the wildcard cert (`*.example.test`), that was created with `example.test` intentionally excluded from SAN. # We want to maintain the same FQDN (`mail.example.test`) between the _acme_ecdsa and _acme_rsa tests. - local LOCAL_BASE_PATH="${PWD}/test/test-files/ssl/example.test/with_ca/rsa" + local LOCAL_BASE_PATH="${PWD}/test/files/ssl/example.test/with_ca/rsa" function _prepare() { # Default `acme.json` for _acme_ecdsa test: @@ -240,7 +240,7 @@ function _copy_to_letsencrypt_storage() { FQDN_DIR=$(echo "${DEST}" | cut -d '/' -f1) mkdir -p "${TEST_TMP_CONFIG}/letsencrypt/${FQDN_DIR}" - if ! cp "${PWD}/test/test-files/ssl/${SRC}" "${TEST_TMP_CONFIG}/letsencrypt/${DEST}"; then + if ! cp "${PWD}/test/files/ssl/${SRC}" "${TEST_TMP_CONFIG}/letsencrypt/${DEST}"; then echo "Could not copy cert file '${SRC}'' to '${DEST}'" >&2 exit 1 fi diff --git a/test/tests/parallel/set1/tls/manual.bats b/test/tests/parallel/set1/tls/manual.bats index 2a55f14f593..c082d6ed728 100644 --- a/test/tests/parallel/set1/tls/manual.bats +++ b/test/tests/parallel/set1/tls/manual.bats @@ -20,7 +20,7 @@ function setup_file() { export TEST_DOMAIN='example.test' local CUSTOM_SETUP_ARGUMENTS=( - --volume "${PWD}/test/test-files/ssl/${TEST_DOMAIN}/with_ca/ecdsa/:/config/ssl/:ro" + --volume "${PWD}/test/files/ssl/${TEST_DOMAIN}/with_ca/ecdsa/:/config/ssl/:ro" --env LOG_LEVEL='trace' --env SSL_TYPE='manual' --env TLS_LEVEL='modern' @@ -108,10 +108,10 @@ function teardown_file() { _default_teardown ; } @test "manual cert changes are picked up by check-for-changes" { printf '%s' 'someThingsChangedHere' \ - >>"$(pwd)/test/test-files/ssl/${TEST_DOMAIN}/with_ca/ecdsa/key.ecdsa.pem" + >>"$(pwd)/test/files/ssl/${TEST_DOMAIN}/with_ca/ecdsa/key.ecdsa.pem" run timeout 15 docker exec "${CONTAINER_NAME}" bash -c "tail -F /var/log/supervisor/changedetector.log | sed '/Manual certificates have changed/ q'" assert_success - sed -i '/someThingsChangedHere/d' "$(pwd)/test/test-files/ssl/${TEST_DOMAIN}/with_ca/ecdsa/key.ecdsa.pem" + sed -i '/someThingsChangedHere/d' "$(pwd)/test/files/ssl/${TEST_DOMAIN}/with_ca/ecdsa/key.ecdsa.pem" } diff --git a/test/tests/parallel/set2/tls_cipherlists.bats b/test/tests/parallel/set2/tls_cipherlists.bats index 2b9511b9c78..3429f516f83 100644 --- a/test/tests/parallel/set2/tls_cipherlists.bats +++ b/test/tests/parallel/set2/tls_cipherlists.bats @@ -17,7 +17,7 @@ function setup_file() { # Contains various certs for testing TLS support (read-only): export TLS_CONFIG_VOLUME - TLS_CONFIG_VOLUME="${PWD}/test/test-files/ssl/${TEST_DOMAIN}/:/config/ssl/:ro" + TLS_CONFIG_VOLUME="${PWD}/test/files/ssl/${TEST_DOMAIN}/:/config/ssl/:ro" # Used for connecting testssl and DMS containers via network name `TEST_DOMAIN`: # NOTE: If the network already exists, the test will fail to start diff --git a/test/tests/parallel/set3/container_configuration/hostname.bats b/test/tests/parallel/set3/container_configuration/hostname.bats index fcb84b28e11..f5774eef8ed 100644 --- a/test/tests/parallel/set3/container_configuration/hostname.bats +++ b/test/tests/parallel/set3/container_configuration/hostname.bats @@ -207,7 +207,7 @@ function _should_have_correct_mail_headers() { # (eg: OVERRIDE_HOSTNAME or `--hostname mail --domainname example.test`) local EXPECTED_HOSTNAME=${3:-${EXPECTED_FQDN}} - _send_email 'email-templates/existing-user1' + _send_email --from 'user@external.tld' --data 'existing/user1' _wait_for_empty_mail_queue_in_container _count_files_in_directory_in_container '/var/mail/localhost.localdomain/user1/new/' '1' diff --git a/test/tests/parallel/set3/mta/dsn.bats b/test/tests/parallel/set3/mta/dsn.bats index dcbb79b6603..a5228cfc60d 100644 --- a/test/tests/parallel/set3/mta/dsn.bats +++ b/test/tests/parallel/set3/mta/dsn.bats @@ -47,9 +47,11 @@ function teardown_file() { @test "should always send a DSN when requested" { export CONTAINER_NAME=${CONTAINER1_NAME} - _send_email 'email-templates/dsn-unauthenticated' - _send_email 'email-templates/dsn-authenticated' '0.0.0.0 465' - _send_email 'email-templates/dsn-authenticated' '0.0.0.0 587' + # TODO replace with _send_email as soon as it supports DSN + # TODO ref: https://github.com/jetmore/swaks/issues/41 + _nc_wrapper 'emails/nc_raw/dsn/unauthenticated' + _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 465' + _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 587' _wait_for_empty_mail_queue_in_container _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log @@ -60,7 +62,7 @@ function teardown_file() { @test "should only send a DSN when requested from ports 465/587" { export CONTAINER_NAME=${CONTAINER2_NAME} - _send_email 'email-templates/dsn-unauthenticated' + _nc_wrapper 'emails/nc_raw/dsn/unauthenticated' _wait_for_empty_mail_queue_in_container # DSN requests can now only be made on ports 465 and 587, @@ -72,8 +74,8 @@ function teardown_file() { assert_failure # These ports are excluded via master.cf. - _send_email 'email-templates/dsn-authenticated' '0.0.0.0 465' - _send_email 'email-templates/dsn-authenticated' '0.0.0.0 587' + _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 465' + _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 587' _wait_for_empty_mail_queue_in_container _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log @@ -83,9 +85,9 @@ function teardown_file() { @test "should never send a DSN" { export CONTAINER_NAME=${CONTAINER3_NAME} - _send_email 'email-templates/dsn-unauthenticated' - _send_email 'email-templates/dsn-authenticated' '0.0.0.0 465' - _send_email 'email-templates/dsn-authenticated' '0.0.0.0 587' + _nc_wrapper 'emails/nc_raw/dsn/unauthenticated' + _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 465' + _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 587' _wait_for_empty_mail_queue_in_container # DSN requests are rejected regardless of origin. diff --git a/test/tests/parallel/set3/mta/lmtp_ip.bats b/test/tests/parallel/set3/mta/lmtp_ip.bats index 8d35c062750..d8be42d936d 100644 --- a/test/tests/parallel/set3/mta/lmtp_ip.bats +++ b/test/tests/parallel/set3/mta/lmtp_ip.bats @@ -38,7 +38,7 @@ function teardown_file() { _default_teardown ; } @test "delivers mail to existing account" { _wait_for_smtp_port_in_container - _send_email 'email-templates/existing-user1' # send a test email + _send_email --data 'existing/user1' # send a test email # Verify delivery was successful, log line should look similar to: # postfix/lmtp[1274]: 0EA424ABE7D9: to=, relay=127.0.0.1[127.0.0.1]:24, delay=0.13, delays=0.07/0.01/0.01/0.05, dsn=2.0.0, status=sent (250 2.0.0 ixPpB+Zvv2P7BAAAUi6ngw Saved) diff --git a/test/tests/parallel/set3/mta/privacy.bats b/test/tests/parallel/set3/mta/privacy.bats index f8160827327..4d4d82ba467 100644 --- a/test/tests/parallel/set3/mta/privacy.bats +++ b/test/tests/parallel/set3/mta/privacy.bats @@ -25,7 +25,11 @@ function teardown_file() { _default_teardown ; } # this test covers https://github.com/docker-mailserver/docker-mailserver/issues/681 @test "(Postfix) remove privacy details of the sender" { - _run_in_container_bash "openssl s_client -quiet -starttls smtp -connect 0.0.0.0:587 < /tmp/docker-mailserver-test/email-templates/send-privacy-email.txt" + _send_email \ + --port 587 -tls --auth LOGIN \ + --auth-user user1@localhost.localdomain \ + --auth-password mypassword \ + --data 'privacy' assert_success _run_until_success_or_timeout 120 _exec_in_container_bash '[[ -d /var/mail/localhost.localdomain/user1/new ]]' diff --git a/test/tests/parallel/set3/mta/smtp_delivery.bats b/test/tests/parallel/set3/mta/smtp_delivery.bats index af98b2f4f1b..169f374ad3d 100644 --- a/test/tests/parallel/set3/mta/smtp_delivery.bats +++ b/test/tests/parallel/set3/mta/smtp_delivery.bats @@ -63,34 +63,55 @@ function setup_file() { # TODO: Move to clamav tests (For use when ClamAV is enabled): # _repeat_in_container_until_success_or_timeout 60 "${CONTAINER_NAME}" test -e /var/run/clamav/clamd.ctl - # _send_email 'email-templates/amavis-virus' + # _send_email --from 'virus@external.tld' --data 'amavis/virus' # Required for 'delivers mail to existing alias': - _send_email 'email-templates/existing-alias-external' + _send_email --to alias1@localhost.localdomain --data 'existing/alias-external' # Required for 'delivers mail to existing alias with recipient delimiter': - _send_email 'email-templates/existing-alias-recipient-delimiter' + _send_email --to alias1~test@localhost.localdomain --data 'existing/alias-recipient-delimiter' # Required for 'delivers mail to existing catchall': - _send_email 'email-templates/existing-catchall-local' + _send_email --to wildcard@localdomain2.com --data 'existing/catchall-local' # Required for 'delivers mail to regexp alias': - _send_email 'email-templates/existing-regexp-alias-local' + _send_email --to test123@localhost.localdomain --data 'existing/regexp-alias-local' # Required for 'rejects mail to unknown user': - _send_email 'email-templates/non-existing-user' + _send_email --to nouser@localhost.localdomain --data 'non-existing-user' # Required for 'redirects mail to external aliases': - _send_email 'email-templates/existing-regexp-alias-external' - _send_email 'email-templates/existing-alias-local' + _send_email --to bounce-always@localhost.localdomain --data 'existing/regexp-alias-external' + _send_email --to alias2@localhost.localdomain --data 'existing/alias-local' # Required for 'rejects spam': - _send_email 'email-templates/amavis-spam' + _send_email --from 'spam@external.tld' --data 'amavis/spam' # Required for 'delivers mail to existing account': - _send_email 'email-templates/existing-user1' - _send_email 'email-templates/existing-user2' - _send_email 'email-templates/existing-user3' - _send_email 'email-templates/existing-added' - _send_email 'email-templates/existing-user-and-cc-local-alias' - _send_email 'email-templates/sieve-spam-folder' - _send_email 'email-templates/sieve-pipe' - _run_in_container_bash 'sendmail root < /tmp/docker-mailserver-test/email-templates/root-email.txt' + _send_email --data 'existing/user1' + assert_success + _send_email --to user2@otherdomain.tld + assert_success + _send_email --to user3@localhost.localdomain + assert_success + _send_email --to added@localhost.localdomain --data 'existing/added' + assert_success + _send_email --to user1@localhost.localdomain --data 'existing/user-and-cc-local-alias' + assert_success + _send_email --data 'sieve/spam-folder' + assert_success + _send_email --to user2@otherdomain.tld --data 'sieve/pipe' + assert_success + _run_in_container_bash 'sendmail root < /tmp/docker-mailserver-test/emails/sendmail/root-email.txt' + assert_success +} + +function _unsuccessful() { + _send_email --port 465 --auth "${1}" --auth-user "${2}" --auth-password wrongpassword + assert_failure + assert_output --partial 'authentication failed' + assert_output --partial 'No authentication type succeeded' +} + +function _successful() { + _send_email --port 465 --auth "${1}" --auth-user "${2}" --auth-password mypassword --quit-after AUTH + assert_success + assert_output --partial 'Authentication successful' } @test "should succeed at emptying mail queue" { @@ -103,44 +124,35 @@ function setup_file() { } @test "should successfully authenticate with good password (plain)" { - _send_email 'auth/smtp-auth-plain' '-w 5 0.0.0.0 465' - assert_output --partial 'Authentication successful' + _successful PLAIN user1@localhost.localdomain } @test "should fail to authenticate with wrong password (plain)" { - _send_email 'auth/smtp-auth-plain-wrong' '-w 20 0.0.0.0 465' - assert_output --partial 'authentication failed' + _unsuccessful PLAIN user1@localhost.localdomain } @test "should successfully authenticate with good password (login)" { - _send_email 'auth/smtp-auth-login' '-w 5 0.0.0.0 465' - assert_output --partial 'Authentication successful' + _successful LOGIN user1@localhost.localdomain } @test "should fail to authenticate with wrong password (login)" { - _send_email 'auth/smtp-auth-login-wrong' '-w 20 0.0.0.0 465' - assert_output --partial 'authentication failed' + _unsuccessful LOGIN user1@localhost.localdomain } @test "[user: 'added'] should successfully authenticate with good password (plain)" { - _send_email 'auth/added-smtp-auth-plain' '-w 5 0.0.0.0 465' - assert_output --partial 'Authentication successful' + _successful PLAIN added@localhost.localdomain } @test "[user: 'added'] should fail to authenticate with wrong password (plain)" { - _send_email 'auth/added-smtp-auth-plain-wrong' '-w 20 0.0.0.0 465' - assert_output --partial 'authentication failed' + _unsuccessful PLAIN added@localhost.localdomain } @test "[user: 'added'] should successfully authenticate with good password (login)" { - _send_email 'auth/added-smtp-auth-login' '-w 5 0.0.0.0 465' - assert_success - assert_output --partial 'Authentication successful' + _successful LOGIN added@localhost.localdomain } @test "[user: 'added'] should fail to authenticate with wrong password (login)" { - _send_email 'auth/added-smtp-auth-login-wrong' '-w 20 0.0.0.0 465' - assert_output --partial 'authentication failed' + _unsuccessful LOGIN added@localhost.localdomain } # TODO: Add a test covering case SPAMASSASSIN_SPAM_TO_INBOX=1 (default) @@ -258,7 +270,13 @@ function setup_file() { # Dovecot does not support SMTPUTF8, so while we can send we cannot receive # Better disable SMTPUTF8 support entirely if we can't handle it correctly @test "not advertising smtputf8" { - _send_email 'email-templates/smtp-ehlo' + # Query supported extensions; SMTPUTF8 should not be available. + # - This query requires a EHLO greeting to the destination server. + _send_email \ + --ehlo mail.external.tld \ + --protocol ESMTP \ + --server mail.example.test \ + --quit-after FIRST-EHLO refute_output --partial 'SMTPUTF8' } diff --git a/test/tests/parallel/set3/mta/smtponly.bats b/test/tests/parallel/set3/mta/smtponly.bats index 66123de6914..7b1f8699edc 100644 --- a/test/tests/parallel/set3/mta/smtponly.bats +++ b/test/tests/parallel/set3/mta/smtponly.bats @@ -32,7 +32,16 @@ function teardown_file() { _default_teardown ; } assert_success # it looks as if someone tries to send mail to another domain outside of DMS - _send_email 'email-templates/smtp-only' + _send_email \ + --ehlo mail.origin.test \ + --protocol SSMTPA \ + --server mail.origin.test \ + --from user@origin.test \ + --to user@destination.test \ + --auth PLAIN \ + --auth-user user@origin.test \ + --auth-password secret + assert_success _wait_for_empty_mail_queue_in_container # this seemingly succeeds, but looking at the logs, it doesn't diff --git a/test/tests/serial/mail_pop3.bats b/test/tests/serial/mail_pop3.bats index cb07484a2d2..008921e49f3 100644 --- a/test/tests/serial/mail_pop3.bats +++ b/test/tests/serial/mail_pop3.bats @@ -24,11 +24,13 @@ function teardown_file() { _default_teardown ; } } @test 'authentication works' { - _send_email 'auth/pop3-auth' '-w 1 0.0.0.0 110' + _nc_wrapper 'auth/pop3-auth' '-w 1 0.0.0.0 110' + assert_success } @test 'added user authentication works' { - _send_email 'auth/added-pop3-auth' '-w 1 0.0.0.0 110' + _nc_wrapper 'auth/added-pop3-auth' '-w 1 0.0.0.0 110' + assert_success } @test '/var/log/mail/mail.log is error-free' { diff --git a/test/tests/serial/mail_with_imap.bats b/test/tests/serial/mail_with_imap.bats index d729c142657..eeccf888d0a 100644 --- a/test/tests/serial/mail_with_imap.bats +++ b/test/tests/serial/mail_with_imap.bats @@ -21,7 +21,8 @@ function setup_file() { function teardown_file() { _default_teardown ; } @test '(Dovecot) LDAP RIMAP connection and authentication works' { - _send_email 'auth/imap-auth' '-w 1 0.0.0.0 143' + _nc_wrapper 'auth/imap-auth' '-w 1 0.0.0.0 143' + assert_success } @test '(SASLauthd) SASL RIMAP authentication works' { @@ -30,13 +31,30 @@ function teardown_file() { _default_teardown ; } } @test '(SASLauthd) RIMAP SMTP authentication works' { - _send_email 'auth/smtp-auth-login' '-w 5 0.0.0.0 25' - assert_output --partial 'Error: authentication not enabled' + _send_email \ + --auth LOGIN \ + --auth-user user1@localhost.localdomain \ + --auth-password mypassword \ + --quit-after AUTH + assert_failure + assert_output --partial 'Host did not advertise authentication' - _send_email 'auth/smtp-auth-login' '-w 5 0.0.0.0 465' + _send_email \ + --port 465 \ + --auth LOGIN \ + --auth-user user1@localhost.localdomain \ + --auth-password mypassword \ + --quit-after AUTH + assert_success assert_output --partial 'Authentication successful' - _send_email 'auth/smtp-auth-login' '-w 5 0.0.0.0 587' + _send_email \ + --port 587 \ + --auth LOGIN \ + --auth-user user1@localhost.localdomain \ + --auth-password mypassword \ + --quit-after AUTH + assert_success assert_output --partial 'Authentication successful' } diff --git a/test/tests/serial/mail_with_ldap.bats b/test/tests/serial/mail_with_ldap.bats index b7b3884b767..f2011d229c3 100644 --- a/test/tests/serial/mail_with_ldap.bats +++ b/test/tests/serial/mail_with_ldap.bats @@ -122,7 +122,6 @@ function setup_file() { # Extra ENV needed to support specific test-cases: local ENV_SUPPORT=( - --env PERMIT_DOCKER=container # Required for attempting SMTP auth on port 25 via nc # Required for openssl commands to be successul: # NOTE: snakeoil cert is created (for `docker-mailserver.invalid`) via Debian post-install script for Postfix package. # TODO: Use proper TLS cert @@ -249,7 +248,7 @@ function teardown() { # dovecot @test "dovecot: ldap imap connection and authentication works" { - _run_in_container_bash 'nc -w 1 0.0.0.0 143 < /tmp/docker-mailserver-test/auth/imap-ldap-auth.txt' + _nc_wrapper 'auth/imap-ldap-auth' '-w 1 0.0.0.0 143' assert_success } @@ -327,12 +326,25 @@ function teardown() { @test "spoofing (with LDAP): rejects sender forging" { _wait_for_smtp_port_in_container_to_respond dms-test_ldap - _run_in_container_bash 'openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/ldap-smtp-auth-spoofed.txt' + _send_email \ + --port 465 -tlsc --auth LOGIN \ + --auth-user some.user@localhost.localdomain \ + --auth-password secret \ + --ehlo mail \ + --from ldap@localhost.localdomain \ + --data 'auth/ldap-smtp-auth-spoofed' assert_output --partial 'Sender address rejected: not owned by user' } @test "spoofing (with LDAP): accepts sending as alias" { - _run_in_container_bash 'openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/ldap-smtp-auth-spoofed-alias.txt' + _send_email \ + --port 465 -tlsc --auth LOGIN \ + --auth-user some.user@localhost.localdomain \ + --auth-password secret \ + --ehlo mail \ + --from postmaster@localhost.localdomain \ + --to some.user@localhost.localdomain \ + --data 'auth/ldap-smtp-auth-spoofed-alias' assert_output --partial 'End data with' } @@ -341,19 +353,42 @@ function teardown() { # Template used has invalid AUTH: https://github.com/docker-mailserver/docker-mailserver/pull/3006#discussion_r1073321432 skip 'TODO: This test seems to have been broken from the start (?)' - _run_in_container_bash 'openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt' + _send_email \ + --port 465 -tlsc --auth LOGIN \ + --auth-user some.user.email@localhost.localdomain \ + --auth-password secret \ + --ehlo mail \ + --from randomspoofedaddress@localhost.localdomain \ + --to some.user@localhost.localdomain \ + --data 'auth/ldap-smtp-auth-spoofed-sender-with-filter-exception' assert_output --partial 'Sender address rejected: not owned by user' } @test "saslauthd: ldap smtp authentication" { - # Requires ENV `PERMIT_DOCKER=container` - _send_email 'auth/sasl-ldap-smtp-auth' '-w 5 0.0.0.0 25' - assert_output --partial 'Error: authentication not enabled' - - _run_in_container_bash 'openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/sasl-ldap-smtp-auth.txt' + _send_email \ + --auth LOGIN \ + --auth-user some.user@localhost.localdomain \ + --auth-password wrongpassword \ + --quit-after AUTH + assert_failure + assert_output --partial 'Host did not advertise authentication' + + _send_email \ + --port 465 -tlsc \ + --auth LOGIN \ + --auth-user some.user@localhost.localdomain \ + --auth-password secret \ + --quit-after AUTH + assert_success assert_output --partial 'Authentication successful' - _run_in_container_bash 'openssl s_client -quiet -starttls smtp -connect 0.0.0.0:587 < /tmp/docker-mailserver-test/auth/sasl-ldap-smtp-auth.txt' + _send_email \ + --port 587 -tls \ + --auth LOGIN \ + --auth-user some.user@localhost.localdomain \ + --auth-password secret \ + --quit-after AUTH + assert_success assert_output --partial 'Authentication successful' } @@ -391,7 +426,7 @@ function _should_successfully_deliver_mail_to() { local SENDER_ADDRESS='user@external.tld' local RECIPIENT_ADDRESS=${1:?Recipient address is required} local MAIL_STORAGE_RECIPIENT=${2:?Recipient storage location is required} - local MAIL_TEMPLATE='/tmp/docker-mailserver-test/email-templates/test-email.txt' + local MAIL_TEMPLATE='/tmp/docker-mailserver-test/emails/test-email.txt' _run_in_container_bash "sendmail -f ${SENDER_ADDRESS} ${RECIPIENT_ADDRESS} < ${MAIL_TEMPLATE}" _wait_for_empty_mail_queue_in_container diff --git a/test/tests/serial/permit_docker.bats b/test/tests/serial/permit_docker.bats index 85f00484174..2ebf5e3e70d 100644 --- a/test/tests/serial/permit_docker.bats +++ b/test/tests/serial/permit_docker.bats @@ -13,7 +13,7 @@ setup_file() { PRIVATE_CONFIG=$(duplicate_config_for_container . mail_smtponly_second_network) docker create --name mail_smtponly_second_network \ -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ + -v "$(pwd)/test/files":/tmp/docker-mailserver-test:ro \ -e SMTP_ONLY=1 \ -e PERMIT_DOCKER=connected-networks \ -e OVERRIDE_HOSTNAME=mail.my-domain.com \ @@ -26,7 +26,7 @@ setup_file() { PRIVATE_CONFIG=$(duplicate_config_for_container . mail_smtponly_second_network_sender) docker run -d --name mail_smtponly_second_network_sender \ -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ + -v "$(pwd)/test/files":/tmp/docker-mailserver-test:ro \ -e SMTP_ONLY=1 \ -e PERMIT_DOCKER=connected-networks \ -e OVERRIDE_HOSTNAME=mail.my-domain.com \ @@ -39,7 +39,7 @@ setup_file() { # create another container that enforces authentication even on local connections docker run -d --name mail_smtponly_force_authentication \ -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ + -v "$(pwd)/test/files":/tmp/docker-mailserver-test:ro \ -e SMTP_ONLY=1 \ -e PERMIT_DOCKER=none \ -e OVERRIDE_HOSTNAME=mail.my-domain.com \ @@ -68,7 +68,7 @@ teardown_file() { _reload_postfix mail_smtponly_second_network # we should be able to send from the other container on the second network! - run docker exec mail_smtponly_second_network_sender /bin/sh -c "nc mail_smtponly_second_network 25 < /tmp/docker-mailserver-test/email-templates/smtp-only.txt" + run docker exec mail_smtponly_second_network_sender /bin/sh -c "nc mail_smtponly_second_network 25 < /tmp/docker-mailserver-test/emails/nc_raw/smtp-only.txt" assert_output --partial "250 2.0.0 Ok: queued as " repeat_in_container_until_success_or_timeout 60 mail_smtponly_second_network /bin/sh -c 'grep -cE "to=.*status\=sent" /var/log/mail/mail.log' } @@ -80,7 +80,7 @@ teardown_file() { _reload_postfix mail_smtponly_force_authentication # the mailserver should require authentication and a protocol error should occur when using TLS - run docker exec mail_smtponly_force_authentication /bin/sh -c "nc localhost 25 < /tmp/docker-mailserver-test/email-templates/smtp-only.txt" + run docker exec mail_smtponly_force_authentication /bin/sh -c "nc localhost 25 < /tmp/docker-mailserver-test/emails/nc_raw/smtp-only.txt" assert_output --partial "550 5.5.1 Protocol error" [[ ${status} -ge 0 ]] } diff --git a/test/tests/serial/test_helper.bats b/test/tests/serial/test_helper.bats index ecca3d85cbf..a3ffa6cfa0e 100644 --- a/test/tests/serial/test_helper.bats +++ b/test/tests/serial/test_helper.bats @@ -171,7 +171,7 @@ BATS_TEST_NAME_PREFIX='test helper functions:' # enable ClamAV to make message delivery slower, so we can detect it CONTAINER_NAME=$(docker run -d --rm \ -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ + -v "$(pwd)/test/files":/tmp/docker-mailserver-test:ro \ -e ENABLE_CLAMAV=1 \ -h mail.my-domain.com \ -t "${NAME}") @@ -186,7 +186,7 @@ BATS_TEST_NAME_PREFIX='test helper functions:' [[ ${SECONDS} -lt 5 ]] # fill the queue with a message - docker exec "${CONTAINER_NAME}" /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/amavis-virus.txt" + docker exec "${CONTAINER_NAME}" /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/emails/amavis-virus.txt" # that should still be stuck in the queue ! TEST_TIMEOUT_IN_SECONDS=0 wait_for_empty_mail_queue_in_container "${CONTAINER_NAME}" @@ -203,7 +203,7 @@ BATS_TEST_NAME_PREFIX='test helper functions:' # enable ClamAV to make message delivery slower, so we can detect it CONTAINER_NAME=$(docker run -d --rm \ -v "${PRIVATE_CONFIG}":/tmp/docker-mailserver \ - -v "$(pwd)/test/test-files":/tmp/docker-mailserver-test:ro \ + -v "$(pwd)/test/files":/tmp/docker-mailserver-test:ro \ -e ENABLE_CLAMAV=1 \ -h mail.my-domain.com \ -t "${NAME}") @@ -213,7 +213,7 @@ BATS_TEST_NAME_PREFIX='test helper functions:' wait_for_smtp_port_in_container "${CONTAINER_NAME}" || docker logs "${CONTAINER_NAME}" # fill the queue with a message - docker exec "${CONTAINER_NAME}" /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/email-templates/amavis-virus.txt" + docker exec "${CONTAINER_NAME}" /bin/sh -c "nc 0.0.0.0 25 < /tmp/docker-mailserver-test/emails/amavis-virus.txt" # give it some time to clear the queue SECONDS=0 diff --git a/test/tests/serial/tests.bats b/test/tests/serial/tests.bats index 094454f0040..26deb541612 100644 --- a/test/tests/serial/tests.bats +++ b/test/tests/serial/tests.bats @@ -80,11 +80,13 @@ function teardown_file() { _default_teardown ; } } @test "imap: authentication works" { - _send_email 'auth/imap-auth' '-w 1 0.0.0.0 143' + _nc_wrapper 'auth/imap-auth' '-w 1 0.0.0.0 143' + assert_success } @test "imap: added user authentication works" { - _send_email 'auth/added-imap-auth' '-w 1 0.0.0.0 143' + _nc_wrapper 'auth/added-imap-auth' '-w 1 0.0.0.0 143' + assert_success } # @@ -288,13 +290,34 @@ EOF @test "spoofing: rejects sender forging" { # rejection of spoofed sender _wait_for_smtp_port_in_container_to_respond - _run_in_container_bash "openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/added-smtp-auth-spoofed.txt" + + # An authenticated user cannot use an envelope sender (MAIL FROM) + # address they do not own according to `main.cf:smtpd_sender_login_maps` lookup + _send_email \ + --port 465 -tlsc --auth LOGIN \ + --auth-user added@localhost.localdomain \ + --auth-password mypassword \ + --ehlo mail \ + --from user2@localhost.localdomain \ + --data 'auth/added-smtp-auth-spoofed' assert_output --partial 'Sender address rejected: not owned by user' } @test "spoofing: accepts sending as alias" { - _run_in_container_bash "openssl s_client -quiet -connect 0.0.0.0:465 < /tmp/docker-mailserver-test/auth/added-smtp-auth-spoofed-alias.txt | grep 'End data with'" + # An authenticated account should be able to send mail from an alias, + # Verifies `main.cf:smtpd_sender_login_maps` includes /etc/postfix/virtual + # The envelope sender address (MAIL FROM) is the lookup key + # to each table. Address is authorized when a result that maps to + # the DMS account is returned. + _send_email \ + --port 465 -tlsc --auth LOGIN \ + --auth-user user1@localhost.localdomain \ + --auth-password mypassword \ + --ehlo mail \ + --from alias1@localhost.localdomain \ + --data 'auth/added-smtp-auth-spoofed-alias' assert_success + assert_output --partial 'End data with' } # diff --git a/test/tests/serial/vmail-id.bats b/test/tests/serial/vmail-id.bats index b44670b2081..0f54ea969ae 100644 --- a/test/tests/serial/vmail-id.bats +++ b/test/tests/serial/vmail-id.bats @@ -20,7 +20,7 @@ function setup_file() { function teardown_file() { _default_teardown ; } @test 'should successfully deliver mail' { - _send_email 'email-templates/existing-user1' + _send_email --data 'existing/user1' _wait_for_empty_mail_queue_in_container # Should be successfully sent (received) by Postfix: From 25c7024cc4c7a6ee81be70144f6ecaf4fddf44ca Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 3 Jan 2024 02:02:59 +0100 Subject: [PATCH 227/592] security(Postfix): Protect against "SMTP Smuggling" attack (#3727) View `CHANGELOG.md` entry and PR for details. --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 15 ++++++++++++++- target/postfix/main.cf | 6 ++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eeeb843d6e9..b6e6d9067bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,23 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v13.0.0...HEAD) +## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v13.1.0...HEAD) > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Security + +DMS is now secured against the [recently published spoofing attack "SMTP Smuggling"](https://www.postfix.org/smtp-smuggling.html) that affected Postfix ([#3727](https://github.com/docker-mailserver/docker-mailserver/pull/3727)): +- Postfix upgraded from `3.5.18` to `3.5.23` which provides the [long-term fix with `smtpd_forbid_bare_newline = yes`](https://www.postfix.org/smtp-smuggling.html#long) +- If you are unable to upgrade to this release of DMS, you may follow [these instructions](https://github.com/docker-mailserver/docker-mailserver/issues/3719#issuecomment-1870865118) for applying the [short-term workaround](https://www.postfix.org/smtp-smuggling.html#short). +- This change should not cause compatibility concerns for legitimate mail clients, however if you use software like `netcat` to send mail to DMS (_like our test-suite previously did_) it may now be rejected (_especially with the the short-term workaround `smtpd_data_restrictions = reject_unauth_pipelining`_). +- **NOTE:** This Postfix update also includes the new parameter [`smtpd_forbid_bare_newline_exclusions`](https://www.postfix.org/postconf.5.html#smtpd_forbid_bare_newline_exclusions) which defaults to `$mynetworks` for excluding trusted mail clients excluded from the restriction. + - With our default `PERMIT_DOCKER=none` this is not a concern. + - Presently the Docker daemon config has `user-proxy: true` enabled by default. + - On a host that can be reached by IPv6, this will route to a DMS IPv4 only container implicitly through the Docker network bridge gateway which rewrites the source address. + - If your `PERMIT_DOCKER` setting allows that gateway IP, then it is part of `$mynetworks` and this attack would not be prevented from such connections. + - If this affects your deployment, refer to [our IPv6 docs](https://docker-mailserver.github.io/docker-mailserver/v13.2/config/advanced/ipv6/) for advice on handling IPv6 correctly in Docker. Alternatively [use our `postfix-main.cf`](https://docker-mailserver.github.io/docker-mailserver/v13.2/config/advanced/override-defaults/postfix/) to set `smtpd_forbid_bare_newline_exclusions=` as empty. + ### Updates - The test suite now uses `swaks` instead of `nc`, which has multiple benefits ([#3732](https://github.com/docker-mailserver/docker-mailserver/pull/3732)): diff --git a/target/postfix/main.cf b/target/postfix/main.cf index 8c329c943d5..a923034721f 100644 --- a/target/postfix/main.cf +++ b/target/postfix/main.cf @@ -57,6 +57,12 @@ smtpd_sender_restrictions = $dms_smtpd_sender_restrictions smtpd_discard_ehlo_keywords = silent-discard, dsn disable_vrfy_command = yes +# Security - Prevent SMTP Smuggling attack +# https://www.postfix.org/smtp-smuggling.html#long +smtpd_forbid_bare_newline = yes +# It is possible to exclude clients on trusted networks from this restriction (the upstream default is `$mynetwork`): +# smtpd_forbid_bare_newline_exclusions = $mynetworks + # Custom defined parameters for DMS: dms_smtpd_sender_restrictions = permit_sasl_authenticated, permit_mynetworks, reject_unknown_sender_domain # Submission ports 587 and 465 support for SPOOF_PROTECTION=1 From bf69ef248ead4bc4e9304383736f52e35183b48b Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Thu, 4 Jan 2024 22:13:13 +0100 Subject: [PATCH 228/592] Postfix: add `smtpd_data_restrictions = reject_unauth_pipelining` (#3744) * add `smtpd_data_restrictions = reject_unauth_pipelining` * fix: Skip restriction if trusted * add changelog entry * revert change to `postfix-amavis.cf` * Update CHANGELOG.md --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 4 ++++ target/postfix/main.cf | 5 +++-- test/tests/parallel/set1/spam_virus/postgrey_enabled.bats | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6e6d9067bf..0c490092136 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,10 @@ DMS is now secured against the [recently published spoofing attack "SMTP Smuggli - `swaks` handles pipelining correctly, hence we can now use `reject_unauth_pipelining` in Postfix's configuration. - `swaks` provides better CLI options that make many files superflous. - `swaks` can also replace `openssl s_client` and handles authentication on submission ports better. +- **Postfix:** + - We now defer rejection from unauthorized pipelining until the SMTP `DATA` command via `smtpd_data_restrictions` (_i.e. at the end of the mail transfer transaction_) ([#3744](https://github.com/docker-mailserver/docker-mailserver/pull/3744)) + - Prevously our configuration only handled this during the client and recipient restriction stages. Postfix will flag this activity when encountered, but the rejection now is handled at `DATA` where unauthorized pipelining would have been valid from this point. + - If you had the Amavis service enabled (default), this restriction was already in place. Otherwise the concerns expressed with `smtpd_data_restrictions = reject_unauth_pipelining` from the security section above apply. We have permitted trusted clients (_`$mynetworks` or authenticated_) to bypass this restriction. ## [v13.1.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.1.0) diff --git a/target/postfix/main.cf b/target/postfix/main.cf index a923034721f..1dc7bdbc5d2 100644 --- a/target/postfix/main.cf +++ b/target/postfix/main.cf @@ -51,10 +51,11 @@ smtpd_helo_required = yes smtpd_delay_reject = yes smtpd_helo_restrictions = permit_mynetworks, reject_invalid_helo_hostname, permit smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination -smtpd_recipient_restrictions = permit_sasl_authenticated, permit_mynetworks, reject_unauth_destination, reject_unauth_pipelining, reject_invalid_helo_hostname, reject_non_fqdn_helo_hostname, reject_unknown_recipient_domain -smtpd_client_restrictions = permit_mynetworks, permit_sasl_authenticated, reject_unauth_destination, reject_unauth_pipelining +smtpd_recipient_restrictions = permit_sasl_authenticated, permit_mynetworks, reject_unauth_destination, reject_invalid_helo_hostname, reject_non_fqdn_helo_hostname, reject_unknown_recipient_domain +smtpd_client_restrictions = permit_mynetworks, permit_sasl_authenticated, reject_unauth_destination smtpd_sender_restrictions = $dms_smtpd_sender_restrictions smtpd_discard_ehlo_keywords = silent-discard, dsn +smtpd_data_restrictions = permit_mynetworks, permit_sasl_authenticated, reject_unauth_pipelining disable_vrfy_command = yes # Security - Prevent SMTP Smuggling attack diff --git a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats index 316e33500da..389fc183a42 100644 --- a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats +++ b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats @@ -44,7 +44,7 @@ function teardown_file() { _default_teardown ; } # The other spam checks in `main.cf:smtpd_recipient_restrictions` would interfere with testing postgrey. _run_in_container sed -i \ -e 's/permit_sasl_authenticated.*policyd-spf,$//g' \ - -e 's/reject_unauth_pipelining.*reject_unknown_recipient_domain,$//g' \ + -e 's/reject_invalid_helo_hostname.*reject_unknown_recipient_domain,$//g' \ -e 's/reject_rbl_client.*inet:127\.0\.0\.1:10023$//g' \ -e 's/smtpd_recipient_restrictions =/smtpd_recipient_restrictions = check_policy_service inet:127.0.0.1:10023/g' \ /etc/postfix/main.cf From 8f391e4d5aa6449fcf8a4762df23cba98c52bc4c Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Thu, 4 Jan 2024 22:14:51 +0100 Subject: [PATCH 229/592] release: v13.2.0 (#3746) * bump `VERSION` & adjust `CHANGELOG.md` * chore: Bump the unreleased base compare tag in `CHANGELOG.md` --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 5 ++++- VERSION | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c490092136..842b60c1c9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,13 +2,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v13.1.0...HEAD) +## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v13.2.0...HEAD) > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +## [v13.2.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.2.0) + ### Security DMS is now secured against the [recently published spoofing attack "SMTP Smuggling"](https://www.postfix.org/smtp-smuggling.html) that affected Postfix ([#3727](https://github.com/docker-mailserver/docker-mailserver/pull/3727)): + - Postfix upgraded from `3.5.18` to `3.5.23` which provides the [long-term fix with `smtpd_forbid_bare_newline = yes`](https://www.postfix.org/smtp-smuggling.html#long) - If you are unable to upgrade to this release of DMS, you may follow [these instructions](https://github.com/docker-mailserver/docker-mailserver/issues/3719#issuecomment-1870865118) for applying the [short-term workaround](https://www.postfix.org/smtp-smuggling.html#short). - This change should not cause compatibility concerns for legitimate mail clients, however if you use software like `netcat` to send mail to DMS (_like our test-suite previously did_) it may now be rejected (_especially with the the short-term workaround `smtpd_data_restrictions = reject_unauth_pipelining`_). diff --git a/VERSION b/VERSION index e6ba351366d..67aee23940e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -13.1.0 +13.2.0 From 04f4ae4569a2b5dcaffb510cd09ef1fb6ba038a2 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 5 Jan 2024 09:07:31 +0100 Subject: [PATCH 230/592] Rspamd: add custom symbol scores for SPF, DKIM & DMARC (#3726) --- CHANGELOG.md | 5 + Dockerfile | 1 + target/rspamd/local.d/actions.conf | 11 ++- target/rspamd/scores.d/policies_group.conf | 108 +++++++++++++++++++++ 4 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 target/rspamd/scores.d/policies_group.conf diff --git a/CHANGELOG.md b/CHANGELOG.md index 842b60c1c9f..bccbe0e9034 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Updates + +- **Rspamd** ([#3726](https://github.com/docker-mailserver/docker-mailserver/pull/3726)): + - symbol scores for SPF, DKIM & DMARC were updated to more closely align with [RFC7489](https://www.rfc-editor.org/rfc/rfc7489#page-24); please note though that complete alignment is undesirable, because other symbols might be added as well, which changes the overall score calculation again, see [this issue](https://github.com/docker-mailserver/docker-mailserver/issues/3690#issuecomment-1866871996) + ## [v13.2.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.2.0) ### Security diff --git a/Dockerfile b/Dockerfile index 4d0e3568868..f9802c2aef5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -106,6 +106,7 @@ EOF # ----------------------------------------------- COPY target/rspamd/local.d/ /etc/rspamd/local.d/ +COPY target/rspamd/scores.d/* /etc/rspamd/scores.d/ # ----------------------------------------------- # --- LDAP & SpamAssassin's Cron ---------------- diff --git a/target/rspamd/local.d/actions.conf b/target/rspamd/local.d/actions.conf index b214c33987e..fb4c15b9bce 100644 --- a/target/rspamd/local.d/actions.conf +++ b/target/rspamd/local.d/actions.conf @@ -1,9 +1,12 @@ # documentation: https://rspamd.com/doc/configuration/metrics.html#actions # and https://rspamd.com/doc/configuration/metrics.html -#greylist = 4; -#add_header = 6; -#rewrite_subject = 7; -#reject = 15; +# These values work in conjunction with the symbol scores in +# `scores.d/*.conf`. When adjusting them, make sure to understand +# and to be able to explain the impact on the whole system. +greylist = 4; +add_header = 6; +rewrite_subject = 7; +reject = 11; subject = "***SPAM*** %s" diff --git a/target/rspamd/scores.d/policies_group.conf b/target/rspamd/scores.d/policies_group.conf new file mode 100644 index 00000000000..5f9426e99fc --- /dev/null +++ b/target/rspamd/scores.d/policies_group.conf @@ -0,0 +1,108 @@ +# Please refer to +# https://github.com/docker-mailserver/docker-mailserver/issues/3690 +# for understanding this file and its scores' values. + +symbols = { + # SPF + "R_SPF_ALLOW" { + weight = -1; + description = "SPF verification allows sending"; + groups = ["spf"]; + } + "R_SPF_NA" { + weight = 1.5; + description = "Missing SPF record"; + one_shot = true; + groups = ["spf"]; + } + "R_SPF_SOFTFAIL" { + weight = 2.5; + description = "SPF verification soft-failed"; + groups = ["spf"]; + } + "R_SPF_FAIL" { + weight = 4.5; + description = "SPF verification failed"; + groups = ["spf"]; + } + + "R_SPF_NEUTRAL" { # == R_SPF_NA + weight = 1.5; + description = "SPF policy is neutral"; + groups = ["spf"]; + } + "R_SPF_DNSFAIL" { # == R_SPF_SOFTFAIL + weight = 2.5; + description = "SPF DNS failure"; + groups = ["spf"]; + } + "R_SPF_PERMFAIL" { # == R_SPF_FAIL + weight = 4.5; + description = "SPF record is malformed or persistent DNS error"; + groups = ["spf"]; + } + + # DKIM + "R_DKIM_ALLOW" { + weight = -1; + description = "DKIM verification succeed"; + one_shot = true; + groups = ["dkim"]; + } + "R_DKIM_NA" { + weight = 0; + description = "Missing DKIM signature"; + one_shot = true; + groups = ["dkim"]; + } + "R_DKIM_TEMPFAIL" { + weight = 1.5; + description = "DKIM verification soft-failed"; + groups = ["dkim"]; + } + "R_DKIM_PERMFAIL" { + weight = 4.5; + description = "DKIM verification hard-failed (invalid)"; + groups = ["dkim"]; + } + + "R_DKIM_REJECT" { # == R_DKIM_PERMFAIL + weight = 4.5; + description = "DKIM verification failed"; + one_shot = true; + groups = ["dkim"]; + } + + # DMARC + "DMARC_NA" { + weight = 1; + description = "No DMARC record"; + groups = ["dmarc"]; + } + "DMARC_POLICY_QUARANTINE" { + weight = 1.5; + description = "DMARC quarantine policy"; + groups = ["dmarc"]; + } + "DMARC_POLICY_REJECT" { + weight = 2; + description = "DMARC reject policy"; + groups = ["dmarc"]; + } + + "DMARC_POLICY_ALLOW" { # no equivalent + weight = -1; + description = "DMARC permit policy"; + groups = ["dmarc"]; + } + "DMARC_POLICY_ALLOW_WITH_FAILURES" { # no equivalent + weight = -0.5; + description = "DMARC permit policy with DKIM/SPF failure"; + groups = ["dmarc"]; + } + "DMARC_POLICY_SOFTFAIL" { # == DMARC_POLICY_QUARANTINE + weight = 1.5; + description = "DMARC soft-failed"; + groups = ["dmarc"]; + } +} From 6082d5f8d04c353dc1f9de723f3acbdfb4a8fa32 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 6 Jan 2024 11:18:30 +1300 Subject: [PATCH 231/592] chore: Disable `smtputf8` support in config directly (#3750) * chore: Disable `smtputf8` support in config This was always configured disabled at runtime, better to just set explicitly in `main.cf` unless config diverges when Dovecot is enabled to opt-out of this feature. --- CHANGELOG.md | 2 ++ target/postfix/main.cf | 3 +++ target/scripts/startup/setup.d/postfix.sh | 3 --- test/tests/parallel/set3/mta/smtp_delivery.bats | 4 ++++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bccbe0e9034..5f229530158 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ All notable changes to this project will be documented in this file. The format ### Updates +- **Internal:** + - Postfix is now configured with `smtputf8_enable = no` in our default `main.cf` config (_instead of during container startup_). ([#3750](https://github.com/docker-mailserver/docker-mailserver/pull/3750)) - **Rspamd** ([#3726](https://github.com/docker-mailserver/docker-mailserver/pull/3726)): - symbol scores for SPF, DKIM & DMARC were updated to more closely align with [RFC7489](https://www.rfc-editor.org/rfc/rfc7489#page-24); please note though that complete alignment is undesirable, because other symbols might be added as well, which changes the overall score calculation again, see [this issue](https://github.com/docker-mailserver/docker-mailserver/issues/3690#issuecomment-1866871996) diff --git a/target/postfix/main.cf b/target/postfix/main.cf index 1dc7bdbc5d2..495ad8a94fe 100644 --- a/target/postfix/main.cf +++ b/target/postfix/main.cf @@ -5,6 +5,9 @@ biff = no append_dot_mydomain = no readme_directory = no +# Disabled as not compatible with Dovecot +smtputf8_enable = no + # Basic configuration # myhostname = alias_maps = hash:/etc/aliases diff --git a/target/scripts/startup/setup.d/postfix.sh b/target/scripts/startup/setup.d/postfix.sh index 58998376a79..5aec86365e6 100644 --- a/target/scripts/startup/setup.d/postfix.sh +++ b/target/scripts/startup/setup.d/postfix.sh @@ -19,9 +19,6 @@ function _setup_postfix_early() { postconf "inet_protocols = ${POSTFIX_INET_PROTOCOLS}" fi - __postfix__log 'trace' "Disabling SMTPUTF8 support" - postconf 'smtputf8_enable = no' - __postfix__log 'trace' "Configuring SASLauthd" if [[ ${ENABLE_SASLAUTHD} -eq 1 ]] && [[ ! -f /etc/postfix/sasl/smtpd.conf ]]; then cat >/etc/postfix/sasl/smtpd.conf << EOF diff --git a/test/tests/parallel/set3/mta/smtp_delivery.bats b/test/tests/parallel/set3/mta/smtp_delivery.bats index 169f374ad3d..f87f11ed4a1 100644 --- a/test/tests/parallel/set3/mta/smtp_delivery.bats +++ b/test/tests/parallel/set3/mta/smtp_delivery.bats @@ -277,6 +277,10 @@ function _successful() { --protocol ESMTP \ --server mail.example.test \ --quit-after FIRST-EHLO + + # Ensure the output is actually related to what we want to refute against: + assert_output --partial 'EHLO mail.external.tld' + assert_output --partial '221 2.0.0 Bye' refute_output --partial 'SMTPUTF8' } From 0eb4ac771498e1eff91944e79c19c8605995bc07 Mon Sep 17 00:00:00 2001 From: Casper Date: Sat, 6 Jan 2024 17:07:00 +0100 Subject: [PATCH 232/592] tests: Replace `wc -l` with `grep -c` (#3752) --- CHANGELOG.md | 1 + .../parallel/set1/spam_virus/clamav.bats | 2 +- .../parallel/set3/scripts/setup_cli.bats | 20 +++++++++---------- test/tests/serial/tests.bats | 2 +- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f229530158..d8f8a0d040a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ All notable changes to this project will be documented in this file. The format ### Updates - **Internal:** + - tests: Replace `wc -l` with `grep -c` ([#3752](https://github.com/docker-mailserver/docker-mailserver/pull/3752)) - Postfix is now configured with `smtputf8_enable = no` in our default `main.cf` config (_instead of during container startup_). ([#3750](https://github.com/docker-mailserver/docker-mailserver/pull/3750)) - **Rspamd** ([#3726](https://github.com/docker-mailserver/docker-mailserver/pull/3726)): - symbol scores for SPF, DKIM & DMARC were updated to more closely align with [RFC7489](https://www.rfc-editor.org/rfc/rfc7489#page-24); please note though that complete alignment is undesirable, because other symbols might be added as well, which changes the overall score calculation again, see [this issue](https://github.com/docker-mailserver/docker-mailserver/issues/3690#issuecomment-1866871996) diff --git a/test/tests/parallel/set1/spam_virus/clamav.bats b/test/tests/parallel/set1/spam_virus/clamav.bats index 9232f90fa74..9c035f5b690 100644 --- a/test/tests/parallel/set1/spam_virus/clamav.bats +++ b/test/tests/parallel/set1/spam_virus/clamav.bats @@ -33,7 +33,7 @@ function setup_file() { function teardown_file() { _default_teardown ; } @test 'log files exist at /var/log/mail directory' { - _run_in_container_bash "ls -1 /var/log/mail/ | grep -E 'clamav|freshclam|mail.log' | wc -l" + _run_in_container_bash "ls -1 /var/log/mail/ | grep -c -E 'clamav|freshclam|mail.log'" assert_success assert_output 3 } diff --git a/test/tests/parallel/set3/scripts/setup_cli.bats b/test/tests/parallel/set3/scripts/setup_cli.bats index dca613586a7..76f16dcefe6 100644 --- a/test/tests/parallel/set3/scripts/setup_cli.bats +++ b/test/tests/parallel/set3/scripts/setup_cli.bats @@ -204,12 +204,12 @@ function teardown_file() { _default_teardown ; } run ./setup.sh -c "${CONTAINER_NAME}" quota set quota_user2 51M assert_failure - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -E '^quota_user@example.com\:12M\$' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -c -E '^quota_user@example.com\:12M\$' | grep 1" assert_success run ./setup.sh -c "${CONTAINER_NAME}" quota set quota_user@example.com 26M assert_success - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -E '^quota_user@example.com\:26M\$' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -c -E '^quota_user@example.com\:26M\$' | grep 1" assert_success run grep "quota_user2@example.com" "${TEST_TMP_CONFIG}/dovecot-quotas.cf" @@ -220,12 +220,12 @@ function teardown_file() { _default_teardown ; } @test "delquota" { run ./setup.sh -c "${CONTAINER_NAME}" quota set quota_user@example.com 12M assert_success - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -E '^quota_user@example.com\:12M\$' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -c -E '^quota_user@example.com\:12M\$' | grep 1" assert_success run ./setup.sh -c "${CONTAINER_NAME}" quota del unknown@domain.com assert_failure - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -E '^quota_user@example.com\:12M\$' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/dovecot-quotas.cf | grep -c -E '^quota_user@example.com\:12M\$' | grep 1" assert_success run ./setup.sh -c "${CONTAINER_NAME}" quota del quota_user@example.com @@ -260,13 +260,13 @@ function teardown_file() { _default_teardown ; } ./setup.sh -c "${CONTAINER_NAME}" relay add-domain example3.org smtp.relay.com 587 # check adding - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -e '^@example1.org\s\+\[smtp.relay1.com\]:2525' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -c -e '^@example1.org\s\+\[smtp.relay1.com\]:2525' | grep 1" assert_success # test default port - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -e '^@example2.org\s\+\[smtp.relay2.com\]:25' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -c -e '^@example2.org\s\+\[smtp.relay2.com\]:25' | grep 1" assert_success # test modifying - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -e '^@example3.org\s\+\[smtp.relay.com\]:587' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -c -e '^@example3.org\s\+\[smtp.relay.com\]:587' | grep 1" assert_success } @@ -276,16 +276,16 @@ function teardown_file() { _default_teardown ; } ./setup.sh -c "${CONTAINER_NAME}" relay add-auth example2.org smtp_user2 smtp_pass_new # test adding - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-sasl-password.cf | grep -e '^@example.org\s\+smtp_user:smtp_pass' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-sasl-password.cf | grep -c -e '^@example.org\s\+smtp_user:smtp_pass' | grep 1" assert_success # test updating - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-sasl-password.cf | grep -e '^@example2.org\s\+smtp_user2:smtp_pass_new' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-sasl-password.cf | grep -c -e '^@example2.org\s\+smtp_user2:smtp_pass_new' | grep 1" assert_success } @test "relay exclude-domain" { ./setup.sh -c "${CONTAINER_NAME}" relay exclude-domain example.org - run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -e '^@example.org\s*$' | wc -l | grep 1" + run /bin/sh -c "cat ${TEST_TMP_CONFIG}/postfix-relaymap.cf | grep -c -e '^@example.org\s*$' | grep 1" assert_success } diff --git a/test/tests/serial/tests.bats b/test/tests/serial/tests.bats index 26deb541612..752e325e487 100644 --- a/test/tests/serial/tests.bats +++ b/test/tests/serial/tests.bats @@ -264,7 +264,7 @@ EOF # @test "amavis: config overrides" { - _run_in_container_bash "grep 'Test Verification' /etc/amavis/conf.d/50-user | wc -l" + _run_in_container_bash "grep -c 'Test Verification' /etc/amavis/conf.d/50-user" assert_success assert_output 1 } From 6d666512c1c2f8307b5d851d066a38eeb367d306 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Mon, 8 Jan 2024 09:34:24 +1300 Subject: [PATCH 233/592] ci: `.gitattributes` - Ensure `eol=lf` for shell scripts (#3755) * ci: `.gitattributes` - Ensure `eol=lf` for shell scripts - These files should always use LF for line endings during a checkout. - `Dockerfile` does not like building with HereDoc `RUN` scripts that expect LF. --- .gitattributes | 7 +++---- CHANGELOG.md | 5 +++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.gitattributes b/.gitattributes index 869c153e500..da5dadda4f2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -20,9 +20,8 @@ ## BUILD: .dockerignore text -Dockerfile text +Dockerfile text eol=lf Makefile -VERSION ## EXAMPLE (RUNTIME): *.env text @@ -75,8 +74,8 @@ target/postsrsd/** text ################################################# ## BATS -*.bash text -*.bats text +*.bash text eol=lf +*.bats text eol=lf ## CONFIG (test/config/) ### OpenLDAP image diff --git a/CHANGELOG.md b/CHANGELOG.md index d8f8a0d040a..39a0ab661e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,11 @@ All notable changes to this project will be documented in this file. The format - **Rspamd** ([#3726](https://github.com/docker-mailserver/docker-mailserver/pull/3726)): - symbol scores for SPF, DKIM & DMARC were updated to more closely align with [RFC7489](https://www.rfc-editor.org/rfc/rfc7489#page-24); please note though that complete alignment is undesirable, because other symbols might be added as well, which changes the overall score calculation again, see [this issue](https://github.com/docker-mailserver/docker-mailserver/issues/3690#issuecomment-1866871996) +### Fixes + +- **Internal:** + - `.gitattributes`: Always use LF line endings on checkout for files with shell script content ([#3755](https://github.com/docker-mailserver/docker-mailserver/pull/3755)) + ## [v13.2.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.2.0) ### Security From 8e2017320092a2db2cb56ae7255271774e0470c1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 8 Jan 2024 09:40:01 +1300 Subject: [PATCH 234/592] docs: updated `CONTRIBUTORS.md` (#3757) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 1470 +++++++++++++++++++++++------------------------ 1 file changed, 735 insertions(+), 735 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 4103ad5f549..05a4e6e4dc1 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -56,17 +56,17 @@ Thanks goes to these wonderful people ✨ - - erik-wramner + + polarathene
- erik-wramner + polarathene - - polarathene + + erik-wramner
- polarathene + erik-wramner @@ -163,17 +163,17 @@ Thanks goes to these wonderful people ✨ - - swiesend + + svenyonson
- swiesend + svenyonson - - svenyonson + + swiesend
- svenyonson + swiesend @@ -213,10 +213,10 @@ Thanks goes to these wonderful people ✨ - - KyleOndy + + mindrunner
- KyleOndy + mindrunner @@ -228,24 +228,24 @@ Thanks goes to these wonderful people ✨ - - mindrunner + + KyleOndy
- mindrunner + KyleOndy - - m-a-v + + bilak
- m-a-v + bilak - - bilak + + m-a-v
- bilak + m-a-v @@ -278,10 +278,46 @@ Thanks goes to these wonderful people ✨ - - dashohoxha + + moqmar
- dashohoxha + moqmar + + + + + pyy +
+ pyy + + + + + dennis95stumm +
+ dennis95stumm + + + + + arneke +
+ arneke + + + + + + akmet +
+ akmet + + + + + diiigle +
+ diiigle @@ -298,6 +334,13 @@ Thanks goes to these wonderful people ✨ mathuin + + + dashohoxha +
+ dashohoxha + + jamebus @@ -335,46 +378,68 @@ Thanks goes to these wonderful people ✨ - - diiigle + + m-schmoock
- diiigle + m-schmoock - - akmet + + mjung
- akmet + mjung - - arneke + + VanVan
- arneke + VanVan - - dennis95stumm + + andreasgerstmayr
- dennis95stumm + andreasgerstmayr - - moqmar + + davidszp
- moqmar + davidszp - - pyy + + kamuri
- pyy + kamuri + + + + + guardiande +
+ guardiande + + + + + Zehir +
+ Zehir + + + + + + weo +
+ weo @@ -390,8 +455,7 @@ Thanks goes to these wonderful people ✨
Birkenstab - - + BrandonSchmitt @@ -412,7 +476,8 @@ Thanks goes to these wonderful people ✨
citec - + + yajo @@ -420,6 +485,13 @@ Thanks goes to these wonderful people ✨ yajo + + + analogue +
+ analogue + + MakerMatrix @@ -427,6 +499,20 @@ Thanks goes to these wonderful people ✨ MakerMatrix + + + Rubytastic2 +
+ Rubytastic2 + + + + + reneploetz +
+ reneploetz + + jsonn @@ -443,117 +529,132 @@ Thanks goes to these wonderful people ✨ - - reneploetz + + keslerm
- reneploetz + keslerm - - Rubytastic2 + + castorinop
- Rubytastic2 + castorinop - - analogue + + p-fruck
- analogue + p-fruck - - weo + + Rillke
- weo + Rillke - - Zehir + + bobbravo2
- Zehir + bobbravo2 - - guardiande + + r-pufky
- guardiande + r-pufky - - kamuri + + vincentDcmps
- kamuri + vincentDcmps - - davidszp + + andymel123
- davidszp + andymel123 - - andreasgerstmayr + + bigpigeon
- andreasgerstmayr + bigpigeon - - mjung + + engelant
- mjung + engelant - - m-schmoock + + j-marz
- m-schmoock + j-marz - - VanVan + + lokipo
- VanVan + lokipo - - elbracht + + msheakoski
- elbracht + msheakoski - - aminvakil + + GoliathLabs
- aminvakil + GoliathLabs - - andrewlow + + tbutter
- andrewlow + tbutter - - abh + + yogo1212
- abh + yogo1212 + + + + + mpanneck +
+ mpanneck + + + + + + willtho89 +
+ willtho89 @@ -562,6 +663,34 @@ Thanks goes to these wonderful people ✨
ubenmackin + + + + abh +
+ abh + + + + + andrewlow +
+ andrewlow + + + + + aminvakil +
+ aminvakil + + + + + elbracht +
+ elbracht + @@ -608,38 +737,38 @@ Thanks goes to these wonderful people ✨ - - fl42 + + nueaf
- fl42 + nueaf - - ipernet + + martinwepner
- ipernet + martinwepner - - H4R0 + + artonge
- H4R0 + artonge - - eltociear + + spacecowboy
- eltociear + spacecowboy - - jamesfryer + + jedateach
- jamesfryer + jedateach @@ -651,167 +780,167 @@ Thanks goes to these wonderful people ✨ - - jedateach + + fl42
- jedateach + fl42 - - spacecowboy + + H4R0
- spacecowboy + H4R0 - - artonge + + ipernet
- artonge + ipernet - - martinwepner + + jamesfryer
- martinwepner + jamesfryer - - nueaf + + eltociear
- nueaf + eltociear - - keslerm + + stigok
- keslerm + stigok - - castorinop + + 5ven
- castorinop + 5ven - - p-fruck + + syl20bnr
- p-fruck + syl20bnr - - Rillke + + sylvaindumont
- Rillke + sylvaindumont - - bobbravo2 + + TechnicLab
- bobbravo2 + TechnicLab - - r-pufky + + thomasschmit
- r-pufky + thomasschmit - - vincentDcmps + + Thiritin
- vincentDcmps + Thiritin - - andymel123 + + tweibert
- andymel123 + tweibert - - bigpigeon + + torus
- bigpigeon + torus - - engelant + + VictorKoenders
- engelant + VictorKoenders - - j-marz + + Twist235
- j-marz + Twist235 - - lokipo + + k3it
- lokipo + k3it - - msheakoski + + Drakulix
- msheakoski + Drakulix - - GoliathLabs + + vilisas
- GoliathLabs + vilisas - - yogo1212 + + 42wim
- yogo1212 + 42wim - - mpanneck + + ShiriNmi1520
- mpanneck + ShiriNmi1520 - - willtho89 + + Zepmann
- willtho89 + Zepmann - - tbutter + + allddd
- tbutter + allddd @@ -823,1149 +952,1020 @@ Thanks goes to these wonderful people ✨ - - ifokeev + + nknapp
- ifokeev + nknapp - - 20th + + pcqnt
- 20th + pcqnt - - 2b + + OrvilleQ
- 2b + OrvilleQ - - askz + + ovidiucp
- askz + ovidiucp - - acch + + mrPjer
- acch + mrPjer - - vifino + + p3dda
- vifino + p3dda - - kachkaev + + peter-hartmann
- kachkaev + peter-hartmann - - alexanderneu + + piwai
- alexanderneu + piwai - - ch3sh1r + + remoe
- ch3sh1r + remoe - - eglia + + romansey
- eglia + romansey - - groupmsl + + MightySCollins
- groupmsl + MightySCollins - - green-anger + + 501st-alpha1
- green-anger + 501st-alpha1 - - iRhonin + + klamann
- iRhonin + klamann - - MrFreezeex + + svdb0
- MrFreezeex + svdb0 - - arunvc + + 3ap
- arunvc + 3ap - - astrocket + + shyim
- astrocket + shyim - - baxerus + + sjmudd
- baxerus + sjmudd - - spock + + simonsystem
- spock + simonsystem - - erdos4d + + stephan-devop
- erdos4d + stephan-devop - - crash7 + + millerjason
- crash7 + millerjason - - auchri + + mplx
- auchri + mplx - - arkanovicz + + odinis
- arkanovicz + odinis - - CBeerta + + okamidash
- CBeerta + okamidash - - damianmoore + + olaf-mandel
- damianmoore + olaf-mandel - - espitall + + ontheair81
- espitall + ontheair81 - - dkarski + + pravynandas
- dkarski + pravynandas - - dbellavista + + presocratics
- dbellavista + presocratics - - danielvandenberg95 + + rhyst
- danielvandenberg95 + rhyst - - mlatorre31 + + rmlhuk
- mlatorre31 + rmlhuk - - mazzz1y + + rriski
- mazzz1y + rriski - - aydodo + + schnippl0r
- aydodo + schnippl0r - - vedtam + + smargold476
- vedtam + smargold476 - - edvorg + + sportshead
- edvorg + sportshead - - eliroca + + squash
- eliroca + squash - - ekkis + + strarsis
- ekkis + strarsis - - ErikEngerd + + tamueller
- ErikEngerd + tamueller - - huncode + + vivacarvajalito
- huncode + vivacarvajalito - - felixn + + wligtenberg
- felixn + wligtenberg - - flole + + wolkenschieber
- flole + wolkenschieber - - froks + + worldworm
- froks + worldworm - - fkefer + + arcaine2
- fkefer + arcaine2 - - frugan-dev + + awb99
- frugan-dev + awb99 - - Marsu31 + + brainkiller
- Marsu31 + brainkiller - - glandais + + cternes
- glandais + cternes - - GiovanH + + dborowy
- GiovanH + dborowy - - harryyoud + + dimalo
- harryyoud + dimalo - - HeySora -
- HeySora - - - - - sirgantrithon -
- sirgantrithon - - - - - - Influencer -
- Influencer - - - - - jcalfee -
- jcalfee - - - - - mivek -
- mivek - - - - - init-js -
- init-js - - - - - Jeidnx -
- Jeidnx - - - - - JiLleON -
- JiLleON - - - - - - jirislav -
- jirislav - - - - - jmccl -
- jmccl - - - - - jurekbarth -
- jurekbarth - - - - - JOduMonT -
- JOduMonT - - - - - Kaan88 -
- Kaan88 - - - - - akkumar -
- akkumar - - - - - - KCrawley -
- KCrawley - - - - - khuedoan -
- khuedoan - - - - - JustAnother1 -
- JustAnother1 - - - - - LeoWinterDE -
- LeoWinterDE - - - - - linhandev + + eleith
- linhandev + eleith - - luke- + + ghnp5
- luke- + ghnp5 - - LucidityCrash + + helmutundarnold
- LucidityCrash + helmutundarnold - - MadsRC + + hnws
- MadsRC + hnws - - madmath03 + + i-C-o-d-e-r
- madmath03 + i-C-o-d-e-r - - maxemann96 + + idaadi
- maxemann96 + idaadi - - dragetd + + ixeft
- dragetd + ixeft - - michaeljensen + + jjtt
- michaeljensen + jjtt - - exhuma + + paralax
- exhuma + paralax - - milas + + jpduyx
- milas + jpduyx - - mcchots + + landergate
- mcchots + landergate - - MohammedNoureldin + + callmemagnus
- MohammedNoureldin + callmemagnus - - mpldr + + marios88
- mpldr + marios88 - - naveensrinivasan + + matrixes
- naveensrinivasan + matrixes - - neuralp + + mchamplain
- neuralp + mchamplain - - radicand + + auchri
- radicand + auchri - - nilshoell + + arkanovicz
- nilshoell + arkanovicz - - nknapp + + CBeerta
- nknapp + CBeerta - - pcqnt + + damianmoore
- pcqnt + damianmoore - - OrvilleQ + + espitall
- OrvilleQ + espitall - - ovidiucp + + dkarski
- ovidiucp + dkarski - - mrPjer + + dbellavista
- mrPjer + dbellavista - - p3dda + + danielvandenberg95
- p3dda + danielvandenberg95 - - peter-hartmann + + mlatorre31
- peter-hartmann + mlatorre31 - - piwai + + mazzz1y
- piwai + mazzz1y - - remoe + + aydodo
- remoe + aydodo - - romansey + + vedtam
- romansey + vedtam - - MightySCollins + + edvorg
- MightySCollins + edvorg - - 501st-alpha1 + + eliroca
- 501st-alpha1 + eliroca - - klamann + + ekkis
- klamann + ekkis - - svdb0 + + ErikEngerd
- svdb0 + ErikEngerd - - 3ap + + huncode
- 3ap + huncode - - shyim + + felixn
- shyim + felixn - - sjmudd + + flole
- sjmudd + flole - - simonsystem + + froks
- simonsystem + froks - - stephan-devop + + ifokeev
- stephan-devop + ifokeev - - stigok + + 20th
- stigok + 20th - - 5ven + + 2b
- 5ven + 2b - - syl20bnr + + askz
- syl20bnr + askz - - sylvaindumont + + acch
- sylvaindumont + acch - - TechnicLab + + vifino
- TechnicLab + vifino - - thomasschmit + + kachkaev
- thomasschmit + kachkaev - - Thiritin + + alexanderneu
- Thiritin + alexanderneu - - tweibert + + ch3sh1r
- tweibert + ch3sh1r - - torus + + eglia
- torus + eglia - - VictorKoenders + + groupmsl
- VictorKoenders + groupmsl - - Twist235 + + green-anger
- Twist235 + green-anger - - k3it + + iRhonin
- k3it + iRhonin - - Drakulix + + MrFreezeex
- Drakulix + MrFreezeex - - vilisas + + arunvc
- vilisas + arunvc - - 42wim + + astrocket
- 42wim + astrocket - - ShiriNmi1520 + + baxerus
- ShiriNmi1520 + baxerus - - Zepmann + + spock
- Zepmann + spock - - allddd + + erdos4d
- allddd + erdos4d - - arcaine2 + + crash7
- arcaine2 + crash7 - - awb99 + + fkefer
- awb99 + fkefer - - brainkiller + + khuedoan
- brainkiller + khuedoan - - cternes + + JustAnother1
- cternes + JustAnother1 - - dborowy + + LeoWinterDE
- dborowy + LeoWinterDE - - dimalo + + linhandev
- dimalo + linhandev - - eleith + + luke-
- eleith + luke- - - ghnp5 + + LucidityCrash
- ghnp5 + LucidityCrash - - helmutundarnold + + MadsRC
- helmutundarnold + MadsRC - - hnws + + madmath03
- hnws + madmath03 - - i-C-o-d-e-r + + maxemann96
- i-C-o-d-e-r + maxemann96 - - idaadi + + dragetd
- idaadi + dragetd - - ixeft + + michaeljensen
- ixeft + michaeljensen - - jjtt + + exhuma
- jjtt + exhuma - - paralax + + milas
- paralax + milas - - jpduyx + + mcchots
- jpduyx + mcchots - - landergate + + MohammedNoureldin
- landergate + MohammedNoureldin - - callmemagnus + + mpldr
- callmemagnus + mpldr - - marios88 + + naveensrinivasan
- marios88 + naveensrinivasan - - matrixes + + neuralp
- matrixes + neuralp - - mchamplain + + radicand
- mchamplain + radicand - - millerjason + + nilshoell
- millerjason + nilshoell - - mplx + + frugan-dev
- mplx + frugan-dev - - odinis + + Marsu31
- odinis + Marsu31 - - okamidash + + glandais
- okamidash + glandais - - olaf-mandel + + GiovanH
- olaf-mandel + GiovanH - - ontheair81 + + harryyoud
- ontheair81 + harryyoud - - pravynandas + + HeySora
- pravynandas + HeySora - - presocratics + + sirgantrithon
- presocratics + sirgantrithon - - rhyst + + Influencer
- rhyst + Influencer - - rmlhuk + + jcalfee
- rmlhuk + jcalfee - - rriski + + mivek
- rriski + mivek - - schnippl0r + + init-js
- schnippl0r + init-js - - smargold476 + + Jeidnx
- smargold476 + Jeidnx - - sportshead + + JiLleON
- sportshead + JiLleON - - squash + + jirislav
- squash + jirislav - - strarsis + + jmccl
- strarsis + jmccl - - tamueller + + jurekbarth
- tamueller + jurekbarth - - vivacarvajalito + + JOduMonT
- vivacarvajalito + JOduMonT - - wligtenberg + + Kaan88
- wligtenberg + Kaan88 - - wolkenschieber + + akkumar
- wolkenschieber + akkumar - - worldworm + + KCrawley
- worldworm + KCrawley From 5e28c17cf4c72d6b1ebda35c5af9f411d2652159 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Mon, 8 Jan 2024 15:07:38 +1300 Subject: [PATCH 235/592] docs: SpamAssassin ENV docs refactor (#3756) * chore: Log `SPAMASSASSIN_SPAM_TO_INBOX=1` ENV correctly ENV name logged was incomplete. * docs: Update SA related ENV docs * fix: Log level `warning` should be `warn` * docs: FAQ - Revise outdated SA entry * chore: Antispam / Antivirus => Anti-spam / Anti-virus * docs: ENV - Additional revisions to SA ENV * docs: ENV - Move `ENABLE_SPAMASSASSIN_KAM` --- CHANGELOG.md | 3 + Dockerfile | 2 +- README.md | 2 +- .../config/advanced/optional-config.md | 2 +- docs/content/config/environment.md | 173 ++++++++++++++---- docs/content/faq.md | 70 ++----- docs/content/index.md | 2 +- docs/mkdocs.yml | 2 +- mailserver.env | 12 +- .../scripts/startup/setup.d/security/misc.sh | 6 +- 10 files changed, 178 insertions(+), 96 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39a0ab661e9..074c3ead3a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ All notable changes to this project will be documented in this file. The format - Postfix is now configured with `smtputf8_enable = no` in our default `main.cf` config (_instead of during container startup_). ([#3750](https://github.com/docker-mailserver/docker-mailserver/pull/3750)) - **Rspamd** ([#3726](https://github.com/docker-mailserver/docker-mailserver/pull/3726)): - symbol scores for SPF, DKIM & DMARC were updated to more closely align with [RFC7489](https://www.rfc-editor.org/rfc/rfc7489#page-24); please note though that complete alignment is undesirable, because other symbols might be added as well, which changes the overall score calculation again, see [this issue](https://github.com/docker-mailserver/docker-mailserver/issues/3690#issuecomment-1866871996) +- **Docs:** + - Revised the SpamAssassin ENV docs to better communicate configuration and their relation to other ENV settings. ([#3756](https://github.com/docker-mailserver/docker-mailserver/pull/3756)) + ### Fixes diff --git a/Dockerfile b/Dockerfile index f9802c2aef5..65d818ab773 100644 --- a/Dockerfile +++ b/Dockerfile @@ -319,7 +319,7 @@ LABEL org.opencontainers.image.title="docker-mailserver" LABEL org.opencontainers.image.vendor="The Docker Mailserver Organization" LABEL org.opencontainers.image.authors="The Docker Mailserver Organization on GitHub" LABEL org.opencontainers.image.licenses="MIT" -LABEL org.opencontainers.image.description="A fullstack but simple mail server (SMTP, IMAP, LDAP, Antispam, Antivirus, etc.). Only configuration files, no SQL database." +LABEL org.opencontainers.image.description="A fullstack but simple mail server (SMTP, IMAP, LDAP, Anti-spam, Anti-virus, etc.). Only configuration files, no SQL database." LABEL org.opencontainers.image.url="https://github.com/docker-mailserver" LABEL org.opencontainers.image.documentation="https://github.com/docker-mailserver/docker-mailserver/blob/master/README.md" LABEL org.opencontainers.image.source="https://github.com/docker-mailserver/docker-mailserver" diff --git a/README.md b/README.md index b3072cd9528..8918c2e9d8d 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ## :page_with_curl: About -A production-ready fullstack but simple containerized mail server (SMTP, IMAP, LDAP, Antispam, Antivirus, etc.). Only configuration files, no SQL database. Keep it simple and versioned. Easy to deploy and upgrade. Originally created by @tomav, this project is now maintained by volunteers since January 2021. +A production-ready fullstack but simple containerized mail server (SMTP, IMAP, LDAP, Anti-spam, Anti-virus, etc.). Only configuration files, no SQL database. Keep it simple and versioned. Easy to deploy and upgrade. Originally created by @tomav, this project is now maintained by volunteers since January 2021. ## :bulb: Documentation diff --git a/docs/content/config/advanced/optional-config.md b/docs/content/config/advanced/optional-config.md index 8a43e4db537..21f82a3b869 100644 --- a/docs/content/config/advanced/optional-config.md +++ b/docs/content/config/advanced/optional-config.md @@ -33,7 +33,7 @@ This is a list of all configuration files and directories which are optional or - **ldap-aliases.cf:** Configuration for the virtual alias mapping `virtual_alias_maps`. See the [`setup-stack.sh`][github-commit-setup-stack.sh-L411] script. - **ldap-domains.cf:** Configuration for the virtual domain mapping `virtual_mailbox_domains`. See the [`setup-stack.sh`][github-commit-setup-stack.sh-L411] script. - **whitelist_clients.local:** Whitelisted domains, not considered by postgrey. Enter one host or domain per line. -- **spamassassin-rules.cf:** Antispam rules for Spamassassin. (Docs: [FAQ - SpamAssassin Rules][docs-faq-spamrules]) +- **spamassassin-rules.cf:** Anti-spam rules for Spamassassin. (Docs: [FAQ - SpamAssassin Rules][docs-faq-spamrules]) - **fail2ban-fail2ban.cf:** Additional config options for `fail2ban.cf`. (Docs: [Fail2Ban][docs-fail2ban]) - **fail2ban-jail.cf:** Additional config options for fail2ban's jail behaviour. (Docs: [Fail2Ban][docs-fail2ban]) - **amavis.cf:** replaces the `/etc/amavis/conf.d/50-user` file diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 71807c87401..c3c074fdf0e 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -316,27 +316,31 @@ Note: More information at ##### MOVE_SPAM_TO_JUNK -When enabled, e-mails marked with the +- 0 => Spam messages will be delivered in the mailbox. +- **1** => Spam messages will be delivered in the `Junk` folder. -1. `X-Spam: Yes` header added by Rspamd -2. `X-Spam-Flag: YES` header added by SpamAssassin (requires [`SPAMASSASSIN_SPAM_TO_INBOX=1`](#spamassassin_spam_to_inbox)) +Routes mail identified as spam into the recipient(s) Junk folder (_via a Dovecot Sieve script_). -will be automatically moved to the Junk folder (with the help of a Sieve script). +!!! info -- 0 => Spam messages will be delivered in the mailbox. -- **1** => Spam messages will be delivered in the `Junk` folder. + Mail is received as spam when it has been marked with either header: + + - `X-Spam: Yes` (_added by Rspamd_) + - `X-Spam-Flag: YES` (_added by SpamAssassin - requires [`SPAMASSASSIN_SPAM_TO_INBOX=1`](#spamassassin_spam_to_inbox)_) ##### MARK_SPAM_AS_READ +- **0** => disabled +- 1 => Spam messages will be marked as read + Enable to treat received spam as "read" (_avoids notification to MUA client of new mail_). -Mail is received as spam when it has been marked with either header: +!!! info -1. `X-Spam: Yes` (_by Rspamd_) -2. `X-Spam-Flag: YES` (_by SpamAssassin - requires [`SPAMASSASSIN_SPAM_TO_INBOX=1`](#spamassassin_spam_to_inbox)_) + Mail is received as spam when it has been marked with either header: -- **0** => disabled -- 1 => Spam messages will be marked as read + - `X-Spam: Yes` (_added by Rspamd_) + - `X-Spam-Flag: YES` (_added by SpamAssassin - requires [`SPAMASSASSIN_SPAM_TO_INBOX=1`](#spamassassin_spam_to_inbox)_) #### Rspamd @@ -515,63 +519,170 @@ Changes the interval in which log files are rotated. - **0** => SpamAssassin is disabled - 1 => SpamAssassin is enabled -##### SPAMASSASSIN_SPAM_TO_INBOX +??? info "SpamAssassin analyzes incoming mail and assigns a spam score" -- 0 => Spam messages will be bounced (_rejected_) without any notification (_dangerous_). -- **1** => Spam messages will be delivered to the inbox and tagged as spam using `SA_SPAM_SUBJECT`. + Integration with Amavis involves processing mail based on the assigned spam score via [`SA_TAG`, `SA_TAG2` and `SA_KILL`][amavis-docs::spam-score]. -##### ENABLE_SPAMASSASSIN_KAM + These settings have equivalent ENV supported by DMS for easy adjustments, as documented below. -[KAM](https://mcgrail.com/template/projects#KAM1) is a 3rd party SpamAssassin ruleset, provided by the McGrail Foundation. If SpamAssassin is enabled, KAM can be used in addition to the default ruleset. +[amavis-docs::spam-score]: https://www.ijs.si/software/amavisd/amavisd-new-docs.html#tagkill + +##### ENABLE_SPAMASSASSIN_KAM - **0** => KAM disabled - 1 => KAM enabled +[KAM](https://mcgrail.com/template/projects#KAM1) is a 3rd party SpamAssassin ruleset, provided by the McGrail Foundation. If SpamAssassin is enabled, KAM can be used in addition to the default ruleset. + +##### SPAMASSASSIN_SPAM_TO_INBOX + +- 0 => (_Amavis action: `D_BOUNCE`_): Spam messages will be bounced (_rejected_) without any notification (_dangerous_). +- **1** => (_Amavis action: `D_PASS`_): Spam messages will be delivered to the inbox. + +!!! note + + The Amavis action configured by this setting: + + - Influences the behaviour of the [`SA_KILL`](#sa_kill) setting. + - Applies to the Amavis config parameters `$final_spam_destiny` and `$final_bad_header_destiny`. + +!!! note "This ENV setting is related to" + + - [`MOVE_SPAM_TO_JUNK=1`](#move_spam_to_junk) + - [`MARK_SPAM_AS_READ=1`](#mark_spam_as_read) + - [`SA_SPAM_SUBJECT`](#sa_spam_subject) + ##### SA_TAG -- **2.0** => add spam info headers if at, or above that level +- **2.0** => add 'spam info' headers at, or above this spam score + +Mail is not yet considered spam at this spam score, but for purposes like diagnostics it can be useful to identify mail with a spam score at a lower bound than `SA_TAG2`. + +??? example "`X-Spam` headers appended to mail" -Note: this SpamAssassin setting needs `ENABLE_SPAMASSASSIN=1` + Send a simple mail to a local DMS account `hello@example.com`: + + ```bash + docker exec dms swaks --server 0.0.0.0 --to hello@example.com --body 'spam' + ``` + + Inspecting the raw mail you will notice several `X-Spam` headers were added to the mail like this: + + ``` + X-Spam-Flag: NO + X-Spam-Score: 4.162 + X-Spam-Level: **** + X-Spam-Status: No, score=4.162 tagged_above=2 required=4 + tests=[BODY_SINGLE_WORD=1, DKIM_ADSP_NXDOMAIN=0.8, + NO_DNS_FOR_FROM=0.379, NO_RECEIVED=-0.001, NO_RELAYS=-0.001, + PYZOR_CHECK=1.985] autolearn=no autolearn_force=no + ``` + + !!! info "The `X-Spam-Score` is `4.162`" + + High enough for `SA_TAG` to trigger adding these headers, but not high enough for `SA_TAG2` (_which would set `X-Spam-Flag: YES` instead_). ##### SA_TAG2 -- **6.31** => add 'spam detected' headers at that level +- **6.31** => add 'spam detected' headers at, or above this level + +When a spam score is high enough, mark mail as spam (_Appends the mail header: `X-Spam-Flag: YES`_). -Note: this SpamAssassin setting needs `ENABLE_SPAMASSASSIN=1` +!!! info "Interaction with other ENV" + + - [`SA_SPAM_SUBJECT`](#sa_spam_subject) modifies the mail subject to better communicate spam mail to the user. + - [`MOVE_SPAM_TO_JUNK=1`](#move_spam_to_junk): The mail is still delivered, but to the recipient(s) junk folder instead. This feature reduces the usefulness of `SA_SPAM_SUBJECT`. ##### SA_KILL -- **10.0** => triggers spam evasive actions +- **10.0** => quarantine + triggers action to handle spam + +Controls the spam score threshold for triggering an action on mail that has a high spam score. + +??? tip "Choosing an appropriate `SA_KILL` value" + + The value should be high enough to be represent confidence in mail as spam: + + - Too low: The action taken may prevent legitimate mail (ham) that was incorrectly detected as spam from being delivered successfully. + - Too high: Allows more spam to bypass the `SA_KILL` trigger (_how to treat mail with high confidence that it is actually spam_). + + Experiences from DMS users with these settings has been [collected here][gh-issue::sa-tunables-insights], along with [some direct configuration guides][gh-issue::sa-tunables-guides] (_under "Resources for references"_). + +[gh-issue::sa-tunables-insights]: https://github.com/docker-mailserver/docker-mailserver/pull/3058#issuecomment-1420268148 +[gh-issue::sa-tunables-guides]: https://github.com/docker-mailserver/docker-mailserver/pull/3058#issuecomment-1416547911 -!!! note "This SpamAssassin setting needs `ENABLE_SPAMASSASSIN=1`" +??? info "Trigger action" - By default, DMS is configured to quarantine spam emails. + DMS will configure Amavis with either of these actions based on the DMS [`SPAMASSASSIN_SPAM_TO_INBOX`](#spamassassin_spam_to_inbox) ENV setting: - If emails are quarantined, they are compressed and stored in a location dependent on the `ONE_DIR` setting above. To inhibit this behaviour and deliver spam emails, set this to a very high value e.g. `100.0`. + - `D_PASS` (**default**): + - Accept mail and deliver it to the recipient(s), despite the high spam score. A copy is still stored in quarantine. + - This is a good default to start with until you are more confident in an `SA_KILL` threshold that won't accidentally discard / bounce legitimate mail users are expecting to arrive but is detected as spam. + - `D_BOUNCE`: + - Additionally sends a bounce notification (DSN). + - The [DSN is suppressed][amavis-docs::actions] (_no bounce sent_) when the spam score exceeds the Amavis `$sa_dsn_cutoff_level` config setting (default: `10`). With the DMS `SA_KILL` default also being `10`, no DSN will ever be sent. + - `D_REJECT` / `D_DISCARD`: + - These two aren't configured by DMS, but are valid alternative action values if configuring Amavis directly. - If `ONE_DIR=1` (default) the location is `/var/mail-state/lib-amavis/virusmails/`, or if `ONE_DIR=0`: `/var/lib/amavis/virusmails/`. These paths are inside the docker container. +??? note "Quarantined mail" + + When mail has a spam score that reaches the `SA_KILL` threshold: + + - [It will be quarantined][amavis-docs::quarantine] regardless of the `SA_KILL` action to perform. + - With `D_PASS` the delivered mail also appends an `X-Quarantine-ID` mail header. The ID value of this header is part of the quarantined file name. + + If emails are quarantined, they are compressed and stored at a location dependent on the [`ONE_DIR`](#one_dir) setting: + + - `ONE_DIR=1` (default): `/var/mail-state/lib-amavis/virusmails/` + - `ONE_DIR=0`: `/var/lib/amavis/virusmails/` + + !!! tip + + Easily list mail stored in quarantine with `find` and the quarantine path: + + ```bash + find /var/lib/amavis/virusmails -type f + ``` + +[amavis-docs::actions]: https://www.ijs.si/software/amavisd/amavisd-new-docs.html#actions +[amavis-docs::quarantine]: https://www.ijs.si/software/amavisd/amavisd-new-docs.html#quarantine ##### SA_SPAM_SUBJECT -- **\*\*\*SPAM\*\*\*** => add tag to subject if spam detected +Adds a prefix to the subject header when mail is marked as spam (_via [`SA_TAG2`](#sa_tag2)_). -Note: this SpamAssassin setting needs `ENABLE_SPAMASSASSIN=1`. Add the SpamAssassin score to the subject line by inserting the keyword \_SCORE\_: **\*\*\*SPAM(\_SCORE\_)\*\*\***. +- **`'***SPAM*** '`** => A string value to use as a mail subject prefix. +- `undef` => Opt-out of modifying the subject for mail marked as spam. + +??? example "Including trailing white-space" + + Add trailing white-space by quote wrapping the value: `SA_SPAM_SUBJECT='[SPAM] '` + +??? example "Including the associated spam score" + + The [`_SCORE_` tag][sa-docs::score-tag] will be substituted with the SpamAssassin score: `SA_SPAM_SUBJECT=***SPAM(_SCORE_)***`. + +[sa-docs::score-tag]: https://spamassassin.apache.org/full/4.0.x/doc/Mail_SpamAssassin_Conf.html#rewrite_header-subject-from-to-STRING ##### SA_SHORTCIRCUIT_BAYES_SPAM - **1** => will activate SpamAssassin short circuiting for bayes spam detection. -This will uncomment the respective line in ```/etc/spamassasin/local.cf``` +This will uncomment the respective line in `/etc/spamassasin/local.cf` + +!!! warning -Note: activate this only if you are confident in your bayes database for identifying spam. + Activate this only if you are confident in your bayes database for identifying spam. ##### SA_SHORTCIRCUIT_BAYES_HAM - **1** => will activate SpamAssassin short circuiting for bayes ham detection -This will uncomment the respective line in ```/etc/spamassasin/local.cf``` +This will uncomment the respective line in `/etc/spamassasin/local.cf` + +!!! warning -Note: activate this only if you are confident in your bayes database for identifying ham. + Activate this only if you are confident in your bayes database for identifying ham. #### Fetchmail diff --git a/docs/content/faq.md b/docs/content/faq.md index 4da64b60081..f666b10226c 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -378,18 +378,7 @@ When you run DMS with the ENV variable `ONE_DIR=1` (default), this directory wil #### How can I manage my custom SpamAssassin rules? -Antispam rules are managed in `docker-data/dms/config/spamassassin-rules.cf`. - -#### What are acceptable `SA_SPAM_SUBJECT` values? - -For no subject set `SA_SPAM_SUBJECT=undef`. - -For a trailing white-space subject one can define the whole variable with quotes in `compose.yaml`: - -```yaml -environment: - - "SA_SPAM_SUBJECT=[SPAM] " -``` +Anti-spam rules are managed in `docker-data/dms/config/spamassassin-rules.cf`. #### Why are SpamAssassin `x-headers` not inserted into my `subdomain.example.com` subdomain emails? @@ -479,59 +468,39 @@ The following configuration works nicely: file: ./docker-data/dms/cron/sa-learn ``` -With the default settings, SpamAssassin will require 200 mails trained for spam (for example with the method explained above) and 200 mails trained for ham (using the same command as above but using `--ham` and providing it with some ham mails). Until you provided these 200+200 mails, SpamAssassin will not take the learned mails into account. For further reference, see the [SpamAssassin Wiki](https://wiki.apache.org/spamassassin/BayesNotWorking). - -#### How do I have more control about what SpamAssassin is filtering? +With the default settings, SpamAssassin will require 200 mails trained for spam (for example with the method explained above) and 200 mails trained for ham (using the same command as above but using `--ham` and providing it with some ham mails). -By default, SPAM and INFECTED emails are put to a quarantine which is not very straight forward to access. Several config settings are affecting this behavior: +- Until you provided these 200+200 mails, SpamAssassin will not take the learned mails into account. +- For further reference, see the [SpamAssassin Wiki](https://wiki.apache.org/spamassassin/BayesNotWorking). -First, make sure you have the proper thresholds set: - -```conf -SA_TAG=-100000.0 -SA_TAG2=3.75 -SA_KILL=100000.0 -``` +#### How do I have more control about what SpamAssassin is filtering? -- The very negative value in `SA_TAG` makes sure, that all emails have the SpamAssassin headers included. -- `SA_TAG2` is the actual threshold to set the YES/NO flag for spam detection. -- `SA_KILL` needs to be very high, to make sure nothing is bounced at all (`SA_KILL` superseeds `SPAMASSASSIN_SPAM_TO_INBOX`) +This is related to Amavis processing the mail after SpamAssassin has analyzed it and assigned a spam score. -Make sure everything (including SPAM) is delivered to the inbox and not quarantined: +- DMS provides some [common SA tunables via ENV][docs::env::sa_env]. +- Additional configuration can be managed with the DMS config volume by providing `docker-data/dms/config/amavis.cf`. -```conf -SPAMASSASSIN_SPAM_TO_INBOX=1 -``` +#### How can I send quarantined mail to a mailbox? -Use `MOVE_SPAM_TO_JUNK=1` or create a sieve script which puts spam to the Junk folder: +SPAM and INFECTED emails that [reach the `SA_KILL` threshold are archived into quarantine][docs::env::sa_kill]. -```sieve -require ["comparator-i;ascii-numeric","relational","fileinto"]; - -if header :contains "X-Spam-Flag" "YES" { - fileinto "Junk"; -} elsif allof ( - not header :matches "x-spam-score" "-*", - header :value "ge" :comparator "i;ascii-numeric" "x-spam-score" "3.75" -) { - fileinto "Junk"; -} -``` - -Create a dedicated mailbox for emails which are infected/bad header and everything amavis is blocking by default and put its address into `docker-data/dms/config/amavis.cf` +Instead of a quarantine folder, you can use a dedicated mailbox instead. Create an account like `quarantine@example.com` and create `docker-data/dms/config/amavis.cf`: ```cf -$clean_quarantine_to = "amavis\@example.com"; -$virus_quarantine_to = "amavis\@example.com"; -$banned_quarantine_to = "amavis\@example.com"; -$bad_header_quarantine_to = "amavis\@example.com"; -$spam_quarantine_to = "amavis\@example.com"; +$clean_quarantine_to = "quarantine\@example.com"; +$virus_quarantine_to = "quarantine\@example.com"; +$banned_quarantine_to = "quarantine\@example.com"; +$bad_header_quarantine_to = "quarantine\@example.com"; +$spam_quarantine_to = "quarantine\@example.com"; ``` [fail2ban-customize]: ./config/security/fail2ban.md [docs-maintenance]: ./config/advanced/maintenance/update-and-cleanup.md [docs-override-postfix]: ./config/advanced/override-defaults/postfix.md [docs-userpatches]: ./config/advanced/override-defaults/user-patches.md +[docs-optional-configuration]: ./config/advanced/optional-config.md +[docs::env::sa_env]: ./config/environment.md#spamassassin +[docs::env::sa_kill]: ./config/environment.md#sa_kill [github-comment-baredomain]: https://github.com/docker-mailserver/docker-mailserver/issues/3048#issuecomment-1432358353 [github-comment-override-hostname]: https://github.com/docker-mailserver/docker-mailserver/issues/1731#issuecomment-753968425 [github-issue-95]: https://github.com/docker-mailserver/docker-mailserver/issues/95 @@ -542,4 +511,3 @@ $spam_quarantine_to = "amavis\@example.com"; [github-issue-1792]: https://github.com/docker-mailserver/docker-mailserver/pull/1792 [hanscees-userpatches]: https://github.com/hanscees/dockerscripts/blob/master/scripts/tomav-user-patches.sh [mail-state-folders]: https://github.com/docker-mailserver/docker-mailserver/blob/c7e498194546416fb7231cb03254e77e085d18df/target/scripts/startup/misc-stack.sh#L24-L33 -[docs-optional-configuration]: ./config/advanced/optional-config.md diff --git a/docs/content/index.md b/docs/content/index.md index 51be4fb8585..ff1214b133e 100644 --- a/docs/content/index.md +++ b/docs/content/index.md @@ -14,7 +14,7 @@ This documentation provides you not only with the basic setup and configuration ## About -`docker-mailserver`, or DMS for short, is a production-ready fullstack but simple mail server (SMTP, IMAP, LDAP, Antispam, Antivirus, etc.). It employs only configuration files, no SQL database. The image is focused around the slogan "Keep it simple and versioned". +`docker-mailserver`, or DMS for short, is a production-ready fullstack but simple mail server (SMTP, IMAP, LDAP, Anti-spam, Anti-virus, etc.). It employs only configuration files, no SQL database. The image is focused around the slogan "Keep it simple and versioned". ## Contents diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index bd41a798c4e..6441dffe685 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -1,6 +1,6 @@ # Site specific: site_name: 'Docker Mailserver' -site_description: 'A fullstack but simple mail-server (SMTP, IMAP, LDAP, Antispam, Antivirus, etc.) using Docker.' +site_description: 'A fullstack but simple mail-server (SMTP, IMAP, LDAP, Anti-spam, Anti-virus, etc.) using Docker.' site_author: 'docker-mailserver (Github Organization)' copyright: '

© Docker Mailserver Organization
This project is licensed under the MIT license.

' diff --git a/mailserver.env b/mailserver.env index e84157a2331..0c2e1e40ebc 100644 --- a/mailserver.env +++ b/mailserver.env @@ -368,9 +368,6 @@ DOVECOT_INET_PROTOCOLS=all ENABLE_SPAMASSASSIN=0 -# deliver spam messages in the inbox (eventually tagged using SA_SPAM_SUBJECT) -SPAMASSASSIN_SPAM_TO_INBOX=1 - # KAM is a 3rd party SpamAssassin ruleset, provided by the McGrail Foundation. # If SpamAssassin is enabled, KAM can be used in addition to the default ruleset. # - **0** => KAM disabled @@ -379,23 +376,26 @@ SPAMASSASSIN_SPAM_TO_INBOX=1 # Note: only has an effect if `ENABLE_SPAMASSASSIN=1` ENABLE_SPAMASSASSIN_KAM=0 +# deliver spam messages to the inbox (tagged using SA_SPAM_SUBJECT) +SPAMASSASSIN_SPAM_TO_INBOX=1 + # spam messages will be moved in the Junk folder (SPAMASSASSIN_SPAM_TO_INBOX=1 required) MOVE_SPAM_TO_JUNK=1 # spam messages wil be marked as read MARK_SPAM_AS_READ=0 -# add spam info headers if at, or above that level: +# add 'spam info' headers at, or above this level SA_TAG=2.0 -# add 'spam detected' headers at that level +# add 'spam detected' headers at, or above this level SA_TAG2=6.31 # triggers spam evasive actions SA_KILL=10.0 # add tag to subject if spam detected -SA_SPAM_SUBJECT=***SPAM***** +SA_SPAM_SUBJECT='***SPAM*** ' # ----------------------------------------------- # --- Fetchmail Section ------------------------- diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index 78c1e60adf2..170f46fbec2 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -111,7 +111,7 @@ function __setup__security__spamassassin() { if [[ ${SPAMASSASSIN_SPAM_TO_INBOX} -eq 1 ]]; then _log 'trace' 'Configuring Spamassassin/Amavis to send SPAM to inbox' - _log 'debug' 'SPAM_TO_INBOX=1 is set. SA_KILL will be ignored.' + _log 'debug' "'SPAMASSASSIN_SPAM_TO_INBOX=1' is set. The 'SA_KILL' ENV will be ignored." sed -i "s|\$final_spam_destiny.*=.*$|\$final_spam_destiny = D_PASS;|g" /etc/amavis/conf.d/49-docker-mailserver sed -i "s|\$final_bad_header_destiny.*=.*$|\$final_bad_header_destiny = D_PASS;|g" /etc/amavis/conf.d/49-docker-mailserver @@ -265,7 +265,7 @@ EOF chown dovecot:root /usr/lib/dovecot/sieve-global/after/spam_to_junk.{sieve,svbin} if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] && [[ ${SPAMASSASSIN_SPAM_TO_INBOX} -eq 0 ]]; then - _log 'warning' "'SPAMASSASSIN_SPAM_TO_INBOX=0' but it is required to be 1 for 'MOVE_SPAM_TO_JUNK=1' to work" + _log 'warn' "'SPAMASSASSIN_SPAM_TO_INBOX=0' but it is required to be 1 for 'MOVE_SPAM_TO_JUNK=1' to work" fi else _log 'debug' 'Spam emails will not be moved to the Junk folder' @@ -290,7 +290,7 @@ EOF chown dovecot:root /usr/lib/dovecot/sieve-global/after/spam_mark_as_read.{sieve,svbin} if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] && [[ ${SPAMASSASSIN_SPAM_TO_INBOX} -eq 0 ]]; then - _log 'warning' "'SPAMASSASSIN_SPAM_TO_INBOX=0' but it is required to be 1 for 'MARK_SPAM_AS_READ=1' to work" + _log 'warn' "'SPAMASSASSIN_SPAM_TO_INBOX=0' but it is required to be 1 for 'MARK_SPAM_AS_READ=1' to work" fi else _log 'debug' 'Spam emails will not be marked as read' From ac25fb495bce9b21c1b643126aca7b0b26556ea2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jan 2024 18:39:28 +0100 Subject: [PATCH 236/592] chore(deps): Bump docker/metadata-action from 5.4.0 to 5.5.0 (#3762) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index b0a32b93b12..007712214e0 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -23,7 +23,7 @@ jobs: - name: 'Prepare tags' id: prep - uses: docker/metadata-action@v5.4.0 + uses: docker/metadata-action@v5.5.0 with: images: | ${{ secrets.DOCKER_REPOSITORY }} From 8d8936dfac81d762c03177d83876c69cb321e342 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jan 2024 17:40:50 +0000 Subject: [PATCH 237/592] chore(deps): Bump anchore/scan-action from 3.3.8 to 3.4.0 (#3761) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 08b4fd29fc4..d7cf6932d83 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v3.3.8 + uses: anchore/scan-action@v3.4.0 id: scan with: image: mailserver-testing:ci From aba218e6d7f30fbf4717178581c3e04537bea178 Mon Sep 17 00:00:00 2001 From: Casper Date: Wed, 10 Jan 2024 00:31:30 +0100 Subject: [PATCH 238/592] Fix jaq: Download platform specific binary (#3766) * choose architecture dynamically --- CHANGELOG.md | 1 + target/scripts/build/packages.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 074c3ead3a1..86302566317 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ All notable changes to this project will be documented in this file. The format - **Internal:** - `.gitattributes`: Always use LF line endings on checkout for files with shell script content ([#3755](https://github.com/docker-mailserver/docker-mailserver/pull/3755)) + - Fix missing 'jaq' binary for ARM architecture ([#3766](https://github.com/docker-mailserver/docker-mailserver/pull/3766)) ## [v13.2.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.2.0) diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 566d5441dff..7a4b60f6732 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -192,7 +192,7 @@ function _install_getmail() { function _install_utils() { _log 'debug' 'Installing utils sourced from Github' - curl -sL https://github.com/01mf02/jaq/releases/latest/download/jaq-v1.2.0-x86_64-unknown-linux-musl -o /usr/bin/jaq && chmod +x /usr/bin/jaq + curl -sL "https://github.com/01mf02/jaq/releases/latest/download/jaq-v1.2.0-$(uname -m)-unknown-linux-gnu" -o /usr/bin/jaq && chmod +x /usr/bin/jaq } function _remove_data_after_package_installations() { From 06fab3f12964b9ce5d5d6e889de28598635c6cf5 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Thu, 11 Jan 2024 10:34:08 +0100 Subject: [PATCH 239/592] tests: streamline tests and helpers further (#3747) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 7 + target/scripts/build/packages.sh | 10 +- test/files/emails/existing/added.txt | 5 - test/files/emails/existing/alias-external.txt | 5 - test/files/emails/existing/alias-local.txt | 5 - .../existing/alias-recipient-delimiter.txt | 5 - test/files/emails/existing/catchall-local.txt | 5 - .../emails/existing/regexp-alias-external.txt | 5 - .../emails/existing/regexp-alias-local.txt | 5 - .../existing/user-and-cc-local-alias.txt | 6 - test/files/emails/existing/user1.txt | 5 - test/files/emails/non-existing-user.txt | 5 - test/files/emails/rspamd/pass.txt | 5 - test/files/emails/rspamd/spam-header.txt | 5 - test/files/emails/rspamd/spam.txt | 5 - test/files/emails/rspamd/virus.txt | 5 - test/helper/common.bash | 4 +- test/helper/sending.bash | 177 ++++++++++++------ .../parallel/set1/dovecot/dovecot_quotas.bats | 9 +- .../parallel/set1/dovecot/dovecot_sieve.bats | 6 +- .../set1/dovecot/mailbox_format_dbox.bats | 6 +- .../set1/dovecot/special_use_folders.bats | 5 +- .../parallel/set1/spam_virus/clamav.bats | 3 +- .../disabled_clamav_spamassassin.bats | 3 +- .../parallel/set1/spam_virus/fail2ban.bats | 2 +- .../set1/spam_virus/postgrey_enabled.bats | 9 +- .../parallel/set1/spam_virus/postscreen.bats | 14 +- .../parallel/set1/spam_virus/rspamd_full.bats | 40 ++-- .../set1/spam_virus/spam_junk_folder.bats | 2 +- .../container_configuration/hostname.bats | 2 +- test/tests/parallel/set3/mta/dsn.bats | 18 +- test/tests/parallel/set3/mta/lmtp_ip.bats | 2 +- test/tests/parallel/set3/mta/privacy.bats | 5 +- .../parallel/set3/mta/smtp_delivery.bats | 42 ++--- test/tests/serial/mail_pop3.bats | 4 +- test/tests/serial/mail_with_imap.bats | 12 +- test/tests/serial/mail_with_ldap.bats | 27 +-- test/tests/serial/tests.bats | 14 +- test/tests/serial/vmail-id.bats | 4 +- 39 files changed, 247 insertions(+), 251 deletions(-) delete mode 100644 test/files/emails/existing/added.txt delete mode 100644 test/files/emails/existing/alias-external.txt delete mode 100644 test/files/emails/existing/alias-local.txt delete mode 100644 test/files/emails/existing/alias-recipient-delimiter.txt delete mode 100644 test/files/emails/existing/catchall-local.txt delete mode 100644 test/files/emails/existing/regexp-alias-external.txt delete mode 100644 test/files/emails/existing/regexp-alias-local.txt delete mode 100644 test/files/emails/existing/user-and-cc-local-alias.txt delete mode 100644 test/files/emails/existing/user1.txt delete mode 100644 test/files/emails/non-existing-user.txt delete mode 100644 test/files/emails/rspamd/pass.txt delete mode 100644 test/files/emails/rspamd/spam-header.txt delete mode 100644 test/files/emails/rspamd/spam.txt delete mode 100644 test/files/emails/rspamd/virus.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 86302566317..382b9efaae5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ All notable changes to this project will be documented in this file. The format ### Updates +- **Tests**: + - Refactored mail sending ([#3747](https://github.com/docker-mailserver/docker-mailserver/pull/3747)): + - This change is a follow-up to [#3732](https://github.com/docker-mailserver/docker-mailserver/pull/3732) from DMS v13.2. + - `swaks` version is now the latest from Github releases instead of the Debian package. + - `_nc_wrapper`, `_send_mail` and related helpers expect the `.txt` filepath extension again. + - `sending.bash` helper methods were refactored to better integrate `swaks` and accommodate different usage contexts. + - `test/files/emails/existing/` files were removed similar to previous removal of SMTP auth files as they became redundant with `swaks`. - **Internal:** - tests: Replace `wc -l` with `grep -c` ([#3752](https://github.com/docker-mailserver/docker-mailserver/pull/3752)) - Postfix is now configured with `smtputf8_enable = no` in our default `main.cf` config (_instead of during container startup_). ([#3750](https://github.com/docker-mailserver/docker-mailserver/pull/3750)) diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 7a4b60f6732..e3607f4882b 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -80,7 +80,7 @@ function _install_packages() { # `bind9-dnsutils` provides the `dig` command # `iputils-ping` provides the `ping` command DEBUG_PACKAGES=( - bind9-dnsutils iputils-ping less nano swaks + bind9-dnsutils iputils-ping less nano ) apt-get "${QUIET}" --no-install-recommends install \ @@ -192,7 +192,15 @@ function _install_getmail() { function _install_utils() { _log 'debug' 'Installing utils sourced from Github' + _log 'trace' 'Installing jaq' curl -sL "https://github.com/01mf02/jaq/releases/latest/download/jaq-v1.2.0-$(uname -m)-unknown-linux-gnu" -o /usr/bin/jaq && chmod +x /usr/bin/jaq + + _log 'trace' 'Installing swaks' + local SWAKS_VERSION='20240103.0' + local SWAKS_RELEASE="swaks-${SWAKS_VERSION}" + curl -sSfL "https://github.com/jetmore/swaks/releases/download/v${SWAKS_VERSION}/${SWAKS_RELEASE}.tar.gz" | tar -xz + mv "${SWAKS_RELEASE}/swaks" /usr/local/bin + rm -r "${SWAKS_RELEASE}" } function _remove_data_after_package_installations() { diff --git a/test/files/emails/existing/added.txt b/test/files/emails/existing/added.txt deleted file mode 100644 index 827b681f0e6..00000000000 --- a/test/files/emails/existing/added.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-added.txt -This is a test mail. diff --git a/test/files/emails/existing/alias-external.txt b/test/files/emails/existing/alias-external.txt deleted file mode 100644 index 03f1af6c618..00000000000 --- a/test/files/emails/existing/alias-external.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-alias-external.txt -This is a test mail. diff --git a/test/files/emails/existing/alias-local.txt b/test/files/emails/existing/alias-local.txt deleted file mode 100644 index 9b481a98550..00000000000 --- a/test/files/emails/existing/alias-local.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local Alias -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-alias-local.txt -This is a test mail. diff --git a/test/files/emails/existing/alias-recipient-delimiter.txt b/test/files/emails/existing/alias-recipient-delimiter.txt deleted file mode 100644 index 07cb8d40f1a..00000000000 --- a/test/files/emails/existing/alias-recipient-delimiter.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local Alias With Delimiter -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-alias-recipient-delimiter.txt -This is a test mail. diff --git a/test/files/emails/existing/catchall-local.txt b/test/files/emails/existing/catchall-local.txt deleted file mode 100644 index ab3e1988b52..00000000000 --- a/test/files/emails/existing/catchall-local.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-catchall-local.txt -This is a test mail. diff --git a/test/files/emails/existing/regexp-alias-external.txt b/test/files/emails/existing/regexp-alias-external.txt deleted file mode 100644 index b50ac90f1b3..00000000000 --- a/test/files/emails/existing/regexp-alias-external.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-regexp-alias-external.txt -This is a test mail. diff --git a/test/files/emails/existing/regexp-alias-local.txt b/test/files/emails/existing/regexp-alias-local.txt deleted file mode 100644 index e45b7c6c79a..00000000000 --- a/test/files/emails/existing/regexp-alias-local.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-regexp-alias-local.txt -This is a test mail. diff --git a/test/files/emails/existing/user-and-cc-local-alias.txt b/test/files/emails/existing/user-and-cc-local-alias.txt deleted file mode 100644 index 37814f91013..00000000000 --- a/test/files/emails/existing/user-and-cc-local-alias.txt +++ /dev/null @@ -1,6 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Cc: Existing Local Alias -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-user-and-cc-local-alias.txt -This is a test mail. diff --git a/test/files/emails/existing/user1.txt b/test/files/emails/existing/user1.txt deleted file mode 100644 index 23d49dc92de..00000000000 --- a/test/files/emails/existing/user1.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message existing-user1.txt -This is a test mail. diff --git a/test/files/emails/non-existing-user.txt b/test/files/emails/non-existing-user.txt deleted file mode 100644 index 3d92470e9f9..00000000000 --- a/test/files/emails/non-existing-user.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message non-existing-user.txt -This is a test mail. diff --git a/test/files/emails/rspamd/pass.txt b/test/files/emails/rspamd/pass.txt deleted file mode 100644 index ce9286b1460..00000000000 --- a/test/files/emails/rspamd/pass.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message rspamd/pass.txt -This mail should pass and Rspamd should not mark it. diff --git a/test/files/emails/rspamd/spam-header.txt b/test/files/emails/rspamd/spam-header.txt deleted file mode 100644 index 8722e42f0c9..00000000000 --- a/test/files/emails/rspamd/spam-header.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 21 Jan 2023 11:11:11 +0000 -Subject: Test Message rspamd-spam-header.txt -YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X diff --git a/test/files/emails/rspamd/spam.txt b/test/files/emails/rspamd/spam.txt deleted file mode 100644 index c561e779bb0..00000000000 --- a/test/files/emails/rspamd/spam.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 21 Jan 2023 11:11:11 +0000 -Subject: Test Message rspamd-spam.txt -XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X diff --git a/test/files/emails/rspamd/virus.txt b/test/files/emails/rspamd/virus.txt deleted file mode 100644 index cb18927d5d3..00000000000 --- a/test/files/emails/rspamd/virus.txt +++ /dev/null @@ -1,5 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 21 Jan 2023 11:11:11 +0000 -Subject: Test Message rspamd-virus.txt -X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H* diff --git a/test/helper/common.bash b/test/helper/common.bash index ab21ef60a15..0891bf8ce43 100644 --- a/test/helper/common.bash +++ b/test/helper/common.bash @@ -466,7 +466,7 @@ function _print_mail_log_for_id() { local MAIL_ID=${1:?Mail ID must be provided} local CONTAINER_NAME=$(__handle_container_name "${2:-}") - _run_in_container grep -F "${MAIL_ID}" /var/log/mail.log + _run_in_container grep -E "${MAIL_ID}" /var/log/mail.log } # A simple wrapper for netcat (`nc`). This is useful when sending @@ -480,7 +480,7 @@ function _nc_wrapper() { [[ -v CONTAINER_NAME ]] || return 1 - _run_in_container_bash "nc ${NC_PARAMETERS} < /tmp/docker-mailserver-test/${FILE}.txt" + _run_in_container_bash "nc ${NC_PARAMETERS} < /tmp/docker-mailserver-test/${FILE}" } # ? << Miscellaneous helper functions diff --git a/test/helper/sending.bash b/test/helper/sending.bash index 48012178c89..529e36aa365 100644 --- a/test/helper/sending.bash +++ b/test/helper/sending.bash @@ -7,68 +7,118 @@ # ! ATTENTION: This file is loaded by `common.sh` - do not load it yourself! # ! ATTENTION: This file requires helper functions from `common.sh`! -# Sends a mail from localhost (127.0.0.1) to a container. To send -# a custom email, create a file at `test/files/`, -# and provide `` as an argument to this function. +# Sends an e-mail from the container named by the environment variable `CONTAINER_NAME` +# to the same or another container. # -# Parameters include all options that one can supply to `swaks` -# itself. The `--data` parameter expects a relative path from `emails/` -# where the contents will be implicitly provided to `swaks` via STDIN. +# To send a custom email, you can +# +# 1. create a file at `test/files/` and provide `` via `--data` as an argument to this function; +# 2. use this function without the `--data` argument, in which case we provide a default; +# 3. provide data inline (`--data `). +# +# The very first parameter **may** be `--expect-rejection` - use it of you expect the mail transaction to not finish +# successfully. All other (following) parameters include all options that one can supply to `swaks` itself. +# As mentioned before, the `--data` parameter expects a value of either: +# +# - A relative path from `test/files/emails/` +# - An "inline" data string (e.g., `Date: 1 Jan 2024\nSubject: This is a test`) +# +# ## Output +# +# This functions prints the output of the transaction that `swaks` prints. # # ## Attention # # This function assumes `CONTAINER_NAME` to be properly set (to the container # name the command should be executed in)! # -# This function will just send the email in an "asynchronous" fashion, i.e. it will -# send the email but it will not make sure the mail queue is empty after the mail -# has been sent. +# This function will send the email in an "asynchronous" fashion, +# it will return without waiting for the Postfix mail queue to be emptied. function _send_email() { - [[ -v CONTAINER_NAME ]] || return 1 - - # Parameter defaults common to our testing needs: - local EHLO='mail.external.tld' - local FROM='user@external.tld' - local TO='user1@localhost.localdomain' - local SERVER='0.0.0.0' - local PORT=25 - # Extra options for `swaks` that aren't covered by the default options above: - local ADDITIONAL_SWAKS_OPTIONS=() - # Specifically for handling `--data` option below: - local FINAL_SWAKS_OPTIONS=() - - while [[ ${#} -gt 0 ]]; do - case "${1}" in - ( '--ehlo' ) EHLO=${2:?--ehlo given but no argument} ; shift 2 ;; - ( '--from' ) FROM=${2:?--from given but no argument} ; shift 2 ;; - ( '--to' ) TO=${2:?--to given but no argument} ; shift 2 ;; - ( '--server' ) SERVER=${2:?--server given but no argument} ; shift 2 ;; - ( '--port' ) PORT=${2:?--port given but no argument} ; shift 2 ;; - ( '--data' ) - local TEMPLATE_FILE="/tmp/docker-mailserver-test/emails/${2:?--data given but no argument provided}.txt" - FINAL_SWAKS_OPTIONS+=('--data') - FINAL_SWAKS_OPTIONS+=('-') - FINAL_SWAKS_OPTIONS+=('<') - FINAL_SWAKS_OPTIONS+=("${TEMPLATE_FILE}") - shift 2 - ;; - ( * ) ADDITIONAL_SWAKS_OPTIONS+=("${1}") ; shift 1 ;; - esac - done - - _run_in_container_bash "swaks --server ${SERVER} --port ${PORT} --ehlo ${EHLO} --from ${FROM} --to ${TO} ${ADDITIONAL_SWAKS_OPTIONS[*]} ${FINAL_SWAKS_OPTIONS[*]}" + local RETURN_VALUE=0 + local COMMAND_STRING + + function __parse_arguments() { + [[ -v CONTAINER_NAME ]] || return 1 + + # Parameter defaults common to our testing needs: + local EHLO='mail.external.tld' + local FROM='user@external.tld' + local TO='user1@localhost.localdomain' + local SERVER='0.0.0.0' + local PORT=25 + # Extra options for `swaks` that aren't covered by the default options above: + local ADDITIONAL_SWAKS_OPTIONS=() + local DATA_WAS_SUPPLIED=0 + + while [[ ${#} -gt 0 ]]; do + case "${1}" in + ( '--ehlo' ) EHLO=${2:?--ehlo given but no argument} ; shift 2 ;; + ( '--from' ) FROM=${2:?--from given but no argument} ; shift 2 ;; + ( '--to' ) TO=${2:?--to given but no argument} ; shift 2 ;; + ( '--server' ) SERVER=${2:?--server given but no argument} ; shift 2 ;; + ( '--port' ) PORT=${2:?--port given but no argument} ; shift 2 ;; + ( '--data' ) + ADDITIONAL_SWAKS_OPTIONS+=('--data') + local FILE_PATH="/tmp/docker-mailserver-test/emails/${2:?--data given but no argument provided}" + if _exec_in_container_bash "[[ -e ${FILE_PATH} ]]"; then + ADDITIONAL_SWAKS_OPTIONS+=("@${FILE_PATH}") + else + ADDITIONAL_SWAKS_OPTIONS+=("'${2}'") + fi + shift 2 + DATA_WAS_SUPPLIED=1 + ;; + ( * ) ADDITIONAL_SWAKS_OPTIONS+=("'${1}'") ; shift 1 ;; + esac + done + + if [[ ${DATA_WAS_SUPPLIED} -eq 0 ]]; then + # Fallback template (without the implicit `Message-Id` + `X-Mailer` headers from swaks): + # NOTE: It is better to let Postfix generate and append the `Message-Id` header itself, + # as it will contain the Queue ID for tracking in logs (which is also returned in swaks output). + ADDITIONAL_SWAKS_OPTIONS+=('--data') + ADDITIONAL_SWAKS_OPTIONS+=("'Date: %DATE%\nTo: %TO_ADDRESS%\nFrom: %FROM_ADDRESS%\nSubject: test %DATE%\n%NEW_HEADERS%\n%BODY%\n'") + fi + + echo "swaks --server '${SERVER}' --port '${PORT}' --ehlo '${EHLO}' --from '${FROM}' --to '${TO}' ${ADDITIONAL_SWAKS_OPTIONS[*]}" + } + + if [[ ${1:-} == --expect-rejection ]]; then + shift 1 + COMMAND_STRING=$(__parse_arguments "${@}") + _run_in_container_bash "${COMMAND_STRING}" + RETURN_VALUE=${?} + else + COMMAND_STRING=$(__parse_arguments "${@}") + _run_in_container_bash "${COMMAND_STRING}" + assert_success + fi + + # shellcheck disable=SC2154 + echo "${output}" + return "${RETURN_VALUE}" } # Like `_send_email` with two major differences: # # 1. this function waits for the mail to be processed; there is no asynchronicity -# because filtering the logs in a synchronous way is easier and safer! -# 2. this function prints an ID one can later filter by to check logs +# because filtering the logs in a synchronous way is easier and safer; +# 2. this function takes the name of a variable and inserts IDs one can later +# filter by to check logs. # # No. 2 is especially useful in case you send more than one email in a single # test file and need to assert certain log entries for each mail individually. # -# This function takes the same arguments as `_send_mail`. +# The first argument has to be the name of the variable that the e-mail ID is stored in. +# The second argument **can** be the flag `--expect-rejection`. +# +# - If this flag is supplied, the function does not check whether the whole mail delivery +# transaction was successful. Additionally the queue ID will be retrieved differently. +# - CAUTION: It must still be possible to `grep` for the Message-ID that Postfix +# generated in the mail log; otherwise this function fails. +# +# The rest of the arguments are the same as `_send_email`. # # ## Attention # @@ -82,20 +132,35 @@ function _send_email() { # chosen. Sending more than one mail at any given point in time with this function # is UNDEFINED BEHAVIOR! function _send_email_and_get_id() { - [[ -v CONTAINER_NAME ]] || return 1 - - _wait_for_empty_mail_queue_in_container - _send_email "${@}" - _wait_for_empty_mail_queue_in_container + # Export the variable denoted by ${1} so everyone has access + export "${1:?Mail ID must be set for _send_email_and_get_id}" + # Get a "reference" to the content of the variable denoted by ${1} so we can manipulate the content + local -n ID_ENV_VAR_REF=${1:?} + # Prepare the message ID header here because we will shift away ${1} later + local MID="<${1}@dms-tests>" + # Get rid of ${1} so only the arguments for swaks remain + shift 1 - local MAIL_ID + local QUEUE_ID # The unique ID Postfix (and other services) use may be different in length # on different systems (e.g. amd64 (11) vs aarch64 (10)). Hence, we use a # range to safely capture it. - MAIL_ID=$(_exec_in_container tac /var/log/mail.log \ - | grep -E -m 1 'postfix/smtpd.*: [A-Z0-9]+: client=localhost' \ - | grep -E -o '[A-Z0-9]{9,12}' || true) + local QUEUE_ID_REGEX='[A-Z0-9]{9,12}' + + _wait_for_empty_mail_queue_in_container + local OUTPUT=$(_send_email "${@}" --header "Message-Id: ${MID}") + _wait_for_empty_mail_queue_in_container + + # We store Postfix's queue ID first + ID_ENV_VAR_REF=$(_exec_in_container tac /var/log/mail.log \ + | grep -E "postfix/cleanup.*: ${QUEUE_ID_REGEX}:.*message-id=${MID}" \ + | grep -E --only-matching --max-count 1 "${QUEUE_ID_REGEX}" || :) + # But we also requre potential Dovecot sieve output, which requires the mesage ID, + # so we need to provide the message ID too. + ID_ENV_VAR_REF+="|${MID}" - assert_not_equal "${MAIL_ID}" '' - echo "${MAIL_ID}" + # Last but not least, we perform plausibility checks on the IDs. + assert_not_equal "${ID_ENV_VAR_REF}" '' + run echo "${ID_ENV_VAR_REF}" + assert_line --regexp "^${QUEUE_ID_REGEX}\|${MID}$" } diff --git a/test/tests/parallel/set1/dovecot/dovecot_quotas.bats b/test/tests/parallel/set1/dovecot/dovecot_quotas.bats index 81cf9bc15cf..8e71bf7327b 100644 --- a/test/tests/parallel/set1/dovecot/dovecot_quotas.bats +++ b/test/tests/parallel/set1/dovecot/dovecot_quotas.bats @@ -225,12 +225,9 @@ function teardown_file() { _default_teardown ; } sleep 10 # send some big emails - _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded' - assert_success - _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded' - assert_success - _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded' - assert_success + _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded.txt' + _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded.txt' + _send_email --to 'quotauser@otherdomain.tld' --data 'quota-exceeded.txt' # check for quota warn message existence run _repeat_until_success_or_timeout 20 _exec_in_container grep -R 'Subject: quota warning' /var/mail/otherdomain.tld/quotauser/new/ assert_success diff --git a/test/tests/parallel/set1/dovecot/dovecot_sieve.bats b/test/tests/parallel/set1/dovecot/dovecot_sieve.bats index e3e076a5267..6acbc4a57b7 100644 --- a/test/tests/parallel/set1/dovecot/dovecot_sieve.bats +++ b/test/tests/parallel/set1/dovecot/dovecot_sieve.bats @@ -26,11 +26,9 @@ function setup_file() { _wait_for_smtp_port_in_container # Single mail sent from 'spam@spam.com' that is handled by User (relocate) and Global (copy) sieves for user1: - _send_email --data 'sieve/spam-folder' - assert_success + _send_email --data 'sieve/spam-folder.txt' # Mail for user2 triggers the sieve-pipe: - _send_email --to 'user2@otherdomain.tld' --data 'sieve/pipe' - assert_success + _send_email --to 'user2@otherdomain.tld' --data 'sieve/pipe.txt' _wait_for_empty_mail_queue_in_container } diff --git a/test/tests/parallel/set1/dovecot/mailbox_format_dbox.bats b/test/tests/parallel/set1/dovecot/mailbox_format_dbox.bats index 033a5bdec6b..b9f6d8f6efc 100644 --- a/test/tests/parallel/set1/dovecot/mailbox_format_dbox.bats +++ b/test/tests/parallel/set1/dovecot/mailbox_format_dbox.bats @@ -26,8 +26,7 @@ function teardown() { _default_teardown ; } _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' _wait_for_smtp_port_in_container - _send_email --data 'existing/user1' - assert_success + _send_email _wait_for_empty_mail_queue_in_container # Mail received should be stored as `u.1` (one file per message) @@ -48,8 +47,7 @@ function teardown() { _default_teardown ; } _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' _wait_for_smtp_port_in_container - _send_email --data 'existing/user1' - assert_success + _send_email _wait_for_empty_mail_queue_in_container # Mail received should be stored in `m.1` (1 or more messages) diff --git a/test/tests/parallel/set1/dovecot/special_use_folders.bats b/test/tests/parallel/set1/dovecot/special_use_folders.bats index fe1f554e5a2..4965b84452c 100644 --- a/test/tests/parallel/set1/dovecot/special_use_folders.bats +++ b/test/tests/parallel/set1/dovecot/special_use_folders.bats @@ -14,8 +14,7 @@ function setup_file() { function teardown_file() { _default_teardown ; } @test 'normal delivery works' { - _send_email --data 'existing/user1' - assert_success + _send_email _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new 1 } @@ -27,7 +26,7 @@ function teardown_file() { _default_teardown ; } } @test "(IMAP) special-use folders should be created when necessary" { - _nc_wrapper 'nc/imap_special_use_folders' '-w 8 0.0.0.0 143' + _nc_wrapper 'nc/imap_special_use_folders.txt' '-w 8 0.0.0.0 143' assert_output --partial 'Drafts' assert_output --partial 'Junk' assert_output --partial 'Trash' diff --git a/test/tests/parallel/set1/spam_virus/clamav.bats b/test/tests/parallel/set1/spam_virus/clamav.bats index 9c035f5b690..a916896af66 100644 --- a/test/tests/parallel/set1/spam_virus/clamav.bats +++ b/test/tests/parallel/set1/spam_virus/clamav.bats @@ -25,8 +25,7 @@ function setup_file() { _wait_for_service postfix _wait_for_smtp_port_in_container - _send_email --from 'virus@external.tld' --data 'amavis/virus' - assert_success + _send_email --from 'virus@external.tld' --data 'amavis/virus.txt' _wait_for_empty_mail_queue_in_container } diff --git a/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats b/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats index f2474cc07a1..5ec28396717 100644 --- a/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats +++ b/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats @@ -18,8 +18,7 @@ function setup_file() { _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' _wait_for_smtp_port_in_container - _send_email --data 'existing/user1' - assert_success + _send_email _wait_for_empty_mail_queue_in_container } diff --git a/test/tests/parallel/set1/spam_virus/fail2ban.bats b/test/tests/parallel/set1/spam_virus/fail2ban.bats index 8a03ba04acd..f451befd59d 100644 --- a/test/tests/parallel/set1/spam_virus/fail2ban.bats +++ b/test/tests/parallel/set1/spam_virus/fail2ban.bats @@ -74,7 +74,7 @@ function teardown_file() { CONTAINER1_IP=$(_get_container_ip "${CONTAINER1_NAME}") # Trigger a ban by failing to login twice: for _ in {1..2}; do - CONTAINER_NAME=${CONTAINER2_NAME} _send_email \ + CONTAINER_NAME=${CONTAINER2_NAME} _send_email --expect-rejection \ --server "${CONTAINER1_IP}" \ --port 465 \ --auth PLAIN \ diff --git a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats index 389fc183a42..5538d3bf30a 100644 --- a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats +++ b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats @@ -51,7 +51,7 @@ function teardown_file() { _default_teardown ; } _reload_postfix # Send test mail (it should fail to deliver): - _send_email --from 'user@external.tld' --port 25 --data 'postgrey' + _send_email --expect-rejection --from 'user@external.tld' --port 25 --data 'postgrey.txt' assert_failure assert_output --partial 'Recipient address rejected: Delayed by Postgrey' @@ -67,8 +67,7 @@ function teardown_file() { _default_teardown ; } # Wait until `$POSTGREY_DELAY` seconds pass before trying again: sleep 3 # Retry delivering test mail (it should be trusted this time): - _send_email --from 'user@external.tld' --port 25 --data 'postgrey' - assert_success + _send_email --from 'user@external.tld' --port 25 --data 'postgrey.txt' # Confirm postgrey permitted delivery (triplet is now trusted): _should_have_log_entry \ @@ -87,7 +86,7 @@ function teardown_file() { _default_teardown ; } # - It'd also cause the earlier greylist test to fail. # - TODO: Actually confirm whitelist feature works correctly as these test cases are using a workaround: @test "should whitelist sender 'user@whitelist.tld'" { - _nc_wrapper 'nc/postgrey_whitelist' '-w 0 0.0.0.0 10023' + _nc_wrapper 'nc/postgrey_whitelist.txt' '-w 0 0.0.0.0 10023' _should_have_log_entry \ 'action=pass' \ @@ -96,7 +95,7 @@ function teardown_file() { _default_teardown ; } } @test "should whitelist recipient 'user2@otherdomain.tld'" { - _nc_wrapper 'nc/postgrey_whitelist_recipients' '-w 0 0.0.0.0 10023' + _nc_wrapper 'nc/postgrey_whitelist_recipients.txt' '-w 0 0.0.0.0 10023' _should_have_log_entry \ 'action=pass' \ diff --git a/test/tests/parallel/set1/spam_virus/postscreen.bats b/test/tests/parallel/set1/spam_virus/postscreen.bats index 377b2479357..92333f98d67 100644 --- a/test/tests/parallel/set1/spam_virus/postscreen.bats +++ b/test/tests/parallel/set1/spam_virus/postscreen.bats @@ -44,7 +44,7 @@ function teardown_file() { # Use `nc` to send all SMTP commands at once instead (emulate a misbehaving client that should be rejected) # NOTE: Postscreen only runs on port 25, avoid implicit ports in test methods @test 'should fail send when talking out of turn' { - CONTAINER_NAME=${CONTAINER2_NAME} _nc_wrapper 'emails/nc_raw/postscreen' "${CONTAINER1_IP} 25" + CONTAINER_NAME=${CONTAINER2_NAME} _nc_wrapper 'emails/nc_raw/postscreen.txt' "${CONTAINER1_IP} 25" # Expected postscreen log entry: assert_output --partial 'Protocol error' @@ -56,14 +56,10 @@ function teardown_file() { @test "should successfully pass postscreen and get postfix greeting message (respecting postscreen_greet_wait time)" { # Configure `send_email()` to send from the mail client container (CONTAINER2_NAME) via ENV override, # mail is sent to the DMS server container (CONTAINER1_NAME) via `--server` parameter: - CONTAINER_NAME=${CONTAINER2_NAME} _send_email --server "${CONTAINER1_IP}" --port 25 --data 'postscreen' - # NOTE: Cannot assert_success due to sender address not being resolvable. - # TODO: Uncomment when proper resolution of domain names is possible: - # assert_success - - # TODO: Prefer this approach when `_send_email_and_get_id()` can support separate client and server containers: - # local MAIL_ID=$(_send_email_and_get_id --port 25 --data 'postscreen') - # _print_mail_log_for_id "${MAIL_ID}" + # TODO: Use _send_email_and_get_id when proper resolution of domain names is possible: + CONTAINER_NAME=${CONTAINER2_NAME} _send_email --expect-rejection --server "${CONTAINER1_IP}" --port 25 --data 'postscreen.txt' + # CONTAINER_NAME=${CONTAINER2_NAME} _send_email_and_get_id MAIL_ID_POSTSCREEN --server "${CONTAINER1_IP}" --data 'postscreen.txt' + # _print_mail_log_for_id "${MAIL_ID_POSTSCREEN}" # assert_output --partial "stored mail into mailbox 'INBOX'" _run_in_container cat /var/log/mail.log diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index 2e610d72618..b83ac3535c3 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -43,16 +43,24 @@ function setup_file() { _wait_for_service postfix _wait_for_smtp_port_in_container - # We will send 3 emails: the first one should pass just fine; the second one should - # be rejected due to spam; the third one should be rejected due to a virus. - export MAIL_ID1=$(_send_email_and_get_id --from 'rspamd-pass@example.test' --data 'rspamd/pass') - export MAIL_ID2=$(_send_email_and_get_id --from 'rspamd-spam@example.test' --data 'rspamd/spam') - export MAIL_ID3=$(_send_email_and_get_id --from 'rspamd-virus@example.test' --data 'rspamd/virus') - export MAIL_ID4=$(_send_email_and_get_id --from 'rspamd-spam-header@example.test' --data 'rspamd/spam-header') - - for ID in MAIL_ID{1,2,3,4}; do - [[ -n ${!ID} ]] || { echo "${ID} is empty - aborting!" ; return 1 ; } - done + # ref: https://rspamd.com/doc/gtube_patterns.html + local GTUBE_SUFFIX='*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X' + + # We will send 4 emails: + # 1. The first one should pass just fine + _send_email_and_get_id MAIL_ID_PASS + # 2. The second one should be rejected (GTUBE pattern) + _send_email_and_get_id MAIL_ID_REJECT --expect-rejection --body "XJS${GTUBE_SUFFIX}" + # 3. The third one should be rejected due to a virus (ClamAV EICAR pattern) + # shellcheck disable=SC2016 + _send_email_and_get_id MAIL_ID_VIRUS --expect-rejection \ + --body 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*' + # 4. The fourth one will receive an added header (GTUBE pattern) + _send_email_and_get_id MAIL_ID_HEADER --body "YJS${GTUBE_SUFFIX}" + + _run_in_container cat /var/log/mail.log + assert_success + refute_output --partial 'inet:localhost:11332: Connection refused' } function teardown_file() { _default_teardown ; } @@ -104,7 +112,7 @@ function teardown_file() { _default_teardown ; } @test 'normal mail passes fine' { _service_log_should_contain_string 'rspamd' 'F \(no action\)' - _print_mail_log_for_id "${MAIL_ID1}" + _print_mail_log_for_id "${MAIL_ID_PASS}" assert_output --partial "stored mail into mailbox 'INBOX'" _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 @@ -114,7 +122,7 @@ function teardown_file() { _default_teardown ; } _service_log_should_contain_string 'rspamd' 'S \(reject\)' _service_log_should_contain_string 'rspamd' 'reject "Gtube pattern"' - _print_mail_log_for_id "${MAIL_ID2}" + _print_mail_log_for_id "${MAIL_ID_REJECT}" assert_output --partial 'milter-reject' assert_output --partial '5.7.1 Gtube pattern' @@ -125,7 +133,7 @@ function teardown_file() { _default_teardown ; } _service_log_should_contain_string 'rspamd' 'T \(reject\)' _service_log_should_contain_string 'rspamd' 'reject "ClamAV FOUND VIRUS "Eicar-Signature"' - _print_mail_log_for_id "${MAIL_ID3}" + _print_mail_log_for_id "${MAIL_ID_VIRUS}" assert_output --partial 'milter-reject' assert_output --partial '5.7.1 ClamAV FOUND VIRUS "Eicar-Signature"' refute_output --partial "stored mail into mailbox 'INBOX'" @@ -214,7 +222,7 @@ function teardown_file() { _default_teardown ; } _service_log_should_contain_string 'rspamd' 'S \(add header\)' _service_log_should_contain_string 'rspamd' 'add header "Gtube pattern"' - _print_mail_log_for_id "${MAIL_ID4}" + _print_mail_log_for_id "${MAIL_ID_HEADER}" assert_output --partial "fileinto action: stored mail into mailbox 'Junk'" _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 @@ -256,7 +264,7 @@ function teardown_file() { _default_teardown ; } # Move an email to the "Junk" folder from "INBOX"; the first email we # sent should pass fine, hence we can now move it. - _nc_wrapper 'nc/rspamd_imap_move_to_junk' '0.0.0.0 143' + _nc_wrapper 'nc/rspamd_imap_move_to_junk.txt' '0.0.0.0 143' sleep 1 # wait for the transaction to finish _run_in_container cat /var/log/mail/mail.log @@ -270,7 +278,7 @@ function teardown_file() { _default_teardown ; } # Move an email to the "INBOX" folder from "Junk"; there should be two mails # in the "Junk" folder, since the second email we sent during setup should # have landed in the Junk folder already. - _nc_wrapper 'nc/rspamd_imap_move_to_inbox' '0.0.0.0 143' + _nc_wrapper 'nc/rspamd_imap_move_to_inbox.txt' '0.0.0.0 143' sleep 1 # wait for the transaction to finish _run_in_container cat /var/log/mail/mail.log diff --git a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats index fea23b0bdc7..2c3a522d70b 100644 --- a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats +++ b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats @@ -95,7 +95,7 @@ function teardown() { _default_teardown ; } function _should_send_spam_message() { _wait_for_smtp_port_in_container _wait_for_tcp_port_in_container 10024 # port 10024 is for Amavis - _send_email --from 'spam@external.tld' --data 'amavis/spam' + _send_email --from 'spam@external.tld' --data 'amavis/spam.txt' } function _should_be_received_by_amavis() { diff --git a/test/tests/parallel/set3/container_configuration/hostname.bats b/test/tests/parallel/set3/container_configuration/hostname.bats index f5774eef8ed..a525ecb2d90 100644 --- a/test/tests/parallel/set3/container_configuration/hostname.bats +++ b/test/tests/parallel/set3/container_configuration/hostname.bats @@ -207,7 +207,7 @@ function _should_have_correct_mail_headers() { # (eg: OVERRIDE_HOSTNAME or `--hostname mail --domainname example.test`) local EXPECTED_HOSTNAME=${3:-${EXPECTED_FQDN}} - _send_email --from 'user@external.tld' --data 'existing/user1' + _send_email --from 'user@external.tld' _wait_for_empty_mail_queue_in_container _count_files_in_directory_in_container '/var/mail/localhost.localdomain/user1/new/' '1' diff --git a/test/tests/parallel/set3/mta/dsn.bats b/test/tests/parallel/set3/mta/dsn.bats index a5228cfc60d..9a65b147c10 100644 --- a/test/tests/parallel/set3/mta/dsn.bats +++ b/test/tests/parallel/set3/mta/dsn.bats @@ -49,9 +49,9 @@ function teardown_file() { # TODO replace with _send_email as soon as it supports DSN # TODO ref: https://github.com/jetmore/swaks/issues/41 - _nc_wrapper 'emails/nc_raw/dsn/unauthenticated' - _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 465' - _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 587' + _nc_wrapper 'emails/nc_raw/dsn/unauthenticated.txt' + _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 465' + _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 587' _wait_for_empty_mail_queue_in_container _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log @@ -62,7 +62,7 @@ function teardown_file() { @test "should only send a DSN when requested from ports 465/587" { export CONTAINER_NAME=${CONTAINER2_NAME} - _nc_wrapper 'emails/nc_raw/dsn/unauthenticated' + _nc_wrapper 'emails/nc_raw/dsn/unauthenticated.txt' _wait_for_empty_mail_queue_in_container # DSN requests can now only be made on ports 465 and 587, @@ -74,8 +74,8 @@ function teardown_file() { assert_failure # These ports are excluded via master.cf. - _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 465' - _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 587' + _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 465' + _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 587' _wait_for_empty_mail_queue_in_container _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log @@ -85,9 +85,9 @@ function teardown_file() { @test "should never send a DSN" { export CONTAINER_NAME=${CONTAINER3_NAME} - _nc_wrapper 'emails/nc_raw/dsn/unauthenticated' - _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 465' - _nc_wrapper 'emails/nc_raw/dsn/authenticated' '0.0.0.0 587' + _nc_wrapper 'emails/nc_raw/dsn/unauthenticated.txt' + _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 465' + _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 587' _wait_for_empty_mail_queue_in_container # DSN requests are rejected regardless of origin. diff --git a/test/tests/parallel/set3/mta/lmtp_ip.bats b/test/tests/parallel/set3/mta/lmtp_ip.bats index d8be42d936d..201cb2379d5 100644 --- a/test/tests/parallel/set3/mta/lmtp_ip.bats +++ b/test/tests/parallel/set3/mta/lmtp_ip.bats @@ -38,7 +38,7 @@ function teardown_file() { _default_teardown ; } @test "delivers mail to existing account" { _wait_for_smtp_port_in_container - _send_email --data 'existing/user1' # send a test email + _send_email # Verify delivery was successful, log line should look similar to: # postfix/lmtp[1274]: 0EA424ABE7D9: to=, relay=127.0.0.1[127.0.0.1]:24, delay=0.13, delays=0.07/0.01/0.01/0.05, dsn=2.0.0, status=sent (250 2.0.0 ixPpB+Zvv2P7BAAAUi6ngw Saved) diff --git a/test/tests/parallel/set3/mta/privacy.bats b/test/tests/parallel/set3/mta/privacy.bats index 4d4d82ba467..614e2e87db0 100644 --- a/test/tests/parallel/set3/mta/privacy.bats +++ b/test/tests/parallel/set3/mta/privacy.bats @@ -26,11 +26,10 @@ function teardown_file() { _default_teardown ; } # this test covers https://github.com/docker-mailserver/docker-mailserver/issues/681 @test "(Postfix) remove privacy details of the sender" { _send_email \ - --port 587 -tls --auth LOGIN \ + --port 587 -tls --auth PLAIN \ --auth-user user1@localhost.localdomain \ --auth-password mypassword \ - --data 'privacy' - assert_success + --data 'privacy.txt' _run_until_success_or_timeout 120 _exec_in_container_bash '[[ -d /var/mail/localhost.localdomain/user1/new ]]' assert_success diff --git a/test/tests/parallel/set3/mta/smtp_delivery.bats b/test/tests/parallel/set3/mta/smtp_delivery.bats index f87f11ed4a1..329f36b2b6e 100644 --- a/test/tests/parallel/set3/mta/smtp_delivery.bats +++ b/test/tests/parallel/set3/mta/smtp_delivery.bats @@ -63,46 +63,43 @@ function setup_file() { # TODO: Move to clamav tests (For use when ClamAV is enabled): # _repeat_in_container_until_success_or_timeout 60 "${CONTAINER_NAME}" test -e /var/run/clamav/clamd.ctl - # _send_email --from 'virus@external.tld' --data 'amavis/virus' + # _send_email --from 'virus@external.tld' --data 'amavis/virus.txt' # Required for 'delivers mail to existing alias': - _send_email --to alias1@localhost.localdomain --data 'existing/alias-external' + _send_email --to alias1@localhost.localdomain --header "Subject: Test Message existing-alias-external" # Required for 'delivers mail to existing alias with recipient delimiter': - _send_email --to alias1~test@localhost.localdomain --data 'existing/alias-recipient-delimiter' + _send_email --to alias1~test@localhost.localdomain --header 'Subject: Test Message existing-alias-recipient-delimiter' # Required for 'delivers mail to existing catchall': - _send_email --to wildcard@localdomain2.com --data 'existing/catchall-local' + _send_email --to wildcard@localdomain2.com --header 'Subject: Test Message existing-catchall-local' # Required for 'delivers mail to regexp alias': - _send_email --to test123@localhost.localdomain --data 'existing/regexp-alias-local' + _send_email --to test123@localhost.localdomain --header 'Subject: Test Message existing-regexp-alias-local' # Required for 'rejects mail to unknown user': - _send_email --to nouser@localhost.localdomain --data 'non-existing-user' + _send_email --expect-rejection --to nouser@localhost.localdomain + assert_failure # Required for 'redirects mail to external aliases': - _send_email --to bounce-always@localhost.localdomain --data 'existing/regexp-alias-external' - _send_email --to alias2@localhost.localdomain --data 'existing/alias-local' + _send_email --to bounce-always@localhost.localdomain + _send_email --to alias2@localhost.localdomain # Required for 'rejects spam': - _send_email --from 'spam@external.tld' --data 'amavis/spam' + _send_email --from 'spam@external.tld' --data 'amavis/spam.txt' # Required for 'delivers mail to existing account': - _send_email --data 'existing/user1' - assert_success + _send_email --header 'Subject: Test Message existing-user1' _send_email --to user2@otherdomain.tld - assert_success _send_email --to user3@localhost.localdomain - assert_success - _send_email --to added@localhost.localdomain --data 'existing/added' - assert_success - _send_email --to user1@localhost.localdomain --data 'existing/user-and-cc-local-alias' - assert_success - _send_email --data 'sieve/spam-folder' - assert_success - _send_email --to user2@otherdomain.tld --data 'sieve/pipe' - assert_success + _send_email --to added@localhost.localdomain --header 'Subject: Test Message existing-added' + _send_email \ + --to user1@localhost.localdomain \ + --header 'Subject: Test Message existing-user-and-cc-local-alias' \ + --cc 'alias2@localhost.localdomain' + _send_email --data 'sieve/spam-folder.txt' + _send_email --to user2@otherdomain.tld --data 'sieve/pipe.txt' _run_in_container_bash 'sendmail root < /tmp/docker-mailserver-test/emails/sendmail/root-email.txt' assert_success } function _unsuccessful() { - _send_email --port 465 --auth "${1}" --auth-user "${2}" --auth-password wrongpassword + _send_email --expect-rejection --port 465 --auth "${1}" --auth-user "${2}" --auth-password wrongpassword --quit-after AUTH assert_failure assert_output --partial 'authentication failed' assert_output --partial 'No authentication type succeeded' @@ -110,7 +107,6 @@ function _unsuccessful() { function _successful() { _send_email --port 465 --auth "${1}" --auth-user "${2}" --auth-password mypassword --quit-after AUTH - assert_success assert_output --partial 'Authentication successful' } diff --git a/test/tests/serial/mail_pop3.bats b/test/tests/serial/mail_pop3.bats index 008921e49f3..d815e1eed9e 100644 --- a/test/tests/serial/mail_pop3.bats +++ b/test/tests/serial/mail_pop3.bats @@ -24,12 +24,12 @@ function teardown_file() { _default_teardown ; } } @test 'authentication works' { - _nc_wrapper 'auth/pop3-auth' '-w 1 0.0.0.0 110' + _nc_wrapper 'auth/pop3-auth.txt' '-w 1 0.0.0.0 110' assert_success } @test 'added user authentication works' { - _nc_wrapper 'auth/added-pop3-auth' '-w 1 0.0.0.0 110' + _nc_wrapper 'auth/added-pop3-auth.txt' '-w 1 0.0.0.0 110' assert_success } diff --git a/test/tests/serial/mail_with_imap.bats b/test/tests/serial/mail_with_imap.bats index eeccf888d0a..94f1d5199c5 100644 --- a/test/tests/serial/mail_with_imap.bats +++ b/test/tests/serial/mail_with_imap.bats @@ -21,7 +21,7 @@ function setup_file() { function teardown_file() { _default_teardown ; } @test '(Dovecot) LDAP RIMAP connection and authentication works' { - _nc_wrapper 'auth/imap-auth' '-w 1 0.0.0.0 143' + _nc_wrapper 'auth/imap-auth.txt' '-w 1 0.0.0.0 143' assert_success } @@ -31,8 +31,8 @@ function teardown_file() { _default_teardown ; } } @test '(SASLauthd) RIMAP SMTP authentication works' { - _send_email \ - --auth LOGIN \ + _send_email --expect-rejection \ + --auth PLAIN \ --auth-user user1@localhost.localdomain \ --auth-password mypassword \ --quit-after AUTH @@ -41,20 +41,18 @@ function teardown_file() { _default_teardown ; } _send_email \ --port 465 \ - --auth LOGIN \ + --auth PLAIN \ --auth-user user1@localhost.localdomain \ --auth-password mypassword \ --quit-after AUTH - assert_success assert_output --partial 'Authentication successful' _send_email \ --port 587 \ - --auth LOGIN \ + --auth PLAIN \ --auth-user user1@localhost.localdomain \ --auth-password mypassword \ --quit-after AUTH - assert_success assert_output --partial 'Authentication successful' } diff --git a/test/tests/serial/mail_with_ldap.bats b/test/tests/serial/mail_with_ldap.bats index f2011d229c3..f3ee1cc11bf 100644 --- a/test/tests/serial/mail_with_ldap.bats +++ b/test/tests/serial/mail_with_ldap.bats @@ -248,7 +248,7 @@ function teardown() { # dovecot @test "dovecot: ldap imap connection and authentication works" { - _nc_wrapper 'auth/imap-ldap-auth' '-w 1 0.0.0.0 143' + _nc_wrapper 'auth/imap-ldap-auth.txt' '-w 1 0.0.0.0 143' assert_success } @@ -326,25 +326,26 @@ function teardown() { @test "spoofing (with LDAP): rejects sender forging" { _wait_for_smtp_port_in_container_to_respond dms-test_ldap - _send_email \ - --port 465 -tlsc --auth LOGIN \ + _send_email --expect-rejection \ + --port 465 -tlsc --auth PLAIN \ --auth-user some.user@localhost.localdomain \ --auth-password secret \ --ehlo mail \ --from ldap@localhost.localdomain \ - --data 'auth/ldap-smtp-auth-spoofed' + --data 'auth/ldap-smtp-auth-spoofed.txt' + assert_failure assert_output --partial 'Sender address rejected: not owned by user' } @test "spoofing (with LDAP): accepts sending as alias" { _send_email \ - --port 465 -tlsc --auth LOGIN \ + --port 465 -tlsc --auth PLAIN \ --auth-user some.user@localhost.localdomain \ --auth-password secret \ --ehlo mail \ --from postmaster@localhost.localdomain \ --to some.user@localhost.localdomain \ - --data 'auth/ldap-smtp-auth-spoofed-alias' + --data 'auth/ldap-smtp-auth-spoofed-alias.txt' assert_output --partial 'End data with' } @@ -353,20 +354,21 @@ function teardown() { # Template used has invalid AUTH: https://github.com/docker-mailserver/docker-mailserver/pull/3006#discussion_r1073321432 skip 'TODO: This test seems to have been broken from the start (?)' - _send_email \ - --port 465 -tlsc --auth LOGIN \ + _send_email --expect-rejection \ + --port 465 -tlsc --auth PLAIN \ --auth-user some.user.email@localhost.localdomain \ --auth-password secret \ --ehlo mail \ --from randomspoofedaddress@localhost.localdomain \ --to some.user@localhost.localdomain \ - --data 'auth/ldap-smtp-auth-spoofed-sender-with-filter-exception' + --data 'auth/ldap-smtp-auth-spoofed-sender-with-filter-exception.txt' + assert_failure assert_output --partial 'Sender address rejected: not owned by user' } @test "saslauthd: ldap smtp authentication" { - _send_email \ - --auth LOGIN \ + _send_email --expect-rejection \ + --auth PLAIN \ --auth-user some.user@localhost.localdomain \ --auth-password wrongpassword \ --quit-after AUTH @@ -379,12 +381,11 @@ function teardown() { --auth-user some.user@localhost.localdomain \ --auth-password secret \ --quit-after AUTH - assert_success assert_output --partial 'Authentication successful' _send_email \ --port 587 -tls \ - --auth LOGIN \ + --auth PLAIN \ --auth-user some.user@localhost.localdomain \ --auth-password secret \ --quit-after AUTH diff --git a/test/tests/serial/tests.bats b/test/tests/serial/tests.bats index 752e325e487..a344bd8d481 100644 --- a/test/tests/serial/tests.bats +++ b/test/tests/serial/tests.bats @@ -80,12 +80,12 @@ function teardown_file() { _default_teardown ; } } @test "imap: authentication works" { - _nc_wrapper 'auth/imap-auth' '-w 1 0.0.0.0 143' + _nc_wrapper 'auth/imap-auth.txt' '-w 1 0.0.0.0 143' assert_success } @test "imap: added user authentication works" { - _nc_wrapper 'auth/added-imap-auth' '-w 1 0.0.0.0 143' + _nc_wrapper 'auth/added-imap-auth.txt' '-w 1 0.0.0.0 143' assert_success } @@ -293,13 +293,13 @@ EOF # An authenticated user cannot use an envelope sender (MAIL FROM) # address they do not own according to `main.cf:smtpd_sender_login_maps` lookup - _send_email \ - --port 465 -tlsc --auth LOGIN \ + _send_email --expect-rejection \ + --port 465 -tlsc --auth PLAIN \ --auth-user added@localhost.localdomain \ --auth-password mypassword \ --ehlo mail \ --from user2@localhost.localdomain \ - --data 'auth/added-smtp-auth-spoofed' + --data 'auth/added-smtp-auth-spoofed.txt' assert_output --partial 'Sender address rejected: not owned by user' } @@ -310,12 +310,12 @@ EOF # to each table. Address is authorized when a result that maps to # the DMS account is returned. _send_email \ - --port 465 -tlsc --auth LOGIN \ + --port 465 -tlsc --auth PLAIN \ --auth-user user1@localhost.localdomain \ --auth-password mypassword \ --ehlo mail \ --from alias1@localhost.localdomain \ - --data 'auth/added-smtp-auth-spoofed-alias' + --data 'auth/added-smtp-auth-spoofed-alias.txt' assert_success assert_output --partial 'End data with' } diff --git a/test/tests/serial/vmail-id.bats b/test/tests/serial/vmail-id.bats index 0f54ea969ae..2541f4f8ea6 100644 --- a/test/tests/serial/vmail-id.bats +++ b/test/tests/serial/vmail-id.bats @@ -20,7 +20,7 @@ function setup_file() { function teardown_file() { _default_teardown ; } @test 'should successfully deliver mail' { - _send_email --data 'existing/user1' + _send_email --header 'Subject: Test Message existing-user1' _wait_for_empty_mail_queue_in_container # Should be successfully sent (received) by Postfix: @@ -31,7 +31,7 @@ function teardown_file() { _default_teardown ; } # Verify successful delivery via Dovecot to `/var/mail` account by searching for the subject: _repeat_in_container_until_success_or_timeout 20 "${CONTAINER_NAME}" grep -R \ - 'Subject: Test Message existing-user1.txt' \ + 'Subject: Test Message existing-user1' \ '/var/mail/localhost.localdomain/user1/new/' assert_success _should_output_number_of_lines 1 From 52c4582f7b431e69fd8eae67b97bbb330614d0b5 Mon Sep 17 00:00:00 2001 From: Keval Kapdee Date: Fri, 12 Jan 2024 20:45:14 +0000 Subject: [PATCH 240/592] feat: Auth - OAuth2 (Dovecot PassDB) (#3480) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .gitignore | 1 + CHANGELOG.md | 7 +++ Dockerfile | 7 +++ README.md | 1 + docs/content/config/advanced/auth-oauth2.md | 69 +++++++++++++++++++++ docs/content/config/environment.md | 22 ++++++- docs/mkdocs.yml | 1 + mailserver.env | 12 ++++ target/dovecot/10-auth.conf | 1 + target/dovecot/auth-oauth2.conf.ext | 7 +++ target/dovecot/dovecot-oauth2.conf.ext | 4 ++ target/scripts/start-mailserver.sh | 5 ++ target/scripts/startup/setup.d/oauth2.sh | 11 ++++ target/scripts/startup/variables-stack.sh | 6 ++ test/config/oauth2/provider.py | 56 +++++++++++++++++ test/files/auth/imap-oauth2-auth.txt | 4 ++ test/tests/serial/mail_with_oauth2.bats | 66 ++++++++++++++++++++ 17 files changed, 278 insertions(+), 2 deletions(-) create mode 100644 docs/content/config/advanced/auth-oauth2.md create mode 100644 target/dovecot/auth-oauth2.conf.ext create mode 100644 target/dovecot/dovecot-oauth2.conf.ext create mode 100644 target/scripts/startup/setup.d/oauth2.sh create mode 100644 test/config/oauth2/provider.py create mode 100644 test/files/auth/imap-oauth2-auth.txt create mode 100644 test/tests/serial/mail_with_oauth2.bats diff --git a/.gitignore b/.gitignore index 50d22a228ec..79a4dc3cd85 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ ################################################# .env +compose.override.yaml docs/site/ docker-data/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 382b9efaae5..e388194183c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Features + + - **Authentication with OIDC / OAuth 2.0** 🎉 + - DMS now supports authentication via OAuth2 (_via `XOAUTH2` or `OAUTHBEARER` SASL mechanisms_) from capable services (_like Roundcube_). + - This does not replace the need for an `ACCOUNT_PROVISIONER` (`FILE` / `LDAP`), which is required for an account to receive or send mail. + - Successful authentication (_via Dovecot PassDB_) still requires an existing account (_lookup via Dovecot UserDB_). + ### Updates - **Tests**: diff --git a/Dockerfile b/Dockerfile index 65d818ab773..2118622d77c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -108,6 +108,13 @@ EOF COPY target/rspamd/local.d/ /etc/rspamd/local.d/ COPY target/rspamd/scores.d/* /etc/rspamd/scores.d/ +# ----------------------------------------------- +# --- OAUTH2 ------------------------------------ +# ----------------------------------------------- + +COPY target/dovecot/auth-oauth2.conf.ext /etc/dovecot/conf.d +COPY target/dovecot/dovecot-oauth2.conf.ext /etc/dovecot + # ----------------------------------------------- # --- LDAP & SpamAssassin's Cron ---------------- # ----------------------------------------------- diff --git a/README.md b/README.md index 8918c2e9d8d..e81d21add5d 100644 --- a/README.md +++ b/README.md @@ -48,3 +48,4 @@ If you have issues, please search through [the documentation][documentation::web - Support for [LetsEncrypt](https://letsencrypt.org/), manual and self-signed certificates - A [setup script](https://docker-mailserver.github.io/docker-mailserver/latest/config/setup.sh) for easy configuration and maintenance - SASLauthd with LDAP authentication +- OAuth2 authentication (_via `XOAUTH2` or `OAUTHBEARER` SASL mechanisms_) diff --git a/docs/content/config/advanced/auth-oauth2.md b/docs/content/config/advanced/auth-oauth2.md new file mode 100644 index 00000000000..963a6c2ca09 --- /dev/null +++ b/docs/content/config/advanced/auth-oauth2.md @@ -0,0 +1,69 @@ +--- +title: 'Advanced | Basic OAuth2 Authentication' +--- + +## Introduction + +!!! warning "This is only a supplement to the existing account provisioners" + + Accounts must still be managed via the configured [`ACCOUNT_PROVISIONER`][env::account-provisioner] (FILE or LDAP). + + Reasoning for this can be found in [#3480][gh-pr::oauth2]. Future iterations on this feature may allow it to become a full account provisioner. + +[gh-pr::oauth2]: https://github.com/docker-mailserver/docker-mailserver/pull/3480 +[env::account-provisioner]: ../environment.md#account_provisioner + +The present OAuth2 support provides the capability for 3rd-party applications such as Roundcube to authenticate with DMS (dovecot) by using a token obtained from an OAuth2 provider, instead of passing passwords around. + +## Example (Authentik & Roundcube) + +This example assumes you have: + +- A working DMS server set up +- An Authentik server set up ([documentation](https://goauthentik.io/docs/installation/)) +- A Roundcube server set up (either [docker](https://hub.docker.com/r/roundcube/roundcubemail/) or [bare metal](https://github.com/roundcube/roundcubemail/wiki/Installation)) + +!!! example "Setup Instructions" + + === "1. Docker Mailserver" + Edit the following values in `mailserver.env`: + ```env + # ----------------------------------------------- + # --- OAUTH2 Section ---------------------------- + # ----------------------------------------------- + + # empty => OAUTH2 authentication is disabled + # 1 => OAUTH2 authentication is enabled + ENABLE_OAUTH2=1 + + # Specify the user info endpoint URL of the oauth2 provider + OAUTH2_INTROSPECTION_URL=https://authentik.example.com/application/o/userinfo/ + ``` + + === "2. Authentik" + 1. Create a new OAuth2 provider + 2. Note the client id and client secret + 3. Set the allowed redirect url to the equivalent of `https://roundcube.example.com/index.php/login/oauth` for your RoundCube instance. + + === "3. Roundcube" + Add the following to `oauth2.inc.php` ([documentation](https://github.com/roundcube/roundcubemail/wiki/Configuration)): + + ```php + $config['oauth_provider'] = 'generic'; + $config['oauth_provider_name'] = 'Authentik'; + $config['oauth_client_id'] = ''; + $config['oauth_client_secret'] = ''; + $config['oauth_auth_uri'] = 'https://authentik.example.com/application/o/authorize/'; + $config['oauth_token_uri'] = 'https://authentik.example.com/application/o/token/'; + $config['oauth_identity_uri'] = 'https://authentik.example.com/application/o/userinfo/'; + + // Optional: disable SSL certificate check on HTTP requests to OAuth server. For possible values, see: + // http://docs.guzzlephp.org/en/stable/request-options.html#verify + $config['oauth_verify_peer'] = false; + + $config['oauth_scope'] = 'email openid profile'; + $config['oauth_identity_fields'] = ['email']; + + // Boolean: automatically redirect to OAuth login when opening Roundcube without a valid session + $config['oauth_login_redirect'] = false; + ``` diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index c3c074fdf0e..61e604aa5df 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -54,7 +54,15 @@ The Group ID assigned to the static vmail group for `/var/mail` (_Mail storage m Configures the provisioning source of user accounts (including aliases) for user queries and authentication by services managed by DMS (_Postfix and Dovecot_). -User provisioning via OIDC is planned for the future, see [this tracking issue](https://github.com/docker-mailserver/docker-mailserver/issues/2713). +!!! tip "OAuth2 Support" + + Presently DMS supports OAuth2 only as an supplementary authentication method. + + - A third-party service must provide a valid token for the user which Dovecot validates with the authentication service provider. To enable this feature reference the [OAuth2 configuration example guide][docs::auth::oauth2-config-guide]. + - User accounts must be provisioned to receive mail via one of the supported `ACCOUNT_PROVISIONER` providers. + - User provisioning via OIDC is planned for the future, see [this tracking issue](https://github.com/docker-mailserver/docker-mailserver/issues/2713). + +[docs::auth::oauth2-config-guide]: ./advanced/auth-oauth2.md - **empty** => use FILE - LDAP => use LDAP authentication @@ -716,9 +724,19 @@ Enable or disable `getmail`. - **5** => `getmail` The number of minutes for the interval. Min: 1; Max: 30; Default: 5. -#### LDAP +#### OAUTH2 + +##### ENABLE_OAUTH2 +- **empty** => OAUTH2 authentication is disabled +- 1 => OAUTH2 authentication is enabled + +##### OAUTH2_INTROSPECTION_URL + +- => Specify the user info endpoint URL of the oauth2 provider (_eg: `https://oauth2.example.com/userinfo/`_) + +#### LDAP ##### LDAP_START_TLS diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 6441dffe685..7df1acbe67c 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -142,6 +142,7 @@ nav: - 'Postfix': config/advanced/override-defaults/postfix.md - 'Modifications via Script': config/advanced/override-defaults/user-patches.md - 'LDAP Authentication': config/advanced/auth-ldap.md + - 'OAuth2 Authentication': config/advanced/auth-oauth2.md - 'Email Filtering with Sieve': config/advanced/mail-sieve.md - 'Email Gathering with Fetchmail': config/advanced/mail-fetchmail.md - 'Email Gathering with Getmail': config/advanced/mail-getmail.md diff --git a/mailserver.env b/mailserver.env index 0c2e1e40ebc..c9793a99704 100644 --- a/mailserver.env +++ b/mailserver.env @@ -419,6 +419,18 @@ ENABLE_GETMAIL=0 # The number of minutes for the interval. Min: 1; Max: 30. GETMAIL_POLL=5 +# ----------------------------------------------- +# --- OAUTH2 Section ---------------------------- +# ----------------------------------------------- + +# empty => OAUTH2 authentication is disabled +# 1 => OAUTH2 authentication is enabled +ENABLE_OAUTH2= + +# Specify the user info endpoint URL of the oauth2 provider +# Example: https://oauth2.example.com/userinfo/ +OAUTH2_INTROSPECTION_URL= + # ----------------------------------------------- # --- LDAP Section ------------------------------ # ----------------------------------------------- diff --git a/target/dovecot/10-auth.conf b/target/dovecot/10-auth.conf index f71289e97fb..260832fb3cf 100644 --- a/target/dovecot/10-auth.conf +++ b/target/dovecot/10-auth.conf @@ -123,6 +123,7 @@ auth_mechanisms = plain login #!include auth-sql.conf.ext #!include auth-ldap.conf.ext !include auth-passwdfile.inc +#!include auth-oauth2.conf.ext #!include auth-checkpassword.conf.ext #!include auth-vpopmail.conf.ext #!include auth-static.conf.ext diff --git a/target/dovecot/auth-oauth2.conf.ext b/target/dovecot/auth-oauth2.conf.ext new file mode 100644 index 00000000000..6096d1e4794 --- /dev/null +++ b/target/dovecot/auth-oauth2.conf.ext @@ -0,0 +1,7 @@ +auth_mechanisms = $auth_mechanisms oauthbearer xoauth2 + +passdb { + driver = oauth2 + mechanisms = xoauth2 oauthbearer + args = /etc/dovecot/dovecot-oauth2.conf.ext +} diff --git a/target/dovecot/dovecot-oauth2.conf.ext b/target/dovecot/dovecot-oauth2.conf.ext new file mode 100644 index 00000000000..6998ed08ab6 --- /dev/null +++ b/target/dovecot/dovecot-oauth2.conf.ext @@ -0,0 +1,4 @@ +introspection_url = +# Dovecot defaults: +introspection_mode = auth +username_attribute = email diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index 2129b74aa70..1f3522296bf 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -71,6 +71,11 @@ function _register_functions() { ;; esac + if [[ ${ENABLE_OAUTH2} -eq 1 ]]; then + _environment_variables_oauth2 + _register_setup_function '_setup_oauth2' + fi + if [[ ${ENABLE_SASLAUTHD} -eq 1 ]]; then _environment_variables_saslauthd _register_setup_function '_setup_saslauthd' diff --git a/target/scripts/startup/setup.d/oauth2.sh b/target/scripts/startup/setup.d/oauth2.sh new file mode 100644 index 00000000000..20e9ffd1366 --- /dev/null +++ b/target/scripts/startup/setup.d/oauth2.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +function _setup_oauth2() { + _log 'debug' 'Setting up OAUTH2' + + # Enable OAuth2 PassDB (Authentication): + sedfile -i -e '/\!include auth-oauth2\.conf\.ext/s/^#//' /etc/dovecot/conf.d/10-auth.conf + _replace_by_env_in_file 'OAUTH2_' '/etc/dovecot/dovecot-oauth2.conf.ext' + + return 0 +} diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index 2660ce89761..fc38a39db88 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -151,6 +151,12 @@ function __environment_variables_general_setup() { VARS[UPDATE_CHECK_INTERVAL]="${UPDATE_CHECK_INTERVAL:=1d}" } +function _environment_variables_oauth2() { + _log 'debug' 'Setting OAUTH2-related environment variables now' + + VARS[OAUTH2_INTROSPECTION_URL]="${OAUTH2_INTROSPECTION_URL:=}" +} + # This function handles environment variables related to LDAP. # NOTE: SASLAuthd and Dovecot LDAP support inherit these common ENV. function _environment_variables_ldap() { diff --git a/test/config/oauth2/provider.py b/test/config/oauth2/provider.py new file mode 100644 index 00000000000..22fc8129daa --- /dev/null +++ b/test/config/oauth2/provider.py @@ -0,0 +1,56 @@ +# OAuth2 mock service +# +# Dovecot will query this service with the token it was provided. +# If the session for the token is valid, a response provides an attribute to perform a UserDB lookup on (default: email). + +import json +import base64 +from http.server import BaseHTTPRequestHandler, HTTPServer + +# OAuth2.0 Bearer token (paste into https://jwt.io/ to check it's contents). +# You should never need to edit this unless you REALLY need to change the issuer. +token = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vcHJvdmlkZXIuZXhhbXBsZS50ZXN0OjgwMDAvIiwic3ViIjoiODJjMWMzMzRkY2M2ZTMxMWFlNGFhZWJmZTk0NmM1ZTg1OGYwNTVhZmYxY2U1YTM3YWE3Y2M5MWFhYjE3ZTM1YyIsImF1ZCI6Im1haWxzZXJ2ZXIiLCJ1aWQiOiI4OU4zR0NuN1M1Y090WkZNRTVBeVhNbmxURFdVcnEzRmd4YWlyWWhFIn0.zuCytArbphhJn9XT_y9cBdGqDCNo68tBrtOwPIsuKNyF340SaOuZa0xarZofygytdDpLtYr56QlPTKImi-n1ZWrHkRZkwrQi5jQ-j_n2hEAL0vUToLbDnXYfc5q2w7z7X0aoCmiK8-fV7Kx4CVTM7riBgpElf6F3wNAIcX6R1ijUh6ISCL0XYsdogf8WUNZipXY-O4R7YHXdOENuOp3G48hWhxuUh9PsUqE5yxDwLsOVzCTqg9S5gxPQzF2eCN9J0I2XiIlLKvLQPIZ2Y_K7iYvVwjpNdgb4xhm9wuKoIVinYkF_6CwIzAawBWIDJAbix1IslkUPQMGbupTDtOgTiQ" + +# This is the string the user-facing client (e.g. Roundcube) should send via IMAP to Dovecot. +# We include the user and the above token separated by '\1' chars as per the XOAUTH2 spec. +xoauth2 = base64.b64encode(f"user=user1@localhost.localdomain\1auth=Bearer {token}\1\1".encode("utf-8")) +# If changing the user above, use the new output from the below line with the contents of the AUTHENTICATE command in test/test-files/auth/imap-oauth2-auth.txt +print("XOAUTH2 string: " + str(xoauth2)) + + +class HTTPRequestHandler(BaseHTTPRequestHandler): + def do_GET(self): + auth = self.headers.get("Authorization") + if auth is None: + self.send_response(401) + self.end_headers() + return + if len(auth.split()) != 2: + self.send_response(401) + self.end_headers() + return + auth = auth.split()[1] + # Valid session, respond with JSON containing the expected `email` claim to match as Dovecot username: + if auth == token: + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps({ + "email": "user1@localhost.localdomain", + "email_verified": True, + "sub": "82c1c334dcc6e311ae4aaebfe946c5e858f055aff1ce5a37aa7cc91aab17e35c" + }).encode("utf-8")) + else: + self.send_response(401) + self.end_headers() + +server = HTTPServer(('', 80), HTTPRequestHandler) +print("Starting server", flush=True) + +try: + server.serve_forever() +except KeyboardInterrupt: + print() + print("Received keyboard interrupt") +finally: + print("Exiting") diff --git a/test/files/auth/imap-oauth2-auth.txt b/test/files/auth/imap-oauth2-auth.txt new file mode 100644 index 00000000000..825fabdacc2 --- /dev/null +++ b/test/files/auth/imap-oauth2-auth.txt @@ -0,0 +1,4 @@ +a0 NOOP See test/config/oauth2/provider.py to generate the below XOAUTH2 string +a1 AUTHENTICATE XOAUTH2 dXNlcj11c2VyMUBsb2NhbGhvc3QubG9jYWxkb21haW4BYXV0aD1CZWFyZXIgZXlKaGJHY2lPaUpTVXpJMU5pSXNJblI1Y0NJNklrcFhWQ0o5LmV5SnBjM01pT2lKb2RIUndPaTh2Y0hKdmRtbGtaWEl1WlhoaGJYQnNaUzUwWlhOME9qZ3dNREF2SWl3aWMzVmlJam9pT0RKak1XTXpNelJrWTJNMlpUTXhNV0ZsTkdGaFpXSm1aVGswTm1NMVpUZzFPR1l3TlRWaFptWXhZMlUxWVRNM1lXRTNZMk01TVdGaFlqRTNaVE0xWXlJc0ltRjFaQ0k2SW0xaGFXeHpaWEoyWlhJaUxDSjFhV1FpT2lJNE9VNHpSME51TjFNMVkwOTBXa1pOUlRWQmVWaE5ibXhVUkZkVmNuRXpSbWQ0WVdseVdXaEZJbjAuenVDeXRBcmJwaGhKbjlYVF95OWNCZEdxRENObzY4dEJydE93UElzdUtOeUYzNDBTYU91WmEweGFyWm9meWd5dGREcEx0WXI1NlFsUFRLSW1pLW4xWldySGtSWmt3clFpNWpRLWpfbjJoRUFMMHZVVG9MYkRuWFlmYzVxMnc3ejdYMGFvQ21pSzgtZlY3S3g0Q1ZUTTdyaUJncEVsZjZGM3dOQUljWDZSMWlqVWg2SVNDTDBYWXNkb2dmOFdVTlppcFhZLU80UjdZSFhkT0VOdU9wM0c0OGhXaHh1VWg5UHNVcUU1eXhEd0xzT1Z6Q1RxZzlTNWd4UFF6RjJlQ045SjBJMlhpSWxMS3ZMUVBJWjJZX0s3aVl2VndqcE5kZ2I0eGhtOXd1S29JVmluWWtGXzZDd0l6QWF3QldJREpBYml4MUlzbGtVUFFNR2J1cFREdE9nVGlRAQE= +a2 EXAMINE INBOX +a3 LOGOUT diff --git a/test/tests/serial/mail_with_oauth2.bats b/test/tests/serial/mail_with_oauth2.bats new file mode 100644 index 00000000000..0d73bc549a4 --- /dev/null +++ b/test/tests/serial/mail_with_oauth2.bats @@ -0,0 +1,66 @@ +load "${REPOSITORY_ROOT}/test/helper/setup" +load "${REPOSITORY_ROOT}/test/helper/common" + +BATS_TEST_NAME_PREFIX='[OAuth2] ' +CONTAINER1_NAME='dms-test_oauth2' +CONTAINER2_NAME='dms-test_oauth2_provider' + +function setup_file() { + export DMS_TEST_NETWORK='test-network-oauth2' + export DMS_DOMAIN='example.test' + export FQDN_MAIL="mail.${DMS_DOMAIN}" + export FQDN_OAUTH2="oauth2.${DMS_DOMAIN}" + + # Link the test containers to separate network: + # NOTE: If the network already exists, test will fail to start. + docker network create "${DMS_TEST_NETWORK}" + + # Setup local oauth2 provider service: + docker run --rm -d --name "${CONTAINER2_NAME}" \ + --hostname "${FQDN_OAUTH2}" \ + --network "${DMS_TEST_NETWORK}" \ + --volume "${REPOSITORY_ROOT}/test/config/oauth2/:/app/" \ + docker.io/library/python:latest \ + python /app/provider.py + + _run_until_success_or_timeout 20 sh -c "docker logs ${CONTAINER2_NAME} 2>&1 | grep 'Starting server'" + + # + # Setup DMS container + # + + # Add OAUTH2 configuration so that Dovecot can reach out to our mock provider (CONTAINER2) + local ENV_OAUTH2_CONFIG=( + --env ENABLE_OAUTH2=1 + --env OAUTH2_INTROSPECTION_URL=http://oauth2.example.test/userinfo/ + ) + + export CONTAINER_NAME=${CONTAINER1_NAME} + local CUSTOM_SETUP_ARGUMENTS=( + "${ENV_OAUTH2_CONFIG[@]}" + + --hostname "${FQDN_MAIL}" + --network "${DMS_TEST_NETWORK}" + ) + + _init_with_defaults + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + _wait_for_tcp_port_in_container 143 + + # Set default implicit container fallback for helpers: + export CONTAINER_NAME=${CONTAINER1_NAME} +} + +function teardown_file() { + docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}" + docker network rm "${DMS_TEST_NETWORK}" +} + + +@test "oauth2: imap connect and authentication works" { + # An initial connection needs to be made first, otherwise the auth attempt fails + _run_in_container_bash 'nc -vz 0.0.0.0 143' + + _nc_wrapper 'auth/imap-oauth2-auth.txt' '-w 1 0.0.0.0 143' + assert_output --partial 'Examine completed' +} From 71e1102749680545f1ac6cb1a2863ea1714e989c Mon Sep 17 00:00:00 2001 From: Casper Date: Fri, 12 Jan 2024 23:48:14 +0100 Subject: [PATCH 241/592] Tiny #3480 follow up: Add missing ENABLE_OAUTH2 var (#3775) --- target/scripts/startup/variables-stack.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index fc38a39db88..0b351a9ec82 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -83,6 +83,7 @@ function __environment_variables_general_setup() { VARS[ENABLE_FETCHMAIL]="${ENABLE_FETCHMAIL:=0}" VARS[ENABLE_GETMAIL]="${ENABLE_GETMAIL:=0}" VARS[ENABLE_MANAGESIEVE]="${ENABLE_MANAGESIEVE:=0}" + VARS[ENABLE_OAUTH2]="${ENABLE_OAUTH2:=0}" VARS[ENABLE_OPENDKIM]="${ENABLE_OPENDKIM:=1}" VARS[ENABLE_OPENDMARC]="${ENABLE_OPENDMARC:=1}" VARS[ENABLE_POLICYD_SPF]="${ENABLE_POLICYD_SPF:=1}" From e3331b0f44b1d9c1c513389ad5b6f3b09e51dc16 Mon Sep 17 00:00:00 2001 From: Joerg Sonnenberger Date: Sat, 13 Jan 2024 09:37:20 +0100 Subject: [PATCH 242/592] feat: Add MTA-STS support for outbound mail (#3592) * feat: add support for MTA-STS for outgoing mails * Hook-up mta-sts-daemon into basic process handling test * fix: Call python script directly The python3 shebang will run it, which will now meet the expectations of the process testing via pgrep. fail2ban has the same approach. --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 8 +++-- Dockerfile | 9 ++++++ docs/content/config/best-practices/mta-sts.md | 30 +++++++++++++++++++ docs/content/config/environment.md | 9 ++++++ docs/mkdocs.yml | 3 +- mailserver.env | 6 ++++ target/mta-sts-daemon/mta-sts-daemon.yml | 7 +++++ target/scripts/build/packages.sh | 2 +- target/scripts/start-mailserver.sh | 5 ++++ target/scripts/startup/daemons-stack.sh | 1 + target/scripts/startup/setup.d/mail_state.sh | 2 ++ target/scripts/startup/setup.d/mta-sts.sh | 7 +++++ target/supervisor/conf.d/supervisor-app.conf | 12 ++++++++ .../process_check_restart.bats | 4 +++ 14 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 docs/content/config/best-practices/mta-sts.md create mode 100644 target/mta-sts-daemon/mta-sts-daemon.yml create mode 100644 target/scripts/startup/setup.d/mta-sts.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index e388194183c..df9c698721e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,14 @@ All notable changes to this project will be documented in this file. The format ### Features - - **Authentication with OIDC / OAuth 2.0** 🎉 - - DMS now supports authentication via OAuth2 (_via `XOAUTH2` or `OAUTHBEARER` SASL mechanisms_) from capable services (_like Roundcube_). +- **Authentication with OIDC / OAuth 2.0** 🎉 + - DMS now supports authentication via OAuth2 (_via `XOAUTH2` or `OAUTHBEARER` SASL mechanisms_) from capable services (_like Roundcube_). - This does not replace the need for an `ACCOUNT_PROVISIONER` (`FILE` / `LDAP`), which is required for an account to receive or send mail. - Successful authentication (_via Dovecot PassDB_) still requires an existing account (_lookup via Dovecot UserDB_). +- **MTA-STS** (_Optional support for mandatory outgoing TLS encryption_) + - If enabled and the outbound recipient has an MTA-STS policy set, TLS is mandatory for delivering to that recipient. + - Enable via the ENV `ENABLE_MTA_STS=1` + - Supported by major email service providers like Gmail, Yahoo and Outlook. ### Updates diff --git a/Dockerfile b/Dockerfile index 2118622d77c..e822632a022 100644 --- a/Dockerfile +++ b/Dockerfile @@ -199,6 +199,15 @@ COPY target/opendmarc/opendmarc.conf /etc/opendmarc.conf COPY target/opendmarc/default-opendmarc /etc/default/opendmarc COPY target/opendmarc/ignore.hosts /etc/opendmarc/ignore.hosts +# -------------------------------------------------- +# --- postfix-mta-sts-daemon ----------------------- +# -------------------------------------------------- +COPY target/mta-sts-daemon/mta-sts-daemon.yml /etc/mta-sts-daemon.yml +RUN < DNS block lists are disabled - 1 => DNS block lists are enabled +##### ENABLE_MTA_STS + +Enables MTA-STS support for outbound mail. + +- **0** => Disabled +- 1 => Enabled + +See [MTA-STS](best-practices/mta-sts.md) for further explanation. + ##### ENABLE_OPENDKIM Enables the OpenDKIM service. diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 7df1acbe67c..aaaaf51b183 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -122,8 +122,9 @@ nav: - 'Environment Variables': config/environment.md - 'User Management': config/user-management.md - 'Best Practices': - - 'DKIM, DMARC & SPF': config/best-practices/dkim_dmarc_spf.md - 'Auto-discovery': config/best-practices/autodiscover.md + - 'DKIM, DMARC & SPF': config/best-practices/dkim_dmarc_spf.md + - 'MTA-STS': config/best-practices/mta-sts.md - 'Security': - 'Understanding the Ports': config/security/understanding-the-ports.md - 'SSL/TLS': config/security/ssl.md diff --git a/mailserver.env b/mailserver.env index c9793a99704..49bc2cca801 100644 --- a/mailserver.env +++ b/mailserver.env @@ -354,6 +354,12 @@ POSTFIX_REJECT_UNKNOWN_CLIENT_HOSTNAME=0 # Note: More details at http://www.postfix.org/postconf.5.html#inet_protocols POSTFIX_INET_PROTOCOLS=all +# Enables MTA-STS support for outbound mail. +# More details: https://docker-mailserver.github.io/docker-mailserver/latest/config/advanced/mail-mta-sts/ +# - **0** ==> MTA-STS disabled +# - 1 => MTA-STS enabled +ENABLE_MTA_STS=0 + # Choose TCP/IP protocols for dovecot to use # **all** => Listen on all interfaces # ipv4 => Listen only on IPv4 interfaces. Most likely you want this behind Docker. diff --git a/target/mta-sts-daemon/mta-sts-daemon.yml b/target/mta-sts-daemon/mta-sts-daemon.yml new file mode 100644 index 00000000000..4d5d5e556b5 --- /dev/null +++ b/target/mta-sts-daemon/mta-sts-daemon.yml @@ -0,0 +1,7 @@ +# Docs: https://github.com/Snawoot/postfix-mta-sts-resolver/blob/master/man/mta-sts-daemon.yml.5.adoc +path: /var/run/mta-sts/daemon.sock +mode: 0666 +cache: + type: sqlite + options: + filename: "/var/lib/mta-sts/cache.db" diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index e3607f4882b..e57cfe0761b 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -68,7 +68,7 @@ function _install_packages() { ) POSTFIX_PACKAGES=( - pflogsumm postgrey postfix-ldap + pflogsumm postgrey postfix-ldap postfix-mta-sts-resolver postfix-pcre postfix-policyd-spf-python postsrsd ) diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index 1f3522296bf..56dfa1fb301 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -120,6 +120,11 @@ function _register_functions() { _register_setup_function '_setup_apply_fixes_after_configuration' _register_setup_function '_environment_variables_export' + if [[ ${ENABLE_MTA_STS} -eq 1 ]]; then + _register_setup_function '_setup_mta_sts' + _register_start_daemon '_start_daemon_mta_sts_daemon' + fi + # ? >> Daemons _register_start_daemon '_start_daemon_cron' diff --git a/target/scripts/startup/daemons-stack.sh b/target/scripts/startup/daemons-stack.sh index 5476fc9fec9..a4cecf67ad1 100644 --- a/target/scripts/startup/daemons-stack.sh +++ b/target/scripts/startup/daemons-stack.sh @@ -38,6 +38,7 @@ function _start_daemon_opendkim { _default_start_daemon 'opendkim' ; function _start_daemon_opendmarc { _default_start_daemon 'opendmarc' ; } function _start_daemon_postgrey { _default_start_daemon 'postgrey' ; } function _start_daemon_postsrsd { _default_start_daemon 'postsrsd' ; } +function _start_daemon_mta_sts_daemon { _default_start_daemon 'mta-sts-daemon' ; } function _start_daemon_rspamd { _default_start_daemon 'rspamd' ; } function _start_daemon_rspamd_redis { _default_start_daemon 'rspamd-redis' ; } function _start_daemon_rsyslog { _default_start_daemon 'rsyslog' ; } diff --git a/target/scripts/startup/setup.d/mail_state.sh b/target/scripts/startup/setup.d/mail_state.sh index 73c2515be30..9963bbcc82e 100644 --- a/target/scripts/startup/setup.d/mail_state.sh +++ b/target/scripts/startup/setup.d/mail_state.sh @@ -24,6 +24,7 @@ function _setup_save_states() { [[ ${ENABLE_FAIL2BAN} -eq 1 ]] && SERVICEDIRS+=('lib/fail2ban') [[ ${ENABLE_FETCHMAIL} -eq 1 ]] && SERVICEDIRS+=('lib/fetchmail') [[ ${ENABLE_GETMAIL} -eq 1 ]] && SERVICEDIRS+=('lib/getmail') + [[ ${ENABLE_MTA_STS} -eq 1 ]] && SERVICEDIRS+=('lib/mta-sts') [[ ${ENABLE_POSTGREY} -eq 1 ]] && SERVICEDIRS+=('lib/postgrey') [[ ${ENABLE_RSPAMD} -eq 1 ]] && SERVICEDIRS+=('lib/rspamd') [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] && SERVICEDIRS+=('lib/redis') @@ -84,6 +85,7 @@ function _setup_save_states() { [[ ${ENABLE_AMAVIS} -eq 1 ]] && chown -R amavis:amavis "${STATEDIR}/lib-amavis" [[ ${ENABLE_CLAMAV} -eq 1 ]] && chown -R clamav:clamav "${STATEDIR}/lib-clamav" [[ ${ENABLE_FETCHMAIL} -eq 1 ]] && chown -R fetchmail:nogroup "${STATEDIR}/lib-fetchmail" + [[ ${ENABLE_MTA_STS} -eq 1 ]] && chown -R _mta-sts:_mta-sts "${STATEDIR}/lib-mta-sts" [[ ${ENABLE_POSTGREY} -eq 1 ]] && chown -R postgrey:postgrey "${STATEDIR}/lib-postgrey" [[ ${ENABLE_RSPAMD} -eq 1 ]] && chown -R _rspamd:_rspamd "${STATEDIR}/lib-rspamd" [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] && chown -R redis:redis "${STATEDIR}/lib-redis" diff --git a/target/scripts/startup/setup.d/mta-sts.sh b/target/scripts/startup/setup.d/mta-sts.sh new file mode 100644 index 00000000000..7d1f88eaf1e --- /dev/null +++ b/target/scripts/startup/setup.d/mta-sts.sh @@ -0,0 +1,7 @@ +#!/bin/bash + + +function _setup_mta_sts() { + _log 'trace' 'Adding MTA-STS lookup to the Postfix TLS policy map' + _add_to_or_update_postfix_main smtp_tls_policy_maps 'socketmap:unix:/var/run/mta-sts/daemon.sock:postfix' +} diff --git a/target/supervisor/conf.d/supervisor-app.conf b/target/supervisor/conf.d/supervisor-app.conf index 431357d8c52..d64d3d72fdf 100644 --- a/target/supervisor/conf.d/supervisor-app.conf +++ b/target/supervisor/conf.d/supervisor-app.conf @@ -157,3 +157,15 @@ autostart=false stdout_logfile=/var/log/supervisor/%(program_name)s.log stderr_logfile=/var/log/supervisor/%(program_name)s.log command=/bin/bash -l -c /usr/local/bin/update-check.sh + +# Docs: https://github.com/Snawoot/postfix-mta-sts-resolver/blob/master/man/mta-sts-daemon.1.adoc +[program:mta-sts-daemon] +startsecs=0 +stopwaitsecs=55 +autostart=false +autorestart=true +stdout_logfile=/var/log/supervisor/%(program_name)s.log +stderr_logfile=/var/log/supervisor/%(program_name)s.log +command=/usr/bin/mta-sts-daemon --config /etc/mta-sts-daemon.yml +user=_mta-sts +environment=HOME=/var/lib/mta-sts diff --git a/test/tests/parallel/set3/container_configuration/process_check_restart.bats b/test/tests/parallel/set3/container_configuration/process_check_restart.bats index b559d21d7f7..4b01454ecd6 100644 --- a/test/tests/parallel/set3/container_configuration/process_check_restart.bats +++ b/test/tests/parallel/set3/container_configuration/process_check_restart.bats @@ -21,6 +21,7 @@ function teardown() { _default_teardown ; } # dovecot (/usr/sbin/dovecot) # fetchmail (/usr/bin/fetchmail) # fail2ban-server (/usr/bin/python3 /usr/bin/fail2ban-server) - Started by fail2ban-wrapper.sh +# mta-sts-daemon (/usr/bin/bin/python3 /usr/bin/mta-sts-daemon) # postgrey (postgrey) - NOTE: This process lacks path information to match with `--full` in pgrep / pkill # postsrsd (/usr/sbin/postsrsd) - NOTE: Also matches the wrapper: `/bin/bash /usr/local/bin/postsrsd-wrapper.sh` # saslauthd (/usr/sbin/saslauthd) - x5 of the same process are found running (1 is a parent of 4) @@ -44,6 +45,7 @@ ENV_PROCESS_LIST=( dovecot fail2ban-server fetchmail + mta-sts-daemon opendkim opendmarc postgrey @@ -58,6 +60,7 @@ ENV_PROCESS_LIST=( --env ENABLE_CLAMAV=0 --env ENABLE_FAIL2BAN=0 --env ENABLE_FETCHMAIL=0 + --env ENABLE_MTA_STS=0 --env ENABLE_OPENDKIM=0 --env ENABLE_OPENDMARC=0 --env ENABLE_POSTGREY=0 @@ -93,6 +96,7 @@ ENV_PROCESS_LIST=( --env ENABLE_AMAVIS=1 --env ENABLE_FAIL2BAN=1 --env ENABLE_FETCHMAIL=1 + --env ENABLE_MTA_STS=1 --env ENABLE_OPENDKIM=1 --env ENABLE_OPENDMARC=1 --env FETCHMAIL_PARALLEL=1 From f794f65caa5479763fe9ce149b175d08f9f37b18 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 14 Jan 2024 14:08:27 +0100 Subject: [PATCH 243/592] docs: updated `CONTRIBUTORS.md` (#3777) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 301 +++++++++++++++++++++++++----------------------- 1 file changed, 154 insertions(+), 147 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 05a4e6e4dc1..53254c78ce2 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -321,125 +321,132 @@ Thanks goes to these wonderful people ✨ - - egavard + + kiliant
- egavard + kiliant - - mathuin + + robertdolca
- mathuin + robertdolca - - dashohoxha + + okainov
- dashohoxha + okainov - - jamebus + + lukecyca
- jamebus + lukecyca - - lukecyca + + jsonn
- lukecyca + jsonn - - okainov + + jamebus
- okainov + jamebus - - robertdolca + + dashohoxha
- robertdolca + dashohoxha - - kiliant + + mathuin
- kiliant + mathuin - - m-schmoock + + egavard
- m-schmoock + egavard - - mjung + + weo
- mjung + weo - - VanVan + + Zehir
- VanVan + Zehir - - andreasgerstmayr + + guardiande
- andreasgerstmayr + guardiande - - davidszp + + kamuri
- davidszp + kamuri - - kamuri + + davidszp
- kamuri + davidszp - - guardiande + + andreasgerstmayr
- guardiande + andreasgerstmayr - - Zehir + + VanVan
- Zehir + VanVan - - weo + + mjung
- weo + mjung + + + + + m-schmoock +
+ m-schmoock @@ -469,15 +476,15 @@ Thanks goes to these wonderful people ✨
Starbix - + + citec
citec - - + yajo @@ -492,13 +499,6 @@ Thanks goes to these wonderful people ✨ analogue - - - MakerMatrix -
- MakerMatrix - - Rubytastic2 @@ -514,10 +514,10 @@ Thanks goes to these wonderful people ✨ - - jsonn + + MakerMatrix
- jsonn + MakerMatrix @@ -642,26 +642,33 @@ Thanks goes to these wonderful people ✨ yogo1212 + + + willtho89 +
+ willtho89 + + + mpanneck
mpanneck - - + - - willtho89 + + aminvakil
- willtho89 + aminvakil - - ubenmackin + + elbracht
- ubenmackin + elbracht @@ -679,17 +686,10 @@ Thanks goes to these wonderful people ✨ - - aminvakil -
- aminvakil - - - - - elbracht + + ubenmackin
- elbracht + ubenmackin @@ -780,10 +780,17 @@ Thanks goes to these wonderful people ✨ - - fl42 + + jamesfryer
- fl42 + jamesfryer + + + + + eltociear +
+ eltociear @@ -801,17 +808,10 @@ Thanks goes to these wonderful people ✨ - - jamesfryer -
- jamesfryer - - - - - eltociear + + fl42
- eltociear + fl42 @@ -944,10 +944,10 @@ Thanks goes to these wonderful people ✨ - - 0xflotus + + nilshoell
- 0xflotus + nilshoell @@ -1388,6 +1388,13 @@ Thanks goes to these wonderful people ✨ mchamplain + + + 0xflotus +
+ 0xflotus + + auchri @@ -1415,15 +1422,15 @@ Thanks goes to these wonderful people ✨
damianmoore - + + espitall
espitall - - + dkarski @@ -1458,15 +1465,15 @@ Thanks goes to these wonderful people ✨
mazzz1y - + + aydodo
aydodo - - + vedtam @@ -1501,15 +1508,15 @@ Thanks goes to these wonderful people ✨
ErikEngerd - + + huncode
huncode - - + felixn @@ -1544,15 +1551,15 @@ Thanks goes to these wonderful people ✨
20th - + + 2b
2b - - + askz @@ -1587,15 +1594,15 @@ Thanks goes to these wonderful people ✨
alexanderneu - + + ch3sh1r
ch3sh1r - - + eglia @@ -1630,15 +1637,15 @@ Thanks goes to these wonderful people ✨
MrFreezeex - + + arunvc
arunvc - - + astrocket @@ -1673,15 +1680,22 @@ Thanks goes to these wonderful people ✨
crash7 - + + fkefer
fkefer - - + + + + KCrawley +
+ KCrawley + + khuedoan @@ -1709,7 +1723,8 @@ Thanks goes to these wonderful people ✨
linhandev - + + luke- @@ -1723,8 +1738,7 @@ Thanks goes to these wonderful people ✨
LucidityCrash - - + MadsRC @@ -1752,7 +1766,8 @@ Thanks goes to these wonderful people ✨
dragetd - + + michaeljensen @@ -1766,8 +1781,7 @@ Thanks goes to these wonderful people ✨
exhuma - - + milas @@ -1795,7 +1809,8 @@ Thanks goes to these wonderful people ✨
mpldr - + + naveensrinivasan @@ -1809,8 +1824,7 @@ Thanks goes to these wonderful people ✨
neuralp - - + radicand @@ -1818,13 +1832,6 @@ Thanks goes to these wonderful people ✨ radicand - - - nilshoell -
- nilshoell - - frugan-dev @@ -1845,15 +1852,15 @@ Thanks goes to these wonderful people ✨
glandais - + + GiovanH
GiovanH - - + harryyoud @@ -1888,15 +1895,15 @@ Thanks goes to these wonderful people ✨
jcalfee - + + mivek
mivek - - + init-js @@ -1931,15 +1938,15 @@ Thanks goes to these wonderful people ✨
jmccl - + + jurekbarth
jurekbarth - - + JOduMonT @@ -1962,10 +1969,10 @@ Thanks goes to these wonderful people ✨ - - KCrawley + + thechubbypanda
- KCrawley + thechubbypanda From 3a142f97264eff31e3d72a3ce7420ccdb8e3aa4d Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 14 Jan 2024 23:14:03 +0100 Subject: [PATCH 244/592] tests: small adjustments (#3772) --- CHANGELOG.md | 2 +- test/files/emails/amavis/spam.txt | 6 ------ test/helper/sending.bash | 15 +++++++++++---- .../parallel/set1/spam_virus/rspamd_full.bats | 18 ++++++++---------- .../set1/spam_virus/spam_junk_folder.bats | 2 +- .../tests/parallel/set3/mta/smtp_delivery.bats | 2 +- 6 files changed, 22 insertions(+), 23 deletions(-) delete mode 100644 test/files/emails/amavis/spam.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index df9c698721e..0ade3c9ba27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ All notable changes to this project will be documented in this file. The format ### Updates - **Tests**: - - Refactored mail sending ([#3747](https://github.com/docker-mailserver/docker-mailserver/pull/3747)): + - Refactored mail sending ([#3747](https://github.com/docker-mailserver/docker-mailserver/pull/3747) & [#3772](https://github.com/docker-mailserver/docker-mailserver/pull/3772)): - This change is a follow-up to [#3732](https://github.com/docker-mailserver/docker-mailserver/pull/3732) from DMS v13.2. - `swaks` version is now the latest from Github releases instead of the Debian package. - `_nc_wrapper`, `_send_mail` and related helpers expect the `.txt` filepath extension again. diff --git a/test/files/emails/amavis/spam.txt b/test/files/emails/amavis/spam.txt deleted file mode 100644 index e8d26138668..00000000000 --- a/test/files/emails/amavis/spam.txt +++ /dev/null @@ -1,6 +0,0 @@ -From: Docker Mail Server -To: Existing Local User -Date: Sat, 22 May 2010 07:43:25 -0400 -Subject: Test Message amavis/spam.txt -This is a test mail. -XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X diff --git a/test/helper/sending.bash b/test/helper/sending.bash index 529e36aa365..e18dc1ac9cd 100644 --- a/test/helper/sending.bash +++ b/test/helper/sending.bash @@ -141,14 +141,12 @@ function _send_email_and_get_id() { # Get rid of ${1} so only the arguments for swaks remain shift 1 - local QUEUE_ID # The unique ID Postfix (and other services) use may be different in length - # on different systems (e.g. amd64 (11) vs aarch64 (10)). Hence, we use a - # range to safely capture it. + # on different systems. Hence, we use a range to safely capture it. local QUEUE_ID_REGEX='[A-Z0-9]{9,12}' _wait_for_empty_mail_queue_in_container - local OUTPUT=$(_send_email "${@}" --header "Message-Id: ${MID}") + _send_email "${@}" --header "Message-Id: ${MID}" _wait_for_empty_mail_queue_in_container # We store Postfix's queue ID first @@ -164,3 +162,12 @@ function _send_email_and_get_id() { run echo "${ID_ENV_VAR_REF}" assert_line --regexp "^${QUEUE_ID_REGEX}\|${MID}$" } + +# Send a spam e-mail by utilizing GTUBE. +# +# Extra arguments given to this function will be supplied by `_send_email_and_get_id` directly. +function _send_spam() { + _send_email_and_get_id MAIL_ID_SPAM "${@}" \ + --from 'spam@external.tld' \ + --body 'XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X' +} diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index b83ac3535c3..f66e923185f 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -43,20 +43,18 @@ function setup_file() { _wait_for_service postfix _wait_for_smtp_port_in_container - # ref: https://rspamd.com/doc/gtube_patterns.html - local GTUBE_SUFFIX='*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X' - # We will send 4 emails: - # 1. The first one should pass just fine + # 1. The first one should pass just fine _send_email_and_get_id MAIL_ID_PASS - # 2. The second one should be rejected (GTUBE pattern) - _send_email_and_get_id MAIL_ID_REJECT --expect-rejection --body "XJS${GTUBE_SUFFIX}" - # 3. The third one should be rejected due to a virus (ClamAV EICAR pattern) + # 2. The second one should be rejected (Rspamd-specific GTUBE pattern for rejection) + _send_spam --expect-rejection + # 3. The third one should be rejected due to a virus (ClamAV EICAR pattern) # shellcheck disable=SC2016 _send_email_and_get_id MAIL_ID_VIRUS --expect-rejection \ --body 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*' - # 4. The fourth one will receive an added header (GTUBE pattern) - _send_email_and_get_id MAIL_ID_HEADER --body "YJS${GTUBE_SUFFIX}" + # 4. The fourth one will receive an added header (Rspamd-specific GTUBE pattern for adding a spam header) + # ref: https://rspamd.com/doc/gtube_patterns.html + _send_email_and_get_id MAIL_ID_HEADER --body "YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X" _run_in_container cat /var/log/mail.log assert_success @@ -122,7 +120,7 @@ function teardown_file() { _default_teardown ; } _service_log_should_contain_string 'rspamd' 'S \(reject\)' _service_log_should_contain_string 'rspamd' 'reject "Gtube pattern"' - _print_mail_log_for_id "${MAIL_ID_REJECT}" + _print_mail_log_for_id "${MAIL_ID_SPAM}" assert_output --partial 'milter-reject' assert_output --partial '5.7.1 Gtube pattern' diff --git a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats index 2c3a522d70b..15ec4fe106e 100644 --- a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats +++ b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats @@ -95,7 +95,7 @@ function teardown() { _default_teardown ; } function _should_send_spam_message() { _wait_for_smtp_port_in_container _wait_for_tcp_port_in_container 10024 # port 10024 is for Amavis - _send_email --from 'spam@external.tld' --data 'amavis/spam.txt' + _send_spam } function _should_be_received_by_amavis() { diff --git a/test/tests/parallel/set3/mta/smtp_delivery.bats b/test/tests/parallel/set3/mta/smtp_delivery.bats index 329f36b2b6e..e851d94e9b4 100644 --- a/test/tests/parallel/set3/mta/smtp_delivery.bats +++ b/test/tests/parallel/set3/mta/smtp_delivery.bats @@ -81,7 +81,7 @@ function setup_file() { _send_email --to bounce-always@localhost.localdomain _send_email --to alias2@localhost.localdomain # Required for 'rejects spam': - _send_email --from 'spam@external.tld' --data 'amavis/spam.txt' + _send_spam # Required for 'delivers mail to existing account': _send_email --header 'Subject: Test Message existing-user1' From 1449629479ebf9dbd1e63fc78b0d7b9c1b6cd2ff Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Mon, 15 Jan 2024 11:23:23 +1300 Subject: [PATCH 245/592] fix: Revert quoting `SA_SPAM_SUBJECT` in `mailserver.env` (#3767) In Docker Compose `.env` files are parsed properly when values are wrapped with quotes. Trailing white-space is also discarded, like it would be with shell variables. This is not the case with `docker run` or other CRI like `podman` (_including it's compose equivalent support_). Those will parse the quotes to be included in a literal string value. Trailing white-space is also retained. Hence a default with a trailing space is not compatible across CRI. This change documents the default with additional context on how to include a trailing white-space with a custom value for the users CRI choice. It additionally clearly communicates the opt-out value for this feature. --- mailserver.env | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mailserver.env b/mailserver.env index 49bc2cca801..9b085c9f058 100644 --- a/mailserver.env +++ b/mailserver.env @@ -401,7 +401,10 @@ SA_TAG2=6.31 SA_KILL=10.0 # add tag to subject if spam detected -SA_SPAM_SUBJECT='***SPAM*** ' +# The value `undef` opts-out of this feature. The value shown below is the default. +# NOTE: By default spam is delivered to a junk folder, reducing the value of adding a subject prefix. +# NOTE: If not using Docker Compose, other CRI may require the single quotes removed. +#SA_SPAM_SUBJECT='***SPAM*** ' # ----------------------------------------------- # --- Fetchmail Section ------------------------- From ce6ebcc0219eca96ead8d758973f14d94dbb7efe Mon Sep 17 00:00:00 2001 From: Den Date: Mon, 15 Jan 2024 02:10:03 +0200 Subject: [PATCH 246/592] docs: Rspamd DKIM config simplify via `path` setting (#3702) docs: Rspamd DKIM config (`dkim_signing.conf`) example has been simplified via `path` + `selector` settings. --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/config/best-practices/dkim_dmarc_spf.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md index da496aa5d25..ed56504c995 100644 --- a/docs/content/config/best-practices/dkim_dmarc_spf.md +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -156,6 +156,13 @@ DKIM is currently supported by either OpenDKIM or Rspamd: use_esld = true; check_pubkey = true; # you want to use this in the beginning + selector = "mail"; + # The path location is searched for a DKIM key with these variables: + # - `$domain` is sourced from the MIME mail message `From` header + # - `$selector` is configured for `mail` (as a default fallback) + path = "/tmp/docker-mailserver/dkim/keys/$domain/$selector.private"; + + # domain specific configurations can be provided below: domain { example.com { path = "/tmp/docker-mailserver/rspamd/dkim/mail.private"; From 265440b2bbc77971b579bd00ad92a89e8c02d9f3 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Mon, 15 Jan 2024 22:34:15 +1300 Subject: [PATCH 247/592] fix: Ensure `.svbin` files are newer than `.sieve` source files (#3779) --- CHANGELOG.md | 3 ++- target/scripts/startup/setup-stack.sh | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ade3c9ba27..31c4819f786 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,9 +34,10 @@ All notable changes to this project will be documented in this file. The format - **Docs:** - Revised the SpamAssassin ENV docs to better communicate configuration and their relation to other ENV settings. ([#3756](https://github.com/docker-mailserver/docker-mailserver/pull/3756)) - ### Fixes +- **Dovecot:** + - During container startup for Dovecot Sieve, `.sievec` source files compiled to `.svbin` now have their `mtime` adjusted post setup to ensure it is always older than the associated `.svbin` file. This avoids superfluous error logs for sieve scripts that don't actually need to be compiled again ([#3779](https://github.com/docker-mailserver/docker-mailserver/pull/3779)) - **Internal:** - `.gitattributes`: Always use LF line endings on checkout for files with shell script content ([#3755](https://github.com/docker-mailserver/docker-mailserver/pull/3755)) - Fix missing 'jaq' binary for ARM architecture ([#3766](https://github.com/docker-mailserver/docker-mailserver/pull/3766)) diff --git a/target/scripts/startup/setup-stack.sh b/target/scripts/startup/setup-stack.sh index 060dadb2947..f55cb548416 100644 --- a/target/scripts/startup/setup-stack.sh +++ b/target/scripts/startup/setup-stack.sh @@ -20,6 +20,16 @@ function _setup() { ${FUNC} done + _setup_post +} + +function _setup_post() { + # Dovecot `.svbin` files must have a newer mtime than their `.sieve` source files, + # Modifications during setup to these files sometimes results in a common mtime value. + # Handled during post-setup as setup of Dovecot Sieve scripts is not centralized. + find /usr/lib/dovecot/ -iname '*.sieve' -exec touch -d '2 seconds ago' {} + + find /usr/lib/dovecot/ -iname '*.svbin' -exec touch -d '1 seconds ago' {} + + # All startup modifications to configs should have taken place before calling this: _prepare_for_change_detection } From 2bf52342500726e86362ee1ea5954b7d96533d32 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jan 2024 10:18:13 +1300 Subject: [PATCH 248/592] chore(deps): Bump anchore/scan-action from 3.4.0 to 3.5.0 (#3782) Bumps [anchore/scan-action](https://github.com/anchore/scan-action) from 3.4.0 to 3.5.0. - [Release notes](https://github.com/anchore/scan-action/releases) - [Changelog](https://github.com/anchore/scan-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/anchore/scan-action/compare/v3.4.0...v3.5.0) --- updated-dependencies: - dependency-name: anchore/scan-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index d7cf6932d83..896ee80eb99 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v3.4.0 + uses: anchore/scan-action@v3.5.0 id: scan with: image: mailserver-testing:ci From 068ceb1d1aaba8b682bf0be34dbc2a134d823643 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 16 Jan 2024 09:38:08 +0100 Subject: [PATCH 249/592] docs: misc improvements (#3773) * correct misc typos We also seem to be favoring `behavior` over `behaviour`. * bump MkDocs version * resolve errors shown when buildg docs * improve the Rspamd page * behaviour -> behavior Streamline the usage of this word. The majority used behavior, so I opted to go with this way of spelling it. * Apply suggestions from code review --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .github/workflows/scripts/docs/build-docs.sh | 2 +- CHANGELOG.md | 1 + docs/content/config/advanced/auth-ldap.md | 2 +- docs/content/config/advanced/ipv6.md | 6 +- .../config/advanced/optional-config.md | 2 +- docs/content/config/advanced/podman.md | 2 +- docs/content/config/debugging.md | 4 +- docs/content/config/environment.md | 4 +- docs/content/config/security/rspamd.md | 61 ++++++++++++++++--- .../examples/use-cases/imap-folders.md | 2 +- docs/content/faq.md | 6 +- docs/content/index.md | 10 +-- docs/mkdocs.yml | 4 +- 13 files changed, 77 insertions(+), 29 deletions(-) diff --git a/.github/workflows/scripts/docs/build-docs.sh b/.github/workflows/scripts/docs/build-docs.sh index aff66ab5812..5d1cab527c9 100755 --- a/.github/workflows/scripts/docs/build-docs.sh +++ b/.github/workflows/scripts/docs/build-docs.sh @@ -10,7 +10,7 @@ docker run \ --user "$(id -u):$(id -g)" \ --volume "${PWD}:/docs" \ --name "build-docs" \ - squidfunk/mkdocs-material:9.2 build --strict + squidfunk/mkdocs-material:9.5 build --strict # Remove unnecessary build artifacts: https://github.com/squidfunk/mkdocs-material/issues/2519 # site/ is the build output folder. diff --git a/CHANGELOG.md b/CHANGELOG.md index 31c4819f786..b871f0ab9a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ All notable changes to this project will be documented in this file. The format - symbol scores for SPF, DKIM & DMARC were updated to more closely align with [RFC7489](https://www.rfc-editor.org/rfc/rfc7489#page-24); please note though that complete alignment is undesirable, because other symbols might be added as well, which changes the overall score calculation again, see [this issue](https://github.com/docker-mailserver/docker-mailserver/issues/3690#issuecomment-1866871996) - **Docs:** - Revised the SpamAssassin ENV docs to better communicate configuration and their relation to other ENV settings. ([#3756](https://github.com/docker-mailserver/docker-mailserver/pull/3756)) + - Detailed how mail received is assigned a spam score by Rspamd and processed accordingly ([#3773](https://github.com/docker-mailserver/docker-mailserver/pull/3773)) ### Fixes diff --git a/docs/content/config/advanced/auth-ldap.md b/docs/content/config/advanced/auth-ldap.md index c275fd6ce22..ea24526d4b6 100644 --- a/docs/content/config/advanced/auth-ldap.md +++ b/docs/content/config/advanced/auth-ldap.md @@ -56,7 +56,7 @@ These variables specify the LDAP filters that dovecot uses to determine if a use This is split into the following two lookups, both using `%u` as the placeholder for the full login name ([see dovecot documentation for a full list of placeholders](https://doc.dovecot.org/configuration_manual/config_file/config_variables/)). Usually you only need to set `DOVECOT_USER_FILTER`, in which case it will be used for both filters. - `DOVECOT_USER_FILTER` is used to get the account details (uid, gid, home directory, quota, ...) of a user. -- `DOVECOT_PASS_FILTER` is used to get the password information of the user, and is in pretty much all cases identical to `DOVECOT_USER_FILTER` (which is the default behaviour if left away). +- `DOVECOT_PASS_FILTER` is used to get the password information of the user, and is in pretty much all cases identical to `DOVECOT_USER_FILTER` (which is the default behavior if left away). If your directory doesn't have the [postfix-book schema](https://github.com/variablenix/ldap-mail-schema/blob/master/postfix-book.schema) installed, then you must change the internal attribute handling for dovecot. For this you have to change the `pass_attr` and the `user_attr` mapping, as shown in the example below: diff --git a/docs/content/config/advanced/ipv6.md b/docs/content/config/advanced/ipv6.md index f035523a3f2..77dfb0c97f9 100644 --- a/docs/content/config/advanced/ipv6.md +++ b/docs/content/config/advanced/ipv6.md @@ -60,7 +60,7 @@ Enable `ip6tables` support so that Docker will manage IPv6 networking rules as w ``` - `experimental: true` is currently required for `ip6tables: true` to work. - - `userland-proxy` setting [can potentially affect connection behaviour][gh-pull-3244-proxy] for local connections. + - `userland-proxy` setting [can potentially affect connection behavior][gh-pull-3244-proxy] for local connections. Now restart the daemon if it's running: `systemctl restart docker`. @@ -99,7 +99,7 @@ Next, configure a network with an IPv6 subnet for your container with any of the ??? tip "Override the implicit `default` network" - You can optionally avoid the service assignment by [overriding the `default` user-defined network that Docker Compose generates](docker-docs-network-compose-default). Just replace `dms-ipv6` with `default`. + You can optionally avoid the service assignment by [overriding the `default` user-defined network that Docker Compose generates][docker-docs-network-compose-default]. Just replace `dms-ipv6` with `default`. The Docker Compose `default` bridge is not affected by settings for the default `bridge` (aka `docker0`) in `/etc/docker/daemon.json`. @@ -180,7 +180,7 @@ curl --max-time 5 http://[2001:db8::1]:80 !!! info "IPv6 ULA address priority" DNS lookups that have records for both IPv4 and IPv6 addresses (_eg: `localhost`_) may prefer IPv4 over IPv6 (ULA) for private addresses, whereas for public addresses IPv6 has priority. This shouldn't be anything to worry about, but can come across as a surprise when testing your IPv6 setup on the same host instead of from a remote client. - + The preference can be controlled with [`/etc/gai.conf`][networking-gai], and appears was configured this way based on [the assumption that IPv6 ULA would never be used with NAT][networking-gai-blog]. It should only affect the destination resolved for outgoing connections, which for IPv6 ULA should only really affect connections between your containers / host. In future [IPv6 ULA may also be prioritized][networking-gai-rfc]. [docker-subnets]: https://straz.to/2021-09-08-docker-address-pools/#what-are-the-default-address-pools-when-no-configuration-is-given-vanilla-pools diff --git a/docs/content/config/advanced/optional-config.md b/docs/content/config/advanced/optional-config.md index 21f82a3b869..31090050f9a 100644 --- a/docs/content/config/advanced/optional-config.md +++ b/docs/content/config/advanced/optional-config.md @@ -35,7 +35,7 @@ This is a list of all configuration files and directories which are optional or - **whitelist_clients.local:** Whitelisted domains, not considered by postgrey. Enter one host or domain per line. - **spamassassin-rules.cf:** Anti-spam rules for Spamassassin. (Docs: [FAQ - SpamAssassin Rules][docs-faq-spamrules]) - **fail2ban-fail2ban.cf:** Additional config options for `fail2ban.cf`. (Docs: [Fail2Ban][docs-fail2ban]) -- **fail2ban-jail.cf:** Additional config options for fail2ban's jail behaviour. (Docs: [Fail2Ban][docs-fail2ban]) +- **fail2ban-jail.cf:** Additional config options for fail2ban's jail behavior. (Docs: [Fail2Ban][docs-fail2ban]) - **amavis.cf:** replaces the `/etc/amavis/conf.d/50-user` file - **dovecot.cf:** replaces `/etc/dovecot/local.conf`. (Docs: [Override Dovecot Defaults][docs-override-dovecot]) - **dovecot-quotas.cf:** list of custom quotas per mailbox. (Docs: [Accounts][docs-accounts-quota]) diff --git a/docs/content/config/advanced/podman.md b/docs/content/config/advanced/podman.md index b32288c1b8b..4cb60e77952 100644 --- a/docs/content/config/advanced/podman.md +++ b/docs/content/config/advanced/podman.md @@ -107,7 +107,7 @@ The `PERMIT_DOCKER` variable in the `mailserver.env` file allows to specify trus #### Use the slip4netns network driver The second workaround is slightly more complicated because the `compose.yaml` has to be modified. -As shown in the [fail2ban section](../../security/fail2ban/#podman-with-slirp4netns-port-driver) the `slirp4netns` network driver has to be enabled. +As shown in the [fail2ban section](../security/fail2ban.md#podman-with-slirp4netns-port-driver) the `slirp4netns` network driver has to be enabled. This network driver enables podman to correctly resolve IP addresses but it is not compatible with user defined networks which might be a problem depending on your setup. diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index d58430e1f2d..2c89948db07 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -107,9 +107,9 @@ This could be from outdated software, or running a system that isn't able to pro - **macOS:** DMS has limited support for macOS. Often an issue encountered is due to permissions related to the `volumes` config in `compose.yaml`. You may have luck [trying `gRPC FUSE`][gh-macos-support] as the file sharing implementation; [`VirtioFS` is the successor][docker-macos-virtiofs] but presently appears incompatible with DMS. - **Kernel:** Some systems provide [kernels with modifications (_replacing defaults and backporting patches_)][network::kernels-modified] to support running legacy software or kernels, complicating compatibility. This can be commonly experienced with products like NAS. -- **CGroups v2:** Hosts running older kernels (prior to 5.2) and systemd (prior to v244) are not likely to leverage cgroup v2, or have not defaulted to the cgroup v2 `unified` hierarchy. Not meeting this baseline may influence the behaviour of your DMS container, even with the latest Docker Engine installed. +- **CGroups v2:** Hosts running older kernels (prior to 5.2) and systemd (prior to v244) are not likely to leverage cgroup v2, or have not defaulted to the cgroup v2 `unified` hierarchy. Not meeting this baseline may influence the behavior of your DMS container, even with the latest Docker Engine installed. - **Container runtime:** Docker and Podman for example have subtle differences. DMS docs are primarily focused on Docker, but we try to document known issues where relevant. -- **Rootless containers:** Introduces additional differences in behaviour or requirements: +- **Rootless containers:** Introduces additional differences in behavior or requirements: - cgroup v2 is required for supporting rootless containers. - Differences such as for container networking which may further affect support for IPv6 and preserving the client IP (Remote address). Example with Docker rootless are [binding a port to a specific interface][docker-rootless-interface] and the choice of [port forwarding driver][docs-rootless-portdriver]. diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 7ae96fcdc49..1aa6799d46d 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -560,7 +560,7 @@ Changes the interval in which log files are rotated. The Amavis action configured by this setting: - - Influences the behaviour of the [`SA_KILL`](#sa_kill) setting. + - Influences the behavior of the [`SA_KILL`](#sa_kill) setting. - Applies to the Amavis config parameters `$final_spam_destiny` and `$final_bad_header_destiny`. !!! note "This ENV setting is related to" @@ -596,7 +596,7 @@ Mail is not yet considered spam at this spam score, but for purposes like diagno ``` !!! info "The `X-Spam-Score` is `4.162`" - + High enough for `SA_TAG` to trigger adding these headers, but not high enough for `SA_TAG2` (_which would set `X-Spam-Flag: YES` instead_). ##### SA_TAG2 diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index fe9bd5ea669..283c2a9597b 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -22,15 +22,59 @@ The following environment variables are related to Rspamd: 5. [`RSPAMD_HFILTER`](../environment.md#rspamd_hfilter) 6. [`RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE`](../environment.md#rspamd_hfilter_hostname_unknown_score) 7. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) -8. [`MOVE_SPAM_TO_JUNK`](../environment.md#move_spam_to_junk) +8. [`MOVE_SPAM_TO_JUNK`][docs-spam-to-junk] 9. [`MARK_SPAM_AS_READ`](../environment.md#mark_spam_as_read) -With these variables, you can enable Rspamd itself and you can enable / disable certain features related to Rspamd. +With these variables, you can enable Rspamd itself, and you can enable / disable certain features related to Rspamd. + +[docs-spam-to-junk]: ../environment.md#move_spam_to_junk ## The Default Configuration ### Mode of Operation +!!! tip "Attention" + + Read this section carefully if you want to understand how Rspamd is integrated into DMS and how it works (on a surface level). + +Rspamd is integrated as a milter into DMS. When enabled, Postfix's `main.cf` configuration file includes the parameter `rspamd_milter = inet:localhost:11332`, which is added to `smtpd_milters`. As a milter, Rspamd can inspect incoming and outgoing e-mails. + +Each mail is assigned what Rspamd calls symbols: when an e-mail matches a specific criterion, the mail receives a symbol. Afterwards, Rspamd applies a _spam score_ (as usual with anti-spam software) to the e-mail. + +- The score itself is calculated by adding the values of the individual symbols applied earlier. The higher the spam score is, the more likely the e-mail is spam. +- Symbol values can be negative (i.e., these symbols indicate the mail is legitimate, maybe because [SPF and DKIM][docs-dkim-dmarc-spf] are verified successfully) or the symbol can be positive (i.e., these symbols indicate the e-mail is spam, maybe because the e-mail contains a lot of links). + +Rspamd then adds (a few) headers to the e-mail based on the spam score. Most important are `X-Spamd-Result`, which contains an overview of which symbols were applied. It could look like this: + +```txt +X-Spamd-Result default: False [-2.80 / 11.00]; R_SPF_NA(1.50)[no SPF record]; R_DKIM_ALLOW(-1.00)[example.com:s=dtag1]; DWL_DNSWL_LOW(-1.00)[example.com:dkim]; RWL_AMI_LASTHOP(-1.00)[192.0.2.42:from]; DMARC_POLICY_ALLOW(-1.00)[example.com,none]; RWL_MAILSPIKE_EXCELLENT(-0.40)[192.0.2.42:from]; FORGED_SENDER(0.30)[noreply@example.com,some-reply-address@bounce.example.com]; RCVD_IN_DNSWL_LOW(-0.10)[192.0.2.42:from]; MIME_GOOD(-0.10)[multipart/mixed,multipart/related,multipart/alternative,text/plain]; MIME_TRACE(0.00)[0:+,1:+,2:+,3:+,4:~,5:~,6:~]; RCVD_COUNT_THREE(0.00)[3]; RCPT_COUNT_ONE(0.00)[1]; REPLYTO_DN_EQ_FROM_DN(0.00)[]; ARC_NA(0.00)[]; TO_MATCH_ENVRCPT_ALL(0.00)[]; RCVD_TLS_LAST(0.00)[]; DKIM_TRACE(0.00)[example.com:+]; HAS_ATTACHMENT(0.00)[]; TO_DN_NONE(0.00)[]; FROM_NEQ_ENVFROM(0.00)[noreply@example.com,some-reply-address@bounce.example.com]; FROM_HAS_DN(0.00)[]; REPLYTO_DOM_NEQ_FROM_DOM(0.00)[]; PREVIOUSLY_DELIVERED(0.00)[receiver@anotherexample.com]; ASN(0.00)[asn:3320, ipnet:192.0.2.0/24, country:DE]; MID_RHS_MATCH_FROM(0.00)[]; MISSING_XM_UA(0.00)[]; HAS_REPLYTO(0.00)[some-reply-address@dms-reply.example.com] +``` + +And then there is a corresponding `X-Rspamd-Action` header, which shows the overall result and the action that is taken. In our example, it would be: + +```txt +X-Rspamd-Action no action +``` + +Since the score is `-2.80`, nothing will happen and the e-mail is not classified as spam. Our custom [`actions.conf`][rspamd-actions-config] defines what to do at certain scores: + +1. At a score of 4, the e-mail is to be _greylisted_; +2. At a score of 6, the e-mail is _marked with a header_ (`X-Spam: Yes`); +3. At a score of 7, the e-mail will additionally have their _subject re-written_ (appending a prefix like `[SPAM]`); +4. At a score of 11, the e-mail is outright _rejected_. + +--- + +There is more to spam analysis than meets the eye: we have not covered the [Bayes training and filters][rspamc-docs-bayes] here, nor have we talked about [Sieve rules for e-mails that are marked as spam][docs-spam-to-junk]. + +Even the calculation of the score with the individual symbols has been presented to you in a simplified manner. But with the knowledge from above, you're equipped to read on and use Rspamd confidently. Keep on reading to understand the integration even better - you will want to know about your anti-spam software, not only to keep the bad e-mail out, but also to make sure the good e-mail arrive properly! + +[docs-dkim-dmarc-spf]: ../best-practices/dkim_dmarc_spf.md +[rspamd-actions-config]: https://github.com/docker-mailserver/docker-mailserver/blob/master/target/rspamd/local.d/actions.conf +[rspamc-docs-bayes]: https://rspamd.com/doc/configuration/statistic.html + +### Workers + The proxy worker operates in [self-scan mode][rspamd-docs-proxy-self-scan-mode]. This simplifies the setup as we do not require a normal worker. You can easily change this though by [overriding the configuration by DMS](#providing-custom-settings-overriding-settings). DMS does not set a default password for the controller worker. You may want to do that yourself. In setups where you already have an authentication provider in front of the Rspamd webpage, you may want to [set the `secure_ip ` option to `"0.0.0.0/0"` for the controller worker](#with-the-help-of-a-custom-file) to disable password authentication inside Rspamd completely. @@ -39,9 +83,9 @@ DMS does not set a default password for the controller worker. You may want to d ### Persistence with Redis -When Rspamd is enabled, we implicitly also start an instance of Redis in the container. Redis is configured to persist it's data via RDB snapshots to disk in the directory `/var/lib/redis` (_which is a symbolic link to `/var/mail-state/lib-redis/` when [`ONE_DIR=1`](../environment.md#one_dir) and a volume is mounted to `/var/mail-state/`_). With the volume mount the snapshot will restore the Redis data across container restarts, and provide a way to keep backup. +When Rspamd is enabled, we implicitly also start an instance of Redis in the container. Redis is configured to persist its data via RDB snapshots to disk in the directory `/var/lib/redis` (_which is a symbolic link to `/var/mail-state/lib-redis/` when [`ONE_DIR=1`](../environment.md#one_dir) and a volume is mounted to `/var/mail-state/`_). With the volume mount the snapshot will restore the Redis data across container restarts, and provide a way to keep backup. -Redis uses `/etc/redis/redis.conf` for configuration. We adjust this file when enabling the internal Redis service. If you have an external instance of Redis to use, the internal Redis service can be opt-out via setting the ENV [`ENABLE_RSPAMD_REDIS=0`](../environment.md#enable_rspamd_redis) (_link also details required changes to the DMS rspamd config_). +Redis uses `/etc/redis/redis.conf` for configuration. We adjust this file when enabling the internal Redis service. If you have an external instance of Redis to use, the internal Redis service can be opt-out via setting the ENV [`ENABLE_RSPAMD_REDIS=0`](../environment.md#enable_rspamd_redis) (_link also details required changes to the DMS Rspamd config_). ### Web Interface @@ -77,13 +121,13 @@ You can find a list of all Rspamd modules [on their website][rspamd-docs-modules #### Disabled By Default -DMS disables certain modules (clickhouse, elastic, neural, reputation, spamassassin, url_redirector, metric_exporter) by default. We believe these are not required in a standard setup, and they would otherwise needlessly use system resources. +DMS disables certain modules (`clickhouse`, `elastic`, `neural`, `reputation`, `spamassassin`, `url_redirector`, `metric_exporter`) by default. We believe these are not required in a standard setup, and they would otherwise needlessly use system resources. #### Anti-Virus (ClamAV) You can choose to enable ClamAV, and Rspamd will then use it to check for viruses. Just set the environment variable `ENABLE_CLAMAV=1`. -#### RBLs (Realtime Blacklists) / DNSBLs (DNS-based Blacklists) +#### RBLs (Real-time Blacklists) / DNSBLs (DNS-based Blacklists) The [RBL module](https://rspamd.com/doc/modules/rbl.html) is enabled by default. As a consequence, Rspamd will perform DNS lookups to a variety of blacklists. Whether an RBL or a DNSBL is queried depends on where the domain name was obtained: RBL servers are queried with IP addresses extracted from message headers, DNSBL server are queried with domains and IP addresses extracted from the message body \[[source][rbl-vs-dnsbl]\]. @@ -105,12 +149,15 @@ DMS brings sane default settings for Rspamd. They are located at `/etc/rspamd/lo If you want to overwrite the default settings and / or provide your own settings, you can place files at `docker-data/dms/config/rspamd/override.d/`. Files from this directory are copied to `/etc/rspamd/override.d/` during startup. These files [forcibly override][rspamd-docs-override-dir] Rspamd and DMS default settings. +!!! question "What is the [`local.d` directory and how does it compare to `override.d`][rspamd-docs-config-directories]?" + !!! warning "Clashing Overrides" Note that when also [using the `custom-commands.conf` file](#with-the-help-of-a-custom-file), files in `override.d` may be overwritten in case you adjust them manually and with the help of the file. [rspamd-docs-override-dir]: https://www.rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories [docs-dms-config-volume]: ../../faq.md#what-about-the-docker-datadmsconfig-directory +[rspamd-docs-config-directories]: https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories ### With the Help of a Custom File @@ -127,7 +174,7 @@ where `COMMAND` can be: 3. `set-option-for-module`: sets the value for option `ARGUMENT2` to `ARGUMENT3` inside module `ARGUMENT1` 4. `set-option-for-controller`: set the value of option `ARGUMENT1` to `ARGUMENT2` for the controller worker 5. `set-option-for-proxy`: set the value of option `ARGUMENT1` to `ARGUMENT2` for the proxy worker -6. `set-common-option`: set the option `ARGUMENT1` that [defines basic Rspamd behaviour][rspamd-docs-basic-options] to value `ARGUMENT2` +6. `set-common-option`: set the option `ARGUMENT1` that [defines basic Rspamd behavior][rspamd-docs-basic-options] to value `ARGUMENT2` 7. `add-line`: this will add the complete line after `ARGUMENT1` (with all characters) to the file `/etc/rspamd/override.d/` !!! example "An Example Is [Shown Down Below](#adjusting-and-extending-the-very-basic-configuration)" diff --git a/docs/content/examples/use-cases/imap-folders.md b/docs/content/examples/use-cases/imap-folders.md index ad5f3ed3436..ce90355a1d2 100644 --- a/docs/content/examples/use-cases/imap-folders.md +++ b/docs/content/examples/use-cases/imap-folders.md @@ -61,7 +61,7 @@ Take care to test localized names work well as well. This information is provided by the community. - It presently lacks references to confirm the behaviour. If any information is incorrect please let us know! :smile: + It presently lacks references to confirm the behavior. If any information is incorrect please let us know! :smile: [docs-config-overrides-dovecot]: ../../config/advanced/override-defaults/dovecot.md#override-configuration diff --git a/docs/content/faq.md b/docs/content/faq.md index f666b10226c..0985c0a94f6 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -12,7 +12,7 @@ Mails are stored in `/var/mail/${domain}/${username}`. Since `v9.0.0` it is poss ### How are IMAP mailboxes (_aka IMAP Folders_) set up? -`INBOX` is setup by default with the special IMAP folders `Drafts`, `Sent`, `Junk` and `Trash`. You can learn how to modify or add your own folders (_including additional special folders like `Archive`_) by visiting our docs page [_Customizing IMAP Folders_](../examples/use-cases/imap-folders) for more information. +`INBOX` is setup by default with the special IMAP folders `Drafts`, `Sent`, `Junk` and `Trash`. You can learn how to modify or add your own folders (_including additional special folders like `Archive`_) by visiting our docs page [_Customizing IMAP Folders_](./examples/use-cases/imap-folders.md) for more information. ### How do I update DMS? @@ -128,7 +128,7 @@ find "${PWD}/docker-data/dms-backups/" -type f -mtime +30 -delete ### I Want to Know More About the Ports -See [this part of the documentation](../config/security/understanding-the-ports/) for further details and best practice advice, **especially regarding security concerns**. +See [this part of the documentation](./config/security/understanding-the-ports.md) for further details and best practice advice, **especially regarding security concerns**. ### How can I configure my email client? @@ -210,7 +210,7 @@ baduser@example.com devnull ### What kind of SSL certificates can I use? -Both RSA and ECDSA certs are supported. You can provide your own cert files manually, or mount a `letsencrypt` generated directory (_with alternative support for Traefik's `acme.json`_). Check out the [`SSL_TYPE` documentation](../config/environment/#ssl_type) for more details. +Both RSA and ECDSA certs are supported. You can provide your own cert files manually, or mount a `letsencrypt` generated directory (_with alternative support for Traefik's `acme.json`_). Check out the [`SSL_TYPE` documentation](./config/environment.md#ssl_type) for more details. ### I just moved from my old mail server to DMS, but "it doesn't work"? diff --git a/docs/content/index.md b/docs/content/index.md index ff1214b133e..3739423cf60 100644 --- a/docs/content/index.md +++ b/docs/content/index.md @@ -24,9 +24,9 @@ If you're completely new to mail servers or you want to read up on them, check o There is also a script - [`setup.sh`][github-file-setupsh] - supplied with this project. It supports you in configuring and administrating your server. Information on how to get it and how to use it is available [on a dedicated page][docs-setupsh]. -[docs-introduction]: ./introduction/ -[docs-usage]: ./usage/ -[docs-examples]: ./examples/tutorials/basic-installation/ +[docs-introduction]: ./introduction.md +[docs-usage]: ./usage.md +[docs-examples]: ./examples/tutorials/basic-installation.md [github-file-setupsh]: https://github.com/docker-mailserver/docker-mailserver/blob/master/setup.sh [docs-setupsh]: ./config/setup.sh/ @@ -59,10 +59,10 @@ You might also want to check out: DMS employs a variety of tests. If you want to know more about our test suite, view our [testing docs][docs-tests]. -[docs-tests]: ./contributing/tests/ +[docs-tests]: ./contributing/tests.md ### Contributing We are always happy to welcome new contributors. For guidelines and entrypoints please have a look at the [Contributing section][docs-contributing]. -[docs-contributing]: ./contributing/issues-and-pull-requests/ +[docs-contributing]: ./contributing/issues-and-pull-requests.md diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index aaaaf51b183..8a6a24b02ee 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -86,8 +86,8 @@ markdown_extensions: - pymdownx.inlinehilite - pymdownx.tilde - pymdownx.emoji: - emoji_index: !!python/name:materialx.emoji.twemoji - emoji_generator: !!python/name:materialx.emoji.to_svg + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg - pymdownx.highlight: extend_pygments_lang: - name: yml From 2d59aac5a1f3058287e7563884917cbcca9a5c53 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 17 Jan 2024 20:54:27 +1300 Subject: [PATCH 250/592] chore: Add maintenance comment for `sed` usage (#3789) This is a more explicit reminder for any future contributors that get thrown off by the usage of `sed` here and may be inclined to change it. Add a link to reference a comment where it's already been explored what the alternative `sed` invocations available are. --- target/scripts/startup/setup.d/security/misc.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index 170f46fbec2..36d8905fc1f 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -70,6 +70,8 @@ function __setup__security__spamassassin() { if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]]; then _log 'debug' 'Enabling and configuring SpamAssassin' + # Maintainers should take care in attempting to change these sed commands. Alternatives were already explored: + # https://github.com/docker-mailserver/docker-mailserver/pull/3767#issuecomment-1885989591 # shellcheck disable=SC2016 sed -i -r 's|^\$sa_tag_level_deflt (.*);|\$sa_tag_level_deflt = '"${SA_TAG}"';|g' /etc/amavis/conf.d/20-debian_defaults From 437114c5dd7928037deea1a2e31535b5a20746ad Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 17 Jan 2024 22:46:22 +1300 Subject: [PATCH 251/592] tests: Revise `process_check_restart.bats` (#3780) --- CHANGELOG.md | 1 + .../process_check_restart.bats | 37 ++++++++++--------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b871f0ab9a1..cfdfd314478 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ All notable changes to this project will be documented in this file. The format ### Updates - **Tests**: + - Revised testing of service process management (supervisord) to be more robust ([#3780](https://github.com/docker-mailserver/docker-mailserver/pull/3780)) - Refactored mail sending ([#3747](https://github.com/docker-mailserver/docker-mailserver/pull/3747) & [#3772](https://github.com/docker-mailserver/docker-mailserver/pull/3772)): - This change is a follow-up to [#3732](https://github.com/docker-mailserver/docker-mailserver/pull/3732) from DMS v13.2. - `swaks` version is now the latest from Github releases instead of the Debian package. diff --git a/test/tests/parallel/set3/container_configuration/process_check_restart.bats b/test/tests/parallel/set3/container_configuration/process_check_restart.bats index 4b01454ecd6..0a54a60f7ba 100644 --- a/test/tests/parallel/set3/container_configuration/process_check_restart.bats +++ b/test/tests/parallel/set3/container_configuration/process_check_restart.bats @@ -11,19 +11,16 @@ function teardown() { _default_teardown ; } # Process matching notes: # opendkim (/usr/sbin/opendkim) - x2 of the same process are found running (1 is the parent) # opendmarc (/usr/sbin/opendmarc) -# master (/usr/lib/postfix/sbin/master) - Postfix main process (Can take a few seconds running to be ready) -# NOTE: pgrep or pkill used with `--full` would also match `/usr/sbin/amavisd-new (master)` -# -# amavi (/usr/sbin/amavi) - Matches three processes, the main process is `/usr/sbin/amavisd-new (master)` -# NOTE: `amavisd-new` can only be matched with `--full`, regardless pkill would return `/usr/sbin/amavi` +# postfix (/usr/lib/postfix/sbin/master) - Postfix main process (two ancestors, launched via pidproxy python3 script) # +# amavisd-new (usr/sbin/amavisd-new) # clamd (/usr/sbin/clamd) # dovecot (/usr/sbin/dovecot) # fetchmail (/usr/bin/fetchmail) -# fail2ban-server (/usr/bin/python3 /usr/bin/fail2ban-server) - Started by fail2ban-wrapper.sh +# fail2ban-server (/usr/bin/python3 /usr/bin/fail2ban-server) - NOTE: python3 is due to the shebang # mta-sts-daemon (/usr/bin/bin/python3 /usr/bin/mta-sts-daemon) -# postgrey (postgrey) - NOTE: This process lacks path information to match with `--full` in pgrep / pkill -# postsrsd (/usr/sbin/postsrsd) - NOTE: Also matches the wrapper: `/bin/bash /usr/local/bin/postsrsd-wrapper.sh` +# postgrey (postgrey) - NOTE: This process command uses perl via shebang, but unlike python3 the context is missing +# postsrsd (/usr/sbin/postsrsd) # saslauthd (/usr/sbin/saslauthd) - x5 of the same process are found running (1 is a parent of 4) # Delays: @@ -31,17 +28,16 @@ function teardown() { _default_teardown ; } # dovecot + fail2ban, take approx 1 sec to kill properly # opendkim + opendmarc can take up to 6 sec to kill properly # clamd + postsrsd sometimes take 1-3 sec to restart after old process is killed. -# postfix + fail2ban (due to Wrapper scripts) can delay a restart by up to 5 seconds from usage of sleep. # These processes should always be running: CORE_PROCESS_LIST=( - master + postfix ) # These processes can be toggled via ENV: # NOTE: clamd handled in separate test case ENV_PROCESS_LIST=( - amavi + amavisd-new dovecot fail2ban-server fetchmail @@ -89,7 +85,7 @@ ENV_PROCESS_LIST=( done } -# Average time: 23 seconds (29 with wrapper scripts) +# Average time: 23 seconds @test "(enabled ENV) should restart processes when killed" { export CONTAINER_NAME=${CONTAINER2_NAME} local CONTAINER_ARGS_ENV_CUSTOM=( @@ -153,14 +149,18 @@ function _should_restart_when_killed() { # Wait until process has been running for at least MIN_PROCESS_AGE: # (this allows us to more confidently check the process was restarted) _run_until_success_or_timeout 30 _check_if_process_is_running "${PROCESS}" "${MIN_PROCESS_AGE}" - # NOTE: refute_output doesn't have output to compare to when a run failure is due to a timeout + # NOTE: refute_output will not have any output to compare against if a `run` failure is caused by a timeout assert_success assert_output --partial "${PROCESS}" # Should kill the process successfully: # (which should then get restarted by supervisord) - _run_in_container pkill --echo "${PROCESS}" - assert_output --partial "${PROCESS}" + # NOTE: The process name from `pkill --echo` does not always match the equivalent processs name from `pgrep --list-full`. + # The oldest process returned (if multiple) should be the top-level process launched by supervisord, + # the PID will verify the target process was killed correctly: + local PID=$(_exec_in_container pgrep --full --oldest "${PROCESS}") + _run_in_container pkill --echo --full "${PROCESS}" + assert_output --partial "killed (pid ${PID})" assert_success # Wait until original process is not running: @@ -176,13 +176,16 @@ function _should_restart_when_killed() { assert_output --partial "${PROCESS}" } -# NOTE: CONTAINER_NAME is implicit; it should have be set prior to calling. +# NOTE: CONTAINER_NAME is implicit; it should have been set prior to calling. function _check_if_process_is_running() { local PROCESS=${1} local MIN_SECS_RUNNING [[ -n ${2:-} ]] && MIN_SECS_RUNNING=('--older' "${2}") - local IS_RUNNING=$(docker exec "${CONTAINER_NAME}" pgrep --list-full "${MIN_SECS_RUNNING[@]}" "${PROCESS}") + # `--list-full` provides information for matching against (full process path) + # `--full` allows matching the process against the full path (required if a process is not the exec command, such as started by python3 command without a shebang) + # `--oldest` should select the parent process when there are multiple results, typically the command defined in `supervisor-app.conf` + local IS_RUNNING=$(_exec_in_container pgrep --full --list-full "${MIN_SECS_RUNNING[@]}" --oldest "${PROCESS}") # When no matches are found, nothing is returned. Provide something we can assert on (helpful for debugging): if [[ ! ${IS_RUNNING} =~ ${PROCESS} ]]; then From 9cdbef2b369fb4fb0f1b4e534da8703daf92abc9 Mon Sep 17 00:00:00 2001 From: Andreas Perhab Date: Thu, 18 Jan 2024 10:41:55 +0100 Subject: [PATCH 252/592] setup/dkim: chown created dkim directories and keys to config user (#3783) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 2 ++ target/bin/open-dkim | 3 +++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfdfd314478..3ecf1251c3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,8 @@ All notable changes to this project will be documented in this file. The format ### Fixes +- **Setup:** + - `setup` CLI - `setup dkim domain` now creates the keys files with the user owning the key directory ([#3783](https://github.com/docker-mailserver/docker-mailserver/pull/3783)) - **Dovecot:** - During container startup for Dovecot Sieve, `.sievec` source files compiled to `.svbin` now have their `mtime` adjusted post setup to ensure it is always older than the associated `.svbin` file. This avoids superfluous error logs for sieve scripts that don't actually need to be compiled again ([#3779](https://github.com/docker-mailserver/docker-mailserver/pull/3779)) - **Internal:** diff --git a/target/bin/open-dkim b/target/bin/open-dkim index 86fbfb8181a..808ef8cc9aa 100755 --- a/target/bin/open-dkim +++ b/target/bin/open-dkim @@ -144,6 +144,9 @@ while read -r DKIM_DOMAIN; do --directory="/tmp/docker-mailserver/opendkim/keys/${DKIM_DOMAIN}" fi + # fix permissions to use the same user:group as /tmp/docker-mailserver/opendkim/keys + chown -R "$(stat -c '%U:%G' /tmp/docker-mailserver/opendkim/keys)" "/tmp/docker-mailserver/opendkim/keys/${DKIM_DOMAIN}" + # write to KeyTable if necessary KEYTABLEENTRY="${SELECTOR}._domainkey.${DKIM_DOMAIN} ${DKIM_DOMAIN}:${SELECTOR}:/etc/opendkim/keys/${DKIM_DOMAIN}/${SELECTOR}.private" if [[ ! -f "/tmp/docker-mailserver/opendkim/KeyTable" ]]; then From deb0d2d09a98bd8bc0b0f977dbed590fde28c706 Mon Sep 17 00:00:00 2001 From: Roy Sindre Norangshol Date: Fri, 19 Jan 2024 02:58:20 +0100 Subject: [PATCH 253/592] docs: Guidance for binding outbound SMTP with multiple interfaces available (#3465) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 9 ++- .../use-cases/bind-smtp-network-interface.md | 68 +++++++++++++++++++ docs/mkdocs.yml | 1 + 3 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 docs/content/examples/use-cases/bind-smtp-network-interface.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ecf1251c3d..b6126956c64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,9 +17,15 @@ All notable changes to this project will be documented in this file. The format - Enable via the ENV `ENABLE_MTA_STS=1` - Supported by major email service providers like Gmail, Yahoo and Outlook. +### Added + +- **Docs:** + - An example for how to bind outbound SMTP connections to a specific network interface ([#3465](https://github.com/docker-mailserver/docker-mailserver/pull/3465)) + ### Updates - **Tests**: + - Replace `wc -l` with `grep -c` ([#3752](https://github.com/docker-mailserver/docker-mailserver/pull/3752)) - Revised testing of service process management (supervisord) to be more robust ([#3780](https://github.com/docker-mailserver/docker-mailserver/pull/3780)) - Refactored mail sending ([#3747](https://github.com/docker-mailserver/docker-mailserver/pull/3747) & [#3772](https://github.com/docker-mailserver/docker-mailserver/pull/3772)): - This change is a follow-up to [#3732](https://github.com/docker-mailserver/docker-mailserver/pull/3732) from DMS v13.2. @@ -28,10 +34,9 @@ All notable changes to this project will be documented in this file. The format - `sending.bash` helper methods were refactored to better integrate `swaks` and accommodate different usage contexts. - `test/files/emails/existing/` files were removed similar to previous removal of SMTP auth files as they became redundant with `swaks`. - **Internal:** - - tests: Replace `wc -l` with `grep -c` ([#3752](https://github.com/docker-mailserver/docker-mailserver/pull/3752)) - Postfix is now configured with `smtputf8_enable = no` in our default `main.cf` config (_instead of during container startup_). ([#3750](https://github.com/docker-mailserver/docker-mailserver/pull/3750)) - **Rspamd** ([#3726](https://github.com/docker-mailserver/docker-mailserver/pull/3726)): - - symbol scores for SPF, DKIM & DMARC were updated to more closely align with [RFC7489](https://www.rfc-editor.org/rfc/rfc7489#page-24); please note though that complete alignment is undesirable, because other symbols might be added as well, which changes the overall score calculation again, see [this issue](https://github.com/docker-mailserver/docker-mailserver/issues/3690#issuecomment-1866871996) + - Symbol scores for SPF, DKIM & DMARC were updated to more closely align with [RFC7489](https://www.rfc-editor.org/rfc/rfc7489#page-24). Please note that complete alignment is undesirable as other symbols may be added as well, which changes the overall score calculation again, see [this issue](https://github.com/docker-mailserver/docker-mailserver/issues/3690#issuecomment-1866871996) - **Docs:** - Revised the SpamAssassin ENV docs to better communicate configuration and their relation to other ENV settings. ([#3756](https://github.com/docker-mailserver/docker-mailserver/pull/3756)) - Detailed how mail received is assigned a spam score by Rspamd and processed accordingly ([#3773](https://github.com/docker-mailserver/docker-mailserver/pull/3773)) diff --git a/docs/content/examples/use-cases/bind-smtp-network-interface.md b/docs/content/examples/use-cases/bind-smtp-network-interface.md new file mode 100644 index 00000000000..b12e21dea81 --- /dev/null +++ b/docs/content/examples/use-cases/bind-smtp-network-interface.md @@ -0,0 +1,68 @@ +--- +title: 'Use Cases | Binding outbound SMTP to a specific network' +hide: + - toc +--- + +!!! warning "Advice not extensively tested" + + This configuration advice is a community contribution which has only been verified as a solution when using `network: host`, where you have direct access to the host interfaces. + + It may be applicable in other network modes if the container has control of the outbound IPs to bind to. This is not the case with bridge networks that typically bind to a private range network for containers which are bridged to a public interface via Docker. + +If your Docker host is running multiple IPv4 and IPv6 IP-addresses, it may be beneficial to bind outgoing SMTP connections to specific IP-address / interface. + +- When a mail is sent outbound from DMS, it greets the MTA it is connecting to with a EHLO (DMS FQDN) which might be verified against the IP resolved, and that a `PTR` record for that IP resolves an address back to the same IP. +- A similar check with SPF can be against the envelope-sender address which may verify a DNS record like MX / A is valid (_or a similar restriction check from an MTA like [Postfix has with `reject_unknown_sender`][gh-pr::3465::comment-restrictions]_). +- If the IP address is inconsistent for those connections from DMS, these DNS checks are likely to fail. + +This can be configured by [overriding the default Postfix configurations][docs::overrides-postfix] DMS provides. Create `postfix-master.cf` and `postfix-main.cf` files for your config volume (`docker-data/dms/config`). + +In `postfix-main.cf` you'll have to set the [`smtp_bind_address`][postfix-docs::smtp-bind-address-ipv4] and [`smtp_bind_address6`][postfix-docs::smtp-bind-address-ipv6] +to the respective IP-address on the server you want to use. + +[docs::overrides-postfix]: ../../config/advanced/override-defaults/postfix.md +[postfix-docs::smtp-bind-address-ipv4]: https://www.postfix.org/postconf.5.html#smtp_bind_address +[postfix-docs::smtp-bind-address-ipv6]: https://www.postfix.org/postconf.5.html#smtp_bind_address6 + +!!! example + + === "Contributed solution" + + ```title="postfix-main.cf" + smtp_bind_address = 198.51.100.42 + smtp_bind_address6 = 2001:DB8::42 + ``` + + !!! bug "Inheriting the bind from `main.cf` can misconfigure services" + + One problem when setting `smtp_bind_address` in `main.cf` is that it will be inherited by any services in `master.cf` that extend the `smtp` transport. One of these is `smtp-amavis`, which is explicitly configured to listen / connect via loopback (localhost / `127.0.0.1`). + + A `postfix-master.cf` override can workaround that issue by ensuring `smtp-amavis` binds to the expected internal IP: + + ```title="postfix-master.cf" + smtp-amavis/unix/smtp_bind_address=127.0.0.1 + smtp-amavis/unix/smtp_bind_address6=::1 + ``` + + === "Alternative (unverified)" + + A potentially better solution might be to instead [explicitly set the `smtp_bind_address` override on the `smtp` transport service][gh-pr::3465::alternative-solution]: + + ```title="postfix-master.cf" + smtp/inet/smtp_bind_address = 198.51.100.42 + smtp/inet/smtp_bind_address6 = 2001:DB8::42 + ``` + + If that avoids the concern with `smtp-amavis`, you may still need to additionally override for the [`relay` transport][gh-src::postfix-master-cf::relay-transport] as well if you have configured DMS to relay mail. + +!!! note "IP addresses for documentation" + + IP addresses shown in above examples are placeholders, they are IP addresses reserved for documentation by IANA (_[RFC-5737 (IPv4)][rfc-5737] and [RFC-3849 (IPv6)][rfc-3849]_). Replace them with the IP addresses you want DMS to send mail through. + +[rfc-5737]: https://datatracker.ietf.org/doc/html/rfc5737 +[rfc-3849]: https://datatracker.ietf.org/doc/html/rfc3849 + +[gh-pr::3465::comment-restrictions]: https://github.com/docker-mailserver/docker-mailserver/pull/3465#discussion_r1458114528 +[gh-pr::3465::alternative-solution]: https://github.com/docker-mailserver/docker-mailserver/pull/3465#issuecomment-1678107233 +[gh-src::postfix-master-cf::relay-transport]: https://github.com/docker-mailserver/docker-mailserver/blob/9cdbef2b369fb4fb0f1b4e534da8703daf92abc9/target/postfix/master.cf#L65 diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 8a6a24b02ee..0d34b407ba3 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -167,6 +167,7 @@ nav: - 'Customize IMAP Folders': examples/use-cases/imap-folders.md - 'iOS Mail Push Support': examples/use-cases/ios-mail-push-support.md - 'Lua Authentication': examples/use-cases/auth-lua.md + - 'Bind outbound SMTP to a specific network': examples/use-cases/bind-smtp-network-interface.md - 'FAQ' : faq.md - 'Contributing': - 'General Information': contributing/general.md From a5d536201b84a3617bc2d2722ed454d03ae168be Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 20 Jan 2024 17:51:32 +1300 Subject: [PATCH 254/592] docs: Add maintenance comment for `reject_unknown_sender_domain` (#3793) I figured this was a useful comment to reference related to the setting if it's ever being changed or needs to be better understood (linked issue is a common failure that can be encountered related to this restriction). --- target/postfix/main.cf | 1 + 1 file changed, 1 insertion(+) diff --git a/target/postfix/main.cf b/target/postfix/main.cf index 495ad8a94fe..8c1d4c894e1 100644 --- a/target/postfix/main.cf +++ b/target/postfix/main.cf @@ -68,6 +68,7 @@ smtpd_forbid_bare_newline = yes # smtpd_forbid_bare_newline_exclusions = $mynetworks # Custom defined parameters for DMS: +# reject_unknown_sender_domain: https://github.com/docker-mailserver/docker-mailserver/issues/3716#issuecomment-1868033234 dms_smtpd_sender_restrictions = permit_sasl_authenticated, permit_mynetworks, reject_unknown_sender_domain # Submission ports 587 and 465 support for SPOOF_PROTECTION=1 mua_sender_restrictions = reject_authenticated_sender_login_mismatch, $dms_smtpd_sender_restrictions From f3a7f08f9615de40df800c6179e7563c1a394910 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 20 Jan 2024 22:49:09 +1300 Subject: [PATCH 255/592] tests: Revise OAuth2 tests (#3795) * tests: OAuth2 - Replace Python `/userinfo` endpoint with Caddy Better documented, easier flow and separation of concerns via Caddy. The python code had additional noise related to setting up a basic API which is abstracted away via `Caddyfile` config that's dedicated to this task. * tests: OAuth2 - Minimize noise + Improve test assertion Caddyfile can use an Access Token instead of a JWT. Much smaller and correct for this OAuth2 configuration. This new value has been documented inline. Likewise the `sub` field returned is not important to this test. `email_verified` is kept as it may be helpful for further coverage testing. The actual test-case has better assertions for success and failure by checking for Dovecot logs we expect instead of netcat response. `oauth2` to `auth` for the Caddy container hostname is not necessary, just a more generic subdomain choice. * tests: OAuth2 - Caddyfile `imap/xoauth2` route dynamic via query string This way is more flexible and doesn't require modifying the `Caddyfile` directly, while still easy to use. Additionally simplifies understanding the Caddyfile to maintainers by removing the `route` directive that was required to ensure a deterministic order of vars. * tests: OAuth2 - `/imap/xoauth2` respond with IMAP commands for netcat Since this is the only intended usage, might as well have it respond with the full file content. * tests: OAuth2 - Implement coverage for `OAUTHBEARER` Caddyfile route for `/imap/` now accepts any subpath to support handling both `xoauth2` and `oauthbearer` subpaths. Both SASL mechanisms represent the same information, with `XOAUTH2` being a common mechanism to encounter defined by Google, whilst `OAUTHBEARER` is the newer variant standardized by RFC 7628 but not yet as widely adopted. The request to `/userinfo` endpoint will be the same, only the `credentials` value to be encoded differs. Instead of repeating the block for a similar route, this difference is handled via the Caddyfile `map` directive. We match the path context (_`/xoauth2` or `/oauthbearer`, the `/imap` prefix was stripped by `handle_path` earlier_), when there is a valid match, `sasl_mechanism` and `credentials` map vars are created and assigned to be referenced by the later `respond` directive. --- Repeat the same test-case logic, DRY with log asserts extracted to a common function call. This should be fine as the auth method will be sufficient to match against or a common failure caught. * tests: OAuth2 - Minor revisions Separate test cases and additional comment on creating the same base64 encoded credentials via CLI as an alternative to running Caddy. Added a simple `compose.yaml` for troubleshooting or running the container for the `/imap/xoauth2` / `/imap/oauthbearer` endpoints. * tests: OAuth2 - Route endpoints in Caddyfile with snippets instead `reverse_proxy` was a bit more convenient, but the additional internal ports weren't really relevant. It also added noise to logging when troubleshooting. The `import` directive with Snippet blocks instead is a bit cleaner, but when used in a single file snippets must be defined prior to referencing them with the `import` directive. --- `compose.yaml` inlines the examples, with slight modification to `localhost:80`, since the Caddyfile examples `auth.example.test` is more relevant to the tests which can use it, and not applicable to troubleshooting locally outside of tests. * chore: Add entry to `CHANGELOG.md` * chore: Additional context on access token --- CHANGELOG.md | 1 + test/config/oauth2/Caddyfile | 88 +++++++++++++++++++++ test/config/oauth2/compose.yaml | 15 ++++ test/config/oauth2/provider.py | 56 ------------- test/files/auth/imap-oauth2-auth.txt | 4 - test/files/auth/imap-oauth2-oauthbearer.txt | 4 + test/files/auth/imap-oauth2-xoauth2.txt | 4 + test/tests/serial/mail_with_oauth2.bats | 36 ++++++--- 8 files changed, 136 insertions(+), 72 deletions(-) create mode 100644 test/config/oauth2/Caddyfile create mode 100644 test/config/oauth2/compose.yaml delete mode 100644 test/config/oauth2/provider.py delete mode 100644 test/files/auth/imap-oauth2-auth.txt create mode 100644 test/files/auth/imap-oauth2-oauthbearer.txt create mode 100644 test/files/auth/imap-oauth2-xoauth2.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index b6126956c64..90a905f2102 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ All notable changes to this project will be documented in this file. The format ### Updates - **Tests**: + - Revised OAuth2 test ([#3795](https://github.com/docker-mailserver/docker-mailserver/pull/3795)) - Replace `wc -l` with `grep -c` ([#3752](https://github.com/docker-mailserver/docker-mailserver/pull/3752)) - Revised testing of service process management (supervisord) to be more robust ([#3780](https://github.com/docker-mailserver/docker-mailserver/pull/3780)) - Refactored mail sending ([#3747](https://github.com/docker-mailserver/docker-mailserver/pull/3747) & [#3772](https://github.com/docker-mailserver/docker-mailserver/pull/3772)): diff --git a/test/config/oauth2/Caddyfile b/test/config/oauth2/Caddyfile new file mode 100644 index 00000000000..e116aa5549e --- /dev/null +++ b/test/config/oauth2/Caddyfile @@ -0,0 +1,88 @@ +# Mocked OAuth2 /userinfo endpoint normally provided via an Authorization Server (AS) / Identity Provider (IdP) +# +# Dovecot will query the mocked `/userinfo` endpoint with the OAuth2 bearer token it was provided during login. +# If the session for the token is valid, a response returns an attribute to perform a UserDB lookup on (default: email). + +# `DMS_YWNjZXNzX3Rva2Vu` is the access token our OAuth2 tests expect for an authorization request to be successful. +# - The token was created by base64 encoding the string `access_token`, followed by adding `DMS_` as a prefix. +# - Normally an access token is a short-lived value associated to a login session. The value does not encode any real data. +# It is an opaque token: https://oauth.net/2/bearer-tokens/ + +# NOTE: The main server config is at the end within the `:80 { ... }` block. +# This is because the endpoints are extracted out into Caddy snippets, which must be defined before they're referenced. + +# /userinfo +(route-userinfo) { + vars token "DMS_YWNjZXNzX3Rva2Vu" + + # Expects to match an authorization header with a specific bearer token: + # https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication#authentication_schemes + @auth header Authorization "Bearer {vars.token}" + + # If the provided authorization header has the expected value (bearer token), respond with this JSON payload: + handle @auth { + # JSON inlined via HereDoc string feature: + # Dovecot OAuth2 defaults to `username_attribute = email`, which must be returned in the response to match + # with the `user` credentials field that Dovecot received via base64 encoded IMAP `AUTHENTICATE` value. + respond <&1 | grep 'Starting server'" + _run_until_success_or_timeout 20 bash -c "docker logs ${CONTAINER2_NAME} 2>&1 | grep 'serving initial configuration'" # # Setup DMS container # - # Add OAUTH2 configuration so that Dovecot can reach out to our mock provider (CONTAINER2) + # Add OAuth2 configuration so that Dovecot can query our mocked identity provider (CONTAINER2) local ENV_OAUTH2_CONFIG=( --env ENABLE_OAUTH2=1 - --env OAUTH2_INTROSPECTION_URL=http://oauth2.example.test/userinfo/ + --env OAUTH2_INTROSPECTION_URL=http://auth.example.test/userinfo ) export CONTAINER_NAME=${CONTAINER1_NAME} @@ -49,6 +48,9 @@ function setup_file() { # Set default implicit container fallback for helpers: export CONTAINER_NAME=${CONTAINER1_NAME} + + # An initial connection needs to be made first, otherwise the auth attempts fail + _run_in_container_bash 'nc -vz 0.0.0.0 143' } function teardown_file() { @@ -56,11 +58,21 @@ function teardown_file() { docker network rm "${DMS_TEST_NETWORK}" } +@test "should authenticate with XOAUTH2 over IMAP" { + _nc_wrapper 'auth/imap-oauth2-xoauth2.txt' '-w 1 0.0.0.0 143' + __verify_successful_login 'XOAUTH2' +} + +@test "should authenticate with OAUTHBEARER over IMAP" { + _nc_wrapper 'auth/imap-oauth2-oauthbearer.txt' '-w 1 0.0.0.0 143' + __verify_successful_login 'OAUTHBEARER' +} -@test "oauth2: imap connect and authentication works" { - # An initial connection needs to be made first, otherwise the auth attempt fails - _run_in_container_bash 'nc -vz 0.0.0.0 143' +function __verify_successful_login() { + local AUTH_METHOD=${1} - _nc_wrapper 'auth/imap-oauth2-auth.txt' '-w 1 0.0.0.0 143' - assert_output --partial 'Examine completed' + # Inspect the relevant Dovecot logs to catch failure / success: + _run_in_container grep 'dovecot:' /var/log/mail.log + refute_output --partial 'oauth2 failed: Introspection failed' + assert_output --partial "dovecot: imap-login: Login: user=, method=${AUTH_METHOD}" } From b78978caedfa67e96425a57d4226cd1768474d17 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sat, 20 Jan 2024 12:33:05 +0100 Subject: [PATCH 256/592] release: v13.3.0 (#3781) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 4 +++- VERSION | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90a905f2102..74acdc61776 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v13.2.0...HEAD) +## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v13.3.0...HEAD) > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +## [v13.3.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.3.0) + ### Features - **Authentication with OIDC / OAuth 2.0** 🎉 diff --git a/VERSION b/VERSION index 67aee23940e..ac565bc1cab 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -13.2.0 +13.3.0 From 41b471fb140fc8fe381c11e7d1bd538356fcacba Mon Sep 17 00:00:00 2001 From: Jam Balaya Date: Sun, 21 Jan 2024 20:21:29 +0900 Subject: [PATCH 257/592] fix(typo): comment on mailserver.env (#3799) Thanks! --- mailserver.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mailserver.env b/mailserver.env index 9b085c9f058..1d131696e61 100644 --- a/mailserver.env +++ b/mailserver.env @@ -388,7 +388,7 @@ SPAMASSASSIN_SPAM_TO_INBOX=1 # spam messages will be moved in the Junk folder (SPAMASSASSIN_SPAM_TO_INBOX=1 required) MOVE_SPAM_TO_JUNK=1 -# spam messages wil be marked as read +# spam messages will be marked as read MARK_SPAM_AS_READ=0 # add 'spam info' headers at, or above this level From 3cbcdb2d6517b6c61818afb49c0eb58ee82b6202 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 21 Jan 2024 12:31:10 +0100 Subject: [PATCH 258/592] docs: update `CONTRIBUTORS.md` (#3798) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Casper --- CONTRIBUTORS.md | 215 ++++++++++++++++++++++++++---------------------- 1 file changed, 115 insertions(+), 100 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 53254c78ce2..f86435f4616 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -814,14 +814,21 @@ Thanks goes to these wonderful people ✨ fl42 + + + nilshoell +
+ nilshoell + + + stigok
stigok - - + 5ven @@ -856,15 +863,15 @@ Thanks goes to these wonderful people ✨
thomasschmit - + + Thiritin
Thiritin - - + tweibert @@ -899,15 +906,15 @@ Thanks goes to these wonderful people ✨
k3it - + + Drakulix
Drakulix - - + vilisas @@ -942,13 +949,6 @@ Thanks goes to these wonderful people ✨
allddd - - - - nilshoell -
- nilshoell - @@ -1022,21 +1022,28 @@ Thanks goes to these wonderful people ✨ romansey + + + norrs +
+ norrs + + MightySCollins
MightySCollins - + + 501st-alpha1
501st-alpha1 - - + klamann @@ -1071,15 +1078,15 @@ Thanks goes to these wonderful people ✨
sjmudd - + + simonsystem
simonsystem - - + stephan-devop @@ -1114,15 +1121,15 @@ Thanks goes to these wonderful people ✨
okamidash - + + olaf-mandel
olaf-mandel - - + ontheair81 @@ -1157,15 +1164,15 @@ Thanks goes to these wonderful people ✨
rmlhuk - + + rriski
rriski - - + schnippl0r @@ -1200,15 +1207,15 @@ Thanks goes to these wonderful people ✨
strarsis - + + tamueller
tamueller - - + vivacarvajalito @@ -1243,15 +1250,15 @@ Thanks goes to these wonderful people ✨
arcaine2 - + + awb99
awb99 - - + brainkiller @@ -1286,15 +1293,15 @@ Thanks goes to these wonderful people ✨
eleith - + + ghnp5
ghnp5 - - + helmutundarnold @@ -1329,15 +1336,15 @@ Thanks goes to these wonderful people ✨
ixeft - + + jjtt
jjtt - - + paralax @@ -1372,15 +1379,15 @@ Thanks goes to these wonderful people ✨
marios88 - + + matrixes
matrixes - - + mchamplain @@ -1388,13 +1395,6 @@ Thanks goes to these wonderful people ✨ mchamplain - - - 0xflotus -
- 0xflotus - - auchri @@ -1452,21 +1452,28 @@ Thanks goes to these wonderful people ✨ danielvandenberg95 + + + denisix +
+ denisix + + mlatorre31
mlatorre31 - + + mazzz1y
mazzz1y - - + aydodo @@ -1501,15 +1508,15 @@ Thanks goes to these wonderful people ✨
ekkis - + + ErikEngerd
ErikEngerd - - + huncode @@ -1532,12 +1539,20 @@ Thanks goes to these wonderful people ✨ - - froks + + thechubbypanda
- froks + thechubbypanda + + + 0xflotus +
+ 0xflotus + + + ifokeev @@ -1551,8 +1566,7 @@ Thanks goes to these wonderful people ✨
20th - - + 2b @@ -1580,7 +1594,8 @@ Thanks goes to these wonderful people ✨
vifino - + + kachkaev @@ -1594,8 +1609,7 @@ Thanks goes to these wonderful people ✨
alexanderneu - - + ch3sh1r @@ -1623,7 +1637,8 @@ Thanks goes to these wonderful people ✨
green-anger - + + iRhonin @@ -1637,8 +1652,7 @@ Thanks goes to these wonderful people ✨
MrFreezeex - - + arunvc @@ -1666,7 +1680,8 @@ Thanks goes to these wonderful people ✨
spock - + + erdos4d @@ -1680,14 +1695,6 @@ Thanks goes to these wonderful people ✨
crash7 - - - - - fkefer -
- fkefer - @@ -1716,15 +1723,15 @@ Thanks goes to these wonderful people ✨
LeoWinterDE - + + linhandev
linhandev - - + luke- @@ -1759,15 +1766,15 @@ Thanks goes to these wonderful people ✨
maxemann96 - + + dragetd
dragetd - - + michaeljensen @@ -1802,15 +1809,15 @@ Thanks goes to these wonderful people ✨
MohammedNoureldin - + + mpldr
mpldr - - + naveensrinivasan @@ -1832,6 +1839,21 @@ Thanks goes to these wonderful people ✨ radicand + + + froks +
+ froks + + + + + fkefer +
+ fkefer + + + frugan-dev @@ -1852,8 +1874,7 @@ Thanks goes to these wonderful people ✨
glandais - - + GiovanH @@ -1874,7 +1895,8 @@ Thanks goes to these wonderful people ✨
HeySora - + + sirgantrithon @@ -1895,8 +1917,7 @@ Thanks goes to these wonderful people ✨
jcalfee - - + mivek @@ -1917,7 +1938,8 @@ Thanks goes to these wonderful people ✨
Jeidnx - + + JiLleON @@ -1938,8 +1960,7 @@ Thanks goes to these wonderful people ✨
jmccl - - + jurekbarth @@ -1960,20 +1981,14 @@ Thanks goes to these wonderful people ✨
Kaan88 - + + akkumar
akkumar - - - - thechubbypanda -
- thechubbypanda - From 37f4c853b2e319d33ed5eed63837fc94f2976010 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jan 2024 21:02:49 +0100 Subject: [PATCH 259/592] chore(deps): Bump actions/cache from 3 to 4 (#3807) --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 0f375d4570a..ccef46f5f06 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -64,7 +64,7 @@ jobs: # When full, the least accessed cache upload is evicted to free up storage. # https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows - name: 'Handle Docker build layer cache' - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: /tmp/.buildx-cache key: cache-buildx-${{ steps.derive-image-cache-key.outputs.digest }} diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 007712214e0..d7a791c595c 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -46,7 +46,7 @@ jobs: # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), # only AMD64 image is expected to be cached, ARM images will build from scratch. - name: 'Retrieve image build from build cache' - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: /tmp/.buildx-cache key: cache-buildx-${{ inputs.cache-key }} diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 5b8bac62825..2c1d1045fd8 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -29,7 +29,7 @@ jobs: # This should always be a cache-hit, thus `restore-keys` fallback is not used. # No new cache uploads should ever happen for this job. - name: 'Retrieve image built from build cache' - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: /tmp/.buildx-cache key: cache-buildx-${{ inputs.cache-key }} diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 896ee80eb99..f8f87ba3d14 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -28,7 +28,7 @@ jobs: # This should always be a cache-hit, thus `restore-keys` fallback is not used. # No new cache uploads should ever happen for this job. - name: 'Retrieve image built from build cache' - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: /tmp/.buildx-cache key: cache-buildx-${{ inputs.cache-key }} From 315f33c9fe8fad64bee1f8f4a93e67e887e870ae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jan 2024 15:36:53 +1300 Subject: [PATCH 260/592] chore(deps): Bump anchore/scan-action from 3.5.0 to 3.6.0 (#3808) Bumps [anchore/scan-action](https://github.com/anchore/scan-action) from 3.5.0 to 3.6.0. - [Release notes](https://github.com/anchore/scan-action/releases) - [Changelog](https://github.com/anchore/scan-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/anchore/scan-action/compare/v3.5.0...v3.6.0) --- updated-dependencies: - dependency-name: anchore/scan-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index f8f87ba3d14..b261de913e7 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v3.5.0 + uses: anchore/scan-action@v3.6.0 id: scan with: image: mailserver-testing:ci From d40a17f7e071005faf71e39a09dd040b370f5fb4 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 23 Jan 2024 23:51:10 +1300 Subject: [PATCH 261/592] fix: Ensure correct ownership for the Rspamd DKIM directory (#3813) The UID / GID shifted during a new release. Until DKIM handling is refactored in a new major release, this fix ensures the content maintains the expected `_rspamd` ownership. --- target/scripts/startup/setup-stack.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/target/scripts/startup/setup-stack.sh b/target/scripts/startup/setup-stack.sh index f55cb548416..c3c54cc37df 100644 --- a/target/scripts/startup/setup-stack.sh +++ b/target/scripts/startup/setup-stack.sh @@ -94,6 +94,10 @@ function _setup_apply_fixes_after_configuration() { _log 'debug' 'Removing files and directories from older versions' rm -rf /var/mail-state/spool-postfix/{dev,etc,lib,pid,usr,private/auth} + + # /tmp/docker-mailserver/rspamd/dkim + _log 'debug' "Ensuring ${RSPAMD_DMS_DKIM_D} is owned by '_rspamd:_rspamd'" + chown -R _rspamd:_rspamd "${RSPAMD_DMS_DKIM_D}" } function _run_user_patches() { From 611a66bf98ab14065e9915105154bc53bb9012fb Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 24 Jan 2024 07:11:05 +1300 Subject: [PATCH 262/592] fix: Correctly support multiple Dovecot PassDBs (#3812) * fix: Dovecot PassDB should restrict allowed auth mechanisms This prevents PassDBs incompatible with certain auth mechanisms from logging failures which accidentally triggers Fail2Ban. Instead only allow the PassDB to be authenticated against when it's compatible with the auth mechanism used. * tests: Use `curl` for OAuth2 login test-cases instead of netcat `curl` provides this capability for both IMAP and SMTP authentication with a bearer token. It supports both `XOAUTH2` and `OAUTHBEARER` mechanisms, as these updated test-cases demonstrate. * chore: Add entry to `CHANGELOG.md` --- CHANGELOG.md | 7 +++ target/dovecot/auth-ldap.conf.ext | 21 ++++++++ target/dovecot/auth-oauth2.conf.ext | 7 +++ target/dovecot/auth-passwdfile.inc | 1 + test/config/oauth2/Caddyfile | 3 ++ test/files/auth/imap-oauth2-oauthbearer.txt | 4 -- test/files/auth/imap-oauth2-xoauth2.txt | 4 -- test/tests/serial/mail_with_oauth2.bats | 54 ++++++++++++++++++--- 8 files changed, 85 insertions(+), 16 deletions(-) create mode 100644 target/dovecot/auth-ldap.conf.ext delete mode 100644 test/files/auth/imap-oauth2-oauthbearer.txt delete mode 100644 test/files/auth/imap-oauth2-xoauth2.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 74acdc61776..ffb966364f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Fixes + +**Dovecot:** + - Restrict the auth mechanisms for PassDB configs we manage (oauth2, passwd-file, ldap) ([#3812](https://github.com/docker-mailserver/docker-mailserver/pull/3812)) + - Prevents misleading auth failures from attempting to authenticate against a PassDB with incompatible auth mechanisms. + - When the new OAuth2 feature was enabled, it introduced false-positives with logged auth failures which triggered Fail2Ban to ban the IP. + ## [v13.3.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.3.0) ### Features diff --git a/target/dovecot/auth-ldap.conf.ext b/target/dovecot/auth-ldap.conf.ext new file mode 100644 index 00000000000..222769aa2c5 --- /dev/null +++ b/target/dovecot/auth-ldap.conf.ext @@ -0,0 +1,21 @@ +# NOTE: This is effectively the same default LDAP config shipped by Dovecot +# The only difference is the addition of the passdb mechanisms field, +# which restricts what auth mechanisms are supported / expected. +# This prevents unnecessary auth failure logs triggering Fail2Ban when +# additional passdb are enabled (OAuth2). + +passdb { + driver = ldap + mechanism = plain login + + # Path for LDAP configuration file, see example-config/dovecot-ldap.conf.ext + args = /etc/dovecot/dovecot-ldap.conf.ext +} + +userdb { + driver = ldap + args = /etc/dovecot/dovecot-ldap.conf.ext + + # Default fields can be used to specify defaults that LDAP may override + #default_fields = home=/home/virtual/%u +} diff --git a/target/dovecot/auth-oauth2.conf.ext b/target/dovecot/auth-oauth2.conf.ext index 6096d1e4794..99a7986b978 100644 --- a/target/dovecot/auth-oauth2.conf.ext +++ b/target/dovecot/auth-oauth2.conf.ext @@ -1,5 +1,12 @@ +# Allow clients to use these additional mechanisms: auth_mechanisms = $auth_mechanisms oauthbearer xoauth2 +# Dovecot docs consider the oauth2 driver as a "success/failure" type PassDB: +# https://doc.dovecot.org/configuration_manual/authentication/password_databases_passdb/#success-failure-database +# Which implies it cannot be configured for the non-plaintext SASL mechanisms listed here: +# https://doc.dovecot.org/configuration_manual/authentication/authentication_mechanisms/#dovecot-supports-the-following-non-plaintext-mechanisms +# However that is not the case, these mechanisms are still valid to prevent trying other incompatible mechanisms (like `plain`). + passdb { driver = oauth2 mechanisms = xoauth2 oauthbearer diff --git a/target/dovecot/auth-passwdfile.inc b/target/dovecot/auth-passwdfile.inc index 6bbf8258d68..38be4e5f1ab 100644 --- a/target/dovecot/auth-passwdfile.inc +++ b/target/dovecot/auth-passwdfile.inc @@ -9,6 +9,7 @@ passdb { driver = passwd-file + mechanisms = plain login args = scheme=SHA512-CRYPT username_format=%u /etc/dovecot/userdb } diff --git a/test/config/oauth2/Caddyfile b/test/config/oauth2/Caddyfile index e116aa5549e..f87ffc80c86 100644 --- a/test/config/oauth2/Caddyfile +++ b/test/config/oauth2/Caddyfile @@ -38,6 +38,9 @@ } } +# NOTE: This portion of config is only relevant for understanding what happens seamlesssly, +# DMS tests no longer use raw IMAP commands with netcat, thus none of this is relevant beyond reference for troubleshooting. +# # /imap/xoauth2 # Generate IMAP commands for authentication testing # Base64 encoded credentials can alternative be done via CLI with: diff --git a/test/files/auth/imap-oauth2-oauthbearer.txt b/test/files/auth/imap-oauth2-oauthbearer.txt deleted file mode 100644 index d85c63e8d6e..00000000000 --- a/test/files/auth/imap-oauth2-oauthbearer.txt +++ /dev/null @@ -1,4 +0,0 @@ -a0 NOOP See test/config/oauth2/Caddyfile to generate the below OAUTHBEARER string -a1 AUTHENTICATE OAUTHBEARER bixhPXVzZXIxQGxvY2FsaG9zdC5sb2NhbGRvbWFpbiwBaG9zdD1sb2NhbGhvc3QBcG9ydD0xNDMBYXV0aD1CZWFyZXIgRE1TX1lXTmpaWE56WDNSdmEyVnUBAQ== -a2 EXAMINE INBOX -a3 LOGOUT diff --git a/test/files/auth/imap-oauth2-xoauth2.txt b/test/files/auth/imap-oauth2-xoauth2.txt deleted file mode 100644 index 0371b0cf09e..00000000000 --- a/test/files/auth/imap-oauth2-xoauth2.txt +++ /dev/null @@ -1,4 +0,0 @@ -a0 NOOP See test/config/oauth2/Caddyfile to generate the below XOAUTH2 string -a1 AUTHENTICATE XOAUTH2 dXNlcj11c2VyMUBsb2NhbGhvc3QubG9jYWxkb21haW4BYXV0aD1CZWFyZXIgRE1TX1lXTmpaWE56WDNSdmEyVnUBAQ== -a2 EXAMINE INBOX -a3 LOGOUT diff --git a/test/tests/serial/mail_with_oauth2.bats b/test/tests/serial/mail_with_oauth2.bats index 0cc34a01de2..9b4d12feb70 100644 --- a/test/tests/serial/mail_with_oauth2.bats +++ b/test/tests/serial/mail_with_oauth2.bats @@ -58,21 +58,59 @@ function teardown_file() { docker network rm "${DMS_TEST_NETWORK}" } -@test "should authenticate with XOAUTH2 over IMAP" { - _nc_wrapper 'auth/imap-oauth2-xoauth2.txt' '-w 1 0.0.0.0 143' - __verify_successful_login 'XOAUTH2' +@test "should authenticate with XOAUTH2" { + __should_login_successfully_with 'XOAUTH2' } -@test "should authenticate with OAUTHBEARER over IMAP" { - _nc_wrapper 'auth/imap-oauth2-oauthbearer.txt' '-w 1 0.0.0.0 143' - __verify_successful_login 'OAUTHBEARER' +@test "should authenticate with OAUTHBEARER" { + __should_login_successfully_with 'OAUTHBEARER' } -function __verify_successful_login() { +function __should_login_successfully_with() { local AUTH_METHOD=${1} + # These values are the auth credentials checked against the Caddy `/userinfo` endpoint: + local USER_ACCOUNT='user1@localhost.localdomain' + local ACCESS_TOKEN='DMS_YWNjZXNzX3Rva2Vu' + __verify_auth_with_imap + __verify_auth_with_smtp +} + +# Dovecot direct auth verification via IMAP: +function __verify_auth_with_imap() { + # NOTE: Include the `--verbose` option if you're troubleshooting and want to see the protocol exchange messages + # NOTE: `--user username:password` is valid for testing `PLAIN` auth mechanism, but you should prefer swaks instead. + _run_in_container curl --silent \ + --login-options "AUTH=${AUTH_METHOD}" --oauth2-bearer "${ACCESS_TOKEN}" --user "${USER_ACCOUNT}" \ + --url 'imap://localhost:143' -X 'LOGOUT' + + __dovecot_logs_should_verify_success +} + +# Postfix delegates by default to Dovecot via SASL: +# NOTE: This won't be compatible with LDAP if `ENABLE_SASLAUTHD=1` with `ldap` SASL mechanism: +function __verify_auth_with_smtp() { + # NOTE: `--upload-file` with some mail content seems required for using curl to test OAuth2 authentication. + # TODO: Replace with swaks and early exit option when it supports XOAUTH2 + OAUTHBEARER: + _run_in_container curl --silent \ + --login-options "AUTH=${AUTH_METHOD}" --oauth2-bearer "${ACCESS_TOKEN}" --user "${USER_ACCOUNT}" \ + --url 'smtp://localhost:587' --mail-from "${USER_ACCOUNT}" --mail-rcpt "${USER_ACCOUNT}" --upload-file - <<< 'RFC 5322 content - not important' + + # Postfix specific auth logs: + _run_in_container grep 'postfix/submission/smtpd' /var/log/mail.log + assert_output --partial "sasl_method=${AUTH_METHOD}, sasl_username=${USER_ACCOUNT}" + + # Dovecot logs should still be checked as it is handling the actual auth process under the hood: + __dovecot_logs_should_verify_success +} + +function __dovecot_logs_should_verify_success() { # Inspect the relevant Dovecot logs to catch failure / success: _run_in_container grep 'dovecot:' /var/log/mail.log refute_output --partial 'oauth2 failed: Introspection failed' - assert_output --partial "dovecot: imap-login: Login: user=, method=${AUTH_METHOD}" + assert_output --partial "dovecot: imap-login: Login: user=<${USER_ACCOUNT}>, method=${AUTH_METHOD}" + + # If another PassDB is enabled, it should not have been attempted with the XOAUTH2 / OAUTHBEARER mechanisms: + # dovecot: auth: passwd-file(${USER_ACCOUNT},127.0.0.1): Password mismatch (SHA1 of given password: d390c1) - trying the next passdb + refute_output --partial 'trying the next passdb' } From 9a53fb0463a12c8357c9b020f5293ac847df9f1e Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 24 Jan 2024 11:47:32 +1300 Subject: [PATCH 263/592] docs: Minor revisions to Dovecot Sieve page (#3811) - The old Dovecot wiki link needed to be updated to the new location. - The new docs are not entirely compatible AFAIK, thus making the existing examples/docs a bit outdated / incompatible. A warning admonition has been added early on to raise awareness to the reader. - Minor formatting revisions to the content. --- docs/content/config/advanced/mail-sieve.md | 34 +++++++++++++++++----- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/docs/content/config/advanced/mail-sieve.md b/docs/content/config/advanced/mail-sieve.md index b3cef249c24..759e4f7880b 100644 --- a/docs/content/config/advanced/mail-sieve.md +++ b/docs/content/config/advanced/mail-sieve.md @@ -4,14 +4,22 @@ title: 'Advanced | Email Filtering with Sieve' ## User-Defined Sieve Filters -[Sieve](http://sieve.info/) allows to specify filtering rules for incoming emails that allow for example sorting mails into different folders depending on the title of an email. -There are global and user specific filters which are filtering the incoming emails in the following order: +!!! warning "Advice may be outdated" -- Global-before -> User specific -> Global-after + This section was contributed by the community some time ago and some configuration examples may be outdated. + +[Sieve][sieve-info] allows to specify filtering rules for incoming emails that allow for example sorting mails into different folders depending on the title of an email. + +!!! info "Global vs User order" + + There are global and user specific filters which are filtering the incoming emails in the following order: + + Global-before -> User specific -> Global-after Global filters are applied to EVERY incoming mail for EVERY email address. -To specify a global Sieve filter provide a `docker-data/dms/config/before.dovecot.sieve` or a `docker-data/dms/config/after.dovecot.sieve` file with your filter rules. -If any filter in this filtering chain discards an incoming mail, the delivery process will stop as well and the mail will not reach any following filters (e.g. global-before stops an incoming spam mail: The mail will get discarded and a user-specific filter won't get applied.) + +- To specify a global Sieve filter provide a `docker-data/dms/config/before.dovecot.sieve` or a `docker-data/dms/config/after.dovecot.sieve` file with your filter rules. +- If any filter in this filtering chain discards an incoming mail, the delivery process will stop as well and the mail will not reach any following filters (e.g. global-before stops an incoming spam mail: The mail will get discarded and a user-specific filter won't get applied.) To specify a user-defined Sieve filter place a `.dovecot.sieve` file into a virtual user's mail folder (e.g. `/var/mail/example.com/user1/home/.dovecot.sieve`). If this file exists dovecot will apply the filtering rules. @@ -32,6 +40,7 @@ An example of a sieve filter that moves mails to a folder `INBOX/spam` depending ``` !!! warning + That folders have to exist beforehand if sieve should move them. Another example of a sieve filter that forward mails to a different address: @@ -52,14 +61,25 @@ Just forward all incoming emails and do not save them locally: redirect "user2@not-example.com"; ``` -You can also use external programs to filter or pipe (process) messages by adding executable scripts in `docker-data/dms/config/sieve-pipe` or `docker-data/dms/config/sieve-filter`. This can be used in lieu of a local alias file, for instance to forward an email to a webservice. These programs can then be referenced by filename, by all users. Note that the process running the scripts run as a privileged user. For further information see [Dovecot's wiki](https://wiki.dovecot.org/Pigeonhole/Sieve/Plugins/Pipe). +You can also use external programs to filter or pipe (process) messages by adding executable scripts in `docker-data/dms/config/sieve-pipe` or `docker-data/dms/config/sieve-filter`. + +This can be used in lieu of a local alias file, for instance to forward an email to a webservice. + +- These programs can then be referenced by filename, by all users. +- Note that the process running the scripts run as a privileged user. +- For further information see [Dovecot's docs][dovecot-docs::sieve-pipe]. ```sieve require ["vnd.dovecot.pipe"]; pipe "external-program"; ``` -For more examples or a detailed description of the Sieve language have a look at [the official site](http://sieve.info/examplescripts). Other resources are available on the internet where you can find several [examples](https://support.tigertech.net/sieve#sieve-example-rules-jmp). +For more examples or a detailed description of the Sieve language have a look at [the official site][sieve-info::examples]. Other resources are available on the internet where you can find several [examples][third-party::sieve-examples]. + +[dovecot-docs::sieve-pipe]: https://doc.dovecot.org/configuration_manual/sieve/plugins/extprograms/#pigeonhole-plugin-extprograms +[sieve-info]: http://sieve.info/ +[sieve-info::examples]: http://sieve.info/examplescripts +[third-party::sieve-examples]: https://support.tigertech.net/sieve#sieve-example-rules-jmp ## Automatic Sorting Based on Subaddresses From 2cf58569614618474cc5623559f1ff409154c835 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 25 Jan 2024 00:46:56 +1300 Subject: [PATCH 264/592] chore: Raise awareness of v13 breaking change better (#3818) Several issues have been raised where this was not an obvious breaking change to the reader. Additional context on impact relevance has been included. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffb966364f6..b6f94a178d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -153,6 +153,7 @@ This patch release fixes two bugs that Rspamd users encountered with the `v13.0. - **Postfix:** - `/etc/postfix/master.cf` has renamed the "smtps" service to "submissions" ([#3235](https://github.com/docker-mailserver/docker-mailserver/pull/3235)) - This is the modern `/etc/services` name for port 465, aligning with the similar "submission" port 587. + - If you have configured Proxy Protocol support with a reverse proxy via `postfix-master.cf` (_as [per our docs guide](https://docker-mailserver.github.io/docker-mailserver/v13.0/examples/tutorials/mailserver-behind-proxy/)_), you will want to update `smtps` to `submissions` there. - Postfix now defaults to supporting DSNs (_[Delivery Status Notifications](https://github.com/docker-mailserver/docker-mailserver/pull/3572#issuecomment-1751880574)_) only for authenticated users (_via ports 465 + 587_). This is a security measure to reduce spammer abuse of your DMS instance as a backscatter source. ([#3572](https://github.com/docker-mailserver/docker-mailserver/pull/3572)) - If you need to modify this change, please let us know by opening an issue / discussion. - You can [opt out (_enable DSNs_) via the `postfix-main.cf` override support](https://docker-mailserver.github.io/docker-mailserver/v12.1/config/advanced/override-defaults/postfix/) using the contents: `smtpd_discard_ehlo_keywords =`. From 0c7e49e654fb64a0b23abae2b423f54e8312beeb Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 25 Jan 2024 02:25:13 +1300 Subject: [PATCH 265/592] release: v13.3.1 (#3817) * chore: Bump `VERSION` * chore: Update `CHANGELOG.md` --------- Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- CHANGELOG.md | 8 ++++++-- VERSION | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6f94a178d3..9a77b28f9a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,16 +2,20 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v13.3.0...HEAD) +## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v13.3.1...HEAD) > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +## [v13.3.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.3.1) + ### Fixes -**Dovecot:** +- **Dovecot:** - Restrict the auth mechanisms for PassDB configs we manage (oauth2, passwd-file, ldap) ([#3812](https://github.com/docker-mailserver/docker-mailserver/pull/3812)) - Prevents misleading auth failures from attempting to authenticate against a PassDB with incompatible auth mechanisms. - When the new OAuth2 feature was enabled, it introduced false-positives with logged auth failures which triggered Fail2Ban to ban the IP. +- **Rspamd:** + - Ensure correct ownership (`_rspamd:_rspamd`) for the Rspamd DKIM directory + files `/tmp/docker-mailserver/rspamd/dkim/` ([#3813](https://github.com/docker-mailserver/docker-mailserver/pull/3813)) ## [v13.3.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.3.0) diff --git a/VERSION b/VERSION index ac565bc1cab..c3d10c59d83 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -13.3.0 +13.3.1 From 00018e7e2b226fafb3a5ab50db078885ca6c8ddc Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 24 Jan 2024 17:05:55 +0100 Subject: [PATCH 266/592] general: update base image to Debian 12 ("Bookworm") (#3403) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Co-authored-by: Casper --- CHANGELOG.md | 27 +++ Dockerfile | 15 +- target/bin/debug-getmail | 2 +- target/bin/getmail-cron | 2 +- target/postfix/main.cf | 7 +- target/scripts/build/compile.sh | 2 +- target/scripts/build/packages.sh | 183 ++++++++---------- target/scripts/check-for-changes.sh | 2 +- .../scripts/startup/setup.d/security/misc.sh | 5 - target/supervisor/conf.d/supervisor-app.conf | 2 +- test/tests/parallel/set1/getmail.bats | 4 +- .../parallel/set1/spam_virus/amavis.bats | 4 +- .../disabled_clamav_spamassassin.bats | 6 +- .../set1/spam_virus/postgrey_enabled.bats | 8 +- .../parallel/set1/spam_virus/rspamd_dkim.bats | 4 +- .../process_check_restart.bats | 4 +- test/tests/serial/mail_with_oauth2.bats | 4 + test/tests/serial/open_dkim.bats | 4 +- 18 files changed, 141 insertions(+), 144 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a77b28f9a9..35ee73a9b63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,33 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +The most noteworthy change of this release is the update of the container's base image from Debian 11 ("Bullseye") to Debian 12 ("Bookworm"). This update alone involves breaking changes and requires a careful update! + +### Breaking + +- **Updated base image to Debian 12** ([#3403](https://github.com/docker-mailserver/docker-mailserver/pull/3403)) + - Changed the default of `DOVECOT_COMMUNITY_REPO` to `0` (disabled) - the Dovecot community repo will (for now) not be the default when building the DMS. + - While Debian 12 (Bookworm) was released in June 2023 and the latest Dovecot `2.3.21` in Sep 2023, as of Jan 2024 there is no [Dovecot community repo available for Debian 12](https://repo.dovecot.org). + - This results in the Dovecot version being downgraded from `2.3.21` (DMS v13.3) to `2.3.19`, which [may affect functionality when you've explicitly configured for these features](https://github.com/dovecot/core/blob/30cde20f63650d8dcc4c7ad45418986f03159946/NEWS#L1-L158): + - OAuth2 (_mostly regarding JWT usage, or POST requests (`introspection_mode = post`) with `client_id` + `client_secret`_). + - Lua HTTP client (_DNS related_). + - Updated packages. For an overview, [we have a review comment on the PR that introduces Debian 12](https://github.com/docker-mailserver/docker-mailserver/pull/3403#issuecomment-1694563615) + - Notable major version bump: `openssl 3`, `clamav 1`, `spamassassin 4`, `redis-server 7`. + - Notable minor version bump: `postfix 3.5.23 => 3.7.9` + - Notable minor version bump + downgrade: `dovecot 2.3.13 => 2.3.19` (_Previous release provided `2.3.21` via community repo, `2.3.19` is now the default_) + - Updates to `packages.sh`: + - The script now uses `/etc/os-release` to determine the release name of Debian + - Removed custom installations of Fail2Ban, getmail6 and Rspamd + - Updated packages lists and added comments for maintainability +- **Postfix:** + - Postfix upgrade from 3.5 to 3.7 ([#3403](https://github.com/docker-mailserver/docker-mailserver/pull/3403)) + - `compatibility_level` was raised from `2` to `3.6` + - Postfix has deprecated the usage of `whitelist` / `blacklist` in config parameters and logging in favor of `allowlist` / `denylist` and similar variations. ([#3403](https://github.com/docker-mailserver/docker-mailserver/pull/3403/files#r1306356328)) + - This [may affect monitoring / analysis of logs output from Postfix](https://www.postfix.org/COMPATIBILITY_README.html#respectful_logging) that expects to match patterns on the prior terminology used. + - DMS `main.cf` has renamed `postscreen_dnsbl_whitelist_threshold` to `postscreen_dnsbl_allowlist_threshold` as part of this change. + - `smtpd_relay_restrictions` (relay policy) is now evaluated after `smtpd_recipient_restrictions` (spam policy). Previously it was evaluated before `smtpd_recipient_restrictions`. Mail to be relayed via DMS must now pass through the spam policy first. + - The TLS fingerprint policy has changed the default from MD5 to SHA256 (_DMS does not modify this Postfix parameter, but may affect any user customizations that do_). + ## [v13.3.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.3.1) ### Fixes diff --git a/Dockerfile b/Dockerfile index e822632a022..b8bf89bcfaf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,10 +4,10 @@ # This is in preparation for more granular stages (eg ClamAV and Fail2Ban split into their own) ARG DEBIAN_FRONTEND=noninteractive -ARG DOVECOT_COMMUNITY_REPO=1 +ARG DOVECOT_COMMUNITY_REPO=0 ARG LOG_LEVEL=trace -FROM docker.io/debian:11-slim AS stage-base +FROM docker.io/debian:12-slim AS stage-base ARG DEBIAN_FRONTEND ARG DOVECOT_COMMUNITY_REPO @@ -30,8 +30,6 @@ COPY target/scripts/helpers/log.sh /usr/local/bin/helpers/log.sh RUN /bin/bash /build/packages.sh && rm -r /build - - # ----------------------------------------------- # --- Compile deb packages ---------------------- # ----------------------------------------------- @@ -130,7 +128,8 @@ COPY \ # hadolint ignore=SC2016 RUN </etc/default/spamassassin sedfile -i -r 's/^\$INIT restart/supervisorctl restart amavis/g' /etc/spamassassin/sa-update-hooks.d/amavisd-new mkdir /etc/spamassassin/kam/ curl -sSfLo /etc/spamassassin/kam/kam.sa-channels.mcgrail.com.key https://mcgrail.com/downloads/kam.sa-channels.mcgrail.com.key @@ -189,7 +188,6 @@ RUN < /dev/null|/usr/bin/supervisorctl signal hup rsyslog >/dev/null|g' /usr/lib/rsyslog/rsyslog-rotate + # this change is for our alternative process manager rather than part of + # a fix related to the change preceding it. + echo -e '\n/usr/bin/supervisorctl signal hup rsyslog >/dev/null' >>/usr/lib/rsyslog/rsyslog-rotate EOF # ----------------------------------------------- diff --git a/target/bin/debug-getmail b/target/bin/debug-getmail index c007ebd2bb7..27834475a5a 100644 --- a/target/bin/debug-getmail +++ b/target/bin/debug-getmail @@ -15,5 +15,5 @@ else fi for FILE in /etc/getmailrc.d/getmailrc*; do - /usr/local/bin/getmail --getmaildir "${GETMAILDIR}" --rcfile "${FILE}" --dump | tail -n +7 + getmail --getmaildir "${GETMAILDIR}" --rcfile "${FILE}" --dump | tail -n +6 done diff --git a/target/bin/getmail-cron b/target/bin/getmail-cron index 8f31e3b1a47..8e1e474927f 100644 --- a/target/bin/getmail-cron +++ b/target/bin/getmail-cron @@ -2,6 +2,6 @@ for FILE in /etc/getmailrc.d/getmailrc*; do if ! pgrep -f "${FILE}$" &>/dev/null; then - /usr/local/bin/getmail --getmaildir /var/lib/getmail --rcfile "${FILE}" + getmail --getmaildir /var/lib/getmail --rcfile "${FILE}" fi done diff --git a/target/postfix/main.cf b/target/postfix/main.cf index 8c1d4c894e1..a0d805cc838 100644 --- a/target/postfix/main.cf +++ b/target/postfix/main.cf @@ -86,7 +86,7 @@ postscreen_dnsbl_sites = list.dnswl.org=127.0.[0..255].1*-3 list.dnswl.org=127.0.[0..255].[2..3]*-4 postscreen_dnsbl_threshold = 3 -postscreen_dnsbl_whitelist_threshold = -1 +postscreen_dnsbl_allowlist_threshold = -1 postscreen_greet_action = enforce postscreen_bare_newline_action = enforce @@ -121,7 +121,4 @@ smtp_header_checks = pcre:/etc/postfix/maps/sender_header_filter.pcre # http://www.postfix.org/COMPATIBILITY_README.html # If backwards-compaitibilty log messages appear, fix them by explicitly adding # the legacy or new default value (alternatively raise the compatibility_level) -# -# TODO: The next compatibility_level is 3.6, when Postfix 3.6 is available consider -# bumping this value after taking the compaitibilty changes into account. -compatibility_level = 2 +compatibility_level = 3.6 diff --git a/target/scripts/build/compile.sh b/target/scripts/build/compile.sh index a7010ba5b90..56feea72993 100644 --- a/target/scripts/build/compile.sh +++ b/target/scripts/build/compile.sh @@ -17,7 +17,7 @@ function _compile_dovecot_fts_xapian() { tar xzvf dovecot-fts-xapian.tar.gz cd fts-xapian-1.5.5 USER=root dh_make -p dovecot-fts-xapian-1.5.5 --single --native --copyright gpl2 -y - rm debian/*.ex debian/*.EX + rm debian/*.ex cp PACKAGES/DEB/control debian/ cp PACKAGES/DEB/changelog debian/ cp PACKAGES/DEB/compat debian/ diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index e57cfe0761b..24dae8a70c8 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -1,10 +1,13 @@ #!/bin/bash -# -eE :: exit on error (do this in functions as well) -# -u :: show (and exit) when using unset variables +# -e :: exit on error (do this in functions as well) +# -E :: inherit the ERR trap to functions, command substitutions and sub-shells +# -u :: show (and exit) when using unset variables # -o pipefail :: exit on error in pipes set -eE -u -o pipefail +VERSION_CODENAME='bookworm' + # shellcheck source=../helpers/log.sh source /usr/local/bin/helpers/log.sh @@ -17,17 +20,38 @@ function _pre_installation_steps() { _log 'trace' 'Updating package signatures' apt-get "${QUIET}" update - _log 'trace' 'Installing packages that are needed early' - apt-get "${QUIET}" install --no-install-recommends apt-utils 2>/dev/null - _log 'trace' 'Upgrading packages' apt-get "${QUIET}" upgrade + + _log 'trace' 'Installing packages that are needed early' + # add packages usually required by apt to + # - not log unnecessary warnings + # - be able to add PPAs early (e.g., Rspamd) + local EARLY_PACKAGES=( + apt-utils # avoid useless warnings + apt-transport-https ca-certificates curl gnupg # required for adding PPAs + systemd-standalone-sysusers # avoid problems with SA / Amavis (https://github.com/docker-mailserver/docker-mailserver/pull/3403#pullrequestreview-1596689953) + ) + apt-get "${QUIET}" install --no-install-recommends "${EARLY_PACKAGES[@]}" 2>/dev/null +} + +function _install_utils() { + _log 'debug' 'Installing utils sourced from Github' + _log 'trace' 'Installing jaq' + curl -sSfL "https://github.com/01mf02/jaq/releases/latest/download/jaq-v1.2.0-$(uname -m)-unknown-linux-gnu" -o /usr/bin/jaq && chmod +x /usr/bin/jaq + + _log 'trace' 'Installing swaks' + local SWAKS_VERSION='20240103.0' + local SWAKS_RELEASE="swaks-${SWAKS_VERSION}" + curl -sSfL "https://github.com/jetmore/swaks/releases/download/v${SWAKS_VERSION}/${SWAKS_RELEASE}.tar.gz" | tar -xz + mv "${SWAKS_RELEASE}/swaks" /usr/local/bin + rm -r "${SWAKS_RELEASE}" } function _install_postfix() { _log 'debug' 'Installing Postfix' - _log 'warn' 'Applying workaround for Postfix bug (see https://github.com//issues/2023#issuecomment-855326403)' + _log 'warn' 'Applying workaround for Postfix bug (see https://github.com/docker-mailserver/docker-mailserver/issues/2023#issuecomment-855326403)' # Debians postfix package has a post-install script that expects a valid FQDN hostname to work: mv /bin/hostname /bin/hostname.bak @@ -43,12 +67,17 @@ function _install_postfix() { function _install_packages() { _log 'debug' 'Installing all packages now' - ANTI_VIRUS_SPAM_PACKAGES=( - amavisd-new clamav clamav-daemon - pyzor razor spamassassin + local ANTI_VIRUS_SPAM_PACKAGES=( + clamav clamav-daemon + # spamassassin is used only with amavisd-new, while pyzor + razor are used by spamassasin + amavisd-new spamassassin pyzor razor + # the following packages are all for Fail2Ban + # https://github.com/docker-mailserver/docker-mailserver/pull/3403#discussion_r1306581431 + fail2ban python3-pyinotify python3-dnspython ) - CODECS_PACKAGES=( + # predominantly for Amavis support + local CODECS_PACKAGES=( altermime arj bzip2 cabextract cpio file gzip lhasa liblz4-tool @@ -57,26 +86,33 @@ function _install_packages() { unrar-free unzip xz-utils ) - MISCELLANEOUS_PACKAGES=( - apt-transport-https binutils bsd-mailx - ca-certificates curl dbconfig-no-thanks - dumb-init gnupg iproute2 libdate-manip-perl - libldap-common libmail-spf-perl - libnet-dns-perl locales logwatch - netcat-openbsd nftables rsyslog - supervisor uuid whois + local MISCELLANEOUS_PACKAGES=( + binutils bsd-mailx + dbconfig-no-thanks dumb-init iproute2 + libdate-manip-perl libldap-common libmail-spf-perl libnet-dns-perl + locales logwatch netcat-openbsd + nftables # primarily for Fail2Ban + rsyslog supervisor + uuid # used for file-locking + whois ) - POSTFIX_PACKAGES=( + local POSTFIX_PACKAGES=( pflogsumm postgrey postfix-ldap postfix-mta-sts-resolver postfix-pcre postfix-policyd-spf-python postsrsd ) - MAIL_PROGRAMS_PACKAGES=( - fetchmail opendkim opendkim-tools + local MAIL_PROGRAMS_PACKAGES=( + opendkim opendkim-tools opendmarc libsasl2-modules sasl2-bin ) + # These packages support community contributed features. + # If they cause too much maintenance burden in future, they are liable for removal. + local COMMUNITY_PACKAGES=( + fetchmail getmail6 + ) + # `bind9-dnsutils` provides the `dig` command # `iputils-ping` provides the `ping` command DEBUG_PACKAGES=( @@ -89,14 +125,12 @@ function _install_packages() { "${MISCELLANEOUS_PACKAGES[@]}" \ "${POSTFIX_PACKAGES[@]}" \ "${MAIL_PROGRAMS_PACKAGES[@]}" \ - "${DEBUG_PACKAGES[@]}" + "${DEBUG_PACKAGES[@]}" \ + "${COMMUNITY_PACKAGES[@]}" } function _install_dovecot() { - declare -a DOVECOT_PACKAGES - - # Dovecot packages for officially supported features. - DOVECOT_PACKAGES=( + local DOVECOT_PACKAGES=( dovecot-core dovecot-imapd dovecot-ldap dovecot-lmtpd dovecot-managesieved dovecot-pop3d dovecot-sieve dovecot-solr @@ -111,7 +145,8 @@ function _install_dovecot() { _log 'trace' 'Using Dovecot community repository' curl https://repo.dovecot.org/DOVECOT-REPO-GPG | gpg --import gpg --export ED409DA1 > /etc/apt/trusted.gpg.d/dovecot.gpg - echo "deb https://repo.dovecot.org/ce-2.3-latest/debian/bullseye bullseye main" > /etc/apt/sources.list.d/dovecot.list + # VERSION_CODENAME sourced from /etc/os-release + echo "deb https://repo.dovecot.org/ce-2.3-latest/debian/${VERSION_CODENAME} ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/dovecot.list _log 'trace' 'Updating Dovecot package signatures' apt-get "${QUIET}" update @@ -128,104 +163,42 @@ function _install_dovecot() { } function _install_rspamd() { - _log 'trace' 'Adding Rspamd package signatures' - local DEB_FILE='/etc/apt/sources.list.d/rspamd.list' - + _log 'debug' 'Installing Rspamd' + _log 'trace' 'Adding Rspamd PPA' curl -sSfL https://rspamd.com/apt-stable/gpg.key | gpg --dearmor >/etc/apt/trusted.gpg.d/rspamd.gpg - local URL='[signed-by=/etc/apt/trusted.gpg.d/rspamd.gpg] http://rspamd.com/apt-stable/ bullseye main' - echo "deb ${URL}" >"${DEB_FILE}" + echo \ + "deb [signed-by=/etc/apt/trusted.gpg.d/rspamd.gpg] http://rspamd.com/apt-stable/ ${VERSION_CODENAME} main" \ + >/etc/apt/sources.list.d/rspamd.list - _log 'debug' 'Installing Rspamd' + _log 'trace' 'Updating package index after adding PPAs' apt-get "${QUIET}" update - apt-get "${QUIET}" --no-install-recommends install 'rspamd' 'redis-server' -} - -function _install_fail2ban() { - local FAIL2BAN_DEB_URL='https://github.com/fail2ban/fail2ban/releases/download/1.0.2/fail2ban_1.0.2-1.upstream1_all.deb' - local FAIL2BAN_DEB_ASC_URL="${FAIL2BAN_DEB_URL}.asc" - local FAIL2BAN_GPG_FINGERPRINT='8738 559E 26F6 71DF 9E2C 6D9E 683B F1BE BD0A 882C' - local FAIL2BAN_GPG_PUBLIC_KEY_ID='0x683BF1BEBD0A882C' - local FAIL2BAN_GPG_PUBLIC_KEY_SERVER='hkps://keyserver.ubuntu.com' - - _log 'debug' 'Installing Fail2ban' - apt-get "${QUIET}" --no-install-recommends install python3-pyinotify python3-dnspython - - gpg --keyserver "${FAIL2BAN_GPG_PUBLIC_KEY_SERVER}" --recv-keys "${FAIL2BAN_GPG_PUBLIC_KEY_ID}" 2>&1 - - curl -Lkso fail2ban.deb "${FAIL2BAN_DEB_URL}" - curl -Lkso fail2ban.deb.asc "${FAIL2BAN_DEB_ASC_URL}" - - FINGERPRINT=$(LANG=C gpg --verify fail2ban.deb.asc fail2ban.deb |& sed -n 's#Primary key fingerprint: \(.*\)#\1#p') - - if [[ -z ${FINGERPRINT} ]]; then - echo 'ERROR: Invalid GPG signature!' >&2 - exit 1 - fi - if [[ ${FINGERPRINT} != "${FAIL2BAN_GPG_FINGERPRINT}" ]]; then - echo "ERROR: Wrong GPG fingerprint!" >&2 - exit 1 - fi - - dpkg -i fail2ban.deb 2>&1 - rm fail2ban.deb fail2ban.deb.asc - - _log 'debug' 'Patching Fail2ban to enable network bans' - # Enable network bans - # https://github.com/docker-mailserver/docker-mailserver/issues/2669 - sedfile -i -r 's/^_nft_add_set = .+/_nft_add_set = add set \\{ type \\; flags interval\\; \\}/' /etc/fail2ban/action.d/nftables.conf -} - -# Presently the getmail6 package is v6.14, which is too old. -# v6.18 contains fixes for Google and Microsoft OAuth support. -# using pip to install getmail. -# TODO This can be removed when the base image is updated to Debian 12 (Bookworm) -function _install_getmail() { - _log 'debug' 'Installing getmail6' - apt-get "${QUIET}" --no-install-recommends install python3-pip - pip3 install --no-cache-dir 'getmail6~=6.18.12' - ln -s /usr/local/bin/getmail /usr/bin/getmail - ln -s /usr/local/bin/getmail-gmail-xoauth-tokens /usr/bin/getmail-gmail-xoauth-tokens - apt-get "${QUIET}" purge python3-pip - apt-get "${QUIET}" autoremove -} - -function _install_utils() { - _log 'debug' 'Installing utils sourced from Github' - _log 'trace' 'Installing jaq' - curl -sL "https://github.com/01mf02/jaq/releases/latest/download/jaq-v1.2.0-$(uname -m)-unknown-linux-gnu" -o /usr/bin/jaq && chmod +x /usr/bin/jaq - - _log 'trace' 'Installing swaks' - local SWAKS_VERSION='20240103.0' - local SWAKS_RELEASE="swaks-${SWAKS_VERSION}" - curl -sSfL "https://github.com/jetmore/swaks/releases/download/v${SWAKS_VERSION}/${SWAKS_RELEASE}.tar.gz" | tar -xz - mv "${SWAKS_RELEASE}/swaks" /usr/local/bin - rm -r "${SWAKS_RELEASE}" + _log 'trace' 'Installing actual package' + apt-get "${QUIET}" install rspamd redis-server } -function _remove_data_after_package_installations() { +function _post_installation_steps() { + _log 'debug' 'Running post-installation steps (cleanup)' _log 'debug' 'Deleting sensitive files (secrets)' rm /etc/postsrsd.secret _log 'debug' 'Deleting default logwatch cronjob' rm /etc/cron.daily/00logwatch -} -function _post_installation_steps() { - _log 'debug' 'Running post-installation steps (cleanup)' + _log 'trace' 'Removing leftovers from APT' apt-get "${QUIET}" clean rm -rf /var/lib/apt/lists/* - _log 'info' 'Finished installing packages' + _log 'debug' 'Patching Fail2ban to enable network bans' + # Enable network bans + # https://github.com/docker-mailserver/docker-mailserver/issues/2669 + sedfile -i -r 's/^_nft_add_set = .+/_nft_add_set = add set
\\{ type \\; flags interval\\; \\}/' /etc/fail2ban/action.d/nftables.conf } _pre_installation_steps +_install_utils _install_postfix _install_packages _install_dovecot _install_rspamd -_install_fail2ban -_install_getmail -_install_utils -_remove_data_after_package_installations _post_installation_steps diff --git a/target/scripts/check-for-changes.sh b/target/scripts/check-for-changes.sh index 66417c09f32..2fa004b00c7 100755 --- a/target/scripts/check-for-changes.sh +++ b/target/scripts/check-for-changes.sh @@ -88,7 +88,7 @@ function _reload_amavis() { if [[ ${CHANGED} =~ ${DMS_DIR}/postfix-accounts.cf ]] || [[ ${CHANGED} =~ ${DMS_DIR}/postfix-virtual.cf ]]; then # /etc/postfix/vhost was updated, amavis must refresh it's config by # reading this file again in case of new domains, otherwise they will be ignored. - amavisd-new reload + amavisd reload fi } diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index 36d8905fc1f..ec32a1e6d84 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -81,11 +81,6 @@ function __setup__security__spamassassin() { # shellcheck disable=SC2016 sed -i -r 's|^\$sa_kill_level_deflt (.*);|\$sa_kill_level_deflt = '"${SA_KILL}"';|g' /etc/amavis/conf.d/20-debian_defaults - # fix cron.daily for spamassassin - sed -i \ - 's|invoke-rc.d spamassassin reload|/etc/init\.d/spamassassin reload|g' \ - /etc/cron.daily/spamassassin - if [[ ${SA_SPAM_SUBJECT} == 'undef' ]]; then # shellcheck disable=SC2016 sed -i -r 's|^\$sa_spam_subject_tag (.*);|\$sa_spam_subject_tag = undef;|g' /etc/amavis/conf.d/20-debian_defaults diff --git a/target/supervisor/conf.d/supervisor-app.conf b/target/supervisor/conf.d/supervisor-app.conf index d64d3d72fdf..50a7acc3e13 100644 --- a/target/supervisor/conf.d/supervisor-app.conf +++ b/target/supervisor/conf.d/supervisor-app.conf @@ -94,7 +94,7 @@ autostart=false autorestart=true stdout_logfile=/var/log/supervisor/%(program_name)s.log stderr_logfile=/var/log/supervisor/%(program_name)s.log -command=/usr/sbin/amavisd-new foreground +command=/usr/sbin/amavisd foreground [program:rspamd] startsecs=0 diff --git a/test/tests/parallel/set1/getmail.bats b/test/tests/parallel/set1/getmail.bats index 4b5c528a171..ca1fdf87d6a 100644 --- a/test/tests/parallel/set1/getmail.bats +++ b/test/tests/parallel/set1/getmail.bats @@ -25,9 +25,9 @@ function teardown_file() { _default_teardown ; } assert_line 'received = false' assert_line 'delivered_to = false' - _run_in_container stat /usr/local/bin/debug-getmail + _run_in_container_bash '[[ -f /usr/local/bin/debug-getmail ]]' assert_success - _run_in_container stat /usr/local/bin/getmail-cron + _run_in_container_bash '[[ -f /usr/local/bin/getmail-cron ]]' assert_success } diff --git a/test/tests/parallel/set1/spam_virus/amavis.bats b/test/tests/parallel/set1/spam_virus/amavis.bats index bc920234490..d7a59cb253a 100644 --- a/test/tests/parallel/set1/spam_virus/amavis.bats +++ b/test/tests/parallel/set1/spam_virus/amavis.bats @@ -51,10 +51,8 @@ function teardown_file() { export CONTAINER_NAME=${CONTAINER1_NAME} # give Amavis just a bit of time to print out its full debug log - run _repeat_in_container_until_success_or_timeout 5 "${CONTAINER_NAME}" grep 'ANTI-SPAM-SA' /var/log/mail/mail.log + run _repeat_in_container_until_success_or_timeout 20 "${CONTAINER_NAME}" grep 'SpamControl: init_pre_fork on SpamAssassin done' /var/log/mail/mail.log assert_success - assert_output --partial 'loaded' - refute_output --partial 'NOT loaded' } @test '(Amavis enabled) SA ENV should update Amavis config' { diff --git a/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats b/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats index 5ec28396717..1711950fb6d 100644 --- a/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats +++ b/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats @@ -30,8 +30,12 @@ function teardown_file() { _default_teardown ; } } @test "SA - Amavis integration should not be active" { - _run_in_container_bash "grep -i 'ANTI-SPAM-SA code' /var/log/mail/mail.log | grep 'NOT loaded'" + # Wait until Amavis has finished initializing: + run _repeat_in_container_until_success_or_timeout 20 "${CONTAINER_NAME}" grep 'Deleting db files in /var/lib/amavis/db' /var/log/mail/mail.log assert_success + # Amavis module for SA should not be loaded (`SpamControl: scanner SpamAssassin, module Amavis::SpamControl::SpamAssassin`): + _run_in_container grep 'scanner SpamAssassin' /var/log/mail/mail.log + assert_failure } @test "SA - should not have been called" { diff --git a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats index 5538d3bf30a..2609812e90b 100644 --- a/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats +++ b/test/tests/parallel/set1/spam_virus/postgrey_enabled.bats @@ -59,7 +59,7 @@ function teardown_file() { _default_teardown ; } _should_have_log_entry \ 'action=greylist' \ 'reason=new' \ - 'client_address=127.0.0.1/32, sender=user@external.tld, recipient=user1@localhost.localdomain' + 'client_address=127.0.0.1, sender=user@external.tld, recipient=user1@localhost.localdomain' } # NOTE: This test case depends on the previous one @@ -73,7 +73,7 @@ function teardown_file() { _default_teardown ; } _should_have_log_entry \ 'action=pass' \ 'reason=triplet found' \ - 'client_address=127.0.0.1/32, sender=user@external.tld, recipient=user1@localhost.localdomain' + 'client_address=127.0.0.1, sender=user@external.tld, recipient=user1@localhost.localdomain' } # NOTE: These two whitelist tests use `files/nc/` instead of `files/emails`. @@ -91,7 +91,7 @@ function teardown_file() { _default_teardown ; } _should_have_log_entry \ 'action=pass' \ 'reason=client whitelist' \ - 'client_address=127.0.0.1/32, sender=test@whitelist.tld, recipient=user1@localhost.localdomain' + 'client_address=127.0.0.1, sender=test@whitelist.tld, recipient=user1@localhost.localdomain' } @test "should whitelist recipient 'user2@otherdomain.tld'" { @@ -100,7 +100,7 @@ function teardown_file() { _default_teardown ; } _should_have_log_entry \ 'action=pass' \ 'reason=recipient whitelist' \ - 'client_address=127.0.0.1/32, sender=test@nonwhitelist.tld, recipient=user2@otherdomain.tld' + 'client_address=127.0.0.1, sender=test@nonwhitelist.tld, recipient=user2@otherdomain.tld' } function _should_have_log_entry() { diff --git a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats index 215b334deb6..044e38fbf26 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats @@ -222,7 +222,7 @@ function __check_rsa_keys() { # Check the private key matches the specification _run_in_container_bash "openssl rsa -in '${BASE_FILE_NAME}.private.txt' -noout -text" assert_success - assert_line --index 0 "RSA Private-Key: (${KEYSIZE} bit, 2 primes)" + assert_line --index 0 "Private-Key: (${KEYSIZE} bit, 2 primes)" # Check the public key matches the specification # @@ -232,7 +232,7 @@ function __check_rsa_keys() { PUBKEY=$(_exec_in_container_bash "grep -o 'p=.*' ${BASE_FILE_NAME}.public.dns.txt") _run_in_container_bash "openssl enc -base64 -d <<< ${PUBKEY#p=} | openssl pkey -inform DER -pubin -noout -text" assert_success - assert_line --index 0 "RSA Public-Key: (${KEYSIZE} bit)" + assert_line --index 0 "Public-Key: (${KEYSIZE} bit)" } # Verify that all DKIM key files are present. diff --git a/test/tests/parallel/set3/container_configuration/process_check_restart.bats b/test/tests/parallel/set3/container_configuration/process_check_restart.bats index 0a54a60f7ba..4f0fd90fb0b 100644 --- a/test/tests/parallel/set3/container_configuration/process_check_restart.bats +++ b/test/tests/parallel/set3/container_configuration/process_check_restart.bats @@ -13,7 +13,7 @@ function teardown() { _default_teardown ; } # opendmarc (/usr/sbin/opendmarc) # postfix (/usr/lib/postfix/sbin/master) - Postfix main process (two ancestors, launched via pidproxy python3 script) # -# amavisd-new (usr/sbin/amavisd-new) +# amavisd (usr/sbin/amavisd) # clamd (/usr/sbin/clamd) # dovecot (/usr/sbin/dovecot) # fetchmail (/usr/bin/fetchmail) @@ -37,7 +37,7 @@ CORE_PROCESS_LIST=( # These processes can be toggled via ENV: # NOTE: clamd handled in separate test case ENV_PROCESS_LIST=( - amavisd-new + amavisd dovecot fail2ban-server fetchmail diff --git a/test/tests/serial/mail_with_oauth2.bats b/test/tests/serial/mail_with_oauth2.bats index 9b4d12feb70..8da87980907 100644 --- a/test/tests/serial/mail_with_oauth2.bats +++ b/test/tests/serial/mail_with_oauth2.bats @@ -59,6 +59,10 @@ function teardown_file() { } @test "should authenticate with XOAUTH2" { + # curl packaged in Debian 12 (and the latest release as of Jan 2024) broke XOAUTH2 support + # https://github.com/docker-mailserver/docker-mailserver/pull/3403#issuecomment-1907100624 + skip 'unable to test XOAUTH mechanism due to bug since curl 7.80' + __should_login_successfully_with 'XOAUTH2' } diff --git a/test/tests/serial/open_dkim.bats b/test/tests/serial/open_dkim.bats index b4950d1f1c7..e51fdedbc3a 100644 --- a/test/tests/serial/open_dkim.bats +++ b/test/tests/serial/open_dkim.bats @@ -226,7 +226,7 @@ function __should_have_expected_files() { # DKIM private key for signing, parse it to verify private key size is correct: _run_in_container_bash "openssl rsa -in '${TARGET_DIR}/mail.private' -noout -text" assert_success - assert_line --index 0 "RSA Private-Key: (${EXPECTED_KEYSIZE} bit, 2 primes)" + assert_line --index 0 "Private-Key: (${EXPECTED_KEYSIZE} bit, 2 primes)" # DKIM record, extract public key (base64 encoded, potentially multi-line) # - tail to exclude first line, @@ -240,7 +240,7 @@ function __should_have_expected_files() { ) | openssl enc -base64 -d | openssl pkey -inform DER -pubin -noout -text " assert_success - assert_line --index 0 "RSA Public-Key: (${EXPECTED_KEYSIZE} bit)" + assert_line --index 0 "Public-Key: (${EXPECTED_KEYSIZE} bit)" # Contents is for expected DKIM_DOMAIN and selector (mail): _run_in_container cat "${TARGET_DIR}/mail.txt" From ed1e1ebbd3eeb61b059967d94c54d8881112fba4 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 24 Jan 2024 23:06:05 +0100 Subject: [PATCH 267/592] tests: new sending and filtering functions (#3786) * move log/filter functions into own file * add ShellCheck global directives * use new function for tracking logs The new function, called `_send_email_with_mid`, aligns with suggestions from @polarethene and is heavily simplified compared to its predecessor `_send_email_and_get_id`. New helpers will be introduced to filter logs according to the MID constructed in this function. * new filters for searching logs with MID * use new filters (and sending) functions * add new helper for asserting non-existence of log message * use new filters in tests * Apply suggestions from code review - `_mid` / `MID` => `_msgid` / `MSG_ID` - Revised documentation / tooltip comments * Apply suggestions from code review * fix tests * use more distinct names for MSG_ID headers * update `_filter_service_log` to not use `-i -E` Moreover, I added a function to print the whole mail log. Appropriate comments were added to this function to indicate that one should only use this function when necessary. * adjust helpers to new helper filter * follow-up of previous commit * add CHANGELOG entry * Apply suggestions from code review * chore: Update OAuth2 to use new log helper * Apply suggestions from code review Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> * added explicit `_regexp` filters for logs * Apply suggestions from code review --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 5 + test/helper/common.bash | 68 +--------- test/helper/log_and_filtering.bash | 119 ++++++++++++++++++ test/helper/sending.bash | 83 ++++-------- .../parallel/set1/spam_virus/clamav.bats | 7 +- .../disabled_clamav_spamassassin.bats | 10 +- .../parallel/set1/spam_virus/postscreen.bats | 9 +- .../parallel/set1/spam_virus/rspamd_full.bats | 46 ++++--- test/tests/parallel/set3/mta/dsn.bats | 10 +- .../parallel/set3/mta/smtp_delivery.bats | 26 ++-- test/tests/serial/mail_pop3.bats | 9 +- test/tests/serial/mail_with_oauth2.bats | 2 +- test/tests/serial/tests.bats | 30 ++--- test/tests/serial/vmail-id.bats | 30 ++--- 14 files changed, 227 insertions(+), 227 deletions(-) create mode 100644 test/helper/log_and_filtering.bash diff --git a/CHANGELOG.md b/CHANGELOG.md index 35ee73a9b63..12f07561c7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,11 @@ The most noteworthy change of this release is the update of the container's base - `smtpd_relay_restrictions` (relay policy) is now evaluated after `smtpd_recipient_restrictions` (spam policy). Previously it was evaluated before `smtpd_recipient_restrictions`. Mail to be relayed via DMS must now pass through the spam policy first. - The TLS fingerprint policy has changed the default from MD5 to SHA256 (_DMS does not modify this Postfix parameter, but may affect any user customizations that do_). +### Updates + +- **Tests:** + - Refactored helper methods for sending e-mails with specific `Message-ID` headers and the helpers for retrieving + filtering logs, which together help isolate logs relevant to specific mail when multiple mails have been processed within a single test. ([#3786](https://github.com/docker-mailserver/docker-mailserver/pull/3786)) + ## [v13.3.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.3.1) ### Fixes diff --git a/test/helper/common.bash b/test/helper/common.bash index 0891bf8ce43..3c54862cc1d 100644 --- a/test/helper/common.bash +++ b/test/helper/common.bash @@ -18,6 +18,7 @@ function __load_bats_helper() { load "${REPOSITORY_ROOT}/test/test_helper/bats-support/load" load "${REPOSITORY_ROOT}/test/test_helper/bats-assert/load" load "${REPOSITORY_ROOT}/test/helper/sending" + load "${REPOSITORY_ROOT}/test/helper/log_and_filtering" } __load_bats_helper @@ -228,7 +229,6 @@ function _run_until_success_or_timeout() { # ! ------------------------------------------------------------------- # ? >> Functions to wait until a condition is met - # Wait until a port is ready. # # @param ${1} = port @@ -351,15 +351,6 @@ function _add_mail_account_then_wait_until_ready() { _wait_until_account_maildir_exists "${MAIL_ACCOUNT}" } -# Assert that the number of lines output by a previous command matches the given -# amount (${1}). `lines` is a special BATS variable updated via `run`. -# -# @param ${1} = number of lines that the output should have -function _should_output_number_of_lines() { - # shellcheck disable=SC2154 - assert_equal "${#lines[@]}" "${1:?Number of lines not provided}" -} - # Reloads the postfix service. # # @param ${1} = container name [OPTIONAL] @@ -372,7 +363,6 @@ function _reload_postfix() { _exec_in_container postfix reload } - # Get the IP of the container (${1}). # # @param ${1} = container name [OPTIONAL] @@ -413,62 +403,6 @@ function _should_have_content_in_directory() { assert_success } -# Filters a service's logs (under `/var/log/supervisor/.log`) given -# a specific string. -# -# @param ${1} = service name -# @param ${2} = string to filter by -# @param ${3} = container name [OPTIONAL] -# -# ## Attention -# -# The string given to this function is interpreted by `grep -E`, i.e. -# as a regular expression. In case you use characters that are special -# in regular expressions, you need to escape them! -function _filter_service_log() { - local SERVICE=${1:?Service name must be provided} - local STRING=${2:?String to match must be provided} - local CONTAINER_NAME=$(__handle_container_name "${3:-}") - local FILE="/var/log/supervisor/${SERVICE}.log" - - # Fallback to alternative log location: - [[ -f ${FILE} ]] || FILE="/var/log/mail/${SERVICE}.log" - _run_in_container grep -E "${STRING}" "${FILE}" -} - -# Like `_filter_service_log` but asserts that the string was found. -# -# @param ${1} = service name -# @param ${2} = string to filter by -# @param ${3} = container name [OPTIONAL] -# -# ## Attention -# -# The string given to this function is interpreted by `grep -E`, i.e. -# as a regular expression. In case you use characters that are special -# in regular expressions, you need to escape them! -function _service_log_should_contain_string() { - local SERVICE=${1:?Service name must be provided} - local STRING=${2:?String to match must be provided} - local CONTAINER_NAME=$(__handle_container_name "${3:-}") - - _filter_service_log "${SERVICE}" "${STRING}" - assert_success -} - -# Filters the mail log for lines that belong to a certain email identified -# by its ID. You can obtain the ID of an email you want to send by using -# `_send_email_and_get_id`. -# -# @param ${1} = email ID -# @param ${2} = container name [OPTIONAL] -function _print_mail_log_for_id() { - local MAIL_ID=${1:?Mail ID must be provided} - local CONTAINER_NAME=$(__handle_container_name "${2:-}") - - _run_in_container grep -E "${MAIL_ID}" /var/log/mail.log -} - # A simple wrapper for netcat (`nc`). This is useful when sending # "raw" e-mails or doing IMAP-related work. # diff --git a/test/helper/log_and_filtering.bash b/test/helper/log_and_filtering.bash new file mode 100644 index 00000000000..415a203b3a6 --- /dev/null +++ b/test/helper/log_and_filtering.bash @@ -0,0 +1,119 @@ +#!/bin/bash + +# ? ABOUT: Functions defined here aid in working with logs and filtering them. + +# ! ATTENTION: This file is loaded by `common.sh` - do not load it yourself! +# ! ATTENTION: This file requires helper functions from `common.sh`! + +# shellcheck disable=SC2034,SC2155 + +# Assert that the number of lines output by a previous command matches the given amount (${1}). +# `lines` is a special BATS variable updated via `run`. +# +# @param ${1} = number of lines that the output should have +function _should_output_number_of_lines() { + # shellcheck disable=SC2154 + assert_equal "${#lines[@]}" "${1:?Number of lines not provided}" +} + +# Filters a service's logs (under `/var/log/supervisor/.log`) given a specific string. +# +# @param ${1} = service name +# @param ${2} = string to filter by +# @param ... = options given to `grep` (which is used to filter logs) +function _filter_service_log() { + local SERVICE=${1:?Service name must be provided} + local STRING=${2:?String to match must be provided} + shift 2 + + local FILE="/var/log/supervisor/${SERVICE}.log" + # Alternative log location fallback: + [[ -f ${FILE} ]] || FILE="/var/log/mail/${SERVICE}.log" + _run_in_container grep "${@}" "${STRING}" "${FILE}" +} + +# Prints the entirety of the primary mail log. +# Avoid using this method when you could filter more specific log lines with: +# +# 1. _filter_service_log +# 2. _service_log_should[_not]_contain_string +function _show_complete_mail_log() { + _run_in_container cat /var/log/mail/mail.log +} + +# Like `_filter_service_log` but asserts that the string was found. +# +# @param ${1} = service name +# @param ${2} = string to filter by +function _service_log_should_contain_string() { + _filter_service_log "${1}" "${2}" --fixed-strings + assert_success +} + +# Like `_filter_service_log` but asserts that the string was _not_ found. +# +# @param ${1} = service name +# @param ${2} = string to filter by +function _service_log_should_not_contain_string() { + _filter_service_log "${1}" "${2}" --fixed-strings + assert_failure +} + +# Like `_filter_service_log` but asserts that the string was found. +# Uses regular expressions under the hood for pattern matching. +# +# @param ${1} = service name +# @param ${2} = regular expression to filter by +function _service_log_should_contain_string_regexp() { + _filter_service_log "${1}" "${2}" --extended-regexp + assert_success +} + +# Like `_filter_service_log` but asserts that the string was _not_ found. +# Uses regular expressions under the hood for pattern matching. +# +# @param ${1} = service name +# @param ${2} = regular expression to filter by +function _service_log_should_not_contain_string_regexp() { + _filter_service_log "${1}" "${2}" --extended-regexp + assert_failure +} + +# Filters the mail log by the given MSG_ID (Message-ID) parameter, +# printing log lines which include the associated Postfix Queue ID. +# +# @param ${1} = The local-part of a Message-ID header value (``) +function _print_mail_log_of_queue_id_from_msgid() { + # A unique ID Postfix generates for tracking queued mail as it's processed. + # The length can vary (as per the postfix docs). Hence, we use a range to safely capture it. + # https://github.com/docker-mailserver/docker-mailserver/pull/3747#discussion_r1446679671 + local QUEUE_ID_REGEX='[A-Z0-9]{9,12}' + + local MSG_ID=$(__construct_msgid "${1:?The local-part for MSG_ID was not provided}") + shift 1 + + _wait_for_empty_mail_queue_in_container + + QUEUE_ID=$(_exec_in_container tac /var/log/mail.log \ + | grep -E "postfix/cleanup.*: ${QUEUE_ID_REGEX}:.*message-id=${MSG_ID}" \ + | grep -E --only-matching --max-count 1 "${QUEUE_ID_REGEX}" || :) + + # We perform plausibility checks on the IDs. + assert_not_equal "${QUEUE_ID}" '' + run echo "${QUEUE_ID}" + assert_line --regexp "^${QUEUE_ID_REGEX}$" + + # Postfix specific logs: + _filter_service_log 'mail' "${QUEUE_ID}" +} + +# A convenience method that filters for Dovecot specific logs with a `msgid` field that matches the MSG_ID input. +# +# @param ${1} = The local-part of a Message-ID header value (``) +function _print_mail_log_for_msgid() { + local MSG_ID=$(__construct_msgid "${1:?The local-part for MSG_ID was not provided}") + shift 1 + + # Dovecot specific logs: + _filter_service_log 'mail' "msgid=${MSG_ID}" +} diff --git a/test/helper/sending.bash b/test/helper/sending.bash index e18dc1ac9cd..1c5d844a4ae 100644 --- a/test/helper/sending.bash +++ b/test/helper/sending.bash @@ -1,11 +1,13 @@ #!/bin/bash -# shellcheck disable=SC2034,SC2155 - # ? ABOUT: Functions defined here help with sending emails in tests. # ! ATTENTION: This file is loaded by `common.sh` - do not load it yourself! # ! ATTENTION: This file requires helper functions from `common.sh`! +# ! ATTENTION: Functions prefixed with `__` are intended for internal use within +# ! this file (or other helpers) only, not in tests. + +# shellcheck disable=SC2034,SC2155 # Sends an e-mail from the container named by the environment variable `CONTAINER_NAME` # to the same or another container. @@ -100,74 +102,35 @@ function _send_email() { return "${RETURN_VALUE}" } -# Like `_send_email` with two major differences: -# -# 1. this function waits for the mail to be processed; there is no asynchronicity -# because filtering the logs in a synchronous way is easier and safer; -# 2. this function takes the name of a variable and inserts IDs one can later -# filter by to check logs. -# -# No. 2 is especially useful in case you send more than one email in a single -# test file and need to assert certain log entries for each mail individually. -# -# The first argument has to be the name of the variable that the e-mail ID is stored in. -# The second argument **can** be the flag `--expect-rejection`. -# -# - If this flag is supplied, the function does not check whether the whole mail delivery -# transaction was successful. Additionally the queue ID will be retrieved differently. -# - CAUTION: It must still be possible to `grep` for the Message-ID that Postfix -# generated in the mail log; otherwise this function fails. -# -# The rest of the arguments are the same as `_send_email`. +# Construct the value for a 'Message-ID' header. +# For tests we use only the local-part to identify mail activity in logs. The rest of the value is fixed. # -# ## Attention +# A Message-ID header value should be in the form of: `` +# https://en.wikipedia.org/wiki/Message-ID +# https://datatracker.ietf.org/doc/html/rfc5322#section-3.6.4 # -# This function assumes `CONTAINER_NAME` to be properly set (to the container -# name the command should be executed in)! -# -# ## Safety +# @param ${1} = The local-part of a Message-ID header value (``) +function __construct_msgid() { + local MSG_ID_LOCALPART=${1:?The local-part for MSG_ID was not provided} + echo "<${MSG_ID_LOCALPART}@dms-tests>" +} + +# Like `_send_email` but adds a "Message-ID: ${1}@dms-tests>" header, +# which allows for filtering logs later. # -# This functions assumes **no concurrent sending of emails to the same container**! -# If two clients send simultaneously, there is no guarantee the correct ID is -# chosen. Sending more than one mail at any given point in time with this function -# is UNDEFINED BEHAVIOR! -function _send_email_and_get_id() { - # Export the variable denoted by ${1} so everyone has access - export "${1:?Mail ID must be set for _send_email_and_get_id}" - # Get a "reference" to the content of the variable denoted by ${1} so we can manipulate the content - local -n ID_ENV_VAR_REF=${1:?} - # Prepare the message ID header here because we will shift away ${1} later - local MID="<${1}@dms-tests>" - # Get rid of ${1} so only the arguments for swaks remain +# @param ${1} = The local-part of a Message-ID header value (``) +function _send_email_with_msgid() { + local MSG_ID=$(__construct_msgid "${1:?The local-part for MSG_ID was not provided}") shift 1 - # The unique ID Postfix (and other services) use may be different in length - # on different systems. Hence, we use a range to safely capture it. - local QUEUE_ID_REGEX='[A-Z0-9]{9,12}' - - _wait_for_empty_mail_queue_in_container - _send_email "${@}" --header "Message-Id: ${MID}" - _wait_for_empty_mail_queue_in_container - - # We store Postfix's queue ID first - ID_ENV_VAR_REF=$(_exec_in_container tac /var/log/mail.log \ - | grep -E "postfix/cleanup.*: ${QUEUE_ID_REGEX}:.*message-id=${MID}" \ - | grep -E --only-matching --max-count 1 "${QUEUE_ID_REGEX}" || :) - # But we also requre potential Dovecot sieve output, which requires the mesage ID, - # so we need to provide the message ID too. - ID_ENV_VAR_REF+="|${MID}" - - # Last but not least, we perform plausibility checks on the IDs. - assert_not_equal "${ID_ENV_VAR_REF}" '' - run echo "${ID_ENV_VAR_REF}" - assert_line --regexp "^${QUEUE_ID_REGEX}\|${MID}$" + _send_email "${@}" --header "Message-ID: ${MSG_ID}" } # Send a spam e-mail by utilizing GTUBE. # -# Extra arguments given to this function will be supplied by `_send_email_and_get_id` directly. +# Extra arguments given to this function will be supplied by `_send_email_with_msgid` directly. function _send_spam() { - _send_email_and_get_id MAIL_ID_SPAM "${@}" \ + _send_email_with_msgid 'dms-test-email-spam' "${@}" \ --from 'spam@external.tld' \ --body 'XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X' } diff --git a/test/tests/parallel/set1/spam_virus/clamav.bats b/test/tests/parallel/set1/spam_virus/clamav.bats index a916896af66..36149edc3b4 100644 --- a/test/tests/parallel/set1/spam_virus/clamav.bats +++ b/test/tests/parallel/set1/spam_virus/clamav.bats @@ -38,8 +38,7 @@ function teardown_file() { _default_teardown ; } } @test 'should be identified by Amavis' { - _run_in_container grep -i 'Found secondary av scanner ClamAV-clamscan' /var/log/mail/mail.log - assert_success + _service_log_should_contain_string 'mail' 'Found secondary av scanner ClamAV-clamscan' } @test 'freshclam cron is enabled' { @@ -53,6 +52,6 @@ function teardown_file() { _default_teardown ; } } @test 'rejects virus' { - _run_in_container_bash "grep 'Blocked INFECTED' /var/log/mail/mail.log | grep ' -> '" - assert_success + _service_log_should_contain_string 'mail' 'Blocked INFECTED' + assert_output --partial ' -> ' } diff --git a/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats b/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats index 1711950fb6d..0f3a7b5b296 100644 --- a/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats +++ b/test/tests/parallel/set1/spam_virus/disabled_clamav_spamassassin.bats @@ -25,20 +25,18 @@ function setup_file() { function teardown_file() { _default_teardown ; } @test "ClamAV - Amavis integration should not be active" { - _run_in_container grep -i 'Found secondary av scanner ClamAV-clamscan' /var/log/mail/mail.log - assert_failure + _service_log_should_not_contain_string 'mail' 'Found secondary av scanner ClamAV-clamscan' } @test "SA - Amavis integration should not be active" { # Wait until Amavis has finished initializing: run _repeat_in_container_until_success_or_timeout 20 "${CONTAINER_NAME}" grep 'Deleting db files in /var/lib/amavis/db' /var/log/mail/mail.log assert_success + # Amavis module for SA should not be loaded (`SpamControl: scanner SpamAssassin, module Amavis::SpamControl::SpamAssassin`): - _run_in_container grep 'scanner SpamAssassin' /var/log/mail/mail.log - assert_failure + _service_log_should_not_contain_string 'mail' 'scanner SpamAssassin' } @test "SA - should not have been called" { - _run_in_container grep -i 'connect to /var/run/clamav/clamd.ctl failed' /var/log/mail/mail.log - assert_failure + _service_log_should_not_contain_string 'mail' 'connect to /var/run/clamav/clamd.ctl failed' } diff --git a/test/tests/parallel/set1/spam_virus/postscreen.bats b/test/tests/parallel/set1/spam_virus/postscreen.bats index 92333f98d67..3f93fe9d566 100644 --- a/test/tests/parallel/set1/spam_virus/postscreen.bats +++ b/test/tests/parallel/set1/spam_virus/postscreen.bats @@ -56,12 +56,11 @@ function teardown_file() { @test "should successfully pass postscreen and get postfix greeting message (respecting postscreen_greet_wait time)" { # Configure `send_email()` to send from the mail client container (CONTAINER2_NAME) via ENV override, # mail is sent to the DMS server container (CONTAINER1_NAME) via `--server` parameter: - # TODO: Use _send_email_and_get_id when proper resolution of domain names is possible: CONTAINER_NAME=${CONTAINER2_NAME} _send_email --expect-rejection --server "${CONTAINER1_IP}" --port 25 --data 'postscreen.txt' - # CONTAINER_NAME=${CONTAINER2_NAME} _send_email_and_get_id MAIL_ID_POSTSCREEN --server "${CONTAINER1_IP}" --data 'postscreen.txt' - # _print_mail_log_for_id "${MAIL_ID_POSTSCREEN}" + # TODO: Use _send_email_with_msgid when proper resolution of domain names is possible: + # CONTAINER_NAME=${CONTAINER2_NAME} _send_email_with_msgid 'msgid-postscreen' --server "${CONTAINER1_IP}" --data 'postscreen.txt' + # _print_mail_log_for_msgid 'msgid-postscreen' # assert_output --partial "stored mail into mailbox 'INBOX'" - _run_in_container cat /var/log/mail.log - assert_output --partial 'PASS NEW' + _service_log_should_contain_string 'mail' 'PASS NEW' } diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index f66e923185f..9e6d62222e5 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -45,16 +45,17 @@ function setup_file() { # We will send 4 emails: # 1. The first one should pass just fine - _send_email_and_get_id MAIL_ID_PASS + _send_email_with_msgid 'rspamd-test-email-pass' # 2. The second one should be rejected (Rspamd-specific GTUBE pattern for rejection) _send_spam --expect-rejection # 3. The third one should be rejected due to a virus (ClamAV EICAR pattern) # shellcheck disable=SC2016 - _send_email_and_get_id MAIL_ID_VIRUS --expect-rejection \ + _send_email_with_msgid 'rspamd-test-email-virus' --expect-rejection \ --body 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*' # 4. The fourth one will receive an added header (Rspamd-specific GTUBE pattern for adding a spam header) # ref: https://rspamd.com/doc/gtube_patterns.html - _send_email_and_get_id MAIL_ID_HEADER --body "YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X" + _send_email_with_msgid 'rspamd-test-email-header' \ + --body "YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X" _run_in_container cat /var/log/mail.log assert_success @@ -92,7 +93,7 @@ function teardown_file() { _default_teardown ; } } @test 'service log exist and contains proper content' { - _service_log_should_contain_string 'rspamd' 'rspamd .* is loading configuration' + _service_log_should_contain_string_regexp 'rspamd' 'rspamd .* is loading configuration' _service_log_should_contain_string 'rspamd' 'lua module clickhouse is disabled in the configuration' _service_log_should_contain_string 'rspamd' 'lua module elastic is disabled in the configuration' _service_log_should_contain_string 'rspamd' 'lua module neural is disabled in the configuration' @@ -108,33 +109,40 @@ function teardown_file() { _default_teardown ; } } @test 'normal mail passes fine' { - _service_log_should_contain_string 'rspamd' 'F \(no action\)' + _service_log_should_contain_string 'rspamd' 'F (no action)' - _print_mail_log_for_id "${MAIL_ID_PASS}" + _print_mail_log_for_msgid 'rspamd-test-email-pass' assert_output --partial "stored mail into mailbox 'INBOX'" _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 } @test 'detects and rejects spam' { - _service_log_should_contain_string 'rspamd' 'S \(reject\)' + _service_log_should_contain_string 'rspamd' 'S (reject)' _service_log_should_contain_string 'rspamd' 'reject "Gtube pattern"' - _print_mail_log_for_id "${MAIL_ID_SPAM}" + _print_mail_log_of_queue_id_from_msgid 'dms-test-email-spam' assert_output --partial 'milter-reject' assert_output --partial '5.7.1 Gtube pattern' + _print_mail_log_for_msgid 'dms-test-email-spam' + refute_output --partial "stored mail into mailbox 'INBOX'" + assert_failure + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 } @test 'detects and rejects virus' { - _service_log_should_contain_string 'rspamd' 'T \(reject\)' + _service_log_should_contain_string 'rspamd' 'T (reject)' _service_log_should_contain_string 'rspamd' 'reject "ClamAV FOUND VIRUS "Eicar-Signature"' - _print_mail_log_for_id "${MAIL_ID_VIRUS}" + _print_mail_log_of_queue_id_from_msgid 'rspamd-test-email-virus' assert_output --partial 'milter-reject' assert_output --partial '5.7.1 ClamAV FOUND VIRUS "Eicar-Signature"' + + _print_mail_log_for_msgid 'dms-test-email-spam' refute_output --partial "stored mail into mailbox 'INBOX'" + assert_failure _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 } @@ -217,10 +225,10 @@ function teardown_file() { _default_teardown ; } _run_in_container_bash '[[ -f /usr/lib/dovecot/sieve-global/after/spam_to_junk.svbin ]]' assert_success - _service_log_should_contain_string 'rspamd' 'S \(add header\)' + _service_log_should_contain_string 'rspamd' 'S (add header)' _service_log_should_contain_string 'rspamd' 'add header "Gtube pattern"' - _print_mail_log_for_id "${MAIL_ID_HEADER}" + _print_mail_log_for_msgid 'rspamd-test-email-header' assert_output --partial "fileinto action: stored mail into mailbox 'Junk'" _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 @@ -265,10 +273,10 @@ function teardown_file() { _default_teardown ; } _nc_wrapper 'nc/rspamd_imap_move_to_junk.txt' '0.0.0.0 143' sleep 1 # wait for the transaction to finish - _run_in_container cat /var/log/mail/mail.log - assert_success - assert_output --partial 'imapsieve: Matched static mailbox rule [1]' - refute_output --partial 'imapsieve: Matched static mailbox rule [2]' + _service_log_should_contain_string 'mail' 'imapsieve: Matched static mailbox rule [1]' + _service_log_should_not_contain_string 'mail' 'imapsieve: Matched static mailbox rule [2]' + + _show_complete_mail_log for LINE in "${LEARN_SPAM_LINES[@]}"; do assert_output --partial "${LINE}" done @@ -279,9 +287,9 @@ function teardown_file() { _default_teardown ; } _nc_wrapper 'nc/rspamd_imap_move_to_inbox.txt' '0.0.0.0 143' sleep 1 # wait for the transaction to finish - _run_in_container cat /var/log/mail/mail.log - assert_success - assert_output --partial 'imapsieve: Matched static mailbox rule [2]' + _service_log_should_contain_string 'mail' 'imapsieve: Matched static mailbox rule [2]' + + _show_complete_mail_log for LINE in "${LEARN_HAM_LINES[@]}"; do assert_output --partial "${LINE}" done diff --git a/test/tests/parallel/set3/mta/dsn.bats b/test/tests/parallel/set3/mta/dsn.bats index 9a65b147c10..7a6af76c1c9 100644 --- a/test/tests/parallel/set3/mta/dsn.bats +++ b/test/tests/parallel/set3/mta/dsn.bats @@ -54,7 +54,7 @@ function teardown_file() { _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 587' _wait_for_empty_mail_queue_in_container - _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log + _filter_service_log 'mail' "${LOG_DSN}" _should_output_number_of_lines 3 } @@ -70,15 +70,14 @@ function teardown_file() { # # Although external requests are discarded, anyone who has requested a DSN # will still receive it, but it will come from the sending mail server, not this one. - _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log - assert_failure + _service_log_should_not_contain_string 'mail' "${LOG_DSN}" # These ports are excluded via master.cf. _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 465' _nc_wrapper 'emails/nc_raw/dsn/authenticated.txt' '0.0.0.0 587' _wait_for_empty_mail_queue_in_container - _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log + _service_log_should_contain_string 'mail' "${LOG_DSN}" _should_output_number_of_lines 2 } @@ -92,6 +91,5 @@ function teardown_file() { # DSN requests are rejected regardless of origin. # This is usually a bad idea, as you won't get them either. - _run_in_container grep "${LOG_DSN}" /var/log/mail/mail.log - assert_failure + _service_log_should_not_contain_string 'mail' "${LOG_DSN}" } diff --git a/test/tests/parallel/set3/mta/smtp_delivery.bats b/test/tests/parallel/set3/mta/smtp_delivery.bats index e851d94e9b4..d183694b06e 100644 --- a/test/tests/parallel/set3/mta/smtp_delivery.bats +++ b/test/tests/parallel/set3/mta/smtp_delivery.bats @@ -174,33 +174,28 @@ function _successful() { } @test "delivers mail to existing alias" { - _run_in_container grep 'to=, orig_to=' /var/log/mail/mail.log - assert_success + _service_log_should_contain_string 'mail' 'to=, orig_to=' assert_output --partial 'status=sent' _should_output_number_of_lines 1 } @test "delivers mail to existing alias with recipient delimiter" { - _run_in_container grep 'to=, orig_to=' /var/log/mail/mail.log - assert_success + _service_log_should_contain_string 'mail' 'to=, orig_to=' assert_output --partial 'status=sent' _should_output_number_of_lines 1 - _run_in_container grep 'to=' /var/log/mail/mail.log - assert_success + _service_log_should_contain_string 'mail' 'to=' refute_output --partial 'status=bounced' } @test "delivers mail to existing catchall" { - _run_in_container grep 'to=, orig_to=' /var/log/mail/mail.log - assert_success + _service_log_should_contain_string 'mail' 'to=, orig_to=' assert_output --partial 'status=sent' _should_output_number_of_lines 1 } @test "delivers mail to regexp alias" { - _run_in_container grep 'to=, orig_to=' /var/log/mail/mail.log - assert_success + _service_log_should_contain_string 'mail' 'to=, orig_to=' assert_output --partial 'status=sent' _should_output_number_of_lines 1 } @@ -227,23 +222,20 @@ function _successful() { } @test "rejects mail to unknown user" { - _run_in_container grep ': Recipient address rejected: User unknown in virtual mailbox table' /var/log/mail/mail.log - assert_success + _service_log_should_contain_string 'mail' ': Recipient address rejected: User unknown in virtual mailbox table' _should_output_number_of_lines 1 } @test "redirects mail to external aliases" { - _run_in_container_bash "grep 'Passed CLEAN {RelayedInbound}' /var/log/mail/mail.log | grep -- '-> '" - assert_success - assert_output --partial ' -> ' + _service_log_should_contain_string 'mail' 'Passed CLEAN {RelayedInbound}' + run bash -c "grep ' -> ' <<< '${output}'" _should_output_number_of_lines 2 # assert_output --partial 'external.tld=user@example.test> -> ' } # TODO: Add a test covering case SPAMASSASSIN_SPAM_TO_INBOX=1 (default) @test "rejects spam" { - _run_in_container grep 'Blocked SPAM {NoBounceInbound,Quarantined}' /var/log/mail/mail.log - assert_success + _service_log_should_contain_string 'mail' 'Blocked SPAM {NoBounceInbound,Quarantined}' assert_output --partial ' -> ' _should_output_number_of_lines 1 diff --git a/test/tests/serial/mail_pop3.bats b/test/tests/serial/mail_pop3.bats index d815e1eed9e..95ae14aa123 100644 --- a/test/tests/serial/mail_pop3.bats +++ b/test/tests/serial/mail_pop3.bats @@ -33,11 +33,10 @@ function teardown_file() { _default_teardown ; } assert_success } -@test '/var/log/mail/mail.log is error-free' { - _run_in_container grep 'non-null host address bits in' /var/log/mail/mail.log - assert_failure - _run_in_container grep ': error:' /var/log/mail/mail.log - assert_failure +# TODO: Remove in favor of a common helper method, as described in vmail-id.bats equivalent test-case +@test 'Mail log is error free' { + _service_log_should_not_contain_string 'mail' 'non-null host address bits in' + _service_log_should_not_contain_string 'mail' ': Error:' } @test '(Manage Sieve) disabled per default' { diff --git a/test/tests/serial/mail_with_oauth2.bats b/test/tests/serial/mail_with_oauth2.bats index 8da87980907..c9a02eeaded 100644 --- a/test/tests/serial/mail_with_oauth2.bats +++ b/test/tests/serial/mail_with_oauth2.bats @@ -110,7 +110,7 @@ function __verify_auth_with_smtp() { function __dovecot_logs_should_verify_success() { # Inspect the relevant Dovecot logs to catch failure / success: - _run_in_container grep 'dovecot:' /var/log/mail.log + _service_log_should_contain_string 'mail' 'dovecot:' refute_output --partial 'oauth2 failed: Introspection failed' assert_output --partial "dovecot: imap-login: Login: user=<${USER_ACCOUNT}>, method=${AUTH_METHOD}" diff --git a/test/tests/serial/tests.bats b/test/tests/serial/tests.bats index a344bd8d481..eff151dfc12 100644 --- a/test/tests/serial/tests.bats +++ b/test/tests/serial/tests.bats @@ -182,23 +182,16 @@ function teardown_file() { _default_teardown ; } assert_success } -@test "system: /var/log/mail/mail.log is error free" { - _run_in_container grep 'non-null host address bits in' /var/log/mail/mail.log - assert_failure - _run_in_container grep 'mail system configuration error' /var/log/mail/mail.log - assert_failure - _run_in_container grep ': error:' /var/log/mail/mail.log - assert_failure - _run_in_container grep -i 'is not writable' /var/log/mail/mail.log - assert_failure - _run_in_container grep -i 'permission denied' /var/log/mail/mail.log - assert_failure - _run_in_container grep -i '(!)connect' /var/log/mail/mail.log - assert_failure - _run_in_container grep -i 'using backwards-compatible default setting' /var/log/mail/mail.log - assert_failure - _run_in_container grep -i 'connect to 127.0.0.1:10023: Connection refused' /var/log/mail/mail.log - assert_failure +# TODO: Remove in favor of a common helper method, as described in vmail-id.bats equivalent test-case +@test "system: Mail log is error free" { + _service_log_should_not_contain_string 'mail' 'non-null host address bits in' + _service_log_should_not_contain_string 'mail' 'mail system configuration error' + _service_log_should_not_contain_string 'mail' ': Error:' + _service_log_should_not_contain_string 'mail' 'is not writable' + _service_log_should_not_contain_string 'mail' 'Permission denied' + _service_log_should_not_contain_string 'mail' '(!)connect' + _service_log_should_not_contain_string 'mail' 'using backwards-compatible default setting' + _service_log_should_not_contain_string 'mail' 'connect to 127.0.0.1:10023: Connection refused' } @test "system: /var/log/auth.log is error free" { @@ -212,7 +205,8 @@ function teardown_file() { _default_teardown ; } } @test "system: amavis decoders installed and available" { - _run_in_container_bash "grep -E '.*(Internal decoder|Found decoder) for\s+\..*' /var/log/mail/mail.log*|grep -Eo '(mail|Z|gz|bz2|xz|lzma|lrz|lzo|lz4|rpm|cpio|tar|deb|rar|arj|arc|zoo|doc|cab|tnef|zip|kmz|7z|jar|swf|lha|iso|exe)' | sort | uniq" + _service_log_should_contain_string_regexp 'mail' '.*(Internal decoder|Found decoder) for\s+\..*' + run bash -c "grep -Eo '(mail|Z|gz|bz2|xz|lzma|lrz|lzo|lz4|rpm|cpio|tar|deb|rar|arj|arc|zoo|doc|cab|tnef|zip|kmz|7z|jar|swf|lha|iso|exe)' <<< '${output}' | sort | uniq" assert_success # Support for doc and zoo removed in buster cat <<'EOF' | assert_output diff --git a/test/tests/serial/vmail-id.bats b/test/tests/serial/vmail-id.bats index 2541f4f8ea6..a0fb85372d2 100644 --- a/test/tests/serial/vmail-id.bats +++ b/test/tests/serial/vmail-id.bats @@ -24,8 +24,7 @@ function teardown_file() { _default_teardown ; } _wait_for_empty_mail_queue_in_container # Should be successfully sent (received) by Postfix: - _run_in_container grep 'to=' /var/log/mail/mail.log - assert_success + _service_log_should_contain_string 'mail' 'to=' assert_output --partial 'status=sent' _should_output_number_of_lines 1 @@ -41,35 +40,28 @@ function teardown_file() { _default_teardown ; } # This test case is shared with tests.bats, but provides context on errors + some minor edits # TODO: Could improve in future with keywords from https://github.com/docker-mailserver/docker-mailserver/pull/3550#issuecomment-1738509088 # Potentially via a helper that allows an optional fixed number of errors to be present if they were intentional -@test '/var/log/mail/mail.log is error free' { +@test 'Mail log is error free' { # Postfix: https://serverfault.com/questions/934703/postfix-451-4-3-0-temporary-lookup-failure - _run_in_container grep 'non-null host address bits in' /var/log/mail/mail.log - assert_failure + _service_log_should_not_contain_string 'mail' 'non-null host address bits in' # Postfix delivery failure: https://github.com/docker-mailserver/docker-mailserver/issues/230 - _run_in_container grep 'mail system configuration error' /var/log/mail/mail.log - assert_failure + _service_log_should_not_contain_string 'mail' 'mail system configuration error' # Unknown error source: https://github.com/docker-mailserver/docker-mailserver/pull/85 - _run_in_container grep -i ': error:' /var/log/mail/mail.log - assert_failure + _service_log_should_not_contain_string 'mail' ': Error:' # Unknown error source: https://github.com/docker-mailserver/docker-mailserver/pull/320 - _run_in_container grep -i 'not writable' /var/log/mail/mail.log - assert_failure - _run_in_container grep -i 'permission denied' /var/log/mail/mail.log - assert_failure + _service_log_should_not_contain_string 'mail' 'not writable' + _service_log_should_not_contain_string 'mail' 'Permission denied' # Amavis: https://forum.howtoforge.com/threads/postfix-smtp-error-caused-by-clamav-cant-connect-to-a-unix-socket-var-run-clamav-clamd-ctl.81002/ - _run_in_container grep -i '(!)connect' /var/log/mail/mail.log - assert_failure + _service_log_should_not_contain_string 'mail' '(!)connect' # Postfix: https://github.com/docker-mailserver/docker-mailserver/pull/2597 - _run_in_container grep -i 'using backwards-compatible default setting' /var/log/mail/mail.log - assert_failure + # Log line match example: https://github.com/docker-mailserver/docker-mailserver/pull/2598#issuecomment-1141176633 + _service_log_should_not_contain_string 'mail' 'using backwards-compatible default setting' # Postgrey: https://github.com/docker-mailserver/docker-mailserver/pull/612#discussion_r117635774 - _run_in_container grep -i 'connect to 127.0.0.1:10023: Connection refused' /var/log/mail/mail.log - assert_failure + _service_log_should_not_contain_string 'mail' 'connect to 127.0.0.1:10023: Connection refused' } From 303ca82fb925ab9a0e0aa8bf68a2caa402afe0aa Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 26 Jan 2024 01:02:19 +1300 Subject: [PATCH 268/592] docs(fix): New external link icon workaround for mkdocs-material 9.5.5 (#3823) This is the easiest to maintain workaround now available. Upstream continues to reject the value such a feature for accessibility. --- docs/content/assets/css/customizations.css | 8 ++++++-- docs/mkdocs.yml | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/content/assets/css/customizations.css b/docs/content/assets/css/customizations.css index 49ede009896..320da152db7 100644 --- a/docs/content/assets/css/customizations.css +++ b/docs/content/assets/css/customizations.css @@ -16,12 +16,16 @@ If you want to append instead, switch `::before` to `::after`. src: url('../fonts/external-link.woff') format('woff'); } -/* Matches the two nav link classes that start with `http` `href` values, regular docs pages use relative URLs instead. */ -.md-tabs__link[href^="http"]::before, .md-nav__link[href^="http"]::before { +/* + Since mkdocs-material 9.5.5 broke support in our docs from DMS v13.3.1, we now use our own class name, + which has been included for the two external nav links in mkdocs.yml via workaround (insert HTML). +*/ +.icon-external-link::before { display: inline-block; /* treat similar to text */ font-family: 'external-link'; content:'\0041'; /* represents "A" which our font renders as an icon instead of the "A" glyph */ font-size: 80%; /* icon is a little too big by default, scale it down */ + margin-right: 4px; } /* ============================================================================================================= */ diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 0d34b407ba3..15d2c3b2dcc 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -173,5 +173,5 @@ nav: - 'General Information': contributing/general.md - 'Tests': contributing/tests.md - 'Issues and Pull Requests': contributing/issues-and-pull-requests.md - - 'DockerHub': https://hub.docker.com/r/mailserver/docker-mailserver/ - - 'GHCR': https://github.com/docker-mailserver/docker-mailserver/pkgs/container/docker-mailserver + - 'DockerHub': https://hub.docker.com/r/mailserver/docker-mailserver/ + - 'GHCR': https://github.com/docker-mailserver/docker-mailserver/pkgs/container/docker-mailserver From 22c6daee32ad0769002564054654a4e989c9eb10 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 26 Jan 2024 01:21:24 +1300 Subject: [PATCH 269/592] chore: Revise improper restart message (#3826) Improved guidance. --- target/scripts/startup/check-stack.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/scripts/startup/check-stack.sh b/target/scripts/startup/check-stack.sh index 61f21d1ce2c..cf1c40f5f05 100644 --- a/target/scripts/startup/check-stack.sh +++ b/target/scripts/startup/check-stack.sh @@ -19,7 +19,7 @@ function _check_improper_restart() { if [[ -f /CONTAINER_START ]]; then _log 'warn' 'This container was (likely) improperly restarted which can result in undefined behavior' - _log 'warn' 'Please destroy the container properly and then start DMS again' + _log 'warn' "Please use 'docker compose up --force-recreate' or equivalent (view our troubleshooting docs)" fi } From 47f8d50bebaa0b1f1cb85d45c1d9210d9ce80a49 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 26 Jan 2024 10:28:26 +1300 Subject: [PATCH 270/592] fix: Ensure configs are sanitized for parsing (#3819) * chore: Detect missing final newline in configs read These lines will be not be processed by `read`, emit a warning to raise awareness. * fix: Ensure parsed config has final newline appended (when possible) This functionality was handled in `accounts.sh` via a similar sed command (that the linked references also offer). `printf` is better for this, no shellcheck comment required either. We additionally don't attempt to modify files that are read-only. * fix: Ensure parsed configs have CRLF to LF corrected (where possible) Likewise, this runtime fix was only covering two config files. It now applies to all callers of this method. * fix: Sanitize `postfix-master.cf` via helper This feature should have been using the helper to avoid user error from their config updates accidentally introducing subtle breakage implicitly (due to CRLF or missing final newline). * tests: Add test cases for new helpers * tests: `rm` is redundant when using `BATS_TEST_TMPDIR` This temporary directory is created and removed implicitly. Even after a test failure. * chore: Remove old `postfix-virtual.cf` migration logic This was introduced in 2018, there should be no one needing to rely on this anymore? * tests: Remove comment on sed failure concern * chore: Add entry to `CHANGELOG.md` * Apply suggestions from code review Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --------- Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- CHANGELOG.md | 4 ++ target/scripts/helpers/accounts.sh | 14 ------- target/scripts/helpers/aliases.sh | 5 --- target/scripts/helpers/utils.sh | 35 ++++++++++++++++ target/scripts/startup/setup.d/postfix.sh | 22 +++++----- test/helper/setup.bash | 1 - .../set3/scripts/helper_functions.bats | 42 +++++++++++++++++++ 7 files changed, 92 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12f07561c7e..9f440e31b63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,10 @@ The most noteworthy change of this release is the update of the container's base - **Tests:** - Refactored helper methods for sending e-mails with specific `Message-ID` headers and the helpers for retrieving + filtering logs, which together help isolate logs relevant to specific mail when multiple mails have been processed within a single test. ([#3786](https://github.com/docker-mailserver/docker-mailserver/pull/3786)) +### Fixes + +- DMS config files that are parsed line by line are now more robust to parse by detecting and fixing line-endings ([#3819](https://github.com/docker-mailserver/docker-mailserver/pull/3819)) + ## [v13.3.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.3.1) ### Fixes diff --git a/target/scripts/helpers/accounts.sh b/target/scripts/helpers/accounts.sh index 7499a2ec8e5..31ded04a6ba 100644 --- a/target/scripts/helpers/accounts.sh +++ b/target/scripts/helpers/accounts.sh @@ -19,16 +19,9 @@ function _create_accounts() { _create_masters if [[ -f ${DATABASE_ACCOUNTS} ]]; then - _log 'trace' "Checking file line endings" - sed -i 's|\r||g' "${DATABASE_ACCOUNTS}" - _log 'trace' "Regenerating postfix user list" echo "# WARNING: this file is auto-generated. Modify ${DATABASE_ACCOUNTS} to edit the user list." > /etc/postfix/vmailbox - # checking that ${DATABASE_ACCOUNTS} ends with a newline - # shellcheck disable=SC1003 - sed -i -e '$a\' "${DATABASE_ACCOUNTS}" - chown dovecot:dovecot "${DOVECOT_USERDB_FILE}" chmod 640 "${DOVECOT_USERDB_FILE}" @@ -158,15 +151,8 @@ function _create_masters() { local DATABASE_DOVECOT_MASTERS='/tmp/docker-mailserver/dovecot-masters.cf' if [[ -f ${DATABASE_DOVECOT_MASTERS} ]]; then - _log 'trace' "Checking file line endings" - sed -i 's|\r||g' "${DATABASE_DOVECOT_MASTERS}" - _log 'trace' "Regenerating dovecot masters list" - # checking that ${DATABASE_DOVECOT_MASTERS} ends with a newline - # shellcheck disable=SC1003 - sed -i -e '$a\' "${DATABASE_DOVECOT_MASTERS}" - chown dovecot:dovecot "${DOVECOT_MASTERDB_FILE}" chmod 640 "${DOVECOT_MASTERDB_FILE}" diff --git a/target/scripts/helpers/aliases.sh b/target/scripts/helpers/aliases.sh index 04a56da3061..b0f2fa1af95 100644 --- a/target/scripts/helpers/aliases.sh +++ b/target/scripts/helpers/aliases.sh @@ -12,11 +12,6 @@ function _handle_postfix_virtual_config() { local DATABASE_VIRTUAL=/tmp/docker-mailserver/postfix-virtual.cf if [[ -f ${DATABASE_VIRTUAL} ]]; then - # fixing old virtual user file - if grep -q ",$" "${DATABASE_VIRTUAL}"; then - sed -i -e "s|, |,|g" -e "s|,$||g" "${DATABASE_VIRTUAL}" - fi - cp -f "${DATABASE_VIRTUAL}" /etc/postfix/virtual else _log 'debug' "'${DATABASE_VIRTUAL}' not provided - no mail alias/forward created" diff --git a/target/scripts/helpers/utils.sh b/target/scripts/helpers/utils.sh index f7095bf3690..c848e13e775 100644 --- a/target/scripts/helpers/utils.sh +++ b/target/scripts/helpers/utils.sh @@ -17,9 +17,44 @@ function _escape_for_sed() { # Returns input after filtering out lines that are: # empty, white-space, comments (`#` as the first non-whitespace character) function _get_valid_lines_from_file() { + _convert_crlf_to_lf_if_necessary "${1}" + _append_final_newline_if_missing "${1}" + grep --extended-regexp --invert-match "^\s*$|^\s*#" "${1}" || true } +# This is to sanitize configs from users that unknowingly introduced CRLF: +function _convert_crlf_to_lf_if_necessary() { + if [[ $(file "${1}") =~ 'CRLF' ]]; then + _log 'warn' "File '${1}' contains CRLF line-endings" + + if [[ -w ${1} ]]; then + _log 'debug' 'Converting CRLF to LF' + sed -i 's|\r||g' "${1}" + else + _log 'warn' "File '${1}' is not writable - cannot change CRLF to LF" + fi + fi +} + +# This is to sanitize configs from users that unknowingly removed the end-of-file LF: +function _append_final_newline_if_missing() { + # Correctly detect a missing final newline and fix it: + # https://stackoverflow.com/questions/38746/how-to-detect-file-ends-in-newline#comment82380232_25749716 + # https://unix.stackexchange.com/questions/31947/how-to-add-a-newline-to-the-end-of-a-file/441200#441200 + # https://unix.stackexchange.com/questions/159557/how-to-non-invasively-test-for-write-access-to-a-file + if [[ $(tail -c1 "${1}" | wc -l) -eq 0 ]]; then + # Avoid fixing when the destination is read-only: + if [[ -w ${1} ]]; then + printf '\n' >> "${1}" + + _log 'info' "File '${1}' was missing a final newline - this has been fixed" + else + _log 'warn' "File '${1}' is missing a final newline - it is not writable, hence it was not fixed - the last line will not be processed!" + fi + fi +} + # Provide the name of an environment variable to this function # and it will return its value stored in /etc/dms-settings function _get_dms_env_value() { diff --git a/target/scripts/startup/setup.d/postfix.sh b/target/scripts/startup/setup.d/postfix.sh index 5aec86365e6..05052faa5de 100644 --- a/target/scripts/startup/setup.d/postfix.sh +++ b/target/scripts/startup/setup.d/postfix.sh @@ -109,8 +109,9 @@ function _setup_postfix_late() { function __postfix__setup_override_configuration() { __postfix__log 'debug' 'Overriding / adjusting configuration with user-supplied values' - if [[ -f /tmp/docker-mailserver/postfix-main.cf ]]; then - cat /tmp/docker-mailserver/postfix-main.cf >>/etc/postfix/main.cf + local OVERRIDE_CONFIG_POSTFIX_MAIN='/tmp/docker-mailserver/postfix-main.cf' + if [[ -f ${OVERRIDE_CONFIG_POSTFIX_MAIN} ]]; then + cat "${OVERRIDE_CONFIG_POSTFIX_MAIN}" >>/etc/postfix/main.cf _adjust_mtime_for_postfix_maincf # do not directly output to 'main.cf' as this causes a read-write-conflict @@ -118,20 +119,19 @@ function __postfix__setup_override_configuration() { mv /tmp/postfix-main-new.cf /etc/postfix/main.cf _adjust_mtime_for_postfix_maincf - __postfix__log 'trace' "Adjusted '/etc/postfix/main.cf' according to '/tmp/docker-mailserver/postfix-main.cf'" + __postfix__log 'trace' "Adjusted '/etc/postfix/main.cf' according to '${OVERRIDE_CONFIG_POSTFIX_MAIN}'" else - __postfix__log 'trace' "No extra Postfix settings loaded because optional '/tmp/docker-mailserver/postfix-main.cf' was not provided" + __postfix__log 'trace' "No extra Postfix settings loaded because optional '${OVERRIDE_CONFIG_POSTFIX_MAIN}' was not provided" fi - if [[ -f /tmp/docker-mailserver/postfix-master.cf ]]; then + local OVERRIDE_CONFIG_POSTFIX_MASTER='/tmp/docker-mailserver/postfix-master.cf' + if [[ -f ${OVERRIDE_CONFIG_POSTFIX_MASTER} ]]; then while read -r LINE; do - if [[ ${LINE} =~ ^[0-9a-z] ]]; then - postconf -P "${LINE}" - fi - done < /tmp/docker-mailserver/postfix-master.cf - __postfix__log 'trace' "Adjusted '/etc/postfix/master.cf' according to '/tmp/docker-mailserver/postfix-master.cf'" + [[ ${LINE} =~ ^[0-9a-z] ]] && postconf -P "${LINE}" + done < <(_get_valid_lines_from_file "${OVERRIDE_CONFIG_POSTFIX_MASTER}") + __postfix__log 'trace' "Adjusted '/etc/postfix/master.cf' according to '${OVERRIDE_CONFIG_POSTFIX_MASTER}'" else - __postfix__log 'trace' "No extra Postfix settings loaded because optional '/tmp/docker-mailserver/postfix-master.cf' was not provided" + __postfix__log 'trace' "No extra Postfix settings loaded because optional '${OVERRIDE_CONFIG_POSTFIX_MASTER}' was not provided" fi } diff --git a/test/helper/setup.bash b/test/helper/setup.bash index 0dd57bd6b80..ed2e4e32c37 100644 --- a/test/helper/setup.bash +++ b/test/helper/setup.bash @@ -102,7 +102,6 @@ function _init_with_defaults() { # The config volume cannot be read-only as some data needs to be written at container startup # - # - two sed failures (unknown lines) # - dovecot-quotas.cf (setup-stack.sh:_setup_dovecot_quotas) # - postfix-aliases.cf (setup-stack.sh:_setup_postfix_aliases) # TODO: Check how many tests need write access. Consider using `docker create` + `docker cp` for easier cleanup. diff --git a/test/tests/parallel/set3/scripts/helper_functions.bats b/test/tests/parallel/set3/scripts/helper_functions.bats index 332de448e2a..518f87170f1 100644 --- a/test/tests/parallel/set3/scripts/helper_functions.bats +++ b/test/tests/parallel/set3/scripts/helper_functions.bats @@ -70,3 +70,45 @@ SOURCE_BASE_PATH="${REPOSITORY_ROOT:?Expected REPOSITORY_ROOT to be set}/target/ assert_failure assert_output --partial "ENV var name must be provided to _env_var_expect_integer" } + +@test '(utils.sh) _convert_crlf_to_lf_if_necessary' { + # shellcheck source=../../../../../target/scripts/helpers/log.sh + source "${SOURCE_BASE_PATH}/log.sh" + # shellcheck source=../../../../../target/scripts/helpers/utils.sh + source "${SOURCE_BASE_PATH}/utils.sh" + + # Create a temporary file in the BATS test-case folder: + local TMP_DMS_CONFIG=$(mktemp -p "${BATS_TEST_TMPDIR}" -t 'dms_XXX.cf') + # A file with mixed line-endings including CRLF: + echo -en 'line one\nline two\r\n' > "${TMP_DMS_CONFIG}" + + # Confirm CRLF detected: + run file "${TMP_DMS_CONFIG}" + assert_output --partial 'CRLF' + + # Helper method detects and fixes: + _convert_crlf_to_lf_if_necessary "${TMP_DMS_CONFIG}" + run file "${TMP_DMS_CONFIG}" + refute_output --partial 'CRLF' +} + +@test '(utils.sh) _append_final_newline_if_missing' { + # shellcheck source=../../../../../target/scripts/helpers/log.sh + source "${SOURCE_BASE_PATH}/log.sh" + # shellcheck source=../../../../../target/scripts/helpers/utils.sh + source "${SOURCE_BASE_PATH}/utils.sh" + + # Create a temporary file in the BATS test-case folder: + local TMP_DMS_CONFIG=$(mktemp -p "${BATS_TEST_TMPDIR}" -t 'dms_XXX.cf') + # A file missing a final newline: + echo -en 'line one\nline two' > "${TMP_DMS_CONFIG}" + + # Confirm missing newline: + run bash -c "tail -c 1 '${TMP_DMS_CONFIG}' | wc -l" + assert_output '0' + + # Helper method detects and fixes: + _append_final_newline_if_missing "${TMP_DMS_CONFIG}" + run bash -c "tail -c 1 '${TMP_DMS_CONFIG}' | wc -l" + assert_output '1' +} From 487867285bd1c7c6d3157afde45adce08d3c0409 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 26 Jan 2024 23:32:18 +1300 Subject: [PATCH 271/592] docs: UX Improvement - Better distinguish side nav page categories (#3835) --- docs/content/assets/css/customizations.css | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/content/assets/css/customizations.css b/docs/content/assets/css/customizations.css index 320da152db7..25cb0274992 100644 --- a/docs/content/assets/css/customizations.css +++ b/docs/content/assets/css/customizations.css @@ -102,3 +102,8 @@ div.md-content article.md-content__inner a.toclink code { .highlight.no-copy .md-clipboard { display: none; } /* ============================================================================================================= */ + +/* Make the left-sidebar nav categories better distinguished from page links (bold text) */ +.md-nav__item--nested > .md-nav__link { + font-weight: 700; +} From 7d9eb1e4a7ddd7b78b7bda2928b802af990af5ec Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 26 Jan 2024 23:32:49 +1300 Subject: [PATCH 272/592] docs: Add context to `sender-cleanup` in Postfix `master.cf` (#3834) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- target/postfix/master.cf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/target/postfix/master.cf b/target/postfix/master.cf index e5b955a48dc..7d2b21d6716 100644 --- a/target/postfix/master.cf +++ b/target/postfix/master.cf @@ -46,6 +46,8 @@ pickup fifo n - n 60 1 pickup -o content_filter= -o receive_override_options=no_header_body_checks +# This relates to submission(s) services defined above: +# https://www.postfix.org/BUILTIN_FILTER_README.html#mx_submission sender-cleanup unix n - n - 0 cleanup -o syslog_name=postfix/sender-cleanup -o header_checks=pcre:/etc/postfix/maps/sender_header_filter.pcre From ba27edc801c0b94cfb7d975dd2fb918a75ac9ea3 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 26 Jan 2024 14:07:46 +0100 Subject: [PATCH 273/592] Rspamd: only declare Rspamd variables when not already declared (#3837) * only declare Rspamd vars when not already declared * update CHANGELOG * Update CHANGELOG.md --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 1 + target/scripts/helpers/rspamd.sh | 21 +++++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f440e31b63..a9c03f790ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ The most noteworthy change of this release is the update of the container's base ### Fixes - DMS config files that are parsed line by line are now more robust to parse by detecting and fixing line-endings ([#3819](https://github.com/docker-mailserver/docker-mailserver/pull/3819)) +- Variables related to Rspamd are declared as `readonly`, which would cause warnings in the log when being re-declared; we now guard against this issue ([#3837](https://github.com/docker-mailserver/docker-mailserver/pull/3837)) ## [v13.3.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.3.1) diff --git a/target/scripts/helpers/rspamd.sh b/target/scripts/helpers/rspamd.sh index 2f4dcc46137..8d1fd6685f9 100644 --- a/target/scripts/helpers/rspamd.sh +++ b/target/scripts/helpers/rspamd.sh @@ -15,14 +15,19 @@ function __do_as_rspamd_user() { # they cannot be modified. Use this function when you require common directory # names, file names, etc. function _rspamd_get_envs() { - readonly RSPAMD_LOCAL_D='/etc/rspamd/local.d' - readonly RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' - - readonly RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' - readonly RSPAMD_DMS_DKIM_D="${RSPAMD_DMS_D}/dkim" - readonly RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d" - - readonly RSPAMD_DMS_CUSTOM_COMMANDS_F="${RSPAMD_DMS_D}/custom-commands.conf" + # If the variables are already set, we cannot set them again as they are declared + # with `readonly`. Checking whether one is declared suffices, because either all + # are declared at once, or none. + if [[ ! -v RSPAMD_LOCAL_D ]]; then + readonly RSPAMD_LOCAL_D='/etc/rspamd/local.d' + readonly RSPAMD_OVERRIDE_D='/etc/rspamd/override.d' + + readonly RSPAMD_DMS_D='/tmp/docker-mailserver/rspamd' + readonly RSPAMD_DMS_DKIM_D="${RSPAMD_DMS_D}/dkim" + readonly RSPAMD_DMS_OVERRIDE_D="${RSPAMD_DMS_D}/override.d" + + readonly RSPAMD_DMS_CUSTOM_COMMANDS_F="${RSPAMD_DMS_D}/custom-commands.conf" + fi } # Parses `RSPAMD_DMS_CUSTOM_COMMANDS_F` and executed the directives given by the file. From 9ac11021e1c33d87ec706f3f76b6c3197f1f000d Mon Sep 17 00:00:00 2001 From: Andreas Perhab Date: Fri, 26 Jan 2024 14:40:29 +0100 Subject: [PATCH 274/592] setup-stack: fix error when RSPAMD_DMS_DKIM_D is not set (#3827) * setup-stack: fix error when RSPAMD_DMS_DKIM_D is not set prevent messages like this chown: cannot access '': No such file or directory when RSPAMD_DMS_DKIM_D has no value * Update target/scripts/startup/setup-stack.sh --------- Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- target/scripts/startup/setup-stack.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/target/scripts/startup/setup-stack.sh b/target/scripts/startup/setup-stack.sh index c3c54cc37df..24ef1581104 100644 --- a/target/scripts/startup/setup-stack.sh +++ b/target/scripts/startup/setup-stack.sh @@ -95,8 +95,9 @@ function _setup_apply_fixes_after_configuration() { _log 'debug' 'Removing files and directories from older versions' rm -rf /var/mail-state/spool-postfix/{dev,etc,lib,pid,usr,private/auth} + _rspamd_get_envs # /tmp/docker-mailserver/rspamd/dkim - _log 'debug' "Ensuring ${RSPAMD_DMS_DKIM_D} is owned by '_rspamd:_rspamd'" + _log 'debug' "Ensuring '${RSPAMD_DMS_DKIM_D}' is owned by '_rspamd:_rspamd'" chown -R _rspamd:_rspamd "${RSPAMD_DMS_DKIM_D}" } From a8ccd54da57111f6e619c987da1ca9d807fa4fcc Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sun, 28 Jan 2024 01:50:01 +1300 Subject: [PATCH 275/592] ci: `docs-preview-deploy.yml` - Switch to official `download-artifact` action (#3838) v4 of the official action now supports this use-case. --- .github/workflows/docs-preview-deploy.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index 93819efaeb4..6f6904d54a3 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -23,14 +23,13 @@ jobs: # Restore workflow context # # ======================== # - # The official Github Action for downloading artifacts does not support multi-workflow + # Retrieve the artifact uploaded from `docs-preview-prepare.yml` workflow run that triggered this deployment - name: 'Download build artifact' - uses: dawidd6/action-download-artifact@v3 + uses: actions/download-artifact@v4 with: + name: preview-build github_token: ${{ secrets.GITHUB_TOKEN }} run_id: ${{ github.event.workflow_run.id }} - workflow: docs-preview-prepare.yml - name: preview-build - name: 'Extract build artifact' run: tar -xf artifact.tar.zst From f27629be4e24a1fc288e947662f8294743063d5f Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sun, 28 Jan 2024 01:51:37 +1300 Subject: [PATCH 276/592] docs: Minor revisions to `README.md` (#3839) --- README.md | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e81d21add5d..cfd453ff558 100644 --- a/README.md +++ b/README.md @@ -4,22 +4,27 @@ [ci::status]: https://img.shields.io/github/actions/workflow/status/docker-mailserver/docker-mailserver/default_on_push.yml?branch=master&color=blue&label=CI&logo=github&logoColor=white&style=for-the-badge [ci::github]: https://github.com/docker-mailserver/docker-mailserver/actions -[docker::pulls]: https://img.shields.io/docker/pulls/mailserver/docker-mailserver.svg?style=for-the-badge&logo=docker&logoColor=white +[docker::pulls]: https://img.shields.io/docker/pulls/mailserver/docker-mailserver.svg?style=for-the-badge&logo=docker&logoColor=white&color=blue [docker::hub]: https://hub.docker.com/r/mailserver/docker-mailserver/ -[documentation::badge]: https://img.shields.io/badge/DOCUMENTATION-GH%20PAGES-0078D4?style=for-the-badge&logo=git&logoColor=white +[documentation::badge]: https://img.shields.io/badge/DOCUMENTATION-GH%20PAGES-0078D4?style=for-the-badge&logo=googledocs&logoColor=white [documentation::web]: https://docker-mailserver.github.io/docker-mailserver/latest/ ## :page_with_curl: About -A production-ready fullstack but simple containerized mail server (SMTP, IMAP, LDAP, Anti-spam, Anti-virus, etc.). Only configuration files, no SQL database. Keep it simple and versioned. Easy to deploy and upgrade. Originally created by @tomav, this project is now maintained by volunteers since January 2021. +A production-ready fullstack but simple containerized mail server (SMTP, IMAP, LDAP, Anti-spam, Anti-virus, etc.). +- Only configuration files, no SQL database. Keep it simple and versioned. Easy to deploy and upgrade. +- Originally created by [@tomav](https://github.com/tomav), this project is now maintained by volunteers since January 2021. -## :bulb: Documentation +## -We provide a [dedicated documentation][documentation::web] hosted on GitHub Pages. Make sure to read it as it contains all the information necessary to set up and configure your mail server. The documentation is crafted with Markdown & [MkDocs Material](https://squidfunk.github.io/mkdocs-material/). +> [!TIP] +> Be sure to read [our documentation][documentation::web]. It provides guidance on initial setup of your mail server. -## :boom: Issues - -If you have issues, please search through [the documentation][documentation::web] **for your version** before opening an issue. The issue tracker is for issues, not for personal support. Make sure the version of the documentation matches the image version you're using! +> [!IMPORTANT] +> If you have issues, please search through [the documentation][documentation::web] **for your version** before opening an issue. +> +> The issue tracker is for issues, not for personal support. +> Make sure the version of the documentation matches the image version you're using! ## :link: Links to Useful Resources From 4a05d7bb7caeb67347034f5280f41f2a42170410 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sun, 28 Jan 2024 10:23:49 +1300 Subject: [PATCH 277/592] docs: Add Debian 12 breaking change for `opendmarc` package (#3841) --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9c03f790ba..e785cac0d21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,11 @@ The most noteworthy change of this release is the update of the container's base - The script now uses `/etc/os-release` to determine the release name of Debian - Removed custom installations of Fail2Ban, getmail6 and Rspamd - Updated packages lists and added comments for maintainability +- OpenDMARC upgrade: `v1.4.0` => `v1.4.2` ([#3841](https://github.com/docker-mailserver/docker-mailserver/pull/3841)) + - Previous versions of OpenDMARC would place incoming mail from domains announcing `p=quarantaine` (_that fail the DMARC check_) into the [Postfix "hold" queue](https://www.postfix.org/QSHAPE_README.html#hold_queue) until administrative intervention. + - [OpenDMARC v1.4.2 has disabled that feature by default](https://github.com/trusteddomainproject/OpenDMARC/issues/105), but it can be enabled again by adding the setting `HoldQuarantinedMessages true` to [`/etc/opendmarc.conf`](https://github.com/docker-mailserver/docker-mailserver/blob/v13.3.1/target/opendmarc/opendmarc.conf) (_provided from DMS_). + - [Our `user-patches.sh` feature](https://docker-mailserver.github.io/docker-mailserver/latest/config/advanced/override-defaults/user-patches/) provides a convenient approach to updating that config file. + - Please let us know if you disagree with the upstream default being carried with DMS, or the value of providing alternative configuration support within DMS. - **Postfix:** - Postfix upgrade from 3.5 to 3.7 ([#3403](https://github.com/docker-mailserver/docker-mailserver/pull/3403)) - `compatibility_level` was raised from `2` to `3.6` From 204825fa5a2476a0b185df85365b5cad6b73b570 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sun, 28 Jan 2024 20:41:19 +1300 Subject: [PATCH 278/592] ci(fix): `docs-preview-deploy.yml` - Use the correct setting names (#3843) --- .github/workflows/docs-preview-deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index 6f6904d54a3..3b6f3c677f2 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -28,8 +28,8 @@ jobs: uses: actions/download-artifact@v4 with: name: preview-build - github_token: ${{ secrets.GITHUB_TOKEN }} - run_id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} - name: 'Extract build artifact' run: tar -xf artifact.tar.zst From 11c508cd11c9c850527c54b49631374052c67303 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 28 Jan 2024 11:15:47 +0000 Subject: [PATCH 279/592] docs: update `CONTRIBUTORS.md` (#3844) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Casper --- CONTRIBUTORS.md | 179 +++++++++++++++++++++++++----------------------- 1 file changed, 93 insertions(+), 86 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index f86435f4616..72784ab01ef 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -135,17 +135,17 @@ Thanks goes to these wonderful people ✨ + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + - + + + - + + + - + + + - + - + + + - + + + - + + + - + - + + + + - + + + - + + + - + - + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + + + - + -
- - alinmear + + ap-wtioit
- alinmear + ap-wtioit 		- - ap-wtioit + + alinmear
- ap-wtioit + alinmear 		@@ -815,13 +815,20 @@ Thanks goes to these wonderful people ✨ - - nilshoell + + auchri
- nilshoell + auchri
+ + stephan-devop +
+ stephan-devop + + 		stigok @@ -856,15 +863,15 @@ Thanks goes to these wonderful people ✨
TechnicLab -
thomasschmit
thomasschmit -
Thiritin @@ -899,15 +906,15 @@ Thanks goes to these wonderful people ✨
Twist235 -
k3it
k3it -
Drakulix @@ -942,15 +949,15 @@ Thanks goes to these wonderful people ✨
Zepmann -
- - allddd + + nilshoell
- allddd + nilshoell -
nknapp @@ -985,15 +992,15 @@ Thanks goes to these wonderful people ✨
mrPjer -
p3dda
p3dda -
peter-hartmann @@ -1028,15 +1035,15 @@ Thanks goes to these wonderful people ✨
norrs -
MightySCollins
MightySCollins -
501st-alpha1 @@ -1071,15 +1078,15 @@ Thanks goes to these wonderful people ✨
shyim -
sjmudd
sjmudd -
simonsystem @@ -1088,10 +1095,10 @@ Thanks goes to these wonderful people ✨ - - stephan-devop + + allddd
- stephan-devop + allddd 		@@ -1114,15 +1121,15 @@ Thanks goes to these wonderful people ✨
odinis -
okamidash
okamidash -
olaf-mandel @@ -1157,15 +1164,15 @@ Thanks goes to these wonderful people ✨
rhyst -
rmlhuk
rmlhuk -
rriski @@ -1200,15 +1207,15 @@ Thanks goes to these wonderful people ✨
squash -
strarsis
strarsis -
tamueller @@ -1243,15 +1250,15 @@ Thanks goes to these wonderful people ✨
worldworm -
arcaine2
arcaine2 -
awb99 @@ -1286,15 +1293,15 @@ Thanks goes to these wonderful people ✨
dimalo -
eleith
eleith -
ghnp5 @@ -1329,15 +1336,15 @@ Thanks goes to these wonderful people ✨
idaadi -
ixeft
ixeft -
jjtt @@ -1372,15 +1379,15 @@ Thanks goes to these wonderful people ✨
callmemagnus -
marios88
marios88 -
matrixes @@ -1395,13 +1402,6 @@ Thanks goes to these wonderful people ✨ mchamplain - - auchri -
- auchri - - 		arkanovicz @@ -1539,10 +1539,10 @@ Thanks goes to these wonderful people ✨ - - thechubbypanda + + froks
- thechubbypanda + froks 		@@ -1696,6 +1696,13 @@ Thanks goes to these wonderful people ✨ crash7 + + thechubbypanda +
+ thechubbypanda + + 		KCrawley @@ -1716,15 +1723,15 @@ Thanks goes to these wonderful people ✨
JustAnother1 -
LeoWinterDE
LeoWinterDE -
linhandev @@ -1759,15 +1766,15 @@ Thanks goes to these wonderful people ✨
madmath03 -
maxemann96
maxemann96 -
dragetd @@ -1802,15 +1809,15 @@ Thanks goes to these wonderful people ✨
mcchots -
MohammedNoureldin
MohammedNoureldin -
mpldr @@ -1839,13 +1846,6 @@ Thanks goes to these wonderful people ✨ radicand - - froks -
- froks - - 		fkefer @@ -1911,6 +1911,13 @@ Thanks goes to these wonderful people ✨ Influencer + + JamBalaya56562 +
+ JamBalaya56562 + + 		jcalfee @@ -1931,15 +1938,15 @@ Thanks goes to these wonderful people ✨
init-js -
Jeidnx
Jeidnx -
JiLleON @@ -1974,15 +1981,15 @@ Thanks goes to these wonderful people ✨
JOduMonT -
Kaan88
Kaan88 -
akkumar From 3b11a8305ee2e303b7e14faeb942e140d0434b3b Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Mon, 29 Jan 2024 10:35:19 +1300 Subject: [PATCH 280/592] docs: Remove ENV `ONE_DIR` (#3840) * docs: Better document DMS volumes * docs: Remove any mention of `ONE_DIR` ENV * chore: Remove `ONE_DIR` ENV from scripts Only `ONE_DIR=0` has any effect. As the actual feature is now dependent upon the `/var/mail-state` location existing. It is advised not mounting anything there instead if wanting to avoid runtime state consolidation. * docs: Adjust link ref convention This is more search friendly / organized to find references to all DMS volumes. * lint: Ensure final newline is present VSCode by default excludes this if the last line rendered is removed (rendered as a separate blank line). A separate setting can enforce adding the final newline upon save regardless. --- CHANGELOG.md | 3 + docs/content/config/advanced/kubernetes.md | 1 - .../config/advanced/optional-config.md | 62 +++++++++++++++++-- .../config/best-practices/dkim_dmarc_spf.md | 2 +- docs/content/config/environment.md | 13 ++-- docs/content/config/security/fail2ban.md | 4 +- docs/content/config/security/rspamd.md | 18 ++++-- docs/content/config/security/ssl.md | 4 +- .../tutorials/mailserver-behind-proxy.md | 2 - docs/content/faq.md | 39 ++++-------- docs/content/usage.md | 4 +- mailserver.env | 4 -- target/scripts/startup/setup.d/mail_state.sh | 6 +- .../startup/setup.d/security/rspamd.sh | 6 +- target/scripts/startup/variables-stack.sh | 1 - 15 files changed, 102 insertions(+), 67 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e785cac0d21..41e6d5e9d0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,9 @@ The most noteworthy change of this release is the update of the container's base ### Updates +- **Environment Variables:** + - `ONE_DIR` has been removed (legacy ENV) ([#3840](https://github.com/docker-mailserver/docker-mailserver/pull/3840)) + - It's only functionality remaining was to opt-out of run-time state consolidation with `ONE_DIR=0` (_when a volume was already mounted to `/var/mail-state`_). - **Tests:** - Refactored helper methods for sending e-mails with specific `Message-ID` headers and the helpers for retrieving + filtering logs, which together help isolate logs relevant to specific mail when multiple mails have been processed within a single test. ([#3786](https://github.com/docker-mailserver/docker-mailserver/pull/3786)) diff --git a/docs/content/config/advanced/kubernetes.md b/docs/content/config/advanced/kubernetes.md index 8a47bffcace..c277852a8a2 100644 --- a/docs/content/config/advanced/kubernetes.md +++ b/docs/content/config/advanced/kubernetes.md @@ -40,7 +40,6 @@ data: POSTMASTER_ADDRESS: postmaster@example.com UPDATE_CHECK_INTERVAL: 10d POSTFIX_INET_PROTOCOLS: ipv4 - ONE_DIR: '1' ENABLE_CLAMAV: '1' ENABLE_POSTGREY: '0' ENABLE_FAIL2BAN: '1' diff --git a/docs/content/config/advanced/optional-config.md b/docs/content/config/advanced/optional-config.md index 31090050f9a..fb1ab29dc27 100644 --- a/docs/content/config/advanced/optional-config.md +++ b/docs/content/config/advanced/optional-config.md @@ -4,9 +4,64 @@ hide: - toc # Hide Table of Contents for this page --- -This is a list of all configuration files and directories which are optional or automatically generated in your [`docker-data/dms/config/`][docs-dms-config-volume] directory. +## Volumes -## Directories +DMS has several locations in the container which may be worth persisting externally via [Docker Volumes][docker-docs::volumes]. + +- Often you will want to prefer [bind mount volumes][docker-docs::volumes::bind-mount] for easy access to files at a local location on your filesystem. +- As a convention for our docs and example configs, the local location has the common prefix `docker-data/dms/` for grouping these related volumes. + +!!! info "Reference - Volmes for DMS" + + Our docs may refer to these DMS specific volumes only by name, or the host/container path for brevity. + + - [Config](#volumes-config): `docker-data/dms/config/` => `/tmp/docker-mailserver/` + - [Mail Storage](#volumes-mail): `docker-data/dms/mail-data/` => `/var/mail/` + - [State](#volumes-state): `docker-data/dms/mail-state/` => `/var/mail-state/` + - [Logs](#volumes-logs): `docker-data/dms/mail-logs/` => `/var/log/mail/` + +[docker-docs::volumes]: https://docs.docker.com/storage/volumes/ +[docker-docs::volumes::bind-mount]: https://docs.docker.com/storage/bind-mounts/ + +### Mail Storage Volume { #volumes-mail } + +This is the location where mail is delivered to your mailboxes. + +### State Volume { #volumes-state } + +Run-time specific state lives here, but so does some data you may want to keep if a failure event occurs (_crash, power loss_). + +!!! example "Examples of relevant data" + + - The Postfix queue (eg: mail pending delivery attempt) + - Fail2Ban blocks. + - ClamAV signature updates. + - Redis storage for Rspamd. + +!!! info "When a volume is mounted to `/var/mail-state/`" + + - Service run-time data is [consolidated into the `/var/mail-state/` directory][mail-state-folders]. Otherwise the original locations vary and would need to be mounted individually. + - The original locations are updated with symlinks to redirect to their new path in `/var/mail-state/` (_eg: `/var/lib/redis` => `/var/mail-state/lib-redis/`_). + + Supported services: Postfix, Dovecot, Fail2Ban, Amavis, PostGrey, ClamAV, SpamAssassin, Rspamd & Redis, Fetchmail, Getmail, LogRotate, PostSRSd, MTA-STS. + +!!! tip + + Sometimes it is helpful to disable this volume when troubleshooting to verify if the data stored here is in a bad state (_eg: caused by a failure event_). + +[mail-state-folders]: https://github.com/docker-mailserver/docker-mailserver/blob/v13.3.1/target/scripts/startup/setup.d/mail_state.sh#L13-L33 + +### Logs Volume { #volumes-log } + +This can be a useful volume to persist for troubleshooting needs for the full set of log files. + +### Config Volume { #volumes-config } + +Most configuration files for Postfix, Dovecot, etc. are persisted here. + +This is a list of all configuration files and directories which are optional, automatically generated / updated by our `setup` CLI, or other internal scripts. + +#### Directories - **sieve-filter:** directory for sieve filter scripts. (Docs: [Sieve][docs-sieve]) - **sieve-pipe:** directory for sieve pipe scripts. (Docs: [Sieve][docs-sieve]) @@ -14,7 +69,7 @@ This is a list of all configuration files and directories which are optional or - **ssl:** SSL Certificate directory if `SSL_TYPE` is set to `self-signed` or `custom`. (Docs: [SSL][docs-ssl]) - **rspamd:** Override directory for custom settings when using Rspamd (Docs: [Rspamd][docs-rspamd-override-d]) -## Files +#### Files - **{user_email_address}.dovecot.sieve:** User specific Sieve filter file. (Docs: [Sieve][docs-sieve]) - **before.dovecot.sieve:** Global Sieve filter file, applied prior to the `${login}.dovecot.sieve` filter. (Docs: [Sieve][docs-sieve]) @@ -42,7 +97,6 @@ This is a list of all configuration files and directories which are optional or - **user-patches.sh:** this file will be run after all configuration files are set up, but before the postfix, amavis and other daemons are started. (Docs: [FAQ - How to adjust settings with the `user-patches.sh` script][docs-faq-userpatches]) - **rspamd/custom-commands.conf:** list of simple commands to adjust Rspamd modules in an easy way (Docs: [Rspamd][docs-rspamd-commands]) -[docs-dms-config-volume]: ../../faq.md#what-about-the-docker-datadmsconfig-directory [docs-accounts-quota]: ../../config/user-management.md#quotas [docs-aliases-regex]: ../../config/user-management.md#configuring-regexp-aliases [docs-dkim]: ../../config/best-practices/dkim_dmarc_spf.md#dkim diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md index ed56504c995..290ac5cd400 100644 --- a/docs/content/config/best-practices/dkim_dmarc_spf.md +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -345,7 +345,7 @@ volumes: ``` [docs-accounts-add]: ../user-management.md#adding-a-new-account -[docs-volumes-config]: ../advanced/optional-config.md +[docs-volumes-config]: ../advanced/optional-config.md#volumes-config [docs-env-opendkim]: ../environment.md#enable_opendkim [docs-env-rspamd]: ../environment.md#enable_rspamd [docs-rspamd-config-dropin]: ../security/rspamd.md#manually diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 1aa6799d46d..b1c3e2b6044 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -45,11 +45,6 @@ Default: 5000 The Group ID assigned to the static vmail group for `/var/mail` (_Mail storage managed by Dovecot_). -##### ONE_DIR - -- 0 => state in default directories. -- **1** => consolidate all states into a single directory (`/var/mail-state`) to allow persistence using docker volumes. See the [related FAQ entry][docs-faq-onedir] for more information. - ##### ACCOUNT_PROVISIONER Configures the provisioning source of user accounts (including aliases) for user queries and authentication by services managed by DMS (_Postfix and Dovecot_). @@ -648,10 +643,10 @@ Controls the spam score threshold for triggering an action on mail that has a hi - [It will be quarantined][amavis-docs::quarantine] regardless of the `SA_KILL` action to perform. - With `D_PASS` the delivered mail also appends an `X-Quarantine-ID` mail header. The ID value of this header is part of the quarantined file name. - If emails are quarantined, they are compressed and stored at a location dependent on the [`ONE_DIR`](#one_dir) setting: + If emails are quarantined, they are compressed and stored at a location: - - `ONE_DIR=1` (default): `/var/mail-state/lib-amavis/virusmails/` - - `ONE_DIR=0`: `/var/lib/amavis/virusmails/` + - Default: `/var/lib/amavis/virusmails/` + - When the [`/var/mail-state/` volume][docs::dms-volumes-state] is present: `/var/mail-state/lib-amavis/virusmails/` !!! tip @@ -1056,9 +1051,9 @@ you to replace both instead of just the envelope sender. - password for default relay user [docs-rspamd]: ./security/rspamd.md -[docs-faq-onedir]: ../faq.md#what-about-docker-datadmsmail-state-folder-varmail-state-internally [docs-tls]: ./security/ssl.md [docs-tls-letsencrypt]: ./security/ssl.md#lets-encrypt-recommended [docs-tls-manual]: ./security/ssl.md#bring-your-own-certificates [docs-tls-selfsigned]: ./security/ssl.md#self-signed-certificates [docs-accounts-quota]: ./user-management.md#quotas +[docs::dms-volumes-state]: ./advanced/optional-config.md#volumes-state diff --git a/docs/content/config/security/fail2ban.md b/docs/content/config/security/fail2ban.md index 04e9a6814a4..375f440cb11 100644 --- a/docs/content/config/security/fail2ban.md +++ b/docs/content/config/security/fail2ban.md @@ -33,7 +33,7 @@ DMS will automatically ban IP addresses of hosts that have generated 6 failed at ### Custom Files -!!! question "What is [`docker-data/dms/config/`][docs-dms-config-volume]?" +!!! question "What is [`docker-data/dms/config/`][docs::dms-volumes-config]?" This following configuration files inside the `docker-data/dms/config/` volume will be copied inside the container during startup @@ -44,7 +44,7 @@ This following configuration files inside the `docker-data/dms/config/` volume w - with this file, you can adjust F2B behavior in general - there is an example provided [in our repository on GitHub][github-file-f2bconfig] -[docs-dms-config-volume]: ../../faq.md#what-about-the-docker-datadmsconfig-directory +[docs::dms-volumes-config]: ../advanced/optional-config.md#volumes-config [github-file-f2bjail]: https://github.com/docker-mailserver/docker-mailserver/blob/master/config-examples/fail2ban-jail.cf [github-file-f2bconfig]: https://github.com/docker-mailserver/docker-mailserver/blob/master/config-examples/fail2ban-fail2ban.cf diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 283c2a9597b..0c457ec3512 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -83,9 +83,15 @@ DMS does not set a default password for the controller worker. You may want to d ### Persistence with Redis -When Rspamd is enabled, we implicitly also start an instance of Redis in the container. Redis is configured to persist its data via RDB snapshots to disk in the directory `/var/lib/redis` (_which is a symbolic link to `/var/mail-state/lib-redis/` when [`ONE_DIR=1`](../environment.md#one_dir) and a volume is mounted to `/var/mail-state/`_). With the volume mount the snapshot will restore the Redis data across container restarts, and provide a way to keep backup. +When Rspamd is enabled, we implicitly also start an instance of Redis in the container: -Redis uses `/etc/redis/redis.conf` for configuration. We adjust this file when enabling the internal Redis service. If you have an external instance of Redis to use, the internal Redis service can be opt-out via setting the ENV [`ENABLE_RSPAMD_REDIS=0`](../environment.md#enable_rspamd_redis) (_link also details required changes to the DMS Rspamd config_). +- Redis is configured to persist its data via RDB snapshots to disk in the directory `/var/lib/redis` (_or the [`/var/mail-state/`][docs::dms-volumes-state] volume when present_). +- With the volume mount the snapshot will restore the Redis data across container restarts, and provide a way to keep backup. + +Redis uses `/etc/redis/redis.conf` for configuration: + +- We adjust this file when enabling the internal Redis service. +- If you have an external instance of Redis to use, the internal Redis service can be opt-out via setting the ENV [`ENABLE_RSPAMD_REDIS=0`](../environment.md#enable_rspamd_redis) (_link also details required changes to the DMS Rspamd config_). ### Web Interface @@ -145,7 +151,7 @@ DMS brings sane default settings for Rspamd. They are located at `/etc/rspamd/lo ### Manually -!!! question "What is [`docker-data/dms/config/`][docs-dms-config-volume]?" +!!! question "What is [`docker-data/dms/config/`][docs::dms-volumes-config]?" If you want to overwrite the default settings and / or provide your own settings, you can place files at `docker-data/dms/config/rspamd/override.d/`. Files from this directory are copied to `/etc/rspamd/override.d/` during startup. These files [forcibly override][rspamd-docs-override-dir] Rspamd and DMS default settings. @@ -156,7 +162,6 @@ If you want to overwrite the default settings and / or provide your own settings Note that when also [using the `custom-commands.conf` file](#with-the-help-of-a-custom-file), files in `override.d` may be overwritten in case you adjust them manually and with the help of the file. [rspamd-docs-override-dir]: https://www.rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories -[docs-dms-config-volume]: ../../faq.md#what-about-the-docker-datadmsconfig-directory [rspamd-docs-config-directories]: https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories ### With the Help of a Custom File @@ -197,7 +202,7 @@ You can also have comments (the line starts with `#`) and blank lines in `custom You want to start using Rspamd? Rspamd is disabled by default, so you need to set the following environment variables: -```cf +```env ENABLE_RSPAMD=1 ENABLE_OPENDKIM=0 ENABLE_OPENDMARC=0 @@ -252,3 +257,6 @@ While _Abusix_ can be integrated into Postfix, Postscreen and a multitude of oth [Abusix]: https://abusix.com/ [abusix-rspamd-integration]: https://docs.abusix.com/abusix-mail-intelligence/gbG8EcJ3x3fSUv8cMZLiwA/getting-started/dmw9dcwSGSNQiLTssFAnBW#rspamd + +[docs::dms-volumes-config]: ../advanced/optional-config.md#volumes-config +[docs::dms-volumes-state]: ../advanced/optional-config.md#volumes-state diff --git a/docs/content/config/security/ssl.md b/docs/content/config/security/ssl.md index c0c615cc14a..10f7adc4851 100644 --- a/docs/content/config/security/ssl.md +++ b/docs/content/config/security/ssl.md @@ -634,7 +634,7 @@ This setup only comes with one caveat: The domain has to be configured on anothe Use self-signed certificates only for testing purposes! -This feature requires you to provide the following files into your [`docker-data/dms/config/ssl/` directory][docs-optional-config] (_internal location: `/tmp/docker-mailserver/ssl/`_): +This feature requires you to provide the following files into your [`docker-data/dms/config/ssl/` directory][docs::dms-volumes-config] (_internal location: `/tmp/docker-mailserver/ssl/`_): - `-key.pem` - `-cert.pem` @@ -876,7 +876,7 @@ By default DMS uses [`ffdhe4096`][ffdhe4096-src] from [IETF RFC 7919][ietf::rfc: Despite this, if you must use non-standard DH parameters or you would like to swap `ffdhe4096` for a different group (eg `ffdhe2048`); Add your own PEM encoded DH params file via a volume to `/tmp/docker-mailserver/dhparams.pem`. This will replace DH params for both Dovecot and Postfix services during container startup. [docs-env::ssl-type]: ../environment.md#ssl_type -[docs-optional-config]: ../advanced/optional-config.md +[docs::dms-volumes-config]: ../advanced/optional-config.md#volumes-config [docs-faq-baredomain]: ../../faq.md#can-i-use-a-nakedbare-domain-ie-no-hostname [github-file-compose]: https://github.com/docker-mailserver/docker-mailserver/blob/master/compose.yaml diff --git a/docs/content/examples/tutorials/mailserver-behind-proxy.md b/docs/content/examples/tutorials/mailserver-behind-proxy.md index 7bdef5d604e..999395426bb 100644 --- a/docs/content/examples/tutorials/mailserver-behind-proxy.md +++ b/docs/content/examples/tutorials/mailserver-behind-proxy.md @@ -125,5 +125,3 @@ service imap-login { !!! note Port `10993` is used here to avoid conflicts with internal systems like `postscreen` and `amavis` as they will exchange messages on the default port and obviously have a different origin then compared to the proxy. - -[docs-optionalconfig]: ../../config/advanced/optional-config.md diff --git a/docs/content/faq.md b/docs/content/faq.md index 0985c0a94f6..6b1782e3190 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -360,20 +360,6 @@ DMS does not manage those concerns, verify they are not causing your delivery pr - [mail-tester](https://www.mail-tester.com/) can test your deliverability. - [helloinbox](https://www.helloinbox.email/) provides a checklist of things to improve your deliverability. -### Special Directories - -#### What About the `docker-data/dms/config/` Directory? - -This documentation and all example configuration files in the GitHub repository use `docker-data/dms/config/` to refer to the directory in the host that is mounted (e.g. via a bind mount) to `/tmp/docker-mailserver/` inside the container. - -Most configuration files for Postfix, Dovecot, etc. are persisted here. [Optional configuration][docs-optional-configuration] is stored here as well. - -#### What About the `docker-data/dms/mail-state/` Directory? - -This documentation and all example configuration files in the GitHub repository use `docker-data/dms/mail-state/` to refer to the directory in the host that is mounted (e.g. via a bind mount) to `/var/mail-state/` inside the container. - -When you run DMS with the ENV variable `ONE_DIR=1` (default), this directory will provide support to persist Fail2Ban blocks, ClamAV signature updates, and the like when the container is restarted or recreated. Service data is [relocated to the `mail-state` folder][mail-state-folders] for the following services: Postfix, Dovecot, Fail2Ban, Amavis, PostGrey, ClamAV, SpamAssassin, Rspamd & Redis. - ### SpamAssasin #### How can I manage my custom SpamAssassin rules? @@ -390,14 +376,15 @@ The default setup `@local_domains_acl = ( ".$mydomain" );` does not match subdom Put received spams in `.Junk/` imap folder using `SPAMASSASSIN_SPAM_TO_INBOX=1` and `MOVE_SPAM_TO_JUNK=1` and add a _user_ cron like the following: -```conf -# This assumes you're having `environment: ONE_DIR=1` in the `mailserver.env`, -# with a consolidated config in `/var/mail-state` -# -# m h dom mon dow command -# Everyday 2:00AM, learn spam from a specific user -0 2 * * * docker exec mailserver sa-learn --spam /var/mail/example.com/username/.Junk --dbpath /var/mail-state/lib-amavis/.spamassassin -``` +!!! example + + **NOTE:** This example assumes you have a [`/var/mail-state` volume][docs::dms-volumes-state] mounted. + + ```conf + # m h dom mon dow command + # Everyday 2:00AM, learn spam from a specific user + 0 2 * * * docker exec mailserver sa-learn --spam /var/mail/example.com/username/.Junk --dbpath /var/mail-state/lib-amavis/.spamassassin + ``` With `docker-compose` you can more easily use the internal instance of `cron` within DMS. This is less problematic than the simple solution shown above, because it decouples the learning from the host on which DMS is running, and avoids errors if the mail server is not running. @@ -405,6 +392,8 @@ The following configuration works nicely: ??? example + **NOTE:** This example assumes you have a [`/var/mail-state` volume][docs::dms-volumes-state] mounted. + Create a _system_ cron file: ```sh @@ -418,9 +407,6 @@ The following configuration works nicely: Edit the system cron file `nano ./docker-data/dms/cron/sa-learn`, and set an appropriate configuration: ```conf - # This assumes you're having `environment: ONE_DIR=1` in the env-mailserver, - # with a consolidated config in `/var/mail-state` - # # '> /dev/null' to send error notifications from 'stderr' to 'postmaster@example.com' # # m h dom mon dow user command @@ -495,10 +481,10 @@ $spam_quarantine_to = "quarantine\@example.com"; ``` [fail2ban-customize]: ./config/security/fail2ban.md +[docs::dms-volumes-state]: ./config/advanced/optional-config.md#volumes-state [docs-maintenance]: ./config/advanced/maintenance/update-and-cleanup.md [docs-override-postfix]: ./config/advanced/override-defaults/postfix.md [docs-userpatches]: ./config/advanced/override-defaults/user-patches.md -[docs-optional-configuration]: ./config/advanced/optional-config.md [docs::env::sa_env]: ./config/environment.md#spamassassin [docs::env::sa_kill]: ./config/environment.md#sa_kill [github-comment-baredomain]: https://github.com/docker-mailserver/docker-mailserver/issues/3048#issuecomment-1432358353 @@ -510,4 +496,3 @@ $spam_quarantine_to = "quarantine\@example.com"; [github-issue-1639]: https://github.com/docker-mailserver/docker-mailserver/issues/1639 [github-issue-1792]: https://github.com/docker-mailserver/docker-mailserver/pull/1792 [hanscees-userpatches]: https://github.com/hanscees/dockerscripts/blob/master/scripts/tomav-user-patches.sh -[mail-state-folders]: https://github.com/docker-mailserver/docker-mailserver/blob/c7e498194546416fb7231cb03254e77e085d18df/target/scripts/startup/misc-stack.sh#L24-L33 diff --git a/docs/content/usage.md b/docs/content/usage.md index 0550c24e936..30f6e822688 100644 --- a/docs/content/usage.md +++ b/docs/content/usage.md @@ -2,9 +2,9 @@ title: Usage --- -This pages explains how to get started with DMS. The guide uses Docker Compose as a reference. In our examples, a volume mounts the host location [`docker-data/dms/config/`][docs-dms-config-volume] to `/tmp/docker-mailserver/` inside the container. +This pages explains how to get started with DMS. The guide uses Docker Compose as a reference. In our examples, a volume mounts the host location [`docker-data/dms/config/`][docs::dms-volumes-config] to `/tmp/docker-mailserver/` inside the container. -[docs-dms-config-volume]: ./faq.md#what-about-the-docker-datadmsconfig-directory +[docs::dms-volumes-config]: ./config/advanced/optional-config.md#volumes-config ## Preliminary Steps diff --git a/mailserver.env b/mailserver.env index 1d131696e61..c171faed125 100644 --- a/mailserver.env +++ b/mailserver.env @@ -30,10 +30,6 @@ LOG_LEVEL=info # debug => Also show debug messages SUPERVISOR_LOGLEVEL= -# 0 => mail state in default directories -# 1 => consolidate all states into a single directory (`/var/mail-state`) to allow persistence using docker volumes -ONE_DIR=1 - # Support for deployment where these defaults are not compatible (eg: some NAS appliances): # /var/mail vmail User ID (default: 5000) DMS_VMAIL_UID= diff --git a/target/scripts/startup/setup.d/mail_state.sh b/target/scripts/startup/setup.d/mail_state.sh index 9963bbcc82e..9c43fea4339 100644 --- a/target/scripts/startup/setup.d/mail_state.sh +++ b/target/scripts/startup/setup.d/mail_state.sh @@ -7,7 +7,7 @@ function _setup_save_states() { STATEDIR='/var/mail-state' - if [[ ${ONE_DIR} -eq 1 ]] && [[ -d ${STATEDIR} ]]; then + if [[ -d ${STATEDIR} ]]; then _log 'debug' "Consolidating all state onto ${STATEDIR}" # Always enabled features: @@ -111,9 +111,7 @@ function _setup_save_states() { # Ref: https://github.com/docker-mailserver/docker-mailserver/pull/3625 chmod 730 "${STATEDIR}/spool-postfix/maildrop" chmod 710 "${STATEDIR}/spool-postfix/public" - elif [[ ${ONE_DIR} -eq 1 ]]; then - _log 'warn' "'ONE_DIR=1' but no volume was mounted to '${STATEDIR}'" else - _log 'debug' 'Not consolidating state (because it has been disabled)' + _log 'debug' "'${STATEDIR}' is not present; Not consolidating state" fi } diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 86786932393..a05a798ef95 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -127,9 +127,9 @@ expand_keys = true; EOF - # Here we adjust the Redis default configuration that we supply to Redis - # when starting it. Note that `/var/lib/redis/` is linked to - # `/var/mail-state/redis/` (for persisting it) if `ONE_DIR=1`. + # Here we adjust the Redis default configuration that we supply to Redis when starting it. + # NOTE: `/var/lib/redis/` is symlinked to `/var/mail-state/redis/` when DMS is started + # with a volume mounted to `/var/mail-state/` for data persistence. sedfile -i -E \ -e 's|^(bind).*|\1 127.0.0.1|g' \ -e 's|^(daemonize).*|\1 no|g' \ diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index 0b351a9ec82..d14aeb3fb49 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -141,7 +141,6 @@ function __environment_variables_general_setup() { VARS[LOGWATCH_INTERVAL]="${LOGWATCH_INTERVAL:=none}" VARS[LOGWATCH_RECIPIENT]="${LOGWATCH_RECIPIENT:=${REPORT_RECIPIENT}}" VARS[LOGWATCH_SENDER]="${LOGWATCH_SENDER:=${REPORT_SENDER}}" - VARS[ONE_DIR]="${ONE_DIR:=1}" VARS[PERMIT_DOCKER]="${PERMIT_DOCKER:=none}" VARS[PFLOGSUMM_RECIPIENT]="${PFLOGSUMM_RECIPIENT:=${REPORT_RECIPIENT}}" VARS[PFLOGSUMM_SENDER]="${PFLOGSUMM_SENDER:=${REPORT_SENDER}}" From afb00939391e409a7bd9759d7346256198262144 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 29 Jan 2024 13:38:01 +0100 Subject: [PATCH 281/592] spam: use Sieve for rewriting subject with Rspamd & SA/Amavis (#3820) --- CHANGELOG.md | 10 ++ docs/content/config/environment.md | 35 +++---- docs/content/config/security/rspamd.md | 5 +- mailserver.env | 13 ++- target/rspamd/local.d/actions.conf | 8 +- target/scripts/start-mailserver.sh | 2 + target/scripts/startup/check-stack.sh | 8 ++ .../scripts/startup/setup.d/security/misc.sh | 61 ++++++++++-- target/scripts/startup/variables-stack.sh | 8 +- test/helper/common.bash | 16 +++ .../parallel/set1/spam_virus/amavis.bats | 99 +++++++++++++------ .../parallel/set1/spam_virus/rspamd_dkim.bats | 9 +- .../parallel/set1/spam_virus/rspamd_full.bats | 89 +++++++++++------ .../set1/spam_virus/rspamd_partly.bats | 9 +- .../set1/spam_virus/spam_junk_folder.bats | 6 +- .../set1/spam_virus/undef_spam_subject.bats | 61 ------------ 16 files changed, 261 insertions(+), 178 deletions(-) delete mode 100644 test/tests/parallel/set1/spam_virus/undef_spam_subject.bats diff --git a/CHANGELOG.md b/CHANGELOG.md index 41e6d5e9d0a..d16c03ec1cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,14 @@ The most noteworthy change of this release is the update of the container's base - DMS `main.cf` has renamed `postscreen_dnsbl_whitelist_threshold` to `postscreen_dnsbl_allowlist_threshold` as part of this change. - `smtpd_relay_restrictions` (relay policy) is now evaluated after `smtpd_recipient_restrictions` (spam policy). Previously it was evaluated before `smtpd_recipient_restrictions`. Mail to be relayed via DMS must now pass through the spam policy first. - The TLS fingerprint policy has changed the default from MD5 to SHA256 (_DMS does not modify this Postfix parameter, but may affect any user customizations that do_). +- **Environment Variables**: + - `SA_SPAM_SUBJECT` has been renamed into `SPAM_SUBJECT` to become anti-spam service agnostic. ([3820](https://github.com/docker-mailserver/docker-mailserver/pull/3820)) + - As this functionality is now handled in Dovecot via a Sieve script instead of the respective anti-spam service during Postfix processing, this feature will only apply to mail stored in Dovecot. If you have relied on this feature in a different context, it will no longer be available. + - Rspamd previously handled this functionality via the `rewrite_subject` action which as now been disabled by default in favor of the new approach with `SPAM_SUBJECT`. + - `SA_SPAM_SUBJECT` is now deprecated and will log a warning if used. The value is copied as a fallback to `SPAM_SUBJECT`. + - The default has changed to not prepend any prefix to the subject unless configured to do so. If you relied on the implicit prefix, you will now need to provide one explicitly. + - `undef` was previously supported as an opt-out with `SA_SPAM_SUBJECT`. This is no longer valid, the equivalent opt-out value is now an empty value (_or rather the omission of this ENV being configured_). + - The feature to include [`_SCORE_` tag](https://spamassassin.apache.org/full/4.0.x/doc/Mail_SpamAssassin_Conf.html#rewrite_header-subject-from-to-STRING) in your value to be replaced by the associated spam score is no longer available. ### Updates @@ -45,6 +53,8 @@ The most noteworthy change of this release is the update of the container's base - It's only functionality remaining was to opt-out of run-time state consolidation with `ONE_DIR=0` (_when a volume was already mounted to `/var/mail-state`_). - **Tests:** - Refactored helper methods for sending e-mails with specific `Message-ID` headers and the helpers for retrieving + filtering logs, which together help isolate logs relevant to specific mail when multiple mails have been processed within a single test. ([#3786](https://github.com/docker-mailserver/docker-mailserver/pull/3786)) +- **Rspamd**: + - The `rewrite_subject` action, is now disabled by default. It has been replaced with the new `SPAM_SUBJECT` environment variable, which implements the functionality via a Sieve script instead in favor of being anti-spam service agnostic ([3820](https://github.com/docker-mailserver/docker-mailserver/pull/3820)) ### Fixes diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index b1c3e2b6044..c98016d15bd 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -51,7 +51,7 @@ Configures the provisioning source of user accounts (including aliases) for user !!! tip "OAuth2 Support" - Presently DMS supports OAuth2 only as an supplementary authentication method. + Presently DMS supports OAuth2 only as an supplementary authentication method. - A third-party service must provide a valid token for the user which Dovecot validates with the authentication service provider. To enable this feature reference the [OAuth2 configuration example guide][docs::auth::oauth2-config-guide]. - User accounts must be provisioned to receive mail via one of the supported `ACCOUNT_PROVISIONER` providers. @@ -354,6 +354,16 @@ Enable to treat received spam as "read" (_avoids notification to MUA client of n - `X-Spam: Yes` (_added by Rspamd_) - `X-Spam-Flag: YES` (_added by SpamAssassin - requires [`SPAMASSASSIN_SPAM_TO_INBOX=1`](#spamassassin_spam_to_inbox)_) +##### SPAM_SUBJECT + +This variable defines a prefix for e-mails tagged with the `X-Spam: Yes` (Rspamd) or `X-Spam-Flag: YES` (SpamAssassin/Amavis) header. + +Default: empty (no prefix will be added to e-mails) + +??? example "Including trailing white-space" + + Add trailing white-space by quote wrapping the value: `SPAM_SUBJECT='[SPAM] '` + #### Rspamd ##### ENABLE_RSPAMD @@ -562,7 +572,7 @@ Changes the interval in which log files are rotated. - [`MOVE_SPAM_TO_JUNK=1`](#move_spam_to_junk) - [`MARK_SPAM_AS_READ=1`](#mark_spam_as_read) - - [`SA_SPAM_SUBJECT`](#sa_spam_subject) + - [`SPAM_SUBJECT`](#spam_subject) ##### SA_TAG @@ -602,8 +612,8 @@ When a spam score is high enough, mark mail as spam (_Appends the mail header: ` !!! info "Interaction with other ENV" - - [`SA_SPAM_SUBJECT`](#sa_spam_subject) modifies the mail subject to better communicate spam mail to the user. - - [`MOVE_SPAM_TO_JUNK=1`](#move_spam_to_junk): The mail is still delivered, but to the recipient(s) junk folder instead. This feature reduces the usefulness of `SA_SPAM_SUBJECT`. + - [`SPAM_SUBJECT`](#spam_subject) modifies the mail subject to better communicate spam mail to the user. + - [`MOVE_SPAM_TO_JUNK=1`](#move_spam_to_junk): The mail is still delivered, but to the recipient(s) junk folder instead. This feature reduces the usefulness of `SPAM_SUBJECT`. ##### SA_KILL @@ -659,23 +669,6 @@ Controls the spam score threshold for triggering an action on mail that has a hi [amavis-docs::actions]: https://www.ijs.si/software/amavisd/amavisd-new-docs.html#actions [amavis-docs::quarantine]: https://www.ijs.si/software/amavisd/amavisd-new-docs.html#quarantine -##### SA_SPAM_SUBJECT - -Adds a prefix to the subject header when mail is marked as spam (_via [`SA_TAG2`](#sa_tag2)_). - -- **`'***SPAM*** '`** => A string value to use as a mail subject prefix. -- `undef` => Opt-out of modifying the subject for mail marked as spam. - -??? example "Including trailing white-space" - - Add trailing white-space by quote wrapping the value: `SA_SPAM_SUBJECT='[SPAM] '` - -??? example "Including the associated spam score" - - The [`_SCORE_` tag][sa-docs::score-tag] will be substituted with the SpamAssassin score: `SA_SPAM_SUBJECT=***SPAM(_SCORE_)***`. - -[sa-docs::score-tag]: https://spamassassin.apache.org/full/4.0.x/doc/Mail_SpamAssassin_Conf.html#rewrite_header-subject-from-to-STRING - ##### SA_SHORTCIRCUIT_BAYES_SPAM - **1** => will activate SpamAssassin short circuiting for bayes spam detection. diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 0c457ec3512..4c466c5de72 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -22,8 +22,9 @@ The following environment variables are related to Rspamd: 5. [`RSPAMD_HFILTER`](../environment.md#rspamd_hfilter) 6. [`RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE`](../environment.md#rspamd_hfilter_hostname_unknown_score) 7. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) -8. [`MOVE_SPAM_TO_JUNK`][docs-spam-to-junk] -9. [`MARK_SPAM_AS_READ`](../environment.md#mark_spam_as_read) +8. [`SPAM_SUBJECT`](../environment.md#spam_subject) +9. [`MOVE_SPAM_TO_JUNK`][docs-spam-to-junk] +10. [`MARK_SPAM_AS_READ`](../environment.md#mark_spam_as_read) With these variables, you can enable Rspamd itself, and you can enable / disable certain features related to Rspamd. diff --git a/mailserver.env b/mailserver.env index c171faed125..38b9adcabf4 100644 --- a/mailserver.env +++ b/mailserver.env @@ -130,6 +130,11 @@ ENABLE_IMAP=1 # **0** => Disabled ENABLE_CLAMAV=0 +# Add the value as a prefix to the mail subject when spam is detected. +# NOTE: By default spam is delivered to a junk folder, reducing the value of a subject prefix for spam. +# NOTE: When not using Docker Compose, other CRI may not support quote-wrapping the value here to preserve any trailing white-space. +SPAM_SUBJECT= + # Enables Rspamd # **0** => Disabled # 1 => Enabled @@ -378,7 +383,7 @@ ENABLE_SPAMASSASSIN=0 # Note: only has an effect if `ENABLE_SPAMASSASSIN=1` ENABLE_SPAMASSASSIN_KAM=0 -# deliver spam messages to the inbox (tagged using SA_SPAM_SUBJECT) +# deliver spam messages to the inbox (tagged using SPAM_SUBJECT) SPAMASSASSIN_SPAM_TO_INBOX=1 # spam messages will be moved in the Junk folder (SPAMASSASSIN_SPAM_TO_INBOX=1 required) @@ -396,12 +401,6 @@ SA_TAG2=6.31 # triggers spam evasive actions SA_KILL=10.0 -# add tag to subject if spam detected -# The value `undef` opts-out of this feature. The value shown below is the default. -# NOTE: By default spam is delivered to a junk folder, reducing the value of adding a subject prefix. -# NOTE: If not using Docker Compose, other CRI may require the single quotes removed. -#SA_SPAM_SUBJECT='***SPAM*** ' - # ----------------------------------------------- # --- Fetchmail Section ------------------------- # ----------------------------------------------- diff --git a/target/rspamd/local.d/actions.conf b/target/rspamd/local.d/actions.conf index fb4c15b9bce..6ecb6d4be70 100644 --- a/target/rspamd/local.d/actions.conf +++ b/target/rspamd/local.d/actions.conf @@ -6,7 +6,11 @@ # and to be able to explain the impact on the whole system. greylist = 4; add_header = 6; -rewrite_subject = 7; reject = 11; -subject = "***SPAM*** %s" +# The value `null` disabled the action. A subject rewrite is handled by `SPAM_SUBJECT`: +# https://docker-mailserver.github.io/docker-mailserver/latest/config/environment/#spam_subject +# +# The reasoning for this can be found in +# https://github.com/docker-mailserver/docker-mailserver/issues/3804 +rewrite_subject = null; diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index 56dfa1fb301..f18abdf649a 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -36,6 +36,7 @@ function _register_functions() { _register_check_function '_check_improper_restart' _register_check_function '_check_hostname' _register_check_function '_check_log_level' + _register_check_function '_check_spam_prefix' # ? >> Setup @@ -48,6 +49,7 @@ function _register_functions() { _register_setup_function '_setup_dovecot_sieve' _register_setup_function '_setup_dovecot_dhparam' _register_setup_function '_setup_dovecot_quota' + _register_setup_function '_setup_spam_subject' _register_setup_function '_setup_spam_to_junk' _register_setup_function '_setup_spam_mark_as_read' fi diff --git a/target/scripts/startup/check-stack.sh b/target/scripts/startup/check-stack.sh index cf1c40f5f05..cc98cfe2a55 100644 --- a/target/scripts/startup/check-stack.sh +++ b/target/scripts/startup/check-stack.sh @@ -52,3 +52,11 @@ function _check_log_level() { LOG_LEVEL="${DEFAULT_LOG_LEVEL}" fi } + +function _check_spam_prefix() { + # This check should be independent of ENABLE_POP3 and ENABLE_IMAP + if [[ ${MOVE_SPAM_TO_JUNK} -eq 0 ]] \ + && [[ -z ${SPAM_SUBJECT} ]]; then + _log 'warn' "'MOVE_SPAM_TO_JUNK=0' and 'SPAM_SUBJECT' is empty - make sure this is intended: spam e-mails might not be immediately recognizable in this configuration" + fi +} diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index ec32a1e6d84..aefeba20bce 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -81,13 +81,9 @@ function __setup__security__spamassassin() { # shellcheck disable=SC2016 sed -i -r 's|^\$sa_kill_level_deflt (.*);|\$sa_kill_level_deflt = '"${SA_KILL}"';|g' /etc/amavis/conf.d/20-debian_defaults - if [[ ${SA_SPAM_SUBJECT} == 'undef' ]]; then - # shellcheck disable=SC2016 - sed -i -r 's|^\$sa_spam_subject_tag (.*);|\$sa_spam_subject_tag = undef;|g' /etc/amavis/conf.d/20-debian_defaults - else - # shellcheck disable=SC2016 - sed -i -r 's|^\$sa_spam_subject_tag (.*);|\$sa_spam_subject_tag = '"'${SA_SPAM_SUBJECT}'"';|g' /etc/amavis/conf.d/20-debian_defaults - fi + # disable rewriting the subject as this is handles by _setup_spam_subject (which uses Dovecot Sieve) + # shellcheck disable=SC2016 + sed -i -r 's|^\$sa_spam_subject_tag (.*);|\$sa_spam_subject_tag = undef;|g' /etc/amavis/conf.d/20-debian_defaults # activate short circuits when SA BAYES is certain it has spam or ham. if [[ ${SA_SHORTCIRCUIT_BAYES_SPAM} -eq 1 ]]; then @@ -245,6 +241,57 @@ function __setup__security__amavis() { fi } +# If `SPAM_SUBJECT` is not empty, we create a Sieve script that alters the `Subject` +# header, in order to prepend a user-defined string. +function _setup_spam_subject() { + if [[ -z ${SPAM_SUBJECT} ]] + then + _log 'debug' 'Spam subject is not set - no prefix will be added to spam e-mails' + else + _log 'debug' "Spam subject is set - the prefix '${SPAM_SUBJECT}' will be added to spam e-mails" + + _log 'trace' "Enabling '+editheader' Sieve extension" + # check whether sieve_global_extensions is disabled (and enabled it if so) + sed -i -E 's|#(sieve_global_extensions.*)|\1|' /etc/dovecot/conf.d/90-sieve.conf + # then append the extension + sedfile -i -E 's|(sieve_global_extensions.*)|\1 +editheader|' /etc/dovecot/conf.d/90-sieve.conf + + _log 'trace' "Adding global (before) Sieve script for subject rewrite" + # This directory contains Sieve scripts that are executed before user-defined Sieve + # scripts run. + local DOVECOT_SIEVE_GLOBAL_BEFORE_DIR='/usr/lib/dovecot/sieve-global/before' + local DOVECOT_SIEVE_FILE='spam_subject' + readonly DOVECOT_SIEVE_GLOBAL_BEFORE_DIR DOVECOT_SIEVE_FILE + + mkdir -p "${DOVECOT_SIEVE_GLOBAL_BEFORE_DIR}" + # ref: https://superuser.com/a/1502589 + cat >"${DOVECOT_SIEVE_GLOBAL_BEFORE_DIR}/${DOVECOT_SIEVE_FILE}.sieve" << EOF +require ["editheader","variables"]; + +if anyof (header :contains "X-Spam-Flag" "YES", + header :contains "X-Spam" "Yes") +{ + # Match the entire subject ... + if header :matches "Subject" "*" { + # ... to get it in a match group that can then be stored in a variable: + set "subject" "\${1}"; + } + + # We can't "replace" a header, but we can delete (all instances of) it and + # re-add (a single instance of) it: + deleteheader "Subject"; + + # Note that the header is added ":last" (so it won't appear before possible + # "Received" headers). + addheader :last "Subject" "${SPAM_SUBJECT}\${subject}"; +} +EOF + + sievec "${DOVECOT_SIEVE_GLOBAL_BEFORE_DIR}/${DOVECOT_SIEVE_FILE}.sieve" + chown dovecot:root "${DOVECOT_SIEVE_GLOBAL_BEFORE_DIR}/${DOVECOT_SIEVE_FILE}."{sieve,svbin} + fi +} + # We can use Sieve to move spam emails to the "Junk" folder. function _setup_spam_to_junk() { if [[ ${MOVE_SPAM_TO_JUNK} -eq 1 ]]; then diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index d14aeb3fb49..219ce6c8550 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -25,6 +25,12 @@ function __environment_variables_backwards_compatibility() { _log 'error' "The ENV for which LDAP host to connect to must include the URI scheme ('ldap://', 'ldaps://', 'ldapi://')" fi + if [[ -n ${SA_SPAM_SUBJECT:-} ]]; then + _log 'warn' "'SA_SPAM_SUBJECT' has been renamed to 'SPAM_SUBJECT' - this warning will block startup on v15.0.0" + _log 'info' "Copying value of 'SA_SPAM_SUBJECT' into 'SPAM_SUBJECT' if 'SPAM_SUBJECT' has not been set explicitly" + SPAM_SUBJECT=${SPAM_SUBJECT:-${SA_SPAM_SUBJECT}} + fi + # TODO this can be uncommented in a PR handling the HOSTNAME/DOMAINNAME issue # TODO see check_for_changes.sh and dns.sh # if [[ -n ${OVERRIDE_HOSTNAME:-} ]] @@ -67,7 +73,7 @@ function __environment_variables_general_setup() { VARS[RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE]="${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE:=6}" VARS[RSPAMD_LEARN]="${RSPAMD_LEARN:=0}" VARS[SA_KILL]=${SA_KILL:="10.0"} - VARS[SA_SPAM_SUBJECT]=${SA_SPAM_SUBJECT:="***SPAM*** "} + VARS[SPAM_SUBJECT]=${SPAM_SUBJECT:=} VARS[SA_TAG]=${SA_TAG:="2.0"} VARS[SA_TAG2]=${SA_TAG2:="6.31"} VARS[SPAMASSASSIN_SPAM_TO_INBOX]="${SPAMASSASSIN_SPAM_TO_INBOX:=1}" diff --git a/test/helper/common.bash b/test/helper/common.bash index 3c54862cc1d..35f4128334c 100644 --- a/test/helper/common.bash +++ b/test/helper/common.bash @@ -417,5 +417,21 @@ function _nc_wrapper() { _run_in_container_bash "nc ${NC_PARAMETERS} < /tmp/docker-mailserver-test/${FILE}" } +# A simple wrapper for a test that checks whether a file exists. +# +# @param ${1} = the path to the file inside the container +function _file_exists_in_container() { + _run_in_container_bash "[[ -f ${1} ]]" + assert_success +} + +# A simple wrapper for a test that checks whether a file does not exist. +# +# @param ${1} = the path to the file (that should not exists) inside the container +function _file_does_not_exist_in_container() { + _run_in_container_bash "[[ -f ${1} ]]" + assert_failure +} + # ? << Miscellaneous helper functions # ! ------------------------------------------------------------------- diff --git a/test/tests/parallel/set1/spam_virus/amavis.bats b/test/tests/parallel/set1/spam_virus/amavis.bats index d7a59cb253a..73c5b2ae1bb 100644 --- a/test/tests/parallel/set1/spam_virus/amavis.bats +++ b/test/tests/parallel/set1/spam_virus/amavis.bats @@ -2,8 +2,9 @@ load "${REPOSITORY_ROOT}/test/helper/common" load "${REPOSITORY_ROOT}/test/helper/setup" BATS_TEST_NAME_PREFIX='[Amavis + SA] ' -CONTAINER1_NAME='dms-test_amavis_enabled' -CONTAINER2_NAME='dms-test_amavis_disabled' +CONTAINER1_NAME='dms-test_amavis-enabled-default' +CONTAINER2_NAME='dms-test_amavis-enabled-custom' +CONTAINER3_NAME='dms-test_amavis-disabled' function setup_file() { export CONTAINER_NAME @@ -12,13 +13,25 @@ function setup_file() { _init_with_defaults local CUSTOM_SETUP_ARGUMENTS=( --env ENABLE_AMAVIS=1 - --env AMAVIS_LOGLEVEL=2 --env ENABLE_SPAMASSASSIN=1 ) _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' CONTAINER_NAME=${CONTAINER2_NAME} _init_with_defaults + local CUSTOM_SETUP_ARGUMENTS=( + --env ENABLE_AMAVIS=1 + --env AMAVIS_LOGLEVEL=2 + --env ENABLE_SPAMASSASSIN=1 + --env SA_TAG=-5.0 + --env SA_TAG2=2.0 + --env SA_KILL=3.0 + --env SPAM_SUBJECT='***SPAM*** ' + ) + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' + + CONTAINER_NAME=${CONTAINER3_NAME} + _init_with_defaults local CUSTOM_SETUP_ARGUMENTS=( --env ENABLE_AMAVIS=0 --env ENABLE_SPAMASSASSIN=0 @@ -27,11 +40,37 @@ function setup_file() { } function teardown_file() { - docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}" + docker rm -f "${CONTAINER1_NAME}" "${CONTAINER2_NAME}" "${CONTAINER3_NAME}" } -@test '(Amavis enabled) configuration should be correct' { +@test '(Amavis enabled - defaults) default Amavis config is correct' { export CONTAINER_NAME=${CONTAINER1_NAME} + local AMAVIS_DEFAULTS_FILE='/etc/amavis/conf.d/20-debian_defaults' + + _run_in_container grep 'sa_tag_level_deflt' "${AMAVIS_DEFAULTS_FILE}" + assert_success + assert_output --partial 'sa_tag_level_deflt = 2.0;' + + _run_in_container grep 'sa_tag2_level_deflt' "${AMAVIS_DEFAULTS_FILE}" + assert_success + # shellcheck disable=SC2016 + assert_output --partial '$sa_tag2_level_deflt = 6.31;' + + _run_in_container grep 'sa_kill_level_deflt' "${AMAVIS_DEFAULTS_FILE}" + assert_success + # shellcheck disable=SC2016 + assert_output --partial '$sa_kill_level_deflt = 10.0;' + + # This feature is handled by our SPAM_SUBJECT ENV through a sieve script instead. + # Thus the feature here should always be disabled via the 'undef' value. + _run_in_container grep 'sa_spam_subject_tag' "${AMAVIS_DEFAULTS_FILE}" + assert_success + # shellcheck disable=SC2016 + assert_output --partial '$sa_spam_subject_tag = undef;' +} + +@test '(Amavis enabled - custom) configuration should be correct' { + export CONTAINER_NAME=${CONTAINER2_NAME} _run_in_container postconf -h content_filter assert_success @@ -41,47 +80,47 @@ function teardown_file() { _run_in_container grep -F '127.0.0.1:10025' /etc/postfix/master.cf assert_success - _run_in_container_bash '[[ -f /etc/cron.d/amavisd-new.disabled ]]' - assert_failure - _run_in_container_bash '[[ -f /etc/cron.d/amavisd-new ]]' - assert_success + _file_does_not_exist_in_container /etc/cron.d/amavisd-new.disabled + _file_exists_in_container /etc/cron.d/amavisd-new } -@test '(Amavis enabled) SA integration should be active' { - export CONTAINER_NAME=${CONTAINER1_NAME} +@test '(Amavis enabled - custom) SA integration should be active' { + export CONTAINER_NAME=${CONTAINER2_NAME} # give Amavis just a bit of time to print out its full debug log run _repeat_in_container_until_success_or_timeout 20 "${CONTAINER_NAME}" grep 'SpamControl: init_pre_fork on SpamAssassin done' /var/log/mail/mail.log assert_success } -@test '(Amavis enabled) SA ENV should update Amavis config' { - export CONTAINER_NAME=${CONTAINER1_NAME} - +@test '(Amavis enabled - custom) ENV should update Amavis config' { + export CONTAINER_NAME=${CONTAINER2_NAME} local AMAVIS_DEFAULTS_FILE='/etc/amavis/conf.d/20-debian_defaults' - # shellcheck disable=SC2016 - _run_in_container grep '\$sa_tag_level_deflt' "${AMAVIS_DEFAULTS_FILE}" - assert_success - assert_output --partial '= 2.0' - # shellcheck disable=SC2016 - _run_in_container grep '\$sa_tag2_level_deflt' "${AMAVIS_DEFAULTS_FILE}" + _run_in_container grep 'sa_tag_level_deflt' "${AMAVIS_DEFAULTS_FILE}" assert_success - assert_output --partial '= 6.31' - # shellcheck disable=SC2016 - _run_in_container grep '\$sa_kill_level_deflt' "${AMAVIS_DEFAULTS_FILE}" + assert_output --partial '$sa_tag_level_deflt = -5.0;' + + _run_in_container grep 'sa_tag2_level_deflt' "${AMAVIS_DEFAULTS_FILE}" assert_success - assert_output --partial '= 10.0' + # shellcheck disable=SC2016 + assert_output --partial '$sa_tag2_level_deflt = 2.0;' + _run_in_container grep 'sa_kill_level_deflt' "${AMAVIS_DEFAULTS_FILE}" + assert_success # shellcheck disable=SC2016 - _run_in_container grep '\$sa_spam_subject_tag' "${AMAVIS_DEFAULTS_FILE}" + assert_output --partial '$sa_kill_level_deflt = 3.0;' + + # This feature is handled by our SPAM_SUBJECT ENV through a sieve script instead. + # Thus the feature here should always be disabled via the 'undef' value. + _run_in_container grep 'sa_spam_subject_tag' "${AMAVIS_DEFAULTS_FILE}" assert_success - assert_output --partial "= '***SPAM*** ';" + # shellcheck disable=SC2016 + assert_output --partial '$sa_spam_subject_tag = undef;' } @test '(Amavis disabled) configuration should be correct' { - export CONTAINER_NAME=${CONTAINER2_NAME} + export CONTAINER_NAME=${CONTAINER3_NAME} _run_in_container postconf -h content_filter assert_success @@ -91,8 +130,6 @@ function teardown_file() { _run_in_container grep -F '127.0.0.1:10025' /etc/postfix/master.cf assert_failure - _run_in_container_bash '[[ -f /etc/cron.d/amavisd-new.disabled ]]' - assert_success - _run_in_container_bash '[[ -f /etc/cron.d/amavisd-new ]]' - assert_failure + _file_exists_in_container /etc/cron.d/amavisd-new.disabled + _file_does_not_exist_in_container /etc/cron.d/amavisd-new } diff --git a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats index 044e38fbf26..20939f032e5 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_dkim.bats @@ -62,8 +62,7 @@ function teardown_file() { _default_teardown ; } assert_output --partial "Supplying a default configuration (to '${SIGNING_CONF_FILE}')" refute_output --partial "'${SIGNING_CONF_FILE}' exists, not supplying a default" assert_output --partial "Finished DKIM key creation" - _run_in_container_bash "[[ -f ${SIGNING_CONF_FILE} ]]" - assert_success + _file_exists_in_container "${SIGNING_CONF_FILE}" _exec_in_container_bash "echo 'blabla' >${SIGNING_CONF_FILE}" local INITIAL_SHA512_SUM=$(_exec_in_container sha512sum "${SIGNING_CONF_FILE}") @@ -87,8 +86,7 @@ function teardown_file() { _default_teardown ; } assert_success _count_files_in_directory_in_container /tmp/docker-mailserver/rspamd/dkim/ 3 - _run_in_container_bash "[[ -f ${SIGNING_CONF_FILE} ]]" - assert_success + _file_exists_in_container "${SIGNING_CONF_FILE}" __check_path_in_signing_config "/tmp/docker-mailserver/rspamd/dkim/rsa-2048-mail-${DOMAIN_NAME}.private.txt" __check_selector_in_signing_config 'mail' @@ -241,8 +239,7 @@ function __check_rsa_keys() { function __check_key_files_are_present() { local BASE_FILE_NAME="${1:?Base file name must be supplied to __check_key_files_are_present}" for FILE in ${BASE_FILE_NAME}.{public.txt,public.dns.txt,private.txt}; do - _run_in_container_bash "[[ -f ${FILE} ]]" - assert_success + _file_exists_in_container "${FILE}" done } diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index 9e6d62222e5..9b14860b315 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -29,6 +29,7 @@ function setup_file() { --env RSPAMD_GREYLISTING=1 --env RSPAMD_HFILTER=1 --env RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE=7 + --env SPAM_SUBJECT='[POTENTIAL SPAM] ' ) cp -r "${TEST_TMP_CONFIG}"/rspamd_full/* "${TEST_TMP_CONFIG}/" @@ -43,7 +44,7 @@ function setup_file() { _wait_for_service postfix _wait_for_smtp_port_in_container - # We will send 4 emails: + # We will send 5 emails: # 1. The first one should pass just fine _send_email_with_msgid 'rspamd-test-email-pass' # 2. The second one should be rejected (Rspamd-specific GTUBE pattern for rejection) @@ -56,6 +57,9 @@ function setup_file() { # ref: https://rspamd.com/doc/gtube_patterns.html _send_email_with_msgid 'rspamd-test-email-header' \ --body "YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X" + # 5. The fifth one will have its subject rewritten, but now spam header is applied. + _send_email_with_msgid 'rspamd-test-email-rewrite_subject' \ + --body "ZJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X" _run_in_container cat /var/log/mail.log assert_success @@ -73,11 +77,18 @@ function teardown_file() { _default_teardown ; } assert_output 'rspamd_milter = inet:localhost:11332' } +@test 'Rspamd base configuration is correct' { + _run_in_container rspamadm configdump actions + assert_success + assert_line 'greylist = 4;' + assert_line 'reject = 11;' + assert_line 'add_header = 6;' + refute_line --regexp 'rewrite_subject = [0-9]+;' +} + @test "contents of '/etc/rspamd/override.d/' are copied" { local OVERRIDE_D='/etc/rspamd/override.d' - - _run_in_container_bash "[[ -f ${OVERRIDE_D}/testmodule_complicated.conf ]]" - assert_success + _file_exists_in_container "${OVERRIDE_D}/testmodule_complicated.conf" } @test 'startup log shows all features as properly enabled' { @@ -89,6 +100,7 @@ function teardown_file() { _default_teardown ; } assert_line --partial 'Enabling greylisting' assert_line --partial 'Hfilter (group) module is enabled' assert_line --partial "Adjusting score for 'HFILTER_HOSTNAME_UNKNOWN' in Hfilter group module to" + assert_line --partial "Spam subject is set - the prefix '[POTENTIAL SPAM] ' will be added to spam e-mails" assert_line --partial "Found file '/tmp/docker-mailserver/rspamd/custom-commands.conf' - parsing and applying it" } @@ -114,7 +126,7 @@ function teardown_file() { _default_teardown ; } _print_mail_log_for_msgid 'rspamd-test-email-pass' assert_output --partial "stored mail into mailbox 'INBOX'" - _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 2 } @test 'detects and rejects spam' { @@ -129,7 +141,7 @@ function teardown_file() { _default_teardown ; } refute_output --partial "stored mail into mailbox 'INBOX'" assert_failure - _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 2 } @test 'detects and rejects virus' { @@ -144,14 +156,13 @@ function teardown_file() { _default_teardown ; } refute_output --partial "stored mail into mailbox 'INBOX'" assert_failure - _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 2 } @test 'custom commands work correctly' { # check `testmodule1` which should be disabled local MODULE_PATH='/etc/rspamd/override.d/testmodule1.conf' - _run_in_container_bash "[[ -f ${MODULE_PATH} ]]" - assert_success + _file_exists_in_container "${MODULE_PATH}" _run_in_container grep -F '# documentation: https://rspamd.com/doc/modules/testmodule1.html' "${MODULE_PATH}" assert_success _run_in_container grep -F 'enabled = false;' "${MODULE_PATH}" @@ -161,8 +172,7 @@ function teardown_file() { _default_teardown ; } # check `testmodule2` which should be enabled and it should have extra options set MODULE_PATH='/etc/rspamd/override.d/testmodule2.conf' - _run_in_container_bash "[[ -f ${MODULE_PATH} ]]" - assert_success + _file_exists_in_container "${MODULE_PATH}" _run_in_container grep -F '# documentation: https://rspamd.com/doc/modules/testmodule2.html' "${MODULE_PATH}" assert_success _run_in_container grep -F 'enabled = true;' "${MODULE_PATH}" @@ -181,8 +191,7 @@ function teardown_file() { _default_teardown ; } # check whether adding a single line writes the line properly in `testmodule4.something` MODULE_PATH='/etc/rspamd/override.d/testmodule4.something' - _run_in_container_bash "[[ -f ${MODULE_PATH} ]]" - assert_success + _file_exists_in_container "${MODULE_PATH}" # shellcheck disable=SC2016 _run_in_container grep -F 'some very long line with "weird $charact"ers' "${MODULE_PATH}" assert_success @@ -193,37 +202,31 @@ function teardown_file() { _default_teardown ; } # check whether spaces in front of options are handles properly in `testmodule_complicated` MODULE_PATH='/etc/rspamd/override.d/testmodule_complicated.conf' - _run_in_container_bash "[[ -f ${MODULE_PATH} ]]" - assert_success + _file_exists_in_container "${MODULE_PATH}" _run_in_container grep -F ' anOption = anotherValue;' "${MODULE_PATH}" # check whether controller option was written properly MODULE_PATH='/etc/rspamd/override.d/worker-controller.inc' - _run_in_container_bash "[[ -f ${MODULE_PATH} ]]" - assert_success + _file_exists_in_container "${MODULE_PATH}" _run_in_container grep -F 'someOption = someValue42;' "${MODULE_PATH}" assert_success # check whether controller option was written properly MODULE_PATH='/etc/rspamd/override.d/worker-proxy.inc' - _run_in_container_bash "[[ -f ${MODULE_PATH} ]]" - assert_success + _file_exists_in_container "${MODULE_PATH}" _run_in_container grep -F 'abcdefg71 = RAAAANdooM;' "${MODULE_PATH}" assert_success # check whether basic options are written properly MODULE_PATH='/etc/rspamd/override.d/options.inc' - _run_in_container_bash "[[ -f ${MODULE_PATH} ]]" - assert_success + _file_exists_in_container "${MODULE_PATH}" _run_in_container grep -F 'OhMy = "PraiseBeLinters !";' "${MODULE_PATH}" assert_success } @test 'MOVE_SPAM_TO_JUNK works for Rspamd' { - _run_in_container_bash '[[ -f /usr/lib/dovecot/sieve-global/after/spam_to_junk.sieve ]]' - assert_success - _run_in_container_bash '[[ -f /usr/lib/dovecot/sieve-global/after/spam_to_junk.svbin ]]' - assert_success + _file_exists_in_container /usr/lib/dovecot/sieve-global/after/spam_to_junk.sieve + _file_exists_in_container /usr/lib/dovecot/sieve-global/after/spam_to_junk.svbin _service_log_should_contain_string 'rspamd' 'S (add header)' _service_log_should_contain_string 'rspamd' 'add header "Gtube pattern"' @@ -231,14 +234,38 @@ function teardown_file() { _default_teardown ; } _print_mail_log_for_msgid 'rspamd-test-email-header' assert_output --partial "fileinto action: stored mail into mailbox 'Junk'" - _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 1 + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 2 _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/.Junk/new/ 1 } +@test 'Rewriting subject works when enforcing it via GTUBE' { + _service_log_should_contain_string 'rspamd' 'S (rewrite subject)' + _service_log_should_contain_string 'rspamd' 'rewrite subject "Gtube pattern"' + + _print_mail_log_for_msgid 'rspamd-test-email-rewrite_subject' + assert_output --partial "stored mail into mailbox 'INBOX'" + + # check that the inbox contains the subject-rewritten e-mail + _run_in_container_bash "grep --fixed-strings 'Subject: *** SPAM ***' /var/mail/localhost.localdomain/user1/new/*" + assert_success + + # check that the inbox contains the normal e-mail (that passes just fine) + _run_in_container_bash "grep --fixed-strings 'Subject: test' /var/mail/localhost.localdomain/user1/new/*" + assert_success +} + +@test 'SPAM_SUBJECT works' { + _file_exists_in_container /usr/lib/dovecot/sieve-global/before/spam_subject.sieve + _file_exists_in_container /usr/lib/dovecot/sieve-global/before/spam_subject.svbin + + # we only have one e-mail in the junk folder, hence using '*' is fine + _run_in_container_bash "grep --fixed-strings 'Subject: [POTENTIAL SPAM]' /var/mail/localhost.localdomain/user1/.Junk/new/*" + assert_success +} + @test 'RSPAMD_LEARN works' { for FILE in learn-{ham,spam}.{sieve,svbin}; do - _run_in_container_bash "[[ -f /usr/lib/dovecot/sieve-pipe/${FILE} ]]" - assert_success + _file_exists_in_container "/usr/lib/dovecot/sieve-pipe/${FILE}" done _run_in_container grep 'mail_plugins.*imap_sieve' /etc/dovecot/conf.d/20-imap.conf @@ -305,8 +332,7 @@ function teardown_file() { _default_teardown ; } @test 'hfilter group module is configured correctly' { local MODULE_FILE='/etc/rspamd/local.d/hfilter_group.conf' - _run_in_container_bash "[[ -f ${MODULE_FILE} ]]" - assert_success + _file_exists_in_container "${MODULE_FILE}" _run_in_container grep '__TAG__HFILTER_HOSTNAME_UNKNOWN' "${MODULE_FILE}" assert_success @@ -315,8 +341,7 @@ function teardown_file() { _default_teardown ; } @test 'checks on authenticated users are disabled' { local MODULE_FILE='/etc/rspamd/local.d/settings.conf' - _run_in_container_bash "[[ -f ${MODULE_FILE} ]]" - assert_success + _file_exists_in_container "${MODULE_FILE}" _run_in_container grep -E -A 6 'authenticated \{' "${MODULE_FILE}" assert_success diff --git a/test/tests/parallel/set1/spam_virus/rspamd_partly.bats b/test/tests/parallel/set1/spam_virus/rspamd_partly.bats index 9fc8af31570..b647f562110 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_partly.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_partly.bats @@ -55,6 +55,7 @@ function teardown_file() { _default_teardown ; } assert_line --partial 'Intelligent learning of spam and ham is disabled' assert_line --partial 'Greylisting is disabled' assert_line --partial 'Disabling Hfilter (group) module' + assert_line --partial 'Spam subject is not set' } @test 'antivirus maximum size was not adjusted unnecessarily' { @@ -64,8 +65,7 @@ function teardown_file() { _default_teardown ; } @test 'learning is properly disabled' { for FILE in learn-{ham,spam}.{sieve,svbin}; do - _run_in_container_bash "[[ -f /usr/lib/dovecot/sieve-pipe/${FILE} ]]" - assert_failure + _file_does_not_exist_in_container "/usr/lib/dovecot/sieve-pipe/${FILE}" done _run_in_container grep 'mail_plugins.*imap_sieve' /etc/dovecot/conf.d/20-imap.conf @@ -83,14 +83,13 @@ function teardown_file() { _default_teardown ; } } @test 'hfilter group module configuration is deleted' { - _run_in_container_bash '[[ -f /etc/rspamd/local.d/hfilter_group.conf ]]' + _file_does_not_exist_in_container /etc/rspamd/local.d/hfilter_group.conf assert_failure } @test 'checks on authenticated users are enabled' { local MODULE_FILE='/etc/rspamd/local.d/settings.conf' - _run_in_container_bash "[[ -f ${MODULE_FILE} ]]" - assert_success + _file_exists_in_container "${MODULE_FILE}" _run_in_container grep -E 'authenticated \{' "${MODULE_FILE}" assert_failure diff --git a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats index 15ec4fe106e..0c873382d74 100644 --- a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats +++ b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats @@ -34,7 +34,7 @@ function teardown() { _default_teardown ; } local CUSTOM_SETUP_ARGUMENTS=( --env ENABLE_AMAVIS=1 --env ENABLE_SPAMASSASSIN=1 - --env SA_SPAM_SUBJECT="SPAM: " + --env SPAM_SUBJECT="SPAM: " --env SPAMASSASSIN_SPAM_TO_INBOX=1 --env MOVE_SPAM_TO_JUNK=0 --env PERMIT_DOCKER=container @@ -55,7 +55,7 @@ function teardown() { _default_teardown ; } local CUSTOM_SETUP_ARGUMENTS=( --env ENABLE_AMAVIS=1 --env ENABLE_SPAMASSASSIN=1 - --env SA_SPAM_SUBJECT="SPAM: " + --env SPAM_SUBJECT="SPAM: " --env SPAMASSASSIN_SPAM_TO_INBOX=1 --env MOVE_SPAM_TO_JUNK=1 --env PERMIT_DOCKER=container @@ -77,7 +77,7 @@ function teardown() { _default_teardown ; } local CUSTOM_SETUP_ARGUMENTS=( --env ENABLE_AMAVIS=1 --env ENABLE_SPAMASSASSIN=1 - --env SA_SPAM_SUBJECT="SPAM: " + --env SPAM_SUBJECT="SPAM: " --env SPAMASSASSIN_SPAM_TO_INBOX=1 --env MARK_SPAM_AS_READ=1 --env PERMIT_DOCKER=container diff --git a/test/tests/parallel/set1/spam_virus/undef_spam_subject.bats b/test/tests/parallel/set1/spam_virus/undef_spam_subject.bats deleted file mode 100644 index 35260d51e84..00000000000 --- a/test/tests/parallel/set1/spam_virus/undef_spam_subject.bats +++ /dev/null @@ -1,61 +0,0 @@ -load "${REPOSITORY_ROOT}/test/helper/setup" -load "${REPOSITORY_ROOT}/test/helper/common" - -BATS_TEST_NAME_PREFIX='[Spam] (undefined subject) ' - -CONTAINER1_NAME='dms-test_spam-undef-subject_1' -CONTAINER2_NAME='dms-test_spam-undef-subject_2' -CONTAINER_NAME=${CONTAINER2_NAME} - -function teardown() { _default_teardown ; } - -@test "'SA_SPAM_SUBJECT=undef' should update Amavis config" { - export CONTAINER_NAME=${CONTAINER1_NAME} - local CUSTOM_SETUP_ARGUMENTS=( - --env ENABLE_AMAVIS=1 - --env ENABLE_SPAMASSASSIN=1 - --env SA_SPAM_SUBJECT='undef' - ) - _init_with_defaults - _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' - - _run_in_container_bash "grep '\$sa_spam_subject_tag' /etc/amavis/conf.d/20-debian_defaults | grep '= undef'" - assert_success -} - -# TODO: Unclear why some of these ENV are relevant for the test? -@test "Docker env variables are set correctly (custom)" { - export CONTAINER_NAME=${CONTAINER2_NAME} - - local CUSTOM_SETUP_ARGUMENTS=( - --env ENABLE_CLAMAV=1 - --env SPOOF_PROTECTION=1 - --env ENABLE_SPAMASSASSIN=1 - --env REPORT_RECIPIENT=user1@localhost.localdomain - --env REPORT_SENDER=report1@mail.my-domain.com - --env SA_TAG=-5.0 - --env SA_TAG2=2.0 - --env SA_KILL=3.0 - --env SA_SPAM_SUBJECT="SPAM: " - --env VIRUSMAILS_DELETE_DELAY=7 - --env ENABLE_SRS=1 - --env ENABLE_MANAGESIEVE=1 - --env PERMIT_DOCKER=host - # NOTE: ulimit required for `ENABLE_SRS=1` until running a newer `postsrsd` - --ulimit "nofile=$(ulimit -Sn):$(ulimit -Hn)" - ) - _init_with_defaults - _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' - - _run_in_container_bash "grep '\$sa_tag_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= -5.0'" - assert_success - - _run_in_container_bash "grep '\$sa_tag2_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 2.0'" - assert_success - - _run_in_container_bash "grep '\$sa_kill_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 3.0'" - assert_success - - _run_in_container_bash "grep '\$sa_spam_subject_tag' /etc/amavis/conf.d/20-debian_defaults | grep '= .SPAM: .'" - assert_success -} From 2018be7fdc04ce313667d365427fa9e79b4db2eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jan 2024 14:08:59 +0100 Subject: [PATCH 282/592] chore(deps): Bump anchore/scan-action from 3.6.0 to 3.6.1 (#3848) Bumps [anchore/scan-action](https://github.com/anchore/scan-action) from 3.6.0 to 3.6.1. - [Release notes](https://github.com/anchore/scan-action/releases) - [Changelog](https://github.com/anchore/scan-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/anchore/scan-action/compare/v3.6.0...v3.6.1) --- updated-dependencies: - dependency-name: anchore/scan-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index b261de913e7..cb1a5994877 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v3.6.0 + uses: anchore/scan-action@v3.6.1 id: scan with: image: mailserver-testing:ci From 4162d608e40370198662e7006ce6d5a81b94671c Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Mon, 29 Jan 2024 22:10:03 +0100 Subject: [PATCH 283/592] Rspamd scripts: only correct permissions when directory exists (#3849) --- target/scripts/startup/setup-stack.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/target/scripts/startup/setup-stack.sh b/target/scripts/startup/setup-stack.sh index 24ef1581104..81e1a98e059 100644 --- a/target/scripts/startup/setup-stack.sh +++ b/target/scripts/startup/setup-stack.sh @@ -97,8 +97,10 @@ function _setup_apply_fixes_after_configuration() { _rspamd_get_envs # /tmp/docker-mailserver/rspamd/dkim - _log 'debug' "Ensuring '${RSPAMD_DMS_DKIM_D}' is owned by '_rspamd:_rspamd'" - chown -R _rspamd:_rspamd "${RSPAMD_DMS_DKIM_D}" + if [[ -d ${RSPAMD_DMS_DKIM_D} ]]; then + _log 'debug' "Ensuring '${RSPAMD_DMS_DKIM_D}' is owned by '_rspamd:_rspamd'" + chown -R _rspamd:_rspamd "${RSPAMD_DMS_DKIM_D}" + fi } function _run_user_patches() { From 244c455ca1b92371a5c29a47ce28588f3eb5baf7 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 30 Jan 2024 11:11:45 +1300 Subject: [PATCH 284/592] fix: `packages.sh` - Download `jaq` via release `tag` not `latest` (#3852) As the filename includes the version / tag, we cannot rely on the latest URL to be stable. --- target/scripts/build/packages.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 24dae8a70c8..afff4cadd12 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -38,7 +38,8 @@ function _pre_installation_steps() { function _install_utils() { _log 'debug' 'Installing utils sourced from Github' _log 'trace' 'Installing jaq' - curl -sSfL "https://github.com/01mf02/jaq/releases/latest/download/jaq-v1.2.0-$(uname -m)-unknown-linux-gnu" -o /usr/bin/jaq && chmod +x /usr/bin/jaq + local JAQ_TAG='v1.3.0' + curl -sSfL "https://github.com/01mf02/jaq/releases/download/${JAQ_TAG}/jaq-${JAQ_TAG}-$(uname -m)-unknown-linux-gnu" -o /usr/bin/jaq && chmod +x /usr/bin/jaq _log 'trace' 'Installing swaks' local SWAKS_VERSION='20240103.0' From 23705e6712cd5b9cacf4cf8b5e7bfc3375e72e2e Mon Sep 17 00:00:00 2001 From: Casper Date: Tue, 30 Jan 2024 07:34:26 +0100 Subject: [PATCH 285/592] fix: abort when (jaq) curl fails (#3853) --- target/scripts/build/packages.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index afff4cadd12..1f769739c78 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -39,7 +39,8 @@ function _install_utils() { _log 'debug' 'Installing utils sourced from Github' _log 'trace' 'Installing jaq' local JAQ_TAG='v1.3.0' - curl -sSfL "https://github.com/01mf02/jaq/releases/download/${JAQ_TAG}/jaq-${JAQ_TAG}-$(uname -m)-unknown-linux-gnu" -o /usr/bin/jaq && chmod +x /usr/bin/jaq + curl -sSfL "https://github.com/01mf02/jaq/releases/download/${JAQ_TAG}/jaq-${JAQ_TAG}-$(uname -m)-unknown-linux-gnu" -o /usr/bin/jaq + chmod +x /usr/bin/jaq _log 'trace' 'Installing swaks' local SWAKS_VERSION='20240103.0' @@ -144,7 +145,7 @@ function _install_dovecot() { # when building for another architecture. if [[ ${DOVECOT_COMMUNITY_REPO} -eq 1 ]] && [[ "$(uname --machine)" == "x86_64" ]]; then _log 'trace' 'Using Dovecot community repository' - curl https://repo.dovecot.org/DOVECOT-REPO-GPG | gpg --import + curl -sSfL https://repo.dovecot.org/DOVECOT-REPO-GPG | gpg --import gpg --export ED409DA1 > /etc/apt/trusted.gpg.d/dovecot.gpg # VERSION_CODENAME sourced from /etc/os-release echo "deb https://repo.dovecot.org/ce-2.3-latest/debian/${VERSION_CODENAME} ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/dovecot.list From dfd5edc000f41ba46b39c469e4b5075b7a4ef1a9 Mon Sep 17 00:00:00 2001 From: Andreas Perhab Date: Tue, 30 Jan 2024 11:17:58 +0100 Subject: [PATCH 286/592] docs: Add new local dependency (`file`) for running tests (#3856) --- docs/content/contributing/tests.md | 2 +- target/postfix/main.cf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/contributing/tests.md b/docs/content/contributing/tests.md index 8816a228bb0..dcd7350583a 100644 --- a/docs/content/contributing/tests.md +++ b/docs/content/contributing/tests.md @@ -51,7 +51,7 @@ Parallel tests are further partitioned into smaller sets. If your system has the To run the test suite, you will need to: 1. [Install Docker][get-docker] -2. Install `jq` and (GNU) `parallel` (under Ubuntu, use `sudo apt-get -y install jq parallel`) +2. Install `jq` , (GNU) `parallel` and `file` (under Ubuntu, use `sudo apt-get -y install jq parallel file`) 3. Execute `git submodule update --init --recursive` if you haven't already initialized the git submodules ### Executing Test(s) diff --git a/target/postfix/main.cf b/target/postfix/main.cf index a0d805cc838..d501eec0a96 100644 --- a/target/postfix/main.cf +++ b/target/postfix/main.cf @@ -119,6 +119,6 @@ smtp_header_checks = pcre:/etc/postfix/maps/sender_header_filter.pcre # The default compatibility_level is 0 - which retains legacy settings defaults: # http://www.postfix.org/COMPATIBILITY_README.html -# If backwards-compaitibilty log messages appear, fix them by explicitly adding +# If backwards-compatibility log messages appear, fix them by explicitly adding # the legacy or new default value (alternatively raise the compatibility_level) compatibility_level = 3.6 From 5b54d1d32ee1f786c6c9a7f6bf4db07ceea871e2 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 31 Jan 2024 10:24:43 +1300 Subject: [PATCH 287/592] refactor: `relay.sh` (#3845) * chore: `relay.sh` helper - Reference user config paths via variables * chore: Better document postfix helper `_vhost_collect_postfix_domains()` The functionality is effectively the same for the two configs for the most part when it comes to parsing out a domain from the target value. Virtual aliases is more flexible in value, which may not have a domain-part present (manual user edit). * chore: `check-for-change.sh` - Support VHOST change visibility - Moves the "handle changes" logic into it's own scoped function, out of the main change detection loop logic. - This will be benefit a future commit change that will rely on `VHOST_UPDATED=1`. * chore: `relay.sh` - Minor revisions to minimize diff noise - Better phrasing of the current logic comments. - Regex patterns assigned to variables (easier to grok intention) - Bulk of the logic for generating `/etc/postfix/relayhost_map` wrapped into a separate function with Postfix config setting handled separately. * refactor: `relay.sh` opt-out logic - Split the two distinct features that configure `/etc/postfix/relayhost_map` into separate functions (_`MATCH_VALID` var no longer needed for legacy support_). - Instead of extracting domains from `postfix-accounts.cf` + `postfix-virtual.cf`, this has already been handled at `/etc/postfix/vhost`, sourcing from there is far less complicated. - Rename loop var `DOMAIN_PART`to `SENDER_DOMAIN` for better context of what it represents when appended to the config file. - Revised maintenance notes + guidance towards a future refactor of this relayhost feature support. * docs: `relay.sh` - Additional comment revisions * feat: `DEFAULT_RELAY_HOST` can now also use relay credentials ENV - Remove comment regarding `smtp_sasl_password_maps = static:${RELAY_USER}:${RELAY_PASSWORD}`, it could be used but `main.cf` presently has `644` permissions vs the `sasl_passwd` file permissions of `600`, less secure at preventing leaking of secrets (ignoring the ENV exposure itself). - Move the `main.cf` settings specific to relayhost credentials support / security into to the relevant function scope instead. This also allows for the configuration to be applied by a change detection event without container restart requirement. - Outer functions for setup and change detection to call have a clearer config dependency guard, as does the `_legacy_support()`. - These changes now support `DEFAULT_RELAY_HOST` to leverage the relay credentials ENV as well. - `DATABASE_RELAYHOSTS` is available in scope to the functions called here that reference it. * docs: Revised ENV docs on relay host config Better quality guidance on configuring relay hosts. * chore: Add entry to `CHANGELOG.md` * fix: `relay.sh` - `grep` regex compatibility with `+` requires `-E` * chore: `postfix.sh` - `FIRST_FIELD` => More descriptive field name --- CHANGELOG.md | 11 ++ docs/content/config/environment.md | 87 ++++++++++--- mailserver.env | 16 ++- target/scripts/check-for-changes.sh | 55 +++++---- target/scripts/helpers/postfix.sh | 23 ++-- target/scripts/helpers/relay.sh | 185 ++++++++++++++-------------- 6 files changed, 232 insertions(+), 145 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d16c03ec1cc..1f426a5bc42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,13 @@ The most noteworthy change of this release is the update of the container's base - DMS `main.cf` has renamed `postscreen_dnsbl_whitelist_threshold` to `postscreen_dnsbl_allowlist_threshold` as part of this change. - `smtpd_relay_restrictions` (relay policy) is now evaluated after `smtpd_recipient_restrictions` (spam policy). Previously it was evaluated before `smtpd_recipient_restrictions`. Mail to be relayed via DMS must now pass through the spam policy first. - The TLS fingerprint policy has changed the default from MD5 to SHA256 (_DMS does not modify this Postfix parameter, but may affect any user customizations that do_). +- **Features:** + - The relay host feature was refactored ([#3845](https://github.com/docker-mailserver/docker-mailserver/pull/3845)) + - The only breaking change this should introduce is with the Change Detection service (`check-for-changes.sh`). + - When credentials are configured for relays, change events that trigger the relayhost logic now reapply the relevant Postfix settings: + - `smtp_sasl_auth_enable = yes` (_SASL auth to outbound MTA connections is enabled_) + - `smtp_sasl_security_options = noanonymous` (_credentials are mandatory for outbound mail delivery_) + - `smtp_tls_security_level = encrypt` (_the outbound MTA connection must always be secure due to credentials sent_) - **Environment Variables**: - `SA_SPAM_SUBJECT` has been renamed into `SPAM_SUBJECT` to become anti-spam service agnostic. ([3820](https://github.com/docker-mailserver/docker-mailserver/pull/3820)) - As this functionality is now handled in Dovecot via a Sieve script instead of the respective anti-spam service during Postfix processing, this feature will only apply to mail stored in Dovecot. If you have relied on this feature in a different context, it will no longer be available. @@ -60,6 +67,10 @@ The most noteworthy change of this release is the update of the container's base - DMS config files that are parsed line by line are now more robust to parse by detecting and fixing line-endings ([#3819](https://github.com/docker-mailserver/docker-mailserver/pull/3819)) - Variables related to Rspamd are declared as `readonly`, which would cause warnings in the log when being re-declared; we now guard against this issue ([#3837](https://github.com/docker-mailserver/docker-mailserver/pull/3837)) +- Relay host feature refactored ([#3845](https://github.com/docker-mailserver/docker-mailserver/pull/3845)) + - `DEFAULT_RELAY_HOST` ENV can now also use the `RELAY_USER` + `RELAY_PASSWORD` ENV for supplying credentials. + - `RELAY_HOST` ENV no longer enforces configuring outbound SMTP to require credentials. Like `DEFAULT_RELAY_HOST` it can now configure a relay where credentials are optional. + - Restarting DMS should not be required when configuring relay hosts without these ENV, but solely via `setup relay ...`, as change detection events now apply relevant Postfix setting changes for supporting credentials too. ## [v13.3.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.3.1) diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index c98016d15bd..5db1c2c3d6b 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -1012,36 +1012,90 @@ you to replace both instead of just the envelope sender. - **empty** => Derived from [`OVERRIDE_HOSTNAME`](#override_hostname), `$DOMAINNAME` (internal), or the container's hostname - Set this if auto-detection fails, isn't what you want, or you wish to have a separate container handle DSNs -#### Default Relay Host +#### Relay Host + +!!! tip "`RELAY_HOST` vs `DEFAULT_RELAY_HOST`" + + `DEFAULT_RELAY_HOST` is encouraged, but presently does not support sender domain opt-out (`setup relay exclude-domain`). + +!!! tip "Opt-in for relay host support" + + If you only want to enable relay for specific sender domains, use can use opt-in via `setup relay add-domain`. ##### DEFAULT_RELAY_HOST -- **empty** => don't set default relayhost setting in main.cf -- default host and port to relay all mail through. - Format: `[example.com]:587` (don't forget the brackets if you need this to - be compatible with `$RELAY_USER` and `$RELAY_PASSWORD`, explained below). +Configures a default relay host. -#### Multi-domain Relay Hosts +!!! info + + - All mail sent outbound from DMS will be relayed through the configured host, unless sender-dependent relayhost maps have been configured (_which have precedence_). + - The host value may optionally be wrapped in brackets (_skips DNS query for MX record_): `[mail.example.com]:587` vs `example.com:587` + +!!! abstract "Technical Details" + + Configures the Postfix `main.cf` setting: [`relayhost`][postfix-config::relayhost] ##### RELAY_HOST -- **empty** => don't configure relay host -- default host to relay mail through +Configures a default relay host. + +!!! info + + - This is a legacy ENV. It is however required for the opt-out feature of `postfix-relaymap.cf` to work. + - When configured, all known mail domains managed by DMS will be configured to relay outbound mail, just like `DEFAULT_RELAY_HOST`. + +!!! note + + Expects a value like `mail.example.com`. Internally this will be wrapped to `[mail.example.com]`, so it should resolve to the MTA directly. + + Do not use with `DEFAULT_RELAY_HOST`. `RELAY_HOST` has precedence as it is configured with `sender_dependent_relayhost_maps`. + +!!! abstract "Technical Details" + + This feature is configured internally using the: + + - Postfix setting with config: [`sender_dependent_relayhost_maps = texthash:/etc/postfix/relayhost_map`][postfix-config::relayhost_maps] + - DMS Config volume support via: `postfix-relaymap.cf` (_generates `/etc/postfix/relayhost_map`_) ##### RELAY_PORT -- **empty** => 25 -- default port to relay mail through +Default => 25 -##### RELAY_USER +Support for configuring a different port than 25 for `RELAY_HOST` to use. + +!!! note + + Requires `RELAY_HOST`. + +#### Relay Host Credentials + +!!! warning "Configuring relay host credentials make outbound authentication mandatory" + + Presently when `RELAY_USER` + `RELAY_PASSWORD` or `postfix-sasl-password.cf` are configured, all outbound mail traffic is configured to require a secure connection established and forbids the omission of credentials. + + Additional feature work is required to only enforce these requirements on mail sent through a configured relay host. -- **empty** => no default -- default relay username (if no specific entry exists in postfix-sasl-password.cf) +##### RELAY_USER ##### RELAY_PASSWORD -- **empty** => no default -- password for default relay user +Provide the credentials to use with `RELAY_HOST` or `DEFAULT_RELAY_HOST`. + +!!! tip "Alternative credentials config" + + You may prefer to use `setup relay add-auth` to avoid exposure of secrets in ENV. + + - With the CLI command you must provide each sender domain relay credentials. + - Alternatively manually edit `postfix-sasl-password.cf` with the correct relayhost entry (_`DEFAULT_RELAY_HOST` value or as defined in `/etc/postfix/relayhost_map`_) to provide credentials per relayhost configured. + +!!! abstract "Technical Details" + + Credentials for relay hosts are configured internally using the: + + - Postfix setting with config: [`smtp_sasl_password_maps = texthash:/etc/postfix/sasl_passwd`][postfix-config::sasl_passwd] + - DMS Config volume support via: `postfix-sasl-password.cf` (_generates `/etc/postfix/sasl_passwd`_) + + This file has relay hosts that must match the `host:port` of `/etc/postfix/relayhost_map` or `main.cf:relayhost`. DMS support handles this for you. [docs-rspamd]: ./security/rspamd.md [docs-tls]: ./security/ssl.md @@ -1050,3 +1104,6 @@ you to replace both instead of just the envelope sender. [docs-tls-selfsigned]: ./security/ssl.md#self-signed-certificates [docs-accounts-quota]: ./user-management.md#quotas [docs::dms-volumes-state]: ./advanced/optional-config.md#volumes-state +[postfix-config::relayhost]: https://www.postfix.org/postconf.5.html#relayhost +[postfix-config::relayhost_maps]: https://www.postfix.org/postconf.5.html#sender_dependent_relayhost_maps +[postfix-config::sasl_passwd]: https://www.postfix.org/postconf.5.html#smtp_sasl_password_maps diff --git a/mailserver.env b/mailserver.env index 38b9adcabf4..6fc9c7c2558 100644 --- a/mailserver.env +++ b/mailserver.env @@ -619,8 +619,8 @@ SRS_SECRET= # Setup relaying all mail through a default relay host # -# empty => don't configure default relay host -# default host and optional port to relay all mail through +# Set a default host to relay all mail through (optionally include a port) +# Example: [mail.example.com]:587 DEFAULT_RELAY_HOST= # ----------------------------------------------- @@ -630,18 +630,22 @@ DEFAULT_RELAY_HOST= # Setup relaying for multiple domains based on the domain name of the sender # optionally uses usernames and passwords in postfix-sasl-password.cf and relay host mappings in postfix-relaymap.cf # -# empty => don't configure relay host -# default host to relay mail through +# Set a default host to relay mail through +# Example: mail.example.com RELAY_HOST= # empty => 25 # default port to relay mail RELAY_PORT=25 +# ----------------------------------------------- +# --- Relay Host Credentials Section ------------ +# ----------------------------------------------- + +# Configure a relay user and password to use with RELAY_HOST / DEFAULT_RELAY_HOST + # empty => no default -# default relay username (if no specific entry exists in postfix-sasl-password.cf) RELAY_USER= # empty => no default -# password for default relay user RELAY_PASSWORD= diff --git a/target/scripts/check-for-changes.sh b/target/scripts/check-for-changes.sh index 2fa004b00c7..3f268710c7e 100755 --- a/target/scripts/check-for-changes.sh +++ b/target/scripts/check-for-changes.sh @@ -49,17 +49,7 @@ function _check_for_changes() { local CHANGED CHANGED=$(_get_changed_files "${CHKSUM_FILE}" "${CHKSUM_FILE}.new") - - # Handle any changes - _ssl_changes - _postfix_dovecot_changes - _rspamd_changes - - _log_with_date 'debug' 'Reloading services due to detected changes' - - [[ ${ENABLE_AMAVIS} -eq 1 ]] && _reload_amavis - _reload_postfix - [[ ${SMTP_ONLY} -ne 1 ]] && dovecot reload + _handle_changes _remove_lock _log_with_date 'debug' 'Completed handling of detected change' @@ -69,6 +59,29 @@ function _check_for_changes() { fi } +function _handle_changes() { + # Variable to identify any config updates dependent upon vhost changes. + local VHOST_UPDATED=0 + # These two configs are the source for /etc/postfix/vhost (managed mail domains) + if [[ ${CHANGED} =~ ${DMS_DIR}/postfix-(accounts|virtual).cf ]]; then + _log_with_date 'trace' 'Regenerating vhosts (Postfix)' + # Regenerate via `helpers/postfix.sh`: + _create_postfix_vhost + + VHOST_UPDATED=1 + fi + + _ssl_changes + _postfix_dovecot_changes + _rspamd_changes + + _log_with_date 'debug' 'Reloading services due to detected changes' + + [[ ${ENABLE_AMAVIS} -eq 1 ]] && _reload_amavis + _reload_postfix + [[ ${SMTP_ONLY} -ne 1 ]] && dovecot reload +} + function _get_changed_files() { local CHKSUM_CURRENT=${1} local CHKSUM_NEW=${2} @@ -85,9 +98,9 @@ function _get_changed_files() { } function _reload_amavis() { - if [[ ${CHANGED} =~ ${DMS_DIR}/postfix-accounts.cf ]] || [[ ${CHANGED} =~ ${DMS_DIR}/postfix-virtual.cf ]]; then - # /etc/postfix/vhost was updated, amavis must refresh it's config by - # reading this file again in case of new domains, otherwise they will be ignored. + # /etc/postfix/vhost was updated, amavis must refresh it's config by + # reading this file again in case of new domains, otherwise they will be ignored. + if [[ ${VHOST_UPDATED} -eq 1 ]]; then amavisd reload fi } @@ -114,14 +127,12 @@ function _postfix_dovecot_changes() { # - postfix-sasl-password.cf used by _relayhost_sasl # - _populate_relayhost_map relies on: # - postfix-relaymap.cf - # - postfix-accounts.cf + postfix-virtual.cf (both will be dropped in future) - if [[ ${CHANGED} =~ ${DMS_DIR}/postfix-accounts.cf ]] \ - || [[ ${CHANGED} =~ ${DMS_DIR}/postfix-virtual.cf ]] \ + if [[ ${VHOST_UPDATED} -eq 1 ]] \ || [[ ${CHANGED} =~ ${DMS_DIR}/postfix-relaymap.cf ]] \ || [[ ${CHANGED} =~ ${DMS_DIR}/postfix-sasl-password.cf ]] then _log_with_date 'trace' 'Regenerating relay config (Postfix)' - _rebuild_relayhost + _process_relayhost_configs fi # Regenerate system + virtual account aliases via `helpers/aliases.sh`: @@ -129,14 +140,6 @@ function _postfix_dovecot_changes() { [[ ${CHANGED} =~ ${DMS_DIR}/postfix-regexp.cf ]] && _handle_postfix_regexp_config [[ ${CHANGED} =~ ${DMS_DIR}/postfix-aliases.cf ]] && _handle_postfix_aliases_config - # Regenerate `/etc/postfix/vhost` (managed mail domains) via `helpers/postfix.sh`: - if [[ ${CHANGED} =~ ${DMS_DIR}/postfix-accounts.cf ]] \ - || [[ ${CHANGED} =~ ${DMS_DIR}/postfix-virtual.cf ]] - then - _log_with_date 'trace' 'Regenerating vhosts (Postfix)' - _create_postfix_vhost - fi - # Legacy workaround handled here, only seems necessary for _create_accounts: # - `helpers/accounts.sh` logic creates folders/files with wrong ownership. _chown_var_mail_if_necessary diff --git a/target/scripts/helpers/postfix.sh b/target/scripts/helpers/postfix.sh index 5fa4fa83313..0bb1c15898d 100644 --- a/target/scripts/helpers/postfix.sh +++ b/target/scripts/helpers/postfix.sh @@ -43,21 +43,28 @@ function _vhost_collect_postfix_domains() { local DATABASE_VIRTUAL='/tmp/docker-mailserver/postfix-virtual.cf' local DOMAIN UNAME - # getting domains FROM mail accounts + # Extract domains from mail accounts: if [[ -f ${DATABASE_ACCOUNTS} ]]; then - while IFS=$'|' read -r LOGIN _; do - DOMAIN=$(echo "${LOGIN}" | cut -d @ -f2) + while IFS=$'|' read -r MAIL_ACCOUNT _; do + # It is expected valid lines have the format local-part@domain-part: + DOMAIN=$(cut -d '@' -f 2 <<< "${MAIL_ACCOUNT}") + echo "${DOMAIN}" >>"${TMP_VHOST}" done < <(_get_valid_lines_from_file "${DATABASE_ACCOUNTS}") fi - # getting domains FROM mail aliases + # TODO: Consider if virtual aliases should be configured to the same vhost file: + # https://github.com/docker-mailserver/docker-mailserver/issues/2813#issuecomment-1272394563 + # Extract domains from virtual alias config: + # Aliases may have the forms: 'local-part@domain-part', only 'local-part', or '@domain-part' (wildcard catch-all) if [[ -f ${DATABASE_VIRTUAL} ]]; then - while read -r FROM _; do - UNAME=$(echo "${FROM}" | cut -d @ -f1) - DOMAIN=$(echo "${FROM}" | cut -d @ -f2) + while read -r ALIAS_FIELD _; do + UNAME=$(cut -d '@' -f 1 <<< "${ALIAS_FIELD}") + DOMAIN=$(cut -d '@' -f 2 <<< "${ALIAS_FIELD}") - # if they are equal it means the line looks like: "user1 other@domain.tld" + # Only add valid domain-parts found: + # The '@' is optional for an alias key (eg: "user1 other@domain.tld"), + # but cut with -f2 would output the same value as it would -f1 when '@' is missing. [[ ${UNAME} != "${DOMAIN}" ]] && echo "${DOMAIN}" >>"${TMP_VHOST}" done < <(_get_valid_lines_from_file "${DATABASE_VIRTUAL}") fi diff --git a/target/scripts/helpers/relay.sh b/target/scripts/helpers/relay.sh index de29dddc7a6..b92da3b42c4 100644 --- a/target/scripts/helpers/relay.sh +++ b/target/scripts/helpers/relay.sh @@ -4,9 +4,9 @@ # Description: # This helper is responsible for configuring outbound SMTP (delivery) through relay-hosts. # -# When mail is sent from Postfix, it is considered relaying to that destination (or the next hop). -# By default delivery external of the container would be direct to the MTA of the recipient address (destination). -# Alternatively mail can be indirectly delivered to the destination by routing through a different MTA (relay-host service). +# When mail is sent to Postfix and the destination is not a domain DMS manages, this requires relaying to that destination (or the next hop). +# By default outbound mail delivery would be direct to the MTA of the recipient address (destination). +# Alternatively mail can be delivered indirectly to that destination by routing through a different MTA (relay-host service). # # This helper is only concerned with relaying mail from authenticated submission (ports 587 + 465). # Thus it does not deal with `relay_domains` (which routes through `relay_transport` transport, default: `master.cf:relay`), @@ -37,22 +37,23 @@ # `postfix reload` or `supervisorctl restart postfix` should be run to properly apply config (which it is). # Otherwise use another table type such as `hash` and run `postmap` on the table after modification. # -# WARNING: Databases (tables above) are rebuilt during change detection. There is a minor chance of -# a lookup occurring during a rebuild of these files that may affect or delay delivery? +# WARNING: Databases (tables above) are rebuilt during change detection. +# There is a minor chance of a lookup occurring during a rebuild of these files that may affect or delay delivery? # TODO: Should instead perform an atomic operation with a temporary file + `mv` to replace? # Or switch back to using `hash` table type if plaintext access is not needed (unless retaining file for postmap). # Either way, plaintext copy is likely accessible if using our supported configs for providing them to the container. -# NOTE: Present support has enforced wrapping the relay host with `[]` (prevents DNS MX record lookup), -# which restricts what is supported by RELAY_HOST, although you usually do want to provide MX host directly. -# NOTE: Present support expects to always append a port with an implicit default of `25`. -# NOTE: DEFAULT_RELAY_HOST imposes neither restriction. +# NOTE: Present support has enforced wrapping the `RELAY_HOST` value with `[]` (prevents DNS MX record lookup), +# shouldn't be an issue as you typically do want to provide the MX host directly? This was presumably for config convenience. +# NOTE: Present support expects to always append a port (_with an implicit default of `25`_). +# NOTE: The `DEFAULT_RELAY_HOST` ENV imposes neither restriction. # -# TODO: RELAY_PORT should be optional, it will use the transport default port (`postconf smtp_tcp_port`), +# TODO: `RELAY_PORT` should be optional (Postfix would fallback to the transports default port (`postconf smtp_tcp_port`), # That shouldn't be a breaking change, as long as the mapping is maintained correctly. -# TODO: RELAY_HOST should consider dropping `[]` and require the user to include that? -# Future refactor for _populate_relayhost_map may warrant dropping these two ENV in favor of DEFAULT_RELAY_HOST? +# TODO: `RELAY_HOST` should consider dropping the implicit `[]` and require the user to include that? +# +# A future refactor of `_populate_relayhost_map()` may warrant dropping those two ENV in favor of `DEFAULT_RELAY_HOST`? function _env_relay_host() { echo "[${RELAY_HOST}]:${RELAY_PORT:-25}" } @@ -60,10 +61,12 @@ function _env_relay_host() { # Responsible for `postfix-sasl-password.cf` support: # `/etc/postfix/sasl_passwd` example at end of file. function _relayhost_sasl() { - if [[ ! -f /tmp/docker-mailserver/postfix-sasl-password.cf ]] \ - && [[ -z ${RELAY_USER} || -z ${RELAY_PASSWORD} ]] - then - _log 'warn' "Missing relay-host mapped credentials provided via ENV, or from postfix-sasl-password.cf" + local DATABASE_SASL_PASSWD='/tmp/docker-mailserver/postfix-sasl-password.cf' + + # Only relevant when required credential sources are provided: + if [[ ! -f ${DATABASE_SASL_PASSWD} ]] \ + && [[ -z ${RELAY_USER} || -z ${RELAY_PASSWORD} ]]; then + _log 'warn' "Missing relay-host mapped credentials provided via ENV, or from ${DATABASE_SASL_PASSWD}" return 1 fi @@ -74,7 +77,6 @@ function _relayhost_sasl() { chown root:root /etc/postfix/sasl_passwd chmod 0600 /etc/postfix/sasl_passwd - local DATABASE_SASL_PASSWD='/tmp/docker-mailserver/postfix-sasl-password.cf' if [[ -f ${DATABASE_SASL_PASSWD} ]]; then # Add domain-specific auth from config file: _get_valid_lines_from_file "${DATABASE_SASL_PASSWD}" >> /etc/postfix/sasl_passwd @@ -83,90 +85,93 @@ function _relayhost_sasl() { postconf 'smtp_sender_dependent_authentication = yes' fi - # Add an authenticated relay host defined via ENV config: - if [[ -n ${RELAY_USER} ]] && [[ -n ${RELAY_PASSWORD} ]]; then - echo "$(_env_relay_host) ${RELAY_USER}:${RELAY_PASSWORD}" >> /etc/postfix/sasl_passwd + # Support authentication to a primary relayhost (when configured with credentials via ENV): + if [[ -n ${DEFAULT_RELAY_HOST} || -n ${RELAY_HOST} ]] \ + && [[ -n ${RELAY_USER} && -n ${RELAY_PASSWORD} ]]; then + echo "${DEFAULT_RELAY_HOST:-$(_env_relay_host)} ${RELAY_USER}:${RELAY_PASSWORD}" >>/etc/postfix/sasl_passwd fi - # Technically if only a single relay host is configured, a `static` lookup table could be used instead?: - # postconf "smtp_sasl_password_maps = static:${RELAY_USER}:${RELAY_PASSWORD}" - postconf 'smtp_sasl_password_maps = texthash:/etc/postfix/sasl_passwd' + # Enable credential lookup + SASL authentication to relayhost: + # - `noanonymous` enforces authentication requirement + # - `encrypt` enforces requirement for a secure connection (prevents sending credentials over cleartext, aka mandatory TLS) + postconf \ + 'smtp_sasl_password_maps = texthash:/etc/postfix/sasl_passwd' \ + 'smtp_sasl_auth_enable = yes' \ + 'smtp_sasl_security_options = noanonymous' \ + 'smtp_tls_security_level = encrypt' } # Responsible for `postfix-relaymap.cf` support: # `/etc/postfix/relayhost_map` example at end of file. # -# Present support uses a table lookup for sender address or domain mapping to relay-hosts, -# Populated via `postfix-relaymap.cf `, which also features a non-standard way to exclude implicitly added internal domains from the feature. -# It also maps all known sender domains (from configs postfix-accounts + postfix-virtual.cf) to the same ENV configured relay-host. +# `postfix-relaymap.cf` represents table syntax expected for `/etc/postfix/relayhost_map`, except that it adds an opt-out parsing feature. +# All known mail domains managed by DMS (/etc/postfix/vhost) are implicitly configured to use `RELAY_HOST` + `RELAY_PORT` as the default relay. +# This approach is effectively equivalent to using `main.cf:relayhost`, but with an excessive workaround to support the explicit opt-out feature. # -# TODO: The account + virtual config parsing and appending to /etc/postfix/relayhost_map seems to be an excessive `main.cf:relayhost` -# implementation, rather than leveraging that for the same purpose and selectively overriding only when needed with `/etc/postfix/relayhost_map`. -# If the issue was to opt-out select domains, if avoiding a default relay-host was not an option, then mapping those sender domains or addresses -# to a separate transport (which can drop the `relayhost` setting) would be more appropriate. -# TODO: With `sender_dependent_default_transport_maps`, we can extract out the excluded domains and route them through a separate transport. -# while deprecating that support in favor of a transport config, similar to what is offered currently via sasl_passwd and relayhost_map. +# TODO: Refactor this feature support so that in `main.cf`: +# - Relay all outbound mail through an external MTA by default (works without credentials): +# `relayhost = ${DEFAULT_RELAY_HOST}` +# - Opt-in to relaying - Selectively relay outbound mail by sender/domain to an external MTA (relayhost can vary): +# `sender_dependent_relayhost_maps = texthash:/etc/postfix/relayhost_map` +# - Opt-out from relaying - Selectively prevent outbound mail from relaying via separate transport mappings (where relayhost is not configured): +# By sender: `sender_dependent_default_transport_maps = texthash:/etc/postfix/sender_transport_map` (the current opt-out feature could utilize this instead) +# By recipient (has precedence): `transport_maps = texthash:/etc/postfix/recipient_transport_map` +# +# Support for relaying via port 465 or equivalent requires additional config support (as needed for 465 vs 587 transports extending smtpd) +# - Default relay transport is configured by `relay_transport`, with default transport port configured by `smtp_tcp_port`. +# - The `relay` transport itself extends from `smtp` transport. More than one can be configured with separate settings via `master.cf`. + function _populate_relayhost_map() { # Create the relayhost_map config file: : >/etc/postfix/relayhost_map chown root:root /etc/postfix/relayhost_map chmod 0600 /etc/postfix/relayhost_map - # Matches lines that are not comments or only white-space: - local MATCH_VALID='^\s*[^#[:space:]]' + _multiple_relayhosts + _legacy_support - # This config is mostly compatible with `/etc/postfix/relayhost_map`, but additionally supports - # not providing a relay host for a sender domain to opt-out of RELAY_HOST? (2nd half of function) - if [[ -f /tmp/docker-mailserver/postfix-relaymap.cf ]]; then - _log 'trace' "Adding relay mappings from postfix-relaymap.cf" + postconf 'sender_dependent_relayhost_maps = texthash:/etc/postfix/relayhost_map' +} +function _multiple_relayhosts() { + if [[ -f ${DATABASE_RELAYHOSTS} ]]; then + _log 'trace' "Adding relay mappings from ${DATABASE_RELAYHOSTS}" + + # Matches lines that are not comments or only white-space: + local MATCH_VALID='^\s*[^#[:space:]]' # Match two values with some white-space between them (eg: `@example.test [relay.service.test]:465`): local MATCH_VALUE_PAIR='\S*\s+\S' - # Copy over lines which are not a comment *and* have a destination. - sed -n -r "/${MATCH_VALID}${MATCH_VALUE_PAIR}/p" /tmp/docker-mailserver/postfix-relaymap.cf >>/etc/postfix/relayhost_map + # Copy over lines which are not a comment *and* have a relay destination. + # Extra condition is due to legacy support (due to opt-out feature), otherwise `_get_valid_lines_from_file()` would be valid. + sed -n -r "/${MATCH_VALID}${MATCH_VALUE_PAIR}/p" "${DATABASE_RELAYHOSTS}" >> /etc/postfix/relayhost_map fi +} - # Everything below here is to parse `postfix-accounts.cf` and `postfix-virtual.cf`, - # extracting out the domain parts (value of email address after `@`), and then - # adding those as mappings to ENV configured RELAY_HOST for lookup in `/etc/postfix/relayhost_map`. - # Provided `postfix-relaymap.cf` didn't exclude any of the domains, - # and they don't already exist within `/etc/postfix/relayhost_map`. - # - # TODO: Breaking change. Replace this lower half and remove the opt-out feature from `postfix-relaymap.cf`. - # Leverage `main.cf:relayhost` for setting a default relayhost as it was prior to this feature addition. - # Any sender domains or addresses that need to opt-out of that default relay-host can either - # map to a different relay-host, or use a separate transport (needs feature support added). - - # Args: - function _list_domain_parts() { - [[ -f $2 ]] && sed -n -r "/${MATCH_VALID}/ ${1}" "${2}" - } - # Matches and outputs (capture group via `/\1/p`) the domain part (value of address after `@`) in the config file. - local PRINT_DOMAIN_PART_ACCOUNTS='s/^[^@|]*@([^\|]+)\|.*$/\1/p' - local PRINT_DOMAIN_PART_VIRTUAL='s/^\s*[^@[:space:]]*@(\S+)\s.*/\1/p' - - { - _list_domain_parts "${PRINT_DOMAIN_PART_ACCOUNTS}" /tmp/docker-mailserver/postfix-accounts.cf - _list_domain_parts "${PRINT_DOMAIN_PART_VIRTUAL}" /tmp/docker-mailserver/postfix-virtual.cf - } | sort -u | while read -r DOMAIN_PART; do - # DOMAIN_PART not already present in `/etc/postfix/relayhost_map`, and not listed as a relay opt-out domain in `postfix-relaymap.cf` - # `^@${DOMAIN_PART}\b` - To check for existing entry, the `\b` avoids accidental partial matches on similar domain parts. - # `^\s*@${DOMAIN_PART}\s*$` - Matches line with only a domain part (eg: @example.test) to avoid including a mapping for those domains to the RELAY_HOST. - if ! grep -q -e "^@${DOMAIN_PART}\b" /etc/postfix/relayhost_map && ! grep -qs -e "^\s*@${DOMAIN_PART}\s*$" /tmp/docker-mailserver/postfix-relaymap.cf; then - _log 'trace' "Adding relay mapping for ${DOMAIN_PART}" - echo "@${DOMAIN_PART} $(_env_relay_host)" >> /etc/postfix/relayhost_map - fi - done +# Implicitly force configure all domains DMS manages to be relayed that haven't yet been configured or provided an explicit opt-out. +# This would normally be handled via an opt-in approach, or through `main.cf:relayhost` with an opt-out approach (sender_dependent_default_transport_maps) +function _legacy_support() { + local DATABASE_VHOST='/etc/postfix/vhost' - postconf 'sender_dependent_relayhost_maps = texthash:/etc/postfix/relayhost_map' -} + # Only relevant when `RELAY_HOST` is configured: + [[ -z ${RELAY_HOST} ]] && return 1 -function _relayhost_configure_postfix() { - postconf \ - 'smtp_sasl_auth_enable = yes' \ - 'smtp_sasl_security_options = noanonymous' \ - 'smtp_tls_security_level = encrypt' + # Configures each `SENDER_DOMAIN` to send outbound mail through the default `RELAY_HOST` + `RELAY_PORT` + # (by adding an entry in `/etc/postfix/relayhost_map`) provided it: + # - `/etc/postfix/relayhost_map` doesn't already have it as an existing entry. + # - `postfix-relaymap.cf` has no explicit opt-out (SENDER_DOMAIN key exists, but with no relayhost value assigned) + # + # NOTE: /etc/postfix/vhost represents managed mail domains sourced from `postfix-accounts.cf` and `postfix-virtual.cf`. + while read -r SENDER_DOMAIN; do + local MATCH_EXISTING_ENTRY="^@${SENDER_DOMAIN}\s+" + local MATCH_OPT_OUT_LINE="^\s*@${SENDER_DOMAIN}\s*$" + + # NOTE: `-E` is required for `\s+` syntax to avoid escaping `+` + if ! grep -q -E "${MATCH_EXISTING_ENTRY}" /etc/postfix/relayhost_map && ! grep -qs "${MATCH_OPT_OUT_LINE}" "${DATABASE_RELAYHOSTS}"; then + _log 'trace' "Configuring '${SENDER_DOMAIN}' for the default relayhost '${RELAY_HOST}'" + echo "@${SENDER_DOMAIN} $(_env_relay_host)" >> /etc/postfix/relayhost_map + fi + done < <(_get_valid_lines_from_file "${DATABASE_VHOST}") } function _setup_relayhost() { @@ -177,24 +182,24 @@ function _setup_relayhost() { postconf "relayhost = ${DEFAULT_RELAY_HOST}" fi - if [[ -n ${RELAY_HOST} ]]; then - _log 'trace' "Setting up relay hosts (default: ${RELAY_HOST})" + _process_relayhost_configs +} - _relayhost_sasl - _populate_relayhost_map +# Called during initial container setup, or by change detection event: +function _process_relayhost_configs() { + local DATABASE_RELAYHOSTS='/tmp/docker-mailserver/postfix-relaymap.cf' - _relayhost_configure_postfix + # One of these must configure a relayhost for the feature to relevant: + if [[ ! -f ${DATABASE_RELAYHOSTS} ]] \ + && [[ -z ${DEFAULT_RELAY_HOST} ]] \ + && [[ -z ${RELAY_HOST} ]]; then + return 1 fi -} -function _rebuild_relayhost() { - if [[ -n ${RELAY_HOST} ]]; then - _relayhost_sasl - _populate_relayhost_map - fi + _relayhost_sasl + _populate_relayhost_map } - # # Config examples for reference # From d65b2f35a7122ce2a9620c6179e59e540e2ff347 Mon Sep 17 00:00:00 2001 From: Casper Date: Tue, 30 Jan 2024 23:04:42 +0100 Subject: [PATCH 288/592] chore: `CHANGELOG.md` - Add `rsyslog` breaking changes for v14 (#3854) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f426a5bc42..29da7935433 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,14 @@ The most noteworthy change of this release is the update of the container's base - DMS `main.cf` has renamed `postscreen_dnsbl_whitelist_threshold` to `postscreen_dnsbl_allowlist_threshold` as part of this change. - `smtpd_relay_restrictions` (relay policy) is now evaluated after `smtpd_recipient_restrictions` (spam policy). Previously it was evaluated before `smtpd_recipient_restrictions`. Mail to be relayed via DMS must now pass through the spam policy first. - The TLS fingerprint policy has changed the default from MD5 to SHA256 (_DMS does not modify this Postfix parameter, but may affect any user customizations that do_). +- **rsyslog:** + - Debian 12 adjusted the `rsyslog` configuration for the default file template from `RSYSLOG_TraditionalFileFormat` to `RSYSLOG_FileFormat` (_upstream default since 2012_). This change may affect you if you have any monitoring / analysis of log output (_eg: `mail.log` / `docker logs`_). + - The two formats are roughly equivalent to [RFC 3164](https://www.rfc-editor.org/rfc/rfc3164)) and [RFC 5424](https://datatracker.ietf.org/doc/html/rfc5424#section-1) respectively. + - A notable difference is the change to [RFC 3339](https://www.rfc-editor.org/rfc/rfc3339.html#appendix-A) timestamps (_a strict subset of ISO 8601_). The [previous non-standardized timestamp format was defined in RFC 3164](https://www.rfc-editor.org/rfc/rfc3164.html#section-4.1.2) as `Mmm dd hh:mm:ss`. + - To revert this change you can add `sedfile -i '1i module(load="builtin:omfile" template="RSYSLOG_TraditionalFileFormat")' /etc/rsyslog.conf` via [our `user-patches.sh` feature](https://docker-mailserver.github.io/docker-mailserver/v14.0/config/advanced/override-defaults/user-patches/). + - Rsyslog now creates fewer log files: + - The files `/var/log/mail/mail.{info,warn,err}` are no longer created. These files represented `/var/log/mail.log` filtered into separate priority levels. As `/var/log/mail.log` contains all mail related messages, these files (_and their rotated counterparts_) can be deleted safely. + - `/var/log/messages`, `/var/log/debug` and several other log files not relevant to DMS were configured by default by Debian previously. These are not part of the `/var/log/mail/` volume mount, so should not impact anyone. - **Features:** - The relay host feature was refactored ([#3845](https://github.com/docker-mailserver/docker-mailserver/pull/3845)) - The only breaking change this should introduce is with the Change Detection service (`check-for-changes.sh`). From d426f724cd1f2a7474ad46233eea13df5fe5ea42 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 31 Jan 2024 23:11:19 +1300 Subject: [PATCH 289/592] docs: Complete rewrite of Relay Host pages (#3861) * docs: Complete rewrite on relay host docs - Both relay docs pages have had heavy refactor / rewrite. - ENV docs page relay host section revised. * docs: Revise relay host page with technical details section * docs: Add LDAP compatibility caveat for `RELAY_HOST` --- .../advanced/mail-forwarding/aws-ses.md | 53 ++++-- .../advanced/mail-forwarding/relay-hosts.md | 175 +++++++++++++----- docs/content/config/environment.md | 52 ++++-- docs/mkdocs.yml | 10 +- 4 files changed, 203 insertions(+), 87 deletions(-) diff --git a/docs/content/config/advanced/mail-forwarding/aws-ses.md b/docs/content/config/advanced/mail-forwarding/aws-ses.md index 2a4dab170d9..00ca2a13908 100644 --- a/docs/content/config/advanced/mail-forwarding/aws-ses.md +++ b/docs/content/config/advanced/mail-forwarding/aws-ses.md @@ -2,29 +2,46 @@ title: 'Mail Forwarding | AWS SES' --- -[Amazon SES (Simple Email Service)](https://aws.amazon.com/ses/) is intended to provide a simple way for cloud based applications to send email and receive email. For the purposes of this project only sending email via SES is supported. Older versions of docker-mailserver used `AWS_SES_HOST` and `AWS_SES_USERPASS` to configure sending, this has changed and the setup is managed through [Configure Relay Hosts][docs-relay]. +[Amazon SES (Simple Email Service)][aws-ses] provides a simple way for cloud based applications to send and receive email. -You will need to create some [Amazon SES SMTP credentials](https://docs.aws.amazon.com/ses/latest/DeveloperGuide/smtp-credentials.html). The SMTP credentials you create will be used to populate the `RELAY_USER` and `RELAY_PASSWORD` environment variables. +!!! example "Configuration via ENV" -The `RELAY_HOST` should match your [AWS SES region](https://docs.aws.amazon.com/general/latest/gr/ses.html), the `RELAY_PORT` will be 587. + [Configure a relay host in DMS][docs::relay] to forward all your mail through AWS SES: -If all of your email is being forwarded through AWS SES, `DEFAULT_RELAY_HOST` should be set accordingly. + - `RELAY_HOST` should match your [AWS SES region][aws-ses::region]. + - `RELAY_PORT` should be set to [one of the supported AWS SES SMTP ports][aws-ses::smtp-ports] (_eg: 587 for STARTTLS_). + - `RELAY_USER` and `RELAY_PASSWORD` should be set to your [Amazon SES SMTP credentials][aws-ses::credentials]. -Example: -``` -DEFAULT_RELAY_HOST=[email-smtp.us-west-2.amazonaws.com]:587 -``` + ```env + RELAY_HOST=email-smtp.us-west-2.amazonaws.com + RELAY_PORT=587 + # Alternative to RELAY_HOST + RELAY_PORT which is compatible with LDAP: + DEFAULT_RELAY_HOST=[email-smtp.us-west-2.amazonaws.com]:587 -!!! note - If you set up [AWS Easy DKIM](https://docs.aws.amazon.com/ses/latest/DeveloperGuide/send-email-authentication-dkim-easy.html) you can safely skip setting up DKIM as the AWS SES will take care of signing your outgoing email. + RELAY_USER=aws-user + RELAY_PASSWORD=secret + ``` -To verify proper operation, send an email to some external account of yours and inspect the mail headers. You will also see the connection to SES in the mail logs. For example: +!!! tip -```log -May 23 07:09:36 mail postfix/smtp[692]: Trusted TLS connection established to email-smtp.us-east-1.amazonaws.com[107.20.142.169]:25: -TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits) -May 23 07:09:36 mail postfix/smtp[692]: 8C82A7E7: to=, relay=email-smtp.us-east-1.amazonaws.com[107.20.142.169]:25, -delay=0.35, delays=0/0.02/0.13/0.2, dsn=2.0.0, status=sent (250 Ok 01000154dc729264-93fdd7ea-f039-43d6-91ed-653e8547867c-000000) -``` + If you have set up [AWS Easy DKIM][aws-ses::easy-dkim], you can safely skip setting up DKIM as AWS SES will take care of signing your outbound mail. -[docs-relay]: ./relay-hosts.md +!!! note "Verify the relay host is configured correctly" + + To verify proper operation, send an email to some external account of yours and inspect the mail headers. + + You will also see the connection to SES in the mail logs: + + ```log + postfix/smtp[692]: Trusted TLS connection established to email-smtp.us-west-1.amazonaws.com[107.20.142.169]:25: + TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits) + postfix/smtp[692]: 8C82A7E7: to=, relay=email-smtp.us-west-1.amazonaws.com[107.20.142.169]:25, + delay=0.35, delays=0/0.02/0.13/0.2, dsn=2.0.0, status=sent (250 Ok 01000154dc729264-93fdd7ea-f039-43d6-91ed-653e8547867c-000000) + ``` + +[docs::relay]: ./relay-hosts.md +[aws-ses]: https://aws.amazon.com/ses/ +[aws-ses::credentials]: https://docs.aws.amazon.com/ses/latest/dg/smtp-credentials.html +[aws-ses::smtp-ports]: https://docs.aws.amazon.com/ses/latest/dg/smtp-connect.html +[aws-ses::region]: https://docs.aws.amazon.com/general/latest/gr/ses.html +[aws-ses::easy-dkim]: https://docs.aws.amazon.com/ses/latest/DeveloperGuide/send-email-authentication-dkim-easy.html diff --git a/docs/content/config/advanced/mail-forwarding/relay-hosts.md b/docs/content/config/advanced/mail-forwarding/relay-hosts.md index db2528e5ba3..7ef8238c442 100644 --- a/docs/content/config/advanced/mail-forwarding/relay-hosts.md +++ b/docs/content/config/advanced/mail-forwarding/relay-hosts.md @@ -2,82 +2,155 @@ title: 'Mail Forwarding | Relay Hosts' --- -## Introduction +## What is a Relay Host? -Rather than having Postfix deliver mail directly, you can configure Postfix to send mail via another mail relay (smarthost). Examples include [Mailgun](https://www.mailgun.com/), [Sendgrid](https://sendgrid.com/) and [AWS SES](https://aws.amazon.com/ses/). +An SMTP relay service (_aka relay host / [smarthost][wikipedia::smarthost]_) is an MTA that relays (_forwards_) mail on behalf of third-parties (_it does not manage the mail domains_). -Depending on the domain of the sender, you may want to send via a different relay, or authenticate in a different way. +- Instead of DMS handling SMTP delivery directly itself (_via Postfix_), it can be configured to delegate delivery by sending all outbound mail through a relay service. +- Examples of popular mail relay services: [AWS SES][smarthost::aws-ses], [Mailgun][smarthost::mailgun], [Mailjet][smarthost::mailjet], [SendGrid][smarthost::sendgrid] -## Basic Configuration +!!! info "When can a relay service can be helpful?" -Basic configuration is done via environment variables: + - Your network provider has blocked outbound connections on port 25 (_required for direct delivery_). + - To improve delivery success via better established reputation (trust) of a relay service. -- `RELAY_HOST`: _default host to relay mail through, `empty` (aka '', or no ENV set) will disable this feature_ -- `RELAY_PORT`: _port on default relay, defaults to port 25_ -- `RELAY_USER`: _username for the default relay_ -- `RELAY_PASSWORD`: _password for the default user_ +## Configuration -Setting these environment variables will cause mail for all sender domains to be routed via the specified host, authenticating with the user/password combination. +All mail sent outbound from DMS (_where the sender address is a DMS account or a virtual alias_) will be relayed through the configured relay host. -!!! warning - For users of the previous `AWS_SES_*` variables: please update your configuration to use these new variables, no other configuration is required. +!!! info "Configuration via ENV" -## Advanced Configuration + Configure the default relayhost with either of these ENV: -### Sender-dependent Authentication + - Preferable (_LDAP compatible_): `DEFAULT_RELAY_HOST` (eg: `[mail.relay-service.com]:25`) + - `RELAY_HOST` (eg: `mail.relay-service.com`) + `RELAY_PORT` (default: 25) -Sender dependent authentication is done in `docker-data/dms/config/postfix-sasl-password.cf`. You can create this file manually, or use: + Most relay services also require authentication configured: -```sh -setup.sh relay add-auth [] -``` + - `RELAY_USER` + `RELAY_PASSWORD` provides credentials for authenticating with the default relayhost. -An example configuration file looks like this: + !!! warning "Providing secrets via ENV" -```txt -@domain1.com relay_user_1:password_1 -@domain2.com relay_user_2:password_2 -``` + While ENV is convenient, the risk of exposing secrets is higher. -If there is no other configuration, this will cause Postfix to deliver email through the relay specified in `RELAY_HOST` env variable, authenticating as `relay_user_1` when sent from `domain1.com` and authenticating as `relay_user_2` when sending from `domain2.com`. + `setup relay add-auth` is a better alternative, which manages the credentials via a config file. -!!! note - To activate the configuration you must either restart the container, or you can also trigger an update by modifying a mail account. +??? tip "Excluding specific sender domains from relay" -### Sender-dependent Relay Host + You can opt-out with: `setup relay exclude-domain ` -Sender dependent relay hosts are configured in `docker-data/dms/config/postfix-relaymap.cf`. You can create this file manually, or use: + Outbound mail from senders of that domain will be sent normally (_instead of through the configured `RELAY_HOST`_). -```sh -setup.sh relay add-domain [] -``` + !!! warning "When any relay host credentials are configured" -An example configuration file looks like this: + It will still be expected that mail is sent over a secure connection with credentials provided. -```txt -@domain1.com [relay1.org]:587 -@domain2.com [relay2.org]:2525 -``` + Thus this opt-out feature is rarely practical. -Combined with the previous configuration in `docker-data/dms/config/postfix-sasl-password.cf`, this will cause Postfix to deliver mail sent from `domain1.com` via `relay1.org:587`, authenticating as `relay_user_1`, and mail sent from `domain2.com` via `relay2.org:2525` authenticating as `relay_user_2`. +### Advanced Configuration -!!! note - You still have to define `RELAY_HOST` to activate the feature +When mail is sent, there is support to change the relay service or the credentials configured based on the sender address domain used. -### Excluding Sender Domains +We provide this support via two config files: -If you want mail sent from some domains to be delivered directly, you can exclude them from being delivered via the default relay by adding them to `docker-data/dms/config/postfix-relaymap.cf` with no destination. You can also do this via: +- Sender-dependent Relay Host: `docker-data/dms/config/postfix-relaymap.cf` +- Sender-dependent Authentication: `docker-data/dms/config/postfix-sasl-password.cf` -```sh -setup.sh relay exclude-domain -``` +!!! tip "Configure with our `setup relay` commands" -Extending the configuration file from above: + While you can edit those configs directly, DMS provides these helpful config management commands: -```txt -@domain1.com [relay1.org]:587 -@domain2.com [relay2.org]:2525 -@domain3.com -``` + ```cli-syntax + # Configure a sender domain to use a specific relay host: + setup relay add-domain [] -This will cause email sent from `domain3.com` to be delivered directly. + # Configure relay host credentials for a sender domain to use: + setup relay add-auth [] + + # Optionally avoid relaying from senders of this domain: + # NOTE: Only supported when configured with the `RELAY_HOST` ENV! + setup relay exclude-domain + ``` + +!!! example "Config file: `postfix-sasl-password.cf`" + + ```cf-extra title="docker-data/dms/config/postfix-sasl-password.cf" + @domain1.com mailgun-user:secret + @domain2.com sendgrid-user:secret + + # NOTE: This must have an exact match with the relay host in `postfix-relaymap.cf`, + # `/etc/postfix/relayhost_map`, or the `DEFAULT_RELAY_HOST` ENV. + # NOTE: Not supported via our setup CLI, but valid config for Postfix. + [email-smtp.us-west-2.amazonaws.com]:2587 aws-user:secret + ``` + + When Postfix needs to lookup credentials for mail sent outbound, the above config will: + + - Authenticate as `mailgun-user` for mail sent with a sender belonging to `@domain1.com` + - Authenticate as `sendgrid-user` for mail sent with a sender belonging to `@domain2.com` + - Authenticate as `aws-user` for mail sent through a configured AWS SES relay host (any sender domain). + +!!! example "Config file: `postfix-relaymap.cf`" + + ```cf-extra title="docker-data/dms/config/postfix-relaymap.cf" + @domain1.com [smtp.mailgun.org]:587 + @domain2.com [smtp.sendgrid.net]:2525 + + # Opt-out of relaying: + @domain3.com + ``` + + When Postfix sends mail outbound from these sender domains, the above config will: + + - Relay mail through `[smtp.mailgun.org]:587` when mail is sent from a sender of `@domain1.com` + - Relay mail through `[smtp.sendgrid.net]:2525` when mail is sent from a sender of `@domain1.com` + - Mail with a sender from `@domain3.com` is not sent through a relay (_**Only applicable** when using `RELAY_HOST`_) + +### Technical Details + +- Both the supported ENV and config files for this feature have additional details covered in our ENV docs [Relay Host section][docs::env-relay]. +- For troubleshooting, a [minimal `compose.yaml` config with several DMS instances][dms-gh::relay-example] demonstrates this feature for local testing. +- [Subscribe to this tracking issue][dms-gh::pr-3607] for future improvements intended for this feature. + +!!! abstract "Postfix Settings" + + Internally this feature is implemented in DMS by [`relay.sh`][dms-repo::helpers-relay]. + + The `relay.sh` script manages configuring these Postfix settings: + + ```cf-extra + # Send all outbound mail through this relay service: + relayhost = [smtp.relay-service.com]:587 + + # Credentials to use: + smtp_sasl_password_maps = texthash:/etc/postfix/sasl_passwd + # Alternative table type examples which do not require a separate file: + #smtp_sasl_password_maps = static:john.doe@relay-service.com:secret + #smtp_sasl_password_maps = inline:{ [smtp.relay-service.com]:587=john.doe@relay-service.com:secret } + + ## Authentication support: + # Required to provide credentials to the relay service: + smtp_sasl_auth_enable = yes + # Enforces requiring credentials when sending mail outbound: + smtp_sasl_security_options = noanonymous + # Enforces a secure connection (TLS required) to the relay service: + smtp_tls_security_level = encrypt + + ## Support for advanced requirements: + # Relay service(s) to use instead of direct delivery for specific sender domains: + sender_dependent_relayhost_maps = texthash:/etc/postfix/relayhost_map + # Support credentials to a relay service(s) that vary by relay host used or sender domain: + smtp_sender_dependent_authentication = yes + ``` + + +[smarthost::mailgun]: https://www.mailgun.com/ +[smarthost::mailjet]: https://www.mailjet.com +[smarthost::sendgrid]: https://sendgrid.com/ +[smarthost::aws-ses]: https://aws.amazon.com/ses/ +[wikipedia::smarthost]: https://en.wikipedia.org/wiki/Smart_host + +[docs::env-relay]: ../../environment.md#relay-host +[dms-repo::helpers-relay]: https://github.com/docker-mailserver/docker-mailserver/blob/v14.0.0/target/scripts/helpers/relay.sh +[dms-gh::pr-3607]: https://github.com/docker-mailserver/docker-mailserver/issues/3607 +[dms-gh::relay-example]: https://github.com/docker-mailserver/docker-mailserver/issues/3842#issuecomment-1913380639 diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 5db1c2c3d6b..d04da022784 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -1014,13 +1014,19 @@ you to replace both instead of just the envelope sender. #### Relay Host -!!! tip "`RELAY_HOST` vs `DEFAULT_RELAY_HOST`" +Supported ENV for the [Relay Host][docs::relay-host] feature. - `DEFAULT_RELAY_HOST` is encouraged, but presently does not support sender domain opt-out (`setup relay exclude-domain`). +!!! note "Prefer `DEFAULT_RELAY_HOST` instead of `RELAY_HOST`" + + This is advised unless you need support for sender domain opt-out (via `setup relay exclude-domain`). + + The implementation for `RELAY_HOST` is not compatible with LDAP. !!! tip "Opt-in for relay host support" - If you only want to enable relay for specific sender domains, use can use opt-in via `setup relay add-domain`. + Enable relaying only for specific sender domains instead by using `setup relay add-domain`. + + **NOTE:** Presently there is a caveat when relay host credentials are configured (_which is incompatible with opt-in_). ##### DEFAULT_RELAY_HOST @@ -1033,22 +1039,24 @@ Configures a default relay host. !!! abstract "Technical Details" - Configures the Postfix `main.cf` setting: [`relayhost`][postfix-config::relayhost] + This ENV internally configures the Postfix `main.cf` setting: [`relayhost`][postfix-config::relayhost] ##### RELAY_HOST Configures a default relay host. -!!! info - - - This is a legacy ENV. It is however required for the opt-out feature of `postfix-relaymap.cf` to work. - - When configured, all known mail domains managed by DMS will be configured to relay outbound mail, just like `DEFAULT_RELAY_HOST`. - !!! note Expects a value like `mail.example.com`. Internally this will be wrapped to `[mail.example.com]`, so it should resolve to the MTA directly. - Do not use with `DEFAULT_RELAY_HOST`. `RELAY_HOST` has precedence as it is configured with `sender_dependent_relayhost_maps`. + !!! warning "Do not use with `DEFAULT_RELAY_HOST`" + + `RELAY_HOST` has precedence as it is configured with `sender_dependent_relayhost_maps`. + +!!! info + + - This is a legacy ENV. It is however required for the opt-out feature of `postfix-relaymap.cf` to work. + - Internal configuration however differs from `DEFAULT_RELAY_HOST`. !!! abstract "Technical Details" @@ -1057,6 +1065,8 @@ Configures a default relay host. - Postfix setting with config: [`sender_dependent_relayhost_maps = texthash:/etc/postfix/relayhost_map`][postfix-config::relayhost_maps] - DMS Config volume support via: `postfix-relaymap.cf` (_generates `/etc/postfix/relayhost_map`_) + All known mail domains managed by DMS will be configured to relay outbound mail to `RELAY_HOST` by adding them implicitly to `/etc/postfix/relayhost_map`, except for domains using the opt-out feature of `postfix-relaymap.cf`. + ##### RELAY_PORT Default => 25 @@ -1069,10 +1079,10 @@ Support for configuring a different port than 25 for `RELAY_HOST` to use. #### Relay Host Credentials -!!! warning "Configuring relay host credentials make outbound authentication mandatory" +!!! warning "Configuring relay host credentials enforces outbound authentication" Presently when `RELAY_USER` + `RELAY_PASSWORD` or `postfix-sasl-password.cf` are configured, all outbound mail traffic is configured to require a secure connection established and forbids the omission of credentials. - + Additional feature work is required to only enforce these requirements on mail sent through a configured relay host. ##### RELAY_USER @@ -1083,10 +1093,10 @@ Provide the credentials to use with `RELAY_HOST` or `DEFAULT_RELAY_HOST`. !!! tip "Alternative credentials config" - You may prefer to use `setup relay add-auth` to avoid exposure of secrets in ENV. + You may prefer to use `setup relay add-auth` to avoid risking ENV exposing secrets. - - With the CLI command you must provide each sender domain relay credentials. - - Alternatively manually edit `postfix-sasl-password.cf` with the correct relayhost entry (_`DEFAULT_RELAY_HOST` value or as defined in `/etc/postfix/relayhost_map`_) to provide credentials per relayhost configured. + - With the CLI command, you must provide relay credentials for each of your sender domains. + - Alternatively manually edit `postfix-sasl-password.cf` with the correct relayhost entry (_`DEFAULT_RELAY_HOST` value, or as defined in `/etc/postfix/relayhost_map`_) to provide credentials per relayhost configured. !!! abstract "Technical Details" @@ -1095,7 +1105,16 @@ Provide the credentials to use with `RELAY_HOST` or `DEFAULT_RELAY_HOST`. - Postfix setting with config: [`smtp_sasl_password_maps = texthash:/etc/postfix/sasl_passwd`][postfix-config::sasl_passwd] - DMS Config volume support via: `postfix-sasl-password.cf` (_generates `/etc/postfix/sasl_passwd`_) - This file has relay hosts that must match the `host:port` of `/etc/postfix/relayhost_map` or `main.cf:relayhost`. DMS support handles this for you. + --- + + When `postfix-sasl-password.cf` is present, DMS will copy it internally to `/etc/postfix/sasl_passwd`. + + - DMS provides support for mapping credentials by sender domain: + - Explicitly via `setup relay add-auth` (_creates / updates `postfix-sasl-password.cf`_). + - Implicitly via the relay ENV support (_configures all known DMS managed domains to use the relay ENV_). + - Credentials can be explicitly configured for specific relay hosts instead of sender domains: + - Add the exact relayhost value (`host:port` / `[host]:port`) from the generated `/etc/postfix/relayhost_map`, or `main.cf:relayhost` (`DEFAULT_RELAY_HOST`). + - `setup relay ...` is missing support, you must instead add these manually to `postfix-sasl-password.cf`. [docs-rspamd]: ./security/rspamd.md [docs-tls]: ./security/ssl.md @@ -1103,6 +1122,7 @@ Provide the credentials to use with `RELAY_HOST` or `DEFAULT_RELAY_HOST`. [docs-tls-manual]: ./security/ssl.md#bring-your-own-certificates [docs-tls-selfsigned]: ./security/ssl.md#self-signed-certificates [docs-accounts-quota]: ./user-management.md#quotas +[docs::relay-host]: ./advanced/mail-forwarding/relay-hosts.md [docs::dms-volumes-state]: ./advanced/optional-config.md#volumes-state [postfix-config::relayhost]: https://www.postfix.org/postconf.5.html#relayhost [postfix-config::relayhost_maps]: https://www.postfix.org/postconf.5.html#sender_dependent_relayhost_maps diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 15d2c3b2dcc..84366423fd6 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -89,6 +89,7 @@ markdown_extensions: emoji_index: !!python/name:material.extensions.emoji.twemoji emoji_generator: !!python/name:material.extensions.emoji.to_svg - pymdownx.highlight: + # Configures an alias (name) to a supported syntax (lang): extend_pygments_lang: - name: yml lang: yaml @@ -98,8 +99,13 @@ markdown_extensions: lang: cfg - name: env lang: properties - # Not helpful with Python Pygments lexer highlighting, but we might change to a JS highlighter in future - # Ideally, this type of codefence might also have word-wrap enabled (CSS: {white-space: pre-wrap}) + # A variant that sometimes has nicer syntax highlighting: + - name: cf-extra + lang: linuxconfig + - name: cli-syntax + lang: linuxconfig + # These formats aren't supported by Python Pygments lexer, + # but we use them when the context is appropriate. - name: log lang: shell-session - name: fetchmailrc From 05fbcf68890bbf8e3acd3214457af8f55f9d125e Mon Sep 17 00:00:00 2001 From: Aaron Spettl Date: Wed, 31 Jan 2024 11:50:58 +0100 Subject: [PATCH 290/592] fix(rspamd): Add missing comma to `local_networks` setting (#3862) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 1 + target/rspamd/local.d/options.inc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29da7935433..d80bbf8ab08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,7 @@ The most noteworthy change of this release is the update of the container's base - `DEFAULT_RELAY_HOST` ENV can now also use the `RELAY_USER` + `RELAY_PASSWORD` ENV for supplying credentials. - `RELAY_HOST` ENV no longer enforces configuring outbound SMTP to require credentials. Like `DEFAULT_RELAY_HOST` it can now configure a relay where credentials are optional. - Restarting DMS should not be required when configuring relay hosts without these ENV, but solely via `setup relay ...`, as change detection events now apply relevant Postfix setting changes for supporting credentials too. +- Rspamd configuration: Add a missing comma in `local_networks` so that all internal IP addresses are actually considered as internal ([#3862](https://github.com/docker-mailserver/docker-mailserver/pull/3862)) ## [v13.3.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.3.1) diff --git a/target/rspamd/local.d/options.inc b/target/rspamd/local.d/options.inc index 8755cfade77..8d67fe74baa 100644 --- a/target/rspamd/local.d/options.inc +++ b/target/rspamd/local.d/options.inc @@ -1,3 +1,3 @@ pidfile = false; soft_reject_on_timeout = true; -local_networks = "127.0.0.1/8, 10.0.0.0/8, 172.16.0.0/12 192.168.0.0/16"; +local_networks = "127.0.0.1/8, 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16"; From 45935f5fb81733ddf7a6afd1503e5b8fce30ef89 Mon Sep 17 00:00:00 2001 From: Hans-Cees Speel Date: Thu, 1 Feb 2024 17:34:33 +0100 Subject: [PATCH 291/592] rspamd: add neural module config (#3833) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 1 + docs/content/config/environment.md | 11 +++++ mailserver.env | 6 +++ target/rspamd/local.d/neural.conf | 42 +++++++++++++++++++ target/rspamd/local.d/neural_group.conf | 26 ++++++++++++ .../startup/setup.d/security/rspamd.sh | 19 ++++++++- target/scripts/startup/variables-stack.sh | 1 + 7 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 target/rspamd/local.d/neural.conf create mode 100644 target/rspamd/local.d/neural_group.conf diff --git a/CHANGELOG.md b/CHANGELOG.md index d80bbf8ab08..09afaba4bb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,7 @@ The most noteworthy change of this release is the update of the container's base - Refactored helper methods for sending e-mails with specific `Message-ID` headers and the helpers for retrieving + filtering logs, which together help isolate logs relevant to specific mail when multiple mails have been processed within a single test. ([#3786](https://github.com/docker-mailserver/docker-mailserver/pull/3786)) - **Rspamd**: - The `rewrite_subject` action, is now disabled by default. It has been replaced with the new `SPAM_SUBJECT` environment variable, which implements the functionality via a Sieve script instead in favor of being anti-spam service agnostic ([3820](https://github.com/docker-mailserver/docker-mailserver/pull/3820)) + - `RSPAMD_NEURAL` was added and is disabled by default. If switched on it wil enable the experimental Rspamd Neural network module to add a layer of analysis to spam detection using neural network technology. ([3833](https://github.com/docker-mailserver/docker-mailserver/pull/3833)) ### Fixes diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index d04da022784..323b6321e7f 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -451,6 +451,17 @@ Can be used to control the score when the [`HFILTER_HOSTNAME_UNKNOWN` symbol](#r Default: 6 (which corresponds to the `add_header` action) + +##### RSPAMD_NEURAL + +Can be used to enable or disable the [Neural network module][rspamd-docs-neural-network]. This is an experimental anti-spam weigh method using three neuaral networks in the configuration added here. As far as we can tell it trains itsself by using other modules to find out what spam is. It will take a while (a week or more) to train its first neural network. The config trains new networks all the time and discards of old networks. +Since it is experimental it is switched of by default. + +- **0** => Disabled +- 1 => Enabled + +[rspamd-docs-neural-network]: https://www.rspamd.com/doc/modules/neural.html + #### Reports ##### PFLOGSUMM_TRIGGER diff --git a/mailserver.env b/mailserver.env index 6fc9c7c2558..7f493a3aed2 100644 --- a/mailserver.env +++ b/mailserver.env @@ -183,6 +183,12 @@ RSPAMD_HFILTER=1 # Default: 6 RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE=6 +# Can be used to enable or disable the (still experimental) neural module. +# +# - **0** => Disabled +# - 1 => Enabled +RSPAMD_NEURAL=0 + # Amavis content filter (used for ClamAV & SpamAssassin) # 0 => Disabled # 1 => Enabled diff --git a/target/rspamd/local.d/neural.conf b/target/rspamd/local.d/neural.conf new file mode 100644 index 00000000000..bfc22befa06 --- /dev/null +++ b/target/rspamd/local.d/neural.conf @@ -0,0 +1,42 @@ +#https://github.com/rspamd/rspamd/issues/3099 +rules { + "NEURAL_WEEK_1000" { + train { + max_trains = 1000; + max_usages = 50; + max_iterations = 25; + learning_rate = 0.01, + spam_score = 8; + ham_score = -2; + } + symbol_spam = "NEURAL_WEEK_SPAM"; + symbol_ham = "NEURAL_WEEK_HAM"; + ann_expire = 300d; + } + "NEURAL_DAYS_200" { + train { + max_trains = 200; + max_usages = 10; + max_iterations = 25; + learning_rate = 0.01, + spam_score = 8; + ham_score = -2; + } + symbol_spam = "NEURAL_DAYS_SPAM"; + symbol_ham = "NEURAL_DAYS_HAM"; + ann_expire = 100d; + } + "NEURAL_HALF_DAY_50" { + train { + max_trains = 50; + max_usages = 4; + max_iterations = 25; + learning_rate = 0.01, + spam_score = 8; + ham_score = -2; + } + symbol_spam = "NEURAL_HALF_DAY_SPAM"; + symbol_ham = "NEURAL_HALF_DAY_HAM"; + ann_expire = 13d; + } +} diff --git a/target/rspamd/local.d/neural_group.conf b/target/rspamd/local.d/neural_group.conf new file mode 100644 index 00000000000..90ba6871f5f --- /dev/null +++ b/target/rspamd/local.d/neural_group.conf @@ -0,0 +1,26 @@ +symbols = { + "NEURAL_WEEK_SPAM" { + weight = 3.0; # sample weight + description = "Neural network spam (long)"; + } + "NEURAL_WEEK_HAM" { + weight = -3.0; # sample weight + description = "Neural network ham (long)"; + } + "NEURAL_DAYS_SPAM" { + weight = 2.5; # sample weight + description = "Neural network spam (medium)"; + } + "NEURAL_DAYS_HAM" { + weight = -1.5; # sample weight + description = "Neural network ham (medium)"; + } + "NEURAL_HALF_DAY_SPAM" { + weight = 2.0; # sample weight + description = "Neural network spam (short)"; + } + "NEURAL_HALF_DAY_HAM" { + weight = -1.0; # sample weight + description = "Neural network ham (short)"; + } +} diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index a05a798ef95..98f83de6528 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -20,6 +20,7 @@ function _setup_rspamd() { __rspamd__setup_learning __rspamd__setup_greylisting __rspamd__setup_hfilter_group + __rspamd__setup_neural __rspamd__setup_check_authenticated _rspamd_handle_user_modules_adjustments # must run last @@ -186,7 +187,6 @@ function __rspamd__setup_default_modules() { local DISABLE_MODULES=( clickhouse elastic - neural reputation spamassassin url_redirector @@ -283,6 +283,23 @@ function __rspamd__setup_hfilter_group() { fi } + +# This function handles setup of the neural module (see +# https://www.rspamd.com/doc/modules/neural.html). This module is experimental +# but can enhance anti-spam scoring possibly. +function __rspamd__setup_neural() { + if _env_var_expect_zero_or_one 'RSPAMD_NEURAL' && [[ ${RSPAMD_NEURAL} -eq 1 ]]; then + __rspamd__log 'debug' 'Enabling Neural module' + __rspamd__log 'warn' 'The Neural module is still experimental (in Rspamd) and hence not tested in DMS' + else + __rspamd__log 'debug' 'Neural module is disabled' + rm -f "${RSPAMD_LOCAL_D}/neural.conf" + rm -f "${RSPAMD_LOCAL_D}/neural_group.conf" + __rspamd__helper__enable_disable_module 'neural' 'false' + fi +} + + # If 'RSPAMD_CHECK_AUTHENTICATED' is enabled, then content checks for all users, i.e. # also for authenticated users, are performed. # diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index 219ce6c8550..eb5bf149bcf 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -71,6 +71,7 @@ function __environment_variables_general_setup() { VARS[RSPAMD_GREYLISTING]="${RSPAMD_GREYLISTING:=0}" VARS[RSPAMD_HFILTER]="${RSPAMD_HFILTER:=1}" VARS[RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE]="${RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE:=6}" + VARS[RSPAMD_NEURAL]="${RSPAMD_NEURAL:=0}" VARS[RSPAMD_LEARN]="${RSPAMD_LEARN:=0}" VARS[SA_KILL]=${SA_KILL:="10.0"} VARS[SPAM_SUBJECT]=${SPAM_SUBJECT:=} From db661bf3ac4a8b61bd92fbeb45466d7ab62af42a Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 2 Feb 2024 18:38:22 +0100 Subject: [PATCH 292/592] docs: misc improvements (but mostly related to Rspamd) (#3858) * remove leftover statement on `/etc/os-release` * update wording on the PR template * add section about other services to Rspamd docs * remove more outdated information from Rspamd docs * moved links and minor rewording in Rspamd docs --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .github/pull_request_template.md | 6 +- CHANGELOG.md | 1 - docs/content/config/security/rspamd.md | 111 +++++++++++++------------ 3 files changed, 60 insertions(+), 58 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 040d67dd751..104f9433b73 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -18,12 +18,12 @@ Fixes # - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] This change requires a documentation update -## Checklist: +## Checklist - [ ] My code follows the style guidelines of this project -- [ ] I have performed a self-review of my own code +- [ ] I have performed a self-review of my code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation (README.md or the documentation under `docs/`) -- [ ] If necessary I have added tests that prove my fix is effective or that my feature works +- [ ] If necessary, I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [ ] **I have added information about changes made in this PR to `CHANGELOG.md`** diff --git a/CHANGELOG.md b/CHANGELOG.md index 09afaba4bb4..7918d810301 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,6 @@ The most noteworthy change of this release is the update of the container's base - Notable minor version bump: `postfix 3.5.23 => 3.7.9` - Notable minor version bump + downgrade: `dovecot 2.3.13 => 2.3.19` (_Previous release provided `2.3.21` via community repo, `2.3.19` is now the default_) - Updates to `packages.sh`: - - The script now uses `/etc/os-release` to determine the release name of Debian - Removed custom installations of Fail2Ban, getmail6 and Rspamd - Updated packages lists and added comments for maintainability - OpenDMARC upgrade: `v1.4.0` => `v1.4.2` ([#3841](https://github.com/docker-mailserver/docker-mailserver/pull/3841)) diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 4c466c5de72..6dc5f202c7c 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -4,12 +4,9 @@ title: 'Security | Rspamd' ## About -Rspamd is a ["fast, free and open-source spam filtering system"][rspamd-homepage]. DMS integrates Rspamd like any other service. We provide a very simple but easy to maintain setup of Rspamd. +Rspamd is a ["fast, free and open-source spam filtering system"][www::rspamd-homepage]. DMS integrates Rspamd like any other service. We provide a basic but easy to maintain setup of Rspamd. -If you want to have a look at the default configuration files for Rspamd that DMS packs, navigate to [`target/rspamd/` inside the repository][dms-default-configuration]. Please consult the [section "The Default Configuration"](#the-default-configuration) section down below for a written overview. - -[rspamd-homepage]: https://rspamd.com/ -[dms-default-configuration]: https://github.com/docker-mailserver/docker-mailserver/tree/master/target/rspamd +If you want to take a look at the default configuration files for Rspamd that DMS packs, navigate to [`target/rspamd/` inside the repository][repo::dms-default-configuration]. Please consult the [section "The Default Configuration"](#the-default-configuration) section down below for a written overview. ## Related Environment Variables @@ -23,15 +20,17 @@ The following environment variables are related to Rspamd: 6. [`RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE`](../environment.md#rspamd_hfilter_hostname_unknown_score) 7. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) 8. [`SPAM_SUBJECT`](../environment.md#spam_subject) -9. [`MOVE_SPAM_TO_JUNK`][docs-spam-to-junk] +9. [`MOVE_SPAM_TO_JUNK`][docs::spam-to-junk] 10. [`MARK_SPAM_AS_READ`](../environment.md#mark_spam_as_read) With these variables, you can enable Rspamd itself, and you can enable / disable certain features related to Rspamd. -[docs-spam-to-junk]: ../environment.md#move_spam_to_junk - ## The Default Configuration +### Other Anti-Spam-Services + +DMS packs other anti-spam services, like SpamAssassin or Amavis, next to Rspamd. There exist services, like ClamAV (`ENABLE_CLAMAV`), that Rspamd can utilize to improve the scanning. Except for ClamAV, we recommend disabling **all other** anti-spam services when using Rspamd. The [basic configuration shown below](#a-very-basic-configuration) provides a good starting point. + ### Mode of Operation !!! tip "Attention" @@ -40,12 +39,12 @@ With these variables, you can enable Rspamd itself, and you can enable / disable Rspamd is integrated as a milter into DMS. When enabled, Postfix's `main.cf` configuration file includes the parameter `rspamd_milter = inet:localhost:11332`, which is added to `smtpd_milters`. As a milter, Rspamd can inspect incoming and outgoing e-mails. -Each mail is assigned what Rspamd calls symbols: when an e-mail matches a specific criterion, the mail receives a symbol. Afterwards, Rspamd applies a _spam score_ (as usual with anti-spam software) to the e-mail. +Each mail is assigned what Rspamd calls symbols: when an e-mail matches a specific criterion, the e-mail receives a symbol. Afterward, Rspamd applies a _spam score_ (as usual with anti-spam software) to the e-mail. - The score itself is calculated by adding the values of the individual symbols applied earlier. The higher the spam score is, the more likely the e-mail is spam. -- Symbol values can be negative (i.e., these symbols indicate the mail is legitimate, maybe because [SPF and DKIM][docs-dkim-dmarc-spf] are verified successfully) or the symbol can be positive (i.e., these symbols indicate the e-mail is spam, maybe because the e-mail contains a lot of links). +- Symbol values can be negative (i.e., these symbols indicate the mail is legitimate, maybe because [SPF and DKIM][docs::dkim-dmarc-spf] are verified successfully). On the other hand, symbol scores can be positive (i.e., these symbols indicate the e-mail is spam, perhaps because the e-mail contains numerous links). -Rspamd then adds (a few) headers to the e-mail based on the spam score. Most important are `X-Spamd-Result`, which contains an overview of which symbols were applied. It could look like this: +Rspamd then adds (a few) headers to the e-mail based on the spam score. Most important is `X-Spamd-Result`, which contains an overview of which symbols were applied. It could look like this: ```txt X-Spamd-Result default: False [-2.80 / 11.00]; R_SPF_NA(1.50)[no SPF record]; R_DKIM_ALLOW(-1.00)[example.com:s=dtag1]; DWL_DNSWL_LOW(-1.00)[example.com:dkim]; RWL_AMI_LASTHOP(-1.00)[192.0.2.42:from]; DMARC_POLICY_ALLOW(-1.00)[example.com,none]; RWL_MAILSPIKE_EXCELLENT(-0.40)[192.0.2.42:from]; FORGED_SENDER(0.30)[noreply@example.com,some-reply-address@bounce.example.com]; RCVD_IN_DNSWL_LOW(-0.10)[192.0.2.42:from]; MIME_GOOD(-0.10)[multipart/mixed,multipart/related,multipart/alternative,text/plain]; MIME_TRACE(0.00)[0:+,1:+,2:+,3:+,4:~,5:~,6:~]; RCVD_COUNT_THREE(0.00)[3]; RCPT_COUNT_ONE(0.00)[1]; REPLYTO_DN_EQ_FROM_DN(0.00)[]; ARC_NA(0.00)[]; TO_MATCH_ENVRCPT_ALL(0.00)[]; RCVD_TLS_LAST(0.00)[]; DKIM_TRACE(0.00)[example.com:+]; HAS_ATTACHMENT(0.00)[]; TO_DN_NONE(0.00)[]; FROM_NEQ_ENVFROM(0.00)[noreply@example.com,some-reply-address@bounce.example.com]; FROM_HAS_DN(0.00)[]; REPLYTO_DOM_NEQ_FROM_DOM(0.00)[]; PREVIOUSLY_DELIVERED(0.00)[receiver@anotherexample.com]; ASN(0.00)[asn:3320, ipnet:192.0.2.0/24, country:DE]; MID_RHS_MATCH_FROM(0.00)[]; MISSING_XM_UA(0.00)[]; HAS_REPLYTO(0.00)[some-reply-address@dms-reply.example.com] @@ -57,37 +56,30 @@ And then there is a corresponding `X-Rspamd-Action` header, which shows the over X-Rspamd-Action no action ``` -Since the score is `-2.80`, nothing will happen and the e-mail is not classified as spam. Our custom [`actions.conf`][rspamd-actions-config] defines what to do at certain scores: +Since the score is `-2.80`, nothing will happen and the e-mail is not classified as spam. Our custom [`actions.conf`][www::rspamd-actions-config] defines what to do at certain scores: 1. At a score of 4, the e-mail is to be _greylisted_; 2. At a score of 6, the e-mail is _marked with a header_ (`X-Spam: Yes`); -3. At a score of 7, the e-mail will additionally have their _subject re-written_ (appending a prefix like `[SPAM]`); -4. At a score of 11, the e-mail is outright _rejected_. +3. At a score of 11, the e-mail is outright _rejected_. --- -There is more to spam analysis than meets the eye: we have not covered the [Bayes training and filters][rspamc-docs-bayes] here, nor have we talked about [Sieve rules for e-mails that are marked as spam][docs-spam-to-junk]. +There is more to spam analysis than meets the eye: we have not covered the [Bayes training and filters][www::rspamd-docs-bayes] here, nor have we discussed [Sieve rules for e-mails that are marked as spam][docs::spam-to-junk]. Even the calculation of the score with the individual symbols has been presented to you in a simplified manner. But with the knowledge from above, you're equipped to read on and use Rspamd confidently. Keep on reading to understand the integration even better - you will want to know about your anti-spam software, not only to keep the bad e-mail out, but also to make sure the good e-mail arrive properly! -[docs-dkim-dmarc-spf]: ../best-practices/dkim_dmarc_spf.md -[rspamd-actions-config]: https://github.com/docker-mailserver/docker-mailserver/blob/master/target/rspamd/local.d/actions.conf -[rspamc-docs-bayes]: https://rspamd.com/doc/configuration/statistic.html - ### Workers -The proxy worker operates in [self-scan mode][rspamd-docs-proxy-self-scan-mode]. This simplifies the setup as we do not require a normal worker. You can easily change this though by [overriding the configuration by DMS](#providing-custom-settings-overriding-settings). +The proxy worker operates in [self-scan mode][www::rspamd-docs-proxy-self-scan-mode]. This simplifies the setup as we do not require a normal worker. You can easily change this though by [overriding the configuration by DMS](#providing-custom-settings-overriding-settings). DMS does not set a default password for the controller worker. You may want to do that yourself. In setups where you already have an authentication provider in front of the Rspamd webpage, you may want to [set the `secure_ip ` option to `"0.0.0.0/0"` for the controller worker](#with-the-help-of-a-custom-file) to disable password authentication inside Rspamd completely. -[rspamd-docs-proxy-self-scan-mode]: https://rspamd.com/doc/workers/rspamd_proxy.html#self-scan-mode - ### Persistence with Redis When Rspamd is enabled, we implicitly also start an instance of Redis in the container: - Redis is configured to persist its data via RDB snapshots to disk in the directory `/var/lib/redis` (_or the [`/var/mail-state/`][docs::dms-volumes-state] volume when present_). -- With the volume mount the snapshot will restore the Redis data across container restarts, and provide a way to keep backup. +- With the volume mount, the snapshot will restore the Redis data across container restarts, and provide a way to keep backup. Redis uses `/etc/redis/redis.conf` for configuration: @@ -96,15 +88,13 @@ Redis uses `/etc/redis/redis.conf` for configuration: ### Web Interface -Rspamd provides a [web interface][rspamc-docs-web-interface], which contains statistics and data Rspamd collects. The interface is enabled by default and reachable on port 11334. +Rspamd provides a [web interface][www::rspamd-docs-web-interface], which contains statistics and data Rspamd collects. The interface is enabled by default and reachable on port 11334. ![Rspamd Web Interface](https://rspamd.com/img/webui.png) -[rspamc-docs-web-interface]: https://rspamd.com/webui/ - ### DNS -DMS does not supply custom values for DNS servers to Rspamd. If you need to use custom DNS servers, which could be required when using [DNS-based black/whitelists](#rbls-realtime-blacklists-dnsbls-dns-based-blacklists), you need to adjust [`options.inc`][rspamd-docs-basic-options] yourself. +DMS does not supply custom values for DNS servers to Rspamd. If you need to use custom DNS servers, which could be required when using [DNS-based black/whitelists](#rbls-realtime-blacklists-dnsbls-dns-based-blacklists), you need to adjust [`options.inc`][www::rspamd-docs-basic-options] yourself. !!! tip "Making DNS Servers Configurable" @@ -122,9 +112,7 @@ You can find the Rspamd logs at `/var/log/mail/rspamd.log`, and the correspondin ### Modules -You can find a list of all Rspamd modules [on their website][rspamd-docs-modules]. - -[rspamd-docs-modules]: https://rspamd.com/doc/modules/ +You can find a list of all Rspamd modules [on their website][www::rspamd-docs-modules]. #### Disabled By Default @@ -136,7 +124,7 @@ You can choose to enable ClamAV, and Rspamd will then use it to check for viruse #### RBLs (Real-time Blacklists) / DNSBLs (DNS-based Blacklists) -The [RBL module](https://rspamd.com/doc/modules/rbl.html) is enabled by default. As a consequence, Rspamd will perform DNS lookups to a variety of blacklists. Whether an RBL or a DNSBL is queried depends on where the domain name was obtained: RBL servers are queried with IP addresses extracted from message headers, DNSBL server are queried with domains and IP addresses extracted from the message body \[[source][rbl-vs-dnsbl]\]. +The [RBL module](https://rspamd.com/doc/modules/rbl.html) is enabled by default. As a consequence, Rspamd will perform DNS lookups to various blacklists. Whether an RBL or a DNSBL is queried depends on where the domain name was obtained: RBL servers are queried with IP addresses extracted from message headers, DNSBL server are queried with domains and IP addresses extracted from the message body \[[source][www::rbl-vs-dnsbl]\]. !!! danger "Rspamd and DNS Block Lists" @@ -144,8 +132,6 @@ The [RBL module](https://rspamd.com/doc/modules/rbl.html) is enabled by default. If you want to use DNSBLs, **try to use your own DNS resolver** and make sure it is set up correctly, i.e. it should be a non-public & **recursive** resolver. Otherwise, you might not be able ([see this Spamhaus post](https://www.spamhaus.org/faq/section/DNSBL%20Usage#365)) to make use of the block lists. -[rbl-vs-dnsbl]: https://forum.eset.com/topic/25277-dnsbl-vs-rbl-mail-security/?do=findComment&comment=119818 - ## Providing Custom Settings & Overriding Settings DMS brings sane default settings for Rspamd. They are located at `/etc/rspamd/local.d/` inside the container (or `target/rspamd/local.d/` in the repository). @@ -154,20 +140,17 @@ DMS brings sane default settings for Rspamd. They are located at `/etc/rspamd/lo !!! question "What is [`docker-data/dms/config/`][docs::dms-volumes-config]?" -If you want to overwrite the default settings and / or provide your own settings, you can place files at `docker-data/dms/config/rspamd/override.d/`. Files from this directory are copied to `/etc/rspamd/override.d/` during startup. These files [forcibly override][rspamd-docs-override-dir] Rspamd and DMS default settings. +If you want to overwrite the default settings or provide your settings, you can place files at `docker-data/dms/config/rspamd/override.d/`. Files from this directory are copied to `/etc/rspamd/override.d/` during startup. These files [forcibly override][www::rspamd-docs-override-dir] Rspamd and DMS default settings. -!!! question "What is the [`local.d` directory and how does it compare to `override.d`][rspamd-docs-config-directories]?" +!!! question "What is the [`local.d` directory and how does it compare to `override.d`][www::rspamd-docs-config-directories]?" !!! warning "Clashing Overrides" Note that when also [using the `custom-commands.conf` file](#with-the-help-of-a-custom-file), files in `override.d` may be overwritten in case you adjust them manually and with the help of the file. -[rspamd-docs-override-dir]: https://www.rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories -[rspamd-docs-config-directories]: https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories - ### With the Help of a Custom File -DMS provides the ability to do simple adjustments to Rspamd modules with the help of a single file. Just place a file called `custom-commands.conf` into `docker-data/dms/config/rspamd/`. If this file is present, DMS will evaluate it. The structure is _very_ simple. Each line in the file looks like this: +DMS provides the ability to do simple adjustments to Rspamd modules with the help of a single file. Just place a file called `custom-commands.conf` into `docker-data/dms/config/rspamd/`. If this file is present, DMS will evaluate it. The structure is simple, as each line in the file looks like this: ```txt COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3 @@ -180,7 +163,7 @@ where `COMMAND` can be: 3. `set-option-for-module`: sets the value for option `ARGUMENT2` to `ARGUMENT3` inside module `ARGUMENT1` 4. `set-option-for-controller`: set the value of option `ARGUMENT1` to `ARGUMENT2` for the controller worker 5. `set-option-for-proxy`: set the value of option `ARGUMENT1` to `ARGUMENT2` for the proxy worker -6. `set-common-option`: set the option `ARGUMENT1` that [defines basic Rspamd behavior][rspamd-docs-basic-options] to value `ARGUMENT2` +6. `set-common-option`: set the option `ARGUMENT1` that [defines basic Rspamd behavior][www::rspamd-docs-basic-options] to value `ARGUMENT2` 7. `add-line`: this will add the complete line after `ARGUMENT1` (with all characters) to the file `/etc/rspamd/override.d/` !!! example "An Example Is [Shown Down Below](#adjusting-and-extending-the-very-basic-configuration)" @@ -195,31 +178,36 @@ You can also have comments (the line starts with `#`) and blank lines in `custom These simple commands are meant to give users the ability to _easily_ alter modules and their options. As a consequence, they are not powerful enough to enable multi-line adjustments. If you need to do something more complex, we advise to do that [manually](#manually)! -[rspamd-docs-basic-options]: https://rspamd.com/doc/configuration/options.html - ## Examples & Advanced Configuration ### A Very Basic Configuration -You want to start using Rspamd? Rspamd is disabled by default, so you need to set the following environment variables: +Do you want to start using Rspamd? Rspamd is disabled by default, so you need to set the following environment variables: ```env ENABLE_RSPAMD=1 +# ClamAV is compatible with Rspamd. Optionally enable it for anti-virus support: +ENABLE_CLAMAV=1 + +# Rspamd replaces the functionality of all these anti-spam services, disable them: ENABLE_OPENDKIM=0 ENABLE_OPENDMARC=0 ENABLE_POLICYD_SPF=0 ENABLE_AMAVIS=0 ENABLE_SPAMASSASSIN=0 + +# Provided you've set `RSPAMD_GREYLISTING=1`, also disable Postgrey: +ENABLE_POSTGREY=0 ``` This will enable Rspamd and disable services you don't need when using Rspamd. ### Adjusting and Extending The Very Basic Configuration -Rspamd is running, but you want or need to adjust it? First, create a file named `custom-commands.conf` under `docker-data/dms/config/rspamd` (which translates to `/tmp/docker-mailserver/rspamd/` inside the container). Then add you changes: +Rspamd is running, but you want or need to adjust it? First, create a file named `custom-commands.conf` under `docker-data/dms/config/rspamd` (which translates to `/tmp/docker-mailserver/rspamd/` inside the container). Then add your changes: -1. Say you want to be able to easily look at the frontend Rspamd provides on port 11334 (default) without the need to enter a password (maybe because you already provide authorization and authentication). You will need to adjust the controller worker: `set-option-for-controller secure_ip "0.0.0.0/0"`. -2. You additionally want to enable the auto-spam-learning for the Bayes module? No problem: `set-option-for-module classifier-bayes autolearn true`. +1. Say you want to be able to easily look at the frontend Rspamd provides on port 11334 (default) without the need to enter a password (maybe because you already provide authorization and authentication). You will have to adjust the controller worker: `set-option-for-controller secure_ip "0.0.0.0/0"`. +2. Do you additionally want to enable the auto-spam-learning for the Bayes module? No problem: `set-option-for-module classifier-bayes autolearn true`. 3. But the chartable module gets on your nerves? Easy: `disable-module chartable`. ??? example "What Does the Result Look Like?" @@ -239,25 +227,40 @@ Rspamd is running, but you want or need to adjust it? First, create a file named ### DKIM Signing -There is a dedicated [section for setting up DKIM with Rspamd in our documentation][docs-dkim-with-rspamd]. - -[docs-dkim-with-rspamd]: ../best-practices/dkim_dmarc_spf.md#dkim +There is a dedicated [section for setting up DKIM with Rspamd in our documentation][docs::dkim-with-rspamd]. ### _Abusix_ Integration -This subsection gives information about the integration of [Abusix], "a set of blocklists that work as an additional email security layer for your existing mail environment". The setup is straight-forward and well documented: +This subsection provides information about the integration of [Abusix][www::abusix], "a set of blocklists that work as an additional email security layer for your existing mail environment". The setup is straight-forward and well documented: 1. [Create an account](https://app.abusix.com/signup) 2. Retrieve your API key -3. Navigate to the ["Getting Started" documentation for Rspamd][abusix-rspamd-integration] and follow the steps described there +3. Navigate to the ["Getting Started" documentation for Rspamd][www::abusix-rspamd-integration] and follow the steps described there 4. Make sure to change `` to your private API key -We recommend mounting the files directly into the container, as they are rather big and not manageable with the [modules script](#with-the-help-of-a-custom-file). If mounted to the correct location, Rspamd will automatically pick them up. +We recommend mounting the files directly into the container, as they are rather big and not manageable with our [`custom-command.conf` script](#with-the-help-of-a-custom-file). If mounted to the correct location, Rspamd will automatically pick them up. While _Abusix_ can be integrated into Postfix, Postscreen and a multitude of other software, we recommend integrating _Abusix_ only into a single piece of software running in your mail server - everything else would be excessive and wasting queries. Moreover, we recommend the integration into suitable filtering software and not Postfix itself, as software like Postscreen or Rspamd can properly evaluate the return codes and other configuration. -[Abusix]: https://abusix.com/ -[abusix-rspamd-integration]: https://docs.abusix.com/abusix-mail-intelligence/gbG8EcJ3x3fSUv8cMZLiwA/getting-started/dmw9dcwSGSNQiLTssFAnBW#rspamd +[rspamd-web]: https://rspamd.com/ +[rspamd-docs::bayes]: https://rspamd.com/doc/configuration/statistic.html +[rspamd-docs::proxy-self-scan-mode]: https://rspamd.com/doc/workers/rspamd_proxy.html#self-scan-mode +[rspamd-docs::web-interface]: https://rspamd.com/webui/ +[rspamd-docs::modules]: https://rspamd.com/doc/modules/ +[rspamd-docs::override-dir]: https://www.rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories +[rspamd-docs::config-directories]: https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories +[rspamd-docs::basic-options]: https://rspamd.com/doc/configuration/options.html + +[www::rbl-vs-dnsbl]: https://forum.eset.com/topic/25277-dnsbl-vs-rbl-mail-security/?do=findComment&comment=119818 +[abusix-web]: https://abusix.com/ +[abusix-docs::rspamd-integration]: https://docs.abusix.com/abusix-mail-intelligence/gbG8EcJ3x3fSUv8cMZLiwA/getting-started/dmw9dcwSGSNQiLTssFAnBW#rspamd + +[dms-repo::rspamd-actions-config]: https://github.com/docker-mailserver/docker-mailserver/blob/v14.0.0/target/rspamd/local.d/actions.conf +[dms-repo::default-rspamd-configuration]: https://github.com/docker-mailserver/docker-mailserver/tree/v14.0.0/target/rspamd + +[docs::spam-to-junk]: ../environment.md#move_spam_to_junk +[docs::dkim-dmarc-spf]: ../best-practices/dkim_dmarc_spf.md +[docs::dkim-with-rspamd]: ../best-practices/dkim_dmarc_spf.md#dkim [docs::dms-volumes-config]: ../advanced/optional-config.md#volumes-config [docs::dms-volumes-state]: ../advanced/optional-config.md#volumes-state From 2c026715874c9240a1e0c4f34c38f9f612eceb5f Mon Sep 17 00:00:00 2001 From: Jackson Zheng <60581068+JacksonZ03@users.noreply.github.com> Date: Sun, 4 Feb 2024 10:04:07 +0000 Subject: [PATCH 293/592] Minor spelling correction (#3870) --- docs/content/config/best-practices/dkim_dmarc_spf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md index 290ac5cd400..52d01deac54 100644 --- a/docs/content/config/best-practices/dkim_dmarc_spf.md +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -82,7 +82,7 @@ You should have: When the DMS FQDN is `mail.example.com` or `example.com`, by default this command will generate DKIM keys for `example.com` as the primary domain for your users mail accounts (eg: `hello@example.com`). - The DKIM generation does not have support to query LDAP for additionanl mail domains it should know about. If the primary mail domain is not sufficient, then you must explicitly specify any extra domains via the `domain` option: + The DKIM generation does not have support to query LDAP for additional mail domains it should know about. If the primary mail domain is not sufficient, then you must explicitly specify any extra domains via the `domain` option: ```sh # ENABLE_OPENDKIM=1 (default): From 32dcabe82684e527fd705d469c7a3c91140e097c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 4 Feb 2024 10:05:24 +0000 Subject: [PATCH 294/592] docs: update `CONTRIBUTORS.md` (#3869) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 309 +++++++++++++++++++++++++----------------------- 1 file changed, 158 insertions(+), 151 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 72784ab01ef..a7c4e661dc1 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -127,18 +127,18 @@ Thanks goes to these wonderful people ✨ - - 00angus + + ap-wtioit
- 00angus + ap-wtioit
- - ap-wtioit + + 00angus
- ap-wtioit + 00angus 		@@ -212,14 +212,21 @@ Thanks goes to these wonderful people ✨ tyranron + + KyleOndy +
+ KyleOndy + +
mindrunner
mindrunner -
MichaelSp @@ -228,10 +235,10 @@ Thanks goes to these wonderful people ✨ - - KyleOndy + + m-a-v
- KyleOndy + m-a-v 		@@ -241,13 +248,6 @@ Thanks goes to these wonderful people ✨ bilak - - m-a-v -
- m-a-v - - 		vortex852456 @@ -256,18 +256,18 @@ Thanks goes to these wonderful people ✨ - - chris54721 + + hanscees
- chris54721 + hanscees
- - hanscees + + chris54721
- hanscees + chris54721 		@@ -658,31 +658,31 @@ Thanks goes to these wonderful people ✨ - - aminvakil + + andrewlow
- aminvakil + andrewlow 		- - elbracht + + abh
- elbracht + abh 		- - abh + + aminvakil
- abh + aminvakil 		- - andrewlow + + elbracht
- andrewlow + elbracht 		@@ -737,38 +737,38 @@ Thanks goes to these wonderful people ✨
- - nueaf + + fl42
- nueaf + fl42 		- - martinwepner + + ipernet
- martinwepner + ipernet 		- - artonge + + H4R0
- artonge + H4R0 		- - spacecowboy + + eltociear
- spacecowboy + eltociear 		- - jedateach + + jamesfryer
- jedateach + jamesfryer 		@@ -780,67 +780,67 @@ Thanks goes to these wonderful people ✨
- - jamesfryer + + artonge
- jamesfryer + artonge 		- - eltociear + + jedateach
- eltociear + jedateach 		- - H4R0 + + spacecowboy
- H4R0 + spacecowboy 		- - ipernet + + martinwepner
- ipernet + martinwepner 		- - fl42 + + nueaf
- fl42 + nueaf 		- - auchri + + Thiritin
- auchri + Thiritin
- - stephan-devop + + thomasschmit
- stephan-devop + thomasschmit 		- - stigok + + TechnicLab
- stigok + TechnicLab 		- - 5ven + + sylvaindumont
- 5ven + sylvaindumont 		@@ -851,32 +851,32 @@ Thanks goes to these wonderful people ✨ - - sylvaindumont + + 5ven
- sylvaindumont + 5ven 		- - TechnicLab + + stigok
- TechnicLab + stigok
- - thomasschmit + + stephan-devop
- thomasschmit + stephan-devop 		- - Thiritin + + radicand
- Thiritin + radicand 		@@ -1402,13 +1402,28 @@ Thanks goes to these wonderful people ✨ mchamplain + + crash7 +
+ crash7 + + 		+ + auchri +
+ auchri + + 		arkanovicz
arkanovicz -
CBeerta @@ -1422,8 +1437,7 @@ Thanks goes to these wonderful people ✨
damianmoore -
espitall @@ -1451,7 +1465,8 @@ Thanks goes to these wonderful people ✨
danielvandenberg95 -
denisix @@ -1465,8 +1480,7 @@ Thanks goes to these wonderful people ✨
mlatorre31 -
mazzz1y @@ -1494,7 +1508,8 @@ Thanks goes to these wonderful people ✨
edvorg -
eliroca @@ -1508,8 +1523,7 @@ Thanks goes to these wonderful people ✨
ekkis -
ErikEngerd @@ -1537,22 +1551,15 @@ Thanks goes to these wonderful people ✨
flole - 		- - froks -
- froks - -
0xflotus
0xflotus -
ifokeev @@ -1581,6 +1588,14 @@ Thanks goes to these wonderful people ✨ askz + + aspettl +
+ aspettl + +
acch @@ -1594,8 +1609,7 @@ Thanks goes to these wonderful people ✨
vifino -
kachkaev @@ -1623,7 +1637,8 @@ Thanks goes to these wonderful people ✨
eglia -
groupmsl @@ -1637,8 +1652,7 @@ Thanks goes to these wonderful people ✨
green-anger -
iRhonin @@ -1666,7 +1680,8 @@ Thanks goes to these wonderful people ✨
astrocket -
baxerus @@ -1680,8 +1695,7 @@ Thanks goes to these wonderful people ✨
spock -
erdos4d @@ -1690,10 +1704,10 @@ Thanks goes to these wonderful people ✨ - - crash7 + + akkumar
- crash7 + akkumar 		@@ -1709,7 +1723,8 @@ Thanks goes to these wonderful people ✨
KCrawley -
khuedoan @@ -1723,8 +1738,7 @@ Thanks goes to these wonderful people ✨
JustAnother1 -
LeoWinterDE @@ -1752,7 +1766,8 @@ Thanks goes to these wonderful people ✨
LucidityCrash -
MadsRC @@ -1766,8 +1781,7 @@ Thanks goes to these wonderful people ✨
madmath03 -
maxemann96 @@ -1795,7 +1809,8 @@ Thanks goes to these wonderful people ✨
exhuma -
milas @@ -1809,8 +1824,7 @@ Thanks goes to these wonderful people ✨
mcchots -
MohammedNoureldin @@ -1838,12 +1852,13 @@ Thanks goes to these wonderful people ✨
neuralp -
- - radicand + + froks
- radicand + froks 		@@ -1852,8 +1867,7 @@ Thanks goes to these wonderful people ✨
fkefer -
frugan-dev @@ -1881,7 +1895,8 @@ Thanks goes to these wonderful people ✨
GiovanH -
harryyoud @@ -1895,8 +1910,7 @@ Thanks goes to these wonderful people ✨
HeySora -
sirgantrithon @@ -1924,7 +1938,8 @@ Thanks goes to these wonderful people ✨
jcalfee -
mivek @@ -1938,8 +1953,7 @@ Thanks goes to these wonderful people ✨
init-js -
Jeidnx @@ -1967,7 +1981,8 @@ Thanks goes to these wonderful people ✨
jmccl -
jurekbarth @@ -1981,21 +1996,13 @@ Thanks goes to these wonderful people ✨
JOduMonT -
Kaan88
Kaan88 - 		- - akkumar -
- akkumar -
From 9fc7f979507a4045b60627a985a26487b7b2de74 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Feb 2024 01:34:20 +1300 Subject: [PATCH 295/592] chore(deps): Bump docker/metadata-action from 5.5.0 to 5.5.1 (#3878) Bumps [docker/metadata-action](https://github.com/docker/metadata-action) from 5.5.0 to 5.5.1. - [Release notes](https://github.com/docker/metadata-action/releases) - [Commits](https://github.com/docker/metadata-action/compare/v5.5.0...v5.5.1) --- updated-dependencies: - dependency-name: docker/metadata-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index d7a791c595c..83357062015 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -23,7 +23,7 @@ jobs: - name: 'Prepare tags' id: prep - uses: docker/metadata-action@v5.5.0 + uses: docker/metadata-action@v5.5.1 with: images: | ${{ secrets.DOCKER_REPOSITORY }} From d5efaf95c3a9239fa5ad3728be1525f505cc38b9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 12:35:57 +0000 Subject: [PATCH 296/592] chore(deps): Bump anchore/scan-action from 3.6.1 to 3.6.4 (#3877) Bumps [anchore/scan-action](https://github.com/anchore/scan-action) from 3.6.1 to 3.6.4. - [Release notes](https://github.com/anchore/scan-action/releases) - [Changelog](https://github.com/anchore/scan-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/anchore/scan-action/compare/v3.6.1...v3.6.4) --- updated-dependencies: - dependency-name: anchore/scan-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index cb1a5994877..78af5ded299 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v3.6.1 + uses: anchore/scan-action@v3.6.4 id: scan with: image: mailserver-testing:ci From 51a391525775058b0f0344f417da4f0c4c8d1ccc Mon Sep 17 00:00:00 2001 From: Rahil Bhimjiani Date: Tue, 6 Feb 2024 04:26:47 +0530 Subject: [PATCH 297/592] docs: fix 404 in mailserver.env and default to RSA 2048 for TLS certs (#3875) * fix 404: broken MTA-STS link in comment of mailserver.env Signed-off-by: Rahil Bhimjiani * docs: recommend and default to RSA 2048 for ssl certs Signed-off-by: Rahil Bhimjiani --------- Signed-off-by: Rahil Bhimjiani --- docs/content/config/security/ssl.md | 8 ++++---- mailserver.env | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/content/config/security/ssl.md b/docs/content/config/security/ssl.md index 10f7adc4851..1b84c4dfe42 100644 --- a/docs/content/config/security/ssl.md +++ b/docs/content/config/security/ssl.md @@ -408,7 +408,7 @@ The following example is the [basic setup][acme-companion::basic-setup] you need - `LETSENCRYPT_TEST=true`: _Recommended during initial setup_. Otherwise the default production endpoint has a [rate limit of 5 duplicate certificates per week][letsencrypt::limits]. Overrides `ACME_CA_URI` to use the _Let's Encrypt_ staging endpoint. - `LETSENCRYPT_EMAIL`: For when you don't use `DEFAULT_EMAIL` on `acme-companion`, or want to assign a different email contact for this container. - - `LETSENCRYPT_KEYSIZE`: Allows you to configure the type (RSA or ECDSA) and size of the private key for your certificate. Default is RSA 4096. + - `LETSENCRYPT_KEYSIZE`: Allows you to configure the type (RSA or ECDSA) and size of the private key for your certificate. Default is RSA 4096, but RSA 2048 is recommended. - `LETSENCRYPT_RESTART_CONTAINER=true`: When the certificate is renewed, the entire container will be restarted to ensure the new certificate is used. [`acme-companion` ENV for default settings][acme-companion::env-config] that apply to all containers using `LETSENCRYPT_HOST`: @@ -450,8 +450,8 @@ The following example is the [basic setup][acme-companion::basic-setup] you need # Optional variables: LETSENCRYPT_mail_TEST=true LETSENCRYPT_mail_EMAIL='admin@example.com' - # RSA-4096 => `4096`, ECDSA-256 => `ec-256`: - LETSENCRYPT_mail_KEYSIZE=4096 + # Supported values are `2048`, `3072` and `4096` for RSA keys, and `ec-256` or `ec-384` for elliptic curve keys. + LETSENCRYPT_mail_KEYSIZE=2048 ``` Unlike with the equivalent ENV for containers, [changes to this file will **not** be detected automatically][acme-companion::standalone-changes]. You would need to wait until the next renewal check by `acme-companion` (_every hour by default_), restart `acme-companion`, or [manually invoke the _service loop_][acme-companion::service-loop]: @@ -488,7 +488,7 @@ For Caddy v2 you can specify the `key_type` in your server's global settings, wh http_port 80 https_port 443 default_sni example.com - key_type rsa4096 + key_type rsa2048 } ``` diff --git a/mailserver.env b/mailserver.env index 7f493a3aed2..44a9a484eea 100644 --- a/mailserver.env +++ b/mailserver.env @@ -362,7 +362,7 @@ POSTFIX_REJECT_UNKNOWN_CLIENT_HOSTNAME=0 POSTFIX_INET_PROTOCOLS=all # Enables MTA-STS support for outbound mail. -# More details: https://docker-mailserver.github.io/docker-mailserver/latest/config/advanced/mail-mta-sts/ +# More details: https://docker-mailserver.github.io/docker-mailserver/v13.3/config/best-practices/mta-sts/ # - **0** ==> MTA-STS disabled # - 1 => MTA-STS enabled ENABLE_MTA_STS=0 From 4f222fe256edc25509b053545b8848c073b535dd Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 6 Feb 2024 00:30:22 +0100 Subject: [PATCH 298/592] Rspamd: improve DKIM key generation (#3876) * correct removal of old files with `--force` `rm` would fail when one of the files is not present, which is quite undesirable log (not harmful until `set -e` is introduced). * use tmp log file ref: https://github.com/docker-mailserver/docker-mailserver/issues/3873#issuecomment-1926736020 * correct indentation --- target/bin/rspamd-dkim | 16 +++++++---- target/scripts/helpers/rspamd.sh | 46 ++++++++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/target/bin/rspamd-dkim b/target/bin/rspamd-dkim index 6dfcc1a0803..689aa7d51b7 100755 --- a/target/bin/rspamd-dkim +++ b/target/bin/rspamd-dkim @@ -177,10 +177,14 @@ function _create_keys() { exit 1 else _log 'info' "Overwriting existing files as the '--force' option was supplied" - rm "${PUBLIC_KEY_FILE}" "${PUBLIC_KEY_DNS_FILE}" "${PRIVATE_KEY_FILE}" + [[ -f ${PUBLIC_KEY_FILE} ]] && rm "${PUBLIC_KEY_FILE}" + [[ -f ${PUBLIC_KEY_DNS_FILE} ]] && rm "${PUBLIC_KEY_DNS_FILE}" + [[ -f ${PRIVATE_KEY_FILE} ]] && rm "${PRIVATE_KEY_FILE}" fi fi + __create_rspamd_err_log + # shellcheck disable=SC2310 if __do_as_rspamd_user rspamadm \ dkim_keygen \ @@ -188,12 +192,14 @@ function _create_keys() { -d "${DOMAIN}" \ "${KEYTYPE_OPTIONS[@]}" \ -k "${PRIVATE_KEY_FILE}" \ - >"${PUBLIC_KEY_FILE}" + >"${PUBLIC_KEY_FILE}" \ + && ! __filter_rspamd_err_log 'Permission denied' # we also need to check the log for error messages then - _log 'info' 'Successfully created DKIM keys' - _log 'debug' "Public key written to '${PUBLIC_KEY_FILE}'" - _log 'debug' "Private key written to '${PRIVATE_KEY_FILE}'" + _log 'info' 'Successfully created DKIM keys' + _log 'debug' "Public key written to '${PUBLIC_KEY_FILE}'" + _log 'debug' "Private key written to '${PRIVATE_KEY_FILE}'" else + __print_rspamd_err_log _exit_with_error 'Creating keys failed' fi } diff --git a/target/scripts/helpers/rspamd.sh b/target/scripts/helpers/rspamd.sh index 8d1fd6685f9..1d3e1417bb3 100644 --- a/target/scripts/helpers/rspamd.sh +++ b/target/scripts/helpers/rspamd.sh @@ -5,9 +5,51 @@ # Perform a specific command as the Rspamd user (`_rspamd`). This is useful # in case you want to have correct permissions on newly created files or if # you want to check whether Rspamd can perform a specific action. +# +# @flag ${1} = '--quiet' to indicate whether log should be disabled [OPTIONAL] function __do_as_rspamd_user() { - _log 'trace' "Running '${*}' as user '_rspamd'" - su _rspamd -s /bin/bash -c "${*}" + if [[ ${1:-} != '--quiet' ]]; then + _log 'trace' "Running '${*}' as user '_rspamd'" + else + shift 1 + fi + + su _rspamd -s /bin/bash -c "${*} 2>${__RSPAMD_ERR_LOG_FILE:-/dev/null}" +} + +# Create a temporary log file (with `mktemp`) that one can filter to search +# for error messages. This is required as `rspamadm` sometimes prints an error +# but does not exit with an error. +# +# The file created is managed in the ENV `__RSPAMD_ERR_LOG_FILE`. This ENV is +# meant for internal usage; do not use it on your scripts. The log file is cleaned +# up when the script exits. +function __create_rspamd_err_log() { + _log 'trace' "Creating Rspamd error log" + trap 'rm -f "${__RSPAMD_ERR_LOG_FILE}"' EXIT # cleanup when we exit + __RSPAMD_ERR_LOG_FILE=$(__do_as_rspamd_user --quiet mktemp) +} + +# Print the Rspamd temporary error log. This will succeed only when the log has been +# created before. +function __print_rspamd_err_log() { + [[ -v __RSPAMD_ERR_LOG_FILE ]] && __do_as_rspamd_user cat "${__RSPAMD_ERR_LOG_FILE}" +} + +# Print the Rspamd temporary error log. We use `grep` but with "fixed strings", which +# means the message you provide is evaluated as-is, not as a regular expression. This +# will succeed only when the log has been created before. +# +# @param ${1} = message to filter by +function __filter_rspamd_err_log() { + if [[ -v __RSPAMD_ERR_LOG_FILE ]]; then + __do_as_rspamd_user grep \ + --quiet \ + --ignore-case \ + --fixed-strings \ + "${1:?A message for filtering is required}" \ + "${__RSPAMD_ERR_LOG_FILE}" + fi } # Calling this function brings common Rspamd-related environment variables From 34654c7e20ccae05a4299669aa395afebea68f40 Mon Sep 17 00:00:00 2001 From: Frugan <7957714+frugan-dev@users.noreply.github.com> Date: Thu, 8 Feb 2024 22:23:58 +0100 Subject: [PATCH 299/592] chore: Source Postgrey `whitelist_clients` config from Github (#3879) Use a more updated list for Postgrey `whitelist_clients` --- CHANGELOG.md | 2 ++ Dockerfile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7918d810301..c8030f43dbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,8 @@ The most noteworthy change of this release is the update of the container's base - **Environment Variables:** - `ONE_DIR` has been removed (legacy ENV) ([#3840](https://github.com/docker-mailserver/docker-mailserver/pull/3840)) - It's only functionality remaining was to opt-out of run-time state consolidation with `ONE_DIR=0` (_when a volume was already mounted to `/var/mail-state`_). +- **Internal:** + - Changed the Postgrey whitelist retrieved during build to [source directly from Github](https://github.com/schweikert/postgrey/blob/master/postgrey_whitelist_clients) as the list is updated more frequently than the [author publishes at their website](https://postgrey.schweikert.ch) ([#3879](https://github.com/docker-mailserver/docker-mailserver/pull/3879)) - **Tests:** - Refactored helper methods for sending e-mails with specific `Message-ID` headers and the helpers for retrieving + filtering logs, which together help isolate logs relevant to specific mail when multiple mails have been processed within a single test. ([#3786](https://github.com/docker-mailserver/docker-mailserver/pull/3786)) - **Rspamd**: diff --git a/Dockerfile b/Dockerfile index b8bf89bcfaf..8110cebfd54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -144,7 +144,7 @@ COPY target/postgrey/postgrey /etc/default/postgrey RUN < Date: Tue, 13 Feb 2024 19:42:17 +1300 Subject: [PATCH 300/592] docs: Complete rewrite of PROXY protocol guide (#3882) --- .../tutorials/mailserver-behind-proxy.md | 454 ++++++++++++++---- 1 file changed, 369 insertions(+), 85 deletions(-) diff --git a/docs/content/examples/tutorials/mailserver-behind-proxy.md b/docs/content/examples/tutorials/mailserver-behind-proxy.md index 999395426bb..47116fde3c1 100644 --- a/docs/content/examples/tutorials/mailserver-behind-proxy.md +++ b/docs/content/examples/tutorials/mailserver-behind-proxy.md @@ -2,126 +2,410 @@ title: 'Tutorials | Mail Server behind a Proxy' --- -## Using DMS behind a Proxy +## Using a Reverse Proxy -### Information +Guidance is provided via a Traefik config example, however if you're only familiar with configuring a reverse proxy for web services there are some differences to keep in mind. -If you are hiding your container behind a proxy service you might have discovered that the proxied requests from now on contain the proxy IP as the request origin. Whilst this behavior is technical correct it produces certain problems on the containers behind the proxy as they cannot distinguish the real origin of the requests anymore. +- A security concern where preserving the client IP is important but needs to be handled at Layer 4 (TCP). +- TLS will be handled differently due protocols like STARTTLS and the need to comply with standards for interoperability with other MTAs. +- The ability to route the same port to different containers by FQDN can be limited. -To solve this problem on TCP connections we can make use of the [proxy protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt). Compared to other workarounds that exist (`X-Forwarded-For` which only works for HTTP requests or `Tproxy` that requires you to recompile your kernel) the proxy protocol: +This reduces many of the benefits for why you might use a reverse proxy, but they can still be useful. -- It is protocol agnostic (can work with any layer 7 protocols, even when encrypted). -- It does not require any infrastructure changes. -- NAT-ing firewalls have no impact it. -- It is scalable. +Some deployments may require a service to route traffic (kubernetes) when deploying, in which case the below advice is important to understand well. -There is only one condition: **both endpoints** of the connection MUST be compatible with proxy protocol. +## What can go wrong? -Luckily `dovecot` and `postfix` are both Proxy-Protocol ready softwares so it depends only on your used reverse-proxy / loadbalancer. +Without a reverse proxy involved, a service is typically aware of the client IP for a connection. -### Configuration of the used Proxy Software +However when a reverse proxy routes the connection this information can be lost, and the proxied service mistakenly treats the client IP as the reverse proxy handling the connection. -The configuration depends on the used proxy system. I will provide the configuration examples of [traefik v2](https://traefik.io/) using IMAP and SMTP with implicit TLS. +- That can be problematic when the client IP is meaningful information for the proxied service to act upon, especially when it [impacts security](#security-concerns). +- The [PROXY protocol][networking::spec:proxy-protocol] is a well established solution to preserve the client IP when both the proxy and service have enabled the support. -Feel free to add your configuration if you achieved the same goal using different proxy software below: +??? abstract "Technical Details - HTTP vs TCP proxying" -??? "Traefik v2" + A key difference for how the network is proxied relates to the [OSI Model][networking::osi-model]: - Truncated configuration of traefik itself: + - Layer 7 (_Application layer protocols: SMTP / IMAP / HTTP / etc_) + - Layer 4 (_Transport layer protocols: TCP / UDP_) - ```yaml + When working with Layer 7 and a protocol like HTTP, it is possible to inspect a protocol header like [`Forwarded`][networking::http-header::forwarded] (_or it's predecessor: [`X-Forwarded-For`][networking::http-header::x-forwarded-for]_). At a lower level with Layer 4, that information is not available and we are routing traffic agnostic to the application protocol being proxied. + + A proxy can prepend the [PROXY protocol][networking::spec:proxy-protocol] header to the TCP/UDP connection as it is routed to the service, which must be configured to be compatible with PROXY protocol (_often this adds a restriction that connections must provide the header, otherwise they're rejected_). + + Beyond your own proxy, traffic may be routed in the network by other means that would also rewrite this information such as Docker's own network management via `iptables` and `userland-proxy` (NAT). The PROXY header ensures the original source and destination IP addresses, along with their ports is preserved across transit. + +## Configuration + +### Reverse Proxy + +The below guidance is focused on configuring [Traefik][traefik-web], but the advice should be roughly applicable elsewhere (_eg: [NGINX][nginx-docs::proxyprotocol], [Caddy][caddy::plugin::l4]_). + +- Support requires the capability to proxy TCP (Layer 4) connections with PROXY protocol enabled for the upstream (DMS). The upstream must also support enabling PROXY protocol (_which for DMS services rejects any connection not using the protocol_). +- TLS should not be terminated at the proxy, that should be delegated to DMS (_which should be configured with the TLS certs_). Reasoning is covered under the [ports section](#ports). + +???+ example "Traefik service" + + The Traefik service config is fairly standard, just define the necessary entrypoints: + + ```yaml title="compose.yaml" services: reverse-proxy: - image: docker.io/traefik:latest # v2.5 - container_name: docker-traefik - restart: always + image: docker.io/traefik:latest # 2.10 / 3.0 + # CAUTION: In production you should configure the Docker API endpoint securely: + # https://doc.traefik.io/traefik/providers/docker/#docker-api-access + volumes: + - /var/run/docker.sock:/var/run/docker.sock command: - - "--providers.docker" - - "--providers.docker.exposedbydefault=false" - - "--providers.docker.network=proxy" - - "--entrypoints.web.address=:80" - - "--entryPoints.websecure.address=:443" - - "--entryPoints.smtp.address=:25" - - "--entryPoints.smtp-ssl.address=:465" - - "--entryPoints.imap-ssl.address=:993" - - "--entryPoints.sieve.address=:4190" + # Docker provider config: + - --providers.docker=true + - --providers.docker.exposedbydefault=false + # DMS ports you want to proxy: + - --entryPoints.mail-smtp.address=:25 + - --entryPoints.mail-submission.address=:587 + - --entryPoints.mail-submissions.address=:465 + - --entryPoints.mail-imap.address=:143 + - --entryPoints.mail-imaps.address=:993 + - --entryPoints.mail-pop3.address=:110 + - --entryPoints.mail-pop3s.address=:995 + - --entryPoints.mail-managesieve.address=:4190 + # Publish external access ports mapped to traefik entrypoint ports: ports: - "25:25" + - "587:587" - "465:465" + - "143:143" - "993:993" + - "110:110" + - "995:995" - "4190:4190" - [...] + # An IP is assigned here for other services (Dovecot) to trust for PROXY protocol: + networks: + default: + ipv4_address: 172.16.42.2 + + # Specifying a subnet to assign a fixed container IP to the reverse proxy: + networks: + default: + name: my-network + ipam: + config: + - subnet: "172.16.42.0/24" ``` - Truncated list of necessary labels on the DMS container: + !!! note "Extra considerations" + + - [`--providers.docker.network=my-network`][traefik-docs::provider-docker::network] is useful when there is more than one network to consider. + - If your deployment has any other hops (an edge proxy, load balancer, etc) between the reverse proxy and the client, you'll need PROXY protocol support throughout that chain. For Traefik this additionally requires [enabling PROXY protocol on your entry points][traefik-docs::entrypoint::proxyprotocol]. + +???+ example "Traefik labels for DMS" - ```yaml + ```yaml title="compose.yaml" services: - mailserver: + dms: image: ghcr.io/docker-mailserver/docker-mailserver:latest - container_name: mailserver hostname: mail.example.com - restart: always - networks: - - proxy labels: - - "traefik.enable=true" - - "traefik.tcp.routers.smtp.rule=HostSNI(`*`)" - - "traefik.tcp.routers.smtp.entrypoints=smtp" - - "traefik.tcp.routers.smtp.service=smtp" - - "traefik.tcp.services.smtp.loadbalancer.server.port=25" - - "traefik.tcp.services.smtp.loadbalancer.proxyProtocol.version=1" - - "traefik.tcp.routers.smtp-ssl.rule=HostSNI(`*`)" - - "traefik.tcp.routers.smtp-ssl.entrypoints=smtp-ssl" - - "traefik.tcp.routers.smtp-ssl.tls.passthrough=true" - - "traefik.tcp.routers.smtp-ssl.service=smtp-ssl" - - "traefik.tcp.services.smtp-ssl.loadbalancer.server.port=465" - - "traefik.tcp.services.smtp-ssl.loadbalancer.proxyProtocol.version=1" - - "traefik.tcp.routers.imap-ssl.rule=HostSNI(`*`)" - - "traefik.tcp.routers.imap-ssl.entrypoints=imap-ssl" - - "traefik.tcp.routers.imap-ssl.service=imap-ssl" - - "traefik.tcp.routers.imap-ssl.tls.passthrough=true" - - "traefik.tcp.services.imap-ssl.loadbalancer.server.port=10993" - - "traefik.tcp.services.imap-ssl.loadbalancer.proxyProtocol.version=2" - - "traefik.tcp.routers.sieve.rule=HostSNI(`*`)" - - "traefik.tcp.routers.sieve.entrypoints=sieve" - - "traefik.tcp.routers.sieve.service=sieve" - - "traefik.tcp.services.sieve.loadbalancer.server.port=4190" - [...] + - traefik.enable=true + + # These are examples, configure the equivalent for any additional ports you proxy. + # Explicit TLS (STARTTLS): + - traefik.tcp.routers.mail-smtp.rule=HostSNI(`*`) + - traefik.tcp.routers.mail-smtp.entrypoints=smtp + - traefik.tcp.routers.mail-smtp.service=smtp + - traefik.tcp.services.mail-smtp.loadbalancer.server.port=25 + - traefik.tcp.services.mail-smtp.loadbalancer.proxyProtocol.version=2 + + # Implicit TLS is no different, except for optional HostSNI support: + - traefik.tcp.routers.mail-submissions.rule=HostSNI(`*`) + - traefik.tcp.routers.mail-submissions.entrypoints=smtp-submissions + - traefik.tcp.routers.mail-submissions.service=smtp-submissions + - traefik.tcp.services.mail-submissions.loadbalancer.server.port=465 + - traefik.tcp.services.mail-submissions.loadbalancer.proxyProtocol.version=2 + # NOTE: Optionally match by SNI rule, this requires TLS passthrough (not compatible with STARTTLS): + #- traefik.tcp.routers.mail-submissions.rule=HostSNI(`mail.example.com`) + #- traefik.tcp.routers.mail-submissions.tls.passthrough=true + ``` + + !!! note "PROXY protocol compatibility" + + Only TCP routers support enabling PROXY Protocol (via [`proxyProtocol.version=2`][traefik-docs::service-tcp::proxyprotocol]) + + Postfix and Dovecot are both compatible with PROXY protocol v1 and v2. + +??? abstract "Technical Details - Ports (Traefik config)" + + !!! info "Explicit TLS (STARTTLS)" + + **Service Ports:** `mail-smtp` (25), `mail-submission` (587), `mail-imap` (143), `mail-pop3` (110), `mail-managesieve` (4190) + + --- + + - [Traefik expects the TCP router to not enable TLS][traefik-docs::router-tcp::server-first-protocols] (_see "Server First protocols"_) for these connections. They begin in plaintext and potentially upgrade the connection to TLS, Traefik has no involvement in STARTTLS. + - Without an initial TLS connection, the [`HostSNI` router rule is not usable][traefik-docs::router-tcp::host-sni] (_see "HostSNI & TLS"_). This limits routing flexibility for these ports (_eg: routing these ports by the FQDN to different DMS containers_). + + !!! info "Implicit TLS" + + **Service Ports:** `mail-submissions` (465), `mail-imaps` (993), `mail-pop3s` (995) + + --- + + The `HostSNI` router rule could specify the DMS FQDN instead of `*`: + + - This requires the router to have TLS enabled, so that Traefik can inspect the server name sent by the client. + - Traefik can only match the SNI to `*` when the client does not provide a server name. Some clients must explicitly opt-in, such as CLI clients `openssl` (`-servername`) and `swaks` (`--tls-sni`). + - Add [`tls.passthrough=true` to the router][traefik-docs::router-tcp::passthrough] (_this implicitly enables TLS_). + - Traefik should not terminate TLS, decryption should occur within DMS instead when proxying to the same implicit TLS ports. + - Passthrough ignores any certificates configured for Traefik; DMS must be configured with the certificates instead (_[DMS can use `acme.json` from Traefik][docs::tls::traefik]_). + + Unlike proxying HTTPS (port 443) to a container via HTTP (port 80), the equivalent for DMS service ports is not supported: + + - Port 25 must secure the connection via STARTTLS to be reached publicly. + - STARTTLS ports requiring authentication for Postfix (587) and Dovecot (110, 143, 4190) are configured to only permit authentication over an encrypted connection. + - Support would require routing the implicit TLS ports to their explicit TLS equivalent ports with auth restrictions removed. `tls.passthrough.true` would not be required, additionally port 25 would always be unencrypted (_if the proxy exclusively manages TLS/certs_), or unreachable by public MTAs attempting delivery if the proxy enables implicit TLS for this port. + +### DMS (Postfix + Dovecot) + +???+ example "Enable PROXY protocol on existing service ports" + + This can be handled via our config override support. + + --- + + Postfix via [`postfix-master.cf`][docs::overrides::postfix]: + + ```cf title="docker-data/dms/config/postfix-master.cf" + smtp/inet/postscreen_upstream_proxy_protocol=haproxy + submission/inet/smtpd_upstream_proxy_protocol=haproxy + submissions/inet/smtpd_upstream_proxy_protocol=haproxy ``` - Keep in mind that it is necessary to use port `10993` here. More information below at `dovecot` configuration. + [`postscreen_upstream_proxy_protocol`][postfix-docs::settings::postscreen_upstream_proxy_protocol] and [`smtpd_upstream_proxy_protocol`][postfix-docs::settings::smtpd_upstream_proxy_protocol] both specify the protocol type used by a proxy. `haproxy` represents the PROXY protocol. + + --- + + Dovecot via [`dovecot.cf`][docs::overrides::dovecot]: + + ```cf title="docker-data/dms/config/dovecot.cf" + haproxy_trusted_networks = 172.16.42.2 + + service imap-login { + inet_listener imap { + haproxy = yes + } + + inet_listener imaps { + haproxy = yes + } + } + + service pop3-login { + inet_listener pop3 { + haproxy = yes + } + + inet_listener pop3s { + haproxy = yes + } + } + + service managesieve-login { + inet_listener sieve { + haproxy = yes + } + } + ``` + + - [`haproxy_trusted_networks`][dovecot-docs::settings::haproxy-trusted-networks] must reference the reverse proxy IP, or a wider subnet using CIDR notation. + - [`haproxy = yes`][dovecot-docs::service-config::haproxy] for the TCP listeners of each login service. + +!!! warning "Internal traffic (_within the network or DMS itself_)" + + - Direct connections to DMS from other containers within the internal network will be rejected when they don't provide the required PROXY header. + - This can also affect services running within the DMS container itself if they attempt to make a connection and aren't PROXY protocol capable. + + --- + + A solution is to configure alternative service ports that offer PROXY protocol support (as shown next). + + Alternatively routing connections to DMS through the local reverse proxy via [DNS query rewriting][gh-dms::dns-rewrite-example] can work too. + +??? example "Configuring services with separate ports for PROXY protocol" + + In this example we'll take the original service ports and add `10000` for the new PROXY protocol service ports. + + Traefik labels will need to update their service ports accordingly (eg: `.loadbalancer.server.port=10465`). + + --- + + Postfix config now requires [our `user-patches.sh` support][docs::overrides::user-patches] to add new services in `/etc/postfix/master.cf`: + + ```bash title="docker-data/dms/config/user-patches.sh" + #!/bin/bash + + # Duplicate the config for the submission(s) service ports (587 / 465) with adjustments for the PROXY ports (10587 / 10465) and `syslog_name` setting: + postconf -Mf submission/inet | sed -e s/^submission/10587/ -e 's/submission/submission-proxyprotocol/' >> /etc/postfix/master.cf + postconf -Mf submissions/inet | sed -e s/^submissions/10465/ -e 's/submissions/submissions-proxyprotocol/' >> /etc/postfix/master.cf + # Enable PROXY Protocol support for these new service variants: + postconf -P 10587/inet/smtpd_upstream_proxy_protocol=haproxy + postconf -P 10465/inet/smtpd_upstream_proxy_protocol=haproxy + + # Create a variant for port 25 too (NOTE: Port 10025 is already assigned in DMS to Amavis): + postconf -Mf smtp/inet | sed -e s/^smtp/12525/ >> /etc/postfix/master.cf + # Enable PROXY Protocol support (different setting as port 25 is handled via postscreen), optionally configure a `syslog_name` to distinguish in logs: + postconf -P 12525/inet/postscreen_upstream_proxy_protocol=haproxy 12525/inet/syslog_name=smtp-proxyprotocol + ``` + + --- + + Dovecot is mostly the same as before: + + - A new service name instead of targeting one to modify. + - Add the new port assignment. + - Set [`ssl = yes`][dovecot-docs::service-config::ssl] when implicit TLS is needed. + + ```cf title="docker-data/dms/config/dovecot.cf" + haproxy_trusted_networks = 172.16.42.2 + + service imap-login { + inet_listener imap-proxied { + haproxy = yes + port = 10143 + } + + inet_listener imaps-proxied { + haproxy = yes + port = 10993 + ssl = yes + } + } + + service pop3-login { + inet_listener pop3-proxied { + haproxy = yes + port = 10110 + } + + inet_listener pop3s-proxied { + haproxy = yes + port = 10995 + ssl = yes + } + } + + service managesieve-login { + inet_listener sieve-proxied { + haproxy = yes + port = 14190 + } + } + ``` + +## Verification + +Send an email through the reverse proxy. If you do not use the DNS query rewriting approach, you'll need to do this from an external client. + +??? example "Sending a generic test mail through `swaks` CLI" + + Run a `swaks` command and then check your DMS logs for the expected client IP, it should no longer be using the reverse proxy IP. + + ```bash + # NOTE: It is common to find port 25 is blocked from outbound connections, you may only be able to test the submission(s) ports. + swaks --helo not-relevant.test --server mail.example.com --port 25 -tls --from hello@not-relevant.test --to user@example.com + ``` + + - You can specify the `--server` as the DMS FQDN or an IP address, where either should connect to the reverse proxy service. + - `not-relevant.test` technically may be subject to some tests, at least for port 25. With the submission(s) ports those should be exempt. + - `-tls` will use STARTTLS on port 25, you can exclude it to send unencrypted, but it would still go through the same port/route being tested. + - To test the submission ports use `--port 587 -tls` or `--port 465 -tlsc` with your credentials `--auth-user user@example.com --auth-password secret` + - Add `--tls-sni mail.example.com` if you have configured `HostSNI` in Traefik router rules (_SNI routing is only valid for implicit TLS ports_). + +??? warning "Do not rely on local testing alone" + + Testing from the Docker host technically works, however the IP is likely subject to more manipulation via `iptables` than an external client. + + The IP will likely appear as from the gateway IP of the Docker network associated to the reverse proxy, where that gateway IP then becomes the client IP when writing the PROXY protocol header. + +## Security concerns + +### Forgery + +Since the PROXY protocol sends a header with the client IP rewritten for software to use instead, this could be abused by bad actors. + +Software on the receiving end of the connection often supports configuring an IP or CIDR range of clients to trust receiving the PROXY protocol header from. + +??? warning "Risk exposure" + + If you trust more than the reverse proxy IP, you must consider the risk exposure: + + - Any container within the network that is compromised could impersonate another IP (_container or external client_) which may have been configured to have additional access/exceptions granted. + - If the reverse proxy is on a separate network/host than DMS, exposure of the PROXY protocol enabled ports outside the network increases the importance of narrowing trust. For example with the [known IPv6 to subnet Gateway IP routing gotcha][docs::ipv6::security-risks] in Docker, trusting the entire subnet DMS belongs to would wrongly trust external clients that have the subnet Gateway IP to impersonate any client IP. + - There is a [known risk with Layer 2 switching][docker::networking::l2-switch-gotcha] (_applicable to VPC networks, impact varies by cloud vendor_): + - Neighbouring hosts can indirectly route to ports published on the interfaces of a separate host system that shouldn't be reachable (_eg: localhost `127.0.0.1`, or a private subnet `172.16.0.0/12`_). + - The scope of this in Docker is limited to published ports only when Docker uses `iptables` with the kernel tunable `sysctl net.ipv4.ip_forward=1` (enabled implicitly). Port access is via `HOST:CONTAINER` ports published to their respective interface(s), that includes the container IP + port. + + While some concerns raised above are rather specific, these type of issues aren't exclusive to Docker and difficult to keep on top of as software is constantly changing. Limit the trusted networks where possible. + +??? warning "Postfix has no concept of trusted proxies" + + Postfix does not appear to have a way to configure trusted proxies like Dovecot does (`haproxy_trusted_networks`). + + [`postscreen_access_list`][postfix-docs::settings::postscreen_access_list] (_or [`smtpd_client_restrictions`][postfix-docs::settings::smtpd_client_restrictions] with [`check_client_access`][postfix-docs::settings::check_client_access] for ports 587/465_) can both restrict access by IP via a [CIDR lookup table][postfix-docs::config-table::cidr], however the client IP is already rewritten at this point via PROXY protocol. + + Thus those settings cannot be used for restricting access to only trusted proxies, only to the actual clients. + + A similar setting [`mynetworks`][postfix-docs::settings::mynetworks] / [`PERMIT_DOCKER`][docs::env::permit_docker] manages elevated trust for bypassing security restrictions. While it is intended for trusted clients, it has no relevance to trusting proxies for the same reasons. + + +### Monitoring + +While PROXY protocol works well with the reverse proxy, you may have some containers internally that interact with DMS on behalf of multiple clients. + +??? example "Roundcube + Fail2Ban" + + You may have other services with functionality like an API to send mail through DMS that likewise delegates credentials through DMS. -### Configuration of the Backend (`dovecot` and `postfix`) + Roundcube is an example of this where authentication is delegated to DMS, which introduces the same concern with loss of client IP. -The following changes can be achieved completely by adding the content to the appropriate files by using the projects [function to overwrite config files][docs-optionalconfig]. + - While this service does implement some support for preserving the client IP, it is limited. + - This may be problematic when monitoring services like Fail2Ban are enabled that scan logs for multiple failed authentication attempts which triggers a ban on the shared IP address. -Changes for `postfix` can be applied by adding the following content to `docker-data/dms/config/postfix-main.cf`: + You should adjust configuration of these monitoring services to monitor for auth failures from those services directly instead, adding an exclusion for that service IP from any DMS logs monitored (_but be mindful of PROXY header forgery risks_). -```cf -postscreen_upstream_proxy_protocol = haproxy -``` +[docs::overrides::dovecot]: ../../config/advanced/override-defaults/dovecot.md +[docs::overrides::postfix]: ../../config/advanced/override-defaults/postfix.md +[docs::overrides::user-patches]: ../../config/advanced/override-defaults/user-patches.md +[docs::ipv6::security-risks]: ../../config/advanced/ipv6.md#what-can-go-wrong +[docs::tls::traefik]: ../../config/security/ssl.md#traefik-v2 +[docs::env::permit_docker]: ../../config/environment.md#permit_docker +[gh-dms::dns-rewrite-example]: https://github.com/docker-mailserver/docker-mailserver/issues/3866#issuecomment-1928877236 -and to `docker-data/dms/config/postfix-master.cf`: +[nginx-docs::proxyprotocol]: https://docs.nginx.com/nginx/admin-guide/load-balancer/using-proxy-protocol +[caddy::plugin::l4]: https://github.com/mholt/caddy-l4 -```cf -submission/inet/smtpd_upstream_proxy_protocol=haproxy -submissions/inet/smtpd_upstream_proxy_protocol=haproxy -``` +[traefik-web]: https://traefik.io +[traefik-docs::entrypoint::proxyprotocol]: https://doc.traefik.io/traefik/routing/entrypoints/#proxyprotocol +[traefik-docs::provider-docker::network]: https://doc.traefik.io/traefik/providers/docker/#network +[traefik-docs::router-tcp::server-first-protocols]: https://doc.traefik.io/traefik/routing/routers/#entrypoints_1 +[traefik-docs::router-tcp::host-sni]: https://doc.traefik.io/traefik/routing/routers/#rule_1 +[traefik-docs::router-tcp::passthrough]: https://doc.traefik.io/traefik/routing/routers/#passthrough +[traefik-docs::service-tcp::proxyprotocol]:https://doc.traefik.io/traefik/routing/services/#proxy-protocol -Changes for `dovecot` can be applied by adding the following content to `docker-data/dms/config/dovecot.cf`: +[dovecot-docs::settings::haproxy-trusted-networks]: https://doc.dovecot.org/settings/core/#core_setting-haproxy_trusted_networks +[dovecot-docs::service-config::haproxy]: https://doc.dovecot.org/configuration_manual/service_configuration/#haproxy-v2-2-19 +[dovecot-docs::service-config::ssl]: https://doc.dovecot.org/configuration_manual/service_configuration/#ssl -```cf -haproxy_trusted_networks = , -haproxy_timeout = 3 secs -service imap-login { - inet_listener imaps { - haproxy = yes - ssl = yes - port = 10993 - } -} -``` +[postfix-docs::config-table::cidr]: https://www.postfix.org/cidr_table.5.html +[postfix-docs::settings::check_client_access]: https://www.postfix.org/postconf.5.html#check_client_access +[postfix-docs::settings::mynetworks]: https://www.postfix.org/postconf.5.html#mynetworks +[postfix-docs::settings::postscreen_access_list]: https://www.postfix.org/postconf.5.html#postscreen_access_list +[postfix-docs::settings::postscreen_upstream_proxy_protocol]: https://www.postfix.org/postconf.5.html#postscreen_upstream_proxy_protocol +[postfix-docs::settings::smtpd_client_restrictions]: https://www.postfix.org/postconf.5.html#smtpd_client_restrictions +[postfix-docs::settings::smtpd_upstream_proxy_protocol]: https://www.postfix.org/postconf.5.html#smtpd_upstream_proxy_protocol -!!! note - Port `10993` is used here to avoid conflicts with internal systems like `postscreen` and `amavis` as they will exchange messages on the default port and obviously have a different origin then compared to the proxy. +[docker::networking::l2-switch-gotcha]: https://github.com/moby/moby/issues/45610 +[networking::spec:proxy-protocol]: https://github.com/haproxy/haproxy/blob/master/doc/proxy-protocol.txt +[networking::http-header::x-forwarded-for]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For +[networking::http-header::forwarded]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded +[networking::osi-model]: https://www.cloudflare.com/learning/ddos/glossary/open-systems-interconnection-model-osi/ From 79a9656f4876e50264060eafbc066287a036a045 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 14 Feb 2024 09:48:35 +0000 Subject: [PATCH 301/592] docs: update `CONTRIBUTORS.md` (#3883) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 446 +++++++++++++++++++++++++----------------------- 1 file changed, 230 insertions(+), 216 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index a7c4e661dc1..1e252291891 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -628,6 +628,13 @@ Thanks goes to these wonderful people ✨ GoliathLabs + + + frugan-dev +
+ frugan-dev + + tbutter @@ -641,15 +648,15 @@ Thanks goes to these wonderful people ✨
yogo1212 - + + willtho89
willtho89 - - + mpanneck @@ -658,10 +665,17 @@ Thanks goes to these wonderful people ✨ - - andrewlow + + ubenmackin
- andrewlow + ubenmackin + + + + + craue +
+ craue @@ -671,6 +685,14 @@ Thanks goes to these wonderful people ✨ abh + + + andrewlow +
+ andrewlow + + + aminvakil @@ -685,21 +707,6 @@ Thanks goes to these wonderful people ✨ elbracht - - - ubenmackin -
- ubenmackin - - - - - - craue -
- craue - - danielpanteleit @@ -727,120 +734,120 @@ Thanks goes to these wonderful people ✨
DuncanvR - + + emazzotta
emazzotta - - - - - fl42 -
- fl42 - - - ipernet + + nueaf
- ipernet + nueaf - - H4R0 + + martinwepner
- H4R0 + martinwepner - - eltociear + + artonge
- eltociear + artonge - - jamesfryer + + spacecowboy
- jamesfryer + spacecowboy - - millaguie + + jedateach
- millaguie + jedateach - - artonge + + millaguie
- artonge + millaguie - - jedateach + + eltociear
- jedateach + eltociear - - spacecowboy + + H4R0
- spacecowboy + H4R0 - - martinwepner + + jamesfryer
- martinwepner + jamesfryer - - nueaf + + ipernet
- nueaf + ipernet - - Thiritin + + fl42
- Thiritin + fl42 - - thomasschmit + + simonsystem
- thomasschmit + simonsystem - - TechnicLab + + stephan-devop
- TechnicLab + stephan-devop - - sylvaindumont + + stigok
- sylvaindumont + stigok + + + + + 5ven +
+ 5ven @@ -851,32 +858,46 @@ Thanks goes to these wonderful people ✨ - - 5ven + + sylvaindumont
- 5ven + sylvaindumont + + + + + + TechnicLab +
+ TechnicLab - - stigok + + ShiriNmi1520
- stigok + ShiriNmi1520 - - + - - stephan-devop + + thomasschmit
- stephan-devop + thomasschmit - - radicand + + Thiritin
- radicand + Thiritin + + + + + 42wim +
+ 42wim @@ -885,7 +906,8 @@ Thanks goes to these wonderful people ✨
tweibert - + + torus @@ -906,8 +928,7 @@ Thanks goes to these wonderful people ✨
Twist235 - - + k3it @@ -928,27 +949,6 @@ Thanks goes to these wonderful people ✨
vilisas - - - - 42wim -
- 42wim - - - - - ShiriNmi1520 -
- ShiriNmi1520 - - - - - Zepmann -
- Zepmann - @@ -1015,13 +1015,28 @@ Thanks goes to these wonderful people ✨ piwai + + + rahilarious +
+ rahilarious + + + + + 0xflotus +
+ 0xflotus + + remoe
remoe - + + romansey @@ -1035,8 +1050,7 @@ Thanks goes to these wonderful people ✨
norrs - - + MightySCollins @@ -1064,7 +1078,8 @@ Thanks goes to these wonderful people ✨
svdb0 - + + 3ap @@ -1078,8 +1093,7 @@ Thanks goes to these wonderful people ✨
shyim - - + sjmudd @@ -1088,17 +1102,10 @@ Thanks goes to these wonderful people ✨ - - simonsystem -
- simonsystem - - - - - allddd + + mchamplain
- allddd + mchamplain @@ -1114,15 +1121,15 @@ Thanks goes to these wonderful people ✨
mplx - + + odinis
odinis - - + okamidash @@ -1157,15 +1164,15 @@ Thanks goes to these wonderful people ✨
presocratics - + + rhyst
rhyst - - + rmlhuk @@ -1200,15 +1207,15 @@ Thanks goes to these wonderful people ✨
sportshead - + + squash
squash - - + strarsis @@ -1243,15 +1250,29 @@ Thanks goes to these wonderful people ✨
wolkenschieber - + + worldworm
worldworm - - + + + + Zepmann +
+ Zepmann + + + + + allddd +
+ allddd + + arcaine2 @@ -1272,7 +1293,8 @@ Thanks goes to these wonderful people ✨
brainkiller - + + cternes @@ -1293,8 +1315,7 @@ Thanks goes to these wonderful people ✨
dimalo - - + eleith @@ -1315,7 +1336,8 @@ Thanks goes to these wonderful people ✨
helmutundarnold - + + hnws @@ -1336,8 +1358,7 @@ Thanks goes to these wonderful people ✨
idaadi - - + ixeft @@ -1358,7 +1379,8 @@ Thanks goes to these wonderful people ✨
paralax - + + jpduyx @@ -1379,8 +1401,7 @@ Thanks goes to these wonderful people ✨
callmemagnus - - + marios88 @@ -1395,35 +1416,21 @@ Thanks goes to these wonderful people ✨ matrixes - - - mchamplain -
- mchamplain - - - - - crash7 -
- crash7 - - auchri
auchri - + + arkanovicz
arkanovicz - - + CBeerta @@ -1458,15 +1465,15 @@ Thanks goes to these wonderful people ✨
dbellavista - + + danielvandenberg95
danielvandenberg95 - - + denisix @@ -1501,15 +1508,15 @@ Thanks goes to these wonderful people ✨
vedtam - + + edvorg
edvorg - - + eliroca @@ -1544,20 +1551,20 @@ Thanks goes to these wonderful people ✨
felixn - + + flole
flole - - + - - 0xflotus + + froks
- 0xflotus + froks @@ -1587,15 +1594,15 @@ Thanks goes to these wonderful people ✨
askz - + + aspettl
aspettl - - + acch @@ -1630,15 +1637,15 @@ Thanks goes to these wonderful people ✨
ch3sh1r - + + eglia
eglia - - + groupmsl @@ -1673,15 +1680,15 @@ Thanks goes to these wonderful people ✨
arunvc - + + astrocket
astrocket - - + baxerus @@ -1704,10 +1711,10 @@ Thanks goes to these wonderful people ✨ - - akkumar + + crash7
- akkumar + crash7 @@ -1716,15 +1723,15 @@ Thanks goes to these wonderful people ✨
thechubbypanda - + + KCrawley
KCrawley - - + khuedoan @@ -1759,15 +1766,15 @@ Thanks goes to these wonderful people ✨
luke- - + + LucidityCrash
LucidityCrash - - + MadsRC @@ -1802,15 +1809,15 @@ Thanks goes to these wonderful people ✨
michaeljensen - + + exhuma
exhuma - - + milas @@ -1845,20 +1852,20 @@ Thanks goes to these wonderful people ✨
naveensrinivasan - + + neuralp
neuralp - - + - - froks + + radicand
- froks + radicand @@ -1868,13 +1875,6 @@ Thanks goes to these wonderful people ✨ fkefer - - - frugan-dev -
- frugan-dev - - Marsu31 @@ -1925,21 +1925,28 @@ Thanks goes to these wonderful people ✨ Influencer + + + JacksonZ03 +
+ JacksonZ03 + + JamBalaya56562
JamBalaya56562 - + + jcalfee
jcalfee - - + mivek @@ -1974,15 +1981,15 @@ Thanks goes to these wonderful people ✨
jirislav - + + jmccl
jmccl - - + jurekbarth @@ -2003,6 +2010,13 @@ Thanks goes to these wonderful people ✨
Kaan88 + + + + akkumar +
+ akkumar + From a815bf5ab44388baecb597a0b83f71872a2d34bc Mon Sep 17 00:00:00 2001 From: Robbert Klarenbeek Date: Fri, 16 Feb 2024 08:24:39 +0100 Subject: [PATCH 302/592] fix: Apply SELinux security context after moving to mail-state (#3890) * fix: Apply SELinux security context after moving to mail-state * fix: Ignore failing chcon on non-SELinux systems --- CHANGELOG.md | 1 + target/scripts/startup/setup.d/mail_state.sh | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8030f43dbc..2faf36f778a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,7 @@ The most noteworthy change of this release is the update of the container's base - `RELAY_HOST` ENV no longer enforces configuring outbound SMTP to require credentials. Like `DEFAULT_RELAY_HOST` it can now configure a relay where credentials are optional. - Restarting DMS should not be required when configuring relay hosts without these ENV, but solely via `setup relay ...`, as change detection events now apply relevant Postfix setting changes for supporting credentials too. - Rspamd configuration: Add a missing comma in `local_networks` so that all internal IP addresses are actually considered as internal ([#3862](https://github.com/docker-mailserver/docker-mailserver/pull/3862)) +- Ensure correct SELinux security context labels for files and directories moved to the mail-state volume during setup ([#3890](https://github.com/docker-mailserver/docker-mailserver/pull/3890)) ## [v13.3.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.3.1) diff --git a/target/scripts/startup/setup.d/mail_state.sh b/target/scripts/startup/setup.d/mail_state.sh index 9c43fea4339..5acf6762d69 100644 --- a/target/scripts/startup/setup.d/mail_state.sh +++ b/target/scripts/startup/setup.d/mail_state.sh @@ -48,6 +48,9 @@ function _setup_save_states() { _log 'trace' "Moving ${SERVICEFILE} to ${DEST}" # Empty volume was mounted, or new content from enabling a feature ENV: mv "${SERVICEFILE}" "${DEST}" + # Apply SELinux security context to match the state directory, so access + # is not restricted to the current running container: + chcon -R --reference="${STATEDIR}" "${DEST}" 2>/dev/null || true fi # Symlink the original file in the container ($SERVICEFILE) to be @@ -69,6 +72,9 @@ function _setup_save_states() { _log 'trace' "Moving contents of ${SERVICEDIR} to ${DEST}" # Empty volume was mounted, or new content from enabling a feature ENV: mv "${SERVICEDIR}" "${DEST}" + # Apply SELinux security context to match the state directory, so access + # is not restricted to the current running container: + chcon -R --reference="${STATEDIR}" "${DEST}" 2>/dev/null || true fi # Symlink the original path in the container ($SERVICEDIR) to be From d86c3cb1598d44afed1db60d3de8e3d1438dcd7c Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 20 Feb 2024 21:21:22 +1300 Subject: [PATCH 303/592] chore: `packages.sh` - Remove redundant comment (#3900) --- target/scripts/build/packages.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 1f769739c78..5360ae957bb 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -147,7 +147,6 @@ function _install_dovecot() { _log 'trace' 'Using Dovecot community repository' curl -sSfL https://repo.dovecot.org/DOVECOT-REPO-GPG | gpg --import gpg --export ED409DA1 > /etc/apt/trusted.gpg.d/dovecot.gpg - # VERSION_CODENAME sourced from /etc/os-release echo "deb https://repo.dovecot.org/ce-2.3-latest/debian/${VERSION_CODENAME} ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/dovecot.list _log 'trace' 'Updating Dovecot package signatures' From 67faa95b0b0e355afc84b86cf1b69ab89c674ab1 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 20 Feb 2024 21:33:04 +1300 Subject: [PATCH 304/592] fix(`setup`): `open-dkim` log for conflicting implementations (#3899) --- target/bin/open-dkim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/bin/open-dkim b/target/bin/open-dkim index 808ef8cc9aa..9410ecfab3a 100755 --- a/target/bin/open-dkim +++ b/target/bin/open-dkim @@ -5,7 +5,7 @@ source /usr/local/bin/helpers/index.sh if [[ -f /etc/dms-settings ]] && [[ $(_get_dms_env_value 'ENABLE_RSPAMD') -eq 1 ]]; then if [[ $(_get_dms_env_value 'ENABLE_OPENDKIM') -eq 1 ]]; then - log 'error' "You enabled Rspamd and OpenDKIM - OpenDKIM will be implicitly used for DKIM keys" + _log 'warn' "Conflicting DKIM support, both Rspamd and OpenDKIM enabled - OpenDKIM will manage DKIM keys" else /usr/local/bin/rspamd-dkim "${@}" exit From e232e43d32b4693a43a841250f2bd0c350f48c89 Mon Sep 17 00:00:00 2001 From: Casper Date: Wed, 21 Feb 2024 11:19:41 +0100 Subject: [PATCH 305/592] fix: fetchmail environment variables (#3901) --- CHANGELOG.md | 1 + target/supervisor/conf.d/supervisor-app.conf | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2faf36f778a..5b05b5d1d42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,7 @@ The most noteworthy change of this release is the update of the container's base - Restarting DMS should not be required when configuring relay hosts without these ENV, but solely via `setup relay ...`, as change detection events now apply relevant Postfix setting changes for supporting credentials too. - Rspamd configuration: Add a missing comma in `local_networks` so that all internal IP addresses are actually considered as internal ([#3862](https://github.com/docker-mailserver/docker-mailserver/pull/3862)) - Ensure correct SELinux security context labels for files and directories moved to the mail-state volume during setup ([#3890](https://github.com/docker-mailserver/docker-mailserver/pull/3890)) +- Use correct environment variable for fetchmail ([#3901](https://github.com/docker-mailserver/docker-mailserver/pull/3901)) ## [v13.3.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.3.1) diff --git a/target/supervisor/conf.d/supervisor-app.conf b/target/supervisor/conf.d/supervisor-app.conf index 50a7acc3e13..7f106456015 100644 --- a/target/supervisor/conf.d/supervisor-app.conf +++ b/target/supervisor/conf.d/supervisor-app.conf @@ -122,6 +122,7 @@ autorestart=true stdout_logfile=/var/log/supervisor/%(program_name)s.log stderr_logfile=/var/log/supervisor/%(program_name)s.log user=fetchmail +environment=HOME="/var/lib/fetchmail",USER="fetchmail" command=/usr/bin/fetchmail -f /etc/fetchmailrc --nodetach --daemon "%(ENV_FETCHMAIL_POLL)s" -i /var/lib/fetchmail/.fetchmail-UIDL-cache --pidfile /var/run/fetchmail/fetchmail.pid [program:postfix] From 95dfc71b54a6445190555b2f192cc4e195fe38dc Mon Sep 17 00:00:00 2001 From: Jesse Portnoy Date: Sat, 24 Feb 2024 23:06:58 +0000 Subject: [PATCH 306/592] Fix typo and broken README link (#3906) --- docs/content/config/environment.md | 2 +- docs/content/usage.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 323b6321e7f..a0916aa360d 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -212,7 +212,7 @@ Configures the handling of creating mails with forged sender addresses. ##### ENABLE_SRS -Enables the Sender Rewriting Scheme. SRS is needed if DMS acts as forwarder. See [postsrsd](https://github.com/roehling/postsrsd/blob/master/README.md#sender-rewriting-scheme-crash-course) for further explanation. +Enables the Sender Rewriting Scheme. SRS is needed if DMS acts as forwarder. See [postsrsd](https://github.com/roehling/postsrsd/blob/main/README.rst) for further explanation. - **0** => Disabled - 1 => Enabled diff --git a/docs/content/usage.md b/docs/content/usage.md index 30f6e822688..76011fe7690 100644 --- a/docs/content/usage.md +++ b/docs/content/usage.md @@ -2,7 +2,7 @@ title: Usage --- -This pages explains how to get started with DMS. The guide uses Docker Compose as a reference. In our examples, a volume mounts the host location [`docker-data/dms/config/`][docs::dms-volumes-config] to `/tmp/docker-mailserver/` inside the container. +This page explains how to get started with DMS. The guide uses Docker Compose as a reference. In our examples, a volume mounts the host location [`docker-data/dms/config/`][docs::dms-volumes-config] to `/tmp/docker-mailserver/` inside the container. [docs::dms-volumes-config]: ./config/advanced/optional-config.md#volumes-config From d3ccaddb708dcb427850e0dfe5bbbfee3047f8ec Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 25 Feb 2024 12:54:49 +0100 Subject: [PATCH 307/592] docs: updated `CONTRIBUTORS.md` (#3909) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 219 +++++++++++++++++++++++++----------------------- 1 file changed, 113 insertions(+), 106 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 1e252291891..c387a654e92 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -291,28 +291,21 @@ Thanks goes to these wonderful people ✨ pyy - - - dennis95stumm -
- dennis95stumm - - arneke
arneke - - + akmet
akmet - + + diiigle @@ -347,15 +340,15 @@ Thanks goes to these wonderful people ✨
lukecyca - - + jsonn
jsonn - + + jamebus @@ -364,17 +357,17 @@ Thanks goes to these wonderful people ✨ - - dashohoxha + + mathuin
- dashohoxha + mathuin - - mathuin + + dashohoxha
- mathuin + dashohoxha @@ -390,15 +383,15 @@ Thanks goes to these wonderful people ✨
weo - - + Zehir
Zehir - + + guardiande @@ -433,15 +426,15 @@ Thanks goes to these wonderful people ✨
VanVan - - + mjung
mjung - + + m-schmoock @@ -476,15 +469,15 @@ Thanks goes to these wonderful people ✨
Starbix - - + citec
citec - + + yajo @@ -519,15 +512,15 @@ Thanks goes to these wonderful people ✨
MakerMatrix - - + pbek
pbek - + + keslerm @@ -562,15 +555,15 @@ Thanks goes to these wonderful people ✨
bobbravo2 - - + r-pufky
r-pufky - + + vincentDcmps @@ -605,15 +598,15 @@ Thanks goes to these wonderful people ✨
j-marz - - + lokipo
lokipo - + + msheakoski @@ -648,15 +641,15 @@ Thanks goes to these wonderful people ✨
yogo1212 - - + willtho89
willtho89 - + + mpanneck @@ -691,15 +684,15 @@ Thanks goes to these wonderful people ✨
andrewlow - - + aminvakil
aminvakil - + + elbracht @@ -734,15 +727,15 @@ Thanks goes to these wonderful people ✨
DuncanvR - - + emazzotta
emazzotta - + + nueaf @@ -777,27 +770,34 @@ Thanks goes to these wonderful people ✨
jedateach - - + millaguie
millaguie + + + + + fl42 +
+ fl42 + - - eltociear + + ipernet
- eltociear + ipernet - - H4R0 + + eltociear
- H4R0 + eltociear @@ -808,17 +808,17 @@ Thanks goes to these wonderful people ✨ - - ipernet + + H4R0
- ipernet + H4R0 - - fl42 + + nilshoell
- fl42 + nilshoell @@ -872,13 +872,6 @@ Thanks goes to these wonderful people ✨ TechnicLab - - - ShiriNmi1520 -
- ShiriNmi1520 - - thomasschmit @@ -893,21 +886,13 @@ Thanks goes to these wonderful people ✨ Thiritin - - - 42wim -
- 42wim - - tweibert
tweibert - - + torus @@ -921,7 +906,8 @@ Thanks goes to these wonderful people ✨
VictorKoenders - + + Twist235 @@ -949,15 +935,22 @@ Thanks goes to these wonderful people ✨
vilisas - - + - - nilshoell + + 42wim
- nilshoell + 42wim + + + ShiriNmi1520 +
+ ShiriNmi1520 + + + nknapp @@ -992,15 +985,15 @@ Thanks goes to these wonderful people ✨
mrPjer - - + p3dda
p3dda - + + peter-hartmann @@ -1035,6 +1028,13 @@ Thanks goes to these wonderful people ✨
remoe + + + + robbertkl +
+ robbertkl + @@ -1717,14 +1717,21 @@ Thanks goes to these wonderful people ✨ crash7 + + + fkefer +
+ fkefer + + + thechubbypanda
thechubbypanda - - + KCrawley @@ -1759,15 +1766,15 @@ Thanks goes to these wonderful people ✨
linhandev - + + luke-
luke- - - + LucidityCrash @@ -1802,15 +1809,15 @@ Thanks goes to these wonderful people ✨
dragetd - + + michaeljensen
michaeljensen - - + exhuma @@ -1845,15 +1852,15 @@ Thanks goes to these wonderful people ✨
mpldr - + + naveensrinivasan
naveensrinivasan - - + neuralp @@ -1868,13 +1875,6 @@ Thanks goes to these wonderful people ✨ radicand - - - fkefer -
- fkefer - - Marsu31 @@ -1968,21 +1968,28 @@ Thanks goes to these wonderful people ✨ Jeidnx + + + jessp01 +
+ jessp01 + + JiLleON
JiLleON - + + jirislav
jirislav - - + jmccl From 2c1faa72444a132785362f61a08b9cf659a5d5a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Feb 2024 13:45:50 +0100 Subject: [PATCH 308/592] chore(deps): Bump myrotvorets/set-commit-status-action (#3911) --- .github/workflows/docs-preview-deploy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index 3b6f3c677f2..dbe57752f19 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -45,7 +45,7 @@ jobs: # but presently does not work correctly via split workflow. It is useful in a split workflow as the 1st stage # no longer indicates if the entire workflow/deployment was successful. - name: 'Commit Status: Set Workflow Status as Pending' - uses: myrotvorets/set-commit-status-action@v2.0.0 + uses: myrotvorets/set-commit-status-action@v2.0.1 with: token: ${{ secrets.GITHUB_TOKEN }} status: pending @@ -105,7 +105,7 @@ jobs: Built with commit: ${{ env.PR_HEADSHA }} - name: 'Commit Status: Update deployment status' - uses: myrotvorets/set-commit-status-action@v2.0.0 + uses: myrotvorets/set-commit-status-action@v2.0.1 # Always run this step regardless of job failing early: if: ${{ always() }} env: From 512f39c7ebc6edce8b5ae5302a2777c4f9173bea Mon Sep 17 00:00:00 2001 From: Dominic Germain Date: Wed, 28 Feb 2024 09:34:30 -0500 Subject: [PATCH 309/592] feat: Configurable number of rotated log files (#3907) --- CHANGELOG.md | 5 +++++ docs/content/config/environment.md | 6 ++++++ mailserver.env | 3 +++ target/scripts/startup/setup.d/log.sh | 8 +++++++- target/scripts/startup/setup.d/security/rspamd.sh | 2 +- target/scripts/startup/variables-stack.sh | 1 + 6 files changed, 23 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b05b5d1d42..009e2355774 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,11 @@ The most noteworthy change of this release is the update of the container's base - `undef` was previously supported as an opt-out with `SA_SPAM_SUBJECT`. This is no longer valid, the equivalent opt-out value is now an empty value (_or rather the omission of this ENV being configured_). - The feature to include [`_SCORE_` tag](https://spamassassin.apache.org/full/4.0.x/doc/Mail_SpamAssassin_Conf.html#rewrite_header-subject-from-to-STRING) in your value to be replaced by the associated spam score is no longer available. +### Added + +- **Environment Variables:** + - `LOGROTATE_COUNT` defines the number of files kept by logrotate ([#3907](https://github.com/docker-mailserver/docker-mailserver/pull/3907)) + ### Updates - **Environment Variables:** diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index a0916aa360d..83fcdfa607e 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -545,6 +545,12 @@ Changes the interval in which log files are rotated. This variable can also determine the interval for Postfix's log summary reports, see [`PFLOGSUMM_TRIGGER`](#pflogsumm_trigger). +##### LOGROTATE_COUNT + +Defines how many files are kept by logrotate. + +- **4** => Number of files + #### SpamAssassin ##### ENABLE_SPAMASSASSIN diff --git a/mailserver.env b/mailserver.env index 44a9a484eea..0d00de0afff 100644 --- a/mailserver.env +++ b/mailserver.env @@ -346,6 +346,9 @@ REPORT_SENDER= # Note: This variable can also determine the interval for Postfix's log summary reports, see [`PFLOGSUMM_TRIGGER`](#pflogsumm_trigger). LOGROTATE_INTERVAL=weekly +# Defines how many log files are kept by logrorate +LOGROTATE_COUNT=4 + # If enabled, employs `reject_unknown_client_hostname` to sender restrictions in Postfix's configuration. # diff --git a/target/scripts/startup/setup.d/log.sh b/target/scripts/startup/setup.d/log.sh index cf282966485..06aa679d706 100644 --- a/target/scripts/startup/setup.d/log.sh +++ b/target/scripts/startup/setup.d/log.sh @@ -19,13 +19,19 @@ function _setup_logrotate() { _dms_panic__invalid_value 'LOGROTATE_INTERVAL' 'Setup -> Logrotate' fi + if [[ ${LOGROTATE_COUNT} =~ ^[0-9]+$ ]]; then + _log 'trace' "Logrotate count set to ${LOGROTATE_COUNT}" + else + _dms_panic__invalid_value 'LOGROTATE_COUNT' 'Setup -> Logrotate' + fi + cat >/etc/logrotate.d/maillog << EOF /var/log/mail/mail.log { compress copytruncate delaycompress - rotate 4 + rotate ${LOGROTATE_COUNT} ${LOGROTATE_INTERVAL} } EOF diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 98f83de6528..6b54addd71a 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -109,7 +109,7 @@ function __rspamd__setup_logfile() { compress copytruncate delaycompress - rotate 4 + rotate ${LOGROTATE_COUNT} ${LOGROTATE_INTERVAL} } EOF diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index eb5bf149bcf..a3be72b84f8 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -145,6 +145,7 @@ function __environment_variables_general_setup() { VARS[GETMAIL_POLL]="${GETMAIL_POLL:=5}" VARS[LOG_LEVEL]="${LOG_LEVEL:=info}" VARS[LOGROTATE_INTERVAL]="${LOGROTATE_INTERVAL:=weekly}" + VARS[LOGROTATE_COUNT]="${LOGROTATE_COUNT:=4}" VARS[LOGWATCH_INTERVAL]="${LOGWATCH_INTERVAL:=none}" VARS[LOGWATCH_RECIPIENT]="${LOGWATCH_RECIPIENT:=${REPORT_RECIPIENT}}" VARS[LOGWATCH_SENDER]="${LOGWATCH_SENDER:=${REPORT_SENDER}}" From aa9465773c99a25dea7ca9534f1928ed6301693a Mon Sep 17 00:00:00 2001 From: Casper Date: Wed, 28 Feb 2024 22:08:19 +0100 Subject: [PATCH 310/592] Rename supervisor-app.conf to dms-services.conf (#3908) * rename supervisor-app.conf to dms-services.conf * changelog added --- CHANGELOG.md | 2 ++ .../conf.d/{supervisor-app.conf => dms-services.conf} | 0 .../set3/container_configuration/process_check_restart.bats | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) rename target/supervisor/conf.d/{supervisor-app.conf => dms-services.conf} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 009e2355774..62da354f0c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,8 @@ The most noteworthy change of this release is the update of the container's base - The default has changed to not prepend any prefix to the subject unless configured to do so. If you relied on the implicit prefix, you will now need to provide one explicitly. - `undef` was previously supported as an opt-out with `SA_SPAM_SUBJECT`. This is no longer valid, the equivalent opt-out value is now an empty value (_or rather the omission of this ENV being configured_). - The feature to include [`_SCORE_` tag](https://spamassassin.apache.org/full/4.0.x/doc/Mail_SpamAssassin_Conf.html#rewrite_header-subject-from-to-STRING) in your value to be replaced by the associated spam score is no longer available. +- **Supervisord**: + - `supervisor-app.conf` renamed to `dms-services.conf` ### Added diff --git a/target/supervisor/conf.d/supervisor-app.conf b/target/supervisor/conf.d/dms-services.conf similarity index 100% rename from target/supervisor/conf.d/supervisor-app.conf rename to target/supervisor/conf.d/dms-services.conf diff --git a/test/tests/parallel/set3/container_configuration/process_check_restart.bats b/test/tests/parallel/set3/container_configuration/process_check_restart.bats index 4f0fd90fb0b..78eabd7a704 100644 --- a/test/tests/parallel/set3/container_configuration/process_check_restart.bats +++ b/test/tests/parallel/set3/container_configuration/process_check_restart.bats @@ -184,7 +184,7 @@ function _check_if_process_is_running() { # `--list-full` provides information for matching against (full process path) # `--full` allows matching the process against the full path (required if a process is not the exec command, such as started by python3 command without a shebang) - # `--oldest` should select the parent process when there are multiple results, typically the command defined in `supervisor-app.conf` + # `--oldest` should select the parent process when there are multiple results, typically the command defined in `dms-services.conf` local IS_RUNNING=$(_exec_in_container pgrep --full --list-full "${MIN_SECS_RUNNING[@]}" --oldest "${PROCESS}") # When no matches are found, nothing is returned. Provide something we can assert on (helpful for debugging): @@ -199,7 +199,7 @@ function _check_if_process_is_running() { # The process manager (supervisord) should perform a graceful shutdown: # NOTE: Time limit should never be below these configured values: -# - supervisor-app.conf:stopwaitsecs +# - dms-services.conf:stopwaitsecs # - compose.yaml:stop_grace_period function _should_stop_cleanly() { run docker stop -t 60 "${CONTAINER_NAME}" From 736f2e44bcddac31b0bed506e5e60756d8dd032a Mon Sep 17 00:00:00 2001 From: Casper Date: Fri, 1 Mar 2024 01:00:23 +0100 Subject: [PATCH 311/592] Fail2Ban: Align logrotate count & interval (#3915) --- CHANGELOG.md | 1 + target/scripts/startup/setup.d/security/misc.sh | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62da354f0c4..6b663f469d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,7 @@ The most noteworthy change of this release is the update of the container's base - **Environment Variables:** - `LOGROTATE_COUNT` defines the number of files kept by logrotate ([#3907](https://github.com/docker-mailserver/docker-mailserver/pull/3907)) + - The fail2ban log file is now also taken into account by `LOGROTATE_COUNT` and `LOGROTATE_INTERVAL` ([#3915](https://github.com/docker-mailserver/docker-mailserver/pull/3915)) ### Updates diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index aefeba20bce..df810b2e766 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -189,14 +189,17 @@ function __setup__security__fail2ban() { _log 'debug' 'Enabling and configuring Fail2Ban' if [[ -e /tmp/docker-mailserver/fail2ban-fail2ban.cf ]]; then + _log 'trace' 'Custom fail2ban-fail2ban.cf found' cp /tmp/docker-mailserver/fail2ban-fail2ban.cf /etc/fail2ban/fail2ban.local fi if [[ -e /tmp/docker-mailserver/fail2ban-jail.cf ]]; then + _log 'trace' 'Custom fail2ban-jail.cf found' cp /tmp/docker-mailserver/fail2ban-jail.cf /etc/fail2ban/jail.d/user-jail.local fi if [[ ${FAIL2BAN_BLOCKTYPE} != 'reject' ]]; then + _log 'trace' "Setting fail2ban blocktype to 'drop'" echo -e '[Init]\nblocktype = drop' >/etc/fail2ban/action.d/nftables-common.local fi @@ -205,6 +208,9 @@ function __setup__security__fail2ban() { _log 'debug' 'Fail2Ban is disabled' rm -f /etc/logrotate.d/fail2ban fi + _log 'trace' 'Configuring fail2ban logrotate rotate count and interval' + sedfile -i "s|rotate 4$|rotate ${LOGROTATE_COUNT}|" /etc/logrotate.d/fail2ban + sedfile -i "s|weekly$|${LOGROTATE_INTERVAL}|" /etc/logrotate.d/fail2ban } function __setup__security__amavis() { From 12f5101d84331e23dea2192b05c502de5e610116 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sat, 2 Mar 2024 02:42:47 +0100 Subject: [PATCH 312/592] Rspamd: improve SPF, DKIM and DMARC Symbol Weights (#3913) --- CHANGELOG.md | 5 +- docs/content/config/security/rspamd.md | 32 +++++---- target/rspamd/scores.d/policies_group.conf | 84 +++++++++++----------- 3 files changed, 65 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b663f469d5..30c7dd0cb85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,8 +78,9 @@ The most noteworthy change of this release is the update of the container's base - **Tests:** - Refactored helper methods for sending e-mails with specific `Message-ID` headers and the helpers for retrieving + filtering logs, which together help isolate logs relevant to specific mail when multiple mails have been processed within a single test. ([#3786](https://github.com/docker-mailserver/docker-mailserver/pull/3786)) - **Rspamd**: - - The `rewrite_subject` action, is now disabled by default. It has been replaced with the new `SPAM_SUBJECT` environment variable, which implements the functionality via a Sieve script instead in favor of being anti-spam service agnostic ([3820](https://github.com/docker-mailserver/docker-mailserver/pull/3820)) - - `RSPAMD_NEURAL` was added and is disabled by default. If switched on it wil enable the experimental Rspamd Neural network module to add a layer of analysis to spam detection using neural network technology. ([3833](https://github.com/docker-mailserver/docker-mailserver/pull/3833)) + - The `rewrite_subject` action, is now disabled by default. It has been replaced with the new `SPAM_SUBJECT` environment variable, which implements the functionality via a Sieve script instead which is anti-spam service agnostic ([#3820](https://github.com/docker-mailserver/docker-mailserver/pull/3820)) + - `RSPAMD_NEURAL` was added and is disabled by default. If switched on it will enable the experimental Rspamd "Neural network" module to add a layer of analysis to spam detection ([#3833](https://github.com/docker-mailserver/docker-mailserver/pull/3833)) + - The symbol weights of SPF, DKIM and DMARC have been adjusted again. Fixes a bug and includes more appropriate combinations of symbols ([#3913](https://github.com/docker-mailserver/docker-mailserver/pull/3913)) ### Fixes diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 6dc5f202c7c..5cb901b740e 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -4,9 +4,9 @@ title: 'Security | Rspamd' ## About -Rspamd is a ["fast, free and open-source spam filtering system"][www::rspamd-homepage]. DMS integrates Rspamd like any other service. We provide a basic but easy to maintain setup of Rspamd. +Rspamd is a ["fast, free and open-source spam filtering system"][rspamd-web]. DMS integrates Rspamd like any other service. We provide a basic but easy to maintain setup of Rspamd. -If you want to take a look at the default configuration files for Rspamd that DMS packs, navigate to [`target/rspamd/` inside the repository][repo::dms-default-configuration]. Please consult the [section "The Default Configuration"](#the-default-configuration) section down below for a written overview. +If you want to take a look at the default configuration files for Rspamd that DMS packs, navigate to [`target/rspamd/` inside the repository][dms-repo::default-rspamd-configuration]. Please consult the [section "The Default Configuration"](#the-default-configuration) section down below for a written overview. ## Related Environment Variables @@ -56,7 +56,7 @@ And then there is a corresponding `X-Rspamd-Action` header, which shows the over X-Rspamd-Action no action ``` -Since the score is `-2.80`, nothing will happen and the e-mail is not classified as spam. Our custom [`actions.conf`][www::rspamd-actions-config] defines what to do at certain scores: +Since the score is `-2.80`, nothing will happen and the e-mail is not classified as spam. Our custom [`actions.conf`][dms-repo::rspamd-actions-config] defines what to do at certain scores: 1. At a score of 4, the e-mail is to be _greylisted_; 2. At a score of 6, the e-mail is _marked with a header_ (`X-Spam: Yes`); @@ -64,13 +64,13 @@ Since the score is `-2.80`, nothing will happen and the e-mail is not classified --- -There is more to spam analysis than meets the eye: we have not covered the [Bayes training and filters][www::rspamd-docs-bayes] here, nor have we discussed [Sieve rules for e-mails that are marked as spam][docs::spam-to-junk]. +There is more to spam analysis than meets the eye: we have not covered the [Bayes training and filters][rspamd-docs::bayes] here, nor have we discussed [Sieve rules for e-mails that are marked as spam][docs::spam-to-junk]. Even the calculation of the score with the individual symbols has been presented to you in a simplified manner. But with the knowledge from above, you're equipped to read on and use Rspamd confidently. Keep on reading to understand the integration even better - you will want to know about your anti-spam software, not only to keep the bad e-mail out, but also to make sure the good e-mail arrive properly! ### Workers -The proxy worker operates in [self-scan mode][www::rspamd-docs-proxy-self-scan-mode]. This simplifies the setup as we do not require a normal worker. You can easily change this though by [overriding the configuration by DMS](#providing-custom-settings-overriding-settings). +The proxy worker operates in [self-scan mode][rspamd-docs::proxy-self-scan-mode]. This simplifies the setup as we do not require a normal worker. You can easily change this though by [overriding the configuration by DMS](#providing-custom-settings-overriding-settings). DMS does not set a default password for the controller worker. You may want to do that yourself. In setups where you already have an authentication provider in front of the Rspamd webpage, you may want to [set the `secure_ip ` option to `"0.0.0.0/0"` for the controller worker](#with-the-help-of-a-custom-file) to disable password authentication inside Rspamd completely. @@ -88,18 +88,24 @@ Redis uses `/etc/redis/redis.conf` for configuration: ### Web Interface -Rspamd provides a [web interface][www::rspamd-docs-web-interface], which contains statistics and data Rspamd collects. The interface is enabled by default and reachable on port 11334. +Rspamd provides a [web interface][rspamd-docs::web-interface], which contains statistics and data Rspamd collects. The interface is enabled by default and reachable on port 11334. ![Rspamd Web Interface](https://rspamd.com/img/webui.png) ### DNS -DMS does not supply custom values for DNS servers to Rspamd. If you need to use custom DNS servers, which could be required when using [DNS-based black/whitelists](#rbls-realtime-blacklists-dnsbls-dns-based-blacklists), you need to adjust [`options.inc`][www::rspamd-docs-basic-options] yourself. +DMS does not supply custom values for DNS servers to Rspamd. If you need to use custom DNS servers, which could be required when using [DNS-based black/whitelists](#rbls-realtime-blacklists-dnsbls-dns-based-blacklists), you need to adjust [`options.inc`][rspamd-docs::basic-options] yourself. !!! tip "Making DNS Servers Configurable" If you want to see an environment variable (like `RSPAMD_DNS_SERVERS`) to support custom DNS servers for Rspamd being added to DMS, please raise a feature request issue. +!!! warning + + Rspamd heavily relies on a properly working DNS server that it can use to resolve DNS queries. If your DNS server is misconfigured, you will encounter issues when Rspamd queries DNS to assess if mail is spam. Legitimate mail is then unintentionally marked as spam or worse, rejected entirely. + + When Rspamd is deciding if mail is spam, it will check DNS records for SPF, DKIM and DMARC. Each of those has an associated symbol for DNS temporary errors with a non-zero weight assigned. That weight contributes towards the spam score assessed by Rspamd which is normally desirable - provided your network DNS is functioning correctly, otherwise when DNS is broken all mail is biased towards spam due to these failed DNS lookups. + !!! danger While we do not provide values for custom DNS servers by default, we set `soft_reject_on_timeout = true;` by default. This setting will cause a soft reject if a task (presumably a DNS request) timeout takes place. @@ -112,7 +118,7 @@ You can find the Rspamd logs at `/var/log/mail/rspamd.log`, and the correspondin ### Modules -You can find a list of all Rspamd modules [on their website][www::rspamd-docs-modules]. +You can find a list of all Rspamd modules [on their website][rspamd-docs::modules]. #### Disabled By Default @@ -140,9 +146,9 @@ DMS brings sane default settings for Rspamd. They are located at `/etc/rspamd/lo !!! question "What is [`docker-data/dms/config/`][docs::dms-volumes-config]?" -If you want to overwrite the default settings or provide your settings, you can place files at `docker-data/dms/config/rspamd/override.d/`. Files from this directory are copied to `/etc/rspamd/override.d/` during startup. These files [forcibly override][www::rspamd-docs-override-dir] Rspamd and DMS default settings. +If you want to overwrite the default settings or provide your settings, you can place files at `docker-data/dms/config/rspamd/override.d/`. Files from this directory are copied to `/etc/rspamd/override.d/` during startup. These files [forcibly override][rspamd-docs::override-dir] Rspamd and DMS default settings. -!!! question "What is the [`local.d` directory and how does it compare to `override.d`][www::rspamd-docs-config-directories]?" +!!! question "What is the [`local.d` directory and how does it compare to `override.d`][rspamd-docs::config-directories]?" !!! warning "Clashing Overrides" @@ -163,7 +169,7 @@ where `COMMAND` can be: 3. `set-option-for-module`: sets the value for option `ARGUMENT2` to `ARGUMENT3` inside module `ARGUMENT1` 4. `set-option-for-controller`: set the value of option `ARGUMENT1` to `ARGUMENT2` for the controller worker 5. `set-option-for-proxy`: set the value of option `ARGUMENT1` to `ARGUMENT2` for the proxy worker -6. `set-common-option`: set the option `ARGUMENT1` that [defines basic Rspamd behavior][www::rspamd-docs-basic-options] to value `ARGUMENT2` +6. `set-common-option`: set the option `ARGUMENT1` that [defines basic Rspamd behavior][rspamd-docs::basic-options] to value `ARGUMENT2` 7. `add-line`: this will add the complete line after `ARGUMENT1` (with all characters) to the file `/etc/rspamd/override.d/` !!! example "An Example Is [Shown Down Below](#adjusting-and-extending-the-very-basic-configuration)" @@ -231,11 +237,11 @@ There is a dedicated [section for setting up DKIM with Rspamd in our documentati ### _Abusix_ Integration -This subsection provides information about the integration of [Abusix][www::abusix], "a set of blocklists that work as an additional email security layer for your existing mail environment". The setup is straight-forward and well documented: +This subsection provides information about the integration of [Abusix][abusix-web], "a set of blocklists that work as an additional email security layer for your existing mail environment". The setup is straight-forward and well documented: 1. [Create an account](https://app.abusix.com/signup) 2. Retrieve your API key -3. Navigate to the ["Getting Started" documentation for Rspamd][www::abusix-rspamd-integration] and follow the steps described there +3. Navigate to the ["Getting Started" documentation for Rspamd][abusix-docs::rspamd-integration] and follow the steps described there 4. Make sure to change `` to your private API key We recommend mounting the files directly into the container, as they are rather big and not manageable with our [`custom-command.conf` script](#with-the-help-of-a-custom-file). If mounted to the correct location, Rspamd will automatically pick them up. diff --git a/target/rspamd/scores.d/policies_group.conf b/target/rspamd/scores.d/policies_group.conf index 5f9426e99fc..f2048665809 100644 --- a/target/rspamd/scores.d/policies_group.conf +++ b/target/rspamd/scores.d/policies_group.conf @@ -1,72 +1,75 @@ # Please refer to # https://github.com/docker-mailserver/docker-mailserver/issues/3690 # for understanding this file and its scores' values. +# +# This configuration is not 100% compliant with RFC7489. +# This is intentional! Rspamd has additional symbols than those defined in this file. +# 100% compliance is not desirable as those symbols will change the overall spam score. symbols = { # SPF - "R_SPF_ALLOW" { + "R_SPF_ALLOW" { # SPF check succeeded weight = -1; description = "SPF verification allows sending"; groups = ["spf"]; } - "R_SPF_NA" { + "R_SPF_NA" { # SPF is not available for this domain weight = 1.5; description = "Missing SPF record"; one_shot = true; groups = ["spf"]; } - "R_SPF_SOFTFAIL" { - weight = 2.5; - description = "SPF verification soft-failed"; - groups = ["spf"]; - } - "R_SPF_FAIL" { - weight = 4.5; - description = "SPF verification failed"; - groups = ["spf"]; - } - - "R_SPF_NEUTRAL" { # == R_SPF_NA + "R_SPF_NEUTRAL" { # same as R_SPF_NA weight = 1.5; description = "SPF policy is neutral"; groups = ["spf"]; } - "R_SPF_DNSFAIL" { # == R_SPF_SOFTFAIL + "R_SPF_SOFTFAIL" { # there was a temporary DNS issue and SPF could not be checked + weight = 2.5; + description = "SPF verification soft-failed"; + groups = ["spf"]; + } + "R_SPF_DNSFAIL" { # same as R_SPF_SOFTFAIL weight = 2.5; description = "SPF DNS failure"; groups = ["spf"]; } - "R_SPF_PERMFAIL" { # == R_SPF_FAIL + "R_SPF_FAIL" { # SPF check failed + weight = 4.5; + description = "SPF verification failed"; + groups = ["spf"]; + } + "R_SPF_PERMFAIL" { # same as R_SPF_FAIL weight = 4.5; description = "SPF record is malformed or persistent DNS error"; groups = ["spf"]; } # DKIM - "R_DKIM_ALLOW" { + "R_DKIM_ALLOW" { # DKIM check succeeded weight = -1; description = "DKIM verification succeed"; one_shot = true; groups = ["dkim"]; } - "R_DKIM_NA" { - weight = 0; + "R_DKIM_NA" { # DKIM is not available for this domain + weight = 1; description = "Missing DKIM signature"; one_shot = true; groups = ["dkim"]; } - "R_DKIM_TEMPFAIL" { + "R_DKIM_TEMPFAIL" { # there was a temporary DNS issue and DKIM could not be checked weight = 1.5; description = "DKIM verification soft-failed"; groups = ["dkim"]; } - "R_DKIM_PERMFAIL" { + "R_DKIM_PERMFAIL" { # DKIM check failed weight = 4.5; description = "DKIM verification hard-failed (invalid)"; groups = ["dkim"]; } - "R_DKIM_REJECT" { # == R_DKIM_PERMFAIL + "R_DKIM_REJECT" { # same as R_DKIM_PERMFAIL weight = 4.5; description = "DKIM verification failed"; one_shot = true; @@ -74,35 +77,34 @@ symbols = { } # DMARC - "DMARC_NA" { - weight = 1; - description = "No DMARC record"; + "DMARC_POLICY_ALLOW" { # DMARC check succeeded + weight = -1; + description = "DMARC permit policy"; groups = ["dmarc"]; } - "DMARC_POLICY_QUARANTINE" { - weight = 1.5; - description = "DMARC quarantine policy"; + "DMARC_POLICY_ALLOW_WITH_FAILURES" { # DMARC check succeeded but either SPF or DKIM was not successful + weight = 0; + description = "DMARC permit policy with DKIM/SPF failure"; groups = ["dmarc"]; } - "DMARC_POLICY_REJECT" { - weight = 2; - description = "DMARC reject policy"; + "DMARC_NA" { # DMARC is not available for this domain + weight = 0.5; + description = "No DMARC record"; groups = ["dmarc"]; } - - "DMARC_POLICY_ALLOW" { # no equivalent - weight = -1; - description = "DMARC permit policy"; + "DMARC_POLICY_SOFTFAIL" { # there was a temporary DNS issue and DMARC could not be checked + weight = 1.5; + description = "DMARC soft-failed"; groups = ["dmarc"]; } - "DMARC_POLICY_ALLOW_WITH_FAILURES" { # no equivalent - weight = -0.5; - description = "DMARC permit policy with DKIM/SPF failure"; + "DMARC_POLICY_QUARANTINE" { # DMARC check failed and the policy is to quarantine + weight = 3; + description = "DMARC quarantine policy"; groups = ["dmarc"]; } - "DMARC_POLICY_SOFTFAIL" { # == DMARC_POLICY_QUARANTINE - weight = 1.5; - description = "DMARC soft-failed"; + "DMARC_POLICY_REJECT" { # DMARC check failed and the policy is to reject + weight = 5.5; + description = "DMARC reject policy"; groups = ["dmarc"]; } } From 83a48e8958c0fcba9778117f235e6064b401f3a1 Mon Sep 17 00:00:00 2001 From: Casper Date: Sun, 3 Mar 2024 22:48:42 +0100 Subject: [PATCH 313/592] Fail2ban logrotate interval/count: substitute only when necessary (#3919) --- CHANGELOG.md | 2 +- target/scripts/startup/setup.d/security/misc.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30c7dd0cb85..b60a7afa2e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,7 +66,7 @@ The most noteworthy change of this release is the update of the container's base - **Environment Variables:** - `LOGROTATE_COUNT` defines the number of files kept by logrotate ([#3907](https://github.com/docker-mailserver/docker-mailserver/pull/3907)) - - The fail2ban log file is now also taken into account by `LOGROTATE_COUNT` and `LOGROTATE_INTERVAL` ([#3915](https://github.com/docker-mailserver/docker-mailserver/pull/3915)) + - The fail2ban log file is now also taken into account by `LOGROTATE_COUNT` and `LOGROTATE_INTERVAL` ([#3915](https://github.com/docker-mailserver/docker-mailserver/pull/3915), [#3919](https://github.com/docker-mailserver/docker-mailserver/pull/3919)) ### Updates diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index df810b2e766..303ae62dec4 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -209,8 +209,8 @@ function __setup__security__fail2ban() { rm -f /etc/logrotate.d/fail2ban fi _log 'trace' 'Configuring fail2ban logrotate rotate count and interval' - sedfile -i "s|rotate 4$|rotate ${LOGROTATE_COUNT}|" /etc/logrotate.d/fail2ban - sedfile -i "s|weekly$|${LOGROTATE_INTERVAL}|" /etc/logrotate.d/fail2ban + [[ ${LOGROTATE_COUNT} -ne 4 ]] && sedfile -i "s|rotate 4$|rotate ${LOGROTATE_COUNT}|" /etc/logrotate.d/fail2ban + [[ ${LOGROTATE_INTERVAL} != "weekly" ]] && sedfile -i "s|weekly$|${LOGROTATE_INTERVAL}|" /etc/logrotate.d/fail2ban } function __setup__security__amavis() { From 0c8d8f26d9323789569cab9ceac33e0b541380e0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 3 Mar 2024 22:50:06 +0100 Subject: [PATCH 314/592] docs: updated `CONTRIBUTORS.md` (#3916) --- CONTRIBUTORS.md | 304 +++++++++++++++++++++++++----------------------- 1 file changed, 156 insertions(+), 148 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index c387a654e92..ffe5cecd8f9 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -642,33 +642,26 @@ Thanks goes to these wonderful people ✨ yogo1212 - - - willtho89 -
- willtho89 - - - mpanneck
mpanneck - + + - - ubenmackin + + willtho89
- ubenmackin + willtho89 - - craue + + andrewlow
- craue + andrewlow @@ -678,21 +671,13 @@ Thanks goes to these wonderful people ✨ abh - - - andrewlow -
- andrewlow - - aminvakil
aminvakil - - + elbracht @@ -700,6 +685,21 @@ Thanks goes to these wonderful people ✨ elbracht + + + craue +
+ craue + + + + + + ubenmackin +
+ ubenmackin + + danielpanteleit @@ -737,38 +737,38 @@ Thanks goes to these wonderful people ✨ - - nueaf + + fl42
- nueaf + fl42 - - martinwepner + + ipernet
- martinwepner + ipernet - - artonge + + H4R0
- artonge + H4R0 - - spacecowboy + + eltociear
- spacecowboy + eltociear - - jedateach + + jamesfryer
- jedateach + jamesfryer @@ -780,67 +780,67 @@ Thanks goes to these wonderful people ✨ - - fl42 + + martinwepner
- fl42 + martinwepner - - ipernet + + artonge
- ipernet + artonge - - eltociear + + jedateach
- eltociear + jedateach - - jamesfryer + + spacecowboy
- jamesfryer + spacecowboy - - H4R0 + + nueaf
- H4R0 + nueaf - - nilshoell + + thomasschmit
- nilshoell + thomasschmit - - simonsystem + + TechnicLab
- simonsystem + TechnicLab - - stephan-devop + + sylvaindumont
- stephan-devop + sylvaindumont - - stigok + + syl20bnr
- stigok + syl20bnr @@ -851,32 +851,32 @@ Thanks goes to these wonderful people ✨ - - syl20bnr + + stigok
- syl20bnr + stigok - - sylvaindumont + + stephan-devop
- sylvaindumont + stephan-devop - - TechnicLab + + simonsystem
- TechnicLab + simonsystem - - thomasschmit + + radicand
- thomasschmit + radicand @@ -951,6 +951,13 @@ Thanks goes to these wonderful people ✨ + + + nilshoell +
+ nilshoell + + nknapp @@ -985,15 +992,15 @@ Thanks goes to these wonderful people ✨
mrPjer - + + p3dda
p3dda - - + peter-hartmann @@ -1015,13 +1022,6 @@ Thanks goes to these wonderful people ✨ rahilarious - - - 0xflotus -
- 0xflotus - - remoe @@ -1416,14 +1416,21 @@ Thanks goes to these wonderful people ✨ matrixes + + + 0xflotus +
+ 0xflotus + + + auchri
auchri - - + arkanovicz @@ -1458,15 +1465,15 @@ Thanks goes to these wonderful people ✨
dkarski - + + dbellavista
dbellavista - - + danielvandenberg95 @@ -1495,6 +1502,14 @@ Thanks goes to these wonderful people ✨ mazzz1y + + + doominator42 +
+ doominator42 + + + aydodo @@ -1508,8 +1523,7 @@ Thanks goes to these wonderful people ✨
vedtam - - + edvorg @@ -1537,7 +1551,8 @@ Thanks goes to these wonderful people ✨
ErikEngerd - + + huncode @@ -1551,8 +1566,7 @@ Thanks goes to these wonderful people ✨
felixn - - + flole @@ -1560,13 +1574,6 @@ Thanks goes to these wonderful people ✨ flole - - - froks -
- froks - - ifokeev @@ -1587,15 +1594,15 @@ Thanks goes to these wonderful people ✨
2b - + + askz
askz - - + aspettl @@ -1630,15 +1637,15 @@ Thanks goes to these wonderful people ✨
alexanderneu - + + ch3sh1r
ch3sh1r - - + eglia @@ -1673,15 +1680,15 @@ Thanks goes to these wonderful people ✨
MrFreezeex - + + arunvc
arunvc - - + astrocket @@ -1716,15 +1723,22 @@ Thanks goes to these wonderful people ✨
crash7 + + + + + froks +
+ froks + - - fkefer + + akkumar
- fkefer + akkumar - - + thechubbypanda @@ -1752,7 +1766,8 @@ Thanks goes to these wonderful people ✨
JustAnother1 - + + LeoWinterDE @@ -1766,8 +1781,7 @@ Thanks goes to these wonderful people ✨
linhandev - - + luke- @@ -1795,7 +1809,8 @@ Thanks goes to these wonderful people ✨
madmath03 - + + maxemann96 @@ -1809,8 +1824,7 @@ Thanks goes to these wonderful people ✨
dragetd - - + michaeljensen @@ -1838,7 +1852,8 @@ Thanks goes to these wonderful people ✨
mcchots - + + MohammedNoureldin @@ -1852,8 +1867,7 @@ Thanks goes to these wonderful people ✨
mpldr - - + naveensrinivasan @@ -1869,10 +1883,10 @@ Thanks goes to these wonderful people ✨ - - radicand + + fkefer
- radicand + fkefer @@ -1881,7 +1895,8 @@ Thanks goes to these wonderful people ✨
Marsu31 - + + glandais @@ -1895,8 +1910,7 @@ Thanks goes to these wonderful people ✨
GiovanH - - + harryyoud @@ -1924,7 +1938,8 @@ Thanks goes to these wonderful people ✨
Influencer - + + JacksonZ03 @@ -1938,8 +1953,7 @@ Thanks goes to these wonderful people ✨
JamBalaya56562 - - + jcalfee @@ -1967,7 +1981,8 @@ Thanks goes to these wonderful people ✨
Jeidnx - + + jessp01 @@ -1981,8 +1996,7 @@ Thanks goes to these wonderful people ✨
JiLleON - - + jirislav @@ -2010,20 +2024,14 @@ Thanks goes to these wonderful people ✨
JOduMonT - + + Kaan88
Kaan88 - - - - akkumar -
- akkumar - From 899b644a04cfbfed10d03d67a6864a2ef988ee32 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 18:19:57 +0100 Subject: [PATCH 315/592] chore(deps): Bump docker/setup-buildx-action from 3.0.0 to 3.1.0 (#3924) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index ccef46f5f06..660b0a2b803 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -79,7 +79,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.0.0 + uses: docker/setup-buildx-action@v3.1.0 # NOTE: AMD64 can build within 2 minutes - name: 'Build images' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 83357062015..bd776cfdd73 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -40,7 +40,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.0.0 + uses: docker/setup-buildx-action@v3.1.0 # Try get the cached build layers from a prior `generic_build.yml` job. # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 2c1d1045fd8..bc26ef4f64e 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -38,7 +38,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.0.0 + uses: docker/setup-buildx-action@v3.1.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 78af5ded299..f800cfb4b16 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -37,7 +37,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.0.0 + uses: docker/setup-buildx-action@v3.1.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. From d227d6dc7368274fa5c2b206e77b33274b4569bc Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Tue, 5 Mar 2024 20:33:04 +1300 Subject: [PATCH 316/592] docs: Reference systemd timer example (`cerbot renew`) (#3921) --- docs/content/config/security/ssl.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/content/config/security/ssl.md b/docs/content/config/security/ssl.md index 1b84c4dfe42..a917448361c 100644 --- a/docs/content/config/security/ssl.md +++ b/docs/content/config/security/ssl.md @@ -134,6 +134,8 @@ Certbot provisions certificates to `/etc/letsencrypt`. Add a volume to store the ``` This process can also be [automated via _cron_ or _systemd timers_][certbot::automated-renewal]. + + - [Example with a systemd timer][certbot::automated-renewal::example-systemd-timer] !!! note "Using a different ACME CA" @@ -903,6 +905,7 @@ Despite this, if you must use non-standard DH parameters or you would like to sw [certbot::standalone]: https://certbot.eff.org/docs/using.html#standalone [certbot::renew]: https://certbot.eff.org/docs/using.html#renewing-certificates [certbot::automated-renewal]: https://certbot.eff.org/docs/using.html#automated-renewals +[certbot::automated-renewal::example-systemd-timer]: https://github.com/orgs/docker-mailserver/discussions/3917#discussioncomment-8661690 [certbot::custom-ca]: https://certbot.eff.org/docs/using.htmlchanging-the-acme-server [certbot::webroot]: https://certbot.eff.org/docs/using.html#webroot From b5b193ca4ca114dd14e725ab4980cf011ed58633 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 5 Mar 2024 10:48:49 +0100 Subject: [PATCH 317/592] Rspamd: minor tweaks and follow-up for SPF, DKIM and DMARC symbols (#3923) * move `policies_group.conf` to correct location I originally assumed the file had to be placed into `scores.d`, but I now know that `local.d` is actually correct. * add configuration for composite symbols See updates to #3690: Additional Rspamd Symbols Rspamd has so-called composite symbols that trigger when a condition is met. Especially AUTH_NA and AUTH_NA_OR_FAIL will adjust the scores of various lines in the table above. This needs to be taken into account. * update CHANGELOG --- CHANGELOG.md | 2 +- Dockerfile | 1 - target/rspamd/local.d/composites.conf | 18 ++++++++++++++++++ .../{scores.d => local.d}/policies_group.conf | 0 4 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 target/rspamd/local.d/composites.conf rename target/rspamd/{scores.d => local.d}/policies_group.conf (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index b60a7afa2e2..1ec71deda30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,7 +80,7 @@ The most noteworthy change of this release is the update of the container's base - **Rspamd**: - The `rewrite_subject` action, is now disabled by default. It has been replaced with the new `SPAM_SUBJECT` environment variable, which implements the functionality via a Sieve script instead which is anti-spam service agnostic ([#3820](https://github.com/docker-mailserver/docker-mailserver/pull/3820)) - `RSPAMD_NEURAL` was added and is disabled by default. If switched on it will enable the experimental Rspamd "Neural network" module to add a layer of analysis to spam detection ([#3833](https://github.com/docker-mailserver/docker-mailserver/pull/3833)) - - The symbol weights of SPF, DKIM and DMARC have been adjusted again. Fixes a bug and includes more appropriate combinations of symbols ([#3913](https://github.com/docker-mailserver/docker-mailserver/pull/3913)) + - The symbol weights of SPF, DKIM and DMARC have been adjusted again. Fixes a bug and includes more appropriate combinations of symbols ([#3913](https://github.com/docker-mailserver/docker-mailserver/pull/3913), [#3923](https://github.com/docker-mailserver/docker-mailserver/pull/3923)) ### Fixes diff --git a/Dockerfile b/Dockerfile index 8110cebfd54..8b2a502c4e4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -104,7 +104,6 @@ EOF # ----------------------------------------------- COPY target/rspamd/local.d/ /etc/rspamd/local.d/ -COPY target/rspamd/scores.d/* /etc/rspamd/scores.d/ # ----------------------------------------------- # --- OAUTH2 ------------------------------------ diff --git a/target/rspamd/local.d/composites.conf b/target/rspamd/local.d/composites.conf new file mode 100644 index 00000000000..ab58f45ddd4 --- /dev/null +++ b/target/rspamd/local.d/composites.conf @@ -0,0 +1,18 @@ +# In addition to `policies_group.conf`, this file contains +# symbols that are applied when certain other symbols are +# applied (or not applied). +# +# We are especially interested in the `policy` field, because +# there are cases in which `remove_weight` is undesirable. + +# When neither SPF, DKIM, nor DMARC are available, we want +# to increase the base score so we apply at least greylisting. +AUTH_NA { + score = 2.5; + policy = "leave"; +} + +AUTH_NA_OR_FAIL { + score = 1; + policy = "leave"; +} diff --git a/target/rspamd/scores.d/policies_group.conf b/target/rspamd/local.d/policies_group.conf similarity index 100% rename from target/rspamd/scores.d/policies_group.conf rename to target/rspamd/local.d/policies_group.conf From e21e5e04902432079418e96fc5ccba3400dd5582 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 6 Mar 2024 08:44:34 +0100 Subject: [PATCH 318/592] Rspamd: update history key in Redis configuration (#3927) --- CHANGELOG.md | 4 +++- target/scripts/startup/setup.d/security/rspamd.sh | 9 +++++++++ test/tests/parallel/set1/spam_virus/rspamd_full.bats | 12 ++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ec71deda30..e1ba19b492d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,7 +52,7 @@ The most noteworthy change of this release is the update of the container's base - `smtp_sasl_security_options = noanonymous` (_credentials are mandatory for outbound mail delivery_) - `smtp_tls_security_level = encrypt` (_the outbound MTA connection must always be secure due to credentials sent_) - **Environment Variables**: - - `SA_SPAM_SUBJECT` has been renamed into `SPAM_SUBJECT` to become anti-spam service agnostic. ([3820](https://github.com/docker-mailserver/docker-mailserver/pull/3820)) + - `SA_SPAM_SUBJECT` has been renamed into `SPAM_SUBJECT` to become anti-spam service agnostic. ([#3820](https://github.com/docker-mailserver/docker-mailserver/pull/3820)) - As this functionality is now handled in Dovecot via a Sieve script instead of the respective anti-spam service during Postfix processing, this feature will only apply to mail stored in Dovecot. If you have relied on this feature in a different context, it will no longer be available. - Rspamd previously handled this functionality via the `rewrite_subject` action which as now been disabled by default in favor of the new approach with `SPAM_SUBJECT`. - `SA_SPAM_SUBJECT` is now deprecated and will log a warning if used. The value is copied as a fallback to `SPAM_SUBJECT`. @@ -61,6 +61,8 @@ The most noteworthy change of this release is the update of the container's base - The feature to include [`_SCORE_` tag](https://spamassassin.apache.org/full/4.0.x/doc/Mail_SpamAssassin_Conf.html#rewrite_header-subject-from-to-STRING) in your value to be replaced by the associated spam score is no longer available. - **Supervisord**: - `supervisor-app.conf` renamed to `dms-services.conf` +- **Rspamd**: + - the Redis history key has been changed in order to not incorporate the hostname of the container (which is desirable in Kubernetes environments) ([#3927](https://github.com/docker-mailserver/docker-mailserver/pull/3927)) ### Added diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 6b54addd71a..702a27ee7ad 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -126,6 +126,15 @@ function __rspamd__setup_redis() { servers = "127.0.0.1:6379"; expand_keys = true; +EOF + + # We do not use `{{HOSTNAME}}` but only `{{COMPRES}}` to better support + # Kubernetes, see https://github.com/orgs/docker-mailserver/discussions/3922 + cat >"${RSPAMD_LOCAL_D}/history_redis.conf" << "EOF" +# documentation: https://rspamd.com/doc/modules/history_redis.html + +key_prefix = "rs_history{{COMPRESS}}"; + EOF # Here we adjust the Redis default configuration that we supply to Redis when starting it. diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index 9b14860b315..32891a2746e 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -86,6 +86,18 @@ function teardown_file() { _default_teardown ; } refute_line --regexp 'rewrite_subject = [0-9]+;' } +@test 'Rspamd Redis configuration is correct' { + _run_in_container rspamadm configdump redis + assert_success + assert_line 'expand_keys = true;' + assert_line 'servers = "127.0.0.1:6379";' + + _run_in_container rspamadm configdump history_redis + assert_success + assert_line 'compress = true;' + assert_line 'key_prefix = "rs_history{{COMPRESS}}";' +} + @test "contents of '/etc/rspamd/override.d/' are copied" { local OVERRIDE_D='/etc/rspamd/override.d' _file_exists_in_container "${OVERRIDE_D}/testmodule_complicated.conf" From 364969919757216f97a205cf05e98dfae0c7c924 Mon Sep 17 00:00:00 2001 From: Kirill Kirilenko Date: Thu, 7 Mar 2024 01:13:22 +0300 Subject: [PATCH 319/592] fix: Move spam to mailbox associated to the `\Junk` special-use attribute (#3925) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 4 ++++ docs/content/config/environment.md | 8 ++++--- target/dovecot/90-sieve.conf | 2 +- .../scripts/startup/setup.d/security/misc.sh | 4 ++-- test/config/junk-mailbox/user-patches.sh | 21 +++++++++++++++++++ .../parallel/set1/spam_virus/rspamd_full.bats | 2 +- .../set1/spam_virus/spam_junk_folder.bats | 13 ++++++++---- 7 files changed, 43 insertions(+), 11 deletions(-) create mode 100755 test/config/junk-mailbox/user-patches.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index e1ba19b492d..de5642bf8f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,10 @@ The most noteworthy change of this release is the update of the container's base - DMS `main.cf` has renamed `postscreen_dnsbl_whitelist_threshold` to `postscreen_dnsbl_allowlist_threshold` as part of this change. - `smtpd_relay_restrictions` (relay policy) is now evaluated after `smtpd_recipient_restrictions` (spam policy). Previously it was evaluated before `smtpd_recipient_restrictions`. Mail to be relayed via DMS must now pass through the spam policy first. - The TLS fingerprint policy has changed the default from MD5 to SHA256 (_DMS does not modify this Postfix parameter, but may affect any user customizations that do_). +- **Dovecot** + - The "Junk" mailbox (folder) is now referenced by it's [special-use flag `\Junk`](https://docker-mailserver.github.io/docker-mailserver/v13.3/examples/use-cases/imap-folders/) instead of an explicit mailbox. ([#3925](https://github.com/docker-mailserver/docker-mailserver/pull/3925)) + - This provides compatibility for the Junk mailbox when it's folder name differs (_eg: Renamed to "Spam"_). + - Potential breakage if your deployment modifies our `spam_to_junk.sieve` sieve script (_which is created during container startup when ENV `MOVE_SPAM_TO_JUNK=1`_) that handles storing spam mail into a users "Junk" mailbox folder. - **rsyslog:** - Debian 12 adjusted the `rsyslog` configuration for the default file template from `RSYSLOG_TraditionalFileFormat` to `RSYSLOG_FileFormat` (_upstream default since 2012_). This change may affect you if you have any monitoring / analysis of log output (_eg: `mail.log` / `docker logs`_). - The two formats are roughly equivalent to [RFC 3164](https://www.rfc-editor.org/rfc/rfc3164)) and [RFC 5424](https://datatracker.ietf.org/doc/html/rfc5424#section-1) respectively. diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 83fcdfa607e..9adfd5d970c 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -328,10 +328,12 @@ Note: More information at ##### MOVE_SPAM_TO_JUNK -- 0 => Spam messages will be delivered in the mailbox. -- **1** => Spam messages will be delivered in the `Junk` folder. +- 0 => Spam messages will be delivered to the inbox. +- **1** => Spam messages will be delivered to the Junk mailbox. -Routes mail identified as spam into the recipient(s) Junk folder (_via a Dovecot Sieve script_). +Routes mail identified as spam into the recipient(s) Junk mailbox (_a specialized folder for junk associated to the [special-use flag `\Junk`][docs::dovecot::special-use-flag], handled via a Dovecot sieve script internally_). + +[docs::dovecot::special-use-flag]: ../examples/use-cases/imap-folders.md !!! info diff --git a/target/dovecot/90-sieve.conf b/target/dovecot/90-sieve.conf index 8b559be23d8..f388009a7c2 100644 --- a/target/dovecot/90-sieve.conf +++ b/target/dovecot/90-sieve.conf @@ -51,7 +51,7 @@ plugin { # deprecated imapflags extension in addition to all extensions were already # enabled by default. #sieve_extensions = +notify +imapflags - sieve_extensions = +notify +imapflags +vnd.dovecot.pipe +vnd.dovecot.filter + sieve_extensions = +notify +imapflags +special-use +vnd.dovecot.pipe +vnd.dovecot.filter # Which Sieve language extensions are ONLY available in global scripts. This # can be used to restrict the use of certain Sieve extensions to administrator diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index 303ae62dec4..bb460716784 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -304,11 +304,11 @@ function _setup_spam_to_junk() { _log 'debug' 'Spam emails will be moved to the Junk folder' mkdir -p /usr/lib/dovecot/sieve-global/after/ cat >/usr/lib/dovecot/sieve-global/after/spam_to_junk.sieve << EOF -require ["fileinto","mailbox"]; +require ["fileinto","special-use"]; if anyof (header :contains "X-Spam-Flag" "YES", header :contains "X-Spam" "Yes") { - fileinto "Junk"; + fileinto :specialuse "\\\\Junk" "Junk"; } EOF sievec /usr/lib/dovecot/sieve-global/after/spam_to_junk.sieve diff --git a/test/config/junk-mailbox/user-patches.sh b/test/config/junk-mailbox/user-patches.sh new file mode 100755 index 00000000000..0a16a21fca4 --- /dev/null +++ b/test/config/junk-mailbox/user-patches.sh @@ -0,0 +1,21 @@ +#!/bin/bash +## +# This user script will be executed between configuration and starting daemons +# To enable it you must save it in your config directory as "user-patches.sh" +## + +echo "[user-patches.sh] Adjusting 'Junk' mailbox name to verify delivery to Junk mailbox based on special-use flag instead of mailbox's name" + +sed -i -e 's/mailbox Junk/mailbox Spam/' /etc/dovecot/conf.d/15-mailboxes.conf + +### Before / After ### + +# mailbox Junk { +# auto = subscribe +# special_use = \Junk +# } + +# mailbox Spam { +# auto = subscribe +# special_use = \Junk +# } diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index 32891a2746e..3bc9fb1059d 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -244,7 +244,7 @@ function teardown_file() { _default_teardown ; } _service_log_should_contain_string 'rspamd' 'add header "Gtube pattern"' _print_mail_log_for_msgid 'rspamd-test-email-header' - assert_output --partial "fileinto action: stored mail into mailbox 'Junk'" + assert_output --partial "fileinto action: stored mail into mailbox [SPECIAL-USE \\Junk]" _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 2 _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/.Junk/new/ 1 diff --git a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats index 0c873382d74..5590d6f7deb 100644 --- a/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats +++ b/test/tests/parallel/set1/spam_virus/spam_junk_folder.bats @@ -49,9 +49,11 @@ function teardown() { _default_teardown ; } _should_receive_spam_at '/var/mail/localhost.localdomain/user1/new/' } -@test "(enabled + MOVE_SPAM_TO_JUNK=1) should deliver spam message into Junk folder" { +@test "(enabled + MOVE_SPAM_TO_JUNK=1) should deliver spam message into Junk mailbox" { export CONTAINER_NAME=${CONTAINER3_NAME} + _init_with_defaults + local CUSTOM_SETUP_ARGUMENTS=( --env ENABLE_AMAVIS=1 --env ENABLE_SPAMASSASSIN=1 @@ -60,14 +62,17 @@ function teardown() { _default_teardown ; } --env MOVE_SPAM_TO_JUNK=1 --env PERMIT_DOCKER=container ) - _init_with_defaults + + # Adjust 'Junk' mailbox name to verify delivery to Junk mailbox based on special-use flag instead of mailbox's name + mv "${TEST_TMP_CONFIG}/junk-mailbox/user-patches.sh" "${TEST_TMP_CONFIG}/" + _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' _should_send_spam_message _should_be_received_by_amavis 'Passed SPAM {RelayedTaggedInbound,Quarantined}' - # Should move delivered spam to Junk folder - _should_receive_spam_at '/var/mail/localhost.localdomain/user1/.Junk/new/' + # Should move delivered spam to the Junk mailbox (adjusted to be located at '.Spam/') + _should_receive_spam_at '/var/mail/localhost.localdomain/user1/.Spam/new/' } # NOTE: Same as test for `CONTAINER3_NAME`, only differing by ENV `MARK_SPAM_AS_READ=1` + `_should_receive_spam_at` location From 267fc552d220414be9f8ba7dbb2f0ed2431f3228 Mon Sep 17 00:00:00 2001 From: Casper Date: Sat, 9 Mar 2024 14:21:02 +0100 Subject: [PATCH 320/592] getmail: remove temp file usage (#3920) --- target/scripts/startup/setup.d/getmail.sh | 8 ++++---- target/scripts/startup/setup.d/security/rspamd.sh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/target/scripts/startup/setup.d/getmail.sh b/target/scripts/startup/setup.d/getmail.sh index a6c304c6b96..6f2801184d9 100644 --- a/target/scripts/startup/setup.d/getmail.sh +++ b/target/scripts/startup/setup.d/getmail.sh @@ -18,10 +18,10 @@ function _setup_getmail() { CONFIGS=1 ID=$(cut -d '-' -f 3 <<< "${FILE}" | cut -d '.' -f 1) local GETMAIL_CONFIG="${GETMAILRC}/getmailrc-${ID}" - cat /etc/getmailrc_general >"${GETMAIL_CONFIG}.tmp" - echo -e "message_log = /var/log/mail/getmail-${ID}.log\n" >>"${GETMAIL_CONFIG}.tmp" - cat "${GETMAIL_CONFIG}.tmp" "${FILE}" >"${GETMAIL_CONFIG}" - rm "${GETMAIL_CONFIG}.tmp" + + cat /etc/getmailrc_general >"${GETMAIL_CONFIG}" + echo -e "message_log = /var/log/mail/getmail-${ID}.log\n" >>"${GETMAIL_CONFIG}" + cat "${FILE}" >>"${GETMAIL_CONFIG}" fi done diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 702a27ee7ad..379162822f8 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -128,7 +128,7 @@ expand_keys = true; EOF - # We do not use `{{HOSTNAME}}` but only `{{COMPRES}}` to better support + # We do not use `{{HOSTNAME}}` but only `{{COMPRESS}}` to better support # Kubernetes, see https://github.com/orgs/docker-mailserver/discussions/3922 cat >"${RSPAMD_LOCAL_D}/history_redis.conf" << "EOF" # documentation: https://rspamd.com/doc/modules/history_redis.html From 8bdda5f43325c826f5e0bd42bf74d19bb83e0a8e Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Mon, 11 Mar 2024 20:02:22 +0900 Subject: [PATCH 321/592] Update user-patches.sh (#3932) --- test/config/rspamd_full/user-patches.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/config/rspamd_full/user-patches.sh b/test/config/rspamd_full/user-patches.sh index 9f4dc9fb985..f731ff36660 100644 --- a/test/config/rspamd_full/user-patches.sh +++ b/test/config/rspamd_full/user-patches.sh @@ -8,7 +8,7 @@ echo "enable_test_patterns = true;" >>/etc/rspamd/local.d/options.inc # We want Dovecot to be very detailed about what it is doing, -# specificially for Sieve because we need to check whether the +# specifically for Sieve because we need to check whether the # Sieve scripts are executed so Rspamd is trained when using # `RSPAMD_LEARN=1`. echo 'mail_debug = yes' >>/etc/dovecot/dovecot.conf From 9bc8869715f1d431274067c8b830d72cf861924e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 19:40:48 +0100 Subject: [PATCH 322/592] chore(deps): Bump docker/build-push-action from 5.1.0 to 5.2.0 (#3934) --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 660b0a2b803..aa3051c8e1b 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v5.1.0 + uses: docker/build-push-action@v5.2.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index bd776cfdd73..a729e5b8b2b 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -67,7 +67,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: 'Build and publish images' - uses: docker/build-push-action@v5.1.0 + uses: docker/build-push-action@v5.2.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index bc26ef4f64e..73db6f6c754 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v5.1.0 + uses: docker/build-push-action@v5.2.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index f800cfb4b16..8ca0b8b2f31 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v5.1.0 + uses: docker/build-push-action@v5.2.0 with: context: . tags: mailserver-testing:ci From a04b53f4f8ba39217739043059a5272e2ba84fd6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Mar 2024 11:46:34 +1300 Subject: [PATCH 323/592] chore(deps): Bump nwtgck/actions-netlify from 2.1 to 3.0 (#3933) Bumps [nwtgck/actions-netlify](https://github.com/nwtgck/actions-netlify) from 2.1 to 3.0. - [Release notes](https://github.com/nwtgck/actions-netlify/releases) - [Changelog](https://github.com/nwtgck/actions-netlify/blob/develop/CHANGELOG.md) - [Commits](https://github.com/nwtgck/actions-netlify/compare/v2.1...v3.0) --- updated-dependencies: - dependency-name: nwtgck/actions-netlify dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs-preview-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index dbe57752f19..7c924f8655b 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -55,7 +55,7 @@ jobs: context: 'Deploy Preview (pull_request => workflow_run)' - name: 'Send preview build to Netlify' - uses: nwtgck/actions-netlify@v2.1 + uses: nwtgck/actions-netlify@v3.0 id: preview timeout-minutes: 1 env: From 2133b51e782ae207475788a9b168b0008a5b450b Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Tue, 12 Mar 2024 09:31:44 +0100 Subject: [PATCH 324/592] docs: rewrite Kubernetes page (#3928) --- docs/content/assets/css/customizations.css | 34 + docs/content/config/advanced/kubernetes.md | 1200 ++++++++++------- .../tutorials/mailserver-behind-proxy.md | 7 +- docs/mkdocs.yml | 5 + 4 files changed, 782 insertions(+), 464 deletions(-) diff --git a/docs/content/assets/css/customizations.css b/docs/content/assets/css/customizations.css index 25cb0274992..7255a8377a5 100644 --- a/docs/content/assets/css/customizations.css +++ b/docs/content/assets/css/customizations.css @@ -107,3 +107,37 @@ div.md-content article.md-content__inner a.toclink code { .md-nav__item--nested > .md-nav__link { font-weight: 700; } + +/* ============================================================================================================= */ + +/* + TaskList style for a pro/con list. Presently only used for this type of list in the kubernetes docs. + Uses a custom icon for the unchecked (con) state: :octicons-x-circle-fill-24: + https://github.com/squidfunk/mkdocs-material/discussions/6811#discussioncomment-8700795 + + TODO: Can better scope the style under a class name when migrating to block extension syntax: + https://github.com/facelessuser/pymdown-extensions/discussions/1973 +*/ + +:root { + --md-tasklist-icon--failed: url('data:image/svg+xml;charset=utf-8,'); +} + +.md-typeset [type="checkbox"] + .task-list-indicator::before { + background-color: rgb(216, 87, 48); + -webkit-mask-image: var(--md-tasklist-icon--failed); + mask-image: var(--md-tasklist-icon--failed); +} + +/* More suitable shade of green */ +.md-typeset [type=checkbox]:checked+.task-list-indicator:before { + background-color: rgb(97, 216, 42); +} + +/* Tiny layout shift */ +[dir=ltr] .md-typeset .task-list-indicator:before { + left: -1.6em; + top: 1px; +} + +/* ============================================================================================================= */ diff --git a/docs/content/config/advanced/kubernetes.md b/docs/content/config/advanced/kubernetes.md index c277852a8a2..97a7e414e97 100644 --- a/docs/content/config/advanced/kubernetes.md +++ b/docs/content/config/advanced/kubernetes.md @@ -4,518 +4,794 @@ title: 'Advanced | Kubernetes' ## Introduction -This article describes how to deploy DMS to Kubernetes. Please note that there is also a [Helm chart] available. +This article describes how to deploy DMS to Kubernetes. We highly recommend everyone to use our community [DMS Helm chart][github-web::docker-mailserver-helm]. -!!! attention "Requirements" +!!! note "Requirements" - We assume basic knowledge about Kubernetes from the reader. Moreover, we assume the reader to have a basic understanding of mail servers. Ideally, the reader has deployed DMS before in an easier setup with Docker (Compose). + 1. Basic knowledge about Kubernetes from the reader. + 2. A basic understanding of mail servers. + 3. Ideally, the reader has already deployed DMS before with a simpler setup (_`docker run` or Docker Compose_). -!!! warning "About Support for Kubernetes" +!!! warning "Limited Support" - Please note that Kubernetes **is not** officially supported and we do not build images specifically designed for it. When opening an issue, please remember that only Docker & Docker Compose are officially supported. + DMS **does not officially support Kubernetes**. This content is entirely community-supported. If you find errors, please open an issue and raise a PR. - This content is entirely community-supported. If you find errors, please open an issue and provide a PR. +## Manually Writing Manifests -## Manifests +If using our Helm chart is not viable for you, here is some guidance to start with your own manifests. -### Configuration + +!!! quote "" -We want to provide the basic configuration in the form of environment variables with a `ConfigMap`. Note that this is just an example configuration; tune the `ConfigMap` to your needs. + === "`ConfigMap`" -```yaml ---- -apiVersion: v1 -kind: ConfigMap - -metadata: - name: mailserver.environment - -immutable: false - -data: - TLS_LEVEL: modern - POSTSCREEN_ACTION: drop - OVERRIDE_HOSTNAME: mail.example.com - FAIL2BAN_BLOCKTYPE: drop - POSTMASTER_ADDRESS: postmaster@example.com - UPDATE_CHECK_INTERVAL: 10d - POSTFIX_INET_PROTOCOLS: ipv4 - ENABLE_CLAMAV: '1' - ENABLE_POSTGREY: '0' - ENABLE_FAIL2BAN: '1' - AMAVIS_LOGLEVEL: '-1' - SPOOF_PROTECTION: '1' - MOVE_SPAM_TO_JUNK: '1' - ENABLE_UPDATE_CHECK: '1' - ENABLE_SPAMASSASSIN: '1' - SUPERVISOR_LOGLEVEL: warn - SPAMASSASSIN_SPAM_TO_INBOX: '1' - - # here, we provide an example for the SSL configuration - SSL_TYPE: manual - SSL_CERT_PATH: /secrets/ssl/rsa/tls.crt - SSL_KEY_PATH: /secrets/ssl/rsa/tls.key -``` - -We can also make use of user-provided configuration files, e.g. `user-patches.sh`, `postfix-accounts.cf` and more, to adjust DMS to our likings. We encourage you to have a look at [Kustomize][kustomize] for creating `ConfigMap`s from multiple files, but for now, we will provide a simple, hand-written example. This example is absolutely minimal and only goes to show what can be done. - -```yaml ---- -apiVersion: v1 -kind: ConfigMap + Provide the basic configuration via environment variables with a `ConfigMap`. + + !!! example -metadata: - name: mailserver.files + Below is only an example configuration, adjust the `ConfigMap` to your own needs. -data: - postfix-accounts.cf: | - test@example.com|{SHA512-CRYPT}$6$someHashValueHere - other@example.com|{SHA512-CRYPT}$6$someOtherHashValueHere -``` + ```yaml + --- + apiVersion: v1 + kind: ConfigMap -!!! attention "Static Configuration" + metadata: + name: mailserver.environment - With the configuration shown above, you can **not** dynamically add accounts as the configuration file mounted into the mail server can not be written to. + immutable: false - Use persistent volumes for production deployments. + data: + TLS_LEVEL: modern + POSTSCREEN_ACTION: drop + OVERRIDE_HOSTNAME: mail.example.com + FAIL2BAN_BLOCKTYPE: drop + POSTMASTER_ADDRESS: postmaster@example.com + UPDATE_CHECK_INTERVAL: 10d + POSTFIX_INET_PROTOCOLS: ipv4 + ENABLE_CLAMAV: '1' + ENABLE_POSTGREY: '0' + ENABLE_FAIL2BAN: '1' + AMAVIS_LOGLEVEL: '-1' + SPOOF_PROTECTION: '1' + MOVE_SPAM_TO_JUNK: '1' + ENABLE_UPDATE_CHECK: '1' + ENABLE_SPAMASSASSIN: '1' + SUPERVISOR_LOGLEVEL: warn + SPAMASSASSIN_SPAM_TO_INBOX: '1' -### Persistence + # here, we provide an example for the SSL configuration + SSL_TYPE: manual + SSL_CERT_PATH: /secrets/ssl/rsa/tls.crt + SSL_KEY_PATH: /secrets/ssl/rsa/tls.key + ``` -Thereafter, we need persistence for our data. Make sure you have a storage provisioner and that you choose the correct `storageClassName`. + You can also make use of user-provided configuration files (_e.g. `user-patches.sh`, `postfix-accounts.cf`, etc_), to customize DMS to your needs. -```yaml ---- -apiVersion: v1 -kind: PersistentVolumeClaim + ??? example "Providing config files" -metadata: - name: data + Here is a minimal example that supplies a `postfix-accounts.cf` file inline with two users: -spec: - storageClassName: local-path - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 25Gi -``` + ```yaml + --- + apiVersion: v1 + kind: ConfigMap -### Service + metadata: + name: mailserver.files -A `Service` is required for getting the traffic to the pod itself. The service is somewhat crucial. Its configuration determines whether the original IP from the sender will be kept. [More about this further down below](#exposing-your-mail-server-to-the-outside-world). + data: + postfix-accounts.cf: | + test@example.com|{SHA512-CRYPT}$6$someHashValueHere + other@example.com|{SHA512-CRYPT}$6$someOtherHashValueHere + ``` -The configuration you're seeing does keep the original IP, but you will not be able to scale this way. We have chosen to go this route in this case because we think most Kubernetes users will only want to have one instance. + !!! warning "Static Configuration" -```yaml ---- -apiVersion: v1 -kind: Service - -metadata: - name: mailserver - labels: - app: mailserver - -spec: - type: LoadBalancer - - selector: - app: mailserver - - ports: - # Transfer - - name: transfer - port: 25 - targetPort: transfer - protocol: TCP - # ESMTP with implicit TLS - - name: esmtp-implicit - port: 465 - targetPort: esmtp-implicit - protocol: TCP - # ESMTP with explicit TLS (STARTTLS) - - name: esmtp-explicit - port: 587 - targetPort: esmtp-explicit - protocol: TCP - # IMAPS with implicit TLS - - name: imap-implicit - port: 993 - targetPort: imap-implicit - protocol: TCP - -``` - -### Deployments - -Last but not least, the `Deployment` becomes the most complex component. It instructs Kubernetes how to run the DMS container and how to apply your `ConfigMaps`, persisted storage, etc. Additionally, we can set options to enforce runtime security here. - -```yaml ---- -apiVersion: apps/v1 -kind: Deployment - -metadata: - name: mailserver - - annotations: - ignore-check.kube-linter.io/run-as-non-root: >- - 'mailserver' needs to run as root - ignore-check.kube-linter.io/privileged-ports: >- - 'mailserver' needs privileged ports - ignore-check.kube-linter.io/no-read-only-root-fs: >- - There are too many files written to make The - root FS read-only - -spec: - replicas: 1 - selector: - matchLabels: - app: mailserver - - template: - metadata: - labels: - app: mailserver - - annotations: - container.apparmor.security.beta.kubernetes.io/mailserver: runtime/default - - spec: - hostname: mail - containers: - - name: mailserver - image: ghcr.io/docker-mailserver/docker-mailserver:latest - imagePullPolicy: IfNotPresent - - securityContext: - # Required to support SGID via `postdrop` executable - # in `/var/mail-state` for Postfix (maildrop + public dirs): - # https://github.com/docker-mailserver/docker-mailserver/pull/3625 - allowPrivilegeEscalation: true - readOnlyRootFilesystem: false - runAsUser: 0 - runAsGroup: 0 - runAsNonRoot: false - privileged: false - capabilities: - add: - # file permission capabilities - - CHOWN - - FOWNER - - MKNOD - - SETGID - - SETUID - - DAC_OVERRIDE - # network capabilities - - NET_ADMIN # needed for F2B - - NET_RAW # needed for F2B - - NET_BIND_SERVICE - # miscellaneous capabilities - - SYS_CHROOT - - KILL - drop: [ALL] - seccompProfile: - type: RuntimeDefault - - # You want to tune this to your needs. If you disable ClamAV, - # you can use less RAM and CPU. This becomes important in - # case you're low on resources and Kubernetes refuses to - # schedule new pods. - resources: - limits: - memory: 4Gi - cpu: 1500m - requests: - memory: 2Gi - cpu: 600m - - volumeMounts: - - name: files - subPath: postfix-accounts.cf - mountPath: /tmp/docker-mailserver/postfix-accounts.cf - readOnly: true - - # PVCs - - name: data - mountPath: /var/mail - subPath: data - readOnly: false - - name: data - mountPath: /var/mail-state - subPath: state - readOnly: false - - name: data - mountPath: /var/log/mail - subPath: log - readOnly: false - - # certificates - - name: certificates-rsa - mountPath: /secrets/ssl/rsa/ - readOnly: true - - # other - - name: tmp-files - mountPath: /tmp - readOnly: false - - ports: - - name: transfer - containerPort: 25 - protocol: TCP - - name: esmtp-implicit - containerPort: 465 - protocol: TCP - - name: esmtp-explicit - containerPort: 587 - - name: imap-implicit - containerPort: 993 - protocol: TCP - - envFrom: - - configMapRef: - name: mailserver.environment - - restartPolicy: Always - - volumes: - # configuration files - - name: files - configMap: - name: mailserver.files - - # PVCs - - name: data - persistentVolumeClaim: - claimName: data - - # certificates - - name: certificates-rsa - secret: - secretName: mail-tls-certificate-rsa - items: - - key: tls.key - path: tls.key - - key: tls.crt - path: tls.crt - - # other - - name: tmp-files - emptyDir: {} -``` - -### Certificates - An Example - -In this example, we use [`cert-manager`][cert-manager] to supply RSA certificates. You can also supply RSA certificates as fallback certificates, which DMS supports out of the box with `SSL_ALT_CERT_PATH` and `SSL_ALT_KEY_PATH`, and provide ECDSA as the proper certificates. - -```yaml ---- -apiVersion: cert-manager.io/v1 -kind: Certificate + The inline `postfix-accounts.cf` config example above provides file content that is static. It is mounted as read-only at runtime, thus cannot support modifications. + + For production deployments, use persistent volumes instead (via `PersistentVolumeClaim`). That will enable files like `postfix-account.cf` to add and remove accounts, while also persisting those changes externally from the container. + + !!! tip "Modularize your `ConfigMap`" + + [Kustomize][kustomize] can be a useful tool as it supports creating a `ConfigMap` from multiple files. + + === "`PersistentVolumeClaim`" + + To persist data externally from the DMS container, configure a `PersistentVolumeClaim` (PVC). + + Make sure you have a storage system (like Longhorn, Rook, etc.) and that you choose the correct `storageClassName` (according to your storage system). + + !!! example + + ```yaml + --- + apiVersion: v1 + kind: PersistentVolumeClaim + + metadata: + name: data -metadata: - name: mail-tls-certificate-rsa + spec: + storageClassName: local-path + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 25Gi + ``` -spec: - secretName: mail-tls-certificate-rsa - isCA: false - privateKey: - algorithm: RSA - encoding: PKCS1 - size: 2048 - dnsNames: [mail.example.com] - issuerRef: - name: mail-issuer - kind: Issuer -``` + === "`Service`" -!!! attention + A [`Service`][k8s-docs::config::service] is required for getting the traffic to the pod itself. It configures a load balancer with the ports you'll need. - You will need to have [`cert-manager`][cert-manager] configured. Especially the issue will need to be configured. Since we do not know how you want or need your certificates to be supplied, we do not provide more configuration here. The documentation for [`cert-manager`][cert-manager] is excellent. + The configuration for a `Service` affects if the original IP from a connecting client is preserved (_this is important_). [More about this further down below](#exposing-your-mail-server-to-the-outside-world). -### Sensitive Data + !!! example -!!! attention "Sensitive Data" + ```yaml + --- + apiVersion: v1 + kind: Service - For storing OpenDKIM keys, TLS certificates or any sort of sensitive data, you should be using `Secret`s. You can mount secrets like `ConfigMap`s and use them the same way. + metadata: + name: mailserver + labels: + app: mailserver -The [TLS docs page][docs-tls] provides guidance when it comes to certificates and transport layer security. Always provide sensitive information vai `Secrets`. + spec: + type: LoadBalancer + + selector: + app: mailserver + + ports: + # smtp + - name: smtp + port: 25 + targetPort: smtp + protocol: TCP + # submissions (ESMTP with implicit TLS) + - name: submission + port: 465 + targetPort: submissions + protocol: TCP + # submission (ESMTP with explicit TLS) + - name: submission + port: 587 + targetPort: submission + protocol: TCP + # imaps (implicit TLS) + - name: imaps + port: 993 + targetPort: imaps + protocol: TCP + ``` + + === "`Certificate`" + + !!! example "Using [`cert-manager`][cert-manager] to supply TLS certificates" + + ```yaml + --- + apiVersion: cert-manager.io/v1 + kind: Certificate + + metadata: + name: mail-tls-certificate-rsa + + spec: + secretName: mail-tls-certificate-rsa + isCA: false + privateKey: + algorithm: RSA + encoding: PKCS1 + size: 2048 + dnsNames: [mail.example.com] + issuerRef: + name: mail-issuer + kind: Issuer + ``` + + The [TLS docs page][docs-tls] provides guidance when it comes to certificates and transport layer security. + + !!! tip "ECDSA + RSA (fallback)" + + You could supply RSA certificates as fallback certificates instead, with ECDSA as the primary. DMS supports dual certificates via the ENV `SSL_ALT_CERT_PATH` and `SSL_ALT_KEY_PATH`. + + !!! warning "Always provide sensitive information via a `Secret`" + + For storing OpenDKIM keys, TLS certificates, or any sort of sensitive data - you should be using `Secret`s. + + A `Secret` is similar to `ConfigMap`, it can be used and mounted as a volume as demonstrated in the [`Deployment` manifest][docs::k8s::config-deployment] tab. + + === "`Deployment`" + + The [`Deployment`][k8s-docs::config::deployment] config is the most complex component. + + - It instructs Kubernetes how to run the DMS container and how to apply your `ConfigMap`s, persisted storage, etc. + - Additional options can be set to enforce runtime security. + + ???+ example + + ```yaml + --- + apiVersion: apps/v1 + kind: Deployment + + metadata: + name: mailserver + + annotations: + ignore-check.kube-linter.io/run-as-non-root: >- + 'mailserver' needs to run as root + ignore-check.kube-linter.io/privileged-ports: >- + 'mailserver' needs privileged ports + ignore-check.kube-linter.io/no-read-only-root-fs: >- + There are too many files written to make the root FS read-only + + spec: + replicas: 1 + selector: + matchLabels: + app: mailserver + + template: + metadata: + labels: + app: mailserver + + annotations: + container.apparmor.security.beta.kubernetes.io/mailserver: runtime/default + + spec: + hostname: mail + containers: + - name: mailserver + image: ghcr.io/docker-mailserver/docker-mailserver:latest + imagePullPolicy: IfNotPresent + + securityContext: + # `allowPrivilegeEscalation: true` is required to support SGID via the `postdrop` + # executable in `/var/mail-state` for Postfix (maildrop + public dirs): + # https://github.com/docker-mailserver/docker-mailserver/pull/3625 + allowPrivilegeEscalation: true + readOnlyRootFilesystem: false + runAsUser: 0 + runAsGroup: 0 + runAsNonRoot: false + privileged: false + capabilities: + add: + # file permission capabilities + - CHOWN + - FOWNER + - MKNOD + - SETGID + - SETUID + - DAC_OVERRIDE + # network capabilities + - NET_ADMIN # needed for F2B + - NET_RAW # needed for F2B + - NET_BIND_SERVICE + # miscellaneous capabilities + - SYS_CHROOT + - KILL + drop: [ALL] + seccompProfile: + type: RuntimeDefault + + # Tune this to your needs. + # If you disable ClamAV, you can use less RAM and CPU. + # This becomes important in case you're low on resources + # and Kubernetes refuses to schedule new pods. + resources: + limits: + memory: 4Gi + cpu: 1500m + requests: + memory: 2Gi + cpu: 600m + + volumeMounts: + - name: files + subPath: postfix-accounts.cf + mountPath: /tmp/docker-mailserver/postfix-accounts.cf + readOnly: true + + # PVCs + - name: data + mountPath: /var/mail + subPath: data + readOnly: false + - name: data + mountPath: /var/mail-state + subPath: state + readOnly: false + - name: data + mountPath: /var/log/mail + subPath: log + readOnly: false + + # certificates + - name: certificates-rsa + mountPath: /secrets/ssl/rsa/ + readOnly: true + + ports: + - name: smtp + containerPort: 25 + protocol: TCP + - name: submissions + containerPort: 465 + protocol: TCP + - name: submission + containerPort: 587 + - name: imaps + containerPort: 993 + protocol: TCP + + envFrom: + - configMapRef: + name: mailserver.environment + + restartPolicy: Always + + volumes: + # configuration files + - name: files + configMap: + name: mailserver.files + + # PVCs + - name: data + persistentVolumeClaim: + claimName: data + + # certificates + - name: certificates-rsa + secret: + secretName: mail-tls-certificate-rsa + items: + - key: tls.key + path: tls.key + - key: tls.crt + path: tls.crt + ``` ## Exposing your Mail Server to the Outside World -The more difficult part with Kubernetes is to expose a deployed DMS to the outside world. Kubernetes provides multiple ways for doing that; each has downsides and complexity. The major problem with exposing DMS to outside world in Kubernetes is to [preserve the real client IP][Kubernetes-service-source-ip]. The real client IP is required by DMS for performing IP-based SPF checks and spam checks. If you do not require SPF checks for incoming mails, you may disable them in your [Postfix configuration][docs-postfix] by dropping the line that states: `check_policy_service unix:private/policyd-spf`. +The more difficult part with Kubernetes is to expose a deployed DMS instance to the outside world. -The easiest approach was covered above, using `#!yaml externalTrafficPolicy: Local`, which disables the service proxy, but makes the service local as well (which does not scale). This approach only works when you are given the correct (that is, a public and routable) IP address by a load balancer (like MetalLB). In this sense, the approach above is similar to the next example below. We want to provide you with a few alternatives too. **But** we also want to communicate the idea of another simple method: you could use a load-balancer without an external IP and DNAT the network traffic to the mail server. After all, this does not interfere with SPF checks because it keeps the origin IP address. If no dedicated external IP address is available, you could try the latter approach, if one is available, use the former. +The major problem with exposing DMS to the outside world in Kubernetes is to [preserve the real client IP][k8s-docs::service-source-ip]. The real client IP is required by DMS for performing IP-based DNS and spam checks. -### External IPs Service +Kubernetes provides multiple ways to address this; each has its upsides and downsides. -The simplest way is to expose DMS as a [Service][Kubernetes-network-service] with [external IPs][Kubernetes-network-external-ip]. This is very similar to the approach taken above. Here, an external IP is given to the service directly by you. With the approach above, you tell your load-balancer to do this. + +!!! quote "" -```yaml ---- -apiVersion: v1 -kind: Service + === "Configure IP Manually" -metadata: - name: mailserver - labels: - app: mailserver + ???+ abstract "Advantages / Disadvantages" -spec: - selector: - app: mailserver - ports: - - name: smtp - port: 25 - targetPort: smtp - # ... + - [x] Simple + - [ ] Requires the node to have a dedicated, publicly routable IP address + - [ ] Limited to a single node (_associated to the dedicated IP address_) + - [ ] Your deployment requires an explicit IP in your configuration (_or an entire Load Balancer_). - externalIPs: - - 80.11.12.10 -``` + !!! info "Requirements" -This approach + 1. You can dedicate a **publicly routable IP** address for the DMS configured `Service`. + 2. A dedicated IP is required to allow your mail server to have matching `A` and `PTR` records (_which other mail servers will use to verify trust when they receive mail sent from your DMS instance_). -- does not preserve the real client IP, so SPF check of incoming mail will fail. -- requires you to specify the exposed IPs explicitly. + !!! example -### Proxy port to Service + Assign the DMS `Service` an external IP directly, or delegate an LB to assign the IP on your behalf. -The [proxy pod][Kubernetes-proxy-service] helps to avoid the necessity of specifying external IPs explicitly. This comes at the cost of complexity; you must deploy a proxy pod on each [Node][Kubernetes-nodes] you want to expose DMS on. + === "External-IP Service" -This approach + The DMS `Service` is configured with an "[external IP][k8s-docs::network-external-ip]" manually. Append your externally reachable IP address to `spec.externalIPs`. -- does not preserve the real client IP, so SPF check of incoming mail will fail. + ```yaml + --- + apiVersion: v1 + kind: Service -### Bind to concrete Node and use host network + metadata: + name: mailserver + labels: + app: mailserver -One way to preserve the real client IP is to use `hostPort` and `hostNetwork: true`. This comes at the cost of availability; you can reach DMS from the outside world only via IPs of [Node][Kubernetes-nodes] where DMS is deployed. + spec: + selector: + app: mailserver + ports: + - name: smtp + port: 25 + targetPort: smtp + # ... -```yaml ---- -apiVersion: extensions/v1beta1 -kind: Deployment - -metadata: - name: mailserver - -# ... - spec: - hostNetwork: true - - # ... - containers: - # ... - ports: - - name: smtp - containerPort: 25 - hostPort: 25 - - name: smtp-auth - containerPort: 587 - hostPort: 587 - - name: imap-secure - containerPort: 993 - hostPort: 993 - # ... -``` - -With this approach, - -- it is not possible to access DMS via other cluster Nodes, only via the Node DMS was deployed at. -- every Port within the Container is exposed on the Host side. - -### Proxy Port to Service via PROXY Protocol - -This way is ideologically the same as [using a proxy pod](#proxy-port-to-service), but instead of a separate proxy pod, you configure your ingress to proxy TCP traffic to the DMS pod using the PROXY protocol, which preserves the real client IP. - -#### Configure your Ingress - -With an [NGINX ingress controller][Kubernetes-nginx], set `externalTrafficPolicy: Local` for its service, and add the following to the TCP services config map (as described [here][Kubernetes-nginx-expose]): - -```yaml -25: "mailserver/mailserver:25::PROXY" -465: "mailserver/mailserver:465::PROXY" -587: "mailserver/mailserver:587::PROXY" -993: "mailserver/mailserver:993::PROXY" -``` - -!!! help "HAProxy" - With [HAProxy][dockerhub-haproxy], the configuration should look similar to the above. If you know what it actually looks like, add an example here. :smiley: - -#### Configure the Mailserver - -Then, configure both [Postfix][docs-postfix] and [Dovecot][docs-dovecot] to expect the PROXY protocol: - -??? example "HAProxy Example" - - ```yaml - kind: ConfigMap - apiVersion: v1 - metadata: - name: mailserver.config - labels: - app: mailserver - data: - postfix-main.cf: | - postscreen_upstream_proxy_protocol = haproxy - postfix-master.cf: | - smtp/inet/postscreen_upstream_proxy_protocol=haproxy - submission/inet/smtpd_upstream_proxy_protocol=haproxy - submissions/inet/smtpd_upstream_proxy_protocol=haproxy - dovecot.cf: | - # Assuming your ingress controller is bound to 10.0.0.0/8 - haproxy_trusted_networks = 10.0.0.0/8, 127.0.0.0/8 - service imap-login { - inet_listener imap { - haproxy = yes - } - inet_listener imaps { - haproxy = yes - } - } - # ... - --- - - kind: Deployment - apiVersion: extensions/v1beta1 - metadata: - name: mailserver - spec: - template: - spec: - containers: - - name: docker-mailserver - volumeMounts: - - name: config - subPath: postfix-main.cf - mountPath: /tmp/docker-mailserver/postfix-main.cf - readOnly: true - - name: config - subPath: postfix-master.cf - mountPath: /tmp/docker-mailserver/postfix-master.cf - readOnly: true - - name: config - subPath: dovecot.cf - mountPath: /tmp/docker-mailserver/dovecot.cf - readOnly: true - ``` - -With this approach, - -- it is not possible to access DMS via cluster-DNS, as the PROXY protocol is required for incoming connections. - -[Helm chart]: https://github.com/docker-mailserver/docker-mailserver-helm -[kustomize]: https://kustomize.io/ -[cert-manager]: https://cert-manager.io/docs/ + externalIPs: + - 10.20.30.40 + ``` + + === "Load-Balancer" + + The config differs depending on your choice of load balancer. This example uses [MetalLB][metallb-web]. + + ```yaml + --- + apiVersion: v1 + kind: Service + + metadata: + name: mailserver + labels: + app: mailserver + annotations: + metallb.universe.tf/address-pool: mailserver + + # ... + + --- + apiVersion: metallb.io/v1beta1 + kind: IPAddressPool + + metadata: + name: mail + namespace: metallb-system + + spec: + addresses: [ ] + autoAssign: true + + --- + apiVersion: metallb.io/v1beta1 + kind: L2Advertisement + + metadata: + name: mail + namespace: metallb-system + + spec: + ipAddressPools: [ mailserver ] + ``` + + === "Host network" + + ???+ abstract "Advantages / Disadvantages" + + - [x] Simple + - [ ] Requires the node to have a dedicated, publicly routable IP address + - [ ] Limited to a single node (_associated to the dedicated IP address_) + - [ ] It is not possible to access DMS via other cluster nodes, only via the node that DMS was deployed on + - [ ] Every port within the container is exposed on the host side + + !!! example + + Using `hostPort` and `hostNetwork: true` is a similar approach to [`network_mode: host` with Docker Compose][docker-docs::compose::network_mode]. + + ```yaml + --- + apiVersion: apps/v1 + kind: Deployment + + metadata: + name: mailserver + + # ... + spec: + hostNetwork: true + # ... + containers: + # ... + ports: + - name: smtp + containerPort: 25 + hostPort: 25 + - name: submissions + containerPort: 465 + hostPort: 465 + - name: submission + containerPort: 587 + hostPort: 587 + - name: imaps + containerPort: 993 + hostPort: 993 + ``` + + === "Using the PROXY Protocol" + + ???+ abstract "Advantages / Disadvantages" + + - [x] Preserves the origin IP address of clients (_which is crucial for DNS related checks_) + - [x] Aligns with a best practice for Kubernetes by using a dedicated ingress, routing external traffic to the k8s cluster (_with the benefits of flexible routing rules_) + - [x] Avoids the restraint of a single [node][k8s-docs::nodes] (_as a workaround to preserve the original client IP_) + - [ ] Introduces complexity by requiring: + - A reverse-proxy / ingress controller (_potentially extra setup_) + - Kubernetes manifest changes for the DMS configured `Service` + - DMS configuration changes for Postfix and Dovecot + - [ ] To keep support for direct connections to DMS services internally within cluster, service ports must be "duplicated" to offer an alternative port for connections using PROXY protocol + + ??? question "What is the PROXY protocol?" + + PROXY protocol is a network protocol for preserving a client’s IP address when the client’s TCP connection passes through a proxy. + + It is a common feature supported among reverse-proxy services (_NGINX, HAProxy, Traefik_), which you may already have handling ingress traffic for your cluster. + + ```mermaid + flowchart LR + A(External Mail Server) -->|Incoming connection| B + subgraph cluster + B("Ingress Acting as a Proxy") -->|PROXY protocol connection| C(DMS) + end + ``` + + For more information on the PROXY protocol, refer to [our dedicated docs page][docs-mailserver-behind-proxy] on the topic. + + ???+ example "Configure the Ingress Controller" + + === "Traefik" + + On Traefik's side, the configuration is very simple. + + - Create an entrypoint for each port that you want to expose (_probably 25, 465, 587 and 993_). + - Each entrypoint should configure an [`IngressRouteTCP`][traefik-docs::k8s::ingress-route-tcp] that routes to the equivalent internal DMS `Service` port which supports PROXY protocol connections. + + The below snippet demonstrates an example for two entrypoints, `submissions` (port 465) and `imaps` (port 993). + + ```yaml + --- + apiVersion: v1 + kind: Service + + metadata: + name: mailserver + + spec: + # This an optimization to get rid of additional routing steps. + # Previously "type: LoadBalancer" + type: ClusterIP + + --- + apiVersion: traefik.io/v1alpha1 + kind: IngressRouteTCP + + metadata: + name: smtp + + spec: + entryPoints: [ submissions ] + routes: + - match: HostSNI(`*`) + services: + - name: mailserver + namespace: mail + port: subs-proxy # note the 15 character limit here + proxyProtocol: + version: 2 + + --- + apiVersion: traefik.io/v1alpha1 + kind: IngressRouteTCP + + metadata: + name: imaps + + spec: + entryPoints: [ imaps ] + routes: + - match: HostSNI(`*`) + services: + - name: mailserver + namespace: mail + port: imaps-proxy + proxyProtocol: + version: 2 + ``` + + !!! info "`*-proxy` port name suffix" + + The `IngressRouteTCP` example configs above reference ports with a `*-proxy` suffix. + + - These port variants will be defined in the [`Deployment` manifest][docs::k8s::config-deployment], and are scoped to the `mailserver` service (via `spec.routes.services.name`). + - The suffix is used to distinguish that these ports are only compatible with connections using the PROXY protocol, which is what your ingress controller should be managing for you by adding the correct PROXY protocol headers to TCP connections it routes to DMS. + + === "NGINX" + + With an [NGINX ingress controller][k8s-docs::nginx], add the following to the TCP services config map (_as described [here][k8s-docs::nginx-expose]_): + + ```yaml + 25: "mailserver/mailserver:25::PROXY" + 465: "mailserver/mailserver:465::PROXY" + 587: "mailserver/mailserver:587::PROXY" + 993: "mailserver/mailserver:993::PROXY" + ``` + + ???+ example "Adjust DMS config for Dovecot + Postfix" + + ??? warning "Only ingress should connect to DMS with PROXY protocol" + + While Dovecot will restrict connections via PROXY protocol to only clients trusted configured via `haproxy_trusted_networks`, Postfix does not have an equivalent setting. Public clients should always route through ingress to establish a PROXY protocol connection. + + You are responsible for properly managing traffic inside your cluster and to **ensure that only trustworthy entities** can connect to the designated PROXY protocol ports. + + With Kubernetes, this is usually the task of the CNI (_container network interface_). + + !!! tip "Advised approach" + + The _"Separate PROXY protocol ports"_ tab below introduces a little more complexity, but provides better compatibility for internal connections to DMS. + + === "Only accept connections with PROXY protocol" + + !!! warning "Connections to DMS within the internal cluster will be rejected" + + The services for these ports can only enable PROXY protocol support by mandating the protocol on all connections for these ports. + + This can be problematic when you also need to support internal cluster traffic directly to DMS (_instead of routing indirectly through the ingress controller_). + + Here is an example configuration for [Postfix][docs-postfix], [Dovecot][docs-dovecot], and the required adjustments for the [`Deployment` manifest][docs::k8s::config-deployment]. The port names are adjusted here only to convey the additional context described earlier. + + ```yaml + kind: ConfigMap + apiVersion: v1 + metadata: + name: mailserver-extra-config + labels: + app: mailserver + data: + postfix-main.cf: | + postscreen_upstream_proxy_protocol = haproxy + postfix-master.cf: | + smtp/inet/postscreen_upstream_proxy_protocol=haproxy + submission/inet/smtpd_upstream_proxy_protocol=haproxy + submissions/inet/smtpd_upstream_proxy_protocol=haproxy + dovecot.cf: | + haproxy_trusted_networks = + service imap-login { + inet_listener imap { + haproxy = yes + } + inet_listener imaps { + haproxy = yes + } + } + # ... + + --- + kind: Deployment + apiVersion: apps/v1 + metadata: + name: mailserver + spec: + template: + spec: + containers: + - name: docker-mailserver + # ... + ports: + - name: smtp-proxy + containerPort: 25 + protocol: TCP + - name: imap-proxy + containerPort: 143 + protocol: TCP + - name: subs-proxy + containerPort: 465 + protocol: TCP + - name: sub-proxy + containerPort: 587 + protocol: TCP + - name: imaps-proxy + containerPort: 993 + protocol: TCP + # ... + volumeMounts: + - name: config + subPath: postfix-main.cf + mountPath: /tmp/docker-mailserver/postfix-main.cf + readOnly: true + - name: config + subPath: postfix-master.cf + mountPath: /tmp/docker-mailserver/postfix-master.cf + readOnly: true + - name: config + subPath: dovecot.cf + mountPath: /tmp/docker-mailserver/dovecot.cf + readOnly: true + ``` + + === "Separate PROXY protocol ports for ingress" + + !!! info + + Supporting internal cluster connections to DMS without using PROXY protocol requires both Postfix and Dovecot to be configured with alternative ports for each service port (_which only differ by enforcing PROXY protocol connections_). + + - The ingress controller will route public connections to the internal alternative ports for DMS (`*-proxy` variants). + - Internal cluster connections will instead use the original ports configured for the DMS container directly (_which are private to the cluster network_). + + In this example we'll create a copy of the original service ports with PROXY protocol enabled, and increment the port number assigned by `10000`. + + Create a `user-patches.sh` file to apply these config changes during container startup: + + ```bash + #!/bin/bash + + # Duplicate the config for the submission(s) service ports (587 / 465) with adjustments for the PROXY ports (10587 / 10465) and `syslog_name` setting: + postconf -Mf submission/inet | sed -e s/^submission/10587/ -e 's/submission/submission-proxyprotocol/' >> /etc/postfix/master.cf + postconf -Mf submissions/inet | sed -e s/^submissions/10465/ -e 's/submissions/submissions-proxyprotocol/' >> /etc/postfix/master.cf + # Enable PROXY Protocol support for these new service variants: + postconf -P 10587/inet/smtpd_upstream_proxy_protocol=haproxy + postconf -P 10465/inet/smtpd_upstream_proxy_protocol=haproxy + + # Create a variant for port 25 too (NOTE: Port 10025 is already assigned in DMS to Amavis): + postconf -Mf smtp/inet | sed -e s/^smtp/12525/ >> /etc/postfix/master.cf + # Enable PROXY Protocol support (different setting as port 25 is handled via postscreen), optionally configure a `syslog_name` to distinguish in logs: + postconf -P 12525/inet/postscreen_upstream_proxy_protocol=haproxy 12525/inet/syslog_name=smtp-proxyprotocol + ``` + + For Dovecot, you can configure [`dovecot.cf`][docs-dovecot] to look like this: + + ```cf + haproxy_trusted_networks = + + service imap-login { + inet_listener imap-proxied { + haproxy = yes + port = 10143 + } + + inet_listener imaps-proxied { + haproxy = yes + port = 10993 + ssl = yes + } + } + ``` + + Update the [`Deployment` manifest][docs::k8s::config-deployment] `ports` section by appending these new ports: + + ```yaml + - name: smtp-proxy + # not 10025 in this example due to a possible clash with Amavis + containerPort: 12525 + protocol: TCP + - name: imap-proxy + containerPort: 10143 + protocol: TCP + - name: subs-proxy + containerPort: 10465 + protocol: TCP + - name: sub-proxy + containerPort: 10587 + protocol: TCP + - name: imaps-proxy + containerPort: 10993 + protocol: TCP + ``` + + !!! note + + If you use other Dovecot ports (110, 995, 4190), you may want to configure those similar to above. The `dovecot.cf` config for these ports is [documented here][docs-mailserver-behind-proxy] (_in the equivalent section of that page_). + +[docs::k8s::config-deployment]: #deployment [docs-tls]: ../security/ssl.md [docs-dovecot]: ./override-defaults/dovecot.md [docs-postfix]: ./override-defaults/postfix.md -[dockerhub-haproxy]: https://hub.docker.com/_/haproxy -[Kubernetes-nginx]: https://kubernetes.github.io/ingress-nginx -[Kubernetes-nginx-expose]: https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services -[Kubernetes-network-service]: https://kubernetes.io/docs/concepts/services-networking/service -[Kubernetes-network-external-ip]: https://kubernetes.io/docs/concepts/services-networking/service/#external-ips -[Kubernetes-nodes]: https://kubernetes.io/docs/concepts/architecture/nodes -[Kubernetes-proxy-service]: https://github.com/kubernetes/contrib/tree/master/for-demos/proxy-to-service -[Kubernetes-service-source-ip]: https://kubernetes.io/docs/tutorials/services/source-ip +[docs-mailserver-behind-proxy]: ../../examples/tutorials/mailserver-behind-proxy.md + +[github-web::docker-mailserver-helm]: https://github.com/docker-mailserver/docker-mailserver-helm +[docker-docs::compose::network_mode]: https://docs.docker.com/compose/compose-file/compose-file-v3/#network_mode +[kustomize]: https://kustomize.io/ +[cert-manager]: https://cert-manager.io/docs/ +[metallb-web]: https://metallb.universe.tf/ + +[k8s-docs::config::service]: https://kubernetes.io/docs/concepts/services-networking/service +[k8s-docs::config::deployment]: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#creating-a-deployment +[k8s-docs::nodes]: https://kubernetes.io/docs/concepts/architecture/nodes +[k8s-docs::nginx]: https://kubernetes.github.io/ingress-nginx +[k8s-docs::nginx-expose]: https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services +[k8s-docs::service-source-ip]: https://kubernetes.io/docs/tutorials/services/source-ip +[k8s-docs::network-external-ip]: https://kubernetes.io/docs/concepts/services-networking/service/#external-ips + +[traefik-docs::k8s::ingress-route-tcp]: https://doc.traefik.io/traefik/routing/providers/kubernetes-crd/#kind-ingressroutetcp diff --git a/docs/content/examples/tutorials/mailserver-behind-proxy.md b/docs/content/examples/tutorials/mailserver-behind-proxy.md index 47116fde3c1..60e4c539c25 100644 --- a/docs/content/examples/tutorials/mailserver-behind-proxy.md +++ b/docs/content/examples/tutorials/mailserver-behind-proxy.md @@ -14,6 +14,8 @@ This reduces many of the benefits for why you might use a reverse proxy, but the Some deployments may require a service to route traffic (kubernetes) when deploying, in which case the below advice is important to understand well. +The guide here has also been adapted for [our Kubernetes docs][docs::kubernetes]. + ## What can go wrong? Without a reverse proxy involved, a service is typically aware of the client IP for a connection. @@ -354,9 +356,8 @@ Software on the receiving end of the connection often supports configuring an IP [`postscreen_access_list`][postfix-docs::settings::postscreen_access_list] (_or [`smtpd_client_restrictions`][postfix-docs::settings::smtpd_client_restrictions] with [`check_client_access`][postfix-docs::settings::check_client_access] for ports 587/465_) can both restrict access by IP via a [CIDR lookup table][postfix-docs::config-table::cidr], however the client IP is already rewritten at this point via PROXY protocol. Thus those settings cannot be used for restricting access to only trusted proxies, only to the actual clients. - - A similar setting [`mynetworks`][postfix-docs::settings::mynetworks] / [`PERMIT_DOCKER`][docs::env::permit_docker] manages elevated trust for bypassing security restrictions. While it is intended for trusted clients, it has no relevance to trusting proxies for the same reasons. + A similar setting [`mynetworks`][postfix-docs::settings::mynetworks] / [`PERMIT_DOCKER`][docs::env::permit_docker] manages elevated trust for bypassing security restrictions. While it is intended for trusted clients, it has no relevance to trusting proxies for the same reasons. ### Monitoring @@ -373,6 +374,8 @@ While PROXY protocol works well with the reverse proxy, you may have some contai You should adjust configuration of these monitoring services to monitor for auth failures from those services directly instead, adding an exclusion for that service IP from any DMS logs monitored (_but be mindful of PROXY header forgery risks_). +[docs::kubernetes]: ../../config/advanced/kubernetes.md#using-the-proxy-protocol + [docs::overrides::dovecot]: ../../config/advanced/override-defaults/dovecot.md [docs::overrides::postfix]: ../../config/advanced/override-defaults/postfix.md [docs::overrides::user-patches]: ../../config/advanced/override-defaults/user-patches.md diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 84366423fd6..e0ff50c9868 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -82,6 +82,11 @@ markdown_extensions: format: !!python/name:pymdownx.superfences.fence_code_format - pymdownx.tabbed: alternate_style: true + slugify: !!python/object/apply:pymdownx.slugs.slugify + kwds: + case: lower + - pymdownx.tasklist: + custom_checkbox: true - pymdownx.magiclink - pymdownx.inlinehilite - pymdownx.tilde From ede95e6f7fca0b02b756bec62d9f6cf81b64dfb1 Mon Sep 17 00:00:00 2001 From: Rahil Bhimjiani Date: Thu, 14 Mar 2024 02:44:14 +0530 Subject: [PATCH 325/592] docs: Update links for account management in `README.md` (#3937) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cfd453ff558..e34aeb12a4c 100644 --- a/README.md +++ b/README.md @@ -38,8 +38,8 @@ A production-ready fullstack but simple containerized mail server (SMTP, IMAP, L ## :package: Included Services -- [Postfix](http://www.postfix.org) with SMTP or LDAP authentication and support for [extension delimiters](https://docker-mailserver.github.io/docker-mailserver/latest/config/user-management/aliases/#address-tags-extension-delimiters-an-alternative-to-aliases) -- [Dovecot](https://www.dovecot.org) with SASL, IMAP, POP3, LDAP, [basic Sieve support](https://docker-mailserver.github.io/docker-mailserver/latest/config/advanced/mail-sieve) and [quotas](https://docker-mailserver.github.io/docker-mailserver/latest/config/user-management/accounts#notes) +- [Postfix](http://www.postfix.org) with SMTP or LDAP authentication and support for [extension delimiters](https://docker-mailserver.github.io/docker-mailserver/v13.3/config/user-management/#address-tags-extension-delimiters-as-an-alternative-to-aliases) +- [Dovecot](https://www.dovecot.org) with SASL, IMAP, POP3, LDAP, [basic Sieve support](https://docker-mailserver.github.io/docker-mailserver/latest/config/advanced/mail-sieve) and [quotas](https://docker-mailserver.github.io/docker-mailserver/v13.3/config/user-management/#quotas) - [Rspamd](https://rspamd.com/) - [Amavis](https://www.amavis.org/) - [SpamAssassin](http://spamassassin.apache.org/) supporting custom rules From cdcd86420e83984228dbb027d5f148f3cea9e0c6 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 14 Mar 2024 15:24:33 +1300 Subject: [PATCH 326/592] docs: Add IPv6 troubleshooting tip (#3938) Sometimes a user may have a configuration error and get halfway there. This should help point them in the right direction. --- docs/content/config/advanced/ipv6.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/content/config/advanced/ipv6.md b/docs/content/config/advanced/ipv6.md index 77dfb0c97f9..7e9c904cffb 100644 --- a/docs/content/config/advanced/ipv6.md +++ b/docs/content/config/advanced/ipv6.md @@ -177,6 +177,12 @@ docker run --rm -d --network dms-ipv6 -p 80:80 traefik/whoami curl --max-time 5 http://[2001:db8::1]:80 ``` +!!! warning "IPv6 gateway IP" + + If instead of the remote IPv6 address, you may notice the gateway IP for the IPv6 subnet your DMS container belongs to. + + This will happen when DMS has an IPv6 IP address assigned, for the same reason as with IPv4, `userland-proxy: true`. It indicates that your `daemon.json` has not been configured correctly or had the updated config applied for `ip6tables :true` + `experimental: true`. Make sure you used `systemctl restart docker` after updating `daemon.json`. + !!! info "IPv6 ULA address priority" DNS lookups that have records for both IPv4 and IPv6 addresses (_eg: `localhost`_) may prefer IPv4 over IPv6 (ULA) for private addresses, whereas for public addresses IPv6 has priority. This shouldn't be anything to worry about, but can come across as a surprise when testing your IPv6 setup on the same host instead of from a remote client. From 910667d5866cfdb2e04e893960bb009b626882fd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 13:22:43 +0100 Subject: [PATCH 327/592] docs: updated `CONTRIBUTORS.md` (#3930) --- CONTRIBUTORS.md | 383 ++++++++++++++++++++++++------------------------ 1 file changed, 195 insertions(+), 188 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index ffe5cecd8f9..9b8e40432ab 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -658,46 +658,46 @@ Thanks goes to these wonderful people ✨ - - andrewlow + + ubenmackin
- andrewlow + ubenmackin - - abh + + craue
- abh + craue - - aminvakil + + abh
- aminvakil + abh - - elbracht + + andrewlow
- elbracht + andrewlow - - craue + + aminvakil
- craue + aminvakil - - ubenmackin + + elbracht
- ubenmackin + elbracht @@ -737,38 +737,38 @@ Thanks goes to these wonderful people ✨ - - fl42 + + nueaf
- fl42 + nueaf - - ipernet + + martinwepner
- ipernet + martinwepner - - H4R0 + + artonge
- H4R0 + artonge - - eltociear + + spacecowboy
- eltociear + spacecowboy - - jamesfryer + + jedateach
- jamesfryer + jedateach @@ -780,67 +780,74 @@ Thanks goes to these wonderful people ✨ - - martinwepner + + eltociear
- martinwepner + eltociear - - artonge + + H4R0
- artonge + H4R0 - - jedateach + + jamesfryer
- jedateach + jamesfryer - - spacecowboy + + ipernet
- spacecowboy + ipernet - - nueaf + + fl42
- nueaf + fl42 - - thomasschmit + + radicand
- thomasschmit + radicand - - TechnicLab + + sjmudd
- TechnicLab + sjmudd - - sylvaindumont + + simonsystem
- sylvaindumont + simonsystem - - syl20bnr + + stephan-devop
- syl20bnr + stephan-devop + + + + + stigok +
+ stigok @@ -851,32 +858,46 @@ Thanks goes to these wonderful people ✨ - - stigok + + syl20bnr
- stigok + syl20bnr + + + + + + sylvaindumont +
+ sylvaindumont - - stephan-devop + + TechnicLab
- stephan-devop + TechnicLab - - + - - simonsystem + + 42wim
- simonsystem + 42wim - - radicand + + vilisas
- radicand + vilisas + + + + + thomasschmit +
+ thomasschmit @@ -885,7 +906,8 @@ Thanks goes to these wonderful people ✨
Thiritin - + + tweibert @@ -906,8 +928,7 @@ Thanks goes to these wonderful people ✨
VictorKoenders - - + Twist235 @@ -928,27 +949,6 @@ Thanks goes to these wonderful people ✨
Drakulix - - - - vilisas -
- vilisas - - - - - 42wim -
- 42wim - - - - - ShiriNmi1520 -
- ShiriNmi1520 - @@ -1015,6 +1015,13 @@ Thanks goes to these wonderful people ✨ piwai + + + auchri +
+ auchri + + rahilarious @@ -1028,15 +1035,15 @@ Thanks goes to these wonderful people ✨
remoe - + + robbertkl
robbertkl - - + romansey @@ -1071,15 +1078,15 @@ Thanks goes to these wonderful people ✨
klamann - + + svdb0
svdb0 - - + 3ap @@ -1095,10 +1102,10 @@ Thanks goes to these wonderful people ✨ - - sjmudd + + ShiriNmi1520
- sjmudd + ShiriNmi1520 @@ -1114,15 +1121,15 @@ Thanks goes to these wonderful people ✨
millerjason - + + mplx
mplx - - + odinis @@ -1157,15 +1164,15 @@ Thanks goes to these wonderful people ✨
pravynandas - + + presocratics
presocratics - - + rhyst @@ -1200,15 +1207,15 @@ Thanks goes to these wonderful people ✨
smargold476 - + + sportshead
sportshead - - + squash @@ -1243,15 +1250,15 @@ Thanks goes to these wonderful people ✨
wligtenberg - + + wolkenschieber
wolkenschieber - - + worldworm @@ -1286,15 +1293,15 @@ Thanks goes to these wonderful people ✨
awb99 - + + brainkiller
brainkiller - - + cternes @@ -1329,15 +1336,15 @@ Thanks goes to these wonderful people ✨
ghnp5 - + + helmutundarnold
helmutundarnold - - + hnws @@ -1372,15 +1379,15 @@ Thanks goes to these wonderful people ✨
jjtt - + + paralax
paralax - - + jpduyx @@ -1415,22 +1422,8 @@ Thanks goes to these wonderful people ✨
matrixes - - - - 0xflotus -
- 0xflotus - - - - auchri -
- auchri - - arkanovicz @@ -1465,15 +1458,15 @@ Thanks goes to these wonderful people ✨
dkarski - - + dbellavista
dbellavista - + + danielvandenberg95 @@ -1508,15 +1501,15 @@ Thanks goes to these wonderful people ✨
doominator42 - - + aydodo
aydodo - + + vedtam @@ -1551,15 +1544,15 @@ Thanks goes to these wonderful people ✨
ErikEngerd - - + huncode
huncode - + + felixn @@ -1574,6 +1567,20 @@ Thanks goes to these wonderful people ✨ flole + + + akkumar +
+ akkumar + + + + + 0xflotus +
+ 0xflotus + + ifokeev @@ -1587,15 +1594,15 @@ Thanks goes to these wonderful people ✨
20th - + + 2b
2b - - + askz @@ -1630,15 +1637,15 @@ Thanks goes to these wonderful people ✨
kachkaev - + + alexanderneu
alexanderneu - - + ch3sh1r @@ -1673,15 +1680,15 @@ Thanks goes to these wonderful people ✨
iRhonin - + + MrFreezeex
MrFreezeex - - + arunvc @@ -1716,28 +1723,14 @@ Thanks goes to these wonderful people ✨
erdos4d - + + crash7
crash7 - - - - - froks -
- froks - - - - - akkumar -
- akkumar - @@ -1760,6 +1753,13 @@ Thanks goes to these wonderful people ✨ khuedoan + + + UltraCoderRU +
+ UltraCoderRU + + JustAnother1 @@ -1882,21 +1882,28 @@ Thanks goes to these wonderful people ✨ neuralp + + + froks +
+ froks + + fkefer
fkefer - + + Marsu31
Marsu31 - - + glandais @@ -1931,15 +1938,15 @@ Thanks goes to these wonderful people ✨
sirgantrithon - + + Influencer
Influencer - - + JacksonZ03 @@ -1974,15 +1981,15 @@ Thanks goes to these wonderful people ✨
init-js - + + Jeidnx
Jeidnx - - + jessp01 @@ -2017,15 +2024,15 @@ Thanks goes to these wonderful people ✨
jurekbarth - + + JOduMonT
JOduMonT - - + Kaan88 From 066773e79fe8e49d7f9b4ae4ff85f1da75abab96 Mon Sep 17 00:00:00 2001 From: Casper Date: Sun, 17 Mar 2024 16:31:55 +0100 Subject: [PATCH 328/592] Better support regular container restarts (#3929) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- CHANGELOG.md | 3 +++ target/scripts/start-mailserver.sh | 36 +++++++++++++++++---------- target/scripts/startup/check-stack.sh | 9 ------- target/scripts/startup/setup-stack.sh | 2 +- 4 files changed, 27 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de5642bf8f2..31f906d5e80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,9 @@ The most noteworthy change of this release is the update of the container's base - `LOGROTATE_COUNT` defines the number of files kept by logrotate ([#3907](https://github.com/docker-mailserver/docker-mailserver/pull/3907)) - The fail2ban log file is now also taken into account by `LOGROTATE_COUNT` and `LOGROTATE_INTERVAL` ([#3915](https://github.com/docker-mailserver/docker-mailserver/pull/3915), [#3919](https://github.com/docker-mailserver/docker-mailserver/pull/3919)) +- **Internal:** + - Regular container restarts are now better supported. Setup scripts that ran previously will now be skipped ([#3929](https://github.com/docker-mailserver/docker-mailserver/pull/3929)) + ### Updates - **Environment Variables:** diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index f18abdf649a..cbe38da9251 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -33,7 +33,6 @@ function _register_functions() { # ? >> Checks - _register_check_function '_check_improper_restart' _register_check_function '_check_hostname' _register_check_function '_check_log_level' _register_check_function '_check_spam_prefix' @@ -170,24 +169,35 @@ function _register_functions() { # ? >> Executing all stacks / actual start of DMS # ------------------------------------------------------------ -_early_supervisor_setup -_early_variables_setup +# Ensure DMS only adjusts config files for a new container. +# Container restarts should skip as they retain the modified config. +if [[ ! -f /CONTAINER_START ]]; then + _early_supervisor_setup + _early_variables_setup -_log 'info' "Welcome to docker-mailserver ${DMS_RELEASE}" + _log 'info' "Welcome to docker-mailserver ${DMS_RELEASE}" -_register_functions -_check -_setup -[[ ${LOG_LEVEL} =~ (debug|trace) ]] && print-environment -_run_user_patches -_start_daemons + _register_functions + _check + _setup + _run_user_patches +else + # container was restarted + _early_variables_setup + + _log 'info' 'Container was restarted. Skipping setup routines.' + _log 'info' "Welcome to docker-mailserver ${DMS_RELEASE}" + + _register_functions +fi # marker to check if container was restarted date >/CONTAINER_START +[[ ${LOG_LEVEL} =~ (debug|trace) ]] && print-environment +_start_daemons + _log 'info' "${HOSTNAME} is up and running" touch /var/log/mail/mail.log -tail -Fn 0 /var/log/mail/mail.log - -exit 0 +exec tail -Fn 0 /var/log/mail/mail.log diff --git a/target/scripts/startup/check-stack.sh b/target/scripts/startup/check-stack.sh index cc98cfe2a55..766fcccf688 100644 --- a/target/scripts/startup/check-stack.sh +++ b/target/scripts/startup/check-stack.sh @@ -14,15 +14,6 @@ function _check() { done } -function _check_improper_restart() { - _log 'debug' 'Checking for improper restart' - - if [[ -f /CONTAINER_START ]]; then - _log 'warn' 'This container was (likely) improperly restarted which can result in undefined behavior' - _log 'warn' "Please use 'docker compose up --force-recreate' or equivalent (view our troubleshooting docs)" - fi -} - function _check_hostname() { _log 'debug' 'Checking that hostname/domainname is provided or overridden' diff --git a/target/scripts/startup/setup-stack.sh b/target/scripts/startup/setup-stack.sh index 81e1a98e059..8c8e646115e 100644 --- a/target/scripts/startup/setup-stack.sh +++ b/target/scripts/startup/setup-stack.sh @@ -40,7 +40,7 @@ function _early_supervisor_setup() { if ! grep -q "loglevel = ${SUPERVISOR_LOGLEVEL}" /etc/supervisor/supervisord.conf; then case "${SUPERVISOR_LOGLEVEL}" in ( 'critical' | 'error' | 'info' | 'debug' ) - sed -i -E \ + sedfile -i -E \ "s|(loglevel).*|\1 = ${SUPERVISOR_LOGLEVEL}|g" \ /etc/supervisor/supervisord.conf From 7017f4c0817efbfe810b8f61e343d026adff7d26 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Mar 2024 09:46:14 +1300 Subject: [PATCH 329/592] chore(deps): Bump docker/build-push-action from 5.2.0 to 5.3.0 (#3947) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 5.2.0 to 5.3.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v5.2.0...v5.3.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index aa3051c8e1b..6a29126dd71 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v5.2.0 + uses: docker/build-push-action@v5.3.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index a729e5b8b2b..7d9b61665f6 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -67,7 +67,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: 'Build and publish images' - uses: docker/build-push-action@v5.2.0 + uses: docker/build-push-action@v5.3.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 73db6f6c754..7b60f3196c1 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v5.2.0 + uses: docker/build-push-action@v5.3.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 8ca0b8b2f31..d9d5639b6d3 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v5.2.0 + uses: docker/build-push-action@v5.3.0 with: context: . tags: mailserver-testing:ci From 849293f88c8a5faebbe2c704badae92b923b2ac5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Mar 2024 20:48:35 +0000 Subject: [PATCH 330/592] chore(deps): Bump docker/setup-buildx-action from 3.1.0 to 3.2.0 (#3946) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3.1.0 to 3.2.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v3.1.0...v3.2.0) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 6a29126dd71..f0bd211123d 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -79,7 +79,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.1.0 + uses: docker/setup-buildx-action@v3.2.0 # NOTE: AMD64 can build within 2 minutes - name: 'Build images' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 7d9b61665f6..266090b6176 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -40,7 +40,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.1.0 + uses: docker/setup-buildx-action@v3.2.0 # Try get the cached build layers from a prior `generic_build.yml` job. # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 7b60f3196c1..4c49ca3e88b 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -38,7 +38,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.1.0 + uses: docker/setup-buildx-action@v3.2.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index d9d5639b6d3..23544820fc0 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -37,7 +37,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.1.0 + uses: docker/setup-buildx-action@v3.2.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. From 3125cad45a7895e93e4779aed1a7c86ad8cf3ee8 Mon Sep 17 00:00:00 2001 From: Casper Date: Thu, 21 Mar 2024 00:53:04 +0100 Subject: [PATCH 331/592] Enable spamassassin only, when amavis is enabled too. (#3943) --- CHANGELOG.md | 1 + target/scripts/startup/setup.d/security/misc.sh | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31f906d5e80..89efb03042c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,7 @@ The most noteworthy change of this release is the update of the container's base - It's only functionality remaining was to opt-out of run-time state consolidation with `ONE_DIR=0` (_when a volume was already mounted to `/var/mail-state`_). - **Internal:** - Changed the Postgrey whitelist retrieved during build to [source directly from Github](https://github.com/schweikert/postgrey/blob/master/postgrey_whitelist_clients) as the list is updated more frequently than the [author publishes at their website](https://postgrey.schweikert.ch) ([#3879](https://github.com/docker-mailserver/docker-mailserver/pull/3879)) + - Enable spamassassin only, when amavis is enabled too. ([#3943](https://github.com/docker-mailserver/docker-mailserver/pull/3943)) - **Tests:** - Refactored helper methods for sending e-mails with specific `Message-ID` headers and the helpers for retrieving + filtering logs, which together help isolate logs relevant to specific mail when multiple mails have been processed within a single test. ([#3786](https://github.com/docker-mailserver/docker-mailserver/pull/3786)) - **Rspamd**: diff --git a/target/scripts/startup/setup.d/security/misc.sh b/target/scripts/startup/setup.d/security/misc.sh index bb460716784..444589df49a 100644 --- a/target/scripts/startup/setup.d/security/misc.sh +++ b/target/scripts/startup/setup.d/security/misc.sh @@ -67,6 +67,11 @@ function __setup__security__postscreen() { } function __setup__security__spamassassin() { + if [[ ${ENABLE_AMAVIS} -ne 1 && ${ENABLE_SPAMASSASSIN} -eq 1 ]]; then + _log 'warn' 'Spamassassin does not work when Amavis is disabled. Enable Amavis to fix it.' + ENABLE_SPAMASSASSIN=0 + fi + if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]]; then _log 'debug' 'Enabling and configuring SpamAssassin' @@ -240,10 +245,6 @@ function __setup__security__amavis() { if [[ ${ENABLE_CLAMAV} -eq 1 ]] && [[ ${ENABLE_RSPAMD} -eq 0 ]]; then _log 'warn' 'ClamAV will not work when Amavis & rspamd are disabled. Enable either Amavis or rspamd to fix it.' fi - - if [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]]; then - _log 'warn' 'Spamassassin will not work when Amavis is disabled. Enable Amavis to fix it.' - fi fi } From 0dad7c49a4f4e6880c091ef0cdbed828cfd86068 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 21 Mar 2024 20:53:00 +0100 Subject: [PATCH 332/592] docs: updated `CONTRIBUTORS.md` (#3944) --- CONTRIBUTORS.md | 438 ++++++++++++++++++++++++------------------------ 1 file changed, 219 insertions(+), 219 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 9b8e40432ab..7c81c96882a 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -321,96 +321,96 @@ Thanks goes to these wonderful people ✨ - - robertdolca + + dashohoxha
- robertdolca + dashohoxha - - okainov + + egavard
- okainov + egavard - - lukecyca + + mathuin
- lukecyca + mathuin - - jsonn + + jamebus
- jsonn + jamebus - - jamebus + + robertdolca
- jamebus + robertdolca - - mathuin + + okainov
- mathuin + okainov - - dashohoxha + + jsonn
- dashohoxha + jsonn - - egavard + + lukecyca
- egavard + lukecyca - - weo + + m-schmoock
- weo + m-schmoock - - Zehir + + mjung
- Zehir + mjung - - guardiande + + eltociear
- guardiande + eltociear - - kamuri + + VanVan
- kamuri + VanVan - - davidszp + + voordev
- davidszp + voordev @@ -421,32 +421,32 @@ Thanks goes to these wonderful people ✨ - - VanVan + + davidszp
- VanVan + davidszp - - mjung + + kamuri
- mjung + kamuri - - m-schmoock + + guardiande
- m-schmoock + guardiande - - voordev + + Zehir
- voordev + Zehir @@ -485,6 +485,20 @@ Thanks goes to these wonderful people ✨ yajo + + + MakerMatrix +
+ MakerMatrix + + + + + weo +
+ weo + + analogue @@ -505,27 +519,20 @@ Thanks goes to these wonderful people ✨
reneploetz - - - - MakerMatrix -
- MakerMatrix - - + + pbek
pbek - - + - - keslerm + + yogo1212
- keslerm + yogo1212 @@ -542,6 +549,21 @@ Thanks goes to these wonderful people ✨ p-fruck + + + tbutter +
+ tbutter + + + + + rahilarious +
+ rahilarious + + + Rillke @@ -562,8 +584,7 @@ Thanks goes to these wonderful people ✨
r-pufky - - + vincentDcmps @@ -571,6 +592,21 @@ Thanks goes to these wonderful people ✨ vincentDcmps + + + frugan-dev +
+ frugan-dev + + + + + mpanneck +
+ mpanneck + + + andymel123 @@ -605,49 +641,13 @@ Thanks goes to these wonderful people ✨
lokipo - - + msheakoski
msheakoski - - - - GoliathLabs -
- GoliathLabs - - - - - frugan-dev -
- frugan-dev - - - - - tbutter -
- tbutter - - - - - yogo1212 -
- yogo1212 - - - - - mpanneck -
- mpanneck - @@ -657,6 +657,13 @@ Thanks goes to these wonderful people ✨ willtho89 + + + GoliathLabs +
+ GoliathLabs + + ubenmackin @@ -684,15 +691,15 @@ Thanks goes to these wonderful people ✨
andrewlow - + + aminvakil
aminvakil - - + elbracht @@ -727,21 +734,14 @@ Thanks goes to these wonderful people ✨
DuncanvR - + + emazzotta
emazzotta - - - - - nueaf -
- nueaf - @@ -772,25 +772,25 @@ Thanks goes to these wonderful people ✨ - - millaguie + + nueaf
- millaguie + nueaf - - eltociear + + keslerm
- eltociear + keslerm - - H4R0 + + millaguie
- H4R0 + millaguie @@ -801,32 +801,32 @@ Thanks goes to these wonderful people ✨ - - ipernet + + fl42
- ipernet + fl42 - - fl42 + + H4R0
- fl42 + H4R0 - - radicand + + ipernet
- radicand + ipernet - - sjmudd + + neuralp
- sjmudd + neuralp @@ -879,20 +879,6 @@ Thanks goes to these wonderful people ✨ TechnicLab - - - 42wim -
- 42wim - - - - - vilisas -
- vilisas - - thomasschmit @@ -906,8 +892,7 @@ Thanks goes to these wonderful people ✨
Thiritin - - + tweibert @@ -921,7 +906,8 @@ Thanks goes to these wonderful people ✨
torus - + + VictorKoenders @@ -949,8 +935,36 @@ Thanks goes to these wonderful people ✨
Drakulix + + + + vilisas +
+ vilisas + + + + + 42wim +
+ 42wim + + + + ShiriNmi1520 +
+ ShiriNmi1520 + + + + + radicand +
+ radicand + + nilshoell @@ -978,7 +992,8 @@ Thanks goes to these wonderful people ✨
OrvilleQ - + + ovidiucp @@ -992,8 +1007,7 @@ Thanks goes to these wonderful people ✨
mrPjer - - + p3dda @@ -1015,20 +1029,6 @@ Thanks goes to these wonderful people ✨ piwai - - - auchri -
- auchri - - - - - rahilarious -
- rahilarious - - remoe @@ -1102,10 +1102,10 @@ Thanks goes to these wonderful people ✨ - - ShiriNmi1520 + + sjmudd
- ShiriNmi1520 + sjmudd @@ -1424,6 +1424,20 @@ Thanks goes to these wonderful people ✨ + + + 0xflotus +
+ 0xflotus + + + + + auchri +
+ auchri + + arkanovicz @@ -1451,7 +1465,8 @@ Thanks goes to these wonderful people ✨
espitall - + + dkarski @@ -1465,8 +1480,7 @@ Thanks goes to these wonderful people ✨
dbellavista - - + danielvandenberg95 @@ -1494,7 +1508,8 @@ Thanks goes to these wonderful people ✨
mazzz1y - + + doominator42 @@ -1508,8 +1523,7 @@ Thanks goes to these wonderful people ✨
aydodo - - + vedtam @@ -1537,7 +1551,8 @@ Thanks goes to these wonderful people ✨
ekkis - + + ErikEngerd @@ -1551,8 +1566,7 @@ Thanks goes to these wonderful people ✨
huncode - - + felixn @@ -1567,20 +1581,6 @@ Thanks goes to these wonderful people ✨ flole - - - akkumar -
- akkumar - - - - - 0xflotus -
- 0xflotus - - ifokeev @@ -1732,6 +1732,20 @@ Thanks goes to these wonderful people ✨ crash7 + + + froks +
+ froks + + + + + akkumar +
+ akkumar + + thechubbypanda @@ -1752,7 +1766,8 @@ Thanks goes to these wonderful people ✨
khuedoan - + + UltraCoderRU @@ -1766,8 +1781,7 @@ Thanks goes to these wonderful people ✨
JustAnother1 - - + LeoWinterDE @@ -1795,7 +1809,8 @@ Thanks goes to these wonderful people ✨
LucidityCrash - + + MadsRC @@ -1809,8 +1824,7 @@ Thanks goes to these wonderful people ✨
madmath03 - - + maxemann96 @@ -1838,7 +1852,8 @@ Thanks goes to these wonderful people ✨
exhuma - + + milas @@ -1852,8 +1867,7 @@ Thanks goes to these wonderful people ✨
mcchots - - + MohammedNoureldin @@ -1875,20 +1889,6 @@ Thanks goes to these wonderful people ✨ naveensrinivasan - - - neuralp -
- neuralp - - - - - froks -
- froks - - fkefer From 082e076377b48378761890a6a39acfda2a0db68d Mon Sep 17 00:00:00 2001 From: Inseo Song Date: Thu, 28 Mar 2024 13:02:11 +0900 Subject: [PATCH 333/592] docs: Add relay host config guide for Gmail (#3958) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .../advanced/mail-forwarding/gmail-smtp.md | 50 +++++++++++++++++++ docs/mkdocs.yml | 1 + 2 files changed, 51 insertions(+) create mode 100644 docs/content/config/advanced/mail-forwarding/gmail-smtp.md diff --git a/docs/content/config/advanced/mail-forwarding/gmail-smtp.md b/docs/content/config/advanced/mail-forwarding/gmail-smtp.md new file mode 100644 index 00000000000..a581acd17ed --- /dev/null +++ b/docs/content/config/advanced/mail-forwarding/gmail-smtp.md @@ -0,0 +1,50 @@ +--- +title: 'Mail Forwarding | Configure Gmail as a relay host' +--- + +This page provides a guide for configuring DMS to use [GMAIL as an SMTP relay host][gmail-smtp]. + +!!! example "Configuration via ENV" + + [Configure a relay host in DMS][docs::relay]. This example shows how the related ENV settings map to the Gmail service config: + + - `RELAY_HOST` should be configured as [advised by Gmail][gmail-smtp::relay-host], there are two SMTP endpoints to choose: + - `smtp.gmail.com` (_for a personal Gmail account_) + - `smtp-relay.gmail.com` (_when using Google Workspace_) + - `RELAY_PORT` should be set to [one of the supported Gmail SMTP ports][gmail-smtp::relay-port] (_eg: 587 for STARTTLS_). + - `RELAY_USER` should be your gmail address (`user@gmail.com`). + - `RELAY_PASSWORD` should be your [App Password][gmail-smtp::app-password], **not** your personal gmail account password. + + ```env + RELAY_HOST=smtp.gmail.com + RELAY_PORT=587 + # Alternative to RELAY_HOST + RELAY_PORT which is compatible with LDAP: + DEFAULT_RELAY_HOST=[smtp.gmail.com]:587 + + RELAY_USER=username@gmail.com + RELAY_PASSWORD=secret + ``` + +!!! tip + + - As per our main [relay host docs page][docs::relay], you may prefer to configure your credentials via `setup relay add-auth` instead of the `RELAY_USER` + `RELAY_PASSWORD` ENV. + - If you configure for `smtp-relay.gmail.com`, the `DEFAULT_RELAY_HOST` ENV should be all you need as shown in the above example. Credentials can be optional when using Google Workspace (`smtp-relay.gmail.com`), which supports restricting connections to trusted IP addresses. + +!!! note "Verify the relay host is configured correctly" + + To verify proper operation, send an email to an external account of yours and inspect the mail headers. + + You will also see the connection to the Gmail relay host (`smtp.gmail.com`) in the mail logs: + + ```log + postfix/smtp[910]: Trusted TLS connection established to smtp.gmail.com[64.233.188.109]:587: + TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) + postfix/smtp[910]: 4BCB547D9D: to=, relay=smtp.gmail.com[64.233.188.109]:587, + delay=2.9, delays=0.01/0.02/1.7/1.2, dsn=2.0.0, status=sent (250 2.0.0 OK 17... - gsmtp) + ``` + +[docs::relay]: ./relay-hosts.md +[gmail-smtp]: https://support.google.com/a/answer/2956491 +[gmail-smtp::relay-host]: https://support.google.com/a/answer/176600 +[gmail-smtp::relay-port]: https://support.google.com/a/answer/2956491 +[gmail-smtp::app-password]: https://support.google.com/accounts/answer/185833 diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index e0ff50c9868..9d7fc81d33c 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -161,6 +161,7 @@ nav: - 'Email Forwarding': - 'Relay Hosts': config/advanced/mail-forwarding/relay-hosts.md - 'AWS SES': config/advanced/mail-forwarding/aws-ses.md + - 'Configure Gmail as a relay host': config/advanced/mail-forwarding/gmail-smtp.md - 'Full-Text Search': config/advanced/full-text-search.md - 'Kubernetes': config/advanced/kubernetes.md - 'IPv6': config/advanced/ipv6.md From 4f10089c90625eb4ccb8bff1cf6e40630e0d8b6c Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 29 Mar 2024 02:07:13 +0100 Subject: [PATCH 334/592] docs: add note about custom F2B setup with PROXY protocol (#3964) --- docs/content/config/advanced/kubernetes.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/content/config/advanced/kubernetes.md b/docs/content/config/advanced/kubernetes.md index 97a7e414e97..8f5ac901ce5 100644 --- a/docs/content/config/advanced/kubernetes.md +++ b/docs/content/config/advanced/kubernetes.md @@ -26,7 +26,7 @@ If using our Helm chart is not viable for you, here is some guidance to start wi === "`ConfigMap`" Provide the basic configuration via environment variables with a `ConfigMap`. - + !!! example Below is only an example configuration, adjust the `ConfigMap` to your own needs. @@ -512,6 +512,7 @@ Kubernetes provides multiple ways to address this; each has its upsides and down - Kubernetes manifest changes for the DMS configured `Service` - DMS configuration changes for Postfix and Dovecot - [ ] To keep support for direct connections to DMS services internally within cluster, service ports must be "duplicated" to offer an alternative port for connections using PROXY protocol + - [ ] Custom Fail2Ban required: Because the traffic to DMS is now coming from the proxy, banning the origin IP address will have no effect; you'll need to implement a [custom solution for your setup][github-web::docker-mailserver::proxy-protocol-fail2ban]. ??? question "What is the PROXY protocol?" @@ -795,3 +796,4 @@ Kubernetes provides multiple ways to address this; each has its upsides and down [k8s-docs::network-external-ip]: https://kubernetes.io/docs/concepts/services-networking/service/#external-ips [traefik-docs::k8s::ingress-route-tcp]: https://doc.traefik.io/traefik/routing/providers/kubernetes-crd/#kind-ingressroutetcp +[github-web::docker-mailserver::proxy-protocol-fail2ban]: https://github.com/docker-mailserver/docker-mailserver/issues/1761#issuecomment-2016879319 From 6733a172d77480e1cbc919125c7d9b7b7fd467ac Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 31 Mar 2024 04:14:02 +0200 Subject: [PATCH 335/592] docs: add FAQ entry about DNS servers and drop feature request on custom DNS servers for Rspamd (#3966) * add FAQ entry about DNS servers I also opted for including a quote from @polarthene which illustrates how DNS servers are a difficult topic and should not be DMS' responsibility. * link to DNS FAQ from Rspamd page & drop feature request The feature request annotation has been removed because we decided it's not DMS responsibility to ensure correctly working DNS servers. --- docs/content/config/security/rspamd.md | 8 +++----- docs/content/faq.md | 13 ++++++++++++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 5cb901b740e..fd0fe25e8b8 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -94,11 +94,7 @@ Rspamd provides a [web interface][rspamd-docs::web-interface], which contains st ### DNS -DMS does not supply custom values for DNS servers to Rspamd. If you need to use custom DNS servers, which could be required when using [DNS-based black/whitelists](#rbls-realtime-blacklists-dnsbls-dns-based-blacklists), you need to adjust [`options.inc`][rspamd-docs::basic-options] yourself. - -!!! tip "Making DNS Servers Configurable" - - If you want to see an environment variable (like `RSPAMD_DNS_SERVERS`) to support custom DNS servers for Rspamd being added to DMS, please raise a feature request issue. +DMS does not supply custom values for DNS servers (to Rspamd). If you need to use custom DNS servers, which could be required when using [DNS-based deny/allowlists](#rbls-real-time-blacklists-dnsbls-dns-based-blacklists), you need to adjust [`options.inc`][rspamd-docs::basic-options] yourself. Make sure to also read our [FAQ page on DNS servers][docs::faq::dns-servers]. !!! warning @@ -270,3 +266,5 @@ While _Abusix_ can be integrated into Postfix, Postscreen and a multitude of oth [docs::dms-volumes-config]: ../advanced/optional-config.md#volumes-config [docs::dms-volumes-state]: ../advanced/optional-config.md#volumes-state + +[docs::faq::dns-servers]: ../../faq.md#what-about-dns-servers diff --git a/docs/content/faq.md b/docs/content/faq.md index 6b1782e3190..4add05896a8 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -79,6 +79,14 @@ volumes: Optionally, you can set the `TZ` ENV variable; e.g. `TZ=Europe/Berlin`. Check [this list](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) for which values are allowed. +### What About DNS Servers? + +Properly working DNS servers are crucial for differentiating spam from legitimate e-mails. Records like `SPF`, `DKIM` and `DMARC` records, as well as working name (resolving `A` records) and reverse name (resolving `PTR` records) resolution ensures legitimate e-mails arrive while e-mails that are more likely phishing and spam do not. + +Anti-spam measures (like SpamAssassin or Rspamd) make use of DNS block lists. To learn more check out our [Rspamd documentation on this topic][docs::rspamd-rbl-dnsbl]. In case you want to utilize RBL/DNSBLs, you need a recursive DNS resolver (_not big custom resolvers like Cloudflare, Quad9, Google, etc._). + +DMS does not integrate support for an internal DNS service as this is a [responsibility that is sensitive to the host environment][gh-discussion::dms-avoid-maintaining-internal-dns]. You can configure internal services within DMS to use your own managed DNS server, or configure for such at the host or container level (_such as with [`compose.yaml`][docker-compose::docs::config-dns]_). + ### What is the file format? All files are using the Unix format with `LF` line endings. Please do not use `CRLF`. @@ -376,7 +384,7 @@ The default setup `@local_domains_acl = ( ".$mydomain" );` does not match subdom Put received spams in `.Junk/` imap folder using `SPAMASSASSIN_SPAM_TO_INBOX=1` and `MOVE_SPAM_TO_JUNK=1` and add a _user_ cron like the following: -!!! example +!!! example **NOTE:** This example assumes you have a [`/var/mail-state` volume][docs::dms-volumes-state] mounted. @@ -482,6 +490,7 @@ $spam_quarantine_to = "quarantine\@example.com"; [fail2ban-customize]: ./config/security/fail2ban.md [docs::dms-volumes-state]: ./config/advanced/optional-config.md#volumes-state +[docs::rspamd-rbl-dnsbl]: ./config/security/rspamd.md#rbls-real-time-blacklists-dnsbls-dns-based-blacklists [docs-maintenance]: ./config/advanced/maintenance/update-and-cleanup.md [docs-override-postfix]: ./config/advanced/override-defaults/postfix.md [docs-userpatches]: ./config/advanced/override-defaults/user-patches.md @@ -495,4 +504,6 @@ $spam_quarantine_to = "quarantine\@example.com"; [github-issue-1405-comment]: https://github.com/docker-mailserver/docker-mailserver/issues/1405#issuecomment-590106498 [github-issue-1639]: https://github.com/docker-mailserver/docker-mailserver/issues/1639 [github-issue-1792]: https://github.com/docker-mailserver/docker-mailserver/pull/1792 +[gh-discussion::dms-avoid-maintaining-internal-dns]: https://github.com/orgs/docker-mailserver/discussions/3959#discussioncomment-8956322 +[docker-compose::docs::config-dns]: https://docs.docker.com/compose/compose-file/compose-file-v3/#dns [hanscees-userpatches]: https://github.com/hanscees/dockerscripts/blob/master/scripts/tomav-user-patches.sh From d502dae0687b25c81aa3f010d88ad5552bc9ab41 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 17:20:35 +0200 Subject: [PATCH 336/592] docs: updated `CONTRIBUTORS.md` (#3967) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 321 +++++++++++++++++++++++++----------------------- 1 file changed, 164 insertions(+), 157 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 7c81c96882a..ea523c9477b 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -19,6 +19,13 @@ Thanks goes to these wonderful people ✨ fbartels + + + polarathene +
+ polarathene + + NorseGaud @@ -39,15 +46,15 @@ Thanks goes to these wonderful people ✨
wernerfred - + + georglauterbach
georglauterbach - - + tomav @@ -55,13 +62,6 @@ Thanks goes to these wonderful people ✨ tomav - - - polarathene -
- polarathene - - erik-wramner @@ -743,6 +743,20 @@ Thanks goes to these wonderful people ✨ emazzotta + + + keslerm +
+ keslerm + + + + + nueaf +
+ nueaf + + martinwepner @@ -763,7 +777,8 @@ Thanks goes to these wonderful people ✨
spacecowboy - + + jedateach @@ -772,18 +787,17 @@ Thanks goes to these wonderful people ✨ - - nueaf + + jamesfryer
- nueaf + jamesfryer - - + - - keslerm + + H4R0
- keslerm + H4R0 @@ -794,10 +808,10 @@ Thanks goes to these wonderful people ✨ - - jamesfryer + + ipernet
- jamesfryer + ipernet @@ -806,20 +820,6 @@ Thanks goes to these wonderful people ✨
fl42 - - - - H4R0 -
- H4R0 - - - - - ipernet -
- ipernet - @@ -829,6 +829,13 @@ Thanks goes to these wonderful people ✨ neuralp + + + sjmudd +
+ sjmudd + + simonsystem @@ -856,15 +863,15 @@ Thanks goes to these wonderful people ✨
5ven - + + syl20bnr
syl20bnr - - + sylvaindumont @@ -879,13 +886,28 @@ Thanks goes to these wonderful people ✨ TechnicLab + + + 42wim +
+ 42wim + + + + + vilisas +
+ vilisas + + thomasschmit
thomasschmit - + + Thiritin @@ -906,8 +928,7 @@ Thanks goes to these wonderful people ✨
torus - - + VictorKoenders @@ -928,7 +949,8 @@ Thanks goes to these wonderful people ✨
k3it - + + Drakulix @@ -936,28 +958,6 @@ Thanks goes to these wonderful people ✨ Drakulix - - - vilisas -
- vilisas - - - - - 42wim -
- 42wim - - - - - - ShiriNmi1520 -
- ShiriNmi1520 - - radicand @@ -1015,6 +1015,13 @@ Thanks goes to these wonderful people ✨ p3dda + + + auchri +
+ auchri + + peter-hartmann @@ -1028,15 +1035,15 @@ Thanks goes to these wonderful people ✨
piwai - + + remoe
remoe - - + robbertkl @@ -1071,15 +1078,15 @@ Thanks goes to these wonderful people ✨
501st-alpha1 - + + klamann
klamann - - + svdb0 @@ -1102,10 +1109,10 @@ Thanks goes to these wonderful people ✨ - - sjmudd + + ShiriNmi1520
- sjmudd + ShiriNmi1520 @@ -1114,15 +1121,15 @@ Thanks goes to these wonderful people ✨
mchamplain - + + millerjason
millerjason - - + mplx @@ -1157,15 +1164,15 @@ Thanks goes to these wonderful people ✨
ontheair81 - + + pravynandas
pravynandas - - + presocratics @@ -1200,15 +1207,15 @@ Thanks goes to these wonderful people ✨
schnippl0r - + + smargold476
smargold476 - - + sportshead @@ -1243,15 +1250,15 @@ Thanks goes to these wonderful people ✨
vivacarvajalito - + + wligtenberg
wligtenberg - - + wolkenschieber @@ -1286,15 +1293,15 @@ Thanks goes to these wonderful people ✨
arcaine2 - + + awb99
awb99 - - + brainkiller @@ -1329,15 +1336,15 @@ Thanks goes to these wonderful people ✨
eleith - + + ghnp5
ghnp5 - - + helmutundarnold @@ -1372,15 +1379,15 @@ Thanks goes to these wonderful people ✨
ixeft - + + jjtt
jjtt - - + paralax @@ -1415,28 +1422,14 @@ Thanks goes to these wonderful people ✨
marios88 - + + matrixes
matrixes - - - - - 0xflotus -
- 0xflotus - - - - - auchri -
- auchri - @@ -1465,15 +1458,15 @@ Thanks goes to these wonderful people ✨
espitall - - + dkarski
dkarski - + + dbellavista @@ -1508,15 +1501,15 @@ Thanks goes to these wonderful people ✨
mazzz1y - - + doominator42
doominator42 - + + aydodo @@ -1551,15 +1544,15 @@ Thanks goes to these wonderful people ✨
ekkis - - + ErikEngerd
ErikEngerd - + + huncode @@ -1581,21 +1574,35 @@ Thanks goes to these wonderful people ✨ flole + + + Kaan88 +
+ Kaan88 + + + + + 0xflotus +
+ 0xflotus + + ifokeev
ifokeev - + + 20th
20th - - + 2b @@ -1630,15 +1637,15 @@ Thanks goes to these wonderful people ✨
vifino - + + kachkaev
kachkaev - - + alexanderneu @@ -1673,15 +1680,15 @@ Thanks goes to these wonderful people ✨
green-anger - + + iRhonin
iRhonin - - + MrFreezeex @@ -1716,15 +1723,15 @@ Thanks goes to these wonderful people ✨
spock - + + erdos4d
erdos4d - - + crash7 @@ -1732,13 +1739,6 @@ Thanks goes to these wonderful people ✨ crash7 - - - froks -
- froks - - akkumar @@ -1889,14 +1889,21 @@ Thanks goes to these wonderful people ✨ naveensrinivasan + + + froks +
+ froks + + + fkefer
fkefer - - + Marsu31 @@ -1931,15 +1938,15 @@ Thanks goes to these wonderful people ✨
HeySora - + + sirgantrithon
sirgantrithon - - + Influencer @@ -1947,6 +1954,13 @@ Thanks goes to these wonderful people ✨ Influencer + + + in-seo +
+ in-seo + + JacksonZ03 @@ -1967,7 +1981,8 @@ Thanks goes to these wonderful people ✨
jcalfee - + + mivek @@ -1981,8 +1996,7 @@ Thanks goes to these wonderful people ✨
init-js - - + Jeidnx @@ -2010,7 +2024,8 @@ Thanks goes to these wonderful people ✨
jirislav - + + jmccl @@ -2024,21 +2039,13 @@ Thanks goes to these wonderful people ✨
jurekbarth - - + JOduMonT
JOduMonT - - - - Kaan88 -
- Kaan88 - From 8c5cf03203ff164a89f45848e542256f2725bc21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 17:56:12 +0200 Subject: [PATCH 337/592] chore(deps): Bump docker/setup-buildx-action from 3.2.0 to 3.3.0 (#3972) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index f0bd211123d..da5d51c15f3 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -79,7 +79,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.2.0 + uses: docker/setup-buildx-action@v3.3.0 # NOTE: AMD64 can build within 2 minutes - name: 'Build images' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 266090b6176..ead38a3f6a9 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -40,7 +40,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.2.0 + uses: docker/setup-buildx-action@v3.3.0 # Try get the cached build layers from a prior `generic_build.yml` job. # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 4c49ca3e88b..4e648e32480 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -38,7 +38,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.2.0 + uses: docker/setup-buildx-action@v3.3.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 23544820fc0..2f1a38034a1 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -37,7 +37,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.2.0 + uses: docker/setup-buildx-action@v3.3.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. From ad5d1011f82401415421b0738209d855bcb12443 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 10:18:17 +0200 Subject: [PATCH 338/592] docs: updated `CONTRIBUTORS.md` (#3971) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index ea523c9477b..01c7897fc37 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1783,10 +1783,10 @@ Thanks goes to these wonderful people ✨ - - LeoWinterDE + + leowinterde
- LeoWinterDE + leowinterde From f2314259829d170984eacd5790406a6f745ae032 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 18:05:07 +0200 Subject: [PATCH 339/592] chore(deps): Bump peaceiris/actions-gh-pages from 3.9.3 to 4.0.0 (#3978) --- .github/workflows/docs-production-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-production-deploy.yml b/.github/workflows/docs-production-deploy.yml index cb8bdbed226..2c6c1e2cf1d 100644 --- a/.github/workflows/docs-production-deploy.yml +++ b/.github/workflows/docs-production-deploy.yml @@ -59,7 +59,7 @@ jobs: {} + - name: 'Deploy to Github Pages' - uses: peaceiris/actions-gh-pages@v3.9.3 + uses: peaceiris/actions-gh-pages@v4.0.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} # Build directory contents to publish to the `gh-pages` branch: From dc5185003058dbdb1d18947a870a09c43a275d85 Mon Sep 17 00:00:00 2001 From: fanqiaojun <166898955+fanqiaojun@users.noreply.github.com> Date: Tue, 16 Apr 2024 03:48:55 +0800 Subject: [PATCH 340/592] chore: remove repetitive words (#3977) --- docs/content/config/advanced/auth-ldap.md | 2 +- docs/content/examples/use-cases/auth-lua.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/config/advanced/auth-ldap.md b/docs/content/config/advanced/auth-ldap.md index ea24526d4b6..397e42ebe11 100644 --- a/docs/content/config/advanced/auth-ldap.md +++ b/docs/content/config/advanced/auth-ldap.md @@ -26,7 +26,7 @@ Those variables contain the LDAP lookup filters for postfix, using `%s` as the p - Technically, there is no difference between `ALIAS` and `GROUP`, but ideally you should use `ALIAS` for personal aliases for a singular person (like `ceo@example.org`) and `GROUP` for multiple people (like `hr@example.org`). - ...for outgoing email, the sender address is put through the `SENDERS` filter, and only if the authenticated user is one of the returned entries, the email can be sent. - This only applies if `SPOOF_PROTECTION=1`. - - If the `SENDERS` filter is missing, the `USER`, `ALIAS` and `GROUP` filters will be used in in a disjunction (OR). + - If the `SENDERS` filter is missing, the `USER`, `ALIAS` and `GROUP` filters will be used in a disjunction (OR). - To for example allow users from the `admin` group to spoof any sender email address, and to force everyone else to only use their personal mailbox address for outgoing email, you can use something like this: `(|(memberOf=cn=admin,*)(mail=%s))` ???+ example diff --git a/docs/content/examples/use-cases/auth-lua.md b/docs/content/examples/use-cases/auth-lua.md index 8258688589e..e34ed3b0de2 100644 --- a/docs/content/examples/use-cases/auth-lua.md +++ b/docs/content/examples/use-cases/auth-lua.md @@ -34,7 +34,7 @@ A drawback of this method is that any (compromised) Nextcloud application passwo To answer the questions asked earlier for this specific scenario: -1. Do I want to use Lua to identify mailboxes and verify that users are are authorized to use mail services? **No. Provisioning is done through LDAP.** +1. Do I want to use Lua to identify mailboxes and verify that users are authorized to use mail services? **No. Provisioning is done through LDAP.** 1. Do I want to use Lua to verify passwords that users authenticate with for IMAP/POP3/SMTP in their mail clients? **Yes. Password authentication is done through Lua against Nextcloud.** 1. If the answer is 'yes' to question 1 or 2: are there other methods that better facilitate my use case instead of custom scripts which rely on me being a developer and not just a user? **No. Only HTTP can be used to authenticate against Nextcloud, which is not supported natively by Dovecot or DMS.** From d87e4d3bfd7780def3c187b0f31c9034954cdfe7 Mon Sep 17 00:00:00 2001 From: Iztok Fister Jr Date: Tue, 16 Apr 2024 22:25:45 +0200 Subject: [PATCH 341/592] docs: Fix typos (#3979) --- docs/content/config/environment.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 9adfd5d970c..867c7459a0f 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -456,8 +456,8 @@ Default: 6 (which corresponds to the `add_header` action) ##### RSPAMD_NEURAL -Can be used to enable or disable the [Neural network module][rspamd-docs-neural-network]. This is an experimental anti-spam weigh method using three neuaral networks in the configuration added here. As far as we can tell it trains itsself by using other modules to find out what spam is. It will take a while (a week or more) to train its first neural network. The config trains new networks all the time and discards of old networks. -Since it is experimental it is switched of by default. +Can be used to enable or disable the [Neural network module][rspamd-docs-neural-network]. This is an experimental anti-spam weigh method using three neural networks in the configuration added here. As far as we can tell it trains itself by using other modules to find out what spam is. It will take a while (a week or more) to train its first neural network. The config trains new networks all the time and discards old networks. +Since it is experimental, it is switched off by default. - **0** => Disabled - 1 => Enabled From 942920615c0ab3d9946d6ff1db6a83f9034351cc Mon Sep 17 00:00:00 2001 From: Tobia Bocchi <29007647+tobiabocchi@users.noreply.github.com> Date: Thu, 18 Apr 2024 03:08:26 +0200 Subject: [PATCH 342/592] docs: Fix typo on usage page (#3980) --- docs/content/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/usage.md b/docs/content/usage.md index 76011fe7690..087547ea5ad 100644 --- a/docs/content/usage.md +++ b/docs/content/usage.md @@ -11,7 +11,7 @@ This page explains how to get started with DMS. The guide uses Docker Compose as Before you can get started with deploying your own mail server, there are some requirements to be met: 1. You need to have a host that you can manage. -2. You need to own a domain, and you need to able to manage DNS for this domain. +2. You need to own a domain, and you need to be able to manage DNS for this domain. ### Host Setup From ac22caf74eff031ad8086ac147949c0b15c24398 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 20 Apr 2024 11:25:02 +1200 Subject: [PATCH 343/592] docs: Updates to TLS page (Caddy, testing, etc) (#3981) --- docs/content/config/security/ssl.md | 233 +++++++++++++++------------- 1 file changed, 127 insertions(+), 106 deletions(-) diff --git a/docs/content/config/security/ssl.md b/docs/content/config/security/ssl.md index a917448361c..a5ba005ad30 100644 --- a/docs/content/config/security/ssl.md +++ b/docs/content/config/security/ssl.md @@ -481,113 +481,108 @@ DSM-generated letsencrypt certificates get auto-renewed every three months. ### Caddy -For Caddy v2 you can specify the `key_type` in your server's global settings, which would end up looking something like this if you're using a `Caddyfile`: - -```caddyfile -{ - debug - admin localhost:2019 - http_port 80 - https_port 443 - default_sni example.com - key_type rsa2048 -} -``` +[Caddy][web::caddy] is an open-source web server with built-in TLS certificate generation. You can use the [official Docker image][dockerhub::caddy] and write your own `Caddyfile`. + +!!! example + + ```yaml title="compose.yaml" + services: + # Basic Caddy service to provision certs: + reverse-proxy: + image: caddy:2.7 + ports: + - 80:80 + - 443:443 + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + - ${CADDY_DATA_DIR}:/data + + # Share the Caddy data volume for certs and configure SSL_TYPE to `letsencrypt` + mailserver: + image: ghcr.io/docker-mailserver/docker-mailserver:latest + hostname: mail.example.com + environment: + SSL_TYPE: letsencrypt + # While you could use a named data volume instead of a bind mount volume, it would require the long-syntax to rename cert files: + # https://docs.docker.com/compose/compose-file/05-services/#volumes + volumes: + - ${CADDY_DATA_DIR}/certificates/acme-v02.api.letsencrypt.org-directory/mail.example.com/mail.example.com.crt:/etc/letsencrypt/live/mail.example.com/fullchain.pem + - ${CADDY_DATA_DIR}/certificates/acme-v02.api.letsencrypt.org-directory/mail.example.com/mail.example.com.key:/etc/letsencrypt/live/mail.example.com/privkey.pem + ``` -If you are instead using a json config for Caddy v2, you can set it in your site's TLS automation policies: - -??? example "Caddy v2 JSON example snippet" - - ```json - { - "apps": { - "http": { - "servers": { - "srv0": { - "listen": [ - ":443" - ], - "routes": [ - { - "match": [ - { - "host": [ - "mail.example.com", - ] - } - ], - "handle": [ - { - "handler": "subroute", - "routes": [ - { - "handle": [ - { - "body": "", - "handler": "static_response" - } - ] - } - ] - } - ], - "terminal": true - }, - ] - } - } - }, - "tls": { - "automation": { - "policies": [ - { - "subjects": [ - "mail.example.com", - ], - "key_type": "rsa2048", - "issuer": { - "email": "admin@example.com", - "module": "acme" - } - }, - { - "issuer": { - "email": "admin@example.com", - "module": "acme" - } - } - ] - } - } + ```caddyfile title="Caddyfile" + mail.example.com { + tls internal { + key_type rsa2048 } + + # Optional, can be useful for troubleshooting + # connection to Caddy with correct certificate: + respond "Hello DMS" } ``` -The generated certificates can then be mounted: + While DMS does not need a webserver to work, this workaround will provision a TLS certificate for DMS to use. -```yaml -volumes: - - ${CADDY_DATA_DIR}/certificates/acme-v02.api.letsencrypt.org-directory/mail.example.com/mail.example.com.crt:/etc/letsencrypt/live/mail.example.com/fullchain.pem - - ${CADDY_DATA_DIR}/certificates/acme-v02.api.letsencrypt.org-directory/mail.example.com/mail.example.com.key:/etc/letsencrypt/live/mail.example.com/privkey.pem -``` + - [`tls internal`][caddy-docs::tls-internal] will create a local self-signed cert for testing. This targets only the site-address, unlike the global `local_certs` option. + - [`key_type`][caddy-docs::key-type] can be used in the `tls` block if you need to enforce RSA as the key type for certificates provisioned. The default is currently ECDSA (P-256). -### Traefik v2 +??? example "With `caddy-docker-proxy`" -[Traefik][traefik::github] is an open-source application proxy using the [ACME protocol][ietf::rfc::acme]. [Traefik][traefik::github] can request certificates for domains and subdomains, and it will take care of renewals, challenge negotiations, etc. We strongly recommend to use [Traefik][traefik::github]'s major version 2. + Using [`lucaslorentz/caddy-docker-proxy`][github::caddy-docker-proxy] allows you to generate a `Caddyfile` by adding labels to your services in `compose.yaml`: -[Traefik][traefik::github]'s storage format is natively supported if the `acme.json` store is mounted into the container at `/etc/letsencrypt/acme.json`. The file is also monitored for changes and will trigger a reload of the mail services (Postfix and Dovecot). + ```yaml title="compose.yaml" + services: + reverse-proxy: + image: lucaslorentz/caddy-docker-proxy:2.8 + ports: + - 80:80 + - 443:443 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ${CADDY_DATA_DIR}:/data + labels: + # Set global config here, this option has an empty value to enable self-signed certs for local testing: + # NOTE: Remove this label when going to production. + caddy.local_certs: "" -Wildcard certificates are supported. If your FQDN is `mail.example.com` and your wildcard certificate is `*.example.com`, add the ENV: `#!bash SSL_DOMAIN=example.com`. + # Use labels to configure Caddy to provision DMS certs + mailserver: + image: ghcr.io/docker-mailserver/docker-mailserver:latest + hostname: mail.example.com + environment: + SSL_TYPE: letsencrypt + volumes: + - ${CADDY_DATA_DIR}/certificates/acme-v02.api.letsencrypt.org-directory/mail.example.com/mail.example.com.crt:/etc/letsencrypt/live/mail.example.com/fullchain.pem + - ${CADDY_DATA_DIR}/certificates/acme-v02.api.letsencrypt.org-directory/mail.example.com/mail.example.com.key:/etc/letsencrypt/live/mail.example.com/privkey.pem + labels: + # Set your DMS FQDN here to add the site-address into the generated Caddyfile: + caddy_0: mail.example.com + # Add a dummy directive is required: + caddy_0.respond: "Hello DMS" + # Uncomment to make a proxy for Rspamd + # caddy_1: rspamd.example.com + # caddy_1.reverse_proxy: "{{upstreams 11334}}" + ``` + +!!! warning "Caddy certificate location varies" + + The path contains the certificate provisioner used. This path may be different from the example above for you and may change over time when multiple provisioner services are used][dms-pr-feedback::caddy-provisioning-gotcha]. -DMS will select it's certificate from `acme.json` checking these ENV for a matching FQDN (_in order of priority_): + This can make the volume mounting for DMS to find the certificates non-deterministic, but you can [restrict provisioning to single service via the `acme_ca` setting][caddy::restrict-acme-provisioner]. -1. `#!bash ${SSL_DOMAIN}` -2. `#!bash ${HOSTNAME}` -3. `#!bash ${DOMAINNAME}` +### Traefik -This setup only comes with one caveat: The domain has to be configured on another service for [Traefik][traefik::github] to actually request it from _Let's Encrypt_, i.e. [Traefik][traefik::github] will not issue a certificate without a service / router demanding it. +[Traefik][traefik::github] is an open-source application proxy using the [ACME protocol][ietf::rfc::acme]. Traefik can request certificates for domains and subdomains, and it will take care of renewals, challenge negotiations, etc. + +Traefik's storage format is natively supported if the `acme.json` store is mounted into the container at `/etc/letsencrypt/acme.json`. The file is also monitored for changes and will trigger a reload of the mail services (Postfix and Dovecot). + +DMS will select it's certificate from `acme.json` prioritizing a match for the DMS FQDN (hostname), while also checking one DNS level up (_eg: `mail.example.com` => `example.com`_). Wildcard certificates are supported. + +This setup only comes with one caveat - The domain has to be configured on another service for Traefik to actually request it from _Let's Encrypt_ (_i.e. Traefik will not issue a certificate without a service / router demanding it_). ???+ example "Example Code" + Here is an example setup for [`docker-compose`](https://docs.docker.com/compose/): ```yaml @@ -716,7 +711,7 @@ The local and internal paths may be whatever you prefer, so long as both `SSL_CE ## Testing a Certificate is Valid -- From your host: +!!! example "Connect to DMS on port 25" ```sh docker exec mailserver openssl s_client \ @@ -725,26 +720,42 @@ The local and internal paths may be whatever you prefer, so long as both `SSL_CE -CApath /etc/ssl/certs/ ``` -- Or: + The response should show the certificate chain with a line further down: `Verify return code: 0 (ok)` + + --- + + This example runs within the DMS container itself to verify the cert is working locally. + + - Adjust the `-connect` IP if testing externally from another system. Additionally testing for port 143 (Dovecot IMAP) is encouraged (_change the protocol for `-starttls` from `smtp` to `imap`_). + - `-CApath` will help verify the certificate chain, provided the location contains the root CA that signed your TLS cert for DMS. + +??? example "Verify certificate dates" ```sh docker exec mailserver openssl s_client \ - -connect 0.0.0.0:143 \ - -starttls imap \ - -CApath /etc/ssl/certs/ + -connect 0.0.0.0:25 \ + -starttls smtp \ + -CApath /etc/ssl/certs/ \ + 2>/dev/null | openssl x509 -noout -dates ``` -And you should see the certificate chain, the server certificate and: `Verify return code: 0 (ok)` +!!! tip "Testing and troubleshooting" -In addition, to verify certificate dates: + If you need to test a connection without resolving DNS, `curl` can connect with `--resolve` option to map an FQDN + Port to an IP address, instead of the request address provided. -```sh -docker exec mailserver openssl s_client \ - -connect 0.0.0.0:25 \ - -starttls smtp \ - -CApath /etc/ssl/certs/ \ - 2>/dev/null | openssl x509 -noout -dates -``` + ```bash + # NOTE: You may want to use `--insecure` if the cert was provisioned with a private CA not present on the curl client: + # Use `--verbose` for additional insights on the connection. + curl --resolve mail.example.com:443:127.0.0.1 https://mail.example.com + ``` + + Similarly with `openssl` you can connect to an IP as shown previously, but provide an explicit SNI if necessary with `-servername mail.example.com`. + + --- + + Both `curl` and `openssl` also support `-4` and `-6` for enforcing IPv4 or IPv6 lookup. + + This can be useful, such as when [DNS resolves the IP to different servers leading to different certificates returned][dms-discussion::gotcha-fqdn-bad-dns]. As shown in that link, `step certificate inspect` is also handy for viewing details of the cert returned or on disk. ## Plain-Text Access @@ -919,3 +930,13 @@ Despite this, if you must use non-standard DH parameters or you would like to sw [acme-companion::standalone]: https://github.com/nginx-proxy/acme-companion/blob/main/docs/Standalone-certificates.md [acme-companion::standalone-changes]: https://github.com/nginx-proxy/acme-companion/blob/main/docs/Standalone-certificates.md#picking-up-changes-to-letsencrypt_user_data [acme-companion::service-loop]: https://github.com/nginx-proxy/acme-companion/blob/main/docs/Container-utilities.md + +[web::caddy]: https://caddyserver.com +[dockerhub::caddy]: https://hub.docker.com/_/caddy +[github::caddy-docker-proxy]: https://github.com/lucaslorentz/caddy-docker-proxy +[dms-pr-feedback::caddy-provisioning-gotcha]: https://github.com/docker-mailserver/docker-mailserver/pull/3485/files#r1297512818 +[caddy-docs::tls-internal]: https://caddyserver.com/docs/caddyfile/directives/tls#syntax +[caddy-docs::key-type]: https://caddyserver.com/docs/caddyfile/options#key-type +[caddy::restrict-acme-provisioner]: https://caddy.community/t/is-there-a-way-on-a-caddyfile-to-force-a-specific-acme-ca/14506 + +[dms-discussion::gotcha-fqdn-bad-dns]: https://github.com/docker-mailserver/docker-mailserver/issues/3955#issuecomment-2027882633 From d739fe3785ec16dcda1dd2213f64298e2269ce72 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Mon, 22 Apr 2024 11:28:11 +1200 Subject: [PATCH 344/592] chore: Remove base-60 port quote warning from example `compose.yaml` (#3982) This should not be relevant to users of `docker compose` which is the primary demographic. --- compose.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index e9c49596234..8f5bfdb2508 100644 --- a/compose.yaml +++ b/compose.yaml @@ -7,7 +7,6 @@ services: env_file: mailserver.env # More information about the mail-server ports: # https://docker-mailserver.github.io/docker-mailserver/latest/config/security/understanding-the-ports/ - # To avoid conflicts with yaml base-60 float, DO NOT remove the quotation marks. ports: - "25:25" # SMTP (explicit TLS => STARTTLS, Authentication is DISABLED => use port 465/587 instead) - "143:143" # IMAP4 (explicit TLS => STARTTLS) From df360516ff221e6c5c6380d62000100178038be8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=BCrst?= <7149167+furstblumier@users.noreply.github.com> Date: Mon, 22 Apr 2024 01:50:02 +0200 Subject: [PATCH 345/592] docs: Add config guide for relaying to and from a private DMS instance (#3973) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Co-authored-by: Casper --- CHANGELOG.md | 2 + .../external-relay-only-mailserver.md | 233 ++++++++++++++++++ docs/mkdocs.yml | 1 + 3 files changed, 236 insertions(+) create mode 100644 docs/content/examples/use-cases/external-relay-only-mailserver.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 89efb03042c..10d3001614f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,8 @@ The most noteworthy change of this release is the update of the container's base ### Added +- **Docs:** + - A guide for configuring a public server to relay inbound and outbound mail from DMS on a private server ([#3973](https://github.com/docker-mailserver/docker-mailserver/pull/3973)) - **Environment Variables:** - `LOGROTATE_COUNT` defines the number of files kept by logrotate ([#3907](https://github.com/docker-mailserver/docker-mailserver/pull/3907)) - The fail2ban log file is now also taken into account by `LOGROTATE_COUNT` and `LOGROTATE_INTERVAL` ([#3915](https://github.com/docker-mailserver/docker-mailserver/pull/3915), [#3919](https://github.com/docker-mailserver/docker-mailserver/pull/3919)) diff --git a/docs/content/examples/use-cases/external-relay-only-mailserver.md b/docs/content/examples/use-cases/external-relay-only-mailserver.md new file mode 100644 index 00000000000..44e83c91221 --- /dev/null +++ b/docs/content/examples/use-cases/external-relay-only-mailserver.md @@ -0,0 +1,233 @@ +--- +title: 'Use Cases | Relay inbound and outbound mail for an internal DMS' +hide: + - toc +--- + +## Introduction + +!!! info "Community contributed guide" + + Adapted into a guide from [this discussion](https://github.com/orgs/docker-mailserver/discussions/3965). + + **Requirements:** + + - A _public server_ with a static IP, like many VPS providers offer. It will only relay mail to DMS, no mail is stored on this system. + - A _private server_ (e.g.: a local system at home) that will run DMS. + - Both servers are connected to the same network via a VPN (_optional convenience for trust via the `mynetworks` setting_). + + --- + + The guide below will assume the VPN is setup on `192.168.2.0/24` with: + + - The _public server_ is using `192.168.2.2` + - The _private server_ is using `192.168.2.3` + +The goal of this guide is to configure a _public server_ that can receive inbound mail and relay that over to DMS on a _private server_, which can likewise submit mail outbound through a _public server_ or service. + +The primary motivation is to keep your mail storage private instead of storing it to disk unencrypted on a VPS host. + +## DNS setup + +Follow our [standard guidance][docs::usage-dns-setup] for DNS setup. + +Set your A, MX and PTR records for the _public server_ as if it were running DMS. + +!!! example "DNS Zone file example" + + For this guide, we assume DNS is configured with: + + - A public reachable IP address of `11.22.33.44` + - Mail for `@example.com` addresses must have an MX record pointing to `mail.example.com`. + - An A record for `mail.example.com` pointing to the IP address of your _public server_. + + ```txt + $ORIGIN example.com + @ IN A 11.22.33.44 + mail IN A 11.22.33.44 + + ; mail server for example.com + @ IN MX 10 mail.example.com. + ``` + + SPF records should also be set up as you normally would for `mail.example.com`. + +## Public Server (Basic Postfix setup) + +You will need to install Postfix on your _public server_. The functionality that is needed for this setup is not yet implemented in DMS, so a vanilla Postfix will probably be easier to work with, especially since this server will only be used as an inbound and outbound relay. + +It's necessary to adjust some settings afterwards. + + +!!! quote "" + + === "Postfix main config" + + ??? example "Create or replace `/etc/postfix/main.cf`" + + ```cf + # See /usr/share/postfix/main.cf.dist for a commented, more complete version + + smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU) + biff = no + + # appending .domain is the MUA's job. + append_dot_mydomain = no + + # Uncomment the next line to generate "delayed mail" warnings + #delay_warning_time = 4h + + # See http://www.postfix.org/COMPATIBILITY_README.html -- default to 3.6 on + # fresh installs. + compatibility_level = 3.6 + + # TLS parameters + smtpd_tls_cert_file=/etc/postfix/certificates/mail.example.com.crt + smtpd_tls_key_file=/etc/postfix/certificates/mail.example.com.key + smtpd_tls_security_level=may + smtp_tls_CApath=/etc/ssl/certs + smtp_tls_security_level=may + smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache + + alias_database = hash:/etc/aliases + alias_maps = hash:/etc/aliases + maillog_file = /var/log/postfix.log + mailbox_size_limit = 0 + inet_interfaces = all + inet_protocols = ipv4 + readme_directory = no + recipient_delimiter = + + + # Customizations relevant to this guide: + myhostname = mail.example.com + myorigin = example.com + mydestination = localhost + mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 192.168.2.0/24 + smtpd_relay_restrictions = permit_mynetworks permit_sasl_authenticated defer_unauth_destination + transport_maps = hash:/etc/postfix/transport + relay_domains = $mydestination, hash:/etc/postfix/relay + + # Disable local system accounts and delivery: + local_recipient_maps = + local_transport = error:local mail delivery is disabled + ``` + + Let's highlight some of the important parts: + + - Avoid including `mail.example.com` in `mydestination`, in fact you can just set `localhost` or nothing at all here as we want all mail to be relayed to our _private server_ (DMS). + - `mynetworks` should contain your VPN network (_eg: `192.168.2.0/24` subnet_). + - Important are `transport_maps = hash:/etc/postfix/transport` and `relay_domains = $mydestination, hash:/etc/postfix/relay`, with their file contents covered below. + - For good measure, also disable `local_recipient_maps`. + - You should have a valid certificate configured for `mail.example.com`. + + !!! warning "Open relay" + + Please be aware that setting `mynetworks` to a public CIDR will leave you with an open relay. **Only** set it to the CIDR of your VPN beyond the localhost ranges. + + === "Route outbound mail through a separate transport" + + When mail arrives to the _public server_ for an `@example.com` address, we want to send it via the `relay` transport to our _private server_ over port 25 for delivery to DMS. + + [`transport_maps`][postfix-docs::transport_maps] is configured with a [`transport` table][postfix-docs::transport_table] file that matches recipient addresses and assigns a non-default transport. This setting has priority over [`relay_transport`][postfix-docs::relay_transport]. + + !!! example "Create `/etc/postfix/transport`" + + ```txt + example.com relay:[192.168.2.3]:25 + ``` + + **Other considerations:** + + - If you have multiple domains, you can add them here too (on separate lines). + - If you use a smarthost add `* relay:[X.X.X.X]:port` to the bottom (eg: `* relay:[relay1.org]:587`), which will relay everything outbound via this relay host. + + !!! tip + + Instead of a file, you could alternatively configure `main.cf` with `transport_maps = inline:{ example.com=relay:[192.168.2.3]:25 }` + + === "Configure recipient domains to relay mail" + + We want `example.com` to be relayed inbound and everything else relayed outbound. + + [`relay_domains`][postfix-docs::relay_domains] is configured with a file with a list of domains that should be relayed (one per line), the 2nd value is required but can be anything. + + !!! example "Create `/etc/postfix/relay`" + + ```txt + example.com OK + ``` + + !!! tip + + Instead of a file, you could alternatively configure `main.cf` with `relay_domains = example.com`. + +!!! note "Files configured with `hash:` table type must run `postmap` to apply changes" + + Run `postmap /etc/postfix/transport` and `postmap /etc/postfix/relay` after creating or updating either of these files, this processes them into a separate file for Postfix to use. + +## Private Server (Running DMS) + +You can set up your DMS instance as you normally would. + +- Be careful not to give it a hostname of `mail.example.com`. Instead, use `internal-mail.example.com` or something similar. +- DKIM can be setup as usual since it considers checks whether the message body has been tampered with, which our public relay doesn't do. Set DKIM up for `mail.example.com`. + +Next, we need to configure our _private server_ to relay all outbound mail through the _public server_ (or a separate smarthost service). The setup is [similar to the default relay setup][docs::relay-host-details]. + + +!!! quote "" + + === "Configure the relay host" + + !!! example "Create `postfix-relaymap.cf`" + + ```txt + @example.com [192.168.2.2]:25 + ``` + + Meaning all mail sent outbound from `@example.com` addresses will be relayed through the _public server_ at that VPN IP. + + The _public server_ `mynetworks` setting from earlier trusts any mail received on port 25 from the VPN network, which is what allows the mail to be sent outbound when it'd otherwise be denied. + + === "Trust the _public server_" + + !!! example "Create `postfix-main.cf`" + + ```txt + mynetworks = 192.168.2.0/24 + ``` + + This will trust any connection from the VPN network to DMS, such as from the _public server_ when relaying mail over to DMS at the _private server_. + + This step is necessary to skip some security measures that DMS normally checks for, like verifying DNS records like SPF are valid. As the mail is being relayed, those checks would fail otherwise as the IP of your _public server_ would not be authorized to send mail on behalf of the sender address in mail being relayed. + + ??? tip "Alternative to `mynetworks` setting" + + Instead of trusting connections by their IP with the `mynetworks` setting, those same security measures can be skipped for any authenticated deliveries to DMS over port 587 instead. + + This is a bit more work. `mynetworks` on the _public server_ `main.cf` Postfix config is for trusting DMS when it sends mail from the _private server_, thus you'll need to have that public Postfix service configured with a login account that DMS can use. + + On the _private server_, DMS needs to know the credentials for that login account, that is handled with `postfix-sasl-password.cf`: + + ```txt + @example.com user:secret + ``` + + You could also relay mail through SendGrid, AWS SES or similar instead of the _public server_ you're running to receive mail from. Login credentials for those relay services are provided via the same `postfix-sasl-password.cf` file. + + --- + + Likewise for the _public server_ to send mail to DMS, it would need to be configured to relay mail with credentials too, removing the need for `mynetworks` on the DMS `postfix-main.cf` config. + + The extra effort to require authentication instead of blind trust of your private subnet can be beneficial at reducing the impact of a compromised system or service on that network that wasn't expected to be permitted to send mail. + +## IMAP / POP3 + +IMAP and POP3 need to point towards your _private server_, since that is where the mailboxes are located, which means you need to have a way for your MUA to connect to it. + +[docs::usage-dns-setup]: ../../usage.md#minimal-dns-setup +[docs::relay-host-details]: ../../config/advanced/mail-forwarding/relay-hosts.md#technical-details +[postfix-docs::relay_domains]: https://www.postfix.org/postconf.5.html#relay_domains +[postfix-docs::relay_transport]: https://www.postfix.org/postconf.5.html#relay_transport +[postfix-docs::transport_maps]: https://www.postfix.org/postconf.5.html#transport_maps +[postfix-docs::transport_table]: https://www.postfix.org/transport.5.html diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 9d7fc81d33c..592634436d6 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -180,6 +180,7 @@ nav: - 'iOS Mail Push Support': examples/use-cases/ios-mail-push-support.md - 'Lua Authentication': examples/use-cases/auth-lua.md - 'Bind outbound SMTP to a specific network': examples/use-cases/bind-smtp-network-interface.md + - 'Relay inbound and outbound mail for an internal DMS': examples/use-cases/external-relay-only-mailserver.md - 'FAQ' : faq.md - 'Contributing': - 'General Information': contributing/general.md From 1051a5d921a07bae4cfe2f43e981d058baa05fb4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 15:46:34 +0200 Subject: [PATCH 346/592] chore(deps): Bump akhilmhdh/contributors-readme-action (#3987) Bumps [akhilmhdh/contributors-readme-action](https://github.com/akhilmhdh/contributors-readme-action) from 2.3.6 to 2.3.8. - [Release notes](https://github.com/akhilmhdh/contributors-readme-action/releases) - [Commits](https://github.com/akhilmhdh/contributors-readme-action/compare/v2.3.6...v2.3.8) --- updated-dependencies: - dependency-name: akhilmhdh/contributors-readme-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/contributors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index 6f8476f274b..9a71b64880f 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v4 - name: 'Update CONTRIBUTORS.md' - uses: akhilmhdh/contributors-readme-action@v2.3.6 + uses: akhilmhdh/contributors-readme-action@v2.3.8 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: From 162e66276a4d4c836d6eb4434fdfb3cfb35e2fad Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 25 Apr 2024 08:38:38 +0200 Subject: [PATCH 347/592] docs: updated `CONTRIBUTORS.md` (#3984) --- CONTRIBUTORS.md | 285 ++++++++++++++++++++++++++---------------------- 1 file changed, 153 insertions(+), 132 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 01c7897fc37..e1027231aea 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -787,10 +787,17 @@ Thanks goes to these wonderful people ✨ - - jamesfryer + + fl42
- jamesfryer + fl42 + + + + + ipernet +
+ ipernet @@ -808,34 +815,13 @@ Thanks goes to these wonderful people ✨ - - ipernet -
- ipernet - - - - - fl42 + + jamesfryer
- fl42 + jamesfryer - - - neuralp -
- neuralp - - - - - sjmudd -
- sjmudd - - simonsystem @@ -863,8 +849,7 @@ Thanks goes to these wonderful people ✨
5ven - - + syl20bnr @@ -878,7 +863,8 @@ Thanks goes to these wonderful people ✨
sylvaindumont - + + TechnicLab @@ -886,28 +872,13 @@ Thanks goes to these wonderful people ✨ TechnicLab - - - 42wim -
- 42wim - - - - - vilisas -
- vilisas - - thomasschmit
thomasschmit - - + Thiritin @@ -915,6 +886,13 @@ Thanks goes to these wonderful people ✨ Thiritin + + + tobiabocchi +
+ tobiabocchi + + tweibert @@ -928,7 +906,8 @@ Thanks goes to these wonderful people ✨
torus - + + VictorKoenders @@ -949,8 +928,7 @@ Thanks goes to these wonderful people ✨
k3it - - + Drakulix @@ -958,6 +936,28 @@ Thanks goes to these wonderful people ✨ Drakulix + + + vilisas +
+ vilisas + + + + + 42wim +
+ 42wim + + + + + + ShiriNmi1520 +
+ ShiriNmi1520 + + radicand @@ -1016,17 +1016,17 @@ Thanks goes to these wonderful people ✨ - - auchri + + peter-hartmann
- auchri + peter-hartmann - - peter-hartmann + + neuralp
- peter-hartmann + neuralp @@ -1109,20 +1109,27 @@ Thanks goes to these wonderful people ✨ - - ShiriNmi1520 + + sjmudd
- ShiriNmi1520 + sjmudd + + + Zepmann +
+ Zepmann + + + mchamplain
mchamplain - - + millerjason @@ -1157,15 +1164,15 @@ Thanks goes to these wonderful people ✨
olaf-mandel - + + ontheair81
ontheair81 - - + pravynandas @@ -1200,15 +1207,15 @@ Thanks goes to these wonderful people ✨
rriski - + + schnippl0r
schnippl0r - - + smargold476 @@ -1243,15 +1250,15 @@ Thanks goes to these wonderful people ✨
tamueller - + + vivacarvajalito
vivacarvajalito - - + wligtenberg @@ -1273,13 +1280,6 @@ Thanks goes to these wonderful people ✨ worldworm - - - Zepmann -
- Zepmann - - allddd @@ -1338,6 +1338,13 @@ Thanks goes to these wonderful people ✨ + + + fanqiaojun +
+ fanqiaojun + + ghnp5 @@ -1372,15 +1379,15 @@ Thanks goes to these wonderful people ✨
idaadi - + + ixeft
ixeft - - + jjtt @@ -1415,15 +1422,15 @@ Thanks goes to these wonderful people ✨
callmemagnus - + + marios88
marios88 - - + matrixes @@ -1431,6 +1438,13 @@ Thanks goes to these wonderful people ✨ matrixes + + + auchri +
+ auchri + + arkanovicz @@ -1451,7 +1465,8 @@ Thanks goes to these wonderful people ✨
damianmoore - + + espitall @@ -1465,8 +1480,7 @@ Thanks goes to these wonderful people ✨
dkarski - - + dbellavista @@ -1494,7 +1508,8 @@ Thanks goes to these wonderful people ✨
mlatorre31 - + + mazzz1y @@ -1508,8 +1523,7 @@ Thanks goes to these wonderful people ✨
doominator42 - - + aydodo @@ -1537,7 +1551,8 @@ Thanks goes to these wonderful people ✨
eliroca - + + ekkis @@ -1551,8 +1566,7 @@ Thanks goes to these wonderful people ✨
ErikEngerd - - + huncode @@ -1575,12 +1589,13 @@ Thanks goes to these wonderful people ✨ - - Kaan88 + + froks
- Kaan88 + froks - + + 0xflotus @@ -1594,8 +1609,7 @@ Thanks goes to these wonderful people ✨
ifokeev - - + 20th @@ -1623,7 +1637,8 @@ Thanks goes to these wonderful people ✨
aspettl - + + acch @@ -1637,8 +1652,7 @@ Thanks goes to these wonderful people ✨
vifino - - + kachkaev @@ -1666,7 +1680,8 @@ Thanks goes to these wonderful people ✨
eglia - + + groupmsl @@ -1680,8 +1695,7 @@ Thanks goes to these wonderful people ✨
green-anger - - + iRhonin @@ -1709,7 +1723,8 @@ Thanks goes to these wonderful people ✨
astrocket - + + baxerus @@ -1723,8 +1738,7 @@ Thanks goes to these wonderful people ✨
spock - - + erdos4d @@ -1739,13 +1753,21 @@ Thanks goes to these wonderful people ✨ crash7 + + + Kaan88 +
+ Kaan88 + + akkumar
akkumar - + + thechubbypanda @@ -1766,8 +1788,7 @@ Thanks goes to these wonderful people ✨
khuedoan - - + UltraCoderRU @@ -1788,7 +1809,8 @@ Thanks goes to these wonderful people ✨
leowinterde - + + linhandev @@ -1809,8 +1831,7 @@ Thanks goes to these wonderful people ✨
LucidityCrash - - + MadsRC @@ -1831,7 +1852,8 @@ Thanks goes to these wonderful people ✨
maxemann96 - + + dragetd @@ -1852,8 +1874,7 @@ Thanks goes to these wonderful people ✨
exhuma - - + milas @@ -1874,7 +1895,8 @@ Thanks goes to these wonderful people ✨
MohammedNoureldin - + + mpldr @@ -1889,14 +1911,6 @@ Thanks goes to these wonderful people ✨ naveensrinivasan - - - froks -
- froks - - - fkefer @@ -1924,7 +1938,8 @@ Thanks goes to these wonderful people ✨
GiovanH - + + harryyoud @@ -1938,8 +1953,7 @@ Thanks goes to these wonderful people ✨
HeySora - - + sirgantrithon @@ -1961,6 +1975,14 @@ Thanks goes to these wonderful people ✨ in-seo + + + firefly-cpp +
+ firefly-cpp + + + JacksonZ03 @@ -1981,8 +2003,7 @@ Thanks goes to these wonderful people ✨
jcalfee - - + mivek @@ -2003,7 +2024,8 @@ Thanks goes to these wonderful people ✨
Jeidnx - + + jessp01 @@ -2024,8 +2046,7 @@ Thanks goes to these wonderful people ✨
jirislav - - + jmccl From be8615f129b4616ea1d4f318da878e7fc01673e2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 28 Apr 2024 19:06:50 +0200 Subject: [PATCH 348/592] docs: updated `CONTRIBUTORS.md` (#3992) --- CONTRIBUTORS.md | 238 +++++++++++++++++++++++++----------------------- 1 file changed, 123 insertions(+), 115 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index e1027231aea..a96b2659359 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -665,17 +665,17 @@ Thanks goes to these wonderful people ✨ - - ubenmackin + + andrewlow
- ubenmackin + andrewlow - - craue + + aminvakil
- craue + aminvakil @@ -686,25 +686,25 @@ Thanks goes to these wonderful people ✨ - - andrewlow + + elbracht
- andrewlow + elbracht - - aminvakil + + craue
- aminvakil + craue - - elbracht + + ubenmackin
- elbracht + ubenmackin @@ -787,17 +787,17 @@ Thanks goes to these wonderful people ✨ - - fl42 + + millaguie
- fl42 + millaguie - - ipernet + + jamesfryer
- ipernet + jamesfryer @@ -808,20 +808,27 @@ Thanks goes to these wonderful people ✨ - - millaguie + + ipernet
- millaguie + ipernet - - jamesfryer + + fl42
- jamesfryer + fl42 + + + 0xflotus +
+ 0xflotus + + simonsystem @@ -856,15 +863,15 @@ Thanks goes to these wonderful people ✨
syl20bnr - + + sylvaindumont
sylvaindumont - - + TechnicLab @@ -899,15 +906,15 @@ Thanks goes to these wonderful people ✨
tweibert - + + torus
torus - - + VictorKoenders @@ -942,20 +949,27 @@ Thanks goes to these wonderful people ✨
vilisas - + + 42wim
42wim - - + - - ShiriNmi1520 + + matrixes
- ShiriNmi1520 + matrixes + + + + + neuralp +
+ neuralp @@ -978,7 +992,8 @@ Thanks goes to these wonderful people ✨
nknapp - + + pcqnt @@ -992,8 +1007,7 @@ Thanks goes to these wonderful people ✨
OrvilleQ - - + ovidiucp @@ -1021,22 +1035,15 @@ Thanks goes to these wonderful people ✨
peter-hartmann - - - - neuralp -
- neuralp - - + + piwai
piwai - - + remoe @@ -1071,15 +1078,15 @@ Thanks goes to these wonderful people ✨
MightySCollins - + + 501st-alpha1
501st-alpha1 - - + klamann @@ -1114,13 +1121,6 @@ Thanks goes to these wonderful people ✨
sjmudd - - - - Zepmann -
- Zepmann - @@ -1280,6 +1280,21 @@ Thanks goes to these wonderful people ✨ worldworm + + + ShiriNmi1520 +
+ ShiriNmi1520 + + + + + Zepmann +
+ Zepmann + + + allddd @@ -1293,8 +1308,7 @@ Thanks goes to these wonderful people ✨
arcaine2 - - + awb99 @@ -1322,7 +1336,8 @@ Thanks goes to these wonderful people ✨
dborowy - + + dimalo @@ -1336,8 +1351,7 @@ Thanks goes to these wonderful people ✨
eleith - - + fanqiaojun @@ -1365,7 +1379,8 @@ Thanks goes to these wonderful people ✨
hnws - + + i-C-o-d-e-r @@ -1379,8 +1394,7 @@ Thanks goes to these wonderful people ✨
idaadi - - + ixeft @@ -1408,7 +1422,8 @@ Thanks goes to these wonderful people ✨
jpduyx - + + landergate @@ -1422,8 +1437,7 @@ Thanks goes to these wonderful people ✨
callmemagnus - - + marios88 @@ -1431,20 +1445,6 @@ Thanks goes to these wonderful people ✨ marios88 - - - matrixes -
- matrixes - - - - - auchri -
- auchri - - arkanovicz @@ -1597,10 +1597,10 @@ Thanks goes to these wonderful people ✨ - - 0xflotus + + fkefer
- 0xflotus + fkefer @@ -1753,21 +1753,28 @@ Thanks goes to these wonderful people ✨ crash7 + + + auchri +
+ auchri + + Kaan88
Kaan88 - + + akkumar
akkumar - - + thechubbypanda @@ -1802,15 +1809,15 @@ Thanks goes to these wonderful people ✨
JustAnother1 - + + leowinterde
leowinterde - - + linhandev @@ -1845,15 +1852,15 @@ Thanks goes to these wonderful people ✨
madmath03 - + + maxemann96
maxemann96 - - + dragetd @@ -1888,15 +1895,15 @@ Thanks goes to these wonderful people ✨
mcchots - + + MohammedNoureldin
MohammedNoureldin - - + mpldr @@ -1912,10 +1919,10 @@ Thanks goes to these wonderful people ✨ - - fkefer + + furstblumier
- fkefer + furstblumier @@ -1931,15 +1938,15 @@ Thanks goes to these wonderful people ✨
glandais - + + GiovanH
GiovanH - - + harryyoud @@ -1974,15 +1981,15 @@ Thanks goes to these wonderful people ✨
in-seo - + + firefly-cpp
firefly-cpp - - + JacksonZ03 @@ -2017,15 +2024,15 @@ Thanks goes to these wonderful people ✨
init-js - + + Jeidnx
Jeidnx - - + jessp01 @@ -2060,7 +2067,8 @@ Thanks goes to these wonderful people ✨
jurekbarth - + + JOduMonT From 83da191f3a4cbe1d8ee458860de6e93ea152b08f Mon Sep 17 00:00:00 2001 From: Wael <8544289+forzagreen@users.noreply.github.com> Date: Thu, 2 May 2024 02:08:29 +0200 Subject: [PATCH 349/592] docs: Fix link for `getmail6` (#3996) --- docs/content/config/advanced/mail-getmail.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/config/advanced/mail-getmail.md b/docs/content/config/advanced/mail-getmail.md index d08f5d0733d..f62388b0226 100644 --- a/docs/content/config/advanced/mail-getmail.md +++ b/docs/content/config/advanced/mail-getmail.md @@ -95,7 +95,7 @@ environment: It is possible to utilize the `getmail-gmail-xoauth-tokens` helper to provide authentication using `xoauth2` for [gmail (example 12)][getmail-docs-xoauth-12] or [Microsoft Office 365 (example 13)][getmail-docs-xoauth-13] -[getmail-website]: https://www.getmail.org +[getmail-website]: https://www.getmail6.org [getmail-docs]: https://getmail6.org/configuration.html [getmail-docs-xoauth-12]: https://github.com/getmail6/getmail6/blob/1f95606156231f1e074ba62a9baa64f892a92ef8/docs/getmailrc-examples#L286 [getmail-docs-xoauth-13]: https://github.com/getmail6/getmail6/blob/1f95606156231f1e074ba62a9baa64f892a92ef8/docs/getmailrc-examples#L351 From 7dcbbd7173a8fd1713a77d55ed224d14c8970aa5 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 2 May 2024 19:41:25 +1200 Subject: [PATCH 350/592] fix(`accounts.sh`): Sync user home location for alias workaround (#3997) --- CHANGELOG.md | 1 + target/scripts/helpers/accounts.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10d3001614f..20ce0ed466d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,7 @@ The most noteworthy change of this release is the update of the container's base - Rspamd configuration: Add a missing comma in `local_networks` so that all internal IP addresses are actually considered as internal ([#3862](https://github.com/docker-mailserver/docker-mailserver/pull/3862)) - Ensure correct SELinux security context labels for files and directories moved to the mail-state volume during setup ([#3890](https://github.com/docker-mailserver/docker-mailserver/pull/3890)) - Use correct environment variable for fetchmail ([#3901](https://github.com/docker-mailserver/docker-mailserver/pull/3901)) +- Dovecot dummy accounts (_virtual alias workaround for dovecot feature `ENABLE_QUOTAS=1`_) now correctly matches the home location of the user for that alias ([#3997](https://github.com/docker-mailserver/docker-mailserver/pull/3997)) ## [v13.3.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.3.1) diff --git a/target/scripts/helpers/accounts.sh b/target/scripts/helpers/accounts.sh index 31ded04a6ba..78464b88893 100644 --- a/target/scripts/helpers/accounts.sh +++ b/target/scripts/helpers/accounts.sh @@ -134,7 +134,7 @@ function _create_dovecot_alias_dummy_accounts() { fi fi - DOVECOT_USERDB_LINE="${ALIAS}:${REAL_ACC[1]}:${DMS_VMAIL_UID}:${DMS_VMAIL_GID}::/var/mail/${REAL_DOMAINNAME}/${REAL_USERNAME}::${REAL_ACC[2]:-}" + DOVECOT_USERDB_LINE="${ALIAS}:${REAL_ACC[1]}:${DMS_VMAIL_UID}:${DMS_VMAIL_GID}::/var/mail/${REAL_DOMAINNAME}/${REAL_USERNAME}/home::${REAL_ACC[2]:-}" if grep -qi "^${ALIAS}:" "${DOVECOT_USERDB_FILE}"; then _log 'warn' "Alias '${ALIAS}' will not be added to '${DOVECOT_USERDB_FILE}' twice" else From d00edd7209d80f8f9b45bf9274fcb6cf858e6867 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 2 May 2024 19:44:54 +1200 Subject: [PATCH 351/592] docs: Revise fetchmail page (#3998) --- .../content/config/advanced/mail-fetchmail.md | 183 ++++++++++-------- 1 file changed, 105 insertions(+), 78 deletions(-) diff --git a/docs/content/config/advanced/mail-fetchmail.md b/docs/content/config/advanced/mail-fetchmail.md index 254196a191e..936d5188c9b 100644 --- a/docs/content/config/advanced/mail-fetchmail.md +++ b/docs/content/config/advanced/mail-fetchmail.md @@ -2,7 +2,7 @@ title: 'Advanced | Email Gathering with Fetchmail' --- -To enable the [fetchmail][fetchmail-website] service to retrieve e-mails set the environment variable `ENABLE_FETCHMAIL` to `1`. Your `compose.yaml` file should look like following snippet: +To enable the [fetchmail][fetchmail-website] service to retrieve e-mails, set the environment variable `ENABLE_FETCHMAIL` to `1`. Your `compose.yaml` file should look like following snippet: ```yaml environment: @@ -18,108 +18,135 @@ Generate a file called `fetchmail.cf` and place it in the `docker-data/dms/confi │   ├── fetchmail.cf │   ├── postfix-accounts.cf │   └── postfix-virtual.cf -├── compose.yaml -└── README.md +└── compose.yaml ``` ## Configuration -A detailed description of the configuration options can be found in the [online version of the manual page][fetchmail-docs]. +Configuration options for `fetchmail.cf` are covered at the [official fetchmail docs][fetchmail-docs-config] (_see the section "The run control file" and the table with "keyword" column for all settings_). -### IMAP Configuration +!!! example "Basic `fetchmail.cf` configuration" -!!! example + Retrieve mail from `remote-user@somewhere.com` and deliver it to `dms-user@example.com`: ```fetchmailrc - poll 'imap.gmail.com' proto imap - user 'username' - pass 'secret' - is 'user1@example.com' - ssl + poll 'mail.somewhere.com' + proto imap + user 'remote-user' + pass 'secret' + is 'dms-user@example.com' ``` -### POP3 Configuration + - `poll` sets the remote mail server to connect to retrieve mail from. + - `proto` lets you connect via IMAP or POP3. + - `user` and `pass` provide the login credentials for the remote mail service account to access. + - `is` configures where the fetched mail will be sent to (_eg: your local DMS account in `docker-data/dms/config/postfix-accounts.cf`_). -!!! example + --- - ```fetchmailrc - poll 'pop3.gmail.com' proto pop3 - user 'username' - pass 'secret' - is 'user2@example.com' - ssl - ``` + ??? warning "`proto imap` will still delete remote mail once fetched" -!!! caution + This is due to a separate default setting `no keep`. Adding the setting `keep` to your config on a new line will prevent deleting the remote copy. - Don’t forget the last line! (_eg: `is 'user1@example.com'`_). After `is`, you have to specify an email address from the configuration file: `docker-data/dms/config/postfix-accounts.cf`. +??? example "Multiple users or remote servers" -More details how to configure fetchmail can be found in the [fetchmail man page in the chapter “The run control file”][fetchmail-docs-run]. + The official docs [config examples][fetchmail-config-examples] show a common convention to indent settings on subsequent lines for visually grouping per server. -### Polling Interval + === "Minimal syntax" -By default the fetchmail service searches every 5 minutes for new mails on your external mail accounts. You can override this default value by changing the ENV variable `FETCHMAIL_POLL`: + ```fetchmailrc + poll 'mail.somewhere.com' proto imap + user 'john.doe' pass 'secret' is 'johnny@example.com' + user 'jane.doe' pass 'secret' is 'jane@example.com' -```yaml -environment: - - FETCHMAIL_POLL=60 -``` + poll 'mail.somewhere-else.com' proto pop3 + user 'john.doe@somewhere-else.com' pass 'secret' is 'johnny@example.com' + ``` + + === "With optional syntax" + + - `#` for adding comments. + - The config file may include "noise" keywords to improve readability. + + ```fetchmailrc + # Retrieve mail for users `john.doe` and `jane.doe` via IMAP at this remote mail server: + poll 'mail.somewhere.com' with proto imap wants: + user 'john.doe' with pass 'secret', is 'johnny@example.com' here + user 'jane.doe' with pass 'secret', is 'jane@example.com' here -You must specify a numeric argument which is a polling interval in seconds. The example above polls every minute for new mails. + # Also retrieve mail from this mail server (but via POP3). + # NOTE: This could also be all on a single line, or with each key + value as a separate line. + # Notice how the remote username includes a full email address, + # Some mail servers like DMS use the full email address as the username: + poll 'mail.somewhere-else.com' with proto pop3 wants: + user 'john.doe@somewhere-else.com' with pass 'secret', is 'johnny@example.com' here + ``` + +!!! tip "`FETCHMAIL_POLL` ENV: Override default polling interval" + + By default the fetchmail service will check every 5 minutes for new mail at the configured mail accounts. + + ```yaml + environment: + # The fetchmail polling interval in seconds: + FETCHMAIL_POLL: 60 + ``` ## Debugging -To debug your `fetchmail.cf` configuration run this command: +To debug your `fetchmail.cf` configuration run this `setup debug` command: ```sh -./setup.sh debug fetchmail +docker exec -it dms-container-name setup debug fetchmail ``` -For more information about the configuration script `setup.sh` [read the corresponding docs][docs-setup]. - -Here a sample output of `./setup.sh debug fetchmail`: - -```log -fetchmail: 6.3.26 querying outlook.office365.com (protocol POP3) at Mon Aug 29 22:11:09 2016: poll started -Trying to connect to 132.245.48.18/995...connected. -fetchmail: Server certificate: -fetchmail: Issuer Organization: Microsoft Corporation -fetchmail: Issuer CommonName: Microsoft IT SSL SHA2 -fetchmail: Subject CommonName: outlook.com -fetchmail: Subject Alternative Name: outlook.com -fetchmail: Subject Alternative Name: *.outlook.com -fetchmail: Subject Alternative Name: office365.com -fetchmail: Subject Alternative Name: *.office365.com -fetchmail: Subject Alternative Name: *.live.com -fetchmail: Subject Alternative Name: *.internal.outlook.com -fetchmail: Subject Alternative Name: *.outlook.office365.com -fetchmail: Subject Alternative Name: outlook.office.com -fetchmail: Subject Alternative Name: attachment.outlook.office.net -fetchmail: Subject Alternative Name: attachment.outlook.officeppe.net -fetchmail: Subject Alternative Name: *.office.com -fetchmail: outlook.office365.com key fingerprint: 3A:A4:58:42:56:CD:BD:11:19:5B:CF:1E:85:16:8E:4D -fetchmail: POP3< +OK The Microsoft Exchange POP3 service is ready. [SABFADEAUABSADAAMQBDAEEAMAAwADAANwAuAGUAdQByAHAAcgBkADAAMQAuAHAAcgBvAGQALgBlAHgAYwBoAGEAbgBnAGUAbABhAGIAcwAuAGMAbwBtAA==] -fetchmail: POP3> CAPA -fetchmail: POP3< +OK -fetchmail: POP3< TOP -fetchmail: POP3< UIDL -fetchmail: POP3< SASL PLAIN -fetchmail: POP3< USER -fetchmail: POP3< . -fetchmail: POP3> USER user1@outlook.com -fetchmail: POP3< +OK -fetchmail: POP3> PASS * -fetchmail: POP3< +OK User successfully logged on. -fetchmail: POP3> STAT -fetchmail: POP3< +OK 0 0 -fetchmail: No mail for user1@outlook.com at outlook.office365.com -fetchmail: POP3> QUIT -fetchmail: POP3< +OK Microsoft Exchange Server 2016 POP3 server signing off. -fetchmail: 6.3.26 querying outlook.office365.com (protocol POP3) at Mon Aug 29 22:11:11 2016: poll completed -fetchmail: normal termination, status 1 -``` +??? example "Sample output of `setup debug fetchmail`" + + ```log + fetchmail: 6.3.26 querying outlook.office365.com (protocol POP3) at Mon Aug 29 22:11:09 2016: poll started + Trying to connect to 132.245.48.18/995...connected. + fetchmail: Server certificate: + fetchmail: Issuer Organization: Microsoft Corporation + fetchmail: Issuer CommonName: Microsoft IT SSL SHA2 + fetchmail: Subject CommonName: outlook.com + fetchmail: Subject Alternative Name: outlook.com + fetchmail: Subject Alternative Name: *.outlook.com + fetchmail: Subject Alternative Name: office365.com + fetchmail: Subject Alternative Name: *.office365.com + fetchmail: Subject Alternative Name: *.live.com + fetchmail: Subject Alternative Name: *.internal.outlook.com + fetchmail: Subject Alternative Name: *.outlook.office365.com + fetchmail: Subject Alternative Name: outlook.office.com + fetchmail: Subject Alternative Name: attachment.outlook.office.net + fetchmail: Subject Alternative Name: attachment.outlook.officeppe.net + fetchmail: Subject Alternative Name: *.office.com + fetchmail: outlook.office365.com key fingerprint: 3A:A4:58:42:56:CD:BD:11:19:5B:CF:1E:85:16:8E:4D + fetchmail: POP3< +OK The Microsoft Exchange POP3 service is ready. [SABFADEAUABSADAAMQBDAEEAMAAwADAANwAuAGUAdQByAHAAcgBkADAAMQAuAHAAcgBvAGQALgBlAHgAYwBoAGEAbgBnAGUAbABhAGIAcwAuAGMAbwBtAA==] + fetchmail: POP3> CAPA + fetchmail: POP3< +OK + fetchmail: POP3< TOP + fetchmail: POP3< UIDL + fetchmail: POP3< SASL PLAIN + fetchmail: POP3< USER + fetchmail: POP3< . + fetchmail: POP3> USER user1@outlook.com + fetchmail: POP3< +OK + fetchmail: POP3> PASS * + fetchmail: POP3< +OK User successfully logged on. + fetchmail: POP3> STAT + fetchmail: POP3< +OK 0 0 + fetchmail: No mail for user1@outlook.com at outlook.office365.com + fetchmail: POP3> QUIT + fetchmail: POP3< +OK Microsoft Exchange Server 2016 POP3 server signing off. + fetchmail: 6.3.26 querying outlook.office365.com (protocol POP3) at Mon Aug 29 22:11:11 2016: poll completed + fetchmail: normal termination, status 1 + ``` + +!!! tip "Troubleshoot with this reference `compose.yaml`" + + [A minimal `compose.yaml` example][fetchmail-compose-example] demonstrates how to run two instances of DMS locally, with one instance configured with `fetchmail.cf` and the other to simulate a remote mail server to fetch from. -[docs-setup]: ../../config/setup.sh.md [fetchmail-website]: https://www.fetchmail.info -[fetchmail-docs]: https://www.fetchmail.info/fetchmail-man.html -[fetchmail-docs-run]: https://www.fetchmail.info/fetchmail-man.html#31 +[fetchmail-docs-config]: https://www.fetchmail.info/fetchmail-man.html#the-run-control-file +[fetchmail-config-examples]: https://www.fetchmail.info/fetchmail-man.html#configuration-examples +[fetchmail-compose-example]: https://github.com/orgs/docker-mailserver/discussions/3994#discussioncomment-9290570 From 7822a9743034c486cf2ceeebde64eed6c3ace8c2 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 2 May 2024 19:48:05 +1200 Subject: [PATCH 352/592] docs(FAQ): Add advice for restricting login by IP (#3999) --- docs/content/faq.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/content/faq.md b/docs/content/faq.md index 4add05896a8..a350f93786a 100644 --- a/docs/content/faq.md +++ b/docs/content/faq.md @@ -291,6 +291,12 @@ mydestination = localhost.$mydomain, localhost proxy_interfaces = X.X.X.X (your public IP) ``` +For reverse proxy support you will want to view [our dedicated guide][docs::examples::reverse-proxy]. + +### How to restrict login by IP? + +There are a few ways you could approach this, see [this discussion answer][gh-discussion::restrict-login-by-ip] for advice. + ### How to adjust settings with the `user-patches.sh` script Suppose you want to change a number of settings that are not listed as variables or add things to the server that are not included? @@ -496,6 +502,7 @@ $spam_quarantine_to = "quarantine\@example.com"; [docs-userpatches]: ./config/advanced/override-defaults/user-patches.md [docs::env::sa_env]: ./config/environment.md#spamassassin [docs::env::sa_kill]: ./config/environment.md#sa_kill +[docs::examples::reverse-proxy]: ./examples/tutorials/mailserver-behind-proxy.md [github-comment-baredomain]: https://github.com/docker-mailserver/docker-mailserver/issues/3048#issuecomment-1432358353 [github-comment-override-hostname]: https://github.com/docker-mailserver/docker-mailserver/issues/1731#issuecomment-753968425 [github-issue-95]: https://github.com/docker-mailserver/docker-mailserver/issues/95 @@ -505,5 +512,6 @@ $spam_quarantine_to = "quarantine\@example.com"; [github-issue-1639]: https://github.com/docker-mailserver/docker-mailserver/issues/1639 [github-issue-1792]: https://github.com/docker-mailserver/docker-mailserver/pull/1792 [gh-discussion::dms-avoid-maintaining-internal-dns]: https://github.com/orgs/docker-mailserver/discussions/3959#discussioncomment-8956322 +[gh-discussion::restrict-login-by-ip]: https://github.com/orgs/docker-mailserver/discussions/3847 [docker-compose::docs::config-dns]: https://docs.docker.com/compose/compose-file/compose-file-v3/#dns [hanscees-userpatches]: https://github.com/hanscees/dockerscripts/blob/master/scripts/tomav-user-patches.sh From e2c2a22dcf2d8adc12f303b84b694945e8dc072e Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 3 May 2024 11:12:34 +1200 Subject: [PATCH 353/592] fix: `postfix-main.cf` may depend upon `postfix-master.cf` (#3880) Custom parameters must be referenced to be retained when `postconf -n` is run. If those parameters are referenced by `postfix-master.cf` this needs to update `master.cf` before updating `main.cf`. --- CHANGELOG.md | 4 ++- target/scripts/startup/setup.d/postfix.sh | 30 +++++++++++-------- test/config/override-configs/postfix-main.cf | 3 ++ .../config/override-configs/postfix-master.cf | 4 +++ .../tests/parallel/set1/config_overrides.bats | 21 +++++++++++++ 5 files changed, 48 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20ce0ed466d..d651b0b59bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,7 +96,9 @@ The most noteworthy change of this release is the update of the container's base ### Fixes -- DMS config files that are parsed line by line are now more robust to parse by detecting and fixing line-endings ([#3819](https://github.com/docker-mailserver/docker-mailserver/pull/3819)) +- DMS config: + - Files that are parsed line by line are now more robust to parse by detecting and fixing line-endings ([#3819](https://github.com/docker-mailserver/docker-mailserver/pull/3819)) + - The override config `postfix-main.cf` now retains custom parameters intended for use with `postfix-master.cf` ([#3880](https://github.com/docker-mailserver/docker-mailserver/pull/3880)) - Variables related to Rspamd are declared as `readonly`, which would cause warnings in the log when being re-declared; we now guard against this issue ([#3837](https://github.com/docker-mailserver/docker-mailserver/pull/3837)) - Relay host feature refactored ([#3845](https://github.com/docker-mailserver/docker-mailserver/pull/3845)) - `DEFAULT_RELAY_HOST` ENV can now also use the `RELAY_USER` + `RELAY_PASSWORD` ENV for supplying credentials. diff --git a/target/scripts/startup/setup.d/postfix.sh b/target/scripts/startup/setup.d/postfix.sh index 05052faa5de..3ebdeb9d95b 100644 --- a/target/scripts/startup/setup.d/postfix.sh +++ b/target/scripts/startup/setup.d/postfix.sh @@ -109,29 +109,33 @@ function _setup_postfix_late() { function __postfix__setup_override_configuration() { __postfix__log 'debug' 'Overriding / adjusting configuration with user-supplied values' + local OVERRIDE_CONFIG_POSTFIX_MASTER='/tmp/docker-mailserver/postfix-master.cf' + if [[ -f ${OVERRIDE_CONFIG_POSTFIX_MASTER} ]]; then + while read -r LINE; do + [[ ${LINE} =~ ^[0-9a-z] ]] && postconf -P "${LINE}" + done < <(_get_valid_lines_from_file "${OVERRIDE_CONFIG_POSTFIX_MASTER}") + __postfix__log 'trace' "Adjusted '/etc/postfix/master.cf' according to '${OVERRIDE_CONFIG_POSTFIX_MASTER}'" + else + __postfix__log 'trace' "No extra Postfix master settings loaded because optional '${OVERRIDE_CONFIG_POSTFIX_MASTER}' was not provided" + fi + + # NOTE: `postfix-main.cf` should be handled after `postfix-master.cf` as custom parameters require an existing reference + # in either `main.cf` or `master.cf` prior to `postconf` reading `main.cf`, otherwise it is discarded from output. local OVERRIDE_CONFIG_POSTFIX_MAIN='/tmp/docker-mailserver/postfix-main.cf' if [[ -f ${OVERRIDE_CONFIG_POSTFIX_MAIN} ]]; then cat "${OVERRIDE_CONFIG_POSTFIX_MAIN}" >>/etc/postfix/main.cf _adjust_mtime_for_postfix_maincf - # do not directly output to 'main.cf' as this causes a read-write-conflict - postconf -n >/tmp/postfix-main-new.cf 2>/dev/null + # Do not directly output to 'main.cf' as this causes a read-write-conflict. + # `postconf` output is filtered to skip expected warnings regarding overrides: + # https://github.com/docker-mailserver/docker-mailserver/pull/3880#discussion_r1510414576 + postconf -n >/tmp/postfix-main-new.cf 2> >(grep -v 'overriding earlier entry') mv /tmp/postfix-main-new.cf /etc/postfix/main.cf _adjust_mtime_for_postfix_maincf __postfix__log 'trace' "Adjusted '/etc/postfix/main.cf' according to '${OVERRIDE_CONFIG_POSTFIX_MAIN}'" else - __postfix__log 'trace' "No extra Postfix settings loaded because optional '${OVERRIDE_CONFIG_POSTFIX_MAIN}' was not provided" - fi - - local OVERRIDE_CONFIG_POSTFIX_MASTER='/tmp/docker-mailserver/postfix-master.cf' - if [[ -f ${OVERRIDE_CONFIG_POSTFIX_MASTER} ]]; then - while read -r LINE; do - [[ ${LINE} =~ ^[0-9a-z] ]] && postconf -P "${LINE}" - done < <(_get_valid_lines_from_file "${OVERRIDE_CONFIG_POSTFIX_MASTER}") - __postfix__log 'trace' "Adjusted '/etc/postfix/master.cf' according to '${OVERRIDE_CONFIG_POSTFIX_MASTER}'" - else - __postfix__log 'trace' "No extra Postfix settings loaded because optional '${OVERRIDE_CONFIG_POSTFIX_MASTER}' was not provided" + __postfix__log 'trace' "No extra Postfix main settings loaded because optional '${OVERRIDE_CONFIG_POSTFIX_MAIN}' was not provided" fi } diff --git a/test/config/override-configs/postfix-main.cf b/test/config/override-configs/postfix-main.cf index ce07bd56eb6..fb29f3ccfa3 100644 --- a/test/config/override-configs/postfix-main.cf +++ b/test/config/override-configs/postfix-main.cf @@ -2,3 +2,6 @@ max_idle = 600s # this is a comment # this is also a comment readme_directory = /tmp + +# This parameter is referenced by an override in `postfix-master.cf`: +custom_parameter = cidr:{{!172.16.0.42 REJECT}}, permit_sasl_authenticated, reject diff --git a/test/config/override-configs/postfix-master.cf b/test/config/override-configs/postfix-master.cf index 516fea81b9d..f54d37d32be 100644 --- a/test/config/override-configs/postfix-master.cf +++ b/test/config/override-configs/postfix-master.cf @@ -1,3 +1,7 @@ submission/inet/smtpd_sasl_security_options=noanonymous # this is a test comment, please don't delete me :'( # this is also a test comment, :O + +# DMS must first process `postfix-master.cf` to `master.cf` when any custom parameters are from `postfix-main.cf` +# before an updated `main.cf` is read via `postconf`, otherwise without an existing reference the parameter is excluded. +submission/inet/smtpd_client_restrictions=$custom_parameter diff --git a/test/tests/parallel/set1/config_overrides.bats b/test/tests/parallel/set1/config_overrides.bats index 0a7a5e7f4ad..0281d197722 100644 --- a/test/tests/parallel/set1/config_overrides.bats +++ b/test/tests/parallel/set1/config_overrides.bats @@ -15,6 +15,9 @@ function setup_file() { function teardown_file() { _default_teardown ; } +# The `postconf` command can query both `main.cf` and `master.cf` at `/etc/postfix/`. +# Reference: http://www.postfix.org/postconf.1.html + @test "Postfix - 'postfix-main.cf' overrides applied to '/etc/postfix/main.cf'" { _run_in_container grep -q 'max_idle = 600s' /tmp/docker-mailserver/postfix-main.cf assert_success @@ -37,6 +40,24 @@ function teardown_file() { _default_teardown ; } assert_output --partial '-o smtpd_sasl_security_options=noanonymous' } +# Custom parameter support works correctly: +# NOTE: This would only fail on a fresh container state, any restart would pass successfully: +# https://github.com/docker-mailserver/docker-mailserver/pull/3880 +@test "Postfix - 'postfix-master.cf' should apply before 'postfix-main.cf'" { + # Retrieve the value for this setting, `postfix-master.cf` should have the override set: + _run_in_container postconf -Ph 'submission/inet/smtpd_client_restrictions' + assert_success + refute_output --partial 'postconf: warning: /etc/postfix/master.cf: undefined parameter: custom_parameter' + #shellcheck disable=SC2016 + assert_output '$custom_parameter' + + # As it's a custom parameter (`$` prefix), ensure the parameter value expands correctly: + _run_in_container postconf -Phx 'submission/inet/smtpd_client_restrictions' + assert_success + refute_output --partial 'postconf: warning: /etc/postfix/master.cf: undefined parameter: custom_parameter' + assert_output 'cidr:{{!172.16.0.42 REJECT}}, permit_sasl_authenticated, reject' +} + @test "Dovecot - 'dovecot.cf' overrides applied to '/etc/dovecot/local.conf'" { _run_in_container grep -q 'mail_max_userip_connections = 69' /tmp/docker-mailserver/dovecot.cf assert_success From d52b813cd90833f5a0f0df01128c5bbd362c7870 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 5 May 2024 11:42:59 +0200 Subject: [PATCH 354/592] docs: updated `CONTRIBUTORS.md` (#4002) --- CONTRIBUTORS.md | 173 +++++++++++++++++++++++++----------------------- 1 file changed, 90 insertions(+), 83 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index a96b2659359..3ecae553821 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -679,17 +679,17 @@ Thanks goes to these wonderful people ✨ - - abh + + elbracht
- abh + elbracht - - elbracht + + abh
- elbracht + abh @@ -823,17 +823,10 @@ Thanks goes to these wonderful people ✨ - - 0xflotus -
- 0xflotus - - - - - simonsystem + + neuralp
- simonsystem + neuralp @@ -863,15 +856,15 @@ Thanks goes to these wonderful people ✨
syl20bnr - - + sylvaindumont
sylvaindumont - + + TechnicLab @@ -906,15 +899,15 @@ Thanks goes to these wonderful people ✨
tweibert - - + torus
torus - + + VictorKoenders @@ -949,6 +942,13 @@ Thanks goes to these wonderful people ✨
vilisas + + + + forzagreen +
+ forzagreen + @@ -965,13 +965,6 @@ Thanks goes to these wonderful people ✨ matrixes - - - neuralp -
- neuralp - - radicand @@ -992,15 +985,15 @@ Thanks goes to these wonderful people ✨
nknapp - - + pcqnt
pcqnt - + + OrvilleQ @@ -1035,15 +1028,15 @@ Thanks goes to these wonderful people ✨
peter-hartmann - - + piwai
piwai - + + remoe @@ -1078,15 +1071,15 @@ Thanks goes to these wonderful people ✨
MightySCollins - - + 501st-alpha1
501st-alpha1 - + + klamann @@ -1121,6 +1114,13 @@ Thanks goes to these wonderful people ✨
sjmudd + + + + simonsystem +
+ simonsystem + @@ -1445,6 +1445,13 @@ Thanks goes to these wonderful people ✨ marios88 + + + 0xflotus +
+ 0xflotus + + arkanovicz @@ -1458,15 +1465,15 @@ Thanks goes to these wonderful people ✨
CBeerta - + + damianmoore
damianmoore - - + espitall @@ -1501,15 +1508,15 @@ Thanks goes to these wonderful people ✨
denisix - + + mlatorre31
mlatorre31 - - + mazzz1y @@ -1544,15 +1551,15 @@ Thanks goes to these wonderful people ✨
edvorg - + + eliroca
eliroca - - + ekkis @@ -1587,15 +1594,15 @@ Thanks goes to these wonderful people ✨
flole - + + froks
froks - - + fkefer @@ -1630,15 +1637,15 @@ Thanks goes to these wonderful people ✨
askz - + + aspettl
aspettl - - + acch @@ -1673,15 +1680,15 @@ Thanks goes to these wonderful people ✨
ch3sh1r - + + eglia
eglia - - + groupmsl @@ -1716,15 +1723,15 @@ Thanks goes to these wonderful people ✨
arunvc - + + astrocket
astrocket - - + baxerus @@ -1759,15 +1766,15 @@ Thanks goes to these wonderful people ✨
auchri - + + Kaan88
Kaan88 - - + akkumar @@ -1802,15 +1809,15 @@ Thanks goes to these wonderful people ✨
UltraCoderRU - + + JustAnother1
JustAnother1 - - + leowinterde @@ -1845,15 +1852,15 @@ Thanks goes to these wonderful people ✨
MadsRC - + + madmath03
madmath03 - - + maxemann96 @@ -1888,15 +1895,15 @@ Thanks goes to these wonderful people ✨
milas - + + mcchots
mcchots - - + MohammedNoureldin @@ -1931,15 +1938,15 @@ Thanks goes to these wonderful people ✨
Marsu31 - + + glandais
glandais - - + GiovanH @@ -1974,15 +1981,15 @@ Thanks goes to these wonderful people ✨
Influencer - + + in-seo
in-seo - - + firefly-cpp @@ -2017,15 +2024,15 @@ Thanks goes to these wonderful people ✨
mivek - + + init-js
init-js - - + Jeidnx @@ -2060,15 +2067,15 @@ Thanks goes to these wonderful people ✨
jmccl - + + jurekbarth
jurekbarth - - + JOduMonT From 10f72224ca9ee3d2f82b187270d7099cf46537e1 Mon Sep 17 00:00:00 2001 From: "F. Eber" <138527669+gitfeber@users.noreply.github.com> Date: Sun, 5 May 2024 19:26:58 +0200 Subject: [PATCH 355/592] Update typo in kubernetes.md (#4003) Added the missing "s" on "submissions", otherwise this error comes up: The Service "mailserver" is invalid: spec.ports[2].name: Duplicate value: "submission" --- docs/content/config/advanced/kubernetes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/config/advanced/kubernetes.md b/docs/content/config/advanced/kubernetes.md index 8f5ac901ce5..268094a6c24 100644 --- a/docs/content/config/advanced/kubernetes.md +++ b/docs/content/config/advanced/kubernetes.md @@ -152,7 +152,7 @@ If using our Helm chart is not viable for you, here is some guidance to start wi targetPort: smtp protocol: TCP # submissions (ESMTP with implicit TLS) - - name: submission + - name: submissions port: 465 targetPort: submissions protocol: TCP From 016d6b5255be6945b08616a79f92f496a583b775 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 19:02:41 +0200 Subject: [PATCH 356/592] chore(deps): Bump akhilmhdh/contributors-readme-action (#4005) --- .github/workflows/contributors.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml index 9a71b64880f..6ca6027bfa8 100644 --- a/.github/workflows/contributors.yml +++ b/.github/workflows/contributors.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v4 - name: 'Update CONTRIBUTORS.md' - uses: akhilmhdh/contributors-readme-action@v2.3.8 + uses: akhilmhdh/contributors-readme-action@v2.3.10 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: From dab3d9fe71091e6f8cc4c20ae49116de76056935 Mon Sep 17 00:00:00 2001 From: mmehnert Date: Sun, 12 May 2024 09:59:22 +0200 Subject: [PATCH 357/592] chore(logwatch): Add `ignore.conf` to ignore logs from Dovecot `index-worker` (#4012) --- CHANGELOG.md | 2 ++ Dockerfile | 1 + target/logwatch/ignore.conf | 2 ++ 3 files changed, 5 insertions(+) create mode 100644 target/logwatch/ignore.conf diff --git a/CHANGELOG.md b/CHANGELOG.md index d651b0b59bd..0ead51f5377 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,8 @@ The most noteworthy change of this release is the update of the container's base - The `rewrite_subject` action, is now disabled by default. It has been replaced with the new `SPAM_SUBJECT` environment variable, which implements the functionality via a Sieve script instead which is anti-spam service agnostic ([#3820](https://github.com/docker-mailserver/docker-mailserver/pull/3820)) - `RSPAMD_NEURAL` was added and is disabled by default. If switched on it will enable the experimental Rspamd "Neural network" module to add a layer of analysis to spam detection ([#3833](https://github.com/docker-mailserver/docker-mailserver/pull/3833)) - The symbol weights of SPF, DKIM and DMARC have been adjusted again. Fixes a bug and includes more appropriate combinations of symbols ([#3913](https://github.com/docker-mailserver/docker-mailserver/pull/3913), [#3923](https://github.com/docker-mailserver/docker-mailserver/pull/3923)) +- **Dovecot:** + - `logwatch` now filters out non-error logs related to the status of the `index-worker` process for FTS indexing. ([#4012](https://github.com/docker-mailserver/docker-mailserver/pull/4012)) ### Fixes diff --git a/Dockerfile b/Dockerfile index 8b2a502c4e4..bf688ec1e9d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -273,6 +273,7 @@ EOF # ----------------------------------------------- COPY target/logwatch/maillog.conf /etc/logwatch/conf/logfiles/maillog.conf +COPY target/logwatch/ignore.conf /etc/logwatch/conf/ignore.conf # ----------------------------------------------- # --- Supervisord & Start ----------------------- diff --git a/target/logwatch/ignore.conf b/target/logwatch/ignore.conf new file mode 100644 index 00000000000..5b7e9312f78 --- /dev/null +++ b/target/logwatch/ignore.conf @@ -0,0 +1,2 @@ +# ignore output from dovecot-fts-xapian about successfully indexed messages +dovecot: indexer-worker\([^\)]+\).*Indexed From 5bd8df68eb06b0f8879f7a953a6da607b77a4730 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 12 May 2024 11:55:22 +0200 Subject: [PATCH 358/592] docs: updated `CONTRIBUTORS.md` (#4014) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 4216 ++++++++++++++++++++++++----------------------- 1 file changed, 2137 insertions(+), 2079 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 3ecae553821..4275d39d72f 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -4,2084 +4,2142 @@ Thanks goes to these wonderful people ✨ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
- - casperklein -
- casperklein - - 		- - fbartels -
- fbartels - - 		- - polarathene -
- polarathene - - 		- - NorseGaud -
- NorseGaud - - 		- - williamdes -
- williamdes - - 		- - wernerfred -
- wernerfred - -
- - georglauterbach -
- georglauterbach - - 		- - tomav -
- tomav - - 		- - erik-wramner -
- erik-wramner - - 		- - chikamichi -
- chikamichi - - 		- - martin-schulze-vireso -
- martin-schulze-vireso - - 		- - Josef-Friedrich -
- Josef-Friedrich - -
- - johansmitsnl -
- johansmitsnl - - 		- - youtous -
- youtous - - 		- - 17Halbe -
- 17Halbe - - 		- - tve -
- tve - - 		- - gmasse -
- gmasse - - 		- - ap-wtioit -
- ap-wtioit - -
- - 00angus -
- 00angus - - 		- - alinmear -
- alinmear - - 		- - dominikwinter -
- dominikwinter - - 		- - crazystick -
- crazystick - - 		- - svenyonson -
- svenyonson - - 		- - swiesend -
- swiesend - -
- - stonemaster -
- stonemaster - - 		- - omarc1492 -
- omarc1492 - - 		- - phish108 -
- phish108 - - 		- - mwlczk -
- mwlczk - - 		- - tyranron -
- tyranron - - 		- - KyleOndy -
- KyleOndy - -
- - mindrunner -
- mindrunner - - 		- - MichaelSp -
- MichaelSp - - 		- - m-a-v -
- m-a-v - - 		- - bilak -
- bilak - - 		- - vortex852456 -
- vortex852456 - - 		- - hanscees -
- hanscees - -
- - chris54721 -
- chris54721 - - 		- - jrpear -
- jrpear - - 		- - moqmar -
- moqmar - - 		- - pyy -
- pyy - - 		- - arneke -
- arneke - - 		- - akmet -
- akmet - -
- - diiigle -
- diiigle - - 		- - kiliant -
- kiliant - - 		- - dashohoxha -
- dashohoxha - - 		- - egavard -
- egavard - - 		- - mathuin -
- mathuin - - 		- - jamebus -
- jamebus - -
- - robertdolca -
- robertdolca - - 		- - okainov -
- okainov - - 		- - jsonn -
- jsonn - - 		- - lukecyca -
- lukecyca - - 		- - m-schmoock -
- m-schmoock - - 		- - mjung -
- mjung - -
- - eltociear -
- eltociear - - 		- - VanVan -
- VanVan - - 		- - voordev -
- voordev - - 		- - andreasgerstmayr -
- andreasgerstmayr - - 		- - davidszp -
- davidszp - - 		- - kamuri -
- kamuri - -
- - guardiande -
- guardiande - - 		- - Zehir -
- Zehir - - 		- - Birkenstab -
- Birkenstab - - 		- - BrandonSchmitt -
- BrandonSchmitt - - 		- - Starbix -
- Starbix - - 		- - citec -
- citec - -
- - yajo -
- yajo - - 		- - MakerMatrix -
- MakerMatrix - - 		- - weo -
- weo - - 		- - analogue -
- analogue - - 		- - Rubytastic2 -
- Rubytastic2 - - 		- - reneploetz -
- reneploetz - -
- - pbek -
- pbek - - 		- - yogo1212 -
- yogo1212 - - 		- - castorinop -
- castorinop - - 		- - p-fruck -
- p-fruck - - 		- - tbutter -
- tbutter - - 		- - rahilarious -
- rahilarious - -
- - Rillke -
- Rillke - - 		- - bobbravo2 -
- bobbravo2 - - 		- - r-pufky -
- r-pufky - - 		- - vincentDcmps -
- vincentDcmps - - 		- - frugan-dev -
- frugan-dev - - 		- - mpanneck -
- mpanneck - -
- - andymel123 -
- andymel123 - - 		- - bigpigeon -
- bigpigeon - - 		- - engelant -
- engelant - - 		- - j-marz -
- j-marz - - 		- - lokipo -
- lokipo - - 		- - msheakoski -
- msheakoski - -
- - willtho89 -
- willtho89 - - 		- - GoliathLabs -
- GoliathLabs - - 		- - andrewlow -
- andrewlow - - 		- - aminvakil -
- aminvakil - - 		- - elbracht -
- elbracht - - 		- - abh -
- abh - -
- - craue -
- craue - - 		- - ubenmackin -
- ubenmackin - - 		- - danielpanteleit -
- danielpanteleit - - 		- - dmcgrandle -
- dmcgrandle - - 		- - theomega -
- theomega - - 		- - DuncanvR -
- DuncanvR - -
- - emazzotta -
- emazzotta - - 		- - keslerm -
- keslerm - - 		- - nueaf -
- nueaf - - 		- - martinwepner -
- martinwepner - - 		- - artonge -
- artonge - - 		- - spacecowboy -
- spacecowboy - -
- - jedateach -
- jedateach - - 		- - millaguie -
- millaguie - - 		- - jamesfryer -
- jamesfryer - - 		- - H4R0 -
- H4R0 - - 		- - ipernet -
- ipernet - - 		- - fl42 -
- fl42 - -
- - neuralp -
- neuralp - - 		- - stephan-devop -
- stephan-devop - - 		- - stigok -
- stigok - - 		- - 5ven -
- 5ven - - 		- - syl20bnr -
- syl20bnr - - 		- - sylvaindumont -
- sylvaindumont - -
- - TechnicLab -
- TechnicLab - - 		- - thomasschmit -
- thomasschmit - - 		- - Thiritin -
- Thiritin - - 		- - tobiabocchi -
- tobiabocchi - - 		- - tweibert -
- tweibert - - 		- - torus -
- torus - -
- - VictorKoenders -
- VictorKoenders - - 		- - Twist235 -
- Twist235 - - 		- - k3it -
- k3it - - 		- - Drakulix -
- Drakulix - - 		- - vilisas -
- vilisas - - 		- - forzagreen -
- forzagreen - -
- - 42wim -
- 42wim - - 		- - matrixes -
- matrixes - - 		- - radicand -
- radicand - - 		- - nilshoell -
- nilshoell - - 		- - nknapp -
- nknapp - - 		- - pcqnt -
- pcqnt - -
- - OrvilleQ -
- OrvilleQ - - 		- - ovidiucp -
- ovidiucp - - 		- - mrPjer -
- mrPjer - - 		- - p3dda -
- p3dda - - 		- - peter-hartmann -
- peter-hartmann - - 		- - piwai -
- piwai - -
- - remoe -
- remoe - - 		- - robbertkl -
- robbertkl - - 		- - romansey -
- romansey - - 		- - norrs -
- norrs - - 		- - MightySCollins -
- MightySCollins - - 		- - 501st-alpha1 -
- 501st-alpha1 - -
- - klamann -
- klamann - - 		- - svdb0 -
- svdb0 - - 		- - 3ap -
- 3ap - - 		- - shyim -
- shyim - - 		- - sjmudd -
- sjmudd - - 		- - simonsystem -
- simonsystem - -
- - mchamplain -
- mchamplain - - 		- - millerjason -
- millerjason - - 		- - mplx -
- mplx - - 		- - odinis -
- odinis - - 		- - okamidash -
- okamidash - - 		- - olaf-mandel -
- olaf-mandel - -
- - ontheair81 -
- ontheair81 - - 		- - pravynandas -
- pravynandas - - 		- - presocratics -
- presocratics - - 		- - rhyst -
- rhyst - - 		- - rmlhuk -
- rmlhuk - - 		- - rriski -
- rriski - -
- - schnippl0r -
- schnippl0r - - 		- - smargold476 -
- smargold476 - - 		- - sportshead -
- sportshead - - 		- - squash -
- squash - - 		- - strarsis -
- strarsis - - 		- - tamueller -
- tamueller - -
- - vivacarvajalito -
- vivacarvajalito - - 		- - wligtenberg -
- wligtenberg - - 		- - wolkenschieber -
- wolkenschieber - - 		- - worldworm -
- worldworm - - 		- - ShiriNmi1520 -
- ShiriNmi1520 - - 		- - Zepmann -
- Zepmann - -
- - allddd -
- allddd - - 		- - arcaine2 -
- arcaine2 - - 		- - awb99 -
- awb99 - - 		- - brainkiller -
- brainkiller - - 		- - cternes -
- cternes - - 		- - dborowy -
- dborowy - -
- - dimalo -
- dimalo - - 		- - eleith -
- eleith - - 		- - fanqiaojun -
- fanqiaojun - - 		- - ghnp5 -
- ghnp5 - - 		- - helmutundarnold -
- helmutundarnold - - 		- - hnws -
- hnws - -
- - i-C-o-d-e-r -
- i-C-o-d-e-r - - 		- - idaadi -
- idaadi - - 		- - ixeft -
- ixeft - - 		- - jjtt -
- jjtt - - 		- - paralax -
- paralax - - 		- - jpduyx -
- jpduyx - -
- - landergate -
- landergate - - 		- - callmemagnus -
- callmemagnus - - 		- - marios88 -
- marios88 - - 		- - 0xflotus -
- 0xflotus - - 		- - arkanovicz -
- arkanovicz - - 		- - CBeerta -
- CBeerta - -
- - damianmoore -
- damianmoore - - 		- - espitall -
- espitall - - 		- - dkarski -
- dkarski - - 		- - dbellavista -
- dbellavista - - 		- - danielvandenberg95 -
- danielvandenberg95 - - 		- - denisix -
- denisix - -
- - mlatorre31 -
- mlatorre31 - - 		- - mazzz1y -
- mazzz1y - - 		- - doominator42 -
- doominator42 - - 		- - aydodo -
- aydodo - - 		- - vedtam -
- vedtam - - 		- - edvorg -
- edvorg - -
- - eliroca -
- eliroca - - 		- - ekkis -
- ekkis - - 		- - ErikEngerd -
- ErikEngerd - - 		- - huncode -
- huncode - - 		- - felixn -
- felixn - - 		- - flole -
- flole - -
- - froks -
- froks - - 		- - fkefer -
- fkefer - - 		- - ifokeev -
- ifokeev - - 		- - 20th -
- 20th - - 		- - 2b -
- 2b - - 		- - askz -
- askz - -
- - aspettl -
- aspettl - - 		- - acch -
- acch - - 		- - vifino -
- vifino - - 		- - kachkaev -
- kachkaev - - 		- - alexanderneu -
- alexanderneu - - 		- - ch3sh1r -
- ch3sh1r - -
- - eglia -
- eglia - - 		- - groupmsl -
- groupmsl - - 		- - green-anger -
- green-anger - - 		- - iRhonin -
- iRhonin - - 		- - MrFreezeex -
- MrFreezeex - - 		- - arunvc -
- arunvc - -
- - astrocket -
- astrocket - - 		- - baxerus -
- baxerus - - 		- - spock -
- spock - - 		- - erdos4d -
- erdos4d - - 		- - crash7 -
- crash7 - - 		- - auchri -
- auchri - -
- - Kaan88 -
- Kaan88 - - 		- - akkumar -
- akkumar - - 		- - thechubbypanda -
- thechubbypanda - - 		- - KCrawley -
- KCrawley - - 		- - khuedoan -
- khuedoan - - 		- - UltraCoderRU -
- UltraCoderRU - -
- - JustAnother1 -
- JustAnother1 - - 		- - leowinterde -
- leowinterde - - 		- - linhandev -
- linhandev - - 		- - luke- -
- luke- - - 		- - LucidityCrash -
- LucidityCrash - - 		- - MadsRC -
- MadsRC - -
- - madmath03 -
- madmath03 - - 		- - maxemann96 -
- maxemann96 - - 		- - dragetd -
- dragetd - - 		- - michaeljensen -
- michaeljensen - - 		- - exhuma -
- exhuma - - 		- - milas -
- milas - -
- - mcchots -
- mcchots - - 		- - MohammedNoureldin -
- MohammedNoureldin - - 		- - mpldr -
- mpldr - - 		- - naveensrinivasan -
- naveensrinivasan - - 		- - furstblumier -
- furstblumier - - 		- - Marsu31 -
- Marsu31 - -
- - glandais -
- glandais - - 		- - GiovanH -
- GiovanH - - 		- - harryyoud -
- harryyoud - - 		- - HeySora -
- HeySora - - 		- - sirgantrithon -
- sirgantrithon - - 		- - Influencer -
- Influencer - -
- - in-seo -
- in-seo - - 		- - firefly-cpp -
- firefly-cpp - - 		- - JacksonZ03 -
- JacksonZ03 - - 		- - JamBalaya56562 -
- JamBalaya56562 - - 		- - jcalfee -
- jcalfee - - 		- - mivek -
- mivek - -
- - init-js -
- init-js - - 		- - Jeidnx -
- Jeidnx - - 		- - jessp01 -
- jessp01 - - 		- - JiLleON -
- JiLleON - - 		- - jirislav -
- jirislav - - 		- - jmccl -
- jmccl - -
- - jurekbarth -
- jurekbarth - - 		- - JOduMonT -
- JOduMonT - -
+ + casperklein +
+ casperklein + + 		+ + fbartels +
+ fbartels + + 		+ + polarathene +
+ polarathene + + 		+ + NorseGaud +
+ NorseGaud + + 		+ + williamdes +
+ williamdes + + 		+ + wernerfred +
+ wernerfred + +
+ + georglauterbach +
+ georglauterbach + + 		+ + tomav +
+ tomav + + 		+ + erik-wramner +
+ erik-wramner + + 		+ + chikamichi +
+ chikamichi + + 		+ + martin-schulze-vireso +
+ martin-schulze-vireso + + 		+ + Josef-Friedrich +
+ Josef-Friedrich + +
+ + johansmitsnl +
+ johansmitsnl + + 		+ + youtous +
+ youtous + + 		+ + 17Halbe +
+ 17Halbe + + 		+ + tve +
+ tve + + 		+ + gmasse +
+ gmasse + + 		+ + ap-wtioit +
+ ap-wtioit + +
+ + 00angus +
+ 00angus + + 		+ + alinmear +
+ alinmear + + 		+ + dominikwinter +
+ dominikwinter + + 		+ + crazystick +
+ crazystick + + 		+ + svenyonson +
+ svenyonson + + 		+ + swiesend +
+ swiesend + +
+ + stonemaster +
+ stonemaster + + 		+ + omarc1492 +
+ omarc1492 + + 		+ + phish108 +
+ phish108 + + 		+ + mwlczk +
+ mwlczk + + 		+ + tyranron +
+ tyranron + + 		+ + KyleOndy +
+ KyleOndy + +
+ + mindrunner +
+ mindrunner + + 		+ + MichaelSp +
+ MichaelSp + + 		+ + m-a-v +
+ m-a-v + + 		+ + bilak +
+ bilak + + 		+ + vortex852456 +
+ vortex852456 + + 		+ + hanscees +
+ hanscees + +
+ + chris54721 +
+ chris54721 + + 		+ + jrpear +
+ jrpear + + 		+ + moqmar +
+ moqmar + + 		+ + pyy +
+ pyy + + 		+ + arneke +
+ arneke + + 		+ + akmet +
+ akmet + +
+ + diiigle +
+ diiigle + + 		+ + kiliant +
+ kiliant + + 		+ + dashohoxha +
+ dashohoxha + + 		+ + egavard +
+ egavard + + 		+ + mathuin +
+ mathuin + + 		+ + jamebus +
+ jamebus + +
+ + robertdolca +
+ robertdolca + + 		+ + okainov +
+ okainov + + 		+ + jsonn +
+ jsonn + + 		+ + lukecyca +
+ lukecyca + + 		+ + m-schmoock +
+ m-schmoock + + 		+ + mjung +
+ mjung + +
+ + eltociear +
+ eltociear + + 		+ + VanVan +
+ VanVan + + 		+ + voordev +
+ voordev + + 		+ + andreasgerstmayr +
+ andreasgerstmayr + + 		+ + davidszp +
+ davidszp + + 		+ + kamuri +
+ kamuri + +
+ + guardiande +
+ guardiande + + 		+ + Zehir +
+ Zehir + + 		+ + Birkenstab +
+ Birkenstab + + 		+ + BrandonSchmitt +
+ BrandonSchmitt + + 		+ + Starbix +
+ Starbix + + 		+ + citec +
+ citec + +
+ + yajo +
+ yajo + + 		+ + MakerMatrix +
+ MakerMatrix + + 		+ + weo +
+ weo + + 		+ + analogue +
+ analogue + + 		+ + Rubytastic2 +
+ Rubytastic2 + + 		+ + reneploetz +
+ reneploetz + +
+ + pbek +
+ pbek + + 		+ + yogo1212 +
+ yogo1212 + + 		+ + castorinop +
+ castorinop + + 		+ + p-fruck +
+ p-fruck + + 		+ + tbutter +
+ tbutter + + 		+ + rahilarious +
+ rahilarious + +
+ + Rillke +
+ Rillke + + 		+ + bobbravo2 +
+ bobbravo2 + + 		+ + r-pufky +
+ r-pufky + + 		+ + vincentDcmps +
+ vincentDcmps + + 		+ + frugan-dev +
+ frugan-dev + + 		+ + mpanneck +
+ mpanneck + +
+ + andymel123 +
+ andymel123 + + 		+ + bigpigeon +
+ bigpigeon + + 		+ + engelant +
+ engelant + + 		+ + j-marz +
+ j-marz + + 		+ + lokipo +
+ lokipo + + 		+ + msheakoski +
+ msheakoski + +
+ + willtho89 +
+ willtho89 + + 		+ + GoliathLabs +
+ GoliathLabs + + 		+ + ubenmackin +
+ ubenmackin + + 		+ + craue +
+ craue + + 		+ + abh +
+ abh + + 		+ + andrewlow +
+ andrewlow + +
+ + aminvakil +
+ aminvakil + + 		+ + elbracht +
+ elbracht + + 		+ + danielpanteleit +
+ danielpanteleit + + 		+ + dmcgrandle +
+ dmcgrandle + + 		+ + theomega +
+ theomega + + 		+ + DuncanvR +
+ DuncanvR + +
+ + emazzotta +
+ emazzotta + + 		+ + keslerm +
+ keslerm + + 		+ + nueaf +
+ nueaf + + 		+ + martinwepner +
+ martinwepner + + 		+ + artonge +
+ artonge + + 		+ + spacecowboy +
+ spacecowboy + +
+ + jedateach +
+ jedateach + + 		+ + H4R0 +
+ H4R0 + + 		+ + jamesfryer +
+ jamesfryer + + 		+ + millaguie +
+ millaguie + + 		+ + ipernet +
+ ipernet + + 		+ + fl42 +
+ fl42 + +
+ + neuralp +
+ neuralp + + 		+ + simonsystem +
+ simonsystem + + 		+ + stephan-devop +
+ stephan-devop + + 		+ + stigok +
+ stigok + + 		+ + 5ven +
+ 5ven + + 		+ + syl20bnr +
+ syl20bnr + +
+ + sylvaindumont +
+ sylvaindumont + + 		+ + TechnicLab +
+ TechnicLab + + 		+ + thomasschmit +
+ thomasschmit + + 		+ + 42wim +
+ 42wim + + 		+ + Thiritin +
+ Thiritin + + 		+ + forzagreen +
+ forzagreen + +
+ + tobiabocchi +
+ tobiabocchi + + 		+ + tweibert +
+ tweibert + + 		+ + torus +
+ torus + + 		+ + VictorKoenders +
+ VictorKoenders + + 		+ + Twist235 +
+ Twist235 + + 		+ + k3it +
+ k3it + +
+ + vilisas +
+ vilisas + + 		+ + Drakulix +
+ Drakulix + + 		+ + radicand +
+ radicand + + 		+ + nilshoell +
+ nilshoell + + 		+ + nknapp +
+ nknapp + + 		+ + pcqnt +
+ pcqnt + +
+ + OrvilleQ +
+ OrvilleQ + + 		+ + ovidiucp +
+ ovidiucp + + 		+ + mrPjer +
+ mrPjer + + 		+ + p3dda +
+ p3dda + + 		+ + 0xflotus +
+ 0xflotus + + 		+ + peter-hartmann +
+ peter-hartmann + +
+ + piwai +
+ piwai + + 		+ + remoe +
+ remoe + + 		+ + robbertkl +
+ robbertkl + + 		+ + romansey +
+ romansey + + 		+ + norrs +
+ norrs + + 		+ + MightySCollins +
+ MightySCollins + +
+ + 501st-alpha1 +
+ 501st-alpha1 + + 		+ + klamann +
+ klamann + + 		+ + svdb0 +
+ svdb0 + + 		+ + 3ap +
+ 3ap + + 		+ + shyim +
+ shyim + + 		+ + sjmudd +
+ sjmudd + +
+ + matrixes +
+ matrixes + + 		+ + mchamplain +
+ mchamplain + + 		+ + millerjason +
+ millerjason + + 		+ + mplx +
+ mplx + + 		+ + odinis +
+ odinis + + 		+ + okamidash +
+ okamidash + +
+ + olaf-mandel +
+ olaf-mandel + + 		+ + ontheair81 +
+ ontheair81 + + 		+ + pravynandas +
+ pravynandas + + 		+ + presocratics +
+ presocratics + + 		+ + rhyst +
+ rhyst + + 		+ + rmlhuk +
+ rmlhuk + +
+ + rriski +
+ rriski + + 		+ + schnippl0r +
+ schnippl0r + + 		+ + smargold476 +
+ smargold476 + + 		+ + sportshead +
+ sportshead + + 		+ + squash +
+ squash + + 		+ + strarsis +
+ strarsis + +
+ + tamueller +
+ tamueller + + 		+ + vivacarvajalito +
+ vivacarvajalito + + 		+ + wligtenberg +
+ wligtenberg + + 		+ + wolkenschieber +
+ wolkenschieber + + 		+ + worldworm +
+ worldworm + + 		+ + ShiriNmi1520 +
+ ShiriNmi1520 + +
+ + Zepmann +
+ Zepmann + + 		+ + allddd +
+ allddd + + 		+ + arcaine2 +
+ arcaine2 + + 		+ + awb99 +
+ awb99 + + 		+ + brainkiller +
+ brainkiller + + 		+ + cternes +
+ cternes + +
+ + dborowy +
+ dborowy + + 		+ + dimalo +
+ dimalo + + 		+ + eleith +
+ eleith + + 		+ + fanqiaojun +
+ fanqiaojun + + 		+ + ghnp5 +
+ ghnp5 + + 		+ + helmutundarnold +
+ helmutundarnold + +
+ + hnws +
+ hnws + + 		+ + i-C-o-d-e-r +
+ i-C-o-d-e-r + + 		+ + idaadi +
+ idaadi + + 		+ + ixeft +
+ ixeft + + 		+ + jjtt +
+ jjtt + + 		+ + paralax +
+ paralax + +
+ + jpduyx +
+ jpduyx + + 		+ + landergate +
+ landergate + + 		+ + callmemagnus +
+ callmemagnus + + 		+ + marios88 +
+ marios88 + + 		+ + arkanovicz +
+ arkanovicz + + 		+ + CBeerta +
+ CBeerta + +
+ + damianmoore +
+ damianmoore + + 		+ + espitall +
+ espitall + + 		+ + dkarski +
+ dkarski + + 		+ + dbellavista +
+ dbellavista + + 		+ + danielvandenberg95 +
+ danielvandenberg95 + + 		+ + denisix +
+ denisix + +
+ + mlatorre31 +
+ mlatorre31 + + 		+ + mazzz1y +
+ mazzz1y + + 		+ + doominator42 +
+ doominator42 + + 		+ + aydodo +
+ aydodo + + 		+ + vedtam +
+ vedtam + + 		+ + edvorg +
+ edvorg + +
+ + eliroca +
+ eliroca + + 		+ + ekkis +
+ ekkis + + 		+ + ErikEngerd +
+ ErikEngerd + + 		+ + huncode +
+ huncode + + 		+ + gitfeber +
+ gitfeber + + 		+ + felixn +
+ felixn + +
+ + flole +
+ flole + + 		+ + froks +
+ froks + + 		+ + ifokeev +
+ ifokeev + + 		+ + 20th +
+ 20th + + 		+ + 2b +
+ 2b + + 		+ + askz +
+ askz + +
+ + aspettl +
+ aspettl + + 		+ + acch +
+ acch + + 		+ + vifino +
+ vifino + + 		+ + kachkaev +
+ kachkaev + + 		+ + alexanderneu +
+ alexanderneu + + 		+ + ch3sh1r +
+ ch3sh1r + +
+ + eglia +
+ eglia + + 		+ + groupmsl +
+ groupmsl + + 		+ + green-anger +
+ green-anger + + 		+ + iRhonin +
+ iRhonin + + 		+ + MrFreezeex +
+ MrFreezeex + + 		+ + arunvc +
+ arunvc + +
+ + astrocket +
+ astrocket + + 		+ + baxerus +
+ baxerus + + 		+ + spock +
+ spock + + 		+ + erdos4d +
+ erdos4d + + 		+ + crash7 +
+ crash7 + + 		+ + auchri +
+ auchri + +
+ + fkefer +
+ fkefer + + 		+ + Kaan88 +
+ Kaan88 + + 		+ + akkumar +
+ akkumar + + 		+ + thechubbypanda +
+ thechubbypanda + + 		+ + KCrawley +
+ KCrawley + + 		+ + khuedoan +
+ khuedoan + +
+ + UltraCoderRU +
+ UltraCoderRU + + 		+ + JustAnother1 +
+ JustAnother1 + + 		+ + leowinterde +
+ leowinterde + + 		+ + linhandev +
+ linhandev + + 		+ + luke- +
+ luke- + + 		+ + LucidityCrash +
+ LucidityCrash + +
+ + MadsRC +
+ MadsRC + + 		+ + madmath03 +
+ madmath03 + + 		+ + maxemann96 +
+ maxemann96 + + 		+ + dragetd +
+ dragetd + + 		+ + michaeljensen +
+ michaeljensen + + 		+ + exhuma +
+ exhuma + +
+ + milas +
+ milas + + 		+ + mcchots +
+ mcchots + + 		+ + MohammedNoureldin +
+ MohammedNoureldin + + 		+ + mpldr +
+ mpldr + + 		+ + naveensrinivasan +
+ naveensrinivasan + + 		+ + furstblumier +
+ furstblumier + +
+ + Marsu31 +
+ Marsu31 + + 		+ + glandais +
+ glandais + + 		+ + GiovanH +
+ GiovanH + + 		+ + harryyoud +
+ harryyoud + + 		+ + HeySora +
+ HeySora + + 		+ + sirgantrithon +
+ sirgantrithon + +
+ + Influencer +
+ Influencer + + 		+ + in-seo +
+ in-seo + + 		+ + firefly-cpp +
+ firefly-cpp + + 		+ + JacksonZ03 +
+ JacksonZ03 + + 		+ + JamBalaya56562 +
+ JamBalaya56562 + + 		+ + jcalfee +
+ jcalfee + +
+ + mivek +
+ mivek + + 		+ + init-js +
+ init-js + + 		+ + Jeidnx +
+ Jeidnx + + 		+ + jessp01 +
+ jessp01 + + 		+ + JiLleON +
+ JiLleON + + 		+ + jirislav +
+ jirislav + +
+ + jmccl +
+ jmccl + + 		+ + jurekbarth +
+ jurekbarth + + 		+ + JOduMonT +
+ JOduMonT + +
From 006f442cd0bdedb386acf328146b630887a88f84 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Fri, 17 May 2024 09:57:07 +0200 Subject: [PATCH 359/592] Update `.ecrc.json` to exclude `CONTRIBUTORS.md` (#4020) --- test/linting/.ecrc.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/linting/.ecrc.json b/test/linting/.ecrc.json index 374615fec3a..44d501ba13d 100644 --- a/test/linting/.ecrc.json +++ b/test/linting/.ecrc.json @@ -3,6 +3,7 @@ "Exclude": [ "^test/bats/", "^test/test_helper/bats-(assert|support)", - "\\.git/" + "\\.git/", + "CONTRIBUTORS\\.md" ] } From a780fb331169dbd6a2d521b5d81900a4ef8636e7 Mon Sep 17 00:00:00 2001 From: pyy <4225935+pyy@users.noreply.github.com> Date: Fri, 17 May 2024 11:38:02 +0200 Subject: [PATCH 360/592] docs: Add tip for disabling the default SPF service (#4019) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/config/best-practices/dkim_dmarc_spf.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md index 52d01deac54..d035556f123 100644 --- a/docs/content/config/best-practices/dkim_dmarc_spf.md +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -302,9 +302,9 @@ The DMARC status may not be displayed instantly due to delays in DNS (caches). D [Source][wikipedia-spf] -!!! note "Disabling `policyd-spf`?" +!!! tip "Disabling the default SPF service `policy-spf`" - As of now, `policyd-spf` cannot be disabled. This is WIP. + Set [`ENABLE_POLICYD_SPF=0`][docs-env-spf-policyd] to opt-out of the default SPF service. Advised when Rspamd is configured to handle SPF instead. ### Adding an SPF Record @@ -348,6 +348,7 @@ volumes: [docs-volumes-config]: ../advanced/optional-config.md#volumes-config [docs-env-opendkim]: ../environment.md#enable_opendkim [docs-env-rspamd]: ../environment.md#enable_rspamd +[docs-env-spf-policyd]: ../environment.md#enable_policyd_spf [docs-rspamd-config-dropin]: ../security/rspamd.md#manually [cloudflare-dkim-dmarc-spf]: https://www.cloudflare.com/learning/email-security/dmarc-dkim-spf/ [rfc-8301]: https://datatracker.ietf.org/doc/html/rfc8301#section-3.2 From ed669bd3148d1565635a02cebcf4cf5f0cf9b780 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sun, 19 May 2024 22:32:53 +1200 Subject: [PATCH 361/592] fix: `/var/mail-state` should not symlink non-existing directories (#4018) Fixes an issue with the Getmail service, view PR thread for additional details. - Log an error when the expected service state directory doesn't exist. - The location `/var/lib/getmail/` doesn't seem like it should have been introduced. Drop it in favor of `/tmp/docker-mailserver/getmail`. It appears to be for storing remote mail that was retrieved if not configured to send to Dovecot like our docs advise. This location was never valid anyway (_as referenced issue covers_). --- CHANGELOG.md | 1 + target/bin/debug-fetchmail | 6 ++++++ target/bin/debug-getmail | 8 +------- target/bin/getmail-cron | 3 ++- target/scripts/startup/setup.d/getmail.sh | 5 +++++ target/scripts/startup/setup.d/mail_state.sh | 5 +++-- 6 files changed, 18 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ead51f5377..5564d8aa335 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,6 +109,7 @@ The most noteworthy change of this release is the update of the container's base - Rspamd configuration: Add a missing comma in `local_networks` so that all internal IP addresses are actually considered as internal ([#3862](https://github.com/docker-mailserver/docker-mailserver/pull/3862)) - Ensure correct SELinux security context labels for files and directories moved to the mail-state volume during setup ([#3890](https://github.com/docker-mailserver/docker-mailserver/pull/3890)) - Use correct environment variable for fetchmail ([#3901](https://github.com/docker-mailserver/docker-mailserver/pull/3901)) +- When using `ENABLE_GETMAIL=1` the undocumented internal location `/var/lib/getmail/` usage has been removed. Only the config volume `/tmp/docker-mailserver/getmail/` location is supported when Getmail has not been configured to deliver mail to Dovecot as advised in the DMS docs ([#4018](https://github.com/docker-mailserver/docker-mailserver/pull/4018)) - Dovecot dummy accounts (_virtual alias workaround for dovecot feature `ENABLE_QUOTAS=1`_) now correctly matches the home location of the user for that alias ([#3997](https://github.com/docker-mailserver/docker-mailserver/pull/3997)) ## [v13.3.1](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v13.3.1) diff --git a/target/bin/debug-fetchmail b/target/bin/debug-fetchmail index 7e267184321..87d30e2be10 100755 --- a/target/bin/debug-fetchmail +++ b/target/bin/debug-fetchmail @@ -5,6 +5,12 @@ source /usr/local/bin/helpers/log.sh # shellcheck source=../scripts/startup/setup.d/fetchmail.sh source /usr/local/bin/setup.d/fetchmail.sh +# TODO: This should probably not implicitly enable the feature. +# The setup method will feature gate and output a debug log if +# the feature is not enabled. +# +# Dropping the ENV here will require updating legacy test: +# test/tests/parallel/set3/scripts/setup_cli.bats ENABLE_FETCHMAIL=1 _setup_fetchmail su -s /bin/sh -c "/usr/bin/fetchmail \ diff --git a/target/bin/debug-getmail b/target/bin/debug-getmail index 27834475a5a..60270215469 100644 --- a/target/bin/debug-getmail +++ b/target/bin/debug-getmail @@ -7,13 +7,7 @@ source /usr/local/bin/setup.d/getmail.sh _setup_getmail -if [[ -d /var/lib/getmail ]]; then - GETMAILDIR=/var/lib/getmail -else - mkdir -p /tmp/docker-mailserver/getmail - GETMAILDIR=/tmp/docker-mailserver/getmail -fi - +GETMAILDIR=/tmp/docker-mailserver/getmail for FILE in /etc/getmailrc.d/getmailrc*; do getmail --getmaildir "${GETMAILDIR}" --rcfile "${FILE}" --dump | tail -n +6 done diff --git a/target/bin/getmail-cron b/target/bin/getmail-cron index 8e1e474927f..fec03906841 100644 --- a/target/bin/getmail-cron +++ b/target/bin/getmail-cron @@ -1,7 +1,8 @@ #! /bin/bash +GETMAILDIR=/tmp/docker-mailserver/getmail for FILE in /etc/getmailrc.d/getmailrc*; do if ! pgrep -f "${FILE}$" &>/dev/null; then - getmail --getmaildir /var/lib/getmail --rcfile "${FILE}" + getmail --getmaildir "${GETMAILDIR}" --rcfile "${FILE}" fi done diff --git a/target/scripts/startup/setup.d/getmail.sh b/target/scripts/startup/setup.d/getmail.sh index 6f2801184d9..c1345063de9 100644 --- a/target/scripts/startup/setup.d/getmail.sh +++ b/target/scripts/startup/setup.d/getmail.sh @@ -31,6 +31,11 @@ function _setup_getmail() { EOF chmod -R 600 "${GETMAILRC}" fi + + # Both the debug command and cron job (that runs getmail) for getmail + # expect this location to exist. + GETMAILDIR=/tmp/docker-mailserver/getmail + mkdir -p "${GETMAILDIR}" else _log 'debug' 'Getmail is disabled' fi diff --git a/target/scripts/startup/setup.d/mail_state.sh b/target/scripts/startup/setup.d/mail_state.sh index 5acf6762d69..de15ee3ba55 100644 --- a/target/scripts/startup/setup.d/mail_state.sh +++ b/target/scripts/startup/setup.d/mail_state.sh @@ -23,7 +23,6 @@ function _setup_save_states() { [[ ${ENABLE_CLAMAV} -eq 1 ]] && SERVICEDIRS+=('lib/clamav') [[ ${ENABLE_FAIL2BAN} -eq 1 ]] && SERVICEDIRS+=('lib/fail2ban') [[ ${ENABLE_FETCHMAIL} -eq 1 ]] && SERVICEDIRS+=('lib/fetchmail') - [[ ${ENABLE_GETMAIL} -eq 1 ]] && SERVICEDIRS+=('lib/getmail') [[ ${ENABLE_MTA_STS} -eq 1 ]] && SERVICEDIRS+=('lib/mta-sts') [[ ${ENABLE_POSTGREY} -eq 1 ]] && SERVICEDIRS+=('lib/postgrey') [[ ${ENABLE_RSPAMD} -eq 1 ]] && SERVICEDIRS+=('lib/rspamd') @@ -70,11 +69,13 @@ function _setup_save_states() { rm -rf "${SERVICEDIR}" elif [[ -d ${SERVICEDIR} ]]; then _log 'trace' "Moving contents of ${SERVICEDIR} to ${DEST}" - # Empty volume was mounted, or new content from enabling a feature ENV: + # An empty volume was mounted, or new content dir now exists from enabling a feature ENV: mv "${SERVICEDIR}" "${DEST}" # Apply SELinux security context to match the state directory, so access # is not restricted to the current running container: chcon -R --reference="${STATEDIR}" "${DEST}" 2>/dev/null || true + else + _log 'error' "${SERVICEDIR} should exist but is missing" fi # Symlink the original path in the container ($SERVICEDIR) to be From 03c905e6f11fa8c14416acd1d73fa6d75020e820 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 19 May 2024 14:06:59 +0200 Subject: [PATCH 362/592] docs: updated `CONTRIBUTORS.md` (#4021) --- CONTRIBUTORS.md | 421 ++++++++++++++++++++++++------------------------ 1 file changed, 214 insertions(+), 207 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 4275d39d72f..1600440c95c 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -285,17 +285,17 @@ Thanks goes to these wonderful people ✨ - - moqmar + + pyy
- moqmar + pyy - - pyy + + moqmar
- pyy + moqmar @@ -329,17 +329,17 @@ Thanks goes to these wonderful people ✨ - - dashohoxha + + egavard
- dashohoxha + egavard - - egavard + + dashohoxha
- egavard + dashohoxha @@ -358,20 +358,6 @@ Thanks goes to these wonderful people ✨ - - - robertdolca -
- robertdolca - - - - - okainov -
- okainov - - jsonn @@ -387,40 +373,40 @@ Thanks goes to these wonderful people ✨ - - m-schmoock + + okainov
- m-schmoock + okainov - - mjung + + robertdolca
- mjung + robertdolca - - - - eltociear + + VanVan
- eltociear + VanVan - - VanVan + + eltociear
- VanVan + eltociear + + - - voordev + + mjung
- voordev + mjung @@ -430,6 +416,13 @@ Thanks goes to these wonderful people ✨ andreasgerstmayr + + + m-schmoock +
+ m-schmoock + + davidszp @@ -444,8 +437,6 @@ Thanks goes to these wonderful people ✨ kamuri - - guardiande @@ -453,6 +444,8 @@ Thanks goes to these wonderful people ✨ guardiande + + Zehir @@ -460,6 +453,13 @@ Thanks goes to these wonderful people ✨ Zehir + + + voordev +
+ voordev + + Birkenstab @@ -505,17 +505,17 @@ Thanks goes to these wonderful people ✨ - - weo + + pbek
- weo + pbek - - analogue + + reneploetz
- analogue + reneploetz @@ -526,26 +526,26 @@ Thanks goes to these wonderful people ✨ - - reneploetz + + analogue
- reneploetz + analogue - - pbek + + weo
- pbek + weo - - yogo1212 + + ubenmackin
- yogo1212 + ubenmackin @@ -562,13 +562,6 @@ Thanks goes to these wonderful people ✨ p-fruck - - - tbutter -
- tbutter - - rahilarious @@ -576,8 +569,6 @@ Thanks goes to these wonderful people ✨ rahilarious - - Rillke @@ -585,6 +576,8 @@ Thanks goes to these wonderful people ✨ Rillke + + bobbravo2 @@ -606,22 +599,6 @@ Thanks goes to these wonderful people ✨ vincentDcmps - - - frugan-dev -
- frugan-dev - - - - - mpanneck -
- mpanneck - - - - andymel123 @@ -643,6 +620,8 @@ Thanks goes to these wonderful people ✨ engelant + + j-marz @@ -663,28 +642,49 @@ Thanks goes to these wonderful people ✨
msheakoski + + + + GoliathLabs +
+ GoliathLabs + + + + + frugan-dev +
+ frugan-dev + + + + + tbutter +
+ tbutter + - - willtho89 + + yogo1212
- willtho89 + yogo1212 - - GoliathLabs + + willtho89
- GoliathLabs + willtho89 - - ubenmackin + + mpanneck
- ubenmackin + mpanneck @@ -806,54 +806,47 @@ Thanks goes to these wonderful people ✨ - - H4R0 + + fl42
- H4R0 + fl42 - - jamesfryer + + ipernet
- jamesfryer + ipernet - - millaguie + + H4R0
- millaguie + H4R0 - - ipernet + + millaguie
- ipernet + millaguie - - fl42 + + jamesfryer
- fl42 + jamesfryer - - neuralp -
- neuralp - - - - - simonsystem + + radicand
- simonsystem + radicand @@ -884,8 +877,6 @@ Thanks goes to these wonderful people ✨ syl20bnr - - sylvaindumont @@ -893,6 +884,8 @@ Thanks goes to these wonderful people ✨ sylvaindumont + + TechnicLab @@ -907,13 +900,6 @@ Thanks goes to these wonderful people ✨ thomasschmit - - - 42wim -
- 42wim - - Thiritin @@ -921,15 +907,6 @@ Thanks goes to these wonderful people ✨ Thiritin - - - forzagreen -
- forzagreen - - - - tobiabocchi @@ -951,6 +928,8 @@ Thanks goes to these wonderful people ✨ torus + + VictorKoenders @@ -972,8 +951,13 @@ Thanks goes to these wonderful people ✨ k3it - - + + + Drakulix +
+ Drakulix + + vilisas @@ -982,17 +966,26 @@ Thanks goes to these wonderful people ✨ - - Drakulix + + forzagreen
- Drakulix + forzagreen + + - - radicand + + 42wim
- radicand + 42wim + + + + + ShiriNmi1520 +
+ ShiriNmi1520 @@ -1016,8 +1009,6 @@ Thanks goes to these wonderful people ✨ pcqnt - - OrvilleQ @@ -1025,6 +1016,8 @@ Thanks goes to these wonderful people ✨ OrvilleQ + + ovidiucp @@ -1047,21 +1040,19 @@ Thanks goes to these wonderful people ✨ - - 0xflotus + + peter-hartmann
- 0xflotus + peter-hartmann - - peter-hartmann + + neuralp
- peter-hartmann + neuralp - - piwai @@ -1069,6 +1060,8 @@ Thanks goes to these wonderful people ✨ piwai + + remoe @@ -1104,8 +1097,6 @@ Thanks goes to these wonderful people ✨ MightySCollins - - 501st-alpha1 @@ -1113,6 +1104,8 @@ Thanks goes to these wonderful people ✨ 501st-alpha1 + + klamann @@ -1148,15 +1141,15 @@ Thanks goes to these wonderful people ✨ sjmudd - - - - matrixes + + simonsystem
- matrixes + simonsystem + + mchamplain @@ -1171,6 +1164,13 @@ Thanks goes to these wonderful people ✨ millerjason + + + mmehnert +
+ mmehnert + + mplx @@ -1317,15 +1317,6 @@ Thanks goes to these wonderful people ✨ worldworm - - - ShiriNmi1520 -
- ShiriNmi1520 - - - - Zepmann @@ -1333,6 +1324,8 @@ Thanks goes to these wonderful people ✨ Zepmann + + allddd @@ -1368,8 +1361,6 @@ Thanks goes to these wonderful people ✨ cternes - - dborowy @@ -1377,6 +1368,8 @@ Thanks goes to these wonderful people ✨ dborowy + + dimalo @@ -1412,8 +1405,6 @@ Thanks goes to these wonderful people ✨ helmutundarnold - - hnws @@ -1421,6 +1412,8 @@ Thanks goes to these wonderful people ✨ hnws + + i-C-o-d-e-r @@ -1456,8 +1449,6 @@ Thanks goes to these wonderful people ✨ paralax - - jpduyx @@ -1465,6 +1456,8 @@ Thanks goes to these wonderful people ✨ jpduyx + + landergate @@ -1486,6 +1479,20 @@ Thanks goes to these wonderful people ✨ marios88 + + + matrixes +
+ matrixes + + + + + 0xflotus +
+ 0xflotus + + arkanovicz @@ -1493,6 +1500,8 @@ Thanks goes to these wonderful people ✨ arkanovicz + + CBeerta @@ -1500,8 +1509,6 @@ Thanks goes to these wonderful people ✨ CBeerta - - damianmoore @@ -1537,6 +1544,8 @@ Thanks goes to these wonderful people ✨ danielvandenberg95 + + denisix @@ -1544,8 +1553,6 @@ Thanks goes to these wonderful people ✨ denisix - - mlatorre31 @@ -1581,6 +1588,8 @@ Thanks goes to these wonderful people ✨ vedtam + + edvorg @@ -1588,8 +1597,6 @@ Thanks goes to these wonderful people ✨ edvorg - - eliroca @@ -1625,6 +1632,8 @@ Thanks goes to these wonderful people ✨ gitfeber + + felixn @@ -1632,8 +1641,6 @@ Thanks goes to these wonderful people ✨ felixn - - flole @@ -1669,6 +1676,8 @@ Thanks goes to these wonderful people ✨ 2b + + askz @@ -1676,8 +1685,6 @@ Thanks goes to these wonderful people ✨ askz - - aspettl @@ -1713,6 +1720,8 @@ Thanks goes to these wonderful people ✨ alexanderneu + + ch3sh1r @@ -1720,8 +1729,6 @@ Thanks goes to these wonderful people ✨ ch3sh1r - - eglia @@ -1757,6 +1764,8 @@ Thanks goes to these wonderful people ✨ MrFreezeex + + arunvc @@ -1764,8 +1773,6 @@ Thanks goes to these wonderful people ✨ arunvc - - astrocket @@ -1801,6 +1808,8 @@ Thanks goes to these wonderful people ✨ crash7 + + auchri @@ -1808,8 +1817,6 @@ Thanks goes to these wonderful people ✨ auchri - - fkefer @@ -1845,6 +1852,8 @@ Thanks goes to these wonderful people ✨ KCrawley + + khuedoan @@ -1852,8 +1861,6 @@ Thanks goes to these wonderful people ✨ khuedoan - - UltraCoderRU @@ -1889,6 +1896,8 @@ Thanks goes to these wonderful people ✨ luke- + + LucidityCrash @@ -1896,8 +1905,6 @@ Thanks goes to these wonderful people ✨ LucidityCrash - - MadsRC @@ -1933,6 +1940,8 @@ Thanks goes to these wonderful people ✨ michaeljensen + + exhuma @@ -1940,8 +1949,6 @@ Thanks goes to these wonderful people ✨ exhuma - - milas @@ -1977,6 +1984,8 @@ Thanks goes to these wonderful people ✨ naveensrinivasan + + furstblumier @@ -1984,8 +1993,6 @@ Thanks goes to these wonderful people ✨ furstblumier - - Marsu31 @@ -2021,6 +2028,8 @@ Thanks goes to these wonderful people ✨ HeySora + + sirgantrithon @@ -2028,8 +2037,6 @@ Thanks goes to these wonderful people ✨ sirgantrithon - - Influencer @@ -2065,6 +2072,8 @@ Thanks goes to these wonderful people ✨ JamBalaya56562 + + jcalfee @@ -2072,8 +2081,6 @@ Thanks goes to these wonderful people ✨ jcalfee - - mivek @@ -2109,6 +2116,8 @@ Thanks goes to these wonderful people ✨ JiLleON + + jirislav @@ -2116,8 +2125,6 @@ Thanks goes to these wonderful people ✨ jirislav - - jmccl From 92b06c40204d97d03a1d3172bb1964a069690759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C5=A0tefka?= Date: Mon, 20 May 2024 15:07:42 +0200 Subject: [PATCH 363/592] docs(rspamd): Add guidance for setting WebUI password (#4023) * fix(docs/rspamd): Add section on how to setup WebUI password * Apply review suggestion * Apply suggestions from code review * Update rspamd.md --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/config/security/rspamd.md | 30 +++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index fd0fe25e8b8..b0f65990371 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -84,14 +84,30 @@ When Rspamd is enabled, we implicitly also start an instance of Redis in the con Redis uses `/etc/redis/redis.conf` for configuration: - We adjust this file when enabling the internal Redis service. -- If you have an external instance of Redis to use, the internal Redis service can be opt-out via setting the ENV [`ENABLE_RSPAMD_REDIS=0`](../environment.md#enable_rspamd_redis) (_link also details required changes to the DMS Rspamd config_). +- If you have an external instance of Redis to use, the internal Redis service can be opt-out via setting the ENV [`ENABLE_RSPAMD_REDIS=0`][docs::env::enable-redis] (_link also details required changes to the DMS Rspamd config_). ### Web Interface -Rspamd provides a [web interface][rspamd-docs::web-interface], which contains statistics and data Rspamd collects. The interface is enabled by default and reachable on port 11334. +Rspamd provides a [web interface][rspamd-docs::web-ui], which contains statistics and data Rspamd collects. The interface is enabled by default and reachable on port 11334. ![Rspamd Web Interface](https://rspamd.com/img/webui.png) +To use the web interface you will need to configure a password, [otherwise you won't be able to log in][rspamd-docs::web-ui::password]. + +??? example "Set a custom password" + + Add this line to [your rspamd `custom-commands.conf` config](#with-the-help-of-a-custom-file) which sets the `password` option of the _controller worker_: + + ``` + set-option-for-controller password "your hashed password here" + ``` + + The password hash can be generated via the `rspamadm pw` command: + + ```bash + docker exec -it rspamadm pw + ``` + ### DNS DMS does not supply custom values for DNS servers (to Rspamd). If you need to use custom DNS servers, which could be required when using [DNS-based deny/allowlists](#rbls-real-time-blacklists-dnsbls-dns-based-blacklists), you need to adjust [`options.inc`][rspamd-docs::basic-options] yourself. Make sure to also read our [FAQ page on DNS servers][docs::faq::dns-servers]. @@ -126,13 +142,13 @@ You can choose to enable ClamAV, and Rspamd will then use it to check for viruse #### RBLs (Real-time Blacklists) / DNSBLs (DNS-based Blacklists) -The [RBL module](https://rspamd.com/doc/modules/rbl.html) is enabled by default. As a consequence, Rspamd will perform DNS lookups to various blacklists. Whether an RBL or a DNSBL is queried depends on where the domain name was obtained: RBL servers are queried with IP addresses extracted from message headers, DNSBL server are queried with domains and IP addresses extracted from the message body \[[source][www::rbl-vs-dnsbl]\]. +The [RBL module][rspamd-docs::modules::rbl] is enabled by default. As a consequence, Rspamd will perform DNS lookups to various blacklists. Whether an RBL or a DNSBL is queried depends on where the domain name was obtained: RBL servers are queried with IP addresses extracted from message headers, DNSBL server are queried with domains and IP addresses extracted from the message body \[[source][www::rbl-vs-dnsbl]\]. !!! danger "Rspamd and DNS Block Lists" When the RBL module is enabled, Rspamd will do a variety of DNS requests to (amongst other things) DNSBLs. There are a variety of issues involved when using DNSBLs. Rspamd will try to mitigate some of them by properly evaluating all return codes. This evaluation is a best effort though, so if the DNSBL operators change or add return codes, it may take a while for Rspamd to adjust as well. - If you want to use DNSBLs, **try to use your own DNS resolver** and make sure it is set up correctly, i.e. it should be a non-public & **recursive** resolver. Otherwise, you might not be able ([see this Spamhaus post](https://www.spamhaus.org/faq/section/DNSBL%20Usage#365)) to make use of the block lists. + If you want to use DNSBLs, **try to use your own DNS resolver** and make sure it is set up correctly, i.e. it should be a non-public & **recursive** resolver. Otherwise, you might not be able ([see this Spamhaus post][spamhaus::faq::dnsbl-usage]) to make use of the block lists. ## Providing Custom Settings & Overriding Settings @@ -247,8 +263,10 @@ While _Abusix_ can be integrated into Postfix, Postscreen and a multitude of oth [rspamd-web]: https://rspamd.com/ [rspamd-docs::bayes]: https://rspamd.com/doc/configuration/statistic.html [rspamd-docs::proxy-self-scan-mode]: https://rspamd.com/doc/workers/rspamd_proxy.html#self-scan-mode -[rspamd-docs::web-interface]: https://rspamd.com/webui/ +[rspamd-docs::web-ui]: https://rspamd.com/webui/ +[rspamd-docs::web-ui::password]: https://www.rspamd.com/doc/tutorials/quickstart.html#setting-the-controller-password [rspamd-docs::modules]: https://rspamd.com/doc/modules/ +[rspamd-docs::modules::rbl]: https://rspamd.com/doc/modules/rbl.html [rspamd-docs::override-dir]: https://www.rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories [rspamd-docs::config-directories]: https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories [rspamd-docs::basic-options]: https://rspamd.com/doc/configuration/options.html @@ -256,10 +274,12 @@ While _Abusix_ can be integrated into Postfix, Postscreen and a multitude of oth [www::rbl-vs-dnsbl]: https://forum.eset.com/topic/25277-dnsbl-vs-rbl-mail-security/?do=findComment&comment=119818 [abusix-web]: https://abusix.com/ [abusix-docs::rspamd-integration]: https://docs.abusix.com/abusix-mail-intelligence/gbG8EcJ3x3fSUv8cMZLiwA/getting-started/dmw9dcwSGSNQiLTssFAnBW#rspamd +[spamhaus::faq::dnsbl-usage]: https://www.spamhaus.org/faq/section/DNSBL%20Usage#365 [dms-repo::rspamd-actions-config]: https://github.com/docker-mailserver/docker-mailserver/blob/v14.0.0/target/rspamd/local.d/actions.conf [dms-repo::default-rspamd-configuration]: https://github.com/docker-mailserver/docker-mailserver/tree/v14.0.0/target/rspamd +[docs::env::enable-redis]: ../environment.md#enable_rspamd_redis [docs::spam-to-junk]: ../environment.md#move_spam_to_junk [docs::dkim-dmarc-spf]: ../best-practices/dkim_dmarc_spf.md [docs::dkim-with-rspamd]: ../best-practices/dkim_dmarc_spf.md#dkim From 993c7b044f126aa941ea30d4ac86dd38c5dc5fe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C5=A0tefka?= Date: Tue, 21 May 2024 16:49:39 +0200 Subject: [PATCH 364/592] breaking: Drop Dovecot support for Solr (#4025) --- CHANGELOG.md | 3 ++ .../config/advanced/full-text-search.md | 49 +------------------ target/scripts/build/packages.sh | 2 +- 3 files changed, 5 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5564d8aa335..04f8adb9194 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,9 @@ The most noteworthy change of this release is the update of the container's base - The "Junk" mailbox (folder) is now referenced by it's [special-use flag `\Junk`](https://docker-mailserver.github.io/docker-mailserver/v13.3/examples/use-cases/imap-folders/) instead of an explicit mailbox. ([#3925](https://github.com/docker-mailserver/docker-mailserver/pull/3925)) - This provides compatibility for the Junk mailbox when it's folder name differs (_eg: Renamed to "Spam"_). - Potential breakage if your deployment modifies our `spam_to_junk.sieve` sieve script (_which is created during container startup when ENV `MOVE_SPAM_TO_JUNK=1`_) that handles storing spam mail into a users "Junk" mailbox folder. + - **Removed support for Solr integration:** ([#4025](https://github.com/docker-mailserver/docker-mailserver/pull/4025)) + - This was a community contributed feature for FTS (Full Text Search), the docs advise using an image that has not been maintained for over 2 years and lacks ARM64 support. Based on user engagement over the years this feature has very niche value to continue to support, thus is being removed. + - If you use Solr, support can be restored if you're willing to contribute docs for the feature that resolves the concerns raised - **rsyslog:** - Debian 12 adjusted the `rsyslog` configuration for the default file template from `RSYSLOG_TraditionalFileFormat` to `RSYSLOG_FileFormat` (_upstream default since 2012_). This change may affect you if you have any monitoring / analysis of log output (_eg: `mail.log` / `docker logs`_). - The two formats are roughly equivalent to [RFC 3164](https://www.rfc-editor.org/rfc/rfc3164)) and [RFC 5424](https://datatracker.ietf.org/doc/html/rfc5424#section-1) respectively. diff --git a/docs/content/config/advanced/full-text-search.md b/docs/content/config/advanced/full-text-search.md index 2e29e49690f..45e7ffc7f94 100644 --- a/docs/content/config/advanced/full-text-search.md +++ b/docs/content/config/advanced/full-text-search.md @@ -16,7 +16,7 @@ The [dovecot-fts-xapian](https://github.com/grosjo/fts-xapian) plugin makes use The indexes will be stored as a subfolder named `xapian-indexes` inside your local `mail-data` folder (_`/var/mail` internally_). With the default settings, 10GB of email data may generate around 4GB of indexed data. -While indexing is memory intensive, you can configure the plugin to limit the amount of memory consumed by the index workers. With Xapian being small and fast, this plugin is a good choice for low memory environments (2GB) as compared to Solr. +While indexing is memory intensive, you can configure the plugin to limit the amount of memory consumed by the index workers. With Xapian being small and fast, this plugin is a good choice for low memory environments (2GB). #### Setup @@ -137,53 +137,6 @@ While indexing is memory intensive, you can configure the plugin to limit the am - ./docker-data/dms/cron/fts_xapian:/etc/cron.d/fts_xapian ``` - -### Solr - -The [dovecot-solr Plugin](https://wiki2.dovecot.org/Plugins/FTS/Solr) is used in conjunction with [Apache Solr](https://lucene.apache.org/solr/) running in a separate container. This is quite straightforward to setup using the following instructions. - -Solr is a mature and fast indexing backend that runs on the JVM. The indexes are relatively compact compared to the size of your total email. - -However, Solr also requires a fair bit of RAM. While Solr is [highly tuneable](https://solr.apache.org/guide/7_0/query-settings-in-solrconfig.html), it may require a bit of testing to get it right. - -#### Setup - -1. `compose.yaml`: - - ```yaml - solr: - image: lmmdock/dovecot-solr:latest - volumes: - - ./docker-data/dms/config/dovecot/solr-dovecot:/opt/solr/server/solr/dovecot - restart: always - - mailserver: - depends_on: - - solr - image: ghcr.io/docker-mailserver/docker-mailserver:latest - ... - volumes: - ... - - ./docker-data/dms/config/dovecot/10-plugin.conf:/etc/dovecot/conf.d/10-plugin.conf:ro - ... - ``` - -2. `./docker-data/dms/config/dovecot/10-plugin.conf`: - - ```conf - mail_plugins = $mail_plugins fts fts_solr - - plugin { - fts = solr - fts_autoindex = yes - fts_solr = url=http://solr:8983/solr/dovecot/ - } - ``` - -3. Recreate containers: `docker compose down ; docker compose up -d` - -4. Flag all user mailbox FTS indexes as invalid, so they are rescanned on demand when they are next searched: `docker compose exec mailserver doveadm fts rescan -A` - #### Further Discussion See [#905](https://github.com/docker-mailserver/docker-mailserver/issues/905) diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 5360ae957bb..6b21cc72d02 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -135,7 +135,7 @@ function _install_dovecot() { local DOVECOT_PACKAGES=( dovecot-core dovecot-imapd dovecot-ldap dovecot-lmtpd dovecot-managesieved - dovecot-pop3d dovecot-sieve dovecot-solr + dovecot-pop3d dovecot-sieve ) # Dovecot packages for community supported features. From 411984928415e5cbfdd3c473f171dfd3c3383c96 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sat, 25 May 2024 19:49:45 +0200 Subject: [PATCH 365/592] update: Dovecot FTS Xapian from 1.5.5 to 1.7.12 (#4034) * update `compile.sh` and Dovecot FTS Xapian to 1.7.12 - I updated from 1.5.5. Moreover, I adjusted the script to have what I consider better style. * update Dockerfile to use recent updates * update CHANGELOG --- CHANGELOG.md | 1 + Dockerfile | 4 ++-- target/scripts/build/compile.sh | 22 +++++++++++++++------- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04f8adb9194..c791ac5a3ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,6 +98,7 @@ The most noteworthy change of this release is the update of the container's base - The symbol weights of SPF, DKIM and DMARC have been adjusted again. Fixes a bug and includes more appropriate combinations of symbols ([#3913](https://github.com/docker-mailserver/docker-mailserver/pull/3913), [#3923](https://github.com/docker-mailserver/docker-mailserver/pull/3923)) - **Dovecot:** - `logwatch` now filters out non-error logs related to the status of the `index-worker` process for FTS indexing. ([#4012](https://github.com/docker-mailserver/docker-mailserver/pull/4012)) + - updated FTS Xapian from version 1.5.5 to 1.7.12 ### Fixes diff --git a/Dockerfile b/Dockerfile index bf688ec1e9d..854c233227b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -82,8 +82,8 @@ EOF # install fts_xapian plugin -COPY --from=stage-compile dovecot-fts-xapian-1.5.5_1.5.5_*.deb / -RUN dpkg -i /dovecot-fts-xapian-1.5.5_1.5.5_*.deb && rm /dovecot-fts-xapian-1.5.5_1.5.5_*.deb +COPY --from=stage-compile dovecot-fts-xapian-1.7.12_1.7.12_*.deb / +RUN dpkg -i /dovecot-fts-xapian-1.7.12_1.7.12_*.deb && rm /dovecot-fts-xapian-1.7.12_1.7.12_*.deb COPY target/dovecot/*.inc target/dovecot/*.conf /etc/dovecot/conf.d/ COPY target/dovecot/dovecot-purge.cron /etc/cron.d/dovecot-purge.disabled diff --git a/target/scripts/build/compile.sh b/target/scripts/build/compile.sh index 56feea72993..4c6fba82426 100644 --- a/target/scripts/build/compile.sh +++ b/target/scripts/build/compile.sh @@ -8,21 +8,29 @@ set -eE -u -o pipefail # shellcheck source=../helpers/log.sh source /usr/local/bin/helpers/log.sh +# shellcheck disable=SC2310 _log_level_is 'trace' && QUIET='-y' || QUIET='-qq' function _compile_dovecot_fts_xapian() { apt-get "${QUIET}" update - apt-get "${QUIET}" --no-install-recommends install automake libtool pkg-config libicu-dev libsqlite3-dev libxapian-dev make build-essential dh-make devscripts dovecot-dev - curl -Lso dovecot-fts-xapian.tar.gz https://github.com/grosjo/fts-xapian/releases/download/1.5.5/dovecot-fts-xapian-1.5.5.tar.gz - tar xzvf dovecot-fts-xapian.tar.gz - cd fts-xapian-1.5.5 - USER=root dh_make -p dovecot-fts-xapian-1.5.5 --single --native --copyright gpl2 -y + apt-get "${QUIET}" --no-install-recommends install \ + automake libtool pkg-config libicu-dev libsqlite3-dev libxapian-dev make build-essential dh-make devscripts dovecot-dev + + local XAPIAN_VERSION='1.7.12' + curl -sSfL -o dovecot-fts-xapian.tar.gz \ + "https://github.com/grosjo/fts-xapian/releases/download/${XAPIAN_VERSION}/dovecot-fts-xapian-${XAPIAN_VERSION}.tar.gz" + tar xf dovecot-fts-xapian.tar.gz + + cd "fts-xapian-${XAPIAN_VERSION}" + USER=root dh_make -p "dovecot-fts-xapian-${XAPIAN_VERSION}" --single --native --copyright gpl2 -y + rm debian/*.ex cp PACKAGES/DEB/control debian/ cp PACKAGES/DEB/changelog debian/ cp PACKAGES/DEB/compat debian/ - sed -i 's/1\.4\.11-6/1.5.5/g' debian/control - sed -i 's/1\.4\.11-6/1.5.5/g' debian/changelog + + sed -i -E "s|(dovecot-fts-xapian)-[1-9\.-]+|\1-${XAPIAN_VERSION}|g" debian/control + sed -i -E "s|(dovecot-fts-xapian)-[1-9\.-]+ \(.*\)(.*)|\1-${XAPIAN_VERSION} (${XAPIAN_VERSION})\2|g" debian/changelog debuild -us -uc -B | tee /tmp/debuild.log 2>&1 } From b222035112fa61fa6a9b6fe16a04b8005cc4e0a1 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sat, 25 May 2024 19:56:19 +0200 Subject: [PATCH 366/592] scripts: perform additional checks when updating/adding/deletting accounts (#4033) * normalize accounts to lowercase * update CHANGELOG * add test to verify bug fix works correctly --- CHANGELOG.md | 2 ++ .../helpers/database/manage/postfix-accounts.sh | 10 ++++++++++ .../tests/parallel/set3/mta/account_management.bats | 13 +++++++++++++ 3 files changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c791ac5a3ed..e2285f6d93b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,8 @@ The most noteworthy change of this release is the update of the container's base - `supervisor-app.conf` renamed to `dms-services.conf` - **Rspamd**: - the Redis history key has been changed in order to not incorporate the hostname of the container (which is desirable in Kubernetes environments) ([#3927](https://github.com/docker-mailserver/docker-mailserver/pull/3927)) +- **Account Management** + - addresses (accounts) are now normalized to lowercase automatically and a warning is logged in case uppercase letters are supplied ### Added diff --git a/target/scripts/helpers/database/manage/postfix-accounts.sh b/target/scripts/helpers/database/manage/postfix-accounts.sh index 8a2a6133392..987254fca9d 100644 --- a/target/scripts/helpers/database/manage/postfix-accounts.sh +++ b/target/scripts/helpers/database/manage/postfix-accounts.sh @@ -13,6 +13,7 @@ function _manage_accounts() { local PASSWD=${4} _arg_expect_mail_account + _arg_check_mail_account case "${ACTION}" in ( 'create' | 'update' ) @@ -69,6 +70,15 @@ function _arg_expect_mail_account() { [[ ${MAIL_ACCOUNT} =~ .*\@.* ]] || { __usage ; _exit_with_error "'${MAIL_ACCOUNT}' should include the domain (eg: user@example.com)" ; } } +# Checks the mail account string, e.g. on uppercase letters. +function _arg_check_mail_account() { + if grep -q -E '[[:upper:]]+' <<< "${MAIL_ACCOUNT}"; then + local MAIL_ACCOUNT_NORMALIZED=${MAIL_ACCOUNT,,} + _log 'warn' "Mail account '${MAIL_ACCOUNT}' has uppercase letters and will be normalized to '${MAIL_ACCOUNT_NORMALIZED}'" + MAIL_ACCOUNT=${MAIL_ACCOUNT_NORMALIZED} + fi +} + function _account_should_not_exist_yet() { __account_already_exists && _exit_with_error "'${MAIL_ACCOUNT}' already exists" if [[ -f ${DATABASE_VIRTUAL} ]] && grep -q "^${MAIL_ACCOUNT}" "${DATABASE_VIRTUAL}"; then diff --git a/test/tests/parallel/set3/mta/account_management.bats b/test/tests/parallel/set3/mta/account_management.bats index bc97d710544..f8d5f9de6cc 100644 --- a/test/tests/parallel/set3/mta/account_management.bats +++ b/test/tests/parallel/set3/mta/account_management.bats @@ -74,6 +74,19 @@ function teardown_file() { _default_teardown ; } __should_add_new_user 'user3@domain.tld' } +@test "should add new user 'USeRx@domain.tld' as 'userx@domain.tld' into 'postfix-accounts.cf' and log a warning" { + local MAIL_ACCOUNT='USeRx@domain.tld' + local NORMALIZED_MAIL_ACCOUNT='userx@domain.tld' + + _run_in_container setup email add "${MAIL_ACCOUNT}" mypassword + assert_success + assert_output --partial "'USeRx@domain.tld' has uppercase letters and will be normalized to 'userx@domain.tld'" + + __check_mail_account_exists "${NORMALIZED_MAIL_ACCOUNT}" + assert_success + assert_output "${NORMALIZED_MAIL_ACCOUNT}" +} + # To catch mistakes from substring matching: @test "should add new user 'auser3@domain.tld' into 'postfix-accounts.cf'" { __should_add_new_user 'auser3@domain.tld' From 94bde85ac6273caf186b9efc94df414ea0a6319c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 26 May 2024 10:41:01 +0200 Subject: [PATCH 367/592] docs: updated `CONTRIBUTORS.md` (#4036) --- CONTRIBUTORS.md | 357 ++++++++++++++++++++++++------------------------ 1 file changed, 182 insertions(+), 175 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 1600440c95c..c627f750255 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -329,135 +329,135 @@ Thanks goes to these wonderful people ✨ - - egavard + + robertdolca
- egavard + robertdolca - - dashohoxha + + okainov
- dashohoxha + okainov - - mathuin + + lukecyca
- mathuin + lukecyca - - jamebus + + jsonn
- jamebus + jsonn - - jsonn + + jamebus
- jsonn + jamebus - - lukecyca + + mathuin
- lukecyca + mathuin - - okainov + + dashohoxha
- okainov + dashohoxha - - robertdolca + + egavard
- robertdolca + egavard - - VanVan + + voordev
- VanVan + voordev - - eltociear + + Zehir
- eltociear + Zehir - - mjung + + guardiande
- mjung + guardiande - - andreasgerstmayr + + kamuri
- andreasgerstmayr + kamuri - - m-schmoock + + davidszp
- m-schmoock + davidszp - - davidszp + + andreasgerstmayr
- davidszp + andreasgerstmayr - - kamuri + + VanVan
- kamuri + VanVan - - guardiande + + eltociear
- guardiande + eltociear - - Zehir + + mjung
- Zehir + mjung - - voordev + + m-schmoock
- voordev + m-schmoock @@ -498,17 +498,24 @@ Thanks goes to these wonderful people ✨ - - MakerMatrix + + weo
- MakerMatrix + weo - - pbek + + analogue
- pbek + analogue + + + + + Rubytastic2 +
+ Rubytastic2 @@ -519,33 +526,40 @@ Thanks goes to these wonderful people ✨ - - Rubytastic2 + + pbek
- Rubytastic2 + pbek + + - - analogue + + MakerMatrix
- analogue + MakerMatrix - - - - weo + + mpanneck
- weo + mpanneck - - ubenmackin + + keslerm
- ubenmackin + keslerm + + + + + willtho89 +
+ willtho89 @@ -562,6 +576,8 @@ Thanks goes to these wonderful people ✨ p-fruck + + rahilarious @@ -576,8 +592,6 @@ Thanks goes to these wonderful people ✨ Rillke - - bobbravo2 @@ -592,12 +606,28 @@ Thanks goes to these wonderful people ✨ r-pufky + + + yogo1212 +
+ yogo1212 + + vincentDcmps
vincentDcmps + + + + + + tbutter +
+ tbutter + @@ -620,8 +650,6 @@ Thanks goes to these wonderful people ✨ engelant - - j-marz @@ -636,6 +664,8 @@ Thanks goes to these wonderful people ✨ lokipo + + msheakoski @@ -658,33 +688,10 @@ Thanks goes to these wonderful people ✨ - - tbutter -
- tbutter - - - - - - - yogo1212 -
- yogo1212 - - - - - willtho89 -
- willtho89 - - - - - mpanneck + + ubenmackin
- mpanneck + ubenmackin @@ -701,6 +708,8 @@ Thanks goes to these wonderful people ✨ abh + + andrewlow @@ -708,8 +717,6 @@ Thanks goes to these wonderful people ✨ andrewlow - - aminvakil @@ -745,6 +752,8 @@ Thanks goes to these wonderful people ✨ theomega + + DuncanvR @@ -752,8 +761,6 @@ Thanks goes to these wonderful people ✨ DuncanvR - - emazzotta @@ -761,13 +768,6 @@ Thanks goes to these wonderful people ✨ emazzotta - - - keslerm -
- keslerm - - nueaf @@ -798,6 +798,13 @@ Thanks goes to these wonderful people ✨ + + + jiriks74 +
+ jiriks74 + + jedateach @@ -833,6 +840,8 @@ Thanks goes to these wonderful people ✨ millaguie + + jamesfryer @@ -840,8 +849,6 @@ Thanks goes to these wonderful people ✨ jamesfryer - - radicand @@ -877,6 +884,8 @@ Thanks goes to these wonderful people ✨ syl20bnr + + sylvaindumont @@ -884,8 +893,6 @@ Thanks goes to these wonderful people ✨ sylvaindumont - - TechnicLab @@ -921,6 +928,8 @@ Thanks goes to these wonderful people ✨ tweibert + + torus @@ -928,8 +937,6 @@ Thanks goes to these wonderful people ✨ torus - - VictorKoenders @@ -965,6 +972,8 @@ Thanks goes to these wonderful people ✨ vilisas + + forzagreen @@ -972,8 +981,6 @@ Thanks goes to these wonderful people ✨ forzagreen - - 42wim @@ -1009,6 +1016,8 @@ Thanks goes to these wonderful people ✨ pcqnt + + OrvilleQ @@ -1016,8 +1025,6 @@ Thanks goes to these wonderful people ✨ OrvilleQ - - ovidiucp @@ -1053,6 +1060,8 @@ Thanks goes to these wonderful people ✨ neuralp + + piwai @@ -1060,8 +1069,6 @@ Thanks goes to these wonderful people ✨ piwai - - remoe @@ -1097,6 +1104,8 @@ Thanks goes to these wonderful people ✨ MightySCollins + + 501st-alpha1 @@ -1104,8 +1113,6 @@ Thanks goes to these wonderful people ✨ 501st-alpha1 - - klamann @@ -1141,6 +1148,8 @@ Thanks goes to these wonderful people ✨ sjmudd + + simonsystem @@ -1148,8 +1157,6 @@ Thanks goes to these wonderful people ✨ simonsystem - - mchamplain @@ -1185,6 +1192,8 @@ Thanks goes to these wonderful people ✨ odinis + + okamidash @@ -1192,8 +1201,6 @@ Thanks goes to these wonderful people ✨ okamidash - - olaf-mandel @@ -1229,6 +1236,8 @@ Thanks goes to these wonderful people ✨ rhyst + + rmlhuk @@ -1236,8 +1245,6 @@ Thanks goes to these wonderful people ✨ rmlhuk - - rriski @@ -1273,6 +1280,8 @@ Thanks goes to these wonderful people ✨ squash + + strarsis @@ -1280,8 +1289,6 @@ Thanks goes to these wonderful people ✨ strarsis - - tamueller @@ -1317,6 +1324,8 @@ Thanks goes to these wonderful people ✨ worldworm + + Zepmann @@ -1324,8 +1333,6 @@ Thanks goes to these wonderful people ✨ Zepmann - - allddd @@ -1361,6 +1368,8 @@ Thanks goes to these wonderful people ✨ cternes + + dborowy @@ -1368,8 +1377,6 @@ Thanks goes to these wonderful people ✨ dborowy - - dimalo @@ -1405,6 +1412,8 @@ Thanks goes to these wonderful people ✨ helmutundarnold + + hnws @@ -1412,8 +1421,6 @@ Thanks goes to these wonderful people ✨ hnws - - i-C-o-d-e-r @@ -1449,6 +1456,8 @@ Thanks goes to these wonderful people ✨ paralax + + jpduyx @@ -1456,8 +1465,6 @@ Thanks goes to these wonderful people ✨ jpduyx - - landergate @@ -1493,6 +1500,8 @@ Thanks goes to these wonderful people ✨ 0xflotus + + arkanovicz @@ -1500,8 +1509,6 @@ Thanks goes to these wonderful people ✨ arkanovicz - - CBeerta @@ -1537,6 +1544,8 @@ Thanks goes to these wonderful people ✨ dbellavista + + danielvandenberg95 @@ -1544,8 +1553,6 @@ Thanks goes to these wonderful people ✨ danielvandenberg95 - - denisix @@ -1581,6 +1588,8 @@ Thanks goes to these wonderful people ✨ aydodo + + vedtam @@ -1588,8 +1597,6 @@ Thanks goes to these wonderful people ✨ vedtam - - edvorg @@ -1625,6 +1632,8 @@ Thanks goes to these wonderful people ✨ huncode + + gitfeber @@ -1632,8 +1641,6 @@ Thanks goes to these wonderful people ✨ gitfeber - - felixn @@ -1669,6 +1676,8 @@ Thanks goes to these wonderful people ✨ 20th + + 2b @@ -1676,8 +1685,6 @@ Thanks goes to these wonderful people ✨ 2b - - askz @@ -1713,6 +1720,8 @@ Thanks goes to these wonderful people ✨ kachkaev + + alexanderneu @@ -1720,8 +1729,6 @@ Thanks goes to these wonderful people ✨ alexanderneu - - ch3sh1r @@ -1757,6 +1764,8 @@ Thanks goes to these wonderful people ✨ iRhonin + + MrFreezeex @@ -1764,8 +1773,6 @@ Thanks goes to these wonderful people ✨ MrFreezeex - - arunvc @@ -1801,6 +1808,8 @@ Thanks goes to these wonderful people ✨ erdos4d + + crash7 @@ -1808,8 +1817,6 @@ Thanks goes to these wonderful people ✨ crash7 - - auchri @@ -1845,6 +1852,8 @@ Thanks goes to these wonderful people ✨ thechubbypanda + + KCrawley @@ -1852,8 +1861,6 @@ Thanks goes to these wonderful people ✨ KCrawley - - khuedoan @@ -1889,6 +1896,8 @@ Thanks goes to these wonderful people ✨ linhandev + + luke- @@ -1896,8 +1905,6 @@ Thanks goes to these wonderful people ✨ luke- - - LucidityCrash @@ -1933,6 +1940,8 @@ Thanks goes to these wonderful people ✨ dragetd + + michaeljensen @@ -1940,8 +1949,6 @@ Thanks goes to these wonderful people ✨ michaeljensen - - exhuma @@ -1977,6 +1984,8 @@ Thanks goes to these wonderful people ✨ mpldr + + naveensrinivasan @@ -1984,8 +1993,6 @@ Thanks goes to these wonderful people ✨ naveensrinivasan - - furstblumier @@ -2021,6 +2028,8 @@ Thanks goes to these wonderful people ✨ harryyoud + + HeySora @@ -2028,8 +2037,6 @@ Thanks goes to these wonderful people ✨ HeySora - - sirgantrithon @@ -2065,6 +2072,8 @@ Thanks goes to these wonderful people ✨ JacksonZ03 + + JamBalaya56562 @@ -2072,8 +2081,6 @@ Thanks goes to these wonderful people ✨ JamBalaya56562 - - jcalfee @@ -2109,6 +2116,8 @@ Thanks goes to these wonderful people ✨ jessp01 + + JiLleON @@ -2116,8 +2125,6 @@ Thanks goes to these wonderful people ✨ JiLleON - - jirislav From a96a4e2768216167b7c80f28a3d4323ca1db0244 Mon Sep 17 00:00:00 2001 From: Tobias Knecht Date: Tue, 28 May 2024 13:20:15 +0200 Subject: [PATCH 368/592] Abusix docs links update. (#4038) --- docs/content/config/security/rspamd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index b0f65990371..a98d3123368 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -273,7 +273,7 @@ While _Abusix_ can be integrated into Postfix, Postscreen and a multitude of oth [www::rbl-vs-dnsbl]: https://forum.eset.com/topic/25277-dnsbl-vs-rbl-mail-security/?do=findComment&comment=119818 [abusix-web]: https://abusix.com/ -[abusix-docs::rspamd-integration]: https://docs.abusix.com/abusix-mail-intelligence/gbG8EcJ3x3fSUv8cMZLiwA/getting-started/dmw9dcwSGSNQiLTssFAnBW#rspamd +[abusix-docs::rspamd-integration]: https://abusix.com/docs/rspamd/ [spamhaus::faq::dnsbl-usage]: https://www.spamhaus.org/faq/section/DNSBL%20Usage#365 [dms-repo::rspamd-actions-config]: https://github.com/docker-mailserver/docker-mailserver/blob/v14.0.0/target/rspamd/local.d/actions.conf From 95d965fb768d828667ff1de53cc3b7ba8260299e Mon Sep 17 00:00:00 2001 From: Guillaume VARA Date: Tue, 28 May 2024 17:52:30 +0200 Subject: [PATCH 369/592] docs(k8s): Advise `externalTrafficPolicy: Local` if no PROXY protocol configured (#4039) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- docs/content/config/advanced/kubernetes.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/content/config/advanced/kubernetes.md b/docs/content/config/advanced/kubernetes.md index 268094a6c24..337f1e36b69 100644 --- a/docs/content/config/advanced/kubernetes.md +++ b/docs/content/config/advanced/kubernetes.md @@ -140,6 +140,9 @@ If using our Helm chart is not viable for you, here is some guidance to start wi app: mailserver spec: + # `Local` is most likely required, otherwise every incoming request would be identified by the external IP, + # which will get banned by Fail2Ban when monitored services are not configured for PROXY protocol + externalTrafficPolicy: Local type: LoadBalancer selector: From f8b3f402763a8ac1112971ec53a2041398e7ce4c Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 29 May 2024 02:28:51 +0200 Subject: [PATCH 370/592] scripts: update log format (#4035) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 2 ++ target/scripts/check-for-changes.sh | 30 +++++++++---------- target/scripts/helpers/log.sh | 28 +++++++++-------- target/scripts/update-check.sh | 12 ++++---- .../set1/spam_virus/rspamd_partly.bats | 2 +- test/tests/parallel/set1/tls/dhparams.bats | 2 +- 6 files changed, 40 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2285f6d93b..305119831ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,8 @@ The most noteworthy change of this release is the update of the container's base - **Removed support for Solr integration:** ([#4025](https://github.com/docker-mailserver/docker-mailserver/pull/4025)) - This was a community contributed feature for FTS (Full Text Search), the docs advise using an image that has not been maintained for over 2 years and lacks ARM64 support. Based on user engagement over the years this feature has very niche value to continue to support, thus is being removed. - If you use Solr, support can be restored if you're willing to contribute docs for the feature that resolves the concerns raised +- **Log**: + - The format of DMS specific logs (_from our scripts, not running services_) has been changed. The new format is ` : ` ([#4035](https://github.com/docker-mailserver/docker-mailserver/pull/4035)) - **rsyslog:** - Debian 12 adjusted the `rsyslog` configuration for the default file template from `RSYSLOG_TraditionalFileFormat` to `RSYSLOG_FileFormat` (_upstream default since 2012_). This change may affect you if you have any monitoring / analysis of log output (_eg: `mail.log` / `docker logs`_). - The two formats are roughly equivalent to [RFC 3164](https://www.rfc-editor.org/rfc/rfc3164)) and [RFC 5424](https://datatracker.ietf.org/doc/html/rfc5424#section-1) respectively. diff --git a/target/scripts/check-for-changes.sh b/target/scripts/check-for-changes.sh index 3f268710c7e..bf5cd90b282 100755 --- a/target/scripts/check-for-changes.sh +++ b/target/scripts/check-for-changes.sh @@ -7,7 +7,7 @@ # shellcheck source=./helpers/index.sh source /usr/local/bin/helpers/index.sh -_log_with_date 'debug' 'Starting changedetector' +_log 'debug' 'Starting changedetector' # ATTENTION: Do not remove! # This script requires some environment variables to be properly set. @@ -30,9 +30,9 @@ if [[ ! -f ${CHKSUM_FILE} ]]; then _exit_with_error "'${CHKSUM_FILE}' is missing" 0 fi -_log_with_date 'trace' "Using postmaster address '${POSTMASTER_ADDRESS}'" +_log 'trace' "Using postmaster address '${POSTMASTER_ADDRESS}'" -_log_with_date 'debug' "Changedetector is ready" +_log 'debug' "Changedetector is ready" function _check_for_changes() { # get chksum and check it, no need to lock config yet @@ -44,7 +44,7 @@ function _check_for_changes() { # 1 – files differ # 2 – inaccessible or missing argument if [[ ${?} -eq 1 ]]; then - _log_with_date 'info' 'Change detected' + _log 'info' 'Change detected' _create_lock # Shared config safety lock local CHANGED @@ -52,7 +52,7 @@ function _check_for_changes() { _handle_changes _remove_lock - _log_with_date 'debug' 'Completed handling of detected change' + _log 'debug' 'Completed handling of detected change' # mark changes as applied mv "${CHKSUM_FILE}.new" "${CHKSUM_FILE}" @@ -64,7 +64,7 @@ function _handle_changes() { local VHOST_UPDATED=0 # These two configs are the source for /etc/postfix/vhost (managed mail domains) if [[ ${CHANGED} =~ ${DMS_DIR}/postfix-(accounts|virtual).cf ]]; then - _log_with_date 'trace' 'Regenerating vhosts (Postfix)' + _log 'trace' 'Regenerating vhosts (Postfix)' # Regenerate via `helpers/postfix.sh`: _create_postfix_vhost @@ -75,7 +75,7 @@ function _handle_changes() { _postfix_dovecot_changes _rspamd_changes - _log_with_date 'debug' 'Reloading services due to detected changes' + _log 'debug' 'Reloading services due to detected changes' [[ ${ENABLE_AMAVIS} -eq 1 ]] && _reload_amavis _reload_postfix @@ -119,7 +119,7 @@ function _postfix_dovecot_changes() { || [[ ${CHANGED} =~ ${DMS_DIR}/dovecot-quotas.cf ]] \ || [[ ${CHANGED} =~ ${DMS_DIR}/dovecot-masters.cf ]] then - _log_with_date 'trace' 'Regenerating accounts (Dovecot + Postfix)' + _log 'trace' 'Regenerating accounts (Dovecot + Postfix)' [[ ${SMTP_ONLY} -ne 1 ]] && _create_accounts fi @@ -131,7 +131,7 @@ function _postfix_dovecot_changes() { || [[ ${CHANGED} =~ ${DMS_DIR}/postfix-relaymap.cf ]] \ || [[ ${CHANGED} =~ ${DMS_DIR}/postfix-sasl-password.cf ]] then - _log_with_date 'trace' 'Regenerating relay config (Postfix)' + _log 'trace' 'Regenerating relay config (Postfix)' _process_relayhost_configs fi @@ -159,14 +159,14 @@ function _ssl_changes() { || [[ ${CHANGED} =~ ${SSL_ALT_CERT_PATH:-${REGEX_NEVER_MATCH}} ]] \ || [[ ${CHANGED} =~ ${SSL_ALT_KEY_PATH:-${REGEX_NEVER_MATCH}} ]] then - _log_with_date 'debug' 'Manual certificates have changed - extracting certificates' + _log 'debug' 'Manual certificates have changed - extracting certificates' _setup_ssl fi # `acme.json` is only relevant to Traefik, and is where it stores the certificates it manages. # When a change is detected it's assumed to be a possible cert renewal that needs to be # extracted for `docker-mailserver` services to adjust to. elif [[ ${CHANGED} =~ /etc/letsencrypt/acme.json ]]; then - _log_with_date 'debug' "'/etc/letsencrypt/acme.json' has changed - extracting certificates" + _log 'debug' "'/etc/letsencrypt/acme.json' has changed - extracting certificates" _setup_ssl # Prevent an unnecessary change detection from the newly extracted cert files by updating their hashes in advance: @@ -188,23 +188,23 @@ function _rspamd_changes() { # "${RSPAMD_DMS_D}/override.d" if [[ ${CHANGED} =~ ${RSPAMD_DMS_OVERRIDE_D}/.* ]]; then - _log_with_date 'trace' 'Rspamd - Copying configuration overrides' + _log 'trace' 'Rspamd - Copying configuration overrides' rm "${RSPAMD_OVERRIDE_D}"/* cp "${RSPAMD_DMS_OVERRIDE_D}"/* "${RSPAMD_OVERRIDE_D}" fi # "${RSPAMD_DMS_D}/custom-commands.conf" if [[ ${CHANGED} =~ ${RSPAMD_DMS_CUSTOM_COMMANDS_F} ]]; then - _log_with_date 'trace' 'Rspamd - Generating new configuration from custom commands' + _log 'trace' 'Rspamd - Generating new configuration from custom commands' _rspamd_handle_user_modules_adjustments fi # "${RSPAMD_DMS_D}/dkim" if [[ ${CHANGED} =~ ${RSPAMD_DMS_DKIM_D} ]]; then - _log_with_date 'trace' 'Rspamd - DKIM files updated' + _log 'trace' 'Rspamd - DKIM files updated' fi - _log_with_date 'debug' 'Rspamd configuration has changed - restarting service' + _log 'debug' 'Rspamd configuration has changed - restarting service' supervisorctl restart rspamd fi } diff --git a/target/scripts/helpers/log.sh b/target/scripts/helpers/log.sh index d771a3f563d..d98f96a8213 100644 --- a/target/scripts/helpers/log.sh +++ b/target/scripts/helpers/log.sh @@ -53,8 +53,7 @@ function _log() { return 1 fi - local LEVEL_AS_INT - local MESSAGE="${RESET}[" + local LEVEL_AS_INT LOG_COLOR LOG_LEVEL_NAME MESSAGE case "$(_get_log_level_or_default)" in ( 'trace' ) LEVEL_AS_INT=5 ;; @@ -67,27 +66,35 @@ function _log() { case "${1}" in ( 'trace' ) [[ ${LEVEL_AS_INT} -ge 5 ]] || return 0 - MESSAGE+=" ${CYAN}TRACE " + LOG_COLOR='CYAN' + LOG_LEVEL_NAME='TRACE' ;; ( 'debug' ) [[ ${LEVEL_AS_INT} -ge 4 ]] || return 0 - MESSAGE+=" ${PURPLE}DEBUG " + LOG_COLOR='PURPLE' + LOG_LEVEL_NAME='DEBUG' ;; ( 'info' ) [[ ${LEVEL_AS_INT} -ge 3 ]] || return 0 - MESSAGE+=" ${BLUE}INF " + LOG_COLOR='BLUE' + # the whitespace is intentional (for alignment purposes) + LOG_LEVEL_NAME='INFO ' ;; ( 'warn' ) [[ ${LEVEL_AS_INT} -ge 2 ]] || return 0 - MESSAGE+=" ${LYELLOW}WARNING " + LOG_COLOR='LYELLOW' + # the whitespace is intentional (for alignment purposes) + LOG_LEVEL_NAME='WARN ' ;; ( 'error' ) [[ ${LEVEL_AS_INT} -ge 1 ]] || return 0 - MESSAGE+=" ${LRED}ERROR " ;; + LOG_COLOR='LRED' + LOG_LEVEL_NAME='ERROR' + ;; ( * ) _log 'error' "Call to '_log' with invalid log level argument '${1}'" @@ -95,7 +102,7 @@ function _log() { ;; esac - MESSAGE+="${RESET}] ${2}" + MESSAGE="$(date --rfc-3339='seconds') ${!LOG_COLOR}${LOG_LEVEL_NAME}${RESET} $(basename "${0}"): ${2}" if [[ ${1} =~ ^(warn|error)$ ]]; then echo -e "${MESSAGE}" >&2 @@ -104,11 +111,6 @@ function _log() { fi } -# Like `_log` but adds a timestamp in front of the message. -function _log_with_date() { - _log "${1}" "$(date '+%Y-%m-%d %H:%M:%S') ${2}" -} - # Get the value of the environment variable LOG_LEVEL if # it is set. Otherwise, try to query the common environment # variables file. If this does not yield a value either, diff --git a/target/scripts/update-check.sh b/target/scripts/update-check.sh index 257fc37d380..f00bfcb3d14 100755 --- a/target/scripts/update-check.sh +++ b/target/scripts/update-check.sh @@ -10,8 +10,8 @@ CHANGELOG_URL='https://github.com/docker-mailserver/docker-mailserver/blob/maste # check for correct syntax # number + suffix. suffix must be 's' for seconds, 'm' for minutes, 'h' for hours or 'd' for days. if [[ ! ${UPDATE_CHECK_INTERVAL} =~ ^[0-9]+[smhd]{1}$ ]]; then - _log_with_date 'warn' "Invalid 'UPDATE_CHECK_INTERVAL' value '${UPDATE_CHECK_INTERVAL}'" - _log_with_date 'warn' 'Falling back to daily update checks' + _log 'warn' "Invalid 'UPDATE_CHECK_INTERVAL' value '${UPDATE_CHECK_INTERVAL}'" + _log 'warn' 'Falling back to daily update checks' UPDATE_CHECK_INTERVAL='1d' fi @@ -22,7 +22,7 @@ while true; do # did we get a valid response? if [[ ${LATEST} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - _log_with_date 'debug' 'Remote version information fetched' + _log 'debug' 'Remote version information fetched' # compare versions if dpkg --compare-versions "${VERSION}" lt "${LATEST}"; then @@ -38,15 +38,15 @@ Latest version: ${LATEST} Changelog: ${CHANGELOG_URL}#END EOF - _log_with_date 'info' "Update available [ ${VERSION} --> ${LATEST} ]" + _log 'info' "Update available [ ${VERSION} --> ${LATEST} ]" # only notify once echo "${MAIL}" | mail -s "Mailserver update available! [ ${VERSION} --> ${LATEST} ]" "${POSTMASTER_ADDRESS}" && exit 0 else - _log_with_date 'info' 'No update available' + _log 'info' 'No update available' fi else - _log_with_date 'warn' 'Update check failed' + _log 'warn' 'Update check failed' fi # check again in 'UPDATE_CHECK_INTERVAL' time diff --git a/test/tests/parallel/set1/spam_virus/rspamd_partly.bats b/test/tests/parallel/set1/spam_virus/rspamd_partly.bats index b647f562110..a1afc9a378a 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_partly.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_partly.bats @@ -44,7 +44,7 @@ function teardown_file() { _default_teardown ; } run docker logs "${CONTAINER_NAME}" assert_success for SERVICE in 'Amavis/SA' 'OpenDKIM' 'OpenDMARC' 'policyd-spf'; do - assert_output --regexp ".*WARNING.*Running ${SERVICE} & Rspamd at the same time is discouraged" + assert_output --regexp ".*WARN.*Running ${SERVICE} & Rspamd at the same time is discouraged" done } diff --git a/test/tests/parallel/set1/tls/dhparams.bats b/test/tests/parallel/set1/tls/dhparams.bats index 8b3047d1737..86586f28c49 100644 --- a/test/tests/parallel/set1/tls/dhparams.bats +++ b/test/tests/parallel/set1/tls/dhparams.bats @@ -50,7 +50,7 @@ function teardown() { _default_teardown ; } # Should emit a warning: run docker logs "${CONTAINER_NAME}" assert_success - assert_output --partial '[ WARNING ] Using self-generated dhparams is considered insecure - unless you know what you are doing, please remove' + assert_output --partial 'Using self-generated dhparams is considered insecure - unless you know what you are doing, please remove' } # Ensures the docker image services (Postfix and Dovecot) have the expected DH files: From e78d5c61eefcef4bc47695e4879c6ba4f0787da8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Jun 2024 08:35:07 +0200 Subject: [PATCH 371/592] docs: updated `CONTRIBUTORS.md` (#4043) --- CONTRIBUTORS.md | 268 +++++++++++++++++++++++++----------------------- 1 file changed, 142 insertions(+), 126 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index c627f750255..67bd247a0e4 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -34,13 +34,6 @@ Thanks goes to these wonderful people ✨ NorseGaud - - - williamdes -
- williamdes - - wernerfred @@ -48,8 +41,6 @@ Thanks goes to these wonderful people ✨ wernerfred - - georglauterbach @@ -57,6 +48,8 @@ Thanks goes to these wonderful people ✨ georglauterbach + + tomav @@ -92,8 +85,6 @@ Thanks goes to these wonderful people ✨ Josef-Friedrich - - johansmitsnl @@ -101,6 +92,8 @@ Thanks goes to these wonderful people ✨ johansmitsnl + + youtous @@ -136,8 +129,6 @@ Thanks goes to these wonderful people ✨ ap-wtioit - - 00angus @@ -145,6 +136,8 @@ Thanks goes to these wonderful people ✨ 00angus + + alinmear @@ -180,14 +173,21 @@ Thanks goes to these wonderful people ✨ swiesend - - stonemaster
stonemaster + + + + + + williamdes +
+ williamdes + @@ -688,47 +688,47 @@ Thanks goes to these wonderful people ✨ - - ubenmackin + + andrewlow
- ubenmackin + andrewlow - - craue + + aminvakil
- craue + aminvakil - - abh + + elbracht
- abh + elbracht - - andrewlow + + abh
- andrewlow + abh - - aminvakil + + craue
- aminvakil + craue - - elbracht + + ubenmackin
- elbracht + ubenmackin @@ -813,17 +813,17 @@ Thanks goes to these wonderful people ✨ - - fl42 + + millaguie
- fl42 + millaguie - - ipernet + + jamesfryer
- ipernet + jamesfryer @@ -834,26 +834,19 @@ Thanks goes to these wonderful people ✨ - - millaguie + + ipernet
- millaguie + ipernet - - jamesfryer -
- jamesfryer - - - - - radicand + + fl42
- radicand + fl42 @@ -884,8 +877,6 @@ Thanks goes to these wonderful people ✨ syl20bnr - - sylvaindumont @@ -893,6 +884,8 @@ Thanks goes to these wonderful people ✨ sylvaindumont + + TechnicLab @@ -921,6 +914,13 @@ Thanks goes to these wonderful people ✨ tobiabocchi + + + tknecht +
+ tknecht + + tweibert @@ -995,6 +995,20 @@ Thanks goes to these wonderful people ✨ ShiriNmi1520 + + + neuralp +
+ neuralp + + + + + radicand +
+ radicand + + nilshoell @@ -1002,6 +1016,8 @@ Thanks goes to these wonderful people ✨ nilshoell + + nknapp @@ -1016,8 +1032,6 @@ Thanks goes to these wonderful people ✨ pcqnt - - OrvilleQ @@ -1046,6 +1060,8 @@ Thanks goes to these wonderful people ✨ p3dda + + peter-hartmann @@ -1053,15 +1069,6 @@ Thanks goes to these wonderful people ✨ peter-hartmann - - - neuralp -
- neuralp - - - - piwai @@ -1097,6 +1104,8 @@ Thanks goes to these wonderful people ✨ norrs + + MightySCollins @@ -1104,8 +1113,6 @@ Thanks goes to these wonderful people ✨ MightySCollins - - 501st-alpha1 @@ -1141,6 +1148,8 @@ Thanks goes to these wonderful people ✨ shyim + + sjmudd @@ -1148,8 +1157,6 @@ Thanks goes to these wonderful people ✨ sjmudd - - simonsystem @@ -1185,6 +1192,8 @@ Thanks goes to these wonderful people ✨ mplx + + odinis @@ -1192,8 +1201,6 @@ Thanks goes to these wonderful people ✨ odinis - - okamidash @@ -1229,6 +1236,8 @@ Thanks goes to these wonderful people ✨ presocratics + + rhyst @@ -1236,8 +1245,6 @@ Thanks goes to these wonderful people ✨ rhyst - - rmlhuk @@ -1273,6 +1280,8 @@ Thanks goes to these wonderful people ✨ sportshead + + squash @@ -1280,8 +1289,6 @@ Thanks goes to these wonderful people ✨ squash - - strarsis @@ -1317,6 +1324,8 @@ Thanks goes to these wonderful people ✨ wolkenschieber + + worldworm @@ -1324,8 +1333,6 @@ Thanks goes to these wonderful people ✨ worldworm - - Zepmann @@ -1361,6 +1368,8 @@ Thanks goes to these wonderful people ✨ brainkiller + + cternes @@ -1368,8 +1377,6 @@ Thanks goes to these wonderful people ✨ cternes - - dborowy @@ -1405,6 +1412,8 @@ Thanks goes to these wonderful people ✨ ghnp5 + + helmutundarnold @@ -1412,8 +1421,6 @@ Thanks goes to these wonderful people ✨ helmutundarnold - - hnws @@ -1449,6 +1456,8 @@ Thanks goes to these wonderful people ✨ jjtt + + paralax @@ -1456,8 +1465,6 @@ Thanks goes to these wonderful people ✨ paralax - - jpduyx @@ -1492,13 +1499,6 @@ Thanks goes to these wonderful people ✨
matrixes - - - - 0xflotus -
- 0xflotus - @@ -1662,6 +1662,22 @@ Thanks goes to these wonderful people ✨ froks + + + JOduMonT +
+ JOduMonT + + + + + 0xflotus +
+ 0xflotus + + + + ifokeev @@ -1676,8 +1692,6 @@ Thanks goes to these wonderful people ✨ 20th - - 2b @@ -1706,6 +1720,8 @@ Thanks goes to these wonderful people ✨ acch + + vifino @@ -1720,8 +1736,6 @@ Thanks goes to these wonderful people ✨ kachkaev - - alexanderneu @@ -1750,6 +1764,8 @@ Thanks goes to these wonderful people ✨ groupmsl + + green-anger @@ -1764,8 +1780,6 @@ Thanks goes to these wonderful people ✨ iRhonin - - MrFreezeex @@ -1794,6 +1808,8 @@ Thanks goes to these wonderful people ✨ baxerus + + spock @@ -1808,8 +1824,6 @@ Thanks goes to these wonderful people ✨ erdos4d - - crash7 @@ -1824,13 +1838,6 @@ Thanks goes to these wonderful people ✨ auchri - - - fkefer -
- fkefer - - Kaan88 @@ -1845,6 +1852,8 @@ Thanks goes to these wonderful people ✨ akkumar + + thechubbypanda @@ -1852,8 +1861,6 @@ Thanks goes to these wonderful people ✨ thechubbypanda - - KCrawley @@ -1889,6 +1896,8 @@ Thanks goes to these wonderful people ✨ leowinterde + + linhandev @@ -1896,8 +1905,6 @@ Thanks goes to these wonderful people ✨ linhandev - - luke- @@ -1933,6 +1940,8 @@ Thanks goes to these wonderful people ✨ maxemann96 + + dragetd @@ -1940,8 +1949,6 @@ Thanks goes to these wonderful people ✨ dragetd - - michaeljensen @@ -1977,6 +1984,8 @@ Thanks goes to these wonderful people ✨ MohammedNoureldin + + mpldr @@ -1984,8 +1993,6 @@ Thanks goes to these wonderful people ✨ mpldr - - naveensrinivasan @@ -1993,6 +2000,13 @@ Thanks goes to these wonderful people ✨ naveensrinivasan + + + fkefer +
+ fkefer + + furstblumier @@ -2014,6 +2028,8 @@ Thanks goes to these wonderful people ✨ glandais + + GiovanH @@ -2021,6 +2037,13 @@ Thanks goes to these wonderful people ✨ GiovanH + + + Amphaal +
+ Amphaal + + harryyoud @@ -2028,8 +2051,6 @@ Thanks goes to these wonderful people ✨ harryyoud - - HeySora @@ -2051,6 +2072,8 @@ Thanks goes to these wonderful people ✨ Influencer + + in-seo @@ -2072,8 +2095,6 @@ Thanks goes to these wonderful people ✨ JacksonZ03 - - JamBalaya56562 @@ -2095,6 +2116,8 @@ Thanks goes to these wonderful people ✨ mivek + + init-js @@ -2116,8 +2139,6 @@ Thanks goes to these wonderful people ✨ jessp01 - - JiLleON @@ -2139,19 +2160,14 @@ Thanks goes to these wonderful people ✨ jmccl + + jurekbarth
jurekbarth - - - - JOduMonT -
- JOduMonT - From eb165ded65859794bbeca66e529a3466ceee298b Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 5 Jun 2024 17:14:35 +0200 Subject: [PATCH 372/592] prepare v14.0.0 release (#4013) Co-authored-by: Casper --- CHANGELOG.md | 4 +++- VERSION | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 305119831ed..779c92c1f06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v13.3.1...HEAD) +## [Unreleased](https://github.com/docker-mailserver/docker-mailserver/compare/v14.0.0...HEAD) > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +## [v14.0.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v14.0.0) + The most noteworthy change of this release is the update of the container's base image from Debian 11 ("Bullseye") to Debian 12 ("Bookworm"). This update alone involves breaking changes and requires a careful update! ### Breaking diff --git a/VERSION b/VERSION index c3d10c59d83..4b964e96540 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -13.3.1 +14.0.0 From f1df81a7e43cf2698bd3bd95e2e413bd5acc96e7 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 7 Jun 2024 18:42:05 +1200 Subject: [PATCH 373/592] docs: `mailserver.env` improve description for `SPAM_SUBJECT` (#4050) Co-authored-by: Casper --- mailserver.env | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mailserver.env b/mailserver.env index 0d00de0afff..1ec88dd2de4 100644 --- a/mailserver.env +++ b/mailserver.env @@ -130,8 +130,9 @@ ENABLE_IMAP=1 # **0** => Disabled ENABLE_CLAMAV=0 -# Add the value as a prefix to the mail subject when spam is detected. -# NOTE: By default spam is delivered to a junk folder, reducing the value of a subject prefix for spam. +# Add the value of this ENV as a prefix to the mail subject when spam is detected. +# NOTE: This subject prefix may be redundant (by default spam is delivered to a junk folder). +# It provides value when your junk mail is stored alongside legitimate mail instead of a separate location (like with `SPAMASSASSIN_SPAM_TO_INBOX=1` or `MOVE_SPAM_TO_JUNK=0` or a POP3 only setup, without IMAP). # NOTE: When not using Docker Compose, other CRI may not support quote-wrapping the value here to preserve any trailing white-space. SPAM_SUBJECT= From 18d9d1adcc479b7a38a31780a363c8ed272da32b Mon Sep 17 00:00:00 2001 From: Casper Date: Sat, 8 Jun 2024 13:43:25 +0200 Subject: [PATCH 374/592] Fail2ban 1.1.0 (#4045) --- CHANGELOG.md | 5 ++++ Dockerfile | 2 -- target/scripts/build/packages.sh | 48 ++++++++++++++++++++++++++------ 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 779c92c1f06..8fda6aabef5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Updates + +- **Fail2ban**: + - Bump version to [1.1.0](https://github.com/fail2ban/fail2ban/releases/tag/1.1.0). For more information, check the [changelog](https://github.com/fail2ban/fail2ban/blob/1.1.0/ChangeLog). + ## [v14.0.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v14.0.0) The most noteworthy change of this release is the update of the container's base image from Debian 11 ("Bullseye") to Debian 12 ("Bookworm"). This update alone involves breaking changes and requires a careful update! diff --git a/Dockerfile b/Dockerfile index 854c233227b..da9aa83da24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -185,8 +185,6 @@ COPY target/fail2ban/fail2ban.d/fixes.local /etc/fail2ban/fail2ban.d/fixes.local RUN <&1 + + curl -Lkso fail2ban.deb "${FAIL2BAN_DEB_URL}" + curl -Lkso fail2ban.deb.asc "${FAIL2BAN_DEB_ASC_URL}" + + FINGERPRINT=$(LANG=C gpg --verify fail2ban.deb.asc fail2ban.deb |& sed -n 's#Primary key fingerprint: \(.*\)#\1#p') + + if [[ -z ${FINGERPRINT} ]]; then + echo 'ERROR: Invalid GPG signature!' >&2 + exit 1 + fi + + if [[ ${FINGERPRINT} != "${FAIL2BAN_GPG_FINGERPRINT}" ]]; then + echo "ERROR: Wrong GPG fingerprint!" >&2 + exit 1 + fi + + dpkg -i fail2ban.deb 2>&1 + rm fail2ban.deb fail2ban.deb.asc + + _log 'debug' 'Patching Fail2ban to enable network bans' + # Enable network bans + # https://github.com/docker-mailserver/docker-mailserver/issues/2669 + # https://github.com/fail2ban/fail2ban/issues/3125 + sedfile -i -r 's/^_nft_add_set = .+/_nft_add_set = add set \\{ type \\; flags interval\\; \\}/' /etc/fail2ban/action.d/nftables.conf +} + function _post_installation_steps() { _log 'debug' 'Running post-installation steps (cleanup)' _log 'debug' 'Deleting sensitive files (secrets)' @@ -189,11 +225,6 @@ function _post_installation_steps() { _log 'trace' 'Removing leftovers from APT' apt-get "${QUIET}" clean rm -rf /var/lib/apt/lists/* - - _log 'debug' 'Patching Fail2ban to enable network bans' - # Enable network bans - # https://github.com/docker-mailserver/docker-mailserver/issues/2669 - sedfile -i -r 's/^_nft_add_set = .+/_nft_add_set = add set
\\{ type \\; flags interval\\; \\}/' /etc/fail2ban/action.d/nftables.conf } _pre_installation_steps @@ -202,4 +233,5 @@ _install_postfix _install_packages _install_dovecot _install_rspamd +_install_fail2ban _post_installation_steps From 5c798e68294f1e3c0fc133a1e4b6c745373836b8 Mon Sep 17 00:00:00 2001 From: mmehnert Date: Sun, 9 Jun 2024 15:12:49 +0200 Subject: [PATCH 375/592] Update logwatch ignore.conf to exclude Xapian messages about pending documents (#4060) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- CHANGELOG.md | 4 ++++ target/logwatch/ignore.conf | 1 + 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fda6aabef5..1aa3391e249 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ All notable changes to this project will be documented in this file. The format - **Fail2ban**: - Bump version to [1.1.0](https://github.com/fail2ban/fail2ban/releases/tag/1.1.0). For more information, check the [changelog](https://github.com/fail2ban/fail2ban/blob/1.1.0/ChangeLog). +#### Fixes +- **Dovecot:** + - `logwatch` Update logwatch `ignore.conf` to exclude Xapian messages about pending documents + ## [v14.0.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v14.0.0) The most noteworthy change of this release is the update of the container's base image from Debian 11 ("Bullseye") to Debian 12 ("Bookworm"). This update alone involves breaking changes and requires a careful update! diff --git a/target/logwatch/ignore.conf b/target/logwatch/ignore.conf index 5b7e9312f78..e3138434ed1 100644 --- a/target/logwatch/ignore.conf +++ b/target/logwatch/ignore.conf @@ -1,2 +1,3 @@ # ignore output from dovecot-fts-xapian about successfully indexed messages dovecot: indexer-worker\([^\)]+\).*Indexed +dovecot: indexer-worker\([^\)]+\).*FTS Xapian: Waiting for all pending documents to be processed From e6713a0aecb0ee449862e2a9d9c76200872c461e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Jun 2024 14:33:57 +1200 Subject: [PATCH 376/592] chore(deps): Bump docker/build-push-action from 5.3.0 to 5.4.0 (#4062) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 5.3.0 to 5.4.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v5.3.0...v5.4.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index da5d51c15f3..7f1b7c98777 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v5.3.0 + uses: docker/build-push-action@v5.4.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index ead38a3f6a9..bd54c42877c 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -67,7 +67,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: 'Build and publish images' - uses: docker/build-push-action@v5.3.0 + uses: docker/build-push-action@v5.4.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 4e648e32480..a021e79f6d9 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v5.3.0 + uses: docker/build-push-action@v5.4.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 2f1a38034a1..2b15929b30b 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v5.3.0 + uses: docker/build-push-action@v5.4.0 with: context: . tags: mailserver-testing:ci From 8e32635993d196c2268c7bd97f8f101a66e2e52d Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sun, 16 Jun 2024 00:26:33 +1200 Subject: [PATCH 377/592] docs: Document fix for PROXY protocol with `postscreen` (#4066) --- docs/content/examples/tutorials/mailserver-behind-proxy.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/content/examples/tutorials/mailserver-behind-proxy.md b/docs/content/examples/tutorials/mailserver-behind-proxy.md index 60e4c539c25..0c39c125a3d 100644 --- a/docs/content/examples/tutorials/mailserver-behind-proxy.md +++ b/docs/content/examples/tutorials/mailserver-behind-proxy.md @@ -259,6 +259,12 @@ The below guidance is focused on configuring [Traefik][traefik-web], but the adv postconf -P 12525/inet/postscreen_upstream_proxy_protocol=haproxy 12525/inet/syslog_name=smtp-proxyprotocol ``` + Supporting port 25 with an additional PROXY protocol port will also require a `postfix-main.cf` override line for `postscreen` to work correctly: + + ```cf title="docker-data/dms/config/postfix-main.cf" + postscreen_cache_map = proxy:btree:$data_directory/postscreen_cache + ``` + --- Dovecot is mostly the same as before: From e4aff5531e1eab2721e7161fb2b2497a8a95a714 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 16 Jun 2024 18:36:10 +0200 Subject: [PATCH 378/592] docs: updated `CONTRIBUTORS.md` (#4069) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 264 ++++++++++++++++++++++++------------------------ 1 file changed, 132 insertions(+), 132 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 67bd247a0e4..c4a42c2e3cf 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -387,10 +387,10 @@ Thanks goes to these wonderful people ✨ - - - + + - - - + + - - + + - - - - + + - - + + + + + + + + + - - + + - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - - + + - - + + - - + + - - + + - - + + + - - + + - - + + - - + + - - - - - - - + + - - + + - - + + - - + + - - + + - - + + - - + + + - - - + + + + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - -
- - voordev + + citec
- voordev + citec 		@@ -461,10 +461,10 @@ Thanks goes to these wonderful people ✨ - - Birkenstab + + Starbix
- Birkenstab + Starbix 		@@ -475,21 +475,12 @@ Thanks goes to these wonderful people ✨ - - Starbix -
- Starbix - - 		- - citec + + Birkenstab
- citec + Birkenstab
yajo @@ -497,25 +488,27 @@ Thanks goes to these wonderful people ✨ yajo
- - weo + + voordev
- weo + voordev 		- - analogue + + MakerMatrix
- analogue + MakerMatrix 		- - Rubytastic2 + + pbek
- Rubytastic2 + pbek 		@@ -526,40 +519,33 @@ Thanks goes to these wonderful people ✨ - - pbek -
- pbek - -
- - MakerMatrix + + Rubytastic2
- MakerMatrix + Rubytastic2 		- - mpanneck + + analogue
- mpanneck + analogue
- - keslerm + + weo
- keslerm + weo 		- - willtho89 + + andrewlow
- willtho89 + andrewlow 		@@ -576,8 +562,6 @@ Thanks goes to these wonderful people ✨ p-fruck
rahilarious @@ -592,6 +576,8 @@ Thanks goes to these wonderful people ✨ Rillke
bobbravo2 @@ -606,28 +592,12 @@ Thanks goes to these wonderful people ✨ r-pufky - - yogo1212 -
- yogo1212 - - 		vincentDcmps
vincentDcmps -
- - tbutter -
- tbutter - 		@@ -650,6 +620,8 @@ Thanks goes to these wonderful people ✨ engelant
j-marz @@ -664,8 +636,13 @@ Thanks goes to these wonderful people ✨ lokipo
+ + mmehnert +
+ mmehnert + + 		msheakoski @@ -687,11 +664,34 @@ Thanks goes to these wonderful people ✨ frugan-dev
- - andrewlow + + tbutter
- andrewlow + tbutter + + 		+ + yogo1212 +
+ yogo1212 + + 		+ + willtho89 +
+ willtho89 + + 		+ + mpanneck +
+ mpanneck 		@@ -768,6 +768,13 @@ Thanks goes to these wonderful people ✨ emazzotta + + keslerm +
+ keslerm + + 		nueaf @@ -789,6 +796,8 @@ Thanks goes to these wonderful people ✨ artonge
spacecowboy @@ -796,8 +805,6 @@ Thanks goes to these wonderful people ✨ spacecowboy
jiriks74 @@ -833,6 +840,8 @@ Thanks goes to these wonderful people ✨ H4R0
ipernet @@ -840,8 +849,6 @@ Thanks goes to these wonderful people ✨ ipernet
fl42 @@ -849,6 +856,13 @@ Thanks goes to these wonderful people ✨ fl42 + + simonsystem +
+ simonsystem + + 		stephan-devop @@ -870,6 +884,8 @@ Thanks goes to these wonderful people ✨ 5ven
syl20bnr @@ -884,8 +900,6 @@ Thanks goes to these wonderful people ✨ sylvaindumont
TechnicLab @@ -914,6 +928,8 @@ Thanks goes to these wonderful people ✨ tobiabocchi
tknecht @@ -928,8 +944,6 @@ Thanks goes to these wonderful people ✨ tweibert
torus @@ -958,6 +972,8 @@ Thanks goes to these wonderful people ✨ k3it
Drakulix @@ -972,8 +988,6 @@ Thanks goes to these wonderful people ✨ vilisas
forzagreen @@ -989,10 +1003,10 @@ Thanks goes to these wonderful people ✨ - - ShiriNmi1520 + + arkanovicz
- ShiriNmi1520 + arkanovicz 		@@ -1002,6 +1016,8 @@ Thanks goes to these wonderful people ✨ neuralp
radicand @@ -1016,8 +1032,6 @@ Thanks goes to these wonderful people ✨ nilshoell
nknapp @@ -1046,6 +1060,8 @@ Thanks goes to these wonderful people ✨ ovidiucp
mrPjer @@ -1060,8 +1076,6 @@ Thanks goes to these wonderful people ✨ p3dda
peter-hartmann @@ -1090,6 +1104,8 @@ Thanks goes to these wonderful people ✨ robbertkl
romansey @@ -1104,8 +1120,6 @@ Thanks goes to these wonderful people ✨ norrs
MightySCollins @@ -1134,6 +1148,8 @@ Thanks goes to these wonderful people ✨ svdb0
3ap @@ -1148,8 +1164,6 @@ Thanks goes to these wonderful people ✨ shyim
sjmudd @@ -1158,10 +1172,10 @@ Thanks goes to these wonderful people ✨ - - simonsystem + + matrixes
- simonsystem + matrixes 		@@ -1178,13 +1192,8 @@ Thanks goes to these wonderful people ✨ millerjason - - mmehnert -
- mmehnert - -
mplx @@ -1192,8 +1201,6 @@ Thanks goes to these wonderful people ✨ mplx
odinis @@ -1229,6 +1236,8 @@ Thanks goes to these wonderful people ✨ pravynandas
presocratics @@ -1236,8 +1245,6 @@ Thanks goes to these wonderful people ✨ presocratics
rhyst @@ -1273,6 +1280,8 @@ Thanks goes to these wonderful people ✨ smargold476
sportshead @@ -1280,8 +1289,6 @@ Thanks goes to these wonderful people ✨ sportshead
squash @@ -1317,6 +1324,8 @@ Thanks goes to these wonderful people ✨ wligtenberg
wolkenschieber @@ -1324,8 +1333,6 @@ Thanks goes to these wonderful people ✨ wolkenschieber
worldworm @@ -1333,6 +1340,13 @@ Thanks goes to these wonderful people ✨ worldworm + + ShiriNmi1520 +
+ ShiriNmi1520 + + 		Zepmann @@ -1354,6 +1368,8 @@ Thanks goes to these wonderful people ✨ arcaine2
awb99 @@ -1368,8 +1384,6 @@ Thanks goes to these wonderful people ✨ brainkiller
cternes @@ -1398,6 +1412,8 @@ Thanks goes to these wonderful people ✨ eleith
fanqiaojun @@ -1412,8 +1428,6 @@ Thanks goes to these wonderful people ✨ ghnp5
helmutundarnold @@ -1442,6 +1456,8 @@ Thanks goes to these wonderful people ✨ idaadi
ixeft @@ -1456,8 +1472,6 @@ Thanks goes to these wonderful people ✨ jjtt
paralax @@ -1486,6 +1500,8 @@ Thanks goes to these wonderful people ✨ callmemagnus
marios88 @@ -1493,22 +1509,6 @@ Thanks goes to these wonderful people ✨ marios88 - - matrixes -
- matrixes - -
- - arkanovicz -
- arkanovicz - - 		CBeerta From 40aab6bd1808c521db1e193e970f267ba16ace2f Mon Sep 17 00:00:00 2001 From: beertje44 <101147154+beertje44@users.noreply.github.com> Date: Tue, 18 Jun 2024 08:46:34 +0200 Subject: [PATCH 379/592] docs: Add tutorial for configuring Dovecot FTS with Solr (#4070) Describe how to use Apache Solr as a Dovecot FTS backend. --------- Co-authored-by: Casper Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .../examples/tutorials/dovecot-solr.md | 153 ++++++++++++++++++ docs/mkdocs.yml | 1 + 2 files changed, 154 insertions(+) create mode 100644 docs/content/examples/tutorials/dovecot-solr.md diff --git a/docs/content/examples/tutorials/dovecot-solr.md b/docs/content/examples/tutorials/dovecot-solr.md new file mode 100644 index 00000000000..95287c5fd6d --- /dev/null +++ b/docs/content/examples/tutorials/dovecot-solr.md @@ -0,0 +1,153 @@ +# Dovecot Full Text Search (FTS) using the Solr Backend + +Dovecot supports several FTS backends for providing fast and efficient full text searching of e-mails directly from the IMAP server. + +As the size of your mail storage grows, the benefits of FTS are especially notable: + +- Without FTS, Dovecot would perform a search query by checking each individual email stored for a match, and then repeat this process again from scratch for the exact same query in future. +- Some mail clients (_like Thunderbird_) may provide their own indexing and search features when all mail to search is stored locally, otherwise Dovecot needs to handle the search query (_for example webmail and mobile clients, like Gmail_). +- FTS indexes each mail into a database for querying instead, where it can skip the cost of inspecting irrelevant emails for a query. + +!!! warning "This is a community contributed guide" + + It extends [our official docs for Dovecot FTS][docs::dovecot::full-text-search] with a focus on Apache Solr. DMS does not officially support this integration. + +## Setup Solr for DMS + +An FTS backend supported by Dovecot is [Apache Solr][github-solr], a fast and efficient multi-purpose search indexer. + +### Add the required `dovecot-solr` package + +As the official DMS image does not provide `dovecot-solr`, you'll need to include the package in your own image (_extending a DMS release as a base image_), or via our [`user-patches.sh` feature][docs::user-patches]: + + +!!! quote "" + + === "`user-patches.sh`" + + If you'd prefer to avoid a custom image build. This approach is simpler but with the caveat that any time the container is restarted, you'll have a delay as the package is installed each time. + + ```bash + #!/bin/bash + + apt-get update && apt-get install dovecot-solr + ``` + + === "`compose.yaml`" + + A custom DMS image does not add much friction. You do not need a separate `Dockerfile` as Docker Compose supports building from an inline `Dockerfile` in your `compose.yaml`. + + The `image` key of the service is swapped for the `build` key instead, as shown below: + + ```yaml + services: + mailserver: + hostname: mail.example.com + # Do not use `image` anymore, unless referring to the tagged image build below + # Add this `build` section to your real `compose.yaml` for your DMS service: + build: + tags: + - local/dms:14.0 + dockerfile_inline: | + FROM docker.io/mailserver/docker-mailserver:14.0 + RUN apt-get update && apt-get install dovecot-solr + ``` + + - Just run `docker compose up` and it will pull DMS and build your custom image to run a container. + - Updating to a new DMS release is straight-forward, just adjust the version tag as you normally would. If you make future changes that don't apply, you may need to force a rebuild. + - This approach only needs to install the package once with the image build itself. This minimizes delay of container startup. + +!!! note "Why doesn't DMS include `dovecot-solr`?" + + This integration is not officially supported in DMS as no maintainer is able to provide troubleshooting support. + + Prior to v14, the package was included but the community contributed guide had been outdated for several years that it was non-functional. It was decided that it was better to drop support and docs, however some DMS users voiced active use of Solr and it's benefits over Xapian for FTS which led to these revised docs. + + **ARM64 builds do not have support for `dovecot-solr`**. Additionally the [user demand for including `dovecot-solr` is presently too low][gh-dms::feature-request::dovecot-solr-package] to justify vs the minimal effort to add additional packages as shown above. + +### `compose.yaml` config + +Firstly you need a working Solr container, for this the [official docker image][dockerhub-solr] will do: + +```yaml +services: + solr: + image: solr:latest + container_name: dms-solr + environment: + # As Solr can be quite resource hungry, raise the memory limit to 2GB. + # The default is 512MB, which may be exhausted quickly. + SOLR_JAVA_MEM: "-Xms2g -Xmx2g" + volumes: + - ./docker-data/solr:/var/solr + restart: always +``` + +DMS will connect internally to the `solr` service above. Either have both services in the same `compose.yaml` file, or ensure that the containers are connected to the same docker network. + +### Configure Solr for Dovecot + +1. Once the Solr container is started, you need to configure a "Solr core" for Dovecot: + + ```bash + docker exec -it dms-solr /bin/sh + solr create -c dovecot + cp -R /opt/solr/contrib/analysis-extras/lib /var/solr/data/dovecot + ``` + + Stop the `dms-solr` container and you should now have a `./data/dovecot` folder in the local bind mount volume. + +2. Solr needs a schema that is specifically tailored for Dovecot FTS. + + As of writing of this guide, Solr 9 is the current release. [Dovecot provides the required schema configs][github-dovecot::core-docs] for Solr, copy the following two v9 config files to `./data/dovecot` and rename them accordingly: + + - `solr-config-9.xml` (_rename to `solrconfig.xml`_) + - `solr-schema-9.xml` (_rename to `schema.xml`_) + + Additionally, remove the `managed-schema.xml` file from `./data/dovecot` and ensure the two files you copied have a [UID and GID of `8983`][dockerfile-solr-uidgid] assigned. + + Start the Solr container once again, you should now have a working Solr core specifically for Dovecot FTS. + +3. Configure Dovecot in DMS to connect to this Solr core: + + Create a `10-plugin.conf` file in your `./config/dovecot` folder with this contents: + + ```config + mail_plugins = $mail_plugins fts fts_solr + + plugin { + fts = solr + fts_autoindex = yes + fts_solr = url=http://dms-solr:8983/solr/dovecot/ + } + ``` + + Add a volume mount for that config to your DMS service in `compose.yaml`: + + ```yaml + services: + mailserver: + volumes: + - ./docker-data/config/dovecot/10-plugin.conf:/etc/dovecot/conf.d/10-plugin.conf:ro + ``` + +### Trigger Dovecot FTS indexing + +After following the previous steps, restart DMS and run this command to have Dovecot re-index all mail: + +```bash +docker compose exec mailserver doveadm fts rescan -A +``` + +!!! info "Indexing will take a while depending on how large your mail folders" + + Usually within 15 minutes or so, you should be able to search your mail using the Dovecot FTS feature! :tada: + +[docs::user-patches]: ../../config/advanced/override-defaults/user-patches.md +[docs::dovecot::full-text-search]: ../../config/advanced/full-text-search.md +[gh-dms::feature-request::dovecot-solr-package]: https://github.com/docker-mailserver/docker-mailserver/issues/4052 + +[dockerhub-solr]: https://hub.docker.com/_/solr +[dockerfile-solr-uidgid]: https://github.com/apache/solr-docker/blob/9cd850b72309de05169544395c83a85b329d6b86/9.6/Dockerfile#L89-L92 +[github-solr]: https://github.com/apache/solr +[github-dovecot::core-docs]: https://github.com/dovecot/core/tree/main/doc diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 592634436d6..b04631d206c 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -174,6 +174,7 @@ nav: - 'Crowdsec': examples/tutorials/crowdsec.md - 'Building your own Docker image': examples/tutorials/docker-build.md - 'Blog Posts': examples/tutorials/blog-posts.md + - 'Dovecot FTS with Apache Solr': examples/tutorials/dovecot-solr.md - 'Use Cases': - 'Forward-Only Mail-Server with LDAP': examples/use-cases/forward-only-mailserver-with-ldap-authentication.md - 'Customize IMAP Folders': examples/use-cases/imap-folders.md From 98cbcfc17187805131f0efc66af50cd2a7a7f62b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Jun 2024 22:30:24 +0000 Subject: [PATCH 380/592] chore(deps): Bump docker/build-push-action from 5.4.0 to 6.0.0 (#4074) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Casper --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 7f1b7c98777..0584238c49d 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v5.4.0 + uses: docker/build-push-action@v6.0.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index bd54c42877c..db6fbd0cf26 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -67,7 +67,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: 'Build and publish images' - uses: docker/build-push-action@v5.4.0 + uses: docker/build-push-action@v6.0.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index a021e79f6d9..46a294e692e 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v5.4.0 + uses: docker/build-push-action@v6.0.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 2b15929b30b..aaeb5849f2c 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v5.4.0 + uses: docker/build-push-action@v6.0.0 with: context: . tags: mailserver-testing:ci From d7dab2d20d25abecf42dc0e213b408465b38b3d8 Mon Sep 17 00:00:00 2001 From: Casper Date: Wed, 19 Jun 2024 08:10:00 +0200 Subject: [PATCH 381/592] feat: Add password confirmation (#4072) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- CHANGELOG.md | 5 +++++ target/bin/adddovecotmasteruser | 3 +-- target/bin/addmailuser | 3 +-- target/bin/addsaslpassword | 3 +-- target/bin/updatedovecotmasteruser | 3 +-- target/bin/updatemailuser | 3 +-- target/scripts/helpers/database/manage/postfix-accounts.sh | 5 +++++ 7 files changed, 15 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1aa3391e249..2205ed9b598 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Added + +- **Internal:** + - Add password confirmation to several `setup.sh` commands ([#4072](https://github.com/docker-mailserver/docker-mailserver/pull/4072)) + ### Updates - **Fail2ban**: diff --git a/target/bin/adddovecotmasteruser b/target/bin/adddovecotmasteruser index 41041db2e3b..d327478d8ed 100755 --- a/target/bin/adddovecotmasteruser +++ b/target/bin/adddovecotmasteruser @@ -7,8 +7,7 @@ function _main() { _require_n_parameters_or_print_usage 1 "${@}" local MAIL_ACCOUNT="${1}" - shift - local PASSWD="${*}" + local PASSWD="${2}" _manage_accounts_dovecotmaster_create "${MAIL_ACCOUNT}" "${PASSWD}" } diff --git a/target/bin/addmailuser b/target/bin/addmailuser index 15691c89872..0eee9d8287f 100755 --- a/target/bin/addmailuser +++ b/target/bin/addmailuser @@ -7,8 +7,7 @@ function _main() { _require_n_parameters_or_print_usage 1 "${@}" local MAIL_ACCOUNT="${1}" - shift - local PASSWD="${*}" + local PASSWD="${2}" _manage_accounts_create "${MAIL_ACCOUNT}" "${PASSWD}" diff --git a/target/bin/addsaslpassword b/target/bin/addsaslpassword index 9461c112e22..7cb82dc2af1 100755 --- a/target/bin/addsaslpassword +++ b/target/bin/addsaslpassword @@ -8,8 +8,7 @@ function _main() { local DOMAIN="${1}" local RELAY_ACCOUNT="${2}" - shift 2 - local PASSWD="${*}" + local PASSWD="${3}" _validate_parameters _add_relayhost_credentials diff --git a/target/bin/updatedovecotmasteruser b/target/bin/updatedovecotmasteruser index eeff21a591d..cc1ae9309a9 100755 --- a/target/bin/updatedovecotmasteruser +++ b/target/bin/updatedovecotmasteruser @@ -7,8 +7,7 @@ function _main() { _require_n_parameters_or_print_usage 1 "${@}" local MAIL_ACCOUNT="${1}" - shift - local PASSWD="${*}" + local PASSWD="${2}" _manage_accounts_dovecotmaster_update "${MAIL_ACCOUNT}" "${PASSWD}" } diff --git a/target/bin/updatemailuser b/target/bin/updatemailuser index ce45533e37b..be1b981cc98 100755 --- a/target/bin/updatemailuser +++ b/target/bin/updatemailuser @@ -7,8 +7,7 @@ function _main() { _require_n_parameters_or_print_usage 1 "${@}" local MAIL_ACCOUNT="${1}" - shift - local PASSWD="${*}" + local PASSWD="${2}" _manage_accounts_update "${MAIL_ACCOUNT}" "${PASSWD}" } diff --git a/target/scripts/helpers/database/manage/postfix-accounts.sh b/target/scripts/helpers/database/manage/postfix-accounts.sh index 987254fca9d..e92b2555ffd 100644 --- a/target/scripts/helpers/database/manage/postfix-accounts.sh +++ b/target/scripts/helpers/database/manage/postfix-accounts.sh @@ -98,9 +98,14 @@ function __account_already_exists() { # Also used by addsaslpassword function _password_request_if_missing() { + local PASSWD_CONFIRM if [[ -z ${PASSWD} ]]; then read -r -s -p 'Enter Password: ' PASSWD echo [[ -z ${PASSWD} ]] && _exit_with_error 'Password must not be empty' + + read -r -s -p 'Confirm Password: ' PASSWD_CONFIRM + echo + [[ ${PASSWD} != "${PASSWD_CONFIRM}" ]] && _exit_with_error 'Passwords do not match!' fi } From 8a082be714d165e954f0ad05e65810d706c6b4ec Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 19 Jun 2024 20:39:06 +1200 Subject: [PATCH 382/592] docs: Add info regarding DKIM key rotation and non-expiry (#4076) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Co-authored-by: Casper --- .../config/best-practices/dkim_dmarc_spf.md | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md index d035556f123..d6ba06b7e7e 100644 --- a/docs/content/config/best-practices/dkim_dmarc_spf.md +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -33,6 +33,21 @@ When DKIM is enabled: DKIM requires a public/private key pair to enable **signing (_via private key_)** your outgoing mail, while the receiving end must query DNS to **verify (_via public key_)** that the signature is trustworthy. +??? info "Verification expiry" + + Unlike your TLS certificate, your DKIM keypair does not have a fixed expiry associated to it. + + + Instead, an expiry may be included in your DKIM signature for each mail sent, where a receiver will [refuse to validate the signature for an email after that expiry date][dkim-verification-expiry-refusal]. This is an added precaution to mitigate malicious activity like "DKIM replay attacks", where an already delivered email from a third-party with a trustworthy DKIM signature is leveraged by a spammer when sending mail to an MTA which verifies the DKIM signature successfully, enabling the spammer to bypass spam protections. + + Unlike a TLS handshake where you are authenticating trust with future communications, with DKIM once the mail has been received and trust of the signature has been verified, the value of verifying the signature again at a later date is less meaningful since the signature was to ensure no tampering had occurred during delivery through the network. + +??? tip "DKIM key rotation" + + You can rotate your DKIM keypair by switching to a new DKIM selector (_and DNS updates_), while the previous key and selector remains valid for verification until the last mail signed with that key reaches it's expiry. + + DMS does not provide any automation or support for key rotation, [nor is it likely to provide a notable security benefit][gh-discussion::dkim-key-rotation-expiry] to the typical small scale DMS deployment. + ### Generating Keys You'll need to repeat this process if you add any new domains. @@ -72,7 +87,7 @@ You should have: According to [RFC 8301][rfc-8301], keys are preferably between 1024 and 2048 bits. Keys of size 4096-bit or larger may not be compatible to all systems your mail is intended for. - You [should not need a key length beyond 2048-bit][github-issue-dkimlength]. If 2048-bit does not meet your security needs, you may want to instead consider adopting key rotation or switching from RSA to ECC keys for DKIM. + You [should not need a key length beyond 2048-bit][gh-issue::dkim-length]. If 2048-bit does not meet your security needs, you may want to instead consider adopting key rotation or switching from RSA to ECC keys for DKIM. ??? note "You may need to specify mail domains explicitly" @@ -352,7 +367,8 @@ volumes: [docs-rspamd-config-dropin]: ../security/rspamd.md#manually [cloudflare-dkim-dmarc-spf]: https://www.cloudflare.com/learning/email-security/dmarc-dkim-spf/ [rfc-8301]: https://datatracker.ietf.org/doc/html/rfc8301#section-3.2 -[github-issue-dkimlength]: https://github.com/docker-mailserver/docker-mailserver/issues/1854#issuecomment-806280929 +[gh-discussion::dkim-key-rotation-expiry]: https://github.com/orgs/docker-mailserver/discussions/4068#discussioncomment-9784263 +[gh-issue::dkim-length]: https://github.com/docker-mailserver/docker-mailserver/issues/1854#issuecomment-806280929 [rspamd-docs-dkim-checks]: https://www.rspamd.com/doc/modules/dkim.html [rspamd-docs-dkim-signing]: https://www.rspamd.com/doc/modules/dkim_signing.html [dns::example-webui]: https://www.vultr.com/docs/introduction-to-vultr-dns/ @@ -360,6 +376,7 @@ volumes: [dns::wikipedia-zonefile]: https://en.wikipedia.org/wiki/Zone_file [dns::webui-dkim]: https://serverfault.com/questions/763815/route-53-doesnt-allow-adding-dkim-keys-because-length-is-too-long [dkim-ed25519-support]: https://serverfault.com/questions/1023674/is-ed25519-well-supported-for-the-dkim-validation/1074545#1074545 +[dkim-verification-expiry-refusal]: https://mxtoolbox.com/problem/dkim/dkim-signature-expiration [mxtoolbox-dkim-verifier]: https://mxtoolbox.com/dkim.aspx [dmarc-howto-configtags]: https://github.com/internetstandards/toolbox-wiki/blob/master/DMARC-how-to.md#overview-of-dmarc-configuration-tags [dmarc-tool-gca]: https://dmarcguide.globalcyberalliance.org From e370c0c96a28acfbd66b20ffac327528eda78104 Mon Sep 17 00:00:00 2001 From: Casper Date: Wed, 19 Jun 2024 18:34:18 +0200 Subject: [PATCH 383/592] fail2ban install: remove -k (--insecure) from curl options (#4080) --- CHANGELOG.md | 4 ++++ target/scripts/build/packages.sh | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2205ed9b598..832e25d2a03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Security + +- **Fail2ban**: + - Ensure a secure connection, when downloading the fail2ban package ([#4080](https://github.com/docker-mailserver/docker-mailserver/pull/4080)) ### Added - **Internal:** diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 004c3b8c137..4469f508f1b 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -189,8 +189,8 @@ function _install_fail2ban() { gpg --keyserver "${FAIL2BAN_GPG_PUBLIC_KEY_SERVER}" --recv-keys "${FAIL2BAN_GPG_PUBLIC_KEY_ID}" 2>&1 - curl -Lkso fail2ban.deb "${FAIL2BAN_DEB_URL}" - curl -Lkso fail2ban.deb.asc "${FAIL2BAN_DEB_ASC_URL}" + curl -fsSLo fail2ban.deb "${FAIL2BAN_DEB_URL}" + curl -fsSLo fail2ban.deb.asc "${FAIL2BAN_DEB_ASC_URL}" FINGERPRINT=$(LANG=C gpg --verify fail2ban.deb.asc fail2ban.deb |& sed -n 's#Primary key fingerprint: \(.*\)#\1#p') From ccaa02b8b5f1841b127fe51ac6efe30620173ba6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Jun 2024 22:43:10 +0200 Subject: [PATCH 384/592] chore(deps): Bump docker/build-push-action from 6.0.0 to 6.1.0 (#4086) --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 0584238c49d..ac87d337282 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v6.0.0 + uses: docker/build-push-action@v6.1.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index db6fbd0cf26..b575249b44b 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -67,7 +67,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: 'Build and publish images' - uses: docker/build-push-action@v6.0.0 + uses: docker/build-push-action@v6.1.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 46a294e692e..52d3d00d5bc 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.0.0 + uses: docker/build-push-action@v6.1.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index aaeb5849f2c..b2f74a44b8f 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.0.0 + uses: docker/build-push-action@v6.1.0 with: context: . tags: mailserver-testing:ci From 5a4a136ec5631f9d3bffaf5c40eed571c30da0f1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 29 Jun 2024 12:44:00 +0200 Subject: [PATCH 385/592] docs: updated `CONTRIBUTORS.md` (#4084) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTORS.md | 133 +++++++++++++++++++++++++----------------------- 1 file changed, 70 insertions(+), 63 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index c4a42c2e3cf..9a270918644 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -856,13 +856,6 @@ Thanks goes to these wonderful people ✨ fl42 - - simonsystem -
- simonsystem - - 		stephan-devop @@ -884,8 +877,6 @@ Thanks goes to these wonderful people ✨ 5ven
syl20bnr @@ -893,6 +884,8 @@ Thanks goes to these wonderful people ✨ syl20bnr
sylvaindumont @@ -928,8 +921,6 @@ Thanks goes to these wonderful people ✨ tobiabocchi
tknecht @@ -937,6 +928,8 @@ Thanks goes to these wonderful people ✨ tknecht
tweibert @@ -972,8 +965,6 @@ Thanks goes to these wonderful people ✨ k3it
Drakulix @@ -981,6 +972,8 @@ Thanks goes to these wonderful people ✨ Drakulix
vilisas @@ -1003,10 +996,10 @@ Thanks goes to these wonderful people ✨ - - arkanovicz + + ShiriNmi1520
- arkanovicz + ShiriNmi1520 		@@ -1016,8 +1009,6 @@ Thanks goes to these wonderful people ✨ neuralp
radicand @@ -1025,6 +1016,8 @@ Thanks goes to these wonderful people ✨ radicand
nilshoell @@ -1060,8 +1053,6 @@ Thanks goes to these wonderful people ✨ ovidiucp
mrPjer @@ -1069,6 +1060,8 @@ Thanks goes to these wonderful people ✨ mrPjer
p3dda @@ -1104,8 +1097,6 @@ Thanks goes to these wonderful people ✨ robbertkl
romansey @@ -1113,6 +1104,8 @@ Thanks goes to these wonderful people ✨ romansey
norrs @@ -1148,8 +1141,6 @@ Thanks goes to these wonderful people ✨ svdb0
3ap @@ -1157,6 +1148,8 @@ Thanks goes to these wonderful people ✨ 3ap
shyim @@ -1171,6 +1164,13 @@ Thanks goes to these wonderful people ✨ sjmudd + + simonsystem +
+ simonsystem + + 		matrixes @@ -1340,13 +1340,6 @@ Thanks goes to these wonderful people ✨ worldworm - - ShiriNmi1520 -
- ShiriNmi1520 - - 		Zepmann @@ -1368,14 +1361,21 @@ Thanks goes to these wonderful people ✨ arcaine2
awb99
awb99 +
+ + beertje44 +
+ beertje44 + 		@@ -1509,6 +1509,13 @@ Thanks goes to these wonderful people ✨ marios88 + + arkanovicz +
+ arkanovicz + + 		CBeerta @@ -1537,6 +1544,8 @@ Thanks goes to these wonderful people ✨ dkarski
dbellavista @@ -1544,8 +1553,6 @@ Thanks goes to these wonderful people ✨ dbellavista
danielvandenberg95 @@ -1581,6 +1588,8 @@ Thanks goes to these wonderful people ✨ doominator42
aydodo @@ -1588,8 +1597,6 @@ Thanks goes to these wonderful people ✨ aydodo
vedtam @@ -1625,6 +1632,8 @@ Thanks goes to these wonderful people ✨ ErikEngerd
huncode @@ -1632,8 +1641,6 @@ Thanks goes to these wonderful people ✨ huncode
gitfeber @@ -1669,6 +1676,8 @@ Thanks goes to these wonderful people ✨ JOduMonT
0xflotus @@ -1676,8 +1685,6 @@ Thanks goes to these wonderful people ✨ 0xflotus
ifokeev @@ -1713,6 +1720,8 @@ Thanks goes to these wonderful people ✨ aspettl
acch @@ -1720,8 +1729,6 @@ Thanks goes to these wonderful people ✨ acch
vifino @@ -1757,6 +1764,8 @@ Thanks goes to these wonderful people ✨ eglia
groupmsl @@ -1764,8 +1773,6 @@ Thanks goes to these wonderful people ✨ groupmsl
green-anger @@ -1801,6 +1808,8 @@ Thanks goes to these wonderful people ✨ astrocket
baxerus @@ -1808,8 +1817,6 @@ Thanks goes to these wonderful people ✨ baxerus
spock @@ -1845,6 +1852,8 @@ Thanks goes to these wonderful people ✨ Kaan88
akkumar @@ -1852,8 +1861,6 @@ Thanks goes to these wonderful people ✨ akkumar
thechubbypanda @@ -1889,6 +1896,8 @@ Thanks goes to these wonderful people ✨ JustAnother1
leowinterde @@ -1896,8 +1905,6 @@ Thanks goes to these wonderful people ✨ leowinterde
linhandev @@ -1933,6 +1940,8 @@ Thanks goes to these wonderful people ✨ madmath03
maxemann96 @@ -1940,8 +1949,6 @@ Thanks goes to these wonderful people ✨ maxemann96
dragetd @@ -1977,6 +1984,8 @@ Thanks goes to these wonderful people ✨ mcchots
MohammedNoureldin @@ -1984,8 +1993,6 @@ Thanks goes to these wonderful people ✨ MohammedNoureldin
mpldr @@ -2021,6 +2028,8 @@ Thanks goes to these wonderful people ✨ Marsu31
glandais @@ -2028,8 +2037,6 @@ Thanks goes to these wonderful people ✨ glandais
GiovanH @@ -2065,6 +2072,8 @@ Thanks goes to these wonderful people ✨ sirgantrithon
Influencer @@ -2072,8 +2081,6 @@ Thanks goes to these wonderful people ✨ Influencer
in-seo @@ -2109,6 +2116,8 @@ Thanks goes to these wonderful people ✨ jcalfee
mivek @@ -2116,8 +2125,6 @@ Thanks goes to these wonderful people ✨ mivek
init-js @@ -2153,6 +2160,8 @@ Thanks goes to these wonderful people ✨ jirislav
jmccl @@ -2160,8 +2169,6 @@ Thanks goes to these wonderful people ✨ jmccl
jurekbarth From 22383c28e7cf6d38b674f8a0d28ff522bbad3929 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 30 Jun 2024 11:34:38 +0200 Subject: [PATCH 386/592] CI: Remove reviewer assignment from Dependabot config (#4088) --- .github/dependabot.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 99a0fb33e4a..327014c57d5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,8 +4,6 @@ updates: directory: "/" schedule: interval: "weekly" - reviewers: - - "docker-mailserver/maintainers" labels: - "area/ci" - "kind/update" @@ -15,8 +13,6 @@ updates: directory: / schedule: interval: "weekly" - reviewers: - - "docker-mailserver/maintainers" labels: - "area/ci" - "kind/update" From b3a5e9e4e8eb0de311940c86d233c66fa3d72354 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Jul 2024 12:17:19 +1200 Subject: [PATCH 387/592] chore(deps): Bump docker/build-push-action from 6.1.0 to 6.2.0 (#4089) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.1.0 to 6.2.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v6.1.0...v6.2.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index ac87d337282..51d300e4a93 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v6.1.0 + uses: docker/build-push-action@v6.2.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index b575249b44b..7b7c4fe5b30 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -67,7 +67,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: 'Build and publish images' - uses: docker/build-push-action@v6.1.0 + uses: docker/build-push-action@v6.2.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 52d3d00d5bc..03bebcc96b6 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.1.0 + uses: docker/build-push-action@v6.2.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index b2f74a44b8f..5f7a1014882 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.1.0 + uses: docker/build-push-action@v6.2.0 with: context: . tags: mailserver-testing:ci From 9175424d0f3c67622c030311cc720d7477f1c2b6 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 3 Jul 2024 22:16:02 +1200 Subject: [PATCH 388/592] fix: Update `dovecot-fts-xapian` to `1.7.13` (#4095) * fix: Update `dovecot-fts-xapian` to `1.7.13` Contains a fix to a regression introduced that broke indexing --------- Co-authored-by: casperklein --- CHANGELOG.md | 3 ++- Dockerfile | 4 ++-- target/scripts/build/compile.sh | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 832e25d2a03..2d9e6ceb8d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,8 @@ All notable changes to this project will be documented in this file. The format #### Fixes - **Dovecot:** - - `logwatch` Update logwatch `ignore.conf` to exclude Xapian messages about pending documents + - `logwatch` Update logwatch `ignore.conf` to exclude Xapian messages about pending documents ([#4060](https://github.com/docker-mailserver/docker-mailserver/pull/4060)) + - `dovecot-fts-xapian` plugin was updated to `1.7.13`, fixing a regression with indexing ([#4095](https://github.com/docker-mailserver/docker-mailserver/pull/4095)) ## [v14.0.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v14.0.0) diff --git a/Dockerfile b/Dockerfile index da9aa83da24..f6199fd4d32 100644 --- a/Dockerfile +++ b/Dockerfile @@ -82,8 +82,8 @@ EOF # install fts_xapian plugin -COPY --from=stage-compile dovecot-fts-xapian-1.7.12_1.7.12_*.deb / -RUN dpkg -i /dovecot-fts-xapian-1.7.12_1.7.12_*.deb && rm /dovecot-fts-xapian-1.7.12_1.7.12_*.deb +COPY --from=stage-compile dovecot-fts-xapian-*.deb / +RUN dpkg -i /dovecot-fts-xapian-*.deb && rm /dovecot-fts-xapian-*.deb COPY target/dovecot/*.inc target/dovecot/*.conf /etc/dovecot/conf.d/ COPY target/dovecot/dovecot-purge.cron /etc/cron.d/dovecot-purge.disabled diff --git a/target/scripts/build/compile.sh b/target/scripts/build/compile.sh index 4c6fba82426..299ba7c8de2 100644 --- a/target/scripts/build/compile.sh +++ b/target/scripts/build/compile.sh @@ -16,7 +16,7 @@ function _compile_dovecot_fts_xapian() { apt-get "${QUIET}" --no-install-recommends install \ automake libtool pkg-config libicu-dev libsqlite3-dev libxapian-dev make build-essential dh-make devscripts dovecot-dev - local XAPIAN_VERSION='1.7.12' + local XAPIAN_VERSION='1.7.13' curl -sSfL -o dovecot-fts-xapian.tar.gz \ "https://github.com/grosjo/fts-xapian/releases/download/${XAPIAN_VERSION}/dovecot-fts-xapian-${XAPIAN_VERSION}.tar.gz" tar xf dovecot-fts-xapian.tar.gz From 2d12bbb7fd410393d0b5419f86e91287613d613c Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 5 Jul 2024 23:48:42 +1200 Subject: [PATCH 389/592] docs: Update `compose.yaml` for `dovecot-solr` guide (#4099) The `image` field is used for the default tag, if it's not specified Compose will infer one in addition to any extra `tags` provided. Better to use `image` for the tag assignment, and a clear `pull_policy` to prevent trying to pull a remote image of the same name. --- docs/content/examples/tutorials/dovecot-solr.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/content/examples/tutorials/dovecot-solr.md b/docs/content/examples/tutorials/dovecot-solr.md index 95287c5fd6d..be4c91f6577 100644 --- a/docs/content/examples/tutorials/dovecot-solr.md +++ b/docs/content/examples/tutorials/dovecot-solr.md @@ -43,11 +43,12 @@ As the official DMS image does not provide `dovecot-solr`, you'll need to includ services: mailserver: hostname: mail.example.com - # Do not use `image` anymore, unless referring to the tagged image build below + # The `image` setting now represents the tag for the local build configured below: + image: local/dms:14.0 + # Local build (no need to try pull `image` remotely): + pull_policy: build # Add this `build` section to your real `compose.yaml` for your DMS service: build: - tags: - - local/dms:14.0 dockerfile_inline: | FROM docker.io/mailserver/docker-mailserver:14.0 RUN apt-get update && apt-get install dovecot-solr From 19d52d9dcc5489f6dc8ac9aa51f8a224b48e411b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jul 2024 11:23:02 +1200 Subject: [PATCH 390/592] chore(deps): Bump docker/build-push-action from 6.2.0 to 6.3.0 (#4103) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.2.0 to 6.3.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v6.2.0...v6.3.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 51d300e4a93..2938b522d48 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v6.2.0 + uses: docker/build-push-action@v6.3.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 7b7c4fe5b30..11fd1f2e406 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -67,7 +67,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: 'Build and publish images' - uses: docker/build-push-action@v6.2.0 + uses: docker/build-push-action@v6.3.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 03bebcc96b6..4c36b22b943 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.2.0 + uses: docker/build-push-action@v6.3.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 5f7a1014882..b07ae3c3b30 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.2.0 + uses: docker/build-push-action@v6.3.0 with: context: . tags: mailserver-testing:ci From 2cca907615697cbca2ecc20a91f7664e61abfd11 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jul 2024 23:24:48 +0000 Subject: [PATCH 391/592] chore(deps): Bump docker/setup-buildx-action from 3.3.0 to 3.4.0 (#4104) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3.3.0 to 3.4.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v3.3.0...v3.4.0) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 2938b522d48..29d781515c7 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -79,7 +79,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.3.0 + uses: docker/setup-buildx-action@v3.4.0 # NOTE: AMD64 can build within 2 minutes - name: 'Build images' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 11fd1f2e406..68c423c1c09 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -40,7 +40,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.3.0 + uses: docker/setup-buildx-action@v3.4.0 # Try get the cached build layers from a prior `generic_build.yml` job. # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 4c36b22b943..9d7c6902a62 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -38,7 +38,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.3.0 + uses: docker/setup-buildx-action@v3.4.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index b07ae3c3b30..fe290de3b03 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -37,7 +37,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.3.0 + uses: docker/setup-buildx-action@v3.4.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. From 3a40c457fc7903cbf409609b446632e69bf346ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jul 2024 23:26:20 +0000 Subject: [PATCH 392/592] chore(deps): Bump docker/setup-qemu-action from 3.0.0 to 3.1.0 (#4105) Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 3.0.0 to 3.1.0. - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/v3.0.0...v3.1.0) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 29d781515c7..c54826ae8dc 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -74,7 +74,7 @@ jobs: cache-buildx- - name: 'Set up QEMU' - uses: docker/setup-qemu-action@v3.0.0 + uses: docker/setup-qemu-action@v3.1.0 with: platforms: arm64 diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 68c423c1c09..bd6f5fa9930 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -35,7 +35,7 @@ jobs: type=semver,pattern={{major}}.{{minor}}.{{patch}} - name: 'Set up QEMU' - uses: docker/setup-qemu-action@v3.0.0 + uses: docker/setup-qemu-action@v3.1.0 with: platforms: arm64 From 4778f15fdaf5e94aee364396166fdc2821066c46 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 10 Jul 2024 05:44:09 +1200 Subject: [PATCH 393/592] docs: TLS typo fix (#4106) - Caddy admonition - Missing `[` for annotating a link. - Traefik - `docker-compose` => "Docker Compose" --- docs/content/config/security/ssl.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/config/security/ssl.md b/docs/content/config/security/ssl.md index a5ba005ad30..eeb39a5f069 100644 --- a/docs/content/config/security/ssl.md +++ b/docs/content/config/security/ssl.md @@ -567,7 +567,7 @@ DSM-generated letsencrypt certificates get auto-renewed every three months. !!! warning "Caddy certificate location varies" - The path contains the certificate provisioner used. This path may be different from the example above for you and may change over time when multiple provisioner services are used][dms-pr-feedback::caddy-provisioning-gotcha]. + The path contains the certificate provisioner used. This path may be different from the example above for you and may change over time when [multiple ACME provisioner services are used][dms-pr-feedback::caddy-provisioning-gotcha]. This can make the volume mounting for DMS to find the certificates non-deterministic, but you can [restrict provisioning to single service via the `acme_ca` setting][caddy::restrict-acme-provisioner]. @@ -583,7 +583,7 @@ This setup only comes with one caveat - The domain has to be configured on anoth ???+ example "Example Code" - Here is an example setup for [`docker-compose`](https://docs.docker.com/compose/): + Here is an example setup for [`Docker Compose`](https://docs.docker.com/compose/): ```yaml services: From 755540cacf50193780ee6697f6569efa9c8745d1 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sun, 14 Jul 2024 21:24:33 +1200 Subject: [PATCH 394/592] docs: `docker-build.md` - Update `DOVECOT_COMMUNITY_REPO` default (#4111) --- docs/content/examples/tutorials/docker-build.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/examples/tutorials/docker-build.md b/docs/content/examples/tutorials/docker-build.md index 538da822029..63241e67110 100644 --- a/docs/content/examples/tutorials/docker-build.md +++ b/docs/content/examples/tutorials/docker-build.md @@ -32,7 +32,7 @@ We make use of build features that require a recent version of Docker. v23.0 or The `Dockerfile` includes several build [`ARG`][docker-docs::builder-arg] instructions that can be configured: -- `DOVECOT_COMMUNITY_REPO`: Install Dovecot from the community repo instead of from Debian (default = 1) +- `DOVECOT_COMMUNITY_REPO`: Install Dovecot from the community repo instead of from Debian (default = 0) - `DMS_RELEASE`: The image version (default = edge) - `VCS_REVISION`: The git commit hash used for the build (default = unknown) From bf4ebc2a410691af82315605d3b385ba3a28e1c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jul 2024 13:42:20 +1200 Subject: [PATCH 395/592] chore(deps): Bump anchore/scan-action from 3.6.4 to 4.0.0 (#4114) Bumps [anchore/scan-action](https://github.com/anchore/scan-action) from 3.6.4 to 4.0.0. - [Release notes](https://github.com/anchore/scan-action/releases) - [Changelog](https://github.com/anchore/scan-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/anchore/scan-action/compare/v3.6.4...v4.0.0) --- updated-dependencies: - dependency-name: anchore/scan-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index fe290de3b03..1cb5f0ed07d 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v3.6.4 + uses: anchore/scan-action@v4.0.0 id: scan with: image: mailserver-testing:ci From 34423c2f665b8c7839c40fbc7d1fe8cda6342456 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jul 2024 01:44:17 +0000 Subject: [PATCH 396/592] chore(deps): Bump docker/build-push-action from 6.3.0 to 6.4.0 (#4113) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.3.0 to 6.4.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v6.3.0...v6.4.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index c54826ae8dc..5b10ef87de5 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v6.3.0 + uses: docker/build-push-action@v6.4.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index bd6f5fa9930..426009908bb 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -67,7 +67,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: 'Build and publish images' - uses: docker/build-push-action@v6.3.0 + uses: docker/build-push-action@v6.4.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 9d7c6902a62..1cab38b81ec 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.3.0 + uses: docker/build-push-action@v6.4.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 1cb5f0ed07d..95b6c4f6ddb 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.3.0 + uses: docker/build-push-action@v6.4.0 with: context: . tags: mailserver-testing:ci From c5f125c973d65d0f51000061952ec378b7449c6e Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 18 Jul 2024 10:31:50 +1200 Subject: [PATCH 397/592] tests: Update `curl` note for XOAUTH2 support (#4118) --- test/tests/serial/mail_with_oauth2.bats | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/tests/serial/mail_with_oauth2.bats b/test/tests/serial/mail_with_oauth2.bats index c9a02eeaded..968d63abad4 100644 --- a/test/tests/serial/mail_with_oauth2.bats +++ b/test/tests/serial/mail_with_oauth2.bats @@ -59,9 +59,14 @@ function teardown_file() { } @test "should authenticate with XOAUTH2" { - # curl packaged in Debian 12 (and the latest release as of Jan 2024) broke XOAUTH2 support + # curl 7.80.0 (Nov 2021) broke XOAUTH2 support (DMS v14 release with Debian 12 packages curl 7.88.1) # https://github.com/docker-mailserver/docker-mailserver/pull/3403#issuecomment-1907100624 - skip 'unable to test XOAUTH mechanism due to bug since curl 7.80' + # + # Fixed in curl 8.6.0 (Jan 31 2024): + # - https://github.com/curl/curl/issues/10259 + # - https://github.com/curl/curl/commit/7b2d98dfadf209108aa7772ee21ae42e3dab219f (referenced in release changelog by commit title) + # - https://github.com/curl/curl/releases/tag/curl-8_6_0 + skip 'unable to test XOAUTH2 mechanism due to bug in curl versions 7.80.0 --> 8.5.0' __should_login_successfully_with 'XOAUTH2' } From 0698ad9370de780cccd46c0c6ca9113a45afc775 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Mon, 22 Jul 2024 10:00:53 +1200 Subject: [PATCH 398/592] docs: Refactor pages for Account Management (#4122) * docs: Relocate account / auth pages into a common section * docs: Update references to relocated pages * docs: Add account management overview page Updates remaining links to account sections on this page instead (_for `accounts`, `aliases`, `quotas`_). This page will cover the features and defer to separate pages for more specific content where relevant. * docs: Correct relocated pages titles and links * docs: Accounts (Dovecot Master) - Minor revisions * docs: Fix highlighting roundcube PHP snippet in OAuth2 page * docs: Accounts (File) - Refactor - Manual method not necessary to document. - Condense `setup` example guidance. - Quotas / Aliases content migrated to Overview when not specific about file provisioner. Some of the content is this commit is not a complete revision. * chore: Temporary commit * docs(refactor): Sub-addressing section Much better docs on the sub-addressing feature supported by Postfix and Dovecot, along with the guidance with usage in Sieve. * docs: Revise accounts section Add some context regarding DMS accounts and their distinction/overlap from the email address functionality, and it's relevant context for receiving/sending. File provisioner, minor revisions to referencing associated config files and account management. * docs: Minor adjustments * docs: Refactor the quota section Better documented with links and coverage over the workaround details we've implemented. * docs: Revise the quota section Minor revisions with phrasing, admonitions for structure and better explanation of the feature functionality/purpose. * docs: Alias section refactor Extensively covers known issues and technical details that have been discussed often enough. The improvements should benefit both users and maintainers. * docs: Refactor master accounts page This rewrite should more clearly document the feature, along with a better example and additional links for reference. * docs: OAuth2 revision Minor update to this page: - Links extracted to bottom of page as per convention. - ENV file example converted to preferred `compose.yaml` ENV settings. * docs: Sieve minor revisions - Correct link to subaddressing section - Make the config file example snippets intended filename less ambiguous. - Minor rephrasng. * docs: Revise accounts overview section Revised the account section and added additional clarity for common confusion with relation to sender address and multi-domain support. Top of the page now clarifies it's a technical reference and directs users to the related pages for configuration / caveats. Technical Overview links to Dovecot docs were missing. * docs: Another revision pass File based provisioner docs: - Sections indent with info admonitions. - Accounts section expanded with config format and example. - Quotas section expanded and shifted to bottom (alphabetical sort). - Split into `setup` CLI and config reference groups. Overview page: - Sections indent with info admonitions. - Revised content. * docs(chore): Shift sub-addressing section This is related to accounts and aliases, but not provisioners, thus extract out of the accounts parent section. * docs: Document `postfix-accounts.cf` third column This lacked documentation but was community contributed feature to allow further customization of a Dovecot Account. It has caveats as DMS does not take these into consideration anywhere in scripts. Documenting officially for better awareness. * docs: Revise and expand supplementary pages Better outline the OAuth2 login process, the two supported login mechanisms and their docs/rfcs, along with documenting caveat with mail client compatibility. Add a verification tip for the OAuth2 support, showing how `curl` can be used, along with caveat presently affecting the `curl` in DMS v14. Additionally note the feature still isn't documented fully, providing the user with additional references for more information. `ACCOUNT_PROVISIONER` ENV docs minimized. No `OIDC` provisioner plans, the OAuth2 docs page now mentions SCIM 2.0 API as the next step towards resolving that concern. The tip admonition was removed as it no longer provides value, instead we link to the Account Management overview page. Dovecot Master Accounts docs page now lightly document the `setup` CLI and config format for the feature. * docs: Fix broken anchor links Some anchor links to different parts of our docs have gone stale. This branch also broke a few itself that I missed. The build now only reports issues with anchor links to Content Tabs, which it must not be aware of during the build (_MKDocs Material specific feature?_) * docs(lint): Fix indentation level * chore: Add entry to `CHANGELOG.md` + corrections --- CHANGELOG.md | 41 +-- README.md | 4 +- .../config/account-management/overview.md | 252 ++++++++++++++++++ .../account-management/provisioner/file.md | 206 ++++++++++++++ .../provisioner/ldap.md} | 6 +- .../supplementary/master-accounts.md | 70 +++++ .../supplementary/oauth2.md | 145 ++++++++++ docs/content/config/advanced/auth-oauth2.md | 69 ----- .../advanced/dovecot-master-accounts.md | 21 -- docs/content/config/advanced/mail-sieve.md | 96 +++++-- .../config/advanced/optional-config.md | 19 +- docs/content/config/advanced/podman.md | 6 +- .../config/best-practices/dkim_dmarc_spf.md | 2 +- docs/content/config/debugging.md | 2 +- docs/content/config/environment.md | 21 +- docs/content/config/security/fail2ban.md | 2 +- docs/content/config/security/ssl.md | 2 +- .../security/understanding-the-ports.md | 2 +- docs/content/config/user-management.md | 86 ------ .../contributing/issues-and-pull-requests.md | 8 +- .../tutorials/mailserver-behind-proxy.md | 4 +- docs/content/examples/use-cases/auth-lua.md | 2 +- docs/content/usage.md | 2 +- docs/mkdocs.yml | 19 +- mailserver.env | 2 +- 25 files changed, 829 insertions(+), 260 deletions(-) create mode 100644 docs/content/config/account-management/overview.md create mode 100644 docs/content/config/account-management/provisioner/file.md rename docs/content/config/{advanced/auth-ldap.md => account-management/provisioner/ldap.md} (99%) create mode 100644 docs/content/config/account-management/supplementary/master-accounts.md create mode 100644 docs/content/config/account-management/supplementary/oauth2.md delete mode 100644 docs/content/config/advanced/auth-oauth2.md delete mode 100755 docs/content/config/advanced/dovecot-master-accounts.md delete mode 100644 docs/content/config/user-management.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d9e6ceb8d4..b39f60d8752 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,9 @@ All notable changes to this project will be documented in this file. The format ### Security -- **Fail2ban**: +- **Fail2ban:** - Ensure a secure connection, when downloading the fail2ban package ([#4080](https://github.com/docker-mailserver/docker-mailserver/pull/4080)) + ### Added - **Internal:** @@ -17,10 +18,14 @@ All notable changes to this project will be documented in this file. The format ### Updates -- **Fail2ban**: +- **Fail2ban:** - Bump version to [1.1.0](https://github.com/fail2ban/fail2ban/releases/tag/1.1.0). For more information, check the [changelog](https://github.com/fail2ban/fail2ban/blob/1.1.0/ChangeLog). +- **Documentation:** + - Rewritten and organized the pages for Account Management and Authentication ([#4122](https://github.com/docker-mailserver/docker-mailserver/pull/4122)) + #### Fixes + - **Dovecot:** - `logwatch` Update logwatch `ignore.conf` to exclude Xapian messages about pending documents ([#4060](https://github.com/docker-mailserver/docker-mailserver/pull/4060)) - `dovecot-fts-xapian` plugin was updated to `1.7.13`, fixing a regression with indexing ([#4095](https://github.com/docker-mailserver/docker-mailserver/pull/4095)) @@ -64,7 +69,7 @@ The most noteworthy change of this release is the update of the container's base - **Removed support for Solr integration:** ([#4025](https://github.com/docker-mailserver/docker-mailserver/pull/4025)) - This was a community contributed feature for FTS (Full Text Search), the docs advise using an image that has not been maintained for over 2 years and lacks ARM64 support. Based on user engagement over the years this feature has very niche value to continue to support, thus is being removed. - If you use Solr, support can be restored if you're willing to contribute docs for the feature that resolves the concerns raised -- **Log**: +- **Log:** - The format of DMS specific logs (_from our scripts, not running services_) has been changed. The new format is ` : ` ([#4035](https://github.com/docker-mailserver/docker-mailserver/pull/4035)) - **rsyslog:** - Debian 12 adjusted the `rsyslog` configuration for the default file template from `RSYSLOG_TraditionalFileFormat` to `RSYSLOG_FileFormat` (_upstream default since 2012_). This change may affect you if you have any monitoring / analysis of log output (_eg: `mail.log` / `docker logs`_). @@ -81,7 +86,7 @@ The most noteworthy change of this release is the update of the container's base - `smtp_sasl_auth_enable = yes` (_SASL auth to outbound MTA connections is enabled_) - `smtp_sasl_security_options = noanonymous` (_credentials are mandatory for outbound mail delivery_) - `smtp_tls_security_level = encrypt` (_the outbound MTA connection must always be secure due to credentials sent_) -- **Environment Variables**: +- **Environment Variables:** - `SA_SPAM_SUBJECT` has been renamed into `SPAM_SUBJECT` to become anti-spam service agnostic. ([#3820](https://github.com/docker-mailserver/docker-mailserver/pull/3820)) - As this functionality is now handled in Dovecot via a Sieve script instead of the respective anti-spam service during Postfix processing, this feature will only apply to mail stored in Dovecot. If you have relied on this feature in a different context, it will no longer be available. - Rspamd previously handled this functionality via the `rewrite_subject` action which as now been disabled by default in favor of the new approach with `SPAM_SUBJECT`. @@ -89,16 +94,16 @@ The most noteworthy change of this release is the update of the container's base - The default has changed to not prepend any prefix to the subject unless configured to do so. If you relied on the implicit prefix, you will now need to provide one explicitly. - `undef` was previously supported as an opt-out with `SA_SPAM_SUBJECT`. This is no longer valid, the equivalent opt-out value is now an empty value (_or rather the omission of this ENV being configured_). - The feature to include [`_SCORE_` tag](https://spamassassin.apache.org/full/4.0.x/doc/Mail_SpamAssassin_Conf.html#rewrite_header-subject-from-to-STRING) in your value to be replaced by the associated spam score is no longer available. -- **Supervisord**: - - `supervisor-app.conf` renamed to `dms-services.conf` -- **Rspamd**: - - the Redis history key has been changed in order to not incorporate the hostname of the container (which is desirable in Kubernetes environments) ([#3927](https://github.com/docker-mailserver/docker-mailserver/pull/3927)) -- **Account Management** - - addresses (accounts) are now normalized to lowercase automatically and a warning is logged in case uppercase letters are supplied +- **Supervisord:** + - `supervisor-app.conf` renamed to `dms-services.conf` ([#3908](https://github.com/docker-mailserver/docker-mailserver/pull/3908)) +- **Rspamd:** + - The Redis history key has been changed in order to not incorporate the hostname of the container (which is desirable in Kubernetes environments) ([#3927](https://github.com/docker-mailserver/docker-mailserver/pull/3927)) +- **Account Management:** + - Addresses (accounts) are now normalized to lowercase automatically and a warning is logged in case uppercase letters are supplied ([#4033](https://github.com/docker-mailserver/docker-mailserver/pull/4033)) ### Added -- **Docs:** +- **Documentation:** - A guide for configuring a public server to relay inbound and outbound mail from DMS on a private server ([#3973](https://github.com/docker-mailserver/docker-mailserver/pull/3973)) - **Environment Variables:** - `LOGROTATE_COUNT` defines the number of files kept by logrotate ([#3907](https://github.com/docker-mailserver/docker-mailserver/pull/3907)) @@ -117,7 +122,7 @@ The most noteworthy change of this release is the update of the container's base - Enable spamassassin only, when amavis is enabled too. ([#3943](https://github.com/docker-mailserver/docker-mailserver/pull/3943)) - **Tests:** - Refactored helper methods for sending e-mails with specific `Message-ID` headers and the helpers for retrieving + filtering logs, which together help isolate logs relevant to specific mail when multiple mails have been processed within a single test. ([#3786](https://github.com/docker-mailserver/docker-mailserver/pull/3786)) -- **Rspamd**: +- **Rspamd:** - The `rewrite_subject` action, is now disabled by default. It has been replaced with the new `SPAM_SUBJECT` environment variable, which implements the functionality via a Sieve script instead which is anti-spam service agnostic ([#3820](https://github.com/docker-mailserver/docker-mailserver/pull/3820)) - `RSPAMD_NEURAL` was added and is disabled by default. If switched on it will enable the experimental Rspamd "Neural network" module to add a layer of analysis to spam detection ([#3833](https://github.com/docker-mailserver/docker-mailserver/pull/3833)) - The symbol weights of SPF, DKIM and DMARC have been adjusted again. Fixes a bug and includes more appropriate combinations of symbols ([#3913](https://github.com/docker-mailserver/docker-mailserver/pull/3913), [#3923](https://github.com/docker-mailserver/docker-mailserver/pull/3923)) @@ -167,12 +172,12 @@ The most noteworthy change of this release is the update of the container's base ### Added -- **Docs:** +- **Documentation:** - An example for how to bind outbound SMTP connections to a specific network interface ([#3465](https://github.com/docker-mailserver/docker-mailserver/pull/3465)) ### Updates -- **Tests**: +- **Tests:** - Revised OAuth2 test ([#3795](https://github.com/docker-mailserver/docker-mailserver/pull/3795)) - Replace `wc -l` with `grep -c` ([#3752](https://github.com/docker-mailserver/docker-mailserver/pull/3752)) - Revised testing of service process management (supervisord) to be more robust ([#3780](https://github.com/docker-mailserver/docker-mailserver/pull/3780)) @@ -184,9 +189,9 @@ The most noteworthy change of this release is the update of the container's base - `test/files/emails/existing/` files were removed similar to previous removal of SMTP auth files as they became redundant with `swaks`. - **Internal:** - Postfix is now configured with `smtputf8_enable = no` in our default `main.cf` config (_instead of during container startup_). ([#3750](https://github.com/docker-mailserver/docker-mailserver/pull/3750)) -- **Rspamd** ([#3726](https://github.com/docker-mailserver/docker-mailserver/pull/3726)): +- **Rspamd:** ([#3726](https://github.com/docker-mailserver/docker-mailserver/pull/3726)): - Symbol scores for SPF, DKIM & DMARC were updated to more closely align with [RFC7489](https://www.rfc-editor.org/rfc/rfc7489#page-24). Please note that complete alignment is undesirable as other symbols may be added as well, which changes the overall score calculation again, see [this issue](https://github.com/docker-mailserver/docker-mailserver/issues/3690#issuecomment-1866871996) -- **Docs:** +- **Documentation:** - Revised the SpamAssassin ENV docs to better communicate configuration and their relation to other ENV settings. ([#3756](https://github.com/docker-mailserver/docker-mailserver/pull/3756)) - Detailed how mail received is assigned a spam score by Rspamd and processed accordingly ([#3773](https://github.com/docker-mailserver/docker-mailserver/pull/3773)) @@ -235,7 +240,7 @@ DMS is now secured against the [recently published spoofing attack "SMTP Smuggli - ENV `ENABLE_IMAP` ([#3703](https://github.com/docker-mailserver/docker-mailserver/pull/3703)) - **Tests:** - You can now use `make run-local-instance` to run a DMS image that was built locally to test changes ([#3663](https://github.com/docker-mailserver/docker-mailserver/pull/3663)) -- **Internal**: +- **Internal:** - Log a warning when update-check is enabled, but no stable release image is used ([#3684](https://github.com/docker-mailserver/docker-mailserver/pull/3684)) ### Updates @@ -250,7 +255,7 @@ DMS is now secured against the [recently published spoofing attack "SMTP Smuggli ### Fixed -- **Internal**: +- **Internal:** - The container startup welcome log message now references `DMS_RELEASE` ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) - `VERSION` was incremented for prior releases to be notified of the v13.0.1 patch release ([#3676](https://github.com/docker-mailserver/docker-mailserver/pull/3676)) - `VERSION` is no longer included in the image ([#3711](https://github.com/docker-mailserver/docker-mailserver/pull/3711)) diff --git a/README.md b/README.md index e34aeb12a4c..e4dbf44a984 100644 --- a/README.md +++ b/README.md @@ -38,8 +38,8 @@ A production-ready fullstack but simple containerized mail server (SMTP, IMAP, L ## :package: Included Services -- [Postfix](http://www.postfix.org) with SMTP or LDAP authentication and support for [extension delimiters](https://docker-mailserver.github.io/docker-mailserver/v13.3/config/user-management/#address-tags-extension-delimiters-as-an-alternative-to-aliases) -- [Dovecot](https://www.dovecot.org) with SASL, IMAP, POP3, LDAP, [basic Sieve support](https://docker-mailserver.github.io/docker-mailserver/latest/config/advanced/mail-sieve) and [quotas](https://docker-mailserver.github.io/docker-mailserver/v13.3/config/user-management/#quotas) +- [Postfix](http://www.postfix.org) with SMTP or LDAP authentication and support for [extension delimiters](https://docker-mailserver.github.io/docker-mailserver/latest/config/account-management/overview/#aliases) +- [Dovecot](https://www.dovecot.org) with SASL, IMAP, POP3, LDAP, [basic Sieve support](https://docker-mailserver.github.io/docker-mailserver/latest/config/advanced/mail-sieve) and [quotas](https://docker-mailserver.github.io/docker-mailserver/latest/config/account-management/overview/#quotas) - [Rspamd](https://rspamd.com/) - [Amavis](https://www.amavis.org/) - [SpamAssassin](http://spamassassin.apache.org/) supporting custom rules diff --git a/docs/content/config/account-management/overview.md b/docs/content/config/account-management/overview.md new file mode 100644 index 00000000000..d94104f2237 --- /dev/null +++ b/docs/content/config/account-management/overview.md @@ -0,0 +1,252 @@ +# Account Management - Overview + +This page provides a technical reference for account management in DMS. + +!!! note "Account provisioners and alternative authentication support" + + Each [`ACCOUNT_PROVISIONER`][docs::env::account-provisioner] has a separate page for configuration guidance and caveats: + + - [`FILE` provisioner docs][docs::account-provisioner::file] + - [`LDAP` provisioner docs][docs::account-provisioner::ldap] + + Authentication from the provisioner can be supplemented with additional methods: + + - [OAuth2 / OIDC][docs::account-auth::oauth2] (_allow login from an external authentication service_) + - [Master Accounts][docs::account-auth::master-accounts] (_access the mailbox of any DMS account_) + + --- + + For custom authentication requirements, you could [implement this with Lua][docs::examples::auth-lua]. + +## Accounts + +!!! info + + To receive or send mail, you'll need to provision user accounts into DMS (_as each provisioner page documents_). + + --- + + A DMS account represents a user with their _login username_ + password, and optional config like aliases and quota. + + - Sending mail from different addresses **does not require** aliases or separate accounts. + - Each account is configured with a _primary email address_ that a mailbox is associated to. + +??? info "Primary email address" + + The email address associated to an account creates a mailbox. This address is relevant: + + - When DMS **receives mail** for that address as the recipient (_or an alias that resolves to it_), to identify which mailbox to deliver into. + - With **mail submission**: + - `SPOOF_PROTECTION=1` **restricts the sender address** to the DMS account email address (_unless additional sender addresses have been permitted via supported config_). + - `SPOOF_PROTECTION=0` allows DMS accounts to **use any sender address** (_only a single DMS account is necessary to send mail with different sender addresses_). + + --- + + For more details, see the [Technical Overview](#technical-overview) section. + +??? note "Support for multiple mail domains" + + No extra configuration in DMS is required after provisioning an account with an email address. + + - The DNS records for a domain should direct mail to DMS and allow DMS to send mail on behalf of that domain. + - DMS does not need TLS certificates for your mail domains, only for the DMS FQDN (_the `hostname` setting_). + +??? warning "Choosing a compatible email address" + + An email address should conform to the standard [permitted charset and format][email-syntax::valid-charset-format] (`local-part@domain-part`). + + --- + + DMS has features that need to reserve special characters to work correctly. Ensure those characters are not present in email addresses you configure for DMS, otherwise disable / opt-out of the feature. + + - [Sub-addressing](#sub-addressing) is enabled by default with `+` as the _tag delimiter_. The tag can be changed, feature opt-out when the tag is explicitly unset. + +### Aliases + +!!! info + + Aliases allow receiving mail: + + - As an alternative delivery address for a DMS account mailbox. + - To redirect / forward to an external address outside of DMS like `@gmail.com`. + +??? abstract "Technical Details (_Local vs Virtual aliases_)" + + Aliases are managed through Postfix which supports _local_ and _virtual_ aliases: + + - **Local aliases** are for mail routed to the [`local` delivery agent][postfix::delivery-agent::local] (see [associated alias config format][postfix::config-table::local-alias]) + - You rarely need to configure this. It is used internally for system unix accounts belonging to the services running in DMS (_including `root`_). + - `postmaster` may be a local alias to `root`, and `root` to a virtual alias or real email address. + - Any mail sent through the `local` delivery agent will not be delivered to an inbox managed by Dovecot (_unless you have configured a local alias to redirect mail to a valid address or alias_). + - The domain-part of an these aliases belongs to your DMS FQDN (_`hostname: mail.example.com`, thus `user@mail.example.com`_). Technically there is no domain-part at this point, that context is used when routing delivery, the local delivery agent only knows of the local-part (_an alias or unix account_). + - [**Virtual aliases**][postfix-docs::virtual-alias] are for mail routed to the [`virtual` delivery agent][postfix::delivery-agent::virtual] (see [associated alias config format][postfix::config-table::virtual-alias]) + - When alias support in DMS is discussed without the context of being a local or virtual alias, it's likely the virtual kind (_but could also be agnostic_). + - The domain-part of an these aliases belongs to a mail domain managed by DMS (_like `user@example.com`_). + + !!! tip "Verify alias resolves correctly" + + You can run `postmap -q ` in the container to verify an alias resolves to the expected target. If the target is also an alias, the command will not expand that alias to resolve the actual recipient(s). + + For the `FILE` provisioner, an example would be: `postmap -q alias1@example.com /etc/postfix/virtual`. For the `LDAP` provisioner you'd need to adjust the table path. + + !!! info "Side effect - Dovecot Quotas (`ENABLE_QUOTAS=1`)" + + As a side effect of the alias workaround for the `FILE` provisioner with this feature, aliases can be used for account login. This is not intentional. + +### Quotas + +!!! info + + Enables mail clients with the capability to query a mailbox for disk-space used and capacity limit. + + - This feature is enabled by default, opt-out via [`ENABLE_QUOTAS=0`][docs::env::enable-quotas] + - **Not implemented** for the LDAP provisioner (_PR welcome! View the [feature request for implementation advice][gh-issue::dms-feature-request::dovecot-quotas-ldap]_) + +??? tip "How are quotas useful?" + + Without quota limits for disk storage, a mailbox could fill up the available storage which would cause delivery failures to all mailboxes. + + Quotas help by preventing that abuse, so that only a mailbox exceeding the assigned quota experiences a delivery failure instead of negatively impacting others (_provided disk space is available_). + +??? abstract "Technical Details" + + The [Dovecot Quotas feature][gh-pr::dms-feature::dovecot-quotas] is configured by enabling the [Dovecot `imap-quota` plugin][dovecot-docs::plugin::imap-quota] and using the [`count` quota backend][dovecot-docs::config::quota-backend-count]. + + --- + + **Dovecot workaround for Postfix aliases** + + When mail is delivered to DMS, Postfix will query Dovecot with the recipient(s) to verify quota has not been exceeded. + + This allows early rejection of mail arriving to DMS, preventing a spammer from taking advantage of a [backscatter][wikipedia::backscatter] source if the mail was accepted by Postfix, only to later be rejected by Dovecot for storage when the quota limit was already reached. + + However, Postfix does not resolve aliases until after the incoming mail is accepted. + + 1. Postfix queries Dovecot (_a [`check_policy_service` restriction tied to the Dovecot `quota-status` service][dms::workaround::dovecot-quotas::notes-1]_) with the recipient (_the alias_). + 2. `dovecot: auth: passwd-file(alias@example.com): unknown user` is logged, Postfix is then informed that the recipient mailbox is not full even if it actually was (_since no such user exists in the Dovecot UserDB_). + 3. However, when the real mailbox address that the alias would later resolve into does have a quota that exceeded the configured limit, Dovecot will refuse the mail delivery from Postfix which introduces a backscatter source for spammers. + + As a [workaround to this problem with the `ENABLE_QUOTAS=1` feature][dms::workaround::dovecot-quotas::summary], DMS will add aliases as fake users into Dovecot UserDB (_that are configured with the same data as the real address the alias would resolve to, thus sharing the same mailbox location and quota limit_). This allows Postfix to properly be aware of an aliased mailbox having exceeded the allowed quota. + + **NOTE:** This workaround **only supports** aliases to a single target recipient of a real account address / mailbox. + + - Additionally, aliases that resolve to another alias or to an external address would both fail the UserDB lookup, unable to determine if enough storage is available. + - A proper fix would [implement a Postfix policy service][dms::workaround::dovecot-quotas::notes-2] that could correctly resolve aliases to valid entries in the Dovecot UserDB, querying the `quota-status` service and returning that response to Postfix. + +## Sub-addressing + +!!! info + + [Subaddressing][wikipedia::subaddressing] (_aka **Plus Addressing** or **Address Tags**_) is a feature that allows you to receive mail to an address which includes a tag appended to the `local-part` of a valid account address. + + - A subaddress has a tag delimiter (_default: `+`_), followed by the tag: `+@` + - The subaddress `user+github@example.com` would deliver mail to the same mailbox as `user@example.com`. + - Tags are dynamic. Anything between the `+` and `@` is understood as the tag, no additional configuration required. + - Only the first occurence of the tag delimiter is recognized. Any additional occurences become part of the tag value itself. + +??? tip "When is subaddressing useful?" + + A common use-case is to use a unique tag for each service you register your email address with. + + - Routing delivery to different folders in your mailbox based on the tag (_via a [Sieve filter][docs::sieve::subaddressing]_). + - Data leaks or bulk sales of email addresses. + - If spam / phishing mail you receive has not removed the tag, you will have better insight into where your address was compromised from. + - When the expected tag is missing, this additionally helps identify bad actors. Especially when mail delivery is routed to subfolders by tag. + - For more use-cases, view the end of [this article][web::subaddress-use-cases]. + +??? tip "Changing the tag delimiter" + + Add `recipient_delimiter = +` to these config override files (_replacing `+` with your preferred delimiter_): + + - Postfix: `docker-data/dms/config/postfix-main.cf` + - Dovecot: `docker-data/dms/config/dovecot.cf` + +??? tip "Opt-out of subaddressing" + + Follow the advice to change the tag delimiter, but instead set an empty value (`recipient_delimiter =`). + +??? warning "Only for receiving, not sending" + + Do not attempt to send mail from these tagged addresses, they are not equivalent to aliases. + + This feature is only intended to be used when a mail client sends to a DMS managed recipient address. While DMS does not restrict the sender address you choose to send mail from (_provided `SPOOF_PROTECTION` has not been enabled_), it is often [forbidden by mail services][ms-exchange-docs::limitations]. + +??? abstract "Technical Details" + + The configured tag delimiter (`+`) allows both Postfix and Dovecot to recognize subaddresses. Without this feature configured, the subaddresses would be considered as separate mail accounts rather than routed to a common account address. + + --- + + Internally DMS has the tag delimiter configured by: + + - Applying the Postfix `main.cf` setting: [`recipient_delimiter = +`][postfix-docs::recipient-delimiter] + - Dovecot has the equivalent setting set as `+` by default: [`recipient_delimiter = +`][dovecot-docs::config::recipient-delimiter] + +## Technical Overview + +!!! info + + This section provides insight for understanding how Postfix and Dovecot services are involved. It is intended as a reference for maintainers and contributors. + + - **Postfix** - Handles when mail is delivered (inbound) to DMS, or sent (outbound) from DMS. + - **Dovecot** - Manages access and storage for mail delivered to the DMS account mailboxes of your users. + +??? abstract "Technical Details - Postfix (Inbound vs Outbound)" + + Postfix needs to know how to handle inbound and outbound mail by asking these queries: + + === "Inbound" + + - What mail domains is DMS responsible for handling? (_for accepting mail delivered_) + - What are valid mail addresses for those mail domains? (_reject delivery for users that don't exist_) + - Are there any aliases to redirect mail to 1 or more users, or forward to externally? + + === "Outbound" + + - When `SPOOF_PROTECTION=1`, how should DMS restrict the sender address? (_eg: Users may only send mail from their associated mailbox address_) + +??? abstract "Technical Details - Dovecot (Authentication)" + + Dovecot additionally handles authenticating user accounts for sending and retrieving mail: + + - Over the ports for IMAP and POP3 connections (_110, 143, 993, 995_). + - As the default configured SASL provider, which Postfix delegates user authentication through (_for the submission(s) ports 465 & 587_). Saslauthd can be configured as an alternative SASL provider. + + Dovecot splits all authentication lookups into two categories: + + - A [PassDB][dovecot::docs::passdb] lookup most importantly authenticates the user. It may also provide any other necessary pre-login information. + - A [UserDB][dovecot::docs::userdb] lookup retrieves post-login information specific to a user. + +[docs::env::account-provisioner]: ../environment.md#account_provisioner +[docs::account-provisioner::file]: ./provisioner/file.md +[docs::account-provisioner::ldap]: ./provisioner/ldap.md +[docs::account-auth::oauth2]: ./supplementary/oauth2.md +[docs::account-auth::master-accounts]: ./supplementary/master-accounts.md +[docs::examples::auth-lua]: ../../examples/use-cases/auth-lua.md +[email-syntax::valid-charset-format]: https://stackoverflow.com/questions/2049502/what-characters-are-allowed-in-an-email-address/2049510#2049510 + +[postfix-docs::virtual-alias]: http://www.postfix.org/VIRTUAL_README.html#virtual_alias +[postfix-docs::recipient-delimiter]: http://www.postfix.org/postconf.5.html#recipient_delimiter +[dovecot-docs::config::recipient-delimiter]: https://doc.dovecot.org/settings/core/#core_setting-recipient_delimiter +[postfix::delivery-agent::local]: https://www.postfix.org/local.8.html +[postfix::delivery-agent::virtual]: https://www.postfix.org/virtual.8.html +[postfix::config-table::local-alias]: https://www.postfix.org/aliases.5.html +[postfix::config-table::virtual-alias]: https://www.postfix.org/virtual.5.html + +[docs::env::enable-quotas]: ../environment.md#enable_quotas +[gh-issue::dms-feature-request::dovecot-quotas-ldap]: https://github.com/docker-mailserver/docker-mailserver/issues/2957 +[dovecot-docs::config::quota-backend-count]: https://doc.dovecot.org/configuration_manual/quota/quota_count/#quota-backend-count +[dovecot-docs::plugin::imap-quota]: https://doc.dovecot.org/settings/plugin/imap-quota-plugin/ +[gh-pr::dms-feature::dovecot-quotas]: https://github.com/docker-mailserver/docker-mailserver/pull/1469 +[wikipedia::backscatter]: https://en.wikipedia.org/wiki/Backscatter_%28email%29 +[dms::workaround::dovecot-quotas::notes-1]: https://github.com/docker-mailserver/docker-mailserver/issues/2091#issuecomment-954298788 +[dms::workaround::dovecot-quotas::notes-2]: https://github.com/docker-mailserver/docker-mailserver/pull/2248#issuecomment-953754532 +[dms::workaround::dovecot-quotas::summary]: https://github.com/docker-mailserver/docker-mailserver/pull/2248#issuecomment-955088677 + +[docs::sieve::subaddressing]: ../advanced/mail-sieve.md#subaddress-mailbox-routing +[web::subaddress-use-cases]: https://www.codetwo.com/admins-blog/plus-addressing +[wikipedia::subaddressing]: https://en.wikipedia.org/wiki/Email_address#Sub-addressing +[ms-exchange-docs::limitations]: https://learn.microsoft.com/en-us/exchange/recipients-in-exchange-online/plus-addressing-in-exchange-online#using-plus-addresses + +[dovecot::docs::passdb]: https://doc.dovecot.org/configuration_manual/authentication/password_databases_passdb +[dovecot::docs::userdb]: https://doc.dovecot.org/configuration_manual/authentication/user_databases_userdb diff --git a/docs/content/config/account-management/provisioner/file.md b/docs/content/config/account-management/provisioner/file.md new file mode 100644 index 00000000000..5b74ffcc0a0 --- /dev/null +++ b/docs/content/config/account-management/provisioner/file.md @@ -0,0 +1,206 @@ +--- +title: 'Account Management | Provisioner (File)' +--- + +# Provisioner - File + +## Management via the `setup` CLI + +The best way to manage DMS accounts and related config files is through our `setup` CLI provided within the container. + +!!! example "Using the `setup` CLI" + + Try the following within the DMS container (`docker exec -it bash`): + + - Add an account: `setup email add ` + - Add an alias: `setup alias add ` + - Learn more about the available subcommands via: `setup help` + + ```bash + # Starts a basic DMS instance and then shells into the container to use the `setup` CLI: + docker run --rm -itd --name dms --hostname mail.example.com mailserver/docker-mailserver + docker exec -it dms bash + + # Create an account: + setup email add hello@example.com your-password-here + + # Create an alias: + setup alias add your-alias-here@example.com hello@example.com + + # Limit the mailbox capacity to 10 MiB: + setup quota set hello@example.com 10M + ``` + + ??? tip "Secure password input" + + When you don't provide a password to the command, you will be prompted for one. This avoids the password being captured in your shell history. + + ```bash + # As you input your password it will not update. + # Press the ENTER key to apply the hidden password input. + $ setup email add hello@example.com + Enter Password: + Confirm Password: + ``` + +!!! note "Account removal via `setup email del`" + + When you remove a DMS account with this command, it will also remove any associated aliases and quota. + + The command will also prompt for deleting the account mailbox from disk, or can be forced with the `-y` flag. + +## Config Reference + +These config files belong to the [Config Volume][docs::volumes::config]. + +### Accounts + +!!! info + + **Config file:** `docker-data/dms/config/postfix-accounts.cf` + + --- + + The config format is line-based with two fields separated by the delimiter `|`: + + - **User:** The primary email address for the account mailbox to use. + - **Password:** A SHA512-CRYPT hash of the account password (_in this example it is `secret`_). + + ??? tip "Password hash without the `setup email add` command" + + A compatible password hash can be generated with: + + ```bash + doveadm pw -s SHA512-CRYPT -u hello@example.com -p secret + ``` + +!!! example "`postfix-accounts.cf` config file" + + In this example DMS manages mail for the domain `example.com`: + + ```cf title="postfix-accounts.cf" + hello@example.com|{SHA512-CRYPT}$6$W4rxRQwI6HNMt9n3$riCi5/OqUxnU8eZsOlZwoCnrNgu1gBGPkJc.ER.LhJCu7sOg9i1kBrRIistlBIp938GdBgMlYuoXYUU5A4Qiv0 + ``` + + --- + + **Dovecot "extra fields"** + + [Appending a third column will customize "extra fields"][gh-issue::provisioner-file::accounts-extra-fields] when converting account data into a Dovecot UserDB entry. + + DMS is not aware of these customizations beyond carrying them over, expect potential for bugs when this feature breaks any assumed conventions used in the scripts (_such as changing the mailbox path or type_). + +!!! note + + Account creation will normalize the provided email address to lowercase, as DMS does not support multiple case-sensitive address variants. + + The email address chosen will also represent the _login username_ credential for mail clients to authenticate with. + +### Aliases + +!!! info + + **Config file:** `docker-data/dms/config/postfix-virtual.cf` + + --- + + The config format is line-based with key value pairs (**alias** --> **target address**), with white-space as a delimiter. + +!!! example "`postfix-virtual.cf` config file" + + In this example DMS manages mail for the domain `example.com`: + + ```cf-extra title="postfix-virtual.cf" + # Alias delivers to an existing account: + alias1@example.com hello@example.com + + # Alias forwards to an external email address: + alias2@example.com external-account@gmail.com + ``` + +??? warning "Known Issues" + + **`setup` CLI prevents an alias and account sharing an address:** + + You cannot presently add a new account (`setup email add`) or alias (`setup alias add`) with an address which already exists as an alias or account in DMS. + + This [restriction was enforced][gh-issue::bugs::account-alias-overlap] due to [problems it could cause][gh-issue::bugs::account-alias-overlap-problem], although there are [use-cases where you may legitimately require this functionality][gh-issue::feature-request::allow-account-alias-overlap]. + + For now you must manually edit the `postfix-virtual.cf` file as a workaround. There are no run-time checks outside of the `setup` CLI related to this restriction. + + --- + + **Wildcard catch-all support (`@example.com`):** + + While this type of alias without a local-part is supported, you must keep in mind that aliases in Postfix have a higher precedence than a real address associated to a DMS account. + + As a result, the wildcard is matched first and will direct mail for that entire domain to the alias target address. To work around this, [you will need an alias for each non-alias address of that domain][gh-issue::bugs::wildcard-catchall]. + + Additionally, Postfix will read the alias config and choose the alias value that matches the recipient address first. Ensure your more specific aliases for the domain are declared above the wildcard alias in the config file. + + --- + + **Aliasing to another alias or multiple recipients:** + + [While aliasing to multiple recipients is possible][gh-discussions::no-support::alias-multiple-targets], DMS does not officially support that. + + - You may experience issues when our feature integrations don't expect more than one target per alias. + - These concerns also apply to the usage of nested aliases (_where the recipient target provided is to an alias instead of a real address_). An example is the [incompatibility with `setup alias add`][gh-issue::bugs::alias-nested]. + +#### Configuring RegEx aliases + +!!! info + + **Config file:** `docker-data/dms/config/postfix-regexp.cf` + + --- + + This config file is similar to the above `postfix-virtual.cf`, but the alias value is instead configured with a regex pattern. + + There is **no `setup` CLI support** for this feature, it is config only. + +!!! example "`postfix-regexp.cf` config file" + + Deliver all mail for `test` users to `qa@example.com` instead: + + ```cf-extra title="postfix-regexp.cf" + # Remember to escape regex tokens like `.` => `\.`, otherwise + # your alias pattern may be more permissive than you intended: + /^test[0-9][0-9]*@example\.com/ qa@example.com + ``` + +??? abstract "Technical Details" + + `postfix-virtual.cf` has precedence, `postfix-regexp.cf` will only be checked if no alias match was found in `postfix-virtual.cf`. + + These files are both copied internally to `/etc/postfix/` and configured in `main.cf` for the `virtual_alias_maps` setting. As `postfix-virtual.cf` is declared first for that setting, it will be processed before using `postfix-regexp.cf` as a fallback. + +### Quotas + +!!! info + + **Config file:** `docker-data/dms/config/dovecot-quotas.cf` + + ---- + + The config format is line-based with two fields separated by the delimiter `:`: + + - **Dovecot UserDB account:** The user DMS account. It should have a matching field in `postfix-accounts.cf`. + - **Quota limit:** Expressed in bytes (_binary unit suffix is supported: `M` => `MiB`, `G` => `GiB`_). + +!!! example "`dovecot-quotas.cf` config file" + + For the account with the mailbox address of `hello@example.com`, it may not exceed 5 GiB in storage: + + ```cf-extra title="dovecot-quotas.cf" + hello@example.com:5G + ``` + +[docs::volumes::config]: ../../advanced/optional-config.md#volumes-config +[gh-issue::provisioner-file::accounts-extra-fields]: https://github.com/docker-mailserver/docker-mailserver/issues/4117 +[gh-issue::feature-request::allow-account-alias-overlap]: https://github.com/docker-mailserver/docker-mailserver/issues/3528 +[gh-issue::bugs::account-alias-overlap-problem]: https://github.com/docker-mailserver/docker-mailserver/issues/3350#issuecomment-1550528898 +[gh-issue::bugs::account-alias-overlap]: https://github.com/docker-mailserver/docker-mailserver/issues/3022#issuecomment-1807816689 +[gh-issue::bugs::wildcard-catchall]: https://github.com/docker-mailserver/docker-mailserver/issues/3022#issuecomment-1610452561 +[gh-issue::bugs::alias-nested]: https://github.com/docker-mailserver/docker-mailserver/issues/3622#issuecomment-1794504849 +[gh-discussions::no-support::alias-multiple-targets]: https://github.com/orgs/docker-mailserver/discussions/3805#discussioncomment-8215417 diff --git a/docs/content/config/advanced/auth-ldap.md b/docs/content/config/account-management/provisioner/ldap.md similarity index 99% rename from docs/content/config/advanced/auth-ldap.md rename to docs/content/config/account-management/provisioner/ldap.md index 397e42ebe11..607ee3d5f63 100644 --- a/docs/content/config/advanced/auth-ldap.md +++ b/docs/content/config/account-management/provisioner/ldap.md @@ -1,5 +1,5 @@ --- -title: 'Advanced | LDAP Authentication' +title: 'Account Management | Provisioner (LDAP)' --- ## Introduction @@ -304,5 +304,5 @@ The changes on the configurations necessary to work with Active Directory (**onl - NET_ADMIN ``` -[docs-environment]: ../environment.md -[docs-userpatches]: ./override-defaults/user-patches.md +[docs-environment]: ../../environment.md +[docs-userpatches]: ../../advanced/override-defaults/user-patches.md diff --git a/docs/content/config/account-management/supplementary/master-accounts.md b/docs/content/config/account-management/supplementary/master-accounts.md new file mode 100644 index 00000000000..a8665a83eef --- /dev/null +++ b/docs/content/config/account-management/supplementary/master-accounts.md @@ -0,0 +1,70 @@ +--- +title: 'Account Management | Master Accounts (Dovecot)' +hide: + - toc # Hide Table of Contents for this page +--- + +This feature is useful for administrative tasks like hot backups. + +!!! note + + This feature is presently [not supported with `ACCOUNT_PROVISIONER=LDAP`][dms::feature::dovecot-master-accounts::caveat-ldap]. + +!!! info + + A _Master Account_: + + - Can login as any user (DMS account) and access their mailbox. + - Is not associated to a separate DMS account, nor is it a DMS account itself. + + --- + + **`setup` CLI support** + + Use the `setup dovecot-master ` commands. These are roughly equivalent to the `setup email` subcommands. + + --- + + **Config file:** `docker-data/dms/config/dovecot-masters.cf` + + The config format is the same as [`postfix-accounts.cf` for `ACCOUNT_PROVISIONER=FILE`][docs::account-management::file::accounts]. + + The only difference is the account field has no `@domain-part` suffix, it is only a username. + +??? abstract "Technical Details" + + [The _Master Accounts_ feature][dms::feature::dovecot-master-accounts] in DMS configures the [Dovecot Master Users][dovecot-docs::auth::master-users] feature with the Dovecot setting [`auth_master_user_separator`][dovecot-docs::config::auth-master-user-separator] (_where the default value is `*`_). + +## Login via Master Account + +!!! info + + To login as another DMS account (`user@example.com`) with POP3 or IMAP, use the following credentials format: + + - Username: `*` (`user@example.com*admin`) + - Password: `` + +!!! example "Verify login functionality" + + In the DMS container, you can verify with the `testsaslauthd` command: + + ```bash + # Prerequisites: + # A regular DMS account to test login through a Master Account: + setup email add user@example.com secret + # Add a new Master Account: + setup dovecot-master add admin top-secret + ``` + + ```bash + # Login with credentials format as described earlier: + testsaslauthd -u 'user@example.com*admin' -p 'top-secret' + ``` + + Alternatively, any mail client should be able to login the equivalent credentials. + +[dms::feature::dovecot-master-accounts]: https://github.com/docker-mailserver/docker-mailserver/pull/2535 +[dms::feature::dovecot-master-accounts::caveat-ldap]: https://github.com/docker-mailserver/docker-mailserver/pull/2535#issuecomment-1118056745 +[dovecot-docs::auth::master-users]: https://doc.dovecot.org/configuration_manual/authentication/master_users/ +[dovecot-docs::config::auth-master-user-separator]: https://doc.dovecot.org/settings/core/#core_setting-auth_master_user_separator +[docs::account-management::file::accounts]: ../provisioner/file.md#accounts diff --git a/docs/content/config/account-management/supplementary/oauth2.md b/docs/content/config/account-management/supplementary/oauth2.md new file mode 100644 index 00000000000..fb74aeec44f --- /dev/null +++ b/docs/content/config/account-management/supplementary/oauth2.md @@ -0,0 +1,145 @@ +--- +title: 'Account Management | OAuth2 Support' +hide: + - toc # Hide Table of Contents for this page +--- + +# Authentication - OAuth2 / OIDC + +This feature enables support for delegating DMS account authentication through to an external _Identity Provider_ (IdP). + +!!! warning "Receiving mail requires a DMS account to exist" + + If you expect DMS to receive mail, you must provision an account into DMS in advance. Otherwise DMS has no awareness of your externally manmaged users and will reject delivery. + + There are [plans to implement support to provision users through a SCIM 2.0 API][dms-feature-request::scim-api]. An IdP that can operate as a SCIM Client (eg: Authentik) would then integrate with DMS for user provisioning. Until then you must keep your user accounts in sync manually via your configured [`ACCOUNT_PROVISIONER`][docs::env::account-provisioner]. + +??? info "How the feature works" + + 1. A **mail client must have support** to acquire an OAuth2 token from your IdP (_however many clients lack generic OAuth2 / OIDC provider support_). + 2. The mail client then provides that token as the user password via the login mechanism `XOAUTH2` or `OAUTHBEARER`. + 3. DMS (Dovecot) will then check the validity of that token against the Authentication Service it was configured with. + 4. If the response returned is valid for the user account, authentication is successful. + + [**XOAUTH2**][google::xoauth2-docs] (_Googles widely adopted implementation_) and **OAUTHBEARER** (_the newer variant standardized by [RFC 7628][rfc::7628] in 2015_) are supported as standards for verifying that a OAuth Bearer Token (_[RFC 6750][rfc::6750] from 2012_) is valid at the identity provider that created the token. The token itself in both cases is expected to be can an opaque _Access Token_, but it is possible to use a JWT _ID Token_ (_which encodes additional information into the token itself_). + + A mail client like Thunderbird has limited OAuth2 / OIDC support. The software maintains a hard-coded list of providers supported. Roundcube is a webmail client that does have support for generic providers, allowing you to integrate with a broader range of IdP services. + + --- + + **Documentation for this feature is WIP** + + See the [initial feature support][dms-feature::oauth2-pr] and [existing issues][dms-feature::oidc-issues] for guidance that has not yet been documented officially. + +??? tip "Verify authentication works" + + If you have a compatible mail client you can verify login through that. + + --- + + ??? example "CLI - Verify with `curl`" + + ```bash + # Shell into your DMS container: + docker exec -it dms bash + + # Adjust these variables for the methods below to use: + export AUTH_METHOD='OAUTHBEARER' USER_ACCOUNT='hello@example.com' ACCESS_TOKEN='DMS_YWNjZXNzX3Rva2Vu' + + # Authenticate via IMAP (Dovecot): + curl --silent --url 'imap://localhost:143' \ + --login-options "AUTH=${AUTH_METHOD}" --user "${USER_ACCOUNT}" --oauth2-bearer "${ACCESS_TOKEN}" \ + --request 'LOGOUT' \ + && grep "dovecot: imap-login: Login: user=<${USER_ACCOUNT}>, method=${AUTH_METHOD}" /var/log/mail/mail.log + + # Authenticate via SMTP (Postfix), sending a mail with the same sender(from) and recipient(to) address: + # NOTE: `curl` seems to require `--upload-file` with some mail content provided to test SMTP auth. + curl --silent --url 'smtp://localhost:587' \ + --login-options "AUTH=${AUTH_METHOD}" --user "${USER_ACCOUNT}" --oauth2-bearer "${ACCESS_TOKEN}" \ + --mail-from "${USER_ACCOUNT}" --mail-rcpt "${USER_ACCOUNT}" --upload-file - <<< 'RFC 5322 content - not important' \ + && grep "postfix/submission/smtpd.*, sasl_method=${AUTH_METHOD}, sasl_username=${USER_ACCOUNT}" /var/log/mail/mail.log + ``` + + --- + + **Troubleshooting:** + + - Add `--verbose` to the curl options. This will output the protocol exchange which includes if authentication was successful or failed. + - The above example chains the `curl` commands with `grep` on DMS logs (_for Dovecot and Postfix services_). When not running `curl` from the DMS container, ensure you check the logs correctly, or inspect the `--verbose` output instead. + + !!! warning "`curl` bug with `XOAUTH2`" + + [Older releases of `curl` have a bug with `XOAUTH2` support][gh-issue::curl::xoauth2-bug] since `7.80.0` (Nov 2021) but fixed from `8.6.0` (Jan 2024). It treats `XOAUTH2` as `OAUTHBEARER`. + + If you use `docker exec` to run `curl` from within DMS, the current DMS v14 release (_Debian 12 with curl `7.88.1`_) is affected by this bug. + +## Config Examples + +### Authentik with Roundcube + +This example assumes you have already set up: + +- A working DMS server +- An Authentik server ([documentation][authentik::docs::install]) +- A Roundcube server ([docker image][roundcube::dockerhub-image] or [bare metal install][roundcube::docs::install]) + +!!! example "Setup Instructions" + + === "1. Docker Mailserver" + + Update your Docker Compose ENV config to include: + + ```env title="compose.yaml" + services: + mailserver: + env: + # Enable the feature: + - ENABLE_OAUTH2=1 + # Specify the user info endpoint URL of the oauth2 server for token inspection: + - OAUTH2_INTROSPECTION_URL=https://authentik.example.com/application/o/userinfo/ + ``` + + === "2. Authentik" + + 1. Create a new OAuth2 provider. + 2. Note the client id and client secret. Roundcube will need this. + 3. Set the allowed redirect url to the equivalent of `https://roundcube.example.com/index.php/login/oauth` for your RoundCube instance. + + === "3. Roundcube" + + Add the following to `oauth2.inc.php` ([documentation][roundcube::docs::config]): + + ```php + $config['oauth_provider'] = 'generic'; + $config['oauth_provider_name'] = 'Authentik'; + $config['oauth_client_id'] = ''; + $config['oauth_client_secret'] = ''; + $config['oauth_auth_uri'] = 'https://authentik.example.com/application/o/authorize/'; + $config['oauth_token_uri'] = 'https://authentik.example.com/application/o/token/'; + $config['oauth_identity_uri'] = 'https://authentik.example.com/application/o/userinfo/'; + + // Optional: disable SSL certificate check on HTTP requests to OAuth server. For possible values, see: + // http://docs.guzzlephp.org/en/stable/request-options.html#verify + $config['oauth_verify_peer'] = false; + + $config['oauth_scope'] = 'email openid profile'; + $config['oauth_identity_fields'] = ['email']; + + // Boolean: automatically redirect to OAuth login when opening Roundcube without a valid session + $config['oauth_login_redirect'] = false; + ``` + +[dms-feature::oauth2-pr]: https://github.com/docker-mailserver/docker-mailserver/pull/3480 +[dms-feature::oidc-issues]: https://github.com/docker-mailserver/docker-mailserver/issues?q=label%3Afeature%2Fauth-oidc +[docs::env::account-provisioner]: ../../environment.md#account_provisioner +[dms-feature-request::scim-api]: https://github.com/docker-mailserver/docker-mailserver/issues/4090 + +[google::xoauth2-docs]: https://developers.google.com/gmail/imap/xoauth2-protocol#the_sasl_xoauth2_mechanism +[rfc::6750]: https://datatracker.ietf.org/doc/html/rfc6750 +[rfc::7628]: https://datatracker.ietf.org/doc/html/rfc7628 +[gh-issue::curl::xoauth2-bug]: https://github.com/curl/curl/issues/10259#issuecomment-1907192556 + +[authentik::docs::install]: https://goauthentik.io/docs/installation/ +[roundcube::dockerhub-image]: https://hub.docker.com/r/roundcube/roundcubemail +[roundcube::docs::install]: https://github.com/roundcube/roundcubemail/wiki/Installation +[roundcube::docs::config]: https://github.com/roundcube/roundcubemail/wiki/Configuration diff --git a/docs/content/config/advanced/auth-oauth2.md b/docs/content/config/advanced/auth-oauth2.md deleted file mode 100644 index 963a6c2ca09..00000000000 --- a/docs/content/config/advanced/auth-oauth2.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: 'Advanced | Basic OAuth2 Authentication' ---- - -## Introduction - -!!! warning "This is only a supplement to the existing account provisioners" - - Accounts must still be managed via the configured [`ACCOUNT_PROVISIONER`][env::account-provisioner] (FILE or LDAP). - - Reasoning for this can be found in [#3480][gh-pr::oauth2]. Future iterations on this feature may allow it to become a full account provisioner. - -[gh-pr::oauth2]: https://github.com/docker-mailserver/docker-mailserver/pull/3480 -[env::account-provisioner]: ../environment.md#account_provisioner - -The present OAuth2 support provides the capability for 3rd-party applications such as Roundcube to authenticate with DMS (dovecot) by using a token obtained from an OAuth2 provider, instead of passing passwords around. - -## Example (Authentik & Roundcube) - -This example assumes you have: - -- A working DMS server set up -- An Authentik server set up ([documentation](https://goauthentik.io/docs/installation/)) -- A Roundcube server set up (either [docker](https://hub.docker.com/r/roundcube/roundcubemail/) or [bare metal](https://github.com/roundcube/roundcubemail/wiki/Installation)) - -!!! example "Setup Instructions" - - === "1. Docker Mailserver" - Edit the following values in `mailserver.env`: - ```env - # ----------------------------------------------- - # --- OAUTH2 Section ---------------------------- - # ----------------------------------------------- - - # empty => OAUTH2 authentication is disabled - # 1 => OAUTH2 authentication is enabled - ENABLE_OAUTH2=1 - - # Specify the user info endpoint URL of the oauth2 provider - OAUTH2_INTROSPECTION_URL=https://authentik.example.com/application/o/userinfo/ - ``` - - === "2. Authentik" - 1. Create a new OAuth2 provider - 2. Note the client id and client secret - 3. Set the allowed redirect url to the equivalent of `https://roundcube.example.com/index.php/login/oauth` for your RoundCube instance. - - === "3. Roundcube" - Add the following to `oauth2.inc.php` ([documentation](https://github.com/roundcube/roundcubemail/wiki/Configuration)): - - ```php - $config['oauth_provider'] = 'generic'; - $config['oauth_provider_name'] = 'Authentik'; - $config['oauth_client_id'] = ''; - $config['oauth_client_secret'] = ''; - $config['oauth_auth_uri'] = 'https://authentik.example.com/application/o/authorize/'; - $config['oauth_token_uri'] = 'https://authentik.example.com/application/o/token/'; - $config['oauth_identity_uri'] = 'https://authentik.example.com/application/o/userinfo/'; - - // Optional: disable SSL certificate check on HTTP requests to OAuth server. For possible values, see: - // http://docs.guzzlephp.org/en/stable/request-options.html#verify - $config['oauth_verify_peer'] = false; - - $config['oauth_scope'] = 'email openid profile'; - $config['oauth_identity_fields'] = ['email']; - - // Boolean: automatically redirect to OAuth login when opening Roundcube without a valid session - $config['oauth_login_redirect'] = false; - ``` diff --git a/docs/content/config/advanced/dovecot-master-accounts.md b/docs/content/config/advanced/dovecot-master-accounts.md deleted file mode 100755 index 3b2afe2aca7..00000000000 --- a/docs/content/config/advanced/dovecot-master-accounts.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -title: 'Advanced | Dovecot master accounts' ---- - -## Introduction - -A dovecot master account is able to login as any configured user. This is useful for administrative tasks like hot backups. - -## Configuration - -It is possible to create, update, delete and list dovecot master accounts using `setup.sh`. See `setup.sh help` for usage. - -This feature is presently [not supported with LDAP](https://github.com/docker-mailserver/docker-mailserver/pull/2535). - -## Logging in - -Once a master account is configured, it is possible to connect to any users mailbox using this account. Log in over POP3/IMAP using the following credential scheme: - -Username: `*` - -Password: `` diff --git a/docs/content/config/advanced/mail-sieve.md b/docs/content/config/advanced/mail-sieve.md index 759e4f7880b..93ac86017bf 100644 --- a/docs/content/config/advanced/mail-sieve.md +++ b/docs/content/config/advanced/mail-sieve.md @@ -81,22 +81,76 @@ For more examples or a detailed description of the Sieve language have a look at [sieve-info::examples]: http://sieve.info/examplescripts [third-party::sieve-examples]: https://support.tigertech.net/sieve#sieve-example-rules-jmp -## Automatic Sorting Based on Subaddresses +## Automatic Sorting Based on Sub-addresses { #subaddress-mailbox-routing } -It is possible to sort subaddresses such as `user+mailing-lists@example.com` into a corresponding folder (here: `INBOX/Mailing-lists`) automatically. +When mail is delivered to your account, it is possible to organize storing mail into folders by the [subaddress (tag)][docs::accounts-subaddressing] used. -```sieve -require ["envelope", "fileinto", "mailbox", "subaddress", "variables"]; - -if envelope :detail :matches "to" "*" { - set :lower :upperfirst "tag" "${1}"; - if mailboxexists "INBOX.${1}" { - fileinto "INBOX.${1}"; - } else { - fileinto :create "INBOX.${tag}"; - } -} -``` +!!! example "Example: `user+@example.com` to `INBOX/`" + + This example sorts mail into inbox folders by their tag: + + ```sieve title="docker-data/dms/config/user@example.com.dovecot.sieve" + require ["envelope", "fileinto", "mailbox", "subaddress", "variables"]; + + # Check if the mail recipient address has a tag (:detail) + if envelope :detail :matches "to" "*" { + # Create a variable `tag`, with the the captured `to` value normalized (SoCIAL => Social) + set :lower :upperfirst "tag" "${1}"; + + # Store the mail into a folder with the tag name, nested under your inbox folder: + if mailboxexists "INBOX.${tag}" { + fileinto "INBOX.${tag}"; + } else { + fileinto :create "INBOX.${tag}"; + } + } + ``` + + When receiving mail for `user+social@example.com` it would be delivered into the `INBOX/Social` folder. + +??? tip "Only redirect mail for specific tags" + + If you want to only handle specific tags, you could replace the envelope condition and tag assignment from the prior example with: + + ```sieve title="docker-data/dms/config/user@example.com.dovecot.sieve" + # Instead of `:matches`, use the default comparator `:is` (exact match) + if envelope :detail "to" "social" { + set "tag" "Social"; + ``` + + ```sieve title="docker-data/dms/config/user@example.com.dovecot.sieve" + # Alternatively you can also provide a list of values to match: + if envelope :detail "to" ["azure", "aws"] { + set "tag" "Cloud"; + ``` + + ```sieve title="docker-data/dms/config/user@example.com.dovecot.sieve" + # Similar to `:matches`, except `:regex` provides enhanced pattern matching. + # NOTE: This example needs you to `require` the "regex" extension + if envelope :detail :regex "to" "^cloud-(azure|aws)$" { + # Normalize the captured azure/aws tag as the resolved value is no longer fixed: + set :lower :upperfirst "vendor" "${1}"; + # If a `.` exists in the tag, it will create nested folders: + set "tag" "Cloud.${vendor}"; + ``` + + **NOTE:** There is no need to lowercase the tag in the conditional as the [`to` value is a case-insensitive check][sieve-docs::envelope]. + +??? abstract "Technical Details" + + - Dovecot supports this feature via the _Sieve subaddress extension_ ([RFC 5233][rfc::5233::sieve-subaddress]). + - Only a single tag per subaddress is supported. Any additional tag delimiters are part of the tag value itself. + - The Dovecot setting [`recipient_delimiter`][dovecot-docs::config::recipient_delimiter] (default: `+`) configures the tag delimiter. This is where the `local-part` of the recipient address will split at, providing the `:detail` (tag) value for Sieve. + + --- + + `INBOX` is the [default namespace configured by Dovecot][dovecot-docs::namespace]. + + - If you omit the `INBOX.` prefix from the sieve script above, the mailbox (folder) for that tag is created at the top-level alongside your Trash and Junk folders. + - The `.` between `INBOX` and `${tag}` is important as a [separator to distinguish mailbox names][dovecot-docs::mailbox-names]. This can vary by mailbox format or configuration. DMS uses [`Maildir`][dovecot-docs::mailbox-formats::maildir] by default, which uses `.` as the separator. + - [`lmtp_save_to_detail_mailbox = yes`][dovecot-docs::config::lmtp_save_to_detail_mailbox] can be set in `/etc/dovecot/conf.d/20-lmtp.conf`: + - This implements the feature globally, except for the tag normalization and `INBOX.` prefix parts of the example script. + - However, if the sieve script is also present, the script has precedence and will handle this task instead when the condition is successful, otherwise falling back to the global feature. ## Manage Sieve @@ -104,8 +158,7 @@ The [Manage Sieve](https://doc.dovecot.org/admin_manual/pigeonhole_managesieve_s !!! example - ```yaml - # compose.yaml + ```yaml title="compose.yaml" ports: - "4190:4190" environment: @@ -122,3 +175,14 @@ The extension is known to work with the following ManageSieve clients: - **[Sieve Editor](https://github.com/thsmi/sieve)** a portable standalone application based on the former Thunderbird plugin. - **[Kmail](https://kontact.kde.org/components/kmail/)** the mail client of [KDE](https://kde.org/)'s Kontact Suite. + +[docs::accounts-subaddressing]: ../account-management/overview.md#sub-addressing + +[dovecot-docs::namespace]: https://doc.dovecot.org/configuration_manual/namespace/ +[dovecot-docs::mailbox-names]: https://doc.dovecot.org/configuration_manual/sieve/usage/#mailbox-names +[dovecot-docs::mailbox-formats::maildir]: https://doc.dovecot.org/admin_manual/mailbox_formats/maildir/#maildir-mbox-format +[dovecot-docs::config::lmtp_save_to_detail_mailbox]: https://doc.dovecot.org/settings/core/#core_setting-lmtp_save_to_detail_mailbox +[dovecot-docs::config::recipient_delimiter]: https://doc.dovecot.org/settings/core/#core_setting-recipient_delimiter + +[rfc::5233::sieve-subaddress]: https://datatracker.ietf.org/doc/html/rfc5233 +[sieve-docs::envelope]: https://thsmi.github.io/sieve-reference/en/test/core/envelope.html diff --git a/docs/content/config/advanced/optional-config.md b/docs/content/config/advanced/optional-config.md index fb1ab29dc27..57fa68dee14 100644 --- a/docs/content/config/advanced/optional-config.md +++ b/docs/content/config/advanced/optional-config.md @@ -18,10 +18,7 @@ DMS has several locations in the container which may be worth persisting externa - [Config](#volumes-config): `docker-data/dms/config/` => `/tmp/docker-mailserver/` - [Mail Storage](#volumes-mail): `docker-data/dms/mail-data/` => `/var/mail/` - [State](#volumes-state): `docker-data/dms/mail-state/` => `/var/mail-state/` - - [Logs](#volumes-logs): `docker-data/dms/mail-logs/` => `/var/log/mail/` - -[docker-docs::volumes]: https://docs.docker.com/storage/volumes/ -[docker-docs::volumes::bind-mount]: https://docs.docker.com/storage/bind-mounts/ + - [Logs](#volumes-log): `docker-data/dms/mail-logs/` => `/var/log/mail/` ### Mail Storage Volume { #volumes-mail } @@ -80,8 +77,8 @@ This is a list of all configuration files and directories which are optional, au - **postfix-send-access.cf:** List of users denied sending. Modify via [`setup.sh email restrict`][docs-setupsh]. - **postfix-receive-access.cf:** List of users denied receiving. Modify via [`setup.sh email restrict`][docs-setupsh]. - **postfix-virtual.cf:** Alias configuration file. Modify via [`setup.sh alias`][docs-setupsh]. -- **postfix-sasl-password.cf:** listing of relayed domains with their respective `:`. Modify via `setup.sh relay add-auth []`. (Docs: [Relay-Hosts Auth][docs-relayhosts-senderauth]) -- **postfix-relaymap.cf:** domain-specific relays and exclusions. Modify via `setup.sh relay add-domain` and `setup.sh relay exclude-domain`. (Docs: [Relay-Hosts Senders][docs-relayhosts-senderhost]) +- **postfix-sasl-password.cf:** listing of relayed domains with their respective `:`. Modify via `setup.sh relay add-auth []`. (Docs: [Relay-Hosts Auth][docs::relay-hosts::advanced]) +- **postfix-relaymap.cf:** domain-specific relays and exclusions. Modify via `setup.sh relay add-domain` and `setup.sh relay exclude-domain`. (Docs: [Relay-Hosts Senders][docs::relay-hosts::advanced]) - **postfix-regexp.cf:** Regular expression alias file. (Docs: [Aliases][docs-aliases-regex]) - **ldap-users.cf:** Configuration for the virtual user mapping `virtual_mailbox_maps`. See the [`setup-stack.sh`][github-commit-setup-stack.sh-L411] script. - **ldap-groups.cf:** Configuration for the virtual alias mapping `virtual_alias_maps`. See the [`setup-stack.sh`][github-commit-setup-stack.sh-L411] script. @@ -97,16 +94,18 @@ This is a list of all configuration files and directories which are optional, au - **user-patches.sh:** this file will be run after all configuration files are set up, but before the postfix, amavis and other daemons are started. (Docs: [FAQ - How to adjust settings with the `user-patches.sh` script][docs-faq-userpatches]) - **rspamd/custom-commands.conf:** list of simple commands to adjust Rspamd modules in an easy way (Docs: [Rspamd][docs-rspamd-commands]) -[docs-accounts-quota]: ../../config/user-management.md#quotas -[docs-aliases-regex]: ../../config/user-management.md#configuring-regexp-aliases +[docker-docs::volumes]: https://docs.docker.com/storage/volumes/ +[docker-docs::volumes::bind-mount]: https://docs.docker.com/storage/bind-mounts/ + +[docs-accounts-quota]: ../../config/account-management/provisioner/file.md#quotas +[docs-aliases-regex]: ../../config/account-management/provisioner/file.md#configuring-regex-aliases [docs-dkim]: ../../config/best-practices/dkim_dmarc_spf.md#dkim [docs-fail2ban]: ../../config/security/fail2ban.md [docs-faq-spamrules]: ../../faq.md#how-can-i-manage-my-custom-spamassassin-rules [docs-faq-userpatches]: ../../faq.md#how-to-adjust-settings-with-the-user-patchessh-script [docs-override-postfix]: ./override-defaults/postfix.md [docs-override-dovecot]: ./override-defaults/dovecot.md -[docs-relayhosts-senderauth]: ./mail-forwarding/relay-hosts.md#sender-dependent-authentication -[docs-relayhosts-senderhost]: ./mail-forwarding/relay-hosts.md#sender-dependent-relay-host +[docs::relay-hosts::advanced]: ./mail-forwarding/relay-hosts.md#advanced-configuration [docs-sieve]: ./mail-sieve.md [docs-setupsh]: ../../config/setup.sh.md [docs-ssl]: ../../config/security/ssl.md diff --git a/docs/content/config/advanced/podman.md b/docs/content/config/advanced/podman.md index 4cb60e77952..236eeb62a1c 100644 --- a/docs/content/config/advanced/podman.md +++ b/docs/content/config/advanced/podman.md @@ -107,7 +107,7 @@ The `PERMIT_DOCKER` variable in the `mailserver.env` file allows to specify trus #### Use the slip4netns network driver The second workaround is slightly more complicated because the `compose.yaml` has to be modified. -As shown in the [fail2ban section](../security/fail2ban.md#podman-with-slirp4netns-port-driver) the `slirp4netns` network driver has to be enabled. +As shown in the [fail2ban section][docs::fail2ban::rootless] the `slirp4netns` network driver has to be enabled. This network driver enables podman to correctly resolve IP addresses but it is not compatible with user defined networks which might be a problem depending on your setup. @@ -150,7 +150,7 @@ Remember to run this command as root user. ### Port Forwarding -When it comes to forwarding ports using `firewalld`, see for more information. +When it comes to forwarding ports using `firewalld`, see [these port forwarding docs][firewalld-port-forwarding] for more information. ```bash firewall-cmd --permanent --add-forward-port=port=<25|143|465|587|993>:proto=:toport=<10025|10143|10465|10587|10993> @@ -171,5 +171,7 @@ firewall-cmd --reload Just map all the privilege port with non-privilege port you set in compose.yaml before as root user. +[docs::fail2ban::rootless]: ../security/fail2ban.md#rootless-container [rootless::podman]: https://github.com/containers/podman/blob/v3.4.1/docs/source/markdown/podman-run.1.md#--networkmode---net [rootless::podman::interface]: https://github.com/containers/podman/blob/v3.4.1/libpod/networking_slirp4netns.go#L264 +[firewalld-port-forwarding]: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/8/html/securing_networks/using-and-configuring-firewalld_securing-networks#port-forwarding_using-and-configuring-firewalld diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md index d6ba06b7e7e..2ef8e902751 100644 --- a/docs/content/config/best-practices/dkim_dmarc_spf.md +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -359,7 +359,7 @@ volumes: - ./docker-data/dms/config/postfix-policyd-spf.conf:/etc/postfix-policyd-spf-python/policyd-spf.conf ``` -[docs-accounts-add]: ../user-management.md#adding-a-new-account +[docs-accounts]: ../account-management/overview.md#accounts [docs-volumes-config]: ../advanced/optional-config.md#volumes-config [docs-env-opendkim]: ../environment.md#enable_opendkim [docs-env-rspamd]: ../environment.md#enable_rspamd diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index 2c89948db07..30fee3831e7 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -123,7 +123,7 @@ This could be from outdated software, or running a system that isn't able to pro [docs::faq-bare-domain]: ../faq.md#can-i-use-a-nakedbare-domain-ie-no-hostname [docs-ipv6]: ./advanced/ipv6.md [docs-introduction]: ../introduction.md -[docs-rootless-portdriver]: ./security/fail2ban.md#running-inside-a-rootless-container +[docs::fail2ban::rootless-portdriver]: ./security/fail2ban.md#rootless-container [docs-usage]: ../usage.md [gh-issues]: https://github.com/docker-mailserver/docker-mailserver/issues diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 867c7459a0f..2ebd092c7d5 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -47,24 +47,12 @@ The Group ID assigned to the static vmail group for `/var/mail` (_Mail storage m ##### ACCOUNT_PROVISIONER -Configures the provisioning source of user accounts (including aliases) for user queries and authentication by services managed by DMS (_Postfix and Dovecot_). +Configures the [provisioning source of user accounts][docs::account-management::overview] (including aliases) for user queries and authentication by services managed by DMS (_Postfix and Dovecot_). -!!! tip "OAuth2 Support" - - Presently DMS supports OAuth2 only as an supplementary authentication method. - - - A third-party service must provide a valid token for the user which Dovecot validates with the authentication service provider. To enable this feature reference the [OAuth2 configuration example guide][docs::auth::oauth2-config-guide]. - - User accounts must be provisioned to receive mail via one of the supported `ACCOUNT_PROVISIONER` providers. - - User provisioning via OIDC is planned for the future, see [this tracking issue](https://github.com/docker-mailserver/docker-mailserver/issues/2713). - -[docs::auth::oauth2-config-guide]: ./advanced/auth-oauth2.md - -- **empty** => use FILE +- **FILE** => use local files - LDAP => use LDAP authentication -- OIDC => use OIDC authentication (**not yet implemented**) -- FILE => use local files (this is used as the default) -A second container for the ldap service is necessary (e.g. [`bitnami/openldap`](https://hub.docker.com/r/bitnami/openldap/)). +LDAP requires an external service (e.g. [`bitnami/openldap`](https://hub.docker.com/r/bitnami/openldap/)). ##### PERMIT_DOCKER @@ -1140,7 +1128,8 @@ Provide the credentials to use with `RELAY_HOST` or `DEFAULT_RELAY_HOST`. [docs-tls-letsencrypt]: ./security/ssl.md#lets-encrypt-recommended [docs-tls-manual]: ./security/ssl.md#bring-your-own-certificates [docs-tls-selfsigned]: ./security/ssl.md#self-signed-certificates -[docs-accounts-quota]: ./user-management.md#quotas +[docs-accounts-quota]: ./account-management/overview.md#quotas +[docs::account-management::overview]: ./account-management/overview.md [docs::relay-host]: ./advanced/mail-forwarding/relay-hosts.md [docs::dms-volumes-state]: ./advanced/optional-config.md#volumes-state [postfix-config::relayhost]: https://www.postfix.org/postconf.5.html#relayhost diff --git a/docs/content/config/security/fail2ban.md b/docs/content/config/security/fail2ban.md index 375f440cb11..08852274536 100644 --- a/docs/content/config/security/fail2ban.md +++ b/docs/content/config/security/fail2ban.md @@ -78,7 +78,7 @@ docker exec setup fail2ban [ ] docker exec setup fail2ban log ``` -## Running Inside A Rootless Container +## Running Inside A Rootless Container { #rootless-container } [`RootlessKit`][rootless::rootless-kit] is the _fakeroot_ implementation for supporting _rootless mode_ in Docker and Podman. By default, RootlessKit uses the [`builtin` port forwarding driver][rootless::port-drivers], which does not propagate source IP addresses. diff --git a/docs/content/config/security/ssl.md b/docs/content/config/security/ssl.md index eeb39a5f069..dd633c9d6f7 100644 --- a/docs/content/config/security/ssl.md +++ b/docs/content/config/security/ssl.md @@ -6,7 +6,7 @@ There are multiple options to enable SSL (via [`SSL_TYPE`][docs-env::ssl-type]): - Using [letsencrypt](#lets-encrypt-recommended) (recommended) - Using [Caddy](#caddy) -- Using [Traefik](#traefik-v2) +- Using [Traefik](#traefik) - Using [self-signed certificates](#self-signed-certificates) - Using [your own certificates](#bring-your-own-certificates) diff --git a/docs/content/config/security/understanding-the-ports.md b/docs/content/config/security/understanding-the-ports.md index ede8ca3b995..47e46e95152 100644 --- a/docs/content/config/security/understanding-the-ports.md +++ b/docs/content/config/security/understanding-the-ports.md @@ -145,7 +145,7 @@ Unlike with HTTP where a web browser client communicates directly with the serve Other machines that facilitate a connection that generally aren't taken into account can exist between a client and server, such as those where your connection passes through your ISP provider are capable of compromising a `cleartext` connection through interception. -[docs-accounts]: ../user-management.md#accounts +[docs-accounts]: ../account-management/overview.md#accounts [docs-relays]: ../advanced/mail-forwarding/relay-hosts.md [iana-services-465]: https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml?search=465 [starttls-policy-list]: https://github.com/EFForg/starttls-everywhere#email-security-database-starttls-policy-list diff --git a/docs/content/config/user-management.md b/docs/content/config/user-management.md deleted file mode 100644 index a909157e9ff..00000000000 --- a/docs/content/config/user-management.md +++ /dev/null @@ -1,86 +0,0 @@ -# User Management - -## Accounts - -Users (email accounts) are managed in `/tmp/docker-mailserver/postfix-accounts.cf`. The best way to manage accounts is to use the reliable `setup` command inside the container. Just run `docker exec setup help` and have a look at the section about subcommands, specifically the `email` subcommand. - -### Adding a new Account - -#### Via `setup` inside the container - -You can add an account by running `docker exec -ti setup email add `. This method is strongly preferred. - -#### Manually - -!!! warning - - This method is discouraged! - -Alternatively, you may directly add the full email address and its encrypted password, separated by a pipe. To generate a new mail account data, directly from your host, you could for example run the following: - -```sh -docker run --rm -it \ - --env MAIL_USER=user1@example.com \ - --env MAIL_PASS=mypassword \ - ghcr.io/docker-mailserver/docker-mailserver:latest \ - /bin/bash -c \ - 'echo "${MAIL_USER}|$(doveadm pw -s SHA512-CRYPT -u ${MAIL_USER} -p ${MAIL_PASS})" >>docker-data/dms/config/postfix-accounts.cf' -``` - -You will then be asked for a password, and be given back the data for a new account entry, as text. To actually _add_ this new account, just copy all the output text in `docker-data/dms/config/postfix-accounts.cf` file of your running container. - -The result could look like this: - -```cf -user1@example.com|{SHA512-CRYPT}$6$2YpW1nYtPBs2yLYS$z.5PGH1OEzsHHNhl3gJrc3D.YMZkvKw/vp.r5WIiwya6z7P/CQ9GDEJDr2G2V0cAfjDFeAQPUoopsuWPXLk3u1 -``` - -### Quotas - -- `imap-quota` is enabled and allow clients to query their mailbox usage. -- When the mailbox is deleted, the quota directive is deleted as well. -- Dovecot quotas support LDAP, **but it's not implemented** (_PRs are welcome!_). - -## Aliases - -The best way to manage aliases is to use the reliable `setup` script inside the container. Just run `docker exec setup help` and have a look at the section about subcommands, specifically the `alias`-subcommand. - -### About - -You may read [Postfix's documentation on virtual aliases][postfix-docs-alias] first. Aliases are managed in `/tmp/docker-mailserver/postfix-virtual.cf`. An alias is a full email address that will either be: - -- delivered to an existing account registered in `/tmp/docker-mailserver/postfix-accounts.cf` -- redirected to one or more other email addresses - -Alias and target are space separated. An example on a server with `example.com` as its domain: - -```cf -# Alias delivered to an existing account -alias1@example.com user1@example.com - -# Alias forwarded to an external email address -alias2@example.com external-account@gmail.com -``` - -### Configuring RegExp Aliases - -Additional regexp aliases can be configured by placing them into `docker-data/dms/config/postfix-regexp.cf`. The regexp aliases get evaluated after the virtual aliases (container path: `/tmp/docker-mailserver/postfix-virtual.cf`). For example, the following `docker-data/dms/config/postfix-regexp.cf` causes all email sent to "test" users to be delivered to `qa@example.com` instead: - -```cf -/^test[0-9][0-9]*@example.com/ qa@example.com -``` - -### Address Tags (Extension Delimiters) as an alternative to Aliases - -Postfix supports so-called address tags, in the form of plus (+) tags - i.e. `address+tag@example.com` will end up at `address@example.com`. This is configured by default and the (configurable!) separator is set to `+`. For more info, see [Postfix's official documentation][postfix-docs-extension-delimiters]. - -!!! note - - If you do decide to change the configurable separator, you must add the same line to *both* `docker-data/dms/config/postfix-main.cf` and `docker-data/dms/config/dovecot.cf`, because Dovecot is acting as the delivery agent. For example, to switch to `-`, add: - - ```cf - recipient_delimiter = - - ``` - -[postfix-docs-alias]: http://www.postfix.org/VIRTUAL_README.html#virtual_alias -[postfix-docs-extension-delimiters]: http://www.postfix.org/postconf.5.html#recipient_delimiter diff --git a/docs/content/contributing/issues-and-pull-requests.md b/docs/content/contributing/issues-and-pull-requests.md index 0a32baabb9b..c454e7205f8 100644 --- a/docs/content/contributing/issues-and-pull-requests.md +++ b/docs/content/contributing/issues-and-pull-requests.md @@ -45,15 +45,15 @@ The development workflow is the following: 1. Fork the project and clone your fork with `git clone --recurse-submodules ...` or run `git submodule update --init --recursive` after you cloned your fork 2. Write the code that is needed :D 3. Add integration tests if necessary -4. [Prepare your environment and run linting and tests][docs-general-tests] -5. Document your improvements if necessary (e.g. if you introduced new environment variables, describe those in the [ENV documentation][docs-environment]) and add your changes the changelog under the "Unreleased" section +4. [Prepare your environment and run linting and tests][docs::contributing::tests] +5. Document your improvements if necessary (e.g. if you introduced new environment variables, describe those in the [ENV documentation][docs::env]) and add your changes the changelog under the "Unreleased" section 6. [Commit][commit] (and [sign your commit][gpg]), push and create a pull-request to merge into `master`. Please **use the pull-request template** to provide a minimum of contextual information and make sure to meet the requirements of the checklist. Pull requests are automatically tested against the CI and will be reviewed when tests pass. When your changes are validated, your branch is merged. CI builds the new `:edge` image immediately and your changes will be includes in the next version release. [docs-latest]: https://docker-mailserver.github.io/docker-mailserver/latest [github-file-readme]: https://github.com/docker-mailserver/docker-mailserver/blob/master/README.md -[docs-environment]: ../config/environment.md -[docs-general-tests]: ./general.md#tests +[docs::env]: ../config/environment.md +[docs::contributing::tests]: ./tests.md [commit]: https://help.github.com/articles/closing-issues-via-commit-messages/ [gpg]: https://docs.github.com/en/github/authenticating-to-github/generating-a-new-gpg-key diff --git a/docs/content/examples/tutorials/mailserver-behind-proxy.md b/docs/content/examples/tutorials/mailserver-behind-proxy.md index 0c39c125a3d..114877474ca 100644 --- a/docs/content/examples/tutorials/mailserver-behind-proxy.md +++ b/docs/content/examples/tutorials/mailserver-behind-proxy.md @@ -136,6 +136,8 @@ The below guidance is focused on configuring [Traefik][traefik-web], but the adv Postfix and Dovecot are both compatible with PROXY protocol v1 and v2. +#### Ports + ??? abstract "Technical Details - Ports (Traefik config)" !!! info "Explicit TLS (STARTTLS)" @@ -386,7 +388,7 @@ While PROXY protocol works well with the reverse proxy, you may have some contai [docs::overrides::postfix]: ../../config/advanced/override-defaults/postfix.md [docs::overrides::user-patches]: ../../config/advanced/override-defaults/user-patches.md [docs::ipv6::security-risks]: ../../config/advanced/ipv6.md#what-can-go-wrong -[docs::tls::traefik]: ../../config/security/ssl.md#traefik-v2 +[docs::tls::traefik]: ../../config/security/ssl.md#traefik [docs::env::permit_docker]: ../../config/environment.md#permit_docker [gh-dms::dns-rewrite-example]: https://github.com/docker-mailserver/docker-mailserver/issues/3866#issuecomment-1928877236 diff --git a/docs/content/examples/use-cases/auth-lua.md b/docs/content/examples/use-cases/auth-lua.md index e34ed3b0de2..a7fe50c6be6 100644 --- a/docs/content/examples/use-cases/auth-lua.md +++ b/docs/content/examples/use-cases/auth-lua.md @@ -156,7 +156,7 @@ If working with HTTP in Lua, setting `debug = true;` when initiating `dovecot.ht Note that Lua runs compiled bytecode, and that scripts will be compiled when they are initially started. Once compiled, the bytecode is cached and changes in the Lua script will not be processed automatically. Dovecot will reload its configuration and clear its cached Lua bytecode when running `docker exec CONTAINER_NAME dovecot reload`. A (changed) Lua script will be compiled to bytecode the next time it is executed after running the Dovecot reload command. -[docs::auth-ldap]: ../../config/advanced/auth-ldap.md +[docs::auth-ldap]: ../../config/account-management/provisioner/ldap.md [docs::dovecot-override-configuration]: ../../config/advanced/override-defaults/dovecot.md#override-configuration [docs::dovecot-add-configuration]: ../../config/advanced/override-defaults/dovecot.md#add-configuration [docs::faq-alter-running-dms-instance-without-container-relaunch]: ../../faq.md#how-to-alter-a-running-dms-instance-without-relaunching-the-container diff --git a/docs/content/usage.md b/docs/content/usage.md index 087547ea5ad..ca4e8dfef81 100644 --- a/docs/content/usage.md +++ b/docs/content/usage.md @@ -164,7 +164,7 @@ You definitely want to setup TLS. Please refer to [our documentation about TLS][ You should add at least one [alias][docs-aliases], the [_postmaster alias_][docs-env-postmaster]. This is a common convention, but not strictly required. -[docs-aliases]: ./config/user-management.md#aliases +[docs-aliases]: ./config/account-management/overview.md#aliases [docs-env-postmaster]: ./config/environment.md#postmaster_address ```bash diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index b04631d206c..d8ece917144 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -104,6 +104,13 @@ markdown_extensions: lang: cfg - name: env lang: properties + # We only show PHP snippets, requires config change to work: + # https://squidfunk.github.io/mkdocs-material/setup/extensions/python-markdown-extensions/#highlight + # https://facelessuser.github.io/pymdown-extensions/extensions/highlight/#extended-pygments-lexer-options + - name: php + lang: php + options: + startinline: true # A variant that sometimes has nicer syntax highlighting: - name: cf-extra lang: linuxconfig @@ -131,7 +138,14 @@ nav: - 'Usage': usage.md - 'Configuration': - 'Environment Variables': config/environment.md - - 'User Management': config/user-management.md + - 'Account Management': + - 'Overview': config/account-management/overview.md + - 'Provisioner': + - 'File Based': config/account-management/provisioner/file.md + - 'LDAP Service': config/account-management/provisioner/ldap.md + - 'Supplementary': + - 'Master Accounts': config/account-management/supplementary/master-accounts.md + - 'OAuth2 Authentication': config/account-management/supplementary/oauth2.md - 'Best Practices': - 'Auto-discovery': config/best-practices/autodiscover.md - 'DKIM, DMARC & SPF': config/best-practices/dkim_dmarc_spf.md @@ -153,8 +167,6 @@ nav: - 'Dovecot': config/advanced/override-defaults/dovecot.md - 'Postfix': config/advanced/override-defaults/postfix.md - 'Modifications via Script': config/advanced/override-defaults/user-patches.md - - 'LDAP Authentication': config/advanced/auth-ldap.md - - 'OAuth2 Authentication': config/advanced/auth-oauth2.md - 'Email Filtering with Sieve': config/advanced/mail-sieve.md - 'Email Gathering with Fetchmail': config/advanced/mail-fetchmail.md - 'Email Gathering with Getmail': config/advanced/mail-getmail.md @@ -166,7 +178,6 @@ nav: - 'Kubernetes': config/advanced/kubernetes.md - 'IPv6': config/advanced/ipv6.md - 'Podman': config/advanced/podman.md - - 'Dovecot Master Accounts': config/advanced/dovecot-master-accounts.md - 'Examples': - 'Tutorials': - 'Basic Installation': examples/tutorials/basic-installation.md diff --git a/mailserver.env b/mailserver.env index 1ec88dd2de4..77b863ffb63 100644 --- a/mailserver.env +++ b/mailserver.env @@ -267,7 +267,7 @@ POSTFIX_DAGENT= # empty => 0 POSTFIX_MAILBOX_SIZE_LIMIT= -# See https://docker-mailserver.github.io/docker-mailserver/edge/config/user-management/accounts/#notes +# See https://docker-mailserver.github.io/docker-mailserver/latest/config/account-management/overview/#quotas # 0 => Dovecot quota is disabled # 1 => Dovecot quota is enabled ENABLE_QUOTAS=1 From 009237cc26f76a091d5745e1a26706dd1b937fc7 Mon Sep 17 00:00:00 2001 From: Moritz Poldrack <~git@mp.gy> Date: Mon, 22 Jul 2024 13:43:11 +0200 Subject: [PATCH 399/592] chore: Prevent Microsoft MUAs from sending reactions (#4120) --------- Signed-off-by: Moritz Poldrack <~git@mp.gy> --- CHANGELOG.md | 3 ++- target/postfix/sender_header_filter.pcre | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b39f60d8752..490848c53d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,8 @@ All notable changes to this project will be documented in this file. The format - Bump version to [1.1.0](https://github.com/fail2ban/fail2ban/releases/tag/1.1.0). For more information, check the [changelog](https://github.com/fail2ban/fail2ban/blob/1.1.0/ChangeLog). - **Documentation:** - Rewritten and organized the pages for Account Management and Authentication ([#4122](https://github.com/docker-mailserver/docker-mailserver/pull/4122)) - +- **Postfix** + - Disable Microsoft reactions to outgoing mail. ([#4120](https://github.com/docker-mailserver/docker-mailserver/pull/4120)) #### Fixes diff --git a/target/postfix/sender_header_filter.pcre b/target/postfix/sender_header_filter.pcre index cc0a79abfc3..fe4ff045705 100644 --- a/target/postfix/sender_header_filter.pcre +++ b/target/postfix/sender_header_filter.pcre @@ -8,4 +8,4 @@ /^\s*X-Mailer/ IGNORE /^\s*X-Originating-IP/ IGNORE /^\s*Received: from.*127.0.0.1/ IGNORE - +/^Content-Type:/i PREPEND X-MS-Reactions: disallow From 2edd6936c4af9550a54c49b18ff091c93b0a6ffd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jul 2024 16:48:59 +0200 Subject: [PATCH 400/592] chore(deps): Bump docker/setup-buildx-action from 3.4.0 to 3.5.0 (#4128) --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 5b10ef87de5..a38357fdf44 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -79,7 +79,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.4.0 + uses: docker/setup-buildx-action@v3.5.0 # NOTE: AMD64 can build within 2 minutes - name: 'Build images' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 426009908bb..39e1ca23ce8 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -40,7 +40,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.4.0 + uses: docker/setup-buildx-action@v3.5.0 # Try get the cached build layers from a prior `generic_build.yml` job. # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 1cab38b81ec..5bdc88aab76 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -38,7 +38,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.4.0 + uses: docker/setup-buildx-action@v3.5.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 95b6c4f6ddb..7e67574448f 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -37,7 +37,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.4.0 + uses: docker/setup-buildx-action@v3.5.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. From 720a4a25343fe09743c2ba46a1a09ffaf6e3049c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jul 2024 16:53:57 +0200 Subject: [PATCH 401/592] chore(deps): Bump docker/setup-qemu-action from 3.1.0 to 3.2.0 (#4127) --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index a38357fdf44..db46fcc6a44 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -74,7 +74,7 @@ jobs: cache-buildx- - name: 'Set up QEMU' - uses: docker/setup-qemu-action@v3.1.0 + uses: docker/setup-qemu-action@v3.2.0 with: platforms: arm64 diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 39e1ca23ce8..c10e2ee055f 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -35,7 +35,7 @@ jobs: type=semver,pattern={{major}}.{{minor}}.{{patch}} - name: 'Set up QEMU' - uses: docker/setup-qemu-action@v3.1.0 + uses: docker/setup-qemu-action@v3.2.0 with: platforms: arm64 From 093e43480cedec0261fddc513e959e4fa5d112c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jul 2024 08:46:13 +1200 Subject: [PATCH 402/592] chore(deps): Bump docker/build-push-action from 6.4.0 to 6.5.0 (#4126) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.4.0 to 6.5.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v6.4.0...v6.5.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index db46fcc6a44..dee6ce93ffa 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v6.4.0 + uses: docker/build-push-action@v6.5.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index c10e2ee055f..02ce4ffe888 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -67,7 +67,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: 'Build and publish images' - uses: docker/build-push-action@v6.4.0 + uses: docker/build-push-action@v6.5.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 5bdc88aab76..0c160ad3869 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.4.0 + uses: docker/build-push-action@v6.5.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 7e67574448f..7ccfd995fdb 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.4.0 + uses: docker/build-push-action@v6.5.0 with: context: . tags: mailserver-testing:ci From 32c0a346fcec975ba133443d98e3a1e7c0d78714 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jul 2024 09:12:47 +1200 Subject: [PATCH 403/592] chore(deps): Bump anchore/scan-action from 4.0.0 to 4.1.0 (#4125) Bumps [anchore/scan-action](https://github.com/anchore/scan-action) from 4.0.0 to 4.1.0. - [Release notes](https://github.com/anchore/scan-action/releases) - [Changelog](https://github.com/anchore/scan-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/anchore/scan-action/compare/v4.0.0...v4.1.0) --- updated-dependencies: - dependency-name: anchore/scan-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 7ccfd995fdb..7c231b8eb58 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v4.0.0 + uses: anchore/scan-action@v4.1.0 id: scan with: image: mailserver-testing:ci From c544d770e7f27867e56567bda26a6aa125494638 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 30 Jul 2024 18:34:17 +0200 Subject: [PATCH 404/592] docs: updated `CONTRIBUTORS.md` (#4139) --- CONTRIBUTORS.md | 356 ++++++++++++++++++++++++------------------------ 1 file changed, 178 insertions(+), 178 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 9a270918644..d4cc387b33f 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -329,142 +329,142 @@ Thanks goes to these wonderful people ✨ - - - + + - - + + - - + + + + + + - - - - - - + + + + + + + - - + + - - + + - - + + - - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - -
- - robertdolca + + dashohoxha
- robertdolca + dashohoxha 		- - okainov + + egavard
- okainov + egavard 		- - lukecyca + + mathuin
- lukecyca + mathuin 		- - jsonn + + jamebus
- jsonn + jamebus
- - jamebus + + robertdolca
- jamebus + robertdolca 		- - mathuin + + okainov
- mathuin + okainov 		- - dashohoxha + + jsonn
- dashohoxha + jsonn 		- - egavard + + lukecyca
- egavard + lukecyca 		- - citec + + m-schmoock
- citec + m-schmoock 		- - Zehir + + mjung
- Zehir + mjung
- - guardiande + + VanVan
- guardiande + VanVan 		- - kamuri + + eltociear
- kamuri + eltociear 		- - davidszp + + andreasgerstmayr
- davidszp + andreasgerstmayr 		- - andreasgerstmayr + + voordev
- andreasgerstmayr + voordev 		- - VanVan + + davidszp
- VanVan + davidszp 		- - eltociear + + kamuri
- eltociear + kamuri
- - mjung + + guardiande
- mjung + guardiande 		- - m-schmoock + + Zehir
- m-schmoock + Zehir 		- - Starbix + + Birkenstab
- Starbix + Birkenstab 		@@ -475,26 +475,26 @@ Thanks goes to these wonderful people ✨ - - Birkenstab + + Starbix
- Birkenstab + Starbix 		- - yajo + + citec
- yajo + citec
- - voordev + + yajo
- voordev + yajo 		@@ -505,17 +505,17 @@ Thanks goes to these wonderful people ✨ - - pbek + + weo
- pbek + weo 		- - reneploetz + + analogue
- reneploetz + analogue 		@@ -526,26 +526,19 @@ Thanks goes to these wonderful people ✨ - - analogue + + reneploetz
- analogue + reneploetz
- - weo -
- weo - - 		- - andrewlow + + pbek
- andrewlow + pbek 		@@ -576,8 +569,6 @@ Thanks goes to these wonderful people ✨ Rillke
bobbravo2 @@ -585,6 +576,8 @@ Thanks goes to these wonderful people ✨ bobbravo2
r-pufky @@ -620,8 +613,6 @@ Thanks goes to these wonderful people ✨ engelant
j-marz @@ -629,6 +620,8 @@ Thanks goes to these wonderful people ✨ j-marz
lokipo @@ -664,8 +657,6 @@ Thanks goes to these wonderful people ✨ frugan-dev
tbutter @@ -673,6 +664,8 @@ Thanks goes to these wonderful people ✨ tbutter
yogo1212 @@ -687,6 +680,13 @@ Thanks goes to these wonderful people ✨ willtho89 + + mpldr +
+ mpldr + + 		mpanneck @@ -695,21 +695,28 @@ Thanks goes to these wonderful people ✨ - - aminvakil + + craue
- aminvakil + craue 		- - elbracht + + danielpanteleit
- elbracht + danielpanteleit
+ + ubenmackin +
+ ubenmackin + + 		abh @@ -718,24 +725,24 @@ Thanks goes to these wonderful people ✨ - - craue + + andrewlow
- craue + andrewlow 		- - ubenmackin + + aminvakil
- ubenmackin + aminvakil 		- - danielpanteleit + + elbracht
- danielpanteleit + elbracht 		@@ -745,6 +752,8 @@ Thanks goes to these wonderful people ✨ dmcgrandle
theomega @@ -752,8 +761,6 @@ Thanks goes to these wonderful people ✨ theomega
DuncanvR @@ -768,20 +775,6 @@ Thanks goes to these wonderful people ✨ emazzotta - - keslerm -
- keslerm - - 		- - nueaf -
- nueaf - - 		martinwepner @@ -796,8 +789,6 @@ Thanks goes to these wonderful people ✨ artonge
spacecowboy @@ -805,6 +796,8 @@ Thanks goes to these wonderful people ✨ spacecowboy
jiriks74 @@ -812,6 +805,13 @@ Thanks goes to these wonderful people ✨ jiriks74 + + nueaf +
+ nueaf + + 		jedateach @@ -819,6 +819,13 @@ Thanks goes to these wonderful people ✨ jedateach + + keslerm +
+ keslerm + + 		millaguie @@ -826,6 +833,15 @@ Thanks goes to these wonderful people ✨ millaguie + + fl42 +
+ fl42 + +
jamesfryer @@ -840,8 +856,6 @@ Thanks goes to these wonderful people ✨ H4R0
ipernet @@ -850,10 +864,10 @@ Thanks goes to these wonderful people ✨ - - fl42 + + arkanovicz
- fl42 + arkanovicz 		@@ -870,6 +884,8 @@ Thanks goes to these wonderful people ✨ stigok
5ven @@ -884,8 +900,6 @@ Thanks goes to these wonderful people ✨ syl20bnr
sylvaindumont @@ -914,6 +928,8 @@ Thanks goes to these wonderful people ✨ Thiritin
tobiabocchi @@ -928,8 +944,6 @@ Thanks goes to these wonderful people ✨ tknecht
tweibert @@ -958,6 +972,8 @@ Thanks goes to these wonderful people ✨ Twist235
k3it @@ -972,8 +988,6 @@ Thanks goes to these wonderful people ✨ Drakulix
vilisas @@ -995,20 +1009,6 @@ Thanks goes to these wonderful people ✨ 42wim - - ShiriNmi1520 -
- ShiriNmi1520 - - 		- - neuralp -
- neuralp - - 		radicand @@ -1171,6 +1171,13 @@ Thanks goes to these wonderful people ✨ simonsystem + + ShiriNmi1520 +
+ ShiriNmi1520 + + 		matrixes @@ -1185,6 +1192,8 @@ Thanks goes to these wonderful people ✨ mchamplain
millerjason @@ -1192,8 +1201,6 @@ Thanks goes to these wonderful people ✨ millerjason
mplx @@ -1229,6 +1236,8 @@ Thanks goes to these wonderful people ✨ ontheair81
pravynandas @@ -1236,8 +1245,6 @@ Thanks goes to these wonderful people ✨ pravynandas
presocratics @@ -1273,6 +1280,8 @@ Thanks goes to these wonderful people ✨ schnippl0r
smargold476 @@ -1280,8 +1289,6 @@ Thanks goes to these wonderful people ✨ smargold476
sportshead @@ -1317,6 +1324,8 @@ Thanks goes to these wonderful people ✨ vivacarvajalito
wligtenberg @@ -1324,8 +1333,6 @@ Thanks goes to these wonderful people ✨ wligtenberg
wolkenschieber @@ -1361,6 +1368,8 @@ Thanks goes to these wonderful people ✨ arcaine2
awb99 @@ -1368,8 +1377,6 @@ Thanks goes to these wonderful people ✨ awb99
beertje44 @@ -1405,6 +1412,8 @@ Thanks goes to these wonderful people ✨ dimalo
eleith @@ -1412,8 +1421,6 @@ Thanks goes to these wonderful people ✨ eleith
fanqiaojun @@ -1449,6 +1456,8 @@ Thanks goes to these wonderful people ✨ i-C-o-d-e-r
idaadi @@ -1456,8 +1465,6 @@ Thanks goes to these wonderful people ✨ idaadi
ixeft @@ -1493,6 +1500,8 @@ Thanks goes to these wonderful people ✨ landergate
callmemagnus @@ -1500,8 +1509,6 @@ Thanks goes to these wonderful people ✨ callmemagnus
marios88 @@ -1509,13 +1516,6 @@ Thanks goes to these wonderful people ✨ marios88 - - arkanovicz -
- arkanovicz - - 		CBeerta @@ -1994,17 +1994,17 @@ Thanks goes to these wonderful people ✨ - - mpldr + + naveensrinivasan
- mpldr + naveensrinivasan 		- - naveensrinivasan + + neuralp
- naveensrinivasan + neuralp 		From 01194b7552ec974da58b691c1ee2e238de87333b Mon Sep 17 00:00:00 2001 From: Casper Date: Tue, 30 Jul 2024 18:57:33 +0200 Subject: [PATCH 405/592] docs: Add minimal `compose.yaml` examples that demonstrate specific features (#4138) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- demo-setups/fetchmail-compose.yaml | 60 ++++++++++++ demo-setups/relay-compose.yaml | 145 +++++++++++++++++++++++++++++ 2 files changed, 205 insertions(+) create mode 100644 demo-setups/fetchmail-compose.yaml create mode 100644 demo-setups/relay-compose.yaml diff --git a/demo-setups/fetchmail-compose.yaml b/demo-setups/fetchmail-compose.yaml new file mode 100644 index 00000000000..d8632228056 --- /dev/null +++ b/demo-setups/fetchmail-compose.yaml @@ -0,0 +1,60 @@ +# Docs: https://docker-mailserver.github.io/docker-mailserver/v14.0/config/advanced/mail-fetchmail +# Additional context, with CLI commands for verification: +# https://github.com/orgs/docker-mailserver/discussions/3994#discussioncomment-9290570 + +services: + dms-fetch: + image: ghcr.io/docker-mailserver/docker-mailserver:latest # :14.0 + hostname: mail.example.test + environment: + ENABLE_FETCHMAIL: 1 + # We change this setting to 10 for quicker testing: + FETCHMAIL_POLL: 10 + # Link the DNS lookup `remote.test` to resolve to the `dms-remote` container IP (for `@remote.test` address): + # This is only for this example, since no real DNS service is configured, this is a Docker internal DNS feature: + links: + - "dms-remote:remote.test" + # NOTE: Optional, You only need to publish ports if you want to verify via your own mail client. + #ports: + # - "465:465" # ESMTP (implicit TLS) + # - "993:993" # IMAP4 (implicit TLS) + # You'd normally use `volumes` here but for simplicity of the example, all config is contained within `compose.yaml`: + configs: + - source: dms-accounts-fetch + target: /tmp/docker-mailserver/postfix-accounts.cf + - source: fetchmail + target: /tmp/docker-mailserver/fetchmail.cf + + dms-remote: + image: ghcr.io/docker-mailserver/docker-mailserver:latest # :14.0 + hostname: mail.remote.test + environment: + # Allows for us send a test mail easily by trusting any mail client run within this container (`swaks`): + PERMIT_DOCKER: container + # Alternatively, trust and accept any mail received from clients in same subnet of dms-fetch: + #PERMIT_DOCKER: connected-networks + configs: + - source: dms-accounts-remote + target: /tmp/docker-mailserver/postfix-accounts.cf + +# Using the Docker Compose `configs.content` feature instead of volume mounting separate files. +# NOTE: This feature requires Docker Compose v2.23.1 (Nov 2023) or newer: +# https://github.com/compose-spec/compose-spec/pull/446 +configs: + fetchmail: + content: | + poll 'mail.remote.test' proto imap + user 'jane.doe@remote.test' + pass 'secret' + is 'john.doe@example.test' + no sslcertck + + # DMS requires an account to complete setup, configure one for each instance: + # NOTE: Both accounts are configured with the same password (SHA512-CRYPT hashed), `secret`. + dms-accounts-fetch: + content: | + john.doe@example.test|{SHA512-CRYPT}$$6$$sbgFRCmQ.KWS5ryb$$EsWrlYosiadgdUOxCBHY0DQ3qFbeudDhNMqHs6jZt.8gmxUwiLVy738knqkHD4zj4amkb296HFqQ3yDq4UXt8. + + dms-accounts-remote: + content: | + jane.doe@remote.test|{SHA512-CRYPT}$$6$$sbgFRCmQ.KWS5ryb$$EsWrlYosiadgdUOxCBHY0DQ3qFbeudDhNMqHs6jZt.8gmxUwiLVy738knqkHD4zj4amkb296HFqQ3yDq4UXt8. diff --git a/demo-setups/relay-compose.yaml b/demo-setups/relay-compose.yaml new file mode 100644 index 00000000000..f4df2de058c --- /dev/null +++ b/demo-setups/relay-compose.yaml @@ -0,0 +1,145 @@ +# Docs: https://docker-mailserver.github.io/docker-mailserver/v14.0/config/advanced/mail-forwarding/relay-hosts/ +# Additional context, with CLI commands for verification: +# https://github.com/docker-mailserver/docker-mailserver/issues/4136#issuecomment-2253693490 + +services: + # This would represent your actual DMS container: + dms-sender: + image: mailserver/docker-mailserver:latest # :14.0 + hostname: mail.example.test + environment: + # All outbound mail will be relayed through this host + # (change the port to 587 if you do not want the postfix-main.cf override) + - DEFAULT_RELAY_HOST=[smtp.relay-service.test]:465 + # Your relay host credentials. + # (since the relay in the example is DMS, the relay account username is a full email address) + - RELAY_USER=relay-user@relay-service.test + - RELAY_PASSWORD=secret + # The mail client (swaks) needs to connect with TLS: + - SSL_TYPE=manual + - SSL_KEY_PATH=/tmp/tls/key.pem + - SSL_CERT_PATH=/tmp/tls/cert.pem + # You would usually have `volumes` instead of this `configs`: + configs: + - source: dms-main + target: /tmp/docker-mailserver/postfix-main.cf + - source: dms-accounts + target: /tmp/docker-mailserver/postfix-accounts.cf + # Authenticating on port 587 or 465 enforces TLS requirement: + - source: tls-cert + target: /tmp/tls/cert.pem + - source: tls-key + target: /tmp/tls/key.pem + # This is only needed if you want to verify the TLS cert chain with swaks + # (normally with public CA providers like LetsEncrypt this file is already available to a mail client) + - source: tls-ca-cert + target: /tmp/tls/ca-cert.pem + + # Pretend this is your third-party relay service: + dms-relay: + image: mailserver/docker-mailserver:latest # :14.0 + hostname: smtp.relay-service.test + environment: + # WORKAROUND: Bypass security checks from the mail-client (dms-sender container) + # (avoids needing valid DNS for this example) + - PERMIT_DOCKER=connected-networks + # TLS is required when relaying to dms-relay via ports 587 / 465 + # (dms-relay will then relay the mail to dms-destination over port 25) + - SSL_TYPE=manual + - SSL_KEY_PATH=/tmp/tls/key.pem + - SSL_CERT_PATH=/tmp/tls/cert.pem + # WORKAROUND: `links` is required due to lack of properly configured DNS. + # (resolves destination.test to the IP of the dms-destination container) + links: + - "dms-destination:destination.test" + configs: + - source: dms-accounts-relay + target: /tmp/docker-mailserver/postfix-accounts.cf + - source: tls-cert + target: /tmp/tls/cert.pem + - source: tls-key + target: /tmp/tls/key.pem + + # Pretend this is another mail server that your target recipient belongs to (like Gmail): + dms-destination: + image: mailserver/docker-mailserver:latest # :14.0 + hostname: mail.destination.test + # Same workaround for purposes of the example, with the target recipient provisioned to accept mail + environment: + - PERMIT_DOCKER=connected-networks + configs: + - source: dms-accounts-destination + target: /tmp/docker-mailserver/postfix-accounts.cf + +# Using the Docker Compose `configs.content` feature instead of volume mounting separate files. +# NOTE: This feature requires Docker Compose v2.23.1 (Nov 2023) or newer: +# https://github.com/compose-spec/compose-spec/pull/446 +configs: + # DMS expects an account to be configured to run, this example provides accounts already created. + # Login credentials: + # user: "john.doe@example.test" password: "secret" + # user: "relay-user@relay-service.test" password: "secret" + # user: "jane.doe@destination.test" password: "secret" + dms-accounts: + # NOTE: `$` needed to be repeated to escape it, + # which opts out of the `compose.yaml` variable interpolation feature. + content: | + john.doe@example.test|{SHA512-CRYPT}$$6$$sbgFRCmQ.KWS5ryb$$EsWrlYosiadgdUOxCBHY0DQ3qFbeudDhNMqHs6jZt.8gmxUwiLVy738knqkHD4zj4amkb296HFqQ3yDq4UXt8. + + dms-accounts-relay: + content: | + relay-user@relay-service.test|{SHA512-CRYPT}$$6$$o65y1ZXC4ooOPLwZ$$7TF1nYowEtNJpH6BwJBgdj2pPAxaCvhIKQA6ww5zdHm/AA7aemY9eoHC91DOgYNaKj1HLxSeWNDdvrp6mbtUY. + + dms-accounts-destination: + content: | + jane.doe@destination.test|{SHA512-CRYPT}$$6$$o65y1ZXC4ooOPLwZ$$7TF1nYowEtNJpH6BwJBgdj2pPAxaCvhIKQA6ww5zdHm/AA7aemY9eoHC91DOgYNaKj1HLxSeWNDdvrp6mbtUY. + + # This is `postfix-main.cf`, single line change to make all outbound SMTP connections over port 465 instead of 25 (default) + # If you selectively relay mail, you would need to adjust this on the relay service in `/etc/postfix/master.cf`, + # However DMS presently modifies this when using the DMS Relay Host feature support, which may override `postfix-master.cf` or `user-patches.sh` due to `check-for-changes.sh`. + dms-main: + content: | + smtp_tls_wrappermode=yes + + # TLS files: + # - Use an ECDSA cert that's been signed by a self-signed CA for TLS cert verification. + # - This cert is only valid for mail.example.test, mail.destination.test, smtp.relay-service.test + + # `swaks` run in the container will need to reference this CA cert file for successful verficiation (optional). + tls-ca-cert: + content: | + -----BEGIN CERTIFICATE----- + MIIBfTCCASKgAwIBAgIRAMAZttlRlkcuSun0yV0z4RwwCgYIKoZIzj0EAwIwHDEa + MBgGA1UEAxMRU21hbGxzdGVwIFJvb3QgQ0EwHhcNMjEwMTAxMDAwMDAwWhcNMzEw + MTAxMDAwMDAwWjAcMRowGAYDVQQDExFTbWFsbHN0ZXAgUm9vdCBDQTBZMBMGByqG + SM49AgEGCCqGSM49AwEHA0IABJX2hCtoK3+bM5I3rmyApXLJ1gOcVhtoSSwM8XXR + SEl25Kkc0n6mINuMK8UrBkiBUgexf6CYayx3xVr9TmMkg4KjRTBDMA4GA1UdDwEB + /wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEBMB0GA1UdDgQWBBQD8sBrApbyYyqU + y+/TlwGynx2V5jAKBggqhkjOPQQDAgNJADBGAiEAi8N2eOETI+6hY3+G+kzNMd3K + Sd3Ke8b++/nlwr5Fb/sCIQDYAjpKp/MpTDWICeHC2tcB5ptxoTdWkTBuG4rKcktA + 0w== + -----END CERTIFICATE----- + + tls-key: + content: | + -----BEGIN EC PRIVATE KEY----- + MHcCAQEEIOc6wqZmSDmT336K4O26dMk1RCVc0+cmnsO2eK4P5K5yoAoGCCqGSM49 + AwEHoUQDQgAEFOWNgekKKvUZE89vJ7henUYxODYIvCiHitRc2ylwttjqt1KUY1cp + q3jof2fhURHfBUH3dHPXLHig5V9Jw5gqeg== + -----END EC PRIVATE KEY----- + + tls-cert: + content: | + -----BEGIN CERTIFICATE----- + MIIB9DCCAZqgAwIBAgIQE53a/y2c//YXRsz2kLm6gDAKBggqhkjOPQQDAjAcMRow + GAYDVQQDExFTbWFsbHN0ZXAgUm9vdCBDQTAeFw0yMTAxMDEwMDAwMDBaFw0zMTAx + MDEwMDAwMDBaMBkxFzAVBgNVBAMTDlNtYWxsc3RlcCBMZWFmMFkwEwYHKoZIzj0C + AQYIKoZIzj0DAQcDQgAEFOWNgekKKvUZE89vJ7henUYxODYIvCiHitRc2ylwttjq + t1KUY1cpq3jof2fhURHfBUH3dHPXLHig5V9Jw5gqeqOBwDCBvTAOBgNVHQ8BAf8E + BAMCB4AwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMB0GA1UdDgQWBBSz + w74g+O6dcBbwienD70D8A9ESmDAfBgNVHSMEGDAWgBQD8sBrApbyYyqUy+/TlwGy + nx2V5jBMBgNVHREERTBDghFtYWlsLmV4YW1wbGUudGVzdIIVbWFpbC5kZXN0aW5h + dGlvbi50ZXN0ghdzbXRwLnJlbGF5LXNlcnZpY2UudGVzdDAKBggqhkjOPQQDAgNI + ADBFAiEAoety5oClZtuBMkvlUIWRmWlyg1VIOZ544LSEbplsIhcCIHb6awMwNdXP + m/xHjFkuwH1+UjDDRW53Ih7KZoLrQ6Cp + -----END CERTIFICATE----- From 37e5203a6973956d9afadbd9bc575bc5662e7f56 Mon Sep 17 00:00:00 2001 From: Casper Date: Wed, 31 Jul 2024 09:04:51 +0200 Subject: [PATCH 406/592] ci: Remove CONTRIBUTORS.md (#4141) --- .all-contributorsrc | 26 - .gitattributes | 1 - .github/workflows/contributors.yml | 33 - CHANGELOG.md | 6 +- CONTRIBUTORS.md | 2182 ---------------------------- 5 files changed, 5 insertions(+), 2243 deletions(-) delete mode 100644 .all-contributorsrc delete mode 100644 .github/workflows/contributors.yml delete mode 100644 CONTRIBUTORS.md diff --git a/.all-contributorsrc b/.all-contributorsrc deleted file mode 100644 index c3d39c655d1..00000000000 --- a/.all-contributorsrc +++ /dev/null @@ -1,26 +0,0 @@ -{ - "files": [ - "CONTRIBUTORS.md" - ], - "imageSize": 100, - "commit": false, - "contributors": [ - { - "login": "matrixes", - "name": "matrixes", - "avatar_url": "https://avatars.githubusercontent.com/u/46491408?v=4", - "profile": "https://github.com/matrixes", - "contributions": [ - "blog" - ] - } - ], - "contributorsPerLine": 7, - "badgeTemplate": "", - "projectName": "docker-mailserver", - "projectOwner": "docker-mailserver", - "repoType": "github", - "repoHost": "https://github.com", - "skipCi": false, - "commitConvention": "none" -} diff --git a/.gitattributes b/.gitattributes index da5dadda4f2..8933edd9ab9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -28,7 +28,6 @@ Makefile *.yaml text ## PROJECT -.all-contributorsrc text export-ignore .editorconfig text export-ignore .gitattributes text export-ignore .gitignore text export-ignore diff --git a/.github/workflows/contributors.yml b/.github/workflows/contributors.yml deleted file mode 100644 index 6ca6027bfa8..00000000000 --- a/.github/workflows/contributors.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: 'Update Contributors List' - -on: - workflow_dispatch: - schedule: - - cron: 0 4 * * 0 - -permissions: - contents: write - pull-requests: write - statuses: write - -jobs: - add-contributors: - name: 'Add Contributors' - runs-on: ubuntu-22.04 - steps: - - name: 'Checkout' - uses: actions/checkout@v4 - - - name: 'Update CONTRIBUTORS.md' - uses: akhilmhdh/contributors-readme-action@v2.3.10 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - readme_path: CONTRIBUTORS.md - collaborators: all - use_username: true - commit_message: 'docs: updated `CONTRIBUTORS.md`' - committer_username: github-actions[bot] - committer_email: 41898282+github-actions[bot]@users.noreply.github.com - pr_title_on_protected: 'docs: update `CONTRIBUTORS.md`' - auto_detect_branch_protection: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 490848c53d0..97c48e2a2d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,12 +25,16 @@ All notable changes to this project will be documented in this file. The format - **Postfix** - Disable Microsoft reactions to outgoing mail. ([#4120](https://github.com/docker-mailserver/docker-mailserver/pull/4120)) -#### Fixes +### Fixes - **Dovecot:** - `logwatch` Update logwatch `ignore.conf` to exclude Xapian messages about pending documents ([#4060](https://github.com/docker-mailserver/docker-mailserver/pull/4060)) - `dovecot-fts-xapian` plugin was updated to `1.7.13`, fixing a regression with indexing ([#4095](https://github.com/docker-mailserver/docker-mailserver/pull/4095)) +### CI + +- Workflow for `CONTRIBUTORS.md` updates removed. `CONTRIBUTORS.md` file and dependencies removed. ([#4141](https://github.com/docker-mailserver/docker-mailserver/pull/4141)) + ## [v14.0.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v14.0.0) The most noteworthy change of this release is the update of the container's base image from Debian 11 ("Bullseye") to Debian 12 ("Bookworm"). This update alone involves breaking changes and requires a careful update! diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md deleted file mode 100644 index d4cc387b33f..00000000000 --- a/CONTRIBUTORS.md +++ /dev/null @@ -1,2182 +0,0 @@ -# Contributors - -Thanks goes to these wonderful people ✨ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - casperklein -
- casperklein - - 		- - fbartels -
- fbartels - - 		- - polarathene -
- polarathene - - 		- - NorseGaud -
- NorseGaud - - 		- - wernerfred -
- wernerfred - - 		- - georglauterbach -
- georglauterbach - -
- - tomav -
- tomav - - 		- - erik-wramner -
- erik-wramner - - 		- - chikamichi -
- chikamichi - - 		- - martin-schulze-vireso -
- martin-schulze-vireso - - 		- - Josef-Friedrich -
- Josef-Friedrich - - 		- - johansmitsnl -
- johansmitsnl - -
- - youtous -
- youtous - - 		- - 17Halbe -
- 17Halbe - - 		- - tve -
- tve - - 		- - gmasse -
- gmasse - - 		- - ap-wtioit -
- ap-wtioit - - 		- - 00angus -
- 00angus - -
- - alinmear -
- alinmear - - 		- - dominikwinter -
- dominikwinter - - 		- - crazystick -
- crazystick - - 		- - svenyonson -
- svenyonson - - 		- - swiesend -
- swiesend - - 		- - stonemaster -
- stonemaster - -
- - williamdes -
- williamdes - - 		- - omarc1492 -
- omarc1492 - - 		- - phish108 -
- phish108 - - 		- - mwlczk -
- mwlczk - - 		- - tyranron -
- tyranron - - 		- - KyleOndy -
- KyleOndy - -
- - mindrunner -
- mindrunner - - 		- - MichaelSp -
- MichaelSp - - 		- - m-a-v -
- m-a-v - - 		- - bilak -
- bilak - - 		- - vortex852456 -
- vortex852456 - - 		- - hanscees -
- hanscees - -
- - chris54721 -
- chris54721 - - 		- - jrpear -
- jrpear - - 		- - pyy -
- pyy - - 		- - moqmar -
- moqmar - - 		- - arneke -
- arneke - - 		- - akmet -
- akmet - -
- - diiigle -
- diiigle - - 		- - kiliant -
- kiliant - - 		- - dashohoxha -
- dashohoxha - - 		- - egavard -
- egavard - - 		- - mathuin -
- mathuin - - 		- - jamebus -
- jamebus - -
- - robertdolca -
- robertdolca - - 		- - okainov -
- okainov - - 		- - jsonn -
- jsonn - - 		- - lukecyca -
- lukecyca - - 		- - m-schmoock -
- m-schmoock - - 		- - mjung -
- mjung - -
- - VanVan -
- VanVan - - 		- - eltociear -
- eltociear - - 		- - andreasgerstmayr -
- andreasgerstmayr - - 		- - voordev -
- voordev - - 		- - davidszp -
- davidszp - - 		- - kamuri -
- kamuri - -
- - guardiande -
- guardiande - - 		- - Zehir -
- Zehir - - 		- - Birkenstab -
- Birkenstab - - 		- - BrandonSchmitt -
- BrandonSchmitt - - 		- - Starbix -
- Starbix - - 		- - citec -
- citec - -
- - yajo -
- yajo - - 		- - MakerMatrix -
- MakerMatrix - - 		- - weo -
- weo - - 		- - analogue -
- analogue - - 		- - Rubytastic2 -
- Rubytastic2 - - 		- - reneploetz -
- reneploetz - -
- - pbek -
- pbek - - 		- - castorinop -
- castorinop - - 		- - p-fruck -
- p-fruck - - 		- - rahilarious -
- rahilarious - - 		- - Rillke -
- Rillke - - 		- - bobbravo2 -
- bobbravo2 - -
- - r-pufky -
- r-pufky - - 		- - vincentDcmps -
- vincentDcmps - - 		- - andymel123 -
- andymel123 - - 		- - bigpigeon -
- bigpigeon - - 		- - engelant -
- engelant - - 		- - j-marz -
- j-marz - -
- - lokipo -
- lokipo - - 		- - mmehnert -
- mmehnert - - 		- - msheakoski -
- msheakoski - - 		- - GoliathLabs -
- GoliathLabs - - 		- - frugan-dev -
- frugan-dev - - 		- - tbutter -
- tbutter - -
- - yogo1212 -
- yogo1212 - - 		- - willtho89 -
- willtho89 - - 		- - mpldr -
- mpldr - - 		- - mpanneck -
- mpanneck - - 		- - craue -
- craue - - 		- - danielpanteleit -
- danielpanteleit - -
- - ubenmackin -
- ubenmackin - - 		- - abh -
- abh - - 		- - andrewlow -
- andrewlow - - 		- - aminvakil -
- aminvakil - - 		- - elbracht -
- elbracht - - 		- - dmcgrandle -
- dmcgrandle - -
- - theomega -
- theomega - - 		- - DuncanvR -
- DuncanvR - - 		- - emazzotta -
- emazzotta - - 		- - martinwepner -
- martinwepner - - 		- - artonge -
- artonge - - 		- - spacecowboy -
- spacecowboy - -
- - jiriks74 -
- jiriks74 - - 		- - nueaf -
- nueaf - - 		- - jedateach -
- jedateach - - 		- - keslerm -
- keslerm - - 		- - millaguie -
- millaguie - - 		- - fl42 -
- fl42 - -
- - jamesfryer -
- jamesfryer - - 		- - H4R0 -
- H4R0 - - 		- - ipernet -
- ipernet - - 		- - arkanovicz -
- arkanovicz - - 		- - stephan-devop -
- stephan-devop - - 		- - stigok -
- stigok - -
- - 5ven -
- 5ven - - 		- - syl20bnr -
- syl20bnr - - 		- - sylvaindumont -
- sylvaindumont - - 		- - TechnicLab -
- TechnicLab - - 		- - thomasschmit -
- thomasschmit - - 		- - Thiritin -
- Thiritin - -
- - tobiabocchi -
- tobiabocchi - - 		- - tknecht -
- tknecht - - 		- - tweibert -
- tweibert - - 		- - torus -
- torus - - 		- - VictorKoenders -
- VictorKoenders - - 		- - Twist235 -
- Twist235 - -
- - k3it -
- k3it - - 		- - Drakulix -
- Drakulix - - 		- - vilisas -
- vilisas - - 		- - forzagreen -
- forzagreen - - 		- - 42wim -
- 42wim - - 		- - radicand -
- radicand - -
- - nilshoell -
- nilshoell - - 		- - nknapp -
- nknapp - - 		- - pcqnt -
- pcqnt - - 		- - OrvilleQ -
- OrvilleQ - - 		- - ovidiucp -
- ovidiucp - - 		- - mrPjer -
- mrPjer - -
- - p3dda -
- p3dda - - 		- - peter-hartmann -
- peter-hartmann - - 		- - piwai -
- piwai - - 		- - remoe -
- remoe - - 		- - robbertkl -
- robbertkl - - 		- - romansey -
- romansey - -
- - norrs -
- norrs - - 		- - MightySCollins -
- MightySCollins - - 		- - 501st-alpha1 -
- 501st-alpha1 - - 		- - klamann -
- klamann - - 		- - svdb0 -
- svdb0 - - 		- - 3ap -
- 3ap - -
- - shyim -
- shyim - - 		- - sjmudd -
- sjmudd - - 		- - simonsystem -
- simonsystem - - 		- - ShiriNmi1520 -
- ShiriNmi1520 - - 		- - matrixes -
- matrixes - - 		- - mchamplain -
- mchamplain - -
- - millerjason -
- millerjason - - 		- - mplx -
- mplx - - 		- - odinis -
- odinis - - 		- - okamidash -
- okamidash - - 		- - olaf-mandel -
- olaf-mandel - - 		- - ontheair81 -
- ontheair81 - -
- - pravynandas -
- pravynandas - - 		- - presocratics -
- presocratics - - 		- - rhyst -
- rhyst - - 		- - rmlhuk -
- rmlhuk - - 		- - rriski -
- rriski - - 		- - schnippl0r -
- schnippl0r - -
- - smargold476 -
- smargold476 - - 		- - sportshead -
- sportshead - - 		- - squash -
- squash - - 		- - strarsis -
- strarsis - - 		- - tamueller -
- tamueller - - 		- - vivacarvajalito -
- vivacarvajalito - -
- - wligtenberg -
- wligtenberg - - 		- - wolkenschieber -
- wolkenschieber - - 		- - worldworm -
- worldworm - - 		- - Zepmann -
- Zepmann - - 		- - allddd -
- allddd - - 		- - arcaine2 -
- arcaine2 - -
- - awb99 -
- awb99 - - 		- - beertje44 -
- beertje44 - - 		- - brainkiller -
- brainkiller - - 		- - cternes -
- cternes - - 		- - dborowy -
- dborowy - - 		- - dimalo -
- dimalo - -
- - eleith -
- eleith - - 		- - fanqiaojun -
- fanqiaojun - - 		- - ghnp5 -
- ghnp5 - - 		- - helmutundarnold -
- helmutundarnold - - 		- - hnws -
- hnws - - 		- - i-C-o-d-e-r -
- i-C-o-d-e-r - -
- - idaadi -
- idaadi - - 		- - ixeft -
- ixeft - - 		- - jjtt -
- jjtt - - 		- - paralax -
- paralax - - 		- - jpduyx -
- jpduyx - - 		- - landergate -
- landergate - -
- - callmemagnus -
- callmemagnus - - 		- - marios88 -
- marios88 - - 		- - CBeerta -
- CBeerta - - 		- - damianmoore -
- damianmoore - - 		- - espitall -
- espitall - - 		- - dkarski -
- dkarski - -
- - dbellavista -
- dbellavista - - 		- - danielvandenberg95 -
- danielvandenberg95 - - 		- - denisix -
- denisix - - 		- - mlatorre31 -
- mlatorre31 - - 		- - mazzz1y -
- mazzz1y - - 		- - doominator42 -
- doominator42 - -
- - aydodo -
- aydodo - - 		- - vedtam -
- vedtam - - 		- - edvorg -
- edvorg - - 		- - eliroca -
- eliroca - - 		- - ekkis -
- ekkis - - 		- - ErikEngerd -
- ErikEngerd - -
- - huncode -
- huncode - - 		- - gitfeber -
- gitfeber - - 		- - felixn -
- felixn - - 		- - flole -
- flole - - 		- - froks -
- froks - - 		- - JOduMonT -
- JOduMonT - -
- - 0xflotus -
- 0xflotus - - 		- - ifokeev -
- ifokeev - - 		- - 20th -
- 20th - - 		- - 2b -
- 2b - - 		- - askz -
- askz - - 		- - aspettl -
- aspettl - -
- - acch -
- acch - - 		- - vifino -
- vifino - - 		- - kachkaev -
- kachkaev - - 		- - alexanderneu -
- alexanderneu - - 		- - ch3sh1r -
- ch3sh1r - - 		- - eglia -
- eglia - -
- - groupmsl -
- groupmsl - - 		- - green-anger -
- green-anger - - 		- - iRhonin -
- iRhonin - - 		- - MrFreezeex -
- MrFreezeex - - 		- - arunvc -
- arunvc - - 		- - astrocket -
- astrocket - -
- - baxerus -
- baxerus - - 		- - spock -
- spock - - 		- - erdos4d -
- erdos4d - - 		- - crash7 -
- crash7 - - 		- - auchri -
- auchri - - 		- - Kaan88 -
- Kaan88 - -
- - akkumar -
- akkumar - - 		- - thechubbypanda -
- thechubbypanda - - 		- - KCrawley -
- KCrawley - - 		- - khuedoan -
- khuedoan - - 		- - UltraCoderRU -
- UltraCoderRU - - 		- - JustAnother1 -
- JustAnother1 - -
- - leowinterde -
- leowinterde - - 		- - linhandev -
- linhandev - - 		- - luke- -
- luke- - - 		- - LucidityCrash -
- LucidityCrash - - 		- - MadsRC -
- MadsRC - - 		- - madmath03 -
- madmath03 - -
- - maxemann96 -
- maxemann96 - - 		- - dragetd -
- dragetd - - 		- - michaeljensen -
- michaeljensen - - 		- - exhuma -
- exhuma - - 		- - milas -
- milas - - 		- - mcchots -
- mcchots - -
- - MohammedNoureldin -
- MohammedNoureldin - - 		- - naveensrinivasan -
- naveensrinivasan - - 		- - neuralp -
- neuralp - - 		- - fkefer -
- fkefer - - 		- - furstblumier -
- furstblumier - - 		- - Marsu31 -
- Marsu31 - -
- - glandais -
- glandais - - 		- - GiovanH -
- GiovanH - - 		- - Amphaal -
- Amphaal - - 		- - harryyoud -
- harryyoud - - 		- - HeySora -
- HeySora - - 		- - sirgantrithon -
- sirgantrithon - -
- - Influencer -
- Influencer - - 		- - in-seo -
- in-seo - - 		- - firefly-cpp -
- firefly-cpp - - 		- - JacksonZ03 -
- JacksonZ03 - - 		- - JamBalaya56562 -
- JamBalaya56562 - - 		- - jcalfee -
- jcalfee - -
- - mivek -
- mivek - - 		- - init-js -
- init-js - - 		- - Jeidnx -
- Jeidnx - - 		- - jessp01 -
- jessp01 - - 		- - JiLleON -
- JiLleON - - 		- - jirislav -
- jirislav - -
- - jmccl -
- jmccl - - 		- - jurekbarth -
- jurekbarth - -
- From a338c0663931d621105b42350e44608ffc8abf13 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 1 Aug 2024 08:57:36 +1200 Subject: [PATCH 407/592] docs: Add caveat for ENV `DMS_VMAIL_UID` value compatibility (#4143) --- CHANGELOG.md | 9 +++++---- docs/content/config/environment.md | 8 ++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97c48e2a2d3..e28aa1a5e57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ All notable changes to this project will be documented in this file. The format ### Added - **Internal:** - - Add password confirmation to several `setup.sh` commands ([#4072](https://github.com/docker-mailserver/docker-mailserver/pull/4072)) + - Add password confirmation to several `setup` CLI subcommands ([#4072](https://github.com/docker-mailserver/docker-mailserver/pull/4072)) ### Updates @@ -22,13 +22,14 @@ All notable changes to this project will be documented in this file. The format - Bump version to [1.1.0](https://github.com/fail2ban/fail2ban/releases/tag/1.1.0). For more information, check the [changelog](https://github.com/fail2ban/fail2ban/blob/1.1.0/ChangeLog). - **Documentation:** - Rewritten and organized the pages for Account Management and Authentication ([#4122](https://github.com/docker-mailserver/docker-mailserver/pull/4122)) -- **Postfix** - - Disable Microsoft reactions to outgoing mail. ([#4120](https://github.com/docker-mailserver/docker-mailserver/pull/4120)) + - Add caveat for `DMS_VMAIL_UID` not being compatible with `0` / root ([#4143](https://github.com/docker-mailserver/docker-mailserver/pull/4143)) +- **Postfix:** + - Disable Microsoft reactions to outgoing mail ([#4120](https://github.com/docker-mailserver/docker-mailserver/pull/4120)) ### Fixes - **Dovecot:** - - `logwatch` Update logwatch `ignore.conf` to exclude Xapian messages about pending documents ([#4060](https://github.com/docker-mailserver/docker-mailserver/pull/4060)) + - Update logwatch `ignore.conf` to exclude Xapian messages about pending documents ([#4060](https://github.com/docker-mailserver/docker-mailserver/pull/4060)) - `dovecot-fts-xapian` plugin was updated to `1.7.13`, fixing a regression with indexing ([#4095](https://github.com/docker-mailserver/docker-mailserver/pull/4095)) ### CI diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 2ebd092c7d5..0a65b1e4e5d 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -39,6 +39,12 @@ Default: 5000 The User ID assigned to the static vmail user for `/var/mail` (_Mail storage managed by Dovecot_). +!!! warning "Incompatible UID values" + + - A value of [`0` (root) is not compatible][gh-issue::vmail-uid-cannot-be-root]. + - This feature will attempt to adjust the `uid` for the `docker` user (`/etc/passwd`), hence the error emitted to logs if the UID is already assigned to another user. + - The feature appears to work with other UID values that are already assigned in `/etc/passwd`, even though Dovecot by default has a setting for the minimum UID as `500`. + ##### DMS_VMAIL_GID Default: 5000 @@ -1123,6 +1129,8 @@ Provide the credentials to use with `RELAY_HOST` or `DEFAULT_RELAY_HOST`. - Add the exact relayhost value (`host:port` / `[host]:port`) from the generated `/etc/postfix/relayhost_map`, or `main.cf:relayhost` (`DEFAULT_RELAY_HOST`). - `setup relay ...` is missing support, you must instead add these manually to `postfix-sasl-password.cf`. +[gh-issue::vmail-uid-cannot-be-root]: https://github.com/docker-mailserver/docker-mailserver/issues/4098#issuecomment-2257201025 + [docs-rspamd]: ./security/rspamd.md [docs-tls]: ./security/ssl.md [docs-tls-letsencrypt]: ./security/ssl.md#lets-encrypt-recommended From 2f8ad142eccfe3d6291373629c5b1dbf89b71f52 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 2 Aug 2024 09:42:39 +1200 Subject: [PATCH 408/592] fix: Prevent `stderr` being written to `/etc/postfix/main.cf` (#4147) `stderr` is filtered by `grep` to discard unwanted (expected) log noise when appending the override `postfix-main.cf` content (_updated settings did not replace earlier defined instances_). That `grep` filter introduced a regression into DMS v14 release, since any other `stderr` content not being excluded was now blended into `stdout` and redirected with the original `stdout` output for the `postconf -n` command. The fix is to ensure the `grep` output is redirect to `stderr` to avoid that mishap. --- CHANGELOG.md | 2 ++ target/scripts/startup/setup.d/postfix.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e28aa1a5e57..900e867ac7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,8 @@ All notable changes to this project will be documented in this file. The format - **Dovecot:** - Update logwatch `ignore.conf` to exclude Xapian messages about pending documents ([#4060](https://github.com/docker-mailserver/docker-mailserver/pull/4060)) - `dovecot-fts-xapian` plugin was updated to `1.7.13`, fixing a regression with indexing ([#4095](https://github.com/docker-mailserver/docker-mailserver/pull/4095)) +- **Internal:** + - Fixed a regression introduced in v14 where `postfix-main.cf` appended `stderr` output into `/etc/postfix/main.cf`, causing Postfix startup to fail ([#4147](https://github.com/docker-mailserver/docker-mailserver/pull/4147)) ### CI diff --git a/target/scripts/startup/setup.d/postfix.sh b/target/scripts/startup/setup.d/postfix.sh index 3ebdeb9d95b..e140fbaa27d 100644 --- a/target/scripts/startup/setup.d/postfix.sh +++ b/target/scripts/startup/setup.d/postfix.sh @@ -129,7 +129,7 @@ function __postfix__setup_override_configuration() { # Do not directly output to 'main.cf' as this causes a read-write-conflict. # `postconf` output is filtered to skip expected warnings regarding overrides: # https://github.com/docker-mailserver/docker-mailserver/pull/3880#discussion_r1510414576 - postconf -n >/tmp/postfix-main-new.cf 2> >(grep -v 'overriding earlier entry') + postconf -n >/tmp/postfix-main-new.cf 2> >(grep -v 'overriding earlier entry' >&2) mv /tmp/postfix-main-new.cf /etc/postfix/main.cf _adjust_mtime_for_postfix_maincf From 526fd64d110ae792748a2baa6a582e8bd3e7d150 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 3 Aug 2024 12:04:21 +1200 Subject: [PATCH 409/592] fix: Ensure main log file is tailed from the start (#4146) Co-authored-by: Casper --- CHANGELOG.md | 1 + target/scripts/start-mailserver.sh | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 900e867ac7c..a6edc099aad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ All notable changes to this project will be documented in this file. The format - Update logwatch `ignore.conf` to exclude Xapian messages about pending documents ([#4060](https://github.com/docker-mailserver/docker-mailserver/pull/4060)) - `dovecot-fts-xapian` plugin was updated to `1.7.13`, fixing a regression with indexing ([#4095](https://github.com/docker-mailserver/docker-mailserver/pull/4095)) - **Internal:** + - The main `mail.log` which is piped to stdout via `tail` now correctly begins from the first log line of the active container run. Previously some daemon logs and potential warnings/errors were omitted. ([#4146](https://github.com/docker-mailserver/docker-mailserver/pull/4146)) - Fixed a regression introduced in v14 where `postfix-main.cf` appended `stderr` output into `/etc/postfix/main.cf`, causing Postfix startup to fail ([#4147](https://github.com/docker-mailserver/docker-mailserver/pull/4147)) ### CI diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index cbe38da9251..1c43616dfdf 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -194,10 +194,16 @@ fi # marker to check if container was restarted date >/CONTAINER_START +# Container logs will receive updates from this log file: +MAIN_LOGFILE=/var/log/mail/mail.log +# NOTE: rsyslogd would usually create this later during `_start_daemons`, however it would already exist if the container was restarted. +touch "${MAIN_LOGFILE}" +# Ensure `tail` follows the correct position of the log file for this container start (new logs begin once `_start_daemons` is called) +TAIL_START=$(( $(wc -l < "${MAIN_LOGFILE}") + 1 )) + [[ ${LOG_LEVEL} =~ (debug|trace) ]] && print-environment _start_daemons +# Container start-up scripts completed. `tail` will now pipe the log updates to stdout: _log 'info' "${HOSTNAME} is up and running" - -touch /var/log/mail/mail.log -exec tail -Fn 0 /var/log/mail/mail.log +exec tail -Fn "+${TAIL_START}" "${MAIN_LOGFILE}" From d61909bdea85604a99ce9e211af0c69dadb03073 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Aug 2024 19:15:35 +1200 Subject: [PATCH 410/592] chore(deps): Bump docker/setup-buildx-action from 3.5.0 to 3.6.1 (#4152) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3.5.0 to 3.6.1. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v3.5.0...v3.6.1) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index dee6ce93ffa..8442546cafb 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -79,7 +79,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.5.0 + uses: docker/setup-buildx-action@v3.6.1 # NOTE: AMD64 can build within 2 minutes - name: 'Build images' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 02ce4ffe888..5269131726f 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -40,7 +40,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.5.0 + uses: docker/setup-buildx-action@v3.6.1 # Try get the cached build layers from a prior `generic_build.yml` job. # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 0c160ad3869..59e2998e357 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -38,7 +38,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.5.0 + uses: docker/setup-buildx-action@v3.6.1 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 7c231b8eb58..a9b4144a800 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -37,7 +37,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.5.0 + uses: docker/setup-buildx-action@v3.6.1 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. From fb57905aa3248ee215fc96aad16ec203a8014975 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Aug 2024 13:26:05 +0200 Subject: [PATCH 411/592] chore(deps): Bump docker/build-push-action from 6.5.0 to 6.6.1 (#4158) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 8442546cafb..76f7014b625 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v6.5.0 + uses: docker/build-push-action@v6.6.1 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 5269131726f..ffb8a6774d8 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -67,7 +67,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: 'Build and publish images' - uses: docker/build-push-action@v6.5.0 + uses: docker/build-push-action@v6.6.1 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 59e2998e357..0f2f9031db0 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.5.0 + uses: docker/build-push-action@v6.6.1 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index a9b4144a800..e33217a1262 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.5.0 + uses: docker/build-push-action@v6.6.1 with: context: . tags: mailserver-testing:ci From b2978fd760f5ab4c33751edda8331e3d79aa6cf1 Mon Sep 17 00:00:00 2001 From: Casper Date: Sat, 17 Aug 2024 12:14:59 +0200 Subject: [PATCH 412/592] breaking: Refactor `getmail` support (#4156) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- CHANGELOG.md | 13 ++++ Dockerfile | 3 +- .../getmail/getmailrc_general.cf | 4 ++ config-examples/getmail/imap-example.cf | 13 ++++ config-examples/getmail/pop3-example.cf | 13 ++++ docs/content/config/advanced/mail-getmail.md | 25 +++++--- docs/content/config/environment.md | 4 +- mailserver.env | 2 +- target/bin/debug-getmail | 19 ++++-- target/bin/getmail-cron | 8 --- target/getmail/getmail-service.sh | 47 +++++++++++++++ target/getmail/getmailrc_general | 11 ++++ target/scripts/start-mailserver.sh | 1 + target/scripts/startup/daemons-stack.sh | 1 + target/scripts/startup/setup.d/getmail.sh | 60 +++++++++++-------- target/scripts/startup/setup.d/mail_state.sh | 1 + target/supervisor/conf.d/dms-services.conf | 9 +++ .../getmail/{getmail-user3.cf => user3.cf} | 0 test/tests/parallel/set1/getmail.bats | 17 +++--- .../process_check_restart.bats | 4 ++ 20 files changed, 197 insertions(+), 58 deletions(-) rename target/getmail/getmailrc => config-examples/getmail/getmailrc_general.cf (59%) create mode 100644 config-examples/getmail/imap-example.cf create mode 100644 config-examples/getmail/pop3-example.cf delete mode 100644 target/bin/getmail-cron create mode 100644 target/getmail/getmail-service.sh create mode 100644 target/getmail/getmailrc_general rename test/config/getmail/{getmail-user3.cf => user3.cf} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6edc099aad..1a25323400d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,19 @@ All notable changes to this project will be documented in this file. The format > **Note**: Changes and additions listed here are contained in the `:edge` image tag. These changes may not be as stable as released changes. +### Breaking + +- **getmail6** has been refactored: ([#4156](https://github.com/docker-mailserver/docker-mailserver/pull/4156)) + - The [DMS config volume](https://docker-mailserver.github.io/docker-mailserver/v15.0/config/advanced/optional-config/#volumes) now has support for `getmailrc_general.cf` for overriding [common default settings](https://docker-mailserver.github.io/docker-mailserver/v15.0/config/advanced/mail-getmail/#common-options). If you previously mounted this config file directly to `/etc/getmailrc_general` you should switch to our config volume support. + - IMAP/POP3 example configs added to our [`config-examples`](https://github.com/docker-mailserver/docker-mailserver/tree/v15.0.0/config-examples/getmail). + - ENV [`GETMAIL_POLL`](https://docker-mailserver.github.io/docker-mailserver/v15.0/config/environment/#getmail_poll) now supports values above 30 minutes. + - Added `getmail` as a new service for `supervisor` to manage, replacing cron for periodic polling. + - Generated getmail configuration files no longer set the `message_log` option. Instead of individual log files per config, the [default base settings DMS configures](https://github.com/docker-mailserver/docker-mailserver/tree/v15.0.0/target/getmail/getmailrc_general) now enables `message_log_syslog`. This aligns with how other services in DMS log to syslog where it is captured in `mail.log`. + - Getmail configurations have changed location from the base of the DMS Config Volume, to the `getmail/` subdirectory. Any existing configurations **must be migrated manually.** + - DMS v14 mistakenly relocated the getmail state directory to the DMS Config Volume as a `getmail/` subdirectory. + - This has been corrected to `/var/lib/getmail` (_if you have mounted a DMS State Volume to `/var/mail-state`, `/var/lib/getmail` will be symlinked to `/var/mail-state/lib-getmail`_). + - To preserve this state when upgrading to DMS v15, **you must manually migrate `getmail/` from the _DMS Config Volume_ to `lib-getmail/` in the _DMS State Volume_.** + ### Security - **Fail2ban:** diff --git a/Dockerfile b/Dockerfile index f6199fd4d32..02245ebceb6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -212,7 +212,8 @@ EOF RUN echo 'Reason_Message = Message {rejectdefer} due to: {spf}.' >>/etc/postfix-policyd-spf-python/policyd-spf.conf COPY target/fetchmail/fetchmailrc /etc/fetchmailrc_general -COPY target/getmail/getmailrc /etc/getmailrc_general +COPY target/getmail/getmailrc_general /etc/getmailrc_general +COPY target/getmail/getmail-service.sh /usr/local/bin/ COPY target/postfix/main.cf target/postfix/master.cf /etc/postfix/ # DH parameters for DHE cipher suites, ffdhe4096 is the official standard 4096-bit DH params now part of TLS 1.3 diff --git a/target/getmail/getmailrc b/config-examples/getmail/getmailrc_general.cf similarity index 59% rename from target/getmail/getmailrc rename to config-examples/getmail/getmailrc_general.cf index 57c2b4a98c9..73fc443f222 100644 --- a/target/getmail/getmailrc +++ b/config-examples/getmail/getmailrc_general.cf @@ -1,3 +1,5 @@ +# https://getmail6.org/configuration.html#conf-options + [options] verbose = 0 read_all = false @@ -5,3 +7,5 @@ delete = false max_messages_per_session = 500 received = false delivered_to = false +message_log_syslog = true + diff --git a/config-examples/getmail/imap-example.cf b/config-examples/getmail/imap-example.cf new file mode 100644 index 00000000000..fa4fb7d6eb3 --- /dev/null +++ b/config-examples/getmail/imap-example.cf @@ -0,0 +1,13 @@ +# https://getmail6.org/configuration.html + +[retriever] +type = SimpleIMAPSSLRetriever +server = imap.gmail.com +username = alice +password = notsecure + +[destination] +type = MDA_external +path = /usr/lib/dovecot/deliver +allow_root_commands = true +arguments =("-d","user1@example.com") diff --git a/config-examples/getmail/pop3-example.cf b/config-examples/getmail/pop3-example.cf new file mode 100644 index 00000000000..dde60c8710c --- /dev/null +++ b/config-examples/getmail/pop3-example.cf @@ -0,0 +1,13 @@ +# https://getmail6.org/configuration.html + +[retriever] +type = SimplePOP3SSLRetriever +server = pop3.gmail.com +username = alice +password = notsecure + +[destination] +type = MDA_external +path = /usr/lib/dovecot/deliver +allow_root_commands = true +arguments =("-d","user1@example.com") diff --git a/docs/content/config/advanced/mail-getmail.md b/docs/content/config/advanced/mail-getmail.md index f62388b0226..d423b4f09a3 100644 --- a/docs/content/config/advanced/mail-getmail.md +++ b/docs/content/config/advanced/mail-getmail.md @@ -10,14 +10,19 @@ environment: - GETMAIL_POLL=5 ``` -In your DMS config volume (eg: `docker-data/dms/config/`), create a `getmail-.cf` file for each remote account that you want to retrieve mail and store into a local DMS account. `` should be replaced by you, and is just the rest of the filename (eg: `getmail-example.cf`). The contents of each file should be configuration like documented below. +In your DMS config volume (eg: `docker-data/dms/config/`), add a subdirectory `getmail/` for including your getmail config files (eg: `imap-example.cf`) for each remote account that you want to retrieve mail from and deliver to the mailbox of a DMS account. -The directory structure should similar to this: +The content of these config files is documented in the next section with an IMAP and POP3 example to reference. + +The directory structure should look similar to this: ```txt ├── docker-data/dms/config │   ├── dovecot.cf -│   ├── getmail-example.cf +│ ├── getmail +│   │ ├── getmailrc_general.cf +│   │ ├── remote-account1.cf +│   │ ├── remote-account2.cf │   ├── postfix-accounts.cf │   └── postfix-virtual.cf ├── docker-compose.yml @@ -42,9 +47,13 @@ received = false delivered_to = false ``` -If you want to use a different base config, mount a file to `/etc/getmailrc_general`. This file will replace the default "Common Options" base config above, that all `getmail-.cf` files will extend with their configs when used. +The DMS integration for Getmail generates a `getmailrc` config that prepends the common options of the base config to each remote account config file (`*.cf`) found in the DMS Config Volume `getmail/` directory. + +!!! tip "Change the base options" + + Add your own base config as `getmail/getmailrc_general.cf` into the DMS Config Volume. It will replace the DMS defaults shown above. -??? example "IMAP Configuration" +??? example "IMAP Configuration" This example will: @@ -54,7 +63,7 @@ If you want to use a different base config, mount a file to `/etc/getmailrc_gene ```getmailrc [retriever] - type = SimpleIMAPRetriever + type = SimpleIMAPSSLRetriever server = imap.gmail.com username = alice password = notsecure @@ -71,7 +80,7 @@ If you want to use a different base config, mount a file to `/etc/getmailrc_gene ```getmailrc [retriever] - type = SimplePOP3Retriever + type = SimplePOP3SSLRetriever server = pop3.gmail.com username = alice password = notsecure @@ -84,7 +93,7 @@ If you want to use a different base config, mount a file to `/etc/getmailrc_gene ### Polling Interval -By default the `getmail` service checks external mail accounts for new mail every 5 minutes. That polling interval is configurable via the `GETMAIL_POLL` ENV variable, with a value in minutes (_default: 5, min: 1, max: 30_): +By default the `getmail` service checks external mail accounts for new mail every 5 minutes. That polling interval is configurable via the `GETMAIL_POLL` ENV variable, with a value in minutes (_default: 5, min: 1_): ```yaml environment: diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 0a65b1e4e5d..5a766f53d83 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -450,7 +450,7 @@ Default: 6 (which corresponds to the `add_header` action) ##### RSPAMD_NEURAL -Can be used to enable or disable the [Neural network module][rspamd-docs-neural-network]. This is an experimental anti-spam weigh method using three neural networks in the configuration added here. As far as we can tell it trains itself by using other modules to find out what spam is. It will take a while (a week or more) to train its first neural network. The config trains new networks all the time and discards old networks. +Can be used to enable or disable the [Neural network module][rspamd-docs-neural-network]. This is an experimental anti-spam weigh method using three neural networks in the configuration added here. As far as we can tell it trains itself by using other modules to find out what spam is. It will take a while (a week or more) to train its first neural network. The config trains new networks all the time and discards old networks. Since it is experimental, it is switched off by default. - **0** => Disabled @@ -732,7 +732,7 @@ Enable or disable `getmail`. ##### GETMAIL_POLL -- **5** => `getmail` The number of minutes for the interval. Min: 1; Max: 30; Default: 5. +- **5** => `getmail` The number of minutes for the interval. Min: 1; Default: 5. #### OAUTH2 diff --git a/mailserver.env b/mailserver.env index 77b863ffb63..141b607a7b3 100644 --- a/mailserver.env +++ b/mailserver.env @@ -430,7 +430,7 @@ FETCHMAIL_PARALLEL=0 # - 1 => Enabled ENABLE_GETMAIL=0 -# The number of minutes for the interval. Min: 1; Max: 30. +# The number of minutes for the interval. Min: 1; Default: 5. GETMAIL_POLL=5 # ----------------------------------------------- diff --git a/target/bin/debug-getmail b/target/bin/debug-getmail index 60270215469..33d41893281 100644 --- a/target/bin/debug-getmail +++ b/target/bin/debug-getmail @@ -5,9 +5,20 @@ source /usr/local/bin/helpers/log.sh # shellcheck source=../scripts/startup/setup-stack.sh source /usr/local/bin/setup.d/getmail.sh -_setup_getmail +# Setup getmail, even if not enabled. +ENABLE_GETMAIL=1 _setup_getmail -GETMAILDIR=/tmp/docker-mailserver/getmail -for FILE in /etc/getmailrc.d/getmailrc*; do - getmail --getmaildir "${GETMAILDIR}" --rcfile "${FILE}" --dump | tail -n +6 +# Directory, where "oldmail" files are stored. +# Getmail stores its state - its "memory" of what it has seen in your POP/IMAP account - in the oldmail files. +GETMAIL_DIR=/var/lib/getmail + +# If no matching filenames are found, and the shell option nullglob is disabled, the word is left unchanged. +# If the nullglob option is set, and no matches are found, the word is removed. +shopt -s nullglob + +# Dump configuration from each RC file. +for RC_FILE in /etc/getmailrc.d/*; do + echo "${RC_FILE##*/}:" + echo + getmail --getmaildir "${GETMAIL_DIR}" --rcfile "${RC_FILE}" --dump | tail -n +6 done diff --git a/target/bin/getmail-cron b/target/bin/getmail-cron deleted file mode 100644 index fec03906841..00000000000 --- a/target/bin/getmail-cron +++ /dev/null @@ -1,8 +0,0 @@ -#! /bin/bash - -GETMAILDIR=/tmp/docker-mailserver/getmail -for FILE in /etc/getmailrc.d/getmailrc*; do - if ! pgrep -f "${FILE}$" &>/dev/null; then - getmail --getmaildir "${GETMAILDIR}" --rcfile "${FILE}" - fi -done diff --git a/target/getmail/getmail-service.sh b/target/getmail/getmail-service.sh new file mode 100644 index 00000000000..ff2cc57136f --- /dev/null +++ b/target/getmail/getmail-service.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# shellcheck source=../scripts/helpers/log.sh +source /usr/local/bin/helpers/log.sh + +# Directory, where "oldmail" files are stored. +# getmail stores its state - its "memory" of what it has seen in your POP/IMAP account - in the oldmail files. +GETMAIL_DIR=/var/lib/getmail + +# Kill all child processes on EXIT. +# Otherwise 'supervisorctl restart getmail' leads to zombie 'sleep' processes. +trap 'pkill --parent ${$}' EXIT + +function _syslog_error() { + logger --priority mail.err --tag getmail "${1}" +} + +function _stop_service() { + _syslog_error "Stopping service" + exec supervisorctl stop getmail +} + +# Verify the correct value for GETMAIL_POLL. Valid are any numbers greater than 0. +if [[ ! ${GETMAIL_POLL} =~ ^[0-9]+$ ]] || [[ ${GETMAIL_POLL} -lt 1 ]]; then + _syslog_error "Invalid value for GETMAIL_POLL: ${GETMAIL_POLL}" + _stop_service +fi + +# If no matching filenames are found, and the shell option nullglob is disabled, the word is left unchanged. +# If the nullglob option is set, and no matches are found, the word is removed. +shopt -s nullglob + +# Run each getmailrc periodically. +while :; do + for RC_FILE in /etc/getmailrc.d/*; do + _log 'debug' "Processing ${RC_FILE}" + getmail --getmaildir "${GETMAIL_DIR}" --rcfile "${RC_FILE}" + done + + # Stop service if no configuration is found. + if [[ -z ${RC_FILE} ]]; then + _syslog_error 'No configuration found' + _stop_service + fi + + sleep "${GETMAIL_POLL}m" +done diff --git a/target/getmail/getmailrc_general b/target/getmail/getmailrc_general new file mode 100644 index 00000000000..73fc443f222 --- /dev/null +++ b/target/getmail/getmailrc_general @@ -0,0 +1,11 @@ +# https://getmail6.org/configuration.html#conf-options + +[options] +verbose = 0 +read_all = false +delete = false +max_messages_per_session = 500 +received = false +delivered_to = false +message_log_syslog = true + diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index 1c43616dfdf..71dc32f2dc9 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -161,6 +161,7 @@ function _register_functions() { [[ ${ENABLE_CLAMAV} -eq 1 ]] && _register_start_daemon '_start_daemon_clamav' [[ ${ENABLE_AMAVIS} -eq 1 ]] && _register_start_daemon '_start_daemon_amavis' [[ ${ACCOUNT_PROVISIONER} == 'FILE' ]] && _register_start_daemon '_start_daemon_changedetector' + [[ ${ENABLE_GETMAIL} -eq 1 ]] && _register_start_daemon '_start_daemon_getmail' } # ------------------------------------------------------------ diff --git a/target/scripts/startup/daemons-stack.sh b/target/scripts/startup/daemons-stack.sh index a4cecf67ad1..4cc9b2af067 100644 --- a/target/scripts/startup/daemons-stack.sh +++ b/target/scripts/startup/daemons-stack.sh @@ -34,6 +34,7 @@ function _start_daemon_clamav { _default_start_daemon 'clamav' ; function _start_daemon_cron { _default_start_daemon 'cron' ; } function _start_daemon_dovecot { _default_start_daemon 'dovecot' ; } function _start_daemon_fail2ban { _default_start_daemon 'fail2ban' ; } +function _start_daemon_getmail { _default_start_daemon 'getmail' ; } function _start_daemon_opendkim { _default_start_daemon 'opendkim' ; } function _start_daemon_opendmarc { _default_start_daemon 'opendmarc' ; } function _start_daemon_postgrey { _default_start_daemon 'postgrey' ; } diff --git a/target/scripts/startup/setup.d/getmail.sh b/target/scripts/startup/setup.d/getmail.sh index c1345063de9..e63db13891e 100644 --- a/target/scripts/startup/setup.d/getmail.sh +++ b/target/scripts/startup/setup.d/getmail.sh @@ -4,38 +4,46 @@ function _setup_getmail() { if [[ ${ENABLE_GETMAIL} -eq 1 ]]; then _log 'trace' 'Preparing Getmail configuration' - local GETMAILRC ID CONFIGS + local GETMAIL_RC ID GETMAIL_DIR - GETMAILRC='/etc/getmailrc.d' - CONFIGS=0 + local GETMAIL_CONFIG_DIR='/tmp/docker-mailserver/getmail' + local GETMAIL_RC_DIR='/etc/getmailrc.d' + local GETMAIL_RC_GENERAL_CF="${GETMAIL_CONFIG_DIR}/getmailrc_general.cf" + local GETMAIL_RC_GENERAL='/etc/getmailrc_general' - mkdir -p "${GETMAILRC}" + # Create the directory /etc/getmailrc.d to place the user config in later. + mkdir -p "${GETMAIL_RC_DIR}" - # Generate getmailrc configs, starting with the `/etc/getmailrc_general` base config, - # Add a unique `message_log` config, then append users own config to the end. - for FILE in /tmp/docker-mailserver/getmail-*.cf; do - if [[ -f ${FILE} ]]; then - CONFIGS=1 - ID=$(cut -d '-' -f 3 <<< "${FILE}" | cut -d '.' -f 1) - local GETMAIL_CONFIG="${GETMAILRC}/getmailrc-${ID}" + # Check if custom getmailrc_general.cf file is present. + if [[ -f "${GETMAIL_RC_GENERAL_CF}" ]]; then + _log 'debug' "Custom 'getmailrc_general.cf' found" + cp "${GETMAIL_RC_GENERAL_CF}" "${GETMAIL_RC_GENERAL}" + fi - cat /etc/getmailrc_general >"${GETMAIL_CONFIG}" - echo -e "message_log = /var/log/mail/getmail-${ID}.log\n" >>"${GETMAIL_CONFIG}" - cat "${FILE}" >>"${GETMAIL_CONFIG}" - fi - done + # If no matching filenames are found, and the shell option nullglob is disabled, the word is left unchanged. + # If the nullglob option is set, and no matches are found, the word is removed. + shopt -s nullglob - if [[ ${CONFIGS} -eq 1 ]]; then - cat >/etc/cron.d/getmail << EOF -*/${GETMAIL_POLL} * * * * root /usr/local/bin/getmail-cron -EOF - chmod -R 600 "${GETMAILRC}" - fi + # Generate getmailrc configs, starting with the `/etc/getmailrc_general` base config, then appending users own config to the end. + for FILE in "${GETMAIL_CONFIG_DIR}"/*.cf; do + if [[ ${FILE} =~ /getmail/(.+)\.cf ]] && [[ ${FILE} != "${GETMAIL_RC_GENERAL_CF}" ]]; then + ID=${BASH_REMATCH[1]} - # Both the debug command and cron job (that runs getmail) for getmail - # expect this location to exist. - GETMAILDIR=/tmp/docker-mailserver/getmail - mkdir -p "${GETMAILDIR}" + _log 'debug' "Processing getmail config '${ID}'" + + GETMAIL_RC=${GETMAIL_RC_DIR}/${ID} + cat "${GETMAIL_RC_GENERAL}" "${FILE}" >"${GETMAIL_RC}" + fi + done + # Strip read access from non-root due to files containing secrets: + chmod -R 600 "${GETMAIL_RC_DIR}" + + # Directory, where "oldmail" files are stored. + # For more information see: https://getmail6.org/faq.html#faq-about-oldmail + # The debug command for getmail expects this location to exist. + GETMAIL_DIR=/var/lib/getmail + _log 'debug' "Creating getmail state-dir '${GETMAIL_DIR}'" + mkdir -p "${GETMAIL_DIR}" else _log 'debug' 'Getmail is disabled' fi diff --git a/target/scripts/startup/setup.d/mail_state.sh b/target/scripts/startup/setup.d/mail_state.sh index de15ee3ba55..e819c138295 100644 --- a/target/scripts/startup/setup.d/mail_state.sh +++ b/target/scripts/startup/setup.d/mail_state.sh @@ -23,6 +23,7 @@ function _setup_save_states() { [[ ${ENABLE_CLAMAV} -eq 1 ]] && SERVICEDIRS+=('lib/clamav') [[ ${ENABLE_FAIL2BAN} -eq 1 ]] && SERVICEDIRS+=('lib/fail2ban') [[ ${ENABLE_FETCHMAIL} -eq 1 ]] && SERVICEDIRS+=('lib/fetchmail') + [[ ${ENABLE_GETMAIL} -eq 1 ]] && SERVICEDIRS+=('lib/getmail') [[ ${ENABLE_MTA_STS} -eq 1 ]] && SERVICEDIRS+=('lib/mta-sts') [[ ${ENABLE_POSTGREY} -eq 1 ]] && SERVICEDIRS+=('lib/postgrey') [[ ${ENABLE_RSPAMD} -eq 1 ]] && SERVICEDIRS+=('lib/rspamd') diff --git a/target/supervisor/conf.d/dms-services.conf b/target/supervisor/conf.d/dms-services.conf index 7f106456015..f9db838cc5b 100644 --- a/target/supervisor/conf.d/dms-services.conf +++ b/target/supervisor/conf.d/dms-services.conf @@ -170,3 +170,12 @@ stderr_logfile=/var/log/supervisor/%(program_name)s.log command=/usr/bin/mta-sts-daemon --config /etc/mta-sts-daemon.yml user=_mta-sts environment=HOME=/var/lib/mta-sts + +[program:getmail] +startsecs=0 +stopwaitsecs=55 +autostart=false +autorestart=true +stdout_logfile=/var/log/supervisor/%(program_name)s.log +stderr_logfile=/var/log/supervisor/%(program_name)s.log +command=/bin/bash -l -c /usr/local/bin/getmail-service.sh diff --git a/test/config/getmail/getmail-user3.cf b/test/config/getmail/user3.cf similarity index 100% rename from test/config/getmail/getmail-user3.cf rename to test/config/getmail/user3.cf diff --git a/test/tests/parallel/set1/getmail.bats b/test/tests/parallel/set1/getmail.bats index ca1fdf87d6a..8d69ec3df72 100644 --- a/test/tests/parallel/set1/getmail.bats +++ b/test/tests/parallel/set1/getmail.bats @@ -8,12 +8,13 @@ function setup_file() { local CUSTOM_SETUP_ARGUMENTS=(--env 'ENABLE_GETMAIL=1') _init_with_defaults - mv "${TEST_TMP_CONFIG}/getmail/getmail-user3.cf" "${TEST_TMP_CONFIG}/getmail-user3.cf" _common_container_setup 'CUSTOM_SETUP_ARGUMENTS' } function teardown_file() { _default_teardown ; } +#? The file used in the following tests is placed in test/config/getmail/user3.cf + @test 'default configuration exists and is correct' { _run_in_container cat /etc/getmailrc_general assert_success @@ -24,15 +25,16 @@ function teardown_file() { _default_teardown ; } assert_line 'max_messages_per_session = 500' assert_line 'received = false' assert_line 'delivered_to = false' + assert_line 'message_log_syslog = true' _run_in_container_bash '[[ -f /usr/local/bin/debug-getmail ]]' assert_success - _run_in_container_bash '[[ -f /usr/local/bin/getmail-cron ]]' + _run_in_container_bash '[[ -f /usr/local/bin/getmail-service.sh ]]' assert_success } @test 'debug-getmail works as expected' { - _run_in_container cat /etc/getmailrc.d/getmailrc-user3 + _run_in_container cat /etc/getmailrc.d/user3 assert_success assert_line '[options]' assert_line 'verbose = 0' @@ -41,7 +43,7 @@ function teardown_file() { _default_teardown ; } assert_line 'max_messages_per_session = 500' assert_line 'received = false' assert_line 'delivered_to = false' - assert_line 'message_log = /var/log/mail/getmail-user3.log' + assert_line 'message_log_syslog = true' assert_line '[retriever]' assert_line 'type = SimpleIMAPSSLRetriever' assert_line 'server = imap.remote-service.test' @@ -55,19 +57,18 @@ function teardown_file() { _default_teardown ; } _run_in_container /usr/local/bin/debug-getmail assert_success - assert_line --regexp 'retriever:.*SimpleIMAPSSLRetriever\(ca_certs="None", certfile="None", getmaildir="\/tmp\/docker-mailserver\/getmail", imap_on_delete="None", imap_search="None", keyfile="None", mailboxes="\(.*INBOX.*\)", move_on_delete="None", password="\*", password_command="\(\)", port="993", record_mailbox="True", server="imap.remote-service.test", ssl_cert_hostname="None", ssl_ciphers="None", ssl_fingerprints="\(\)", ssl_version="None", timeout="180", use_cram_md5="False", use_kerberos="False", use_peek="True", use_xoauth2="False", username="user3"\)' + assert_line --regexp 'retriever:.*SimpleIMAPSSLRetriever\(ca_certs="None", certfile="None", getmaildir="\/var\/lib\/getmail", imap_on_delete="None", imap_search="None", keyfile="None", mailboxes="\(.*INBOX.*\)", move_on_delete="None", password="\*", password_command="\(\)", port="993", record_mailbox="True", server="imap.remote-service.test", ssl_cert_hostname="None", ssl_ciphers="None", ssl_fingerprints="\(\)", ssl_version="None", timeout="180", use_cram_md5="False", use_kerberos="False", use_peek="True", use_xoauth2="False", username="user3"\)' assert_line --regexp 'destination:.*MDA_external\(allow_root_commands="True", arguments="\(.*-d.*user3@example.test.*\)", command="deliver", group="None", ignore_stderr="False", path="\/usr\/lib\/dovecot\/deliver", pipe_stdout="True", unixfrom="False", user="None"\)' assert_line ' delete : False' assert_line ' delete_after : 0' assert_line ' delete_bigger_than : 0' assert_line ' delivered_to : False' assert_line ' fingerprint : False' - assert_line ' logfile : logfile(filename="/var/log/mail/getmail-user3.log")' assert_line ' max_bytes_per_session : 0' assert_line ' max_message_size : 0' assert_line ' max_messages_per_session : 500' - assert_line ' message_log : /var/log/mail/getmail-user3.log' - assert_line ' message_log_syslog : False' + assert_line ' message_log : None' + assert_line ' message_log_syslog : True' assert_line ' message_log_verbose : False' assert_line ' netrc_file : None' assert_line ' read_all : False' diff --git a/test/tests/parallel/set3/container_configuration/process_check_restart.bats b/test/tests/parallel/set3/container_configuration/process_check_restart.bats index 78eabd7a704..cc9cd5b3dce 100644 --- a/test/tests/parallel/set3/container_configuration/process_check_restart.bats +++ b/test/tests/parallel/set3/container_configuration/process_check_restart.bats @@ -18,6 +18,7 @@ function teardown() { _default_teardown ; } # dovecot (/usr/sbin/dovecot) # fetchmail (/usr/bin/fetchmail) # fail2ban-server (/usr/bin/python3 /usr/bin/fail2ban-server) - NOTE: python3 is due to the shebang +# getmail (/bin/bash /usr/local/bin/getmail-service.sh) # mta-sts-daemon (/usr/bin/bin/python3 /usr/bin/mta-sts-daemon) # postgrey (postgrey) - NOTE: This process command uses perl via shebang, but unlike python3 the context is missing # postsrsd (/usr/sbin/postsrsd) @@ -41,6 +42,7 @@ ENV_PROCESS_LIST=( dovecot fail2ban-server fetchmail + getmail mta-sts-daemon opendkim opendmarc @@ -56,6 +58,7 @@ ENV_PROCESS_LIST=( --env ENABLE_CLAMAV=0 --env ENABLE_FAIL2BAN=0 --env ENABLE_FETCHMAIL=0 + --env ENABLE_GETMAIL=0 --env ENABLE_MTA_STS=0 --env ENABLE_OPENDKIM=0 --env ENABLE_OPENDMARC=0 @@ -92,6 +95,7 @@ ENV_PROCESS_LIST=( --env ENABLE_AMAVIS=1 --env ENABLE_FAIL2BAN=1 --env ENABLE_FETCHMAIL=1 + --env ENABLE_GETMAIL=1 --env ENABLE_MTA_STS=1 --env ENABLE_OPENDKIM=1 --env ENABLE_OPENDMARC=1 From 310786453bb83b11ffab4f62f5be4a3274c58513 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 17 Aug 2024 22:55:31 +1200 Subject: [PATCH 413/592] chore(Dockerfile): COPY ClamAV database from debian images (#4160) Changes ClamAV image source from DockerHub clamav/clamav (Alpine) to clamav/clamav-debian. Only the Debian variant offers multi-platform images. This isn't too important since we are only interested in taking a copy of the database from the image. It should however resolve a CI warning. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 02245ebceb6..3edf6e95841 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,7 +62,7 @@ SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] # which would require an extra memory of 500MB+ during an image build. # When using `COPY --link`, the `--chown` option is only compatible with numeric ID values. # hadolint ignore=DL3021 -COPY --link --chown=200 --from=docker.io/clamav/clamav:latest /var/lib/clamav /var/lib/clamav +COPY --link --chown=200 --from=docker.io/clamav/clamav-debian:latest /var/lib/clamav /var/lib/clamav RUN < Date: Mon, 19 Aug 2024 00:51:44 +0200 Subject: [PATCH 414/592] chore: Add comments to `start-mailserver.sh` and stop using `inherit_errexit` (#4161) --- CHANGELOG.md | 1 + target/scripts/start-mailserver.sh | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a25323400d..6621da18090 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ All notable changes to this project will be documented in this file. The format - **Internal:** - The main `mail.log` which is piped to stdout via `tail` now correctly begins from the first log line of the active container run. Previously some daemon logs and potential warnings/errors were omitted. ([#4146](https://github.com/docker-mailserver/docker-mailserver/pull/4146)) - Fixed a regression introduced in v14 where `postfix-main.cf` appended `stderr` output into `/etc/postfix/main.cf`, causing Postfix startup to fail ([#4147](https://github.com/docker-mailserver/docker-mailserver/pull/4147)) + - Unused `shopt -s inherit_errexit` removed from `start-mailserver.sh` ([#4161](https://github.com/docker-mailserver/docker-mailserver/pull/4161)) ### CI diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index 71dc32f2dc9..f6ceadf513e 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -1,7 +1,11 @@ #!/bin/bash +# When 'pipefail' is enabled, the exit status of the pipeline reflects the exit status of the last command that fails. +# Without 'pipefail', the exit status of a pipeline is determined by the exit status of the last command in the pipeline. set -o pipefail -shopt -s globstar inherit_errexit + +# Allows the usage of '**' in patterns, e.g. ls **/* +shopt -s globstar # ------------------------------------------------------------ # ? >> Sourcing helpers & stacks From 9589d2192bb41d0176c16b6868133c6edc00b019 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 20:27:27 +0200 Subject: [PATCH 415/592] chore(deps): Bump anchore/scan-action from 4.1.0 to 4.1.1 (#4162) --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index e33217a1262..02fbfab6268 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v4.1.0 + uses: anchore/scan-action@v4.1.1 id: scan with: image: mailserver-testing:ci From cb963a9a8b68c4aa8191cf40c67162a5eb3043e0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 21:44:33 +0200 Subject: [PATCH 416/592] chore(deps): Bump docker/build-push-action from 6.6.1 to 6.7.0 (#4163) --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 76f7014b625..203416febb5 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v6.6.1 + uses: docker/build-push-action@v6.7.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index ffb8a6774d8..3a3d3749cde 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -67,7 +67,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: 'Build and publish images' - uses: docker/build-push-action@v6.6.1 + uses: docker/build-push-action@v6.7.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 0f2f9031db0..1658ddccd91 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.6.1 + uses: docker/build-push-action@v6.7.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 02fbfab6268..5ec5901b2fc 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.6.1 + uses: docker/build-push-action@v6.7.0 with: context: . tags: mailserver-testing:ci From a87d49e8f856cebcb7799d22971a1fc133fa1d3b Mon Sep 17 00:00:00 2001 From: Vetu <59065425+Vetuska@users.noreply.github.com> Date: Sat, 24 Aug 2024 21:42:32 +0300 Subject: [PATCH 417/592] fix: typo in volume pathname (#4165) --- docs/content/config/security/ssl.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/config/security/ssl.md b/docs/content/config/security/ssl.md index dd633c9d6f7..96f4a7175f4 100644 --- a/docs/content/config/security/ssl.md +++ b/docs/content/config/security/ssl.md @@ -240,7 +240,7 @@ After completing the steps above, your certificate should be ready to use. image: certbot/dns-cloudflare:latest command: renew --dns-cloudflare --dns-cloudflare-credentials /run/secrets/cloudflare-api-token volumes: - - ./docker-data/certbot/certs/:/etc/letsencrtypt/ + - ./docker-data/certbot/certs/:/etc/letsencrypt/ - ./docker-data/certbot/logs/:/var/log/letsencrypt/ secrets: - cloudflare-api-token From 3349bba1ffaf0a60d9305f073ec2b8d60246bc68 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 18:30:22 +0200 Subject: [PATCH 418/592] chore(deps): Bump anchore/scan-action from 4.1.1 to 4.1.2 (#4166) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 5ec5901b2fc..0275fc98cc9 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v4.1.1 + uses: anchore/scan-action@v4.1.2 id: scan with: image: mailserver-testing:ci From 4e85f799fc39365d87bbe7bc7b9151dae7f31a31 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Mon, 9 Sep 2024 09:58:12 +1200 Subject: [PATCH 419/592] fix: Dovecot LDAP config should exist --- Dockerfile | 3 ++- target/dovecot/auth-ldap.conf.ext | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3edf6e95841..09295a22710 100644 --- a/Dockerfile +++ b/Dockerfile @@ -109,14 +109,15 @@ COPY target/rspamd/local.d/ /etc/rspamd/local.d/ # --- OAUTH2 ------------------------------------ # ----------------------------------------------- -COPY target/dovecot/auth-oauth2.conf.ext /etc/dovecot/conf.d COPY target/dovecot/dovecot-oauth2.conf.ext /etc/dovecot +COPY target/dovecot/auth-oauth2.conf.ext /etc/dovecot/conf.d # ----------------------------------------------- # --- LDAP & SpamAssassin's Cron ---------------- # ----------------------------------------------- COPY target/dovecot/dovecot-ldap.conf.ext /etc/dovecot +COPY target/dovecot/auth-ldap.conf.ext /etc/dovecot/conf.d COPY \ target/postfix/ldap-users.cf \ target/postfix/ldap-groups.cf \ diff --git a/target/dovecot/auth-ldap.conf.ext b/target/dovecot/auth-ldap.conf.ext index 222769aa2c5..a1cae923adc 100644 --- a/target/dovecot/auth-ldap.conf.ext +++ b/target/dovecot/auth-ldap.conf.ext @@ -6,7 +6,7 @@ passdb { driver = ldap - mechanism = plain login + mechanisms = plain login # Path for LDAP configuration file, see example-config/dovecot-ldap.conf.ext args = /etc/dovecot/dovecot-ldap.conf.ext From cace9c56d932bbd53a36f5bc36f21a676d92a708 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Mon, 9 Sep 2024 19:00:53 +1200 Subject: [PATCH 420/592] fix: Dovecot LDAP config should exist (#4175) The config was not copied over during image build, and the associated auth config had a typo for the `mechanisms` key. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6621da18090..7c9839dbbe5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,8 @@ All notable changes to this project will be documented in this file. The format - **Dovecot:** - Update logwatch `ignore.conf` to exclude Xapian messages about pending documents ([#4060](https://github.com/docker-mailserver/docker-mailserver/pull/4060)) - `dovecot-fts-xapian` plugin was updated to `1.7.13`, fixing a regression with indexing ([#4095](https://github.com/docker-mailserver/docker-mailserver/pull/4095)) +- **LDAP:** + - A previous compatibility fix for OAuth2 in v13.3.1 had not applied the actual LDAP config changes. This has been corrected ([#4175](https://github.com/docker-mailserver/docker-mailserver/pull/4175)) - **Internal:** - The main `mail.log` which is piped to stdout via `tail` now correctly begins from the first log line of the active container run. Previously some daemon logs and potential warnings/errors were omitted. ([#4146](https://github.com/docker-mailserver/docker-mailserver/pull/4146)) - Fixed a regression introduced in v14 where `postfix-main.cf` appended `stderr` output into `/etc/postfix/main.cf`, causing Postfix startup to fail ([#4147](https://github.com/docker-mailserver/docker-mailserver/pull/4147)) From 84180f879e5eaaa15930509d4154248c3248d765 Mon Sep 17 00:00:00 2001 From: GallowsDove <71831019+GallowsDove@users.noreply.github.com> Date: Wed, 18 Sep 2024 19:34:42 +0200 Subject: [PATCH 421/592] fix: fix incorrect link in README.md (#4184) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e4dbf44a984..6e1d2735c19 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ A production-ready fullstack but simple containerized mail server (SMTP, IMAP, L ## :package: Included Services - [Postfix](http://www.postfix.org) with SMTP or LDAP authentication and support for [extension delimiters](https://docker-mailserver.github.io/docker-mailserver/latest/config/account-management/overview/#aliases) -- [Dovecot](https://www.dovecot.org) with SASL, IMAP, POP3, LDAP, [basic Sieve support](https://docker-mailserver.github.io/docker-mailserver/latest/config/advanced/mail-sieve) and [quotas](https://docker-mailserver.github.io/docker-mailserver/latest/config/account-management/overview/#quotas) +- [Dovecot](https://www.dovecot.org) with SASL, IMAP, POP3, LDAP, [basic Sieve support](https://docker-mailserver.github.io/docker-mailserver/latest/config/advanced/mail-sieve) and [quotas](https://docker-mailserver.github.io/docker-mailserver/latest/config/user-management/#quotas) - [Rspamd](https://rspamd.com/) - [Amavis](https://www.amavis.org/) - [SpamAssassin](http://spamassassin.apache.org/) supporting custom rules From 94751e00c9787c5223e39bd0cd5b139a0c1f6255 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sat, 21 Sep 2024 21:04:06 +0200 Subject: [PATCH 422/592] dependency: update `jaq` from `1.3.0` to `1.6.0` (#4190) --- CHANGELOG.md | 1 + target/scripts/build/packages.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c9839dbbe5..21008a64ba1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ All notable changes to this project will be documented in this file. The format - Add caveat for `DMS_VMAIL_UID` not being compatible with `0` / root ([#4143](https://github.com/docker-mailserver/docker-mailserver/pull/4143)) - **Postfix:** - Disable Microsoft reactions to outgoing mail ([#4120](https://github.com/docker-mailserver/docker-mailserver/pull/4120)) +- bumped `jaq` version from 1.3.0 to 1.6.0 ([#4190](https://github.com/docker-mailserver/docker-mailserver/pull/4190)) ### Fixes diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 4469f508f1b..2ac40fbabd5 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -38,7 +38,7 @@ function _pre_installation_steps() { function _install_utils() { _log 'debug' 'Installing utils sourced from Github' _log 'trace' 'Installing jaq' - local JAQ_TAG='v1.3.0' + local JAQ_TAG='v1.6.0' curl -sSfL "https://github.com/01mf02/jaq/releases/download/${JAQ_TAG}/jaq-${JAQ_TAG}-$(uname -m)-unknown-linux-gnu" -o /usr/bin/jaq chmod +x /usr/bin/jaq From 025a38d7366d7e08cf61b5106af762ec69e1505d Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Thu, 26 Sep 2024 20:01:35 +1200 Subject: [PATCH 423/592] chore: Add maintenance note for LMTP (#4199) --- target/scripts/startup/setup.d/postfix.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/target/scripts/startup/setup.d/postfix.sh b/target/scripts/startup/setup.d/postfix.sh index e140fbaa27d..e99e6607c86 100644 --- a/target/scripts/startup/setup.d/postfix.sh +++ b/target/scripts/startup/setup.d/postfix.sh @@ -79,6 +79,8 @@ EOF if [[ ${ACCOUNT_PROVISIONER} == 'FILE' ]]; then postconf 'virtual_mailbox_maps = texthash:/etc/postfix/vmailbox' fi + # Historical context regarding decision to use LMTP instead of LDA (do not change this): + # https://github.com/docker-mailserver/docker-mailserver/issues/4178#issuecomment-2375489302 postconf 'virtual_transport = lmtp:unix:/var/run/dovecot/lmtp' fi From 3bf32a6552a3e2ff2ec55bbdd5708e80413d9df1 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sat, 28 Sep 2024 12:52:49 +0200 Subject: [PATCH 424/592] add dedicated feature requests to new project automatically (#4198) --- .github/ISSUE_TEMPLATE/feature_request.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 0ee66a036af..b2c0d4b8424 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -4,6 +4,8 @@ title: 'feature request: ' labels: - kind/new feature - meta/needs triage +projects: + - DMS Core Backlog body: - type: markdown From 1a938dfb15e94f43f1af5ddaf7b8276aba92a825 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sat, 28 Sep 2024 13:27:34 +0200 Subject: [PATCH 425/592] Rspamd: update GTube patters in tests (#4191) --- CHANGELOG.md | 1 + test/config/rspamd_full/user-patches.sh | 2 +- .../parallel/set1/spam_virus/rspamd_full.bats | 15 +++++++++------ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21008a64ba1..14b0b4c9e87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ All notable changes to this project will be documented in this file. The format - **Postfix:** - Disable Microsoft reactions to outgoing mail ([#4120](https://github.com/docker-mailserver/docker-mailserver/pull/4120)) - bumped `jaq` version from 1.3.0 to 1.6.0 ([#4190](https://github.com/docker-mailserver/docker-mailserver/pull/4190)) +- updated Rspamd GTube settings and tests ([#4191](https://github.com/docker-mailserver/docker-mailserver/pull/4191)) ### Fixes diff --git a/test/config/rspamd_full/user-patches.sh b/test/config/rspamd_full/user-patches.sh index f731ff36660..56906064979 100644 --- a/test/config/rspamd_full/user-patches.sh +++ b/test/config/rspamd_full/user-patches.sh @@ -5,7 +5,7 @@ # # We do not use `custom-commands.conf` because this a feature # we are testing too. -echo "enable_test_patterns = true;" >>/etc/rspamd/local.d/options.inc +echo 'gtube_patterns = "all"' >>/etc/rspamd/local.d/options.inc # We want Dovecot to be very detailed about what it is doing, # specifically for Sieve because we need to check whether the diff --git a/test/tests/parallel/set1/spam_virus/rspamd_full.bats b/test/tests/parallel/set1/spam_virus/rspamd_full.bats index 3bc9fb1059d..961731fa27d 100644 --- a/test/tests/parallel/set1/spam_virus/rspamd_full.bats +++ b/test/tests/parallel/set1/spam_virus/rspamd_full.bats @@ -45,8 +45,10 @@ function setup_file() { _wait_for_smtp_port_in_container # We will send 5 emails: - # 1. The first one should pass just fine + # 1. The first ones should pass just fine _send_email_with_msgid 'rspamd-test-email-pass' + _send_email_with_msgid 'rspamd-test-email-pass-gtube' \ + --body 'AJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X' # 2. The second one should be rejected (Rspamd-specific GTUBE pattern for rejection) _send_spam --expect-rejection # 3. The third one should be rejected due to a virus (ClamAV EICAR pattern) @@ -54,7 +56,7 @@ function setup_file() { _send_email_with_msgid 'rspamd-test-email-virus' --expect-rejection \ --body 'X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*' # 4. The fourth one will receive an added header (Rspamd-specific GTUBE pattern for adding a spam header) - # ref: https://rspamd.com/doc/gtube_patterns.html + # ref: https://rspamd.com/doc/other/gtube_patterns.html _send_email_with_msgid 'rspamd-test-email-header' \ --body "YJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X" # 5. The fifth one will have its subject rewritten, but now spam header is applied. @@ -134,11 +136,12 @@ function teardown_file() { _default_teardown ; } @test 'normal mail passes fine' { _service_log_should_contain_string 'rspamd' 'F (no action)' + _service_log_should_contain_string 'rspamd' 'S (no action)' _print_mail_log_for_msgid 'rspamd-test-email-pass' assert_output --partial "stored mail into mailbox 'INBOX'" - _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 2 + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 3 } @test 'detects and rejects spam' { @@ -153,7 +156,7 @@ function teardown_file() { _default_teardown ; } refute_output --partial "stored mail into mailbox 'INBOX'" assert_failure - _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 2 + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 3 } @test 'detects and rejects virus' { @@ -168,7 +171,7 @@ function teardown_file() { _default_teardown ; } refute_output --partial "stored mail into mailbox 'INBOX'" assert_failure - _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 2 + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 3 } @test 'custom commands work correctly' { @@ -246,7 +249,7 @@ function teardown_file() { _default_teardown ; } _print_mail_log_for_msgid 'rspamd-test-email-header' assert_output --partial "fileinto action: stored mail into mailbox [SPECIAL-USE \\Junk]" - _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 2 + _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/new/ 3 _count_files_in_directory_in_container /var/mail/localhost.localdomain/user1/.Junk/new/ 1 } From 3937e1e719f4058e0a4a846bff39383071f8391d Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 29 Sep 2024 12:18:43 +0200 Subject: [PATCH 426/592] scripts: improve DKIM path scanning in Rspamd setup (#4201) --- CHANGELOG.md | 2 + .../startup/setup.d/security/rspamd.sh | 40 +++++++------------ 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14b0b4c9e87..023758e5979 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,8 @@ All notable changes to this project will be documented in this file. The format - The main `mail.log` which is piped to stdout via `tail` now correctly begins from the first log line of the active container run. Previously some daemon logs and potential warnings/errors were omitted. ([#4146](https://github.com/docker-mailserver/docker-mailserver/pull/4146)) - Fixed a regression introduced in v14 where `postfix-main.cf` appended `stderr` output into `/etc/postfix/main.cf`, causing Postfix startup to fail ([#4147](https://github.com/docker-mailserver/docker-mailserver/pull/4147)) - Unused `shopt -s inherit_errexit` removed from `start-mailserver.sh` ([#4161](https://github.com/docker-mailserver/docker-mailserver/pull/4161)) +- **Rspamd:** + - DKIM private key path checking is now performed only on paths that do not contain "$" ([#4201](https://github.com/docker-mailserver/docker-mailserver/pull/4201)) ### CI diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 379162822f8..18b9703b887 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -76,8 +76,9 @@ function __rspamd__run_early_setup_and_checks() { mkdir -p /var/lib/rspamd/ : >/var/lib/rspamd/stats.ucl - if [[ -d ${RSPAMD_DMS_OVERRIDE_D} ]]; then - cp "${RSPAMD_DMS_OVERRIDE_D}"/* "${RSPAMD_OVERRIDE_D}" + # Copy if directory exists and is not empty + if [[ -d ${RSPAMD_DMS_OVERRIDE_D} ]] && [[ -z $(find "${RSPAMD_DMS_OVERRIDE_D}" -maxdepth 0 -empty) ]]; then + cp "${RSPAMD_DMS_OVERRIDE_D}/"* "${RSPAMD_OVERRIDE_D}" fi if [[ ${ENABLE_AMAVIS} -eq 1 ]] || [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]]; then @@ -319,8 +320,7 @@ function __rspamd__setup_check_authenticated() { local MODULE_FILE="${RSPAMD_LOCAL_D}/settings.conf" readonly MODULE_FILE if _env_var_expect_zero_or_one 'RSPAMD_CHECK_AUTHENTICATED' \ - && [[ ${RSPAMD_CHECK_AUTHENTICATED} -eq 0 ]] - then + && [[ ${RSPAMD_CHECK_AUTHENTICATED} -eq 0 ]]; then __rspamd__log 'debug' 'Content checks for authenticated users are disabled' else __rspamd__log 'debug' 'Enabling content checks for authenticated users' @@ -332,32 +332,22 @@ function __rspamd__setup_check_authenticated() { # This function performs a simple check: go through DKIM configuration files, acquire # all private key file locations and check whether they exist and whether they can be -# accessed by Rspamd. +# accessed by Rspamd. We are not checking paths that conatain the '$' symbol. function __rspamd__check_dkim_permissions() { - local DKIM_CONF_FILES DKIM_KEY_FILES - [[ -f ${RSPAMD_LOCAL_D}/dkim_signing.conf ]] && DKIM_CONF_FILES+=("${RSPAMD_LOCAL_D}/dkim_signing.conf") - [[ -f ${RSPAMD_OVERRIDE_D}/dkim_signing.conf ]] && DKIM_CONF_FILES+=("${RSPAMD_OVERRIDE_D}/dkim_signing.conf") - - # Here, we populate DKIM_KEY_FILES which we later iterate over. DKIM_KEY_FILES - # contains all keys files configured by the user. - local FILE - for FILE in "${DKIM_CONF_FILES[@]}"; do - readarray -t DKIM_KEY_FILES_TMP < <(grep -o -E 'path = .*' "${FILE}" | cut -d '=' -f 2 | tr -d ' ";') - DKIM_KEY_FILES+=("${DKIM_KEY_FILES_TMP[@]}") - done - - for FILE in "${DKIM_KEY_FILES[@]}"; do - if [[ -f ${FILE} ]]; then - __rspamd__log 'trace' "Checking DKIM file '${FILE}'" + local KEY_FILE + while read -r KEY_FILE; do + if [[ -f ${KEY_FILE} ]]; then + __rspamd__log 'trace' "Checking DKIM file '${KEY_FILE}'" # See https://serverfault.com/a/829314 for an explanation on `-exec false {} +` # We additionally resolve symbolic links to check the permissions of the actual files - if find "$(realpath -eL "${FILE}")" \( -user _rspamd -or -group _rspamd -or -perm -o=r \) -exec false {} +; then - __rspamd__log 'warn' "Rspamd DKIM private key file '${FILE}' does not appear to have correct permissions/ownership for Rspamd to use it" + if find "$(realpath -L "${KEY_FILE}")" \( -user _rspamd -or -group _rspamd -or -perm -o=r \) \ + -exec false {} +; then + __rspamd__log 'warn' "Rspamd DKIM private key file '${KEY_FILE}' does not appear to have correct permissions/ownership for Rspamd to use it" else - __rspamd__log 'trace' "DKIM file '${FILE}' permissions and ownership appear correct" + __rspamd__log 'trace' "DKIM file '${KEY_FILE}' permissions and ownership appear correct" fi else - __rspamd__log 'warn' "Rspamd DKIM private key file '${FILE}' is configured for usage, but does not appear to exist" + __rspamd__log 'warn' "Rspamd DKIM private key file '${KEY_FILE}' is configured for usage, but does not appear to exist" fi - done + done < <(rspamadm configdump dkim_signing | grep 'path =' | grep -v -F '$' | awk '{print $3}' | tr -d ';"') } From 2bcc5cf9de61d8a4634d494ab31ad24b992dccfe Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 29 Sep 2024 12:53:10 +0200 Subject: [PATCH 427/592] Rspamd documentation: update Abusix signup link (#4204) --- docs/content/config/security/rspamd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index a98d3123368..8705262cfbe 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -251,7 +251,7 @@ There is a dedicated [section for setting up DKIM with Rspamd in our documentati This subsection provides information about the integration of [Abusix][abusix-web], "a set of blocklists that work as an additional email security layer for your existing mail environment". The setup is straight-forward and well documented: -1. [Create an account](https://app.abusix.com/signup) +1. [Create an account](https://app.abusix.com/) 2. Retrieve your API key 3. Navigate to the ["Getting Started" documentation for Rspamd][abusix-docs::rspamd-integration] and follow the steps described there 4. Make sure to change `` to your private API key From c29fe3ff0bc888016720536228878b3f19131056 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 08:54:57 +1300 Subject: [PATCH 428/592] chore(deps): Bump docker/build-push-action from 6.7.0 to 6.9.0 (#4205) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.7.0 to 6.9.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v6.7.0...v6.9.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 203416febb5..80e5b6d43d2 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v6.7.0 + uses: docker/build-push-action@v6.9.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 3a3d3749cde..9c6828f01e0 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -67,7 +67,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: 'Build and publish images' - uses: docker/build-push-action@v6.7.0 + uses: docker/build-push-action@v6.9.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 1658ddccd91..330b2ba5efa 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.7.0 + uses: docker/build-push-action@v6.9.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 0275fc98cc9..b0e6b71aada 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.7.0 + uses: docker/build-push-action@v6.9.0 with: context: . tags: mailserver-testing:ci From 26a44995a9bba825b1348dbf3551687c2100f7a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 09:01:58 +1300 Subject: [PATCH 429/592] chore(deps): Bump docker/setup-buildx-action from 3.6.1 to 3.7.1 (#4216) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3.6.1 to 3.7.1. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v3.6.1...v3.7.1) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 80e5b6d43d2..4391235770a 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -79,7 +79,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.6.1 + uses: docker/setup-buildx-action@v3.7.1 # NOTE: AMD64 can build within 2 minutes - name: 'Build images' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 9c6828f01e0..4f94cc973f3 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -40,7 +40,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.6.1 + uses: docker/setup-buildx-action@v3.7.1 # Try get the cached build layers from a prior `generic_build.yml` job. # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 330b2ba5efa..9ee070a14f0 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -38,7 +38,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.6.1 + uses: docker/setup-buildx-action@v3.7.1 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index b0e6b71aada..d0088e1cef6 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -37,7 +37,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.6.1 + uses: docker/setup-buildx-action@v3.7.1 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. From 34eb54ac398c4c3846645cf79404d8528869891b Mon Sep 17 00:00:00 2001 From: pitilux <100941507+pitilux@users.noreply.github.com> Date: Sat, 12 Oct 2024 00:34:20 +0200 Subject: [PATCH 430/592] fix: Avoid alias being used as regex during dovecot dummy account userdb detection (#4222) Applies alternative approach previously suggested by @polarathene and adds test cases to prevent future regressions --- CHANGELOG.md | 1 + target/scripts/helpers/accounts.sh | 3 ++- test/config/postfix-virtual.cf | 10 ++++++++++ .../parallel/set3/mta/account_management.bats | 20 ++++++++++++++++++- 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 023758e5979..28a97c321b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ All notable changes to this project will be documented in this file. The format - **Dovecot:** - Update logwatch `ignore.conf` to exclude Xapian messages about pending documents ([#4060](https://github.com/docker-mailserver/docker-mailserver/pull/4060)) - `dovecot-fts-xapian` plugin was updated to `1.7.13`, fixing a regression with indexing ([#4095](https://github.com/docker-mailserver/docker-mailserver/pull/4095)) + - The Dovecot Quota support "dummy account" workaround no longer treats the alias as a regex when checking the Dovecot UserDB ([#4222](https://github.com/docker-mailserver/docker-mailserver/pull/4222)) - **LDAP:** - A previous compatibility fix for OAuth2 in v13.3.1 had not applied the actual LDAP config changes. This has been corrected ([#4175](https://github.com/docker-mailserver/docker-mailserver/pull/4175)) - **Internal:** diff --git a/target/scripts/helpers/accounts.sh b/target/scripts/helpers/accounts.sh index 78464b88893..8510b6afc38 100644 --- a/target/scripts/helpers/accounts.sh +++ b/target/scripts/helpers/accounts.sh @@ -135,7 +135,8 @@ function _create_dovecot_alias_dummy_accounts() { fi DOVECOT_USERDB_LINE="${ALIAS}:${REAL_ACC[1]}:${DMS_VMAIL_UID}:${DMS_VMAIL_GID}::/var/mail/${REAL_DOMAINNAME}/${REAL_USERNAME}/home::${REAL_ACC[2]:-}" - if grep -qi "^${ALIAS}:" "${DOVECOT_USERDB_FILE}"; then + # Match a full line with `-xF` to avoid regex patterns introducing false positives matching `ALIAS`: + if grep -qixF "${DOVECOT_USERDB_LINE}" "${DOVECOT_USERDB_FILE}"; then _log 'warn' "Alias '${ALIAS}' will not be added to '${DOVECOT_USERDB_FILE}' twice" else echo "${DOVECOT_USERDB_LINE}" >>"${DOVECOT_USERDB_FILE}" diff --git a/test/config/postfix-virtual.cf b/test/config/postfix-virtual.cf index 4dec6bbbc56..88e29fb5b5f 100644 --- a/test/config/postfix-virtual.cf +++ b/test/config/postfix-virtual.cf @@ -3,3 +3,13 @@ alias1@localhost.localdomain user1@localhost.localdomain # this is also a test comment, :O alias2@localhost.localdomain external1@otherdomain.tld @localdomain2.com user1@localhost.localdomain + +## Dovecot "dummy accounts" for quota support (handled in `helpers/accounts.sh`) +# Do not filter alias by substring condition (longer prefix must be before substring alias): +# https://github.com/docker-mailserver/docker-mailserver/issues/2639 +prefixtest@localhost.localdomain user2@otherdomain.tld +test@localhost.localdomain user2@otherdomain.tld +# Do not filter alias when input be treated as regex tokens (eg `.`): +# https://github.com/docker-mailserver/docker-mailserver/issues/4170 +first-name@localhost.localdomain user2@otherdomain.tld +first.name@localhost.localdomain user2@otherdomain.tld diff --git a/test/tests/parallel/set3/mta/account_management.bats b/test/tests/parallel/set3/mta/account_management.bats index f8d5f9de6cc..f0d4796878f 100644 --- a/test/tests/parallel/set3/mta/account_management.bats +++ b/test/tests/parallel/set3/mta/account_management.bats @@ -29,7 +29,12 @@ function teardown_file() { _default_teardown ; } assert_line --index 5 'alias1@localhost.localdomain' # TODO: Probably not intentional?: assert_line --index 6 '@localdomain2.com' - _should_output_number_of_lines 7 + # Dovecot "dummy accounts" for quota support, see `test/config/postfix-virtual.cf` for more context + assert_line --index 7 'prefixtest@localhost.localdomain' + assert_line --index 8 'test@localhost.localdomain' + assert_line --index 9 'first-name@localhost.localdomain' + assert_line --index 10 'first.name@localhost.localdomain' + _should_output_number_of_lines 11 # Relevant log output from scripts/helpers/accounts.sh:_create_dovecot_alias_dummy_accounts(): # [ DEBUG ] Adding alias 'alias1@localhost.localdomain' for user 'user1@localhost.localdomain' to Dovecot's userdb @@ -37,6 +42,19 @@ function teardown_file() { _default_teardown ; } # [ DEBUG ] Adding alias '@localdomain2.com' for user 'user1@localhost.localdomain' to Dovecot's userdb } +# Dovecot "dummy accounts" for quota support, see `test/config/postfix-virtual.cf` for more context +@test "should create all dovecot dummy accounts" { + run docker logs "${CONTAINER_NAME}" + assert_success + assert_line --partial "Adding alias 'prefixtest@localhost.localdomain' for user 'user2@otherdomain.tld' to Dovecot's userdb" + assert_line --partial "Adding alias 'test@localhost.localdomain' for user 'user2@otherdomain.tld' to Dovecot's userdb" + refute_line --partial "Alias 'test@localhost.localdomain' will not be added to '/etc/dovecot/userdb' twice" + + assert_line --partial "Adding alias 'first-name@localhost.localdomain' for user 'user2@otherdomain.tld' to Dovecot's userdb" + assert_line --partial "Adding alias 'first.name@localhost.localdomain' for user 'user2@otherdomain.tld' to Dovecot's userdb" + refute_line --partial "Alias 'first.name@localhost.localdomain' will not be added to '/etc/dovecot/userdb' twice" +} + @test "should have created maildir for 'user1@localhost.localdomain'" { _run_in_container_bash '[[ -d /var/mail/localhost.localdomain/user1 ]]' assert_success From ff8fc8013b3d145f7bdb0210d168f973e4af9329 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2024 19:23:45 +0100 Subject: [PATCH 431/592] chore(deps): Bump anchore/scan-action from 4.1.2 to 5.1.0 (#4239) --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index d0088e1cef6..42fa30f3ce8 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v4.1.2 + uses: anchore/scan-action@v5.1.0 id: scan with: image: mailserver-testing:ci From 662afec1d115ad694acbaf869bdb6e24cc98e96c Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Wed, 30 Oct 2024 01:32:17 +0100 Subject: [PATCH 432/592] doc: add an example for using the keytype ed25519 when using DKIM & Rspamd (#4243) --- target/bin/rspamd-dkim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/target/bin/rspamd-dkim b/target/bin/rspamd-dkim index 689aa7d51b7..b376271b4fc 100755 --- a/target/bin/rspamd-dkim +++ b/target/bin/rspamd-dkim @@ -55,6 +55,10 @@ ${ORANGE}EXAMPLES${RESET} ${LWHITE}setup config dkim domain example.com${RESET} Generate the DKIM key for a different domain (example.com). + ${LWHITE}setup config dkim keytype ed25519 domain edward.com selector elliptic-test${RESET} + Generate the DKIM key using the ED25519 elliptic curve for the domain + edward.com and the selector elliptic-test. + ${ORANGE}EXIT STATUS${RESET} Exit status is 0 if command was successful. If wrong arguments are provided or arguments contain errors, the script will exit early with a non-zero exit status. From e6bd0b0a0932f11c60541322e769f3d10b711552 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 1 Nov 2024 14:11:33 +1300 Subject: [PATCH 433/592] ci(pr-docs): Acquire metadata from context (#4244) The metadata needed is available via context, prefer this approach instead. --- .github/workflows/docs-preview-deploy.yml | 29 ++++++++++++++++++---- .github/workflows/docs-preview-prepare.yml | 17 +------------ 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index 7c924f8655b..37b5464e83c 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -16,14 +16,20 @@ jobs: preview: name: 'Deploy Preview' runs-on: ubuntu-22.04 - if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' }} + # Requires a PR event triggered `docs-preview-prepare.yml` workflow run that was successful + ensure the head SHA belongs to an associated PR: + if: | + ${{ + github.event.workflow_run.conclusion == 'success' + && github.event.workflow_run.event == 'pull_request' + && contains(github.event.workflow_run.pull_requests.*.head.sha, github.event.workflow_run.head_sha) + }} steps: # ======================== # # Restore workflow context # # ======================== # - # Retrieve the artifact uploaded from `docs-preview-prepare.yml` workflow run that triggered this deployment + # Retrieve the build artifact uploaded from the `docs-preview-prepare.yml` workflow run (that triggered this deployment workflow): - name: 'Download build artifact' uses: actions/download-artifact@v4 with: @@ -34,8 +40,18 @@ jobs: - name: 'Extract build artifact' run: tar -xf artifact.tar.zst - - name: 'Restore preserved ENV' - run: cat pr.env >> "${GITHUB_ENV}" + # The `workflow_run` metadata contains an array of `pull_requests`, get the `workflow_run` equivalent of `github.event.pull_request.number`. + # There should only be one PR item in the array, verify that it shares the same `head_sha` (latest commit of PR). + - name: 'Get PR number' + env: + head_sha: ${{ github.event.workflow_run.head_sha }} + pull_requests: ${{ tojson(github.event.workflow_run.pull_requests) }} + run: | + PR_NUMBER=$(jq -r '[.[] | select(.head.sha == "${{ env.head_sha }}")][0].number' <<< '${{ env.pull_requests }}') + { + echo "PR_NUMBER=${PR_NUMBER}" + echo 'PR_HEADSHA=${{ env.head_sha }}' + } >> "${GITHUB_ENV}" # ==================== # # Deploy preview build # @@ -61,6 +77,9 @@ jobs: env: NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} + # Keep these two ENV in sync with the `docs-preview-prepare.yml` workflow: + BUILD_DIR: docs/site + NETLIFY_SITE_PREFIX: pullrequest-${{ env.PR_NUMBER }} with: github-token: ${{ secrets.GITHUB_TOKEN }} # Fail the job early if credentials are missing / invalid: @@ -71,7 +90,7 @@ jobs: # Only publish the contents of the build output: publish-dir: ${{ env.BUILD_DIR }} # Custom message for the deploy log on Netlify: - deploy-message: '${{ env.PR_TITLE }} (PR #${{ env.PR_NUMBER }} @ commit: ${{ env.PR_HEADSHA }})' + deploy-message: 'Preview Build (PR #${{ env.PR_NUMBER }} @ commit: ${{ env.PR_HEADSHA }})' # Note: Split workflow incorrectly references latest primary branch commit for deployment. # Assign to non-default Deployment Environment for better management: diff --git a/.github/workflows/docs-preview-prepare.yml b/.github/workflows/docs-preview-prepare.yml index cee5562abfd..bf7ad135eea 100644 --- a/.github/workflows/docs-preview-prepare.yml +++ b/.github/workflows/docs-preview-prepare.yml @@ -54,23 +54,8 @@ jobs: # ============================== # # Minimize risk of upload failure by bundling files to a single compressed archive (tar + zstd). - # Bundles build dir and env file into a compressed archive, nested file paths will be preserved. - name: 'Prepare artifact for transfer' - env: - # As a precaution, reference this value by an interpolated ENV var; - # instead of interpolating user controllable input directly in the shell script.. - # https://github.com/docker-mailserver/docker-mailserver/issues/2332#issuecomment-998326798 - PR_TITLE: ${{ github.event.pull_request.title }} - run: | - # Save ENV for transfer - { - echo "PR_HEADSHA=${{ github.event.pull_request.head.sha }}" - echo "PR_NUMBER=${{ github.event.pull_request.number }}" - echo "PR_TITLE=${PR_TITLE}" - echo "NETLIFY_SITE_PREFIX=${{ env.NETLIFY_SITE_PREFIX }}" - echo "BUILD_DIR=${{ env.BUILD_DIR }}" - } >> pr.env - tar --zstd -cf artifact.tar.zst pr.env ${{ env.BUILD_DIR }} + run: tar --zstd -cf artifact.tar.zst ${{ env.BUILD_DIR }} - name: 'Upload artifact for workflow transfer' uses: actions/upload-artifact@v4 From dc0a6403b22536ce72841b48636abf7a254a4dbd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Nov 2024 09:21:53 +1300 Subject: [PATCH 434/592] chore(deps): Bump anchore/scan-action from 5.1.0 to 5.2.0 (#4249) Bumps [anchore/scan-action](https://github.com/anchore/scan-action) from 5.1.0 to 5.2.0. - [Release notes](https://github.com/anchore/scan-action/releases) - [Changelog](https://github.com/anchore/scan-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/anchore/scan-action/compare/v5.1.0...v5.2.0) --- updated-dependencies: - dependency-name: anchore/scan-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 42fa30f3ce8..b96d8ea59a0 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v5.1.0 + uses: anchore/scan-action@v5.2.0 id: scan with: image: mailserver-testing:ci From 0ff9c0132a8914d6756739a7a3b085e47870b93d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alvaro=20Mu=C3=B1oz?= Date: Tue, 5 Nov 2024 00:50:08 +0100 Subject: [PATCH 435/592] ci: Revise `docs-preview-deploy.yml` (#4247) - Fixes the `if` condition that was recently adjusted. - Better documents concerns for maintainers to be aware of. - Reference the `pull_requests` ENV at runtime instead of embedding content into the script via GHA context expression. This is a better practice which prevent exploits from untrusted inputs (_notably for context objects which might introduce new fields in future_). --------- Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .github/workflows/docs-preview-deploy.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index 37b5464e83c..02dc41835b8 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -17,12 +17,12 @@ jobs: name: 'Deploy Preview' runs-on: ubuntu-22.04 # Requires a PR event triggered `docs-preview-prepare.yml` workflow run that was successful + ensure the head SHA belongs to an associated PR: + # NOTE: A multi-line `if` GHA expression must avoid wrapping with `${{ }}`, otherwise it is unintentionally parsed as a string: + # https://github.com/nikitastupin/pwnhub/blob/main/writings/if-condition.md if: | - ${{ - github.event.workflow_run.conclusion == 'success' - && github.event.workflow_run.event == 'pull_request' - && contains(github.event.workflow_run.pull_requests.*.head.sha, github.event.workflow_run.head_sha) - }} + github.event.workflow_run.conclusion == 'success' + && github.event.workflow_run.event == 'pull_request' + && contains(github.event.workflow_run.pull_requests.*.head.sha, github.event.workflow_run.head_sha) steps: # ======================== # @@ -42,12 +42,14 @@ jobs: # The `workflow_run` metadata contains an array of `pull_requests`, get the `workflow_run` equivalent of `github.event.pull_request.number`. # There should only be one PR item in the array, verify that it shares the same `head_sha` (latest commit of PR). + # NOTE: Careful when using GHA context expressions that may have untrusted input here. The expressions are evaluated before the script content itself is run: + # https://github.com/docker-mailserver/docker-mailserver/pull/4247#discussion_r1827067475 - name: 'Get PR number' env: head_sha: ${{ github.event.workflow_run.head_sha }} pull_requests: ${{ tojson(github.event.workflow_run.pull_requests) }} run: | - PR_NUMBER=$(jq -r '[.[] | select(.head.sha == "${{ env.head_sha }}")][0].number' <<< '${{ env.pull_requests }}') + PR_NUMBER=$(jq -r '[.[] | select(.head.sha == "${{ env.head_sha }}")][0].number' <<< "${pull_requests}") { echo "PR_NUMBER=${PR_NUMBER}" echo 'PR_HEADSHA=${{ env.head_sha }}' From a599936c4b5cfc81d00f7bc9d3c67ddff48f5f52 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sun, 10 Nov 2024 18:57:31 +1300 Subject: [PATCH 436/592] ci: `docs-preview-deploy.yml` - Remove third job condition This condition was added as an additional guard but was preventing the workflow from running when PRs were from forked repos. --- .github/workflows/docs-preview-deploy.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index 02dc41835b8..e7ca9f12d85 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -22,7 +22,6 @@ jobs: if: | github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request' - && contains(github.event.workflow_run.pull_requests.*.head.sha, github.event.workflow_run.head_sha) steps: # ======================== # From b960efad748dbf6fdfe7be2e864eca431296cbf5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Nov 2024 08:49:29 +1300 Subject: [PATCH 437/592] chore(deps): Bump anchore/scan-action from 5.2.0 to 5.2.1 (#4260) Bumps [anchore/scan-action](https://github.com/anchore/scan-action) from 5.2.0 to 5.2.1. - [Release notes](https://github.com/anchore/scan-action/releases) - [Changelog](https://github.com/anchore/scan-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/anchore/scan-action/compare/v5.2.0...v5.2.1) --- updated-dependencies: - dependency-name: anchore/scan-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index b96d8ea59a0..5b597d873f9 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v5.2.0 + uses: anchore/scan-action@v5.2.1 id: scan with: image: mailserver-testing:ci From 6b4627ceab44c65d04ae25926aab05956ca2bafb Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 15 Nov 2024 13:00:40 +1300 Subject: [PATCH 438/592] ci(docs-preview): Refactor workflows (#4262) **Overview of changes:** - Runner bumped from Ubuntu 22.04 => 24.04 - Revised inline documentation for maintainers. - The output of `build-docs.sh` is now grouped in the steps action log, and now hides the noise from pulling the image via `docker run`. - Removed the separate `tar` steps with ZSTD as there is only a directory to archive with recent changes to this workflow. The `upload` + `download` actions are sufficient. - The `workflow_run` job has had the PR context restore step extracted to a separate job to minimize noise. - `actions-netlify` is still effectively the same functionality. - `github-token` is no longer configured as it doesn't appear needed with the functions disabled. - Opt-out of the GH deployments feature which is not needed. --- .github/workflows/docs-preview-deploy.yml | 161 +++++++++++-------- .github/workflows/docs-preview-prepare.yml | 55 ++++--- .github/workflows/scripts/docs/build-docs.sh | 3 +- 3 files changed, 130 insertions(+), 89 deletions(-) diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index e7ca9f12d85..40b5cbd27b7 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -1,133 +1,164 @@ name: 'Documentation (run)' on: + # This workflow runs off the primary branch which provides access to the `secrets` context: workflow_run: workflows: ['Documentation (PR)'] types: - completed -# Note: If limiting concurrency is required for this workflow: -# 1. Add an additional job prior to `preview` to get the PR number make it an output. -# 2. Assign that new job as a `needs` dependency for the `preview` job. -# It is still required for `preview` job to download the artifact so that it can access the preview build files. +permissions: + # Required by `actions/download-artifact`: + actions: read + # Required by `marocchino/sticky-pull-request-comment`: + pull-requests: write + # Required by `myrotvorets/set-commit-status-action`: + statuses: write -# This workflow runs off the primary branch and has access to secrets as expected. jobs: - preview: - name: 'Deploy Preview' - runs-on: ubuntu-22.04 + # This could have been another step in the `deploy-preview` job and used `GITHUB_ENV` instead of `GITHUB_OUTPUT`. + # It was split out into a separate job for a cleaner overview of `deploy-preview` ENV inputs and to minimize noise + # from that job related to this workaround (_that is incompatible with PRs from forks_). + pr-context: + name: 'Restore PR Context' + runs-on: ubuntu-24.04 + outputs: + PR_HEADSHA: ${{ steps.set-pr-context.outputs.PR_HEADSHA }} + PR_NUMBER: ${{ steps.set-pr-context.outputs.PR_NUMBER }} # Requires a PR event triggered `docs-preview-prepare.yml` workflow run that was successful + ensure the head SHA belongs to an associated PR: - # NOTE: A multi-line `if` GHA expression must avoid wrapping with `${{ }}`, otherwise it is unintentionally parsed as a string: - # https://github.com/nikitastupin/pwnhub/blob/main/writings/if-condition.md + # NOTE: + # - The `contains` condition checks for event context that is not available when the PR is from a fork. An alternative method would be needed: + # https://stackoverflow.com/questions/59077079/how-to-get-pull-request-number-within-github-actions-workflow/79017997#79017997 + # - A multi-line `if` GHA expression must avoid wrapping with `${{ }}`, otherwise it is unintentionally parsed as a string: + # https://github.com/nikitastupin/pwnhub/blob/main/writings/if-condition.md if: | github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request' + && contains(github.event.workflow_run.pull_requests.*.head.sha, github.event.workflow_run.head_sha) steps: - - # ======================== # - # Restore workflow context # - # ======================== # - - # Retrieve the build artifact uploaded from the `docs-preview-prepare.yml` workflow run (that triggered this deployment workflow): - - name: 'Download build artifact' - uses: actions/download-artifact@v4 - with: - name: preview-build - github-token: ${{ secrets.GITHUB_TOKEN }} - run-id: ${{ github.event.workflow_run.id }} - - - name: 'Extract build artifact' - run: tar -xf artifact.tar.zst - - # The `workflow_run` metadata contains an array of `pull_requests`, get the `workflow_run` equivalent of `github.event.pull_request.number`. - # There should only be one PR item in the array, verify that it shares the same `head_sha` (latest commit of PR). - # NOTE: Careful when using GHA context expressions that may have untrusted input here. The expressions are evaluated before the script content itself is run: - # https://github.com/docker-mailserver/docker-mailserver/pull/4247#discussion_r1827067475 + # NOTE: + # - The `workflow_run` metadata contains an array of `pull_requests`: + # 1. Take the `workflow_run` equivalent of `github.event.pull_request.number`. + # 2. There should only be one PR item in the array, verify that it shares the same `head_sha` (latest commit of PR). + # - Careful when using GHA context expressions that may have untrusted input here. The expressions are evaluated before the script content itself is run: + # https://github.com/docker-mailserver/docker-mailserver/pull/4247#discussion_r1827067475 - name: 'Get PR number' + id: set-pr-context env: head_sha: ${{ github.event.workflow_run.head_sha }} pull_requests: ${{ tojson(github.event.workflow_run.pull_requests) }} run: | PR_NUMBER=$(jq -r '[.[] | select(.head.sha == "${{ env.head_sha }}")][0].number' <<< "${pull_requests}") { - echo "PR_NUMBER=${PR_NUMBER}" echo 'PR_HEADSHA=${{ env.head_sha }}' - } >> "${GITHUB_ENV}" + echo "PR_NUMBER=${PR_NUMBER}" + } >> "${GITHUB_OUTPUT}" + + deploy-preview: + name: 'Deploy Preview' + runs-on: ubuntu-24.04 + needs: [pr-context] + env: + # NOTE: Keep this in sync with the equivalent ENV in `docs-preview-prepare.yml`: + BUILD_DIR: docs/site/ + # PR head SHA (latest commit): + PR_HEADSHA: ${{ needs.pr-context.outputs.PR_HEADSHA }} + PR_NUMBER: ${{ needs.pr-context.outputs.PR_NUMBER }} + # Deploy URL preview prefix (the site name for this prefix is managed at Netlify): + PREVIEW_SITE_PREFIX: pullrequest-${{ needs.pr-context.outputs.PR_NUMBER }} + steps: + - name: 'Retrieve and extract the built docs preview' + uses: actions/download-artifact@v4 + with: + name: preview-build + path: ${{ env.BUILD_DIR }} + # These are needed due this approach relying on `workflow_run`, so that it can access the build artifact: + # (uploaded from the associated `docs-preview-prepare.yml` workflow run) + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} # ==================== # # Deploy preview build # # ==================== # - # Manage workflow deployment status. `enable-commit-status` from `nwtgck/actions-netlify` would handle this, - # but presently does not work correctly via split workflow. It is useful in a split workflow as the 1st stage - # no longer indicates if the entire workflow/deployment was successful. - - name: 'Commit Status: Set Workflow Status as Pending' + # Manage workflow deployment status (Part 1/2): + # NOTE: + # - `workflow_run` trigger does not appear on the PR/commit checks status, only the initial prepare workflow triggered. + # This adds our own status check for this 2nd half of the workflow starting as `pending`, followed by `success` / `failure` at the end. + # - `enable-commit-status` from `nwtgck/actions-netlify` would have handled this, + # but the context `github.sha` that action tries to use references the primary branch commit that this workflow runs from, not the relevant PR commit. + - name: 'Commit Status (1/2) - Set Workflow Status as Pending' uses: myrotvorets/set-commit-status-action@v2.0.1 with: token: ${{ secrets.GITHUB_TOKEN }} status: pending - # Should match `env.PR_HEADSHA` when triggered by `pull_request` event workflow, - # Avoids failure of ENV being unavailable if job fails early: - sha: ${{ github.event.workflow_run.head_sha }} + sha: ${{ env.PR_HEADSHA }} context: 'Deploy Preview (pull_request => workflow_run)' - name: 'Send preview build to Netlify' uses: nwtgck/actions-netlify@v3.0 - id: preview + id: preview-netlify timeout-minutes: 1 env: NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} - NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} - # Keep these two ENV in sync with the `docs-preview-prepare.yml` workflow: - BUILD_DIR: docs/site - NETLIFY_SITE_PREFIX: pullrequest-${{ env.PR_NUMBER }} + NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} with: - github-token: ${{ secrets.GITHUB_TOKEN }} - # Fail the job early if credentials are missing / invalid: + # Fail the job when the required Netlify credentials are missing from ENV: fails-without-credentials: true - # Sets/creates the Netlify deploy URL prefix. - # Uses the PR number for uniqueness: - alias: ${{ env.NETLIFY_SITE_PREFIX }} + # Set/create the Netlify deploy URL prefix: + alias: ${{ env.PREVIEW_SITE_PREFIX }} # Only publish the contents of the build output: publish-dir: ${{ env.BUILD_DIR }} # Custom message for the deploy log on Netlify: - deploy-message: 'Preview Build (PR #${{ env.PR_NUMBER }} @ commit: ${{ env.PR_HEADSHA }})' - - # Note: Split workflow incorrectly references latest primary branch commit for deployment. - # Assign to non-default Deployment Environment for better management: - github-deployment-environment: documentation-previews - github-deployment-description: 'Preview deploy for documentation PRs' + deploy-message: 'Preview Build (PR #${{ env.PR_NUMBER }} @ commit: ${{ env.PR_HEADSHA }}' - # Note - PR context used by this action is incorrect. These features are broken with split workflow: - # https://github.com/nwtgck/actions-netlify/issues/545 # Disable unwanted action defaults: - # Disable adding deploy comment on pre-merge commit (Github creates this for PR diff): + # This input does not fallback to the GITHUB_TOKEN taken from context, nor log that it will skip extra features of the action when this input is not set: + # https://github.com/nwtgck/actions-netlify/issues/1219 + # github-token: ${{ secrets.GITHUB_TOKEN }} + # NOTE: These features won't work correctly when the triggered workflow is not run from the PR branch due to assumed `pull_request` context: + # https://github.com/nwtgck/actions-netlify/issues/545 + # Disable adding a comment to the commit belonging to context `github.sha` about the successful deployment (redundant and often wrong commit): enable-commit-comment: false - # Disable adding a "Netlify - Netlify deployment" check status: + # Disable adding a "Netlify - Netlify deployment" PR check status (workflow job status is sufficient): enable-commit-status: false - # Disable. We provide a custom PR comment in the next action: + # Disable adding a comment about successful deployment status to the PR. + # Prefer `marocchino/sticky-pull-request-comment` instead (more flexible and allows custom message): enable-pull-request-comment: false + # Opt-out of deployment feature: + # NOTE: + # - When affected by `nwtgck/actions-netlify/issues/545`, the deployments published reference the wrong commit and thus information. + # - While the feature creates or assigns a deployment to associate the build with, it is unrelated to the related environments feature (secrets/vars): + # https://github.com/nwtgck/actions-netlify/issues/538#issuecomment-833983970 + # https://docs.github.com/en/actions/managing-workflow-runs-and-deployments/managing-deployments/viewing-deployment-history + # https://docs.github.com/en/actions/managing-workflow-runs-and-deployments/managing-deployments/managing-environments-for-deployment + enable-github-deployment: false + # Assign to non-default Deployment Environment for better management: + # github-deployment-environment: documentation-previews + # github-deployment-description: 'Preview deploy for documentation PRs' # If a `netlify.toml` config is ever needed, enable this: # netlify-config-path: ./docs/netlify.toml - # If ever switching from Github Pages, enable this conditionally (false by default): + # If ever switching from Github Pages, enable this only when not deploying a preview build (false by default): # production-deploy: false - - name: 'Comment on PR: Add/Update deployment status' + - name: 'Comment on PR with preview link' uses: marocchino/sticky-pull-request-comment@v2 with: number: ${{ env.PR_NUMBER }} header: preview-comment recreate: true message: | - [Documentation preview for this PR](${{ steps.preview.outputs.deploy-url }}) is ready! :tada: + [Documentation preview for this PR](${{ steps.preview-netlify.outputs.deploy-url }}) is ready! :tada: Built with commit: ${{ env.PR_HEADSHA }} - - name: 'Commit Status: Update deployment status' + # Manage workflow deployment status (Part 2/2): + - name: 'Commit Status (2/2) - Update deployment status' uses: myrotvorets/set-commit-status-action@v2.0.1 - # Always run this step regardless of job failing early: + # Always run this step regardless of the job failing early: if: ${{ always() }} + # Custom status descriptions: env: DEPLOY_SUCCESS: Successfully deployed preview. DEPLOY_FAILURE: Failed to deploy preview. diff --git a/.github/workflows/docs-preview-prepare.yml b/.github/workflows/docs-preview-prepare.yml index bf7ad135eea..40b586e048d 100644 --- a/.github/workflows/docs-preview-prepare.yml +++ b/.github/workflows/docs-preview-prepare.yml @@ -7,59 +7,68 @@ on: - '.github/workflows/scripts/docs/build-docs.sh' - '.github/workflows/docs-preview-prepare.yml' -# If the workflow for a PR is triggered multiple times, previous existing runs will be canceled. -# eg: Applying multiple suggestions from a review directly via the Github UI. -# Instances of the 2nd phase of this workflow (via `workflow_run`) presently lack concurrency limits due to added complexity. +# If this workflow is triggered while already running for the PR, cancel any earlier running instances: +# Instances of the 2nd phase of this workflow (via `workflow_run`) lack any concurrency limits due to added complexity. concurrency: group: deploypreview-pullrequest-${{ github.event.pull_request.number }} cancel-in-progress: true +env: + # Build output directory (created by the mkdocs-material container, keep this in sync with `build-docs.sh`): + BUILD_DIR: docs/site/ + # These two are only needed to construct `PREVIEW_URL`: + PREVIEW_SITE_NAME: dms-doc-previews + PREVIEW_SITE_PREFIX: pullrequest-${{ github.event.pull_request.number }} + # `pull_request` workflow is unreliable alone: Non-collaborator contributions lack access to secrets for security reasons. # A separate workflow (docs-preview-deploy.yml) handles the deploy after the potentially untrusted code is first run in this workflow. # See: https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ permissions: + # Required by `actions/checkout` for git checkout: contents: read jobs: prepare-preview: name: 'Build Preview' - runs-on: ubuntu-22.04 - env: - BUILD_DIR: docs/site - NETLIFY_SITE_PREFIX: pullrequest-${{ github.event.pull_request.number }} - NETLIFY_SITE_NAME: dms-doc-previews + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 + # ================== # + # Build docs preview # + # ================== # + - name: 'Build with mkdocs-material via Docker' - working-directory: docs + working-directory: docs/ env: - PREVIEW_URL: 'https://${NETLIFY_SITE_PREFIX}--${NETLIFY_SITE_NAME}.netlify.app/' - NETLIFY_BRANDING: 'Deploys by Netlify' + PREVIEW_URL: 'https://${{ env.PREVIEW_SITE_PREFIX }}--${{ env.PREVIEW_SITE_NAME }}.netlify.app/' run: | - # Adjust mkdocs.yml for preview build - sed -i "s|^site_url:.*|site_url: '${PREVIEW_URL}'|" mkdocs.yml + # Adjust `mkdocs.yml` for the preview build requirements: + # - Replace production `site_url` with the preview URL (only affects the canonical link: https://en.wikipedia.org/wiki/Canonical_link_element#HTML) + # - Prepend Netlify logo link to `copyright` content + sed -i "s|^site_url:.*|site_url: '${{ env.PREVIEW_URL }}'|" mkdocs.yml - # Insert sponsor branding into page content (Provider OSS plan requirement): - # Upstream does not provide a nicer maintainable way to do this.. - # Prepends HTML to copyright text and then aligns to the right side. + # Insert branding into page content (Netlify OSS plan requirement): + # - `mkdocs-material` does not provide a better way to do this. + # - Prepends HTML to the copyright text and then aligns the logo to the right-side of the page. + NETLIFY_BRANDING='Deploys by Netlify' sed -i "s|^copyright: '|copyright: '${NETLIFY_BRANDING}|" mkdocs.yml - # Need to override a CSS media query for parent element to always be full width: + # Override a CSS media query for the parent element to always be full width: echo '.md-footer-copyright { width: 100%; }' >> content/assets/css/customizations.css - ../.github/workflows/scripts/docs/build-docs.sh + # Build and prepare for upload: + echo "::group::Build (stdout)" + bash ../.github/workflows/scripts/docs/build-docs.sh + echo "::endgroup::" # ============================== # # Volley over to secure workflow # # ============================== # - # Minimize risk of upload failure by bundling files to a single compressed archive (tar + zstd). - - name: 'Prepare artifact for transfer' - run: tar --zstd -cf artifact.tar.zst ${{ env.BUILD_DIR }} - + # Archives directory `path` into a ZIP file: - name: 'Upload artifact for workflow transfer' uses: actions/upload-artifact@v4 with: name: preview-build - path: artifact.tar.zst + path: ${{ env.BUILD_DIR }} retention-days: 1 diff --git a/.github/workflows/scripts/docs/build-docs.sh b/.github/workflows/scripts/docs/build-docs.sh index 5d1cab527c9..d4384b5e4c5 100755 --- a/.github/workflows/scripts/docs/build-docs.sh +++ b/.github/workflows/scripts/docs/build-docs.sh @@ -7,8 +7,9 @@ set -ex # `build --strict` ensures the build fails when any warnings are omitted. docker run \ --rm \ + --quiet \ --user "$(id -u):$(id -g)" \ - --volume "${PWD}:/docs" \ + --volume "./:/docs" \ --name "build-docs" \ squidfunk/mkdocs-material:9.5 build --strict From 02f1894f74b629d74dbcb8faac2908c2e95c5237 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 20 Nov 2024 16:37:34 +1300 Subject: [PATCH 439/592] ci(docs-preview): Acquire PR context via `gh` CLI (#4267) --- .github/workflows/docs-preview-deploy.yml | 62 +++++++++++------------ CHANGELOG.md | 1 + 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/.github/workflows/docs-preview-deploy.yml b/.github/workflows/docs-preview-deploy.yml index 40b5cbd27b7..ccb9c5f20a0 100644 --- a/.github/workflows/docs-preview-deploy.yml +++ b/.github/workflows/docs-preview-deploy.yml @@ -1,4 +1,4 @@ -name: 'Documentation (run)' +name: 'Documentation (Deploy)' on: # This workflow runs off the primary branch which provides access to the `secrets` context: @@ -10,49 +10,45 @@ on: permissions: # Required by `actions/download-artifact`: actions: read - # Required by `marocchino/sticky-pull-request-comment`: + # Required by `set-pr-context`: + contents: read + # Required by `marocchino/sticky-pull-request-comment` (write) + `set-pr-context` (read): pull-requests: write # Required by `myrotvorets/set-commit-status-action`: statuses: write jobs: - # This could have been another step in the `deploy-preview` job and used `GITHUB_ENV` instead of `GITHUB_OUTPUT`. - # It was split out into a separate job for a cleaner overview of `deploy-preview` ENV inputs and to minimize noise - # from that job related to this workaround (_that is incompatible with PRs from forks_). + # NOTE: This is handled as pre-requisite job to minimize the noise from acquiring these two outputs needed for `deploy-preview` ENV: pr-context: - name: 'Restore PR Context' + name: 'Acquire PR Context' runs-on: ubuntu-24.04 outputs: - PR_HEADSHA: ${{ steps.set-pr-context.outputs.PR_HEADSHA }} - PR_NUMBER: ${{ steps.set-pr-context.outputs.PR_NUMBER }} - # Requires a PR event triggered `docs-preview-prepare.yml` workflow run that was successful + ensure the head SHA belongs to an associated PR: - # NOTE: - # - The `contains` condition checks for event context that is not available when the PR is from a fork. An alternative method would be needed: - # https://stackoverflow.com/questions/59077079/how-to-get-pull-request-number-within-github-actions-workflow/79017997#79017997 - # - A multi-line `if` GHA expression must avoid wrapping with `${{ }}`, otherwise it is unintentionally parsed as a string: - # https://github.com/nikitastupin/pwnhub/blob/main/writings/if-condition.md - if: | - github.event.workflow_run.conclusion == 'success' - && github.event.workflow_run.event == 'pull_request' - && contains(github.event.workflow_run.pull_requests.*.head.sha, github.event.workflow_run.head_sha) + PR_HEADSHA: ${{ steps.set-pr-context.outputs.head-sha }} + PR_NUMBER: ${{ steps.set-pr-context.outputs.number }} + if: ${{ github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request' }} steps: - # NOTE: - # - The `workflow_run` metadata contains an array of `pull_requests`: - # 1. Take the `workflow_run` equivalent of `github.event.pull_request.number`. - # 2. There should only be one PR item in the array, verify that it shares the same `head_sha` (latest commit of PR). - # - Careful when using GHA context expressions that may have untrusted input here. The expressions are evaluated before the script content itself is run: - # https://github.com/docker-mailserver/docker-mailserver/pull/4247#discussion_r1827067475 - - name: 'Get PR number' + - name: 'Get PR context' id: set-pr-context env: - head_sha: ${{ github.event.workflow_run.head_sha }} - pull_requests: ${{ tojson(github.event.workflow_run.pull_requests) }} + # Token is required for the GH CLI: + GH_TOKEN: ${{ github.token }} + # Best practice for scripts is to reference via ENV at runtime. Avoid using GHA context expressions in the script content directly: + # https://github.com/docker-mailserver/docker-mailserver/pull/4247#discussion_r1827067475 + PR_TARGET_REPO: ${{ github.repository }} + # If the PR is from a fork, prefix it with `:`, otherwise only the PR branch name is relevant: + PR_BRANCH: |- + ${{ + (github.event.workflow_run.head_repository.owner.login != github.event.workflow_run.repository.owner.login) + && format('{0}:{1}', github.event.workflow_run.head_repository.owner.login, github.event.workflow_run.head_branch) + || github.event.workflow_run.head_branch + }} + # Use the GH CLI to query the PR branch, which provides the PR number and head SHA to assign as outputs: + # (`--jq` formats JSON to `key=value` pairs and renames `headRefOid` to `head-sha`) run: | - PR_NUMBER=$(jq -r '[.[] | select(.head.sha == "${{ env.head_sha }}")][0].number' <<< "${pull_requests}") - { - echo 'PR_HEADSHA=${{ env.head_sha }}' - echo "PR_NUMBER=${PR_NUMBER}" - } >> "${GITHUB_OUTPUT}" + gh pr view --repo "${PR_TARGET_REPO}" "${PR_BRANCH}" \ + --json 'number,headRefOid' \ + --jq '"number=\(.number)\nhead-sha=\(.headRefOid)"' \ + >> "${GITHUB_OUTPUT}" deploy-preview: name: 'Deploy Preview' @@ -165,6 +161,6 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} status: ${{ job.status == 'success' && 'success' || 'failure' }} - sha: ${{ github.event.workflow_run.head_sha }} + sha: ${{ env.PR_HEADSHA }} context: 'Deploy Preview (pull_request => workflow_run)' description: ${{ job.status == 'success' && env.DEPLOY_SUCCESS || env.DEPLOY_FAILURE }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 28a97c321b6..0ba86566eb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ All notable changes to this project will be documented in this file. The format ### CI - Workflow for `CONTRIBUTORS.md` updates removed. `CONTRIBUTORS.md` file and dependencies removed. ([#4141](https://github.com/docker-mailserver/docker-mailserver/pull/4141)) +- Refactored the workflows for generating documentation previews on PRs to be more secure ([#4267](https://github.com/docker-mailserver/docker-mailserver/pull/4267), [#4264](https://github.com/docker-mailserver/docker-mailserver/pull/4264), [#4262](https://github.com/docker-mailserver/docker-mailserver/pull/4262), [#4247](https://github.com/docker-mailserver/docker-mailserver/pull/4247), [#4244](https://github.com/docker-mailserver/docker-mailserver/pull/4244)) ## [v14.0.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v14.0.0) From c15354058fec085d331e6c01f0e3d1f498ea6f06 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 20 Nov 2024 17:19:58 +1300 Subject: [PATCH 440/592] fix: SASLAuth - Drop services for `mysql`, `shadow`, `pam` auth mechanisms (#4259) --- CHANGELOG.md | 1 + docs/content/config/environment.md | 22 ++++++++++-------- target/scripts/startup/variables-stack.sh | 5 ++-- target/supervisor/conf.d/saslauth.conf | 28 ----------------------- 4 files changed, 17 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ba86566eb1..44f8fd0d44a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file. The format ### Breaking +- **saslauthd** mechanism support via ENV `SASLAUTHD_MECHANISMS` with `pam`, `shadow`, `mysql` values has been removed. Only `ldap` and `rimap` remain supported ([#4259](https://github.com/docker-mailserver/docker-mailserver/pull/4259)) - **getmail6** has been refactored: ([#4156](https://github.com/docker-mailserver/docker-mailserver/pull/4156)) - The [DMS config volume](https://docker-mailserver.github.io/docker-mailserver/v15.0/config/advanced/optional-config/#volumes) now has support for `getmailrc_general.cf` for overriding [common default settings](https://docker-mailserver.github.io/docker-mailserver/v15.0/config/advanced/mail-getmail/#common-options). If you previously mounted this config file directly to `/etc/getmailrc_general` you should switch to our config volume support. - IMAP/POP3 example configs added to our [`config-examples`](https://github.com/docker-mailserver/docker-mailserver/tree/v15.0.0/config-examples/getmail). diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index 5a766f53d83..b7231b1014b 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -910,22 +910,26 @@ Note: This postgrey setting needs `ENABLE_POSTGREY=1` ##### SASLAUTHD_MECHANISMS -- **empty** => pam -- `ldap` => authenticate against ldap server -- `shadow` => authenticate against local user db -- `mysql` => authenticate against mysql db -- `rimap` => authenticate against imap server -- NOTE: can be a list of mechanisms like pam ldap shadow +DMS only implements support for these mechanisms: + +- **`ldap`** => Authenticate against an LDAP server +- `rimap` => Authenticate against an IMAP server ##### SASLAUTHD_MECH_OPTIONS - **empty** => None -- e.g. with SASLAUTHD_MECHANISMS rimap you need to specify the ip-address/servername of the imap server ==> xxx.xxx.xxx.xxx + +!!! info + + With `SASLAUTHD_MECHANISMS=rimap` you need to specify the ip-address / servername of the IMAP server, such as `SASLAUTHD_MECH_OPTIONS=127.0.0.1`. ##### SASLAUTHD_LDAP_SERVER -- **empty** => same as `LDAP_SERVER_HOST` -- Note: You must include the desired URI scheme (`ldap://`, `ldaps://`, `ldapi://`). +- **empty** => Use the same value as `LDAP_SERVER_HOST` + +!!! note + + You must include the desired URI scheme (`ldap://`, `ldaps://`, `ldapi://`). ##### SASLAUTHD_LDAP_START_TLS diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index a3be72b84f8..3fa4d76165b 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -182,8 +182,9 @@ function _environment_variables_ldap() { function _environment_variables_saslauthd() { _log 'debug' 'Setting SASLAUTHD-related environment variables now' - # Only used by the supervisor service command (upstream default: `/etc/default/saslauthd`) - VARS[SASLAUTHD_MECHANISMS]="${SASLAUTHD_MECHANISMS:=pam}" + # This ENV is only used by the supervisor service config `saslauth.conf`: + # NOTE: `pam` is set as the upstream default in `/etc/default/saslauthd` + VARS[SASLAUTHD_MECHANISMS]="${SASLAUTHD_MECHANISMS:=ldap}" } # This function Writes the contents of the `VARS` map (associative array) diff --git a/target/supervisor/conf.d/saslauth.conf b/target/supervisor/conf.d/saslauth.conf index 508ff83c4fa..e42aa198eb2 100644 --- a/target/supervisor/conf.d/saslauth.conf +++ b/target/supervisor/conf.d/saslauth.conf @@ -7,24 +7,6 @@ stderr_logfile=/var/log/supervisor/%(program_name)s.log command=/usr/sbin/saslauthd -d -a ldap -O /etc/saslauthd.conf pidfile=/var/run/saslauthd/saslauthd.pid -[program:saslauthd_mysql] -startsecs=0 -autostart=false -autorestart=true -stdout_logfile=/var/log/supervisor/%(program_name)s.log -stderr_logfile=/var/log/supervisor/%(program_name)s.log -command=/usr/sbin/saslauthd -d -a mysql -O "%(ENV_SASLAUTHD_MECH_OPTIONS)s" -pidfile=/var/run/saslauthd/saslauthd.pid - -[program:saslauthd_pam] -startsecs=0 -autostart=false -autorestart=true -stdout_logfile=/var/log/supervisor/%(program_name)s.log -stderr_logfile=/var/log/supervisor/%(program_name)s.log -command=/usr/sbin/saslauthd -d -a pam -O "%(ENV_SASLAUTHD_MECH_OPTIONS)s" -pidfile=/var/run/saslauthd/saslauthd.pid - [program:saslauthd_rimap] startsecs=0 autostart=false @@ -33,13 +15,3 @@ stdout_logfile=/var/log/supervisor/%(program_name)s.log stderr_logfile=/var/log/supervisor/%(program_name)s.log command=/usr/sbin/saslauthd -d -a rimap -r -O "%(ENV_SASLAUTHD_MECH_OPTIONS)s" pidfile=/var/run/saslauthd/saslauthd.pid - -[program:saslauthd_shadow] -startsecs=0 -autostart=false -autorestart=true -stdout_logfile=/var/log/supervisor/%(program_name)s.log -stderr_logfile=/var/log/supervisor/%(program_name)s.log -command=/usr/sbin/saslauthd -d -a shadow -O "%(ENV_SASLAUTHD_MECH_OPTIONS)s" -pidfile=/var/run/saslauthd/saslauthd.pid - From ab087d28b398ee5d8f374b101a47e899dbf897ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Nov 2024 09:08:10 +1300 Subject: [PATCH 441/592] chore(deps): Bump docker/metadata-action from 5.5.1 to 5.6.1 (#4273) Bumps [docker/metadata-action](https://github.com/docker/metadata-action) from 5.5.1 to 5.6.1. - [Release notes](https://github.com/docker/metadata-action/releases) - [Commits](https://github.com/docker/metadata-action/compare/v5.5.1...v5.6.1) --- updated-dependencies: - dependency-name: docker/metadata-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 4f94cc973f3..596435b2f4d 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -23,7 +23,7 @@ jobs: - name: 'Prepare tags' id: prep - uses: docker/metadata-action@v5.5.1 + uses: docker/metadata-action@v5.6.1 with: images: | ${{ secrets.DOCKER_REPOSITORY }} From edfecbceb163eda98124fdb10dd37bfe6fa0f46d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Nov 2024 09:11:35 +1300 Subject: [PATCH 442/592] chore(deps): Bump anchore/scan-action from 5.2.1 to 5.3.0 (#4274) Bumps [anchore/scan-action](https://github.com/anchore/scan-action) from 5.2.1 to 5.3.0. - [Release notes](https://github.com/anchore/scan-action/releases) - [Changelog](https://github.com/anchore/scan-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/anchore/scan-action/compare/v5.2.1...v5.3.0) --- updated-dependencies: - dependency-name: anchore/scan-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 5b597d873f9..3db8002c84c 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v5.2.1 + uses: anchore/scan-action@v5.3.0 id: scan with: image: mailserver-testing:ci From d07e6d67d6740ea9e4d43781d4df43a2aa4665af Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 29 Nov 2024 16:12:00 +1300 Subject: [PATCH 443/592] chore: Update `jaq` to `2.0.0` (#4277) - Bump to [`jaq` v2 release](https://github.com/01mf02/jaq/releases/tag/v2.0.0), artifact naming convention changed. - Tidied up the changelog a little bit unrelated to this `jaq` update. - Fixed a typo with an `rspamd.sh` comment + minor revision to the comment. --- CHANGELOG.md | 30 +++++++++---------- target/scripts/build/packages.sh | 4 +-- .../startup/setup.d/security/rspamd.sh | 6 ++-- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44f8fd0d44a..b8dc1830df4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ All notable changes to this project will be documented in this file. The format - Added `getmail` as a new service for `supervisor` to manage, replacing cron for periodic polling. - Generated getmail configuration files no longer set the `message_log` option. Instead of individual log files per config, the [default base settings DMS configures](https://github.com/docker-mailserver/docker-mailserver/tree/v15.0.0/target/getmail/getmailrc_general) now enables `message_log_syslog`. This aligns with how other services in DMS log to syslog where it is captured in `mail.log`. - Getmail configurations have changed location from the base of the DMS Config Volume, to the `getmail/` subdirectory. Any existing configurations **must be migrated manually.** - - DMS v14 mistakenly relocated the getmail state directory to the DMS Config Volume as a `getmail/` subdirectory. + - DMS v14 mistakenly relocated the _getmail state directory_ to the _DMS Config Volume_ as a `getmail/` subdirectory. - This has been corrected to `/var/lib/getmail` (_if you have mounted a DMS State Volume to `/var/mail-state`, `/var/lib/getmail` will be symlinked to `/var/mail-state/lib-getmail`_). - To preserve this state when upgrading to DMS v15, **you must manually migrate `getmail/` from the _DMS Config Volume_ to `lib-getmail/` in the _DMS State Volume_.** @@ -33,34 +33,34 @@ All notable changes to this project will be documented in this file. The format ### Updates - **Fail2ban:** - - Bump version to [1.1.0](https://github.com/fail2ban/fail2ban/releases/tag/1.1.0). For more information, check the [changelog](https://github.com/fail2ban/fail2ban/blob/1.1.0/ChangeLog). + - Updated to version [`1.1.0`](https://github.com/fail2ban/fail2ban/releases/tag/1.1.0) ([#4045](https://github.com/docker-mailserver/docker-mailserver/pull/4045)) - **Documentation:** - - Rewritten and organized the pages for Account Management and Authentication ([#4122](https://github.com/docker-mailserver/docker-mailserver/pull/4122)) - - Add caveat for `DMS_VMAIL_UID` not being compatible with `0` / root ([#4143](https://github.com/docker-mailserver/docker-mailserver/pull/4143)) + - Account Management and Authentication pages have been rewritten and better organized ([#4122](https://github.com/docker-mailserver/docker-mailserver/pull/4122)) + - Add a caveat for `DMS_VMAIL_UID` not being compatible with `0` / root ([#4143](https://github.com/docker-mailserver/docker-mailserver/pull/4143)) - **Postfix:** - - Disable Microsoft reactions to outgoing mail ([#4120](https://github.com/docker-mailserver/docker-mailserver/pull/4120)) -- bumped `jaq` version from 1.3.0 to 1.6.0 ([#4190](https://github.com/docker-mailserver/docker-mailserver/pull/4190)) -- updated Rspamd GTube settings and tests ([#4191](https://github.com/docker-mailserver/docker-mailserver/pull/4191)) + - By default opt-out from _Microsoft reactions_ for outbound mail ([#4120](https://github.com/docker-mailserver/docker-mailserver/pull/4120)) +- Updated `jaq` version from `1.3.0` to `2.0.0` ([#4190](https://github.com/docker-mailserver/docker-mailserver/pull/4190)) +- Updated Rspamd GTube settings and tests ([#4191](https://github.com/docker-mailserver/docker-mailserver/pull/4191)) ### Fixes - **Dovecot:** - - Update logwatch `ignore.conf` to exclude Xapian messages about pending documents ([#4060](https://github.com/docker-mailserver/docker-mailserver/pull/4060)) + - The logwatch `ignore.conf` now also excludes Xapian messages about pending documents ([#4060](https://github.com/docker-mailserver/docker-mailserver/pull/4060)) - `dovecot-fts-xapian` plugin was updated to `1.7.13`, fixing a regression with indexing ([#4095](https://github.com/docker-mailserver/docker-mailserver/pull/4095)) - - The Dovecot Quota support "dummy account" workaround no longer treats the alias as a regex when checking the Dovecot UserDB ([#4222](https://github.com/docker-mailserver/docker-mailserver/pull/4222)) + - The "dummy account" workaround for _Dovecot Quota_ feature support no longer treats the alias as a regex when checking the Dovecot UserDB ([#4222](https://github.com/docker-mailserver/docker-mailserver/pull/4222)) - **LDAP:** - - A previous compatibility fix for OAuth2 in v13.3.1 had not applied the actual LDAP config changes. This has been corrected ([#4175](https://github.com/docker-mailserver/docker-mailserver/pull/4175)) + - Correctly apply a compatibility fix for OAuth2 introduced in DMS v13.3.1 which had not been applied to the actual LDAP config changes ([#4175](https://github.com/docker-mailserver/docker-mailserver/pull/4175)) - **Internal:** - - The main `mail.log` which is piped to stdout via `tail` now correctly begins from the first log line of the active container run. Previously some daemon logs and potential warnings/errors were omitted. ([#4146](https://github.com/docker-mailserver/docker-mailserver/pull/4146)) + - The main `mail.log` (_which is piped to stdout via `tail`_) now correctly begins from the first log line of the active container run. Previously some daemon logs and potential warnings/errors were omitted ([#4146](https://github.com/docker-mailserver/docker-mailserver/pull/4146)) - Fixed a regression introduced in v14 where `postfix-main.cf` appended `stderr` output into `/etc/postfix/main.cf`, causing Postfix startup to fail ([#4147](https://github.com/docker-mailserver/docker-mailserver/pull/4147)) - - Unused `shopt -s inherit_errexit` removed from `start-mailserver.sh` ([#4161](https://github.com/docker-mailserver/docker-mailserver/pull/4161)) + - `start-mailserver.sh` removed unused `shopt -s inherit_errexit` ([#4161](https://github.com/docker-mailserver/docker-mailserver/pull/4161)) - **Rspamd:** - - DKIM private key path checking is now performed only on paths that do not contain "$" ([#4201](https://github.com/docker-mailserver/docker-mailserver/pull/4201)) + - DKIM private key path checking is now performed only on paths that do not contain `$` ([#4201](https://github.com/docker-mailserver/docker-mailserver/pull/4201)) ### CI -- Workflow for `CONTRIBUTORS.md` updates removed. `CONTRIBUTORS.md` file and dependencies removed. ([#4141](https://github.com/docker-mailserver/docker-mailserver/pull/4141)) -- Refactored the workflows for generating documentation previews on PRs to be more secure ([#4267](https://github.com/docker-mailserver/docker-mailserver/pull/4267), [#4264](https://github.com/docker-mailserver/docker-mailserver/pull/4264), [#4262](https://github.com/docker-mailserver/docker-mailserver/pull/4262), [#4247](https://github.com/docker-mailserver/docker-mailserver/pull/4247), [#4244](https://github.com/docker-mailserver/docker-mailserver/pull/4244)) +- Removed `CONTRIBUTORS.md`, `.all-contributorsrc`, and workflow ([#4141](https://github.com/docker-mailserver/docker-mailserver/pull/4141)) +- Refactored the workflows to be more secure for generating documentation previews on PRs ([#4267](https://github.com/docker-mailserver/docker-mailserver/pull/4267), [#4264](https://github.com/docker-mailserver/docker-mailserver/pull/4264), [#4262](https://github.com/docker-mailserver/docker-mailserver/pull/4262), [#4247](https://github.com/docker-mailserver/docker-mailserver/pull/4247), [#4244](https://github.com/docker-mailserver/docker-mailserver/pull/4244)) ## [v14.0.0](https://github.com/docker-mailserver/docker-mailserver/releases/tag/v14.0.0) diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 2ac40fbabd5..1e29efa4e24 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -38,8 +38,8 @@ function _pre_installation_steps() { function _install_utils() { _log 'debug' 'Installing utils sourced from Github' _log 'trace' 'Installing jaq' - local JAQ_TAG='v1.6.0' - curl -sSfL "https://github.com/01mf02/jaq/releases/download/${JAQ_TAG}/jaq-${JAQ_TAG}-$(uname -m)-unknown-linux-gnu" -o /usr/bin/jaq + local JAQ_TAG='v2.0.0' + curl -sSfL "https://github.com/01mf02/jaq/releases/download/${JAQ_TAG}/jaq-$(uname -m)-unknown-linux-gnu" -o /usr/bin/jaq chmod +x /usr/bin/jaq _log 'trace' 'Installing swaks' diff --git a/target/scripts/startup/setup.d/security/rspamd.sh b/target/scripts/startup/setup.d/security/rspamd.sh index 18b9703b887..085197c252f 100644 --- a/target/scripts/startup/setup.d/security/rspamd.sh +++ b/target/scripts/startup/setup.d/security/rspamd.sh @@ -330,9 +330,9 @@ function __rspamd__setup_check_authenticated() { fi } -# This function performs a simple check: go through DKIM configuration files, acquire -# all private key file locations and check whether they exist and whether they can be -# accessed by Rspamd. We are not checking paths that conatain the '$' symbol. +# This function performs a simple check on the queried rspamd DKIM configuration: +# - Acquire all private key file locations and check whether they exist and can be accessed by Rspamd. +# - We are not checking paths that contain the '$' symbol. function __rspamd__check_dkim_permissions() { local KEY_FILE while read -r KEY_FILE; do From 10882f97f23a1244e39771e9987d9443810ace69 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 21:51:52 +0100 Subject: [PATCH 444/592] chore(deps): Bump docker/build-push-action from 6.9.0 to 6.10.0 (#4278) --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 4391235770a..70b5d256388 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v6.9.0 + uses: docker/build-push-action@v6.10.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 596435b2f4d..a6d76a11562 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -67,7 +67,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: 'Build and publish images' - uses: docker/build-push-action@v6.9.0 + uses: docker/build-push-action@v6.10.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 9ee070a14f0..3f1053dacb4 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.9.0 + uses: docker/build-push-action@v6.10.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 3db8002c84c..9cad4293a2c 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.9.0 + uses: docker/build-push-action@v6.10.0 with: context: . tags: mailserver-testing:ci From 9f0918c335f007e60cd9bc979a37c7ff8f61d049 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 7 Dec 2024 02:02:35 +1300 Subject: [PATCH 445/592] fix(`packages.sh`): `swaks --help` (#4282) This command requires the `perl-doc` package to work. --- CHANGELOG.md | 1 + target/scripts/build/packages.sh | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8dc1830df4..b0409511889 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ All notable changes to this project will be documented in this file. The format - `start-mailserver.sh` removed unused `shopt -s inherit_errexit` ([#4161](https://github.com/docker-mailserver/docker-mailserver/pull/4161)) - **Rspamd:** - DKIM private key path checking is now performed only on paths that do not contain `$` ([#4201](https://github.com/docker-mailserver/docker-mailserver/pull/4201)) +- The command `swaks --help` is now functional ([#4282](https://github.com/docker-mailserver/docker-mailserver/pull/4282)) ### CI diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 1e29efa4e24..787c84acd89 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -43,6 +43,7 @@ function _install_utils() { chmod +x /usr/bin/jaq _log 'trace' 'Installing swaks' + apt-get "${QUIET}" install --no-install-recommends perl-doc local SWAKS_VERSION='20240103.0' local SWAKS_RELEASE="swaks-${SWAKS_VERSION}" curl -sSfL "https://github.com/jetmore/swaks/releases/download/v${SWAKS_VERSION}/${SWAKS_RELEASE}.tar.gz" | tar -xz @@ -118,7 +119,7 @@ function _install_packages() { bind9-dnsutils iputils-ping less nano ) - apt-get "${QUIET}" --no-install-recommends install \ + apt-get "${QUIET}" install --no-install-recommends \ "${ANTI_VIRUS_SPAM_PACKAGES[@]}" \ "${CODECS_PACKAGES[@]}" \ "${MISCELLANEOUS_PACKAGES[@]}" \ @@ -154,10 +155,10 @@ function _install_dovecot() { fi _log 'debug' 'Installing Dovecot' - apt-get "${QUIET}" --no-install-recommends install "${DOVECOT_PACKAGES[@]}" + apt-get "${QUIET}" install --no-install-recommends "${DOVECOT_PACKAGES[@]}" # dependency for fts_xapian - apt-get "${QUIET}" --no-install-recommends install libxapian30 + apt-get "${QUIET}" install --no-install-recommends libxapian30 } function _install_rspamd() { @@ -185,7 +186,7 @@ function _install_fail2ban() { _log 'debug' 'Installing Fail2ban' # Dependencies (https://github.com/docker-mailserver/docker-mailserver/pull/3403#discussion_r1306581431) - apt-get "${QUIET}" --no-install-recommends install python3-pyinotify python3-dnspython python3-systemd + apt-get "${QUIET}" install --no-install-recommends python3-pyinotify python3-dnspython python3-systemd gpg --keyserver "${FAIL2BAN_GPG_PUBLIC_KEY_SERVER}" --recv-keys "${FAIL2BAN_GPG_PUBLIC_KEY_ID}" 2>&1 From cd225f1250267f868b0436791ad305d0a39616f5 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sat, 7 Dec 2024 02:04:13 +1300 Subject: [PATCH 446/592] ci(`bug_report.yml`): Drop the feedback field (#4283) This input has not provided much value to us since it's introduction, removing as redundant. --- .github/ISSUE_TEMPLATE/bug_report.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index c12439ce5f5..83982b173eb 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -67,10 +67,3 @@ body: - This field expects only plain text (_rendered as a fenced code block_). - You can enable debug output by setting the environment variable `LOG_LEVEL` to `debug` or `trace`. render: Text - - type: input - id: form-improvements - attributes: - label: Improvements to this form? - description: If you have criticism or general feedback about this issue form, feel free to tell us so we can enhance the experience for everyone. - validations: - required: false From 96bffd79791e1cc85fe81a5e797384236fcca2e0 Mon Sep 17 00:00:00 2001 From: Casper Date: Fri, 6 Dec 2024 21:22:37 +0100 Subject: [PATCH 447/592] chore(compile.sh): Consistent apt-get install command --- target/scripts/build/compile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/scripts/build/compile.sh b/target/scripts/build/compile.sh index 299ba7c8de2..c34a88331c3 100644 --- a/target/scripts/build/compile.sh +++ b/target/scripts/build/compile.sh @@ -13,7 +13,7 @@ _log_level_is 'trace' && QUIET='-y' || QUIET='-qq' function _compile_dovecot_fts_xapian() { apt-get "${QUIET}" update - apt-get "${QUIET}" --no-install-recommends install \ + apt-get "${QUIET}" install --no-install-recommends \ automake libtool pkg-config libicu-dev libsqlite3-dev libxapian-dev make build-essential dh-make devscripts dovecot-dev local XAPIAN_VERSION='1.7.13' From a03ff8ff7c4ecef502e56aa4d79b67c32a999772 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:18:51 +0100 Subject: [PATCH 448/592] chore(deps): Bump docker/setup-buildx-action from 3.7.1 to 3.8.0 (#4293) --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 70b5d256388..0f1b308ec66 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -79,7 +79,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.7.1 + uses: docker/setup-buildx-action@v3.8.0 # NOTE: AMD64 can build within 2 minutes - name: 'Build images' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index a6d76a11562..119809d08c3 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -40,7 +40,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.7.1 + uses: docker/setup-buildx-action@v3.8.0 # Try get the cached build layers from a prior `generic_build.yml` job. # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 3f1053dacb4..31ec616509c 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -38,7 +38,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.7.1 + uses: docker/setup-buildx-action@v3.8.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 9cad4293a2c..1405df73ae9 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -37,7 +37,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.7.1 + uses: docker/setup-buildx-action@v3.8.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. From 02415b03dce763a717747bb737e4698ddcd7aefd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 08:26:14 +0100 Subject: [PATCH 449/592] chore(deps): Bump anchore/scan-action from 5.3.0 to 6.0.0 (#4292) --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 1405df73ae9..86246f2c9f7 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v5.3.0 + uses: anchore/scan-action@v6.0.0 id: scan with: image: mailserver-testing:ci From 259f2031fca6ee8b2f55386cec8c55d30176cfd2 Mon Sep 17 00:00:00 2001 From: Pooyan Khanjankhani Date: Mon, 23 Dec 2024 05:08:46 +0330 Subject: [PATCH 450/592] docs: Fix typo on usage page (#4294) Co-authored-by: Casper --- docs/content/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/usage.md b/docs/content/usage.md index ca4e8dfef81..1c31aa176ca 100644 --- a/docs/content/usage.md +++ b/docs/content/usage.md @@ -48,7 +48,7 @@ We will later dig into DKIM, DMARC & SPF, but for now, these are the records tha - The **MX record** tells everyone which (DNS) name is responsible for e-mails on your domain. Because you want to keep the option of running another service on the domain name itself, you run your mail server on `mail.example.com`. This does not imply your e-mails will look like `test@mail.example.com`, the DNS name of your mail server is decoupled of the domain it serves e-mails for. - In theory, you mail server could even serve e-mails for `test@some-other-domain.com`, if the MX record for `some-other-domain.com` points to `mail.example.com`. + Your mail server could also handle emails for `test@some-other-domain.com`, if the MX record for `some-other-domain.com` points to `mail.example.com`. - The **A record** tells everyone which IP address the DNS name `mail.example.com` resolves to. - The **PTR record** is the counterpart of the A record, telling everyone what name the IP address `11.22.33.44` resolves to. From 24fb65ce7b638b342c1c9f427912756a0e53ed3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20Wo=C5=BAniak?= <92504295+zespere@users.noreply.github.com> Date: Sat, 28 Dec 2024 06:00:27 +0100 Subject: [PATCH 451/592] docs: Environment - Update Dovecot docs URL (#4296) Co-authored-by: Casper --- docs/content/config/environment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/config/environment.md b/docs/content/config/environment.md index b7231b1014b..a16f6bcb9bc 100644 --- a/docs/content/config/environment.md +++ b/docs/content/config/environment.md @@ -295,7 +295,7 @@ Customize the update check interval. Number + Suffix. Suffix must be 's' for sec - sdbox => (experimental) uses Dovecot high-performance mailbox format, one file contains one message - mdbox ==> (experimental) uses Dovecot high-performance mailbox format, multiple messages per file and multiple files per box -This option has been added in November 2019. Using other format than Maildir is considered as experimental in docker-mailserver and should only be used for testing purpose. For more details, please refer to [Dovecot Documentation](https://wiki2.dovecot.org/MailboxFormat). +This option has been added in November 2019. Using other format than Maildir is considered as experimental in docker-mailserver and should only be used for testing purpose. For more details, please refer to [Dovecot Documentation](https://doc.dovecot.org/admin_manual/mailbox_formats/#mailbox-formats). ##### POSTFIX_REJECT_UNKNOWN_CLIENT_HOSTNAME From e6d519b6f82f1d3a1fada4b24edaed6f64f5d3f0 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Mon, 6 Jan 2025 09:02:02 +1300 Subject: [PATCH 452/592] docs: TLS (Caddy) - Revise advice on `tls internal` (#4305) --- docs/content/config/security/ssl.md | 36 ++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/docs/content/config/security/ssl.md b/docs/content/config/security/ssl.md index 96f4a7175f4..9b1fd8f74dd 100644 --- a/docs/content/config/security/ssl.md +++ b/docs/content/config/security/ssl.md @@ -485,6 +485,8 @@ DSM-generated letsencrypt certificates get auto-renewed every three months. !!! example + While DMS does not need a webserver to work, this workaround will provision a TLS certificate for DMS to use by adding a dummy site block to trigger cert provisioning. + ```yaml title="compose.yaml" services: # Basic Caddy service to provision certs: @@ -510,9 +512,12 @@ DSM-generated letsencrypt certificates get auto-renewed every three months. - ${CADDY_DATA_DIR}/certificates/acme-v02.api.letsencrypt.org-directory/mail.example.com/mail.example.com.key:/etc/letsencrypt/live/mail.example.com/privkey.pem ``` + An explicit entry in your `Caddyfile` config will have Caddy provision and renew a certificate for your DMS FQDN: + ```caddyfile title="Caddyfile" mail.example.com { - tls internal { + # Optionally provision RSA 2048-bit certificate instead of ECDSA P-256: + tls { key_type rsa2048 } @@ -522,10 +527,12 @@ DSM-generated letsencrypt certificates get auto-renewed every three months. } ``` - While DMS does not need a webserver to work, this workaround will provision a TLS certificate for DMS to use. + !!! info - - [`tls internal`][caddy-docs::tls-internal] will create a local self-signed cert for testing. This targets only the site-address, unlike the global `local_certs` option. - - [`key_type`][caddy-docs::key-type] can be used in the `tls` block if you need to enforce RSA as the key type for certificates provisioned. The default is currently ECDSA (P-256). + An explicit `tls` directive affects only the site-address block it's used in: + + - Use [`tls internal { ... }`][caddy-docs::tls-internal] if wanting to create a local self-signed cert, which may be useful for testing. This allows opt-in to use self-signed certs unlike the global `local_certs` option. + - [`key_type`][caddy-docs::key-type] can be used in the `tls` block if you need to enforce RSA as the key type for certificates provisioned. The default is currently ECDSA (P-256). This may improve compatibility with legacy clients. ??? example "With `caddy-docker-proxy`" @@ -558,9 +565,9 @@ DSM-generated letsencrypt certificates get auto-renewed every three months. labels: # Set your DMS FQDN here to add the site-address into the generated Caddyfile: caddy_0: mail.example.com - # Add a dummy directive is required: + # Adding a dummy directive is required: caddy_0.respond: "Hello DMS" - # Uncomment to make a proxy for Rspamd + # Uncomment to make a proxy for Rspamd: # caddy_1: rspamd.example.com # caddy_1.reverse_proxy: "{{upstreams 11334}}" ``` @@ -571,6 +578,23 @@ DSM-generated letsencrypt certificates get auto-renewed every three months. This can make the volume mounting for DMS to find the certificates non-deterministic, but you can [restrict provisioning to single service via the `acme_ca` setting][caddy::restrict-acme-provisioner]. + --- + + **NOTE:** Bind mounting a file directly instead of a directory will mount by inode. If the file is updated at renewal and this modifies the inode on the host system, then the container will still point to the old certificate. + + If this happens, consider using our manual TLS type instead: + + ```yaml title="compose.yaml" + services: + mailserver: + environment: + SSL_TYPE: manual + SSL_CERT_PATH: /srv/tls/mail.example.com/mail.example.com.crt + SSL_KEY_PATH: /srv/tls/mail.example.com/mail.example.com.key + volumes: + - ${CADDY_DATA_DIR}/certificates/acme-v02.api.letsencrypt.org-directory/mail.example.com/:/srv/tls/mail.example.com/:ro + ``` + ### Traefik [Traefik][traefik::github] is an open-source application proxy using the [ACME protocol][ietf::rfc::acme]. Traefik can request certificates for domains and subdomains, and it will take care of renewals, challenge negotiations, etc. From a302bd79e3c76c2b501628cfbcee77fee03b9511 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jan 2025 12:36:44 +1300 Subject: [PATCH 453/592] chore(deps): Bump docker/setup-qemu-action from 3.2.0 to 3.3.0 (#4309) Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 3.2.0 to 3.3.0. - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/v3.2.0...v3.3.0) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 0f1b308ec66..c3dce52c1d3 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -74,7 +74,7 @@ jobs: cache-buildx- - name: 'Set up QEMU' - uses: docker/setup-qemu-action@v3.2.0 + uses: docker/setup-qemu-action@v3.3.0 with: platforms: arm64 diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 119809d08c3..2520aade286 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -35,7 +35,7 @@ jobs: type=semver,pattern={{major}}.{{minor}}.{{patch}} - name: 'Set up QEMU' - uses: docker/setup-qemu-action@v3.2.0 + uses: docker/setup-qemu-action@v3.3.0 with: platforms: arm64 From 8df1fba96e1efb1c28317f6e162323bf781918be Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jan 2025 12:38:15 +1300 Subject: [PATCH 454/592] chore(deps): Bump docker/build-push-action from 6.10.0 to 6.11.0 (#4310) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.10.0 to 6.11.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v6.10.0...v6.11.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index c3dce52c1d3..0561587f8f9 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -83,7 +83,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v6.10.0 + uses: docker/build-push-action@v6.11.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 2520aade286..835e2a8128c 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -67,7 +67,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: 'Build and publish images' - uses: docker/build-push-action@v6.10.0 + uses: docker/build-push-action@v6.11.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 31ec616509c..19324f0a8ac 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.10.0 + uses: docker/build-push-action@v6.11.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 86246f2c9f7..bf700139c80 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.10.0 + uses: docker/build-push-action@v6.11.0 with: context: . tags: mailserver-testing:ci From 3ebca5dabaea1469f482d22912e08ec33c16cfed Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sat, 18 Jan 2025 21:26:07 +0100 Subject: [PATCH 455/592] Revert "fix: fix incorrect link in README.md (#4184)" (#4322) This reverts commit 84180f879e5eaaa15930509d4154248c3248d765. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6e1d2735c19..e4dbf44a984 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ A production-ready fullstack but simple containerized mail server (SMTP, IMAP, L ## :package: Included Services - [Postfix](http://www.postfix.org) with SMTP or LDAP authentication and support for [extension delimiters](https://docker-mailserver.github.io/docker-mailserver/latest/config/account-management/overview/#aliases) -- [Dovecot](https://www.dovecot.org) with SASL, IMAP, POP3, LDAP, [basic Sieve support](https://docker-mailserver.github.io/docker-mailserver/latest/config/advanced/mail-sieve) and [quotas](https://docker-mailserver.github.io/docker-mailserver/latest/config/user-management/#quotas) +- [Dovecot](https://www.dovecot.org) with SASL, IMAP, POP3, LDAP, [basic Sieve support](https://docker-mailserver.github.io/docker-mailserver/latest/config/advanced/mail-sieve) and [quotas](https://docker-mailserver.github.io/docker-mailserver/latest/config/account-management/overview/#quotas) - [Rspamd](https://rspamd.com/) - [Amavis](https://www.amavis.org/) - [SpamAssassin](http://spamassassin.apache.org/) supporting custom rules From f0daa1c8abff606bd3fa6f5212b7e77c1e7b38a2 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 19 Jan 2025 01:27:07 +0100 Subject: [PATCH 456/592] chore: remove `VERSION` file (#4321) --- .dockerignore | 1 - .github/workflows/generic_build.yml | 7 ++----- CHANGELOG.md | 1 + VERSION | 1 - 4 files changed, 3 insertions(+), 7 deletions(-) delete mode 100644 VERSION diff --git a/.dockerignore b/.dockerignore index b6eef669ae8..50365994307 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,2 @@ * !target -!VERSION diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 0561587f8f9..b81c94de76d 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -32,7 +32,7 @@ jobs: with: submodules: recursive - # Can potentially be replaced by: `${{ hashFiles('target/**', 'Dockerfile', 'VERSION') }}` + # Can potentially be replaced by: `${{ hashFiles('target/**', 'Dockerfile') }}` # Must not be affected by file metadata changes and have a consistent sort order: # https://docs.github.com/en/actions/learn-github-actions/expressions#hashfiles # Keying by the relevant build context is more re-usable than a commit SHA. @@ -40,10 +40,7 @@ jobs: id: derive-image-cache-key shell: bash run: | - ADDITIONAL_FILES=( - 'Dockerfile' - 'VERSION' - ) + ADDITIONAL_FILES=( 'Dockerfile' ) # Recursively collect file paths from `target/` and pipe a list of # checksums to be sorted (by hash value) and finally generate a checksum diff --git a/CHANGELOG.md b/CHANGELOG.md index b0409511889..8ca4bdcb9bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ All notable changes to this project will be documented in this file. The format - DMS v14 mistakenly relocated the _getmail state directory_ to the _DMS Config Volume_ as a `getmail/` subdirectory. - This has been corrected to `/var/lib/getmail` (_if you have mounted a DMS State Volume to `/var/mail-state`, `/var/lib/getmail` will be symlinked to `/var/mail-state/lib-getmail`_). - To preserve this state when upgrading to DMS v15, **you must manually migrate `getmail/` from the _DMS Config Volume_ to `lib-getmail/` in the _DMS State Volume_.** +- **removed `VERSION`** file that was used for checking version updates ([#3677](https://github.com/docker-mailserver/docker-mailserver/issues/3677),[#4321](https://github.com/docker-mailserver/docker-mailserver/pull/4321)) ### Security diff --git a/VERSION b/VERSION deleted file mode 100644 index 4b964e96540..00000000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -14.0.0 From 2d56210c522410a29d15d0c890667fad2fee1c75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Jan 2025 21:42:25 +0100 Subject: [PATCH 457/592] chore(deps): Bump docker/build-push-action from 6.11.0 to 6.12.0 (#4324) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.11.0 to 6.12.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v6.11.0...v6.12.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index b81c94de76d..5c81963721f 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -80,7 +80,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v6.11.0 + uses: docker/build-push-action@v6.12.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 835e2a8128c..edd642fba9b 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -67,7 +67,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: 'Build and publish images' - uses: docker/build-push-action@v6.11.0 + uses: docker/build-push-action@v6.12.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 19324f0a8ac..190f1f89d76 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.11.0 + uses: docker/build-push-action@v6.12.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index bf700139c80..a74c98277d0 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.11.0 + uses: docker/build-push-action@v6.12.0 with: context: . tags: mailserver-testing:ci From 7cb2fc788f67a90319e2d7ecdd5422bc9cd33afc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Jan 2025 10:56:37 +0100 Subject: [PATCH 458/592] chore(deps): Bump anchore/scan-action from 6.0.0 to 6.1.0 (#4332) Bumps [anchore/scan-action](https://github.com/anchore/scan-action) from 6.0.0 to 6.1.0. - [Release notes](https://github.com/anchore/scan-action/releases) - [Changelog](https://github.com/anchore/scan-action/blob/main/RELEASE.md) - [Commits](https://github.com/anchore/scan-action/compare/v6.0.0...v6.1.0) --- updated-dependencies: - dependency-name: anchore/scan-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/generic_vulnerability-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index a74c98277d0..2e73e0f2852 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -55,7 +55,7 @@ jobs: provenance: false - name: 'Run the Anchore Grype scan action' - uses: anchore/scan-action@v6.0.0 + uses: anchore/scan-action@v6.1.0 id: scan with: image: mailserver-testing:ci From 4d8a56072a88cbacea373be88aa1d805e1eacc59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Jan 2025 11:02:25 +0100 Subject: [PATCH 459/592] chore(deps): Bump docker/build-push-action from 6.12.0 to 6.13.0 (#4331) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.12.0 to 6.13.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v6.12.0...v6.13.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 5c81963721f..6000f5ffd53 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -80,7 +80,7 @@ jobs: # NOTE: AMD64 can build within 2 minutes - name: 'Build images' - uses: docker/build-push-action@v6.12.0 + uses: docker/build-push-action@v6.13.0 with: context: . # Build at least the AMD64 image (which runs against the test suite). diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index edd642fba9b..42cde7ea4d5 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -67,7 +67,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: 'Build and publish images' - uses: docker/build-push-action@v6.12.0 + uses: docker/build-push-action@v6.13.0 with: context: . build-args: | diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index 190f1f89d76..f28dbc9b7ce 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -43,7 +43,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.12.0 + uses: docker/build-push-action@v6.13.0 with: context: . tags: mailserver-testing:ci diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 2e73e0f2852..3b20b2edb3f 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -42,7 +42,7 @@ jobs: # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. - name: 'Build AMD64 image from cache' - uses: docker/build-push-action@v6.12.0 + uses: docker/build-push-action@v6.13.0 with: context: . tags: mailserver-testing:ci From 3faa40bfb5442e4fd6638aa5878fe807feb7e94f Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sun, 2 Feb 2025 22:35:50 +0100 Subject: [PATCH 460/592] docs: add ARC example to Rspamd documentation (#4328) Signed-off-by: georglauterbach <44545919+georglauterbach@users.noreply.github.com> --- docs/content/config/security/rspamd.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index 8705262cfbe..c49baea6648 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -247,6 +247,24 @@ Rspamd is running, but you want or need to adjust it? First, create a file named There is a dedicated [section for setting up DKIM with Rspamd in our documentation][docs::dkim-with-rspamd]. +### ARC (Authenticated Received Chain) + +ARC is not set up by default, but you can easily enable it by adding a file called `arc.conf` to `docker-data/dms/config/rspamd/override.d/`. ARC can use DKIM keys that you should have already created. The configuration file could then contain the following: + +```conf +sign_local = true; +sign_authenticated = true; + +domain { + { + # Change the path here to your actual private key + path = "/tmp/docker-mailserver/rspamd/dkim/rsa-2048-mail-.private.txt"; + # Changhe the selected if you chose a non-default one + selector = "mail"; + } +} +``` + ### _Abusix_ Integration This subsection provides information about the integration of [Abusix][abusix-web], "a set of blocklists that work as an additional email security layer for your existing mail environment". The setup is straight-forward and well documented: From 0e61f170fd552077e797483310a7d61500bbe020 Mon Sep 17 00:00:00 2001 From: Zlatibor Veljkovic Date: Tue, 4 Feb 2025 00:45:43 +0100 Subject: [PATCH 461/592] docs: `bind-smtp-network-interface.md` - Add bridge network config advice (#4330) Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> --- .../use-cases/bind-smtp-network-interface.md | 50 ++++++++++++++++--- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/docs/content/examples/use-cases/bind-smtp-network-interface.md b/docs/content/examples/use-cases/bind-smtp-network-interface.md index b12e21dea81..0da33b19952 100644 --- a/docs/content/examples/use-cases/bind-smtp-network-interface.md +++ b/docs/content/examples/use-cases/bind-smtp-network-interface.md @@ -21,10 +21,6 @@ This can be configured by [overriding the default Postfix configurations][docs:: In `postfix-main.cf` you'll have to set the [`smtp_bind_address`][postfix-docs::smtp-bind-address-ipv4] and [`smtp_bind_address6`][postfix-docs::smtp-bind-address-ipv6] to the respective IP-address on the server you want to use. -[docs::overrides-postfix]: ../../config/advanced/override-defaults/postfix.md -[postfix-docs::smtp-bind-address-ipv4]: https://www.postfix.org/postconf.5.html#smtp_bind_address -[postfix-docs::smtp-bind-address-ipv6]: https://www.postfix.org/postconf.5.html#smtp_bind_address6 - !!! example === "Contributed solution" @@ -55,14 +51,56 @@ to the respective IP-address on the server you want to use. ``` If that avoids the concern with `smtp-amavis`, you may still need to additionally override for the [`relay` transport][gh-src::postfix-master-cf::relay-transport] as well if you have configured DMS to relay mail. + + === "Bridged Networks" + + When your DMS container is using a bridge network, you'll instead need to restrict which IP address inbound and outbound traffic is routed through via the bridged interface. + + For **inbound** traffic, you may configure this at whatever scope is most appropriate for you: + + - **Daemon:** Change the [default bind address][inbound-ip::docker-docs::daemon] configured in `/etc/docker/daemon.json` (default `0.0.0.0`) + - **Network:** Assign the [`host_binding_ipv4` bridge driver option][inbound-ip::docker-docs::network] as shown in the below `compose.yaml` snippet. + - **Container:** Provide an explicit host IP address when [publishing a port][inbound-ip::docker-docs::container]. + + For **outbound** traffic, the bridge network will use the default route. You can change this by either: + + - [Manually routing networks][outbound-ip::route-manually] on the host. + - Use the [`host_ipv4` driver option][outbind-ip::host-ipv4] for Docker networks to force the SNAT (source IP) that the bridged network will route outbound traffic through. + - This IP address must belong to a network interface to be routed through it. + - IPv6 support via `host_ipv6` [requires at least Docker v25][outbind-ip::host-ipv6]. + + --- + + Here is a `compose.yaml` snippet that applies the inbound + outbound settings to the default bridge network Docker Compose creates (_if it already exists, you will need to ensure it's re-created to apply the updated settings_): + + ```yaml title="compose.yaml" + networks: + default: + driver_opts: + # Inbound IP (sets the host IP that published ports receive traffic from): + com.docker.network.bridge.host_binding_ipv4: 198.51.100.42 + # Outbound IP (sets the host IP that external hosts will receive connections from): + com.docker.network.host_ipv4: 198.51.100.42 + ``` !!! note "IP addresses for documentation" - IP addresses shown in above examples are placeholders, they are IP addresses reserved for documentation by IANA (_[RFC-5737 (IPv4)][rfc-5737] and [RFC-3849 (IPv6)][rfc-3849]_). Replace them with the IP addresses you want DMS to send mail through. - + IP addresses shown in above examples (`198.51.100.42` + `2001:DB8::42`) are placeholders, they are IP addresses reserved for documentation by IANA (_[RFC-5737 (IPv4)][rfc-5737] and [RFC-3849 (IPv6)][rfc-3849]_). Replace them with the IP addresses you want DMS to send mail through. + +[docs::overrides-postfix]: ../../config/advanced/override-defaults/postfix.md +[postfix-docs::smtp-bind-address-ipv4]: https://www.postfix.org/postconf.5.html#smtp_bind_address +[postfix-docs::smtp-bind-address-ipv6]: https://www.postfix.org/postconf.5.html#smtp_bind_address6 + [rfc-5737]: https://datatracker.ietf.org/doc/html/rfc5737 [rfc-3849]: https://datatracker.ietf.org/doc/html/rfc3849 [gh-pr::3465::comment-restrictions]: https://github.com/docker-mailserver/docker-mailserver/pull/3465#discussion_r1458114528 [gh-pr::3465::alternative-solution]: https://github.com/docker-mailserver/docker-mailserver/pull/3465#issuecomment-1678107233 [gh-src::postfix-master-cf::relay-transport]: https://github.com/docker-mailserver/docker-mailserver/blob/9cdbef2b369fb4fb0f1b4e534da8703daf92abc9/target/postfix/master.cf#L65 + +[inbound-ip::docker-docs::daemon]: https://docs.docker.com/reference/cli/dockerd/#default-network-options +[inbound-ip::docker-docs::network]: https://docs.docker.com/engine/network/drivers/bridge/#default-host-binding-address +[inbound-ip::docker-docs::container]: https://docs.docker.com/reference/compose-file/services/#ports +[outbound-ip::route-manually]: https://github.com/moby/moby/issues/30053#issuecomment-1077041045 +[outbind-ip::host-ipv4]: https://github.com/moby/libnetwork/pull/2454 +[outbind-ip::host-ipv6]: https://github.com/moby/moby/issues/46469 From e116920f4df64cd55c9cd91e8783180b3fa2aae0 Mon Sep 17 00:00:00 2001 From: Casper Date: Wed, 5 Feb 2025 23:10:57 +0100 Subject: [PATCH 462/592] Add missing "setup debug getmail" command and documentation (#4346) --- CHANGELOG.md | 1 + docs/content/config/advanced/mail-getmail.md | 8 ++++++++ target/bin/setup | 2 ++ 3 files changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ca4bdcb9bb..781ef3e48bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ All notable changes to this project will be documented in this file. The format - DMS v14 mistakenly relocated the _getmail state directory_ to the _DMS Config Volume_ as a `getmail/` subdirectory. - This has been corrected to `/var/lib/getmail` (_if you have mounted a DMS State Volume to `/var/mail-state`, `/var/lib/getmail` will be symlinked to `/var/mail-state/lib-getmail`_). - To preserve this state when upgrading to DMS v15, **you must manually migrate `getmail/` from the _DMS Config Volume_ to `lib-getmail/` in the _DMS State Volume_.** + - Added missing `debug getmail` subcommand to `setup` ([#4346](https://github.com/docker-mailserver/docker-mailserver/pull/4346)) - **removed `VERSION`** file that was used for checking version updates ([#3677](https://github.com/docker-mailserver/docker-mailserver/issues/3677),[#4321](https://github.com/docker-mailserver/docker-mailserver/pull/4321)) ### Security diff --git a/docs/content/config/advanced/mail-getmail.md b/docs/content/config/advanced/mail-getmail.md index d423b4f09a3..f4ce7e63058 100644 --- a/docs/content/config/advanced/mail-getmail.md +++ b/docs/content/config/advanced/mail-getmail.md @@ -108,3 +108,11 @@ It is possible to utilize the `getmail-gmail-xoauth-tokens` helper to provide au [getmail-docs]: https://getmail6.org/configuration.html [getmail-docs-xoauth-12]: https://github.com/getmail6/getmail6/blob/1f95606156231f1e074ba62a9baa64f892a92ef8/docs/getmailrc-examples#L286 [getmail-docs-xoauth-13]: https://github.com/getmail6/getmail6/blob/1f95606156231f1e074ba62a9baa64f892a92ef8/docs/getmailrc-examples#L351 + +## Debugging + +To debug your `getmail` configurations, run this `setup debug` command: + +```sh +docker exec -it dms-container-name setup debug getmail +``` diff --git a/target/bin/setup b/target/bin/setup index ac0d63282c9..c05f59a768c 100755 --- a/target/bin/setup +++ b/target/bin/setup @@ -64,6 +64,7 @@ ${RED}[${ORANGE}SUB${RED}]${ORANGE}COMMANDS${RESET} ${LBLUE}COMMAND${RESET} debug ${RED}:=${RESET} setup debug ${CYAN}fetchmail${RESET} + setup debug ${CYAN}getmail${RESET} setup debug ${CYAN}login${RESET} setup debug ${CYAN}show-mail-logs${RESET} @@ -150,6 +151,7 @@ function _main() { ( debug ) case ${2:-} in ( fetchmail ) debug-fetchmail ;; + ( getmail ) debug-getmail ;; ( show-mail-logs ) cat /var/log/mail/mail.log ;; ( login ) shift 2 From 85793988d6d81ec21e19ccab96070fcd1f3d6208 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 7 Feb 2025 10:10:37 +1300 Subject: [PATCH 463/592] chore: `demo-setups/relay-compose.yaml` should use network alias (#4347) --- demo-setups/relay-compose.yaml | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/demo-setups/relay-compose.yaml b/demo-setups/relay-compose.yaml index f4df2de058c..cb4f0846837 100644 --- a/demo-setups/relay-compose.yaml +++ b/demo-setups/relay-compose.yaml @@ -41,17 +41,13 @@ services: hostname: smtp.relay-service.test environment: # WORKAROUND: Bypass security checks from the mail-client (dms-sender container) - # (avoids needing valid DNS for this example) + # (avoids needing expected DNS records to run this example) - PERMIT_DOCKER=connected-networks # TLS is required when relaying to dms-relay via ports 587 / 465 # (dms-relay will then relay the mail to dms-destination over port 25) - SSL_TYPE=manual - SSL_KEY_PATH=/tmp/tls/key.pem - SSL_CERT_PATH=/tmp/tls/cert.pem - # WORKAROUND: `links` is required due to lack of properly configured DNS. - # (resolves destination.test to the IP of the dms-destination container) - links: - - "dms-destination:destination.test" configs: - source: dms-accounts-relay target: /tmp/docker-mailserver/postfix-accounts.cf @@ -64,8 +60,14 @@ services: dms-destination: image: mailserver/docker-mailserver:latest # :14.0 hostname: mail.destination.test - # Same workaround for purposes of the example, with the target recipient provisioned to accept mail + # WORKAROUND: dms-relay must be able to resolve DNS for `@destination.test` to the IP of this container: + # Normally a MX record would direct mail to the MTA (eg: `mail.destination.test`) + networks: + default: + aliases: + - destination.test environment: + # WORKAROUND: Same workaround as needed for dms-relay - PERMIT_DOCKER=connected-networks configs: - source: dms-accounts-destination @@ -75,6 +77,13 @@ services: # NOTE: This feature requires Docker Compose v2.23.1 (Nov 2023) or newer: # https://github.com/compose-spec/compose-spec/pull/446 configs: + # `postfix-main.cf`, a single line change to make all outbound SMTP connections over implicit TLS instead of the default explicit TLS (StartTLS). + # NOTE: If you need to only selectively relay mail, you would need to instead adjust this on the relay service in `/etc/postfix/master.cf`, + # However DMS presently modifies this when using the DMS Relay Host feature support, which may override `postfix-master.cf` or `user-patches.sh` due to `check-for-changes.sh`. + dms-main: + content: | + smtp_tls_wrappermode=yes + # DMS expects an account to be configured to run, this example provides accounts already created. # Login credentials: # user: "john.doe@example.test" password: "secret" @@ -94,13 +103,6 @@ configs: content: | jane.doe@destination.test|{SHA512-CRYPT}$$6$$o65y1ZXC4ooOPLwZ$$7TF1nYowEtNJpH6BwJBgdj2pPAxaCvhIKQA6ww5zdHm/AA7aemY9eoHC91DOgYNaKj1HLxSeWNDdvrp6mbtUY. - # This is `postfix-main.cf`, single line change to make all outbound SMTP connections over port 465 instead of 25 (default) - # If you selectively relay mail, you would need to adjust this on the relay service in `/etc/postfix/master.cf`, - # However DMS presently modifies this when using the DMS Relay Host feature support, which may override `postfix-master.cf` or `user-patches.sh` due to `check-for-changes.sh`. - dms-main: - content: | - smtp_tls_wrappermode=yes - # TLS files: # - Use an ECDSA cert that's been signed by a self-signed CA for TLS cert verification. # - This cert is only valid for mail.example.test, mail.destination.test, smtp.relay-service.test From 59a379aed781d8e66b9388491aed245a9f5d9af1 Mon Sep 17 00:00:00 2001 From: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> Date: Sat, 8 Feb 2025 22:23:06 +0100 Subject: [PATCH 464/592] scripts: restructure container restart behavior (#4323) Signed-off-by: georglauterbach <44545919+georglauterbach@users.noreply.github.com> Signed-off-by: dependabot[bot] Co-authored-by: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- CHANGELOG.md | 9 +- target/scripts/start-mailserver.sh | 54 ++--- target/scripts/startup/check-stack.sh | 18 -- target/scripts/startup/setup-stack.sh | 4 +- target/scripts/startup/setup.d/mail_state.sh | 242 ++++++++++--------- target/scripts/startup/variables-stack.sh | 56 ++++- 6 files changed, 203 insertions(+), 180 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 781ef3e48bb..379992c3859 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,6 @@ All notable changes to this project will be documented in this file. The format - DMS v14 mistakenly relocated the _getmail state directory_ to the _DMS Config Volume_ as a `getmail/` subdirectory. - This has been corrected to `/var/lib/getmail` (_if you have mounted a DMS State Volume to `/var/mail-state`, `/var/lib/getmail` will be symlinked to `/var/mail-state/lib-getmail`_). - To preserve this state when upgrading to DMS v15, **you must manually migrate `getmail/` from the _DMS Config Volume_ to `lib-getmail/` in the _DMS State Volume_.** - - Added missing `debug getmail` subcommand to `setup` ([#4346](https://github.com/docker-mailserver/docker-mailserver/pull/4346)) -- **removed `VERSION`** file that was used for checking version updates ([#3677](https://github.com/docker-mailserver/docker-mailserver/issues/3677),[#4321](https://github.com/docker-mailserver/docker-mailserver/pull/4321)) ### Security @@ -31,9 +29,11 @@ All notable changes to this project will be documented in this file. The format - **Internal:** - Add password confirmation to several `setup` CLI subcommands ([#4072](https://github.com/docker-mailserver/docker-mailserver/pull/4072)) + - Added a `debug getmail` subcommand to `setup` ([#4346](https://github.com/docker-mailserver/docker-mailserver/pull/4346)) ### Updates +- **Removed `VERSION` file** from the repo that releases of DMS prior to v13 (Nov 2023) would check to detect new releases ([#3677](https://github.com/docker-mailserver/docker-mailserver/issues/3677), [#4321](https://github.com/docker-mailserver/docker-mailserver/pull/4321)) - **Fail2ban:** - Updated to version [`1.1.0`](https://github.com/fail2ban/fail2ban/releases/tag/1.1.0) ([#4045](https://github.com/docker-mailserver/docker-mailserver/pull/4045)) - **Documentation:** @@ -54,11 +54,12 @@ All notable changes to this project will be documented in this file. The format - Correctly apply a compatibility fix for OAuth2 introduced in DMS v13.3.1 which had not been applied to the actual LDAP config changes ([#4175](https://github.com/docker-mailserver/docker-mailserver/pull/4175)) - **Internal:** - The main `mail.log` (_which is piped to stdout via `tail`_) now correctly begins from the first log line of the active container run. Previously some daemon logs and potential warnings/errors were omitted ([#4146](https://github.com/docker-mailserver/docker-mailserver/pull/4146)) - - Fixed a regression introduced in v14 where `postfix-main.cf` appended `stderr` output into `/etc/postfix/main.cf`, causing Postfix startup to fail ([#4147](https://github.com/docker-mailserver/docker-mailserver/pull/4147)) - `start-mailserver.sh` removed unused `shopt -s inherit_errexit` ([#4161](https://github.com/docker-mailserver/docker-mailserver/pull/4161)) + - Fixed a regression introduced in v14 where `postfix-main.cf` appended `stderr` output into `/etc/postfix/main.cf`, causing Postfix startup to fail ([#4147](https://github.com/docker-mailserver/docker-mailserver/pull/4147)) + - Fixed a regression introduced in v14 to better support running `start-mailserver.sh` with container restarts, which now only skip calling `_setup()` ([#4323](https://github.com/docker-mailserver/docker-mailserver/pull/4323#issuecomment-2629559254)) + - The command `swaks --help` is now functional ([#4282](https://github.com/docker-mailserver/docker-mailserver/pull/4282)) - **Rspamd:** - DKIM private key path checking is now performed only on paths that do not contain `$` ([#4201](https://github.com/docker-mailserver/docker-mailserver/pull/4201)) -- The command `swaks --help` is now functional ([#4282](https://github.com/docker-mailserver/docker-mailserver/pull/4282)) ### CI diff --git a/target/scripts/start-mailserver.sh b/target/scripts/start-mailserver.sh index f6ceadf513e..e820f746aac 100755 --- a/target/scripts/start-mailserver.sh +++ b/target/scripts/start-mailserver.sh @@ -38,7 +38,6 @@ function _register_functions() { # ? >> Checks _register_check_function '_check_hostname' - _register_check_function '_check_log_level' _register_check_function '_check_spam_prefix' # ? >> Setup @@ -63,7 +62,6 @@ function _register_functions() { ;; ( 'LDAP' ) - _environment_variables_ldap _register_setup_function '_setup_ldap' ;; @@ -76,15 +74,8 @@ function _register_functions() { ;; esac - if [[ ${ENABLE_OAUTH2} -eq 1 ]]; then - _environment_variables_oauth2 - _register_setup_function '_setup_oauth2' - fi - - if [[ ${ENABLE_SASLAUTHD} -eq 1 ]]; then - _environment_variables_saslauthd - _register_setup_function '_setup_saslauthd' - fi + [[ ${ENABLE_OAUTH2} -eq 1 ]] && _register_setup_function '_setup_oauth2' + [[ ${ENABLE_SASLAUTHD} -eq 1 ]] && _register_setup_function '_setup_saslauthd' _register_setup_function '_setup_dovecot_inet_protocols' @@ -122,20 +113,23 @@ function _register_functions() { _register_setup_function '_setup_logwatch' _register_setup_function '_setup_save_states' - _register_setup_function '_setup_apply_fixes_after_configuration' - _register_setup_function '_environment_variables_export' + _register_setup_function '_setup_adjust_state_permissions' if [[ ${ENABLE_MTA_STS} -eq 1 ]]; then _register_setup_function '_setup_mta_sts' _register_start_daemon '_start_daemon_mta_sts_daemon' fi + # ! The following functions must be executed after all other setup functions + _register_setup_function '_setup_directory_and_file_permissions' + _register_setup_function '_setup_run_user_patches' + # ? >> Daemons _register_start_daemon '_start_daemon_cron' _register_start_daemon '_start_daemon_rsyslog' - [[ ${SMTP_ONLY} -ne 1 ]] && _register_start_daemon '_start_daemon_dovecot' + [[ ${SMTP_ONLY} -ne 1 ]] && _register_start_daemon '_start_daemon_dovecot' if [[ ${ENABLE_UPDATE_CHECK} -eq 1 ]]; then if [[ ${DMS_RELEASE} != 'edge' ]]; then @@ -174,26 +168,24 @@ function _register_functions() { # ? >> Executing all stacks / actual start of DMS # ------------------------------------------------------------ -# Ensure DMS only adjusts config files for a new container. -# Container restarts should skip as they retain the modified config. -if [[ ! -f /CONTAINER_START ]]; then - _early_supervisor_setup - _early_variables_setup +_early_supervisor_setup +_early_variables_setup - _log 'info' "Welcome to docker-mailserver ${DMS_RELEASE}" +_log 'info' "Welcome to docker-mailserver ${DMS_RELEASE}" - _register_functions - _check - _setup - _run_user_patches -else - # container was restarted - _early_variables_setup - - _log 'info' 'Container was restarted. Skipping setup routines.' - _log 'info' "Welcome to docker-mailserver ${DMS_RELEASE}" +_register_functions +_check - _register_functions +# Ensure DMS only adjusts config files for a new container. +# Container restarts should skip as they retain the modified config. +if [[ -f /CONTAINER_START ]]; then + _log 'info' 'Container was restarted. Skipping most setup routines.' + # We cannot skip all setup routines because some need to run _after_ + # the initial setup (and hence, they cannot be moved to the check stack). + _setup_directory_and_file_permissions + _setup_adjust_state_permissions +else + _setup fi # marker to check if container was restarted diff --git a/target/scripts/startup/check-stack.sh b/target/scripts/startup/check-stack.sh index 766fcccf688..52d83ad49c8 100644 --- a/target/scripts/startup/check-stack.sh +++ b/target/scripts/startup/check-stack.sh @@ -26,24 +26,6 @@ function _check_hostname() { fi } -function _check_log_level() { - if [[ ${LOG_LEVEL} == 'trace' ]] \ - || [[ ${LOG_LEVEL} == 'debug' ]] \ - || [[ ${LOG_LEVEL} == 'info' ]] \ - || [[ ${LOG_LEVEL} == 'warn' ]] \ - || [[ ${LOG_LEVEL} == 'error' ]] - then - return 0 - else - local DEFAULT_LOG_LEVEL='info' - _log 'warn' "Log level '${LOG_LEVEL}' is invalid (falling back to default '${DEFAULT_LOG_LEVEL}')" - - # shellcheck disable=SC2034 - VARS[LOG_LEVEL]="${DEFAULT_LOG_LEVEL}" - LOG_LEVEL="${DEFAULT_LOG_LEVEL}" - fi -} - function _check_spam_prefix() { # This check should be independent of ENABLE_POP3 and ENABLE_IMAP if [[ ${MOVE_SPAM_TO_JUNK} -eq 0 ]] \ diff --git a/target/scripts/startup/setup-stack.sh b/target/scripts/startup/setup-stack.sh index 8c8e646115e..789dc0c7c4a 100644 --- a/target/scripts/startup/setup-stack.sh +++ b/target/scripts/startup/setup-stack.sh @@ -82,7 +82,7 @@ function _setup_timezone() { fi } -function _setup_apply_fixes_after_configuration() { +function _setup_directory_and_file_permissions() { _log 'trace' 'Removing leftover PID files from a stop/start' find /var/run/ -not -name 'supervisord.pid' -name '*.pid' -delete touch /dev/shm/supervisor.sock @@ -103,7 +103,7 @@ function _setup_apply_fixes_after_configuration() { fi } -function _run_user_patches() { +function _setup_run_user_patches() { local USER_PATCHES='/tmp/docker-mailserver/user-patches.sh' if [[ -f ${USER_PATCHES} ]]; then diff --git a/target/scripts/startup/setup.d/mail_state.sh b/target/scripts/startup/setup.d/mail_state.sh index e819c138295..931e3c05aa7 100644 --- a/target/scripts/startup/setup.d/mail_state.sh +++ b/target/scripts/startup/setup.d/mail_state.sh @@ -3,123 +3,129 @@ # Consolidate all states into a single directory # (/var/mail-state) to allow persistence using docker volumes function _setup_save_states() { - local DEST DESTDIR STATEDIR SERVICEDIR SERVICEDIRS SERVICEFILE SERVICEFILES - - STATEDIR='/var/mail-state' - - if [[ -d ${STATEDIR} ]]; then - _log 'debug' "Consolidating all state onto ${STATEDIR}" - - # Always enabled features: - SERVICEDIRS=( - lib/logrotate - lib/postfix - spool/postfix - ) - - # Only consolidate state for services that are enabled - # Notably avoids copying over 200MB for the ClamAV database - [[ ${ENABLE_AMAVIS} -eq 1 ]] && SERVICEDIRS+=('lib/amavis') - [[ ${ENABLE_CLAMAV} -eq 1 ]] && SERVICEDIRS+=('lib/clamav') - [[ ${ENABLE_FAIL2BAN} -eq 1 ]] && SERVICEDIRS+=('lib/fail2ban') - [[ ${ENABLE_FETCHMAIL} -eq 1 ]] && SERVICEDIRS+=('lib/fetchmail') - [[ ${ENABLE_GETMAIL} -eq 1 ]] && SERVICEDIRS+=('lib/getmail') - [[ ${ENABLE_MTA_STS} -eq 1 ]] && SERVICEDIRS+=('lib/mta-sts') - [[ ${ENABLE_POSTGREY} -eq 1 ]] && SERVICEDIRS+=('lib/postgrey') - [[ ${ENABLE_RSPAMD} -eq 1 ]] && SERVICEDIRS+=('lib/rspamd') - [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] && SERVICEDIRS+=('lib/redis') - [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] && SERVICEDIRS+=('lib/spamassassin') - [[ ${ENABLE_SRS} -eq 1 ]] && SERVICEDIRS+=('lib/postsrsd') - [[ ${SMTP_ONLY} -ne 1 ]] && SERVICEDIRS+=('lib/dovecot') - - # Single service files - [[ ${ENABLE_SRS} -eq 1 ]] && SERVICEFILES+=('/etc/postsrsd.secret') - - for SERVICEFILE in "${SERVICEFILES[@]}"; do - DEST="${STATEDIR}/${SERVICEFILE}" - DESTDIR="${DEST%/*}" - - mkdir -p "${DESTDIR}" - if [[ -f ${DEST} ]]; then - _log 'trace' "Destination ${DEST} exists, linking ${SERVICEFILE} to it" - # Original content from image no longer relevant, remove it: - rm -f "${SERVICEFILE}" - elif [[ -f "${SERVICEFILE}" ]]; then - _log 'trace' "Moving ${SERVICEFILE} to ${DEST}" - # Empty volume was mounted, or new content from enabling a feature ENV: - mv "${SERVICEFILE}" "${DEST}" - # Apply SELinux security context to match the state directory, so access - # is not restricted to the current running container: - chcon -R --reference="${STATEDIR}" "${DEST}" 2>/dev/null || true - fi - - # Symlink the original file in the container ($SERVICEFILE) to be - # sourced from assocaiated path in /var/mail-state/ ($DEST): - ln -s "${DEST}" "${SERVICEFILE}" - done - - for SERVICEDIR in "${SERVICEDIRS[@]}"; do - DEST="${STATEDIR}/${SERVICEDIR//\//-}" - SERVICEDIR="/var/${SERVICEDIR}" - - # If relevant content is found in /var/mail-state (presumably a volume mount), - # use it instead. Otherwise copy over any missing directories checked. - if [[ -d ${DEST} ]]; then - _log 'trace' "Destination ${DEST} exists, linking ${SERVICEDIR} to it" - # Original content from image no longer relevant, remove it: - rm -rf "${SERVICEDIR}" - elif [[ -d ${SERVICEDIR} ]]; then - _log 'trace' "Moving contents of ${SERVICEDIR} to ${DEST}" - # An empty volume was mounted, or new content dir now exists from enabling a feature ENV: - mv "${SERVICEDIR}" "${DEST}" - # Apply SELinux security context to match the state directory, so access - # is not restricted to the current running container: - chcon -R --reference="${STATEDIR}" "${DEST}" 2>/dev/null || true - else - _log 'error' "${SERVICEDIR} should exist but is missing" - fi - - # Symlink the original path in the container ($SERVICEDIR) to be - # sourced from assocaiated path in /var/mail-state/ ($DEST): - ln -s "${DEST}" "${SERVICEDIR}" - done - - # This ensures the user and group of the files from the external mount have their - # numeric ID values in sync. New releases where the installed packages order changes - # can change the values in the Docker image, causing an ownership mismatch. - # NOTE: More details about users and groups added during image builds are documented here: - # https://github.com/docker-mailserver/docker-mailserver/pull/3011#issuecomment-1399120252 - _log 'trace' "Fixing ${STATEDIR}/* permissions" - [[ ${ENABLE_AMAVIS} -eq 1 ]] && chown -R amavis:amavis "${STATEDIR}/lib-amavis" - [[ ${ENABLE_CLAMAV} -eq 1 ]] && chown -R clamav:clamav "${STATEDIR}/lib-clamav" - [[ ${ENABLE_FETCHMAIL} -eq 1 ]] && chown -R fetchmail:nogroup "${STATEDIR}/lib-fetchmail" - [[ ${ENABLE_MTA_STS} -eq 1 ]] && chown -R _mta-sts:_mta-sts "${STATEDIR}/lib-mta-sts" - [[ ${ENABLE_POSTGREY} -eq 1 ]] && chown -R postgrey:postgrey "${STATEDIR}/lib-postgrey" - [[ ${ENABLE_RSPAMD} -eq 1 ]] && chown -R _rspamd:_rspamd "${STATEDIR}/lib-rspamd" - [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] && chown -R redis:redis "${STATEDIR}/lib-redis" - [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] && chown -R debian-spamd:debian-spamd "${STATEDIR}/lib-spamassassin" - - chown -R root:root "${STATEDIR}/lib-logrotate" - chown -R postfix:postfix "${STATEDIR}/lib-postfix" - - # NOTE: The Postfix spool location has mixed owner/groups to take into account: - # UID = postfix(101): active, bounce, corrupt, defer, deferred, flush, hold, incoming, maildrop, private, public, saved, trace - # UID = root(0): dev, etc, lib, pid, usr - # GID = postdrop(103): maildrop, public - # GID for all other directories is root(0) - # NOTE: `spool-postfix/private/` will be set to `postfix:postfix` when Postfix starts / restarts - # Set most common ownership: - chown -R postfix:root "${STATEDIR}/spool-postfix" - chown root:root "${STATEDIR}/spool-postfix" - - # These two require the postdrop(103) group: - chgrp -R postdrop "${STATEDIR}"/spool-postfix/{maildrop,public} - - # These permissions rely on the `postdrop` binary having the SGID bit set. - # Ref: https://github.com/docker-mailserver/docker-mailserver/pull/3625 - chmod 730 "${STATEDIR}/spool-postfix/maildrop" - chmod 710 "${STATEDIR}/spool-postfix/public" - else - _log 'debug' "'${STATEDIR}' is not present; Not consolidating state" + if [[ ! -d ${DMS_STATE_DIR:?DMS_STATE_DIR is not set} ]]; then + _log 'debug' "'${DMS_STATE_DIR}' is not present - not consolidating state" + return 0 fi + + _log 'debug' "Consolidating all state onto ${DMS_STATE_DIR}" + + local DEST SERVICEDIR SERVICEDIRS SERVICEFILE SERVICEFILES + + # Always enabled features: + SERVICEDIRS=( + 'lib/logrotate' + 'lib/postfix' + 'spool/postfix' + ) + + # Only consolidate state for services that are enabled + # Notably avoids copying over 200MB for the ClamAV database + [[ ${ENABLE_AMAVIS} -eq 1 ]] && SERVICEDIRS+=('lib/amavis') + [[ ${ENABLE_CLAMAV} -eq 1 ]] && SERVICEDIRS+=('lib/clamav') + [[ ${ENABLE_FAIL2BAN} -eq 1 ]] && SERVICEDIRS+=('lib/fail2ban') + [[ ${ENABLE_FETCHMAIL} -eq 1 ]] && SERVICEDIRS+=('lib/fetchmail') + [[ ${ENABLE_GETMAIL} -eq 1 ]] && SERVICEDIRS+=('lib/getmail') + [[ ${ENABLE_MTA_STS} -eq 1 ]] && SERVICEDIRS+=('lib/mta-sts') + [[ ${ENABLE_POSTGREY} -eq 1 ]] && SERVICEDIRS+=('lib/postgrey') + [[ ${ENABLE_RSPAMD} -eq 1 ]] && SERVICEDIRS+=('lib/rspamd') + [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] && SERVICEDIRS+=('lib/redis') + [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] && SERVICEDIRS+=('lib/spamassassin') + [[ ${ENABLE_SRS} -eq 1 ]] && SERVICEDIRS+=('lib/postsrsd') + [[ ${SMTP_ONLY} -ne 1 ]] && SERVICEDIRS+=('lib/dovecot') + + # Single service files + [[ ${ENABLE_SRS} -eq 1 ]] && SERVICEFILES+=('/etc/postsrsd.secret') + + for SERVICEFILE in "${SERVICEFILES[@]}"; do + DEST="${DMS_STATE_DIR}/${SERVICEFILE}" + + # Append service parent dir(s) path to the state dir and ensure it exists: + mkdir -p "${DEST%/*}" + if [[ -f ${DEST} ]]; then + _log 'trace' "Destination ${DEST} exists, linking ${SERVICEFILE} to it" + # Original content from image no longer relevant, remove it: + rm -f "${SERVICEFILE}" + elif [[ -f "${SERVICEFILE}" ]]; then + _log 'trace' "Moving ${SERVICEFILE} to ${DEST}" + # Empty volume was mounted, or new content from enabling a feature ENV: + mv "${SERVICEFILE}" "${DEST}" + # Apply SELinux security context to match the state directory, so access + # is not restricted to the current running container: + chcon -R --reference="${DMS_STATE_DIR}" "${DEST}" 2>/dev/null || true + fi + + # Symlink the original file in the container ($SERVICEFILE) to be + # sourced from assocaiated path in /var/mail-state/ ($DEST): + ln -s "${DEST}" "${SERVICEFILE}" + done + + for SERVICEDIR in "${SERVICEDIRS[@]}"; do + DEST="${DMS_STATE_DIR}/${SERVICEDIR//\//-}" + SERVICEDIR="/var/${SERVICEDIR}" + + # If relevant content is found in /var/mail-state (presumably a volume mount), + # use it instead. Otherwise copy over any missing directories checked. + if [[ -d ${DEST} ]]; then + _log 'trace' "Destination ${DEST} exists, linking ${SERVICEDIR} to it" + # Original content from image no longer relevant, remove it: + rm -rf "${SERVICEDIR}" + elif [[ -d ${SERVICEDIR} ]]; then + _log 'trace' "Moving contents of ${SERVICEDIR} to ${DEST}" + # An empty volume was mounted, or new content dir now exists from enabling a feature ENV: + mv "${SERVICEDIR}" "${DEST}" + # Apply SELinux security context to match the state directory, so access + # is not restricted to the current running container: + # https://github.com/docker-mailserver/docker-mailserver/pull/3890 + chcon -R --reference="${DMS_STATE_DIR}" "${DEST}" 2>/dev/null || true + else + _log 'error' "${SERVICEDIR} should exist but is missing" + fi + + # Symlink the original path in the container ($SERVICEDIR) to be + # sourced from associated path in /var/mail-state/ ($DEST): + ln -s "${DEST}" "${SERVICEDIR}" + done +} + +# These corrections are to fix changes to UID/GID values between upgrades, +# or when ownership/permissions were altered externally on the host (eg: migration or system scripts) +function _setup_adjust_state_permissions() { + [[ ! -d ${DMS_STATE_DIR:?DMS_STATE_DIR is not set} ]] && return 0 + + # This ensures the user and group of the files from the external mount have their + # numeric ID values in sync. New releases where the installed packages order changes + # can change the values in the Docker image, causing an ownership mismatch. + # NOTE: More details about users and groups added during image builds are documented here: + # https://github.com/docker-mailserver/docker-mailserver/pull/3011#issuecomment-1399120252 + _log 'trace' "Ensuring correct ownership + permissions for DMS state dir: '${DMS_STATE_DIR}'" + [[ ${ENABLE_AMAVIS} -eq 1 ]] && chown -R amavis:amavis "${DMS_STATE_DIR}/lib-amavis" + [[ ${ENABLE_CLAMAV} -eq 1 ]] && chown -R clamav:clamav "${DMS_STATE_DIR}/lib-clamav" + [[ ${ENABLE_FETCHMAIL} -eq 1 ]] && chown -R fetchmail:nogroup "${DMS_STATE_DIR}/lib-fetchmail" + [[ ${ENABLE_MTA_STS} -eq 1 ]] && chown -R _mta-sts:_mta-sts "${DMS_STATE_DIR}/lib-mta-sts" + [[ ${ENABLE_POSTGREY} -eq 1 ]] && chown -R postgrey:postgrey "${DMS_STATE_DIR}/lib-postgrey" + [[ ${ENABLE_RSPAMD} -eq 1 ]] && chown -R _rspamd:_rspamd "${DMS_STATE_DIR}/lib-rspamd" + [[ ${ENABLE_RSPAMD_REDIS} -eq 1 ]] && chown -R redis:redis "${DMS_STATE_DIR}/lib-redis" + [[ ${ENABLE_SPAMASSASSIN} -eq 1 ]] && chown -R debian-spamd:debian-spamd "${DMS_STATE_DIR}/lib-spamassassin" + + chown -R root:root "${DMS_STATE_DIR}/lib-logrotate" + chown -R postfix:postfix "${DMS_STATE_DIR}/lib-postfix" + + # NOTE: The Postfix spool location has mixed owner/groups to take into account: + # UID = postfix(101): active, bounce, corrupt, defer, deferred, flush, hold, incoming, maildrop, private, public, saved, trace + # UID = root(0): dev, etc, lib, pid, usr + # GID = postdrop(103): maildrop, public + # GID for all other directories is root(0) + # NOTE: `spool-postfix/private/` will be set to `postfix:postfix` when Postfix starts / restarts + # Set most common ownership: + chown -R postfix:root "${DMS_STATE_DIR}/spool-postfix" + chown root:root "${DMS_STATE_DIR}/spool-postfix" + + # These two require the postdrop(103) group: + chgrp -R postdrop "${DMS_STATE_DIR}"/spool-postfix/{maildrop,public} + + # These permissions rely on the `postdrop` binary having the SGID bit set. + # Ref: https://github.com/docker-mailserver/docker-mailserver/pull/3625 + chmod 730 "${DMS_STATE_DIR}/spool-postfix/maildrop" + chmod 710 "${DMS_STATE_DIR}/spool-postfix/public" } diff --git a/target/scripts/startup/variables-stack.sh b/target/scripts/startup/variables-stack.sh index 3fa4d76165b..f8f5af5e2ea 100644 --- a/target/scripts/startup/variables-stack.sh +++ b/target/scripts/startup/variables-stack.sh @@ -4,9 +4,27 @@ declare -A VARS function _early_variables_setup() { + __environment_variables_log_level _obtain_hostname_and_domainname __environment_variables_backwards_compatibility __environment_variables_general_setup + + [[ ${ACCOUNT_PROVISIONER} == 'LDAP' ]] && __environment_variables_ldap + [[ ${ENABLE_OAUTH2} -eq 1 ]] && __environment_variables_oauth2 + [[ ${ENABLE_SASLAUTHD} -eq 1 ]] && __environment_variables_saslauthd + + __environment_variables_export +} + +# Declare a variable as readonly if it is not already set. +function __declare_readonly() { + local VARIABLE_NAME=${1:?Variable name required when declaring a variable as readonly} + local VARIABLE_VALUE=${2:?Variable value required when declaring a variable as readonly} + + if [[ ! -v ${VARIABLE_NAME} ]]; then + readonly "${VARIABLE_NAME}=${VARIABLE_VALUE}" + VARS[${VARIABLE_NAME}]="${VARIABLE_VALUE}" + fi } # This function handles variables that are deprecated. This allows a @@ -55,6 +73,12 @@ function __environment_variables_general_setup() { VARS[DMS_VMAIL_UID]="${DMS_VMAIL_UID:=5000}" VARS[DMS_VMAIL_GID]="${DMS_VMAIL_GID:=5000}" + # internal variables are next + + __declare_readonly 'DMS_STATE_DIR' '/var/mail-state' + + # user-customizable are last + _log 'trace' 'Setting anti-spam & anti-virus environment variables' VARS[AMAVIS_LOGLEVEL]="${AMAVIS_LOGLEVEL:=0}" @@ -159,15 +183,27 @@ function __environment_variables_general_setup() { VARS[UPDATE_CHECK_INTERVAL]="${UPDATE_CHECK_INTERVAL:=1d}" } -function _environment_variables_oauth2() { - _log 'debug' 'Setting OAUTH2-related environment variables now' - - VARS[OAUTH2_INTROSPECTION_URL]="${OAUTH2_INTROSPECTION_URL:=}" +function __environment_variables_log_level() { + if [[ ${LOG_LEVEL} == 'trace' ]] \ + || [[ ${LOG_LEVEL} == 'debug' ]] \ + || [[ ${LOG_LEVEL} == 'info' ]] \ + || [[ ${LOG_LEVEL} == 'warn' ]] \ + || [[ ${LOG_LEVEL} == 'error' ]] + then + return 0 + else + local DEFAULT_LOG_LEVEL='info' + _log 'warn' "Log level '${LOG_LEVEL}' is invalid (falling back to default '${DEFAULT_LOG_LEVEL}')" + + # shellcheck disable=SC2034 + VARS[LOG_LEVEL]="${DEFAULT_LOG_LEVEL}" + LOG_LEVEL="${DEFAULT_LOG_LEVEL}" + fi } # This function handles environment variables related to LDAP. # NOTE: SASLAuthd and Dovecot LDAP support inherit these common ENV. -function _environment_variables_ldap() { +function __environment_variables_ldap() { _log 'debug' 'Setting LDAP-related environment variables now' VARS[LDAP_BIND_DN]="${LDAP_BIND_DN:=}" @@ -177,9 +213,15 @@ function _environment_variables_ldap() { VARS[LDAP_START_TLS]="${LDAP_START_TLS:=no}" } +function __environment_variables_oauth2() { + _log 'debug' 'Setting OAUTH2-related environment variables now' + + VARS[OAUTH2_INTROSPECTION_URL]="${OAUTH2_INTROSPECTION_URL:=}" +} + # This function handles environment variables related to SASLAUTHD # LDAP specific ENV handled in: `startup/setup.d/saslauthd.sh:_setup_saslauthd()` -function _environment_variables_saslauthd() { +function __environment_variables_saslauthd() { _log 'debug' 'Setting SASLAUTHD-related environment variables now' # This ENV is only used by the supervisor service config `saslauth.conf`: @@ -190,7 +232,7 @@ function _environment_variables_saslauthd() { # This function Writes the contents of the `VARS` map (associative array) # to locations where they can be sourced from (e.g. `/etc/dms-settings`) # or where they can be used by Bash directly (e.g. `/root/.bashrc`). -function _environment_variables_export() { +function __environment_variables_export() { _log 'debug' "Exporting environment variables now (creating '/etc/dms-settings')" : >/root/.bashrc # make DMS variables available in login shells and their subprocesses From c66d8ce40b705f7ce9d67e12e48a26ca1a67be9d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Feb 2025 10:36:20 +0100 Subject: [PATCH 465/592] chore(deps): Bump docker/setup-qemu-action from 3.3.0 to 3.4.0 (#4353) --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 6000f5ffd53..53cb6893001 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -71,7 +71,7 @@ jobs: cache-buildx- - name: 'Set up QEMU' - uses: docker/setup-qemu-action@v3.3.0 + uses: docker/setup-qemu-action@v3.4.0 with: platforms: arm64 diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index 42cde7ea4d5..d5966a2bfca 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -35,7 +35,7 @@ jobs: type=semver,pattern={{major}}.{{minor}}.{{patch}} - name: 'Set up QEMU' - uses: docker/setup-qemu-action@v3.3.0 + uses: docker/setup-qemu-action@v3.4.0 with: platforms: arm64 From 83bfe72d489beda875717130eba9b348218f3bda Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Wed, 12 Feb 2025 11:56:51 +1300 Subject: [PATCH 466/592] chore: Migrate dovecot config from Dockerfile (#4350) --- Dockerfile | 10 ---- target/scripts/start-mailserver.sh | 1 - target/scripts/startup/setup.d/dovecot.sh | 59 ++++++++++++++++++++--- 3 files changed, 51 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index 09295a22710..32ac9ae84a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -88,16 +88,6 @@ RUN dpkg -i /dovecot-fts-xapian-*.deb && rm /dovecot-fts-xapian-*.deb COPY target/dovecot/*.inc target/dovecot/*.conf /etc/dovecot/conf.d/ COPY target/dovecot/dovecot-purge.cron /etc/cron.d/dovecot-purge.disabled RUN chmod 0 /etc/cron.d/dovecot-purge.disabled -WORKDIR /usr/share/dovecot - -# hadolint ignore=SC2016,SC2086,SC2069 -RUN < Date: Wed, 12 Feb 2025 20:34:22 +0100 Subject: [PATCH 467/592] docs: Fix typo in DKIM and utils.sh (#4358) --- docs/content/config/best-practices/dkim_dmarc_spf.md | 4 ++-- target/scripts/helpers/utils.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md index 2ef8e902751..c04d4b7e1bf 100644 --- a/docs/content/config/best-practices/dkim_dmarc_spf.md +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -29,7 +29,7 @@ Cloudflare has written an [article about DKIM, DMARC and SPF][cloudflare-dkim-dm When DKIM is enabled: 1. Inbound mail will verify any included DKIM signatures -2. Outbound mail is signed (_when you're sending domain has a configured DKIM key_) +2. Outbound mail is signed (_when your sending domain has a configured DKIM key_) DKIM requires a public/private key pair to enable **signing (_via private key_)** your outgoing mail, while the receiving end must query DNS to **verify (_via public key_)** that the signature is trustworthy. @@ -76,7 +76,7 @@ You should have: ??? info "Changing the key size" The keypair generated for using with DKIM presently defaults to RSA-2048. This is a good size but you can lower the security to `1024-bit`, or increase it to `4096-bit` (_discouraged as that is excessive_). - + To generate a key with different size (_for RSA 1024-bit_) run: ```sh diff --git a/target/scripts/helpers/utils.sh b/target/scripts/helpers/utils.sh index c848e13e775..90fd8f9b028 100644 --- a/target/scripts/helpers/utils.sh +++ b/target/scripts/helpers/utils.sh @@ -122,7 +122,7 @@ function _reload_postfix() { # you can set the environment variable `POSTFIX_README_DIRECTORY='/new/dir/'` # (`POSTFIX_` is an arbitrary prefix, you can choose the one you like), # and then call this function: -# `_replace_by_env_in_file 'POSTFIX_' 'PATH TO POSTFIX's main.cf>` +# `_replace_by_env_in_file 'POSTFIX_' '` # # ## Panics # From 425d1162aeeeb532e12eeccd9548d31213c9f07f Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Fri, 14 Feb 2025 01:16:31 +1300 Subject: [PATCH 468/592] chore: `packages.sh` - Bump versions + housekeeping (#4357) --- CHANGELOG.md | 38 ++++++++++--------- target/scripts/build/compile.sh | 41 ++++++++++++-------- target/scripts/build/packages.sh | 64 +++++++++++++++++++------------- 3 files changed, 85 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 379992c3859..794c2e64a71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,20 +11,12 @@ All notable changes to this project will be documented in this file. The format - **saslauthd** mechanism support via ENV `SASLAUTHD_MECHANISMS` with `pam`, `shadow`, `mysql` values has been removed. Only `ldap` and `rimap` remain supported ([#4259](https://github.com/docker-mailserver/docker-mailserver/pull/4259)) - **getmail6** has been refactored: ([#4156](https://github.com/docker-mailserver/docker-mailserver/pull/4156)) - The [DMS config volume](https://docker-mailserver.github.io/docker-mailserver/v15.0/config/advanced/optional-config/#volumes) now has support for `getmailrc_general.cf` for overriding [common default settings](https://docker-mailserver.github.io/docker-mailserver/v15.0/config/advanced/mail-getmail/#common-options). If you previously mounted this config file directly to `/etc/getmailrc_general` you should switch to our config volume support. - - IMAP/POP3 example configs added to our [`config-examples`](https://github.com/docker-mailserver/docker-mailserver/tree/v15.0.0/config-examples/getmail). - - ENV [`GETMAIL_POLL`](https://docker-mailserver.github.io/docker-mailserver/v15.0/config/environment/#getmail_poll) now supports values above 30 minutes. - - Added `getmail` as a new service for `supervisor` to manage, replacing cron for periodic polling. - Generated getmail configuration files no longer set the `message_log` option. Instead of individual log files per config, the [default base settings DMS configures](https://github.com/docker-mailserver/docker-mailserver/tree/v15.0.0/target/getmail/getmailrc_general) now enables `message_log_syslog`. This aligns with how other services in DMS log to syslog where it is captured in `mail.log`. - Getmail configurations have changed location from the base of the DMS Config Volume, to the `getmail/` subdirectory. Any existing configurations **must be migrated manually.** - - DMS v14 mistakenly relocated the _getmail state directory_ to the _DMS Config Volume_ as a `getmail/` subdirectory. + - **DMS v14 mistakenly** relocated the _getmail state directory_ to the _DMS Config Volume_ as a `getmail/` subdirectory. - This has been corrected to `/var/lib/getmail` (_if you have mounted a DMS State Volume to `/var/mail-state`, `/var/lib/getmail` will be symlinked to `/var/mail-state/lib-getmail`_). - To preserve this state when upgrading to DMS v15, **you must manually migrate `getmail/` from the _DMS Config Volume_ to `lib-getmail/` in the _DMS State Volume_.** -### Security - -- **Fail2ban:** - - Ensure a secure connection, when downloading the fail2ban package ([#4080](https://github.com/docker-mailserver/docker-mailserver/pull/4080)) - ### Added - **Internal:** @@ -33,30 +25,40 @@ All notable changes to this project will be documented in this file. The format ### Updates -- **Removed `VERSION` file** from the repo that releases of DMS prior to v13 (Nov 2023) would check to detect new releases ([#3677](https://github.com/docker-mailserver/docker-mailserver/issues/3677), [#4321](https://github.com/docker-mailserver/docker-mailserver/pull/4321)) -- **Fail2ban:** - - Updated to version [`1.1.0`](https://github.com/fail2ban/fail2ban/releases/tag/1.1.0) ([#4045](https://github.com/docker-mailserver/docker-mailserver/pull/4045)) +**Internal:** + - **Removed `VERSION` file** from the repo. Releases of DMS prior to v13 (Nov 2023) would check this to detect new releases ([#3677](https://github.com/docker-mailserver/docker-mailserver/issues/3677), [#4321](https://github.com/docker-mailserver/docker-mailserver/pull/4321)) + - During image build, ensure a secure connection when downloading the `fail2ban` package ([#4080](https://github.com/docker-mailserver/docker-mailserver/pull/4080)) - **Documentation:** - Account Management and Authentication pages have been rewritten and better organized ([#4122](https://github.com/docker-mailserver/docker-mailserver/pull/4122)) - Add a caveat for `DMS_VMAIL_UID` not being compatible with `0` / root ([#4143](https://github.com/docker-mailserver/docker-mailserver/pull/4143)) +- **Getmail:** ([#4156](https://github.com/docker-mailserver/docker-mailserver/pull/4156)) + - Added `getmail` as a new service for `supervisor` to manage, replacing cron for periodic polling. + - IMAP/POP3 example configs added to our [`config-examples`](https://github.com/docker-mailserver/docker-mailserver/tree/v15.0.0/config-examples/getmail). + - ENV [`GETMAIL_POLL`](https://docker-mailserver.github.io/docker-mailserver/v15.0/config/environment/#getmail_poll) now supports values above 30 minutes. - **Postfix:** - By default opt-out from _Microsoft reactions_ for outbound mail ([#4120](https://github.com/docker-mailserver/docker-mailserver/pull/4120)) -- Updated `jaq` version from `1.3.0` to `2.0.0` ([#4190](https://github.com/docker-mailserver/docker-mailserver/pull/4190)) -- Updated Rspamd GTube settings and tests ([#4191](https://github.com/docker-mailserver/docker-mailserver/pull/4191)) +- **Rspamd:** + - Updated GTube settings and tests ([#4191](https://github.com/docker-mailserver/docker-mailserver/pull/4191)) +- Updated externally installed software ([#4357](https://github.com/docker-mailserver/docker-mailserver/pull/4357)): + - `DOVECOT_COMMUNITY_REPO=1` custom image build ARG now supports the latest Dovecot [`2.4.x`](https://github.com/dovecot/core/releases/tag/2.4.0) (_DMS provides Dovecot `2.3.19` by default_) + - Dovecot FTS Xapian module (`1.7.12` => [`1.9.0`](https://github.com/grosjo/fts-xapian/releases/tag/1.9)) + - `jaq` (`1.3.0` => [`2.1.0`](https://github.com/01mf02/jaq/releases/tag/v2.1.0)) + - Fail2Ban (`1.0.2-2` => [`1.1.0`](https://github.com/fail2ban/fail2ban/releases/tag/1.1.0)) ([#4045](https://github.com/docker-mailserver/docker-mailserver/pull/4045)) + - Rspamd (`3.8.4` => [`3.11.0`](https://github.com/rspamd/rspamd/releases/tag/3.11.0)) - Implicitly upgraded during image build, as the third-party repo lacks version pinning support. ### Fixes - **Dovecot:** - The logwatch `ignore.conf` now also excludes Xapian messages about pending documents ([#4060](https://github.com/docker-mailserver/docker-mailserver/pull/4060)) - - `dovecot-fts-xapian` plugin was updated to `1.7.13`, fixing a regression with indexing ([#4095](https://github.com/docker-mailserver/docker-mailserver/pull/4095)) + - `dovecot-fts-xapian` plugin was updated, fixing a regression with indexing ([#4095](https://github.com/docker-mailserver/docker-mailserver/pull/4095)) - The "dummy account" workaround for _Dovecot Quota_ feature support no longer treats the alias as a regex when checking the Dovecot UserDB ([#4222](https://github.com/docker-mailserver/docker-mailserver/pull/4222)) - **LDAP:** - - Correctly apply a compatibility fix for OAuth2 introduced in DMS v13.3.1 which had not been applied to the actual LDAP config changes ([#4175](https://github.com/docker-mailserver/docker-mailserver/pull/4175)) + - Correctly apply a compatibility fix for OAuth2 introduced in DMS `v13.3.1` which had not been applied to the actual LDAP config changes ([#4175](https://github.com/docker-mailserver/docker-mailserver/pull/4175)) - **Internal:** - The main `mail.log` (_which is piped to stdout via `tail`_) now correctly begins from the first log line of the active container run. Previously some daemon logs and potential warnings/errors were omitted ([#4146](https://github.com/docker-mailserver/docker-mailserver/pull/4146)) - `start-mailserver.sh` removed unused `shopt -s inherit_errexit` ([#4161](https://github.com/docker-mailserver/docker-mailserver/pull/4161)) - - Fixed a regression introduced in v14 where `postfix-main.cf` appended `stderr` output into `/etc/postfix/main.cf`, causing Postfix startup to fail ([#4147](https://github.com/docker-mailserver/docker-mailserver/pull/4147)) - - Fixed a regression introduced in v14 to better support running `start-mailserver.sh` with container restarts, which now only skip calling `_setup()` ([#4323](https://github.com/docker-mailserver/docker-mailserver/pull/4323#issuecomment-2629559254)) + - Fixed a regression introduced in DMS v14 where `postfix-main.cf` appended `stderr` output into `/etc/postfix/main.cf`, causing Postfix startup to fail ([#4147](https://github.com/docker-mailserver/docker-mailserver/pull/4147)) + - Fixed a regression introduced in DMS v14 to better support running `start-mailserver.sh` with container restarts, which now only skip calling `_setup()` ([#4323](https://github.com/docker-mailserver/docker-mailserver/pull/4323#issuecomment-2629559254)) - The command `swaks --help` is now functional ([#4282](https://github.com/docker-mailserver/docker-mailserver/pull/4282)) - **Rspamd:** - DKIM private key path checking is now performed only on paths that do not contain `$` ([#4201](https://github.com/docker-mailserver/docker-mailserver/pull/4201)) diff --git a/target/scripts/build/compile.sh b/target/scripts/build/compile.sh index c34a88331c3..bad84f2358b 100644 --- a/target/scripts/build/compile.sh +++ b/target/scripts/build/compile.sh @@ -11,28 +11,39 @@ source /usr/local/bin/helpers/log.sh # shellcheck disable=SC2310 _log_level_is 'trace' && QUIET='-y' || QUIET='-qq' -function _compile_dovecot_fts_xapian() { +function _install_build_deps() { apt-get "${QUIET}" update apt-get "${QUIET}" install --no-install-recommends \ automake libtool pkg-config libicu-dev libsqlite3-dev libxapian-dev make build-essential dh-make devscripts dovecot-dev +} - local XAPIAN_VERSION='1.7.13' - curl -sSfL -o dovecot-fts-xapian.tar.gz \ - "https://github.com/grosjo/fts-xapian/releases/download/${XAPIAN_VERSION}/dovecot-fts-xapian-${XAPIAN_VERSION}.tar.gz" - tar xf dovecot-fts-xapian.tar.gz - +function _build_package() { + local XAPIAN_VERSION='1.9' + curl -fsSL "https://github.com/grosjo/fts-xapian/releases/download/${XAPIAN_VERSION}/dovecot-fts-xapian-${XAPIAN_VERSION}.tar.gz" \ + | tar -xz cd "fts-xapian-${XAPIAN_VERSION}" - USER=root dh_make -p "dovecot-fts-xapian-${XAPIAN_VERSION}" --single --native --copyright gpl2 -y + # Prepare for building DEB source package: + # https://manpages.debian.org/bookworm/dh-make/dh_make.1.en.html + # License LGPL 2.1: https://github.com/grosjo/fts-xapian/issues/174#issuecomment-2422404568 + USER=root dh_make --packagename "dovecot-fts-xapian-${XAPIAN_VERSION}" --single --native --copyright lgpl2 -y + # Remove generated example files: rm debian/*.ex - cp PACKAGES/DEB/control debian/ - cp PACKAGES/DEB/changelog debian/ - cp PACKAGES/DEB/compat debian/ - + # Add required package metadata: + # https://www.debian.org/doc/manuals/maint-guide/dreq.en.html#control + curl -fsSL https://raw.githubusercontent.com/grosjo/fts-xapian/refs/tags/1.7.16/PACKAGES/DEB/control > debian/control + # Replace version number: sed -i -E "s|(dovecot-fts-xapian)-[1-9\.-]+|\1-${XAPIAN_VERSION}|g" debian/control - sed -i -E "s|(dovecot-fts-xapian)-[1-9\.-]+ \(.*\)(.*)|\1-${XAPIAN_VERSION} (${XAPIAN_VERSION})\2|g" debian/changelog - - debuild -us -uc -B | tee /tmp/debuild.log 2>&1 + # Required to proceed with debuild: + # https://www.debian.org/doc/manuals/maint-guide/dother.en.html#compat + # (13 is the default debhelper version from the original `dh_make` generated `debian/control`): + echo '13' > debian/compat + + # Build arch specific binary package via debuild: + # https://manpages.debian.org/bookworm/devscripts/debuild.1.en.html + # https://manpages.debian.org/bookworm/dpkg-dev/dpkg-buildpackage.1.en.html + debuild --no-sign --build=any | tee /tmp/debuild.log 2>&1 } -_compile_dovecot_fts_xapian +_install_build_deps +_build_package diff --git a/target/scripts/build/packages.sh b/target/scripts/build/packages.sh index 787c84acd89..bcfdcbcb810 100644 --- a/target/scripts/build/packages.sh +++ b/target/scripts/build/packages.sh @@ -24,13 +24,14 @@ function _pre_installation_steps() { apt-get "${QUIET}" upgrade _log 'trace' 'Installing packages that are needed early' - # add packages usually required by apt to - # - not log unnecessary warnings - # - be able to add PPAs early (e.g., Rspamd) + # Add packages usually required by apt to: local EARLY_PACKAGES=( - apt-utils # avoid useless warnings - apt-transport-https ca-certificates curl gnupg # required for adding PPAs - systemd-standalone-sysusers # avoid problems with SA / Amavis (https://github.com/docker-mailserver/docker-mailserver/pull/3403#pullrequestreview-1596689953) + # Avoid logging unnecessary warnings: + apt-utils + # Required for adding third-party repos (/etc/apt/sources.list.d) as alternative package sources (eg: Dovecot CE and Rspamd): + apt-transport-https ca-certificates curl gnupg + # Avoid problems with SA / Amavis (https://github.com/docker-mailserver/docker-mailserver/pull/3403#pullrequestreview-1596689953): + systemd-standalone-sysusers ) apt-get "${QUIET}" install --no-install-recommends "${EARLY_PACKAGES[@]}" 2>/dev/null } @@ -38,7 +39,7 @@ function _pre_installation_steps() { function _install_utils() { _log 'debug' 'Installing utils sourced from Github' _log 'trace' 'Installing jaq' - local JAQ_TAG='v2.0.0' + local JAQ_TAG='v2.1.0' curl -sSfL "https://github.com/01mf02/jaq/releases/download/${JAQ_TAG}/jaq-$(uname -m)-unknown-linux-gnu" -o /usr/bin/jaq chmod +x /usr/bin/jaq @@ -136,43 +137,56 @@ function _install_dovecot() { dovecot-pop3d dovecot-sieve ) - # Dovecot packages for community supported features. + # Additional Dovecot packages for supporting the DMS community (docs-only guide contributions). DOVECOT_PACKAGES+=(dovecot-auth-lua) - # Dovecot's deb community repository only provides x86_64 packages, so do not include it - # when building for another architecture. + # (Opt-in via ENV) Change repo source for dovecot packages to a third-party repo maintained by Dovecot. + # NOTE: AMD64 / x86_64 is the only supported arch from the Dovecot CE repo (thus noDMS built for ARM64 / aarch64) + # Repo: https://repo.dovecot.org/ce-2.4-latest/debian/bookworm/dists/bookworm/main/ + # Docs: https://repo.dovecot.org/#debian if [[ ${DOVECOT_COMMUNITY_REPO} -eq 1 ]] && [[ "$(uname --machine)" == "x86_64" ]]; then - _log 'trace' 'Using Dovecot community repository' - curl -sSfL https://repo.dovecot.org/DOVECOT-REPO-GPG | gpg --import - gpg --export ED409DA1 > /etc/apt/trusted.gpg.d/dovecot.gpg - echo "deb https://repo.dovecot.org/ce-2.3-latest/debian/${VERSION_CODENAME} ${VERSION_CODENAME} main" > /etc/apt/sources.list.d/dovecot.list - - _log 'trace' 'Updating Dovecot package signatures' + # WARNING: Repo only provides Debian Bookworm package support for Dovecot CE 2.4+. + # As Debian Bookworm only packages Dovecot 2.3.x, building DMS with this alternative package repo may not yet be compatible with DMS: + # - 2.3.19: https://salsa.debian.org/debian/dovecot/-/tree/stable/bookworm + # - 2.3.21: https://salsa.debian.org/debian/dovecot/-/tree/stable/bookworm-backports + + _log 'trace' 'Adding third-party package repository (Dovecot)' + curl -fsSL https://repo.dovecot.org/DOVECOT-REPO-GPG-2.4 | gpg --dearmor > /usr/share/keyrings/upstream-dovecot.gpg + echo \ + "deb [signed-by=/usr/share/keyrings/upstream-dovecot.gpg] https://repo.dovecot.org/ce-2.4-latest/debian/${VERSION_CODENAME} ${VERSION_CODENAME} main" \ + > /etc/apt/sources.list.d/upstream-dovecot.list + + # Refresh package index: apt-get "${QUIET}" update - # Additional community package needed for Lua support if the Dovecot community repository is used. + # This repo instead provides `dovecot-auth-lua` as a transitional package to `dovecot-lua`, + # thus this extra package is required to retain lua support: DOVECOT_PACKAGES+=(dovecot-lua) fi _log 'debug' 'Installing Dovecot' apt-get "${QUIET}" install --no-install-recommends "${DOVECOT_PACKAGES[@]}" - # dependency for fts_xapian + # Runtime dependency for fts_xapian (built via `compile.sh`): apt-get "${QUIET}" install --no-install-recommends libxapian30 } function _install_rspamd() { - _log 'debug' 'Installing Rspamd' - _log 'trace' 'Adding Rspamd PPA' - curl -sSfL https://rspamd.com/apt-stable/gpg.key | gpg --dearmor >/etc/apt/trusted.gpg.d/rspamd.gpg + # NOTE: DMS only supports the rspamd package via using the third-party repo maintained by Rspamd (AMD64 + ARM64): + # Repo: https://rspamd.com/apt-stable/dists/bookworm/main/ + # Docs: https://rspamd.com/downloads.html#debian-and-ubuntu-linux + # NOTE: Debian 12 provides Rspamd 3.4 (too old) and Rspamd discourages it's use + + _log 'trace' 'Adding third-party package repository (Rspamd)' + curl -fsSL https://rspamd.com/apt-stable/gpg.key | gpg --dearmor > /usr/share/keyrings/upstream-rspamd.gpg echo \ - "deb [signed-by=/etc/apt/trusted.gpg.d/rspamd.gpg] http://rspamd.com/apt-stable/ ${VERSION_CODENAME} main" \ - >/etc/apt/sources.list.d/rspamd.list + "deb [signed-by=/usr/share/keyrings/upstream-rspamd.gpg] https://rspamd.com/apt-stable/ ${VERSION_CODENAME} main" \ + > /etc/apt/sources.list.d/upstream-rspamd.list - _log 'trace' 'Updating package index after adding PPAs' + # Refresh package index: apt-get "${QUIET}" update - _log 'trace' 'Installing actual package' + _log 'debug' 'Installing Rspamd' apt-get "${QUIET}" install rspamd redis-server } From f2fedff251a91da3bfc1f780a17b30e1346a53a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Feb 2025 21:46:09 +0100 Subject: [PATCH 469/592] chore(deps): Bump docker/setup-buildx-action from 3.8.0 to 3.9.0 (#4352) --- .github/workflows/generic_build.yml | 2 +- .github/workflows/generic_publish.yml | 2 +- .github/workflows/generic_test.yml | 2 +- .github/workflows/generic_vulnerability-scan.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/generic_build.yml b/.github/workflows/generic_build.yml index 53cb6893001..30d8df46815 100644 --- a/.github/workflows/generic_build.yml +++ b/.github/workflows/generic_build.yml @@ -76,7 +76,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.8.0 + uses: docker/setup-buildx-action@v3.9.0 # NOTE: AMD64 can build within 2 minutes - name: 'Build images' diff --git a/.github/workflows/generic_publish.yml b/.github/workflows/generic_publish.yml index d5966a2bfca..72228d51bb0 100644 --- a/.github/workflows/generic_publish.yml +++ b/.github/workflows/generic_publish.yml @@ -40,7 +40,7 @@ jobs: platforms: arm64 - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.8.0 + uses: docker/setup-buildx-action@v3.9.0 # Try get the cached build layers from a prior `generic_build.yml` job. # NOTE: Until adopting `type=gha` scoped cache exporter (in `docker/build-push-action`), diff --git a/.github/workflows/generic_test.yml b/.github/workflows/generic_test.yml index f28dbc9b7ce..1303c8ada16 100644 --- a/.github/workflows/generic_test.yml +++ b/.github/workflows/generic_test.yml @@ -38,7 +38,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.8.0 + uses: docker/setup-buildx-action@v3.9.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. diff --git a/.github/workflows/generic_vulnerability-scan.yml b/.github/workflows/generic_vulnerability-scan.yml index 3b20b2edb3f..511ade7a4ba 100644 --- a/.github/workflows/generic_vulnerability-scan.yml +++ b/.github/workflows/generic_vulnerability-scan.yml @@ -37,7 +37,7 @@ jobs: # Ensures consistent BuildKit version (not coupled to Docker Engine), # and increased compatibility of the build cache vs mixing buildx drivers. - name: 'Set up Docker Buildx' - uses: docker/setup-buildx-action@v3.8.0 + uses: docker/setup-buildx-action@v3.9.0 # Importing from the cache should create the image within approx 30 seconds: # NOTE: `qemu` step is not needed as we only test for AMD64. From aba92b7bb83929fab296a4c5340133d3f5dadaeb Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sun, 16 Feb 2025 21:53:01 +1300 Subject: [PATCH 470/592] ci: Upgrade `mkdocs-material` to 9.6 (#4368) --- .github/workflows/scripts/docs/build-docs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scripts/docs/build-docs.sh b/.github/workflows/scripts/docs/build-docs.sh index d4384b5e4c5..4eac8c4dd95 100755 --- a/.github/workflows/scripts/docs/build-docs.sh +++ b/.github/workflows/scripts/docs/build-docs.sh @@ -11,7 +11,7 @@ docker run \ --user "$(id -u):$(id -g)" \ --volume "./:/docs" \ --name "build-docs" \ - squidfunk/mkdocs-material:9.5 build --strict + squidfunk/mkdocs-material:9.6 build --strict # Remove unnecessary build artifacts: https://github.com/squidfunk/mkdocs-material/issues/2519 # site/ is the build output folder. From 07e558e4bee524262c8ab18ad04e685b941e782d Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Sun, 16 Feb 2025 22:22:40 +1300 Subject: [PATCH 471/592] docs: Fix broken ref links (#4366) --- docs/content/config/best-practices/dkim_dmarc_spf.md | 2 +- docs/content/config/debugging.md | 2 +- docs/content/contributing/general.md | 11 +++++++++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/content/config/best-practices/dkim_dmarc_spf.md b/docs/content/config/best-practices/dkim_dmarc_spf.md index c04d4b7e1bf..daea09d34ae 100644 --- a/docs/content/config/best-practices/dkim_dmarc_spf.md +++ b/docs/content/config/best-practices/dkim_dmarc_spf.md @@ -54,7 +54,7 @@ You'll need to repeat this process if you add any new domains. You should have: -- At least one [email account setup][docs-accounts-add] +- At least one [email account setup][docs-accounts] - Attached a [volume for config][docs-volumes-config] to persist the generated files to local storage !!! example "Creating DKIM Keys" diff --git a/docs/content/config/debugging.md b/docs/content/config/debugging.md index 30fee3831e7..fca962f16bf 100644 --- a/docs/content/config/debugging.md +++ b/docs/content/config/debugging.md @@ -111,7 +111,7 @@ This could be from outdated software, or running a system that isn't able to pro - **Container runtime:** Docker and Podman for example have subtle differences. DMS docs are primarily focused on Docker, but we try to document known issues where relevant. - **Rootless containers:** Introduces additional differences in behavior or requirements: - cgroup v2 is required for supporting rootless containers. - - Differences such as for container networking which may further affect support for IPv6 and preserving the client IP (Remote address). Example with Docker rootless are [binding a port to a specific interface][docker-rootless-interface] and the choice of [port forwarding driver][docs-rootless-portdriver]. + - Differences such as for container networking which may further affect support for IPv6 and preserving the client IP (Remote address). Example with Docker rootless are [binding a port to a specific interface][docker-rootless-interface] and the choice of [port forwarding driver][docs::fail2ban::rootless-portdriver]. [network::docker-userlandproxy]: https://github.com/moby/moby/issues/44721 [network::docker-nftables]: https://github.com/moby/moby/issues/26824 diff --git a/docs/content/contributing/general.md b/docs/content/contributing/general.md index ea88dc85e19..a540f2576bb 100644 --- a/docs/content/contributing/general.md +++ b/docs/content/contributing/general.md @@ -15,13 +15,20 @@ When refactoring, writing or altering scripts or other files, adhere to these ru Make sure to select `edge` in the dropdown menu at the top. Navigate to the page you would like to edit and click the edit button in the top right. This allows you to make changes and create a pull-request. -Alternatively you can make the changes locally. For that you'll need to have Docker installed. Navigate into the `docs/` directory. Then run: +Alternatively you can make the changes locally. For that you'll need to have Docker installed and run: ```sh -docker run --rm -it -p 8000:8000 -v "${PWD}:/docs" squidfunk/mkdocs-material +# From the root directory of the git clone: +docker run --rm -it -p 8000:8000 -v "./docs:/docs" squidfunk/mkdocs-material ``` This serves the documentation on your local machine on port `8000`. Each change will be hot-reloaded onto the page you view, just edit, save and look at the result. +!!! note + + The container logs will inform you of invalid links detected, but a [few are false-positives][gh-dms::mkdocs-link-error-false-positives] due to our usage of linking to specific [content tabs][mkdocs::content-tabs]. + [get-docker]: https://docs.docker.com/get-docker/ [docs-bats-parallel]: https://bats-core.readthedocs.io/en/v1.8.2/usage.html#parallel-execution +[gh-dms::mkdocs-link-error-false-positives]: https://github.com/docker-mailserver/docker-mailserver/pull/4366 +[mkdocs::content-tabs]: https://squidfunk.github.io/mkdocs-material/reference/content-tabs/#anchor-links From 0ebf820b00fcc7c634229c5df50d5c36cb2a9c8f Mon Sep 17 00:00:00 2001 From: Casper Date: Sun, 16 Feb 2025 10:46:49 +0100 Subject: [PATCH 472/592] Make deletion of mailbox data opt-in (#4365) Co-authored-by: Georg Lauterbach <44545919+georglauterbach@users.noreply.github.com> --- CHANGELOG.md | 1 + target/bin/delmailuser | 26 ++++++++++++++------------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 794c2e64a71..36e51e4193a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ All notable changes to this project will be documented in this file. The format - **DMS v14 mistakenly** relocated the _getmail state directory_ to the _DMS Config Volume_ as a `getmail/` subdirectory. - This has been corrected to `/var/lib/getmail` (_if you have mounted a DMS State Volume to `/var/mail-state`, `/var/lib/getmail` will be symlinked to `/var/mail-state/lib-getmail`_). - To preserve this state when upgrading to DMS v15, **you must manually migrate `getmail/` from the _DMS Config Volume_ to `lib-getmail/` in the _DMS State Volume_.** + - `setup email delete ` now requires explicit confirmation if the mailbox data should be deleted ([#4365](https://github.com/docker-mailserver/docker-mailserver/pull/4365)). ### Added diff --git a/target/bin/delmailuser b/target/bin/delmailuser index c4b68ff3905..9488953e6ec 100755 --- a/target/bin/delmailuser +++ b/target/bin/delmailuser @@ -19,7 +19,11 @@ function _main() { for MAIL_ACCOUNT in "${@}"; do _account_should_already_exist - [[ ${MAILDEL} -eq 1 ]] && _remove_maildir "${MAIL_ACCOUNT}" + if [[ ${MAILDEL} -eq 1 ]]; then + _remove_maildir "${MAIL_ACCOUNT}" + else + _log 'info' "The mailbox data will not be deleted." + fi _manage_virtual_aliases_delete '_' "${MAIL_ACCOUNT}" \ || _exit_with_error "Aliases for '${MAIL_ACCOUNT}' could not be deleted" @@ -31,7 +35,7 @@ function _main() { _manage_accounts_delete "${MAIL_ACCOUNT}" \ || _exit_with_error "'${MAIL_ACCOUNT}' could not be deleted" - _log 'info' "'${MAIL_ACCOUNT}' and associated data deleted" + _log 'info' "'${MAIL_ACCOUNT}' and associated data (aliases, quotas) deleted" done } @@ -43,14 +47,14 @@ ${ORANGE}USAGE${RESET} ${ORANGE}OPTIONS${RESET} -y - Skip prompt by approving to ${LWHITE}delete all mail storage${RESET} for the account(s). + Skip prompt by approving to ${LWHITE}delete all mail data${RESET} for the account(s). ${BLUE}Generic Program Information${RESET} help Print the usage information. ${ORANGE}DESCRIPTION${RESET} Delete a mail account, including associated data (aliases, quotas) and - optionally the mailbox storage for that account. + optionally the mailbox data for that account. ${ORANGE}EXAMPLES${RESET} ${LWHITE}./setup.sh email del user@example.com${RESET} @@ -87,12 +91,10 @@ function _parse_options() { function _maildel_request_if_missing() { if [[ ${MAILDEL} -eq 0 ]]; then local MAILDEL_CHOSEN - read -r -p "Do you want to delete the mailbox as well (removing all mails)? [Y/n] " MAILDEL_CHOSEN + read -r -p "Do you want to delete the mailbox data as well (removing all mails)? [y/N] " MAILDEL_CHOSEN - # TODO: Why would MAILDEL be set to true if MAILDEL_CHOSEN is empty? - if [[ ${MAILDEL_CHOSEN} =~ (y|Y|yes|Yes) ]] || [[ -z ${MAILDEL_CHOSEN} ]]; then - MAILDEL=1 - fi + # Delete mailbox data only if the user provides explicit confirmation. + [[ ${MAILDEL_CHOSEN,,} == "y" ]] && MAILDEL=1 fi } @@ -103,10 +105,10 @@ function _remove_maildir() { local DOMAIN_PART="${MAIL_ACCOUNT#*@}" local MAIL_ACCOUNT_STORAGE_DIR="/var/mail/${DOMAIN_PART}/${LOCAL_PART}" - [[ ! -d ${MAIL_ACCOUNT_STORAGE_DIR} ]] && _exit_with_error "Mailbox directory '${MAIL_ACCOUNT_STORAGE_DIR}' does not exist" + [[ ! -d ${MAIL_ACCOUNT_STORAGE_DIR} ]] && _exit_with_error "Mailbox data directory '${MAIL_ACCOUNT_STORAGE_DIR}' does not exist" - _log 'info' "Deleting Mailbox: '${MAIL_ACCOUNT_STORAGE_DIR}'" - rm -R "${MAIL_ACCOUNT_STORAGE_DIR}" || _exit_with_error 'Mailbox could not be deleted' + _log 'info' "Deleting mailbox data: '${MAIL_ACCOUNT_STORAGE_DIR}'" + rm -R "${MAIL_ACCOUNT_STORAGE_DIR}" || _exit_with_error 'Mailbox data could not be deleted' # Remove parent directory too if it's empty: rmdir "/var/mail/${DOMAIN_PART}" &>/dev/null } From bcee78e2c1c310c4815b3e03c5e9278ad8ef1343 Mon Sep 17 00:00:00 2001 From: Brennan Kinney <5098581+polarathene@users.noreply.github.com> Date: Mon, 17 Feb 2025 11:17:29 +1300 Subject: [PATCH 473/592] docs: Revise Rspamd page (#4360) --- docs/content/config/security/rspamd.md | 265 +++++++++++++++---------- 1 file changed, 163 insertions(+), 102 deletions(-) diff --git a/docs/content/config/security/rspamd.md b/docs/content/config/security/rspamd.md index c49baea6648..45f94c656e4 100644 --- a/docs/content/config/security/rspamd.md +++ b/docs/content/config/security/rspamd.md @@ -6,34 +6,60 @@ title: 'Security | Rspamd' Rspamd is a ["fast, free and open-source spam filtering system"][rspamd-web]. DMS integrates Rspamd like any other service. We provide a basic but easy to maintain setup of Rspamd. -If you want to take a look at the default configuration files for Rspamd that DMS packs, navigate to [`target/rspamd/` inside the repository][dms-repo::default-rspamd-configuration]. Please consult the [section "The Default Configuration"](#the-default-configuration) section down below for a written overview. - -## Related Environment Variables - -The following environment variables are related to Rspamd: - -1. [`ENABLE_RSPAMD`](../environment.md#enable_rspamd) -2. [`ENABLE_RSPAMD_REDIS`](../environment.md#enable_rspamd_redis) -3. [`RSPAMD_CHECK_AUTHENTICATED`](../environment.md#rspamd_check_authenticated) -4. [`RSPAMD_GREYLISTING`](../environment.md#rspamd_greylisting) -5. [`RSPAMD_HFILTER`](../environment.md#rspamd_hfilter) -6. [`RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE`](../environment.md#rspamd_hfilter_hostname_unknown_score) -7. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) -8. [`SPAM_SUBJECT`](../environment.md#spam_subject) -9. [`MOVE_SPAM_TO_JUNK`][docs::spam-to-junk] -10. [`MARK_SPAM_AS_READ`](../environment.md#mark_spam_as_read) - -With these variables, you can enable Rspamd itself, and you can enable / disable certain features related to Rspamd. +If you want to take a look at the default configuration files for Rspamd that DMS adds, navigate to [`target/rspamd/` inside the repository][dms-repo::default-rspamd-configuration]. Please consult the [section "The Default Configuration"](#the-default-configuration) section down below for a written overview. + +### Enable Rspamd + +Rspamd is presently opt-in for DMS, but intended to become the default anti-spam service in a future release. + +DMS offers two anti-spam solutions: + +- Legacy (_Amavis, SpamAssassin, OpenDKIM, OpenDMARC_) +- Rspamd (_Provides equivalent features of software from the legacy solution_) + +While you could configure Rspamd to only replace some of the legacy services, it is advised to only use Rspamd with the legacy services disabled. + +!!! example "Switch to Rspamd" + + To use Rspamd add the following ENV config changes: + + ```env + ENABLE_RSPAMD=1 + + # Rspamd replaces the functionality of all these anti-spam services, disable them: + ENABLE_OPENDKIM=0 + ENABLE_OPENDMARC=0 + ENABLE_POLICYD_SPF=0 + ENABLE_AMAVIS=0 + ENABLE_SPAMASSASSIN=0 + # Greylisting is opt-in, if you had enabled Postgrey switch to the Rspamd equivalent: + ENABLE_POSTGREY=0 + RSPAMD_GREYLISTING=1 + + # Optional: Add anti-virus support with ClamAV (compatible with Rspamd): + ENABLE_CLAMAV=1 + ``` -## The Default Configuration +!!! info "Relevant Environment Variables" -### Other Anti-Spam-Services + The following environment variables are related to Rspamd: + + 1. [`ENABLE_RSPAMD`](../environment.md#enable_rspamd) + 2. [`ENABLE_RSPAMD_REDIS`](../environment.md#enable_rspamd_redis) + 3. [`RSPAMD_CHECK_AUTHENTICATED`](../environment.md#rspamd_check_authenticated) + 4. [`RSPAMD_GREYLISTING`](../environment.md#rspamd_greylisting) + 5. [`RSPAMD_HFILTER`](../environment.md#rspamd_hfilter) + 6. [`RSPAMD_HFILTER_HOSTNAME_UNKNOWN_SCORE`](../environment.md#rspamd_hfilter_hostname_unknown_score) + 7. [`RSPAMD_LEARN`](../environment.md#rspamd_learn) + 8. [`SPAM_SUBJECT`](../environment.md#spam_subject) + 9. [`MOVE_SPAM_TO_JUNK`][docs::spam-to-junk] + 10. [`MARK_SPAM_AS_READ`](../environment.md#mark_spam_as_read) -DMS packs other anti-spam services, like SpamAssassin or Amavis, next to Rspamd. There exist services, like ClamAV (`ENABLE_CLAMAV`), that Rspamd can utilize to improve the scanning. Except for ClamAV, we recommend disabling **all other** anti-spam services when using Rspamd. The [basic configuration shown below](#a-very-basic-configuration) provides a good starting point. +## Overview of Rspamd support ### Mode of Operation -!!! tip "Attention" +!!! note "Attention" Read this section carefully if you want to understand how Rspamd is integrated into DMS and how it works (on a surface level). @@ -79,13 +105,16 @@ DMS does not set a default password for the controller worker. You may want to d When Rspamd is enabled, we implicitly also start an instance of Redis in the container: - Redis is configured to persist its data via RDB snapshots to disk in the directory `/var/lib/redis` (_or the [`/var/mail-state/`][docs::dms-volumes-state] volume when present_). -- With the volume mount, the snapshot will restore the Redis data across container restarts, and provide a way to keep backup. +- With the volume mount, the snapshot will restore the Redis data across container updates, and provide a way to keep a backup. +- Without a volume mount a containers internal state will persist across restarts until the container is recreated due to changes like ENV or upgrading the image for the container. Redis uses `/etc/redis/redis.conf` for configuration: - We adjust this file when enabling the internal Redis service. - If you have an external instance of Redis to use, the internal Redis service can be opt-out via setting the ENV [`ENABLE_RSPAMD_REDIS=0`][docs::env::enable-redis] (_link also details required changes to the DMS Rspamd config_). +If you are interested in using Valkey instead of Redis, please refer to [this guidance][gh-dms::guide::valkey]. + ### Web Interface Rspamd provides a [web interface][rspamd-docs::web-ui], which contains statistics and data Rspamd collects. The interface is enabled by default and reachable on port 11334. @@ -96,7 +125,7 @@ To use the web interface you will need to configure a password, [otherwise you w ??? example "Set a custom password" - Add this line to [your rspamd `custom-commands.conf` config](#with-the-help-of-a-custom-file) which sets the `password` option of the _controller worker_: + Add this line to [your Rspamd `custom-commands.conf` config](#with-the-help-of-a-custom-file) which sets the `password` option of the _controller worker_: ``` set-option-for-controller password "your hashed password here" @@ -108,9 +137,13 @@ To use the web interface you will need to configure a password, [otherwise you w docker exec -it rspamadm pw ``` + --- + + **Related:** A minimal Rspamd `compose.yaml` [example with a reverse-proxy for web access][gh-dms:guide::rspamd-web]. + ### DNS -DMS does not supply custom values for DNS servers (to Rspamd). If you need to use custom DNS servers, which could be required when using [DNS-based deny/allowlists](#rbls-real-time-blacklists-dnsbls-dns-based-blacklists), you need to adjust [`options.inc`][rspamd-docs::basic-options] yourself. Make sure to also read our [FAQ page on DNS servers][docs::faq::dns-servers]. +DMS does not supply custom values for DNS servers (to Rspamd). If you need to use custom DNS servers, which could be required when using [DNS-based deny/allowlists](#rbls-real-time-blacklists-dnsbls-dns-based-blacklists), you need to adjust [`options.inc`][rspamd-docs::config::global] yourself. Make sure to also read our [FAQ page on DNS servers][docs::faq::dns-servers]. !!! warning @@ -142,7 +175,7 @@ You can choose to enable ClamAV, and Rspamd will then use it to check for viruse #### RBLs (Real-time Blacklists) / DNSBLs (DNS-based Blacklists) -The [RBL module][rspamd-docs::modules::rbl] is enabled by default. As a consequence, Rspamd will perform DNS lookups to various blacklists. Whether an RBL or a DNSBL is queried depends on where the domain name was obtained: RBL servers are queried with IP addresses extracted from message headers, DNSBL server are queried with domains and IP addresses extracted from the message body \[[source][www::rbl-vs-dnsbl]\]. +The [RBL module][rspamd-docs::modules::rbl] is enabled by default. As a consequence, Rspamd will perform DNS lookups to various blacklists. Whether an RBL or a DNSBL is queried depends on where the domain name was obtained: RBL servers are queried with IP addresses extracted from message headers, DNSBL server are queried with domains and IP addresses extracted from the message body ([source][www::rbl-vs-dnsbl]). !!! danger "Rspamd and DNS Block Lists" @@ -152,124 +185,141 @@ The [RBL module][rspamd-docs::modules::rbl] is enabled by default. As a conseque ## Providing Custom Settings & Overriding Settings -DMS brings sane default settings for Rspamd. They are located at `/etc/rspamd/local.d/` inside the container (or `target/rspamd/local.d/` in the repository). - -### Manually - -!!! question "What is [`docker-data/dms/config/`][docs::dms-volumes-config]?" +!!! info "Rspamd config overriding precedence" -If you want to overwrite the default settings or provide your settings, you can place files at `docker-data/dms/config/rspamd/override.d/`. Files from this directory are copied to `/etc/rspamd/override.d/` during startup. These files [forcibly override][rspamd-docs::override-dir] Rspamd and DMS default settings. + Rspamd has a layered approach for configuration with [`local.d` and `override.d` config directories][rspamd-docs::config-directories]. -!!! question "What is the [`local.d` directory and how does it compare to `override.d`][rspamd-docs::config-directories]?" + - DMS [extends the Rspamd default configs via `/etc/rspamd/local.d/`][dms-repo::default-rspamd-configuration]. + - User config changes should be handled separately as overrides via the [DMS Config Volume][docs::dms-volumes-config] (`docker-data/dms/config/`) with either: + - `./rspamd/override.d/` - Config files placed here are copied to `/etc/rspamd/override.d/` during container startup. + - [`./rspamd/custom-commands.conf`](#with-the-help-of-a-custom-file) - Applied after copying any provided configs from `rspamd/override.d/` (DMS Config volume) to `/etc/rspamd/override.d/`. -!!! warning "Clashing Overrides" +!!! abstract "Reference docs for Rspamd config" - Note that when also [using the `custom-commands.conf` file](#with-the-help-of-a-custom-file), files in `override.d` may be overwritten in case you adjust them manually and with the help of the file. + - [Config Overview][rspamd-docs::config::overview], [Quickstart guide][rspamd-docs::config::quickstart], and [Config Syntax (UCL)][rspamd-docs::config::ucl-syntax] + - Global Options ([`options.inc`][rspamd-docs::config::global]) + - [Workers][rspamd-docs::config::workers] ([`worker-controller.inc`][rspamd-docs::config::worker-controller], [`worker-proxy.inc`][rspamd-docs::config::worker-proxy]) + - [Modules][rspamd-docs::modules] (_view each module page for their specific config options_) -### With the Help of a Custom File +!!! tip "View rendered config" -DMS provides the ability to do simple adjustments to Rspamd modules with the help of a single file. Just place a file called `custom-commands.conf` into `docker-data/dms/config/rspamd/`. If this file is present, DMS will evaluate it. The structure is simple, as each line in the file looks like this: + `rspamadm configdump` will output the full rspamd configuration that is used should you need it for troubleshooting / inspection. -```txt -COMMAND ARGUMENT1 ARGUMENT2 ARGUMENT3 -``` + - You can also see which modules are enabled / disabled via `rspamadm configdump --modules-state` + - Specific config sections like `dkim` or `worker` can also be used to filter the output to just those sections: `rspamadm configdump dkim worker` + - Use `--show-help` to include inline documentation for many settings. -where `COMMAND` can be: +### Using `custom-commands.conf` { #with-the-help-of-a-custom-file } -1. `disable-module`: disables the module with name `ARGUMENT1` -2. `enable-module`: explicitly enables the module with name `ARGUMENT1` -3. `set-option-for-module`: sets the value for option `ARGUMENT2` to `ARGUMENT3` inside module `ARGUMENT1` -4. `set-option-for-controller`: set the value of option `ARGUMENT1` to `ARGUMENT2` for the controller worker -5. `set-option-for-proxy`: set the value of option `ARGUMENT1` to `ARGUMENT2` for the proxy worker -6. `set-common-option`: set the option `ARGUMENT1` that [defines basic Rspamd behavior][rspamd-docs::basic-options] to value `ARGUMENT2` -7. `add-line`: this will add the complete line after `ARGUMENT1` (with all characters) to the file `/etc/rspamd/override.d/` +For convenience DMS provides a single config file that will directly create or modify multiple configs at `/etc/rspamd/override.d/`. This is handled as the final rspamd configuration step during container startup. -!!! example "An Example Is [Shown Down Below](#adjusting-and-extending-the-very-basic-configuration)" +DMS will apply this config when you provide `rspamd/custom-commands.conf` in your DMS Config volume. Configure it with directive lines as documented below. -!!! note "File Names & Extensions" +!!! note "Only use this feature for `option = value` changes" - For command 1 - 3, we append the `.conf` suffix to the module name to get the correct file name automatically. For commands 4 - 6, the file name is fixed (you don't even need to provide it). For command 7, you will need to provide the whole file name (including the suffix) yourself! + `custom-commands.conf` is only suitable for adding or replacing simple `option = value` settings for configs at `/etc/rspamd/override.d/`. + + - New settings are appended to the associated config file. + - When replacing an existing setting in an override config, that setting may be any matching line (_allowing for nested scopes, instead of only top-level keys_). + + Any changes involving more advanced [UCL config syntax][rspamd-docs::config::ucl-syntax] should instead add UCL config files directly to `rspamd/override.d/` (_in the DMS Config volume_). -You can also have comments (the line starts with `#`) and blank lines in `custom-commands.conf` - they are properly handled and not evaluated. +!!! info "`custom-commands.conf` syntax" -!!! tip "Adjusting Modules This Way" + There are 7 directives available to manage custom Rspamd configurations. Add these directive lines into `custom-commands.conf`, they will be processed sequentially. - These simple commands are meant to give users the ability to _easily_ alter modules and their options. As a consequence, they are not powerful enough to enable multi-line adjustments. If you need to do something more complex, we advise to do that [manually](#manually)! + **Directives:** -## Examples & Advanced Configuration + ```txt + # For /etc/rspamd/override.d/{options.inc,worker-controller.inc,worker-proxy}.inc + set-common-option