diff --git a/Dockerfile b/Dockerfile index 87000b3..0289cb4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y -q --no-install-recommends \ postfix postfix-pgsql postfix-mysql postfix-ldap postfix-pcre libsasl2-modules \ dovecot-core dovecot-imapd dovecot-lmtpd dovecot-pgsql dovecot-mysql dovecot-ldap dovecot-sieve dovecot-managesieved dovecot-pop3d \ fetchmail libdbi-perl libdbd-pg-perl libdbd-mysql-perl liblockfile-simple-perl \ - clamav clamav-daemon \ + clamav clamav-daemon clamdscan \ python3-pip python3-setuptools python3-wheel python3-gpg python3-watchdog \ rsyslog dnsutils curl unbound unbound-anchor jq rsync \ inotify-tools \ diff --git a/Makefile b/Makefile index 93a121c..e58ceac 100644 --- a/Makefile +++ b/Makefile @@ -41,8 +41,11 @@ build-no-cache: build: docker build -t $(NAME) . +# Test containers are removed with -v, so the anonymous volumes of the +# database images (mysql, postgres, redis) go with them instead of +# piling up on every run. init_openldap: - -docker rm -f \ + -docker rm -fv \ openldap || true docker run \ -d \ @@ -58,7 +61,7 @@ init_openldap: || { echo "TIMEOUT: openldap did not become ready"; docker logs --tail 50 openldap; exit 1; } init_redis: - -docker rm -f \ + -docker rm -fv \ redis || true docker run \ -d \ @@ -69,7 +72,7 @@ init_redis: || { echo "TIMEOUT: redis did not become ready"; docker logs --tail 50 redis; exit 1; } init_mariadb: - -docker rm -f \ + -docker rm -fv \ mariadb || true docker run \ -d \ @@ -86,7 +89,7 @@ init_mariadb: || { echo "TIMEOUT: mariadb did not become ready"; docker logs --tail 50 mariadb; exit 1; } init_postgres: - -docker rm -f \ + -docker rm -fv \ postgres || true docker run \ -d \ @@ -101,7 +104,7 @@ init_postgres: || { echo "TIMEOUT: postgres did not become ready"; docker logs --tail 50 postgres; exit 1; } init_ldap: init_openldap init_redis - -docker rm -f \ + -docker rm -fv \ mailserver_ldap || true docker run \ -d \ @@ -179,11 +182,11 @@ fixtures_ldap: run_ldap: ./test/bats/bin/bats test/ldap.bats stop_ldap: - -docker rm -f \ + -docker rm -fv \ mailserver_ldap || true init_ldap2: init_openldap init_redis - -docker rm -f \ + -docker rm -fv \ mailserver_ldap2 || true docker run \ -d \ @@ -257,11 +260,11 @@ fixtures_ldap2: run_ldap2: ./test/bats/bin/bats test/ldap2.bats stop_ldap2: - -docker rm -f \ + -docker rm -fv \ mailserver_ldap2 || true init_default: init_redis init_mariadb - -docker rm -f \ + -docker rm -fv \ mailserver_default || true docker run \ @@ -286,7 +289,7 @@ init_default: init_redis init_mariadb -t $(NAME) init_reverse: init_redis init_postgres - -docker rm -f \ + -docker rm -fv \ mailserver_reverse || true docker run \ -d \ @@ -356,11 +359,11 @@ fixtures_reverse: run_reverse: ./test/bats/bin/bats test/reverse.bats stop_reverse: - -docker rm -f \ + -docker rm -fv \ mailserver_reverse || true init_ecdsa: init_redis init_mariadb - -docker rm -f \ + -docker rm -fv \ mailserver_ecdsa || true docker run \ -d \ @@ -383,11 +386,11 @@ run_ecdsa: docker exec mailserver_ecdsa /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 587 ; do sleep 1 ; done" ./test/bats/bin/bats test/ecdsa.bats stop_ecdsa: - -docker rm -f \ + -docker rm -fv \ mailserver_ecdsa || true init_traefik_acmev1: init_redis init_mariadb - -docker rm -f \ + -docker rm -fv \ mailserver_traefik_acmev1 || true docker run \ -d \ @@ -421,11 +424,11 @@ run_traefik_acmev1: docker exec mailserver_traefik_acmev1 /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 587 ; do sleep 1 ; done" ./test/bats/bin/bats test/traefik_acmev1.bats stop_traefik_acmev1: - -docker rm -f \ + -docker rm -fv \ mailserver_traefik_acmev1 || true init_traefik_acmev2: init_redis init_mariadb - -docker rm -f \ + -docker rm -fv \ mailserver_traefik_acmev2 || true docker run \ -d \ @@ -459,14 +462,14 @@ run_traefik_acmev2: docker exec mailserver_traefik_acmev2 /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 587 ; do sleep 1 ; done" ./test/bats/bin/bats test/traefik_acmev2.bats stop_traefik_acmev2: - -docker rm -f \ + -docker rm -fv \ mailserver_traefik_acmev2 || true # acme.json as written by Traefik 2 and 3 (they share the format), generated # with Traefik 3.6 against a local ACME test server. It holds two certificate # resolvers, the case that used to leave the mailserver without a certificate. init_traefik_v3: init_redis init_mariadb - -docker rm -f \ + -docker rm -fv \ mailserver_traefik_v3 || true docker run \ -d \ @@ -499,13 +502,14 @@ run_traefik_v3: docker exec mailserver_traefik_v3 /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 587 ; do sleep 1 ; done" ./test/bats/bin/bats test/traefik_v3.bats stop_traefik_v3: - -docker rm -f \ + -docker rm -fv \ mailserver_traefik_v3 || true fixtures_default: - # Wait for clamav unofficial sigs database update (default) - docker exec mailserver_default /bin/sh -c "while [ -f /var/lib/clamav-unofficial-sigs/pid/clamav-unofficial-sigs.pid ] ; do sleep 1 ; done" + # Wait for the clamav unofficial sigs update that runs at startup (default). + # Its pid file only appears once the update starts, so wait for the log line. + docker exec mailserver_default /bin/sh -c "until grep -q 'clamav unofficial signature update done' /var/log/mail.log 2>/dev/null ; do sleep 1 ; done" # Wait for clamav load databases (default) docker exec mailserver_default /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 3310 ; do sleep 1 ; done" # Wait for rspamd to start (default) @@ -549,11 +553,11 @@ fixtures_default: run_default: ./test/bats/bin/bats test/default.bats stop_default: - -docker rm -f \ + -docker rm -fv \ mailserver_default || true init_sieve: init_redis init_mariadb - -docker rm -f \ + -docker rm -fv \ mailserver_sieve || true docker run \ -d \ @@ -611,7 +615,7 @@ run_sieve: ./test/bats/bin/bats test/sieve.bats stop_sieve: - -docker rm -f \ + -docker rm -fv \ mailserver_sieve || true clean: diff --git a/README.md b/README.md index 886703a..1136bda 100644 --- a/README.md +++ b/README.md @@ -691,7 +691,7 @@ interserver_enabled="yes" user_configuration_complete="yes" ``` -If the startup script detects this file, clamav-unofficial-sigs is automatically enabled and third-party databases downloaded under `/mnt/docker/mail/clamav` after Clamav startup. Once the databases are downloaded, a SIGUSR2 signal is sent to Clamav to reload the signature databases: +If the startup script detects this file, clamav-unofficial-sigs is automatically enabled and third-party databases downloaded under `/mnt/docker/mail/clamav` after Clamav startup. Once the databases are downloaded and Clamav has finished starting, Clamav is asked to reload the signature databases through its socket (`clamdscan --reload`): ``` docker logs -f mailserver @@ -706,6 +706,8 @@ clamd[xxxxxx]: Reading databases from /var/lib/clamav clamd[xxxxxx]: Database correctly reloaded (6812263 signatures) ``` +After that the update runs every hour from cron, as the `clamav` user. Its errors and alerts go to the container log, tagged `clamav-unofficial-sigs`: errors also land in `/var/log/mail.err`, alerts (such as a newer clamav-unofficial-sigs version being available) in `/var/log/mail.warn`. The full record of each run is in `/var/log/clamav-unofficial-sigs/clamav-unofficial-sigs.log`. +

Back to table of contents :arrow_up_small:

### Unbound DNS resolver diff --git a/rootfs/etc/clamav-unofficial-sigs/os.conf b/rootfs/etc/clamav-unofficial-sigs/os.conf index 4fc70c9..cc24421 100644 --- a/rootfs/etc/clamav-unofficial-sigs/os.conf +++ b/rootfs/etc/clamav-unofficial-sigs/os.conf @@ -29,11 +29,16 @@ clam_dbs="/var/lib/clamav" clamd_pid="/run/clamav/clamd.pid" -clamd_reload_opt="s6-svc -2 /services/clamd" +# Through clamd's socket, which works for clamav, the user the update runs as, +# once clamd has finished starting +clamd_reload_opt="/usr/local/bin/clamav-unofficial-sigs-reload.sh" # this is in case the needed/expected keyword is now restart and not reload clamd_restart_opt="s6-svc -2 /services/clamd" # this is enabled here https://github.com/mailserver2/mailserver/blob/master/rootfs/etc/clamav/unofficial-sigs/os.conf#L6 clamd_socket="/run/clamav/clamd.ctl" +# cron runs the update through this wrapper, which sends its output to syslog +cron_script_full_path="/usr/local/bin/clamav-unofficial-sigs-cron.sh" + # https://eXtremeSHOK.com ###################################################### diff --git a/rootfs/services/_parent/run b/rootfs/services/_parent/run index b7e5d51..565a462 100644 --- a/rootfs/services/_parent/run +++ b/rootfs/services/_parent/run @@ -26,7 +26,7 @@ if [ "$DISABLE_CLAMAV" = false ]; then s6-svc -u /services/freshclam if [ -f "/var/mail/clamav-unofficial-sigs/user.conf" ]; then logger -p mail.info "s6-supervise : clamav unofficial signature update running" - clamav-unofficial-sigs.sh &>/dev/null + s6-setuidgid clamav clamav-unofficial-sigs.sh &>/dev/null logger -p mail.info "s6-supervise : clamav unofficial signature update done" fi s6-svwait -u /services/clamd diff --git a/rootfs/usr/local/bin/clamav-unofficial-sigs-cron.sh b/rootfs/usr/local/bin/clamav-unofficial-sigs-cron.sh new file mode 100644 index 0000000..6148495 --- /dev/null +++ b/rootfs/usr/local/bin/clamav-unofficial-sigs-cron.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# The hourly clamav-unofficial-sigs update, as cron runs it (see +# cron_script_full_path in /etc/clamav-unofficial-sigs/os.conf). +# +# cron mails whatever a job prints to the job's user, clamav@, which +# nobody reads, and the mail is scanned by rspamd on the way. The output goes +# to syslog instead, with the rest of the container's logs: shell errors +# (stderr) at mail.err, the script's own ERROR and ALERT lines (stdout) at +# mail.warning. + +pid_file=/var/lib/clamav-unofficial-sigs/pid/clamav-unofficial-sigs.pid + +# The update at startup and this hourly one are independent, so finding the +# other one running is expected rather than an error. +if [ -f "$pid_file" ] && ps -p "$(cat "$pid_file")" > /dev/null 2>&1; then + logger -t clamav-unofficial-sigs -p mail.info "update already running, skipping this run" + exit 0 +fi + +# The script frames its messages with lines of = * - or +, which are dropped. +{ /usr/local/bin/clamav-unofficial-sigs.sh 2>&1 1>&3 3>&- \ + | grep -v '^[-=*+]*$' | logger -t clamav-unofficial-sigs -p mail.err; } 3>&1 \ + | grep -v '^[-=*+]*$' | logger -t clamav-unofficial-sigs -p mail.warning diff --git a/rootfs/usr/local/bin/clamav-unofficial-sigs-reload.sh b/rootfs/usr/local/bin/clamav-unofficial-sigs-reload.sh new file mode 100644 index 0000000..9e81033 --- /dev/null +++ b/rootfs/usr/local/bin/clamav-unofficial-sigs-reload.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# How clamav-unofficial-sigs asks clamd to reload its databases (see +# clamd_reload_opt in /etc/clamav-unofficial-sigs/os.conf). +# +# Whenever clamd is starting, at container start or after it restarts, it has +# no socket and ignores SIGUSR2, the updater's fallback, until it has loaded +# its databases, so a reload sent then is lost. The update at startup runs +# while clamd starts, so it meets this regularly; an hourly one can too. So +# every reload first waits for clamd to answer. Each failed ping prints an +# ERROR, which the updater would take for a failed reload, so the wait is +# silent: if clamd never answers, the reload's own error is what the updater +# sees. +# +# A reload sent while clamd is already reloading does not wait: clamd answers +# then, and merges it into the reload in progress. Anything that reload missed +# is picked up by clamd's SelfCheck. + +clamdscan --ping 300:1 > /dev/null 2>&1 +exec clamdscan --reload diff --git a/rootfs/usr/local/bin/setup.sh b/rootfs/usr/local/bin/setup.sh index 566d7ba..6afd80a 100644 --- a/rootfs/usr/local/bin/setup.sh +++ b/rootfs/usr/local/bin/setup.sh @@ -799,12 +799,19 @@ chown -R clamav:clamav /var/run/clamav /var/mail/clamav /var/log/clamav # CLAMAV-UNOFFICIAL-SIGS # --------------------------------------------------------------------------------------------- -if [ -f "/var/mail/clamav-unofficial-sigs/user.conf" ]; then +if [ "$DISABLE_CLAMAV" = true ]; then + echo "[INFO] clamav-unofficial-sigs is disabled (ClamAV is disabled)" +elif [ -f "/var/mail/clamav-unofficial-sigs/user.conf" ]; then echo "[INFO] clamav-unofficial-sigs is enabled (user configuration found)" rm -rf /var/lib/clamav-unofficial-sigs ln -s /var/mail/clamav-unofficial-sigs /var/lib/clamav-unofficial-sigs cp -f /var/mail/clamav-unofficial-sigs/user.conf /etc/clamav-unofficial-sigs mkdir -p /var/log/clamav-unofficial-sigs + # The update runs as clamav, at startup and from cron, and has to be able to + # write all of its state, including what a volume holds from runs as root. + # user.conf is the administrator's file and keeps its owner. + find /var/mail/clamav-unofficial-sigs ! -name user.conf -exec chown clamav:clamav {} + + chown -R clamav:clamav /var/log/clamav-unofficial-sigs clamav-unofficial-sigs.sh --install-cron &>/dev/null clamav-unofficial-sigs.sh --install-logrotate &>/dev/null else diff --git a/test/default.bats b/test/default.bats index 118fa49..ef15746 100644 --- a/test/default.bats +++ b/test/default.bats @@ -384,25 +384,31 @@ load 'test_helper/bats-assert/load' # EICAR are rejected; the rest are "no action", including both tiny.quota # messages, which are refused at LMTP after rspamd has passed them. Not counted: # postfix's own bounces (not miltered), quota warnings (dovecot-lda, never enter -# postfix) and the IMAP learn calls (counted as learned). +# postfix) and the IMAP learn calls (counted as learned). Nothing else in the +# container sends mail: the clamav-unofficial-sigs cron job logs to syslog +# instead (tested below). @test "checking rspamd: 9 messages scanned" { - run docker exec mailserver_default /bin/sh -c "rspamc stat | grep -i 'Messages scanned: 9'" + run docker exec mailserver_default rspamc stat assert_success + assert_output --partial 'Messages scanned: 9' } @test "checking rspamd: 7 messages with action no action" { - run docker exec mailserver_default /bin/sh -c "rspamc stat | grep -i 'Messages with action no action: 7'" + run docker exec mailserver_default rspamc stat assert_success + assert_output --partial 'Messages with action no action: 7' } @test "checking rspamd: 2 messages with action reject" { - run docker exec mailserver_default /bin/sh -c "rspamc stat | grep -i 'Messages with action reject: 2'" + run docker exec mailserver_default rspamc stat assert_success + assert_output --partial 'Messages with action reject: 2' } @test "checking rspamd: 2 messages learned" { - run docker exec mailserver_default /bin/sh -c "rspamc stat | grep -i 'Messages learned: 2'" + run docker exec mailserver_default rspamc stat assert_success + assert_output --partial 'Messages learned: 2' } @test "checking rspamd: 1 address whitelisted in default configuration" { @@ -729,6 +735,64 @@ load 'test_helper/bats-assert/load' assert_success } +# The update as cron runs it: the user and command from its cron file, with +# no terminal. Without a terminal the script first sleeps for up to ten +# minutes (enable_random), so that is switched off. +sigs_cron_job() { + docker exec mailserver_default /bin/sh -c "grep -q '^enable_random=\"no\"' /etc/clamav-unofficial-sigs/user.conf || echo 'enable_random=\"no\"' >> /etc/clamav-unofficial-sigs/user.conf" + docker exec mailserver_default /bin/sh -c "grep -v '^#' /etc/cron.d/clamav-unofficial-sigs | grep . | { read -r m h dom mon dow user cmd; su -s /bin/sh \"\$user\" -c \"\$cmd\"; }" +} + +# cron mails anything the job prints to clamav@, which nobody reads and +# which rspamd scans on the way (throwing off its counters above), so the job +# must print nothing. +@test "checking clamav-unofficial-sigs: cron job runs cleanly as its cron user" { + run sigs_cron_job + assert_success + assert_output '' + run docker exec mailserver_default grep clamav-unofficial-sigs /var/log/mail.err + assert_failure +} + +# After an update the job reloads clamd with clamd_reload_opt, as its cron user. +# clamdscan --reload prints nothing, so clamd's own log is the evidence. +@test "checking clamav-unofficial-sigs: cron user can reload clamd" { + reloads() { docker exec mailserver_default grep -c 'Database correctly reloaded' /var/log/clamav/clamav.log || true; } + before=$(reloads) + run docker exec mailserver_default /bin/sh -c ". /etc/clamav-unofficial-sigs/os.conf; user=\$(grep -v '^#' /etc/cron.d/clamav-unofficial-sigs | grep . | awk '{print \$6}'); su -s /bin/sh \"\$user\" -c \"\$clamd_reload_opt\"" + assert_success + for i in $(seq 60); do [ "$(reloads)" -gt "$before" ] && break; sleep 1; done + [ "$(reloads)" -gt "$before" ] +} + +# While clamd is starting, at container start or after it restarts, it has no +# socket and ignores SIGUSR2 until it has loaded its databases, so a reload +# sent then is lost. The updater treats any ERROR in the reload's output as a +# failure. +@test "checking clamav-unofficial-sigs: cron user's reload waits for clamd to start" { + reloads() { docker exec mailserver_default grep -c 'Database correctly reloaded' /var/log/clamav/clamav.log || true; } + before=$(reloads) + docker exec mailserver_default /bin/sh -c "s6-svc -d /services/clamd; while pgrep -x clamd > /dev/null; do sleep 0.2; done; s6-svc -u /services/clamd" + run docker exec mailserver_default /bin/sh -c ". /etc/clamav-unofficial-sigs/os.conf; user=\$(grep -v '^#' /etc/cron.d/clamav-unofficial-sigs | grep . | awk '{print \$6}'); su -s /bin/sh \"\$user\" -c \"\$clamd_reload_opt\"" + assert_success + refute_output --partial 'ERROR' + for i in $(seq 120); do [ "$(reloads)" -gt "$before" ] && break; sleep 1; done + [ "$(reloads)" -gt "$before" ] +} + +# The update at startup and the hourly one are independent, so the cron job +# finding another update running is expected, not an error. PID 1 stands in +# for the running update. +@test "checking clamav-unofficial-sigs: cron job is quiet while another update runs" { + docker exec mailserver_default /bin/sh -c "echo 1 > /var/lib/clamav-unofficial-sigs/pid/clamav-unofficial-sigs.pid" + run sigs_cron_job + docker exec mailserver_default rm -f /var/lib/clamav-unofficial-sigs/pid/clamav-unofficial-sigs.pid + assert_success + assert_output '' + run docker exec mailserver_default grep 'clamav-unofficial-sigs.*ERROR' /var/log/mail.log + assert_failure +} + # @test "checking clamav-unofficial-sigs: TEST 1 — Html.Sanesecurity.TestSig_Type3_Bdy" { # run docker exec mailserver_default /bin/sh -c "clamscan --database=/var/lib/clamav/phish.ndb - < /tmp/tests/clamav/test1.eml" # assert_failure @@ -901,3 +965,27 @@ load 'test_helper/bats-assert/load' run docker exec mailserver_default grep -h "fd limit" /var/log/mail.warn /var/log/mail.log assert_failure } + +# Last on purpose: these make the clamav-unofficial-sigs cron job fail, which +# writes to mail.log, mail.warn and mail.err, all checked clean above. +@test "checking clamav-unofficial-sigs: cron job stderr goes to mail.err" { + docker exec mailserver_default chown root /var/lib/clamav-unofficial-sigs/configs/current-dbs.txt + run sigs_cron_job + docker exec mailserver_default chown clamav /var/lib/clamav-unofficial-sigs/configs/current-dbs.txt + assert_success + assert_output '' + run docker exec mailserver_default grep 'clamav-unofficial-sigs.*current-dbs.txt: Permission denied' /var/log/mail.err + assert_success +} + +@test "checking clamav-unofficial-sigs: cron job stdout goes to mail.warning" { + docker exec mailserver_default chown root /var/log/clamav-unofficial-sigs/clamav-unofficial-sigs.log + run sigs_cron_job + docker exec mailserver_default chown clamav /var/log/clamav-unofficial-sigs/clamav-unofficial-sigs.log + assert_success + assert_output '' + run docker exec mailserver_default grep 'clamav-unofficial-sigs.*Logging Disabled' /var/log/mail.warn + assert_success + run docker exec mailserver_default grep 'clamav-unofficial-sigs.*Logging Disabled' /var/log/mail.err + assert_failure +} diff --git a/test/ldap2.bats b/test/ldap2.bats index eb26d38..0dbd26a 100644 --- a/test/ldap2.bats +++ b/test/ldap2.bats @@ -202,6 +202,17 @@ load 'test_helper/bats-assert/load' assert_success } +# +# clamav-unofficial-sigs +# + +# This configuration mounts a user.conf for the signature updater, but ClamAV +# is disabled, so there is nothing for it to update. +@test "checking clamav-unofficial-sigs: no cron task when ClamAV is disabled (ldap2 configuration)" { + run docker exec mailserver_ldap2 [ -f /etc/cron.d/clamav-unofficial-sigs ] + assert_failure +} + # # logs