Clamav unofficial sigs cron

Dockerfile

@@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y -q --no-install-recommends \
     postfix postfix-pgsql postfix-mysql postfix-ldap postfix-pcre libsasl2-modules \
     dovecot-core dovecot-imapd dovecot-lmtpd dovecot-pgsql dovecot-mysql dovecot-ldap dovecot-sieve dovecot-managesieved dovecot-pop3d \
     fetchmail libdbi-perl libdbd-pg-perl libdbd-mysql-perl liblockfile-simple-perl \
-    clamav clamav-daemon \
+    clamav clamav-daemon clamdscan \
     python3-pip python3-setuptools python3-wheel python3-gpg python3-watchdog \
     rsyslog dnsutils curl unbound unbound-anchor jq rsync \
     inotify-tools \

Makefile

@@ -41,8 +41,11 @@ build-no-cache:
 build:
 	docker build -t $(NAME) .
 
+# Test containers are removed with -v, so the anonymous volumes of the
+# database images (mysql, postgres, redis) go with them instead of
+# piling up on every run.
 init_openldap:
-	-docker rm -f \
+	-docker rm -fv \
 		openldap || true
 	docker run \
 		-d \
@@ -58,7 +61,7 @@ init_openldap:
 		|| { echo "TIMEOUT: openldap did not become ready"; docker logs --tail 50 openldap; exit 1; }
 
 init_redis:
-	-docker rm -f \
+	-docker rm -fv \
 		redis || true
 	docker run \
 		-d \
@@ -69,7 +72,7 @@ init_redis:
 		|| { echo "TIMEOUT: redis did not become ready"; docker logs --tail 50 redis; exit 1; }
 
 init_mariadb:
-	-docker rm -f \
+	-docker rm -fv \
 		mariadb || true
 	docker run \
 		-d \
@@ -86,7 +89,7 @@ init_mariadb:
 		|| { echo "TIMEOUT: mariadb did not become ready"; docker logs --tail 50 mariadb; exit 1; }
 
 init_postgres:
-	-docker rm -f \
+	-docker rm -fv \
 		postgres || true
 	docker run \
 		-d \
@@ -101,7 +104,7 @@ init_postgres:
 		|| { echo "TIMEOUT: postgres did not become ready"; docker logs --tail 50 postgres; exit 1; }
 
 init_ldap: init_openldap init_redis
-	-docker rm -f \
+	-docker rm -fv \
 		mailserver_ldap || true
 	docker run \
 		-d \
@@ -179,11 +182,11 @@ fixtures_ldap:
 run_ldap:
 	./test/bats/bin/bats test/ldap.bats
 stop_ldap:
-	-docker rm -f \
+	-docker rm -fv \
 		mailserver_ldap || true
 
 init_ldap2: init_openldap init_redis
-	-docker rm -f \
+	-docker rm -fv \
 		mailserver_ldap2 || true
 	docker run \
 		-d \
@@ -257,11 +260,11 @@ fixtures_ldap2:
 run_ldap2:
 	./test/bats/bin/bats test/ldap2.bats
 stop_ldap2:
-	-docker rm -f \
+	-docker rm -fv \
 		mailserver_ldap2 || true
 
 init_default: init_redis init_mariadb
-	-docker rm -f \
+	-docker rm -fv \
 		mailserver_default || true
 
 	docker run \
@@ -286,7 +289,7 @@ init_default: init_redis init_mariadb
 		-t $(NAME)
 
 init_reverse: init_redis init_postgres
-	-docker rm -f \
+	-docker rm -fv \
 		mailserver_reverse || true
 	docker run \
 		-d \
@@ -356,11 +359,11 @@ fixtures_reverse:
 run_reverse:
 	./test/bats/bin/bats test/reverse.bats
 stop_reverse:
-	-docker rm -f \
+	-docker rm -fv \
 		mailserver_reverse || true
 
 init_ecdsa: init_redis init_mariadb
-	-docker rm -f \
+	-docker rm -fv \
 		mailserver_ecdsa || true
 	docker run \
 		-d \
@@ -383,11 +386,11 @@ run_ecdsa:
 	docker exec mailserver_ecdsa /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 587 ; do sleep 1 ; done"
 	./test/bats/bin/bats test/ecdsa.bats
 stop_ecdsa:
-	-docker rm -f \
+	-docker rm -fv \
 		mailserver_ecdsa || true
 
 init_traefik_acmev1: init_redis init_mariadb
-	-docker rm -f \
+	-docker rm -fv \
 		mailserver_traefik_acmev1 || true
 	docker run \
 		-d \
@@ -421,11 +424,11 @@ run_traefik_acmev1:
 	docker exec mailserver_traefik_acmev1 /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 587 ; do sleep 1 ; done"
 	./test/bats/bin/bats test/traefik_acmev1.bats
 stop_traefik_acmev1:
-	-docker rm -f \
+	-docker rm -fv \
 		mailserver_traefik_acmev1 || true
 
 init_traefik_acmev2: init_redis init_mariadb
-	-docker rm -f \
+	-docker rm -fv \
 		mailserver_traefik_acmev2 || true
 	docker run \
 		-d \
@@ -459,14 +462,14 @@ run_traefik_acmev2:
 	docker exec mailserver_traefik_acmev2 /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 587 ; do sleep 1 ; done"
 	./test/bats/bin/bats test/traefik_acmev2.bats
 stop_traefik_acmev2:
-	-docker rm -f \
+	-docker rm -fv \
 		mailserver_traefik_acmev2 || true
 
 # acme.json as written by Traefik 2 and 3 (they share the format), generated
 # with Traefik 3.6 against a local ACME test server. It holds two certificate
 # resolvers, the case that used to leave the mailserver without a certificate.
 init_traefik_v3: init_redis init_mariadb
-	-docker rm -f \
+	-docker rm -fv \
 		mailserver_traefik_v3 || true
 	docker run \
 		-d \
@@ -499,13 +502,14 @@ run_traefik_v3:
 	docker exec mailserver_traefik_v3 /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 587 ; do sleep 1 ; done"
 	./test/bats/bin/bats test/traefik_v3.bats
 stop_traefik_v3:
-	-docker rm -f \
+	-docker rm -fv \
 		mailserver_traefik_v3 || true
 
 fixtures_default:
 
-	# Wait for clamav unofficial sigs database update (default)
-	docker exec mailserver_default /bin/sh -c "while [ -f /var/lib/clamav-unofficial-sigs/pid/clamav-unofficial-sigs.pid ] ; do sleep 1 ; done"
+	# Wait for the clamav unofficial sigs update that runs at startup (default).
+	# Its pid file only appears once the update starts, so wait for the log line.
+	docker exec mailserver_default /bin/sh -c "until grep -q 'clamav unofficial signature update done' /var/log/mail.log 2>/dev/null ; do sleep 1 ; done"
 	# Wait for clamav load databases (default)
 	docker exec mailserver_default /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 3310 ; do sleep 1 ; done"
 	# Wait for rspamd to start (default)
@@ -549,11 +553,11 @@ fixtures_default:
 run_default:
 	./test/bats/bin/bats test/default.bats
 stop_default:
-	-docker rm -f \
+	-docker rm -fv \
 		mailserver_default || true
 
 init_sieve: init_redis init_mariadb
-	-docker rm -f \
+	-docker rm -fv \
 		mailserver_sieve || true
 	docker run \
 		-d \
@@ -611,7 +615,7 @@ run_sieve:
 	./test/bats/bin/bats test/sieve.bats
 
 stop_sieve:
-	-docker rm -f \
+	-docker rm -fv \
 		mailserver_sieve || true
 
 clean:

README.md

@@ -691,7 +691,7 @@ interserver_enabled="yes"
 user_configuration_complete="yes"
 ```
 
-If the startup script detects this file, clamav-unofficial-sigs is automatically enabled and third-party databases downloaded under `/mnt/docker/mail/clamav` after Clamav startup. Once the databases are downloaded, a SIGUSR2 signal is sent to Clamav to reload the signature databases:
+If the startup script detects this file, clamav-unofficial-sigs is automatically enabled and third-party databases downloaded under `/mnt/docker/mail/clamav` after Clamav startup. Once the databases are downloaded and Clamav has finished starting, Clamav is asked to reload the signature databases through its socket (`clamdscan --reload`):
 
 ```
 docker logs -f mailserver
@@ -706,6 +706,8 @@ clamd[xxxxxx]: Reading databases from /var/lib/clamav
 clamd[xxxxxx]: Database correctly reloaded (6812263 signatures)
 ```
 
+After that the update runs every hour from cron, as the `clamav` user. Its errors and alerts go to the container log, tagged `clamav-unofficial-sigs`: errors also land in `/var/log/mail.err`, alerts (such as a newer clamav-unofficial-sigs version being available) in `/var/log/mail.warn`. The full record of each run is in `/var/log/clamav-unofficial-sigs/clamav-unofficial-sigs.log`.
+
 <p align="right"><a href="#summary">Back to table of contents :arrow_up_small:</a></p>
 
 ### Unbound DNS resolver

rootfs/etc/clamav-unofficial-sigs/os.conf

@@ -29,11 +29,16 @@ clam_dbs="/var/lib/clamav"
 
 clamd_pid="/run/clamav/clamd.pid"
 
-clamd_reload_opt="s6-svc -2 /services/clamd"
+# Through clamd's socket, which works for clamav, the user the update runs as,
+# once clamd has finished starting
+clamd_reload_opt="/usr/local/bin/clamav-unofficial-sigs-reload.sh"
 # this is in case the needed/expected keyword is now restart and not reload
 clamd_restart_opt="s6-svc -2 /services/clamd"
 
 # this is enabled here https://github.com/mailserver2/mailserver/blob/master/rootfs/etc/clamav/unofficial-sigs/os.conf#L6
 clamd_socket="/run/clamav/clamd.ctl"
 
+# cron runs the update through this wrapper, which sends its output to syslog
+cron_script_full_path="/usr/local/bin/clamav-unofficial-sigs-cron.sh"
+
 # https://eXtremeSHOK.com ######################################################

rootfs/services/_parent/run

@@ -26,7 +26,7 @@ if [ "$DISABLE_CLAMAV" = false ]; then
   s6-svc -u /services/freshclam
   if [ -f "/var/mail/clamav-unofficial-sigs/user.conf" ]; then
     logger -p mail.info "s6-supervise : clamav unofficial signature update running"
-    clamav-unofficial-sigs.sh &>/dev/null
+    s6-setuidgid clamav clamav-unofficial-sigs.sh &>/dev/null
     logger -p mail.info "s6-supervise : clamav unofficial signature update done"
   fi
   s6-svwait -u /services/clamd

rootfs/usr/local/bin/clamav-unofficial-sigs-cron.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# The hourly clamav-unofficial-sigs update, as cron runs it (see
+# cron_script_full_path in /etc/clamav-unofficial-sigs/os.conf).
+#
+# cron mails whatever a job prints to the job's user, clamav@<FQDN>, which
+# nobody reads, and the mail is scanned by rspamd on the way. The output goes
+# to syslog instead, with the rest of the container's logs: shell errors
+# (stderr) at mail.err, the script's own ERROR and ALERT lines (stdout) at
+# mail.warning.
+
+pid_file=/var/lib/clamav-unofficial-sigs/pid/clamav-unofficial-sigs.pid
+
+# The update at startup and this hourly one are independent, so finding the
+# other one running is expected rather than an error.
+if [ -f "$pid_file" ] && ps -p "$(cat "$pid_file")" > /dev/null 2>&1; then
+  logger -t clamav-unofficial-sigs -p mail.info "update already running, skipping this run"
+  exit 0
+fi
+
+# The script frames its messages with lines of = * - or +, which are dropped.
+{ /usr/local/bin/clamav-unofficial-sigs.sh 2>&1 1>&3 3>&- \
+    | grep -v '^[-=*+]*$' | logger -t clamav-unofficial-sigs -p mail.err; } 3>&1 \
+  | grep -v '^[-=*+]*$' | logger -t clamav-unofficial-sigs -p mail.warning

rootfs/usr/local/bin/clamav-unofficial-sigs-reload.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# How clamav-unofficial-sigs asks clamd to reload its databases (see
+# clamd_reload_opt in /etc/clamav-unofficial-sigs/os.conf).
+#
+# Whenever clamd is starting, at container start or after it restarts, it has
+# no socket and ignores SIGUSR2, the updater's fallback, until it has loaded
+# its databases, so a reload sent then is lost. The update at startup runs
+# while clamd starts, so it meets this regularly; an hourly one can too. So
+# every reload first waits for clamd to answer. Each failed ping prints an
+# ERROR, which the updater would take for a failed reload, so the wait is
+# silent: if clamd never answers, the reload's own error is what the updater
+# sees.
+#
+# A reload sent while clamd is already reloading does not wait: clamd answers
+# then, and merges it into the reload in progress. Anything that reload missed
+# is picked up by clamd's SelfCheck.
+
+clamdscan --ping 300:1 > /dev/null 2>&1
+exec clamdscan --reload

rootfs/usr/local/bin/setup.sh

@@ -799,12 +799,19 @@ chown -R clamav:clamav /var/run/clamav /var/mail/clamav /var/log/clamav
 # CLAMAV-UNOFFICIAL-SIGS
 # ---------------------------------------------------------------------------------------------
 
-if [ -f "/var/mail/clamav-unofficial-sigs/user.conf" ]; then
+if [ "$DISABLE_CLAMAV" = true ]; then
+  echo "[INFO] clamav-unofficial-sigs is disabled (ClamAV is disabled)"
+elif [ -f "/var/mail/clamav-unofficial-sigs/user.conf" ]; then
   echo "[INFO] clamav-unofficial-sigs is enabled (user configuration found)"
   rm -rf /var/lib/clamav-unofficial-sigs
   ln -s /var/mail/clamav-unofficial-sigs /var/lib/clamav-unofficial-sigs
   cp -f /var/mail/clamav-unofficial-sigs/user.conf /etc/clamav-unofficial-sigs
   mkdir -p /var/log/clamav-unofficial-sigs
+  # The update runs as clamav, at startup and from cron, and has to be able to
+  # write all of its state, including what a volume holds from runs as root.
+  # user.conf is the administrator's file and keeps its owner.
+  find /var/mail/clamav-unofficial-sigs ! -name user.conf -exec chown clamav:clamav {} +
+  chown -R clamav:clamav /var/log/clamav-unofficial-sigs
   clamav-unofficial-sigs.sh --install-cron &>/dev/null 
   clamav-unofficial-sigs.sh --install-logrotate &>/dev/null
 else