1
0
mirror of https://github.com/mailcow/mailcow-dockerized.git synced 2026-05-16 20:41:55 +00:00

Merge remote-tracking branch 'origin/staging' into nightly

This commit is contained in:
FreddleSpl0it
2024-07-31 10:35:44 +02:00
91 changed files with 3112 additions and 1413 deletions

View File

@@ -1,8 +1,8 @@
FROM alpine:3.18
FROM alpine:3.20
LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
LABEL maintainer "The Infrastructure Company GmbH GmbH <info@servercow.de>"
ARG PIP_BREAK_SYSTEM_PACKAGES=1
RUN apk upgrade --no-cache \
&& apk add --update --no-cache \
bash \
@@ -15,9 +15,7 @@ RUN apk upgrade --no-cache \
tini \
tzdata \
python3 \
py3-pip \
&& pip3 install --upgrade pip \
&& pip3 install acme-tiny
acme-tiny --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community/
COPY acme.sh /srv/acme.sh
COPY functions.sh /srv/functions.sh

View File

@@ -33,6 +33,10 @@ if [[ "${ONLY_MAILCOW_HOSTNAME}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
ONLY_MAILCOW_HOSTNAME=y
fi
if [[ "${AUTODISCOVER_SAN}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
AUTODISCOVER_SAN=y
fi
# Request individual certificate for every domain
if [[ "${ENABLE_SSL_SNI}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
ENABLE_SSL_SNI=y
@@ -211,7 +215,11 @@ while true; do
ADDITIONAL_SAN_ARR+=($i)
fi
done
if [[ ${AUTODISCOVER_SAN} == "y" ]]; then
# Fetch certs for autoconfig and autodiscover subdomains
ADDITIONAL_WC_ARR+=('autodiscover' 'autoconfig')
fi
if [[ ${SKIP_IP_CHECK} != "y" ]]; then
# Start IP detection

View File

@@ -1,3 +1,3 @@
FROM debian:bullseye-slim
FROM debian:bookworm-slim
RUN apt update && apt install pigz

View File

@@ -1,6 +1,6 @@
FROM alpine:3.19
FROM alpine:3.20
LABEL maintainer "The Infrastructure Company GmbH GmbH <info@servercow.de>"
LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
RUN apk upgrade --no-cache \
&& apk add --update --no-cache \

View File

@@ -1,6 +1,6 @@
FROM alpine:3.19
FROM alpine:3.20
LABEL maintainer "The Infrastructure Company GmbH GmbH <info@servercow.de>"
LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
ARG PIP_BREAK_SYSTEM_PACKAGES=1
WORKDIR /app

View File

@@ -358,8 +358,8 @@ class DockerApi:
for line in cmd_response.split("\n"):
if '$2$' in line:
hash = line.strip()
hash_out = re.search('\$2\$.+$', hash).group(0)
rspamd_passphrase_hash = re.sub('[^0-9a-zA-Z\$]+', '', hash_out.rstrip())
hash_out = re.search(r'\$2\$.+$', hash).group(0)
rspamd_passphrase_hash = re.sub(r'[^0-9a-zA-Z\$]+', '', hash_out.rstrip())
rspamd_password_filename = "/etc/rspamd/override.d/worker-controller-password.inc"
cmd = '''/bin/echo 'enable_password = "%s";' > %s && cat %s''' % (rspamd_passphrase_hash, rspamd_password_filename, rspamd_password_filename)
cmd_response = self.exec_cmd_container(container, cmd, user="_rspamd")

View File

@@ -1,5 +1,5 @@
FROM alpine:3.19
LABEL maintainer "The Infrastructure Company GmbH GmbH <info@servercow.de>"
FROM alpine:3.20
LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
# renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=^(?<version>.*)$
ARG GOSU_VERSION=1.16
@@ -23,6 +23,7 @@ RUN addgroup -g 5000 vmail \
envsubst \
ca-certificates \
curl \
coreutils \
jq \
lua \
lua-cjson \
@@ -65,7 +66,7 @@ RUN addgroup -g 5000 vmail \
perl-package-stash-xs \
perl-par-packer \
perl-parse-recdescent \
perl-lockfile-simple --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community/ \
perl-lockfile-simple \
libproc \
perl-readonly \
perl-regexp-common \
@@ -107,13 +108,12 @@ RUN addgroup -g 5000 vmail \
dovecot-pigeonhole-plugin \
dovecot-pop3d \
dovecot-fts-solr \
dovecot-fts-flatcurve \
&& arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) \
&& wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$arch" \
&& chmod +x /usr/local/bin/gosu \
&& gosu nobody true
#RUN cpan LockFile::Simple
COPY trim_logs.sh /usr/local/bin/trim_logs.sh
COPY clean_q_aged.sh /usr/local/bin/clean_q_aged.sh
COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
@@ -132,6 +132,7 @@ COPY stop-supervisor.sh /usr/local/sbin/stop-supervisor.sh
COPY quarantine_notify.py /usr/local/bin/quarantine_notify.py
COPY quota_notify.py /usr/local/bin/quota_notify.py
COPY repl_health.sh /usr/local/bin/repl_health.sh
COPY optimize-fts.sh /usr/local/bin/optimize-fts.sh
ENTRYPOINT ["/docker-entrypoint.sh"]
CMD exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf

View File

@@ -29,6 +29,7 @@ ${REDIS_CMDLINE} SET DOVECOT_REPL_HEALTH 1 > /dev/null
# Create missing directories
[[ ! -d /etc/dovecot/sql/ ]] && mkdir -p /etc/dovecot/sql/
[[ ! -d /etc/dovecot/auth/ ]] && mkdir -p /etc/dovecot/auth/
[[ ! -d /etc/dovecot/conf.d/ ]] && mkdir -p /etc/dovecot/conf.d/
[[ ! -d /var/vmail/_garbage ]] && mkdir -p /var/vmail/_garbage
[[ ! -d /var/vmail/sieve ]] && mkdir -p /var/vmail/sieve
[[ ! -d /etc/sogo ]] && mkdir -p /etc/sogo
@@ -109,7 +110,14 @@ EOF
echo -n ${ACL_ANYONE} > /etc/dovecot/acl_anyone
if [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
if [[ "${FLATCURVE_EXPERIMENTAL}" =~ ^([yY][eE][sS]|[yY]) ]]; then
echo -e "\e[33mActivating Flatcurve as FTS Backend...\e[0m"
echo -e "\e[33mDepending on your previous setup a full reindex might be needed... \e[0m"
echo -e "\e[34mVisit https://docs.mailcow.email/manual-guides/Dovecot/u_e-dovecot-fts/#fts-related-dovecot-commands to learn how to reindex\e[0m"
echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify fts fts_flatcurve listescape replication' > /etc/dovecot/mail_plugins
echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify mail_log fts fts_flatcurve listescape replication' > /etc/dovecot/mail_plugins_imap
echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl fts fts_flatcurve notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
elif [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify listescape replication' > /etc/dovecot/mail_plugins
echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify listescape replication mail_log' > /etc/dovecot/mail_plugins_imap
echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
@@ -128,6 +136,48 @@ user_query = SELECT CONCAT(JSON_UNQUOTE(JSON_VALUE(attributes, '$.mailbox_format
iterate_query = SELECT username FROM mailbox WHERE active = '1' OR active = '2';
EOF
# Temporarily set FTS depending on user choice inside mailcow.conf. Will be removed as soon as Solr is dropped
if [[ "${FLATCURVE_EXPERIMENTAL}" =~ ^([yY][eE][sS]|[yY])$ ]]; then
cat <<EOF > /etc/dovecot/conf.d/fts.conf
# Autogenerated by mailcow
plugin {
fts_autoindex = yes
fts_autoindex_exclude = \Junk
fts_autoindex_exclude2 = \Trash
fts = flatcurve
# These are not flatcurve settings, but required for Dovecot FTS. See
# Dovecot FTS Configuration link above for further information.
fts_languages = en es de
fts_tokenizer_generic = algorithm=simple
fts_tokenizers = generic email-address
# OPTIONAL: Recommended default FTS core configuration
fts_filters = normalizer-icu snowball stopwords
fts_filters_en = lowercase snowball english-possessive stopwords
}
EOF
elif [[ ! "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])$ ]]; then
cat <<EOF > /etc/dovecot/conf.d/fts.conf
# Autogenerated by mailcow
plugin {
fts = solr
fts_autoindex = yes
fts_autoindex_exclude = \Junk
fts_autoindex_exclude2 = \Trash
fts_solr = url=http://solr:8983/solr/dovecot-fts/
fts_tokenizers = generic email-address
fts_tokenizer_generic = algorithm=simple
fts_filters = normalizer-icu snowball stopwords
fts_filters_en = lowercase snowball english-possessive stopwords
}
EOF
fi
# Migrate old sieve_after file
[[ -f /etc/dovecot/sieve_after ]] && mv /etc/dovecot/sieve_after /etc/dovecot/global_sieve_after
# Create global sieve scripts
@@ -218,7 +268,6 @@ mail_replica = tcp:${MAILCOW_REPLICA_IP}:${DOVEADM_REPLICA_PORT}
EOF
fi
# 401 is user dovecot
if [[ ! -s /mail_crypt/ecprivkey.pem || ! -s /mail_crypt/ecpubkey.pem ]]; then
openssl ecparam -name prime256v1 -genkey | openssl pkey -out /mail_crypt/ecprivkey.pem
@@ -262,7 +311,8 @@ chmod +x /usr/lib/dovecot/sieve/rspamd-pipe-ham \
/usr/local/bin/maildir_gc.sh \
/usr/local/sbin/stop-supervisor.sh \
/usr/local/bin/quota_notify.py \
/usr/local/bin/repl_health.sh
/usr/local/bin/repl_health.sh \
/usr/local/bin/optimize-fts.sh
# Prepare environment file for cronjobs
printenv | sed 's/^\(.*\)$/export \1/g' > /source_env.sh

View File

@@ -0,0 +1,7 @@
#!/bin/bash
if [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ && ! "${FLATCURVE_EXPERIMENTAL}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
exit 0
else
doveadm fts optimize -A
fi

View File

@@ -11,7 +11,7 @@ else
fi
# Deploy
curl --connect-timeout 15 --retry 10 --max-time 30 http://www.spamassassin.heinlein-support.de/$(dig txt 1.4.3.spamassassin.heinlein-support.de +short | tr -d '"' | tr -dc '0-9').tar.gz --output /tmp/sa-rules-heinlein.tar.gz
curl --connect-timeout 15 --retry 10 --max-time 30 https://www.spamassassin.heinlein-support.de/$(dig txt 1.4.3.spamassassin.heinlein-support.de +short | tr -d '"' | tr -dc '0-9').tar.gz --output /tmp/sa-rules-heinlein.tar.gz
if gzip -t /tmp/sa-rules-heinlein.tar.gz; then
tar xfvz /tmp/sa-rules-heinlein.tar.gz -C /tmp/sa-rules-heinlein
cat /tmp/sa-rules-heinlein/*cf > /etc/rspamd/custom/sa-rules

View File

@@ -1,4 +1,4 @@
FROM alpine:3.19
FROM alpine:3.20
LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
WORKDIR /app

View File

@@ -80,16 +80,16 @@ def refreshF2bregex():
global exit_code
if not r.get('F2B_REGEX'):
f2bregex = {}
f2bregex[1] = 'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)'
f2bregex[2] = 'Rspamd UI: Invalid password by ([0-9a-f\.:]+)'
f2bregex[3] = 'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+'
f2bregex[4] = 'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+'
f2bregex[5] = 'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+'
f2bregex[6] = '-login: Disconnected.+ \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),'
f2bregex[7] = '-login: Aborted login.+ \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
f2bregex[8] = '-login: Aborted login.+ \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
f2bregex[9] = 'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked'
f2bregex[10] = '([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+'
f2bregex[1] = r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)'
f2bregex[2] = r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)'
f2bregex[3] = r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+'
f2bregex[4] = r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+'
f2bregex[5] = r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+'
f2bregex[6] = r'-login: Disconnected.+ \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),'
f2bregex[7] = r'-login: Aborted login.+ \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
f2bregex[8] = r'-login: Aborted login.+ \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
f2bregex[9] = r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked'
f2bregex[10] = r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+'
r.set('F2B_REGEX', json.dumps(f2bregex, ensure_ascii=False))
else:
try:
@@ -114,8 +114,6 @@ def ban(address):
global lock
refreshF2boptions()
BAN_TIME = int(f2boptions['ban_time'])
BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment'])
MAX_ATTEMPTS = int(f2boptions['max_attempts'])
RETRY_WINDOW = int(f2boptions['retry_window'])
NETBAN_IPV4 = '/' + str(f2boptions['netban_ipv4'])
@@ -150,7 +148,7 @@ def ban(address):
if bans[net]['attempts'] >= MAX_ATTEMPTS:
cur_time = int(round(time.time()))
NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** bans[net]['ban_counter']
NET_BAN_TIME = calcNetBanTime(bans[net]['ban_counter'])
logger.logCrit('Banning %s for %d minutes' % (net, NET_BAN_TIME / 60 ))
if type(ip) is ipaddress.IPv4Address and int(f2boptions['manage_external']) != 1:
with lock:
@@ -277,12 +275,11 @@ def snat6(snat_target):
tables.snat6(snat_target, os.getenv('IPV6_NETWORK', 'fd4d:6169:6c63:6f77::/64'))
def autopurge():
global f2boptions
while not quit_now:
time.sleep(10)
refreshF2boptions()
BAN_TIME = int(f2boptions['ban_time'])
MAX_BAN_TIME = int(f2boptions['max_ban_time'])
BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment'])
MAX_ATTEMPTS = int(f2boptions['max_attempts'])
QUEUE_UNBAN = r.hgetall('F2B_QUEUE_UNBAN')
if QUEUE_UNBAN:
@@ -290,9 +287,9 @@ def autopurge():
unban(str(net))
for net in bans.copy():
if bans[net]['attempts'] >= MAX_ATTEMPTS:
NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** bans[net]['ban_counter']
NET_BAN_TIME = calcNetBanTime(bans[net]['ban_counter'])
TIME_SINCE_LAST_ATTEMPT = time.time() - bans[net]['last_attempt']
if TIME_SINCE_LAST_ATTEMPT > NET_BAN_TIME or TIME_SINCE_LAST_ATTEMPT > MAX_BAN_TIME:
if TIME_SINCE_LAST_ATTEMPT > NET_BAN_TIME:
unban(net)
def mailcowChainOrder():
@@ -306,6 +303,16 @@ def mailcowChainOrder():
if quit_now: return
quit_now, exit_code = tables.checkIPv6ChainOrder()
def calcNetBanTime(ban_counter):
global f2boptions
BAN_TIME = int(f2boptions['ban_time'])
MAX_BAN_TIME = int(f2boptions['max_ban_time'])
BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment'])
NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** ban_counter
NET_BAN_TIME = max([BAN_TIME, min([NET_BAN_TIME, MAX_BAN_TIME])])
return NET_BAN_TIME
def isIpNetwork(address):
try:
ipaddress.ip_network(address, False)

View File

@@ -13,9 +13,12 @@ class Logger:
tolog['time'] = int(round(time.time()))
tolog['priority'] = priority
tolog['message'] = message
if self.r is not None:
self.r.lpush('NETFILTER_LOG', json.dumps(tolog, ensure_ascii=False))
print(message)
if self.r is not None:
try:
self.r.lpush('NETFILTER_LOG', json.dumps(tolog, ensure_ascii=False))
except Exception as ex:
print('Failed logging to redis: %s' % (ex))
def logWarn(self, message):
self.log('warn', message)

View File

@@ -452,6 +452,8 @@ class NFTables:
continue
rule = _object["rule"]["expr"][0]["match"]
if not "payload" in rule["left"]:
continue
left_opt = rule["left"]["payload"]
if not left_opt["protocol"] == _family:
continue

View File

@@ -1,4 +1,4 @@
FROM alpine:3.19
FROM alpine:3.20
LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
ARG PIP_BREAK_SYSTEM_PACKAGES=1

View File

@@ -24,6 +24,7 @@ done
CONTAINER_ID=
until [[ ! -z "${CONTAINER_ID}" ]] && [[ "${CONTAINER_ID}" =~ ^[[:alnum:]]*$ ]]; do
CONTAINER_ID=$(curl --silent --insecure https://dockerapi/containers/json | jq -r ".[] | {name: .Config.Labels[\"com.docker.compose.service\"], project: .Config.Labels[\"com.docker.compose.project\"], id: .Id}" 2> /dev/null | jq -rc "select( .name | tostring | contains(\"mysql-mailcow\")) | select( .project | tostring | contains(\"${COMPOSE_PROJECT_NAME,,}\")) | .id" 2> /dev/null)
echo "Could not get mysql-mailcow container id... trying again"
sleep 2
done
echo "MySQL @ ${CONTAINER_ID}"

View File

@@ -1,5 +1,5 @@
FROM debian:bullseye-slim
LABEL maintainer "The Infrastructure Company GmbH GmbH <info@servercow.de>"
FROM debian:bookworm-slim
LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
ARG DEBIAN_FRONTEND=noninteractive
ENV LC_ALL C

View File

@@ -415,12 +415,6 @@ postscreen_dnsbl_sites = wl.mailspike.net=127.0.0.[18;19;20]*-2
b.barracudacentral.org=127.0.0.2*7
bl.mailspike.net=127.0.0.2*5
bl.mailspike.net=127.0.0.[10;11;12]*4
dnsbl.sorbs.net=127.0.0.10*8
dnsbl.sorbs.net=127.0.0.5*6
dnsbl.sorbs.net=127.0.0.7*3
dnsbl.sorbs.net=127.0.0.8*2
dnsbl.sorbs.net=127.0.0.6*2
dnsbl.sorbs.net=127.0.0.9*2
EOF
fi
DNSBL_CONFIG=$(grep -v '^#' /opt/postfix/conf/dns_blocklists.cf | grep '\S')

View File

@@ -1,4 +1,4 @@
@version: 3.28
@version: 3.38
@include "scl.conf"
options {
chain_hostnames(off);

View File

@@ -1,4 +1,4 @@
@version: 3.28
@version: 3.38
@include "scl.conf"
options {
chain_hostnames(off);

View File

@@ -1,8 +1,9 @@
FROM debian:bookworm-slim
FROM debian:bullseye-slim
LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
ARG DEBIAN_FRONTEND=noninteractive
ARG CODENAME=bookworm
ARG RSPAMD_VER=rspamd_3.7.5-2~8c86c1676
ARG CODENAME=bullseye
ENV LC_ALL C
RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -11,12 +12,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
gnupg2 \
apt-transport-https \
dnsutils \
netcat-traditional \
&& apt-key adv --fetch-keys https://rspamd.com/apt-stable/gpg.key \
&& echo "deb https://rspamd.com/apt-stable/ $CODENAME main" > /etc/apt/sources.list.d/rspamd.list \
&& apt-get update \
&& apt-get --no-install-recommends -y install rspamd redis-tools procps nano \
&& rm -rf /var/lib/apt/lists/* \
netcat \
wget \
redis-tools \
procps \
nano \
&& arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) \
&& wget -P /tmp https://rspamd.com/apt-stable/pool/main/r/rspamd/${RSPAMD_VER}~${CODENAME}_${arch}.deb\
&& apt install -y /tmp/${RSPAMD_VER}~${CODENAME}_${arch}.deb \
&& rm -rf /var/lib/apt/lists/* /tmp/*\
&& apt-get autoremove --purge \
&& apt-get clean \
&& mkdir -p /run/rspamd \
@@ -25,7 +29,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
&& sed -i 's/#analysis_keyword_table > 0/analysis_cat_table.macro_exist == "M"/g' /usr/share/rspamd/lualib/lua_scanners/oletools.lua
COPY settings.conf /etc/rspamd/settings.conf
COPY metadata_exporter.lua /usr/share/rspamd/plugins/metadata_exporter.lua
COPY set_worker_password.sh /set_worker_password.sh
COPY docker-entrypoint.sh /docker-entrypoint.sh

View File

@@ -1,632 +0,0 @@
--[[
Copyright (c) 2016, Andrew Lewis <nerf@judo.za.org>
Copyright (c) 2016, Vsevolod Stakhov <vsevolod@highsecure.ru>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
]]--
if confighelp then
return
end
-- A plugin that pushes metadata (or whole messages) to external services
local redis_params
local lua_util = require "lua_util"
local rspamd_http = require "rspamd_http"
local rspamd_util = require "rspamd_util"
local rspamd_logger = require "rspamd_logger"
local ucl = require "ucl"
local E = {}
local N = 'metadata_exporter'
local settings = {
pusher_enabled = {},
pusher_format = {},
pusher_select = {},
mime_type = 'text/plain',
defer = false,
mail_from = '',
mail_to = 'postmaster@localhost',
helo = 'rspamd',
email_template = [[From: "Rspamd" <$mail_from>
To: $mail_to
Subject: Spam alert
Date: $date
MIME-Version: 1.0
Message-ID: <$our_message_id>
Content-type: text/plain; charset=utf-8
Content-Transfer-Encoding: 8bit
Authenticated username: $user
IP: $ip
Queue ID: $qid
SMTP FROM: $from
SMTP RCPT: $rcpt
MIME From: $header_from
MIME To: $header_to
MIME Date: $header_date
Subject: $header_subject
Message-ID: $message_id
Action: $action
Score: $score
Symbols: $symbols]],
}
local function get_general_metadata(task, flatten, no_content)
local r = {}
local ip = task:get_from_ip()
if ip and ip:is_valid() then
r.ip = tostring(ip)
else
r.ip = 'unknown'
end
r.user = task:get_user() or 'unknown'
r.qid = task:get_queue_id() or 'unknown'
r.subject = task:get_subject() or 'unknown'
r.action = task:get_metric_action('default')
local s = task:get_metric_score('default')[1]
r.score = flatten and string.format('%.2f', s) or s
local fuzzy = task:get_mempool():get_variable("fuzzy_hashes", "fstrings")
if fuzzy and #fuzzy > 0 then
local fz = {}
for _,h in ipairs(fuzzy) do
table.insert(fz, h)
end
if not flatten then
r.fuzzy = fz
else
r.fuzzy = table.concat(fz, ', ')
end
else
r.fuzzy = 'unknown'
end
local rcpt = task:get_recipients('smtp')
if rcpt then
local l = {}
for _, a in ipairs(rcpt) do
table.insert(l, a['addr'])
end
if not flatten then
r.rcpt = l
else
r.rcpt = table.concat(l, ', ')
end
else
r.rcpt = 'unknown'
end
local from = task:get_from('smtp')
if ((from or E)[1] or E).addr then
r.from = from[1].addr
else
r.from = 'unknown'
end
local syminf = task:get_symbols_all()
if flatten then
local l = {}
for _, sym in ipairs(syminf) do
local txt
if sym.options then
local topt = table.concat(sym.options, ', ')
txt = sym.name .. '(' .. string.format('%.2f', sym.score) .. ')' .. ' [' .. topt .. ']'
else
txt = sym.name .. '(' .. string.format('%.2f', sym.score) .. ')'
end
table.insert(l, txt)
end
r.symbols = table.concat(l, '\n\t')
else
r.symbols = syminf
end
local function process_header(name)
local hdr = task:get_header_full(name)
if hdr then
local l = {}
for _, h in ipairs(hdr) do
table.insert(l, h.decoded)
end
if not flatten then
return l
else
return table.concat(l, '\n')
end
else
return 'unknown'
end
end
if not no_content then
r.header_from = process_header('from')
r.header_to = process_header('to')
r.header_subject = process_header('subject')
r.header_date = process_header('date')
r.message_id = task:get_message_id()
end
return r
end
local formatters = {
default = function(task)
return task:get_content(), {}
end,
email_alert = function(task, rule, extra)
local meta = get_general_metadata(task, true)
local display_emails = {}
local mail_targets = {}
meta.mail_from = rule.mail_from or settings.mail_from
local mail_rcpt = rule.mail_to or settings.mail_to
if type(mail_rcpt) ~= 'table' then
table.insert(display_emails, string.format('<%s>', mail_rcpt))
table.insert(mail_targets, mail_rcpt)
else
for _, e in ipairs(mail_rcpt) do
table.insert(display_emails, string.format('<%s>', e))
table.insert(mail_targets, mail_rcpt)
end
end
if rule.email_alert_sender then
local x = task:get_from('smtp')
if x and string.len(x[1].addr) > 0 then
table.insert(mail_targets, x)
table.insert(display_emails, string.format('<%s>', x[1].addr))
end
end
if rule.email_alert_user then
local x = task:get_user()
if x then
table.insert(mail_targets, x)
table.insert(display_emails, string.format('<%s>', x))
end
end
if rule.email_alert_recipients then
local x = task:get_recipients('smtp')
if x then
for _, e in ipairs(x) do
if string.len(e.addr) > 0 then
table.insert(mail_targets, e.addr)
table.insert(display_emails, string.format('<%s>', e.addr))
end
end
end
end
meta.mail_to = table.concat(display_emails, ', ')
meta.our_message_id = rspamd_util.random_hex(12) .. '@rspamd'
meta.date = rspamd_util.time_to_string(rspamd_util.get_time())
return lua_util.template(rule.email_template or settings.email_template, meta), { mail_targets = mail_targets}
end,
json = function(task)
return ucl.to_format(get_general_metadata(task), 'json-compact')
end
}
local function is_spam(action)
return (action == 'reject' or action == 'add header' or action == 'rewrite subject')
end
local selectors = {
default = function(task)
return true
end,
is_spam = function(task)
local action = task:get_metric_action('default')
return is_spam(action)
end,
is_spam_authed = function(task)
if not task:get_user() then
return false
end
local action = task:get_metric_action('default')
return is_spam(action)
end,
is_reject = function(task)
local action = task:get_metric_action('default')
return (action == 'reject')
end,
is_reject_authed = function(task)
if not task:get_user() then
return false
end
local action = task:get_metric_action('default')
return (action == 'reject')
end,
}
local function maybe_defer(task, rule)
if rule.defer then
rspamd_logger.warnx(task, 'deferring message')
task:set_pre_result('soft reject', 'deferred', N)
end
end
local pushers = {
redis_pubsub = function(task, formatted, rule)
local _,ret,upstream
local function redis_pub_cb(err)
if err then
rspamd_logger.errx(task, 'got error %s when publishing on server %s',
err, upstream:get_addr())
return maybe_defer(task, rule)
end
return true
end
ret,_,upstream = rspamd_redis_make_request(task,
redis_params, -- connect params
nil, -- hash key
true, -- is write
redis_pub_cb, --callback
'PUBLISH', -- command
{rule.channel, formatted} -- arguments
)
if not ret then
rspamd_logger.errx(task, 'error connecting to redis')
maybe_defer(task, rule)
end
end,
http = function(task, formatted, rule)
local function http_callback(err, code)
if err then
rspamd_logger.errx(task, 'got error %s in http callback', err)
return maybe_defer(task, rule)
end
if code ~= 200 then
rspamd_logger.errx(task, 'got unexpected http status: %s', code)
return maybe_defer(task, rule)
end
return true
end
local hdrs = {}
if rule.meta_headers then
local gm = get_general_metadata(task, false, true)
local pfx = rule.meta_header_prefix or 'X-Rspamd-'
for k, v in pairs(gm) do
if type(v) == 'table' then
hdrs[pfx .. k] = ucl.to_format(v, 'json-compact')
else
hdrs[pfx .. k] = v
end
end
end
rspamd_http.request({
task=task,
url=rule.url,
body=formatted,
callback=http_callback,
mime_type=rule.mime_type or settings.mime_type,
headers=hdrs,
})
end,
send_mail = function(task, formatted, rule, extra)
local lua_smtp = require "lua_smtp"
local function sendmail_cb(ret, err)
if not ret then
rspamd_logger.errx(task, 'SMTP export error: %s', err)
maybe_defer(task, rule)
end
end
lua_smtp.sendmail({
task = task,
host = rule.smtp,
port = rule.smtp_port or settings.smtp_port or 25,
from = rule.mail_from or settings.mail_from,
recipients = extra.mail_targets or rule.mail_to or settings.mail_to,
helo = rule.helo or settings.helo,
timeout = rule.timeout or settings.timeout,
}, formatted, sendmail_cb)
end,
}
local opts = rspamd_config:get_all_opt(N)
if not opts then return end
local process_settings = {
select = function(val)
selectors.custom = assert(load(val))()
end,
format = function(val)
formatters.custom = assert(load(val))()
end,
push = function(val)
pushers.custom = assert(load(val))()
end,
custom_push = function(val)
if type(val) == 'table' then
for k, v in pairs(val) do
pushers[k] = assert(load(v))()
end
end
end,
custom_select = function(val)
if type(val) == 'table' then
for k, v in pairs(val) do
selectors[k] = assert(load(v))()
end
end
end,
custom_format = function(val)
if type(val) == 'table' then
for k, v in pairs(val) do
formatters[k] = assert(load(v))()
end
end
end,
pusher_enabled = function(val)
if type(val) == 'string' then
if pushers[val] then
settings.pusher_enabled[val] = true
else
rspamd_logger.errx(rspamd_config, 'Pusher type: %s is invalid', val)
end
elseif type(val) == 'table' then
for _, v in ipairs(val) do
if pushers[v] then
settings.pusher_enabled[v] = true
else
rspamd_logger.errx(rspamd_config, 'Pusher type: %s is invalid', val)
end
end
end
end,
}
for k, v in pairs(opts) do
local f = process_settings[k]
if f then
f(opts[k])
else
settings[k] = v
end
end
if type(settings.rules) ~= 'table' then
-- Legacy config
settings.rules = {}
if not next(settings.pusher_enabled) then
if pushers.custom then
rspamd_logger.infox(rspamd_config, 'Custom pusher implicitly enabled')
settings.pusher_enabled.custom = true
else
-- Check legacy options
if settings.url then
rspamd_logger.warnx(rspamd_config, 'HTTP pusher implicitly enabled')
settings.pusher_enabled.http = true
end
if settings.channel then
rspamd_logger.warnx(rspamd_config, 'Redis Pubsub pusher implicitly enabled')
settings.pusher_enabled.redis_pubsub = true
end
if settings.smtp and settings.mail_to then
rspamd_logger.warnx(rspamd_config, 'SMTP pusher implicitly enabled')
settings.pusher_enabled.send_mail = true
end
end
end
if not next(settings.pusher_enabled) then
rspamd_logger.errx(rspamd_config, 'No push backend enabled')
return
end
if settings.formatter then
settings.format = formatters[settings.formatter]
if not settings.format then
rspamd_logger.errx(rspamd_config, 'No such formatter: %s', settings.formatter)
return
end
end
if settings.selector then
settings.select = selectors[settings.selector]
if not settings.select then
rspamd_logger.errx(rspamd_config, 'No such selector: %s', settings.selector)
return
end
end
for k in pairs(settings.pusher_enabled) do
local formatter = settings.pusher_format[k]
local selector = settings.pusher_select[k]
if not formatter then
settings.pusher_format[k] = settings.formatter or 'default'
rspamd_logger.infox(rspamd_config, 'Using default formatter for %s pusher', k)
else
if not formatters[formatter] then
rspamd_logger.errx(rspamd_config, 'No such formatter: %s - disabling %s', formatter, k)
settings.pusher_enabled.k = nil
end
end
if not selector then
settings.pusher_select[k] = settings.selector or 'default'
rspamd_logger.infox(rspamd_config, 'Using default selector for %s pusher', k)
else
if not selectors[selector] then
rspamd_logger.errx(rspamd_config, 'No such selector: %s - disabling %s', selector, k)
settings.pusher_enabled.k = nil
end
end
end
if settings.pusher_enabled.redis_pubsub then
redis_params = rspamd_parse_redis_server(N)
if not redis_params then
rspamd_logger.errx(rspamd_config, 'No redis servers are specified')
settings.pusher_enabled.redis_pubsub = nil
else
local r = {}
r.backend = 'redis_pubsub'
r.channel = settings.channel
r.defer = settings.defer
r.selector = settings.pusher_select.redis_pubsub
r.formatter = settings.pusher_format.redis_pubsub
settings.rules[r.backend:upper()] = r
end
end
if settings.pusher_enabled.http then
if not settings.url then
rspamd_logger.errx(rspamd_config, 'No URL is specified')
settings.pusher_enabled.http = nil
else
local r = {}
r.backend = 'http'
r.url = settings.url
r.mime_type = settings.mime_type
r.defer = settings.defer
r.selector = settings.pusher_select.http
r.formatter = settings.pusher_format.http
settings.rules[r.backend:upper()] = r
end
end
if settings.pusher_enabled.send_mail then
if not (settings.mail_to and settings.smtp) then
rspamd_logger.errx(rspamd_config, 'No mail_to and/or smtp setting is specified')
settings.pusher_enabled.send_mail = nil
else
local r = {}
r.backend = 'send_mail'
r.mail_to = settings.mail_to
r.mail_from = settings.mail_from
r.helo = settings.hello
r.smtp = settings.smtp
r.smtp_port = settings.smtp_port
r.email_template = settings.email_template
r.defer = settings.defer
r.selector = settings.pusher_select.send_mail
r.formatter = settings.pusher_format.send_mail
settings.rules[r.backend:upper()] = r
end
end
if not next(settings.pusher_enabled) then
rspamd_logger.errx(rspamd_config, 'No push backend enabled')
return
end
elseif not next(settings.rules) then
lua_util.debugm(N, rspamd_config, 'No rules enabled')
return
end
if not settings.rules or not next(settings.rules) then
rspamd_logger.errx(rspamd_config, 'No rules enabled')
return
end
local backend_required_elements = {
http = {
'url',
},
smtp = {
'mail_to',
'smtp',
},
redis_pubsub = {
'channel',
},
}
local check_element = {
selector = function(k, v)
if not selectors[v] then
rspamd_logger.errx(rspamd_config, 'Rule %s has invalid selector %s', k, v)
return false
else
return true
end
end,
formatter = function(k, v)
if not formatters[v] then
rspamd_logger.errx(rspamd_config, 'Rule %s has invalid formatter %s', k, v)
return false
else
return true
end
end,
}
local backend_check = {
default = function(k, rule)
local reqset = backend_required_elements[rule.backend]
if reqset then
for _, e in ipairs(reqset) do
if not rule[e] then
rspamd_logger.errx(rspamd_config, 'Rule %s misses required setting %s', k, e)
settings.rules[k] = nil
end
end
end
for sett, v in pairs(rule) do
local f = check_element[sett]
if f then
if not f(sett, v) then
settings.rules[k] = nil
end
end
end
end,
}
backend_check.redis_pubsub = function(k, rule)
if not redis_params then
redis_params = rspamd_parse_redis_server(N)
end
if not redis_params then
rspamd_logger.errx(rspamd_config, 'No redis servers are specified')
settings.rules[k] = nil
else
backend_check.default(k, rule)
end
end
setmetatable(backend_check, {
__index = function()
return backend_check.default
end,
})
for k, v in pairs(settings.rules) do
if type(v) == 'table' then
local backend = v.backend
if not backend then
rspamd_logger.errx(rspamd_config, 'Rule %s has no backend', k)
settings.rules[k] = nil
elseif not pushers[backend] then
rspamd_logger.errx(rspamd_config, 'Rule %s has invalid backend %s', k, backend)
settings.rules[k] = nil
else
local f = backend_check[backend]
f(k, v)
end
else
rspamd_logger.errx(rspamd_config, 'Rule %s has bad type: %s', k, type(v))
settings.rules[k] = nil
end
end
local function gen_exporter(rule)
return function (task)
if task:has_flag('skip') then return end
local selector = rule.selector or 'default'
local selected = selectors[selector](task)
if selected then
lua_util.debugm(N, task, 'Message selected for processing')
local formatter = rule.formatter or 'default'
local formatted, extra = formatters[formatter](task, rule)
if formatted then
pushers[rule.backend](task, formatted, rule, extra)
else
lua_util.debugm(N, task, 'Formatter [%s] returned non-truthy value [%s]', formatter, formatted)
end
else
lua_util.debugm(N, task, 'Selector [%s] returned non-truthy value [%s]', selector, selected)
end
end
end
if not next(settings.rules) then
rspamd_logger.errx(rspamd_config, 'No rules enabled')
lua_util.disable_module(N, "config")
end
for k, r in pairs(settings.rules) do
rspamd_config:register_symbol({
name = 'EXPORT_METADATA_' .. k,
type = 'idempotent',
callback = gen_exporter(r),
priority = 10,
flags = 'empty,explicit_disable,ignore_passthrough',
})
end

View File

@@ -1,5 +1,5 @@
FROM debian:bullseye-slim
LABEL maintainer "The Infrastructure Company GmbH GmbH <info@servercow.de>"
LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
ARG DEBIAN_FRONTEND=noninteractive
ARG DEBIAN_VERSION=bullseye

View File

@@ -1,7 +1,15 @@
#!/bin/bash
if [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
if [[ "${FLATCURVE_EXPERIMENTAL}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
echo "FLATCURVE_EXPERIMENTAL=y, skipping Solr but enabling Flatcurve as FTS for Dovecot!"
echo "Solr will be removed in the future!"
sleep 365d
exit 0
elif [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
echo "SKIP_SOLR=y, skipping Solr..."
echo "HINT: You could try the newer FTS Backend Flatcurve, which is currently in experimental state..."
echo "Simply set FLATCURVE_EXPERIMENTAL=y inside your mailcow.conf and restart the stack afterwards!"
echo "Solr will be removed in the future!"
sleep 365d
exit 0
fi
@@ -57,5 +65,11 @@ if [[ "${1}" == "--bootstrap" ]]; then
exit 0
fi
echo "Starting up Solr..."
echo -e "\e[31mSolr is deprecated! You can try the new FTS System now by enabling FLATCURVE_EXPERIMENTAL=y inside mailcow.conf and restarting the stack\e[0m"
echo -e "\e[31mSolr will be removed completely soon!\e[0m"
sleep 15
exec gosu solr solr-foreground

View File

@@ -1,11 +1,10 @@
FROM alpine:3.18
FROM alpine:3.20
LABEL maintainer "The Infrastructure Company GmbH GmbH <info@servercow.de>"
LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
RUN apk add --update --no-cache \
curl \
bind-tools \
netcat-openbsd \
unbound \
bash \
openssl \
@@ -20,7 +19,7 @@ EXPOSE 53/udp 53/tcp
COPY docker-entrypoint.sh /docker-entrypoint.sh
# healthcheck (dig, ping, nc)
# healthcheck (dig, ping)
COPY healthcheck.sh /healthcheck.sh
RUN chmod +x /healthcheck.sh
HEALTHCHECK --interval=30s --timeout=30s CMD [ "/healthcheck.sh" ]

View File

@@ -5,9 +5,13 @@ if [[ "${SKIP_UNBOUND_HEALTHCHECK}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
SKIP_UNBOUND_HEALTHCHECK=y
fi
# Reset logfile
echo "$(date +"%Y-%m-%d %H:%M:%S"): Starting health check - logs can be found in /var/log/healthcheck.log"
echo "$(date +"%Y-%m-%d %H:%M:%S"): Starting health check" > /var/log/healthcheck.log
# Declare log function for logfile inside container
function log_to_file() {
echo "$(date +"%Y-%m-%d %H:%M:%S"): $1" > /var/log/healthcheck.log
echo "$(date +"%Y-%m-%d %H:%M:%S"): $1" >> /var/log/healthcheck.log
}
# General Ping function to check general pingability
@@ -50,27 +54,6 @@ function check_dns() {
}
# Simple Netcat Check to connect to common webports
function check_netcat() {
declare -a domains=("mailcow.email" "github.com" "hub.docker.com")
declare -a ports=("80" "443")
for domain in "${domains[@]}" ; do
for port in "${ports[@]}" ; do
nc -z -w 2 $domain $port
if [ $? -ne 0 ]; then
log_to_file "Healthcheck: Could not reach $domain on Port $port... Gave up!"
log_to_file "Please check your internet connection or firewall rules to fix this error."
return 1
fi
done
done
log_to_file "Healthcheck: Netcat Checks WORKING properly!"
return 0
}
if [[ ${SKIP_UNBOUND_HEALTHCHECK} == "y" ]]; then
log_to_file "Healthcheck: ALL CHECKS WERE SKIPPED! Unbound is healthy!"
exit 0
@@ -89,11 +72,5 @@ if [ $? -ne 0 ]; then
exit 1
fi
check_netcat
if [ $? -ne 0 ]; then
exit 1
fi
log_to_file "Healthcheck: ALL CHECKS WERE SUCCESSFUL! Unbound is healthy!"
exit 0

View File

@@ -1,4 +1,4 @@
FROM alpine:3.18
FROM alpine:3.20
LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
# Installation