public/mailcow-dockerized
1
0
Fork 0
mirror of https://github.com/mailcow/mailcow-dockerized.git synced 2026-07-23 13:13:29 +00:00
Code Issues Releases Wiki Activity

migrate from redis to valkey

Browse Source
This commit is contained in:
FreddleSpl0it
2025-02-28 15:36:19 +01:00
parent 35a6f81d0d
commit c27000215e
80 changed files with 754 additions and 753 deletions
Download Patch File Download Diff File Expand all files Collapse all files
data/Dockerfiles/acme/acme.sh
+8 -8
View File
@@ -3,14 +3,14 @@ set -o pipefail
exec 5>&1
# Do not attempt to write to slave
if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
export REDIS_CMDLINE="redis-cli -h ${REDIS_SLAVEOF_IP} -p ${REDIS_SLAVEOF_PORT} -a ${REDISPASS} --no-auth-warning"
if [[ ! -z ${VALKEY_SLAVEOF_IP} ]]; then
export VALKEY_CMDLINE="redis-cli -h ${VALKEY_SLAVEOF_IP} -p ${VALKEY_SLAVEOF_PORT} -a ${VALKEYPASS} --no-auth-warning"
else
export REDIS_CMDLINE="redis-cli -h redis -p 6379 -a ${REDISPASS} --no-auth-warning"
export VALKEY_CMDLINE="redis-cli -h valkey-mailcow -p 6379 -a ${VALKEYPASS} --no-auth-warning"
fi
until [[ $(${REDIS_CMDLINE} PING) == "PONG" ]]; do
echo "Waiting for Redis..."
until [[ $(${VALKEY_CMDLINE} PING) == "PONG" ]]; do
echo "Waiting for Valkey..."
sleep 2
done
@@ -360,7 +360,7 @@ while true; do
if [[ -z ${VALIDATED_CERTIFICATES[*]} ]]; then
log_f "Cannot validate any hostnames, skipping Let's Encrypt for 1 hour."
log_f "Use SKIP_LETS_ENCRYPT=y in mailcow.conf to skip it permanently."
${REDIS_CMDLINE} SET ACME_FAIL_TIME "$(date +%s)"
${VALKEY_CMDLINE} SET ACME_FAIL_TIME "$(date +%s)"
sleep 1h
exec $(readlink -f "$0")
fi
@@ -401,7 +401,7 @@ while true; do
DOVECOT_CERT_SERIAL_NEW="$(echo | openssl s_client -connect dovecot:143 -starttls imap 2>/dev/null | openssl x509 -inform pem -noout -serial | cut -d "=" -f 2)"
if [[ ${RELOAD_LOOP_C} -gt 3 ]]; then
log_f "Some services do return old end dates, something went wrong!"
${REDIS_CMDLINE} SET ACME_FAIL_TIME "$(date +%s)"
${VALKEY_CMDLINE} SET ACME_FAIL_TIME "$(date +%s)"
break;
fi
done
@@ -422,7 +422,7 @@ while true; do
;;
*) # non-zero
log_f "Some errors occurred, retrying in 30 minutes..."
${REDIS_CMDLINE} SET ACME_FAIL_TIME "$(date +%s)"
${VALKEY_CMDLINE} SET ACME_FAIL_TIME "$(date +%s)"
sleep 30m
exec $(readlink -f "$0")
;;
data/Dockerfiles/acme/functions.sh
+3 -3
View File
@@ -5,13 +5,13 @@ log_f() {
echo -n "$(date) - ${1}"
elif [[ ${2} == "no_date" ]]; then
echo "${1}"
elif [[ ${2} != "redis_only" ]]; then
elif [[ ${2} != "valkey_only" ]]; then
echo "$(date) - ${1}"
fi
if [[ ${3} == "b64" ]]; then
${REDIS_CMDLINE} LPUSH ACME_LOG "{\"time\":\"$(date +%s)\",\"message\":\"base64,$(printf '%s' "${MAILCOW_HOSTNAME} - ${1}")\"}" > /dev/null
${VALKEY_CMDLINE} LPUSH ACME_LOG "{\"time\":\"$(date +%s)\",\"message\":\"base64,$(printf '%s' "${MAILCOW_HOSTNAME} - ${1}")\"}" > /dev/null
else
${REDIS_CMDLINE} LPUSH ACME_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${MAILCOW_HOSTNAME} - ${1}" | \
${VALKEY_CMDLINE} LPUSH ACME_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${MAILCOW_HOSTNAME} - ${1}" | \
tr '%&;$"[]{}-\r\n' ' ')\"}" > /dev/null
fi
}
data/Dockerfiles/acme/obtain-certificate.sh
+2 -2
View File
@@ -101,7 +101,7 @@ ACME_RESPONSE=$(acme-tiny ${DIRECTORY_URL} ${ACME_CONTACT_PARAMETER} \
--acme-dir /var/www/acme/ 2>&1 > /tmp/_cert.pem | tee /dev/fd/5; exit ${PIPESTATUS[0]})
SUCCESS="$?"
ACME_RESPONSE_B64=$(echo "${ACME_RESPONSE}" | openssl enc -e -A -base64)
log_f "${ACME_RESPONSE_B64}" redis_only b64
log_f "${ACME_RESPONSE_B64}" valkey_only b64
case "$SUCCESS" in
0) # cert requested
log_f "Deploying certificate ${CERT}..."
@@ -124,7 +124,7 @@ case "$SUCCESS" in
;;
*) # non-zero is non-fun
log_f "Failed to obtain certificate ${CERT} for domains '${CERT_DOMAINS[*]}'"
redis-cli -h redis -a ${REDISPASS} --no-auth-warning SET ACME_FAIL_TIME "$(date +%s)"
redis-cli -h valkey-mailcow -a ${VALKEYPASS} --no-auth-warning SET ACME_FAIL_TIME "$(date +%s)"
exit 100${SUCCESS}
;;
esac
data/Dockerfiles/dockerapi/main.py
+14 -14
View File
@@ -32,21 +32,21 @@ async def lifespan(app: FastAPI):
logger.info("Init APP")
# Init redis client
if os.environ['REDIS_SLAVEOF_IP'] != "":
redis_client = redis = await aioredis.from_url(f"redis://{os.environ['REDIS_SLAVEOF_IP']}:{os.environ['REDIS_SLAVEOF_PORT']}/0", password=os.environ['REDISPASS'])
# Init valkey client
if os.environ['VALKEY_SLAVEOF_IP'] != "":
valkey_client = valkey = await aioredis.from_url(f"redis://{os.environ['VALKEY_SLAVEOF_IP']}:{os.environ['VALKEY_SLAVEOF_PORT']}/0", password=os.environ['VALKEYPASS'])
else:
redis_client = redis = await aioredis.from_url("redis://redis-mailcow:6379/0", password=os.environ['REDISPASS'])
valkey_client = valkey = await aioredis.from_url("redis://valkey-mailcow:6379/0", password=os.environ['VALKEYPASS'])
# Init docker clients
sync_docker_client = docker.DockerClient(base_url='unix://var/run/docker.sock', version='auto')
async_docker_client = aiodocker.Docker(url='unix:///var/run/docker.sock')
dockerapi = DockerApi(redis_client, sync_docker_client, async_docker_client, logger)
dockerapi = DockerApi(valkey_client, sync_docker_client, async_docker_client, logger)
logger.info("Subscribe to redis channel")
# Subscribe to redis channel
dockerapi.pubsub = redis.pubsub()
logger.info("Subscribe to valkey channel")
# Subscribe to valkey channel
dockerapi.pubsub = valkey.pubsub()
await dockerapi.pubsub.subscribe("MC_CHANNEL")
asyncio.create_task(handle_pubsub_messages(dockerapi.pubsub))
@@ -57,9 +57,9 @@ async def lifespan(app: FastAPI):
dockerapi.sync_docker_client.close()
await dockerapi.async_docker_client.close()
# Close redis
# Close valkey
await dockerapi.pubsub.unsubscribe("MC_CHANNEL")
await dockerapi.redis_client.close()
await dockerapi.valkey_client.close()
app = FastAPI(lifespan=lifespan)
@@ -73,11 +73,11 @@ async def get_host_update_stats():
dockerapi.host_stats_isUpdating = True
while True:
if await dockerapi.redis_client.exists('host_stats'):
if await dockerapi.valkey_client.exists('host_stats'):
break
await asyncio.sleep(1.5)
stats = json.loads(await dockerapi.redis_client.get('host_stats'))
stats = json.loads(await dockerapi.valkey_client.get('host_stats'))
return Response(content=json.dumps(stats, indent=4), media_type="application/json")
@app.get("/containers/{container_id}/json")
@@ -185,11 +185,11 @@ async def post_container_update_stats(container_id : str):
dockerapi.containerIds_to_update.append(container_id)
while True:
if await dockerapi.redis_client.exists(container_id + '_stats'):
if await dockerapi.valkey_client.exists(container_id + '_stats'):
break
await asyncio.sleep(1.5)
stats = json.loads(await dockerapi.redis_client.get(container_id + '_stats'))
stats = json.loads(await dockerapi.valkey_client.get(container_id + '_stats'))
return Response(content=json.dumps(stats, indent=4), media_type="application/json")
data/Dockerfiles/dockerapi/modules/DockerApi.py
+6 -6
View File
@@ -10,8 +10,8 @@ from datetime import datetime
from fastapi import FastAPI, Response, Request
class DockerApi:
def __init__(self, redis_client, sync_docker_client, async_docker_client, logger):
self.redis_client = redis_client
def __init__(self, valkey_client, sync_docker_client, async_docker_client, logger):
self.valkey_client = valkey_client
self.sync_docker_client = sync_docker_client
self.async_docker_client = async_docker_client
self.logger = logger
@@ -533,7 +533,7 @@ class DockerApi:
"architecture": platform.machine()
}
await self.redis_client.set('host_stats', json.dumps(host_stats), ex=10)
await self.valkey_client.set('host_stats', json.dumps(host_stats), ex=10)
except Exception as e:
res = {
"type": "danger",
@@ -550,14 +550,14 @@ class DockerApi:
if container._id == container_id:
res = await container.stats(stream=False)
if await self.redis_client.exists(container_id + '_stats'):
stats = json.loads(await self.redis_client.get(container_id + '_stats'))
if await self.valkey_client.exists(container_id + '_stats'):
stats = json.loads(await self.valkey_client.get(container_id + '_stats'))
else:
stats = []
stats.append(res[0])
if len(stats) > 3:
del stats[0]
await self.redis_client.set(container_id + '_stats', json.dumps(stats), ex=60)
await self.valkey_client.set(container_id + '_stats', json.dumps(stats), ex=60)
except Exception as e:
res = {
"type": "danger",
data/Dockerfiles/dovecot/Dockerfile
+1 -1
View File
@@ -114,7 +114,7 @@ RUN addgroup -g 5000 vmail \
COPY trim_logs.sh /usr/local/bin/trim_logs.sh
COPY clean_q_aged.sh /usr/local/bin/clean_q_aged.sh
COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
COPY syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng-redis_slave.conf
COPY syslog-ng-valkey_slave.conf /etc/syslog-ng/syslog-ng-valkey_slave.conf
COPY imapsync /usr/local/bin/imapsync
COPY imapsync_runner.pl /usr/local/bin/imapsync_runner.pl
COPY report-spam.sieve /usr/lib/dovecot/sieve/report-spam.sieve
data/Dockerfiles/dovecot/clean_q_aged.sh
+1 -1
View File
@@ -2,7 +2,7 @@
source /source_env.sh
MAX_AGE=$(redis-cli --raw -h redis-mailcow -a ${REDISPASS} --no-auth-warning GET Q_MAX_AGE)
MAX_AGE=$(redis-cli --raw -h valkey-mailcow -a ${VALKEYPASS} --no-auth-warning GET Q_MAX_AGE)
if [[ -z ${MAX_AGE} ]]; then
echo "Max age for quarantine items not defined"
data/Dockerfiles/dovecot/docker-entrypoint.sh
+8 -8
View File
@@ -13,18 +13,18 @@ until dig +short mailcow.email > /dev/null; do
done
# Do not attempt to write to slave
if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
REDIS_CMDLINE="redis-cli -h ${REDIS_SLAVEOF_IP} -p ${REDIS_SLAVEOF_PORT} -a ${REDISPASS} --no-auth-warning"
if [[ ! -z ${VALKEY_SLAVEOF_IP} ]]; then
VALKEY_CMDLINE="redis-cli -h ${VALKEY_SLAVEOF_IP} -p ${VALKEY_SLAVEOF_PORT} -a ${VALKEYPASS} --no-auth-warning"
else
REDIS_CMDLINE="redis-cli -h redis -p 6379 -a ${REDISPASS} --no-auth-warning"
VALKEY_CMDLINE="redis-cli -h valkey-mailcow -p 6379 -a ${VALKEYPASS} --no-auth-warning"
fi
until [[ $(${REDIS_CMDLINE} PING) == "PONG" ]]; do
echo "Waiting for Redis..."
until [[ $(${VALKEY_CMDLINE} PING) == "PONG" ]]; do
echo "Waiting for Valkey..."
sleep 2
done
${REDIS_CMDLINE} SET DOVECOT_REPL_HEALTH 1 > /dev/null
${VALKEY_CMDLINE} SET DOVECOT_REPL_HEALTH 1 > /dev/null
# Create missing directories
[[ ! -d /etc/dovecot/sql/ ]] && mkdir -p /etc/dovecot/sql/
@@ -458,8 +458,8 @@ done
# May be related to something inside Docker, I seriously don't know
touch /etc/dovecot/lua/passwd-verify.lua
if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
cp /etc/syslog-ng/syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng.conf
if [[ ! -z ${VALKEY_SLAVEOF_IP} ]]; then
cp /etc/syslog-ng/syslog-ng-valkey_slave.conf /etc/syslog-ng/syslog-ng.conf
fi
exec "$@"
data/Dockerfiles/dovecot/quarantine_notify.py
+1 -1
View File
@@ -31,7 +31,7 @@ try:
while True:
try:
r = redis.StrictRedis(host='redis', decode_responses=True, port=6379, db=0, password=os.environ['REDISPASS'])
r = redis.StrictRedis(host='valkey-mailcow', decode_responses=True, port=6379, db=0, password=os.environ['VALKEYPASS'])
r.ping()
except Exception as ex:
print('%s - trying again...' % (ex))
data/Dockerfiles/dovecot/quota_notify.py
+1 -1
View File
@@ -23,7 +23,7 @@ else:
while True:
try:
r = redis.StrictRedis(host='redis', decode_responses=True, port=6379, db=0, username='quota_notify', password='')
r = redis.StrictRedis(host='valkey-mailcow', decode_responses=True, port=6379, db=0, username='quota_notify', password='')
r.ping()
except Exception as ex:
print('%s - trying again...' % (ex))
data/Dockerfiles/dovecot/repl_health.sh
+6 -6
View File
@@ -3,16 +3,16 @@
source /source_env.sh
# Do not attempt to write to slave
if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
REDIS_CMDLINE="redis-cli -h ${REDIS_SLAVEOF_IP} -p ${REDIS_SLAVEOF_PORT} -a ${REDISPASS} --no-auth-warning"
if [[ ! -z ${VALKEY_SLAVEOF_IP} ]]; then
VALKEY_CMDLINE="redis-cli -h ${VALKEY_SLAVEOF_IP} -p ${VALKEY_SLAVEOF_PORT} -a ${VALKEYPASS} --no-auth-warning"
else
REDIS_CMDLINE="redis-cli -h redis -p 6379 -a ${REDISPASS} --no-auth-warning"
VALKEY_CMDLINE="redis-cli -h valkey-mailcow -p 6379 -a ${VALKEYPASS} --no-auth-warning"
fi
# Is replication active?
# grep on file is less expensive than doveconf
if [ -n ${MAILCOW_REPLICA_IP} ]; then
${REDIS_CMDLINE} SET DOVECOT_REPL_HEALTH 1 > /dev/null
${VALKEY_CMDLINE} SET DOVECOT_REPL_HEALTH 1 > /dev/null
exit
fi
@@ -22,7 +22,7 @@ FAILED_SYNCS=$(doveadm replicator status | grep "Waiting 'failed' requests" | gr
# 1 failed job for mailcow.local is expected and healthy
if [[ "${FAILED_SYNCS}" != 0 ]] && [[ "${FAILED_SYNCS}" != 1 ]]; then
printf "Dovecot replicator has %d failed jobs\n" "${FAILED_SYNCS}"
${REDIS_CMDLINE} SET DOVECOT_REPL_HEALTH "${FAILED_SYNCS}" > /dev/null
${VALKEY_CMDLINE} SET DOVECOT_REPL_HEALTH "${FAILED_SYNCS}" > /dev/null
else
${REDIS_CMDLINE} SET DOVECOT_REPL_HEALTH 1 > /dev/null
${VALKEY_CMDLINE} SET DOVECOT_REPL_HEALTH 1 > /dev/null
fi
data/Dockerfiles/dovecot/syslog-ng-redis_slave.conf → data/Dockerfiles/dovecot/syslog-ng-valkey_slave.conf
+12 -12
View File
@@ -15,21 +15,21 @@ source s_dgram {
internal();
};
destination d_stdout { pipe("/dev/stdout"); };
destination d_redis_ui_log {
destination d_valkey_ui_log {
redis(
host("`REDIS_SLAVEOF_IP`")
persist-name("redis1")
port(`REDIS_SLAVEOF_PORT`)
auth("`REDISPASS`")
host("`VALKEY_SLAVEOF_IP`")
persist-name("valkey1")
port(`VALKEY_SLAVEOF_PORT`)
auth("`VALKEYPASS`")
command("LPUSH" "DOVECOT_MAILLOG" "$(format-json time=\"$S_UNIXTIME\" priority=\"$PRIORITY\" program=\"$PROGRAM\" message=\"$MESSAGE\")\n")
);
};
destination d_redis_f2b_channel {
destination d_valkey_f2b_channel {
redis(
host("`REDIS_SLAVEOF_IP`")
persist-name("redis2")
port(`REDIS_SLAVEOF_PORT`)
auth("`REDISPASS`")
host("`VALKEY_SLAVEOF_IP`")
persist-name("valkey2")
port(`VALKEY_SLAVEOF_PORT`)
auth("`VALKEYPASS`")
command("PUBLISH" "F2B_CHANNEL" "$(sanitize $MESSAGE)")
);
};
@@ -48,6 +48,6 @@ log {
filter(f_replica);
destination(d_stdout);
filter(f_mail);
destination(d_redis_ui_log);
destination(d_redis_f2b_channel);
destination(d_valkey_ui_log);
destination(d_valkey_f2b_channel);
};
data/Dockerfiles/dovecot/syslog-ng.conf
+10 -10
View File
@@ -15,21 +15,21 @@ source s_dgram {
internal();
};
destination d_stdout { pipe("/dev/stdout"); };
destination d_redis_ui_log {
destination d_valkey_ui_log {
redis(
host("redis-mailcow")
persist-name("redis1")
host("valkey-mailcow")
persist-name("valkey1")
port(6379)
auth("`REDISPASS`")
auth("`VALKEYPASS`")
command("LPUSH" "DOVECOT_MAILLOG" "$(format-json time=\"$S_UNIXTIME\" priority=\"$PRIORITY\" program=\"$PROGRAM\" message=\"$MESSAGE\")\n")
);
};
destination d_redis_f2b_channel {
destination d_valkey_f2b_channel {
redis(
host("redis-mailcow")
persist-name("redis2")
host("valkey-mailcow")
persist-name("valkey2")
port(6379)
auth("`REDISPASS`")
auth("`VALKEYPASS`")
command("PUBLISH" "F2B_CHANNEL" "$(sanitize $MESSAGE)")
);
};
@@ -48,6 +48,6 @@ log {
filter(f_replica);
destination(d_stdout);
filter(f_mail);
destination(d_redis_ui_log);
destination(d_redis_f2b_channel);
destination(d_valkey_ui_log);
destination(d_valkey_f2b_channel);
};
data/Dockerfiles/dovecot/trim_logs.sh
+12 -12
View File
@@ -9,17 +9,17 @@ catch_non_zero() {
}
source /source_env.sh
# Do not attempt to write to slave
if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
REDIS_CMDLINE="redis-cli -h ${REDIS_SLAVEOF_IP} -p ${REDIS_SLAVEOF_PORT} -a ${REDISPASS} --no-auth-warning"
if [[ ! -z ${VALKEY_SLAVEOF_IP} ]]; then
VALKEY_CMDLINE="redis-cli -h ${VALKEY_SLAVEOF_IP} -p ${VALKEY_SLAVEOF_PORT} -a ${VALKEYPASS} --no-auth-warning"
else
REDIS_CMDLINE="redis-cli -h redis -p 6379 -a ${REDISPASS} --no-auth-warning"
VALKEY_CMDLINE="redis-cli -h valkey-mailcow -p 6379 -a ${VALKEYPASS} --no-auth-warning"
fi
catch_non_zero "${REDIS_CMDLINE} LTRIM ACME_LOG 0 ${LOG_LINES}"
catch_non_zero "${REDIS_CMDLINE} LTRIM POSTFIX_MAILLOG 0 ${LOG_LINES}"
catch_non_zero "${REDIS_CMDLINE} LTRIM DOVECOT_MAILLOG 0 ${LOG_LINES}"
catch_non_zero "${REDIS_CMDLINE} LTRIM SOGO_LOG 0 ${LOG_LINES}"
catch_non_zero "${REDIS_CMDLINE} LTRIM NETFILTER_LOG 0 ${LOG_LINES}"
catch_non_zero "${REDIS_CMDLINE} LTRIM AUTODISCOVER_LOG 0 ${LOG_LINES}"
catch_non_zero "${REDIS_CMDLINE} LTRIM API_LOG 0 ${LOG_LINES}"
catch_non_zero "${REDIS_CMDLINE} LTRIM RL_LOG 0 ${LOG_LINES}"
catch_non_zero "${REDIS_CMDLINE} LTRIM WATCHDOG_LOG 0 ${LOG_LINES}"
catch_non_zero "${VALKEY_CMDLINE} LTRIM ACME_LOG 0 ${LOG_LINES}"
catch_non_zero "${VALKEY_CMDLINE} LTRIM POSTFIX_MAILLOG 0 ${LOG_LINES}"
catch_non_zero "${VALKEY_CMDLINE} LTRIM DOVECOT_MAILLOG 0 ${LOG_LINES}"
catch_non_zero "${VALKEY_CMDLINE} LTRIM SOGO_LOG 0 ${LOG_LINES}"
catch_non_zero "${VALKEY_CMDLINE} LTRIM NETFILTER_LOG 0 ${LOG_LINES}"
catch_non_zero "${VALKEY_CMDLINE} LTRIM AUTODISCOVER_LOG 0 ${LOG_LINES}"
catch_non_zero "${VALKEY_CMDLINE} LTRIM API_LOG 0 ${LOG_LINES}"
catch_non_zero "${VALKEY_CMDLINE} LTRIM RL_LOG 0 ${LOG_LINES}"
catch_non_zero "${VALKEY_CMDLINE} LTRIM WATCHDOG_LOG 0 ${LOG_LINES}"
data/Dockerfiles/netfilter/main.py
+40 -40
View File
@@ -41,24 +41,24 @@ def refreshF2boptions():
f2boptions = {}
if not r.get('F2B_OPTIONS'):
f2boptions['ban_time'] = r.get('F2B_BAN_TIME')
f2boptions['max_ban_time'] = r.get('F2B_MAX_BAN_TIME')
f2boptions['ban_time_increment'] = r.get('F2B_BAN_TIME_INCREMENT')
f2boptions['max_attempts'] = r.get('F2B_MAX_ATTEMPTS')
f2boptions['retry_window'] = r.get('F2B_RETRY_WINDOW')
f2boptions['netban_ipv4'] = r.get('F2B_NETBAN_IPV4')
f2boptions['netban_ipv6'] = r.get('F2B_NETBAN_IPV6')
if not valkey.get('F2B_OPTIONS'):
f2boptions['ban_time'] = valkey.get('F2B_BAN_TIME')
f2boptions['max_ban_time'] = valkey.get('F2B_MAX_BAN_TIME')
f2boptions['ban_time_increment'] = valkey.get('F2B_BAN_TIME_INCREMENT')
f2boptions['max_attempts'] = valkey.get('F2B_MAX_ATTEMPTS')
f2boptions['retry_window'] = valkey.get('F2B_RETRY_WINDOW')
f2boptions['netban_ipv4'] = valkey.get('F2B_NETBAN_IPV4')
f2boptions['netban_ipv6'] = valkey.get('F2B_NETBAN_IPV6')
else:
try:
f2boptions = json.loads(r.get('F2B_OPTIONS'))
f2boptions = json.loads(valkey.get('F2B_OPTIONS'))
except ValueError:
logger.logCrit('Error loading F2B options: F2B_OPTIONS is not json')
quit_now = True
exit_code = 2
verifyF2boptions(f2boptions)
r.set('F2B_OPTIONS', json.dumps(f2boptions, ensure_ascii=False))
valkey.set('F2B_OPTIONS', json.dumps(f2boptions, ensure_ascii=False))
def verifyF2boptions(f2boptions):
verifyF2boption(f2boptions,'ban_time', 1800)
@@ -78,7 +78,7 @@ def refreshF2bregex():
global f2bregex
global quit_now
global exit_code
if not r.get('F2B_REGEX'):
if not valkey.get('F2B_REGEX'):
f2bregex = {}
f2bregex[1] = r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)'
f2bregex[2] = r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)'
@@ -89,11 +89,11 @@ def refreshF2bregex():
f2bregex[7] = r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)'
f2bregex[8] = r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked'
f2bregex[9] = r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+'
r.set('F2B_REGEX', json.dumps(f2bregex, ensure_ascii=False))
valkey.set('F2B_REGEX', json.dumps(f2bregex, ensure_ascii=False))
else:
try:
f2bregex = {}
f2bregex = json.loads(r.get('F2B_REGEX'))
f2bregex = json.loads(valkey.get('F2B_REGEX'))
except ValueError:
logger.logCrit('Error loading F2B options: F2B_REGEX is not json')
quit_now = True
@@ -156,7 +156,7 @@ def ban(address):
with lock:
tables.banIPv6(net)
r.hset('F2B_ACTIVE_BANS', '%s' % net, cur_time + NET_BAN_TIME)
valkey.hset('F2B_ACTIVE_BANS', '%s' % net, cur_time + NET_BAN_TIME)
else:
logger.logWarn('%d more attempts in the next %d seconds until %s is banned' % (MAX_ATTEMPTS - bans[net]['attempts'], RETRY_WINDOW, net))
@@ -165,7 +165,7 @@ def unban(net):
if not net in bans:
logger.logInfo('%s is not banned, skipping unban and deleting from queue (if any)' % net)
r.hdel('F2B_QUEUE_UNBAN', '%s' % net)
valkey.hdel('F2B_QUEUE_UNBAN', '%s' % net)
return
logger.logInfo('Unbanning %s' % net)
@@ -176,8 +176,8 @@ def unban(net):
with lock:
tables.unbanIPv6(net)
r.hdel('F2B_ACTIVE_BANS', '%s' % net)
r.hdel('F2B_QUEUE_UNBAN', '%s' % net)
valkey.hdel('F2B_ACTIVE_BANS', '%s' % net)
valkey.hdel('F2B_QUEUE_UNBAN', '%s' % net)
if net in bans:
bans[net]['attempts'] = 0
bans[net]['ban_counter'] += 1
@@ -203,10 +203,10 @@ def permBan(net, unban=False):
if is_unbanned:
r.hdel('F2B_PERM_BANS', '%s' % net)
valkey.hdel('F2B_PERM_BANS', '%s' % net)
logger.logCrit('Removed host/network %s from blacklist' % net)
elif is_banned:
r.hset('F2B_PERM_BANS', '%s' % net, int(round(time.time())))
valkey.hset('F2B_PERM_BANS', '%s' % net, int(round(time.time())))
logger.logCrit('Added host/network %s to blacklist' % net)
def clear():
@@ -219,17 +219,17 @@ def clear():
tables.clearIPv6Table()
try:
if r is not None:
r.delete('F2B_ACTIVE_BANS')
r.delete('F2B_PERM_BANS')
valkey.delete('F2B_ACTIVE_BANS')
valkey.delete('F2B_PERM_BANS')
except Exception as ex:
logger.logWarn('Error clearing redis keys F2B_ACTIVE_BANS and F2B_PERM_BANS: %s' % ex)
logger.logWarn('Error clearing valkey keys F2B_ACTIVE_BANS and F2B_PERM_BANS: %s' % ex)
def watch():
global pubsub
global quit_now
global exit_code
logger.logInfo('Watching Redis channel F2B_CHANNEL')
logger.logInfo('Watching Valkey channel F2B_CHANNEL')
pubsub.subscribe('F2B_CHANNEL')
while not quit_now:
@@ -280,7 +280,7 @@ def autopurge():
time.sleep(10)
refreshF2boptions()
MAX_ATTEMPTS = int(f2boptions['max_attempts'])
QUEUE_UNBAN = r.hgetall('F2B_QUEUE_UNBAN')
QUEUE_UNBAN = valkey.hgetall('F2B_QUEUE_UNBAN')
if QUEUE_UNBAN:
for net in QUEUE_UNBAN:
unban(str(net))
@@ -352,7 +352,7 @@ def whitelistUpdate():
global WHITELIST
while not quit_now:
start_time = time.time()
list = r.hgetall('F2B_WHITELIST')
list = valkey.hgetall('F2B_WHITELIST')
new_whitelist = []
if list:
new_whitelist = genNetworkList(list)
@@ -367,7 +367,7 @@ def blacklistUpdate():
global BLACKLIST
while not quit_now:
start_time = time.time()
list = r.hgetall('F2B_BLACKLIST')
list = valkey.hgetall('F2B_BLACKLIST')
new_blacklist = []
if list:
new_blacklist = genNetworkList(list)
@@ -427,30 +427,30 @@ if __name__ == '__main__':
logger.logInfo(f"Setting {chain_name} isolation")
tables.create_mailcow_isolation_rule("br-mailcow", [3306, 6379, 8983, 12345], os.getenv("MAILCOW_REPLICA_IP"))
# connect to redis
# connect to valkey
while True:
try:
redis_slaveof_ip = os.getenv('REDIS_SLAVEOF_IP', '')
redis_slaveof_port = os.getenv('REDIS_SLAVEOF_PORT', '')
if "".__eq__(redis_slaveof_ip):
r = redis.StrictRedis(host=os.getenv('IPV4_NETWORK', '172.22.1') + '.249', decode_responses=True, port=6379, db=0, password=os.environ['REDISPASS'])
valkey_slaveof_ip = os.getenv('VALKEY_SLAVEOF_IP', '')
valkey_slaveof_port = os.getenv('VALKEY_SLAVEOF_PORT', '')
if "".__eq__(valkey_slaveof_ip):
valkey = redis.StrictRedis(host=os.getenv('IPV4_NETWORK', '172.22.1') + '.249', decode_responses=True, port=6379, db=0, password=os.environ['VALKEYPASS'])
else:
r = redis.StrictRedis(host=redis_slaveof_ip, decode_responses=True, port=redis_slaveof_port, db=0, password=os.environ['REDISPASS'])
r.ping()
pubsub = r.pubsub()
valkey = redis.StrictRedis(host=valkey_slaveof_ip, decode_responses=True, port=valkey_slaveof_port, db=0, password=os.environ['VALKEYPASS'])
valkey.ping()
pubsub = valkey.pubsub()
except Exception as ex:
print('%s - trying again in 3 seconds' % (ex))
time.sleep(3)
else:
break
logger.set_redis(r)
logger.set_valkey(r)
# rename fail2ban to netfilter
if r.exists('F2B_LOG'):
r.rename('F2B_LOG', 'NETFILTER_LOG')
# clear bans in redis
r.delete('F2B_ACTIVE_BANS')
r.delete('F2B_PERM_BANS')
if valkey.exists('F2B_LOG'):
valkey.rename('F2B_LOG', 'NETFILTER_LOG')
# clear bans in valkey
valkey.delete('F2B_ACTIVE_BANS')
valkey.delete('F2B_PERM_BANS')
refreshF2boptions()
data/Dockerfiles/netfilter/modules/Logger.py
+6 -6
View File
@@ -3,10 +3,10 @@ import json
class Logger:
def __init__(self):
self.r = None
self.valkey = None
def set_redis(self, redis):
self.r = redis
def set_valkey(self, valkey):
self.valkey = valkey
def log(self, priority, message):
tolog = {}
@@ -14,11 +14,11 @@ class Logger:
tolog['priority'] = priority
tolog['message'] = message
print(message)
if self.r is not None:
if self.valkey is not None:
try:
self.r.lpush('NETFILTER_LOG', json.dumps(tolog, ensure_ascii=False))
self.valkey.lpush('NETFILTER_LOG', json.dumps(tolog, ensure_ascii=False))
except Exception as ex:
print('Failed logging to redis: %s' % (ex))
print('Failed logging to valkey: %s' % (ex))
def logWarn(self, message):
self.log('warn', message)
data/Dockerfiles/phpfpm/docker-entrypoint.sh
+23 -23
View File
@@ -9,24 +9,24 @@ while ! mariadb-admin status --ssl=false --socket=/var/run/mysqld/mysqld.sock -u
done
# Do not attempt to write to slave
if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
REDIS_HOST=$REDIS_SLAVEOF_IP
REDIS_PORT=$REDIS_SLAVEOF_PORT
if [[ ! -z ${VALKEY_SLAVEOF_IP} ]]; then
VALKEY_HOST=$VALKEY_SLAVEOF_IP
VALKEY_PORT=$VALKEY_SLAVEOF_PORT
else
REDIS_HOST="redis"
REDIS_PORT="6379"
VALKEY_HOST="valkey-mailcow"
VALKEY_PORT="6379"
fi
REDIS_CMDLINE="redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} -a ${REDISPASS} --no-auth-warning"
VALKEY_CMDLINE="redis-cli -h ${VALKEY_HOST} -p ${VALKEY_PORT} -a ${VALKEYPASS} --no-auth-warning"
until [[ $(${REDIS_CMDLINE} PING) == "PONG" ]]; do
echo "Waiting for Redis..."
until [[ $(${VALKEY_CMDLINE} PING) == "PONG" ]]; do
echo "Waiting for Valkey..."
sleep 2
done
# Set redis session store
# Set valkey session store
echo -n '
session.save_handler = redis
session.save_path = "tcp://'${REDIS_HOST}':'${REDIS_PORT}'?auth='${REDISPASS}'"
session.save_path = "tcp://'${VALKEY_HOST}':'${VALKEY_PORT}'?auth='${VALKEYPASS}'"
' > /usr/local/etc/php/conf.d/session_store.ini
# Check mysql_upgrade (master and slave)
@@ -91,22 +91,22 @@ fi
if [[ "${MASTER}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
echo "We are master, preparing..."
# Set a default release format
if [[ -z $(${REDIS_CMDLINE} --raw GET Q_RELEASE_FORMAT) ]]; then
${REDIS_CMDLINE} --raw SET Q_RELEASE_FORMAT raw
if [[ -z $(${VALKEY_CMDLINE} --raw GET Q_RELEASE_FORMAT) ]]; then
${VALKEY_CMDLINE} --raw SET Q_RELEASE_FORMAT raw
fi
# Set max age of q items - if unset
if [[ -z $(${REDIS_CMDLINE} --raw GET Q_MAX_AGE) ]]; then
${REDIS_CMDLINE} --raw SET Q_MAX_AGE 365
if [[ -z $(${VALKEY_CMDLINE} --raw GET Q_MAX_AGE) ]]; then
${VALKEY_CMDLINE} --raw SET Q_MAX_AGE 365
fi
# Set default password policy - if unset
if [[ -z $(${REDIS_CMDLINE} --raw HGET PASSWD_POLICY length) ]]; then
${REDIS_CMDLINE} --raw HSET PASSWD_POLICY length 6
${REDIS_CMDLINE} --raw HSET PASSWD_POLICY chars 0
${REDIS_CMDLINE} --raw HSET PASSWD_POLICY special_chars 0
${REDIS_CMDLINE} --raw HSET PASSWD_POLICY lowerupper 0
${REDIS_CMDLINE} --raw HSET PASSWD_POLICY numbers 0
if [[ -z $(${VALKEY_CMDLINE} --raw HGET PASSWD_POLICY length) ]]; then
${VALKEY_CMDLINE} --raw HSET PASSWD_POLICY length 6
${VALKEY_CMDLINE} --raw HSET PASSWD_POLICY chars 0
${VALKEY_CMDLINE} --raw HSET PASSWD_POLICY special_chars 0
${VALKEY_CMDLINE} --raw HSET PASSWD_POLICY lowerupper 0
${VALKEY_CMDLINE} --raw HSET PASSWD_POLICY numbers 0
fi
# Trigger db init
@@ -114,9 +114,9 @@ if [[ "${MASTER}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
php -c /usr/local/etc/php -f /web/inc/init_db.inc.php
# Recreating domain map
echo "Rebuilding domain map in Redis..."
echo "Rebuilding domain map in Valkey..."
declare -a DOMAIN_ARR
${REDIS_CMDLINE} DEL DOMAIN_MAP > /dev/null
${VALKEY_CMDLINE} DEL DOMAIN_MAP > /dev/null
while read line
do
DOMAIN_ARR+=("$line")
@@ -128,7 +128,7 @@ if [[ "${MASTER}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
if [[ ! -z ${DOMAIN_ARR} ]]; then
for domain in "${DOMAIN_ARR[@]}"; do
${REDIS_CMDLINE} HSET DOMAIN_MAP ${domain} 1 > /dev/null
${VALKEY_CMDLINE} HSET DOMAIN_MAP ${domain} 1 > /dev/null
done
fi
data/Dockerfiles/postfix/Dockerfile
+1 -1
View File
@@ -41,7 +41,7 @@ RUN groupadd -g 102 postfix \
COPY supervisord.conf /etc/supervisor/supervisord.conf
COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
COPY syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng-redis_slave.conf
COPY syslog-ng-valkey_slave.conf /etc/syslog-ng/syslog-ng-valkey_slave.conf
COPY postfix.sh /opt/postfix.sh
COPY rspamd-pipe-ham /usr/local/bin/rspamd-pipe-ham
COPY rspamd-pipe-spam /usr/local/bin/rspamd-pipe-spam
data/Dockerfiles/postfix/docker-entrypoint.sh
+3 -3
View File
@@ -8,8 +8,8 @@ for file in /hooks/*; do
fi
done
if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
cp /etc/syslog-ng/syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng.conf
if [[ ! -z ${VALKEY_SLAVEOF_IP} ]]; then
cp /etc/syslog-ng/syslog-ng-valkey_slave.conf /etc/syslog-ng/syslog-ng.conf
fi
# Fix OpenSSL 3.X TLS1.0, 1.1 support (https://community.mailcow.email/d/4062-hi-all/20)
@@ -21,6 +21,6 @@ if grep -qE '\!SSLv2|\!SSLv3|>=TLSv1(\.[0-1])?$' /opt/postfix/conf/main.cf /opt/
echo "[tls_system_default]" >> /etc/ssl/openssl.cnf
echo "MinProtocol = TLSv1" >> /etc/ssl/openssl.cnf
echo "CipherString = DEFAULT@SECLEVEL=0" >> /etc/ssl/openssl.cnf
fi
fi
exec "$@"
data/Dockerfiles/postfix/syslog-ng-redis_slave.conf → data/Dockerfiles/postfix/syslog-ng-valkey_slave.conf
+12 -12
View File
@@ -15,21 +15,21 @@ source s_src {
internal();
};
destination d_stdout { pipe("/dev/stdout"); };
destination d_redis_ui_log {
destination d_valkey_ui_log {
redis(
host("`REDIS_SLAVEOF_IP`")
persist-name("redis1")
port(`REDIS_SLAVEOF_PORT`)
auth("`REDISPASS`")
host("`VALKEY_SLAVEOF_IP`")
persist-name("valkey1")
port(`VALKEY_SLAVEOF_PORT`)
auth("`VALKEYPASS`")
command("LPUSH" "POSTFIX_MAILLOG" "$(format-json time=\"$S_UNIXTIME\" priority=\"$PRIORITY\" program=\"$PROGRAM\" message=\"$MESSAGE\")\n")
);
};
destination d_redis_f2b_channel {
destination d_valkey_f2b_channel {
redis(
host("`REDIS_SLAVEOF_IP`")
persist-name("redis2")
port(`REDIS_SLAVEOF_PORT`)
auth("`REDISPASS`")
host("`VALKEY_SLAVEOF_IP`")
persist-name("valkey2")
port(`VALKEY_SLAVEOF_PORT`)
auth("`VALKEYPASS`")
command("PUBLISH" "F2B_CHANNEL" "$(sanitize $MESSAGE)")
);
};
@@ -50,6 +50,6 @@ log {
filter(f_ignore);
destination(d_stdout);
filter(f_mail);
destination(d_redis_ui_log);
destination(d_redis_f2b_channel);
destination(d_valkey_ui_log);
destination(d_valkey_f2b_channel);
};
data/Dockerfiles/postfix/syslog-ng.conf
+10 -10
View File
@@ -15,21 +15,21 @@ source s_src {
internal();
};
destination d_stdout { pipe("/dev/stdout"); };
destination d_redis_ui_log {
destination d_valkey_ui_log {
redis(
host("redis-mailcow")
persist-name("redis1")
host("valkey-mailcow")
persist-name("valkey1")
port(6379)
auth("`REDISPASS`")
auth("`VALKEYPASS`")
command("LPUSH" "POSTFIX_MAILLOG" "$(format-json time=\"$S_UNIXTIME\" priority=\"$PRIORITY\" program=\"$PROGRAM\" message=\"$MESSAGE\")\n")
);
};
destination d_redis_f2b_channel {
destination d_valkey_f2b_channel {
redis(
host("redis-mailcow")
persist-name("redis2")
host("valkey-mailcow")
persist-name("valkey2")
port(6379)
auth("`REDISPASS`")
auth("`VALKEYPASS`")
command("PUBLISH" "F2B_CHANNEL" "$(sanitize $MESSAGE)")
);
};
@@ -50,6 +50,6 @@ log {
filter(f_ignore);
destination(d_stdout);
filter(f_mail);
destination(d_redis_ui_log);
destination(d_redis_f2b_channel);
destination(d_valkey_ui_log);
destination(d_valkey_f2b_channel);
};
data/Dockerfiles/rspamd/docker-entrypoint.sh
+14 -14
View File
@@ -52,33 +52,33 @@ if [[ ! -z ${RSPAMD_V6} ]]; then
echo ${RSPAMD_V6}/128 >> /etc/rspamd/custom/rspamd_trusted.map
fi
if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
if [[ ! -z ${VALKEY_SLAVEOF_IP} ]]; then
cat <<EOF > /etc/rspamd/local.d/redis.conf
read_servers = "redis:6379";
write_servers = "${REDIS_SLAVEOF_IP}:${REDIS_SLAVEOF_PORT}";
password = "${REDISPASS}";
read_servers = "valkey-mailcow:6379";
write_servers = "${VALKEY_SLAVEOF_IP}:${VALKEY_SLAVEOF_PORT}";
password = "${VALKEYPASS}";
timeout = 10;
EOF
until [[ $(redis-cli -h redis-mailcow -a ${REDISPASS} --no-auth-warning PING) == "PONG" ]]; do
echo "Waiting for Redis @redis-mailcow..."
until [[ $(redis-cli -h valkey-mailcow -a ${VALKEYPASS} --no-auth-warning PING) == "PONG" ]]; do
echo "Waiting for Valkey @valkey-mailcow..."
sleep 2
done
until [[ $(redis-cli -h ${REDIS_SLAVEOF_IP} -p ${REDIS_SLAVEOF_PORT} -a ${REDISPASS} --no-auth-warning PING) == "PONG" ]]; do
echo "Waiting for Redis @${REDIS_SLAVEOF_IP}..."
until [[ $(redis-cli -h ${VALKEY_SLAVEOF_IP} -p ${VALKEY_SLAVEOF_PORT} -a ${VALKEYPASS} --no-auth-warning PING) == "PONG" ]]; do
echo "Waiting for Valkey @${VALKEY_SLAVEOF_IP}..."
sleep 2
done
redis-cli -h redis-mailcow -a ${REDISPASS} --no-auth-warning SLAVEOF ${REDIS_SLAVEOF_IP} ${REDIS_SLAVEOF_PORT}
redis-cli -h valkey-mailcow -a ${VALKEYPASS} --no-auth-warning SLAVEOF ${VALKEY_SLAVEOF_IP} ${VALKEY_SLAVEOF_PORT}
else
cat <<EOF > /etc/rspamd/local.d/redis.conf
servers = "redis:6379";
password = "${REDISPASS}";
servers = "valkey-mailcow:6379";
password = "${VALKEYPASS}";
timeout = 10;
EOF
until [[ $(redis-cli -h redis-mailcow -a ${REDISPASS} --no-auth-warning PING) == "PONG" ]]; do
echo "Waiting for Redis slave..."
until [[ $(redis-cli -h valkey-mailcow -a ${VALKEYPASS} --no-auth-warning PING) == "PONG" ]]; do
echo "Waiting for Valkey slave..."
sleep 2
done
redis-cli -h redis-mailcow -a ${REDISPASS} --no-auth-warning SLAVEOF NO ONE
redis-cli -h valkey-mailcow -a ${VALKEYPASS} --no-auth-warning SLAVEOF NO ONE
fi
# Provide additional lua modules
data/Dockerfiles/sogo/Dockerfile
+1 -1
View File
@@ -44,7 +44,7 @@ RUN echo "Building from repository $SOGO_DEBIAN_REPOSITORY" \
COPY ./bootstrap-sogo.sh /bootstrap-sogo.sh
COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
COPY syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng-redis_slave.conf
COPY syslog-ng-valkey_slave.conf /etc/syslog-ng/syslog-ng-valkey_slave.conf
COPY supervisord.conf /etc/supervisor/supervisord.conf
COPY acl.diff /acl.diff
COPY stop-supervisor.sh /usr/local/sbin/stop-supervisor.sh
data/Dockerfiles/sogo/docker-entrypoint.sh
+2 -2
View File
@@ -6,8 +6,8 @@ if [[ "${SKIP_SOGO}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
exit 0
fi
if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
cp /etc/syslog-ng/syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng.conf
if [[ ! -z ${VALKEY_SLAVEOF_IP} ]]; then
cp /etc/syslog-ng/syslog-ng-valkey_slave.conf /etc/syslog-ng/syslog-ng.conf
fi
echo "$TZ" > /etc/timezone
data/Dockerfiles/sogo/syslog-ng-redis_slave.conf → data/Dockerfiles/sogo/syslog-ng-valkey_slave.conf
+12 -12
View File
@@ -17,28 +17,28 @@ source s_sogo {
pipe("/dev/sogo_log" owner(sogo) group(sogo));
};
destination d_stdout { pipe("/dev/stdout"); };
destination d_redis_ui_log {
destination d_valkey_ui_log {
redis(
host("`REDIS_SLAVEOF_IP`")
persist-name("redis1")
port(`REDIS_SLAVEOF_PORT`)
auth("`REDISPASS`")
host("`VALKEY_SLAVEOF_IP`")
persist-name("valkey1")
port(`VALKEY_SLAVEOF_PORT`)
auth("`VALKEYPASS`")
command("LPUSH" "SOGO_LOG" "$(format-json time=\"$S_UNIXTIME\" priority=\"$PRIORITY\" program=\"$PROGRAM\" message=\"$MESSAGE\")\n")
);
};
destination d_redis_f2b_channel {
destination d_valkey_f2b_channel {
redis(
host("`REDIS_SLAVEOF_IP`")
persist-name("redis2")
port(`REDIS_SLAVEOF_PORT`)
auth("`REDISPASS`")
host("`VALKEY_SLAVEOF_IP`")
persist-name("valkey2")
port(`VALKEY_SLAVEOF_PORT`)
auth("`VALKEYPASS`")
command("PUBLISH" "F2B_CHANNEL" "$(sanitize $MESSAGE)")
);
};
log {
source(s_sogo);
destination(d_redis_ui_log);
destination(d_redis_f2b_channel);
destination(d_valkey_ui_log);
destination(d_valkey_f2b_channel);
};
log {
source(s_sogo);
data/Dockerfiles/sogo/syslog-ng.conf
+10 -10
View File
@@ -17,28 +17,28 @@ source s_sogo {
pipe("/dev/sogo_log" owner(sogo) group(sogo));
};
destination d_stdout { pipe("/dev/stdout"); };
destination d_redis_ui_log {
destination d_valkey_ui_log {
redis(
host("redis-mailcow")
persist-name("redis1")
host("valkey-mailcow")
persist-name("valkey1")
port(6379)
auth("`REDISPASS`")
auth("`VALKEYPASS`")
command("LPUSH" "SOGO_LOG" "$(format-json time=\"$S_UNIXTIME\" priority=\"$PRIORITY\" program=\"$PROGRAM\" message=\"$MESSAGE\")\n")
);
};
destination d_redis_f2b_channel {
destination d_valkey_f2b_channel {
redis(
host("redis-mailcow")
persist-name("redis2")
host("valkey-mailcow")
persist-name("valkey2")
port(6379)
auth("`REDISPASS`")
auth("`VALKEYPASS`")
command("PUBLISH" "F2B_CHANNEL" "$(sanitize $MESSAGE)")
);
};
log {
source(s_sogo);
destination(d_redis_ui_log);
destination(d_redis_f2b_channel);
destination(d_valkey_ui_log);
destination(d_valkey_f2b_channel);
};
log {
source(s_sogo);
data/Dockerfiles/watchdog/watchdog.sh
+37 -37
View File
@@ -39,18 +39,18 @@ while ! mariadb-admin status --ssl=false --socket=/var/run/mysqld/mysqld.sock -u
done
# Do not attempt to write to slave
if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
REDIS_CMDLINE="redis-cli -h ${REDIS_SLAVEOF_IP} -p ${REDIS_SLAVEOF_PORT} -a ${REDISPASS} --no-auth-warning"
if [[ ! -z ${VALKEY_SLAVEOF_IP} ]]; then
VALKEY_CMDLINE="redis-cli -h ${VALKEY_SLAVEOF_IP} -p ${VALKEY_SLAVEOF_PORT} -a ${VALKEYPASS} --no-auth-warning"
else
REDIS_CMDLINE="redis-cli -h redis -p 6379 -a ${REDISPASS} --no-auth-warning"
VALKEY_CMDLINE="redis-cli -h valkey-mailcow -p 6379 -a ${VALKEYPASS} --no-auth-warning"
fi
until [[ $(${REDIS_CMDLINE} PING) == "PONG" ]]; do
echo "Waiting for Redis..."
until [[ $(${VALKEY_CMDLINE} PING) == "PONG" ]]; do
echo "Waiting for Valkey..."
sleep 2
done
${REDIS_CMDLINE} DEL F2B_RES > /dev/null
${VALKEY_CMDLINE} DEL F2B_RES > /dev/null
# Common functions
get_ipv6(){
@@ -85,15 +85,15 @@ progress() {
[[ ${CURRENT} -gt ${TOTAL} ]] && return
[[ ${CURRENT} -lt 0 ]] && CURRENT=0
PERCENT=$(( 200 * ${CURRENT} / ${TOTAL} % 2 + 100 * ${CURRENT} / ${TOTAL} ))
${REDIS_CMDLINE} LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"service\":\"${SERVICE}\",\"lvl\":\"${PERCENT}\",\"hpnow\":\"${CURRENT}\",\"hptotal\":\"${TOTAL}\",\"hpdiff\":\"${DIFF}\"}" > /dev/null
log_msg "${SERVICE} health level: ${PERCENT}% (${CURRENT}/${TOTAL}), health trend: ${DIFF}" no_redis
${VALKEY_CMDLINE} LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"service\":\"${SERVICE}\",\"lvl\":\"${PERCENT}\",\"hpnow\":\"${CURRENT}\",\"hptotal\":\"${TOTAL}\",\"hpdiff\":\"${DIFF}\"}" > /dev/null
log_msg "${SERVICE} health level: ${PERCENT}% (${CURRENT}/${TOTAL}), health trend: ${DIFF}" no_valkey
# Return 10 to indicate a dead service
[ ${CURRENT} -le 0 ] && return 10
}
log_msg() {
if [[ ${2} != "no_redis" ]]; then
${REDIS_CMDLINE} LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}" | \
if [[ ${2} != "no_valkey" ]]; then
${VALKEY_CMDLINE} LPUSH WATCHDOG_LOG "{\"time\":\"$(date +%s)\",\"message\":\"$(printf '%s' "${1}" | \
tr '\r\n%&;$"_[]{}-' ' ')\"}" > /dev/null
fi
echo $(date) $(printf '%s\n' "${1}")
@@ -109,10 +109,10 @@ function notify_error() {
# If exists, mail will be throttled by argument in seconds
[[ ! -z ${3} ]] && THROTTLE=${3}
if [[ ! -z ${THROTTLE} ]]; then
TTL_LEFT="$(${REDIS_CMDLINE} TTL THROTTLE_${1} 2> /dev/null)"
TTL_LEFT="$(${VALKEY_CMDLINE} TTL THROTTLE_${1} 2> /dev/null)"
if [[ "${TTL_LEFT}" == "-2" ]]; then
# Delay key not found, setting a delay key now
${REDIS_CMDLINE} SET THROTTLE_${1} 1 EX ${THROTTLE}
${VALKEY_CMDLINE} SET THROTTLE_${1} 1 EX ${THROTTLE}
else
log_msg "Not sending notification email now, blocked for ${TTL_LEFT} seconds..."
return 1
@@ -319,21 +319,21 @@ unbound_checks() {
return 1
}
redis_checks() {
# A check for the local redis container
valkey_checks() {
# A check for the local valkey container
err_count=0
diff_c=0
THRESHOLD=${REDIS_THRESHOLD}
THRESHOLD=${VALKEY_THRESHOLD}
# Reduce error count by 2 after restarting an unhealthy container
trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
while [ ${err_count} -lt ${THRESHOLD} ]; do
touch /tmp/redis-mailcow; echo "$(tail -50 /tmp/redis-mailcow)" > /tmp/redis-mailcow
host_ip=$(get_container_ip redis-mailcow)
touch /tmp/valkey-mailcow; echo "$(tail -50 /tmp/valkey-mailcow)" > /tmp/valkey-mailcow
host_ip=$(get_container_ip valkey-mailcow)
err_c_cur=${err_count}
/usr/lib/nagios/plugins/check_tcp -4 -H redis-mailcow -p 6379 -E -s "AUTH ${REDISPASS}\nPING\n" -q "QUIT" -e "PONG" 2>> /tmp/redis-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
/usr/lib/nagios/plugins/check_tcp -4 -H valkey-mailcow -p 6379 -E -s "AUTH ${VALKEYPASS}\nPING\n" -q "QUIT" -e "PONG" 2>> /tmp/valkey-mailcow 1>&2; err_count=$(( ${err_count} + $? ))
[ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
[ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
progress "Redis" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
progress "Valkey" ${THRESHOLD} $(( ${THRESHOLD} - ${err_count} )) ${diff_c}
if [[ $? == 10 ]]; then
diff_c=0
sleep 1
@@ -503,12 +503,12 @@ dovecot_repl_checks() {
err_count=0
diff_c=0
THRESHOLD=${DOVECOT_REPL_THRESHOLD}
D_REPL_STATUS=$(redis-cli -h redis -a ${REDISPASS} --no-auth-warning -r GET DOVECOT_REPL_HEALTH)
D_REPL_STATUS=$(redis-cli -h valkey-mailcow -a ${VALKEYPASS} --no-auth-warning -r GET DOVECOT_REPL_HEALTH)
# Reduce error count by 2 after restarting an unhealthy container
trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
while [ ${err_count} -lt ${THRESHOLD} ]; do
err_c_cur=${err_count}
D_REPL_STATUS=$(redis-cli --raw -h redis -a ${REDISPASS} --no-auth-warning GET DOVECOT_REPL_HEALTH)
D_REPL_STATUS=$(redis-cli --raw -h valkey-mailcow -a ${VALKEYPASS} --no-auth-warning GET DOVECOT_REPL_HEALTH)
if [[ "${D_REPL_STATUS}" != "1" ]]; then
err_count=$(( ${err_count} + 1 ))
fi
@@ -578,19 +578,19 @@ ratelimit_checks() {
err_count=0
diff_c=0
THRESHOLD=${RATELIMIT_THRESHOLD}
RL_LOG_STATUS=$(redis-cli -h redis -a ${REDISPASS} --no-auth-warning LRANGE RL_LOG 0 0 | jq .qid)
RL_LOG_STATUS=$(redis-cli -h valkey-mailcow -a ${VALKEYPASS} --no-auth-warning LRANGE RL_LOG 0 0 | jq .qid)
# Reduce error count by 2 after restarting an unhealthy container
trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
while [ ${err_count} -lt ${THRESHOLD} ]; do
err_c_cur=${err_count}
RL_LOG_STATUS_PREV=${RL_LOG_STATUS}
RL_LOG_STATUS=$(redis-cli -h redis -a ${REDISPASS} --no-auth-warning LRANGE RL_LOG 0 0 | jq .qid)
RL_LOG_STATUS=$(redis-cli -h valkey-mailcow -a ${VALKEYPASS} --no-auth-warning LRANGE RL_LOG 0 0 | jq .qid)
if [[ ${RL_LOG_STATUS_PREV} != ${RL_LOG_STATUS} ]]; then
err_count=$(( ${err_count} + 1 ))
echo 'Last 10 applied ratelimits (may overlap with previous reports).' > /tmp/ratelimit
echo 'Full ratelimit buckets can be emptied by deleting the ratelimit hash from within mailcow UI (see /debug -> Protocols -> Ratelimit):' >> /tmp/ratelimit
echo >> /tmp/ratelimit
redis-cli --raw -h redis -a ${REDISPASS} --no-auth-warning LRANGE RL_LOG 0 10 | jq . >> /tmp/ratelimit
redis-cli --raw -h valkey-mailcow -a ${VALKEYPASS} --no-auth-warning LRANGE RL_LOG 0 10 | jq . >> /tmp/ratelimit
fi
[ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
[ ${err_c_cur} -ne ${err_count} ] && diff_c=$(( ${err_c_cur} - ${err_count} ))
@@ -639,20 +639,20 @@ fail2ban_checks() {
err_count=0
diff_c=0
THRESHOLD=${FAIL2BAN_THRESHOLD}
F2B_LOG_STATUS=($(${REDIS_CMDLINE} --raw HKEYS F2B_ACTIVE_BANS))
F2B_LOG_STATUS=($(${VALKEY_CMDLINE} --raw HKEYS F2B_ACTIVE_BANS))
F2B_RES=
# Reduce error count by 2 after restarting an unhealthy container
trap "[ ${err_count} -gt 1 ] && err_count=$(( ${err_count} - 2 ))" USR1
while [ ${err_count} -lt ${THRESHOLD} ]; do
err_c_cur=${err_count}
F2B_LOG_STATUS_PREV=(${F2B_LOG_STATUS[@]})
F2B_LOG_STATUS=($(${REDIS_CMDLINE} --raw HKEYS F2B_ACTIVE_BANS))
F2B_LOG_STATUS=($(${VALKEY_CMDLINE} --raw HKEYS F2B_ACTIVE_BANS))
array_diff F2B_RES F2B_LOG_STATUS F2B_LOG_STATUS_PREV
if [[ ! -z "${F2B_RES}" ]]; then
err_count=$(( ${err_count} + 1 ))
echo -n "${F2B_RES[@]}" | tr -cd "[a-fA-F0-9.:/] " | timeout 3s ${REDIS_CMDLINE} -x SET F2B_RES > /dev/null
echo -n "${F2B_RES[@]}" | tr -cd "[a-fA-F0-9.:/] " | timeout 3s ${VALKEY_CMDLINE} -x SET F2B_RES > /dev/null
if [ $? -ne 0 ]; then
${REDIS_CMDLINE} -x DEL F2B_RES
${VALKEY_CMDLINE} -x DEL F2B_RES
fi
fi
[ ${err_c_cur} -eq ${err_count} ] && [ ! $((${err_count} - 1)) -lt 0 ] && err_count=$((${err_count} - 1)) diff_c=1
@@ -673,9 +673,9 @@ acme_checks() {
err_count=0
diff_c=0
THRESHOLD=${ACME_THRESHOLD}
ACME_LOG_STATUS=$(redis-cli -h redis -a ${REDISPASS} --no-auth-warning GET ACME_FAIL_TIME)
ACME_LOG_STATUS=$(redis-cli -h valkey-mailcow -a ${VALKEYPASS} --no-auth-warning GET ACME_FAIL_TIME)
if [[ -z "${ACME_LOG_STATUS}" ]]; then
${REDIS_CMDLINE} SET ACME_FAIL_TIME 0
${VALKEY_CMDLINE} SET ACME_FAIL_TIME 0
ACME_LOG_STATUS=0
fi
# Reduce error count by 2 after restarting an unhealthy container
@@ -685,7 +685,7 @@ acme_checks() {
ACME_LOG_STATUS_PREV=${ACME_LOG_STATUS}
ACME_LC=0
until [[ ! -z ${ACME_LOG_STATUS} ]] || [ ${ACME_LC} -ge 3 ]; do
ACME_LOG_STATUS=$(redis-cli -h redis -a ${REDISPASS} --no-auth-warning GET ACME_FAIL_TIME 2> /dev/null)
ACME_LOG_STATUS=$(redis-cli -h valkey-mailcow -a ${VALKEYPASS} --no-auth-warning GET ACME_FAIL_TIME 2> /dev/null)
sleep 3
ACME_LC=$((ACME_LC+1))
done
@@ -834,14 +834,14 @@ BACKGROUND_TASKS+=(${PID})
(
while true; do
if ! redis_checks; then
log_msg "Local Redis hit error limit"
echo redis-mailcow > /tmp/com_pipe
if ! valkey_checks; then
log_msg "Local Valkey hit error limit"
echo valkey-mailcow > /tmp/com_pipe
fi
done
) &
PID=$!
echo "Spawned redis_checks with PID ${PID}"
echo "Spawned valkey_checks with PID ${PID}"
BACKGROUND_TASKS+=(${PID})
(
@@ -1085,9 +1085,9 @@ while true; do
# Define $2 to override message text, else print service was restarted at ...
notify_error "${com_pipe_answer}" "Please check acme-mailcow for further information."
elif [[ ${com_pipe_answer} == "fail2ban" ]]; then
F2B_RES=($(timeout 4s ${REDIS_CMDLINE} --raw GET F2B_RES 2> /dev/null))
F2B_RES=($(timeout 4s ${VALKEY_CMDLINE} --raw GET F2B_RES 2> /dev/null))
if [[ ! -z "${F2B_RES}" ]]; then
${REDIS_CMDLINE} DEL F2B_RES > /dev/null
${VALKEY_CMDLINE} DEL F2B_RES > /dev/null
host=
for host in "${F2B_RES[@]}"; do
log_msg "Banned ${host}"