mirror of
https://github.com/mailcow/mailcow-dockerized.git
synced 2025-12-20 13:21:30 +00:00
[Dovecot] use python bootstrapper to start DOVECOT container
This commit is contained in:
FreddleSpl0it
@@ -9,21 +9,25 @@ import time
|
||||
import socket
|
||||
import signal
|
||||
import re
|
||||
import redis
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import mysql.connector
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
|
||||
class BootstrapBase:
|
||||
def __init__(self, container, db_config, db_table, db_settings):
|
||||
def __init__(self, container, db_config, db_table, db_settings, redis_config):
|
||||
self.container = container
|
||||
self.db_config = db_config
|
||||
self.db_table = db_table
|
||||
self.db_settings = db_settings
|
||||
self.redis_config = redis_config
|
||||
|
||||
self.env = None
|
||||
self.env_vars = None
|
||||
self.mysql_conn = None
|
||||
self.redis_conn = None
|
||||
|
||||
def render_config(self, template_name, output_path):
|
||||
"""
|
||||
@@ -184,16 +188,21 @@ class BootstrapBase:
|
||||
|
||||
Args:
|
||||
path (str or Path): Path to the file or directory.
|
||||
user (str): Username for new owner.
|
||||
group (str, optional): Group name; defaults to user's group if not provided.
|
||||
user (str or int): Username or UID for new owner.
|
||||
group (str or int, optional): Group name or GID; defaults to user's group if not provided.
|
||||
recursive (bool): If True and path is a directory, ownership is applied recursively.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the path does not exist.
|
||||
"""
|
||||
|
||||
uid = pwd.getpwnam(user).pw_uid
|
||||
gid = grp.getgrnam(group or user).gr_gid
|
||||
# Resolve UID
|
||||
uid = int(user) if str(user).isdigit() else pwd.getpwnam(user).pw_uid
|
||||
# Resolve GID
|
||||
if group is not None:
|
||||
gid = int(group) if str(group).isdigit() else grp.getgrnam(group).gr_gid
|
||||
else:
|
||||
gid = uid if isinstance(user, int) or str(user).isdigit() else grp.getgrnam(user).gr_gid
|
||||
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
@@ -231,6 +240,67 @@ class BootstrapBase:
|
||||
|
||||
shutil.move(str(src_path), str(dst_path))
|
||||
|
||||
def copy_file(self, src, dst, overwrite=True):
|
||||
"""
|
||||
Copies a file from src to dst using shutil.
|
||||
|
||||
Args:
|
||||
src (str or Path): Source file path.
|
||||
dst (str or Path): Destination file path.
|
||||
overwrite (bool): Whether to overwrite the destination if it exists.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the source file doesn't exist.
|
||||
FileExistsError: If the destination exists and overwrite is False.
|
||||
IOError: If the copy operation fails.
|
||||
"""
|
||||
|
||||
src_path = Path(src)
|
||||
dst_path = Path(dst)
|
||||
|
||||
if not src_path.is_file():
|
||||
raise FileNotFoundError(f"Source file not found: {src_path}")
|
||||
|
||||
if dst_path.exists() and not overwrite:
|
||||
raise FileExistsError(f"Destination exists: {dst_path}")
|
||||
|
||||
dst_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
shutil.copy2(src_path, dst_path)
|
||||
|
||||
def remove(self, path, recursive=False, wipe_contents=False):
|
||||
"""
|
||||
Removes a file or directory.
|
||||
|
||||
Args:
|
||||
path (str or Path): The file or directory path to remove.
|
||||
recursive (bool): If True, directories will be removed recursively.
|
||||
wipe_contents (bool): If True and path is a directory, only its contents are removed, not the dir itself.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the path does not exist.
|
||||
ValueError: If a directory is passed without recursive or wipe_contents.
|
||||
"""
|
||||
|
||||
path = Path(path)
|
||||
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Cannot remove: {path} does not exist")
|
||||
|
||||
if wipe_contents and path.is_dir():
|
||||
for child in path.iterdir():
|
||||
if child.is_dir():
|
||||
shutil.rmtree(child)
|
||||
else:
|
||||
child.unlink()
|
||||
elif path.is_file():
|
||||
path.unlink()
|
||||
elif path.is_dir():
|
||||
if recursive:
|
||||
shutil.rmtree(path)
|
||||
else:
|
||||
raise ValueError(f"{path} is a directory. Use recursive=True or wipe_contents=True to remove it.")
|
||||
|
||||
def create_dir(self, path):
|
||||
"""
|
||||
Creates a directory if it does not exist.
|
||||
@@ -376,6 +446,49 @@ class BootstrapBase:
|
||||
if self.mysql_conn and self.mysql_conn.is_connected():
|
||||
self.mysql_conn.close()
|
||||
|
||||
def connect_redis(self, retries=10, delay=2):
|
||||
"""
|
||||
Establishes a Redis connection and stores it in `self.redis_conn`.
|
||||
|
||||
Args:
|
||||
retries (int): Number of ping retries before giving up.
|
||||
delay (int): Seconds between retries.
|
||||
"""
|
||||
|
||||
client = redis.Redis(
|
||||
host=self.redis_config['host'],
|
||||
port=self.redis_config['port'],
|
||||
password=self.redis_config['password'],
|
||||
db=self.redis_config['db'],
|
||||
decode_responses=True
|
||||
)
|
||||
|
||||
for _ in range(retries):
|
||||
try:
|
||||
if client.ping():
|
||||
self.redis_conn = client
|
||||
return
|
||||
except redis.RedisError as e:
|
||||
print(f"Waiting for Redis... ({e})")
|
||||
time.sleep(delay)
|
||||
|
||||
raise ConnectionError("Redis is not available after multiple attempts.")
|
||||
|
||||
def close_redis(self):
|
||||
"""
|
||||
Closes the Redis connection if it's open.
|
||||
|
||||
Safe to call even if Redis was never connected or already closed.
|
||||
"""
|
||||
|
||||
if self.redis_conn:
|
||||
try:
|
||||
self.redis_conn.close()
|
||||
except Exception as e:
|
||||
print(f"Error while closing Redis connection: {e}")
|
||||
finally:
|
||||
self.redis_conn = None
|
||||
|
||||
def wait_for_schema_update(self, init_file_path="init_db.inc.php", check_interval=5):
|
||||
"""
|
||||
Waits until the current database schema version matches the expected version
|
||||
@@ -559,4 +672,7 @@ class BootstrapBase:
|
||||
print(e.stderr.strip())
|
||||
if check:
|
||||
raise
|
||||
return e
|
||||
return e
|
||||
|
||||
def sha1_filter(self, value):
|
||||
return hashlib.sha1(value.encode()).hexdigest()
|
||||
|
||||
307
data/Dockerfiles/bootstrap/modules/BootstrapDovecot.py
Normal file
307
data/Dockerfiles/bootstrap/modules/BootstrapDovecot.py
Normal file
@@ -0,0 +1,307 @@
|
||||
from jinja2 import Environment, FileSystemLoader
|
||||
from modules.BootstrapBase import BootstrapBase
|
||||
from pathlib import Path
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import pwd
|
||||
import hashlib
|
||||
|
||||
class Bootstrap(BootstrapBase):
|
||||
def bootstrap(self):
|
||||
# Connect to MySQL
|
||||
self.connect_mysql()
|
||||
self.wait_for_schema_update()
|
||||
|
||||
# Connect to Redis
|
||||
self.connect_redis()
|
||||
self.redis_conn.set("DOVECOT_REPL_HEALTH", 1)
|
||||
|
||||
# Wait for DNS
|
||||
self.wait_for_dns("mailcow.email")
|
||||
|
||||
# Create missing directories
|
||||
self.create_dir("/etc/dovecot/sql/")
|
||||
self.create_dir("/etc/dovecot/auth/")
|
||||
self.create_dir("/var/vmail/_garbage")
|
||||
self.create_dir("/var/vmail/sieve")
|
||||
self.create_dir("/etc/sogo")
|
||||
self.create_dir("/var/volatile")
|
||||
|
||||
# Setup Jinja2 Environment and load vars
|
||||
self.env = Environment(
|
||||
loader=FileSystemLoader('./etc/dovecot/config_templates'),
|
||||
keep_trailing_newline=True,
|
||||
lstrip_blocks=True,
|
||||
trim_blocks=True
|
||||
)
|
||||
extra_vars = {
|
||||
"VALID_CERT_DIRS": self.get_valid_cert_dirs(),
|
||||
"RAND_USER": self.rand_pass(),
|
||||
"RAND_PASS": self.rand_pass(),
|
||||
"RAND_PASS2": self.rand_pass(),
|
||||
"ENV_VARS": dict(os.environ)
|
||||
}
|
||||
self.env_vars = self.prepare_template_vars('/overwrites.json', extra_vars)
|
||||
# Escape DBPASS
|
||||
self.env_vars['DBPASS'] = self.env_vars['DBPASS'].replace('"', r'\"')
|
||||
# Set custom filters
|
||||
self.env.filters['sha1'] = self.sha1_filter
|
||||
|
||||
print("Set Timezone")
|
||||
self.set_timezone()
|
||||
|
||||
print("Render config")
|
||||
self.render_config("dovecot-dict-sql-quota.conf.j2", "/etc/dovecot/sql/dovecot-dict-sql-quota.conf")
|
||||
self.render_config("dovecot-dict-sql-userdb.conf.j2", "/etc/dovecot/sql/dovecot-dict-sql-userdb.conf")
|
||||
self.render_config("dovecot-dict-sql-sieve_before.conf.j2", "/etc/dovecot/sql/dovecot-dict-sql-sieve_before.conf")
|
||||
self.render_config("dovecot-dict-sql-sieve_after.conf.j2", "/etc/dovecot/sql/dovecot-dict-sql-sieve_after.conf")
|
||||
self.render_config("mail_plugins.j2", "/etc/dovecot/mail_plugins")
|
||||
self.render_config("mail_plugins_imap.j2", "/etc/dovecot/mail_plugins_imap")
|
||||
self.render_config("mail_plugins_lmtp.j2", "/etc/dovecot/mail_plugins_lmtp")
|
||||
self.render_config("global_sieve_after.sieve.j2", "/var/vmail/sieve/global_sieve_after.sieve")
|
||||
self.render_config("global_sieve_before.sieve.j2", "/var/vmail/sieve/global_sieve_before.sieve")
|
||||
self.render_config("dovecot-master.passwd.j2", "/etc/dovecot/dovecot-master.passwd")
|
||||
self.render_config("dovecot-master.userdb.j2", "/etc/dovecot/dovecot-master.userdb")
|
||||
self.render_config("sieve.creds.j2", "/etc/sogo/sieve.creds")
|
||||
self.render_config("sogo-sso.pass.j2", "/etc/phpfpm/sogo-sso.pass")
|
||||
self.render_config("cron.creds.j2", "/etc/sogo/cron.creds")
|
||||
self.render_config("source_env.sh.j2", "/source_env.sh")
|
||||
self.render_config("maildir_gc.sh.j2", "/usr/local/bin/maildir_gc.sh")
|
||||
self.render_config("dovecot.conf.j2", "/etc/dovecot/dovecot.conf")
|
||||
|
||||
files = [
|
||||
"/etc/dovecot/mail_plugins",
|
||||
"/etc/dovecot/mail_plugins_imap",
|
||||
"/etc/dovecot/mail_plugins_lmtp",
|
||||
"/templates/quarantine.tpl"
|
||||
]
|
||||
for file in files:
|
||||
self.set_permissions(file, 0o644)
|
||||
|
||||
try:
|
||||
# Migrate old sieve_after file
|
||||
self.move_file("/etc/dovecot/sieve_after", "/var/vmail/sieve/global_sieve_after.sieve")
|
||||
except Exception as e:
|
||||
pass
|
||||
try:
|
||||
# Cleanup random user maildirs
|
||||
self.remove("/var/vmail/mailcow.local", wipe_contents=True)
|
||||
except Exception as e:
|
||||
pass
|
||||
try:
|
||||
# Cleanup PIDs
|
||||
self.remove("/tmp/quarantine_notify.pid")
|
||||
except Exception as e:
|
||||
pass
|
||||
try:
|
||||
self.remove("/var/run/dovecot/master.pid")
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
# Check permissions of vmail/index/garbage directories.
|
||||
# Do not do this every start-up, it may take a very long time. So we use a stat check here.
|
||||
files = [
|
||||
"/var/vmail",
|
||||
"/var/vmail/_garbage",
|
||||
"/var/vmail_index"
|
||||
]
|
||||
for file in files:
|
||||
path = Path(file)
|
||||
try:
|
||||
stat_info = path.stat()
|
||||
current_user = pwd.getpwuid(stat_info.st_uid).pw_name
|
||||
|
||||
if current_user != "vmail":
|
||||
print(f"Ownership of {path} is {current_user}, fixing to vmail:vmail...")
|
||||
self.set_owner(path, user="vmail", group="vmail", recursive=True)
|
||||
else:
|
||||
print(f"Ownership of {path} is already correct (vmail)")
|
||||
except Exception as e:
|
||||
print(f"Error checking ownership of {path}: {e}")
|
||||
|
||||
# Compile sieve scripts
|
||||
files = [
|
||||
"/var/vmail/sieve/global_sieve_before.sieve",
|
||||
"/var/vmail/sieve/global_sieve_after.sieve",
|
||||
"/usr/lib/dovecot/sieve/report-spam.sieve",
|
||||
"/usr/lib/dovecot/sieve/report-ham.sieve",
|
||||
]
|
||||
for file in files:
|
||||
self.run_command(["sievec", file], check=False)
|
||||
|
||||
# Fix permissions
|
||||
for path in Path("/etc/dovecot/sql").glob("*.conf"):
|
||||
self.set_owner(path, "root", "root")
|
||||
self.set_permissions(path, 0o640)
|
||||
|
||||
files = [
|
||||
"/etc/dovecot/auth/passwd-verify.lua",
|
||||
*Path("/etc/dovecot/sql").glob("dovecot-dict-sql-sieve*"),
|
||||
*Path("/etc/dovecot/sql").glob("dovecot-dict-sql-quota*")
|
||||
]
|
||||
for file in files:
|
||||
self.set_owner(file, "root", "dovecot")
|
||||
|
||||
self.set_permissions("/etc/dovecot/auth/passwd-verify.lua", 0o640)
|
||||
|
||||
for file in ["/var/vmail/sieve", "/var/volatile", "/var/vmail_index"]:
|
||||
self.set_owner(file, "vmail", "vmail", recursive=True)
|
||||
|
||||
self.run_command(["adduser", "vmail", "tty"])
|
||||
self.run_command(["chmod", "g+rw", "/dev/console"])
|
||||
self.set_owner("/dev/console", "root", "tty")
|
||||
files = [
|
||||
"/usr/lib/dovecot/sieve/rspamd-pipe-ham",
|
||||
"/usr/lib/dovecot/sieve/rspamd-pipe-spam",
|
||||
"/usr/local/bin/imapsync_runner.pl",
|
||||
"/usr/local/bin/imapsync",
|
||||
"/usr/local/bin/trim_logs.sh",
|
||||
"/usr/local/bin/sa-rules.sh",
|
||||
"/usr/local/bin/clean_q_aged.sh",
|
||||
"/usr/local/bin/maildir_gc.sh",
|
||||
"/usr/local/sbin/stop-supervisor.sh",
|
||||
"/usr/local/bin/quota_notify.py",
|
||||
"/usr/local/bin/repl_health.sh",
|
||||
"/usr/local/bin/optimize-fts.sh"
|
||||
]
|
||||
for file in files:
|
||||
self.set_permissions(file, 0o755)
|
||||
|
||||
# Collect SA rules once now
|
||||
self.run_command(["/usr/local/bin/sa-rules.sh"], check=False)
|
||||
|
||||
self.generate_mail_crypt_keys()
|
||||
self.cleanup_imapsync_jobs()
|
||||
self.generate_guid_version()
|
||||
|
||||
def get_valid_cert_dirs(self):
|
||||
"""
|
||||
Returns a mapping of domains to their certificate directory path.
|
||||
|
||||
Example:
|
||||
{
|
||||
"example.com": "/etc/ssl/mail/example.com/",
|
||||
"www.example.com": "/etc/ssl/mail/example.com/"
|
||||
}
|
||||
"""
|
||||
sni_map = {}
|
||||
base_path = Path("/etc/ssl/mail")
|
||||
if not base_path.exists():
|
||||
return sni_map
|
||||
|
||||
for cert_dir in base_path.iterdir():
|
||||
if not cert_dir.is_dir():
|
||||
continue
|
||||
|
||||
domains_file = cert_dir / "domains"
|
||||
cert_file = cert_dir / "cert.pem"
|
||||
key_file = cert_dir / "key.pem"
|
||||
|
||||
if not (domains_file.exists() and cert_file.exists() and key_file.exists()):
|
||||
continue
|
||||
|
||||
with open(domains_file, "r") as f:
|
||||
domains = [line.strip() for line in f if line.strip()]
|
||||
for domain in domains:
|
||||
sni_map[domain] = str(cert_dir)
|
||||
|
||||
return sni_map
|
||||
|
||||
def generate_mail_crypt_keys(self):
|
||||
"""
|
||||
Ensures mail_crypt EC keypair exists. Generates if missing. Adjusts permissions.
|
||||
"""
|
||||
|
||||
key_dir = Path("/mail_crypt")
|
||||
priv_key = key_dir / "ecprivkey.pem"
|
||||
pub_key = key_dir / "ecpubkey.pem"
|
||||
|
||||
# Generate keys if they don't exist or are empty
|
||||
if not priv_key.exists() or priv_key.stat().st_size == 0 or \
|
||||
not pub_key.exists() or pub_key.stat().st_size == 0:
|
||||
self.run_command(
|
||||
"openssl ecparam -name prime256v1 -genkey | openssl pkey -out /mail_crypt/ecprivkey.pem",
|
||||
shell=True
|
||||
)
|
||||
self.run_command(
|
||||
"openssl pkey -in /mail_crypt/ecprivkey.pem -pubout -out /mail_crypt/ecpubkey.pem",
|
||||
shell=True
|
||||
)
|
||||
|
||||
# Set ownership to UID 401 (dovecot)
|
||||
self.set_owner(priv_key, user='401')
|
||||
self.set_owner(pub_key, user='401')
|
||||
|
||||
def cleanup_imapsync_jobs(self):
|
||||
"""
|
||||
Cleans up stale imapsync locks and resets running status in the database.
|
||||
|
||||
Deletes the imapsync_busy.lock file if present and sets `is_running` to 0
|
||||
in the `imapsync` table, if it exists.
|
||||
|
||||
Logs:
|
||||
Any issues with file operations or SQL execution.
|
||||
"""
|
||||
|
||||
lock_file = Path("/tmp/imapsync_busy.lock")
|
||||
if lock_file.exists():
|
||||
try:
|
||||
lock_file.unlink()
|
||||
except Exception as e:
|
||||
print(f"Failed to remove lock file: {e}")
|
||||
|
||||
try:
|
||||
cursor = self.mysql_conn.cursor()
|
||||
cursor.execute("SHOW TABLES LIKE 'imapsync'")
|
||||
result = cursor.fetchone()
|
||||
if result:
|
||||
cursor.execute("UPDATE imapsync SET is_running='0'")
|
||||
self.mysql_conn.commit()
|
||||
cursor.close()
|
||||
except Exception as e:
|
||||
print(f"Error updating imapsync table: {e}")
|
||||
|
||||
def generate_guid_version(self):
|
||||
"""
|
||||
Waits for the `versions` table to be created, then generates a GUID
|
||||
based on the mail hostname and Dovecot's public key and inserts it
|
||||
into the `versions` table.
|
||||
|
||||
If the key or hash is missing or malformed, marks it as INVALID.
|
||||
"""
|
||||
|
||||
try:
|
||||
result = self.run_command(["doveconf", "-P"], check=True)
|
||||
pubkey_path = None
|
||||
for line in result.stdout.splitlines():
|
||||
if "mail_crypt_global_public_key" in line:
|
||||
parts = line.split('<')
|
||||
if len(parts) > 1:
|
||||
pubkey_path = parts[1].strip()
|
||||
break
|
||||
|
||||
if pubkey_path and Path(pubkey_path).exists():
|
||||
with open(pubkey_path, "rb") as key_file:
|
||||
pubkey_data = key_file.read()
|
||||
|
||||
hostname = self.env_vars.get("MAILCOW_HOSTNAME", "mailcow.local").encode("utf-8")
|
||||
concat = hostname + pubkey_data
|
||||
guid = hashlib.sha256(concat).hexdigest()
|
||||
|
||||
if len(guid) == 64:
|
||||
version_value = guid
|
||||
else:
|
||||
version_value = "INVALID"
|
||||
|
||||
cursor = self.mysql_conn.cursor()
|
||||
cursor.execute(
|
||||
"REPLACE INTO versions (application, version) VALUES (%s, %s)",
|
||||
("GUID", version_value)
|
||||
)
|
||||
self.mysql_conn.commit()
|
||||
cursor.close()
|
||||
else:
|
||||
print("Public key not found or unreadable. GUID not generated.")
|
||||
except Exception as e:
|
||||
print(f"Failed to generate or store GUID: {e}")
|
||||
Reference in New Issue
Block a user