[Rspamd] Remove the URLhaus and MalwareBazaar maps retired by abuse.ch

[Dovecot] Mask Redis password in trim_logs output

_modules/scripts/backup.sh

@@ -1,71 +0,0 @@
-#!/usr/bin/env bash
-# _modules/scripts/backup.sh
-# THIS SCRIPT IS DESIGNED TO BE RUNNING BY MAILCOW SCRIPTS ONLY!
-# DO NOT, AGAIN, NOT TRY TO RUN THIS SCRIPT STANDALONE!!!!!!
-
-# Backup with restic before the update when it is set up (target, confirmed password, initialized repository):
-# interactive: asked for; --backup: without asking; --force: none (with --backup: without asking, a failure or a backup
-# that cannot run stops the update; only "not set up" continues)
-backup_failed() {
-  echo -e "\e[31m${1}\e[0m"
-  if [ "$FORCE" ]; then
-    echo "The update was started with --force --backup, it stops here. Nothing was changed."
-    exit 1
-  fi
-  read -r -p "Continue the update without a backup? [y/N] " backup_response
-  if [[ ! "${backup_response}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
-    echo "OK, exiting."
-    exit 1
-  fi
-}
-
-# Whether to back up (asked with the other questions); the backup itself runs before the first change of the checkout
-backup_decide() {
-  local output rc
-  RUN_BACKUP=
-  if [ "$FORCE" ] && [ ! "$BACKUP" ]; then
-    return 0
-  fi
-  if [[ -z "$($COMPOSE_COMMAND ps -q restic-mailcow 2> /dev/null)" ]]; then
-    output="the backup container does not run"
-    rc=1
-  else
-    # exit code 0: set up, 2: not set up, others: error (e.g. credentials this container cannot read)
-    output=$($COMPOSE_COMMAND exec -T restic-mailcow python3 /usr/local/bin/restic_runner.py --configured 2>&1 < /dev/null)
-    rc=$?
-  fi
-  if [[ ${rc} -eq 2 ]]; then
-    [ "$BACKUP" ] && echo -e "\e[33mNo backup before the update: ${output}.\e[0m"
-    return 0
-  elif [[ ${rc} -ne 0 ]]; then
-    if [ "$BACKUP" ]; then
-      backup_failed "No backup before the update possible: ${output}"
-    elif [[ -n "$($COMPOSE_COMMAND ps -q restic-mailcow 2> /dev/null)" ]]; then
-      echo -e "\e[33mNo backup before the update: ${output}\e[0m"
-    fi
-    return 0
-  fi
-  if [ ! "$BACKUP" ]; then
-    read -r -p "Create a backup before the update? [Y/n] " backup_response
-    if [[ "${backup_response}" =~ ^([nN][oO]|[nN])$ ]]; then
-      echo "OK, no backup."
-      return 0
-    fi
-  fi
-  RUN_BACKUP=y
-}
-
-# The backup chosen by backup_decide, once
-backup_run() {
-  if [ ! "$RUN_BACKUP" ]; then
-    return 0
-  fi
-  RUN_BACKUP=
-  if [[ "${RESTIC_BACKUP_MODE}" =~ ^(data|config)$ ]]; then
-    echo -e "\e[33mRESTIC_BACKUP_MODE=${RESTIC_BACKUP_MODE}: the backup holds no complete mailcow directory, a restore does not bring back this mailcow version.\e[0m"
-  fi
-  echo -e "\e[32mCreating a backup before the update...\e[0m"
-  if ! $COMPOSE_COMMAND exec -T restic-mailcow python3 /usr/local/bin/restic_runner.py --backup < /dev/null; then
-    backup_failed "The backup before the update failed, see the output above and the backup jobs in the mailcow UI."
-  fi
-}

_modules/scripts/new_options.sh

@@ -61,8 +61,6 @@ adapt_new_options() {
   "ACME_DNS_PROVIDER"
   "ACME_ACCOUNT_EMAIL"
   "NGINX_MAX_BODY_SIZE"
-  "RESTIC_LOCAL_PATH"
-  "RESTIC_BACKUP_MODE"
   )
 
   sed -i --follow-symlinks '$a\' mailcow.conf
@@ -317,19 +315,6 @@ adapt_new_options() {
             echo '# Keep it above the message size limit of Postfix (100 MB by default). The mailcow UI and API stay limited to 64m.' >> mailcow.conf
             echo 'NGINX_MAX_BODY_SIZE=128m' >> mailcow.conf
             ;;
-        RESTIC_LOCAL_PATH)
-            echo '# Folder on the host for backups to the target "local" (backup settings in the mailcow UI), as an absolute path,' >> mailcow.conf
-            echo '# e.g. a NAS mounted from another disk. The repository is created in a subfolder of it.' >> mailcow.conf
-            echo '# Empty: the Docker volume restic-repo-vol-1 of mailcow, on the same disk as the mailcow data.' >> mailcow.conf
-            echo 'RESTIC_LOCAL_PATH=' >> mailcow.conf
-            ;;
-        RESTIC_BACKUP_MODE)
-            echo '# What the backups of the mailcow UI hold:' >> mailcow.conf
-            echo '# - all (default),' >> mailcow.conf
-            echo '# - data (vmail, vmail-index, database, redis, mail crypt keys and mailcow.conf)' >> mailcow.conf
-            echo '# - config (database, redis, mail crypt keys, mailcow.conf and data/conf).' >> mailcow.conf
-            echo 'RESTIC_BACKUP_MODE=all' >> mailcow.conf
-            ;;
         *)
             echo "${option}=" >> mailcow.conf
             ;;

data/Dockerfiles/dockerapi/main.py

@@ -159,9 +159,6 @@ async def post_containers(container_id : str, post_action : str, request: Reques
       api_call_method = getattr(dockerapi, api_call_method_name, lambda container_id: Response(content=json.dumps({'type': 'danger', 'msg':'container_post - unknown api call' }, indent=4), media_type="application/json"))
 
       dockerapi.logger.info("api call: %s, container_id: %s" % (api_call_method_name, container_id))
-      if getattr(api_call_method, 'long_running', False):
-        # runs for minutes (e.g. sogo-tool of a backup restore): in a worker thread, the other api calls go on
-        return await asyncio.to_thread(api_call_method, request_json, container_id=container_id)
       return api_call_method(request_json, container_id=container_id)
     except Exception as e:
       dockerapi.logger.error("error - container_post: %s" % str(e))

data/Dockerfiles/dockerapi/modules/DockerApi.py

@@ -6,15 +6,9 @@ import time
 import json
 import asyncio
 import platform
-import docker
 from datetime import datetime
 from fastapi import FastAPI, Response, Request
 
-def long_running(method):
-  """Marks an api call that runs for minutes: main.py runs it in a worker thread, the other api calls go on"""
-  method.long_running = True
-  return method
-
 class DockerApi:
   def __init__(self, redis_client, sync_docker_client, async_docker_client, logger):
     self.redis_client = redis_client
@@ -451,35 +445,6 @@ class DockerApi:
         return Response(content=json.dumps({ 'type': 'danger', 'msg': 'users required' }, indent=4), media_type="application/json")
       init_return = container.exec_run(["/bin/bash", "-c", 'for user in "$@"; do sogo-tool init-user "$user" || exit 1; done', "init_user"] + users, user='sogo')
       return self.exec_run_handler('generic', init_return)
-  # api call: container_post - post_action: exec - cmd: sogo - task: restore
-  # Resets calendars, address books and preferences of a user to the sogo-tool backup file <dir>/<username> in a
-  # restore batch of the backup (/restore/<job>/b<batch>/sogo_backup, only the sogo user can read it); values are
-  # checked and passed as arguments, never through the shell. Runs in a worker thread (main.py) with its own client.
-  @long_running
-  def container_post__exec__sogo__restore(self, request_json, **kwargs):
-    if 'container_id' in kwargs:
-      filters = {"id": kwargs['container_id']}
-    elif 'container_name' in kwargs:
-      filters = {"name": kwargs['container_name']}
-
-    directory = str(request_json.get('dir', ''))
-    username = str(request_json.get('username', ''))
-    if (not re.fullmatch(r'/restore/[0-9]+/b[0-9]+/sogo_backup', directory)
-      or not re.fullmatch(r'[A-Za-z0-9_][A-Za-z0-9._+-]*@[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)+', username)):
-      return Response(content=json.dumps({ 'type': 'danger', 'msg': 'dir and username required' }, indent=4), media_type="application/json")
-    # the folder: no symlink, uid 999, mode 0700; the file: plain, uid 999, mode 0600. sogo-tool restores folders and
-    # preferences in two runs, both stopped before the runner gives up waiting (30 minutes)
-    script = ('[ -d "$1" ] && [ ! -L "$1" ] && [ "$(realpath -e -- "$1")" = "$1" ] && [ "$(stat -c "%u %a" -- "$1")" = "999 700" ] '
-      '&& [ -f "$1/$2" ] && [ ! -L "$1/$2" ] && [ "$(stat -c "%u %a" -- "$1/$2")" = "999 600" ] '
-      '|| { echo "invalid directory or file"; exit 1; }; '
-      'timeout -k 30 850 sogo-tool restore -f ALL "$1" "$2" && exec timeout -k 30 850 sogo-tool restore -p "$1" "$2"')
-    client = docker.DockerClient(base_url='unix:///var/run/docker.sock', version='auto', timeout=1800)
-    try:
-      for container in client.containers.list(filters=filters):
-        restore_return = container.exec_run(["/bin/bash", "-c", script, "restore", directory, username], user='sogo')
-        return self.exec_run_handler('generic', restore_return)
-    finally:
-      client.close()
   # api call: container_post - post_action: exec - cmd: doveadm - task: subscribe_shared
   # Subscribes users to every folder of a mailbox shared with them (Shared/<owner>/...), or unsubscribes them
   def container_post__exec__doveadm__subscribe_shared(self, request_json, **kwargs):
@@ -621,8 +586,8 @@ class DockerApi:
       if not user or id in ('user=', 'group='):
         return Response(content=json.dumps({ 'type': 'danger', 'msg': 'user and id required' }, indent=4), media_type="application/json")
       action = 'doveadm acl set -u "$ACL_USER" -- "$folder" "$ACL_ID" $ACL_RIGHTS' if rights else 'doveadm acl delete -u "$ACL_USER" -- "$folder" "$ACL_ID"'
-      script = ('set -o pipefail; doveadm mailbox list -u "$ACL_USER" | while IFS= read -r folder; do '
-        'case "$folder" in Shared/*) continue;; esac; ' + action + ' || exit 1; done && doveadm acl recalc -u "$ACL_USER"')
+      script = ('set -o pipefail; doveadm -f tab mailbox status -u "$ACL_USER" guid "*" | tail -n +2 | cut -f1 | while IFS= read -r folder; do '
+        'case "$folder" in Shared|Shared/*) continue;; esac; ' + action + ' || exit 1; done && doveadm acl recalc -u "$ACL_USER"')
       acl_return = container.exec_run(["/bin/bash", "-c", script],
         environment={ 'ACL_USER': user, 'ACL_ID': id, 'ACL_RIGHTS': ' '.join(rights) })
       return self.exec_run_handler('generic', acl_return)

data/Dockerfiles/dovecot/Dockerfile

@@ -150,8 +150,6 @@ COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
 COPY syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng-redis_slave.conf
 COPY imapsync /usr/local/bin/imapsync
 COPY imapsync_runner.py /usr/local/bin/imapsync_runner.py
-COPY retention_runner.py /usr/local/bin/retention_runner.py
-COPY restore_runner.py /usr/local/bin/restore_runner.py
 COPY report-spam.sieve /usr/lib/dovecot/sieve/report-spam.sieve
 COPY report-ham.sieve /usr/lib/dovecot/sieve/report-ham.sieve
 COPY rspamd-pipe-ham /usr/lib/dovecot/sieve/rspamd-pipe-ham

data/Dockerfiles/dovecot/docker-entrypoint.sh

@@ -281,8 +281,6 @@ chown root:tty /dev/console
 chmod +x /usr/lib/dovecot/sieve/rspamd-pipe-ham \
   /usr/lib/dovecot/sieve/rspamd-pipe-spam \
   /usr/local/bin/imapsync_runner.py \
-  /usr/local/bin/retention_runner.py \
-  /usr/local/bin/restore_runner.py \
   /usr/local/bin/imapsync \
   /usr/local/bin/trim_logs.sh \
   /usr/local/bin/sa-rules.sh \

data/Dockerfiles/dovecot/restore_runner.py

@@ -1,712 +0,0 @@
-#!/usr/bin/python3
-# Mail part of restores (ofelia, dovecot container, as vmail): the restic container restores a batch of mailboxes
-# into /restore/<job id>/b<n> and sets the job to step 'mails'. Per mailbox this imports the mails folder by folder
-# (into "Restored <snapshot time>" or only the missing ones into the original folders), adds IMAP ACLs and missing
-# Sieve scripts, merges the results into the job and sets step 'mails_done'; the restic runner then removes the staging
-# and goes on with the next batch. Nothing that exists is changed or deleted.
-
-import base64
-import fcntl
-import json
-import os
-import re
-import smtplib
-import stat
-import subprocess
-import sys
-import threading
-import time
-from email.message import EmailMessage
-from email.utils import formatdate, make_msgid
-import MySQLdb
-import MySQLdb.cursors
-
-lock_file = '/tmp/restore_runner.lock'
-restore_dir = '/restore'
-max_log = 1000000
-# import, moves and the removal of the temporary folder ignore the quota (a restore may need more than is free),
-# the quota is recalculated afterwards
-no_quota = ['-o', 'plugin/quota=']
-# uid ranges per import (one argument stays far below the 128 KiB limit), folders per fetch and status
-uid_ranges = 2000
-folder_batch = 200
-# the restic runner waits for the batch while this heartbeat is fresh
-heartbeat_interval = 30
-# seconds to wait for the next batch of a restore
-batch_wait = 120
-rights_by_letter = {'l': 'lookup', 'r': 'read', 'w': 'write', 's': 'write-seen', 't': 'write-deleted', 'i': 'insert',
-  'p': 'post', 'e': 'expunge', 'k': 'create', 'x': 'delete', 'a': 'admin'}
-components_here = {'mails', 'imap_acl', 'filters', 'sieve_activate'}
-
-conn = None
-
-
-def connect():
-  return MySQLdb.connect(unix_socket='/var/run/mysqld/mysqld.sock', user=os.environ['DBUSER'],
-    passwd=os.environ['DBPASS'], db=os.environ['DBNAME'], charset='utf8mb4', autocommit=True)
-
-# Reconnect when the server dropped the connection (imports can outlast wait_timeout)
-def query(sql, args=None):
-  global conn
-  try:
-    conn.ping()
-  except (AttributeError, MySQLdb.Error):
-    conn = connect()
-  cur = conn.cursor(MySQLdb.cursors.DictCursor)
-  cur.execute(sql, args)
-  return cur
-
-# state_restore_seen while a batch runs, in its own connection
-class Heartbeat(threading.Thread):
-  def __init__(self):
-    super().__init__(daemon=True)
-    self.done = threading.Event()
-
-  def run(self):
-    db = None
-    while not self.done.is_set():
-      try:
-        if db is None:
-          db = connect()
-        db.cursor().execute("""INSERT INTO backup_setting (name, value) VALUES ('state_restore_seen', %s)
-          ON DUPLICATE KEY UPDATE value = VALUES(value)""", (str(int(time.time())),))
-      except MySQLdb.Error:
-        db = None
-      self.done.wait(heartbeat_interval)
-
-def log(job_id, text):
-  print('restore job %d: %s' % (job_id, text), flush=True)
-  line = '%s %s\n' % (time.strftime('%Y-%m-%d %H:%M:%S'), text)
-  query("UPDATE backup_job SET log = RIGHT(CONCAT(IFNULL(log, ''), %s), %s) WHERE id = %s", (line, max_log, job_id))
-
-def save(job_id, result):
-  query("UPDATE backup_job SET result = %s WHERE id = %s", (json.dumps(result), job_id))
-
-def doveadm(*args, data=None, check=True, partial=False):
-  result = subprocess.run(['doveadm', *args], input=data, stdin=None if data is not None else subprocess.DEVNULL,
-    capture_output=True, timeout=86400)
-  # mailbox status fails for folders that cannot be opened (implicit parents); the others are listed
-  if check and result.returncode != 0 and not (partial and result.stdout.strip()):
-    # mail_log writes an info line per message, the errors are the other lines
-    lines = [l for l in result.stderr.decode('utf-8', 'replace').splitlines() if ': Info: ' not in l]
-    raise RuntimeError('\n'.join(lines)[-500:] or 'doveadm exit code %d' % result.returncode)
-  return result.stdout.decode('utf-8', 'replace')
-
-def doveadm_json(*args, partial=False):
-  out = doveadm('-f', 'json', *args, partial=partial)
-  return json.loads(out) if out.strip() else []
-
-# free disk space that must stay after the import: max(2 GiB, 5 % of the filesystem), a full disk stops mail
-# (disk_reserve, mib and count_errors as in restic_runner.py of the backup image; tests/python/backup compares them)
-def disk_reserve(total):
-  return max(2 << 30, total // 20)
-
-def import_space_ok(need, free, total):
-  return need * 11 // 10 + disk_reserve(total) <= free
-
-def mib(size):
-  return '%.1f MiB' % (size / 1048576)
-
-def component(errors=None):
-  return {'added': 0, 'skipped': 0, 'errors': errors or []}
-
-# Folder names on disk: modified UTF-7, listescape writes . / \ as \xx
-def mutf7_decode(text):
-  def utf16(m):
-    if not m.group(1):
-      return '&'
-    data = m.group(1).replace(',', '/')
-    return base64.b64decode(data + '=' * (-len(data) % 4)).decode('utf-16-be')
-  return re.sub(r'&([A-Za-z0-9+,]*)-', utf16, text)
-
-def unescape(part):
-  return mutf7_decode(re.sub(r'\\([0-9a-fA-F]{2})', lambda m: chr(int(m.group(1), 16)), part))
-
-# Maildir++: the maildir is INBOX, ".A.B" is A/B
-def folder_name(entry):
-  return '/'.join(unescape(part) for part in entry[1:].split('.')) if entry else 'INBOX'
-
-def is_dir(path):
-  try:
-    return stat.S_ISDIR(os.lstat(path).st_mode)
-  except OSError:
-    return False
-
-def read_file(path):
-  fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
-  with os.fdopen(fd, 'rb') as f:
-    return f.read()
-
-# Prefixes of the shared and public namespaces, their folders belong to others
-def shared_prefixes():
-  prefixes, namespace, depth = {'Shared/'}, None, 0
-  # as vmail doveconf cannot read every included file (mail_crypt keys) and fails, but prints the namespaces
-  for line in subprocess.run(['doveconf', '-n'], capture_output=True, text=True).stdout.splitlines():
-    stripped = line.strip()
-    if stripped.endswith('{'):
-      depth += 1
-      if depth == 1 and stripped.startswith('namespace'):
-        namespace = {}
-    elif stripped == '}':
-      if depth == 1 and namespace is not None:
-        if namespace.get('type') in ('shared', 'public') and namespace.get('prefix'):
-          prefixes.add(namespace['prefix'].split('%')[0])
-        namespace = None
-      depth = max(depth - 1, 0)
-    elif depth == 1 and namespace is not None and '=' in stripped:
-      key, value = (part.strip() for part in stripped.split('=', 1))
-      namespace[key] = value
-  return tuple(prefix for prefix in prefixes if prefix)
-
-# name => guid of the user's own folders (implicit parents have none); the names are looked up literally
-def own_folders(user, prefixes):
-  names = [n for n in doveadm('mailbox', 'list', '-u', user).splitlines() if n and not n.startswith(prefixes)]
-  wanted, guids = set(names), {}
-  for i in range(0, len(names), folder_batch):
-    for row in doveadm_json('mailbox', 'status', '-u', user, 'guid', '--', *names[i:i + folder_batch], partial=True):
-      if row.get('mailbox') in wanted and re.fullmatch(r'[0-9a-f]{32}', row.get('guid') or ''):
-        guids[row['mailbox']] = row['guid']
-  return guids
-
-def masked(user):
-  return '*' in user or '?' in user
-
-def free_name(base, existing):
-  name, count = base, 1
-  while name in existing or any(folder.startswith(name + '/') for folder in existing):
-    count += 1
-    name = '%s (%d)' % (base, count)
-  return name
-
-# Path of the maildir now (/var/vmail/<domain>/<local>/Maildir), None for unknown or inactive mailboxes
-def mail_location(user):
-  result = subprocess.run(['doveadm', 'user', '-f', 'mail', '--', user], capture_output=True, text=True)
-  m = re.match(r'maildir:([^:]+)', result.stdout.strip()) if result.returncode == 0 else None
-  return m.group(1).rstrip('/') if m else None
-
-# The restored maildir of a mailbox (may hold only ACL files), None if the snapshot has nothing of it
-def staged_maildir(staging, home, location):
-  for path in (staging + location, home + '/' + os.environ.get('MAILDIR_SUB', 'Maildir'), home):
-    if is_dir(path):
-      return path
-  return None
-
-def uid_chunks(uids):
-  ranges, start, prev = [], None, None
-  for uid in sorted(uids):
-    if start is None:
-      start = prev = uid
-    elif uid == prev + 1:
-      prev = uid
-    else:
-      ranges.append('%d:%d' % (start, prev) if prev != start else str(start))
-      start = prev = uid
-  if start is not None:
-    ranges.append('%d:%d' % (start, prev) if prev != start else str(start))
-  return [','.join(ranges[i:i + uid_ranges]) for i in range(0, len(ranges), uid_ranges)]
-
-# Same message: same Message-ID, without one the same Date and size
-def message_key(row):
-  message_id = re.sub(r'\s+', '', row.get('hdr.message-id') or '').strip('<>')
-  if message_id:
-    return 'id:' + message_id
-  return 'date:%s:%s' % (row.get('date.sent'), row.get('size.physical'))
-
-def present_keys(user, guids):
-  keys = set()
-  guids = list(guids)
-  for i in range(0, len(guids), folder_batch):
-    search = []
-    for guid in guids[i:i + folder_batch]:
-      search += (['OR'] if search else []) + ['mailbox-guid', guid]
-    keys.update(message_key(row) for row in doveadm_json('fetch', '-u', user, 'hdr.message-id date.sent size.physical',
-      *search))
-  return keys
-
-def location(path):
-  if ':' in path:
-    raise RuntimeError('unsupported path')
-  return 'maildir:%s:INDEX=MEMORY' % path
-
-# The folders of a restored maildir as standalone maildirs (their messages are the INBOX there): the subfolders are
-# moved out of the maildir, so every folder is imported without the name rules of the source (listescape)
-def split_folders(maildir, home):
-  folders, failed = [('INBOX', maildir)], []
-  out = home + '/_restore_folders'
-  for entry in sorted(os.listdir(maildir)):
-    path = os.path.join(maildir, entry)
-    if not entry.startswith('.') or entry in ('.', '..') or not is_dir(path) or not is_dir(path + '/cur'):
-      continue
-    try:
-      name = folder_name(entry)
-    except (ValueError, UnicodeDecodeError):
-      failed.append(entry)
-      continue
-    if not os.path.isdir(out):
-      os.mkdir(out, 0o700)
-    target = '%s/f%d' % (out, len(folders))
-    os.rename(path, target)
-    folders.append((name, target))
-  return folders, failed
-
-def source_messages(user, root):
-  return doveadm_json('-o', 'mail_location=' + location(root), 'fetch', '-u', user,
-    'uid hdr.message-id date.sent size.physical', 'mailbox', 'INBOX')
-
-def remove_tree(user, name):
-  doveadm(*no_quota, 'mailbox', 'delete', '-u', user, '-r', '-s', '--', name, check=False)
-
-def folder_guid(user, name):
-  rows = doveadm_json('mailbox', 'status', '-u', user, 'guid', '--', name)
-  for row in rows:
-    if row.get('mailbox') == name:
-      return row['guid']
-  raise RuntimeError('folder not found after the import')
-
-def quota_room(user):
-  # free bytes of the storage quota, None without a limit
-  for row in doveadm_json('quota', 'get', '-u', user):
-    if row.get('type') == 'STORAGE' and str(row.get('limit', '-')).isdigit():
-      return (int(row['limit']) - int(row.get('value') or 0)) * 1024
-  return None
-
-# Subscriptions of the backup (subscriptions file: "V\t2", then names with \t between the levels)
-def backup_subscriptions(maildir):
-  try:
-    lines = read_file(maildir + '/subscriptions').decode('utf-8', 'replace').splitlines()
-  except OSError:
-    return []
-  if lines and lines[0].startswith('V\t'):
-    return ['/'.join(unescape(part) for part in line.split('\t')) for line in lines[1:] if line.strip()]
-  # version 1: storage names with the Maildir++ separator
-  return [folder_name('.' + line.strip()) for line in lines if line.strip()]
-
-def restore_subscriptions(user, wanted, prefixes):
-  if not wanted:
-    return 0
-  existing = set(own_folders(user, prefixes)) | {'INBOX'}
-  subscribed = set(doveadm('mailbox', 'list', '-u', user, '-s').splitlines())
-  missing = [name for name in dict.fromkeys(wanted) if name in existing and name not in subscribed]
-  if missing:
-    doveadm('mailbox', 'subscribe', '-u', user, '--', *missing)
-  return len(missing)
-
-# Mails folder by folder: original = only messages missing anywhere in the mailbox, into their original folders;
-# folder = all messages below "Restored <snapshot time>"
-def restore_mails(job_id, user, home, maildir, mode, snapshot_time, prefixes, check_quota):
-  result = component()
-  result.update(folders=0, folders_failed=0)
-  if not maildir or not is_dir(maildir + '/cur'):
-    # a key the UI translates
-    result['note'] = 'no_mails'
-    return result
-  tmp = 'Restoring %d' % job_id
-  # a leftover of an interrupted run goes first, it is never counted as present
-  remove_tree(user, tmp)
-  subscriptions = backup_subscriptions(maildir)
-  folders, broken = split_folders(maildir, home)
-  for entry in broken:
-    result['errors'].append('folder %s: name not readable' % entry)
-    result['folders_failed'] += 1
-  live = own_folders(user, prefixes)
-  present = present_keys(user, live.values()) if mode == 'original' else set()
-  plan, need = [], 0
-  for name, root in folders:
-    try:
-      rows = source_messages(user, root)
-    except RuntimeError as e:
-      result['errors'].append('%s: %s' % (name, e))
-      result['folders_failed'] += 1
-      continue
-    wanted = [row for row in rows if mode != 'original' or message_key(row) not in present]
-    result['skipped'] += len(rows) - len(wanted)
-    if wanted:
-      size = sum(int(row.get('size.physical') or 0) for row in wanted)
-      plan.append((name, root, [int(row['uid']) for row in wanted], size))
-      need += size
-  if not plan:
-    if mode == 'original':
-      result['subscribed'] = restore_subscriptions(user, subscriptions, prefixes)
-    return result
-  vfs = os.statvfs('/var/vmail')
-  free, total = vfs.f_bavail * vfs.f_frsize, vfs.f_blocks * vfs.f_frsize
-  if not import_space_ok(need, free, total):
-    raise RuntimeError('not enough free disk space for the mails: %s needed plus %s reserve, %s free' % (
-      mib(need * 11 // 10), mib(disk_reserve(total)), mib(free)))
-  if check_quota:
-    room = quota_room(user)
-    if room is not None and need > room:
-      raise RuntimeError('the mails do not fit into the quota of the mailbox: %s needed, %s free' % (
-        mib(need), mib(max(room, 0))))
-  names = set(live) | {'INBOX'}
-  dest = None
-  if mode != 'original':
-    dest = free_name('Restored ' + snapshot_time, names)
-    doveadm('mailbox', 'create', '-u', user, '-s', '--', dest)
-    names.add(dest)
-    result['folder'] = dest
-  created = []
-  try:
-    for number, (name, root, uids, size) in enumerate(plan):
-      target = dest + '/' + name if dest else name
-      stage = '%s/f%d' % (tmp, number)
-      try:
-        for chunk in uid_chunks(uids):
-          doveadm(*no_quota, 'import', '-u', user, '-s', location(root), stage, 'mailbox', 'INBOX', 'uid', chunk)
-        guid = folder_guid(user, stage + '/INBOX')
-        if target not in names:
-          doveadm('mailbox', 'create', '-u', user, '-s', '--', target)
-          names.add(target)
-          created.append(target)
-        doveadm(*no_quota, 'move', '-u', user, '--', target, 'mailbox-guid', guid, 'all')
-        result['added'] += len(uids)
-        result['folders'] += 1
-      except RuntimeError as e:
-        result['errors'].append('%s: %s' % (name, e))
-        result['folders_failed'] += 1
-  finally:
-    remove_tree(user, tmp)
-  result['folders_created'] = len(created)
-  if mode == 'original':
-    result['subscribed'] = restore_subscriptions(user, subscriptions, prefixes)
-  return result
-
-def acl_entries(path):
-  entries = []
-  try:
-    lines = read_file(path).decode('utf-8', 'replace').splitlines()
-  except OSError:
-    return entries
-  for line in lines:
-    parts = line.split()
-    # negative rights would take rights away
-    if len(parts) < 2 or parts[0].startswith('-'):
-      continue
-    rights = [rights_by_letter[c] for c in parts[1] if c in rights_by_letter] + [p for p in parts[2:] if p.startswith(':')]
-    if rights:
-      entries.append((parts[0], rights))
-  return entries
-
-# (folder, entries) of the dovecot-acl files of a restored maildir
-def maildir_acls(maildir):
-  acls = []
-  if not maildir or not is_dir(maildir):
-    return acls
-  for entry in [''] + sorted(e for e in os.listdir(maildir) if e.startswith('.') and e not in ('.', '..')):
-    path = os.path.join(maildir, entry, 'dovecot-acl') if entry else os.path.join(maildir, 'dovecot-acl')
-    if entry and not is_dir(os.path.join(maildir, entry)):
-      continue
-    try:
-      if stat.S_ISREG(os.lstat(path).st_mode):
-        acls.append((folder_name(entry), acl_entries(path)))
-    except (OSError, ValueError, UnicodeDecodeError):
-      continue
-  return acls
-
-# owner => (folder, entries) of every mailbox in the ACL staging (domain admins: only their domains)
-def staged_acls(acl_dir, allowed):
-  acls = {}
-  base = (acl_dir or '') + '/var/vmail'
-  sub = os.environ.get('MAILDIR_SUB', '')
-  if not acl_dir or not is_dir(base):
-    return acls
-  for domain in os.listdir(base):
-    if '.' not in domain or not is_dir(os.path.join(base, domain)) or (allowed is not None and domain.lower() not in allowed):
-      continue
-    for local in os.listdir(os.path.join(base, domain)):
-      home = os.path.join(base, domain, local)
-      maildir = os.path.join(home, sub) if sub and is_dir(os.path.join(home, sub)) else home
-      found = maildir_acls(maildir)
-      if found:
-        acls['%s@%s' % (local, domain)] = found
-  return acls
-
-class AclRestore:
-  def __init__(self, prefixes, allowed):
-    self.prefixes = prefixes
-    self.allowed = allowed
-    self.users, self.folders = {}, {}
-    # anyone/authenticated only when the configuration allows them
-    self.anyone = os.environ.get('ACL_ANYONE', 'disallow') == 'allow'
-
-  def in_scope(self, address):
-    return self.allowed is None or address.rpartition('@')[2].lower() in self.allowed
-
-  def user_exists(self, user):
-    if user not in self.users:
-      self.users[user] = query("SELECT 1 FROM mailbox WHERE username = %s", (user,)).fetchone() is not None
-    return self.users[user]
-
-  def identifier_ok(self, identifier):
-    if identifier.startswith('user='):
-      return self.user_exists(identifier[5:].lower())
-    # group=mcg_<id> belongs to the mailbox groups, the group grants cron keeps them
-    if identifier.startswith('group=mcg_'):
-      return False
-    if identifier in ('anyone', 'authenticated'):
-      return self.anyone
-    return True
-
-  def shown(self, identifier):
-    # domain admins do not see addresses of other domains
-    if identifier.startswith('user=') and not self.in_scope(identifier[5:]):
-      return 'user=(other domain)'
-    return identifier
-
-  def add(self, owner, folder, identifier, rights, result):
-    if not self.in_scope(owner) or masked(owner):
-      result['skipped'] += 1
-      return
-    try:
-      if owner not in self.folders:
-        self.folders[owner] = set(own_folders(owner, self.prefixes)) | {'INBOX'}
-      if folder not in self.folders[owner] or not self.identifier_ok(identifier):
-        result['skipped'] += 1
-        return
-      have = set()
-      for row in doveadm_json('acl', 'get', '-u', owner, '--', folder):
-        if row.get('id') == identifier:
-          have.update((row.get('rights') or '').split())
-      missing = [right for right in rights if right not in have]
-      if not missing:
-        result['skipped'] += 1
-        return
-      doveadm('acl', 'add', '-u', owner, '--', folder, identifier, *missing)
-      result['added'] += 1
-    except RuntimeError as e:
-      result['errors'].append('%s %s %s: %s' % (owner, folder, self.shown(identifier), e))
-
-# Rights the mailbox granted on its folders and rights other mailboxes granted to it; added, never removed
-def restore_acl(acl, user, own, granted):
-  result = component()
-  for folder, entries in own:
-    for identifier, rights in entries:
-      acl.add(user, folder, identifier, rights, result)
-  if acl.user_exists(user):
-    for owner, folders in granted.items():
-      if owner == user or not acl.in_scope(owner) or not acl.user_exists(owner):
-        continue
-      for folder, entries in folders:
-        for identifier, rights in entries:
-          if identifier.lower() == 'user=' + user:
-            acl.add(owner, folder, identifier, rights, result)
-  return result
-
-# Own Sieve scripts (incl. the SOGo vacation script): missing names are added; with activate the active script of the
-# backup is activated if none is active now
-def restore_sieve(user, home, active_link, activate):
-  result = component()
-  folder = home + '/sieve'
-  scripts = []
-  if is_dir(folder):
-    scripts = sorted(f[:-6] for f in os.listdir(folder) if f.endswith('.sieve') and stat.S_ISREG(os.lstat(os.path.join(folder, f)).st_mode))
-  current, active = set(), None
-  for line in doveadm('sieve', 'list', '-u', user).splitlines():
-    name, _, flag = line.rpartition(' ')
-    if name:
-      current.add(name)
-      if flag == 'ACTIVE':
-        active = name
-  for name in scripts:
-    if name in current:
-      result['skipped'] += 1
-      continue
-    try:
-      doveadm('sieve', 'put', '-u', user, '--', name, data=read_file(os.path.join(folder, name + '.sieve')))
-      current.add(name)
-      result['added'] += 1
-    except (OSError, RuntimeError) as e:
-      result['errors'].append('%s: %s' % (name, e))
-  if not activate:
-    return result
-  target = os.readlink(active_link) if os.path.islink(active_link) else ''
-  backup_active = os.path.basename(target)[:-6] if target.endswith('.sieve') else None
-  if backup_active and not active and backup_active in current:
-    try:
-      doveadm('sieve', 'activate', '-u', user, '--', backup_active)
-      result['activated'] = backup_active
-    except RuntimeError as e:
-      result['errors'].append('%s: %s' % (backup_active, e))
-  return result
-
-# filters: backup.php reported its part already, the counts are added up
-def merge(results, key, part):
-  old = results.get(key)
-  if not isinstance(old, dict):
-    results[key] = part
-    return
-  for count in ('added', 'skipped'):
-    old[count] = (old.get(count) or 0) + part.get(count, 0)
-  old['errors'] = (old.get('errors') or []) + part.get('errors', [])
-  for name, value in part.items():
-    if name not in ('added', 'skipped', 'errors'):
-      old[name] = value
-
-def count_errors(result):
-  count = sum(len(result.get(key, {}).get('errors') or []) for key in ('domain', 'aliases') if isinstance(result.get(key), dict))
-  for entry in (result.get('mailboxes') or {}).values():
-    count += int(bool(entry.get('error')))
-    count += sum(len(value.get('errors') or []) for value in entry.values() if isinstance(value, dict))
-  return count
-
-def restore_mailbox(job_id, user, components, mode, staging, snapshot_time, prefixes, acl, granted, check_quota, result):
-  local, _, domain = user.rpartition('@')
-  home = '%s/var/vmail/%s/%s' % (staging, domain, local)
-  if masked(user):
-    for key in components & {'mails', 'imap_acl', 'filters'}:
-      merge(result, key, component(['username with * or ?, doveadm reads it as a mask']))
-    return
-  location = mail_location(user)
-  if not location:
-    for key in components & {'mails', 'imap_acl', 'filters'}:
-      merge(result, key, component(['the mailbox does not exist now or is inactive']))
-    return
-  maildir = staged_maildir(staging, home, location)
-  # the ACL files are read before the folders of the maildir are moved for the import
-  own = maildir_acls(maildir) if 'imap_acl' in components else []
-  if 'mails' in components:
-    try:
-      result['mails'] = restore_mails(job_id, user, home, maildir, mode, snapshot_time, prefixes, check_quota)
-    except Exception as e:
-      result['mails'] = component([str(e)[:1000]])
-    try:
-      doveadm('quota', 'recalc', '-u', user)
-    except RuntimeError as e:
-      result['mails']['errors'].append('quota recalc: %s' % e)
-    # new folders
-    acl.folders.pop(user, None)
-  if 'imap_acl' in components:
-    try:
-      result['imap_acl'] = restore_acl(acl, user, own, granted)
-    except Exception as e:
-      result['imap_acl'] = component([str(e)[:1000]])
-  if 'filters' in components:
-    try:
-      part = restore_sieve(user, home, '%s/var/vmail/sieve/%s.sieve' % (staging, user), 'sieve_activate' in components)
-    except Exception as e:
-      part = component([str(e)[:1000]])
-    merge(result, 'filters', part)
-
-def fail(job_id, error):
-  query("""UPDATE backup_job SET state = IF(state = 'cancelled', state, 'failed'), error = %s, finished = NOW()
-    WHERE id = %s""", (error, job_id))
-  log(job_id, 'restore job failed: ' + error)
-  notify(job_id, error)
-
-def notify(job_id, error):
-  hostname = os.environ.get('MAILCOW_HOSTNAME', '')
-  row = query("SELECT value FROM backup_setting WHERE name = 'notify_email'").fetchone()
-  rcpts = [r.strip() for r in (row['value'] if row else '').split(',') if re.fullmatch(r'[^@\s<>,]+@[^@\s<>,]+', r.strip())]
-  if not rcpts:
-    return
-  msg = EmailMessage()
-  msg['From'] = 'backup@' + hostname
-  msg['To'] = ', '.join(rcpts)
-  msg['Subject'] = 'mailcow backup on %s: restore job %d failed' % (hostname, job_id)
-  msg['Date'] = formatdate(localtime=True)
-  msg['Message-ID'] = make_msgid(domain=hostname or None)
-  msg.set_content('The restore job %d failed:\n\n%s\n\nThe job log is in the backup section of the mailcow UI.\n'
-    % (job_id, error))
-  try:
-    with smtplib.SMTP('postfix', 588, hostname or None, timeout=60) as smtp:
-      smtp.send_message(msg)
-  except (OSError, smtplib.SMTPException) as e:
-    print('restore job %d: the notification could not be sent: %s' % (job_id, e), file=sys.stderr)
-
-def staging_dir(job_id, path):
-  # the batch folder of this job, never a symlink
-  if not isinstance(path, str) or not re.fullmatch(r'/restore/%d/[A-Za-z0-9_-]+' % job_id, path) or \
-      os.path.realpath(path) != path or not is_dir(path):
-    return None
-  return path
-
-def run_batch(job):
-  job_id = job['id']
-  params = json.loads(job['params'] or '{}')
-  result = json.loads(job['result'] or '{}')
-  batch = result.get('batch') or {}
-  staging = staging_dir(job_id, batch.get('dir'))
-  acl_dir = staging_dir(job_id, batch.get('acl_dir')) if batch.get('acl_dir') else None
-  mailboxes = [m for m in batch.get('mailboxes') or [] if isinstance(m, str)]
-  components = {c for c in params.get('components') or [] if c in components_here}
-  mode = 'original' if params.get('mail_target') == 'original' else 'folder'
-  allowed = {d.lower() for d in params['allowed_domains']} if isinstance(params.get('allowed_domains'), list) else None
-  check_quota = params.get('requested_role', 'admin') != 'admin'
-  snapshot_time = (result.get('snapshot') or {}).get('time') or time.strftime('%Y-%m-%d %H:%M')
-  results = result.setdefault('mailboxes', {})
-  prefixes = shared_prefixes()
-  acl = AclRestore(prefixes, allowed)
-  granted = staged_acls(acl_dir, allowed) if 'imap_acl' in components else {}
-  log(job_id, 'restoring %s for %d mailboxes%s' % (', '.join(sorted(components)), len(mailboxes),
-    ' (mails: %s)' % mode if 'mails' in components else ''))
-  for user in mailboxes:
-    entry = results.setdefault(user, {})
-    if entry.get('error'):
-      continue
-    started = time.monotonic()
-    if not staging:
-      for key in components & {'mails', 'imap_acl', 'filters'}:
-        merge(entry, key, component(['the restored files are missing']))
-    else:
-      restore_mailbox(job_id, user, components, mode, staging, snapshot_time, prefixes, acl, granted, check_quota, entry)
-    summary = ', '.join('%s +%d' % (key, entry[key].get('added', 0)) for key in sorted(components) if isinstance(entry.get(key), dict))
-    errors = sum(len(entry[key].get('errors') or []) for key in components if isinstance(entry.get(key), dict))
-    log(job_id, '%s: %s%s (%d s)' % (user, summary, ', %d errors' % errors if errors else '', time.monotonic() - started))
-    result['errors'] = count_errors(result)
-    save(job_id, result)
-  result['errors'] = count_errors(result)
-  save(job_id, result)
-  # the restic runner removes the staging and goes on
-  query("UPDATE backup_job SET step = 'mails_done' WHERE id = %s AND state = 'running' AND step = 'mails_running'", (job_id,))
-
-def main():
-  global conn
-  if os.geteuid() == 0:
-    # as vmail like the ofelia job: a run as root would leave a lock file the runs as vmail cannot open
-    os.execv('/usr/local/bin/gosu', ['gosu', 'vmail', sys.executable, os.path.abspath(__file__)] + sys.argv[1:])
-  os.umask(0o077)
-  # released by the kernel when the process ends, a crashed run leaves no stale lock
-  lock_fd = os.open(lock_file, os.O_RDWR | os.O_CREAT, 0o600)
-  try:
-    fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-  except OSError:
-    sys.exit("restore: can't lock " + lock_file)
-  conn = connect()
-  heartbeat = Heartbeat()
-  heartbeat.start()
-  try:
-    # under the lock: a batch in mails_running was left by a crashed or stopped run, its temporary folders go
-    for row in query("""SELECT id, result FROM backup_job WHERE type = 'restore' AND state = 'running'
-        AND step = 'mails_running'""").fetchall():
-      try:
-        for user in (json.loads(row['result'] or '{}').get('batch') or {}).get('mailboxes') or []:
-          remove_tree(user, 'Restoring %d' % row['id'])
-          doveadm('quota', 'recalc', '-u', user, check=False)
-      except (ValueError, AttributeError, RuntimeError):
-        pass
-      fail(row['id'], 'interrupted: the restore of the mails was stopped')
-    last, waited = None, 0
-    while True:
-      job = query("""SELECT id, params, result FROM backup_job WHERE type = 'restore' AND state = 'running'
-        AND step = 'mails' ORDER BY id LIMIT 1""").fetchone()
-      if not job:
-        # the next batch of a restore follows within seconds: wait for it instead of the next ofelia run
-        row = query("SELECT state FROM backup_job WHERE id = %s", (last,)).fetchone() if last else None
-        if row and row['state'] == 'running' and waited < batch_wait:
-          time.sleep(2)
-          waited += 2
-          continue
-        break
-      last, waited = job['id'], 0
-      if query("UPDATE backup_job SET step = 'mails_running' WHERE id = %s AND state = 'running' AND step = 'mails'",
-          (job['id'],)).rowcount != 1:
-        continue
-      try:
-        run_batch(job)
-      except Exception as e:
-        fail(job['id'], str(e)[:1000])
-  finally:
-    heartbeat.done.set()
-
-if __name__ == '__main__':
-  main()

data/Dockerfiles/dovecot/retention_runner.py

@@ -1,256 +0,0 @@
-#!/usr/bin/python3
-# Nightly run of the retention rules (ofelia, dovecot container): gets the plan page by page from the mailcow UI
-# (nginx:8081/retention.php), gives every folder of a mailbox to the first entry of its plan that matches it, counts
-# the mail saved in the folders before the period of the rule with doveadm, deletes it unless the rule only
-# simulates, and reports the results back.
-
-import fcntl
-import hashlib
-import hmac
-import json
-import os
-import re
-import subprocess
-import sys
-import time
-import urllib.parse
-import urllib.request
-
-plan_url = 'http://nginx:8081/retention.php'
-lock_file = '/tmp/retention_runner.lock'
-# folder keys of the rules and the special-use flag of their folders in the Dovecot configuration
-special_use = {'trash': '\\Trash', 'junk': '\\Junk', 'sent': '\\Sent', 'drafts': '\\Drafts', 'archive': '\\Archive'}
-own_folder = 'folder:'
-# mailboxes per page of the plan and results per report, a large instance has many mailboxes
-plan_page = 200
-report_batch = 200
-# a request is tried this often, waiting longer each time
-tries = 3
-
-token = hmac.new(os.environ.get('DBPASS', '').encode(), b'retention', hashlib.sha256).hexdigest()
-
-def request(url, body=None):
-  data = None if body is None else json.dumps(body).encode()
-  req = urllib.request.Request(url, data=data, headers={'Content-Type': 'application/json', 'X-Retention-Token': token},
-    method='GET' if body is None else 'POST')
-  with urllib.request.urlopen(req, timeout=300) as response:
-    return json.load(response)
-
-def retried(url, body=None):
-  for attempt in range(1, tries + 1):
-    try:
-      return request(url, body)
-    except Exception as e:
-      if attempt == tries:
-        raise
-      print('retention: request failed, trying again: %s' % e, file=sys.stderr)
-      time.sleep(30 * attempt)
-
-def plan():
-  # one page after the other, so memory stays small; rule changes apply to the pages fetched after them
-  after = ''
-  while after is not None:
-    page = retried(plan_url + '?' + urllib.parse.urlencode({'after': after, 'limit': plan_page}))
-    yield from page['mailboxes']
-    after = page['next']
-
-def post(body):
-  # False if the report could not be sent
-  try:
-    retried(plan_url, body)
-    return True
-  except Exception as e:
-    print('retention: report failed: %s' % e, file=sys.stderr)
-    return False
-
-class Reports:
-  # results go back in parts; once a part cannot be sent, the rest waits for the end of the run
-  def __init__(self):
-    self.pending, self.failed = [], False
-
-  def add(self, result):
-    self.pending.append(result)
-    if len(self.pending) >= report_batch and not self.failed:
-      self.failed = not self.send()
-
-  def send(self, summary=None, run=None):
-    # full parts, at the end all results with the summary and the run in the last part
-    final = summary is not None
-    while len(self.pending) >= report_batch or final:
-      body = {'results': self.pending[:report_batch]}
-      if final and len(self.pending) <= report_batch:
-        body.update(summary=summary, run=run)
-      if not post(body):
-        return False
-      del self.pending[:report_batch]
-      if 'summary' in body:
-        break
-    return True
-
-def doveadm(*args, partial=False):
-  result = subprocess.run(['doveadm', *args], capture_output=True, text=True, timeout=3600)
-  # mailbox status also fails for single folders that cannot be opened; the others are still listed
-  if result.returncode != 0 and not (partial and result.stdout):
-    raise RuntimeError((result.stderr or result.stdout).strip()[:500] or 'doveadm exit %d' % result.returncode)
-  return result.stdout
-
-def configuration():
-  # names of the special folders per key, and the prefixes of the shared and public namespaces (not the user's)
-  names = {key: set() for key in special_use}
-  prefixes = ['Shared/']
-  namespace, mailbox = None, None
-  # as vmail doveconf cannot read every included file (mail_crypt keys) and fails, but prints the namespaces
-  for line in subprocess.run(['doveconf', '-n'], capture_output=True, text=True).stdout.splitlines():
-    stripped = line.strip()
-    if line.startswith('namespace ') and line.endswith('{'):
-      namespace = {}
-    elif line == '}':
-      if namespace and namespace.get('type') in ('shared', 'public') and namespace.get('prefix'):
-        prefixes.append(namespace['prefix'].split('%')[0])
-      namespace = None
-    elif namespace is None:
-      continue
-    elif re.match(r'mailbox .+ \{$', stripped):
-      mailbox = stripped[len('mailbox '):-2].strip().strip('"')
-    elif stripped == '}':
-      mailbox = None
-    elif '=' in stripped:
-      key, value = (part.strip() for part in stripped.split('=', 1))
-      if mailbox is not None and key == 'special_use':
-        for folder, flag in special_use.items():
-          if flag in value.split():
-            names[folder].add(mailbox)
-      elif mailbox is None and key in ('type', 'prefix'):
-        namespace[key] = value
-  if not names['trash']:
-    raise RuntimeError('no special folders in the output of doveconf -n')
-  return names, [prefix for prefix in prefixes if prefix]
-
-def own_folders(user, prefixes):
-  # folder name => guid of the user's own folders; listed first, so folders others share are not opened
-  names = set(name for name in doveadm('-f', 'tab', 'mailbox', 'list', '-u', user).splitlines()[1:]
-    if name and not any((name + '/').startswith(prefix) for prefix in prefixes))
-  folders = {}
-  if not names:
-    return folders
-  for line in doveadm('-f', 'tab', 'mailbox', 'status', '-u', user, '--', 'guid', *sorted(names), partial=True).splitlines()[1:]:
-    name, _, guid = line.rpartition('\t')
-    if name in names and re.fullmatch(r'[0-9a-f]{32}', guid):
-      folders[name] = guid
-  return folders
-
-def canonical(name):
-  # Dovecot writes INBOX in capitals, also as the parent of folders
-  return 'INBOX' + name[5:] if name[:5].upper() == 'INBOX' and name[5:6] in ('', '/') else name
-
-def matches(action, name, names):
-  # match: the folder itself, an own folder name with *, the subfolders of a special folder or the inbox, all folders
-  key, match = action['folder'], action['match']
-  if match == 'all':
-    return True
-  if key.startswith(own_folder):
-    # * is the only wildcard of own folder names
-    pattern = '.*'.join(re.escape(part) for part in canonical(key[len(own_folder):]).split('*'))
-    return re.fullmatch(pattern, name) is not None
-  folders = ['INBOX'] if key == 'inbox' else names.get(key, ())
-  if match == 'subfolders':
-    return any(name.startswith(folder + '/') for folder in folders)
-  return name in folders
-
-def decide(actions, folders, names):
-  # rule id => (its action, the folders it decides about): every folder goes to the first action that matches it
-  decided = {}
-  for name in sorted(folders):
-    for action in actions:
-      if matches(action, canonical(name), names):
-        decided.setdefault(action['rule_id'], (action, []))[1].append(name)
-        break
-  return decided
-
-def found(user, query):
-  return len(doveadm('search', '-u', user, '--', *query).splitlines())
-
-def clean(user, action, selected, folders, result):
-  # name and guid together: doveadm then only opens these folders
-  query = []
-  for name in selected:
-    query += (['OR'] if query else []) + ['(', 'mailbox', name, 'mailbox-guid', folders[name], ')']
-  query = ['('] + query + [')', 'savedbefore', '%dd' % int(action['days'])]
-  if action['keep_flagged']:
-    query += ['NOT', 'FLAGGED']
-  result['matched'] = found(user, query)
-  if not result['matched'] or action['mode'] != 'delete':
-    return
-  try:
-    doveadm('expunge', '-u', user, '--', *query)
-    result['deleted'] = result['matched']
-  except Exception as e:
-    # part of the mail may be gone: what is left is counted
-    result['error'] = str(e)[:1000]
-    try:
-      result['deleted'] = max(0, result['matched'] - found(user, query))
-    except Exception:
-      pass
-
-def main():
-  if os.geteuid() == 0:
-    # as vmail like the nightly job: a run as root would leave a lock file the runs as vmail cannot open
-    os.execv('/usr/local/bin/gosu', ['gosu', 'vmail', sys.executable, os.path.abspath(__file__)] + sys.argv[1:])
-  # released by the kernel when the process ends, a crashed run leaves no stale lock
-  lock_fd = os.open(lock_file, os.O_RDWR | os.O_CREAT, 0o600)
-  try:
-    fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-  except OSError:
-    sys.exit("retention: can't lock " + lock_file)
-  reports = Reports()
-  run = {'started': int(time.time()), 'mailboxes': 0, 'matched': 0, 'deleted': 0, 'errors': 0, 'error': ''}
-  summary = {}
-  try:
-    names, prefixes = configuration()
-    for mailbox in plan():
-      user = mailbox['username']
-      run['mailboxes'] += 1
-      try:
-        if '*' in user or '?' in user:
-          # doveadm -u would run for every user the name matches as a mask
-          raise RuntimeError('username with * or ?, doveadm reads it as a mask')
-        folders, error = own_folders(user, prefixes), ''
-        decided = decide(mailbox['actions'], folders, names)
-      except Exception as e:
-        # the folders are unknown: an error for every rule of the mailbox
-        folders, error, decided = {}, str(e)[:1000], {}
-        for action in mailbox['actions']:
-          decided.setdefault(action['rule_id'], (action, []))
-      for rule_id, (action, selected) in decided.items():
-        rule = summary.setdefault(rule_id, {'rule_id': rule_id, 'mailboxes': 0, 'matched': 0, 'deleted': 0, 'errors': 0,
-          'simulated': int(action['mode'] != 'delete')})
-        # the log keeps 10000 characters of the folder names, more would only make the report too large
-        result = {'username': user, 'rule_id': rule_id, 'folder': '\n'.join(selected)[:10000], 'days': action['days'], 'matched': 0,
-          'deleted': 0, 'simulated': rule['simulated'], 'error': error}
-        if not error:
-          try:
-            clean(user, action, selected, folders, result)
-          except Exception as e:
-            result['error'] = str(e)[:1000]
-        for counter in (rule, run):
-          counter['matched'] += result['matched']
-          counter['deleted'] += result['deleted']
-          counter['errors'] += int(result['error'] != '')
-        rule['mailboxes'] += 1
-        # only mailboxes with found mail or an error are logged, every rule gets a summary
-        if result['matched'] or result['error']:
-          reports.add(result)
-  except Exception as e:
-    run['error'] = str(e)[:1000]
-  reported = reports.send(list(summary.values()), run)
-  print('retention: %d mailboxes, %d mails deleted, %d found by simulating rules, %d errors' % (run['mailboxes'],
-    run['deleted'], sum(rule['matched'] for rule in summary.values() if rule['simulated']), run['errors']))
-  if run['error']:
-    print('retention: %s' % run['error'], file=sys.stderr)
-  if not reported:
-    print('retention: the results could not be reported', file=sys.stderr)
-  if run['error'] or not reported:
-    sys.exit(1)
-
-if __name__ == '__main__':
-  main()

data/Dockerfiles/dovecot/trim_logs.sh

@@ -4,7 +4,7 @@ catch_non_zero() {
   ${CMD} > /dev/null
   EC=$?
   if [ ${EC} -ne 0 ]; then
-    echo "Command ${CMD} failed to execute, exit code was ${EC}"
+    echo "Command ${CMD//${REDISPASS}/***} failed to execute, exit code was ${EC}"
   fi
 }
 source /source_env.sh

data/Dockerfiles/nginx/Dockerfile

@@ -3,7 +3,8 @@ LABEL maintainer "The Infrastructure Company GmbH <info@servercow.de>"
 
 ENV PIP_BREAK_SYSTEM_PACKAGES=1
 
-RUN apk add --no-cache nginx \
+RUN apk upgrade --no-cache && \
+  apk add --no-cache nginx \
   python3 \
   py3-pip && \
   pip install --upgrade pip && \

data/Dockerfiles/postfix-tlspol/Dockerfile

@@ -4,7 +4,7 @@ WORKDIR /src
 ENV CGO_ENABLED=0 \
     GO111MODULE=on \
 		NOOPT=1 \
-    VERSION=1.14.0
+    VERSION=1.14.5
 
 RUN git clone --branch v${VERSION} https://github.com/Zuplu/postfix-tlspol && \
     cd /src/postfix-tlspol && \

data/Dockerfiles/postfix/Dockerfile

@@ -26,7 +26,6 @@ RUN groupadd -g 102 postfix \
 	postfix \
 	postfix-mysql \
 	postfix-pcre \
-	python3 \
 	redis-tools \
 	sasl2-bin \
 	sudo \
@@ -47,7 +46,6 @@ COPY postfix.sh /opt/postfix.sh
 COPY rspamd-pipe-ham /usr/local/bin/rspamd-pipe-ham
 COPY rspamd-pipe-spam /usr/local/bin/rspamd-pipe-spam
 COPY whitelist_forwardinghosts.sh /usr/local/bin/whitelist_forwardinghosts.sh
-COPY mailcow-policy.py /usr/local/bin/mailcow-policy.py
 COPY stop-supervisor.sh /usr/local/sbin/stop-supervisor.sh
 COPY docker-entrypoint.sh /docker-entrypoint.sh
 
@@ -55,7 +53,6 @@ RUN chmod +x /opt/postfix.sh \
   /usr/local/bin/rspamd-pipe-ham \
   /usr/local/bin/rspamd-pipe-spam \
   /usr/local/bin/whitelist_forwardinghosts.sh \
-  /usr/local/bin/mailcow-policy.py \
   /usr/local/sbin/stop-supervisor.sh
 RUN rm -rf /tmp/* /var/tmp/*
 

data/Dockerfiles/postfix/mailcow-policy.py

@@ -1,54 +0,0 @@
-#!/usr/bin/python3
-# Policy service of the mail policies (mailcow-policy in master.cf, spawned by Postfix): reads the policy requests
-# of smtpd and asks the mailcow UI for the action (nginx:8081/mailpolicy.php). Without an answer the recipient is
-# deferred; smtpd only asks while policies or internal aliases exist (mysql_mail_policy_maps.cf).
-
-import http.client
-import re
-import socket
-import sys
-from urllib.parse import urlencode
-
-# one line: DUNNO or a rejection with code and text
-valid_action = re.compile(rb'DUNNO|[45]\d\d [45]\.\d{1,3}\.\d{1,3} [\x20-\x7e]{1,200}')
-unavailable = b'DEFER_IF_PERMIT Mail policy check temporarily unavailable'
-# address of nginx, resolved once per process (a lookup costs more than the check)
-nginx = None
-
-def check(request):
-  global nginx
-  body = urlencode(sorted(request.items())).encode()
-  try:
-    if nginx is None:
-      nginx = socket.gethostbyname('nginx')
-    # http.client sets TCP_NODELAY, so the body is not held back until the headers are acknowledged
-    # below the timeout of Postfix (main.cf), so an answer always arrives
-    conn = http.client.HTTPConnection(nginx, 8081, timeout=8)
-    try:
-      conn.request('POST', '/mailpolicy.php', body, {'Host': 'nginx', 'Content-Type': 'application/x-www-form-urlencoded'})
-      response = conn.getresponse()
-      action = response.read()
-    finally:
-      conn.close()
-  except (OSError, http.client.HTTPException):
-    # nginx may have a new address after a restart
-    nginx = None
-    return unavailable
-  return action if response.status == 200 and valid_action.fullmatch(action) else unavailable
-
-def main():
-  request = {}
-  # attributes as bytes: addresses may contain any 8-bit characters
-  for line in sys.stdin.buffer:
-    line = line.rstrip(b'\r\n')
-    if line:
-      name, separator, value = line.partition(b'=')
-      if separator:
-        request[name] = value
-      continue
-    sys.stdout.buffer.write(b'action=' + check(request) + b'\n\n')
-    sys.stdout.buffer.flush()
-    request = {}
-
-if __name__ == '__main__':
-  main()

data/Dockerfiles/postfix/postfix.sh

@@ -15,21 +15,6 @@ until dig +short mailcow.email > /dev/null; do
   sleep 1
 done
 
-# Wait for updated schema (DB_VERSION from docker-compose.yml), the maps read tables the init_db of php-fpm creates on updates
-DBV_NOW=$(mariadb --ssl=false --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null)
-DBV_WAIT=0
-while [[ "${DBV_NOW}" != "${DB_VERSION}" ]] && [[ ${DBV_WAIT} -lt 60 ]]; do
-  echo "Waiting for schema update..."
-  sleep 5
-  DBV_WAIT=$((DBV_WAIT + 1))
-  DBV_NOW=$(mariadb --ssl=false --socket=/var/run/mysqld/mysqld.sock -u ${DBUSER} -p${DBPASS} ${DBNAME} -e "SELECT version FROM versions WHERE application = 'db_schema';" -BN 2>/dev/null)
-done
-if [[ "${DBV_NOW}" != "${DB_VERSION}" ]]; then
-  echo -e "\e[31mDB schema is ${DBV_NOW}, expected ${DB_VERSION} - starting anyway, check the php-fpm log (init_db)\e[0m"
-else
-  echo "DB schema is ${DBV_NOW}"
-fi
-
 cat <<EOF > /etc/aliases
 # Autogenerated by mailcow
 null: /dev/null
@@ -178,45 +163,6 @@ query = SELECT CONCAT('smtp_via_transport_maps:', nexthop) AS transport FROM tra
   AND destination = '%s';
 EOF
 
-# Archive copies (journal) to the archive set in the admin UI: on port 591 the sender archive-bounce@localhost
-# sends the whole mail through archive_smtp (master.cf), whatever the recipient domain
-cat <<EOF > /opt/postfix/conf/sql/mysql_archive_sender_filter_maps.cf
-# Autogenerated by mailcow
-user = ${DBUSER}
-password = ${DBPASS}
-hosts = unix:/var/run/mysqld/mysqld.sock
-dbname = ${DBNAME}
-query = SELECT CONCAT('FILTER archive_smtp:', nexthop) AS action FROM archive_transport
-  WHERE active = '1'
-  AND '%s' = 'archive-bounce@localhost'
-  LIMIT 1;
-EOF
-
-cat <<EOF > /opt/postfix/conf/sql/mysql_archive_sasl_passwd_maps.cf
-# Autogenerated by mailcow
-user = ${DBUSER}
-password = ${DBPASS}
-hosts = unix:/var/run/mysqld/mysqld.sock
-dbname = ${DBNAME}
-query = SELECT CONCAT_WS(':', username, password) AS auth_data FROM archive_transport
-  WHERE nexthop = '%s'
-  AND active = '1'
-  AND username != ''
-  LIMIT 1;
-EOF
-
-cat <<EOF > /opt/postfix/conf/sql/mysql_archive_tls_policy_maps.cf
-# Autogenerated by mailcow
-user = ${DBUSER}
-password = ${DBPASS}
-hosts = unix:/var/run/mysqld/mysqld.sock
-dbname = ${DBNAME}
-query = SELECT tls_policy FROM archive_transport
-  WHERE nexthop = '%s'
-  AND active = '1'
-  LIMIT 1;
-EOF
-
 cat <<EOF > /opt/postfix/conf/sql/mysql_virtual_resource_maps.cf
 # Autogenerated by mailcow
 user = ${DBUSER}
@@ -278,19 +224,6 @@ query = SELECT CONCAT_WS(':', username, password) AS auth_data FROM transports
   LIMIT 1;
 EOF
 
-# Mail policies: while policies or internal aliases exist, smtpd asks the policy service (mailcow_mail_policy in main.cf)
-cat <<EOF > /opt/postfix/conf/sql/mysql_mail_policy_maps.cf
-# Autogenerated by mailcow
-user = ${DBUSER}
-password = ${DBPASS}
-hosts = unix:/var/run/mysqld/mysqld.sock
-dbname = ${DBNAME}
-query = SELECT 'mailcow_mail_policy' FROM DUAL WHERE '%s' != ''
-  AND (EXISTS (SELECT 1 FROM mail_policy_member JOIN mail_policy ON mail_policy.id = mail_policy_member.policy_id
-      WHERE mail_policy.active = '1')
-    OR EXISTS (SELECT 1 FROM alias WHERE internal = '1' AND active IN ('1', '2')));
-EOF
-
 cat <<EOF > /opt/postfix/conf/sql/mysql_virtual_alias_domain_maps.cf
 # Autogenerated by mailcow
 user = ${DBUSER}

data/Dockerfiles/restic/Dockerfile

@@ -1,28 +0,0 @@
-FROM alpine:3.24
-
-LABEL maintainer="The Infrastructure Company GmbH <info@servercow.de>"
-
-ENV LANG=C.UTF-8
-ENV LC_ALL=C.UTF-8
-# MariaDB Connector/C >= 3.4 refuses DB servers without TLS, the runner connects through the socket
-ENV MARIADB_TLS_DISABLE_PEER_VERIFICATION=1
-
-RUN apk add --no-cache --update \
-  ca-certificates \
-  git \
-  mariadb-client \
-  openssh-client \
-  py3-mysqlclient \
-  py3-pynacl \
-  python3 \
-  restic \
-  tini \
-  tzdata \
-  # dump, export, manifest and the mailcow directory are below /mailcow (paths of the snapshot), only root may read them
-  && mkdir -m 700 /mailcow
-
-COPY restic_runner.py /usr/local/bin/restic_runner.py
-COPY sogo_restore.py /usr/local/bin/sogo_restore.py
-
-ENTRYPOINT ["/sbin/tini", "--"]
-CMD ["/usr/bin/python3", "-u", "/usr/local/bin/restic_runner.py"]

data/Dockerfiles/restic/sogo_restore.py

@@ -1,118 +0,0 @@
-"""SOGo part of a mailbox restore (component sogo, overwrites), called by the restic runner for one mailbox.
-
-Runs `sogo-tool restore -f ALL <dir> <user>` and `sogo-tool restore -p <dir> <user>` in the sogo container through
-dockerapi on the user's file of the nightly sogo-tool backup: missing calendars, address books and entries come back,
-the names and shares of existing folders and all SOGo preferences are replaced by the backup's, entries changed since
-stay. Then the user's entries in SOGo's cache (memcached) are removed, so the change shows at once.
-"""
-
-import json
-import os
-import re
-import socket
-import ssl
-import urllib.request
-
-dockerapi_url = 'https://dockerapi:443'
-memcached_addr = ('memcached-mailcow', 11211)
-# dockerapi stops each of the two sogo-tool runs after 850 s
-restore_timeout = 1800
-# accepted by the dockerapi handler as well
-user_re = re.compile(r'[A-Za-z0-9_][A-Za-z0-9._+-]*@[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)+')
-
-
-def _short(text, limit=300):
-  # sogo-tool may print database URLs: no credentials, one line, the end has the error
-  text = re.sub(r'([A-Za-z][A-Za-z0-9+.-]*://)[^\s/@]*@', r'\1***@', str(text))
-  text = ' '.join(text.split())
-  return text if len(text) <= limit else '...' + text[-limit:]
-
-
-def _api(path, body=None, timeout=60):
-  # dockerapi uses the mail certificate, its name does not match
-  ctx = ssl.create_default_context()
-  ctx.check_hostname = False
-  ctx.verify_mode = ssl.CERT_NONE
-  data = None if body is None else json.dumps(body).encode()
-  req = urllib.request.Request(dockerapi_url + path, data=data, headers={'Content-Type': 'application/json'},
-    method='GET' if body is None else 'POST')
-  with urllib.request.urlopen(req, timeout=timeout, context=ctx) as response:
-    return json.load(response)
-
-
-def _sogo_container():
-  project = os.environ.get('COMPOSE_PROJECT_NAME', '').lower()
-  by_name = None
-  for cid, info in _api('/containers/json').items():
-    if not cid.isalnum() or not isinstance(info, dict):
-      continue
-    labels = (info.get('Config') or {}).get('Labels') or {}
-    if labels.get('com.docker.compose.service') == 'sogo-mailcow':
-      if not project or labels.get('com.docker.compose.project', '').lower() == project:
-        return cid
-    elif by_name is None and 'sogo-mailcow' in str(info.get('Name', '')):
-      by_name = cid
-  if by_name:
-    return by_name
-  raise RuntimeError('sogo container not found')
-
-
-def _memcached_delete(keys):
-  keys = [k for k in keys if 0 < len(k.encode()) <= 250 and not re.search(r'[\s\x00-\x1f\x7f]', k)]
-  if not keys:
-    return
-  with socket.create_connection(memcached_addr, timeout=10) as s:
-    for key in keys:
-      s.sendall(b'delete ' + key.encode() + b'\r\n')
-      reply = b''
-      while not reply.endswith(b'\r\n'):
-        chunk = s.recv(64)
-        if not chunk:
-          raise RuntimeError('memcached closed the connection')
-        reply += chunk
-      if reply.strip() not in (b'DELETED', b'NOT_FOUND'):
-        raise RuntimeError('memcached: %s' % _short(reply.decode('ascii', 'replace')))
-
-
-def _cache_keys(db, username):
-  # preferences and the ACLs of the user's folders (<uid>/Calendar/personal+acl); never read the c_*location
-  # columns of sogo_folder_info (database credentials)
-  keys = ['%s+defaults' % username, '%s+settings' % username]
-  cur = db.cursor()
-  try:
-    cur.execute('SELECT c_path FROM sogo_folder_info WHERE c_path2 = %s', (username,))
-    keys += ['%s+acl' % row[0][len('/Users/'):] for row in cur.fetchall() if row[0].startswith('/Users/')]
-  finally:
-    cur.close()
-  return keys
-
-
-def restore(db, backup_dir, username, log):
-  """Restores the SOGo data of one user from <backup_dir>/<username> (staged readable for the sogo user only).
-  db: open MySQLdb connection; log: callable for short progress lines. Returns {"added": 0|1, "errors": [...]}."""
-  result = {'added': 0, 'errors': []}
-  if not user_re.fullmatch(username):
-    result['errors'].append('username not supported by the SOGo restore')
-    return result
-  path = os.path.join(backup_dir, username)
-  if not os.path.isfile(path) or os.path.islink(path):
-    # sogo-tool backup ALL skips users without SOGo folders
-    # a key the UI translates
-    result['note'] = 'no_sogo_data'
-    log('sogo: no SOGo data in the snapshot')
-    return result
-  try:
-    res = _api('/containers/%s/exec' % _sogo_container(), {'cmd': 'sogo', 'task': 'restore', 'dir': backup_dir,
-      'username': username}, timeout=restore_timeout)
-    if not isinstance(res, dict) or res.get('type') != 'success':
-      raise RuntimeError('sogo-tool restore failed: %s' % _short(res.get('msg', '') if isinstance(res, dict) else res))
-    result['added'] = 1
-    log('sogo: calendars, address books and preferences reset to the backup')
-  except Exception as e:
-    result['errors'].append(_short(e))
-  try:
-    # SOGo caches ACLs and preferences for 15 minutes
-    _memcached_delete(_cache_keys(db, username))
-  except Exception as e:
-    result['errors'].append('SOGo cache not cleared, changes may show up after 15 minutes: %s' % _short(e))
-  return result

data/Dockerfiles/rspamd/docker-entrypoint.sh

@@ -277,9 +277,34 @@ if [[ ! -z ${SPAMHAUS_DQS_KEY} ]]; then
       }
     }
     "DBL" {
-        # override the defaults for DBL defined in modules.d/rbl.conf
+        # full copy of the DBL rule from modules.d/rbl.conf, pointed at DQS:
+        # the override include replaces the stock rule instead of merging into it
         rbl = "${SPAMHAUS_DQS_KEY}.dbl.dq.spamhaus.net";
         disable_monitoring = true;
+        checks = ["emails", "dkim", "helo", "rdns", "replyto", "urls", "content_urls"];
+        no_ip = true;
+        ignore_defaults = true;
+        exclude_users = false;
+        emails_domainonly = true;
+        selector = {
+            from_smtp = "from('smtp'):domain";
+            from_mime = "from('mime'):domain";
+            mid = "header(Message-Id).regexp('@([^\.]+\.[^>]+)').last";
+        }
+        returncodes = {
+            DBL_SPAM = "127.0.1.2";
+            DBL_PHISH = "127.0.1.4";
+            DBL_MALWARE = "127.0.1.5";
+            DBL_BOTNET = "127.0.1.6";
+            DBL_ABUSE = "127.0.1.102";
+            DBL_ABUSE_REDIR = "127.0.1.103";
+            DBL_ABUSE_PHISH = "127.0.1.104";
+            DBL_ABUSE_MALWARE = "127.0.1.105";
+            DBL_ABUSE_BOTNET = "127.0.1.106";
+            DBL_PROHIBIT = "127.0.1.255";
+            DBL_BLOCKED_OPENRESOLVER = "127.255.255.254";
+            DBL_BLOCKED = "127.255.255.255";
+        }
     }
     "ZRD" {
         ignore_defaults = true;

data/conf/dovecot/auth/mailcowauth.php

@@ -116,8 +116,8 @@ if ($result === false){
     exit;
   }
   if ($result) {
-    error_log('MAILCOWAUTH: User auth for user ' . $post['username'] . " with service " . $post['service'] . " from IP " . $client_ip);
-    set_sasl_log($post['username'], $client_ip, $post['service']);
+    error_log('MAILCOWAUTH: User auth for user ' . $post['username'] . " with service " . $service . " from IP " . $client_ip);
+    set_sasl_log($post['username'], $client_ip, $service);
   }
 }
 

data/conf/dovecot/dovecot.conf

@@ -199,9 +199,6 @@ plugin {
   sieve_plugins = sieve_imapsieve sieve_extprograms
   sieve_vacation_send_from_recipient = yes
   sieve_redirect_envelope_from = recipient
-  # Notifications are sent with the address of the mailbox (not :from or postmaster), so mail policies apply;
-  # the old notify extension always sends as postmaster and is on by default, so sieve_extensions disables it
-  sieve_notify_mailto_envelope_from = recipient
   # From elsewhere to Spam folder
   imapsieve_mailbox1_name = Junk
   imapsieve_mailbox1_causes = COPY
@@ -218,7 +215,7 @@ plugin {
   quota_warning2 = storage=80%% quota-warning 80 %u
   sieve_pipe_bin_dir = /usr/lib/dovecot/sieve
   sieve_global_extensions = +vnd.dovecot.pipe +vnd.dovecot.execute
-  sieve_extensions = -notify +imapflags +vacation-seconds +editheader
+  sieve_extensions = +notify +imapflags +vacation-seconds +editheader
   sieve_max_script_size = 1M
   sieve_max_redirects = 100
   sieve_max_actions = 101

data/conf/nginx/templates/nginx.conf.j2

@@ -150,15 +150,6 @@ http {
         access_log /var/log/nginx/access.log;
         root /dynmaps;
 
-        # backup endpoint of the restic runner: large restore requests, long merges of mailboxes and domains
-        location = /backup.php {
-            client_max_body_size 32m;
-            fastcgi_read_timeout 600s;
-            fastcgi_pass {{ PHPFPMHOST }}:9001;
-            include fastcgi_params;
-            fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
-        }
-
         location ~ \.php$ {
             try_files $uri =404;
             fastcgi_split_path_info ^(.+\.php)(/.+)$;
@@ -168,13 +159,6 @@ http {
             fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
             fastcgi_param PATH_INFO $fastcgi_path_info;
         }
-
-        location = /retention.php {
-            client_max_body_size 32m;
-            fastcgi_pass {{ PHPFPMHOST }}:9001;
-            include fastcgi_params;
-            fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
-        }
     }
 
     # rspamd meta_exporter:

data/conf/nginx/templates/sites-default.conf.j2

@@ -94,11 +94,11 @@ location / {
 }
 
 location /qhandler {
-    rewrite ^/qhandler/(.*)/(.*) /qhandler.php?action=$1&hash=$2;
+    rewrite ^/qhandler/([^/]+)/(.+?)/?$ /qhandler.php?action=$1&hash=$2;
 }
 
 location /edit {
-    rewrite ^/edit/(.*)/(.*) /edit.php?$1=$2;
+    rewrite ^/edit/([^/]+)/(.+?)/?$ /edit.php?$1=$2;
 }
 
 location ~ ^/api/v1/(.*)$ {
@@ -187,6 +187,12 @@ location /sogo-auth-verify {
 }
 
 location ^~ /Microsoft-Server-ActiveSync {
+    # Outlook for iOS/Android (Microsoft's sync service) sends EAS commands without credentials
+    # and only authenticates after a 401 Basic challenge. SOGo answers those with 403, so challenge here.
+    if ($http_authorization = "") {
+        add_header WWW-Authenticate 'Basic realm="SOGo"' always;
+        return 401;
+    }
     auth_request /sogo-auth-verify;
     auth_request_set $user $upstream_http_x_user;
     auth_request_set $auth $upstream_http_x_auth;

data/conf/phpfpm/crons/identity-sync.php

@@ -60,7 +60,6 @@ $_SESSION['acl']['sogo_access'] = "1";
 $_SESSION['acl']['protocol_access'] = "1";
 $_SESSION['acl']['mailbox_relayhost'] = "1";
 $_SESSION['acl']['unlimited_quota'] = "1";
-$_SESSION['acl']['archive'] = "1";
 
 // One run at a time, released when the process ends
 $lock = fopen('/tmp/identity-sync.lock', 'c');

data/conf/postfix/local_transport

@@ -1,3 +1,2 @@
 /watchdog@localhost$/   watchdog_discard:
-/^archive-bounce@localhost$/  archive_discard:
 /localhost$/  local:

data/conf/postfix/main.cf

@@ -50,10 +50,6 @@ postscreen_greet_wait = 3s
 postscreen_non_smtp_command_enable = no
 postscreen_pipelining_enable = no
 proxy_read_maps = proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_transport_maps.cf,
-  proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sender_filter_maps.cf,
-  proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sasl_passwd_maps.cf,
-  proxy:mysql:/opt/postfix/conf/sql/mysql_archive_tls_policy_maps.cf,
-  proxy:mysql:/opt/postfix/conf/sql/mysql_mail_policy_maps.cf,
   proxy:mysql:/opt/postfix/conf/sql/mysql_mbr_access_maps.cf,
   proxy:mysql:/opt/postfix/conf/sql/mysql_tls_enforce_in_policy.cf,
   $sender_dependent_default_transport_maps,
@@ -94,7 +90,6 @@ smtpd_hard_error_limit = ${stress?1}${stress:5}
 smtpd_helo_required = yes
 smtpd_proxy_timeout = 600s
 smtpd_recipient_restrictions = check_recipient_mx_access proxy:mysql:/opt/postfix/conf/sql/mysql_mbr_access_maps.cf,
-  check_recipient_access proxy:mysql:/opt/postfix/conf/sql/mysql_mail_policy_maps.cf,
   permit_sasl_authenticated,
   permit_mynetworks,
   check_recipient_access proxy:mysql:/opt/postfix/conf/sql/mysql_tls_enforce_in_policy.cf,
@@ -175,11 +170,5 @@ parent_domain_matches_subdomains = debug_peer_list,fast_flush_domains,mynetworks
 # This Option is added to correctly set the X-Original-To Header when mails are send to lmtp (dovecot)
 lmtp_destination_recipient_limit=1
 
-# Mail policies: while policies or internal aliases exist, mysql_mail_policy_maps.cf returns this class and smtpd
-# asks the policy service for every recipient (mailcow-policy in master.cf); no answer defers the recipient
-smtpd_restriction_classes = mailcow_mail_policy
-mailcow_mail_policy = check_policy_service { unix:private/mailcow-policy, timeout=10s, try_limit=1, default_action=DEFER_IF_PERMIT }
-mailcow-policy_time_limit = 3600s
-
 # DO NOT EDIT ANYTHING BELOW #
 # Overrides #

data/conf/postfix/master.cf

@@ -61,15 +61,13 @@ submission inet n       -       n       -       -       smtpd
   -o non_smtpd_milters=
   -o syslog_name=postfix/quarantine
 
-# used to send bcc mails and archive copies (sender archive-bounce@localhost goes to the archive)
-# the archive FILTER comes after the one of MX-based transports, Postfix uses the last FILTER
+# used to send bcc mails
 591 inet n      -       n       -       -       smtpd
   -o smtpd_helo_restrictions=
   -o smtpd_client_restrictions=permit_mynetworks,reject
   -o smtpd_tls_auth_only=no
   -o smtpd_milters=
   -o non_smtpd_milters=
-  -o smtpd_recipient_restrictions=check_recipient_mx_access,proxy:mysql:/opt/postfix/conf/sql/mysql_mbr_access_maps.cf,check_sender_access,proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sender_filter_maps.cf,permit_mynetworks,reject
   -o syslog_name=postfix/bcc
 
 # enforced smtp connector
@@ -83,27 +81,6 @@ smtp_enforced_tls      unix  -       -       n       -       -       smtp
 smtp_via_transport_maps      unix  -       -       n       -       -       smtp
   -o smtp_sasl_password_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_sasl_passwd_maps_transport_maps.cf
 
-# archive copies (journal) to the archive set in the admin UI: only mails from archive-bounce@localhost on 591 get here
-# (FILTER of mysql_archive_sender_filter_maps), sent unchanged, TLS by default, refusals are retried (soft_bounce)
-# until maximal_queue_lifetime
-archive_smtp      unix  -       -       n       -       -       smtp
-  -o soft_bounce=yes
-  -o smtp_header_checks=
-  -o smtp_mime_header_checks=
-  -o smtp_nested_header_checks=
-  -o smtp_body_checks=
-  -o smtp_line_length_limit=0
-  -o disable_mime_output_conversion=yes
-  -o smtp_tls_security_level=encrypt
-  -o smtp_tls_policy_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_archive_tls_policy_maps.cf
-  -o smtp_sasl_auth_enable=yes
-  -o smtp_sasl_password_maps=proxy:mysql:/opt/postfix/conf/sql/mysql_archive_sasl_passwd_maps.cf
-  -o smtp_sasl_tls_security_options=noanonymous
-  -o syslog_name=postfix/archive
-# notices about archive copies (delays, bounces) go to archive-bounce@localhost and are dropped here
-archive_discard   unix  -       -       n       -       -       discard
-  -o syslog_name=postfix/archive
-
 tlsproxy   unix  -       -       n       -       0       tlsproxy
 dnsblog    unix  -       -       n       -       0       dnsblog
 pickup     fifo  n       -       n       60      1       pickup
@@ -140,14 +117,10 @@ smtp_sender_cleanup unix n - y - 0 cleanup
 127.0.0.1:10027 inet n n n - 0 spawn user=nobody argv=/usr/local/bin/whitelist_forwardinghosts.sh
 # end whitelist_fwd
 
-# mail policies: policy service of smtpd, asks the mailcow UI (mailcow_mail_policy in main.cf)
-mailcow-policy unix - n n - 0 spawn user=nobody argv=/usr/local/bin/mailcow-policy.py
-
 # start watchdog-specific
-# logs to local7 (hidden), no mail policy check
+# logs to local7 (hidden)
 589 inet n      -       n       -       -       smtpd
   -o smtpd_client_restrictions=permit_mynetworks,reject
-  -o smtpd_recipient_restrictions=permit_mynetworks,reject
   -o syslog_name=watchdog
   -o syslog_facility=local7
   -o smtpd_milters=

data/conf/rspamd/dynmaps/archive.php

@@ -1,47 +0,0 @@
-<?php
-// Archive decision for an accepted mail, asked by the MAILCOW_ARCHIVE symbol of rspamd
-// Request: JSON {from, rcpt[], user, bcc, qid, received, sha256}; answer: JSON {archive, rcpt, sender, headers}
-header('Content-Type: application/json');
-require_once "vars.inc.php";
-require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
-// Do not show errors, we log to using error_log
-ini_set('error_reporting', 0);
-// Init database
-$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
-$opt = [
-    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
-    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-    PDO::ATTR_EMULATE_PREPARES   => false,
-];
-try {
-  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
-}
-catch (PDOException $e) {
-  error_log("ARCHIVE: " . $e . PHP_EOL);
-  http_response_code(501);
-  exit;
-}
-// Init Redis, it keeps the global switch; without it nothing is archived
-$redis = new Redis();
-try {
-  $redis->connect('redis-mailcow', 6379);
-  $redis->auth(getenv("REDISPASS"));
-}
-catch (Exception $e) {
-  $redis = null;
-}
-
-$request = json_decode(file_get_contents('php://input'), true);
-if (!is_array($request)) {
-  http_response_code(400);
-  exit;
-}
-
-try {
-  $archive = \Mailcow\Archive\ArchiveModule::create($pdo, \Mailcow\Core\Context::system(), $redis);
-  echo json_encode($archive->journal->prepare($request));
-}
-catch (Throwable $e) {
-  error_log("ARCHIVE: " . $e->getMessage() . PHP_EOL);
-  http_response_code(502);
-}

data/conf/rspamd/dynmaps/backup.php

@@ -1,97 +0,0 @@
-<?php
-// Backup with restic: endpoint of the runner in the restic container (Mailcow\Backup)
-// GET ?domains=1: {"domains": [...]}, all domains
-// GET ?export=<domain>[&after=<username>]: a page of the export of a domain (the first with domain_data and aliases,
-//   every page with a part of the mailboxes and "next_after", the last username to ask for the next page with, null
-//   after the last), incl. password hashes, DKIM keys and secrets; the runner merges the pages into
-//   /mailcow/export/<domain>.json of the snapshot.
-// POST {"action": "restore_domain", "job": <id>, "data": <domain_data>}: recreates a deleted domain
-// POST {"action": "restore_aliases", "job": <id>, "aliases": [...]}: a part (at most 500) of the aliases of a domain restore
-// POST {"action": "restore", "job": <id>, "username": <mailbox>, "data": <its section of the export>}: one mailbox
-//   (created first when it was deleted); all answer {"result": {<component>: {"added", "skipped", "errors"}}}
-// All need the header X-Backup-Token: HMAC-SHA256 of "backup" with the database password as key
-header('Content-Type: application/json');
-require_once "vars.inc.php";
-require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
-// Do not show errors, we log to using error_log
-ini_set('error_reporting', 0);
-if (empty($database_pass) || !hash_equals(hash_hmac('sha256', 'backup', $database_pass), (string)($_SERVER['HTTP_X_BACKUP_TOKEN'] ?? ''))) {
-  http_response_code(403);
-  exit;
-}
-// Init database
-$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
-$opt = [
-    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
-    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-    PDO::ATTR_EMULATE_PREPARES   => false,
-];
-try {
-  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
-}
-catch (PDOException $e) {
-  error_log("BACKUP: " . $e . PHP_EOL);
-  http_response_code(501);
-  exit;
-}
-
-try {
-  // DKIM keys, rate limits and the domain map are kept in Redis
-  $redis = new Redis();
-  if (!empty(getenv('REDIS_SLAVEOF_IP'))) {
-    $redis->connect(getenv('REDIS_SLAVEOF_IP'), getenv('REDIS_SLAVEOF_PORT'));
-  }
-  else {
-    $redis->connect('redis-mailcow', 6379);
-  }
-  $redis->auth(getenv("REDISPASS"));
-  $config = array(
-    'timezone' => getenv('TZ') ?: 'UTC',
-    'alias_disable_external_domains' => ($ALIAS_DISABLE_EXTERNAL_DOMAINS ?? false) === true,
-  );
-  if ($_SERVER['REQUEST_METHOD'] === 'POST') {
-    $body = json_decode((string)file_get_contents('php://input'), true);
-    if (!is_array($body)) {
-      http_response_code(400);
-      exit;
-    }
-    // The restore has the side effects of the UI: SOGo view and restart, Dovecot auth cache, group shares (legacy code)
-    require_once __DIR__ . '/../web/inc/functions.inc.php';
-    require_once __DIR__ . '/../web/inc/functions.auth.inc.php';
-    require_once __DIR__ . '/../web/inc/functions.docker.inc.php';
-    require_once __DIR__ . '/../web/inc/functions.mailbox.inc.php';
-    require_once __DIR__ . '/../web/inc/functions.ratelimit.inc.php';
-    require_once __DIR__ . '/../web/inc/functions.acl.inc.php';
-    // The legacy mailbox functions run as admin, without a session
-    $_SESSION = array(
-      'mailcow_cc_username' => 'admin',
-      'mailcow_cc_role' => 'admin',
-      'access_all_exception' => '0',
-      'acl' => array_fill_keys(array('tls_policy', 'quarantine_notification', 'quarantine_category', 'ratelimit', 'sogo_access',
-        'protocol_access', 'mailbox_relayhost', 'unlimited_quota'), '1'),
-    );
-    $backup = \Mailcow\Backup\BackupModule::create($pdo, \Mailcow\Core\Context::system(), $config, $redis);
-    echo json_encode($backup->api->post($body), JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
-  }
-  else {
-    $backup = \Mailcow\Backup\BackupModule::create($pdo, \Mailcow\Core\Context::system(), $config, $redis);
-    if (isset($_GET['export'])) {
-      echo json_encode($backup->api->export((string)$_GET['export'], (isset($_GET['after']) && is_string($_GET['after'])) ? $_GET['after'] : null),
-        JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR);
-    }
-    elseif (isset($_GET['domains'])) {
-      echo json_encode($backup->api->domains());
-    }
-    else {
-      http_response_code(400);
-    }
-  }
-}
-catch (\Mailcow\Core\UserError $e) {
-  http_response_code(($e->key === 'domain_not_found') ? 404 : 400);
-  echo json_encode(array('error' => $e->key, 'args' => $e->args));
-}
-catch (Throwable $e) {
-  error_log("BACKUP: " . $e->getMessage() . PHP_EOL);
-  http_response_code(500);
-}

data/conf/rspamd/dynmaps/mailpolicy.php

@@ -1,39 +0,0 @@
-<?php
-// Mail policy check for one recipient, asked by the Postfix policy service (mailcow-policy in master.cf)
-// Request: POST with the attributes of the Postfix policy request; answer: the action (DUNNO or the rejection)
-header('Content-Type: text/plain');
-require_once "vars.inc.php";
-require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
-// Do not show errors, we log to using error_log
-ini_set('error_reporting', 0);
-// Init database
-$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
-$opt = [
-    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
-    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-    PDO::ATTR_EMULATE_PREPARES   => false,
-];
-try {
-  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
-}
-catch (PDOException $e) {
-  error_log("MAILPOLICY: " . $e . PHP_EOL);
-  http_response_code(501);
-  exit;
-}
-
-if (!isset($_POST['request']) || $_POST['request'] !== 'smtpd_access_policy') {
-  http_response_code(400);
-  exit;
-}
-$value = fn(string $key): string => is_string($_POST[$key] ?? null) ? $_POST[$key] : '';
-
-try {
-  $policy = \Mailcow\MailPolicy\MailPolicyModule::create($pdo, \Mailcow\Core\Context::system());
-  echo $policy->api->check($value('sender'), $value('recipient'), $value('sasl_username'), (int)$value('server_port'), $value('helo_name'));
-}
-catch (Throwable $e) {
-  // the policy service answers with a temporary error
-  error_log("MAILPOLICY: " . $e->getMessage() . PHP_EOL);
-  http_response_code(500);
-}

data/conf/rspamd/dynmaps/retention.php

@@ -1,52 +0,0 @@
-<?php
-// Retention rules for the nightly run in the dovecot container (retention_runner.py)
-// GET ?after=<username>&limit=<n>: a page of the plan, per mailbox the folders to clean up (JSON)
-// POST (JSON {"results": [...], "summary": [...], "run": {...}}): the results
-// Both need the header X-Retention-Token: HMAC-SHA256 of "retention" with the database password as key
-header('Content-Type: application/json');
-require_once "vars.inc.php";
-require_once __DIR__ . '/../web/inc/lib/vendor/autoload.php';
-// Do not show errors, we log to using error_log
-ini_set('error_reporting', 0);
-if (empty($database_pass) || !hash_equals(hash_hmac('sha256', 'retention', $database_pass), (string)($_SERVER['HTTP_X_RETENTION_TOKEN'] ?? ''))) {
-  http_response_code(403);
-  exit;
-}
-// Init database
-$dsn = $database_type . ":unix_socket=" . $database_sock . ";dbname=" . $database_name;
-$opt = [
-    PDO::ATTR_ERRMODE            => PDO::ERRMODE_EXCEPTION,
-    PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
-    PDO::ATTR_EMULATE_PREPARES   => false,
-];
-try {
-  $pdo = new PDO($dsn, $database_user, $database_pass, $opt);
-}
-catch (PDOException $e) {
-  error_log("RETENTION: " . $e . PHP_EOL);
-  http_response_code(501);
-  exit;
-}
-
-try {
-  $retention = \Mailcow\Retention\RetentionModule::create($pdo, \Mailcow\Core\Context::system());
-  if ($_SERVER['REQUEST_METHOD'] === 'POST') {
-    $body = json_decode((string)file_get_contents('php://input'), true);
-    if (!is_array($body) || !is_array($body['results'] ?? null)) {
-      http_response_code(400);
-      exit;
-    }
-    echo json_encode(['stored' => $retention->api->report($body['results'], is_array($body['summary'] ?? null) ? $body['summary'] : [],
-      is_array($body['run'] ?? null) ? $body['run'] : null)]);
-  }
-  else {
-    echo json_encode($retention->api->plan((string)($_GET['after'] ?? ''), (int)($_GET['limit'] ?? 0)));
-  }
-}
-catch (\Mailcow\Core\UserError $e) {
-  http_response_code(400);
-}
-catch (Throwable $e) {
-  error_log("RETENTION: " . $e->getMessage() . PHP_EOL);
-  http_response_code(500);
-}

data/conf/rspamd/dynmaps/settings.php

@@ -469,5 +469,35 @@ while ($row = array_shift($rows)) {
 }
 ?>
 
+<?php
+// Start internal aliases
+
+$stmt = $pdo->query("SELECT `id`, `address`, `domain` FROM `alias` WHERE `active` = '1' AND `internal` = '1'");
+$aliases = $stmt->fetchAll(PDO::FETCH_ASSOC);
+while ($alias = array_shift($aliases)) {
+  // build allowed_domains regex and add target domain and alias domains
+  $stmt = $pdo->prepare("SELECT `alias_domain` FROM `alias_domain` WHERE `active` = '1' AND `target_domain` = :target_domain");
+  $stmt->execute(array(':target_domain' => $alias['domain']));
+  $allowed_domains = $stmt->fetchAll(PDO::FETCH_ASSOC);
+  $allowed_domains = array_map(function($item) {
+    return str_replace('.', '\.', $item['alias_domain']);
+  }, $allowed_domains);
+  $allowed_domains[] = str_replace('.', '\.', $alias['domain']);
+  $allowed_domains = implode('|', $allowed_domains);
+?>
+  internal_alias_<?=$alias['id'];?> {
+    priority = 10;
+    rcpt = "<?=$alias['address'];?>";
+    from = "/^((?!.*@(<?=$allowed_domains;?>)).)*$/";
+    apply "default" {
+      MAILCOW_INTERNAL_ALIAS = 9999.0;
+    }
+    symbols [
+      "MAILCOW_INTERNAL_ALIAS"
+    ]
+  }
+<?php
+}
+?>
 
 }

data/conf/rspamd/local.d/composites.conf

@@ -1,7 +1,3 @@
-MX_IMPLICIT {
-  expression = "MX_GOOD & MX_MISSING";
-  score = -0.01;
-}
 VIRUS_FOUND {
   expression = "CLAM_VIRUS & !MAILCOW_WHITE";
   score = 2000.0;

data/conf/rspamd/local.d/groups.conf

@@ -23,24 +23,6 @@ symbols {
   }
 }
 
-group "MX" {
-  "MX_INVALID" {
-    score = 0.5;
-    description = "No connectable MX";
-    one_shot = true;
-  }
-  "MX_MISSING" {
-    score = 2.0;
-    description = "No MX record";
-    one_shot = true;
-  }
-  "MX_GOOD" {
-    score = -0.01;
-    description = "MX was ok";
-    one_shot = true;
-  }
-}
-
 group "reputation" {
   symbols = {
     "IP_REPUTATION_HAM" {

data/conf/rspamd/local.d/multimap.conf

@@ -5,12 +5,6 @@ RCPT_MAILCOW_DOMAIN {
   symbols_set = ["RCPT_MAILCOW_DOMAIN"];
 }
 
-WHITELISTED_FWD_HOST {
-  type = "ip";
-  map = "redis://WHITELISTED_FWD_HOST";
-  symbols_set = ["WHITELISTED_FWD_HOST"];
-}
-
 BULK_HEADER {
   type = "content";
   map = "${LOCAL_CONFDIR}/custom/bulk_header.map";
@@ -151,20 +145,6 @@ BAD_LANG {
   score = 5.0;
 }
 
-BAZAAR_ABUSE_CH {
-  type = "selector";
-  selector = "attachments(hex,md5)";
-  map = "https://bazaar.abuse.ch/export/txt/md5/recent/";
-  score = 10.0;
-}
-
-URLHAUS_ABUSE_CH {
-  type = "selector";
-  selector = "urls";
-  map = "https://urlhaus.abuse.ch/downloads/text_online/";
-  score = 10.0;
-}
-
 SMTP_LIMITED_ACCESS {
   type = "user";
   map = "redis://SMTP_LIMITED_ACCESS";

data/conf/rspamd/local.d/mx_check.conf

@@ -1,6 +1,5 @@
 timeout = 8.0;
 symbol_bad_mx = "MX_INVALID";
-symbol_no_mx = "MX_MISSING";
 symbol_good_mx = "MX_GOOD";
 expire = 86400;
 key_prefix = "rmx";

data/conf/rspamd/local.d/options.inc

@@ -9,3 +9,5 @@ disable_monitoring = true;
 # instead of silently accepting the message without further processing.
 soft_reject_on_timeout = true;
 local_addrs = /etc/rspamd/custom/mailcow_networks.map;
+# Must not be lower than message_size_limit in postfix main.cf
+max_message = 134217728;

data/conf/rspamd/lua/rspamd.local.lua

@@ -362,6 +362,61 @@ rspamd_config:register_symbol({
   priority = 19
 })
 
+rspamd_config:register_symbol({
+  name = 'WHITELISTED_FWD_HOST',
+  type = 'prefilter',
+  score = 0.0,
+  callback = function(task)
+    local rspamd_logger = require "rspamd_logger"
+    local redis_params = rspamd_parse_redis_server('fwd_host')
+    local ip = task:get_from_ip()
+
+    if ip == nil or not ip:is_valid() then
+      return false
+    end
+
+    local from_ip_string = tostring(ip)
+    local ip_check_table = {'WHITELISTED_FWD_HOST', from_ip_string}
+
+    local maxbits = 128
+    local minbits = 7
+    if ip:get_version() == 4 then
+        maxbits = 32
+        minbits = 8
+    end
+    for i=maxbits,minbits,-1 do
+      table.insert(ip_check_table, ip:apply_mask(i):to_string() .. "/" .. i)
+    end
+
+    local function fwd_host_cb(err, data)
+      if err or type(data) ~= 'table' then
+        rspamd_logger.infox(rspamd_config, "fwd_host query request for ip %s returned invalid or empty data (\"%s\") or error (\"%s\")", ip, data, err)
+        return
+      end
+      for _,v in pairs(data) do
+        if type(v) == 'string' then
+          task:insert_result('WHITELISTED_FWD_HOST', 1.0, from_ip_string)
+          task:set_flag('pass_all')
+          return
+        end
+      end
+    end
+
+    local redis_ret = rspamd_redis_make_request(task,
+      redis_params, -- connect params
+      'WHITELISTED_FWD_HOST', -- hash key
+      false, -- is write
+      fwd_host_cb, --callback
+      'HMGET', -- command
+      ip_check_table -- arguments
+    )
+    if not redis_ret then
+      rspamd_logger.infox(rspamd_config, "cannot check fwd_host redis map")
+    end
+  end,
+  priority = 18
+})
+
 rspamd_config:register_symbol({
   name = 'TLS_HEADER',
   type = 'postfilter',
@@ -390,9 +445,8 @@ rspamd_config:register_symbol({
     local rspamd_logger = require "rspamd_logger"
     local redis_params = rspamd_parse_redis_server('taghandler')
     local rspamd_http = require "rspamd_http"
-    local rcpts = task:get_recipients('smtp')
+    local rcpts = task:get_recipients({'smtp', 'orig'})
     local lua_util = require "lua_util"
-    local tagged_rcpt = task:get_symbol("TAGGED_RCPT")
 
     local function remove_moo_tag()
       local moo_tag_header = task:get_header('X-Moo-Tag', false)
@@ -417,8 +471,9 @@ rspamd_config:register_symbol({
 
     -- Check if recipient has a tag (contains '+')
     local tag = nil
-    if tagged_rcpt ~= nil then
-      tag = tagged_rcpt
+    local base_user, tag_part = rcpt_user:match('^([^%+]+)%+(.+)$')
+    if base_user and tag_part then
+      tag = tag_part
       rspamd_logger.infox("TAG_MOO: found tag in recipient: %s (base: %s, tag: %s)", rcpt_addr, base_user, tag)
     end
 
@@ -498,8 +553,7 @@ rspamd_config:register_symbol({
           else
             rspamd_logger.infox("TAG_MOO: user wants subject modified for tagged mail")
             local sbj = task:get_header('Subject') or ''
-            local tag_value = tag[1] and tag[1].options and tag[1].options[1] or ''
-            new_sbj = '=?UTF-8?B?' .. tostring(util.encode_base64('[' .. tag_value .. '] ' .. sbj)) .. '?='
+            new_sbj = '=?UTF-8?B?' .. tostring(util.encode_base64('[' .. tag .. '] ' .. sbj)) .. '?='
             task:set_milter_reply({
               remove_headers = {
                 ['Subject'] = 1,
@@ -907,6 +961,9 @@ rspamd_config:register_symbol({
 
             if not seen_cte and rewrite.need_rewrite_ct then
               out[#out + 1] = string.format('%s: %s', 'Content-Transfer-Encoding', new_cte)
+              task:set_milter_reply({
+                add_headers = {['Content-Transfer-Encoding'] = new_cte}
+              })
             end
 
             -- End of headers
@@ -956,134 +1013,3 @@ rspamd_config:register_symbol({
   end,
   priority = 1
 })
-
--- Archive: a copy of every accepted mail of an archived mailbox goes by SMTP to the archive, through Postfix (port 591)
-rspamd_config:register_symbol({
-  name = 'MAILCOW_ARCHIVE',
-  type = 'idempotent',
-  flags = 'empty,explicit_disable,ignore_passthrough',
-  callback = function(task)
-    local rspamd_http = require "rspamd_http"
-    local rspamd_logger = require "rspamd_logger"
-    local rspamd_text = require "rspamd_text"
-    local rspamd_util = require "rspamd_util"
-    local hash = require "rspamd_cryptobox_hash"
-    local lua_smtp = require "lua_smtp"
-    local cjson = require "cjson"
-
-    -- accepted mails only: no reject, soft reject (greylisting included), discard or quarantine
-    local action = task:get_metric_action('default')
-    local accepted = (action == 'no action' or action == 'add header' or action == 'rewrite subject' or action == 'greylist')
-    local pre_accept = false
-    if task.has_pre_result then
-      local has_pre, pre_action = task:has_pre_result()
-      pre_accept = (has_pre and pre_action == 'accept')
-    end
-    if not accepted and not pre_accept then
-      return
-    end
-
-    local rcpts = {}
-    for _, r in ipairs(task:get_recipients('smtp') or {}) do
-      if r.addr and r.addr ~= '' then
-        table.insert(rcpts, r.addr)
-      end
-    end
-    if #rcpts == 0 then
-      return
-    end
-    local from = task:get_from('smtp')
-    local from_addr = (from and from[1] and from[1].addr) or ''
-    local qid = task:get_queue_id() or ''
-    -- the BCC symbol sends copies under the same condition, they bypass rspamd
-    local bcc = (pre_accept or (action ~= 'greylist' and accepted)) and not task:has_symbol('ENCRYPTED_CHAT')
-    local redis_params = rspamd_parse_redis_server('archive')
-
-    -- not archived: counted and listed for the archive status in the UI
-    -- the Message-ID finds the mail in the archive log of the UI
-    local message_id = task:get_message_id()
-    if not message_id or message_id == 'undef' then
-      message_id = ''
-    end
-    local function dropped(reason)
-      rspamd_logger.errx(task, "archive: mail %s is not archived: %s", qid, reason)
-      rspamd_redis_make_request(task, redis_params, 'ARCHIVE_DROPPED', true, function() end, 'INCR', {'ARCHIVE_DROPPED'})
-      rspamd_redis_make_request(task, redis_params, 'ARCHIVE_DROPPED_LOG', true, function() end, 'LPUSH',
-        {'ARCHIVE_DROPPED_LOG', cjson.encode({time = os.time(), qid = qid, reason = tostring(reason), message_id = message_id})})
-      rspamd_redis_make_request(task, redis_params, 'ARCHIVE_DROPPED_LOG', true, function() end, 'LTRIM', {'ARCHIVE_DROPPED_LOG', '0', '99'})
-    end
-
-    local function archive()
-      -- the archive gets the mail as accepted, with CRLF line ends and a final CRLF (also what the hash covers)
-      local content = rspamd_text.fromstring(tostring(task:get_content()))
-      content:normalize_newlines('smtp')
-      if #content < 2 or content:at(#content) ~= 10 or content:at(#content - 1) ~= 13 then
-        content = rspamd_text.fromtable({content, '\r\n'})
-      end
-      local body = {
-        from = from_addr,
-        rcpt = rcpts,
-        user = task:get_user() or '',
-        bcc = bcc,
-        qid = qid,
-        received = rspamd_util.get_time(),
-        sha256 = hash.create_specific('sha256', content):hex()
-      }
-
-      local function sent_cb(ok, err)
-        if not ok then
-          dropped(err)
-        end
-      end
-
-      local function journal_cb(err, code, res_body)
-        if err or code ~= 200 then
-          dropped(string.format('owner lookup failed (%s, %s)', err, code))
-          return
-        end
-        local ok, res = pcall(cjson.decode, tostring(res_body))
-        if not ok or type(res) ~= 'table' then
-          dropped('invalid answer of the owner lookup')
-          return
-        end
-        if res.archive ~= true then
-          return
-        end
-        -- lua_smtp ends the data with CRLF.CRLF, so the last CRLF of the content is left to it
-        lua_smtp.sendmail({
-          task = task,
-          host = os.getenv("IPV4_NETWORK") .. '.253',
-          port = 591,
-          from = res.sender,
-          recipients = res.rcpt,
-          helo = 'archive',
-          timeout = 20,
-        }, {res.headers or '', content:span(1, #content - 2)}, sent_cb)
-      end
-
-      rspamd_http.request({
-        task = task,
-        url = 'http://nginx:8081/archive.php',
-        body = cjson.encode(body),
-        timeout = (#rcpts > 100) and 10.0 or 2.0,
-        callback = journal_cb,
-        headers = {['Content-Type'] = 'application/json'},
-      })
-    end
-
-    local function enabled_cb(err, data)
-      if not err and type(data) == 'string' and data == '1' then
-        archive()
-      end
-    end
-
-    rspamd_redis_make_request(task,
-      redis_params, -- connect params
-      'ARCHIVE_ENABLED', -- hash key
-      false, -- is write
-      enabled_cb, --callback
-      'GET', -- command
-      {'ARCHIVE_ENABLED'} -- arguments
-    )
-  end
-})

data/conf/rspamd/meta_exporter/pipe.php

@@ -109,7 +109,7 @@ foreach ($rcpts as $rcpt) {
   }
 
   // Skip if domain is excluded
-  if (in_array($parsed_rcpt['domain'], $exclude_domains)) {
+  if (is_array($exclude_domains) && in_array($parsed_rcpt['domain'], $exclude_domains)) {
     error_log(sprintf("QUARANTINE: Skipped domain %s", $parsed_rcpt['domain']) . PHP_EOL);
     continue;
   }

data/web/admin/mailbox.php

@@ -11,8 +11,6 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
 
 $js_minifier->add('/web/js/site/imapsync.js');
 $js_minifier->add('/web/js/site/groups.js');
-$js_minifier->add('/web/js/site/mail-policy.js');
-$js_minifier->add('/web/js/site/retention.js');
 $js_minifier->add('/web/js/site/mailbox.js');
 $js_minifier->add('/web/js/presets/sieveMailbox.js');
 $js_minifier->add('/web/js/site/pwgen.js');
@@ -48,9 +46,6 @@ $template_data = [
   'lang_edit' => json_encode($lang['edit']),
   'lang_datatables' => json_encode($lang['datatables']),
   'imapsync_redirect_uri' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->oauth->redirectUri(),
-  'archive_enabled' => \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled(),
-  'mail_policy_groups' => \Mailcow\MailPolicy\MailPolicyModule::fromGlobals()->policies->groupChoices($domains),
-  'retention_groups' => \Mailcow\Retention\RetentionModule::fromGlobals()->rules->groupChoices($domains),
 ];
 
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php';

data/web/admin/restore.php

@@ -1,8 +0,0 @@
-<?php
-require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/prerequisites.inc.php';
-require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/triggers.admin.inc.php';
-
-protect_route(['admin']);
-
-$role = "admin";
-require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/restore.inc.php';

data/web/admin/system.php

@@ -13,8 +13,6 @@ $js_minifier->add('/web/js/site/admin.js');
 $js_minifier->add('/web/js/presets/rspamd.js');
 $js_minifier->add('/web/js/site/pwgen.js');
 $js_minifier->add('/web/js/site/identity.js');
-$js_minifier->add('/web/js/site/archive.js');
-$js_minifier->add('/web/js/site/backup.js');
 
 // all domains
 $domains = mailbox('get', 'domains');
@@ -118,15 +116,11 @@ $template_data = [
   'ui_theme' => customize('get', 'ui_theme'),
   'password_complexity' => password_complexity('get'),
   'imapsync_settings' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->api->getSettings(),
-  'archive_settings' => \Mailcow\Archive\ArchiveModule::fromGlobals()->api->getSettings(),
   'show_rspamd_global_filters' => @$_SESSION['show_rspamd_global_filters'],
   'cors_settings' => $cors_settings,
   'is_https' => isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on',
   'iam_settings' => $iam_settings,
   'mbox_templates' => $mbox_templates,
-  'backup_timezone' => getenv('TZ') ?: 'UTC',
-  'backup_hostname' => getenv('MAILCOW_HOSTNAME') ?: '',
-  'backup_demo_mode' => $DEMO_MODE,
   'lang_admin' => json_encode($lang['admin']),
   'lang_datatables' => json_encode($lang['datatables'])
 ];

data/web/autodiscover-json.php

@@ -8,7 +8,7 @@ if(file_exists('inc/vars.local.inc.php')) {
 $autodiscover_config = array_merge($default_autodiscover_config, $autodiscover_config);
 
 header('Content-type: application/json');
-if (strtolower($_GET['Protocol']) == 'activesync' && getenv('SKIP_SOGO') != "y") {
+if (strtolower($_GET['Protocol']) == 'activesync' && $autodiscover_config['autodiscoverType'] == 'activesync' && getenv('SKIP_SOGO') != "y") {
   echo '{"Protocol":"ActiveSync","Url":"' . $autodiscover_config['activesync']['url'] . '"}';
 }
 elseif (strtolower($_GET['Protocol']) == 'autodiscoverv1') {

data/web/autodiscover.php

@@ -31,7 +31,7 @@ error_reporting(0);
 
 $data = trim(file_get_contents("php://input"));
 
-if (strpos($data, 'autodiscover/outlook/responseschema') !== false) {
+if (strpos($data, 'autodiscover/outlook/responseschema') !== false && $autodiscover_config['autodiscoverType'] == 'activesync') {
   $autodiscover_config['autodiscoverType'] = 'imap';
   if ($autodiscover_config['useEASforOutlook'] == 'yes' &&
     // Office for macOS does not support EAS

data/web/css/site/admin.css

@@ -1,133 +1,83 @@
-.pagination a {
-  text-decoration: none !important;
-}
-.panel.panel-default {
-  overflow: visible !important;
-}
-.table-responsive {
-  overflow: visible !important;
-}
-.table-responsive {
-  overflow-x: scroll !important;
-}
-body {
-  overflow-y:scroll;
-}
-/* Fix modal moving content left */
-body.modal-open {
-  overflow-y:scroll;
-  padding-right: inherit !important;
-}
-@media (min-width: 992px) {
-  .container {
-      width: 80%;
-  }
-}
-.mass-actions-admin {
-  user-select: none;
-}
-.inputMissingAttr {
-  border-color: #FF4136;
-}
-.rotate {
-  -moz-transition: all 0.3s linear;
-  -webkit-transition: all 0.3s linear;
-  transition: all 0.3s linear;
-}
-.rotate.animation {
-  -ms-transform:rotateX(180deg);
-  -moz-transform:rotateX(180deg);
-  -webkit-transform:rotateX(180deg);
-  transform:rotateX(180deg);
-}
-.anchor {
-  display: block;
-  height: 65px;
-  margin-top: -65px;
-  visibility: hidden;
-}
-.thumbnail img {
-  min-height:100px;
-  height:100px;
-}
-.nav-tabs>li>a {
-  z-index: 1;
-}
-.table-condensed .input-sm {
-  width: 100%!important;
-}
-.table-condensed > thead > tr > th, .table-condensed > tbody > tr > th, .table-condensed > tfoot > tr > th, .table-condensed > thead > tr > td, .table-condensed > tbody > tr > td, .table-condensed > tfoot > tr > td {
-  padding: 3px;
-}
-table tbody tr td input[type="checkbox"] {
-  cursor: pointer;
-}
-#quarantine_template {
-  margin:20px;
-}
-.regex-input {
-  font-family: Consolas,monaco,monospace;
-  font-size: 1rem;
-}
-.label-keys {
-  font-size:100%;
-  margin: 0px !important;
-  white-space: normal !important;
-}
-.key-action {
-  font-weight:bold;
-  color:white !important;
-}
-.dkim-label {
-  margin: 0 0 8px !important;
-}
-/* backup schedule: the weekday toggles and the remove buttons of the entries react to the mouse, in every theme */
-.backup-schedule-days .btn:hover {
-  background-image: linear-gradient(rgba(128, 128, 128, .3), rgba(128, 128, 128, .3));
-}
-.backup-schedule-remove {
-  opacity: .6;
-}
-.backup-schedule-remove:hover, .backup-schedule-remove:focus-visible {
-  opacity: 1;
-  color: var(--bs-danger) !important;
-}
-/* result and log of a backup job: the border of the theme in light and dark mode (Bootstrap's .border stays light) */
-.backup-pre {
-  border: 1px solid var(--mc-border);
-}
-/* result of a restore job: an entry to open per mailbox, numbers in columns */
-.backup-result {
-  border: 1px solid var(--mc-border);
-}
-.backup-result > details + details {
-  border-top: 1px solid var(--mc-border);
-}
-.backup-result summary {
-  list-style: none;
-  cursor: pointer;
-}
-.backup-result summary::-webkit-details-marker {
-  display: none;
-}
-.backup-result summary:hover {
-  background-image: linear-gradient(rgba(128, 128, 128, .12), rgba(128, 128, 128, .12));
-}
-.backup-result summary .bi-chevron-right {
-  transition: transform .15s;
-}
-.backup-result details[open] > summary .bi-chevron-right {
-  transform: rotate(90deg);
-}
-.backup-result-body {
-  padding-left: 2.6rem !important;
-}
-.backup-result-body .table {
-  --bs-table-bg: transparent;
-}
-.backup-result-body .num {
-  width: 8.5rem;
-  text-align: right;
-  font-variant-numeric: tabular-nums;
-  white-space: nowrap;
-}
+.pagination a {
+  text-decoration: none !important;
+}
+.panel.panel-default {
+  overflow: visible !important;
+}
+.table-responsive {
+  overflow: visible !important;
+}
+.table-responsive {
+  overflow-x: scroll !important;
+}
+body {
+  overflow-y:scroll;
+}
+/* Fix modal moving content left */
+body.modal-open {
+  overflow-y:scroll;
+  padding-right: inherit !important;
+}
+@media (min-width: 992px) {
+  .container {
+      width: 80%;
+  }
+}
+.mass-actions-admin {
+  user-select: none;
+}
+.inputMissingAttr {
+  border-color: #FF4136;
+}
+.rotate {
+  -moz-transition: all 0.3s linear;
+  -webkit-transition: all 0.3s linear;
+  transition: all 0.3s linear;
+}
+.rotate.animation {
+  -ms-transform:rotateX(180deg);
+  -moz-transform:rotateX(180deg);
+  -webkit-transform:rotateX(180deg);
+  transform:rotateX(180deg);
+}
+.anchor {
+  display: block;
+  height: 65px;
+  margin-top: -65px;
+  visibility: hidden;
+}
+.thumbnail img {
+  min-height:100px;
+  height:100px;
+}
+.nav-tabs>li>a {
+  z-index: 1;
+}
+.table-condensed .input-sm {
+  width: 100%!important;
+}
+.table-condensed > thead > tr > th, .table-condensed > tbody > tr > th, .table-condensed > tfoot > tr > th, .table-condensed > thead > tr > td, .table-condensed > tbody > tr > td, .table-condensed > tfoot > tr > td {
+  padding: 3px;
+}
+table tbody tr td input[type="checkbox"] {
+  cursor: pointer;
+}
+#quarantine_template {
+  margin:20px;
+}
+.regex-input {
+  font-family: Consolas,monaco,monospace;
+  font-size: 1rem;
+}
+.label-keys {
+  font-size:100%;
+  margin: 0px !important;
+  white-space: normal !important;
+}
+.key-action {
+  font-weight:bold;
+  color:white !important;
+}
+.dkim-label {
+  margin: 0 0 8px !important;
+}

data/web/css/themes/mailcow-darkmode.css

@@ -495,16 +495,4 @@ input::placeholder {
 }
 .btn-light, .btn-light:hover {
     background-image: none;
-}
-/* native time picker icon and popup in dark mode */
-input[type="time"] {
-    color-scheme: dark;
-}
-/* switches while off: a light knob and a visible border (Bootstrap's dark knob is for light pages, the dark mode
-   does not set data-bs-theme) */
-.form-switch .form-check-input:not(:checked):not(:focus) {
-    --bs-form-switch-bg: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='rgba%28255, 255, 255, 0.6%29'/%3e%3c/svg%3e");
-}
-.form-switch .form-check-input:not(:checked) {
-    border-color: var(--mc-btn-outline-border);
-}
+}

data/web/domainadmin/mailbox.php

@@ -11,8 +11,6 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
 
 $js_minifier->add('/web/js/site/imapsync.js');
 $js_minifier->add('/web/js/site/groups.js');
-$js_minifier->add('/web/js/site/mail-policy.js');
-$js_minifier->add('/web/js/site/retention.js');
 $js_minifier->add('/web/js/site/mailbox.js');
 $js_minifier->add('/web/js/presets/sieveMailbox.js');
 $js_minifier->add('/web/js/site/pwgen.js');
@@ -47,9 +45,6 @@ $template_data = [
   'lang_edit' => json_encode($lang['edit']),
   'lang_datatables' => json_encode($lang['datatables']),
   'imapsync_redirect_uri' => \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->oauth->redirectUri(),
-  'archive_enabled' => \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled(),
-  'mail_policy_groups' => \Mailcow\MailPolicy\MailPolicyModule::fromGlobals()->policies->groupChoices($domains),
-  'retention_groups' => \Mailcow\Retention\RetentionModule::fromGlobals()->rules->groupChoices($domains),
 ];
 
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php';

data/web/domainadmin/restore.php

@@ -1,8 +0,0 @@
-<?php
-require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/prerequisites.inc.php';
-require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/triggers.domainadmin.inc.php';
-
-protect_route(['domainadmin']);
-
-$role = "domainadmin";
-require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/restore.inc.php';

data/web/edit.php

@@ -65,8 +65,7 @@ if (isset($_SESSION['mailcow_cc_role'])) {
             'mta_sts' => $mta_sts,
             'mailboxes' => mailbox('get', 'mailboxes', $_GET["domain"]),
             'aliases' => mailbox('get', 'aliases', $_GET["domain"], 'address'),
-            'alias_domains' => mailbox('get', 'alias_domains', $_GET["domain"]),
-            'archive_enabled' => \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled()
+            'alias_domains' => mailbox('get', 'alias_domains', $_GET["domain"])
           ];
       }
     }
@@ -76,7 +75,6 @@ if (isset($_SESSION['mailcow_cc_role'])) {
         $template_data = [
           'template' => $domain_template,
           'rl' => ['frame' => $domain_template['attributes']['rl_frame']],
-          'archive_enabled' => \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled(),
         ];
         $template = 'edit/domain-templates.twig';
         $result = true;
@@ -87,7 +85,6 @@ if (isset($_SESSION['mailcow_cc_role'])) {
           $template_data = [
             'template' => $mailbox_template,
             'rl' => ['frame' => $mailbox_template['attributes']['rl_frame']],
-            'archive_enabled' => \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled(),
           ];
           $template = 'edit/mailbox-templates.twig';
           $result = true;
@@ -118,28 +115,6 @@ if (isset($_SESSION['mailcow_cc_role'])) {
           'mbox_templates' => ($_SESSION['mailcow_cc_role'] == 'admin') ? mailbox('get', 'mailbox_templates') : [],
         ];
     }
-    elseif (isset($_GET['mail-policy']) &&
-      is_numeric($_GET['mail-policy'])) {
-        $mail_policy = \Mailcow\MailPolicy\MailPolicyModule::fromGlobals();
-        $result = $mail_policy->api->getPolicy($_GET['mail-policy']);
-        $template = 'edit/mail-policy.twig';
-        // Choices: mailboxes and groups of the domain of the policy
-        $template_data = [
-          'mailboxes' => !empty($result) ? mailbox('get', 'mailboxes', $result['domain']) : [],
-          'mail_policy_groups' => !empty($result) ? $mail_policy->policies->groupChoices([$result['domain']]) : [],
-        ];
-    }
-    elseif (isset($_GET['retention-rule']) &&
-      is_numeric($_GET['retention-rule'])) {
-        $retention = \Mailcow\Retention\RetentionModule::fromGlobals();
-        $result = $retention->api->getRule($_GET['retention-rule']);
-        $template = 'edit/retention-rule.twig';
-        // Choices: mailboxes and groups of the domain of the rule
-        $template_data = [
-          'mailboxes' => !empty($result) ? mailbox('get', 'mailboxes', $result['domain']) : [],
-          'retention_groups' => !empty($result) ? $retention->rules->groupChoices([$result['domain']]) : [],
-        ];
-    }
     elseif (isset($_GET['api-key']) &&
       is_numeric($_GET['api-key'])) {
         $result = \Mailcow\ApiKey\ApiKeyModule::fromGlobals()->api->getKey($_GET['api-key']);
@@ -187,7 +162,6 @@ if (isset($_SESSION['mailcow_cc_role'])) {
           'user_acls' => acl('get', 'user', $mailbox),
           'mailbox_details' => $result,
           'iam_settings' => $iam_settings,
-          'archive_enabled' => \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled(),
         ];
       }
     }

data/web/inc/footer.inc.php

@@ -74,9 +74,6 @@ $globalVariables = [
   'lang_danger' => json_encode($lang['danger']),
   'lang_syncjobs' => json_encode($lang['syncjobs']),
   'lang_groups' => json_encode($lang['groups']),
-  'lang_mail_policy' => json_encode($lang['mail_policy'] ?? []),
-  'lang_retention' => json_encode($lang['retention'] ?? []),
-  'lang_backup' => json_encode($lang['backup'] ?? []),
   'docker_timeout' => $DOCKER_TIMEOUT,
   'session_lifetime' => (int)$SESSION_LIFETIME,
   'csrf_token' => $_SESSION['CSRF']['TOKEN'],

data/web/inc/functions.auth.inc.php

@@ -601,6 +601,13 @@ function keycloak_mbox_login_rest($user, $pass, $extra = null){
 
   // validate mailcow_password
   $mailcow_password = $user_res['attributes']['mailcow_password'][0];
+  // Keycloak returns binary LDAP attributes like userPassword Base64 encoded
+  if (substr($mailcow_password, 0, 1) != '{') {
+    $decoded_password = base64_decode($mailcow_password, true);
+    if ($decoded_password !== false && substr($decoded_password, 0, 1) == '{') {
+      $mailcow_password = $decoded_password;
+    }
+  }
   if (!verify_hash($mailcow_password, $pass)) {
     return false;
   }

data/web/inc/functions.fwdhost.inc.php

@@ -35,6 +35,13 @@ function fwdhost($_action, $_data = null) {
         );
         return false;
       }
+      $hosts = array_unique(array_map(function ($host) {
+        $net = explode('/', $host, 2);
+        if (filter_var($net[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) {
+          $net[0] = inet_ntop(inet_pton($net[0]));
+        }
+        return implode('/', $net);
+      }, $hosts));
       foreach ($hosts as $host) {
         try {
           $redis->hSet('WHITELISTED_FWD_HOST', $host, $source);

data/web/inc/functions.inc.php

@@ -539,7 +539,6 @@ function sys_mail($_data) {
       $mail->Body = $mass_text;
     }
     $mail->XMailer = 'MooMassMail';
-    $archive = \Mailcow\Archive\ArchiveModule::fromGlobals();
     foreach ($rcpts as $rcpt) {
       $mail->AddAddress($rcpt);
       if (!$mail->send()) {
@@ -549,10 +548,6 @@ function sys_mail($_data) {
           'msg' => 'Mailer error (RCPT "' . htmlspecialchars($rcpt) . '"): ' . str_replace('https://github.com/PHPMailer/PHPMailer/wiki/Troubleshooting', '', $mail->ErrorInfo)
         );
       }
-      else {
-        // Delivered by LMTP without rspamd, so the archive gets it here
-        $archive->journal->submitDelivered($mail->getSentMIMEMessage(), $mass_from, array($rcpt), 'massmail');
-      }
       $mail->ClearAllRecipients();
     }
   }
@@ -1421,11 +1416,11 @@ function set_tfa($_data) {
       // Block TFA removal if force_tfa policy is active
       $is_forced_tfa = false;
       if ($_SESSION['mailcow_cc_role'] === 'user') {
-        $stmt_check = $pdo->prepare("SELECT JSON_EXTRACT(`attributes`, '$.force_tfa') FROM `mailbox` WHERE `username` = ?");
+        $stmt_check = $pdo->prepare("SELECT JSON_VALUE(`attributes`, '$.force_tfa') FROM `mailbox` WHERE `username` = ?");
         $stmt_check->execute(array($username));
         $is_forced_tfa = ($stmt_check->fetchColumn() == '1');
       } else {
-        $stmt_check = $pdo->prepare("SELECT JSON_EXTRACT(`attributes`, '$.force_tfa') FROM `admin` WHERE `username` = ?");
+        $stmt_check = $pdo->prepare("SELECT JSON_VALUE(`attributes`, '$.force_tfa') FROM `admin` WHERE `username` = ?");
         $stmt_check->execute(array($username));
         $is_forced_tfa = ($stmt_check->fetchColumn() == '1');
       }
@@ -1692,11 +1687,11 @@ function unset_tfa_key($_data) {
     // Block key removal if force_tfa policy is active
     $is_forced_tfa = false;
     if ($_SESSION['mailcow_cc_role'] === 'user') {
-      $stmt_check = $pdo->prepare("SELECT JSON_EXTRACT(`attributes`, '$.force_tfa') FROM `mailbox` WHERE `username` = ?");
+      $stmt_check = $pdo->prepare("SELECT JSON_VALUE(`attributes`, '$.force_tfa') FROM `mailbox` WHERE `username` = ?");
       $stmt_check->execute(array($username));
       $is_forced_tfa = ($stmt_check->fetchColumn() == '1');
     } else {
-      $stmt_check = $pdo->prepare("SELECT JSON_EXTRACT(`attributes`, '$.force_tfa') FROM `admin` WHERE `username` = ?");
+      $stmt_check = $pdo->prepare("SELECT JSON_VALUE(`attributes`, '$.force_tfa') FROM `admin` WHERE `username` = ?");
       $stmt_check->execute(array($username));
       $is_forced_tfa = ($stmt_check->fetchColumn() == '1');
     }
@@ -3111,7 +3106,17 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
     case "get-keycloak-admin-token":
       // get access_token for service account of mailcow client
       if ($iam_settings['authsource'] !== 'keycloak') return false;
-      if (isset($iam_settings['access_token'])) {
+      // only introspect a cached access_token that is not (about to be) expired,
+      // introspecting an expired token makes Keycloak log INTROSPECT_TOKEN_ERROR
+      $token_not_expired = false;
+      if (!empty($iam_settings['access_token'])) {
+        $token_parts = explode('.', $iam_settings['access_token']);
+        if (count($token_parts) === 3) {
+          $token_payload = json_decode(base64_decode(strtr($token_parts[1], '-_', '+/')), true);
+          $token_not_expired = is_array($token_payload) && isset($token_payload['exp']) && intval($token_payload['exp']) > time() + 30;
+        }
+      }
+      if ($token_not_expired) {
         // check if access_token is valid
         $url = "{$iam_settings['server_url']}/realms/{$iam_settings['realm']}/protocol/openid-connect/token/introspect";
         $req = http_build_query(array(
@@ -3162,6 +3167,8 @@ function identity_provider($_action = null, $_data = null, $_extra = null) {
         ':key' => 'access_token',
         ':value' => $res['access_token']
       ));
+      // keep the cached token in sync, so subsequent calls in the same run reuse it
+      $iam_settings['access_token'] = $res['access_token'];
       return $res['access_token'];
     break;
   }

data/web/inc/functions.mailbox.inc.php

@@ -366,7 +366,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $relay_unknown_only = (isset($_data['relay_unknown_only'])) ? intval($_data['relay_unknown_only']) : $DOMAIN_DEFAULT_ATTRIBUTES['relay_unknown_only'];
           $backupmx = (isset($_data['backupmx'])) ? intval($_data['backupmx']) : $DOMAIN_DEFAULT_ATTRIBUTES['backupmx'];
           $gal = (isset($_data['gal'])) ? intval($_data['gal']) : $DOMAIN_DEFAULT_ATTRIBUTES['gal'];
-          $archive = (intval((isset($_data['archive'])) ? $_data['archive'] : ($DOMAIN_DEFAULT_ATTRIBUTES['archive'] ?? 0)) == 1) ? 1 : 0;
           if ($relay_all_recipients == 1) {
             $backupmx = '1';
           }
@@ -422,8 +421,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             ':domain' => '%@' . $domain
           ));
           // save domain
-          $stmt = $pdo->prepare("INSERT INTO `domain` (`domain`, `description`, `aliases`, `mailboxes`, `defquota`, `maxquota`, `quota`, `backupmx`, `gal`, `active`, `relay_unknown_only`, `relay_all_recipients`, `archive`)
-            VALUES (:domain, :description, :aliases, :mailboxes, :defquota, :maxquota, :quota, :backupmx, :gal, :active, :relay_unknown_only, :relay_all_recipients, :archive)");
+          $stmt = $pdo->prepare("INSERT INTO `domain` (`domain`, `description`, `aliases`, `mailboxes`, `defquota`, `maxquota`, `quota`, `backupmx`, `gal`, `active`, `relay_unknown_only`, `relay_all_recipients`)
+            VALUES (:domain, :description, :aliases, :mailboxes, :defquota, :maxquota, :quota, :backupmx, :gal, :active, :relay_unknown_only, :relay_all_recipients)");
           $stmt->execute(array(
             ':domain' => $domain,
             ':description' => $description,
@@ -436,8 +435,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             ':gal' => $gal,
             ':active' => $active,
             ':relay_unknown_only' => $relay_unknown_only,
-            ':relay_all_recipients' => $relay_all_recipients,
-            ':archive' => $archive
+            ':relay_all_recipients' => $relay_all_recipients
           ));
           // save tags
           foreach($tags as $index => $tag){
@@ -942,8 +940,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $relayhost = (isset($_data['relayhost'])) ? intval($_data['relayhost']) : 0;
           // Shared mailbox: no login of its own, members open it through group shares
           $shared = (isset($_data['shared'])) ? intval($_data['shared']) : 0;
-          // Archiving comes from the template unless the caller may set it
-          $archive = \Mailcow\Archive\Model\Mailbox::setting((isset($_data['archive']) && hasACLAccess("archive")) ? $_data['archive'] : ($MAILBOX_DEFAULT_ATTRIBUTES['archive'] ?? 'inherit'));
           $quarantine_notification = (isset($_data['quarantine_notification'])) ? strval($_data['quarantine_notification']) : strval($MAILBOX_DEFAULT_ATTRIBUTES['quarantine_notification']);
           $quarantine_category = (isset($_data['quarantine_category'])) ? strval($_data['quarantine_category']) : strval($MAILBOX_DEFAULT_ATTRIBUTES['quarantine_category']);
           // Validate quarantine_category
@@ -978,7 +974,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               'dav_access' => strval($dav_access),
               'relayhost' => strval($relayhost),
               'shared' => strval($shared),
-              'archive' => $archive,
               'passwd_update' => time(),
               'mailbox_format' => strval($MAILBOX_DEFAULT_ATTRIBUTES['mailbox_format']),
               'quarantine_notification' => strval($quarantine_notification),
@@ -1530,7 +1525,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $attr['backupmx']                   = (isset($_data['backupmx'])) ? intval($_data['backupmx']) : 0;
           $attr['relay_all_recipients']       = (isset($_data['relay_all_recipients'])) ? intval($_data['relay_all_recipients']) : 0;
           $attr['relay_unknown_only']          = (isset($_data['relay_unknown_only'])) ? intval($_data['relay_unknown_only']) : 0;
-          $attr['archive']                    = (isset($_data['archive']) && intval($_data['archive']) == 1) ? 1 : 0;
           $attr['dkim_selector']              = (isset($_data['dkim_selector'])) ? $_data['dkim_selector'] : "dkim";
           $attr['key_size']                   = isset($_data['key_size']) ? intval($_data['key_size']) : 2048;
 
@@ -1606,7 +1600,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $attr["rl_value"]                    = (!empty($_data['rl_value'])) ? $_data['rl_value'] : "";
           $attr["force_pw_update"]             = isset($_data['force_pw_update']) ? intval($_data['force_pw_update']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['force_pw_update']);
           $attr["force_tfa"]                   = isset($_data['force_tfa']) ? intval($_data['force_tfa']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['force_tfa']);
-          $attr["archive"]                     = \Mailcow\Archive\Model\Mailbox::setting($_data['archive'] ?? $MAILBOX_DEFAULT_ATTRIBUTES['archive']);
           $attr["sogo_access"]                 = isset($_data['sogo_access']) ? intval($_data['sogo_access']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['sogo_access']);
           $attr["active"]                      = isset($_data['active']) ? intval($_data['active']) : 1;
           $attr["tls_enforce_in"]              = isset($_data['tls_enforce_in']) ? intval($_data['tls_enforce_in']) : intval($MAILBOX_DEFAULT_ATTRIBUTES['tls_enforce_in']);
@@ -2439,7 +2432,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               if ($address != $is_now['address']) {
                 // Send-as grants of groups follow the address
                 \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->addressRenamed($is_now['address'], $address);
-                \Mailcow\MailPolicy\MailPolicyModule::fromGlobals()->api->addressRenamed($is_now['address'], $address);
               }
             }
             $_SESSION['return'][] = array(
@@ -2496,7 +2488,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 $gal                  = (isset($_data['gal'])) ? intval($_data['gal']) : $is_now['gal'];
                 $description          = (!empty($_data['description']) && isset($_SESSION['acl']['domain_desc']) && $_SESSION['acl']['domain_desc'] == "1") ? $_data['description'] : $is_now['description'];
                 (int)$relayhost       = (isset($_data['relayhost']) && isset($_SESSION['acl']['domain_relayhost']) && $_SESSION['acl']['domain_relayhost'] == "1") ? intval($_data['relayhost']) : intval($is_now['relayhost']);
-                $archive              = (isset($_data['archive']) && hasACLAccess("archive")) ? ((intval($_data['archive']) == 1) ? 1 : 0) : intval($is_now['archive']);
                 $tags                 = (is_array($_data['tags']) ? $_data['tags'] : array());
               }
               else {
@@ -2510,13 +2501,11 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
 
               $stmt = $pdo->prepare("UPDATE `domain` SET
               `description` = :description,
-              `gal` = :gal,
-              `archive` = :archive
+              `gal` = :gal
                 WHERE `domain` = :domain");
               $stmt->execute(array(
                 ':description' => $description,
                 ':gal' => $gal,
-                ':archive' => $archive,
                 ':domain' => $domain
               ));
               // save tags
@@ -2535,6 +2524,10 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                   ':domain' => $domain,
                   ':tag_name' => $tag,
                 ));
+                $stmt = $pdo->prepare("UPDATE `domain` SET `modified` = NOW() WHERE `domain` = :domain");
+                $stmt->execute(array(
+                  ':domain' => $domain,
+                ));
               }
 
               $_SESSION['return'][] = array(
@@ -2549,7 +2542,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 $active               = (isset($_data['active'])) ? intval($_data['active']) : $is_now['active'];
                 $backupmx             = (isset($_data['backupmx'])) ? intval($_data['backupmx']) : $is_now['backupmx'];
                 $gal                  = (isset($_data['gal'])) ? intval($_data['gal']) : $is_now['gal'];
-                $archive              = (isset($_data['archive'])) ? ((intval($_data['archive']) == 1) ? 1 : 0) : intval($is_now['archive']);
                 $relay_all_recipients = (isset($_data['relay_all_recipients'])) ? intval($_data['relay_all_recipients']) : $is_now['relay_all_recipients'];
                 $relay_unknown_only   = (isset($_data['relay_unknown_only'])) ? intval($_data['relay_unknown_only']) : $is_now['relay_unknown_only'];
                 $relayhost            = (isset($_data['relayhost'])) ? intval($_data['relayhost']) : $is_now['relayhost'];
@@ -2664,7 +2656,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               `relay_unknown_only` = :relay_unknown_only,
               `backupmx` = :backupmx,
               `gal` = :gal,
-              `archive` = :archive,
               `active` = :active,
               `quota` = :quota,
               `defquota` = :defquota,
@@ -2679,7 +2670,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 ':relay_unknown_only' => $relay_unknown_only,
                 ':backupmx' => $backupmx,
                 ':gal' => $gal,
-                ':archive' => $archive,
                 ':active' => $active,
                 ':quota' => $quota,
                 ':defquota' => $defquota,
@@ -2706,6 +2696,10 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                   ':domain' => $domain,
                   ':tag_name' => $tag,
                 ));
+                $stmt = $pdo->prepare("UPDATE `domain` SET `modified` = NOW() WHERE `domain` = :domain");
+                $stmt->execute(array(
+                  ':domain' => $domain,
+                ));
               }
 
               $_SESSION['return'][] = array(
@@ -2767,7 +2761,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             $attr['backupmx']                   = (isset($_data['backupmx'])) ? intval($_data['backupmx']) : 0;
             $attr['relay_all_recipients']       = (isset($_data['relay_all_recipients'])) ? intval($_data['relay_all_recipients']) : 0;
             $attr['relay_unknown_only']          = (isset($_data['relay_unknown_only'])) ? intval($_data['relay_unknown_only']) : 0;
-            $attr['archive']                    = (isset($_data['archive'])) ? ((intval($_data['archive']) == 1) ? 1 : 0) : intval($is_now['attributes']['archive'] ?? 0);
             $attr['dkim_selector']              = (isset($_data['dkim_selector'])) ? $_data['dkim_selector'] : "dkim";
             $attr['key_size']                   = isset($_data['key_size']) ? intval($_data['key_size']) : 2048;
 
@@ -2829,7 +2822,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               (int)$eas_access     = (isset($_data['eas_access']) && hasACLAccess("protocol_access")) ? intval($_data['eas_access']) : intval($is_now['attributes']['eas_access']);
               (int)$dav_access    = (isset($_data['dav_access']) && hasACLAccess("protocol_access")) ? intval($_data['dav_access']) : intval($is_now['attributes']['dav_access']);
               (int)$relayhost       = (isset($_data['relayhost']) && hasACLAccess("mailbox_relayhost")) ? intval($_data['relayhost']) : intval($is_now['attributes']['relayhost']);
-              $archive              = \Mailcow\Archive\Model\Mailbox::setting((isset($_data['archive']) && hasACLAccess("archive")) ? $_data['archive'] : ($is_now['attributes']['archive'] ?? null));
               (int)$shared          = (isset($_data['shared'])) ? intval($_data['shared']) : intval($is_now['attributes']['shared'] ?? 0);
               (int)$quota_m         = (isset_has_content($_data['quota'])) ? intval($_data['quota']) : ($is_now['quota'] / 1048576);
               $name                 = (!empty($_data['name'])) ? ltrim(rtrim($_data['name'], '>'), '<') : $is_now['name'];
@@ -3130,7 +3122,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                   `attributes` = JSON_SET(`attributes`, '$.smtp_access', :smtp_access),
                   `attributes` = JSON_SET(`attributes`, '$.eas_access', :eas_access),
                   `attributes` = JSON_SET(`attributes`, '$.dav_access', :dav_access),
-                  `attributes` = JSON_SET(`attributes`, '$.archive', :archive),
                   `attributes` = JSON_SET(`attributes`, '$.recovery_email', :recovery_email),
                   `attributes` = JSON_SET(`attributes`, '$.attribute_hash', :attribute_hash)
                     WHERE `username` = :username");
@@ -3148,7 +3139,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 ':smtp_access' => $smtp_access,
                 ':eas_access' => $eas_access,
                 ':dav_access' => $dav_access,
-                ':archive' => $archive,
                 ':recovery_email' => $pw_recovery_email,
                 ':relayhost' => $relayhost,
                 ':shared' => strval($shared),
@@ -3188,6 +3178,10 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                   ':username' => $username,
                   ':tag_name' => $tag,
                 ));
+                $stmt = $pdo->prepare("UPDATE `mailbox` SET `modified` = NOW() WHERE `username` = :username");
+                $stmt->execute(array(
+                  ':username' => $username,
+                ));
               } catch (Exception $e) {
               }
             }
@@ -3313,9 +3307,7 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               'mailbox_group_imap_grant' => ['owner'],
               'mailbox_group_sogo_grant' => ['owner'],
               'mailbox_group_sogo_acl' => ['c_uid'],
-              'mailbox_group_subscription' => ['username', 'owner'],
-              'mail_policy_member' => ['username'],
-              'retention_rule_member' => ['username']
+              'mailbox_group_subscription' => ['username', 'owner']
             ];
             foreach ($tables as $table => $columns) {
               foreach ($columns as $column) {
@@ -3331,8 +3323,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             \Mailcow\Imapsync\ImapsyncModule::fromGlobals()->sources->onAccountRenamed($old_username, $new_username);
             // Group memberships and send-as grants follow, group aliases are rebuilt
             \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->hooks->onMailboxRenamed($old_username, $new_username);
-            // Address entries of mail policies follow
-            \Mailcow\MailPolicy\MailPolicyModule::fromGlobals()->api->addressRenamed($old_username, $new_username);
 
             // Update c_uid, c_name and mail in _sogo_static_view table
             $pdo->prepare("UPDATE _sogo_static_view SET c_uid = :new_username, c_name = :new_username2, mail = :new_username3 WHERE c_uid = :old_username")
@@ -3556,7 +3546,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             $attr["rl_value"]                    = (!empty($_data['rl_value'])) ? $_data['rl_value'] : $is_now['rl_value'];
             $attr["force_pw_update"]             = isset($_data['force_pw_update']) ? intval($_data['force_pw_update']) : $is_now['force_pw_update'];
             $attr["force_tfa"]                   = isset($_data['force_tfa']) ? intval($_data['force_tfa']) : $is_now['force_tfa'];
-            $attr["archive"]                     = \Mailcow\Archive\Model\Mailbox::setting($_data['archive'] ?? ($is_now['attributes']['archive'] ?? null));
             $attr["sogo_access"]                 = isset($_data['sogo_access']) ? intval($_data['sogo_access']) : $is_now['sogo_access'];
             $attr["active"]                      = isset($_data['active']) ? intval($_data['active']) : $is_now['active'];
             $attr["tls_enforce_in"]              = isset($_data['tls_enforce_in']) ? intval($_data['tls_enforce_in']) : $is_now['tls_enforce_in'];
@@ -4011,6 +4000,9 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $stmt->execute(array(':goto' => '(^|,)'.preg_quote($_data, '/').'($|,)'));
           $rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
           while ($row = array_shift($rows)) {
+            if (!hasAliasObjectAccess($_SESSION['mailcow_cc_username'], $_SESSION['mailcow_cc_role'], $row['address'])) {
+              continue;
+            }
             // Keep old array for backward compatibility
             $data['fixed_sender_aliases'][] = $row['address'];
             // Split into allowed/blocked for proper display
@@ -4705,7 +4697,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               `relay_unknown_only`,
               `backupmx`,
               `gal`,
-              `archive`,
               `active`
                 FROM `domain` WHERE `domain`= :domain");
           $stmt->execute(array(
@@ -4766,7 +4757,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $domaindata['backupmx_int'] = $row['backupmx'];
           $domaindata['gal'] = $row['gal'];
           $domaindata['gal_int'] = $row['gal'];
-          $domaindata['archive'] = $row['archive'];
           $domaindata['rl'] = $rl;
           $domaindata['active'] = $row['active'];
           $domaindata['active_int'] = $row['active'];
@@ -4854,8 +4844,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           if (preg_match('/y|yes/i', getenv('MASTER'))) {
             $stmt = $pdo->prepare("SELECT
               `domain`.`backupmx`,
-              `domain`.`active` AS `domain_active`,
-              `domain`.`archive` AS `domain_archive`,
               `mailbox`.`username`,
               `mailbox`.`name`,
               `mailbox`.`active`,
@@ -4878,8 +4866,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           else {
             $stmt = $pdo->prepare("SELECT
               `domain`.`backupmx`,
-              `domain`.`active` AS `domain_active`,
-              `domain`.`archive` AS `domain_archive`,
               `mailbox`.`username`,
               `mailbox`.`name`,
               `mailbox`.`active`,
@@ -4919,18 +4905,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           $mailboxdata['created'] = $row['created'];
           $mailboxdata['modified'] = $row['modified'];
           $mailboxdata['authsource'] = ($row['authsource']) ? $row['authsource'] : 'mailcow';
-          // Mails of the mailbox go to the archive: global switch, mailbox setting ("inherit" follows the domain), receives mail
-          static $archive_enabled = null;
-          if ($archive_enabled === null) {
-            try {
-              $archive_enabled = \Mailcow\Archive\ArchiveModule::fromGlobals()->settings->enabled();
-            }
-            catch (PDOException $e) {
-              $archive_enabled = false;
-            }
-          }
-          $mailboxdata['archive'] = ($archive_enabled && $row['active'] != 0 && $row['domain_active'] == 1 && $row['backupmx'] == 0
-            && \Mailcow\Archive\Model\Mailbox::effective($mailboxdata['attributes']['archive'] ?? null, $row['domain_archive'])) ? 1 : 0;
 
           if ($mailboxdata['percent_in_use'] === '- ') {
             $mailboxdata['percent_class'] = "info";
@@ -4964,6 +4938,9 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
 			      else if ($SaslLogs['service'] == 'SSO') {
               $last_sso_login = strtotime($SaslLogs['datetime']);
             }
+            else if ($SaslLogs['service'] == 'SOGO') {
+              $last_sogo_login = strtotime($SaslLogs['datetime']);
+            }
           }
           if (!isset($last_imap_login) || $GLOBALS['SHOW_LAST_LOGIN'] === false) {
             $last_imap_login = 0;
@@ -4977,10 +4954,14 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
 		      if (!isset($last_sso_login) || $GLOBALS['SHOW_LAST_LOGIN'] === false) {
             $last_sso_login = 0;
           }
+          if (!isset($last_sogo_login) || $GLOBALS['SHOW_LAST_LOGIN'] === false) {
+            $last_sogo_login = 0;
+          }
           $mailboxdata['last_imap_login'] = $last_imap_login;
           $mailboxdata['last_smtp_login'] = $last_smtp_login;
           $mailboxdata['last_pop3_login'] = $last_pop3_login;
           $mailboxdata['last_sso_login'] = $last_sso_login;
+          $mailboxdata['last_sogo_login'] = $last_sogo_login;
 
           if (!isset($_extra) || $_extra != 'reduced') {
             $rl = ratelimit('get', 'mailbox', $_data);
@@ -5018,7 +4999,16 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             $stmt->execute(array(':username' => $_data));
             $sender_acl_rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
             foreach ($sender_acl_rows as $sender_acl_row) {
-              $mailboxdata['sender_acl'][] = $sender_acl_row['send_as'];
+              $send_as = $sender_acl_row['send_as'];
+              if ($_SESSION['mailcow_cc_role'] == 'domainadmin' && $send_as != '*') {
+                $in_scope = (substr($send_as, 0, 1) == '@')
+                  ? hasDomainAccess($_SESSION['mailcow_cc_username'], $_SESSION['mailcow_cc_role'], substr($send_as, 1))
+                  : hasAliasObjectAccess($_SESSION['mailcow_cc_username'], $_SESSION['mailcow_cc_role'], $send_as);
+                if (!$in_scope) {
+                  continue;
+                }
+              }
+              $mailboxdata['sender_acl'][] = $send_as;
             }
           }
           $stmt = $pdo->prepare("SELECT `tag_name`
@@ -5389,7 +5379,6 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               ':domain' => $domain,
             ));
             \Mailcow\MailboxGroup\MailboxGroupModule::fromGlobals()->api->domainDeleted($domain);
-            \Mailcow\Retention\RetentionModule::fromGlobals()->api->domainDeleted($domain);
             $stmt = $pdo->prepare("DELETE FROM `imapsync_source_domain` WHERE `domain` = :domain");
             $stmt->execute(array(
               ':domain' => $domain,
@@ -5958,6 +5947,12 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 ':domain' => $domain,
                 ':tag_name' => $tag,
               ));
+              if ($stmt->rowCount() > 0) {
+                $stmt = $pdo->prepare("UPDATE `domain` SET `modified` = NOW() WHERE `domain` = :domain");
+                $stmt->execute(array(
+                  ':domain' => $domain,
+                ));
+              }
             }
           }
 
@@ -6010,6 +6005,12 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 ':username' => $username,
                 ':tag_name' => $tag,
               ));
+              if ($stmt->rowCount() > 0) {
+                $stmt = $pdo->prepare("UPDATE `mailbox` SET `modified` = NOW() WHERE `username` = :username");
+                $stmt->execute(array(
+                  ':username' => $username,
+                ));
+              }
             }
           }
 

data/web/inc/functions.quarantine.inc.php

@@ -154,7 +154,6 @@ function quarantine($_action, $_data = null) {
             $mail->addAttachment($msg_tmpf, $detail_row['qid'] . '.eml');
             $mail->Body = sprintf($lang['quarantine']['release_body']);
             $mail->send();
-            $released_msg = $mail->getSentMIMEMessage();
             unlink($msg_tmpf);
           }
           catch (phpmailerException $e) {
@@ -171,7 +170,6 @@ function quarantine($_action, $_data = null) {
         }
         elseif ($release_format == 'raw') {
           $detail_row['msg'] = preg_replace('/^X-Spam-Flag: (.*)/m', 'X-Pre-Release-Spam-Flag: $1', $detail_row['msg']);
-          $released_msg = $detail_row['msg'];
           // dot-stuffing per RFC5321 4.5.2: escape leading dots
           $detail_row['msg'] = preg_replace('~^\.~m', '..', $detail_row['msg']);
           $postfix_talk = array(
@@ -215,10 +213,6 @@ function quarantine($_action, $_data = null) {
           }
           fclose($smtp_connection);
         }
-        // Port 590 bypasses rspamd, so the archive gets released mails here
-        if (isset($released_msg)) {
-          \Mailcow\Archive\ArchiveModule::fromGlobals()->journal->submitDelivered($released_msg, $sender, array($detail_row['rcpt']), 'quarantine');
-        }
         $stmt = $pdo->prepare("DELETE FROM `quarantine` WHERE id = :id");
         $stmt->execute(array(
           ':id' => $row['id']
@@ -459,7 +453,6 @@ function quarantine($_action, $_data = null) {
               $mail->addAttachment($msg_tmpf, $row['qid'] . '.eml');
               $mail->Body = sprintf($lang['quarantine']['release_body']);
               $mail->send();
-              $released_msg = $mail->getSentMIMEMessage();
               unlink($msg_tmpf);
             }
             catch (phpmailerException $e) {
@@ -474,7 +467,6 @@ function quarantine($_action, $_data = null) {
           }
           elseif ($release_format == 'raw') {
             $row['msg'] = preg_replace('/^X-Spam-Flag: (.*)/m', 'X-Pre-Release-Spam-Flag: $1', $row['msg']);
-            $released_msg = $row['msg'];
             // dot-stuffing per RFC5321 4.5.2: escape leading dots
             $row['msg'] = preg_replace('~^\.~m', '..', $row['msg']);
             $postfix_talk = array(
@@ -514,11 +506,6 @@ function quarantine($_action, $_data = null) {
             }
             fclose($smtp_connection);
           }
-          // Port 590 bypasses rspamd, so the archive gets released mails here
-          if (isset($released_msg)) {
-            \Mailcow\Archive\ArchiveModule::fromGlobals()->journal->submitDelivered($released_msg, $sender, array($row['rcpt']), 'quarantine');
-            unset($released_msg);
-          }
           $stmt = $pdo->prepare("DELETE FROM `quarantine` WHERE `id` = :id");
           $stmt->execute(array(
             ':id' => $id

data/web/inc/init_db.inc.php

@@ -242,7 +242,6 @@ function init_db_schema()
           "gal" => "TINYINT(1) NOT NULL DEFAULT '1'",
           "relay_all_recipients" => "TINYINT(1) NOT NULL DEFAULT '0'",
           "relay_unknown_only" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          "archive" => "TINYINT(1) NOT NULL DEFAULT '0'",
           "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
           "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP",
           "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
@@ -492,257 +491,6 @@ function init_db_schema()
         ),
         "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
       ),
-      // Mail policies: whom the assigned mailboxes and groups may send to and receive from (Mailcow\MailPolicy)
-      "mail_policy" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "domain" => "VARCHAR(255) NOT NULL",
-          "name" => "VARCHAR(100) NOT NULL",
-          "description" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          // all, internal or list; the entries are JSON lists of addresses, domains and group:<id>
-          "send_mode" => "VARCHAR(16) NOT NULL DEFAULT 'all'",
-          "send_allow" => "MEDIUMTEXT NOT NULL",
-          "send_block" => "MEDIUMTEXT NOT NULL",
-          "receive_mode" => "VARCHAR(16) NOT NULL DEFAULT 'all'",
-          "receive_allow" => "MEDIUMTEXT NOT NULL",
-          "receive_block" => "MEDIUMTEXT NOT NULL",
-          "reject_text" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
-          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP",
-          "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "unique" => array(
-            "domain_name" => array("domain", "name")
-          ),
-          "fkey" => array(
-            "fk_mail_policy_domain" => array(
-              "col" => "domain",
-              "ref" => "domain.domain",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      // a mailbox or a group (its members and its address) a policy is assigned to
-      "mail_policy_member" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "policy_id" => "INT NOT NULL",
-          "username" => "VARCHAR(255) NULL DEFAULT NULL",
-          "group_id" => "INT NULL DEFAULT NULL"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "unique" => array(
-            "policy_mailbox" => array("policy_id", "username"),
-            "policy_group" => array("policy_id", "group_id")
-          ),
-          "key" => array(
-            "username" => array("username"),
-            "group_id" => array("group_id")
-          ),
-          "fkey" => array(
-            "fk_mail_policy_member_policy" => array(
-              "col" => "policy_id",
-              "ref" => "mail_policy.id",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            ),
-            "fk_mail_policy_member_mailbox" => array(
-              "col" => "username",
-              "ref" => "mailbox.username",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            ),
-            "fk_mail_policy_member_group" => array(
-              "col" => "group_id",
-              "ref" => "mailbox_group.id",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      "retention_rule" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "domain" => "VARCHAR(255) NOT NULL",
-          "name" => "VARCHAR(100) NOT NULL",
-          "description" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          // JSON list: trash, junk, sent, drafts, archive, inbox, all, folder:<name>
-          "folders" => "TEXT NOT NULL",
-          "days" => "INT NOT NULL DEFAULT '30'",
-          "keep_flagged" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          // simulate (count only) or delete
-          "mode" => "VARCHAR(16) NOT NULL DEFAULT 'simulate'",
-          "all_mailboxes" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
-          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP",
-          "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "unique" => array(
-            "domain_name" => array("domain", "name")
-          ),
-          "fkey" => array(
-            "fk_retention_rule_domain" => array(
-              "col" => "domain",
-              "ref" => "domain.domain",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      // a mailbox or a group (its members of the rule's domain) a retention rule is assigned to
-      "retention_rule_member" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "rule_id" => "INT NOT NULL",
-          "username" => "VARCHAR(255) NULL DEFAULT NULL",
-          "group_id" => "INT NULL DEFAULT NULL"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "unique" => array(
-            "rule_mailbox" => array("rule_id", "username"),
-            "rule_group" => array("rule_id", "group_id")
-          ),
-          "key" => array(
-            "username" => array("username"),
-            "group_id" => array("group_id")
-          ),
-          "fkey" => array(
-            "fk_retention_rule_member_rule" => array(
-              "col" => "rule_id",
-              "ref" => "retention_rule.id",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            ),
-            "fk_retention_rule_member_mailbox" => array(
-              "col" => "username",
-              "ref" => "mailbox.username",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            ),
-            "fk_retention_rule_member_group" => array(
-              "col" => "group_id",
-              "ref" => "mailbox_group.id",
-              "delete" => "CASCADE",
-              "update" => "NO ACTION"
-            )
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      // results of the nightly run: per mailbox and rule with found mail or an error (folder: the folder names), per
-      // rule a summary (username '') and the whole run (username '', rule_id NULL); kept 90 days, of simulating
-      // rules only the last run
-      "retention_log" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "domain" => "VARCHAR(255) NOT NULL",
-          "username" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          "rule_id" => "INT NULL DEFAULT NULL",
-          "rule_name" => "VARCHAR(100) NOT NULL DEFAULT ''",
-          "folder" => "TEXT NULL DEFAULT NULL",
-          "days" => "INT NOT NULL DEFAULT '0'",
-          "mailboxes" => "INT NOT NULL DEFAULT '0'",
-          "matched" => "INT NOT NULL DEFAULT '0'",
-          "deleted" => "INT NOT NULL DEFAULT '0'",
-          "errors" => "INT NOT NULL DEFAULT '0'",
-          "simulated" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          "error" => "TEXT NULL DEFAULT NULL",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "key" => array(
-            "domain_created" => array("domain", "created"),
-            "rule_created" => array("rule_id", "username", "created"),
-            "created" => array("created")
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      "backup_setting" => array(
-        "cols" => array(
-          "name" => "VARCHAR(64) NOT NULL",
-          "value" => "TEXT NOT NULL",
-          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("name")
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      "backup_job" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "type" => "ENUM('backup', 'check', 'prune', 'init', 'restore') NOT NULL",
-          "state" => "ENUM('queued', 'running', 'done', 'failed', 'cancelled') NOT NULL DEFAULT 'queued'",
-          "step" => "VARCHAR(32) NULL DEFAULT NULL",
-          "params" => "MEDIUMTEXT NULL DEFAULT NULL",
-          "result" => "MEDIUMTEXT NULL DEFAULT NULL",
-          "log" => "MEDIUMTEXT NULL DEFAULT NULL",
-          "error" => "TEXT NULL DEFAULT NULL",
-          "domain" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          "requested_by" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
-          "started" => "DATETIME NULL DEFAULT NULL",
-          "finished" => "DATETIME NULL DEFAULT NULL"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "key" => array(
-            "state_type" => array("state", "type"),
-            "domain_created" => array("domain", "created")
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      "backup_snapshot" => array(
-        "cols" => array(
-          "id" => "VARCHAR(64) NOT NULL",
-          "short_id" => "VARCHAR(16) NOT NULL",
-          "time" => "DATETIME NOT NULL",
-          "hostname" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          "tags" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          "summary" => "TEXT NULL DEFAULT NULL",
-          "domains" => "MEDIUMTEXT NULL DEFAULT NULL",
-          "content" => "MEDIUMBLOB NULL DEFAULT NULL"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "key" => array(
-            "time" => array("time")
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
       "mailbox_group_imap_grant" => array(
         "cols" => array(
           "id" => "INT NOT NULL AUTO_INCREMENT",
@@ -955,8 +703,7 @@ function init_db_schema()
             "group_id" => array("group_id")
           ),
           "key" => array(
-            "domain" => array("domain"),
-            "internal" => array("internal", "active")
+            "domain" => array("domain")
           ),
           "fkey" => array(
             "fk_alias_mailbox_group" => array(
@@ -1306,10 +1053,7 @@ function init_db_schema()
           "domain_relayhost" => "TINYINT(1) NOT NULL DEFAULT '1'",
           "domain_desc" => "TINYINT(1) NOT NULL DEFAULT '0'",
           "alias_external_goto" => "TINYINT(1) NOT NULL DEFAULT '1'",
-          "mailbox_groups" => "TINYINT(1) NOT NULL DEFAULT '1'",
-          "archive" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          "mail_policies" => "TINYINT(1) NOT NULL DEFAULT '0'",
-          "retention" => "TINYINT(1) NOT NULL DEFAULT '0'"
+          "mailbox_groups" => "TINYINT(1) NOT NULL DEFAULT '1'"
         ),
         "keys" => array(
           "primary" => array(
@@ -1533,42 +1277,6 @@ function init_db_schema()
         ),
         "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
       ),
-      "archive_settings" => array(
-        "cols" => array(
-          "name" => "VARCHAR(64) NOT NULL",
-          "value" => "TEXT NOT NULL",
-          "created" => "DATETIME(0) NOT NULL DEFAULT NOW(0)",
-          "modified" => "DATETIME ON UPDATE CURRENT_TIMESTAMP"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("name")
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
-      // Archive target the Postfix maps read (FILTER of the copies on port 591, SASL, TLS policy), one row kept in line with archive_settings
-      "archive_transport" => array(
-        "cols" => array(
-          "id" => "INT NOT NULL AUTO_INCREMENT",
-          "address" => "VARCHAR(255) NOT NULL",
-          "nexthop" => "VARCHAR(255) NOT NULL",
-          "username" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          "password" => "VARCHAR(255) NOT NULL DEFAULT ''",
-          "tls_policy" => "VARCHAR(64) NOT NULL DEFAULT 'encrypt'",
-          "active" => "TINYINT(1) NOT NULL DEFAULT '1'"
-        ),
-        "keys" => array(
-          "primary" => array(
-            "" => array("id")
-          ),
-          "key" => array(
-            "address" => array("address"),
-            "nexthop" => array("nexthop")
-          )
-        ),
-        "attr" => "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC"
-      ),
       "bcc_maps" => array(
         "cols" => array(
           "id" => "INT NOT NULL AUTO_INCREMENT",
@@ -2301,7 +2009,6 @@ function init_db_schema()
     $pdo->query("UPDATE `mailbox` SET `attributes` =  JSON_SET(`attributes`, '$.relayhost', \"0\") WHERE JSON_VALUE(`attributes`, '$.relayhost') IS NULL;");
     $pdo->query("UPDATE `mailbox` SET `attributes` =  JSON_SET(`attributes`, '$.force_pw_update', \"0\") WHERE JSON_VALUE(`attributes`, '$.force_pw_update') IS NULL;");
     $pdo->query("UPDATE `mailbox` SET `attributes` =  JSON_SET(`attributes`, '$.force_tfa', \"0\") WHERE JSON_VALUE(`attributes`, '$.force_tfa') IS NULL;");
-    $pdo->query("UPDATE `mailbox` SET `attributes` =  JSON_SET(`attributes`, '$.archive', \"inherit\") WHERE JSON_VALUE(`attributes`, '$.archive') IS NULL;");
     // admin attributes
     $pdo->query("UPDATE `admin` SET `attributes` = '{}' WHERE `attributes` = '' OR `attributes` IS NULL;");
     $pdo->query("UPDATE `admin` SET `attributes` =  JSON_SET(`attributes`, '$.force_tfa', \"0\") WHERE JSON_VALUE(`attributes`, '$.force_tfa') IS NULL;");
@@ -2349,7 +2056,6 @@ function init_db_schema()
         "backupmx" => 0,
         "relay_all_recipients" => 0,
         "relay_unknown_only" => 0,
-        "archive" => 0,
         "dkim_selector" => "dkim",
         "key_size" => 2048,
         "max_quota_for_domain" => 10240 * 1048576,
@@ -2367,7 +2073,6 @@ function init_db_schema()
         "rl_value" => "",
         "force_pw_update" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['force_pw_update']),
         "force_tfa" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['force_tfa']),
-        "archive" => strval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['archive']),
         "sogo_access" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['sogo_access']),
         "active" => 1,
         "tls_enforce_in" => intval($GLOBALS['MAILBOX_DEFAULT_ATTRIBUTES']['tls_enforce_in']),

data/web/inc/lib/mailcow/Archive/ArchiveModule.php

@@ -1,65 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Archive;
-
-use Mailcow\Archive\Controller\ArchiveApiController;
-use Mailcow\Archive\Legacy\LegacyQueue;
-use Mailcow\Archive\Redis\ArchiveState;
-use Mailcow\Archive\Repository\LookupRepository;
-use Mailcow\Archive\Repository\SettingsRepository;
-use Mailcow\Archive\Service\HeaderBuilder;
-use Mailcow\Archive\Service\JournalService;
-use Mailcow\Archive\Service\MailboxListService;
-use Mailcow\Archive\Service\OwnerResolver;
-use Mailcow\Archive\Service\SettingsService;
-use Mailcow\Archive\Service\StatusService;
-use Mailcow\Archive\Smtp\SmtpClient;
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\Context;
-use PDO;
-
-/**
- * Composition root of the archive module (archive copies by SMTP through Postfix).
- *
- *   process_get_return(ArchiveModule::fromGlobals()->api->getMailboxes());
- *   ArchiveModule::create($pdo, Context::system(), $redis)->journal->prepare($request);  // rspamd dynmap
- *   ArchiveModule::fromGlobals()->journal->submitDelivered($message, $from, $rcpts, 'quarantine');
- */
-class ArchiveModule
-{
-    /** Request keys whose values never reach a log */
-    private const SECRET_KEYS = ['password', 'signing_secret'];
-
-    public function __construct(
-        public readonly SettingsService $settings,
-        public readonly JournalService $journal,
-        public readonly StatusService $status,
-        public readonly MailboxListService $mailboxes,
-        public readonly ArchiveApiController $api
-    ) {
-    }
-
-    /** @param ?object $redis Redis connection; without it nothing is read from or written to Redis */
-    public static function create(PDO $pdo, Context $context, ?object $redis = null): self
-    {
-        $settingsRepo = new SettingsRepository($pdo);
-        $lookup = new LookupRepository($pdo);
-        $state = new ArchiveState($redis);
-        $smtp = new SmtpClient();
-        $settings = new SettingsService($settingsRepo, $lookup, $state, $smtp, $context);
-        $journal = new JournalService($settingsRepo, new OwnerResolver($lookup), new HeaderBuilder(), $smtp, $state, $context);
-        $status = new StatusService($settingsRepo, $state, new LegacyQueue(), $context);
-        $mailboxes = new MailboxListService($lookup, $state, $context);
-        $api = new ArchiveApiController($settings, $status, $journal, $mailboxes, new ApiResponse(self::SECRET_KEYS));
-        return new self($settings, $journal, $status, $mailboxes, $api);
-    }
-
-    /** Legacy entry point for procedural callers: reads $pdo, $redis and $_SESSION; $context overrides the session caller */
-    public static function fromGlobals(?Context $context = null): self
-    {
-        $redis = $GLOBALS['redis'] ?? null;
-        return self::create($GLOBALS['pdo'], $context ?? Context::fromSession($_SESSION ?? []), is_object($redis) ? $redis : null);
-    }
-}

data/web/inc/lib/mailcow/Archive/Controller/ArchiveApiController.php

@@ -1,103 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Archive\Controller;
-
-use Mailcow\Archive\Service\JournalService;
-use Mailcow\Archive\Service\MailboxListService;
-use Mailcow\Archive\Service\SettingsService;
-use Mailcow\Archive\Service\StatusService;
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\UserError;
-use PDOException;
-
-/**
- * HTTP API adapter for json_api.php (routes `archive/*`, `archive`).
- *
- * Write methods return what json_api passes to process_*_return() and write their messages via ApiResponse.
- * Messages are logged as ['archive', action, object, data].
- */
-class ArchiveApiController
-{
-    public function __construct(private SettingsService $settings, private StatusService $status, private JournalService $journal,
-        private MailboxListService $mailboxes, private ApiResponse $response)
-    {
-    }
-
-    /** GET archive/settings */
-    public function getSettings(): array
-    {
-        return $this->read(fn(): array => $this->settings->get()->toArray());
-    }
-
-    /** GET archive/status */
-    public function getStatus(): array
-    {
-        return $this->read(fn(): array => $this->status->status());
-    }
-
-    /** GET logs/archive[/count] */
-    public function getLog(int $count = 0): array
-    {
-        return $this->read(fn(): array => $this->status->log($count));
-    }
-
-    /** GET archive/mailboxes */
-    public function getMailboxes(): array
-    {
-        return $this->read(fn(): array => $this->mailboxes->archived());
-    }
-
-    /** POST edit/archive: the settings; `action: test` tests the connection, `action: testmail` sends a test copy */
-    public function edit(array $attr): bool
-    {
-        return match ($attr['action'] ?? null) {
-            'test' => $this->handle('edit', 'test', [], fn(array $log): bool => $this->testResult($this->settings->test(), $log)),
-            'testmail' => $this->handle('edit', 'testmail', [],
-                fn(array $log): bool => $this->response->success(['archive_testmail_queued', htmlspecialchars($this->journal->testMail())], $log)),
-            default => $this->handle('edit', 'settings', $attr, function (array $log) use ($attr): bool {
-                $this->settings->save($attr);
-                return $this->response->success('archive_settings_saved', $log);
-            }),
-        };
-    }
-
-    /** A missing extension is a warning: the archive works, but may refuse some mails */
-    private function testResult(string|array $msg, array $log): bool
-    {
-        if (is_array($msg)) {
-            $this->response->warning($msg, $log);
-            return true;
-        }
-        return $this->response->success($msg, $log);
-    }
-
-    /** A GET answer, empty when the caller may not read it */
-    private function read(callable $work): array
-    {
-        try {
-            return $work();
-        } catch (UserError) {
-            return [];
-        }
-    }
-
-    /**
-     * Runs one write call: $work gets the log entry and returns the result for process_*_return().
-     * A UserError becomes a danger message, a database error stops the call with one `mysql_error` message.
-     */
-    private function handle(string $action, string $object, mixed $data, callable $work): bool
-    {
-        $log = ['archive', $action, $object, $data];
-        try {
-            $result = $work($log);
-        } catch (UserError $e) {
-            $result = $this->response->error($e, $log);
-        } catch (PDOException $e) {
-            $result = $this->response->danger(['mysql_error', $e->getMessage()], $log);
-        }
-        $this->response->flush();
-        return $result;
-    }
-}

data/web/inc/lib/mailcow/Archive/Legacy/LegacyQueue.php

@@ -1,27 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Archive\Legacy;
-
-/**
- * Bridge to the legacy mailq() (Postfix queue through dockerapi) until it is converted.
- */
-class LegacyQueue
-{
-    /**
-     * @return array[] queue entries of $sender: queue_id, queue_name, arrival_time, message_size,
-     *                 recipients ("address (delay reason)")
-     */
-    public function entries(string $sender): array
-    {
-        if (!function_exists('mailq')) {
-            return [];
-        }
-        $list = json_decode((string)mailq('get'), true);
-        if (!is_array($list)) {
-            return [];
-        }
-        return array_values(array_filter($list, fn(mixed $entry): bool => is_array($entry) && ($entry['sender'] ?? '') === $sender));
-    }
-}

data/web/inc/lib/mailcow/Archive/Model/Journal.php

@@ -1,32 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Archive\Model;
-
-/**
- * What the archive learns about one mail besides its content: owners, SMTP envelope, origin and content hash.
- */
-class Journal
-{
-    public const DIRECTIONS = ['in', 'out', 'internal'];
-    public const SOURCES = ['smtp', 'quarantine', 'massmail', 'test'];
-
-    /**
-     * @param string[] $owners mailbox usernames the mail belongs to
-     * @param string[] $rcpts SMTP envelope recipients as received (incl. BCC)
-     * @param float $received time mailcow accepted the mail (unix time with milliseconds)
-     * @param string $sha256 hash of the content as sent to the archive (CRLF line ends)
-     */
-    public function __construct(
-        public readonly array $owners,
-        public readonly array $rcpts,
-        public readonly string $from,
-        public readonly string $qid,
-        public readonly string $direction,
-        public readonly string $source,
-        public readonly float $received,
-        public readonly string $sha256
-    ) {
-    }
-}

data/web/inc/lib/mailcow/Archive/Model/Mailbox.php

@@ -1,74 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Archive\Model;
-
-/**
- * A mailbox that receives mail (regular kind, active 1 or 2, in an active domain that is no backup MX)
- * with its archive settings: the mailbox attribute `archive` ('1', '0' or 'inherit') and the domain default.
- */
-class Mailbox
-{
-    public const INHERIT = 'inherit';
-
-    public function __construct(
-        public readonly string $username,
-        public readonly string $domain,
-        public readonly string $archive,
-        public readonly bool $domainArchive,
-        public readonly string $name = '',
-        public readonly int $active = 1
-    ) {
-    }
-
-    public static function fromRow(array $row): self
-    {
-        return new self(
-            strtolower((string)$row['username']),
-            strtolower((string)$row['domain']),
-            self::setting($row['archive'] ?? null),
-            (int)($row['domain_archive'] ?? 0) === 1,
-            (string)($row['name'] ?? ''),
-            (int)($row['active'] ?? 1)
-        );
-    }
-
-    /** Mailbox setting: '1', '0' (also as number or boolean), anything else (also a missing attribute) is 'inherit' */
-    public static function setting(mixed $value): string
-    {
-        $value = is_bool($value) ? (int)$value : $value;
-        $value = is_scalar($value) ? (string)$value : '';
-        return in_array($value, ['0', '1'], true) ? $value : self::INHERIT;
-    }
-
-    /** Archiving applies by the mailbox setting, 'inherit' follows the domain (without the global switch) */
-    public static function effective(mixed $archive, mixed $domainArchive): bool
-    {
-        $archive = self::setting($archive);
-        return ($archive === self::INHERIT) ? (int)$domainArchive === 1 : $archive === '1';
-    }
-
-    public function archived(): bool
-    {
-        return self::effective($this->archive, $this->domainArchive);
-    }
-
-    /** Why the mailbox is archived: its own setting or the domain default */
-    public function source(): string
-    {
-        return ($this->archive === '1') ? 'mailbox' : 'domain';
-    }
-
-    /** Entry of GET archive/mailboxes */
-    public function toArray(): array
-    {
-        return [
-            'username' => $this->username,
-            'domain' => $this->domain,
-            'name' => $this->name,
-            'active' => $this->active,
-            'source' => $this->source(),
-        ];
-    }
-}

data/web/inc/lib/mailcow/Archive/Model/Settings.php

@@ -1,72 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Archive\Model;
-
-/**
- * The archive target (table `archive_settings`): archive copies of accepted mails go by SMTP to `address`,
- * through the Postfix queue (table `archive_transport`, service archive_smtp in master.cf).
- */
-class Settings
-{
-    public const TLS_MODES = ['encrypt', 'verify', 'may'];
-    /** Metadata headers of a copy: archivecow (X-Archive-*, signed), envelope (sender/recipients), none */
-    public const PROFILES = ['archivecow', 'envelope', 'none'];
-    /** Envelope sender of copies; Postfix drops every notice to it (archive_discard), nothing reaches a real sender */
-    public const SENDER = 'archive-bounce@localhost';
-
-    /** rspamd hands accepted mails to the archive */
-    public bool $enabled = false;
-    /** Address the archive receives copies at */
-    public string $address = '';
-    /** SMTP server of the archive; empty: MX of the address domain */
-    public string $host = '';
-    public int $port = 25;
-    public string $tls = 'encrypt';
-    /** SMTP AUTH at the archive, optional */
-    public string $username = '';
-    /** Never part of toArray() */
-    public string $password = '';
-    public string $profile = 'archivecow';
-    /** Key of X-Archive-Signature (profile archivecow), never part of toArray() */
-    public string $signingKey = '';
-    /** Header names of the envelope profile */
-    public string $headerFrom = 'X-Envelope-From';
-    public string $headerRcpt = 'X-Envelope-To';
-
-    /** Copies can be sent: an address, and a signing key for archivecow */
-    public function canDeliver(): bool
-    {
-        return $this->address !== '' && ($this->profile !== 'archivecow' || $this->signingKey !== '');
-    }
-
-    /** Next hop for Postfix: [host]:port, else the domain of the address (MX lookup), with the port when not 25 */
-    public function nexthop(): string
-    {
-        if ($this->host !== '') {
-            $host = filter_var($this->host, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) ? 'ipv6:' . $this->host : $this->host;
-            return '[' . $host . ']:' . $this->port;
-        }
-        $domain = substr($this->address, (int)strrpos($this->address, '@') + 1);
-        return ($this->port === 25) ? $domain : $domain . ':' . $this->port;
-    }
-
-    /** API representation (GET archive/settings), without secrets */
-    public function toArray(): array
-    {
-        return [
-            'enabled' => (int)$this->enabled,
-            'address' => $this->address,
-            'host' => $this->host,
-            'port' => $this->port,
-            'tls' => $this->tls,
-            'username' => $this->username,
-            'has_password' => (int)($this->password !== ''),
-            'profile' => $this->profile,
-            'has_signing_secret' => (int)($this->signingKey !== ''),
-            'header_from' => $this->headerFrom,
-            'header_rcpt' => $this->headerRcpt,
-        ];
-    }
-}

data/web/inc/lib/mailcow/Archive/Redis/ArchiveState.php

@@ -1,118 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Archive\Redis;
-
-use Mailcow\Archive\Model\Settings;
-use RuntimeException;
-
-/**
- * Redis side of the archive:
- * - ARCHIVE_ENABLED: the global switch, kept here only; rspamd reads it before it asks for owners.
- * - ARCHIVE_DROPPED / ARCHIVE_DROPPED_LOG: copies that could not be handed to Postfix (rspamd.local.lua, release
- *   hooks), counted and the last 100 listed.
- * - POSTFIX_MAILLOG: the Postfix log, read for the archive log: deliveries of the archive_smtp service and expired
- *   copies, with the Message-ID of the mail (a copy keeps the Message-ID of the original).
- *
- * The connection is a plain object, so the class loads without the redis extension; without a connection
- * nothing is read or written.
- */
-class ArchiveState
-{
-    public const ENABLED = 'ARCHIVE_ENABLED';
-    public const DROPPED = 'ARCHIVE_DROPPED';
-    public const DROPPED_LOG = 'ARCHIVE_DROPPED_LOG';
-    private const MAILLOG = 'POSTFIX_MAILLOG';
-
-    public function __construct(private ?object $redis)
-    {
-    }
-
-    public function setEnabled(bool $enabled): void
-    {
-        $this->redis?->set(self::ENABLED, $enabled ? '1' : '0');
-    }
-
-    /** The global switch: on only while ARCHIVE_ENABLED is '1', as rspamd reads it (missing or unreadable is off) */
-    public function enabled(): bool
-    {
-        try {
-            return $this->redis?->get(self::ENABLED) === '1';
-        } catch (RuntimeException) {
-            return false;
-        }
-    }
-
-    /** A mail was not archived: its copy could not be handed to Postfix */
-    public function dropped(string $qid, string $reason, string $messageId = ''): void
-    {
-        if ($this->redis === null) {
-            return;
-        }
-        $this->redis->incr(self::DROPPED);
-        $this->redis->lPush(self::DROPPED_LOG, json_encode(['time' => time(), 'qid' => $qid, 'reason' => $reason, 'message_id' => $messageId]));
-        $this->redis->lTrim(self::DROPPED_LOG, 0, 99);
-    }
-
-    /** @return array{count: int, recent: array[]} count since ever; recent: time, qid, reason, message_id; newest first */
-    public function droppedStats(int $limit = 100): array
-    {
-        if ($this->redis === null) {
-            return ['count' => 0, 'recent' => []];
-        }
-        $recent = [];
-        foreach ((array)$this->redis->lRange(self::DROPPED_LOG, 0, $limit - 1) as $line) {
-            $entry = is_string($line) ? json_decode($line, true) : null;
-            if (is_array($entry)) {
-                $recent[] = ['time' => (int)($entry['time'] ?? 0), 'qid' => (string)($entry['qid'] ?? ''),
-                    'reason' => (string)($entry['reason'] ?? ''), 'message_id' => (string)($entry['message_id'] ?? '')];
-            }
-        }
-        return ['count' => (int)$this->redis->get(self::DROPPED), 'recent' => $recent];
-    }
-
-    /**
-     * Archive log, newest first: deliveries of copies (status sent|deferred|softbounce|bounced, text = answer of the
-     * archive), expired copies (expired) and mails not archived (dropped, text = reason).
-     *
-     * @return array[] time, status, qid (of the copy, of the mail for dropped), message_id, text
-     */
-    public function events(int $limit = 1000): array
-    {
-        if ($this->redis === null) {
-            return [];
-        }
-        $events = [];
-        $messageIds = [];
-        foreach ((array)$this->redis->lRange(self::MAILLOG, 0, -1) as $line) {
-            $entry = is_string($line) ? json_decode($line, true) : null;
-            if (!is_array($entry)) {
-                continue;
-            }
-            $program = (string)($entry['program'] ?? '');
-            $message = (string)($entry['message'] ?? '');
-            $time = (int)($entry['time'] ?? 0);
-            if ($program === 'postfix/cleanup' && preg_match('/^([0-9A-Za-z]+): message-id=(\S*)/', $message, $m)) {
-                $messageIds[$m[1]] = trim($m[2], '<>');
-            }
-            // the archive_discard lines are the dropped notices, not deliveries
-            elseif ($program === 'postfix/archive/smtp' && preg_match('/^([0-9A-Za-z]+): to=<[^>]*>.*\bstatus=(\w+) \((.*)\)$/', $message, $m)) {
-                $events[] = ['time' => $time, 'status' => strtolower($m[2]), 'qid' => $m[1], 'message_id' => '', 'text' => $m[3]];
-            }
-            elseif ($program === 'postfix/qmgr' && preg_match('/^([0-9A-Za-z]+): from=<' . preg_quote(Settings::SENDER, '/') . '>, status=expired/', $message, $m)) {
-                $events[] = ['time' => $time, 'status' => 'expired', 'qid' => $m[1], 'message_id' => '', 'text' => ''];
-            }
-        }
-        foreach ($events as &$event) {
-            $event['message_id'] = $messageIds[$event['qid']] ?? '';
-        }
-        unset($event);
-        foreach ($this->droppedStats()['recent'] as $dropped) {
-            $events[] = ['time' => $dropped['time'], 'status' => 'dropped', 'qid' => $dropped['qid'],
-                'message_id' => $dropped['message_id'], 'text' => $dropped['reason']];
-        }
-        usort($events, fn(array $a, array $b): int => $b['time'] <=> $a['time']);
-        return array_slice($events, 0, $limit);
-    }
-}

data/web/inc/lib/mailcow/Archive/Repository/LookupRepository.php

@@ -1,94 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Archive\Repository;
-
-use Mailcow\Archive\Model\Mailbox;
-use Mailcow\Core\Repository;
-
-/**
- * Read-only lookups of the address tables, with the same conditions as the Postfix maps
- * (data/Dockerfiles/postfix/postfix.sh), and the mailboxes with their archive settings.
- */
-class LookupRepository extends Repository
-{
-    /** Columns of a mailbox that receives mail; the domain must be active and no backup MX */
-    private const MAILBOX = "SELECT m.`username`, m.`domain`, m.`name`, m.`active`,
-        JSON_VALUE(m.`attributes`, '$.archive') AS `archive`, d.`archive` AS `domain_archive`
-      FROM `mailbox` m
-      JOIN `domain` d ON d.`domain` = m.`domain`
-      WHERE m.`kind` = '' AND (m.`active` = '1' OR m.`active` = '2') AND d.`active` = '1' AND d.`backupmx` = '0'";
-
-    /** recipient_canonical_maps: new destination of an address or '@domain' */
-    public function recipientMap(string $key): ?string
-    {
-        return $this->value("SELECT `new_dest` FROM `recipient_maps` WHERE `old_dest` = :key AND `active` = '1' LIMIT 1", $key);
-    }
-
-    /** Targets of an alias row (aliases, catch-alls '@domain', groups, mailbox self rows) */
-    public function aliasGoto(string $key): ?string
-    {
-        return $this->value("SELECT `goto` FROM `alias` WHERE `address` = :key AND (`active` = '1' OR `active` = '2') LIMIT 1", $key);
-    }
-
-    /** Resources (location, thing, group) are discarded by Postfix */
-    public function isResource(string $username): bool
-    {
-        return $this->value("SELECT `username` FROM `mailbox` WHERE `username` = :key AND `kind` REGEXP 'location|thing|group' LIMIT 1", $username) !== null;
-    }
-
-    public function spamAliasGoto(string $key): ?string
-    {
-        return $this->value("SELECT `goto` FROM `spamalias` WHERE `address` = :key
-          AND (`validity` >= UNIX_TIMESTAMP() OR `permanent` != 0) LIMIT 1", $key);
-    }
-
-    /** Mailbox local@target behind local@domain when domain is an alias domain */
-    public function aliasDomainMailbox(string $local, string $domain): ?string
-    {
-        $username = $this->query("SELECT `mailbox`.`username` FROM `mailbox`, `alias_domain`
-          WHERE `alias_domain`.`alias_domain` = :domain AND `mailbox`.`username` = CONCAT(:local, '@', `alias_domain`.`target_domain`)
-            AND (`mailbox`.`active` = '1' OR `mailbox`.`active` = '2') AND `alias_domain`.`active` = '1' LIMIT 1",
-          ['domain' => $domain, 'local' => $local])->fetchColumn();
-        return is_string($username) ? $username : null;
-    }
-
-    /** BCC destination of an address or '@domain' as rspamd's bcc.php finds it; $type 'rcpt' or 'sender' */
-    public function bccDestination(string $key, string $type): ?string
-    {
-        $dest = $this->query("SELECT `bcc_dest` FROM `bcc_maps` WHERE `type` = :type AND `local_dest` = :key AND `active` = '1' LIMIT 1",
-          ['type' => $type, 'key' => $key])->fetchColumn();
-        return is_string($dest) ? $dest : null;
-    }
-
-    /** A domain or alias domain of this mailcow */
-    public function isMailcowDomain(string $domain): bool
-    {
-        return $this->query("SELECT `domain` FROM `domain` WHERE `domain` = :domain
-          UNION SELECT `alias_domain` FROM `alias_domain` WHERE `alias_domain` = :alias LIMIT 1",
-          ['domain' => $domain, 'alias' => $domain])->fetchColumn() !== false;
-    }
-
-    /** The mailbox when it receives mail */
-    public function mailbox(string $username): ?Mailbox
-    {
-        $row = $this->query(self::MAILBOX . " AND m.`username` = :username", ['username' => $username])->fetch();
-        return is_array($row) ? Mailbox::fromRow($row) : null;
-    }
-
-    /** @return Mailbox[] mailboxes archived by their setting or their domain (without the global switch) */
-    public function archivedMailboxes(): array
-    {
-        $rows = $this->query(self::MAILBOX . " AND (JSON_VALUE(m.`attributes`, '$.archive') = '1'
-            OR (COALESCE(JSON_VALUE(m.`attributes`, '$.archive'), '') NOT IN ('0', '1') AND d.`archive` = '1'))
-          ORDER BY m.`username`")->fetchAll();
-        return array_map(fn(array $row): Mailbox => Mailbox::fromRow($row), $rows);
-    }
-
-    private function value(string $sql, string $key): ?string
-    {
-        $value = $this->query($sql, ['key' => $key])->fetchColumn();
-        return is_string($value) ? $value : null;
-    }
-}

data/web/inc/lib/mailcow/Archive/Repository/SettingsRepository.php

@@ -1,71 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Archive\Repository;
-
-use Mailcow\Archive\Model\Settings;
-use Mailcow\Core\Repository;
-use PDO;
-
-/**
- * SQL for the archive target: the settings (table `archive_settings`, name => value) and the row the Postfix maps
- * read (table `archive_transport`: transport, SASL, TLS policy of the archive_smtp service).
- */
-class SettingsRepository extends Repository
-{
-    /** Stored settings; keys that are not stored yet keep the defaults of the model */
-    public function load(): Settings
-    {
-        $rows = $this->query("SELECT `name`, `value` FROM `archive_settings`")->fetchAll(PDO::FETCH_KEY_PAIR);
-        $settings = new Settings();
-        $settings->address = (string)($rows['address'] ?? '');
-        $settings->host = (string)($rows['host'] ?? '');
-        $settings->port = (int)($rows['port'] ?? $settings->port);
-        $settings->tls = (string)($rows['tls'] ?? $settings->tls);
-        $settings->username = (string)($rows['username'] ?? '');
-        $settings->password = (string)($rows['password'] ?? '');
-        $settings->profile = (string)($rows['profile'] ?? $settings->profile);
-        $settings->signingKey = (string)($rows['signing_secret'] ?? '');
-        $settings->headerFrom = (string)($rows['header_from'] ?? $settings->headerFrom);
-        $settings->headerRcpt = (string)($rows['header_rcpt'] ?? $settings->headerRcpt);
-        return $settings;
-    }
-
-    /**
-     * Saves the settings and the transport row. The row stays while an address is set, also with archiving off,
-     * so copies still queued in Postfix are delivered.
-     */
-    public function save(Settings $settings): void
-    {
-        $values = [
-            'address' => $settings->address,
-            'host' => $settings->host,
-            'port' => (string)$settings->port,
-            'tls' => $settings->tls,
-            'username' => $settings->username,
-            'password' => $settings->password,
-            'profile' => $settings->profile,
-            'signing_secret' => $settings->signingKey,
-            'header_from' => $settings->headerFrom,
-            'header_rcpt' => $settings->headerRcpt,
-        ];
-        $sql = "INSERT INTO `archive_settings` (`name`, `value`) VALUES (:name, :value) ON DUPLICATE KEY UPDATE `value` = VALUES(`value`)";
-        $this->transaction(function () use ($sql, $values, $settings): void {
-            foreach ($values as $name => $value) {
-                $this->query($sql, ['name' => $name, 'value' => $value]);
-            }
-            $this->query("DELETE FROM `archive_transport`");
-            if ($settings->address !== '') {
-                $this->insertRow('archive_transport', [
-                    'address' => $settings->address,
-                    'nexthop' => $settings->nexthop(),
-                    'username' => $settings->username,
-                    'password' => $settings->password,
-                    'tls_policy' => $settings->tls,
-                    'active' => 1,
-                ]);
-            }
-        });
-    }
-}

data/web/inc/lib/mailcow/Archive/Service/HeaderBuilder.php

@@ -1,125 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Archive\Service;
-
-use DateTime;
-use Mailcow\Archive\Model\Journal;
-use Mailcow\Archive\Model\Settings;
-
-/**
- * Metadata header block in front of an archive copy, by profile:
- *
- * - archivecow: X-Archive-Owner, -Rcpt, -From, -Queue-ID, -Received, -Direction, -Source, -SHA256 and as the last
- *   line X-Archive-Signature: v=1; a=hmac-sha256; s=<hex>. The signature is HMAC-SHA256 with the signing key over
- *   the lines above it, each as "<lower case name>:<value without any whitespace>", joined by "\n".
- *   The block ends with the signature line, everything after it is the mail as accepted (CRLF line ends).
- * - envelope: SMTP sender and recipients under configurable names (default X-Envelope-From / X-Envelope-To).
- * - none: no headers.
- *
- * Empty values are left out. Lists are folded after commas (CRLF + space).
- */
-class HeaderBuilder
-{
-    /** Fold a list before a line would get longer */
-    private const FOLD_AT = 78;
-
-    public function build(Settings $settings, Journal $journal): string
-    {
-        return match ($settings->profile) {
-            'archivecow' => $this->archivecow($settings, $journal),
-            'envelope' => $this->envelope($settings, $journal),
-            default => '',
-        };
-    }
-
-    /** One line of the signed text */
-    public static function canonical(string $name, string $value): string
-    {
-        return strtolower($name) . ':' . preg_replace('/\s+/', '', $value);
-    }
-
-    private function archivecow(Settings $settings, Journal $journal): string
-    {
-        $values = [
-            'X-Archive-Owner' => $journal->owners,
-            'X-Archive-Rcpt' => $journal->rcpts,
-            'X-Archive-From' => $journal->from,
-            'X-Archive-Queue-ID' => $journal->qid,
-            'X-Archive-Received' => self::time($journal->received),
-            'X-Archive-Direction' => $journal->direction,
-            'X-Archive-Source' => $journal->source,
-            'X-Archive-SHA256' => $journal->sha256,
-        ];
-        $lines = [];
-        $signed = [];
-        foreach ($values as $name => $value) {
-            $value = is_array($value) ? self::cleanList($value) : self::clean($value);
-            if ($value === '' || $value === []) {
-                continue;
-            }
-            $lines[] = self::line($name, $value);
-            $signed[] = self::canonical($name, is_array($value) ? implode(',', $value) : $value);
-        }
-        $lines[] = 'X-Archive-Signature: v=1; a=hmac-sha256; s=' . hash_hmac('sha256', implode("\n", $signed), $settings->signingKey);
-        return implode("\r\n", $lines) . "\r\n";
-    }
-
-    private function envelope(Settings $settings, Journal $journal): string
-    {
-        $from = self::clean($journal->from);
-        $lines = [self::line($settings->headerFrom, ($from === '') ? '<>' : $from)];
-        $rcpts = self::cleanList($journal->rcpts);
-        if ($rcpts !== []) {
-            $lines[] = self::line($settings->headerRcpt, $rcpts);
-        }
-        return implode("\r\n", $lines) . "\r\n";
-    }
-
-    /** Header line; a list is folded after commas */
-    private static function line(string $name, string|array $value): string
-    {
-        if (is_string($value)) {
-            return $name . ': ' . $value;
-        }
-        $out = $name . ': ' . array_shift($value);
-        $length = strlen($out);
-        foreach ($value as $item) {
-            if ($length + strlen($item) + 1 > self::FOLD_AT) {
-                $out .= ",\r\n " . $item;
-                $length = strlen($item) + 1;
-            } else {
-                $out .= ',' . $item;
-                $length += strlen($item) + 1;
-            }
-        }
-        return $out;
-    }
-
-    /** Header value without control characters (no header injection) */
-    private static function clean(string $value): string
-    {
-        return trim((string)preg_replace('/[\x00-\x1f\x7f]+/', ' ', $value));
-    }
-
-    /** @return string[] */
-    private static function cleanList(array $values): array
-    {
-        $list = [];
-        foreach ($values as $value) {
-            $value = is_string($value) ? str_replace(',', '', self::clean($value)) : '';
-            if ($value !== '') {
-                $list[] = $value;
-            }
-        }
-        return $list;
-    }
-
-    /** RFC 3339 in UTC with milliseconds */
-    private static function time(float $time): string
-    {
-        $date = DateTime::createFromFormat('U.u', sprintf('%.3F', $time));
-        return $date ? $date->format('Y-m-d\TH:i:s.v\Z') : '';
-    }
-}

data/web/inc/lib/mailcow/Archive/Service/JournalService.php

@@ -1,144 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Archive\Service;
-
-use Mailcow\Archive\Model\Journal;
-use Mailcow\Archive\Model\Settings;
-use Mailcow\Archive\Redis\ArchiveState;
-use Mailcow\Archive\Repository\SettingsRepository;
-use Mailcow\Archive\Smtp\SmtpClient;
-use Mailcow\Archive\Smtp\SmtpException;
-use Mailcow\Core\Context;
-use Mailcow\Core\UserError;
-use Throwable;
-
-/**
- * Archive copies: a copy of every accepted mail of an archived mailbox goes by SMTP to the archive, through the
- * Postfix queue. rspamd sends the copies of mails it scanned (rspamd.local.lua) and asks prepare() whether to send
- * one and for its header block; mails that bypass rspamd (quarantine release, mass mail) are sent here.
- *
- * Copies have the envelope sender Settings::SENDER, Postfix drops every notice to it: an archive copy never causes
- * a bounce or delay warning to a real sender.
- */
-class JournalService
-{
-    /** Postfix service that takes copies (no milter, mynetworks only) */
-    private const SUBMIT_PORT = 591;
-
-    public function __construct(
-        private SettingsRepository $settings,
-        private OwnerResolver $resolver,
-        private HeaderBuilder $headers,
-        private SmtpClient $smtp,
-        private ArchiveState $state,
-        private Context $context,
-        private string $postfixHost = 'postfix'
-    ) {
-    }
-
-    /**
-     * Archive decision for a mail rspamd accepted (dynmap archive.php), no copy while archiving is off or no
-     * archived mailbox is involved.
-     *
-     * @param array $request from, rcpt[], user, bcc, qid, received, sha256 (of the content rspamd sends)
-     * @return array{archive: bool, rcpt?: string, sender?: string, headers?: string}
-     */
-    public function prepare(array $request): array
-    {
-        $settings = $this->settings->load();
-        if (!$this->state->enabled() || !$settings->canDeliver()) {
-            return ['archive' => false];
-        }
-        $from = self::text($request['from'] ?? '');
-        $rcpts = is_array($request['rcpt'] ?? null) ? array_values(array_filter($request['rcpt'], 'is_string')) : [];
-        $resolved = $this->resolver->resolve($from, $rcpts, self::text($request['user'] ?? ''), ($request['bcc'] ?? false) === true);
-        if ($resolved['owners'] === []) {
-            return ['archive' => false];
-        }
-        $sha256 = self::text($request['sha256'] ?? '');
-        $journal = new Journal($resolved['owners'], $rcpts, $from, mb_strcut(self::text($request['qid'] ?? ''), 0, 64),
-            $resolved['direction'], 'smtp', is_numeric($request['received'] ?? null) ? (float)$request['received'] : microtime(true),
-            preg_match('/^[0-9a-f]{64}$/', $sha256) ? $sha256 : '');
-        return ['archive' => true, 'rcpt' => $settings->address, 'sender' => Settings::SENDER, 'headers' => $this->headers->build($settings, $journal)];
-    }
-
-    /**
-     * Sends the copy of a mail that reached its mailbox without rspamd (quarantine release, mass mail) for its
-     * archived recipients. Errors are logged and counted, never thrown: the mail is delivered already.
-     *
-     * @param string[] $rcpts
-     */
-    public function submitDelivered(string $message, string $from, array $rcpts, string $source): bool
-    {
-        try {
-            $settings = $this->settings->load();
-            if (!$this->state->enabled() || !$settings->canDeliver()) {
-                return false;
-            }
-            $resolved = $this->resolver->resolve($from, $rcpts, '', false);
-            if ($resolved['owners'] === []) {
-                return false;
-            }
-            $this->send($settings, $message, $resolved['owners'], array_values($rcpts), $from, 'in', $source);
-            return true;
-        } catch (Throwable $e) {
-            error_log('ARCHIVE: cannot send the archive copy (' . $source . '): ' . $e->getMessage());
-            $this->state->dropped('', $source . ': ' . $e->getMessage(), self::messageId($message));
-            return false;
-        }
-    }
-
-    /**
-     * Sends a test copy through Postfix to the archive and returns the Postfix queue id.
-     *
-     * @throws UserError
-     */
-    public function testMail(): string
-    {
-        $this->context->requireAdmin();
-        $settings = $this->settings->load();
-        if (!$settings->canDeliver()) {
-            throw new UserError('archive_incomplete');
-        }
-        $message = implode("\r\n", [
-            'From: mailcow <' . Settings::SENDER . '>',
-            'To: <' . $settings->address . '>',
-            'Subject: mailcow archive test',
-            'Date: ' . date(DATE_RFC2822),
-            'Message-ID: <' . bin2hex(random_bytes(12)) . '@archive-test.mailcow>',
-            'MIME-Version: 1.0',
-            'Content-Type: text/plain; charset=utf-8',
-            '',
-            'Test copy from mailcow (System - Configuration - Archive).',
-            '',
-        ]);
-        try {
-            return $this->send($settings, $message, [], [$settings->address], Settings::SENDER, 'in', 'test');
-        } catch (SmtpException $e) {
-            throw new UserError('archive_testmail_failed', [htmlspecialchars($e->describe())]);
-        }
-    }
-
-    /** @throws SmtpException */
-    private function send(Settings $settings, string $message, array $owners, array $rcpts, string $from, string $direction, string $source): string
-    {
-        $message = SmtpClient::crlf($message);
-        $journal = new Journal($owners, $rcpts, $from, '', $direction, $source, microtime(true), hash('sha256', $message));
-        return $this->smtp->submit($this->postfixHost, self::SUBMIT_PORT, Settings::SENDER, $settings->address,
-            $this->headers->build($settings, $journal) . $message);
-    }
-
-    /** Message-ID of a mail, for the archive log */
-    private static function messageId(string $message): string
-    {
-        $headers = explode("\n\n", str_replace("\r\n", "\n", $message), 2)[0];
-        return preg_match('/^Message-ID:\s*<?([^>\s]+)>?/mi', $headers, $m) ? $m[1] : '';
-    }
-
-    private static function text(mixed $value): string
-    {
-        return is_scalar($value) ? trim((string)preg_replace('/[\x00-\x1f\x7f]+/', ' ', (string)$value)) : '';
-    }
-}

data/web/inc/lib/mailcow/Archive/Service/MailboxListService.php

@@ -1,38 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Archive\Service;
-
-use Mailcow\Archive\Model\Mailbox;
-use Mailcow\Archive\Redis\ArchiveState;
-use Mailcow\Archive\Repository\LookupRepository;
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\Context;
-
-/**
- * The mailboxes archived now (GET archive/mailboxes), the list archivecow keeps its mailboxes in line with.
- * Same rule as the owners of a mail: the global switch, then the mailbox setting, 'inherit' follows the domain.
- */
-class MailboxListService
-{
-    public function __construct(private LookupRepository $lookup, private ArchiveState $state, private Context $context)
-    {
-    }
-
-    /**
-     * Admins, read-only API keys included.
-     *
-     * @return array{enabled: int, mailboxes: array[]}
-     * @throws AccessDenied
-     */
-    public function archived(): array
-    {
-        $this->context->requireAdmin(false);
-        $enabled = $this->state->enabled();
-        return [
-            'enabled' => (int)$enabled,
-            'mailboxes' => $enabled ? array_map(fn(Mailbox $mailbox): array => $mailbox->toArray(), $this->lookup->archivedMailboxes()) : [],
-        ];
-    }
-}

data/web/inc/lib/mailcow/Archive/Service/OwnerResolver.php

@@ -1,264 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Archive\Service;
-
-use Mailcow\Archive\Model\Mailbox;
-use Mailcow\Archive\Repository\LookupRepository;
-
-/**
- * Finds the mailboxes a mail belongs to in the archive, following the way Postfix delivers it.
- *
- * Incoming: each envelope recipient goes through recipient_maps (one rewrite) and the virtual alias expansion:
- * alias rows (incl. catch-alls and groups), resources (discarded), spam aliases and mailboxes of alias domains,
- * each tried as u+ext@d, u@d and @d, recursively. The local mailboxes at the end receive the mail. BCC map
- * destinations count as recipients when rspamd sends the BCC copy (it bypasses rspamd).
- *
- * Outgoing: the SASL user and the mailbox the user sent as (its address, with +tag or in an alias domain);
- * Postfix already checked that the login may use that sender. Without a login the sender never counts, so a
- * forged local sender cannot make a mailbox an owner.
- *
- * Only mailboxes with archiving become owners.
- */
-class OwnerResolver
-{
-    /** virtual_alias_recursion_limit and virtual_alias_expansion_limit of Postfix */
-    private const MAX_DEPTH = 1000;
-    private const MAX_ADDRESSES = 1000;
-
-    public function __construct(private LookupRepository $lookup)
-    {
-    }
-
-    /**
-     * @param string[] $rcpts SMTP envelope recipients
-     * @param bool $bcc rspamd sent the BCC copies of this mail
-     * @return array{owners: string[], direction: string} sorted usernames; direction in, out or internal
-     */
-    public function resolve(string $from, array $rcpts, string $user, bool $bcc): array
-    {
-        $from = self::address($from);
-        $incoming = [];
-        foreach ($rcpts as $rcpt) {
-            $rcpt = self::address($rcpt);
-            if ($rcpt !== '') {
-                $this->deliver($rcpt, $incoming);
-            }
-        }
-        if ($bcc) {
-            foreach ($this->bccDestinations($from, $rcpts) as $destination) {
-                $this->deliver($destination, $incoming);
-            }
-        }
-        $outgoing = $this->senders($from, self::address($user));
-        $owners = [];
-        foreach (array_merge(array_values($incoming), array_values($outgoing)) as $mailbox) {
-            if ($mailbox->archived()) {
-                $owners[$mailbox->username] = true;
-            }
-        }
-        $owners = array_keys($owners);
-        sort($owners);
-        $direction = empty($outgoing) ? 'in' : (empty($incoming) ? 'out' : 'internal');
-        return ['owners' => $owners, 'direction' => $direction];
-    }
-
-    /** Adds the local mailboxes a recipient ends in to $mailboxes (username => Mailbox) */
-    private function deliver(string $address, array &$mailboxes): void
-    {
-        $seen = [];
-        $this->expand($this->canonical($address), 0, $mailboxes, $seen);
-    }
-
-    /** recipient_maps: first matching key form, one rewrite */
-    private function canonical(string $address): string
-    {
-        foreach (self::keys($address) as $key) {
-            $target = self::target($this->lookup->recipientMap($key));
-            if ($target !== '') {
-                return self::rewrite($address, $key, $target);
-            }
-        }
-        return $address;
-    }
-
-    private function expand(string $address, int $depth, array &$mailboxes, array &$seen): void
-    {
-        if ($depth > self::MAX_DEPTH || count($seen) >= self::MAX_ADDRESSES || isset($seen[$address])) {
-            return;
-        }
-        $seen[$address] = true;
-        $targets = $this->virtual($address);
-        if ($targets === null) {
-            $this->final($address, $mailboxes);
-            return;
-        }
-        foreach ($targets as $target) {
-            // an address that expands to itself is delivered as it is
-            if ($target === $address) {
-                $this->final($address, $mailboxes);
-            } else {
-                $this->expand($target, $depth + 1, $mailboxes, $seen);
-            }
-        }
-    }
-
-    /** Targets from the first key form a virtual alias map knows, null if none does */
-    private function virtual(string $address): ?array
-    {
-        if (self::split($address)[1] === 'localhost') {
-            // null@, spam@, ham@localhost: local delivery of Postfix, never a mailbox
-            return [];
-        }
-        foreach (self::keys($address) as $key) {
-            $targets = $this->virtualLookup($key);
-            if ($targets !== null) {
-                return array_map(fn(string $target): string => self::rewrite($address, $key, $target), $targets);
-            }
-        }
-        return null;
-    }
-
-    /** virtual_alias_maps in the order of main.cf: alias, resource, spam alias, alias domain mailbox */
-    private function virtualLookup(string $key): ?array
-    {
-        $domainKey = str_starts_with($key, '@');
-        $targets = self::targets($this->lookup->aliasGoto($key));
-        if ($targets === null && !$domainKey && $this->lookup->isResource($key)) {
-            $targets = ['null@localhost'];
-        }
-        $targets ??= self::targets($this->lookup->spamAliasGoto($key));
-        if ($targets === null && !$domainKey) {
-            [$local, $domain] = self::split($key);
-            $targets = self::targets($this->lookup->aliasDomainMailbox($local, $domain));
-        }
-        return $targets;
-    }
-
-    /** A final address: a mailbox (as u+ext@d or u@d) or anything else, which is not ours */
-    private function final(string $address, array &$mailboxes): void
-    {
-        foreach (array_unique([$address, self::untagged($address)]) as $candidate) {
-            $mailbox = $this->lookup->mailbox($candidate);
-            if ($mailbox !== null) {
-                $mailboxes[$mailbox->username] = $mailbox;
-                return;
-            }
-        }
-    }
-
-    /** BCC destinations of rspamd: rcpt maps for every recipient and its domain, sender maps for MAIL FROM and its domain */
-    private function bccDestinations(string $from, array $rcpts): array
-    {
-        $destinations = [];
-        foreach ($rcpts as $rcpt) {
-            $rcpt = self::address($rcpt);
-            if ($rcpt !== '') {
-                $destinations[] = $this->lookup->bccDestination(self::untagged($rcpt), 'rcpt');
-                $destinations[] = $this->lookup->bccDestination('@' . self::split($rcpt)[1], 'rcpt');
-            }
-        }
-        if ($from !== '') {
-            $destinations[] = $this->lookup->bccDestination(self::untagged($from), 'sender');
-            $destinations[] = $this->lookup->bccDestination('@' . self::split($from)[1], 'sender');
-        }
-        return array_values(array_unique(array_filter(array_map(fn(?string $address): string => self::address($address), $destinations))));
-    }
-
-    /** Mailboxes on the sending side: the SASL user and the mailbox the user sent as */
-    private function senders(string $from, string $user): array
-    {
-        $mailbox = ($user !== '') ? $this->lookup->mailbox($user) : null;
-        if ($mailbox === null) {
-            return [];
-        }
-        $senders = [$mailbox->username => $mailbox];
-        $sentAs = ($from !== '') ? $this->sentAs(self::untagged($from)) : null;
-        if ($sentAs !== null) {
-            $senders[$sentAs->username] = $sentAs;
-        }
-        return $senders;
-    }
-
-    /** Mailbox a sender address belongs to: the mailbox itself or the mailbox behind an alias domain address, not aliases */
-    private function sentAs(string $address): ?Mailbox
-    {
-        $mailbox = $this->lookup->mailbox($address);
-        if ($mailbox === null) {
-            [$local, $domain] = self::split($address);
-            $username = $this->lookup->aliasDomainMailbox($local, $domain);
-            $mailbox = ($username !== null) ? $this->lookup->mailbox(strtolower($username)) : null;
-        }
-        return $mailbox;
-    }
-
-    /** Lookup keys in the order of Postfix: u+ext@d, u@d, @d */
-    private static function keys(string $address): array
-    {
-        $keys = [$address, self::untagged($address), '@' . self::split($address)[1]];
-        return array_values(array_unique($keys));
-    }
-
-    /**
-     * Result of a lookup for $address under $key: '@domain' keeps the local part, a match without the extension
-     * passes it on (propagate_unmatched_extensions).
-     */
-    private static function rewrite(string $address, string $key, string $target): string
-    {
-        [$local] = self::split($address);
-        if (str_starts_with($target, '@')) {
-            return $local . $target;
-        }
-        $extension = self::extension($local);
-        if ($key === $address || $extension === '') {
-            return $target;
-        }
-        [$targetLocal, $targetDomain] = self::split($target);
-        return ($targetDomain === 'localhost' || self::extension($targetLocal) !== '')
-            ? $target
-            : $targetLocal . '+' . $extension . '@' . $targetDomain;
-    }
-
-    /** Comma separated lookup result; null when empty (not found) */
-    private static function targets(?string $value): ?array
-    {
-        $targets = array_values(array_filter(array_map(fn(string $target): string => self::target($target), explode(',', (string)$value))));
-        return empty($targets) ? null : $targets;
-    }
-
-    /** Address or '@domain' from a lookup result, lower case; '' if invalid */
-    private static function target(?string $value): string
-    {
-        $value = strtolower(trim((string)$value));
-        return preg_match('/^[^@\s,<>]*@[^@\s,<>]+$/', $value) ? $value : '';
-    }
-
-    /** Lower case address without angle brackets; '' unless local@domain */
-    private static function address(mixed $value): string
-    {
-        $value = is_string($value) ? strtolower(trim($value, " \t\r\n<>")) : '';
-        return preg_match('/^[^@\s,<>]+@[^@\s,<>]+$/', $value) ? $value : '';
-    }
-
-    private static function untagged(string $address): string
-    {
-        [$local, $domain] = self::split($address);
-        $pos = strpos($local, '+');
-        return ($pos === false || $pos === 0) ? $address : substr($local, 0, $pos) . '@' . $domain;
-    }
-
-    /** Address extension after the recipient delimiter "+" */
-    private static function extension(string $local): string
-    {
-        $pos = strpos($local, '+');
-        return ($pos === false || $pos === 0) ? '' : substr($local, $pos + 1);
-    }
-
-    /** [local part, domain] of an address or '@domain' */
-    private static function split(string $address): array
-    {
-        $at = strrpos($address, '@');
-        return ($at === false) ? [$address, ''] : [substr($address, 0, $at), substr($address, $at + 1)];
-    }
-}

data/web/inc/lib/mailcow/Archive/Service/SettingsService.php

@@ -1,164 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Archive\Service;
-
-use Mailcow\Archive\Model\Settings;
-use Mailcow\Archive\Redis\ArchiveState;
-use Mailcow\Archive\Repository\LookupRepository;
-use Mailcow\Archive\Repository\SettingsRepository;
-use Mailcow\Archive\Smtp\SmtpClient;
-use Mailcow\Archive\Smtp\SmtpException;
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\Context;
-use Mailcow\Core\Input;
-use Mailcow\Core\UserError;
-
-/**
- * Admin use cases of the archive target. Read-only API keys may read the settings (never password or key).
- */
-class SettingsService
-{
-    private const MAX_SECRET_LENGTH = 255;
-    /** Extensions the archive should announce: Postfix passes 8-bit mails unchanged (archive_smtp in master.cf) */
-    private const WANTED_EXTENSIONS = ['8BITMIME'];
-
-    public function __construct(private SettingsRepository $repo, private LookupRepository $lookup, private ArchiveState $state,
-        private SmtpClient $smtp, private Context $context)
-    {
-    }
-
-    /** @throws AccessDenied */
-    public function get(): Settings
-    {
-        $this->context->requireAdmin(false);
-        $settings = $this->repo->load();
-        $settings->enabled = $this->state->enabled();
-        return $settings;
-    }
-
-    /** Global switch for internal callers (page templates, mailbox details), no permission check */
-    public function enabled(): bool
-    {
-        return $this->state->enabled();
-    }
-
-    /**
-     * Saves the settings: attributes missing in $input keep their value, an empty password or signing key keeps
-     * the stored one, `password_remove` deletes the password. Switching on needs an address (and a signing key for archivecow).
-     *
-     * @throws UserError
-     */
-    public function save(array $input): Settings
-    {
-        $this->context->requireAdmin();
-        $settings = $this->repo->load();
-        $settings->enabled = $this->state->enabled();
-        if (array_key_exists('address', $input)) {
-            $address = strtolower((string)Input::text($input['address']));
-            if (!is_scalar($input['address']) || ($address !== '' && !filter_var($address, FILTER_VALIDATE_EMAIL))) {
-                throw new UserError('archive_address_invalid');
-            }
-            // Postfix would treat an own address as local (aliases, unknown users) before the copy reaches the archive
-            if ($address !== '' && $this->lookup->isMailcowDomain(substr($address, strrpos($address, '@') + 1))) {
-                throw new UserError('archive_address_local');
-            }
-            $settings->address = $address;
-        }
-        if (array_key_exists('host', $input)) {
-            $host = strtolower(trim((string)Input::text($input['host']), " []"));
-            if (!is_scalar($input['host']) || ($host !== '' && !filter_var($host, FILTER_VALIDATE_IP)
-                && !filter_var($host, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME))) {
-                throw new UserError('archive_host_invalid');
-            }
-            $settings->host = $host;
-        }
-        if (array_key_exists('port', $input)) {
-            $settings->port = Input::intInRange($input['port'], 1, 65535) ?? throw new UserError('archive_port_invalid');
-        }
-        if (array_key_exists('tls', $input)) {
-            $settings->tls = in_array($input['tls'], Settings::TLS_MODES, true) ? $input['tls'] : throw new UserError('archive_tls_invalid');
-        }
-        if (array_key_exists('username', $input)) {
-            $username = Input::text($input['username']);
-            if ($username === null || strlen($username) > self::MAX_SECRET_LENGTH || preg_match('/[\s:]/', $username)) {
-                throw new UserError('archive_username_invalid');
-            }
-            $settings->username = $username;
-        }
-        if (array_key_exists('password', $input)) {
-            $settings->password = self::secret($input['password'], $settings->password, 'archive_password_invalid');
-        }
-        if (self::flag($input['password_remove'] ?? '0')) {
-            $settings->password = '';
-        }
-        if (array_key_exists('profile', $input)) {
-            $settings->profile = in_array($input['profile'], Settings::PROFILES, true) ? $input['profile'] : throw new UserError('archive_profile_invalid');
-        }
-        if (array_key_exists('signing_secret', $input)) {
-            $settings->signingKey = self::secret($input['signing_secret'], $settings->signingKey, 'archive_signing_secret_invalid');
-        }
-        foreach (['header_from' => 'headerFrom', 'header_rcpt' => 'headerRcpt'] as $key => $property) {
-            if (array_key_exists($key, $input)) {
-                $name = Input::text($input[$key]);
-                if ($name === null || !preg_match('/^[A-Za-z][A-Za-z0-9-]{0,63}$/', $name)) {
-                    throw new UserError('archive_header_invalid');
-                }
-                $settings->$property = $name;
-            }
-        }
-        if (array_key_exists('enabled', $input)) {
-            $settings->enabled = self::flag($input['enabled']);
-        }
-        if ($settings->enabled && !$settings->canDeliver()) {
-            throw new UserError('archive_incomplete');
-        }
-        $this->repo->save($settings);
-        $this->state->setEnabled($settings->enabled);
-        return $settings;
-    }
-
-    /**
-     * Tests the stored target: connection, TLS, AUTH and whether the archive accepts the address (no mail is sent).
-     * Returns the success message, with the missing extension when the archive lacks 8BITMIME.
-     *
-     * @throws UserError
-     */
-    public function test(): string|array
-    {
-        $this->context->requireAdmin();
-        $settings = $this->repo->load();
-        if ($settings->address === '') {
-            throw new UserError('archive_incomplete');
-        }
-        $host = ($settings->host !== '') ? $settings->host : $this->smtp->mailHost(substr($settings->address, strrpos($settings->address, '@') + 1));
-        try {
-            $extensions = $this->smtp->probe($host, $settings->port, $settings->tls, $settings->username, $settings->password,
-                Settings::SENDER, $settings->address);
-        } catch (SmtpException $e) {
-            throw new UserError('archive_test_failed', [htmlspecialchars($host . ':' . $settings->port . ' ' . $e->describe())]);
-        }
-        $missing = array_values(array_diff(self::WANTED_EXTENSIONS, $extensions));
-        return empty($missing) ? 'archive_test_ok' : ['archive_test_ok_missing', implode(', ', $missing)];
-    }
-
-    /** A new secret, or the stored one when $value is empty */
-    private static function secret(mixed $value, string $current, string $error): string
-    {
-        $secret = Input::text($value);
-        if ($secret === null || strlen($secret) > self::MAX_SECRET_LENGTH || preg_match('/[^\x21-\x7e]/', $secret)) {
-            throw new UserError($error);
-        }
-        return ($secret === '') ? $current : $secret;
-    }
-
-    /** Checkbox or API flag; a hidden "0" + checkbox pair arrives as ["0", "1"] when checked */
-    private static function flag(mixed $value): bool
-    {
-        if (is_array($value)) {
-            $value = end($value);
-        }
-        return filter_var($value, FILTER_VALIDATE_BOOLEAN);
-    }
-}

data/web/inc/lib/mailcow/Archive/Service/StatusService.php

@@ -1,85 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Archive\Service;
-
-use Mailcow\Archive\Legacy\LegacyQueue;
-use Mailcow\Archive\Model\Settings;
-use Mailcow\Archive\Redis\ArchiveState;
-use Mailcow\Archive\Repository\SettingsRepository;
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\Context;
-
-/**
- * State of the archive for the admin: the summary of the archive tab (copies waiting in the Postfix queue with the
- * reason Postfix gives, last delivery, mails not archived) and the archive log of the dashboard.
- */
-class StatusService
-{
-    /** Mails not archived count as a problem for a day */
-    private const DROPPED_WINDOW = 86400;
-
-    public function __construct(private SettingsRepository $settings, private ArchiveState $state, private LegacyQueue $queue,
-        private Context $context)
-    {
-    }
-
-    /**
-     * GET archive/status (admins, read-only keys included).
-     *
-     * @throws AccessDenied
-     */
-    public function status(): array
-    {
-        $this->context->requireAdmin(false);
-        $settings = $this->settings->load();
-        $copies = $this->queue->entries(Settings::SENDER);
-        $reasons = [];
-        $oldest = null;
-        $bytes = 0;
-        foreach ($copies as $copy) {
-            $arrival = (int)($copy['arrival_time'] ?? 0);
-            $oldest = ($oldest === null) ? $arrival : min($oldest, $arrival);
-            $bytes += (int)($copy['message_size'] ?? 0);
-            foreach ((array)($copy['recipients'] ?? []) as $rcpt) {
-                if (is_string($rcpt) && preg_match('/^\S+ \((.+)\)$/', $rcpt, $m)) {
-                    $reasons[$m[1]] = true;
-                }
-            }
-        }
-        $dropped = $this->state->droppedStats();
-        $recent = array_values(array_filter($dropped['recent'], fn(array $d): bool => $d['time'] >= time() - self::DROPPED_WINDOW));
-        $lastDelivery = null;
-        foreach ($this->state->events() as $event) {
-            if ($event['status'] === 'sent') {
-                $lastDelivery = $event['time'];
-                break;
-            }
-        }
-        return [
-            'enabled' => (int)$this->state->enabled(),
-            'configured' => (int)$settings->canDeliver(),
-            'queued' => count($copies),
-            'queued_bytes' => $bytes,
-            'oldest_queued' => $oldest,
-            'delay_reasons' => array_slice(array_keys($reasons), 0, 5),
-            'last_delivery' => $lastDelivery,
-            'dropped' => $dropped['count'],
-            'dropped_24h' => count($recent),
-            'last_dropped' => $recent ? ['time' => $recent[0]['time'], 'reason' => $recent[0]['reason']] : null,
-        ];
-    }
-
-    /**
-     * GET logs/archive: deliveries, expired copies and mails not archived, newest first (admins, read-only keys included);
-     * $count limits the entries, 0 keeps the default.
-     *
-     * @throws AccessDenied
-     */
-    public function log(int $count = 0): array
-    {
-        $this->context->requireAdmin(false);
-        return ($count > 0) ? $this->state->events($count) : $this->state->events();
-    }
-}

data/web/inc/lib/mailcow/Archive/Smtp/SmtpClient.php

@@ -1,190 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Archive\Smtp;
-
-/**
- * Small SMTP client: hands archive copies to Postfix (port 591: no milter, mynetworks only) and tests the
- * connection to the archive server (EHLO, STARTTLS, AUTH, MAIL/RCPT, no mail is sent).
- *
- * The socket I/O sits in protected methods, so tests can replace it.
- */
-class SmtpClient
-{
-    private const TIMEOUT = 20;
-
-    /** @var resource|null */
-    private $socket = null;
-
-    /**
-     * Sends one mail and returns the queue id Postfix reports. $data must end with CRLF; it is received as it is.
-     *
-     * @throws SmtpException
-     */
-    public function submit(string $host, int $port, string $from, string $rcpt, string $data): string
-    {
-        $this->open($host, $port);
-        try {
-            $this->expect('connect', 220);
-            $this->ehlo();
-            $this->command('sender', 'MAIL FROM:<' . $from . '>', 250);
-            $this->command('recipient', 'RCPT TO:<' . $rcpt . '>', 250);
-            $this->command('data', 'DATA', 354);
-            $this->write(self::dotStuff(self::crlf($data)) . ".\r\n");
-            $reply = $this->expect('data', 250);
-            $this->write("QUIT\r\n");
-            return preg_match('/queued as ([0-9A-Za-z]+)/', $reply, $m) ? $m[1] : '';
-        } finally {
-            $this->close();
-        }
-    }
-
-    /**
-     * Connection test against the archive server: STARTTLS (unless $tls is 'may' and it is not offered), AUTH when
-     * a username is set, then MAIL and RCPT for the archive address, then RSET.
-     *
-     * @return string[] extensions the server announced (after STARTTLS)
-     * @throws SmtpException
-     */
-    public function probe(string $host, int $port, string $tls, string $username, string $password, string $from, string $rcpt): array
-    {
-        $this->open($host, $port);
-        try {
-            $this->expect('connect', 220);
-            $extensions = $this->ehlo();
-            if ($tls !== 'may' || in_array('STARTTLS', $extensions, true)) {
-                if (!in_array('STARTTLS', $extensions, true)) {
-                    throw new SmtpException('tls', 'STARTTLS not offered');
-                }
-                $this->command('tls', 'STARTTLS', 220);
-                $this->startTls($host, $tls === 'verify');
-                $extensions = $this->ehlo();
-            }
-            if ($username !== '') {
-                $this->command('auth', 'AUTH PLAIN ' . base64_encode("\0" . $username . "\0" . $password), 235);
-            }
-            $this->command('sender', 'MAIL FROM:<' . $from . '>', 250);
-            $this->command('recipient', 'RCPT TO:<' . $rcpt . '>', 250);
-            $this->command('recipient', 'RSET', 250);
-            $this->write("QUIT\r\n");
-            return $extensions;
-        } finally {
-            $this->close();
-        }
-    }
-
-    /** Mail server of a domain: the MX with the lowest preference, else the domain itself */
-    public function mailHost(string $domain): string
-    {
-        if (@getmxrr($domain, $hosts, $weights) && !empty($hosts)) {
-            array_multisort($weights, SORT_NUMERIC, $hosts);
-            return rtrim($hosts[0], '.');
-        }
-        return $domain;
-    }
-
-    /** CRLF line ends, ending with CRLF */
-    public static function crlf(string $data): string
-    {
-        $data = (string)preg_replace('/\r\n|\r|\n/', "\r\n", $data);
-        return str_ends_with($data, "\r\n") ? $data : $data . "\r\n";
-    }
-
-    /** RFC 5321 4.5.2: a line starting with "." gets another "." */
-    public static function dotStuff(string $data): string
-    {
-        $data = str_replace("\r\n.", "\r\n..", $data);
-        return str_starts_with($data, '.') ? '.' . $data : $data;
-    }
-
-    /** @throws SmtpException */
-    protected function open(string $host, int $port): void
-    {
-        $target = filter_var($host, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) ? '[' . $host . ']' : $host;
-        $socket = @stream_socket_client('tcp://' . $target . ':' . $port, $errno, $error, self::TIMEOUT);
-        if ($socket === false) {
-            throw new SmtpException('connect', $error !== '' ? $error : 'connection failed');
-        }
-        stream_set_timeout($socket, self::TIMEOUT);
-        $this->socket = $socket;
-    }
-
-    protected function close(): void
-    {
-        if (is_resource($this->socket)) {
-            fclose($this->socket);
-        }
-        $this->socket = null;
-    }
-
-    /** @throws SmtpException */
-    protected function write(string $data): void
-    {
-        for ($written = 0; $written < strlen($data); $written += $count) {
-            $count = @fwrite($this->socket, substr($data, $written, 1048576));
-            if ($count === false || $count === 0) {
-                throw new SmtpException('data', 'connection lost while sending');
-            }
-        }
-    }
-
-    /** One reply line without CRLF, null when the connection ended */
-    protected function readLine(): ?string
-    {
-        $line = fgets($this->socket, 4096);
-        return ($line === false) ? null : rtrim($line, "\r\n");
-    }
-
-    /** @throws SmtpException */
-    protected function startTls(string $host, bool $verify): void
-    {
-        stream_context_set_option($this->socket, 'ssl', 'verify_peer', $verify);
-        stream_context_set_option($this->socket, 'ssl', 'verify_peer_name', $verify);
-        stream_context_set_option($this->socket, 'ssl', 'allow_self_signed', !$verify);
-        stream_context_set_option($this->socket, 'ssl', 'peer_name', $host);
-        if (!@stream_socket_enable_crypto($this->socket, true, STREAM_CRYPTO_METHOD_TLS_CLIENT)) {
-            throw new SmtpException('tls', $verify ? 'TLS handshake failed or certificate not valid' : 'TLS handshake failed');
-        }
-    }
-
-    /** @return string[] extensions in upper case */
-    private function ehlo(): array
-    {
-        $reply = $this->command('connect', 'EHLO ' . (gethostname() ?: 'mailcow'), 250);
-        $extensions = [];
-        foreach (array_slice(explode("\n", $reply), 1) as $line) {
-            $extensions[] = strtoupper((string)strtok(substr($line, 4), ' '));
-        }
-        return $extensions;
-    }
-
-    /** @throws SmtpException */
-    private function command(string $step, string $command, int $code): string
-    {
-        $this->write($command . "\r\n");
-        return $this->expect($step, $code);
-    }
-
-    /**
-     * Reads a (multi-line) reply and returns it, lines joined by "\n".
-     *
-     * @throws SmtpException when the code differs
-     */
-    private function expect(string $step, int $code): string
-    {
-        $lines = [];
-        do {
-            $line = $this->readLine();
-            if ($line === null) {
-                throw new SmtpException($step, 'connection closed');
-            }
-            $lines[] = $line;
-        } while (strlen($line) > 3 && $line[3] === '-');
-        $reply = implode("\n", $lines);
-        if ((int)substr($reply, 0, 3) !== $code && !($code === 250 && (int)substr($reply, 0, 3) === 251)) {
-            throw new SmtpException($step, $reply);
-        }
-        return $reply;
-    }
-}

data/web/inc/lib/mailcow/Archive/Smtp/SmtpException.php

@@ -1,27 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Archive\Smtp;
-
-use RuntimeException;
-
-/**
- * An SMTP step failed: connect, tls, auth, sender, recipient or data, with the reply of the server.
- */
-class SmtpException extends RuntimeException
-{
-    private const MAX_TEXT_LENGTH = 200;
-
-    public function __construct(public readonly string $step, public readonly string $reply)
-    {
-        parent::__construct($step . ': ' . $reply);
-    }
-
-    /** Short text for messages: "<step>: <reply>", without angle brackets (the UI escapes message arguments twice) */
-    public function describe(): string
-    {
-        $text = trim((string)preg_replace('/\s+/', ' ', $this->step . ': ' . str_replace(['<', '>'], '', $this->reply)));
-        return mb_substr($text, 0, self::MAX_TEXT_LENGTH);
-    }
-}

data/web/inc/lib/mailcow/Backup/BackupModule.php

@@ -1,90 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup;
-
-use Mailcow\Backup\Controller\BackupApiController;
-use Mailcow\Backup\Legacy\LegacyDomain;
-use Mailcow\Backup\Legacy\NoTemplates;
-use Mailcow\Backup\Model\Settings;
-use Mailcow\Backup\Repository\ExportRepository;
-use Mailcow\Backup\Repository\JobRepository;
-use Mailcow\Backup\Repository\RestoreRepository;
-use Mailcow\Backup\Repository\SettingsRepository;
-use Mailcow\Backup\Repository\SnapshotRepository;
-use Mailcow\Backup\Service\DomainRestoreService;
-use Mailcow\Backup\Service\ExportService;
-use Mailcow\Backup\Service\GroupMembers;
-use Mailcow\Backup\Service\JobService;
-use Mailcow\Backup\Service\RestoreService;
-use Mailcow\Backup\Service\SettingsService;
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\Context;
-use Mailcow\Dovecot\DovecotModule;
-use Mailcow\MailboxGroup\Repository\GroupRepository;
-use Mailcow\MailboxGroup\Repository\LookupRepository;
-use Mailcow\MailboxGroup\Service\ChangeEffects;
-use Mailcow\MailboxGroup\Service\SharingService;
-use Mailcow\MailboxGroup\Service\TemplateService;
-use Mailcow\Sogo\SogoModule;
-use PDO;
-use Redis;
-
-/**
- * Composition root of the backup with restic: settings, jobs of the runners, snapshots, the export of a domain and the
- * restore of mailboxes and domains. The runner in the restic container reaches the export and the restore actions
- * through data/conf/rspamd/dynmaps/backup.php.
- *
- *   process_get_return(BackupModule::fromGlobals()->api->getSettings());
- *   echo json_encode(BackupModule::create($pdo, Context::system(), $config, $redis)->api->export($domain, $page));  // backup.php
- *
- * $config: timezone (TZ of mailcow, for the schedule), alias_disable_external_domains ($ALIAS_DISABLE_EXTERNAL_DOMAINS),
- * demo_mode ($DEMO_MODE: the API changes nothing).
- * $redis: for DKIM keys, rate limits and the domain map (export and restore).
- */
-class BackupModule
-{
-    public function __construct(
-        public readonly SettingsService $settings,
-        public readonly JobService $jobs,
-        public readonly ExportService $export,
-        public readonly RestoreService $restore,
-        public readonly DomainRestoreService $domainRestore,
-        public readonly BackupApiController $api
-    ) {
-    }
-
-    public static function create(PDO $pdo, Context $context, array $config = [], ?Redis $redis = null): self
-    {
-        $jobRepo = new JobRepository($pdo);
-        $restoreRepo = new RestoreRepository($pdo);
-        $legacy = new LegacyDomain($redis);
-        $settings = new SettingsService(new SettingsRepository($pdo), $context);
-        $jobs = new JobService($jobRepo, new SnapshotRepository($pdo), $settings, $context, (string)($config['timezone'] ?? 'UTC'));
-        $export = new ExportService(new ExportRepository($pdo), $legacy);
-        $sogo = SogoModule::create()->sogo;
-        $dovecot = DovecotModule::create()->dovecot;
-        // the group module without group templates: a restored mailbox keeps the settings of the backup
-        $groupRepo = new GroupRepository($pdo);
-        $lookup = new LookupRepository($pdo);
-        $templates = new TemplateService($lookup, new NoTemplates());
-        $effects = new ChangeEffects($templates, $sogo, $groupRepo, $dovecot);
-        $groups = new GroupMembers($groupRepo, $lookup, $templates, $effects, new SharingService($groupRepo, $dovecot, $sogo, $effects));
-        $restore = new RestoreService($restoreRepo, $jobRepo, $groups, $sogo, $dovecot, $legacy, (bool)($config['alias_disable_external_domains'] ?? false));
-        $domainRestore = new DomainRestoreService($restoreRepo, $jobRepo, $groups, $legacy);
-        return new self($settings, $jobs, $export, $restore, $domainRestore,
-            new BackupApiController($settings, $jobs, $export, $restore, $domainRestore, new ApiResponse(Settings::SECRETS),
-                (bool)($config['demo_mode'] ?? false)));
-    }
-
-    /** Legacy entry point for procedural callers: reads $pdo, $redis, $_SESSION, TZ, $ALIAS_DISABLE_EXTERNAL_DOMAINS and $DEMO_MODE */
-    public static function fromGlobals(?Context $context = null): self
-    {
-        return self::create($GLOBALS['pdo'], $context ?? Context::fromSession($_SESSION ?? []), [
-            'timezone' => getenv('TZ') ?: 'UTC',
-            'alias_disable_external_domains' => ($GLOBALS['ALIAS_DISABLE_EXTERNAL_DOMAINS'] ?? false) === true,
-            'demo_mode' => ($GLOBALS['DEMO_MODE'] ?? false) === true,
-        ],($GLOBALS['redis'] ?? null) instanceof Redis ? $GLOBALS['redis'] : null);
-    }
-}

data/web/inc/lib/mailcow/Backup/Controller/BackupApiController.php

@@ -1,181 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Controller;
-
-use Mailcow\Backup\Model\Job;
-use Mailcow\Backup\Service\DomainRestoreService;
-use Mailcow\Backup\Service\ExportService;
-use Mailcow\Backup\Service\JobService;
-use Mailcow\Backup\Service\RestoreService;
-use Mailcow\Backup\Service\SettingsService;
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\UserError;
-use PDOException;
-
-/**
- * HTTP API adapter for json_api.php (routes backup-settings, backup-status, backup-snapshots, backup-snapshot-content,
- * backup-jobs, backup-job, backup-restore) and the runner's endpoint data/conf/rspamd/dynmaps/backup.php (domains,
- * export pages, restore actions). Messages are logged as ['backup', action, object, data].
- */
-class BackupApiController
-{
-    public function __construct(private SettingsService $settings, private JobService $jobs, private ExportService $export,
-        private RestoreService $restore, private DomainRestoreService $domainRestore, private ApiResponse $response,
-        private bool $demoMode = false)
-    {
-    }
-
-    /** GET backup-settings: secrets as <key>_set; {} for non-admins */
-    public function getSettings(): array
-    {
-        return $this->settings->get()?->toArray() ?? [];
-    }
-
-    /** POST edit/backup-settings */
-    public function editSettings(array $attr): bool
-    {
-        return $this->handle('edit', 'settings', $attr, function (array $log) use ($attr): bool {
-            $this->settings->save($attr);
-            return $this->response->success('backup_settings_saved', $log);
-        });
-    }
-
-    /** GET backup-status */
-    public function getStatus(): array
-    {
-        return $this->jobs->status();
-    }
-
-    /** GET backup-snapshots */
-    public function getSnapshots(): array
-    {
-        return $this->jobs->snapshots();
-    }
-
-    /** GET backup-snapshot-content/<snapshot>[/<domain>]: domains of a snapshot, or the mailboxes of one, with their state now */
-    public function getSnapshotContent(mixed $snapshot, mixed $domain = null): array
-    {
-        return ($domain === null || $domain === '') ? $this->jobs->content($snapshot) : $this->jobs->domainContent($snapshot, $domain);
-    }
-
-    /** GET backup-snapshot-search/<snapshot>/<term>: domains and mailboxes of a snapshot matching the term */
-    public function getSnapshotSearch(mixed $snapshot, mixed $term): array
-    {
-        return $this->jobs->search($snapshot, $term);
-    }
-
-    /** GET backup-jobs[/<count>[/<type>]]: newest first, without logs; count 0 = default */
-    public function getJobs(int $limit = 0, mixed $type = null): array
-    {
-        return array_map(fn(Job $job): array => $this->jobs->view($job), $this->jobs->list($limit, $type));
-    }
-
-    /** GET backup-job/<id>: with its log */
-    public function getJob(mixed $id): array
-    {
-        $id = $this->toId($id);
-        $job = ($id === null) ? null : $this->jobs->get($id);
-        return ($job === null) ? [] : $this->jobs->view($job, true);
-    }
-
-    /** POST add/backup-job: {type: backup|check|prune|init} */
-    public function addJob(array $attr): bool
-    {
-        return $this->handle('add', 'job', $attr, function (array $log) use ($attr): bool {
-            $job = $this->jobs->create($attr);
-            return $this->response->success(['backup_job_added', $job->type, $job->id], $log);
-        });
-    }
-
-    /** POST add/backup-restore: {snapshot, scope, mailboxes | domain, components, mail_target} */
-    public function addRestore(array $attr): bool
-    {
-        return $this->handle('add', 'restore', $attr, function (array $log) use ($attr): bool {
-            $job = $this->jobs->restore($attr);
-            return $this->response->success(['backup_restore_added', $job->id, count($job->mailboxes())], $log);
-        });
-    }
-
-    /** POST delete/backup-job: cancels queued and running jobs, one message per id */
-    public function deleteJobs(array $items): bool
-    {
-        return $this->handle('delete', 'job', ['id' => $items], function (array $log) use ($items): bool {
-            foreach ($items as $id) {
-                try {
-                    $job = $this->jobs->cancel($this->requireId($id));
-                    $this->response->success(['backup_job_cancelled', $job->id], $log);
-                } catch (UserError $e) {
-                    $this->response->error($e, $log);
-                }
-            }
-            return true;
-        });
-    }
-
-    /** backup.php ?domains=1 */
-    public function domains(): array
-    {
-        return ['domains' => $this->export->domains()];
-    }
-
-    /** backup.php ?export=<domain>[&after=<username>] @throws UserError */
-    public function export(string $domain, ?string $after = null): array
-    {
-        return $this->export->export($domain, $after);
-    }
-
-    /**
-     * backup.php POST of the runner, {"result": ...} of
-     *   {"action": "restore_domain", "job": id, "data": <domain_data>}: creates a deleted domain
-     *   {"action": "restore_aliases", "job": id, "aliases": [...]}: a part of the domain's aliases (scope domain)
-     *   {"action": "restore", "job": id, "username": u, "data": <mailbox section>}: one mailbox
-     *
-     * @throws UserError
-     */
-    public function post(array $body): array
-    {
-        return ['result' => match ($body['action'] ?? null) {
-            'restore' => $this->restore->merge($body['job'] ?? null, $body['username'] ?? null, $body['data'] ?? null),
-            'restore_aliases' => $this->restore->mergeAliases($body['job'] ?? null, $body['aliases'] ?? null),
-            'restore_domain' => $this->domainRestore->restore($body['job'] ?? null, $body['data'] ?? null),
-            default => throw new UserError('backup_action_invalid'),
-        }];
-    }
-
-    /**
-     * Runs one write call: $work gets the log entry and returns the result for process_*_return().
-     * A UserError becomes a danger message, a database error stops the call with one `mysql_error` message.
-     * In demo mode no write call runs.
-     */
-    private function handle(string $action, string $object, mixed $data, callable $work): bool
-    {
-        $log = ['backup', $action, $object, $data];
-        try {
-            if ($this->demoMode) {
-                throw new UserError('demo_mode_enabled');
-            }
-            $result = $work($log);
-        } catch (UserError $e) {
-            $result = $this->response->error($e, $log);
-        } catch (PDOException $e) {
-            $result = $this->response->danger(['mysql_error', $e->getMessage()], $log);
-        }
-        $this->response->flush();
-        return $result;
-    }
-
-    /** Integer id from request input ("12", 12); null for anything else */
-    private function toId(mixed $id): ?int
-    {
-        return (is_numeric($id) && (float)$id == (int)(float)$id) ? (int)(float)$id : null;
-    }
-
-    /** @throws AccessDenied for ids that cannot exist */
-    private function requireId(mixed $id): int
-    {
-        return $this->toId($id) ?? throw new AccessDenied();
-    }
-}

data/web/inc/lib/mailcow/Backup/Legacy/LegacyDomain.php

@@ -1,84 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Legacy;
-
-use Redis;
-use RuntimeException;
-
-/**
- * What the legacy code keeps outside the database for domains and mailboxes: DKIM keys, rate limits and the domain
- * map in Redis (functions.dkim/ratelimit/mailbox.inc.php), and the SOGo restart through dockerapi. Redis and docker()
- * sit in protected methods so tests can replace them.
- */
-class LegacyDomain
-{
-    public function __construct(private ?Redis $redis = null)
-    {
-    }
-
-    /** @return ?array{selector: string, pubkey: string, privkey: string} the DKIM key of a domain */
-    public function dkim(string $domain): ?array
-    {
-        $pubkey = $this->redis()->hGet('DKIM_PUB_KEYS', $domain);
-        if (!is_string($pubkey) || $pubkey === '') {
-            return null;
-        }
-        $selector = (string)$this->redis()->hGet('DKIM_SELECTORS', $domain);
-        return ['selector' => $selector, 'pubkey' => $pubkey, 'privkey' => (string)$this->redis()->hGet('DKIM_PRIV_KEYS', $selector . '.' . $domain)];
-    }
-
-    /** Stores a DKIM key like dkim('import') does */
-    public function addDkim(string $domain, string $selector, string $pubkey, string $privkey): void
-    {
-        $this->redis()->hSet('DKIM_PUB_KEYS', $domain, $pubkey);
-        $this->redis()->hSet('DKIM_SELECTORS', $domain, $selector);
-        $this->redis()->hSet('DKIM_PRIV_KEYS', $selector . '.' . $domain, $privkey);
-    }
-
-    /**
-     * @param string[] $objects domains or mailboxes
-     * @return array<string, string> object => rate limit ("10 / 1h") of those with one
-     */
-    public function rateLimits(array $objects): array
-    {
-        if (empty($objects)) {
-            return [];
-        }
-        $values = $this->redis()->hMGet('RL_VALUE', array_values($objects));
-        return array_filter(is_array($values) ? $values : [], fn(mixed $value): bool => is_string($value) && $value !== '');
-    }
-
-    public function setRateLimit(string $object, string $value): void
-    {
-        $this->redis()->hSet('RL_VALUE', $object, $value);
-    }
-
-    /** The domain is local for rspamd (legacy domain and alias domain add) */
-    public function addDomainMap(string $domain): void
-    {
-        $this->redis()->hSet('DOMAIN_MAP', $domain, 1);
-    }
-
-    /**
-     * SOGo reads its domains at start: restart it after a domain was added (like the "restart SOGo" option of the form)
-     *
-     * @return ?string null on success, else the error of dockerapi
-     */
-    public function restartSogo(): ?string
-    {
-        $result = json_decode((string)$this->dockerRestart('sogo-mailcow'), true);
-        return (($result['type'] ?? '') === 'success') ? null : (string)($result['msg'] ?? 'dockerapi did not answer');
-    }
-
-    protected function redis(): Redis
-    {
-        return $this->redis ?? throw new RuntimeException('Redis is not connected');
-    }
-
-    protected function dockerRestart(string $service): mixed
-    {
-        return docker('post', $service, 'restart');
-    }
-}

data/web/inc/lib/mailcow/Backup/Legacy/NoTemplates.php

@@ -1,19 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Legacy;
-
-use Mailcow\MailboxGroup\Legacy\LegacyMailbox;
-
-/**
- * The group module's bridge to the mailbox code for restores: a membership restored from a backup never applies the
- * group's mailbox template (the mailbox gets its settings from the backup).
- */
-class NoTemplates extends LegacyMailbox
-{
-    public function applyTemplate(string $username, string $name, string $template): ?string
-    {
-        return null;
-    }
-}

data/web/inc/lib/mailcow/Backup/Model/Job.php

@@ -1,129 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Model;
-
-/**
- * A job of the backup runners (table `backup_job`): backup, check, prune and init run in the restic container; a
- * restore runs there (step `files`, then the merge through backup.php) and, for mails, IMAP ACLs and Sieve
- * scripts, in the dovecot container (step `mails`). Times are unix timestamps.
- */
-class Job
-{
-    public const TYPES = ['backup', 'check', 'prune', 'init', 'restore'];
-    /** Types an admin starts with add/backup-job */
-    public const MANUAL_TYPES = ['backup', 'check', 'prune', 'init'];
-    public const ACTIVE_STATES = ['queued', 'running'];
-
-    public ?int $id = null;
-    public string $type = 'backup';
-    public string $state = 'queued';
-    public ?string $step = null;
-    /**
-     * restore: snapshot, snapshot_time, scope, domain, mailboxes, components, mail_target, requested_role, create_domain (0|1),
-     * create_mailboxes (missing when requested), acl and allowed_domains (domain admins)
-     */
-    public array $params = [];
-    public ?array $result = null;
-    /** Only loaded for a single job */
-    public ?string $log = null;
-    public ?string $error = null;
-    /** restore: the domain of its mailboxes, so domain admins see their restores */
-    public string $domain = '';
-    public string $requestedBy = '';
-    public ?int $created = null;
-    public ?int $started = null;
-    public ?int $finished = null;
-
-    public function __construct(string $type = 'backup')
-    {
-        $this->type = $type;
-    }
-
-    /** Row of `backup_job` with the times as unix timestamps */
-    public static function fromRow(array $row): self
-    {
-        $job = new self((string)$row['type']);
-        $job->id = (int)$row['id'];
-        $job->state = (string)$row['state'];
-        $job->step = isset($row['step']) ? (string)$row['step'] : null;
-        $params = is_string($row['params'] ?? null) ? json_decode($row['params'], true) : null;
-        $job->params = is_array($params) ? $params : [];
-        $result = is_string($row['result'] ?? null) ? json_decode($row['result'], true) : null;
-        $job->result = is_array($result) ? $result : null;
-        $job->log = isset($row['log']) ? (string)$row['log'] : null;
-        $job->error = isset($row['error']) ? (string)$row['error'] : null;
-        $job->domain = (string)($row['domain'] ?? '');
-        $job->requestedBy = (string)($row['requested_by'] ?? '');
-        foreach (['created', 'started', 'finished'] as $time) {
-            $job->$time = isset($row[$time]) ? (int)$row[$time] : null;
-        }
-        return $job;
-    }
-
-    /** Column values for a new row */
-    public function toRow(): array
-    {
-        return [
-            'type' => $this->type,
-            'state' => $this->state,
-            'step' => $this->step,
-            'params' => json_encode($this->params ?: new \stdClass(), JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
-            'domain' => $this->domain,
-            'requested_by' => $this->requestedBy,
-        ];
-    }
-
-    /** @return string[] mailboxes of a restore */
-    public function mailboxes(): array
-    {
-        return array_values(array_filter((array)($this->params['mailboxes'] ?? []), 'is_string'));
-    }
-
-    /** @return string[] components of a restore */
-    public function components(): array
-    {
-        return array_values(array_filter((array)($this->params['components'] ?? []), 'is_string'));
-    }
-
-    /** @return string[] mailboxes of a restore that did not exist when it was requested */
-    public function createMailboxes(): array
-    {
-        return array_values(array_filter((array)($this->params['create_mailboxes'] ?? []), 'is_string'));
-    }
-
-    /** The restore creates its domain (requested by an admin while the domain did not exist) */
-    public function createsDomain(): bool
-    {
-        return (int)($this->params['create_domain'] ?? 0) === 1;
-    }
-
-    public function isActive(): bool
-    {
-        return in_array($this->state, self::ACTIVE_STATES, true);
-    }
-
-    /** Entry of the API; the log only with $withLog (get/backup-job/<id>) */
-    public function toArray(bool $withLog = false): array
-    {
-        $job = [
-            'id' => $this->id,
-            'type' => $this->type,
-            'state' => $this->state,
-            'step' => $this->step,
-            'params' => $this->params ?: new \stdClass(),
-            'result' => $this->result,
-            'error' => $this->error,
-            'domain' => $this->domain,
-            'requested_by' => $this->requestedBy,
-            'created' => $this->created,
-            'started' => $this->started,
-            'finished' => $this->finished,
-        ];
-        if ($withLog) {
-            $job['log'] = (string)$this->log;
-        }
-        return $job;
-    }
-}

data/web/inc/lib/mailcow/Backup/Model/RestoreRun.php

@@ -1,116 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Model;
-
-use Mailcow\Core\Context;
-
-/**
- * The merge of one mailbox of a restore job: what the backup holds, who asked for the restore and what the merge
- * did per component (added, skipped, errors), plus the caches to refresh afterwards.
- */
-class RestoreRun
-{
-    /** @var array<string, array{added: int, skipped: int, errors: string[]}> */
-    public array $results = [];
-    /** Mailboxes whose SOGo identities (aliases, send-as addresses) changed */
-    public array $sogoUsers = [];
-    /** Mailboxes whose userdb or passdb values changed (Dovecot auth cache) */
-    public array $authUsers = [];
-
-    /**
-     * @param array $mailbox  the mailbox row as it is now
-     * @param array $params   params of the restore job (requested_role, acl of a domain admin, scope, ...)
-     * @param array $data     the mailbox's section of the domain export in the snapshot
-     * @param string[] $domains domains whose aliases and send-as addresses the restore may change, [] = all (admins)
-     * @param string $requestedBy who requested the restore
-     */
-    /** The restore created the mailbox (its own alias gets the forwarding of the backup) */
-    public bool $created = false;
-
-    public function __construct(
-        public readonly string $username,
-        public readonly string $domain,
-        public readonly array $mailbox,
-        public readonly array $params,
-        public readonly array $data,
-        public readonly array $domains = [],
-        public readonly string $requestedBy = ''
-    ) {
-    }
-
-    /**
-     * The caller the group module checks: an admin, or the domain admin who requested the restore with his ACL of then
-     * and his domains now
-     */
-    public function requester(): Context
-    {
-        if ($this->byAdmin()) {
-            return new Context('admin', 'backup', [], true);
-        }
-        return new Context('domainadmin', $this->requestedBy, array_map(fn(mixed $flag): string => (string)(int)$flag, (array)($this->params['acl'] ?? [])));
-    }
-
-    public function start(string $component): void
-    {
-        $this->results[$component] = ['added' => 0, 'skipped' => 0, 'errors' => []];
-    }
-
-    public function added(string $component, int $count = 1): void
-    {
-        $this->results[$component]['added'] += $count;
-    }
-
-    public function skipped(string $component, int $count = 1): void
-    {
-        $this->results[$component]['skipped'] += $count;
-    }
-
-    public function error(string $component, string $error): void
-    {
-        $this->results[$component]['errors'][] = $error;
-    }
-
-    /** The part of a component was rolled back: nothing of it counts */
-    public function failed(string $component, string $error): void
-    {
-        $this->start($component);
-        $this->error($component, $error);
-    }
-
-    /** The restore was requested by an admin (domain admins are limited by their ACL and the job's domain) */
-    public function byAdmin(): bool
-    {
-        return ($this->params['requested_role'] ?? '') === 'admin';
-    }
-
-    /** Admins, or a domain admin with this ACL when he requested the restore */
-    public function may(string $acl): bool
-    {
-        return $this->byAdmin() || (int)($this->params['acl'][$acl] ?? 0) === 1;
-    }
-
-    public function mayChangeDomain(string $domain): bool
-    {
-        return $this->domains === [] || in_array(strtolower($domain), $this->domains, true);
-    }
-
-    /** Rows of a list in the backup (sieve_filters, tfa, ...) */
-    public function rows(string $key): array
-    {
-        return array_values(array_filter((array)($this->data[$key] ?? []), 'is_array'));
-    }
-
-    public function sogo(string ...$usernames): void
-    {
-        array_push($this->sogoUsers, ...$usernames);
-        $this->sogoUsers = array_values(array_unique($this->sogoUsers));
-    }
-
-    public function auth(string ...$usernames): void
-    {
-        array_push($this->authUsers, ...$usernames);
-        $this->authUsers = array_values(array_unique($this->authUsers));
-    }
-}

data/web/inc/lib/mailcow/Backup/Model/Settings.php

@@ -1,189 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Model;
-
-/**
- * Backup settings (table `backup_setting`, name/value pairs) over their defaults. Secrets are stored sealed for the
- * runner (`sealed:v1:<base64>`) and never leave toArray(): the API gets `<key>_set` instead. The state_* values, the
- * SSH public key and the SFTP host key are written by the runner in the restic container and cannot be set through
- * the API (PHP writes state_target_changed when the target changes).
- */
-class Settings
-{
-    public const DEFAULTS = [
-        'enabled' => '0',
-        'target' => 's3',
-        's3_endpoint' => 'https://s3.amazonaws.com',
-        's3_bucket' => '',
-        's3_path' => '',
-        's3_region' => '',
-        's3_access_key' => '',
-        's3_secret_key' => '',
-        's3_path_style' => '0',
-        'sftp_host' => '',
-        'sftp_port' => '22',
-        'sftp_user' => '',
-        'sftp_path' => '',
-        'sftp_public_key' => '',
-        'sftp_host_key' => '',
-        'rest_url' => '',
-        'rest_user' => '',
-        'rest_password' => '',
-        'local_path' => 'mailcow',
-        'password' => '',
-        'password_confirmed' => '0',
-        // JSON list of the scheduled backups: {"days": [weekdays, 0 = Sunday], "time": "HH:MM"}
-        'schedule' => '[{"days":[0,1,2,3,4,5,6],"time":"01:00"}]',
-        'keep_hourly' => '0',
-        'keep_daily' => '7',
-        'keep_weekly' => '4',
-        'keep_monthly' => '12',
-        'keep_yearly' => '0',
-        'prune_weekday' => '0',
-        'max_unused' => '10%',
-        'check_subset' => '30',
-        'include_index' => '1',
-        'include_fts' => '1',
-        'limit_upload' => '0',
-        'notify_email' => '',
-        'state_initialized' => '0',
-        'state_runner_seen' => '',
-        'state_last_backup' => '',
-        'state_last_success' => '',
-        'state_repo_size' => '',
-        'state_repo_stats' => '',
-        'state_schedule_since' => '',
-        'state_schedule' => '',
-        // public key (base64, X25519) the secrets are sealed with; the private key stays in the restic container
-        'state_secrets_pubkey' => '',
-        'state_secrets_error' => '',
-        // JSON {old_target, old_addresses, changed, changed_by, time}: the runner mails the old addresses and clears it
-        'state_target_changed' => '',
-        // RESTIC_BACKUP_MODE of mailcow.conf as the runner sees it
-        'state_backup_mode' => '',
-    ];
-    public const SECRETS = ['password', 's3_secret_key', 'rest_password'];
-    /** Written by the runner only */
-    public const RUNNER = ['sftp_public_key', 'sftp_host_key', 'state_initialized', 'state_runner_seen', 'state_last_backup',
-        'state_last_success', 'state_repo_size', 'state_repo_stats', 'state_schedule_since', 'state_schedule', 'state_secrets_pubkey',
-        'state_secrets_error', 'state_target_changed', 'state_backup_mode'];
-    /** Prefix of a sealed secret (sodium_crypto_box_seal with state_secrets_pubkey) */
-    public const SEALED = 'sealed:v1:';
-    public const TARGETS = ['s3', 'sftp', 'rest', 'local'];
-    /** The connection of each target: a change (or another target or repository password) resets state_initialized */
-    public const CONNECTION = [
-        's3' => ['s3_endpoint', 's3_bucket', 's3_path', 's3_region', 's3_access_key', 's3_secret_key', 's3_path_style'],
-        'sftp' => ['sftp_host', 'sftp_port', 'sftp_user', 'sftp_path'],
-        'rest' => ['rest_url', 'rest_user', 'rest_password'],
-        'local' => ['local_path'],
-    ];
-    /** Values a target cannot work without (besides the repository password) */
-    public const REQUIRED = [
-        's3' => ['s3_endpoint', 's3_bucket', 's3_access_key', 's3_secret_key'],
-        'sftp' => ['sftp_host', 'sftp_port', 'sftp_user', 'sftp_path'],
-        'rest' => ['rest_url'],
-        'local' => ['local_path'],
-    ];
-    /** Returned as numbers by the API ('' = not known yet: null; prune_weekday '' = never stays '') */
-    private const NUMBERS = ['enabled', 's3_path_style', 'sftp_port', 'password_confirmed', 'keep_hourly', 'keep_daily', 'keep_weekly', 'keep_monthly',
-        'keep_yearly', 'prune_weekday', 'check_subset', 'include_index', 'include_fts', 'limit_upload', 'state_initialized',
-        'state_runner_seen', 'state_last_backup', 'state_last_success', 'state_repo_size', 'state_schedule_since'];
-    private const JSON = ['schedule', 'state_repo_stats', 'state_target_changed'];
-
-    /** @param array<string, string> $values every key of DEFAULTS */
-    public function __construct(public array $values = self::DEFAULTS)
-    {
-    }
-
-    /** Stored values (name => value) over the defaults; unknown names are dropped */
-    public static function fromStored(array $stored): self
-    {
-        $values = self::DEFAULTS;
-        foreach ($stored as $name => $value) {
-            if (array_key_exists($name, $values)) {
-                $values[$name] = (string)$value;
-            }
-        }
-        return new self($values);
-    }
-
-    public function get(string $name): string
-    {
-        return $this->values[$name] ?? '';
-    }
-
-    /** The target has its required values and the repository password is set */
-    public function configured(): bool
-    {
-        $required = self::REQUIRED[$this->get('target')] ?? null;
-        if ($required === null || $this->get('password') === '') {
-            return false;
-        }
-        foreach ($required as $name) {
-            if ($this->get($name) === '') {
-                return false;
-            }
-        }
-        return true;
-    }
-
-    /**
-     * The schedule as [weekday (0 = Sunday), hour, minute] slots, sorted and unique
-     *
-     * @return array<int, array{int, int, int}>
-     */
-    public function scheduleSlots(): array
-    {
-        $entries = json_decode($this->get('schedule'), true);
-        $slots = [];
-        foreach (is_array($entries) ? $entries : [] as $entry) {
-            if (!is_array($entry) || !preg_match('/^([01]?\d|2[0-3]):([0-5]\d)$/', (string)($entry['time'] ?? ''), $time) || !is_array($entry['days'] ?? null)) {
-                continue;
-            }
-            foreach ($entry['days'] as $day) {
-                if (is_int($day) && $day >= 0 && $day <= 6) {
-                    $slots[] = $day * 1440 + (int)$time[1] * 60 + (int)$time[2];
-                }
-            }
-        }
-        $slots = array_unique($slots);
-        sort($slots);
-        return array_map(fn(int $slot): array => [intdiv($slot, 1440), intdiv($slot % 1440, 60), $slot % 60], $slots);
-    }
-
-    /** The longest time between two scheduled backups of a week in seconds, null without a schedule */
-    public function scheduleGap(): ?int
-    {
-        $slots = array_map(fn(array $slot): int => $slot[0] * 86400 + $slot[1] * 3600 + $slot[2] * 60, $this->scheduleSlots());
-        if (empty($slots)) {
-            return null;
-        }
-        $gaps = [$slots[0] + 7 * 86400 - end($slots)];
-        for ($i = 1; $i < count($slots); $i++) {
-            $gaps[] = $slots[$i] - $slots[$i - 1];
-        }
-        return max($gaps);
-    }
-
-    /** Entry of get/backup-settings: every key, secrets as <key>_set, plus `configured` */
-    public function toArray(): array
-    {
-        $settings = [];
-        foreach ($this->values as $name => $value) {
-            if (in_array($name, self::SECRETS, true)) {
-                $settings[$name . '_set'] = (int)($value !== '');
-            } elseif (in_array($name, self::JSON, true)) {
-                $decoded = json_decode($value, true);
-                $settings[$name] = is_array($decoded) ? $decoded : null;
-            } elseif (in_array($name, self::NUMBERS, true)) {
-                $settings[$name] = ($value === '') ? (($name === 'prune_weekday') ? '' : null) : (int)$value;
-            } else {
-                $settings[$name] = $value;
-            }
-        }
-        $settings['configured'] = (int)$this->configured();
-        return $settings;
-    }
-}

data/web/inc/lib/mailcow/Backup/Model/Snapshot.php

@@ -1,142 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Model;
-
-/**
- * A snapshot of the repository (table `backup_snapshot`, the runner's cache of `restic snapshots --json`).
- */
-class Snapshot
-{
-    public string $id = '';
-    public string $shortId = '';
-    /** unix timestamp */
-    public int $time = 0;
-    public string $hostname = '';
-    /** @var string[] */
-    public array $tags = [];
-    /** restic's summary of the backup (data_added, files_new, ...) */
-    public ?array $summary = null;
-    /** @var ?string[] domains of its manifest, null for snapshots not made by mailcow */
-    public ?array $domains = null;
-    /** @var array<string, ?int> domain => its creation time (unix) when the snapshot was made, from the snapshot row */
-    public array $created = [];
-    /**
-     * @var ?array<string, array{created: ?int, mailboxes: array<string, string>}> per domain its creation time (unix, null
-     *      when not recorded) and username => name of its mailboxes; null = not known (snapshots this installation did
-     *      not make itself, e.g. of another server in the same repository)
-     */
-    public ?array $content = null;
-
-    /** Row of `backup_snapshot` with the time as unix timestamp */
-    public static function fromRow(array $row): self
-    {
-        $snapshot = new self();
-        $snapshot->id = (string)$row['id'];
-        $snapshot->shortId = (string)$row['short_id'];
-        $snapshot->time = (int)$row['time'];
-        $snapshot->hostname = (string)($row['hostname'] ?? '');
-        $tags = (string)($row['tags'] ?? '');
-        $decoded = json_decode($tags, true);
-        $tags = is_array($decoded) ? $decoded : explode(',', $tags);
-        $snapshot->tags = array_values(array_filter(array_map(fn(mixed $tag): string => is_scalar($tag) ? trim((string)$tag) : '', $tags), 'strlen'));
-        $summary = is_string($row['summary'] ?? null) ? json_decode($row['summary'], true) : null;
-        $snapshot->summary = is_array($summary) ? $summary : null;
-        // {domain: creation time}: lists of snapshots need no content
-        $domains = is_string($row['domains'] ?? null) ? json_decode($row['domains'], true) : null;
-        if (is_array($domains)) {
-            $snapshot->domains = [];
-            foreach ($domains as $domain => $created) {
-                $snapshot->domains[] = (string)$domain;
-                $snapshot->created[strtolower((string)$domain)] = is_numeric($created) ? (int)$created : null;
-            }
-        }
-        if (is_string($row['content'] ?? null) && $row['content'] !== '') {
-            $snapshot->content = self::decodeContent($row['content']);
-        }
-        return $snapshot;
-    }
-
-    /**
-     * `content` (gzip JSON) written by the runner: {"domains": {"<domain>": {"created": <unix>, "mailboxes": {"<username>": "<name>"}}}}.
-     * Keys lower case; null when broken.
-     */
-    public static function decodeContent(string $gzip): ?array
-    {
-        $json = @gzdecode($gzip);
-        $data = is_string($json) ? json_decode($json, true) : null;
-        if (!is_array($data) || !is_array($data['domains'] ?? null)) {
-            return null;
-        }
-        $content = [];
-        foreach ($data['domains'] as $domain => $entry) {
-            $mailboxes = is_array($entry) ? ($entry['mailboxes'] ?? []) : [];
-            $created = is_numeric($entry['created'] ?? null) ? (int)$entry['created'] : null;
-            $names = [];
-            foreach (is_array($mailboxes) ? $mailboxes : [] as $username => $name) {
-                $names[strtolower((string)$username)] = is_scalar($name) ? (string)$name : '';
-            }
-            $content[strtolower((string)$domain)] = ['created' => $created, 'mailboxes' => $names];
-        }
-        return $content;
-    }
-
-    /** Domains of the snapshot: those of its content and of its manifest (an export may have failed) */
-    public function domainList(): array
-    {
-        return array_values(array_unique(array_merge(array_map('strval', array_keys($this->content ?? [])), array_map('strtolower', $this->domains ?? []))));
-    }
-
-    /** @return ?array<string, string> username => name of the domain's mailboxes, null when the content does not list the domain */
-    public function mailboxesOf(string $domain): ?array
-    {
-        return $this->content[strtolower($domain)]['mailboxes'] ?? null;
-    }
-
-    /** Creation time of the domain when the snapshot was made, null when not recorded */
-    public function createdOf(string $domain): ?int
-    {
-        return $this->content[strtolower($domain)]['created'] ?? $this->created[strtolower($domain)] ?? null;
-    }
-
-    public function hasDomain(string $domain): bool
-    {
-        return in_array(strtolower($domain), $this->domainList(), true);
-    }
-
-    /**
-     * RESTIC_BACKUP_MODE of the backup (tag mode:<mode>, none for all): all, data (without the mailcow directory) or
-     * config (without mails)
-     */
-    public function mode(): string
-    {
-        foreach ($this->tags as $tag) {
-            if (str_starts_with($tag, 'mode:')) {
-                return substr($tag, 5);
-            }
-        }
-        return 'all';
-    }
-
-    /** Snapshots not made by mailcow know neither domains nor mailboxes */
-    public function knowsDomains(): bool
-    {
-        return $this->content !== null || $this->domains !== null;
-    }
-
-    /** Entry of the API for admins */
-    public function toArray(): array
-    {
-        return [
-            'id' => $this->id,
-            'short_id' => $this->shortId,
-            'time' => $this->time,
-            'hostname' => $this->hostname,
-            'tags' => $this->tags,
-            'summary' => $this->summary,
-            'domains' => $this->domains,
-            'mode' => $this->mode(),
-        ];
-    }
-}

data/web/inc/lib/mailcow/Backup/Repository/ExportRepository.php

@@ -1,169 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Repository;
-
-use Mailcow\Core\Repository;
-use PDO;
-
-/**
- * Reads what the export of a domain holds: the domain with its settings, its aliases and a page of its mailboxes with
- * every row that belongs to a mailbox. Rows are read with all their columns (SELECT *).
- */
-class ExportRepository extends Repository
-{
-    /** @return string[] all domains */
-    public function domains(): array
-    {
-        return $this->query("SELECT `domain` FROM `domain` ORDER BY `domain`")->fetchAll(PDO::FETCH_COLUMN);
-    }
-
-    public function dbVersion(): string
-    {
-        return (string)$this->query("SELECT `version` FROM `versions` WHERE `application` = 'db_schema'")->fetchColumn();
-    }
-
-    /** @return array[] rows of a table matching column => value (table and column names from code) */
-    public function rows(string $table, string $column, string $value): array
-    {
-        self::identifier($table);
-        self::identifier($column);
-        return $this->query("SELECT * FROM `$table` WHERE `$column` = :value", ['value' => $value])->fetchAll();
-    }
-
-    /** @return string[] usernames of the domain's mailboxes */
-    public function usernames(string $domain): array
-    {
-        return $this->query("SELECT `username` FROM `mailbox` WHERE `domain` = :domain", ['domain' => $domain])->fetchAll(PDO::FETCH_COLUMN);
-    }
-
-    /** @return array[] the domain's mailbox rows after $after in username order (keyset paging), at most $limit */
-    public function mailboxesAfter(string $domain, string $after, int $limit): array
-    {
-        return $this->query("SELECT * FROM `mailbox` WHERE `domain` = :domain AND `username` > :after ORDER BY `username` LIMIT " . max(1, $limit),
-            ['domain' => $domain, 'after' => $after])->fetchAll();
-    }
-
-    /** @return array[] alias rows of the domain and its alias domains */
-    public function aliases(string $domain): array
-    {
-        return $this->query("SELECT * FROM `alias` WHERE `domain` = :domain
-            OR `domain` IN (SELECT `alias_domain` FROM `alias_domain` WHERE `target_domain` = :target) ORDER BY `id`",
-            ['domain' => $domain, 'target' => $domain])->fetchAll();
-    }
-
-    /** @return array[] alias rows of any domain whose goto may name a mailbox of the domain (the caller checks) */
-    public function aliasesPointingTo(string $domain): array
-    {
-        return $this->query("SELECT * FROM `alias` WHERE `goto` LIKE :pattern ORDER BY `id`",
-            ['pattern' => '%@' . addcslashes($domain, '%_\\') . '%'])->fetchAll();
-    }
-
-    /**
-     * @param string $table  table with rows of mailboxes
-     * @param string $column its column holding the username
-     * @param string[] $usernames
-     * @return array[] rows of these mailboxes
-     */
-    public function rowsOfMailboxes(string $table, string $column, array $usernames): array
-    {
-        self::identifier($table);
-        self::identifier($column);
-        if (empty($usernames)) {
-            return [];
-        }
-        [$in, $params] = self::in($usernames);
-        return $this->query("SELECT * FROM `$table` WHERE `$column` IN ($in)", $params)->fetchAll();
-    }
-
-    /** @return array[] username, id, name, domain, source of the group memberships of these mailboxes */
-    public function groupMemberships(array $usernames): array
-    {
-        if (empty($usernames)) {
-            return [];
-        }
-        [$in, $params] = self::in($usernames);
-        return $this->query("SELECT `gm`.`username`, `g`.`id`, `g`.`name`, `g`.`domain`, `gm`.`source` FROM `mailbox_group_member` `gm`
-            JOIN `mailbox_group` `g` ON `g`.`id` = `gm`.`group_id` WHERE `gm`.`username` IN ($in) ORDER BY `g`.`id`", $params)->fetchAll();
-    }
-
-    /**
-     * @param string $table  mail_policy or retention_rule
-     * @param string $member its member table
-     * @param string $key    the member table's column with the id
-     * @return array[] username, id, name, domain of the direct assignments of these mailboxes
-     */
-    public function assignments(string $table, string $member, string $key, array $usernames): array
-    {
-        self::identifier($table);
-        self::identifier($member);
-        self::identifier($key);
-        if (empty($usernames)) {
-            return [];
-        }
-        [$in, $params] = self::in($usernames);
-        return $this->query("SELECT `a`.`username`, `p`.`id`, `p`.`name`, `p`.`domain` FROM `$member` `a`
-            JOIN `$table` `p` ON `p`.`id` = `a`.`$key` WHERE `a`.`username` IN ($in) ORDER BY `p`.`id`", $params)->fetchAll();
-    }
-
-    /** @return array[] sync jobs of these mailboxes (without their log) with host1, port1 and name of the source */
-    public function syncJobs(array $usernames): array
-    {
-        if (empty($usernames)) {
-            return [];
-        }
-        [$in, $params] = self::in($usernames);
-        $rows = $this->query("SELECT `i`.*, `s`.`host1` AS `source_host1`, `s`.`port1` AS `source_port1`, `s`.`name` AS `source_name`
-            FROM `imapsync` `i` LEFT JOIN `imapsync_source` `s` ON `s`.`id` = `i`.`source_id` WHERE `i`.`user2` IN ($in) ORDER BY `i`.`id`", $params)->fetchAll();
-        foreach ($rows as &$row) {
-            unset($row['returned_text']);
-        }
-        return $rows;
-    }
-
-    /** @return array[] BCC maps of the whole domain (local_dest "@domain", older rows "domain") */
-    public function domainBccMaps(string $domain): array
-    {
-        return $this->query("SELECT * FROM `bcc_maps` WHERE `local_dest` IN (:domain, :at)", ['domain' => $domain, 'at' => '@' . $domain])->fetchAll();
-    }
-
-    /**
-     * @param string $table  mail_policy or retention_rule
-     * @param string $member its member table
-     * @param string $key    the member table's column with the id
-     * @return array[] rows of the domain, each with `group_members`: the names of the groups it is assigned to
-     */
-    public function domainRules(string $table, string $member, string $key, string $domain): array
-    {
-        self::identifier($table);
-        self::identifier($member);
-        self::identifier($key);
-        $rows = $this->query("SELECT * FROM `$table` WHERE `domain` = :domain ORDER BY `id`", ['domain' => $domain])->fetchAll();
-        foreach ($rows as &$row) {
-            $row['group_members'] = $this->query("SELECT `g`.`name` FROM `$member` `a` JOIN `mailbox_group` `g` ON `g`.`id` = `a`.`group_id`
-                WHERE `a`.`$key` = :id ORDER BY `g`.`name`", ['id' => $row['id']])->fetchAll(PDO::FETCH_COLUMN);
-        }
-        return $rows;
-    }
-
-    /** @return array{0: string, 1: array} placeholders and parameters of an IN list */
-    private static function in(array $values): array
-    {
-        $names = [];
-        $params = [];
-        foreach (array_values($values) as $i => $value) {
-            $names[] = ':in' . $i;
-            $params['in' . $i] = $value;
-        }
-        return [implode(', ', $names), $params];
-    }
-
-    /** Table and column names come from code only; this keeps it that way */
-    private static function identifier(string $name): void
-    {
-        if (!preg_match('/^[a-z_0-9]+$/i', $name)) {
-            throw new \InvalidArgumentException('invalid identifier');
-        }
-    }
-}

data/web/inc/lib/mailcow/Backup/Repository/JobRepository.php

@@ -1,176 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Repository;
-
-use Mailcow\Backup\Model\Job;
-use Mailcow\Core\Context;
-use Mailcow\Core\Repository;
-use Mailcow\Core\UserError;
-use PDO;
-
-/**
- * Table `backup_job` (the runners claim, log and finish the jobs) and the lookups a new restore job needs.
- */
-class JobRepository extends Repository
-{
-    private const COLUMNS = "`id`, `type`, `state`, `step`, `params`, `result`, `error`, `domain`, `requested_by`,
-        UNIX_TIMESTAMP(`created`) AS `created`, UNIX_TIMESTAMP(`started`) AS `started`, UNIX_TIMESTAMP(`finished`) AS `finished`";
-    /** Serializes the check for active jobs and the insert of a new one */
-    private const LOCK = 'mailcow_backup_job';
-
-    public function find(int $id, bool $withLog = false): ?Job
-    {
-        $row = $this->query("SELECT " . self::COLUMNS . ($withLog ? ", `log`" : "") . " FROM `backup_job` WHERE `id` = :id", ['id' => $id])->fetch();
-        return is_array($row) ? Job::fromRow($row) : null;
-    }
-
-    /**
-     * @param ?string $type only jobs of this type
-     * @return Job[] newest first: all jobs for admins, the restores of their domains (since the domain was created) for
-     *         domain admins
-     */
-    public function visible(Context $context, int $limit, ?string $type = null): array
-    {
-        if ($context->hasAdminRights()) {
-            [$where, $params] = ['1=1', []];
-        } elseif ($context->isDomainAdmin()) {
-            // restores of their domains, requested since the domain was created (not those of a former domain of that name)
-            [$where, $params] = ["`type` = 'restore' AND `domain` IN (SELECT `domain` FROM `domain_admins` WHERE `active` = '1' AND `username` = :da)
-              AND `created` >= (SELECT `d`.`created` FROM `domain` `d` WHERE `d`.`domain` = `backup_job`.`domain`)", ['da' => $context->username]];
-        } else {
-            return [];
-        }
-        if ($type !== null) {
-            $where .= " AND `type` = :type";
-            $params['type'] = $type;
-        }
-        return array_map(fn(array $row): Job => Job::fromRow($row),
-            $this->query("SELECT " . self::COLUMNS . " FROM `backup_job` WHERE $where ORDER BY `id` DESC LIMIT " . max(1, $limit), $params)->fetchAll());
-    }
-
-    /** @return Job[] queued and running jobs */
-    public function active(): array
-    {
-        return array_map(fn(array $row): Job => Job::fromRow($row),
-            $this->query("SELECT " . self::COLUMNS . " FROM `backup_job` WHERE `state` IN ('queued', 'running') ORDER BY `id`")->fetchAll());
-    }
-
-    /** @return array<string, Job> type => its newest job */
-    public function latestOfEachType(): array
-    {
-        $jobs = [];
-        foreach ($this->query("SELECT " . self::COLUMNS . " FROM `backup_job` WHERE `id` IN (SELECT MAX(`id`) FROM `backup_job` GROUP BY `type`)")->fetchAll() as $row) {
-            $jobs[(string)$row['type']] = Job::fromRow($row);
-        }
-        return $jobs;
-    }
-
-    /** A backup job was created by the schedule since this unix time */
-    public function scheduledSince(int $since): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `backup_job` WHERE `type` = 'backup' AND `requested_by` = 'schedule' AND `created` >= FROM_UNIXTIME(:since) LIMIT 1",
-            ['since' => $since])->fetchColumn();
-    }
-
-    /**
-     * Inserts the job after $check(active jobs) passed; $check throws a UserError on a conflict. A named lock keeps two
-     * requests from queueing the same work at once.
-     *
-     * @param callable(Job[]): void $check
-     * @throws UserError
-     */
-    public function insertExclusive(Job $job, callable $check): int
-    {
-        if ((int)$this->query("SELECT GET_LOCK('" . self::LOCK . "', 10)")->fetchColumn() !== 1) {
-            throw new UserError('backup_job_active', [$job->type]);
-        }
-        try {
-            $check($this->active());
-            $job->id = $this->insertRow('backup_job', $job->toRow());
-            return $job->id;
-        } finally {
-            $this->query("SELECT RELEASE_LOCK('" . self::LOCK . "')");
-        }
-    }
-
-    /**
-     * Cancels a queued or running job; the restic runner stops at its next check. A restore in the dovecot step
-     * (mails, mails_running) cannot be cancelled. False when the job finished or reached that step meanwhile.
-     */
-    public function cancel(int $id): bool
-    {
-        return $this->query("UPDATE `backup_job` SET `state` = 'cancelled', `finished` = NOW() WHERE `id` = :id
-            AND `state` IN ('queued', 'running') AND NOT (`type` = 'restore' AND COALESCE(`step`, '') IN ('mails', 'mails_running'))",
-            ['id' => $id])->rowCount() > 0;
-    }
-
-    public function isDomain(string $domain): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `domain` WHERE `domain` = :domain", ['domain' => $domain])->fetchColumn();
-    }
-
-    /** Domain of a mailbox, null if there is none */
-    public function mailboxDomain(string $username): ?string
-    {
-        $domain = $this->query("SELECT `domain` FROM `mailbox` WHERE `username` = :username", ['username' => $username])->fetchColumn();
-        return is_string($domain) ? strtolower($domain) : null;
-    }
-
-    /** Unix time the domain was created, null if it does not exist */
-    public function domainCreated(string $domain): ?int
-    {
-        $created = $this->query("SELECT UNIX_TIMESTAMP(`created`) FROM `domain` WHERE `domain` = :domain", ['domain' => $domain])->fetchColumn();
-        return ($created === false || $created === null) ? null : (int)$created;
-    }
-
-    /**
-     * @param string[] $domains
-     * @return array{0: array<string, true>, 1: array<string, array<string, true>>} the domains that exist, and per domain
-     *         the usernames of its mailboxes; one query each per 500 domains
-     */
-    public function domainsAndUsernames(array $domains): array
-    {
-        $existing = [];
-        $usernames = [];
-        foreach (array_chunk(array_values(array_unique($domains)), 500) as $chunk) {
-            $params = [];
-            foreach ($chunk as $i => $domain) {
-                $params['d' . $i] = $domain;
-            }
-            $in = implode(', ', array_map(fn(string $name): string => ':' . $name, array_keys($params)));
-            foreach ($this->query("SELECT `domain` FROM `domain` WHERE `domain` IN ($in)", $params)->fetchAll(PDO::FETCH_COLUMN) as $domain) {
-                $existing[strtolower((string)$domain)] = true;
-            }
-            foreach ($this->query("SELECT `domain`, `username` FROM `mailbox` WHERE `domain` IN ($in)", $params)->fetchAll() as $row) {
-                $usernames[strtolower((string)$row['domain'])][strtolower((string)$row['username'])] = true;
-            }
-        }
-        return [$existing, $usernames];
-    }
-
-    /** @return array<string, string> username => name of the mailboxes of a domain */
-    public function mailboxNames(string $domain): array
-    {
-        $names = [];
-        foreach ($this->query("SELECT `username`, `name` FROM `mailbox` WHERE `domain` = :domain ORDER BY `username`", ['domain' => $domain])->fetchAll() as $row) {
-            $names[strtolower((string)$row['username'])] = (string)$row['name'];
-        }
-        return $names;
-    }
-
-    /** @return string[] all mailboxes of a domain */
-    public function mailboxesOf(string $domain): array
-    {
-        return array_map('strtolower', $this->query("SELECT `username` FROM `mailbox` WHERE `domain` = :domain ORDER BY `username`",
-            ['domain' => $domain])->fetchAll(PDO::FETCH_COLUMN));
-    }
-
-    /** @return string[] alias domains of a domain */
-    public function aliasDomainsOf(string $domain): array
-    {
-        return array_map('strtolower', $this->query("SELECT `alias_domain` FROM `alias_domain` WHERE `target_domain` = :domain",
-            ['domain' => $domain])->fetchAll(PDO::FETCH_COLUMN));
-    }
-}

data/web/inc/lib/mailcow/Backup/Repository/RestoreRepository.php

@@ -1,237 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Repository;
-
-use Mailcow\Core\Repository;
-use PDO;
-use Throwable;
-
-/**
- * Reads and writes of the restore merge. Rows of a backup may come from an older schema, so inserts and updates only
- * use the columns the table has now. Table and column names come from code, never from the backup.
- */
-class RestoreRepository extends Repository
-{
-    /** @var array<string, string[]> table => columns */
-    private array $columns = [];
-
-    /** Runs $work in one transaction */
-    public function inTransaction(callable $work): mixed
-    {
-        return $this->transaction($work);
-    }
-
-    /** Runs $work in a savepoint of the running transaction: an error undoes only its writes and is thrown on */
-    public function savepoint(string $name, callable $work): mixed
-    {
-        self::identifier($name);
-        $this->pdo->exec("SAVEPOINT `$name`");
-        try {
-            $result = $work();
-            $this->pdo->exec("RELEASE SAVEPOINT `$name`");
-            return $result;
-        } catch (Throwable $e) {
-            $this->pdo->exec("ROLLBACK TO SAVEPOINT `$name`");
-            throw $e;
-        }
-    }
-
-    /**
-     * Runs $work while holding a named lock, so a retried request of the runner waits for the first one instead of
-     * merging the same mailbox twice; null when the lock was not free within $timeout seconds
-     */
-    public function locked(string $key, callable $work, int $timeout = 120): mixed
-    {
-        $name = 'mailcow_backup_' . md5($key);
-        if ((int)$this->query("SELECT GET_LOCK(:name, " . max(0, $timeout) . ")", ['name' => $name])->fetchColumn() !== 1) {
-            return null;
-        }
-        try {
-            return $work();
-        } finally {
-            $this->query("SELECT RELEASE_LOCK(:name)", ['name' => $name]);
-        }
-    }
-
-    /** Quota rows of a new mailbox (like the legacy mailbox add) */
-    public function resetQuota(string $username): void
-    {
-        foreach (['quota2', 'quota2replica'] as $table) {
-            $this->query("INSERT INTO `$table` (`username`, `bytes`, `messages`) VALUES (:username, 0, 0)
-              ON DUPLICATE KEY UPDATE `bytes` = 0, `messages` = 0", ['username' => $username]);
-        }
-    }
-
-    /** Regular mailboxes of a domain (the domain's mailbox limit counts these) */
-    public function mailboxCount(string $domain): int
-    {
-        return (int)$this->query("SELECT COUNT(*) FROM `mailbox` WHERE `domain` = :domain AND `kind` = ''", ['domain' => $domain])->fetchColumn();
-    }
-
-    /** External send-as rights for a domain that is local now (legacy domain and alias domain add) */
-    public function dropExternalSenderAcl(string $domain): void
-    {
-        $this->query("DELETE FROM `sender_acl` WHERE `external` = 1 AND `send_as` LIKE :domain", ['domain' => '%@' . addcslashes($domain, '%_\\')]);
-    }
-
-    /** @return string[] the columns of a table in the current schema */
-    public function columns(string $table): array
-    {
-        self::identifier($table);
-        return $this->columns[$table] ??= $this->query("SELECT `COLUMN_NAME` FROM `information_schema`.`COLUMNS`
-            WHERE `TABLE_SCHEMA` = DATABASE() AND `TABLE_NAME` = :table ORDER BY `ORDINAL_POSITION`", ['table' => $table])->fetchAll(PDO::FETCH_COLUMN);
-    }
-
-    /** @return array[] rows of a table matching every condition (column => value) */
-    public function rows(string $table, array $where): array
-    {
-        [$condition, $params] = $this->where($table, $where);
-        return $this->query("SELECT * FROM `$table` WHERE $condition", $params)->fetchAll();
-    }
-
-    /** Inserts a row with the values of the columns the table has now; returns the new id (0 without one) */
-    public function insert(string $table, array $values): int
-    {
-        return $this->insertRow($table, $this->existing($table, $values));
-    }
-
-    /** Updates the rows matching $where with the values of the columns the table has now */
-    public function update(string $table, array $where, array $values): int
-    {
-        $values = $this->existing($table, $values);
-        if (empty($values)) {
-            return 0;
-        }
-        [$condition, $params] = $this->where($table, $where);
-        $set = [];
-        foreach (array_keys($values) as $i => $column) {
-            $set[] = "`$column` = :v$i";
-            $params['v' . $i] = $values[$column];
-        }
-        return $this->query("UPDATE `$table` SET " . implode(', ', $set) . " WHERE $condition", $params)->rowCount();
-    }
-
-    /** A primary or alias domain */
-    public function domainExists(string $domain): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `domain` WHERE `domain` = :domain UNION SELECT 1 FROM `alias_domain` WHERE `alias_domain` = :alias",
-            ['domain' => $domain, 'alias' => $domain])->fetchColumn();
-    }
-
-    /** The primary domain of a domain or alias domain, null if mailcow has neither */
-    public function primaryDomain(string $domain): ?string
-    {
-        $primary = $this->query("SELECT `domain` FROM `domain` WHERE `domain` = :domain
-            UNION SELECT `target_domain` FROM `alias_domain` WHERE `alias_domain` = :alias", ['domain' => $domain, 'alias' => $domain])->fetchColumn();
-        return is_string($primary) ? strtolower($primary) : null;
-    }
-
-    /** An active primary or alias domain (like the legacy is_local_mailcow_domain()) */
-    public function isLocalDomain(string $domain): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `domain` WHERE `domain` = :domain AND `active` = 1
-            UNION SELECT 1 FROM `alias_domain` WHERE `alias_domain` = :alias AND `active` = 1", ['domain' => $domain, 'alias' => $domain])->fetchColumn();
-    }
-
-    /** @return string[] alias domains of a domain */
-    public function aliasDomainsOf(string $domain): array
-    {
-        return array_map('strtolower', $this->query("SELECT `alias_domain` FROM `alias_domain` WHERE `target_domain` = :domain",
-            ['domain' => $domain])->fetchAll(PDO::FETCH_COLUMN));
-    }
-
-    /**
-     * What an address is now: 'mailbox' (also local@alias-domain of a mailbox), 'resource' (a mailbox of kind
-     * location, thing or group), 'alias', 'spamalias' or null
-     */
-    public function addressKind(string $address): ?string
-    {
-        $kind = $this->query("SELECT `kind` FROM `mailbox` WHERE `username` = :address", ['address' => $address])->fetchColumn();
-        if (is_string($kind)) {
-            return preg_match('/location|thing|group/', $kind) ? 'resource' : 'mailbox';
-        }
-        [$local, $domain] = array_pad(explode('@', $address, 2), 2, '');
-        if ((bool)$this->query("SELECT 1 FROM `mailbox` `m` JOIN `alias_domain` `ad` ON `ad`.`alias_domain` = :domain
-            WHERE `m`.`username` = CONCAT(:local, '@', `ad`.`target_domain`)", ['domain' => $domain, 'local' => $local])->fetchColumn()) {
-            return 'mailbox';
-        }
-        if ((bool)$this->query("SELECT 1 FROM `alias` WHERE `address` = :address", ['address' => $address])->fetchColumn()) {
-            return 'alias';
-        }
-        if ((bool)$this->query("SELECT 1 FROM `spamalias` WHERE `address` = :address", ['address' => $address])->fetchColumn()) {
-            return 'spamalias';
-        }
-        return null;
-    }
-
-    /** Aliases the domain may still get (domain limit minus its aliases and those of its alias domains, like the legacy code) */
-    public function aliasesLeft(string $domain): int
-    {
-        $limit = $this->query("SELECT `aliases` FROM `domain` WHERE `domain` = :domain", ['domain' => $domain])->fetchColumn();
-        if ($limit === false) {
-            return 0;
-        }
-        $used = (int)$this->query("SELECT COUNT(*) FROM `alias` WHERE (`domain` = :domain
-            OR `domain` IN (SELECT `alias_domain` FROM `alias_domain` WHERE `target_domain` = :target))
-            AND `address` NOT IN (SELECT `username` FROM `mailbox`)", ['domain' => $domain, 'target' => $domain])->fetchColumn();
-        return (int)$limit - $used;
-    }
-
-    /** Quota (bytes) the other mailboxes of the domain have */
-    public function quotaInUse(string $domain, string $exceptUsername): int
-    {
-        return (int)$this->query("SELECT COALESCE(SUM(`quota`), 0) FROM `mailbox` WHERE `domain` = :domain AND `kind` = '' AND `username` != :username",
-            ['domain' => $domain, 'username' => $exceptUsername])->fetchColumn();
-    }
-
-    /** @return array[] user1, host1 and port1 of the mailbox's sync jobs */
-    public function syncJobsOf(string $username): array
-    {
-        return $this->query("SELECT `i`.`user1`, `s`.`host1`, `s`.`port1` FROM `imapsync` `i`
-            LEFT JOIN `imapsync_source` `s` ON `s`.`id` = `i`.`source_id` WHERE `i`.`user2` = :username", ['username' => $username])->fetchAll();
-    }
-
-    /** @return int[] ids of the sync sources with this host and port the mailbox may use, oldest first */
-    public function syncSourcesFor(string $host, int $port, string $username, string $domain): array
-    {
-        return array_map('intval', $this->query("SELECT `s`.`id` FROM `imapsync_source` `s` WHERE `s`.`host1` = :host AND `s`.`port1` = :port
-            AND (`s`.`scope` = 'all'
-              OR (`s`.`scope` = 'domain' AND EXISTS (SELECT 1 FROM `imapsync_source_domain` `d` WHERE `d`.`source_id` = `s`.`id` AND `d`.`domain` = :domain))
-              OR (`s`.`scope` = 'user' AND EXISTS (SELECT 1 FROM `imapsync_source_user` `u` WHERE `u`.`source_id` = `s`.`id` AND `u`.`username` = :username)))
-            ORDER BY `s`.`id`", ['host' => $host, 'port' => $port, 'domain' => $domain, 'username' => $username])->fetchAll(PDO::FETCH_COLUMN));
-    }
-
-    /** @return array{0: string, 1: array} condition and parameters of column => value pairs */
-    private function where(string $table, array $where): array
-    {
-        self::identifier($table);
-        $parts = [];
-        $params = [];
-        foreach (array_keys($where) as $i => $column) {
-            self::identifier((string)$column);
-            if ($where[$column] === null) {
-                $parts[] = "`$column` IS NULL";
-                continue;
-            }
-            $parts[] = "`$column` = :w$i";
-            $params['w' . $i] = $where[$column];
-        }
-        return [empty($parts) ? '1=1' : implode(' AND ', $parts), $params];
-    }
-
-    /** The values of the columns the table has now */
-    private function existing(string $table, array $values): array
-    {
-        return array_intersect_key($values, array_flip($this->columns($table)));
-    }
-
-    /** Table, column and savepoint names come from code only; this keeps it that way */
-    private static function identifier(string $name): void
-    {
-        if (!preg_match('/^[a-z_0-9]+$/i', $name)) {
-            throw new \InvalidArgumentException('invalid identifier');
-        }
-    }
-}

data/web/inc/lib/mailcow/Backup/Repository/SettingsRepository.php

@@ -1,31 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Repository;
-
-use Mailcow\Core\Repository;
-use PDO;
-
-/**
- * Table `backup_setting` (name/value pairs; the runner writes its state_* values into it as well).
- */
-class SettingsRepository extends Repository
-{
-    /** @return array<string, string> name => value of the stored settings */
-    public function load(): array
-    {
-        return $this->query("SELECT `name`, `value` FROM `backup_setting`")->fetchAll(PDO::FETCH_KEY_PAIR);
-    }
-
-    /** Stores the given values (name => value) */
-    public function save(array $values): void
-    {
-        $this->transaction(function () use ($values): void {
-            foreach ($values as $name => $value) {
-                $this->query("INSERT INTO `backup_setting` (`name`, `value`) VALUES (:name, :value)
-                  ON DUPLICATE KEY UPDATE `value` = VALUES(`value`)", ['name' => (string)$name, 'value' => (string)$value]);
-            }
-        });
-    }
-}

data/web/inc/lib/mailcow/Backup/Repository/SnapshotRepository.php

@@ -1,31 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Repository;
-
-use Mailcow\Backup\Model\Snapshot;
-use Mailcow\Core\Repository;
-
-/**
- * Table `backup_snapshot`: the runner's cache of the repository's snapshots (read only here).
- */
-class SnapshotRepository extends Repository
-{
-    private const COLUMNS = "`id`, `short_id`, UNIX_TIMESTAMP(`time`) AS `time`, `hostname`, `tags`, `summary`, `domains`";
-
-    /** By id or short id, with its content (the mailboxes in it) */
-    public function find(string $id): ?Snapshot
-    {
-        $row = $this->query("SELECT " . self::COLUMNS . ", `content` FROM `backup_snapshot` WHERE `id` = :id OR `short_id` = :short_id ORDER BY `id` = :exact DESC LIMIT 1",
-            ['id' => $id, 'short_id' => $id, 'exact' => $id])->fetch();
-        return is_array($row) ? Snapshot::fromRow($row) : null;
-    }
-
-    /** @return Snapshot[] newest first; the content (the mailboxes in it) only with $withContent */
-    public function all(bool $withContent = false): array
-    {
-        return array_map(fn(array $row): Snapshot => Snapshot::fromRow($row),
-            $this->query("SELECT " . self::COLUMNS . ($withContent ? ", `content`" : "") . " FROM `backup_snapshot` ORDER BY `time` DESC, `id`")->fetchAll());
-    }
-}

data/web/inc/lib/mailcow/Backup/Service/DomainRestoreService.php

@@ -1,320 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Service;
-
-use Mailcow\Backup\Legacy\LegacyDomain;
-use Mailcow\Backup\Model\Job;
-use Mailcow\Backup\Repository\JobRepository;
-use Mailcow\Backup\Repository\RestoreRepository;
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\Context;
-use Mailcow\Core\Input;
-use Mailcow\Core\UserError;
-use Throwable;
-
-/**
- * Action `restore_domain`: recreates a domain deleted since the snapshot from its export (`domain_data`), before its
- * mailboxes are restored. Only for running restores an admin requested while the domain did not exist
- * (params.create_domain).
- *
- * Creates the domain like the legacy domain add (row, tags, domain map, external send-as rights for it dropped, SOGo
- * restarted so it knows the domain), unless it exists by now. Then adds what is missing: alias domains, DKIM keys, rate
- * limit, spam settings, BCC maps, footer, MTA-STS, domain admin assignments of accounts that still exist, groups (no
- * members, send-as rights or shares; members follow with the mailboxes), mail policies and retention rules with their
- * group assignments (matched by name).
- */
-class DomainRestoreService
-{
-    private const NEW_ROW = ['id', 'created', 'modified'];
-    private const POLICY_LISTS = ['send_allow', 'send_block', 'receive_allow', 'receive_block'];
-    private const RATE_LIMIT = '/^\d+ \/ 1[smhd]$/';
-
-    public function __construct(private RestoreRepository $repo, private JobRepository $jobs, private GroupMembers $groups,
-        private LegacyDomain $legacy)
-    {
-    }
-
-    /**
-     * @return array{domain: array{added: int, skipped: int, errors: string[], created: int}}
-     * @throws UserError
-     */
-    public function restore(mixed $jobId, mixed $data): array
-    {
-        $id = Input::intInRange($jobId, 1, PHP_INT_MAX);
-        $job = ($id === null) ? null : $this->jobs->find($id);
-        if ($job === null || $job->type !== 'restore' || $job->state !== 'running') {
-            throw new UserError('backup_job_not_running');
-        }
-        if (!$job->createsDomain() || ($job->params['requested_role'] ?? '') !== 'admin') {
-            throw new AccessDenied();
-        }
-        $domain = strtolower((string)$job->domain);
-        if (!is_array($data) || !is_array($data['domain'] ?? null) || strtolower((string)($data['domain']['domain'] ?? '')) !== $domain) {
-            throw new UserError('backup_export_invalid');
-        }
-        $result = $this->repo->locked($job->id . ':domain', fn(): array => $this->restoreDomain($job, $domain, $data));
-        return is_array($result) ? ['domain' => $result] : throw new UserError('backup_job_busy');
-    }
-
-    private function restoreDomain(Job $job, string $domain, array $data): array
-    {
-        $result = ['added' => 0, 'skipped' => 0, 'errors' => [], 'created' => 0];
-        if (!empty($this->repo->rows('alias_domain', ['alias_domain' => $domain]))) {
-            $result['errors'][] = "$domain is an alias domain now";
-            return $result;
-        }
-        if (!empty($this->repo->rows('domain', ['domain' => $domain]))) {
-            // created meanwhile: only what is missing is added
-            $result['skipped']++;
-        } else {
-            $this->createDomain($domain, $data, $result);
-        }
-        $this->repo->inTransaction(function () use ($domain, $data, &$result): void {
-            $this->aliasDomains($domain, $data, $result);
-            $this->domainSettings($domain, $data, $result);
-        });
-        $this->redisSettings($domain, $data, $result);
-        $groupIds = $this->createGroups($domain, $data, $result);
-        $this->repo->inTransaction(function () use ($domain, $data, $groupIds, &$result): void {
-            $this->rules($domain, $data, 'mail_policies', 'mail_policy', 'mail_policy_member', 'policy_id', $groupIds, $result);
-            $this->rules($domain, $data, 'retention_rules', 'retention_rule', 'retention_rule_member', 'rule_id', $groupIds, $result);
-        });
-        return $result;
-    }
-
-    /** The domain row, its tags and the effects of the legacy domain add */
-    private function createDomain(string $domain, array $data, array &$result): void
-    {
-        // the domain keeps its creation time: domain admins may use the snapshots made after it
-        $row = array_diff_key(RestoreService::decode($data['domain']), array_flip(['modified']));
-        $relayhost = (string)($row['relayhost'] ?? '0');
-        if ($relayhost !== '0' && $relayhost !== '' && empty($this->repo->rows('relayhosts', ['id' => (int)$relayhost]))) {
-            $row['relayhost'] = '0';
-            $result['errors'][] = 'relayhost: does not exist';
-        }
-        $this->repo->inTransaction(function () use ($domain, $row, $data): void {
-            $this->repo->insert('domain', ['domain' => $domain] + $row);
-            $this->repo->dropExternalSenderAcl($domain);
-            foreach (array_unique(array_filter(array_map(fn(mixed $tag): string => is_scalar($tag) ? trim((string)$tag) : '', (array)($data['tags'] ?? [])), 'strlen')) as $tag) {
-                $this->repo->insert('tags_domain', ['domain' => $domain, 'tag_name' => $tag]);
-            }
-        });
-        $result['added']++;
-        $result['created'] = 1;
-        $this->effect($result, 'domain map', fn() => $this->legacy->addDomainMap($domain));
-        $error = $this->legacy->restartSogo();
-        if ($error !== null) {
-            $result['errors'][] = 'SOGo restart failed: ' . $error;
-        }
-    }
-
-    private function aliasDomains(string $domain, array $data, array &$result): void
-    {
-        foreach (self::rows($data, 'alias_domains') as $row) {
-            $alias = strtolower(trim((string)($row['alias_domain'] ?? '')));
-            if ($alias === '' || $this->repo->domainExists($alias)) {
-                $result['skipped']++;
-                continue;
-            }
-            $this->repo->insert('alias_domain', ['alias_domain' => $alias, 'target_domain' => $domain] + array_diff_key($row, array_flip(self::NEW_ROW)));
-            $this->repo->dropExternalSenderAcl($alias);
-            $this->effect($result, "domain map of $alias", fn() => $this->legacy->addDomainMap($alias));
-            $result['added']++;
-        }
-    }
-
-    /** Spam settings, BCC maps, footer, MTA-STS and domain admins of the domain that are missing */
-    private function domainSettings(string $domain, array $data, array &$result): void
-    {
-        foreach (self::rows($data, 'filterconf') as $row) {
-            $option = (string)($row['option'] ?? '');
-            $value = (string)($row['value'] ?? '');
-            $current = $this->repo->rows('filterconf', ['object' => $domain]);
-            $present = in_array($option, ['whitelist_from', 'blacklist_from'], true)
-                ? array_filter($current, fn(array $c): bool => in_array($c['option'], ['whitelist_from', 'blacklist_from'], true) && strtolower((string)$c['value']) === strtolower($value))
-                : array_filter($current, fn(array $c): bool => $c['option'] === $option && (in_array($option, ['lowspamlevel', 'highspamlevel'], true) || (string)$c['value'] === $value));
-            if ($option === '' || !empty($present)) {
-                $result['skipped']++;
-                continue;
-            }
-            $this->repo->insert('filterconf', ['object' => $domain, 'option' => $option, 'value' => $value]);
-            $result['added']++;
-        }
-        foreach (self::rows($data, 'bcc_maps') as $row) {
-            $localDest = strtolower((string)($row['local_dest'] ?? ''));
-            if (!in_array($localDest, [$domain, '@' . $domain], true) || !filter_var((string)($row['bcc_dest'] ?? ''), FILTER_VALIDATE_EMAIL)
-                || !in_array($row['type'] ?? '', ['sender', 'rcpt'], true)) {
-                $result['errors'][] = "BCC map $localDest: invalid";
-            } elseif (!empty($this->repo->rows('bcc_maps', ['local_dest' => $localDest, 'type' => $row['type']]))) {
-                $result['skipped']++;
-            } else {
-                $this->repo->insert('bcc_maps', ['domain' => $domain] + array_diff_key($row, array_flip(self::NEW_ROW)));
-                $result['added']++;
-            }
-        }
-        foreach (['footer' => 'domain_wide_footer', 'mta_sts' => 'mta_sts'] as $key => $table) {
-            if (!is_array($data[$key] ?? null)) {
-                continue;
-            }
-            if (!empty($this->repo->rows($table, ['domain' => $domain]))) {
-                $result['skipped']++;
-                continue;
-            }
-            $this->repo->insert($table, ['domain' => $domain] + array_diff_key(RestoreService::decode($data[$key]), array_flip(['created', 'modified'])));
-            $result['added']++;
-        }
-        foreach (self::rows($data, 'domain_admins') as $row) {
-            $username = (string)($row['username'] ?? '');
-            if (empty($this->repo->rows('admin', ['username' => $username, 'superadmin' => 0]))) {
-                $result['errors'][] = "domain admin $username: does not exist";
-            } elseif (!empty($this->repo->rows('domain_admins', ['username' => $username, 'domain' => $domain]))) {
-                $result['skipped']++;
-            } else {
-                $this->repo->insert('domain_admins', ['username' => $username, 'domain' => $domain, 'active' => (int)($row['active'] ?? 1)]);
-                $result['added']++;
-            }
-        }
-    }
-
-    /** DKIM keys of the domain and its alias domains, and the rate limit of the domain (Redis), when missing */
-    private function redisSettings(string $domain, array $data, array &$result): void
-    {
-        $domains = array_merge([$domain], $this->repo->aliasDomainsOf($domain));
-        foreach (self::rows($data, 'dkim') as $key) {
-            $name = strtolower((string)($key['domain'] ?? ''));
-            $selector = (string)($key['selector'] ?? '');
-            if (!in_array($name, $domains, true)) {
-                continue;
-            }
-            if (!preg_match('/^[a-z0-9._-]{1,63}$/i', $selector) || (string)($key['pubkey'] ?? '') === '' || (string)($key['privkey'] ?? '') === '') {
-                $result['errors'][] = "DKIM key of $name: invalid";
-                continue;
-            }
-            $this->effect($result, "DKIM key of $name", function () use ($name, $selector, $key, &$result): void {
-                if ($this->legacy->dkim($name) !== null) {
-                    $result['skipped']++;
-                    return;
-                }
-                $this->legacy->addDkim($name, $selector, (string)$key['pubkey'], (string)$key['privkey']);
-                $result['added']++;
-            });
-        }
-        $rateLimit = $data['ratelimit'] ?? null;
-        if (is_string($rateLimit) && preg_match(self::RATE_LIMIT, $rateLimit)) {
-            $this->effect($result, 'rate limit', function () use ($domain, $rateLimit, &$result): void {
-                if (isset($this->legacy->rateLimits([$domain])[$domain])) {
-                    $result['skipped']++;
-                    return;
-                }
-                $this->legacy->setRateLimit($domain, $rateLimit);
-                $result['added']++;
-            });
-        }
-    }
-
-    /**
-     * Groups of the domain missing by name, through the group module (address, flags, priority, IdP link)
-     *
-     * @return array<int, int> id in the backup => id now, of all groups found by name
-     */
-    private function createGroups(string $domain, array $data, array &$result): array
-    {
-        $ids = [];
-        $admin = new Context('admin', 'backup', [], true);
-        foreach (self::rows($data, 'groups') as $row) {
-            $name = (string)($row['name'] ?? '');
-            $current = $this->repo->rows('mailbox_group', ['domain' => $domain, 'name' => $name])[0] ?? null;
-            if ($current !== null) {
-                $ids[(int)($row['id'] ?? 0)] = (int)$current['id'];
-                $result['skipped']++;
-                continue;
-            }
-            $input = ['domain' => $domain, 'name' => $name, 'description' => (string)($row['description'] ?? '')];
-            foreach (['sender_allowed', 'sogo_visible', 'internal', 'active', 'priority'] as $key) {
-                if (isset($row[$key])) {
-                    $input[$key] = (int)$row[$key];
-                }
-            }
-            foreach (['address', 'idp_ref'] as $key) {
-                if (($row[$key] ?? null) !== null && $row[$key] !== '') {
-                    $input[$key] = (string)$row[$key];
-                }
-            }
-            try {
-                $ids[(int)($row['id'] ?? 0)] = $this->groups->create($admin, $input);
-                $result['added']++;
-            } catch (UserError $e) {
-                $result['errors'][] = "group $name: " . $e->key;
-            }
-        }
-        return $ids;
-    }
-
-    /**
-     * Mail policies or retention rules missing by name, with their group assignments; group:<id> entries of policies
-     * follow the groups' new ids
-     */
-    private function rules(string $domain, array $data, string $key, string $table, string $member, string $column, array $groupIds, array &$result): void
-    {
-        foreach (self::rows($data, $key) as $row) {
-            $name = (string)($row['name'] ?? '');
-            if (!empty($this->repo->rows($table, ['domain' => $domain, 'name' => $name]))) {
-                $result['skipped']++;
-                continue;
-            }
-            $values = array_diff_key($row, array_flip([...self::NEW_ROW, 'group_members']));
-            foreach (self::POLICY_LISTS as $list) {
-                if (isset($values[$list])) {
-                    $values[$list] = json_encode($this->remapGroups((string)$values[$list], $groupIds, "$name ($list)", $result));
-                }
-            }
-            $id = $this->repo->insert($table, ['domain' => $domain, 'name' => $name] + $values);
-            foreach ((array)($row['group_members'] ?? []) as $groupName) {
-                $group = $this->repo->rows('mailbox_group', ['domain' => $domain, 'name' => (string)$groupName])[0] ?? null;
-                if ($group === null) {
-                    $result['errors'][] = "$name: group $groupName does not exist";
-                    continue;
-                }
-                $this->repo->insert($member, [$column => $id, 'group_id' => (int)$group['id']]);
-            }
-            $result['added']++;
-        }
-    }
-
-    /** @return string[] entries of a JSON list with group:<id> of the backup turned into the group's id now */
-    private function remapGroups(string $json, array $groupIds, string $what, array &$result): array
-    {
-        $entries = [];
-        foreach ((array)json_decode($json, true) as $entry) {
-            if (!is_string($entry)) {
-                continue;
-            }
-            if (preg_match('/^group:(\d+)$/', $entry, $match)) {
-                if (!isset($groupIds[(int)$match[1]])) {
-                    $result['errors'][] = "$what: $entry dropped (group does not exist)";
-                    continue;
-                }
-                $entry = 'group:' . $groupIds[(int)$match[1]];
-            }
-            $entries[] = $entry;
-        }
-        return $entries;
-    }
-
-    /** A Redis or dockerapi step: a failure is reported and the rest goes on */
-    private function effect(array &$result, string $what, callable $work): void
-    {
-        try {
-            $work();
-        } catch (Throwable) {
-            $result['errors'][] = "$what: not stored (Redis)";
-        }
-    }
-
-    private static function rows(array $data, string $key): array
-    {
-        return array_values(array_filter((array)($data[$key] ?? []), 'is_array'));
-    }
-}

data/web/inc/lib/mailcow/Backup/Service/ExportService.php

@@ -1,227 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Service;
-
-use Mailcow\Backup\Legacy\LegacyDomain;
-use Mailcow\Backup\Repository\ExportRepository;
-use Mailcow\Core\UserError;
-
-/**
- * Export of a domain for the backup (/mailcow/export/<domain>.json in a snapshot), in pages of mailboxes the runner
- * merges into one file. The first page holds version, db_version, domain, domain_data (what recreating the domain
- * needs), the aliases and `mailbox_aliases` (username => the aliases of any domain whose goto names the mailbox,
- * without its own alias and group addresses; the runner moves them into the mailbox sections when it reads the file);
- * every page a part of `mailboxes` and `next_after`, the last username of the page to ask for the next one with
- * (keyset paging; null after the last page).
- *
- * Per mailbox every row a restore needs, with all columns. Password hashes, DKIM keys and the secrets of app
- * passwords, 2FA and sync jobs are part of it on purpose; only the runner gets it (backup.php with the token). Binary
- * values (fido2) are written as {"base64": "..."}. Beside the keys of the spec, a mailbox has `own_alias` (its alias
- * row: forwarding with or without a copy), `aliases` (empty, see mailbox_aliases) and `ratelimit` (Redis).
- */
-class ExportService
-{
-    public const VERSION = 1;
-    public const PAGE_SIZE = 500;
-    /** Lists of a mailbox: key => [table, column with the username] */
-    private const MAILBOX_ROWS = [
-        'sender_acl' => ['sender_acl', 'logged_in_as'],
-        'sender_acl_granted' => ['sender_acl', 'send_as'],
-        'sieve_filters' => ['sieve_filters', 'username'],
-        'filterconf' => ['filterconf', 'object'],
-        'spamalias' => ['spamalias', 'goto'],
-        'bcc_maps' => ['bcc_maps', 'local_dest'],
-        'app_passwd' => ['app_passwd', 'mailbox'],
-        'tfa' => ['tfa', 'username'],
-        // FIDO2 keys of mailbox users (passwordless login on the user page), like those of admins
-        'fido2' => ['fido2', 'username'],
-        'identity_link' => ['identity_link', 'username'],
-    ];
-    /** Single rows of a mailbox: key => [table, column with the username] */
-    private const MAILBOX_ROW = [
-        'user_acl' => ['user_acl', 'username'],
-        'pushover' => ['pushover', 'username'],
-    ];
-
-    public function __construct(private ExportRepository $repo, private LegacyDomain $legacy)
-    {
-    }
-
-    /** @return string[] all domains */
-    public function domains(): array
-    {
-        return array_values(array_map('strval', $this->repo->domains()));
-    }
-
-    /**
-     * @param ?string $after the last username of the previous page (keyset paging), null for the first page
-     * @throws UserError domain_not_found
-     */
-    public function export(string $domain, ?string $after = null, int $pageSize = self::PAGE_SIZE): array
-    {
-        $domain = strtolower(trim($domain));
-        $row = ($domain === '') ? null : ($this->repo->rows('domain', 'domain', $domain)[0] ?? null);
-        if ($row === null) {
-            throw new UserError('domain_not_found', [htmlspecialchars($domain)]);
-        }
-        $rows = $this->repo->mailboxesAfter($domain, strtolower($after ?? ''), $pageSize + 1);
-        $more = count($rows) > $pageSize;
-        $rows = array_slice($rows, 0, $pageSize);
-        $export = ['version' => self::VERSION, 'domain' => $domain];
-        if ($after === null) {
-            $export += [
-                'db_version' => $this->repo->dbVersion(),
-                'domain_data' => $this->domainData($domain, $row),
-                'aliases' => array_map(fn(array $alias): array => self::encode($alias), $this->repo->aliases($domain)),
-                'mailbox_aliases' => $this->mailboxAliases($domain) ?: new \stdClass(),
-            ];
-        }
-        $export['mailboxes'] = $this->mailboxes($domain, $rows) ?: new \stdClass();
-        $export['next_after'] = ($more && !empty($rows)) ? strtolower((string)end($rows)['username']) : null;
-        return $export;
-    }
-
-    /** What recreating the domain needs: its row and settings, alias domains, DKIM keys, groups, policies, rules */
-    private function domainData(string $domain, array $row): array
-    {
-        $aliasDomains = $this->repo->rows('alias_domain', 'target_domain', $domain);
-        $dkim = [];
-        foreach (array_merge([$domain], array_map(fn(array $alias): string => strtolower((string)$alias['alias_domain']), $aliasDomains)) as $name) {
-            $key = $this->legacy->dkim($name);
-            if ($key !== null) {
-                $dkim[] = ['domain' => $name] + $key;
-            }
-        }
-        return [
-            'domain' => self::encode($row),
-            'tags' => array_map(fn(array $tag): string => (string)$tag['tag_name'], $this->repo->rows('tags_domain', 'domain', $domain)),
-            'alias_domains' => $aliasDomains,
-            'dkim' => $dkim,
-            'ratelimit' => $this->legacy->rateLimits([$domain])[$domain] ?? null,
-            'filterconf' => $this->repo->rows('filterconf', 'object', $domain),
-            'bcc_maps' => $this->repo->domainBccMaps($domain),
-            'footer' => $this->repo->rows('domain_wide_footer', 'domain', $domain)[0] ?? null,
-            'mta_sts' => $this->repo->rows('mta_sts', 'domain', $domain)[0] ?? null,
-            'domain_admins' => $this->repo->rows('domain_admins', 'domain', $domain),
-            'groups' => $this->repo->rows('mailbox_group', 'domain', $domain),
-            'mail_policies' => $this->repo->domainRules('mail_policy', 'mail_policy_member', 'policy_id', $domain),
-            'retention_rules' => $this->repo->domainRules('retention_rule', 'retention_rule_member', 'rule_id', $domain),
-        ];
-    }
-
-    /** @return array<string, array> username => its section */
-    private function mailboxes(string $domain, array $rows): array
-    {
-        $mailboxes = [];
-        foreach ($rows as $row) {
-            $mailboxes[strtolower((string)$row['username'])] = [
-                'mailbox' => self::encode($row) + ['tags' => []],
-                'user_acl' => null,
-                'pushover' => null,
-                'ratelimit' => null,
-                'own_alias' => null,
-                'aliases' => [],
-                'groups' => [],
-                'mail_policies' => [],
-                'retention_rules' => [],
-                'imapsync' => [],
-            ] + array_fill_keys(array_keys(self::MAILBOX_ROWS), []);
-        }
-        $usernames = array_map('strval', array_keys($mailboxes));
-        if (empty($usernames)) {
-            return [];
-        }
-        // the mailbox of the page a row belongs to, null for anything else
-        $owner = fn(array $row, string $column): ?string => isset($mailboxes[strtolower((string)$row[$column])]) ? strtolower((string)$row[$column]) : null;
-        foreach ($this->repo->rowsOfMailboxes('tags_mailbox', 'username', $usernames) as $row) {
-            if (($username = $owner($row, 'username')) !== null) {
-                $mailboxes[$username]['mailbox']['tags'][] = (string)$row['tag_name'];
-            }
-        }
-        foreach (self::MAILBOX_ROW as $key => [$table, $column]) {
-            foreach ($this->repo->rowsOfMailboxes($table, $column, $usernames) as $row) {
-                if (($username = $owner($row, $column)) !== null) {
-                    $mailboxes[$username][$key] = self::encode($row);
-                }
-            }
-        }
-        foreach (self::MAILBOX_ROWS as $key => [$table, $column]) {
-            foreach ($this->repo->rowsOfMailboxes($table, $column, $usernames) as $row) {
-                if (($username = $owner($row, $column)) !== null) {
-                    $mailboxes[$username][$key][] = self::encode($row);
-                }
-            }
-        }
-        foreach ($this->legacy->rateLimits($usernames) as $username => $value) {
-            if (isset($mailboxes[strtolower((string)$username)])) {
-                $mailboxes[strtolower((string)$username)]['ratelimit'] = $value;
-            }
-        }
-        foreach ($this->repo->groupMemberships($usernames) as $row) {
-            if (($username = $owner($row, 'username')) !== null) {
-                $mailboxes[$username]['groups'][] = ['id' => (int)$row['id'], 'name' => (string)$row['name'], 'domain' => (string)$row['domain'],
-                    'source' => (string)$row['source']];
-            }
-        }
-        foreach (['mail_policies' => ['mail_policy', 'mail_policy_member', 'policy_id'], 'retention_rules' => ['retention_rule', 'retention_rule_member', 'rule_id']]
-            as $key => [$table, $member, $column]) {
-            foreach ($this->repo->assignments($table, $member, $column, $usernames) as $row) {
-                if (($username = $owner($row, 'username')) !== null) {
-                    $mailboxes[$username][$key][] = ['id' => (int)$row['id'], 'name' => (string)$row['name'], 'domain' => (string)$row['domain']];
-                }
-            }
-        }
-        foreach ($this->repo->syncJobs($usernames) as $row) {
-            if (($username = $owner($row, 'user2')) !== null) {
-                $mailboxes[$username]['imapsync'][] = self::encode($row);
-            }
-        }
-        foreach ($this->repo->rowsOfMailboxes('alias', 'address', $usernames) as $row) {
-            if (($username = $owner($row, 'address')) !== null) {
-                $mailboxes[$username]['own_alias'] = self::encode($row);
-            }
-        }
-        return $mailboxes;
-    }
-
-    /**
-     * @return array<string, array[]> username => aliases of any domain whose goto names the mailbox (once per domain: the
-     * alias table is scanned, not once per page)
-     */
-    private function mailboxAliases(string $domain): array
-    {
-        $usernames = array_fill_keys(array_map(fn(mixed $username): string => strtolower((string)$username), $this->repo->usernames($domain)), true);
-        $aliases = [];
-        foreach ($this->repo->aliasesPointingTo($domain) as $row) {
-            if ($row['group_id'] !== null) {
-                continue;
-            }
-            $address = strtolower((string)$row['address']);
-            foreach (array_unique(array_map('strtolower', self::gotos((string)$row['goto']))) as $goto) {
-                if ($goto !== $address && isset($usernames[$goto])) {
-                    $aliases[$goto][] = self::encode($row);
-                }
-            }
-        }
-        return $aliases;
-    }
-
-    /** @return string[] addresses of a goto list */
-    public static function gotos(string $goto): array
-    {
-        return array_values(array_filter(array_map('trim', explode(',', $goto)), 'strlen'));
-    }
-
-    /** Binary values (not UTF-8) as {"base64": "..."}, so the row survives JSON */
-    public static function encode(array $row): array
-    {
-        foreach ($row as $column => $value) {
-            if (is_string($value) && !mb_check_encoding($value, 'UTF-8')) {
-                $row[$column] = ['base64' => base64_encode($value)];
-            }
-        }
-        return $row;
-    }
-}

data/web/inc/lib/mailcow/Backup/Service/GroupMembers.php

@@ -1,48 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Service;
-
-use Mailcow\Core\Context;
-use Mailcow\Core\UserError;
-use Mailcow\MailboxGroup\Repository\GroupRepository;
-use Mailcow\MailboxGroup\Repository\LookupRepository;
-use Mailcow\MailboxGroup\Service\ChangeEffects;
-use Mailcow\MailboxGroup\Service\GroupService;
-use Mailcow\MailboxGroup\Service\SharingService;
-use Mailcow\MailboxGroup\Service\TemplateService;
-
-/**
- * Group writes of a restore through the group module (addresses, send-as rights, shares and caches follow), as the
- * requester of the restore: an admin, or the domain admin with his rights now (GroupService::canEdit). The module is
- * wired without group templates (Legacy\NoTemplates).
- */
-class GroupMembers
-{
-    public function __construct(private GroupRepository $repo, private LookupRepository $lookup, private TemplateService $templates,
-        private ChangeEffects $effects, private SharingService $sharing)
-    {
-    }
-
-    /** @throws UserError e.g. access_denied when the group is read-only for the requester */
-    public function add(Context $context, int $groupId, string $username): void
-    {
-        $this->service($context)->update($groupId, ['add_members' => [$username]]);
-    }
-
-    /**
-     * Creates a group (restore of a deleted domain); members follow with the mailboxes.
-     *
-     * @throws UserError
-     */
-    public function create(Context $context, array $input): int
-    {
-        return (int)$this->service($context)->create($input)->id;
-    }
-
-    protected function service(Context $context): GroupService
-    {
-        return new GroupService($this->repo, $this->lookup, $this->templates, $this->effects, $this->sharing, $context);
-    }
-}

data/web/inc/lib/mailcow/Backup/Service/JobService.php

@@ -1,666 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Service;
-
-use DateTimeImmutable;
-use DateTimeZone;
-use Mailcow\Backup\Model\Job;
-use Mailcow\Backup\Model\Settings;
-use Mailcow\Backup\Model\Snapshot;
-use Mailcow\Backup\Repository\JobRepository;
-use Mailcow\Backup\Repository\SnapshotRepository;
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\Context;
-use Mailcow\Core\Input;
-use Mailcow\Core\UserError;
-
-/**
- * Jobs of the backup runners, the snapshots with their content and the status page.
- *
- * Backup, check, prune and init jobs: admins (not read-only API keys) queue them, one of each type at a time.
- * Restores: admins, and domain admins for mailboxes or a domain they administer now. Mailboxes and domains deleted
- * since the snapshot are created from it (deleted domains by admins only). A component a domain admin could not
- * manage himself (e.g. sync jobs without the `syncjobs` ACL) is refused; his ACL and domains are stored with the job,
- * so the merge and the runners apply the same limits.
- * Reading: admins see every job and snapshot; domain admins the restores of their domains and, of each snapshot made
- * after their domain was created, their domains in it.
- */
-class JobService
-{
-    /**
-     * Restore components in their order. mails, imap_acl, sieve_activate and the Sieve scripts of filters are restored by
-     * the dovecot runner, sogo by the restic runner, the others by RestoreService.
-     */
-    public const COMPONENTS = ['mails', 'imap_acl', 'sogo', 'aliases', 'groups', 'sender_acl', 'mail_policies', 'retention_rules',
-        'filters', 'sieve_activate', 'spam', 'bcc', 'syncjobs', 'app_passwords', 'tfa', 'settings', 'password'];
-    /** Components that replace current data with the backup's (all others only add what is missing) */
-    public const OVERWRITE = ['sogo', 'settings', 'password'];
-    /** Components that need the mails: not in snapshots without them (RESTIC_BACKUP_MODE=config) */
-    public const MAIL_COMPONENTS = ['mails', 'imap_acl', 'sogo', 'sieve_activate'];
-    public const MAIL_TARGETS = ['folder', 'original'];
-    /** Domain admin ACL a component needs */
-    private const COMPONENT_ACL = [
-        'groups' => 'mailbox_groups',
-        'mail_policies' => 'mail_policies',
-        'retention_rules' => 'retention',
-        'filters' => 'filters',
-        'sieve_activate' => 'filters',
-        'spam' => 'spam_policy',
-        'bcc' => 'bcc_maps',
-        'syncjobs' => 'syncjobs',
-        'app_passwords' => 'app_passwds',
-    ];
-    /** Components that only work together with another one */
-    private const REQUIRES = ['sieve_activate' => 'filters'];
-    /** Domain admin ACLs the merge respects, stored with the job */
-    private const RESTORE_ACL = ['filters', 'mailbox_groups', 'mail_policies', 'retention', 'spam_policy', 'bcc_maps', 'syncjobs',
-        'app_passwds', 'protocol_access', 'mailbox_relayhost', 'unlimited_quota', 'sogo_access', 'pushover', 'extend_sender_acl',
-        'alias_external_goto', 'ratelimit', 'tls_policy', 'quarantine', 'quarantine_notification', 'quarantine_category'];
-    private const MAX_MAILBOXES = 1000;
-    private const LIST_LIMIT = 100;
-    private const MAX_LIST = 1000;
-    /** Mailboxes a search returns at most */
-    private const SEARCH_LIMIT = 500;
-    /** The runner reports every 10 seconds */
-    private const RUNNER_TIMEOUT = 120;
-    /** @var array<string, array{bool, ?int}> domain => [access of the domain admin, creation time now], per request */
-    private array $domainState = [];
-
-    public function __construct(private JobRepository $jobs, private SnapshotRepository $snapshots, private SettingsService $settings,
-        private Context $context, private string $timezone = 'UTC')
-    {
-    }
-
-    /**
-     * @param mixed $type only jobs of this type ('' or null = all; an unknown type finds nothing)
-     * @return Job[] newest first, at most $limit (0 = LIST_LIMIT)
-     */
-    public function list(int $limit = 0, mixed $type = null): array
-    {
-        $type = Input::text($type);
-        if ($type !== '' && !in_array($type, Job::TYPES, true)) {
-            return [];
-        }
-        return $this->jobs->visible($this->context, ($limit > 0) ? min($limit, self::MAX_LIST) : self::LIST_LIMIT, ($type === '') ? null : $type);
-    }
-
-    /**
-     * Entry of the API. Domain admins get it restricted: e-mail addresses outside the job's domain and its alias domains
-     * and repository details (URLs, restic repository strings, paths of the repository) are shown as "…".
-     */
-    public function view(Job $job, bool $withLog = false): array
-    {
-        $entry = $job->toArray($withLog);
-        if ($this->context->hasAdminRights()) {
-            return $entry;
-        }
-        $domains = array_merge([strtolower($job->domain)], $this->jobs->aliasDomainsOf(strtolower($job->domain)));
-        foreach (['result', 'error', 'log'] as $key) {
-            if (isset($entry[$key])) {
-                $entry[$key] = self::restricted($entry[$key], $domains);
-            }
-        }
-        // the account of an admin who requested the restore stays hidden, the UI shows "Administrator"
-        if (($job->params['requested_role'] ?? '') !== 'domainadmin') {
-            $entry['requested_by'] = '';
-            if (isset($entry['log'])) {
-                $entry['log'] = str_replace('(requested by ' . $job->requestedBy . ')', '(requested by an administrator)', $entry['log']);
-            }
-        }
-        return $entry;
-    }
-
-    /** The job with its log, null when it does not exist or the caller may not see it */
-    public function get(int $id): ?Job
-    {
-        $job = $this->jobs->find($id, true);
-        return ($job !== null && $this->visible($job)) ? $job : null;
-    }
-
-    /**
-     * Queues a backup, check, prune or init job (admins).
-     *
-     * @throws UserError
-     */
-    public function create(array $input): Job
-    {
-        $this->context->requireAdmin();
-        $type = Input::text($input['type'] ?? null);
-        if (!in_array($type, Job::MANUAL_TYPES, true)) {
-            throw new UserError('backup_job_type_invalid', [htmlspecialchars((string)$type)]);
-        }
-        $settings = $this->settings->load();
-        if (!$settings->configured()) {
-            throw new UserError('backup_not_configured');
-        }
-        if ($type === 'backup' && $settings->get('password_confirmed') !== '1') {
-            throw new UserError('backup_password_not_confirmed');
-        }
-        $job = new Job($type);
-        $job->requestedBy = (string)$this->context->username;
-        $this->jobs->insertExclusive($job, function (array $active) use ($type): void {
-            foreach ($active as $other) {
-                if ($other->type === $type) {
-                    throw new UserError('backup_job_active', [$type]);
-                }
-            }
-        });
-        return $job;
-    }
-
-    /**
-     * Queues a restore: {snapshot, scope: mailbox|domain, mailboxes | domain, components, mail_target: folder|original}.
-     * All mailboxes belong to one domain. Mailboxes deleted since the snapshot are created from it (create_mailboxes);
-     * an admin may restore a deleted domain (create_domain). Scope domain restores the snapshot's mailboxes of the
-     * domain (the current ones for snapshots without content).
-     *
-     * @throws UserError
-     */
-    public function restore(array $input): Job
-    {
-        $this->requireRestorer();
-        if (!$this->settings->load()->configured()) {
-            throw new UserError('backup_not_configured');
-        }
-        $snapshot = $this->snapshot($input['snapshot'] ?? null);
-        $scope = Input::text($input['scope'] ?? 'mailbox');
-        [$domain, $mailboxes] = match ($scope) {
-            'domain' => $this->domainScope($snapshot, $input['domain'] ?? null),
-            'mailbox' => $this->mailboxScope($snapshot, $input['mailboxes'] ?? null, $input['domain'] ?? null),
-            default => throw new UserError('backup_scope_invalid'),
-        };
-        $components = $this->components($input['components'] ?? null);
-        $missing = array_values(array_intersect($components, self::MAIL_COMPONENTS));
-        if ($snapshot->mode() === 'config' && !empty($missing)) {
-            throw new UserError('backup_components_no_mails', [htmlspecialchars(implode(', ', $missing))]);
-        }
-        $mailTarget = Input::text($input['mail_target'] ?? null) ?: 'folder';
-        if (!in_array($mailTarget, self::MAIL_TARGETS, true)) {
-            throw new UserError('backup_mail_target_invalid');
-        }
-        $job = new Job('restore');
-        $job->step = 'files';
-        $job->domain = $domain;
-        $job->requestedBy = (string)$this->context->username;
-        $job->params = [
-            'snapshot' => $snapshot->id,
-            'snapshot_time' => $snapshot->time,
-            'scope' => $scope,
-            'domain' => $domain,
-            'mailboxes' => $mailboxes,
-            'components' => $components,
-            'mail_target' => $mailTarget,
-            'requested_role' => $this->context->hasAdminRights() ? 'admin' : 'domainadmin',
-            'create_domain' => (int)!$this->jobs->isDomain($domain),
-            'create_mailboxes' => array_values(array_filter($mailboxes, fn(string $username): bool => $this->jobs->mailboxDomain($username) === null)),
-        ];
-        if (!$this->context->hasAdminRights()) {
-            $job->params['acl'] = array_combine(self::RESTORE_ACL, array_map(fn(string $acl): int => (int)$this->context->hasAcl($acl), self::RESTORE_ACL));
-            $job->params['allowed_domains'] = array_values(array_unique(array_merge([$domain], $this->jobs->aliasDomainsOf($domain))));
-        }
-        $domainAdmin = !$this->context->hasAdminRights();
-        $this->jobs->insertExclusive($job, function (array $active) use ($mailboxes, $domain, $domainAdmin): void {
-            foreach ($active as $other) {
-                if ($other->type !== 'restore') {
-                    continue;
-                }
-                // a domain admin has one restore per domain at a time
-                if ($domainAdmin && strtolower($other->domain) === $domain) {
-                    throw new UserError('backup_restore_domain_active', [htmlspecialchars($domain)]);
-                }
-                $both = array_intersect($mailboxes, $other->mailboxes());
-                if (!empty($both)) {
-                    throw new UserError('backup_restore_active', [htmlspecialchars((string)reset($both))]);
-                }
-            }
-        });
-        return $job;
-    }
-
-    /**
-     * Cancels a queued or running job (the restic runner stops at its next check): admins any, domain admins the
-     * restores of their domains that domain admins requested. A restore in the dovecot step (mails, mails_running) runs
-     * to its end.
-     *
-     * @throws UserError
-     */
-    public function cancel(int $id): Job
-    {
-        if ($this->context->isReadOnlyApi()) {
-            throw new AccessDenied();
-        }
-        $job = $this->jobs->find($id);
-        // domain admins cancel only restores domain admins requested
-        if ($job === null || !$this->visible($job) || (!$this->context->hasAdminRights() && ($job->params['requested_role'] ?? '') !== 'domainadmin')) {
-            throw new AccessDenied();
-        }
-        if (!$job->isActive() || ($job->type === 'restore' && in_array($job->step, ['mails', 'mails_running'], true)) || !$this->jobs->cancel($id)) {
-            throw new UserError('backup_job_not_cancellable', [$id]);
-        }
-        $job->state = 'cancelled';
-        return $job;
-    }
-
-    /**
-     * Snapshots, newest first. Domain admins get those holding one of their domains (see mayUse()), with id, short_id,
-     * time and those domains.
-     */
-    public function snapshots(): array
-    {
-        if ($this->context->hasAdminRights()) {
-            return array_map(fn(Snapshot $snapshot): array => $snapshot->toArray(), $this->snapshots->all());
-        }
-        if (!$this->context->isDomainAdmin()) {
-            return [];
-        }
-        $snapshots = [];
-        // the domains with their creation time come from the snapshot row, the content is not needed here
-        foreach ($this->snapshots->all() as $snapshot) {
-            $domains = array_values(array_filter($snapshot->domainList(), fn(string $domain): bool => $this->mayUse($snapshot, $domain)));
-            if (!empty($domains)) {
-                $snapshots[] = ['id' => $snapshot->id, 'short_id' => $snapshot->shortId, 'time' => $snapshot->time, 'domains' => $domains,
-                    'mode' => $snapshot->mode()];
-            }
-        }
-        return $snapshots;
-    }
-
-    /**
-     * Domains of a snapshot with their state now: GET backup-snapshot-content/<snapshot>. `available` 0: the snapshot has
-     * no content (not made by this installation), the list comes from its manifest and only existing mailboxes can be restored. A domain
-     * of the manifest missing in the content (its export failed) lists its current mailboxes as well.
-     *
-     * @return array{snapshot: string, time: int, available: int, domains: array[]}|array{} [] when not found or not visible
-     */
-    public function content(mixed $snapshotId): array
-    {
-        $snapshot = $this->visibleSnapshot($snapshotId);
-        if ($snapshot === null) {
-            return [];
-        }
-        $visible = array_values(array_filter($snapshot->domainList(), fn(string $domain): bool => $this->mayUse($snapshot, $domain)));
-        [$existing, $usernames] = $this->jobs->domainsAndUsernames($visible);
-        $domains = [];
-        foreach ($visible as $domain) {
-            $current = $usernames[$domain] ?? [];
-            $mailboxes = $snapshot->mailboxesOf($domain) ?? $current;
-            $domains[] = ['domain' => $domain, 'exists' => (int)isset($existing[$domain]), 'mailboxes' => count($mailboxes),
-                'missing' => count(array_diff_key($mailboxes, $current))];
-        }
-        return ['snapshot' => $snapshot->id, 'time' => $snapshot->time, 'available' => (int)($snapshot->content !== null), 'domains' => $domains];
-    }
-
-    /**
-     * Search in a snapshot: GET backup-snapshot-search/<snapshot>/<term>. Matches domain names and mailboxes by local
-     * part, address or name ("x@": local part, "@d": domain, "x@d": address), with the rules of content(); at most
-     * SEARCH_LIMIT mailboxes. A matching domain comes with all its mailboxes.
-     *
-     * @return array{domains: array[], truncated: int}|array{} [] when the snapshot is not found or not visible
-     */
-    public function search(mixed $snapshotId, mixed $term): array
-    {
-        $snapshot = $this->visibleSnapshot($snapshotId);
-        $term = mb_strtolower((string)Input::text($term));
-        if ($snapshot === null) {
-            return [];
-        }
-        if (trim($term, '@') === '') {
-            return ['domains' => [], 'truncated' => 0];
-        }
-        $domains = [];
-        $hits = 0;
-        $truncated = 0;
-        [$local, $at, $domainTerm] = str_contains($term, '@') ? [strstr($term, '@', true), true, substr((string)strstr($term, '@'), 1)] : [$term, false, ''];
-        foreach ($snapshot->domainList() as $domain) {
-            if (!$this->mayUse($snapshot, $domain)) {
-                continue;
-            }
-            $domainMatch = $at ? ($domainTerm !== '' && str_contains($domain, $domainTerm)) : str_contains($domain, $term);
-            if ($at && $domainTerm !== '' && !$domainMatch) {
-                continue;
-            }
-            $current = $this->jobs->mailboxNames($domain);
-            $mailboxes = [];
-            foreach (($snapshot->mailboxesOf($domain) ?? $current) as $username => $name) {
-                $username = (string)$username;
-                $part = (string)strstr($username, '@', true);
-                $match = match (true) {
-                    // "x@d": the address, "x@": the local part, "@d": every mailbox of the matching domains
-                    $at && $local !== '' && $domainTerm !== '' => str_contains($username, $term),
-                    $at && $local !== '' => str_contains($part, $local),
-                    $at => true,
-                    default => $domainMatch || str_contains($username, $term) || str_contains(mb_strtolower($name), $term),
-                };
-                if (!$match) {
-                    continue;
-                }
-                if ($hits >= self::SEARCH_LIMIT) {
-                    $truncated = 1;
-                    break;
-                }
-                $hits++;
-                $mailboxes[] = ['username' => $username, 'name' => $name, 'exists' => (int)isset($current[$username])];
-            }
-            // a domain without matching mailboxes only for a search by domain
-            if (!empty($mailboxes) || ($domainMatch && !($at && $local !== '') && !$truncated)) {
-                $domains[] = ['domain' => $domain, 'exists' => (int)$this->jobs->isDomain($domain), 'mailboxes' => $mailboxes];
-            }
-            if ($truncated) {
-                break;
-            }
-        }
-        return ['domains' => $domains, 'truncated' => $truncated];
-    }
-
-    /**
-     * Mailboxes of a domain in a snapshot with their state now: GET backup-snapshot-content/<snapshot>/<domain>
-     *
-     * @return array{domain: string, exists: int, mailboxes: array[]}|array{} [] when not found or not visible
-     */
-    public function domainContent(mixed $snapshotId, mixed $domain): array
-    {
-        $snapshot = $this->visibleSnapshot($snapshotId);
-        $domain = strtolower((string)Input::text($domain));
-        if ($snapshot === null || !$snapshot->hasDomain($domain) || !$this->mayUse($snapshot, $domain)) {
-            return [];
-        }
-        $current = $this->jobs->mailboxNames($domain);
-        $mailboxes = [];
-        foreach (($snapshot->mailboxesOf($domain) ?? $current) as $username => $name) {
-            $mailboxes[] = ['username' => (string)$username, 'name' => $name, 'exists' => (int)isset($current[$username])];
-        }
-        return ['domain' => $domain, 'exists' => (int)$this->jobs->isDomain($domain), 'mailboxes' => $mailboxes];
-    }
-
-    /**
-     * State of the backup for admins: settings state, runner heartbeat, repository, newest job of each type, queued and
-     * running jobs, the next scheduled run. [] for everybody else.
-     */
-    public function status(?int $now = null): array
-    {
-        if (!$this->context->hasAdminRights()) {
-            return [];
-        }
-        $now ??= time();
-        $settings = $this->settings->load();
-        $seen = ($settings->get('state_runner_seen') === '') ? null : (int)$settings->get('state_runner_seen');
-        $snapshots = $this->snapshots->all();
-        $latest = $this->jobs->latestOfEachType();
-        $stats = json_decode($settings->get('state_repo_stats'), true);
-        return [
-            'enabled' => (int)$settings->get('enabled'),
-            'target' => $settings->get('target'),
-            'configured' => (int)$settings->configured(),
-            'password_confirmed' => (int)$settings->get('password_confirmed'),
-            'initialized' => (int)$settings->get('state_initialized'),
-            // the runner cannot decrypt the stored credentials (e.g. a database restored on another server): enter them again
-            'secrets_error' => ($settings->get('state_secrets_error') === '') ? null : $settings->get('state_secrets_error'),
-            // RESTIC_BACKUP_MODE (null until the runner reported it)
-            'mode' => ($settings->get('state_backup_mode') === '') ? null : $settings->get('state_backup_mode'),
-            'runner_seen' => $seen,
-            'runner_age' => ($seen === null) ? null : max(0, $now - $seen),
-            'runner_ok' => (int)($seen !== null && $now - $seen <= self::RUNNER_TIMEOUT),
-            'last_success' => ($settings->get('state_last_success') === '') ? null : (int)$settings->get('state_last_success'),
-            'repo_size' => ($settings->get('state_repo_size') === '') ? null : (int)$settings->get('state_repo_size'),
-            'repo_stats' => is_array($stats) ? $stats : null,
-            'snapshots' => count($snapshots),
-            'latest_snapshot' => empty($snapshots) ? null : $snapshots[0]->time,
-            'next_run' => $this->nextRun($settings, $now),
-            // the longest time between two scheduled backups (seconds): a later last success is overdue
-            'schedule_gap' => $settings->scheduleGap(),
-            'jobs' => array_map(fn(string $type): ?array => isset($latest[$type]) ? $latest[$type]->toArray() : null,
-                array_combine(Job::TYPES, Job::TYPES)),
-            'active' => array_map(fn(Job $job): array => $job->toArray(), $this->jobs->active()),
-        ];
-    }
-
-    /**
-     * Unix time of the next scheduled backup, null while the schedule is off. Like the runner: the schedule is armed
-     * (state_schedule_since/state_schedule) when it is switched on or changed, a run is due at the first schedule time
-     * after that; a due run without a scheduled backup since its time is the next one (now).
-     */
-    private function nextRun(Settings $settings, int $now): ?int
-    {
-        if ($settings->get('enabled') !== '1' || $settings->get('password_confirmed') !== '1' || !$settings->configured()) {
-            return null;
-        }
-        $slots = $settings->scheduleSlots();
-        try {
-            $zone = new DateTimeZone($this->timezone);
-        } catch (\Exception) {
-            $zone = new DateTimeZone('UTC');
-        }
-        $today = (new DateTimeImmutable('@' . $now))->setTimezone($zone);
-        // the last schedule time up to now, and the one after it
-        $due = self::slot($slots, $today, -1);
-        $next = self::slot($slots, $today, 1);
-        $since = $settings->get('state_schedule_since');
-        if ($due === null || $since === '' || $settings->get('state_schedule') !== $settings->get('schedule') || (int)$since > $due) {
-            return $next;
-        }
-        return $this->jobs->scheduledSince($due) ? $next : $due;
-    }
-
-    /**
-     * The newest schedule time up to $now ($direction -1) or the first one after it (1) as Unix time, null without
-     * slots ([weekday, hour, minute], sorted)
-     */
-    private static function slot(array $slots, DateTimeImmutable $now, int $direction): ?int
-    {
-        for ($i = 0; $i <= 7; $i++) {
-            $day = $now->modify(sprintf('%+d day', $direction * $i));
-            foreach (($direction < 0) ? array_reverse($slots) : $slots as [$weekday, $hour, $minute]) {
-                if ($weekday !== (int)$day->format('w')) {
-                    continue;
-                }
-                $slot = $day->setTime($hour, $minute)->getTimestamp();
-                if (($direction < 0) ? $slot <= $now->getTimestamp() : $slot > $now->getTimestamp()) {
-                    return $slot;
-                }
-            }
-        }
-        return null;
-    }
-
-    /** @throws AccessDenied unless the caller may request restores */
-    private function requireRestorer(): void
-    {
-        if ($this->context->isReadOnlyApi() || !($this->context->hasAdminRights() || $this->context->isDomainAdmin())) {
-            throw new AccessDenied();
-        }
-    }
-
-    /** Admins every job; domain admins the restores of their domains requested after the domain was created */
-    private function visible(Job $job): bool
-    {
-        if ($this->context->hasAdminRights()) {
-            return true;
-        }
-        if (!$this->context->isDomainAdmin() || $job->type !== 'restore' || $job->domain === '' || !$this->context->canAccessDomain($job->domain)) {
-            return false;
-        }
-        $created = $this->jobs->domainCreated(strtolower($job->domain));
-        return $created !== null && (int)$job->created >= $created;
-    }
-
-    /**
-     * A value of a job as a domain admin sees it: e-mail addresses outside $domains and repository details become "…"
-     *
-     * @param string[] $domains
-     */
-    private static function restricted(mixed $value, array $domains): mixed
-    {
-        if (is_array($value)) {
-            $restricted = [];
-            foreach ($value as $key => $item) {
-                $restricted[is_string($key) ? (string)self::restricted($key, $domains) : $key] = self::restricted($item, $domains);
-            }
-            return $restricted;
-        }
-        if (!is_string($value)) {
-            return $value;
-        }
-        $value = preg_replace(['~\b(?:s3|sftp|rest|b2|azure|gs|swift|rclone):\S+~i', '~\bhttps?://\S+~i', '~/repository\S*~'], '…', $value);
-        return preg_replace_callback('/[^\s@<>"\'(),;:\[\]]+@([a-z0-9.-]+\.[a-z]{2,})/i',
-            fn(array $match): string => in_array(strtolower($match[1]), $domains, true) ? $match[0] : '…', (string)$value);
-    }
-
-    /** @throws UserError backup_snapshot_not_found (also for snapshots a domain admin may not use) */
-    private function snapshot(mixed $value): Snapshot
-    {
-        $id = (string)Input::text($value);
-        $snapshot = ($id === '') ? null : $this->snapshots->find($id);
-        if ($snapshot === null || (!$this->context->hasAdminRights() && empty(array_filter($snapshot->domainList(),
-            fn(string $domain): bool => $this->mayUse($snapshot, $domain))))) {
-            throw new UserError('backup_snapshot_not_found', [htmlspecialchars($id)]);
-        }
-        return $snapshot;
-    }
-
-    private function visibleSnapshot(mixed $value): ?Snapshot
-    {
-        try {
-            return ($this->context->hasAdminRights() || $this->context->isDomainAdmin()) ? $this->snapshot($value) : null;
-        } catch (UserError) {
-            return null;
-        }
-    }
-
-    /**
-     * Admins every domain of the snapshot; domain admins the domains they administer now, when the snapshot holds this
-     * domain and not a former one of the same name: its recorded creation time is the domain's (snapshots without it:
-     * made after the domain was created)
-     */
-    private function mayUse(Snapshot $snapshot, string $domain): bool
-    {
-        if ($this->context->hasAdminRights()) {
-            return true;
-        }
-        if (!$this->context->isDomainAdmin()) {
-            return false;
-        }
-        $this->domainState[$domain] ??= [$this->context->canAccessDomain($domain), $this->jobs->domainCreated($domain)];
-        [$access, $created] = $this->domainState[$domain];
-        if (!$access) {
-            return false;
-        }
-        $recorded = $snapshot->createdOf($domain);
-        return $created !== null && (($recorded !== null) ? $recorded === $created : $snapshot->time > $created);
-    }
-
-    /**
-     * @return array{0: string, 1: string[]} the domain and its mailboxes in the snapshot (now, for snapshots without content)
-     * @throws UserError
-     */
-    private function domainScope(Snapshot $snapshot, mixed $value): array
-    {
-        $domain = strtolower((string)Input::text($value));
-        if ($domain !== '' && !$this->context->hasAdminRights() && !$this->context->canAccessDomain($domain)) {
-            throw new AccessDenied();
-        }
-        $exists = $domain !== '' && $this->jobs->isDomain($domain);
-        // a deleted domain is created from the snapshot's export: only when the content lists it
-        if (!$exists && !($this->context->hasAdminRights() && $snapshot->mailboxesOf($domain) !== null)) {
-            throw new UserError('domain_not_found', [htmlspecialchars((string)Input::text($value))]);
-        }
-        $this->requireDomain($snapshot, $domain);
-        $listed = $snapshot->mailboxesOf($domain);
-        $mailboxes = ($listed === null) ? $this->jobs->mailboxesOf($domain) : array_map('strval', array_keys($listed));
-        // a deleted domain without mailboxes comes back with its aliases and settings
-        if (empty($mailboxes) && $exists) {
-            throw new UserError('backup_mailboxes_empty');
-        }
-        return [$domain, $mailboxes];
-    }
-
-    /**
-     * @return array{0: string, 1: string[]} the domain of the mailboxes and the mailboxes
-     * @throws UserError
-     */
-    private function mailboxScope(Snapshot $snapshot, mixed $value, mixed $domainValue): array
-    {
-        $mailboxes = array_values(array_unique(array_map('strtolower', self::values($value))));
-        if (empty($mailboxes)) {
-            throw new UserError('backup_mailboxes_empty');
-        }
-        if (count($mailboxes) > self::MAX_MAILBOXES) {
-            throw new UserError('backup_entries_exceeded', [self::MAX_MAILBOXES]);
-        }
-        $domain = strtolower((string)Input::text($domainValue));
-        foreach ($mailboxes as $username) {
-            if (str_contains($username, '@') && !$this->context->hasAdminRights() && !$this->context->canAccessDomain(self::domainOf($username))) {
-                throw new AccessDenied();
-            }
-            // a deleted mailbox can be restored when the snapshot knows it
-            $mailboxDomain = $this->jobs->mailboxDomain($username) ?? (isset($snapshot->mailboxesOf(self::domainOf($username))[$username]) ? self::domainOf($username) : null);
-            if ($mailboxDomain === null) {
-                throw new UserError('backup_mailbox_not_found', [htmlspecialchars($username)]);
-            }
-            if ($domain !== '' && $mailboxDomain !== $domain) {
-                throw new UserError('backup_mailboxes_one_domain');
-            }
-            $domain = $mailboxDomain;
-        }
-        if (!$this->jobs->isDomain($domain) && !$this->context->hasAdminRights()) {
-            throw new AccessDenied();
-        }
-        $this->requireDomain($snapshot, $domain);
-        return [$domain, $mailboxes];
-    }
-
-    /** @throws UserError the snapshot does not hold the domain, or the caller may not restore into it */
-    private function requireDomain(Snapshot $snapshot, string $domain): void
-    {
-        if (!$this->context->hasAdminRights() && !$this->context->canAccessDomain($domain)) {
-            throw new AccessDenied();
-        }
-        if ($snapshot->knowsDomains() && !$snapshot->hasDomain($domain)) {
-            throw new UserError('backup_domain_not_in_snapshot', [htmlspecialchars($domain)]);
-        }
-        if (!$this->mayUse($snapshot, $domain)) {
-            throw new UserError('backup_snapshot_not_found', [htmlspecialchars($snapshot->shortId)]);
-        }
-    }
-
-    /**
-     * @return string[] the requested components in their order
-     * @throws UserError
-     */
-    private function components(mixed $value): array
-    {
-        $requested = array_values(array_unique(self::values($value)));
-        if (empty($requested)) {
-            throw new UserError('backup_components_invalid', ['']);
-        }
-        foreach ($requested as $component) {
-            if (!in_array($component, self::COMPONENTS, true)) {
-                throw new UserError('backup_components_invalid', [htmlspecialchars($component)]);
-            }
-            if (!$this->context->hasAdminRights() && isset(self::COMPONENT_ACL[$component]) && !$this->context->hasAcl(self::COMPONENT_ACL[$component])) {
-                throw new UserError('backup_component_denied', [$component]);
-            }
-            if (isset(self::REQUIRES[$component]) && !in_array(self::REQUIRES[$component], $requested, true)) {
-                throw new UserError('backup_component_requires', [$component, self::REQUIRES[$component]]);
-            }
-        }
-        return array_values(array_intersect(self::COMPONENTS, $requested));
-    }
-
-    private static function domainOf(string $address): string
-    {
-        return strtolower(substr((string)strrchr($address, '@'), 1));
-    }
-
-    /**
-     * Values of a list field: an array (multi-select, JSON) or a string separated by line breaks, commas or spaces.
-     *
-     * @return string[]
-     */
-    private static function values(mixed $value): array
-    {
-        $values = is_array($value) ? $value : preg_split('/[\s,;]+/', (string)Input::text($value));
-        return array_values(array_filter(array_map(fn(mixed $v): string => is_scalar($v) ? trim((string)$v) : '', $values), 'strlen'));
-    }
-}

data/web/inc/lib/mailcow/Backup/Service/SettingsService.php

@@ -1,327 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\Backup\Service;
-
-use Mailcow\Backup\Model\Settings;
-use Mailcow\Backup\Repository\SettingsRepository;
-use Mailcow\Core\Context;
-use Mailcow\Core\Input;
-use Mailcow\Core\UserError;
-
-/**
- * Backup settings: admins read them (also read-only API keys) and save them. Values are checked strictly (at least as
- * strictly as the runner), since the runner passes them to restic and ssh. Secrets are write-only and stored sealed
- * with the public key the runner published (only the runner can decrypt them); an empty secret keeps the stored one,
- * and without a published key no secret can be saved. A changed target, connection or repository password resets
- * state_initialized (the runner tests the repository again); a changed SFTP host or port drops the known host key. A
- * new repository password needs a new confirmation that it is stored outside mailcow. A change of a configured target,
- * its connection or the notify addresses leaves state_target_changed for the runner, which mails the notify addresses
- * of before the change.
- */
-class SettingsService
-{
-    private const FLAGS = ['enabled', 's3_path_style', 'password_confirmed', 'include_index', 'include_fts'];
-    /** Integer settings: name => [min, max] */
-    private const RANGES = [
-        'sftp_port' => [1, 65535],
-        'keep_hourly' => [0, 1000],
-        'keep_daily' => [0, 1000],
-        'keep_weekly' => [0, 1000],
-        'keep_monthly' => [0, 1000],
-        'keep_yearly' => [0, 1000],
-        'prune_weekday' => [0, 6],
-        'check_subset' => [0, 365],
-        'limit_upload' => [0, 100000000],
-    ];
-    private const MIN_PASSWORD = 8;
-    private const MAX_LENGTH = 255;
-    private const MAX_NOTIFY = 20;
-    /** Entries of the schedule */
-    private const MAX_SCHEDULE = 24;
-    private const HOSTNAME = '/^(?=.{1,253}$)[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*$/i';
-
-    public function __construct(private SettingsRepository $repo, private Context $context)
-    {
-    }
-
-    /** The settings for admins, null for everybody else */
-    public function get(): ?Settings
-    {
-        return $this->context->hasAdminRights() ? $this->load() : null;
-    }
-
-    /** The settings without permission check (status, job rules) */
-    public function load(): Settings
-    {
-        return Settings::fromStored($this->repo->load());
-    }
-
-    /**
-     * Saves the given settings; missing ones keep their value, unknown and runner-owned ones are ignored.
-     *
-     * @throws UserError
-     */
-    public function save(array $input): Settings
-    {
-        $this->context->requireAdmin();
-        $current = $this->load();
-        $values = $current->values;
-        foreach ($input as $name => $value) {
-            if (!is_string($name) || !array_key_exists($name, Settings::DEFAULTS) || in_array($name, Settings::RUNNER, true)) {
-                continue;
-            }
-            $value = $this->normalize($name, $value);
-            if (in_array($name, Settings::SECRETS, true)) {
-                if ($value === '') {
-                    continue;
-                }
-                $value = self::seal($value, $current->get('state_secrets_pubkey'));
-            }
-            $values[$name] = $value;
-        }
-        if ($values['password'] !== $current->get('password')
-            && !(array_key_exists('password_confirmed', $input) && $values['password_confirmed'] === '1')) {
-            $values['password_confirmed'] = '0';
-        }
-        foreach (array_merge(['target', 'password'], Settings::CONNECTION[$values['target']]) as $name) {
-            if ($values[$name] !== $current->get($name)) {
-                $values['state_initialized'] = '0';
-                break;
-            }
-        }
-        if ($values['sftp_host'] !== $current->get('sftp_host') || $values['sftp_port'] !== $current->get('sftp_port')) {
-            $values['sftp_host_key'] = '';
-        }
-        $settings = new Settings($values);
-        if ($values['enabled'] === '1' && !$settings->configured()) {
-            throw new UserError('backup_not_configured');
-        }
-        // switched on without a time
-        if ($values['enabled'] === '1' && empty($settings->scheduleSlots())) {
-            throw self::invalid('schedule');
-        }
-        $values['state_target_changed'] = $this->targetChange($current, $values);
-        $settings = new Settings($values);
-        $changed = array_diff_assoc($values, $current->values);
-        if (!empty($changed)) {
-            $this->repo->save($changed);
-        }
-        return $settings;
-    }
-
-    /**
-     * state_target_changed after a save: when the target, its connection or the notify addresses change, what the target
-     * was, whom to tell (the notify addresses before the change) and what changed ("target", "notify_email"). An earlier
-     * change the runner has not mailed yet keeps its state and addresses and only gets the new kinds of change, so new
-     * addresses followed by a new target still go to the original addresses. Nothing is recorded for a target that was
-     * not configured or without notify addresses.
-     */
-    private function targetChange(Settings $current, array $values): string
-    {
-        $pending = json_decode($current->get('state_target_changed'), true);
-        if (!$current->configured() && !is_array($pending)) {
-            return $current->get('state_target_changed');
-        }
-        $old = $current->get('target');
-        $watched = array_unique(array_merge(['target'], Settings::CONNECTION[$old] ?? [], Settings::CONNECTION[$values['target']] ?? []));
-        $changed = [];
-        if (!empty(array_filter($watched, fn(string $name): bool => $values[$name] !== $current->get($name)))) {
-            $changed[] = 'target';
-        }
-        if ($values['notify_email'] !== $current->get('notify_email')) {
-            $changed[] = 'notify_email';
-        }
-        if (is_array($pending)) {
-            $pending['changed'] = array_values(array_unique(array_merge((array)($pending['changed'] ?? ['target']), $changed)));
-            return (string)json_encode($pending, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
-        }
-        $addresses = array_values(array_filter(explode(',', $current->get('notify_email')), 'strlen'));
-        if (empty($changed) || empty($addresses)) {
-            return '';
-        }
-        $summary = ['target' => $old];
-        foreach (array_diff(Settings::CONNECTION[$old] ?? [], Settings::SECRETS) as $name) {
-            $summary[$name] = $current->get($name);
-        }
-        return (string)json_encode(['old_target' => $summary, 'old_addresses' => $addresses, 'changed' => $changed,
-            'changed_by' => (string)$this->context->username, 'time' => time()], JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
-    }
-
-    /**
-     * A secret sealed for the runner: sealed:v1:<base64 of sodium_crypto_box_seal>
-     *
-     * @throws UserError backup_secrets_key_missing while the runner has not published its key
-     */
-    private static function seal(string $secret, string $publicKey): string
-    {
-        $key = base64_decode($publicKey, true);
-        if ($key === false || strlen($key) !== SODIUM_CRYPTO_BOX_PUBLICKEYBYTES) {
-            throw new UserError('backup_secrets_key_missing');
-        }
-        return Settings::SEALED . base64_encode(sodium_crypto_box_seal($secret, $key));
-    }
-
-    /**
-     * The value to store, '' for an empty optional value.
-     *
-     * @throws UserError backup_setting_invalid, backup_password_too_short
-     */
-    private function normalize(string $name, mixed $value): string
-    {
-        if (in_array($name, self::FLAGS, true)) {
-            return self::flag($value) ?? throw self::invalid($name);
-        }
-        if ($name === 'notify_email' && is_string($value)) {
-            // one address per line is fine as well
-            $value = preg_replace('/[\r\n\t]+/', ',', $value);
-        }
-        if ($name === 'schedule') {
-            return self::schedule($value) ?? throw self::invalid($name);
-        }
-        // secrets as typed: spaces at the ends are part of a password
-        $text = (in_array($name, Settings::SECRETS, true) && is_string($value)) ? $value : Input::text($value);
-        if ($text === null || mb_strlen($text) > (($name === 'notify_email') ? self::MAX_LENGTH * self::MAX_NOTIFY : self::MAX_LENGTH) || preg_match('/[\x00-\x1f\x7f]/', $text) || !mb_check_encoding($text, 'UTF-8')) {
-            throw self::invalid($name);
-        }
-        // no scheduled prune (append-only or object-lock targets); a prune job still works
-        if ($name === 'prune_weekday' && $text === '') {
-            return '';
-        }
-        if (isset(self::RANGES[$name])) {
-            $int = Input::intInRange($text, ...self::RANGES[$name]);
-            return ($int === null) ? throw self::invalid($name) : (string)$int;
-        }
-        $valid = match ($name) {
-            'target' => in_array($text, Settings::TARGETS, true),
-            // like the runner: scheme, host and port only
-            's3_endpoint' => self::url($text) && in_array(parse_url($text, PHP_URL_PATH) ?? '', ['', '/'], true),
-            'rest_url' => $text === '' || self::url($text),
-            's3_bucket' => $text === '' || preg_match('/^[a-z0-9][a-z0-9._-]{1,62}$/i', $text),
-            's3_path' => $text === '' || self::path(trim($text, '/'), false),
-            's3_region' => $text === '' || preg_match('/^[a-z0-9-]{1,64}$/i', $text),
-            's3_access_key' => preg_match('/^[\x21-\x7e]*$/', $text),
-            's3_secret_key', 'rest_password' => true,
-            'sftp_host' => $text === '' || preg_match(self::HOSTNAME, $text) || filter_var($text, FILTER_VALIDATE_IP),
-            // the runner's character set (ssh -l)
-            'sftp_user' => $text === '' || preg_match('/^[a-z0-9_][a-z0-9._-]{0,63}$/i', $text),
-            'sftp_path' => $text === '' || self::path($text, true),
-            'rest_user' => $text === '' || preg_match('/^[a-z0-9][a-z0-9._-]{0,63}$/i', $text),
-            // the runner's character set, below the repository folder
-            'local_path' => preg_match('~^[a-z0-9._-]+(/[a-z0-9._-]+)*$~i', $text) && self::path($text, false),
-            'password' => true,
-            'max_unused' => preg_match('/^(\d{1,3})\s*%?$/', $text, $percent) && (int)$percent[1] <= 100,
-            'notify_email' => true,
-            default => false,
-        };
-        if (!$valid) {
-            throw self::invalid($name);
-        }
-        return match ($name) {
-            's3_path' => trim($text, '/'),
-            's3_endpoint' => rtrim($text, '/'),
-            'sftp_path' => rtrim($text, '/'),
-            'password' => self::password($text),
-            'max_unused' => (int)$text . '%',
-            'notify_email' => self::emails($text),
-            default => $text,
-        };
-    }
-
-    /**
-     * The schedule as stored: a list (or its JSON) of {days: weekdays 0 (Sunday) to 6, time: "H:MM"}, up to MAX_SCHEDULE
-     * entries, each with at least one weekday; days sorted, times as HH:MM, repeated entries dropped. Null when invalid.
-     */
-    private static function schedule(mixed $value): ?string
-    {
-        if (is_string($value)) {
-            $value = json_decode($value, true);
-        }
-        if (!is_array($value) || !array_is_list($value) || count($value) > self::MAX_SCHEDULE) {
-            return null;
-        }
-        $entries = [];
-        foreach ($value as $entry) {
-            $days = $entry['days'] ?? null;
-            $time = is_array($entry) && is_string($entry['time'] ?? null) ? $entry['time'] : '';
-            if (!is_array($days) || empty($days) || !preg_match('/^([01]?\d|2[0-3]):([0-5]\d)$/', $time, $match)) {
-                return null;
-            }
-            foreach ($days as $day) {
-                if (Input::intInRange($day, 0, 6) === null) {
-                    return null;
-                }
-            }
-            $days = array_values(array_unique(array_map('intval', $days)));
-            sort($days);
-            $entries[] = ['days' => $days, 'time' => sprintf('%02d:%02d', $match[1], $match[2])];
-        }
-        return (string)json_encode(array_values(array_unique($entries, SORT_REGULAR)));
-    }
-
-    /** http(s) URL without credentials, query or fragment (rest_user/rest_password hold the credentials) */
-    private static function url(string $url): bool
-    {
-        $parts = parse_url($url);
-        return filter_var($url, FILTER_VALIDATE_URL) !== false && is_array($parts) && in_array(strtolower($parts['scheme'] ?? ''), ['http', 'https'], true)
-            && !isset($parts['user']) && !isset($parts['pass']) && !isset($parts['query']) && !isset($parts['fragment'])
-            && (preg_match(self::HOSTNAME, (string)($parts['host'] ?? '')) || filter_var(trim((string)($parts['host'] ?? ''), '[]'), FILTER_VALIDATE_IP));
-    }
-
-    /** A path of plain names separated by /, without . and .. parts; a leading / only where $absolute is allowed */
-    private static function path(string $path, bool $absolute): bool
-    {
-        if ($path === '' || (str_starts_with($path, '/') && !$absolute)) {
-            return false;
-        }
-        foreach (explode('/', trim($path, '/')) as $part) {
-            if ($part === '.' || $part === '..' || !preg_match('/^[a-z0-9_~+@][a-z0-9._~+@-]*$/i', $part)) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-    /** @throws UserError backup_password_too_short ('' keeps the stored password) */
-    private static function password(string $password): string
-    {
-        if ($password !== '' && mb_strlen($password) < self::MIN_PASSWORD) {
-            throw new UserError('backup_password_too_short', [self::MIN_PASSWORD]);
-        }
-        return $password;
-    }
-
-    /** @throws UserError for an invalid address or too many */
-    private static function emails(string $text): string
-    {
-        $emails = array_values(array_unique(array_filter(preg_split('/[\s,;]+/', $text), 'strlen')));
-        if (count($emails) > self::MAX_NOTIFY) {
-            throw self::invalid('notify_email');
-        }
-        foreach ($emails as $email) {
-            if (!filter_var($email, FILTER_VALIDATE_EMAIL)) {
-                throw self::invalid('notify_email');
-            }
-        }
-        return implode(',', $emails);
-    }
-
-    /** Checkbox or API flag as '0'/'1'; a hidden "0" + checkbox pair arrives as ["0", "1"] when checked; null = invalid */
-    private static function flag(mixed $value): ?string
-    {
-        if (is_array($value)) {
-            $value = end($value);
-        }
-        if (!is_scalar($value) && $value !== null) {
-            return null;
-        }
-        $flag = filter_var($value ?? false, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE);
-        return ($flag === null) ? null : (string)(int)$flag;
-    }
-
-    private static function invalid(string $name): UserError
-    {
-        return new UserError('backup_setting_invalid', [$name]);
-    }
-}

data/web/inc/lib/mailcow/MailPolicy/Controller/MailPolicyApiController.php

@@ -1,132 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailPolicy\Controller;
-
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\UserError;
-use Mailcow\MailPolicy\Service\PolicyEngine;
-use Mailcow\MailPolicy\Service\PolicyService;
-use PDOException;
-
-/**
- * HTTP API adapter for json_api.php and edit.php (routes `mail-policy`), and the answer to the Postfix policy
- * service (data/conf/rspamd/dynmaps/mailpolicy.php). Messages are logged as ['mail_policy', action, 'policy', data].
- */
-class MailPolicyApiController
-{
-    public function __construct(private PolicyService $policies, private PolicyEngine $engine, private ApiResponse $response)
-    {
-    }
-
-    /** POST add/mail-policy */
-    public function addPolicy(array $attr): bool
-    {
-        return $this->handle('add', $attr, function (array $log) use ($attr): bool {
-            $policy = $this->policies->create($attr);
-            return $this->response->success(['mail_policy_added', htmlspecialchars($policy->name)], $log);
-        });
-    }
-
-    /** POST edit/mail-policy: one message per id */
-    public function editPolicies(?array $items, array $attr): bool
-    {
-        $data = array_merge(['id' => $items], $attr);
-        return $this->handle('edit', $data, function (array $log) use ($data, $attr): bool {
-            $this->policies->requireEditor();
-            return $this->batch((array)($data['id'] ?? []), $log,
-                fn(int $id): array => ['mail_policy_modified', htmlspecialchars($this->policies->update($id, $attr)->name)]);
-        });
-    }
-
-    /** POST delete/mail-policy: one message per id */
-    public function deletePolicies(array $items): bool
-    {
-        return $this->handle('delete', ['id' => $items], function (array $log) use ($items): bool {
-            $this->policies->requireEditor();
-            return $this->batch($items, $log, fn(int $id): array => ['mail_policy_removed', htmlspecialchars($this->policies->delete($id)->name)]);
-        });
-    }
-
-    /** GET mail-policy/all[/<domain,domain>] */
-    public function getPolicies(?string $domains = null): array
-    {
-        $list = array_values(array_filter(array_map('trim', explode(',', (string)$domains)), 'strlen'));
-        return array_values(array_filter(array_map(fn(int $id): array => $this->getPolicy($id), $this->policies->visibleIds($list))));
-    }
-
-    /** GET mail-policy/<id>, also edit.php */
-    public function getPolicy(mixed $id): array
-    {
-        $id = $this->toId($id);
-        $policy = ($id === null) ? null : $this->policies->get($id);
-        return ($policy === null) ? [] : $this->policies->toArray($policy);
-    }
-
-    /** GET mail-policy/test/<sender>/<recipient>[/login]: would the mail be accepted? Sender "-" = null sender; [] when not allowed to ask */
-    public function test(string $sender, string $recipient, bool $loggedIn): array
-    {
-        try {
-            return $this->policies->test(($sender === '-') ? '' : $sender, $recipient, $loggedIn);
-        } catch (UserError) {
-            return [];
-        }
-    }
-
-    /** functions.mailbox.inc.php: a mailbox or alias was renamed */
-    public function addressRenamed(string $old, string $new): void
-    {
-        $this->policies->addressRenamed($old, $new);
-    }
-
-    /** Answer for the Postfix policy service: DUNNO or the rejection */
-    public function check(string $sender, string $recipient, string $login, int $port, string $helo = ''): string
-    {
-        return $this->engine->check($sender, $recipient, $login, $port, $helo)->action();
-    }
-
-    /**
-     * Runs one write call: $work gets the log entry and returns the result for process_*_return().
-     * A UserError becomes a danger message, a database error stops the call with one `mysql_error` message.
-     */
-    private function handle(string $action, mixed $data, callable $work): bool
-    {
-        $log = ['mail_policy', $action, 'policy', $data];
-        try {
-            $result = $work($log);
-        } catch (UserError $e) {
-            $result = $this->response->error($e, $log);
-        } catch (PDOException $e) {
-            $result = $this->response->danger(['mysql_error', $e->getMessage()], $log);
-        }
-        $this->response->flush();
-        return $result;
-    }
-
-    /** One message per id: the success message returned by $work(id), or the UserError */
-    private function batch(array $ids, array $log, callable $work): bool
-    {
-        foreach ($ids as $id) {
-            try {
-                $this->response->success($work($this->requireId($id)), $log);
-            } catch (UserError $e) {
-                $this->response->error($e, $log);
-            }
-        }
-        return true;
-    }
-
-    /** Integer id from request input ("12", 12); null for anything else */
-    private function toId(mixed $id): ?int
-    {
-        return (is_numeric($id) && (float)$id == (int)(float)$id) ? (int)(float)$id : null;
-    }
-
-    /** @throws AccessDenied for ids that cannot exist */
-    private function requireId(mixed $id): int
-    {
-        return $this->toId($id) ?? throw new AccessDenied();
-    }
-}

data/web/inc/lib/mailcow/MailPolicy/MailPolicyModule.php

@@ -1,44 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailPolicy;
-
-use Mailcow\Core\ApiResponse;
-use Mailcow\Core\Context;
-use Mailcow\MailPolicy\Controller\MailPolicyApiController;
-use Mailcow\MailPolicy\Repository\LookupRepository;
-use Mailcow\MailPolicy\Repository\PolicyRepository;
-use Mailcow\MailPolicy\Service\PolicyEngine;
-use Mailcow\MailPolicy\Service\PolicyService;
-use PDO;
-
-/**
- * Composition root of the mail policies: whom mailboxes and groups may send to and receive from, checked by
- * Postfix at RCPT TO through the policy service (mailcow-policy in master.cf, data/conf/rspamd/dynmaps/mailpolicy.php).
- *
- *   process_get_return(MailPolicyModule::fromGlobals()->api->getPolicies());
- *   echo MailPolicyModule::create($pdo, Context::system())->api->check($sender, $recipient, $login, $port);  // policy service
- */
-class MailPolicyModule
-{
-    public function __construct(
-        public readonly PolicyService $policies,
-        public readonly PolicyEngine $engine,
-        public readonly MailPolicyApiController $api
-    ) {
-    }
-
-    public static function create(PDO $pdo, Context $context): self
-    {
-        $engine = new PolicyEngine(new LookupRepository($pdo));
-        $policies = new PolicyService(new PolicyRepository($pdo), $engine, $context);
-        return new self($policies, $engine, new MailPolicyApiController($policies, $engine, new ApiResponse()));
-    }
-
-    /** Legacy entry point for procedural callers: reads $pdo and $_SESSION */
-    public static function fromGlobals(?Context $context = null): self
-    {
-        return self::create($GLOBALS['pdo'], $context ?? Context::fromSession($_SESSION ?? []));
-    }
-}

data/web/inc/lib/mailcow/MailPolicy/Model/Decision.php

@@ -1,56 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailPolicy\Model;
-
-/**
- * Result of the policy check for one recipient: allowed, refused by the send or receive part of a policy, or refused
- * because the recipient is written in a form the check cannot follow (routing through another address).
- */
-class Decision
-{
-    private const TEXT = [
-        'send' => 'Sending to this recipient is not allowed by the mail policy of the sender',
-        'receive' => 'This recipient does not accept mail from this sender',
-        'address' => 'This form of the recipient address is not allowed by the mail policy',
-    ];
-
-    private function __construct(public readonly bool $allowed, public readonly string $side = '', public readonly ?Policy $policy = null)
-    {
-    }
-
-    public static function allow(): self
-    {
-        return new self(true);
-    }
-
-    /** $side send or receive; without $policy the default text is used */
-    public static function deny(string $side, ?Policy $policy = null): self
-    {
-        return new self(false, $side, $policy);
-    }
-
-    /** The recipient routes through another address (user%domain, domain!user, a second @) */
-    public static function routedAddress(): self
-    {
-        return new self(false, 'address');
-    }
-
-    /** Text of the rejection: the text of the policy or the default */
-    public function text(): string
-    {
-        return ($this->policy !== null && $this->policy->rejectText !== '') ? $this->policy->rejectText : self::defaultText($this->side);
-    }
-
-    public static function defaultText(string $side): string
-    {
-        return self::TEXT[$side] ?? '';
-    }
-
-    /** Answer for the Postfix policy protocol */
-    public function action(): string
-    {
-        return $this->allowed ? 'DUNNO' : '550 5.7.1 ' . $this->text();
-    }
-}

data/web/inc/lib/mailcow/MailPolicy/Model/Policy.php

@@ -1,111 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailPolicy\Model;
-
-/**
- * A mail policy (table `mail_policy`): whom its mailboxes and groups may send to and receive from.
- *
- * Each direction has a mode: `all` (only the blocked entries are refused), `internal` (the own domain and its
- * alias domains, plus the allowed entries) or `list` (only the allowed entries). Blocked entries are always
- * refused. Entries are addresses, domains or `group:<id>`. A sender counts as internal only when it is logged in
- * to this mailcow (SASL) with a mailbox of the domain, never because of its address.
- */
-class Policy
-{
-    public const MODES = ['all', 'internal', 'list'];
-
-    public ?int $id = null;
-    public string $domain = '';
-    public string $name = '';
-    public string $description = '';
-    public string $sendMode = 'all';
-    /** @var string[] */
-    public array $sendAllow = [];
-    /** @var string[] */
-    public array $sendBlock = [];
-    public string $receiveMode = 'all';
-    /** @var string[] */
-    public array $receiveAllow = [];
-    /** @var string[] */
-    public array $receiveBlock = [];
-    /** Text of the rejection, '' = default text */
-    public string $rejectText = '';
-    public int $active = 1;
-    /** @var string[] usernames the policy is assigned to */
-    public array $mailboxes = [];
-    /** @var int[] groups the policy is assigned to: their members and their address */
-    public array $groups = [];
-    public ?string $created = null;
-    public ?string $modified = null;
-
-    /** Row of `mail_policy`; assignments are set by the repository */
-    public static function fromRow(array $row): self
-    {
-        $policy = new self();
-        $policy->id = (int)$row['id'];
-        $policy->domain = (string)$row['domain'];
-        $policy->name = (string)$row['name'];
-        $policy->description = (string)($row['description'] ?? '');
-        $policy->sendMode = (string)$row['send_mode'];
-        $policy->sendAllow = self::list($row['send_allow'] ?? null);
-        $policy->sendBlock = self::list($row['send_block'] ?? null);
-        $policy->receiveMode = (string)$row['receive_mode'];
-        $policy->receiveAllow = self::list($row['receive_allow'] ?? null);
-        $policy->receiveBlock = self::list($row['receive_block'] ?? null);
-        $policy->rejectText = (string)($row['reject_text'] ?? '');
-        $policy->active = (int)$row['active'];
-        $policy->created = $row['created'] ?? null;
-        $policy->modified = $row['modified'] ?? null;
-        return $policy;
-    }
-
-    /** Column values for `mail_policy` */
-    public function toRow(): array
-    {
-        return [
-            'domain' => $this->domain,
-            'name' => $this->name,
-            'description' => $this->description,
-            'send_mode' => $this->sendMode,
-            'send_allow' => json_encode($this->sendAllow),
-            'send_block' => json_encode($this->sendBlock),
-            'receive_mode' => $this->receiveMode,
-            'receive_allow' => json_encode($this->receiveAllow),
-            'receive_block' => json_encode($this->receiveBlock),
-            'reject_text' => $this->rejectText,
-            'active' => $this->active,
-        ];
-    }
-
-    /** Entry of the API and the edit page */
-    public function toArray(): array
-    {
-        return [
-            'id' => $this->id,
-            'domain' => $this->domain,
-            'name' => $this->name,
-            'description' => $this->description,
-            'send_mode' => $this->sendMode,
-            'send_allow' => $this->sendAllow,
-            'send_block' => $this->sendBlock,
-            'receive_mode' => $this->receiveMode,
-            'receive_allow' => $this->receiveAllow,
-            'receive_block' => $this->receiveBlock,
-            'reject_text' => $this->rejectText,
-            'active' => $this->active,
-            'mailboxes' => $this->mailboxes,
-            'groups' => $this->groups,
-            'created' => $this->created,
-            'modified' => $this->modified,
-        ];
-    }
-
-    /** @return string[] entries of a JSON list column */
-    private static function list(mixed $json): array
-    {
-        $list = is_string($json) ? json_decode($json, true) : null;
-        return is_array($list) ? array_values(array_filter($list, 'is_string')) : [];
-    }
-}

data/web/inc/lib/mailcow/MailPolicy/Repository/LookupRepository.php

@@ -1,141 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailPolicy\Repository;
-
-use Mailcow\Core\Repository;
-use Mailcow\MailPolicy\Model\Policy;
-
-/**
- * Read-only lookups of the policy check: mailboxes, aliases, alias domains, groups and the active policies of a
- * mailbox or group. Addresses are passed in lower case.
- */
-class LookupRepository extends Repository
-{
-    /** @return array{username: string, domain: string}|null a regular mailbox that is not disabled */
-    public function mailbox(string $username): ?array
-    {
-        $row = $this->query("SELECT `username`, `domain` FROM `mailbox` WHERE `username` = :username AND `kind` = '' AND `active` != '0'",
-            ['username' => $username])->fetch();
-        return is_array($row) ? ['username' => strtolower((string)$row['username']), 'domain' => strtolower((string)$row['domain'])] : null;
-    }
-
-    /** Target domain of an active alias domain */
-    public function aliasDomainTarget(string $domain): ?string
-    {
-        $target = $this->query("SELECT `target_domain` FROM `alias_domain` WHERE `alias_domain` = :domain AND `active` = '1'",
-            ['domain' => $domain])->fetchColumn();
-        return is_string($target) ? strtolower($target) : null;
-    }
-
-    /** A domain or alias domain Postfix delivers itself (as mysql_virtual_domains_maps.cf: active, no backup MX) */
-    public function isLocalDomain(string $domain): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `alias_domain` WHERE `alias_domain` = :d1 AND `active` = '1'
-          UNION SELECT 1 FROM `domain` WHERE `domain` = :d2 AND `active` = '1' AND `backupmx` = '0'",
-            ['d1' => $domain, 'd2' => $domain])->fetchColumn();
-    }
-
-    /**
-     * Stored spelling of a mailbox, alias or temporary alias address. MySQL compares without case and accents, so
-     * Postfix delivers mail for chéf@ to chef@.
-     */
-    public function storedAddress(string $address): ?string
-    {
-        $stored = $this->query("SELECT `username` FROM `mailbox` WHERE `username` = :a1 UNION SELECT `address` FROM `alias` WHERE `address` = :a2
-          UNION SELECT `address` FROM `spamalias` WHERE `address` = :a3 LIMIT 1", ['a1' => $address, 'a2' => $address, 'a3' => $address])->fetchColumn();
-        return is_string($stored) ? strtolower($stored) : null;
-    }
-
-    /** Stored spelling of a domain or alias domain, see storedAddress() */
-    public function storedDomain(string $domain): ?string
-    {
-        $stored = $this->query("SELECT `domain` FROM `domain` WHERE `domain` = :d1 UNION SELECT `alias_domain` FROM `alias_domain` WHERE `alias_domain` = :d2 LIMIT 1",
-            ['d1' => $domain, 'd2' => $domain])->fetchColumn();
-        return is_string($stored) ? strtolower($stored) : null;
-    }
-
-    /** Active group with this distribution address */
-    public function groupByAddress(string $address): ?int
-    {
-        $id = $this->query("SELECT `id` FROM `mailbox_group` WHERE `address` = :address AND `active` = '1'", ['address' => $address])->fetchColumn();
-        return ($id === false) ? null : (int)$id;
-    }
-
-    public function groupAddress(int $id): ?string
-    {
-        $address = $this->query("SELECT `address` FROM `mailbox_group` WHERE `id` = :id AND `active` = '1'", ['id' => $id])->fetchColumn();
-        return is_string($address) && $address !== '' ? strtolower($address) : null;
-    }
-
-    /** Member of an active group */
-    public function isGroupMember(int $groupId, string $username): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `mailbox_group_member` `gm` JOIN `mailbox_group` `g` ON `g`.`id` = `gm`.`group_id`
-          WHERE `gm`.`group_id` = :id AND `gm`.`username` = :username AND `g`.`active` = '1'",
-            ['id' => $groupId, 'username' => $username])->fetchColumn();
-    }
-
-    /**
-     * What Postfix' virtual alias maps find for this key, in their order: an alias (also the own alias of a mailbox
-     * and the alias of a group address) or a temporary alias.
-     *
-     * @return array{kind: string, domain?: string, internal?: bool, owner?: string}|null
-     */
-    public function aliasTarget(string $address): ?array
-    {
-        $alias = $this->query("SELECT `domain`, `internal` FROM `alias` WHERE `address` = :address AND `active` IN ('1', '2')",
-            ['address' => $address])->fetch();
-        if (is_array($alias)) {
-            return ['kind' => 'alias', 'domain' => strtolower((string)$alias['domain']), 'internal' => $alias['internal'] == '1'];
-        }
-        $owner = $this->query("SELECT `goto` FROM `spamalias` WHERE `address` = :address AND (`validity` >= UNIX_TIMESTAMP() OR `permanent` != 0)",
-            ['address' => $address])->fetchColumn();
-        return is_string($owner) ? ['kind' => 'temporary', 'owner' => strtolower($owner)] : null;
-    }
-
-    /** An active catch-all alias of the domain has the internal flag (applies to the whole domain) */
-    public function internalCatchAll(string $domain): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `alias` WHERE `address` = :address AND `internal` = '1' AND `active` IN ('1', '2')",
-            ['address' => '@' . $domain])->fetchColumn();
-    }
-
-    /** An active policy with members restricts sending */
-    public function hasSendRules(): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `mail_policy` `p` JOIN `mail_policy_member` `pm` ON `pm`.`policy_id` = `p`.`id`
-          WHERE `p`.`active` = '1' AND (`p`.`send_mode` != 'all' OR `p`.`send_block` != '[]') LIMIT 1")->fetchColumn();
-    }
-
-    /**
-     * @return Policy[] active policies of the mailbox: assigned to it or to an active group it is a member of; only
-     *                  policies of its own domain, so a group with members of other domains cannot restrict them
-     */
-    public function policiesForMailbox(string $username): array
-    {
-        return $this->policies("SELECT DISTINCT `p`.* FROM `mail_policy` `p`
-          JOIN `mail_policy_member` `pm` ON `pm`.`policy_id` = `p`.`id`
-          JOIN `mailbox` `m` ON `m`.`username` = :u1 AND `m`.`domain` = `p`.`domain`
-          WHERE `p`.`active` = '1' AND (`pm`.`username` = :u2 OR `pm`.`group_id` IN (
-            SELECT `gm`.`group_id` FROM `mailbox_group_member` `gm` JOIN `mailbox_group` `g` ON `g`.`id` = `gm`.`group_id`
-            WHERE `gm`.`username` = :u3 AND `g`.`active` = '1'))
-          ORDER BY `p`.`id`", ['u1' => $username, 'u2' => $username, 'u3' => $username]);
-    }
-
-    /** @return Policy[] active policies assigned to the group */
-    public function policiesForGroup(int $groupId): array
-    {
-        return $this->policies("SELECT DISTINCT `p`.* FROM `mail_policy` `p`
-          JOIN `mail_policy_member` `pm` ON `pm`.`policy_id` = `p`.`id`
-          WHERE `p`.`active` = '1' AND `pm`.`group_id` = :id
-          ORDER BY `p`.`id`", ['id' => $groupId]);
-    }
-
-    /** @return Policy[] */
-    private function policies(string $sql, array $params): array
-    {
-        return array_map(fn(array $row): Policy => Policy::fromRow($row), $this->query($sql, $params)->fetchAll());
-    }
-}

data/web/inc/lib/mailcow/MailPolicy/Repository/PolicyRepository.php

@@ -1,148 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailPolicy\Repository;
-
-use Mailcow\Core\Context;
-use Mailcow\Core\Repository;
-use Mailcow\MailPolicy\Model\Policy;
-use PDO;
-
-/**
- * Tables `mail_policy` and `mail_policy_member` (the mailboxes and groups a policy is assigned to), plus the
- * lookups the editing needs. Assignments go with their mailbox, group or domain (foreign keys, ON DELETE CASCADE).
- */
-class PolicyRepository extends Repository
-{
-    public function find(int $id): ?Policy
-    {
-        $row = $this->query("SELECT * FROM `mail_policy` WHERE `id` = :id", ['id' => $id])->fetch();
-        return is_array($row) ? $this->withMembers(Policy::fromRow($row)) : null;
-    }
-
-    /**
-     * @param string[] $domains limit to these domains, [] = all visible
-     * @return int[] ids of the policies visible to the caller, ordered by domain and name
-     */
-    public function visibleIds(Context $context, array $domains = []): array
-    {
-        if ($context->hasAdminRights()) {
-            [$visible, $params] = ['1=1', []];
-        } elseif ($context->isDomainAdmin()) {
-            [$visible, $params] = ["`domain` IN (SELECT `domain` FROM `domain_admins` WHERE `active` = '1' AND `username` = :vis_da)",
-                ['vis_da' => $context->username]];
-        } else {
-            return [];
-        }
-        $sql = "SELECT `id` FROM `mail_policy` WHERE $visible";
-        if (!empty($domains)) {
-            $names = [];
-            foreach (array_values($domains) as $i => $domain) {
-                $names[] = ':dom' . $i;
-                $params['dom' . $i] = $domain;
-            }
-            $sql .= " AND `domain` IN (" . implode(', ', $names) . ")";
-        }
-        return array_map('intval', $this->query($sql . " ORDER BY `domain`, `name`", $params)->fetchAll(PDO::FETCH_COLUMN));
-    }
-
-    public function nameExists(string $domain, string $name, ?int $exceptId = null): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `mail_policy` WHERE `domain` = :domain AND `name` = :name AND `id` != :id",
-            ['domain' => $domain, 'name' => $name, 'id' => $exceptId ?? 0])->fetchColumn();
-    }
-
-    /** Inserts or updates the policy with its assignments; returns its id */
-    public function save(Policy $policy): int
-    {
-        return $this->transaction(function () use ($policy): int {
-            if ($policy->id === null) {
-                $policy->id = $this->insertRow('mail_policy', $policy->toRow());
-            } else {
-                $this->updateRow('mail_policy', $policy->id, $policy->toRow());
-                $this->query("DELETE FROM `mail_policy_member` WHERE `policy_id` = :id", ['id' => $policy->id]);
-            }
-            foreach ($policy->mailboxes as $username) {
-                $this->insertRow('mail_policy_member', ['policy_id' => $policy->id, 'username' => $username]);
-            }
-            foreach ($policy->groups as $groupId) {
-                $this->insertRow('mail_policy_member', ['policy_id' => $policy->id, 'group_id' => $groupId]);
-            }
-            return $policy->id;
-        });
-    }
-
-    public function delete(int $id): void
-    {
-        $this->query("DELETE FROM `mail_policy` WHERE `id` = :id", ['id' => $id]);
-    }
-
-    /** Address entries of all lists follow a renamed mailbox or alias */
-    public function renameAddress(string $old, string $new): void
-    {
-        $this->transaction(function () use ($old, $new): void {
-            $columns = ['send_allow', 'send_block', 'receive_allow', 'receive_block'];
-            foreach ($this->query("SELECT `id`, `" . implode('`, `', $columns) . "` FROM `mail_policy`")->fetchAll() as $row) {
-                $changed = [];
-                foreach ($columns as $column) {
-                    $entries = json_decode((string)$row[$column], true);
-                    if (is_array($entries) && in_array($old, $entries, true)) {
-                        $changed[$column] = json_encode(array_values(array_unique(array_map(fn(string $entry): string => ($entry === $old) ? $new : $entry, $entries))));
-                    }
-                }
-                if ($changed !== []) {
-                    $this->updateRow('mail_policy', (int)$row['id'], $changed);
-                }
-            }
-        });
-    }
-
-    /** A primary domain (policies never belong to alias domains) */
-    public function isDomain(string $domain): bool
-    {
-        return (bool)$this->query("SELECT 1 FROM `domain` WHERE `domain` = :domain", ['domain' => $domain])->fetchColumn();
-    }
-
-    /** Domain of a regular mailbox, null if there is none */
-    public function mailboxDomain(string $username): ?string
-    {
-        $domain = $this->query("SELECT `domain` FROM `mailbox` WHERE `username` = :username AND `kind` = ''", ['username' => $username])->fetchColumn();
-        return is_string($domain) ? $domain : null;
-    }
-
-    public function groupDomain(int $id): ?string
-    {
-        $domain = $this->query("SELECT `domain` FROM `mailbox_group` WHERE `id` = :id", ['id' => $id])->fetchColumn();
-        return is_string($domain) ? $domain : null;
-    }
-
-    /** @return array[] id, name, domain, address of the groups in these domains (choices of the forms) */
-    public function groupsIn(array $domains): array
-    {
-        if (empty($domains)) {
-            return [];
-        }
-        $names = [];
-        $params = [];
-        foreach (array_values($domains) as $i => $domain) {
-            $names[] = ':dom' . $i;
-            $params['dom' . $i] = $domain;
-        }
-        return $this->query("SELECT `id`, `name`, `domain`, `address` FROM `mailbox_group` WHERE `domain` IN (" . implode(', ', $names) . ")
-          ORDER BY `domain`, `name`", $params)->fetchAll();
-    }
-
-    private function withMembers(Policy $policy): Policy
-    {
-        foreach ($this->query("SELECT `username`, `group_id` FROM `mail_policy_member` WHERE `policy_id` = :id ORDER BY `username`, `group_id`",
-            ['id' => $policy->id])->fetchAll() as $row) {
-            if ($row['username'] !== null) {
-                $policy->mailboxes[] = (string)$row['username'];
-            } elseif ($row['group_id'] !== null) {
-                $policy->groups[] = (int)$row['group_id'];
-            }
-        }
-        return $policy;
-    }
-}

data/web/inc/lib/mailcow/MailPolicy/Service/PolicyEngine.php

@@ -1,327 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailPolicy\Service;
-
-use Mailcow\MailPolicy\Model\Decision;
-use Mailcow\MailPolicy\Model\Policy;
-use Mailcow\MailPolicy\Repository\LookupRepository;
-
-/**
- * Policy check of one recipient, asked by Postfix at RCPT TO (data/conf/rspamd/dynmaps/mailpolicy.php).
- *
- * Send part: the policies of the logged-in mailbox (assigned to it or to its groups) and of the mailbox the
- * sender address belongs to. Receive part: what Postfix delivers the recipient to, looked up in the order of its
- * virtual alias maps (address, address without +tag, mailbox of an alias domain): the policies of the mailbox, of
- * the group address or of the owner of a temporary alias, and the `internal` flag of the alias (a catch-all alias
- * with the flag covers its whole domain). Mail sent to an alias or group reaches its members without their own
- * policies. Every policy must allow the mail.
- *
- * Internal: a recipient in the domain of the policy or one of its alias domains; a sender logged in with a
- * mailbox of that domain. Entries of a local domain or address count for logged-in senders only, so they cannot
- * be forged; block entries also match the envelope sender. External entries are compared with the envelope sender.
- * Addresses are compared the way Postfix finds them: lower case, without trailing dot, in their stored spelling
- * (MySQL ignores accents). Recipients routed through localhost (user%domain@localhost, domain!user@localhost,
- * user@domain@localhost) are refused, Postfix would deliver them to the routed address.
- *
- * Dovecot submits Sieve redirects, vacation replies and notifications on port 588 without login, SOGo sends
- * reminders with a service login: mail with the address of a mailbox counts as sent by that mailbox. Other mail of
- * these services (bounces redirected with the null sender, Sieve rejects) has no known sender: it is checked like
- * mail from outside and does not leave the mailcow while any policy restricts sending. Quota warnings pass.
- */
-class PolicyEngine
-{
-    /** Ports of mailcow's own deliveries: quarantine release (590), BCC and archive copies (591) */
-    private const INTERNAL_PORTS = [590, 591];
-    /** Submission of SOGo and Dovecot, without login reachable from mailcow's own network only */
-    private const SERVICE_PORT = 588;
-    /** Domain of the Dovecot master login of mailcow's services (SOGo reminders) */
-    private const SERVICE_LOGIN_DOMAIN = 'mailcow.local';
-    /** HELO of the quota warning copies of Dovecot (quota_notify.py), configured by admins */
-    private const QUOTA_HELO = 'quotanotification';
-    /** Recipient without domain every server accepts (RFC 5321) */
-    private const POSTMASTER = 'postmaster';
-    /** Postfix delivers these destinations itself (mydestination), user%domain and domain!user route from there */
-    private const LOCAL_DESTINATIONS = ['localhost', 'localhost.localdomain'];
-
-    /** Lookups of the current check */
-    private array $cache = [];
-
-    public function __construct(private LookupRepository $lookup)
-    {
-    }
-
-    /** $login: SASL username, '' if not logged in; $port: smtpd port of the request, 0 if unknown; $helo: HELO name */
-    public function check(string $sender, string $recipient, string $login, int $port = 0, string $helo = ''): Decision
-    {
-        $this->cache = [];
-        $recipient = self::address($recipient);
-        if (in_array($port, self::INTERNAL_PORTS, true) || $recipient === '' || $recipient === self::POSTMASTER) {
-            return Decision::allow();
-        }
-        if (self::isRouted($recipient)) {
-            return Decision::routedAddress();
-        }
-        $recipient = $this->stored($recipient);
-        $sender = $this->stored(self::address($sender));
-        $login = self::address($login);
-        $serviceLogin = self::domainOf($login) === self::SERVICE_LOGIN_DOMAIN;
-        $account = null;
-        if ($login !== '' && !$serviceLogin) {
-            $account = $this->cached('mailbox', $login);
-        } elseif ($serviceLogin || $port === self::SERVICE_PORT) {
-            if ($port === self::SERVICE_PORT && !$serviceLogin && strtolower($helo) === self::QUOTA_HELO) {
-                return Decision::allow();
-            }
-            // without a mailbox sender the sending mailbox is unknown
-            $account = $this->senderMailbox($sender);
-            if ($account === null && !$this->cached('isLocalDomain', self::domainOf($recipient)) && $this->lookup->hasSendRules()) {
-                return Decision::deny('send');
-            }
-        }
-        foreach ($this->sendingPolicies($account, $sender) as $policy) {
-            if (!$this->allows($policy->sendMode, $policy->sendAllow, $policy->sendBlock, $this->isInternalRecipient($recipient, $policy->domain),
-                fn(string $entry): bool => $this->recipientMatches($entry, $recipient))) {
-                return Decision::deny('send', $policy);
-            }
-        }
-        foreach ($this->receivingPolicies($recipient) as $policy) {
-            if (!$this->allows($policy->receiveMode, $policy->receiveAllow, $policy->receiveBlock,
-                $account !== null && $account['domain'] === $policy->domain,
-                fn(string $entry, bool $block): bool => $this->senderMatches($entry, $sender, $account, $block))) {
-                return Decision::deny('receive', $policy);
-            }
-        }
-        return Decision::allow();
-    }
-
-    /** Blocked entries first; then the mode decides. $matches($entry, $block) */
-    private function allows(string $mode, array $allow, array $block, bool $internal, callable $matches): bool
-    {
-        foreach ($block as $entry) {
-            if ($matches($entry, true)) {
-                return false;
-            }
-        }
-        return match ($mode) {
-            'all' => true,
-            'internal' => $internal || $this->any($allow, $matches),
-            'list' => $this->any($allow, $matches),
-            // an unknown mode lets nothing through
-            default => false,
-        };
-    }
-
-    private function any(array $entries, callable $matches): bool
-    {
-        foreach ($entries as $entry) {
-            if ($matches($entry, false)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    /** @return array{username: string, domain: string}|null mailbox of the sender address */
-    private function senderMailbox(string $sender): ?array
-    {
-        return str_contains($sender, '@') ? $this->cached('mailbox', $this->resolve(self::untagged($sender))) : null;
-    }
-
-    /** @return Policy[] of the logged-in mailbox and of the mailbox it sends as */
-    private function sendingPolicies(?array $account, string $sender): array
-    {
-        if ($account === null) {
-            return [];
-        }
-        $policies = $this->lookup->policiesForMailbox($account['username']);
-        $sentAs = $this->senderMailbox($sender);
-        if ($sentAs !== null && $sentAs['username'] !== $account['username']) {
-            $policies = array_merge($policies, $this->lookup->policiesForMailbox($sentAs['username']));
-        }
-        return self::unique($policies);
-    }
-
-    /** @return Policy[] of what Postfix delivers the recipient to, see the class comment */
-    private function receivingPolicies(string $recipient): array
-    {
-        // the address, then without +tag; an alias domain maps to the mailbox in its target domain and Postfix
-        // looks up the result again with the +tag
-        [$address, $target] = $this->aliasTarget([$recipient, self::untagged($recipient)]);
-        $resolved = $this->resolve($recipient);
-        if ($target === null && $resolved !== $recipient) {
-            [$address, $target] = $this->aliasTarget([$resolved, self::untagged($resolved)]);
-        }
-        $address ??= self::untagged($resolved);
-        if (($target['kind'] ?? '') === 'temporary') {
-            $address = $target['owner'];
-        }
-        $policies = ($this->cached('mailbox', $address) !== null) ? $this->lookup->policiesForMailbox($address) : [];
-        $group = $this->lookup->groupByAddress($address);
-        if ($group !== null) {
-            $policies = array_merge($policies, $this->lookup->policiesForGroup($group));
-        }
-        // the `internal` flag of an alias or group: only senders of its domain
-        if (!empty($target['internal'])) {
-            $policies[] = $this->internalRule($target['domain']);
-        }
-        foreach (array_unique([self::domainOf($recipient), $this->resolveDomain(self::domainOf($recipient))]) as $domain) {
-            if ($this->lookup->internalCatchAll($domain)) {
-                $policies[] = $this->internalRule($domain);
-            }
-        }
-        return self::unique($policies);
-    }
-
-    /** @return array{0: ?string, 1: ?array} the first key Postfix' virtual alias maps find, with what they find */
-    private function aliasTarget(array $keys): array
-    {
-        foreach (array_unique($keys) as $key) {
-            $target = $this->lookup->aliasTarget($key);
-            if ($target !== null) {
-                return [$key, $target];
-            }
-        }
-        return [null, null];
-    }
-
-    /** Receive rule of the `internal` flag */
-    private function internalRule(string $domain): Policy
-    {
-        $policy = new Policy();
-        $policy->domain = $this->resolveDomain($domain);
-        $policy->receiveMode = 'internal';
-        return $policy;
-    }
-
-    private function isInternalRecipient(string $recipient, string $domain): bool
-    {
-        return $this->resolveDomain(self::domainOf($recipient)) === $domain;
-    }
-
-    private function recipientMatches(string $entry, string $recipient): bool
-    {
-        $address = $this->resolve(self::untagged($recipient));
-        if (str_starts_with($entry, 'group:')) {
-            $id = (int)substr($entry, 6);
-            return $this->cached('groupAddress', $id) === $address
-                || ($this->cached('mailbox', $address) !== null && $this->cached('isGroupMember', $id, $address));
-        }
-        if (str_contains($entry, '@')) {
-            return $this->resolve($entry) === $address;
-        }
-        return $this->resolveDomain(self::domainOf($address)) === $this->resolveDomain($entry);
-    }
-
-    /** $block: a block entry, which also matches the envelope sender (refusing a forged sender is no risk) */
-    private function senderMatches(string $entry, string $sender, ?array $account, bool $block): bool
-    {
-        $address = $this->resolve(self::untagged($sender));
-        // the sender address of a local entry only counts when logged in: Postfix checked that the login may use it
-        $useAddress = $account !== null || $block;
-        if (str_starts_with($entry, 'group:')) {
-            $id = (int)substr($entry, 6);
-            return ($account !== null && $this->cached('isGroupMember', $id, $account['username']))
-                || ($useAddress && ($this->cached('groupAddress', $id) === $address
-                    || ($this->cached('mailbox', $address) !== null && $this->cached('isGroupMember', $id, $address))));
-        }
-        if (str_contains($entry, '@')) {
-            return $this->cached('isLocalDomain', self::domainOf($entry))
-                ? ($account !== null && $account['username'] === $this->resolve($entry)) || ($useAddress && $address === $this->resolve($entry))
-                : self::untagged($sender) === $entry;
-        }
-        return $this->cached('isLocalDomain', $entry)
-            ? ($account !== null && $account['domain'] === $this->resolveDomain($entry))
-                || ($useAddress && $this->resolveDomain(self::domainOf($address)) === $this->resolveDomain($entry))
-            : self::domainOf($sender) === $entry;
-    }
-
-    /**
-     * Stored spelling of an address of this mailcow: Postfix finds chéf@ as chef@ (MySQL ignores accents). Only
-     * addresses with non-ASCII characters can differ, as they are already in lower case.
-     */
-    private function stored(string $address): string
-    {
-        if (!preg_match('/[\x80-\xff]/', $address) || !str_contains($address, '@')) {
-            return $address;
-        }
-        foreach (array_unique([$address, self::untagged($address)]) as $key) {
-            $stored = $this->cached('storedAddress', $key);
-            if ($stored !== null) {
-                return $stored;
-            }
-        }
-        $domain = $this->cached('storedDomain', self::domainOf($address));
-        return ($domain === null) ? $address : substr($address, 0, strrpos($address, '@') + 1) . $domain;
-    }
-
-    /** Address in an alias domain as the address in its target domain */
-    private function resolve(string $address): string
-    {
-        $domain = self::domainOf($address);
-        $target = ($domain !== '') ? $this->cached('aliasDomainTarget', $domain) : null;
-        return ($target === null) ? $address : substr($address, 0, strrpos($address, '@') + 1) . $target;
-    }
-
-    private function resolveDomain(string $domain): string
-    {
-        return ($domain === '') ? '' : ($this->cached('aliasDomainTarget', $domain) ?? $domain);
-    }
-
-    /** A lookup, done once per check: long lists ask the same domains and groups again */
-    private function cached(string $method, int|string ...$args): mixed
-    {
-        $key = $method . "\0" . implode("\0", $args);
-        if (!array_key_exists($key, $this->cache)) {
-            $this->cache[$key] = $this->lookup->$method(...$args);
-        }
-        return $this->cache[$key];
-    }
-
-    /** @param Policy[] $policies @return Policy[] without repeated policies */
-    private static function unique(array $policies): array
-    {
-        $unique = [];
-        foreach ($policies as $policy) {
-            $unique[$policy->id ?? 'internal:' . $policy->domain] = $policy;
-        }
-        return array_values($unique);
-    }
-
-    /** Lower case, without angle brackets, quotes and trailing dot of the domain, as Postfix resolves it; '' = null sender */
-    private static function address(string $value): string
-    {
-        $value = strtolower(trim($value, " \t\r\n<>"));
-        $at = strrpos($value, '@');
-        return ($at === false) ? $value : str_replace('"', '', substr($value, 0, $at)) . '@' . rtrim(substr($value, $at + 1), '.');
-    }
-
-    /**
-     * Postfix would deliver the recipient elsewhere: no domain, or routing through a local destination
-     * (localhost, the IP of the server) with %, ! or a second @ in the local part
-     */
-    private static function isRouted(string $address): bool
-    {
-        $at = strrpos($address, '@');
-        if ($at === false || $at === 0 || $at === strlen($address) - 1) {
-            return true;
-        }
-        $domain = substr($address, $at + 1);
-        return (in_array($domain, self::LOCAL_DESTINATIONS, true) || str_starts_with($domain, '['))
-            && strpbrk(substr($address, 0, $at), '@%!') !== false;
-    }
-
-    private static function untagged(string $address): string
-    {
-        $at = strrpos($address, '@');
-        $pos = ($at === false) ? false : strpos(substr($address, 0, $at), '+');
-        return ($pos === false || $pos === 0) ? $address : substr($address, 0, $pos) . substr($address, $at);
-    }
-
-    private static function domainOf(string $address): string
-    {
-        $at = strrpos($address, '@');
-        return ($at === false) ? '' : substr($address, $at + 1);
-    }
-}

data/web/inc/lib/mailcow/MailPolicy/Service/PolicyService.php

@@ -1,356 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace Mailcow\MailPolicy\Service;
-
-use Mailcow\Core\AccessDenied;
-use Mailcow\Core\Context;
-use Mailcow\Core\Input;
-use Mailcow\Core\UserError;
-use Mailcow\MailPolicy\Model\Decision;
-use Mailcow\MailPolicy\Model\Policy;
-use Mailcow\MailPolicy\Repository\PolicyRepository;
-
-/**
- * Use cases of mail policies.
- *
- * Read: admins all policies (also read-only API keys); domain admins with the `mail_policies` ACL the policies
- *       of their domains.
- * Edit: admins; domain admins with the ACL for policies of their domains. A policy is assigned to mailboxes and
- *       groups of its own domain; `group:<id>` entries name groups of that domain. Groups need the `mailbox_groups`
- *       ACL as well: without it a domain admin sees no groups and keeps the group entries and assignments.
- */
-class PolicyService
-{
-    private const MAX_ENTRIES = 500;
-    /** The reject text goes into the SMTP reply: printable ASCII */
-    private const MAX_REJECT_TEXT = 200;
-    private const LISTS = ['send_allow' => 'sendAllow', 'send_block' => 'sendBlock', 'receive_allow' => 'receiveAllow', 'receive_block' => 'receiveBlock'];
-
-    public function __construct(private PolicyRepository $repo, private PolicyEngine $engine, private Context $context)
-    {
-    }
-
-    public function canRead(): bool
-    {
-        return $this->context->hasAdminRights()
-            || ($this->context->isDomainAdmin() && $this->context->hasAcl('mail_policies'));
-    }
-
-    /** @throws AccessDenied unless the caller may edit policies at all */
-    public function requireEditor(): void
-    {
-        if (!$this->canRead() || $this->context->isReadOnlyApi()) {
-            throw new AccessDenied();
-        }
-    }
-
-    public function get(int $id): ?Policy
-    {
-        $policy = $this->canRead() ? $this->repo->find($id) : null;
-        return ($policy !== null && $this->visible($policy)) ? $policy : null;
-    }
-
-    /** @return int[] */
-    public function visibleIds(array $domains = []): array
-    {
-        return $this->canRead() ? $this->repo->visibleIds($this->context, $domains) : [];
-    }
-
-    public function canEdit(Policy $policy): bool
-    {
-        return $this->canRead() && !$this->context->isReadOnlyApi() && $this->visible($policy);
-    }
-
-    public function toArray(Policy $policy): array
-    {
-        return $policy->toArray() + ['can_edit' => (int)$this->canEdit($policy)];
-    }
-
-    /** @throws UserError */
-    public function create(array $input): Policy
-    {
-        $this->requireEditor();
-        $policy = new Policy();
-        $policy->domain = $this->domainFor($input['domain'] ?? null);
-        $this->apply($policy, $input);
-        $this->repo->save($policy);
-        return $policy;
-    }
-
-    /** Attributes missing in $input keep their value @throws UserError */
-    public function update(int $id, array $input): Policy
-    {
-        $policy = $this->editable($id);
-        $this->apply($policy, $input);
-        $this->repo->save($policy);
-        return $policy;
-    }
-
-    /** @throws UserError */
-    public function delete(int $id): Policy
-    {
-        $policy = $this->editable($id);
-        $this->repo->delete($id);
-        return $policy;
-    }
-
-    /** Address entries of the lists follow a renamed mailbox or alias; called after the rename, without permission check */
-    public function addressRenamed(string $old, string $new): void
-    {
-        $this->repo->renameAddress(strtolower($old), strtolower($new));
-    }
-
-    /** @return array[] id, name, domain, address of the groups in these domains (choices of the forms) */
-    public function groupChoices(array $domains): array
-    {
-        return ($this->canRead() && $this->canUseGroups()) ? $this->repo->groupsIn($domains) : [];
-    }
-
-    private function canUseGroups(): bool
-    {
-        return $this->context->hasAdminRights() || $this->context->hasAcl('mailbox_groups');
-    }
-
-    /**
-     * "Test a rule": would the mail be accepted? A sender that is a mailbox counts as logged in when $loggedIn.
-     *
-     * @return array{allowed: int, side: string, policy: ?string, internal_alias: int, text: string}
-     * @throws UserError
-     */
-    public function test(mixed $sender, mixed $recipient, bool $loggedIn): array
-    {
-        if (!$this->canRead()) {
-            throw new AccessDenied();
-        }
-        $sender = strtolower((string)Input::text($sender));
-        $recipient = strtolower((string)Input::text($recipient));
-        if (($sender !== '' && !filter_var($sender, FILTER_VALIDATE_EMAIL)) || !filter_var($recipient, FILTER_VALIDATE_EMAIL)) {
-            throw new UserError('mail_policy_test_invalid');
-        }
-        // domain admins test mail of their domains; a sender of another domain counts as not logged in
-        $ownSender = $this->context->hasAdminRights() || $this->context->canAccessDomain(self::domainOf($sender));
-        if (!$ownSender && !$this->context->canAccessDomain(self::domainOf($recipient))) {
-            throw new AccessDenied();
-        }
-        $login = ($loggedIn && $ownSender && $this->repo->mailboxDomain($sender) !== null) ? $sender : '';
-        $decision = $this->engine->check($sender, $recipient, $login);
-        $policy = $decision->policy;
-        $hidden = $policy !== null && $policy->id !== null && !$this->visible($policy);
-        return [
-            'allowed' => (int)$decision->allowed,
-            'side' => $decision->side,
-            'policy' => ($policy !== null && $policy->id !== null && !$hidden) ? $policy->name : null,
-            'internal_alias' => (int)($policy !== null && $policy->id === null),
-            'logged_in' => (int)($login !== ''),
-            // the text of a policy of another domain stays hidden
-            'text' => $decision->allowed ? '' : ($hidden ? Decision::defaultText($decision->side) : $decision->text()),
-        ];
-    }
-
-    private function visible(Policy $policy): bool
-    {
-        return $this->context->hasAdminRights() || $this->context->canAccessDomain($policy->domain);
-    }
-
-    /** @throws UserError */
-    private function editable(int $id): Policy
-    {
-        $this->requireEditor();
-        $policy = $this->get($id);
-        if ($policy === null || !$this->canEdit($policy)) {
-            throw new AccessDenied();
-        }
-        return $policy;
-    }
-
-    /** @throws UserError */
-    private function domainFor(mixed $value): string
-    {
-        $domain = Input::text($value);
-        $domain = ($domain === null || $domain === '') ? '' : (string)idn_to_ascii(strtolower($domain), 0, INTL_IDNA_VARIANT_UTS46);
-        if ($domain !== '' && !$this->context->hasAdminRights() && !$this->context->canAccessDomain($domain)) {
-            throw new AccessDenied();
-        }
-        if ($domain === '' || !$this->repo->isDomain($domain)) {
-            throw new UserError('domain_not_found', [htmlspecialchars((string)Input::text($value))]);
-        }
-        return $domain;
-    }
-
-    /** @throws UserError */
-    private function apply(Policy $policy, array $input): void
-    {
-        if (array_key_exists('name', $input) || $policy->id === null) {
-            $name = Input::text($input['name'] ?? null);
-            if ($name === null || $name === '' || mb_strlen($name) > 100 || preg_match('/[\x00-\x1f\x7f]/', $name)) {
-                throw new UserError('mail_policy_name_invalid');
-            }
-            if ($this->repo->nameExists($policy->domain, $name, $policy->id)) {
-                throw new UserError('object_exists', [htmlspecialchars($name)]);
-            }
-            $policy->name = $name;
-        }
-        if (array_key_exists('description', $input)) {
-            $description = Input::text($input['description']);
-            if ($description === null || mb_strlen($description) > 255 || preg_match('/[\x00-\x1f\x7f]/', $description)) {
-                throw new UserError('mail_policy_description_invalid');
-            }
-            $policy->description = $description;
-        }
-        foreach (['send_mode' => 'sendMode', 'receive_mode' => 'receiveMode'] as $key => $property) {
-            if (array_key_exists($key, $input)) {
-                $mode = Input::text($input[$key]);
-                $policy->$property = in_array($mode, Policy::MODES, true) ? $mode : throw new UserError('mail_policy_mode_invalid');
-            }
-        }
-        $groupAccess = $this->canUseGroups();
-        foreach (self::LISTS as $key => $property) {
-            // the UI sends the groups of a list separately (<list>_groups); the API may name them as group:<id>
-            if (!array_key_exists($key, $input) && !array_key_exists($key . '_groups', $input)) {
-                continue;
-            }
-            $entries = array_key_exists($key, $input) ? self::values($input[$key]) : self::withoutGroups($policy->$property);
-            $kept = [];
-            if (!$groupAccess) {
-                // without access to groups the group entries stay (also when sent back unchanged); the form shows no group choices
-                $kept = array_values(array_diff($policy->$property, self::withoutGroups($policy->$property)));
-                $entries = array_values(array_diff(array_map('strtolower', $entries), $kept));
-            } elseif (array_key_exists($key . '_groups', $input)) {
-                $entries = array_merge(self::withoutGroups($entries), array_map(fn(string $id): string => 'group:' . $id, self::values($input[$key . '_groups'])));
-            }
-            $entries = array_values(array_unique(array_merge($this->entries($entries, $policy->domain, $groupAccess), $kept)));
-            if (count($entries) > self::MAX_ENTRIES) {
-                throw new UserError('mail_policy_entries_exceeded', [self::MAX_ENTRIES]);
-            }
-            $policy->$property = $entries;
-        }
-        if (array_key_exists('reject_text', $input)) {
-            $text = Input::text($input['reject_text']);
-            if ($text === null || strlen($text) > self::MAX_REJECT_TEXT || preg_match('/[^\x20-\x7e]/', $text)) {
-                throw new UserError('mail_policy_reject_text_invalid');
-            }
-            $policy->rejectText = $text;
-        }
-        if (array_key_exists('active', $input)) {
-            $policy->active = (int)self::flag($input['active']);
-        }
-        if (array_key_exists('mailboxes', $input)) {
-            $policy->mailboxes = [];
-            foreach (self::limited($input['mailboxes']) as $username) {
-                $username = strtolower($username);
-                if ($this->repo->mailboxDomain($username) !== $policy->domain) {
-                    throw new UserError('mail_policy_member_invalid', [htmlspecialchars($username)]);
-                }
-                $policy->mailboxes[$username] = $username;
-            }
-            $policy->mailboxes = array_values($policy->mailboxes);
-        }
-        // without access to groups the assignments stay; only sending them back unchanged is accepted
-        if (array_key_exists('groups', $input) && !$groupAccess && self::values($input['groups']) !== []) {
-            $sent = array_map('intval', self::values($input['groups']));
-            $stored = $policy->groups;
-            sort($sent);
-            sort($stored);
-            if (array_values(array_unique($sent)) !== $stored) {
-                throw new AccessDenied();
-            }
-        }
-        if (array_key_exists('groups', $input) && $groupAccess) {
-            $policy->groups = [];
-            foreach (self::limited($input['groups']) as $value) {
-                $id = Input::intInRange($value, 1, PHP_INT_MAX);
-                if ($id === null || $this->repo->groupDomain($id) !== $policy->domain) {
-                    throw new UserError('mail_policy_member_invalid', [htmlspecialchars($value)]);
-                }
-                $policy->groups[$id] = $id;
-            }
-            $policy->groups = array_values($policy->groups);
-        }
-    }
-
-    /**
-     * Entries of a list: addresses, domains (a leading @ is dropped) and group:<id> of a group of the policy's
-     * domain, lower case, without repeats.
-     *
-     * @param string[] $values
-     * @return string[]
-     * @throws UserError
-     */
-    private function entries(array $values, string $domain, bool $groupAccess): array
-    {
-        $entries = [];
-        foreach (self::limited(array_map('strtolower', $values)) as $entry) {
-            if (str_starts_with($entry, 'group:')) {
-                $id = Input::intInRange(substr($entry, 6), 1, PHP_INT_MAX);
-                if (!$groupAccess || $id === null || $this->repo->groupDomain($id) !== $domain) {
-                    throw new UserError('mail_policy_entry_invalid', [htmlspecialchars($entry)]);
-                }
-                $entry = 'group:' . $id;
-            } elseif (str_contains(ltrim($entry, '@'), '@')) {
-                // in the form the engine compares: domain without the final dot, never a +tag or an IP literal
-                $at = strrpos($entry, '@');
-                $local = substr($entry, 0, $at);
-                $entry = $local . '@' . rtrim(substr($entry, $at + 1), '.');
-                if (!filter_var($entry, FILTER_VALIDATE_EMAIL) || str_contains(substr($local, 1), '+') || str_contains($entry, '@[')) {
-                    throw new UserError('mail_policy_entry_invalid', [htmlspecialchars($entry)]);
-                }
-            } else {
-                $entry = rtrim(ltrim($entry, '@'), '.');
-                if (!str_contains($entry, '.') || !filter_var($entry, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME)) {
-                    throw new UserError('mail_policy_entry_invalid', [htmlspecialchars($entry)]);
-                }
-            }
-            $entries[$entry] = $entry;
-        }
-        return array_values($entries);
-    }
-
-    /**
-     * Values of a list field without repeats, at most MAX_ENTRIES (checked before anything is looked up)
-     *
-     * @return string[]
-     * @throws UserError
-     */
-    private static function limited(mixed $value): array
-    {
-        $values = array_values(array_unique(self::values($value)));
-        if (count($values) > self::MAX_ENTRIES) {
-            throw new UserError('mail_policy_entries_exceeded', [self::MAX_ENTRIES]);
-        }
-        return $values;
-    }
-
-    /** @return string[] entries without group:<id> */
-    private static function withoutGroups(array $entries): array
-    {
-        return array_values(array_filter($entries, fn(string $entry): bool => !str_starts_with(strtolower($entry), 'group:')));
-    }
-
-    /**
-     * Values of a list field: an array (multi-select, JSON) or a string separated by line breaks, commas or spaces.
-     *
-     * @return string[]
-     */
-    private static function values(mixed $value): array
-    {
-        $values = is_array($value) ? $value : preg_split('/[\s,;]+/', (string)Input::text($value));
-        return array_values(array_filter(array_map(fn(mixed $v): string => is_scalar($v) ? trim((string)$v) : '', $values), 'strlen'));
-    }
-
-    /** Checkbox or API flag; a hidden "0" + checkbox pair arrives as ["0", "1"] when checked */
-    private static function flag(mixed $value): bool
-    {
-        if (is_array($value)) {
-            $value = end($value);
-        }
-        return filter_var($value, FILTER_VALIDATE_BOOLEAN);
-    }
-
-    private static function domainOf(string $address): string
-    {
-        $at = strrpos($address, '@');
-        return ($at === false) ? '' : substr($address, $at + 1);
-    }
-}