[Web][Restic][Dovecot][Dockerapi] Add restic backups with restore in the UI

- Scheduled, encrypted backups to S3, SFTP, REST or a local folder (new restic-mailcow container)
- Restore of mailboxes and whole domains in the UI, also deleted ones, for admins and domain admins
- Scope set by RESTIC_BACKUP_MODE in mailcow.conf (all, data, config)
- Optional backup before updates (update.sh), helper-scripts/restic.sh for a full restore
This commit is contained in:
FreddleSpl0it committed 2026-11-08 22:56:27 +01:00
1 parent a64ee13670
commit 6cf0390d58
66 files changed
+15214 -97

No files matched your search

@@ -0,0 +1,129 @@
<?php
declare(strict_types=1);
namespace Mailcow\Backup\Model;
/**
* A job of the backup runners (table `backup_job`): backup, check, prune and init run in the restic container; a
* restore runs there (step `files`, then the merge through backup.php) and, for mails, IMAP ACLs and Sieve
* scripts, in the dovecot container (step `mails`). Times are unix timestamps.
*/
class Job
{
public const TYPES = ['backup', 'check', 'prune', 'init', 'restore'];
/** Types an admin starts with add/backup-job */
public const MANUAL_TYPES = ['backup', 'check', 'prune', 'init'];
public const ACTIVE_STATES = ['queued', 'running'];
public ?int $id = null;
public string $type = 'backup';
public string $state = 'queued';
public ?string $step = null;
/**
* restore: snapshot, snapshot_time, scope, domain, mailboxes, components, mail_target, requested_role, create_domain (0|1),
* create_mailboxes (missing when requested), acl and allowed_domains (domain admins)
*/
public array $params = [];
public ?array $result = null;
/** Only loaded for a single job */
public ?string $log = null;
public ?string $error = null;
/** restore: the domain of its mailboxes, so domain admins see their restores */
public string $domain = '';
public string $requestedBy = '';
public ?int $created = null;
public ?int $started = null;
public ?int $finished = null;
public function __construct(string $type = 'backup')
{
$this->type = $type;
}
/** Row of `backup_job` with the times as unix timestamps */
public static function fromRow(array $row): self
{
$job = new self((string)$row['type']);
$job->id = (int)$row['id'];
$job->state = (string)$row['state'];
$job->step = isset($row['step']) ? (string)$row['step'] : null;
$params = is_string($row['params'] ?? null) ? json_decode($row['params'], true) : null;
$job->params = is_array($params) ? $params : [];
$result = is_string($row['result'] ?? null) ? json_decode($row['result'], true) : null;
$job->result = is_array($result) ? $result : null;
$job->log = isset($row['log']) ? (string)$row['log'] : null;
$job->error = isset($row['error']) ? (string)$row['error'] : null;
$job->domain = (string)($row['domain'] ?? '');
$job->requestedBy = (string)($row['requested_by'] ?? '');
foreach (['created', 'started', 'finished'] as $time) {
$job->$time = isset($row[$time]) ? (int)$row[$time] : null;
}
return $job;
}
/** Column values for a new row */
public function toRow(): array
{
return [
'type' => $this->type,
'state' => $this->state,
'step' => $this->step,
'params' => json_encode($this->params ?: new \stdClass(), JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
'domain' => $this->domain,
'requested_by' => $this->requestedBy,
];
}
/** @return string[] mailboxes of a restore */
public function mailboxes(): array
{
return array_values(array_filter((array)($this->params['mailboxes'] ?? []), 'is_string'));
}
/** @return string[] components of a restore */
public function components(): array
{
return array_values(array_filter((array)($this->params['components'] ?? []), 'is_string'));
}
/** @return string[] mailboxes of a restore that did not exist when it was requested */
public function createMailboxes(): array
{
return array_values(array_filter((array)($this->params['create_mailboxes'] ?? []), 'is_string'));
}
/** The restore creates its domain (requested by an admin while the domain did not exist) */
public function createsDomain(): bool
{
return (int)($this->params['create_domain'] ?? 0) === 1;
}
public function isActive(): bool
{
return in_array($this->state, self::ACTIVE_STATES, true);
}
/** Entry of the API; the log only with $withLog (get/backup-job/<id>) */
public function toArray(bool $withLog = false): array
{
$job = [
'id' => $this->id,
'type' => $this->type,
'state' => $this->state,
'step' => $this->step,
'params' => $this->params ?: new \stdClass(),
'result' => $this->result,
'error' => $this->error,
'domain' => $this->domain,
'requested_by' => $this->requestedBy,
'created' => $this->created,
'started' => $this->started,
'finished' => $this->finished,
];
if ($withLog) {
$job['log'] = (string)$this->log;
}
return $job;
}
}
@@ -0,0 +1,116 @@
<?php
declare(strict_types=1);
namespace Mailcow\Backup\Model;
use Mailcow\Core\Context;
/**
* The merge of one mailbox of a restore job: what the backup holds, who asked for the restore and what the merge
* did per component (added, skipped, errors), plus the caches to refresh afterwards.
*/
class RestoreRun
{
/** @var array<string, array{added: int, skipped: int, errors: string[]}> */
public array $results = [];
/** Mailboxes whose SOGo identities (aliases, send-as addresses) changed */
public array $sogoUsers = [];
/** Mailboxes whose userdb or passdb values changed (Dovecot auth cache) */
public array $authUsers = [];
/**
* @param array $mailbox the mailbox row as it is now
* @param array $params params of the restore job (requested_role, acl of a domain admin, scope, ...)
* @param array $data the mailbox's section of the domain export in the snapshot
* @param string[] $domains domains whose aliases and send-as addresses the restore may change, [] = all (admins)
* @param string $requestedBy who requested the restore
*/
/** The restore created the mailbox (its own alias gets the forwarding of the backup) */
public bool $created = false;
public function __construct(
public readonly string $username,
public readonly string $domain,
public readonly array $mailbox,
public readonly array $params,
public readonly array $data,
public readonly array $domains = [],
public readonly string $requestedBy = ''
) {
}
/**
* The caller the group module checks: an admin, or the domain admin who requested the restore with his ACL of then
* and his domains now
*/
public function requester(): Context
{
if ($this->byAdmin()) {
return new Context('admin', 'backup', [], true);
}
return new Context('domainadmin', $this->requestedBy, array_map(fn(mixed $flag): string => (string)(int)$flag, (array)($this->params['acl'] ?? [])));
}
public function start(string $component): void
{
$this->results[$component] = ['added' => 0, 'skipped' => 0, 'errors' => []];
}
public function added(string $component, int $count = 1): void
{
$this->results[$component]['added'] += $count;
}
public function skipped(string $component, int $count = 1): void
{
$this->results[$component]['skipped'] += $count;
}
public function error(string $component, string $error): void
{
$this->results[$component]['errors'][] = $error;
}
/** The part of a component was rolled back: nothing of it counts */
public function failed(string $component, string $error): void
{
$this->start($component);
$this->error($component, $error);
}
/** The restore was requested by an admin (domain admins are limited by their ACL and the job's domain) */
public function byAdmin(): bool
{
return ($this->params['requested_role'] ?? '') === 'admin';
}
/** Admins, or a domain admin with this ACL when he requested the restore */
public function may(string $acl): bool
{
return $this->byAdmin() || (int)($this->params['acl'][$acl] ?? 0) === 1;
}
public function mayChangeDomain(string $domain): bool
{
return $this->domains === [] || in_array(strtolower($domain), $this->domains, true);
}
/** Rows of a list in the backup (sieve_filters, tfa, ...) */
public function rows(string $key): array
{
return array_values(array_filter((array)($this->data[$key] ?? []), 'is_array'));
}
public function sogo(string ...$usernames): void
{
array_push($this->sogoUsers, ...$usernames);
$this->sogoUsers = array_values(array_unique($this->sogoUsers));
}
public function auth(string ...$usernames): void
{
array_push($this->authUsers, ...$usernames);
$this->authUsers = array_values(array_unique($this->authUsers));
}
}
@@ -0,0 +1,189 @@
<?php
declare(strict_types=1);
namespace Mailcow\Backup\Model;
/**
* Backup settings (table `backup_setting`, name/value pairs) over their defaults. Secrets are stored sealed for the
* runner (`sealed:v1:<base64>`) and never leave toArray(): the API gets `<key>_set` instead. The state_* values, the
* SSH public key and the SFTP host key are written by the runner in the restic container and cannot be set through
* the API (PHP writes state_target_changed when the target changes).
*/
class Settings
{
public const DEFAULTS = [
'enabled' => '0',
'target' => 's3',
's3_endpoint' => 'https://s3.amazonaws.com',
's3_bucket' => '',
's3_path' => '',
's3_region' => '',
's3_access_key' => '',
's3_secret_key' => '',
's3_path_style' => '0',
'sftp_host' => '',
'sftp_port' => '22',
'sftp_user' => '',
'sftp_path' => '',
'sftp_public_key' => '',
'sftp_host_key' => '',
'rest_url' => '',
'rest_user' => '',
'rest_password' => '',
'local_path' => 'mailcow',
'password' => '',
'password_confirmed' => '0',
// JSON list of the scheduled backups: {"days": [weekdays, 0 = Sunday], "time": "HH:MM"}
'schedule' => '[{"days":[0,1,2,3,4,5,6],"time":"01:00"}]',
'keep_hourly' => '0',
'keep_daily' => '7',
'keep_weekly' => '4',
'keep_monthly' => '12',
'keep_yearly' => '0',
'prune_weekday' => '0',
'max_unused' => '10%',
'check_subset' => '30',
'include_index' => '1',
'include_fts' => '1',
'limit_upload' => '0',
'notify_email' => '',
'state_initialized' => '0',
'state_runner_seen' => '',
'state_last_backup' => '',
'state_last_success' => '',
'state_repo_size' => '',
'state_repo_stats' => '',
'state_schedule_since' => '',
'state_schedule' => '',
// public key (base64, X25519) the secrets are sealed with; the private key stays in the restic container
'state_secrets_pubkey' => '',
'state_secrets_error' => '',
// JSON {old_target, old_addresses, changed, changed_by, time}: the runner mails the old addresses and clears it
'state_target_changed' => '',
// RESTIC_BACKUP_MODE of mailcow.conf as the runner sees it
'state_backup_mode' => '',
];
public const SECRETS = ['password', 's3_secret_key', 'rest_password'];
/** Written by the runner only */
public const RUNNER = ['sftp_public_key', 'sftp_host_key', 'state_initialized', 'state_runner_seen', 'state_last_backup',
'state_last_success', 'state_repo_size', 'state_repo_stats', 'state_schedule_since', 'state_schedule', 'state_secrets_pubkey',
'state_secrets_error', 'state_target_changed', 'state_backup_mode'];
/** Prefix of a sealed secret (sodium_crypto_box_seal with state_secrets_pubkey) */
public const SEALED = 'sealed:v1:';
public const TARGETS = ['s3', 'sftp', 'rest', 'local'];
/** The connection of each target: a change (or another target or repository password) resets state_initialized */
public const CONNECTION = [
's3' => ['s3_endpoint', 's3_bucket', 's3_path', 's3_region', 's3_access_key', 's3_secret_key', 's3_path_style'],
'sftp' => ['sftp_host', 'sftp_port', 'sftp_user', 'sftp_path'],
'rest' => ['rest_url', 'rest_user', 'rest_password'],
'local' => ['local_path'],
];
/** Values a target cannot work without (besides the repository password) */
public const REQUIRED = [
's3' => ['s3_endpoint', 's3_bucket', 's3_access_key', 's3_secret_key'],
'sftp' => ['sftp_host', 'sftp_port', 'sftp_user', 'sftp_path'],
'rest' => ['rest_url'],
'local' => ['local_path'],
];
/** Returned as numbers by the API ('' = not known yet: null; prune_weekday '' = never stays '') */
private const NUMBERS = ['enabled', 's3_path_style', 'sftp_port', 'password_confirmed', 'keep_hourly', 'keep_daily', 'keep_weekly', 'keep_monthly',
'keep_yearly', 'prune_weekday', 'check_subset', 'include_index', 'include_fts', 'limit_upload', 'state_initialized',
'state_runner_seen', 'state_last_backup', 'state_last_success', 'state_repo_size', 'state_schedule_since'];
private const JSON = ['schedule', 'state_repo_stats', 'state_target_changed'];
/** @param array<string, string> $values every key of DEFAULTS */
public function __construct(public array $values = self::DEFAULTS)
{
}
/** Stored values (name => value) over the defaults; unknown names are dropped */
public static function fromStored(array $stored): self
{
$values = self::DEFAULTS;
foreach ($stored as $name => $value) {
if (array_key_exists($name, $values)) {
$values[$name] = (string)$value;
}
}
return new self($values);
}
public function get(string $name): string
{
return $this->values[$name] ?? '';
}
/** The target has its required values and the repository password is set */
public function configured(): bool
{
$required = self::REQUIRED[$this->get('target')] ?? null;
if ($required === null || $this->get('password') === '') {
return false;
}
foreach ($required as $name) {
if ($this->get($name) === '') {
return false;
}
}
return true;
}
/**
* The schedule as [weekday (0 = Sunday), hour, minute] slots, sorted and unique
*
* @return array<int, array{int, int, int}>
*/
public function scheduleSlots(): array
{
$entries = json_decode($this->get('schedule'), true);
$slots = [];
foreach (is_array($entries) ? $entries : [] as $entry) {
if (!is_array($entry) || !preg_match('/^([01]?\d|2[0-3]):([0-5]\d)$/', (string)($entry['time'] ?? ''), $time) || !is_array($entry['days'] ?? null)) {
continue;
}
foreach ($entry['days'] as $day) {
if (is_int($day) && $day >= 0 && $day <= 6) {
$slots[] = $day * 1440 + (int)$time[1] * 60 + (int)$time[2];
}
}
}
$slots = array_unique($slots);
sort($slots);
return array_map(fn(int $slot): array => [intdiv($slot, 1440), intdiv($slot % 1440, 60), $slot % 60], $slots);
}
/** The longest time between two scheduled backups of a week in seconds, null without a schedule */
public function scheduleGap(): ?int
{
$slots = array_map(fn(array $slot): int => $slot[0] * 86400 + $slot[1] * 3600 + $slot[2] * 60, $this->scheduleSlots());
if (empty($slots)) {
return null;
}
$gaps = [$slots[0] + 7 * 86400 - end($slots)];
for ($i = 1; $i < count($slots); $i++) {
$gaps[] = $slots[$i] - $slots[$i - 1];
}
return max($gaps);
}
/** Entry of get/backup-settings: every key, secrets as <key>_set, plus `configured` */
public function toArray(): array
{
$settings = [];
foreach ($this->values as $name => $value) {
if (in_array($name, self::SECRETS, true)) {
$settings[$name . '_set'] = (int)($value !== '');
} elseif (in_array($name, self::JSON, true)) {
$decoded = json_decode($value, true);
$settings[$name] = is_array($decoded) ? $decoded : null;
} elseif (in_array($name, self::NUMBERS, true)) {
$settings[$name] = ($value === '') ? (($name === 'prune_weekday') ? '' : null) : (int)$value;
} else {
$settings[$name] = $value;
}
}
$settings['configured'] = (int)$this->configured();
return $settings;
}
}
@@ -0,0 +1,142 @@
<?php
declare(strict_types=1);
namespace Mailcow\Backup\Model;
/**
* A snapshot of the repository (table `backup_snapshot`, the runner's cache of `restic snapshots --json`).
*/
class Snapshot
{
public string $id = '';
public string $shortId = '';
/** unix timestamp */
public int $time = 0;
public string $hostname = '';
/** @var string[] */
public array $tags = [];
/** restic's summary of the backup (data_added, files_new, ...) */
public ?array $summary = null;
/** @var ?string[] domains of its manifest, null for snapshots not made by mailcow */
public ?array $domains = null;
/** @var array<string, ?int> domain => its creation time (unix) when the snapshot was made, from the snapshot row */
public array $created = [];
/**
* @var ?array<string, array{created: ?int, mailboxes: array<string, string>}> per domain its creation time (unix, null
* when not recorded) and username => name of its mailboxes; null = not known (snapshots this installation did
* not make itself, e.g. of another server in the same repository)
*/
public ?array $content = null;
/** Row of `backup_snapshot` with the time as unix timestamp */
public static function fromRow(array $row): self
{
$snapshot = new self();
$snapshot->id = (string)$row['id'];
$snapshot->shortId = (string)$row['short_id'];
$snapshot->time = (int)$row['time'];
$snapshot->hostname = (string)($row['hostname'] ?? '');
$tags = (string)($row['tags'] ?? '');
$decoded = json_decode($tags, true);
$tags = is_array($decoded) ? $decoded : explode(',', $tags);
$snapshot->tags = array_values(array_filter(array_map(fn(mixed $tag): string => is_scalar($tag) ? trim((string)$tag) : '', $tags), 'strlen'));
$summary = is_string($row['summary'] ?? null) ? json_decode($row['summary'], true) : null;
$snapshot->summary = is_array($summary) ? $summary : null;
// {domain: creation time}: lists of snapshots need no content
$domains = is_string($row['domains'] ?? null) ? json_decode($row['domains'], true) : null;
if (is_array($domains)) {
$snapshot->domains = [];
foreach ($domains as $domain => $created) {
$snapshot->domains[] = (string)$domain;
$snapshot->created[strtolower((string)$domain)] = is_numeric($created) ? (int)$created : null;
}
}
if (is_string($row['content'] ?? null) && $row['content'] !== '') {
$snapshot->content = self::decodeContent($row['content']);
}
return $snapshot;
}
/**
* `content` (gzip JSON) written by the runner: {"domains": {"<domain>": {"created": <unix>, "mailboxes": {"<username>": "<name>"}}}}.
* Keys lower case; null when broken.
*/
public static function decodeContent(string $gzip): ?array
{
$json = @gzdecode($gzip);
$data = is_string($json) ? json_decode($json, true) : null;
if (!is_array($data) || !is_array($data['domains'] ?? null)) {
return null;
}
$content = [];
foreach ($data['domains'] as $domain => $entry) {
$mailboxes = is_array($entry) ? ($entry['mailboxes'] ?? []) : [];
$created = is_numeric($entry['created'] ?? null) ? (int)$entry['created'] : null;
$names = [];
foreach (is_array($mailboxes) ? $mailboxes : [] as $username => $name) {
$names[strtolower((string)$username)] = is_scalar($name) ? (string)$name : '';
}
$content[strtolower((string)$domain)] = ['created' => $created, 'mailboxes' => $names];
}
return $content;
}
/** Domains of the snapshot: those of its content and of its manifest (an export may have failed) */
public function domainList(): array
{
return array_values(array_unique(array_merge(array_map('strval', array_keys($this->content ?? [])), array_map('strtolower', $this->domains ?? []))));
}
/** @return ?array<string, string> username => name of the domain's mailboxes, null when the content does not list the domain */
public function mailboxesOf(string $domain): ?array
{
return $this->content[strtolower($domain)]['mailboxes'] ?? null;
}
/** Creation time of the domain when the snapshot was made, null when not recorded */
public function createdOf(string $domain): ?int
{
return $this->content[strtolower($domain)]['created'] ?? $this->created[strtolower($domain)] ?? null;
}
public function hasDomain(string $domain): bool
{
return in_array(strtolower($domain), $this->domainList(), true);
}
/**
* RESTIC_BACKUP_MODE of the backup (tag mode:<mode>, none for all): all, data (without the mailcow directory) or
* config (without mails)
*/
public function mode(): string
{
foreach ($this->tags as $tag) {
if (str_starts_with($tag, 'mode:')) {
return substr($tag, 5);
}
}
return 'all';
}
/** Snapshots not made by mailcow know neither domains nor mailboxes */
public function knowsDomains(): bool
{
return $this->content !== null || $this->domains !== null;
}
/** Entry of the API for admins */
public function toArray(): array
{
return [
'id' => $this->id,
'short_id' => $this->shortId,
'time' => $this->time,
'hostname' => $this->hostname,
'tags' => $this->tags,
'summary' => $this->summary,
'domains' => $this->domains,
'mode' => $this->mode(),
];
}
}