[Web][Dovecot][SOGo][DockerApi] Add mailbox groups, shared mailboxes and identity provider sync

This commit is contained in:
FreddleSpl0it committed 2026-10-27 10:01:33 +01:00
1 parent 8bc8fbdd2c
commit 2c5718c9c9
82 files changed
+6488 -1027

No files matched your search

@@ -0,0 +1,51 @@
<?php
declare(strict_types=1);
namespace Mailcow\Identity\Service;
use Mailcow\Identity\Repository\IdentityRepository;
/**
* Links of mailboxes to IdP users, checked at IdP logins (REST, LDAP bind, OIDC). Mailboxes are found by email,
* so the IdP user id has to match the link: an address taken over by another IdP user must not open the old
* mailbox. Unlinked mailboxes are linked on their first login.
*/
class LinkService
{
public function __construct(private IdentityRepository $repo)
{
}
/** The IdP user may log in to the mailbox; links the mailbox when it has no link yet */
public function verify(string $provider, string $username, string $externalId): bool
{
if ($externalId === '') {
return false;
}
$link = $this->repo->findLink($username);
if ($link !== null && $link['provider'] === $provider && $link['external_id'] !== null) {
return hash_equals((string)$link['external_id'], $externalId);
}
if (!$this->mayCreate($provider, $username, $externalId)) {
return false;
}
$this->repo->link($username, $provider, $externalId);
return true;
}
/** The IdP user owns no other mailbox (checked before a mailbox is created on login) */
public function mayCreate(string $provider, string $username, string $externalId): bool
{
$owner = ($externalId === '') ? null : $this->repo->usernameOf($provider, $externalId);
return $externalId !== '' && ($owner === null || $owner === $username);
}
/** Links a mailbox created on login */
public function link(string $provider, string $username, string $externalId): void
{
if ($externalId !== '') {
$this->repo->link($username, $provider, $externalId);
}
}
}
@@ -0,0 +1,80 @@
<?php
declare(strict_types=1);
namespace Mailcow\Identity\Service;
use Mailcow\Core\AccessDenied;
use Mailcow\Core\Context;
use Mailcow\Core\Input;
use Mailcow\Core\UserError;
use Mailcow\Identity\Model\Settings;
use Mailcow\Identity\Repository\IdentityRepository;
use Mailcow\Identity\Repository\RunRepository;
/**
* Admin use cases of the sync: status, run requests (executed by the cron) and links of mailboxes.
* Admins only, read-only API keys may read. Domain admins may read the link of mailboxes they manage.
*/
class StatusService
{
public const MODES = ['preview', 'run', 'force'];
public function __construct(private Settings $settings, private IdentityRepository $repo, private RunRepository $runs,
private Context $context)
{
}
/** @throws AccessDenied */
public function status(): array
{
$this->context->requireAdmin(false);
return [
'provider' => $this->settings->provider,
'can_sync' => $this->settings->canSync(),
'sync_enabled' => $this->settings->syncEnabled(),
'counts' => $this->repo->stats($this->settings->provider),
'runs' => $this->runs->recent(),
];
}
/**
* Queues a run for the cron: preview (dry run), run, or force (run without the change limit).
*
* @throws UserError
*/
public function requestRun(mixed $mode): int
{
$this->context->requireAdmin();
$mode = Input::text($mode);
if (!in_array($mode, self::MODES, true)) {
throw new UserError('iam_sync_mode_invalid');
}
if (!$this->settings->canSync()) {
throw new UserError('iam_sync_unavailable');
}
return $this->runs->request($this->settings->provider, $mode, (string)$this->context->username);
}
/** Link of a mailbox for the mailbox edit page, null if none or not allowed */
public function link(string $username): ?array
{
$username = strtolower(trim($username));
if (!($this->context->hasAdminRights() || ($this->context->isDomainAdmin() && $this->context->canAccessMailbox($username)))) {
return null;
}
return $this->repo->findLink($username);
}
/**
* Drops the link, the mailbox is linked again by email (next run or login). Resolves conflicts, e.g. when an
* address was given to another IdP user.
*
* @throws AccessDenied
*/
public function unlink(string $username): void
{
$this->context->requireAdmin();
$this->repo->unlink(strtolower(trim($username)));
}
}
@@ -0,0 +1,311 @@
<?php
declare(strict_types=1);
namespace Mailcow\Identity\Service;
use Mailcow\Identity\Model\Account;
use Mailcow\Identity\Model\DirectoryGroup;
use Mailcow\Identity\Model\DirectoryUser;
use Mailcow\Identity\Model\Listing;
use Mailcow\Identity\Model\Settings;
use Mailcow\Identity\Model\State;
use Mailcow\Identity\Model\SyncPlan;
/**
* Decides what a sync run changes, without side effects.
*
* Users: linked mailboxes are found by IdP user id, unlinked ones are adopted by email. A mailbox linked to
* another IdP user, or an IdP user whose email moved away from its mailbox, is a conflict; nothing is
* relinked or created for it.
* Offboarding: a user disabled in the IdP, or not listed by two consecutive complete runs, gets login disabled
* (active = 2), once per IdP disable. Only those mailboxes are activated again when the IdP enables
* the user; manual changes of active always win.
* Groups: IdP groups map to mailbox groups by idp_ref; unknown ones are created in the domain of their mail,
* else in the default group domain. Linked users that are not listed keep their IdP memberships.
* Guard: more deactivations or membership removals than max(5, limit % of active IdP mailboxes resp. IdP
* memberships) block the run unless forced.
*/
class SyncPlanner
{
private const GUARD_MIN = 5;
public function plan(Settings $settings, Listing $listing, State $state, bool $force = false): SyncPlan
{
$plan = new SyncPlan();
$owners = [];
foreach ($state->accounts as $username => $account) {
if ($account->externalId !== null) {
$owners[$account->externalId] = (string)$username;
}
}
/** @var array<string, DirectoryUser> $matched username => listed user */
$matched = [];
/** @var array<string, DirectoryUser> $candidates email => user to create, template still open */
$candidates = [];
$duplicates = [];
foreach ($listing->users as $user) {
if (isset($owners[$user->id])) {
$username = $owners[$user->id];
if ($user->email !== null && $user->email !== $username) {
self::conflict($plan, $username . ' is linked to an IdP user whose email is now ' . $user->email);
}
$matched[$username] = $user;
} elseif ($user->email === null) {
$plan->skipped++;
} elseif (isset($state->accounts[$user->email])) {
$account = $state->accounts[$user->email];
if ($account->externalId !== null || isset($plan->links[$user->email])) {
self::conflict($plan, $user->email . ' is linked to another IdP user');
continue;
}
$plan->links[$user->email] = $user->id;
$matched[$user->email] = $user;
} elseif (isset($state->mailboxes[$user->email]) || !$settings->importUsers || !$user->enabled
|| !in_array(substr((string)strrchr($user->email, '@'), 1), $state->domains, true)) {
// other authsource, import off, disabled or no mailcow domain
$plan->skipped++;
} elseif (isset($candidates[$user->email]) || isset($duplicates[$user->email])) {
self::conflict($plan, 'Several IdP users have the email ' . $user->email);
unset($candidates[$user->email]);
$duplicates[$user->email] = true;
} else {
$candidates[$user->email] = $user;
}
}
$groups = ($settings->groupSync && $listing->groups !== null) ? $listing->groups : null;
$this->planCreates($settings, $candidates, $groups ?? [], $state, $plan);
$usernames = [];
foreach ($matched as $username => $user) {
$usernames[$user->id] = (string)$username;
}
foreach ($plan->creates as $create) {
$usernames[$create['id']] = $create['email'];
}
if ($groups !== null) {
$this->planGroups($settings, $groups, $state, $usernames, $matched, $plan);
}
$this->planStates($settings, $state, $matched, $plan);
if ($settings->periodicSync) {
$this->planUpdates($settings, $state, $matched, $plan);
}
$active = count(array_filter($state->accounts, fn(Account $a): bool => $a->active === 1));
$memberships = array_sum(array_map(fn(array $g): int => count(self::idpMembers($g)), $state->groups));
if (!$force && (count($plan->deactivate) > self::limit($active, $settings->offboardingLimit)
|| $plan->membershipRemovals > self::limit($memberships, $settings->offboardingLimit))) {
$plan->blocked = true;
}
return $plan;
}
/**
* @param array<string, DirectoryGroup> $groups
* @param array<string, string> $usernames user id => username of matched and new users
*/
private function planGroups(Settings $settings, array $groups, State $state, array $usernames, array $matched, SyncPlan $plan): void
{
$linked = [];
$names = [];
foreach ($state->groups as $group) {
if ($group['idp_ref'] !== null) {
$linked[(string)$group['idp_ref']] = $group;
}
$names[$group['domain']][mb_strtolower($group['name'])] = (int)$group['id'];
}
foreach ($groups as $ref => $directoryGroup) {
$ref = (string)$ref;
$members = [];
foreach ($directoryGroup->memberIds as $id) {
if (isset($usernames[$id])) {
$members[] = $usernames[$id];
}
}
$current = $linked[$ref] ?? null;
if ($current === null) {
if (!$settings->groupAutocreate) {
continue;
}
$domain = self::groupDomain($settings, $directoryGroup, $state->domains);
if ($domain === null) {
self::conflict($plan, 'IdP group ' . $directoryGroup->name . ' has no mailcow domain');
continue;
}
if (isset($names[$domain][mb_strtolower($directoryGroup->name)])) {
self::conflict($plan, 'A group ' . $directoryGroup->name . ' exists in ' . $domain . ' and is not linked to the IdP group');
continue;
}
$names[$domain][mb_strtolower($directoryGroup->name)] = 0;
$plan->groupCreates[] = ['ref' => $ref, 'name' => $directoryGroup->name, 'domain' => $domain, 'mail' => $directoryGroup->mail];
$plan->memberships[$ref] = self::sorted($members);
continue;
}
$idpMembers = self::idpMembers($current);
// Linked users missing in the listing keep their memberships until they are listed again
foreach ($idpMembers as $username) {
if (isset($state->accounts[$username]) && !isset($matched[$username])) {
$members[] = $username;
}
}
$members = self::sorted($members);
$removed = array_diff($idpMembers, $members);
if (!empty($removed) || !empty(array_diff($members, array_map('strval', array_keys($current['members']))))) {
$plan->memberships[$ref] = $members;
$plan->membershipRemovals += count($removed);
}
if ($directoryGroup->name !== $current['name']) {
$taken = $names[$current['domain']][mb_strtolower($directoryGroup->name)] ?? null;
if ($taken !== null && $taken !== (int)$current['id']) {
self::conflict($plan, 'IdP group ' . $current['name'] . ' was renamed to ' . $directoryGroup->name . ', which exists in ' . $current['domain']);
} else {
$plan->groupRenames[(int)$current['id']] = $directoryGroup->name;
$names[$current['domain']][mb_strtolower($directoryGroup->name)] = (int)$current['id'];
}
}
}
foreach ($linked as $ref => $group) {
if (!isset($groups[$ref])) {
$plan->orphanedGroups[] = (int)$group['id'];
$plan->membershipRemovals += count(self::idpMembers($group));
}
}
}
/**
* Template of a new mailbox: the template of its highest-priority linked group (as in TemplateService), else
* the attribute mapping, else the default template.
*
* @param array<string, DirectoryUser> $candidates
* @param array<string, DirectoryGroup> $groups listed groups, [] without group sync
*/
private function planCreates(Settings $settings, array $candidates, array $groups, State $state, SyncPlan $plan): void
{
$linked = [];
foreach ($state->groups as $group) {
if ($group['idp_ref'] !== null && $group['template'] !== null && $group['active'] === 1) {
$linked[(string)$group['idp_ref']] = $group;
}
}
$groupsOf = [];
foreach ($groups as $ref => $directoryGroup) {
if (isset($linked[(string)$ref])) {
foreach ($directoryGroup->memberIds as $id) {
$groupsOf[$id][] = $linked[(string)$ref];
}
}
}
foreach ($candidates as $email => $user) {
$best = null;
foreach ($groupsOf[$user->id] ?? [] as $group) {
if ($best === null || $group['priority'] > $best['priority'] || ($group['priority'] === $best['priority'] && $group['id'] < $best['id'])) {
$best = $group;
}
}
$template = $best['template'] ?? $settings->mappedTemplate($user->templateValues)
?? (($settings->defaultTemplate !== '') ? $settings->defaultTemplate : null);
if ($template === null) {
self::conflict($plan, 'No template matches ' . $email);
continue;
}
$plan->creates[] = ['email' => (string)$email, 'name' => self::name($user->name, ''), 'template' => $template, 'id' => $user->id];
}
}
/** @param array<string, DirectoryUser> $matched */
private function planStates(Settings $settings, State $state, array $matched, SyncPlan $plan): void
{
foreach ($state->accounts as $username => $account) {
$username = (string)$username;
$user = $matched[$username] ?? null;
if ($user === null && $account->missingSince === null) {
$plan->missing[] = $username;
continue;
}
if ($user !== null && $account->missingSince !== null) {
$plan->found[] = $username;
}
if (!$settings->offboarding) {
continue;
}
// Second complete run without the user counts as removed from the IdP
$disabled = ($user === null) || !$user->enabled;
if ($disabled && !$account->idpDisabled && $account->active === 1) {
$plan->deactivate[] = $username;
} elseif (!$disabled && $account->idpDisabled) {
if ($account->active === 2) {
$plan->restore[] = $username;
} else {
$plan->release[] = $username;
}
}
}
}
/**
* Name and template of linked mailboxes, like the IdP login does. The planned template is the legacy mapping;
* SyncService applies the effective template again after the group changes of the run.
*
* @param array<string, DirectoryUser> $matched
*/
private function planUpdates(Settings $settings, State $state, array $matched, SyncPlan $plan): void
{
foreach ($matched as $username => $user) {
$account = $state->accounts[$username];
$legacy = $settings->mappedTemplate($user->templateValues);
$plan->profiles[$username] = ['name' => self::name($user->name, $account->name), 'template' => $legacy];
$template = $state->effectiveTemplates[$username] ?? $legacy;
if ($template === null) {
continue;
}
if (!isset($state->templateHashes[$template])) {
self::conflict($plan, 'Template ' . $template . ' does not exist');
continue;
}
$name = self::name($user->name, $account->name);
if ($name !== $account->name || $state->templateHashes[$template] !== $account->attributeHash) {
$plan->updates[] = ['username' => (string)$username, 'name' => $name, 'template' => $legacy];
}
}
}
/** Domain for a new group: the domain of its mail if it is a mailcow domain, else the default group domain */
private static function groupDomain(Settings $settings, DirectoryGroup $group, array $domains): ?string
{
$domain = ($group->mail !== null) ? substr((string)strrchr($group->mail, '@'), 1) : '';
if (in_array($domain, $domains, true)) {
return $domain;
}
return in_array($settings->groupDefaultDomain, $domains, true) ? $settings->groupDefaultDomain : null;
}
/** Mailbox name as mailbox_from_template stores it; an empty IdP name keeps the current one */
private static function name(string $name, string $current): string
{
$name = ltrim(rtrim(trim($name), '>'), '<');
return ($name === '') ? $current : $name;
}
private static function limit(int $base, int $percent): int
{
return max(self::GUARD_MIN, (int)ceil($base * $percent / 100));
}
/** @return string[] members of a state group with source idp */
private static function idpMembers(array $group): array
{
return array_map('strval', array_keys(array_filter($group['members'], fn(string $source): bool => $source === 'idp')));
}
private static function sorted(array $usernames): array
{
$usernames = array_values(array_unique(array_map('strval', $usernames)));
sort($usernames);
return $usernames;
}
private static function conflict(SyncPlan $plan, string $message): void
{
if (!in_array($message, $plan->conflicts, true)) {
$plan->conflicts[] = $message;
}
}
}
@@ -0,0 +1,231 @@
<?php
declare(strict_types=1);
namespace Mailcow\Identity\Service;
use Closure;
use Mailcow\Core\UserError;
use Mailcow\Identity\Directory\Directory;
use Mailcow\Identity\Directory\DirectoryException;
use Mailcow\Identity\Legacy\LegacyAccounts;
use Mailcow\Identity\Model\Settings;
use Mailcow\Identity\Model\State;
use Mailcow\Identity\Model\SyncPlan;
use Mailcow\Identity\Repository\IdentityRepository;
use Mailcow\Identity\Repository\RunRepository;
use Mailcow\MailboxGroup\Service\IdpSyncService;
use Mailcow\MailboxGroup\Service\TemplateService;
use Mailcow\Sogo\Service\SogoService;
use Throwable;
/**
* Sync runs of the IdP (cron only): list the directory, plan, apply. A listing error aborts the run before
* anything is written; a blocked or preview run writes nothing but the run summary.
* The directory is created only when a run is due, so idle cron calls never connect to the IdP.
*/
class SyncService
{
/** A run still `running` after this many minutes crashed */
private const STALE_MINUTES = 60;
/** Errors kept in the run summary */
private const DETAILS = 50;
public function __construct(
private Settings $settings,
/** @var Closure(): ?Directory */
private Closure $directory,
private IdentityRepository $repo,
private RunRepository $runs,
private SyncPlanner $planner,
private IdpSyncService $groups,
private TemplateService $templates,
private LegacyAccounts $legacy,
private SogoService $sogo
) {
}
/**
* Runs the oldest requested run, else a scheduled run when the sync interval is over.
*
* @return ?array the finished run (id, mode, status, summary), null if nothing was due
*/
public function runDue(): ?array
{
$this->runs->failStale(self::STALE_MINUTES);
$run = $this->runs->nextPending();
if ($run === null) {
if (!$this->settings->syncEnabled()) {
return null;
}
$since = $this->runs->secondsSinceScheduled();
if ($since !== null && $since < $this->settings->syncInterval * 60) {
return null;
}
$run = ['id' => $this->runs->create($this->settings->provider, 'run'), 'mode' => 'run'];
}
return $this->execute((int)$run['id'], (string)$run['mode']);
}
/** @return array{id: int, mode: string, status: string, summary: array} */
public function execute(int $id, string $mode): array
{
$this->runs->start($id);
try {
$directory = $this->settings->canSync() ? ($this->directory)() : null;
if ($directory === null) {
throw new DirectoryException('The identity provider cannot be synced');
}
$listing = $directory->listAll($this->settings->groupSync);
$plan = $this->planner->plan($this->settings, $listing, $this->state(), $mode === 'force');
$summary = $plan->summary();
if ($mode === 'preview') {
$status = 'ok';
} elseif ($plan->blocked) {
$status = 'blocked';
} else {
[$errors, $updated] = $this->apply($plan);
$summary['counts']['updates'] = $updated;
$summary['counts']['errors'] = count($errors);
$summary['errors'] = array_slice($errors, 0, self::DETAILS);
$status = 'ok';
}
} catch (DirectoryException $e) {
$status = 'error';
$summary = ['error' => $e->getMessage()];
} catch (Throwable $e) {
$this->runs->finish($id, 'error', ['error' => 'The sync run failed: ' . $e->getMessage(), 'mode' => $mode]);
throw $e;
}
$summary['mode'] = $mode;
$this->runs->finish($id, $status, $summary);
return ['id' => $id, 'mode' => $mode, 'status' => $status, 'summary' => $summary];
}
private function state(): State
{
$accounts = $this->repo->accounts($this->settings->provider);
return new State($accounts, $this->repo->mailboxes(), $this->repo->domains(),
$this->settings->groupSync ? $this->groups->groups() : [],
$this->settings->periodicSync ? $this->templates->effective(array_keys($accounts)) : [],
$this->repo->templateHashes());
}
/** @return array{0: string[], 1: int} errors of single changes (the rest is still applied), updated mailboxes */
private function apply(SyncPlan $plan): array
{
$provider = $this->settings->provider;
$errors = [];
foreach ($plan->links as $username => $externalId) {
$this->repo->link((string)$username, $provider, $externalId);
}
foreach ($plan->found as $username) {
$this->repo->setMissing($username, $provider, false);
}
foreach ($plan->missing as $username) {
$this->repo->setMissing($username, $provider, true);
}
foreach ($plan->creates as $create) {
$error = $this->legacy->createMailbox($create['email'], $create['name'], $provider, $create['template']);
if ($error !== null) {
$errors[] = $create['email'] . ': ' . $error;
continue;
}
$this->repo->link($create['email'], $provider, $create['id']);
}
foreach ($plan->deactivate as $username) {
$this->repo->setActive($username, $provider, 2, true);
}
foreach ($plan->restore as $username) {
$this->repo->setActive($username, $provider, 1, false);
}
foreach ($plan->release as $username) {
$this->repo->setIdpDisabled($username, $provider, false);
}
$this->sogo->refreshStaticView(array_merge($plan->deactivate, $plan->restore));
$changed = [];
$errors = array_merge($errors, $this->applyGroups($plan, $changed));
[$updateErrors, $updated] = $this->applyUpdates($plan, $changed);
return [array_merge($errors, $updateErrors), $updated];
}
/**
* @param string[] $changed collects the members that joined or left a group
* @return string[]
*/
private function applyGroups(SyncPlan $plan, array &$changed): array
{
$errors = [];
foreach ($plan->groupCreates as $create) {
try {
foreach ($this->groups->createFromIdp($create['ref'], $create['name'], $create['domain'], $create['mail']) as $warning) {
$errors[] = $create['name'] . ': ' . $warning[0];
}
} catch (UserError $e) {
$errors[] = $create['name'] . ': ' . $e->key;
}
}
foreach ($plan->groupRenames as $id => $name) {
try {
$this->groups->renameFromIdp((int)$id, $name);
} catch (UserError $e) {
$errors[] = $name . ': ' . $e->key;
}
}
foreach ($plan->memberships as $ref => $usernames) {
$group = $this->groups->findByIdpRef((string)$ref);
if ($group !== null) {
$this->groupResult($group->name, $this->groups->syncGroupMembers((int)$group->id, $usernames), $errors, $changed);
}
}
foreach ($plan->orphanedGroups as $id) {
$this->groupResult((string)$id, $this->groups->syncGroupMembers($id, []), $errors, $changed);
}
return $errors;
}
/**
* Name and template updates after the group changes: planned updates and members that joined or left a group
* in this run (e.g. the legacy mapping applies again after leaving a template group). The effective template
* is resolved and compared again.
*
* @return array{0: string[], 1: int} errors, updated mailboxes
*/
private function applyUpdates(SyncPlan $plan, array $changed): array
{
$candidates = array_values(array_unique(array_merge(array_column($plan->updates, 'username'),
array_intersect($changed, array_keys($plan->profiles)))));
if (empty($candidates)) {
return [[], 0];
}
$errors = [];
$updated = 0;
$accounts = $this->repo->accounts($this->settings->provider);
$hashes = $this->repo->templateHashes();
$effective = $this->templates->effective($candidates);
foreach ($candidates as $username) {
$profile = $plan->profiles[$username] ?? null;
$account = $accounts[$username] ?? null;
$template = $effective[$username] ?? $profile['template'] ?? null;
if ($profile === null || $account === null || $template === null
|| ($profile['name'] === $account->name && ($hashes[$template] ?? null) === $account->attributeHash)) {
continue;
}
$error = $this->legacy->applyTemplate($username, $profile['name'], $template);
if ($error !== null) {
$errors[] = $username . ': ' . $error;
} else {
$updated++;
}
}
return [$errors, $updated];
}
private function groupResult(string $group, array $result, array &$errors, array &$changed): void
{
foreach ($result['warnings'] as $warning) {
$errors[] = $group . ': ' . $warning[0];
}
array_push($changed, ...$result['added'], ...$result['removed']);
}
}