[Web][Dovecot] Allow parallel imapsync execution

data/Dockerfiles/dovecot/Dockerfile

@@ -69,6 +69,8 @@ RUN addgroup -g 5000 vmail \
   perl-par-packer \
   perl-parse-recdescent \
   perl-lockfile-simple \
+  perl-parallel-forkmanager \
+  perl-redis \
   libproc2 \
   perl-readonly \
   perl-regexp-common \

data/Dockerfiles/dovecot/imapsync_runner.pl

@@ -2,21 +2,15 @@
 
 use DBI;
 use LockFile::Simple qw(lock trylock unlock);
-use Proc::ProcessTable;
 use Data::Dumper qw(Dumper);
 use IPC::Run 'run';
 use File::Temp;
 use Try::Tiny;
+use Parallel::ForkManager;
+use Redis;
 use sigtrap 'handler' => \&sig_handler, qw(INT TERM KILL QUIT);
 
 sub trim { my $s = shift; $s =~ s/^\s+|\s+$//g; return $s };
-my $t = Proc::ProcessTable->new;
-my $imapsync_running = grep { $_->{cmndline} =~ /imapsync\s/i } @{$t->table};
-if ($imapsync_running ge 1)
-{
-  print "imapsync is active, exiting...";
-  exit;
-}
 
 sub qqw($) {
   my @params = ();
@@ -35,93 +29,51 @@ sub qqw($) {
   return @params;
 }
 
-$run_dir="/tmp";
+our $pm;
-$dsn = 'DBI:mysql:database=' . $ENV{'DBNAME'} . ';mysql_socket=/var/run/mysqld/mysqld.sock';
-$lock_file = $run_dir . "/imapsync_busy";
-$lockmgr = LockFile::Simple->make(-autoclean => 1, -max => 1);
-$lockmgr->lock($lock_file) || die "can't lock ${lock_file}";
-$dbh = DBI->connect($dsn, $ENV{'DBUSER'}, $ENV{'DBPASS'}, {
-  mysql_auto_reconnect => 1,
-  mysql_enable_utf8mb4 => 1
-});
-$dbh->do("UPDATE imapsync SET is_running = 0");
 
 sub sig_handler {
-  # Send die to force exception in "run"
+  if (defined $pm) {
+    foreach my $child_pid (keys %{ $pm->{processes} }) {
+      kill 'TERM', $child_pid;
+    }
+  }
   die "sig_handler received signal, preparing to exit...\n";
 };
 
-open my $file, '<', "/etc/sogo/sieve.creds"; 
+sub run_one_job {
-my $creds = <$file>; 
+  my ($dbh, $row_ref, $master_user, $master_pass) = @_;
-close $file;
-my ($master_user, $master_pass) = split /:/, $creds;
-my $sth = $dbh->prepare("SELECT id,
-  user1,
-  user2,
-  host1,
-  authmech1,
-  password1,
-  exclude,
-  port1,
-  enc1,
-  delete2duplicates,
-  maxage,
-  subfolder2,
-  delete1,
-  delete2,
-  automap,
-  skipcrossduplicates,
-  maxbytespersecond,
-  custom_params,
-  subscribeall,
-  timeout1,
-  timeout2,
-  dry
-    FROM imapsync
-      WHERE active = 1
-        AND is_running = 0
-        AND (
-          UNIX_TIMESTAMP(NOW()) - UNIX_TIMESTAMP(last_run) > mins_interval * 60
-          OR
-          last_run IS NULL)
-  ORDER BY last_run");
 
-$sth->execute();
+  my $id                  = $row_ref->[0];
-my $row;
+  my $user1               = $row_ref->[1];
-
+  my $user2               = $row_ref->[2];
-while ($row = $sth->fetchrow_arrayref()) {
+  my $host1               = $row_ref->[3];
-
+  my $authmech1           = $row_ref->[4];
-  $id                  = @$row[0];
+  my $password1           = $row_ref->[5];
-  $user1               = @$row[1];
+  my $exclude             = $row_ref->[6];
-  $user2               = @$row[2];
+  my $port1               = $row_ref->[7];
-  $host1               = @$row[3];
+  my $enc1                = $row_ref->[8];
-  $authmech1           = @$row[4];
+  my $delete2duplicates   = $row_ref->[9];
-  $password1           = @$row[5];
+  my $maxage              = $row_ref->[10];
-  $exclude             = @$row[6];
+  my $subfolder2          = $row_ref->[11];
-  $port1               = @$row[7];
+  my $delete1             = $row_ref->[12];
-  $enc1                = @$row[8];
+  my $delete2             = $row_ref->[13];
-  $delete2duplicates   = @$row[9];
+  my $automap             = $row_ref->[14];
-  $maxage              = @$row[10];
+  my $skipcrossduplicates = $row_ref->[15];
-  $subfolder2          = @$row[11];
+  my $maxbytespersecond   = $row_ref->[16];
-  $delete1             = @$row[12];
+  my $custom_params       = $row_ref->[17];
-  $delete2             = @$row[13];
+  my $subscribeall        = $row_ref->[18];
-  $automap             = @$row[14];
+  my $timeout1            = $row_ref->[19];
-  $skipcrossduplicates = @$row[15];
+  my $timeout2            = $row_ref->[20];
-  $maxbytespersecond   = @$row[16];
+  my $dry                 = $row_ref->[21];
-  $custom_params       = @$row[17];
-  $subscribeall        = @$row[18];
-  $timeout1            = @$row[19];
-  $timeout2            = @$row[20];
-  $dry                 = @$row[21];
 
   if ($enc1 eq "TLS") { $enc1 = "--tls1"; } elsif ($enc1 eq "SSL") { $enc1 = "--ssl1"; } else { undef $enc1; }
 
-  my $template = $run_dir . '/imapsync.XXXXXXX';
+  my $template = "/tmp/imapsync.XXXXXXX";
   my $passfile1 = File::Temp->new(TEMPLATE => $template);
   my $passfile2 = File::Temp->new(TEMPLATE => $template);
-  
+
   binmode( $passfile1, ":utf8" );
-  
+
   print $passfile1 "$password1\n";
   print $passfile2 trim($master_pass) . "\n";
 
@@ -157,40 +109,130 @@ while ($row = $sth->fetchrow_arrayref()) {
   '--noreleasecheck'];
 
   try {
-    $is_running = $dbh->prepare("UPDATE imapsync SET is_running = 1, success = NULL, exit_status = NULL WHERE id = ?");
+    my $is_running = $dbh->prepare("UPDATE imapsync SET is_running = 1, success = NULL, exit_status = NULL WHERE id = ?");
-    $is_running->bind_param( 1, ${id} );
+    $is_running->bind_param( 1, $id );
     $is_running->execute();
 
     run [@$generated_cmds, @$custom_params_ref], '&>', \my $stdout;
 
-    # check exit code and status
+    my ($exit_code, $exit_status) = ($stdout =~ m/Exiting\swith\sreturn\svalue\s(\d+)\s\(([^:)]+)/);
-    ($exit_code, $exit_status) = ($stdout =~ m/Exiting\swith\sreturn\svalue\s(\d+)\s\(([^:)]+)/);
 
-    $success = 0;
+    my $success = 0;
     if (defined $exit_code && $exit_code == 0) {
       $success = 1;
     }
 
-    $update = $dbh->prepare("UPDATE imapsync SET returned_text = ?, success = ?, exit_status = ? WHERE id = ?");
+    my $update = $dbh->prepare("UPDATE imapsync SET returned_text = ?, success = ?, exit_status = ? WHERE id = ?");
-    $update->bind_param( 1, ${stdout} );
+    $update->bind_param( 1, $stdout );
-    $update->bind_param( 2, ${success} );
+    $update->bind_param( 2, $success );
-    $update->bind_param( 3, ${exit_status} );
+    $update->bind_param( 3, $exit_status );
-    $update->bind_param( 4, ${id} );
+    $update->bind_param( 4, $id );
     $update->execute();
   } catch {
-    $update = $dbh->prepare("UPDATE imapsync SET returned_text = 'Could not start or finish imapsync', success = 0 WHERE id = ?");
+    my $update = $dbh->prepare("UPDATE imapsync SET returned_text = 'Could not start or finish imapsync', success = 0 WHERE id = ?");
-    $update->bind_param( 1, ${id} );
+    $update->bind_param( 1, $id );
     $update->execute();
   } finally {
-    $update = $dbh->prepare("UPDATE imapsync SET last_run = NOW(), is_running = 0 WHERE id = ?");
+    my $update = $dbh->prepare("UPDATE imapsync SET last_run = NOW(), is_running = 0 WHERE id = ?");
-    $update->bind_param( 1, ${id} );
+    $update->bind_param( 1, $id );
     $update->execute();
   };
-
-
 }
 
+my $run_dir = "/tmp";
+my $dsn = 'DBI:mysql:database=' . $ENV{'DBNAME'} . ';mysql_socket=/var/run/mysqld/mysqld.sock';
+my $lock_file = $run_dir . "/imapsync_busy";
+my $lockmgr = LockFile::Simple->make(-autoclean => 1, -max => 1);
+$lockmgr->lock($lock_file) || die "can't lock ${lock_file}";
+
+my $max_parallel = 1;
+try {
+  my $redis = Redis->new(
+    server    => 'redis-mailcow:6379',
+    password  => $ENV{'REDISPASS'},
+    reconnect => 10,
+    every     => 1_000_000,
+  );
+  my $val = $redis->get('SYNCJOBS_MAX_PARALLEL');
+  if (defined $val && $val =~ /^\d+$/) {
+    $max_parallel = int($val);
+  }
+  $redis->quit();
+} catch {
+  warn "Could not read SYNCJOBS_MAX_PARALLEL from Redis, defaulting to 1: $_";
+};
+$max_parallel = 1  if $max_parallel < 1;
+$max_parallel = 50 if $max_parallel > 50;
+
+my $dbh = DBI->connect($dsn, $ENV{'DBUSER'}, $ENV{'DBPASS'}, {
+  mysql_auto_reconnect => 1,
+  mysql_enable_utf8mb4 => 1
+});
+$dbh->do("UPDATE imapsync SET is_running = 0");
+
+open my $file, '<', "/etc/sogo/sieve.creds";
+my $creds = <$file>;
+close $file;
+my ($master_user, $master_pass) = split /:/, $creds;
+
+my $sth = $dbh->prepare("SELECT id,
+  user1,
+  user2,
+  host1,
+  authmech1,
+  password1,
+  exclude,
+  port1,
+  enc1,
+  delete2duplicates,
+  maxage,
+  subfolder2,
+  delete1,
+  delete2,
+  automap,
+  skipcrossduplicates,
+  maxbytespersecond,
+  custom_params,
+  subscribeall,
+  timeout1,
+  timeout2,
+  dry
+    FROM imapsync
+      WHERE active = 1
+        AND is_running = 0
+        AND (
+          UNIX_TIMESTAMP(NOW()) - UNIX_TIMESTAMP(last_run) > mins_interval * 60
+          OR
+          last_run IS NULL)
+  ORDER BY last_run");
+
+$sth->execute();
+my @jobs;
+while (my $row = $sth->fetchrow_arrayref()) {
+  push @jobs, [ @$row ];
+}
 $sth->finish();
 $dbh->disconnect();
 
+$pm = Parallel::ForkManager->new($max_parallel);
+
+JOB:
+foreach my $job (@jobs) {
+  my $pid = $pm->start;
+  if ($pid) {
+    next JOB;
+  }
+
+  my $child_dbh = DBI->connect($dsn, $ENV{'DBUSER'}, $ENV{'DBPASS'}, {
+    mysql_auto_reconnect => 1,
+    mysql_enable_utf8mb4 => 1
+  });
+  run_one_job($child_dbh, $job, $master_user, $master_pass);
+  $child_dbh->disconnect();
+
+  $pm->finish;
+}
+
+$pm->wait_all_children;
+
 $lockmgr->unlock($lock_file);

data/web/admin/system.php

@@ -106,6 +106,7 @@ $template_data = [
   'f2b_data' => $f2b_data,
   'f2b_banlist_url' => getBaseUrl() . "/f2b-banlist?id=" . $f2b_data['banlist_id'],
   'q_data' => quarantine('settings'),
+  'sj_data' => mailbox('get', 'syncjob_settings'),
   'qn_data' => quota_notification('get'),
   'pw_reset_data' => reset_password('get_notification'),
   'rsettings_map' => file_get_contents('http://nginx:8081/settings.php'),

data/web/api/openapi.yaml

@@ -3931,6 +3931,64 @@ paths:
                   type: object
               type: object
       summary: Update sync job
+  /api/v1/edit/syncjob_settings:
+    post:
+      responses:
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "200":
+          content:
+            application/json:
+              examples:
+                response:
+                  value:
+                    log:
+                      - entity
+                      - action
+                      - object
+                    msg:
+                      - message
+                      - entity name
+                    type: success
+              schema:
+                properties:
+                  log:
+                    description: contains request object
+                    items: {}
+                    type: array
+                  msg:
+                    items: {}
+                    type: array
+                  type:
+                    enum:
+                      - success
+                      - danger
+                      - error
+                    type: string
+                type: object
+          description: OK
+          headers: {}
+      tags:
+        - Sync jobs
+      description: >-
+        Update the global sync job settings. Currently exposes the maximum
+        number of imapsync processes that are allowed to run in parallel.
+        Admin access required.
+      operationId: Update sync job settings
+      requestBody:
+        content:
+          application/json:
+            schema:
+              example:
+                max_parallel: 4
+              properties:
+                max_parallel:
+                  description: >-
+                    Maximum number of imapsync processes allowed to run in
+                    parallel (1 = sequential behavior, max 50)
+                  type: integer
+              type: object
+      summary: Update sync job settings
   /api/v1/edit/user-acl:
     post:
       responses:
@@ -5629,6 +5687,36 @@ paths:
       description: You can list all syn jobs existing in system.
       operationId: Get sync jobs
       summary: Get sync jobs
+  /api/v1/get/syncjob_settings:
+    get:
+      responses:
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "200":
+          content:
+            application/json:
+              examples:
+                response:
+                  value:
+                    max_parallel: 4
+              schema:
+                properties:
+                  max_parallel:
+                    description: >-
+                      Maximum number of imapsync processes allowed to run in
+                      parallel (1 = sequential behavior)
+                    type: integer
+                type: object
+          description: OK
+          headers: {}
+      tags:
+        - Sync jobs
+      description: >-
+        Return the global sync job settings. Currently exposes the maximum
+        number of imapsync processes allowed to run in parallel. Admin access
+        required.
+      operationId: Get sync job settings
+      summary: Get sync job settings
   "/api/v1/get/tls-policy-map/{id}":
     get:
       parameters:

data/web/inc/functions.mailbox.inc.php

@@ -2265,6 +2265,35 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             );
           }
         break;
+        case 'syncjob_settings':
+          if (!isset($_SESSION['mailcow_cc_role']) || $_SESSION['mailcow_cc_role'] != 'admin') {
+            $_SESSION['return'][] = array(
+              'type' => 'danger',
+              'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+              'msg' => 'access_denied'
+            );
+            return false;
+          }
+          $max_parallel = intval($_data['max_parallel']);
+          if ($max_parallel < 1)  { $max_parallel = 1; }
+          if ($max_parallel > 50) { $max_parallel = 50; }
+          try {
+            $redis->Set('SYNCJOBS_MAX_PARALLEL', $max_parallel);
+          }
+          catch (RedisException $e) {
+            $_SESSION['return'][] = array(
+              'type' => 'danger',
+              'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+              'msg' => array('redis_error', $e->getMessage())
+            );
+            return false;
+          }
+          $_SESSION['return'][] = array(
+            'type' => 'success',
+            'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+            'msg' => 'syncjob_settings_saved'
+          );
+        break;
         case 'syncjob':
           if (!is_array($_data['id'])) {
             $ids = array();
@@ -4619,6 +4648,20 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           }
           return $syncjobdetails;
         break;
+        case 'syncjob_settings':
+          if (!isset($_SESSION['mailcow_cc_role']) || $_SESSION['mailcow_cc_role'] != 'admin') {
+            return false;
+          }
+          $settings = array();
+          try {
+            $max_parallel = $redis->Get('SYNCJOBS_MAX_PARALLEL');
+          }
+          catch (RedisException $e) {
+            $max_parallel = null;
+          }
+          $settings['max_parallel'] = intval($max_parallel) ?: 1;
+          return $settings;
+        break;
         case 'syncjobs':
           $syncjobdata = array();
           if (isset($_data) && filter_var($_data, FILTER_VALIDATE_EMAIL)) {

data/web/json_api.php

@@ -1104,6 +1104,10 @@ if (isset($_GET['query'])) {
               break;
             }
           break;
+          case "syncjob_settings":
+            $data = mailbox('get', 'syncjob_settings');
+            process_get_return($data);
+          break;
           case "active-user-sieve":
             if (isset($object)) {
               $sieve_filter = mailbox('get', 'active_user_sieve', $object);
@@ -1970,6 +1974,9 @@ if (isset($_GET['query'])) {
         case "syncjob":
           process_edit_return(mailbox('edit', 'syncjob', array_merge(array('id' => $items), $attr)));
         break;
+        case "syncjob_settings":
+          process_edit_return(mailbox('edit', 'syncjob_settings', $attr));
+        break;
         case "filter":
           process_edit_return(mailbox('edit', 'filter', array_merge(array('id' => $items), $attr)));
         break;

data/web/lang/lang.de-de.json

@@ -372,6 +372,8 @@
         "spamfilter": "Spamfilter",
         "subject": "Betreff",
         "success": "Erfolg",
+        "syncjobs": "Sync-Jobs",
+        "syncjobs_max_parallel": "Maximale Anzahl paralleler Sync-Jobs<br><small>Wie viele imapsync-Prozesse dürfen gleichzeitig laufen. 1 = sequentiell (aktuelles Verhalten).</small>",
         "sys_mails": "System-E-Mails",
         "task": "Aufgabe",
         "text": "Text",
@@ -1200,6 +1202,7 @@
         "template_modified": "Änderungen am Template %s wurden gespeichert",
         "template_removed": "Template ID %s wurde gelöscht",
         "sogo_profile_reset": "ActiveSync-Gerät des Benutzers %s wurde zurückgesetzt",
+        "syncjob_settings_saved": "Sync-Job-Einstellungen wurden gespeichert",
         "tls_policy_map_entry_deleted": "TLS-Richtlinie mit der ID %s wurde gelöscht",
         "tls_policy_map_entry_saved": "TLS-Richtlinieneintrag \"%s\" wurde gespeichert",
         "ui_texts": "Änderungen an UI-Texten",

data/web/lang/lang.en-gb.json

@@ -382,6 +382,8 @@
         "spamfilter": "Spam filter",
         "subject": "Subject",
         "success": "Success",
+        "syncjobs": "Sync jobs",
+        "syncjobs_max_parallel": "Maximum parallel sync jobs<br><small>How many imapsync processes are allowed to run in parallel. 1 = sequential (legacy behavior).</small>",
         "sys_mails": "System mails",
         "task": "Task",
         "text": "Text",
@@ -1204,6 +1206,7 @@
         "settings_map_added": "Added settings map entry",
         "settings_map_removed": "Removed settings map ID %s",
         "sogo_profile_reset": "SOGo profile for user %s was reset",
+        "syncjob_settings_saved": "Sync job settings have been saved",
         "template_added": "Added template %s",
         "template_modified": "Changes to template %s have been saved",
         "template_removed": "Template ID %s has been deleted",

data/web/templates/admin.twig

@@ -21,6 +21,7 @@
         <li><button class="dropdown-item" data-bs-target="#tab-config-fwdhosts" aria-selected="false" aria-controls="tab-config-fwdhosts" role="tab" data-bs-toggle="tab">{{ lang.admin.forwarding_hosts }}</button></li>
         <li><button class="dropdown-item" data-bs-target="#tab-config-f2b" aria-selected="false" aria-controls="tab-config-f2b" role="tab" data-bs-toggle="tab">{{ lang.admin.f2b_parameters }}</button></li>
         <li><button class="dropdown-item" data-bs-target="#tab-config-quarantine" aria-selected="false" aria-controls="tab-config-quarantine" role="tab" data-bs-toggle="tab">{{ lang.admin.quarantine }}</button></li>
+        <li><button class="dropdown-item" data-bs-target="#tab-config-syncjobs" aria-selected="false" aria-controls="tab-config-syncjobs" role="tab" data-bs-toggle="tab">{{ lang.admin.syncjobs }}</button></li>
         <li><button class="dropdown-item" data-bs-target="#tab-config-quota" aria-selected="false" aria-controls="tab-config-quota" role="tab" data-bs-toggle="tab">{{ lang.admin.quota_notifications }}</button></li>
         <li><button class="dropdown-item" data-bs-target="#tab-config-rsettings" aria-selected="false" aria-controls="tab-config-rsettings" role="tab" data-bs-toggle="tab">{{ lang.admin.rspamd_settings_map }}</button></li>
         <li><button class="dropdown-item" data-bs-target="#tab-config-password-settings" aria-selected="false" aria-controls="tab-config-password-settings" role="tab" data-bs-toggle="tab">{{ lang.admin.password_settings }}</button></li>
@@ -50,6 +51,7 @@
         {% include 'admin/tab-config-fwdhosts.twig' %}
         {% include 'admin/tab-config-f2b.twig' %}
         {% include 'admin/tab-config-quarantine.twig' %}
+        {% include 'admin/tab-config-syncjobs.twig' %}
         {% include 'admin/tab-config-quota.twig' %}
         {% include 'admin/tab-config-rsettings.twig' %}
         {% include 'admin/tab-config-customize.twig' %}

data/web/templates/admin/tab-config-syncjobs.twig

@@ -0,0 +1,21 @@
+<div class="tab-pane fade" id="tab-config-syncjobs" role="tabpanel" aria-labelledby="tab-config-syncjobs">
+  <div class="card mb-4">
+    <div class="card-header d-flex fs-5">
+      <button class="btn d-md-none flex-grow-1 text-start" data-bs-target="#collapse-tab-config-syncjobs" data-bs-toggle="collapse" aria-controls="collapse-tab-config-syncjobs">
+        {{ lang.admin.syncjobs }}
+      </button>
+      <span class="d-none d-md-block">{{ lang.admin.syncjobs }}</span>
+    </div>
+    <div id="collapse-tab-config-syncjobs" class="card-body collapse" data-bs-parent="#admin-content">
+      <form class="form-horizontal" data-id="syncjob_settings" role="form" method="post">
+        <div class="row mb-4">
+          <label class="col-sm-4 control-label text-sm-end" for="syncjobs_max_parallel">{{ lang.admin.syncjobs_max_parallel|raw }}</label>
+          <div class="col-sm-8">
+            <input type="number" class="form-control" id="syncjobs_max_parallel" name="max_parallel" value="{{ sj_data.max_parallel }}" min="1" max="50" required>
+          </div>
+        </div>
+        <button class="btn btn-sm d-block d-sm-inline btn-success" data-action="edit_selected" data-item="self" data-id="syncjob_settings" data-api-url='edit/syncjob_settings' href="#"><i class="bi bi-check-lg"></i> {{ lang.admin.save }}</button>
+      </form>
+    </div>
+  </div>
+</div>

docker-compose.yml

@@ -252,7 +252,7 @@ services:
             - sogo
 
     dovecot-mailcow:
-      image: ghcr.io/mailcow/dovecot:2.3.21.1-2
+      image: ghcr.io/mailcow/dovecot:2.3.21.1-3
       depends_on:
         - mysql-mailcow
         - netfilter-mailcow