[Agent] Replace dockerapi container with Redis-based control bus

data/web/admin/dashboard.php

@@ -16,46 +16,92 @@ if (!isset($_SESSION['gal']) && $license_cache = $redis->Get('LICENSE_STATUS_CAC
 
 $js_minifier->add('/web/js/site/dashboard.js');
 
-// vmail df
-$exec_fields = array('cmd' => 'system', 'task' => 'df', 'dir' => '/var/vmail');
-$vmail_df = explode(',', (string)json_decode(docker('post', 'dovecot-mailcow', 'exec', $exec_fields), true));
+$vmail_df_resp = agent('request', 'dovecot', 'exec.df', array('dir' => '/var/vmail'), 5);
+$vmail_df = (!empty($vmail_df_resp['ok']) && is_string($vmail_df_resp['result']))
+  ? explode(',', $vmail_df_resp['result'])
+  : array('', '', '', '', '', '/var/vmail');
 
-// containers
-$containers_info = (array) docker('info');
-if ($clamd_status === false) unset($containers_info['clamd-mailcow']);
-if ($olefy_status === false) unset($containers_info['olefy-mailcow']);
-ksort($containers_info);
-$containers = array();
-foreach ($containers_info as $container => $container_info) {
-  if (!isset($container_info['State']) || !is_array($container_info['State']) || !isset($container_info['State']['StartedAt'])){
-    continue;
-  }
-  date_default_timezone_set('UTC');
-  $StartedAt = date_parse($container_info['State']['StartedAt']);
-  if ($StartedAt['hour'] !== false) {
-    $date = new \DateTime();
-    $date->setTimestamp(mktime(
-      $StartedAt['hour'],
-      $StartedAt['minute'],
-      $StartedAt['second'],
-      $StartedAt['month'],
-      $StartedAt['day'],
-      $StartedAt['year']));
-    try {
-      $user_tz = new DateTimeZone(getenv('TZ'));
-      $date->setTimezone($user_tz);
-      $container_info['State']['StartedAtHR'] = $date->format('r');
-    } catch(Exception $e) {
-      $container_info['State']['StartedAtHR'] = '?';
-    }
-  }
-  else {
-    $container_info['State']['StartedAtHR'] = '?';
-  }
-  $containers[$container] = $container_info;
+$known_services = agent('services');
+
+try {
+  $tz_obj = new DateTimeZone(getenv('TZ') ?: 'UTC');
+}
+catch (Exception $e) {
+  $tz_obj = new DateTimeZone('UTC');
 }
 
-// get mailcow data
+$containers = array();
+foreach ($known_services as $svc) {
+  $live_nodes = agent('live_nodes', $svc);
+  $running = !empty($live_nodes);
+  $first_node = $running ? $live_nodes[0] : '';
+  $first_meta = $running ? (agent('node_meta', $svc, $first_node) ?: array()) : array();
+
+  $started_at_hr = '—';
+  $started_at_iso = isset($first_meta['started_at']) ? $first_meta['started_at'] : '';
+  if ($started_at_iso !== '') {
+    try {
+      $d = new DateTime($started_at_iso);
+      $d->setTimezone($tz_obj);
+      $started_at_hr = $d->format('r');
+    }
+    catch (Exception $e) {}
+  }
+
+  $nodes = array();
+  $unhealthy_nodes = 0;
+  $first_unhealthy_detail = '';
+  foreach ($live_nodes as $n) {
+    $m = agent('node_meta', $svc, $n) ?: array();
+    $s = agent('node_stats', $svc, $n) ?: array();
+    $node_health = isset($m['health']) ? $m['health'] : '';
+    $node_health_detail = isset($m['health_detail']) ? $m['health_detail'] : '';
+    if ($node_health === 'fail') {
+      $unhealthy_nodes++;
+      if ($first_unhealthy_detail === '') {
+        $first_unhealthy_detail = $node_health_detail;
+      }
+    }
+    $nodes[] = array(
+      'NodeId' => $n,
+      'Image' => isset($m['image']) ? $m['image'] : '',
+      'StartedAt' => isset($m['started_at']) ? $m['started_at'] : '',
+      'Version' => isset($m['version']) ? $m['version'] : '',
+      'CPUPercent' => isset($s['cpu_percent']) ? $s['cpu_percent'] : '',
+      'MemoryBytes' => isset($s['memory_bytes']) ? $s['memory_bytes'] : '',
+      'Health' => $node_health,
+      'HealthDetail' => $node_health_detail
+    );
+  }
+
+  $service_health = 'unknown';
+  if ($running) {
+    $service_health = ($unhealthy_nodes === 0) ? 'ok' : (($unhealthy_nodes === count($live_nodes)) ? 'fail' : 'degraded');
+  }
+
+  $containers[$svc . '-mailcow'] = array(
+    'Service' => $svc,
+    'State' => array(
+      'Running' => $running ? 1 : 0,
+      'NodeCount' => count($live_nodes),
+      'UnhealthyCount' => $unhealthy_nodes,
+      'Health' => $service_health,
+      'HealthDetail' => $first_unhealthy_detail,
+      'StartedAt' => $started_at_iso,
+      'StartedAtHR' => $started_at_hr
+    ),
+    'Config' => array(
+      'Image' => isset($first_meta['image']) ? $first_meta['image'] : ''
+    ),
+    'Id' => $first_node,
+    'Nodes' => $nodes,
+    'External' => false
+  );
+}
+
+$infra_containers = infra('status');
+ksort($containers);
+
 $hostname = getenv('MAILCOW_HOSTNAME');
 $timezone = getenv('TZ');
 
@@ -70,6 +116,7 @@ $template_data = [
   'clamd_status' => $clamd_status,
   'olefy_status' => $olefy_status,
   'containers' => $containers,
+  'infra_containers' => $infra_containers,
   'ip_check' => customize('get', 'ip_check'),
   'lang_admin' => json_encode($lang['admin']),
   'lang_debug' => json_encode($lang['debug']),
@@ -77,5 +124,3 @@ $template_data = [
 ];
 
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/footer.inc.php';
-
-

data/web/inc/ajax/container_ctrl.php

@@ -1,59 +1,35 @@
-<?php
-
-// Block requests by checking the 'Sec-Fetch-Dest' header.
-if (isset($_SERVER['HTTP_SEC_FETCH_DEST']) && $_SERVER['HTTP_SEC_FETCH_DEST'] !== 'empty') {
-  header('HTTP/1.1 403 Forbidden');
-  exit;
-}
-
-require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/prerequisites.inc.php';
-if (!isset($_SESSION['mailcow_cc_role']) || $_SESSION['mailcow_cc_role'] != 'admin') {
-	exit();
-}
-
-if (preg_match('/^[a-z\-]{0,}-mailcow/', $_GET['service'])) {
-  if ($_GET['action'] == "start") {
-    header('Content-Type: text/html; charset=utf-8');
-    $retry = 0;
-    while (docker('info', $_GET['service'])['State']['Running'] != 1 && $retry <= 3) {
-      $response = docker('post', $_GET['service'], 'start');
-      $response = json_decode($response, true);
-      $last_response = ($response['type'] == "success") ? '<b><span class="pull-right text-success">OK</span></b>' : '<b><span class="pull-right text-danger">Error: ' . $response['msg'] . '</span></b>';
-      if ($response['type'] == "success") {
-        break;
-      }
-      usleep(1500000);
-      $retry++;
-    }
-    echo (!isset($last_response)) ? '<b><span class="pull-right text-warning">Already running</span></b>' : $last_response;
-  }
-  if ($_GET['action'] == "stop") {
-    header('Content-Type: text/html; charset=utf-8');
-    $retry = 0;
-    while (docker('info', $_GET['service'])['State']['Running'] == 1 && $retry <= 3) {
-      $response = docker('post', $_GET['service'], 'stop');
-      $response = json_decode($response, true);
-      $last_response = ($response['type'] == "success") ? '<b><span class="pull-right text-success">OK</span></b>' : '<b><span class="pull-right text-danger">Error: ' . $response['msg'] . '</span></b>';
-      if ($response['type'] == "success") {
-        break;
-      }
-      usleep(1500000);
-      $retry++;
-    }
-    echo (!isset($last_response)) ? '<b><span class="pull-right text-warning">Not running</span></b>' : $last_response;
-  }
-  if ($_GET['action'] == "restart") {
-    header('Content-Type: text/html; charset=utf-8');
-    $response = docker('post', $_GET['service'], 'restart');
-    $response = json_decode($response, true);
-    $last_response = ($response['type'] == "success") ? '<b><span class="pull-right text-success">OK</span></b>' : '<b><span class="pull-right text-danger">Error: ' . $response['msg'] . '</span></b>';
-    echo (!isset($last_response)) ? '<b><span class="pull-right text-warning">Cannot restart container</span></b>' : $last_response;
-  }
-  if ($_GET['action'] == "logs") {
-    $lines = (empty($_GET['lines']) || !is_numeric($_GET['lines'])) ? 1000 : $_GET['lines'];
-    header('Content-Type: text/plain; charset=utf-8');
-    print_r(preg_split('/\n/', docker('logs', $_GET['service'], $lines)));
-  }
-}
-
-?>
+<?php
+if (isset($_SERVER['HTTP_SEC_FETCH_DEST']) && $_SERVER['HTTP_SEC_FETCH_DEST'] !== 'empty') {
+  header('HTTP/1.1 403 Forbidden');
+  exit;
+}
+
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/prerequisites.inc.php';
+if (!isset($_SESSION['mailcow_cc_role']) || $_SESSION['mailcow_cc_role'] != 'admin') {
+  exit();
+}
+
+if (!preg_match('/^[a-z\-]{0,}-mailcow/', $_GET['service'] ?? '')) {
+  exit();
+}
+
+if (($_GET['action'] ?? '') !== 'restart') {
+  exit();
+}
+
+$service = preg_replace('/-mailcow$/', '', $_GET['service']);
+$node = isset($_GET['node']) ? preg_replace('/[^a-zA-Z0-9._\-]/', '', $_GET['node']) : '';
+
+$args = ($node !== '') ? array('target_node' => $node) : array();
+$resp = agent('request', $service, 'restart', $args, 60);
+header('Content-Type: text/html; charset=utf-8');
+if (agent('ok', $resp)) {
+  echo '<b><span class="pull-right text-success">' . htmlspecialchars($lang['success']['service_restart_ok']) . '</span></b>';
+}
+else {
+  $err_key = agent('error_lang', $resp);
+  $err_msg = isset($lang['danger'][$err_key])
+    ? sprintf($lang['danger'][$err_key], $service)
+    : $lang['danger']['agent_unknown_error'];
+  echo '<b><span class="pull-right text-danger">' . htmlspecialchars($err_msg) . '</span></b>';
+}

data/web/inc/functions.agent.inc.php

@@ -0,0 +1,281 @@
+<?php
+define('AGENT_ERR_NOT_FOUND',   'not_found');
+define('AGENT_ERR_TIMEOUT',     'timeout');
+define('AGENT_ERR_VALIDATION',  'validation');
+define('AGENT_ERR_UNSUPPORTED', 'unsupported_command');
+define('AGENT_ERR_INTERNAL',    'internal');
+
+function agent($_action, $_service = null, $_data = null, $_args = array(), $_timeout = 10) {
+  global $redis;
+  switch ($_action) {
+    case 'request_id':
+      return sprintf('%013d%s', (int)(microtime(true) * 1000), substr(bin2hex(random_bytes(10)), 0, 16));
+    break;
+    case 'services':
+      $list = array(
+        'unbound', 'clamd', 'rspamd', 'php-fpm', 'sogo',
+        'dovecot', 'postfix', 'postfix-tlspol', 'nginx', 'acme',
+        'netfilter', 'watchdog', 'olefy', 'host'
+      );
+      if (preg_match('/^([yY][eE][sS]|[yY])+$/', isset($_ENV['SKIP_CLAMD']) ? $_ENV['SKIP_CLAMD'] : '')) {
+        $list = array_values(array_diff($list, array('clamd')));
+      }
+      if (preg_match('/^([yY][eE][sS]|[yY])+$/', isset($_ENV['SKIP_OLEFY']) ? $_ENV['SKIP_OLEFY'] : '')) {
+        $list = array_values(array_diff($list, array('olefy')));
+      }
+      sort($list);
+      return $list;
+    break;
+    case 'live_nodes':
+      try {
+        $members = $redis->zRangeByScore('mailcow.nodes.' . $_service, (string)(time() - 30), '+inf');
+      }
+      catch (RedisException $e) {
+        return array();
+      }
+      return is_array($members) ? $members : array();
+    break;
+    case 'node_meta':
+      try {
+        $h = $redis->hGetAll('mailcow.node.' . $_service . '.' . $_data);
+      }
+      catch (RedisException $e) {
+        return null;
+      }
+      return $h ?: null;
+    break;
+    case 'node_stats':
+      try {
+        $h = $redis->hGetAll('mailcow.stats.' . $_service . '.' . $_data);
+      }
+      catch (RedisException $e) {
+        return null;
+      }
+      return $h ?: null;
+    break;
+    case 'stats':
+      $out = array();
+      foreach (agent('live_nodes', $_service) as $node_id) {
+        $stats = agent('node_stats', $_service, $node_id);
+        if ($stats) {
+          $out[$node_id] = $stats;
+        }
+      }
+      return $out;
+    break;
+    case 'publish':
+      $env = array(
+        'cmd' => $_data,
+        'request_id' => agent('request_id'),
+        'args' => (object)(is_array($_args) ? $_args : array()),
+        'issued_by' => 'mailcow-php'
+      );
+      try {
+        $redis->publish('mailcow.control.' . $_service, json_encode($env));
+      }
+      catch (RedisException $e) {
+        return false;
+      }
+      return true;
+    break;
+    case 'request':
+      $rid = agent('request_id');
+      $reply_to = 'mailcow.reply.' . $rid;
+      $env = array(
+        'cmd' => $_data,
+        'request_id' => $rid,
+        'args' => (object)(is_array($_args) ? $_args : array()),
+        'reply_to' => $reply_to,
+        'deadline' => gmdate('Y-m-d\TH:i:s\Z', time() + $_timeout),
+        'issued_by' => 'mailcow-php'
+      );
+      try {
+        $subs = $redis->publish('mailcow.control.' . $_service, json_encode($env));
+        if ($subs === 0) {
+          return array('ok' => false, 'result' => null, 'error' => $_service, 'error_code' => AGENT_ERR_NOT_FOUND, 'node' => '', 'duration_ms' => 0);
+        }
+        $popped = $redis->blPop(array($reply_to), $_timeout);
+      }
+      catch (RedisException $e) {
+        return array('ok' => false, 'result' => null, 'error' => $e->getMessage(), 'error_code' => AGENT_ERR_INTERNAL, 'node' => '', 'duration_ms' => 0);
+      }
+      if (!$popped || count($popped) < 2) {
+        return array('ok' => false, 'result' => null, 'error' => '', 'error_code' => AGENT_ERR_TIMEOUT, 'node' => '', 'duration_ms' => 0);
+      }
+      $resp = json_decode($popped[1], true);
+      if (!is_array($resp)) {
+        return array('ok' => false, 'result' => null, 'error' => 'malformed reply', 'error_code' => AGENT_ERR_INTERNAL, 'node' => '', 'duration_ms' => 0);
+      }
+      return array(
+        'ok' => !empty($resp['ok']),
+        'result' => isset($resp['result']) ? $resp['result'] : null,
+        'error' => isset($resp['error']) ? $resp['error'] : '',
+        'error_code' => isset($resp['error_code']) ? $resp['error_code'] : '',
+        'node' => isset($resp['node']) ? $resp['node'] : '',
+        'duration_ms' => isset($resp['duration_ms']) ? $resp['duration_ms'] : 0
+      );
+    break;
+    case 'request_all':
+      $rid = agent('request_id');
+      $reply_to = 'mailcow.reply.' . $rid;
+      $env = array(
+        'cmd' => $_data,
+        'request_id' => $rid,
+        'args' => (object)(is_array($_args) ? $_args : array()),
+        'reply_to' => $reply_to,
+        'deadline' => gmdate('Y-m-d\TH:i:s\Z', time() + $_timeout),
+        'issued_by' => 'mailcow-php'
+      );
+      $expected = max(1, count(agent('live_nodes', $_service)));
+      try {
+        $subs = (int)$redis->publish('mailcow.control.' . $_service, json_encode($env));
+      }
+      catch (RedisException $e) {
+        return array('responses' => array(), 'expected_nodes' => $expected, 'received_nodes' => array(), 'missing_nodes' => array(), 'error' => $e->getMessage());
+      }
+      if ($subs === 0) {
+        return array('responses' => array(), 'expected_nodes' => 0, 'received_nodes' => array(), 'missing_nodes' => array());
+      }
+      $responses = array();
+      $deadline = microtime(true) + $_timeout;
+      for ($i = 0; $i < $subs; $i++) {
+        $remaining = (int)ceil($deadline - microtime(true));
+        if ($remaining <= 0) break;
+        try {
+          $popped = $redis->blPop(array($reply_to), $remaining);
+        }
+        catch (RedisException $e) {
+          break;
+        }
+        if (!$popped || count($popped) < 2) break;
+        $resp = json_decode($popped[1], true);
+        if (is_array($resp)) {
+          $responses[] = array(
+            'ok' => !empty($resp['ok']),
+            'result' => isset($resp['result']) ? $resp['result'] : null,
+            'error' => isset($resp['error']) ? $resp['error'] : '',
+            'error_code' => isset($resp['error_code']) ? $resp['error_code'] : '',
+            'node' => isset($resp['node']) ? $resp['node'] : '',
+            'duration_ms' => isset($resp['duration_ms']) ? $resp['duration_ms'] : 0
+          );
+        }
+      }
+      $received_nodes = array();
+      foreach ($responses as $r) {
+        if (!empty($r['node'])) {
+          $received_nodes[] = $r['node'];
+        }
+      }
+      $live = agent('live_nodes', $_service);
+      return array(
+        'responses' => $responses,
+        'expected_nodes' => $expected,
+        'received_nodes' => array_values(array_unique($received_nodes)),
+        'missing_nodes' => array_values(array_diff($live, $received_nodes))
+      );
+    break;
+    case 'ok':
+      if (isset($_service['responses'])) {
+        foreach ($_service['responses'] as $r) {
+          if (!empty($r['ok'])) return true;
+        }
+        return false;
+      }
+      return !empty($_service['ok']);
+    break;
+    case 'first_error':
+      foreach (isset($_service['responses']) ? $_service['responses'] : array() as $r) {
+        if (empty($r['ok']) && !empty($r['error'])) return $r['error'];
+      }
+      return '';
+    break;
+    case 'error_lang':
+      $code = is_array($_service) && isset($_service['error_code']) ? $_service['error_code'] : '';
+      switch ($code) {
+        case AGENT_ERR_NOT_FOUND:
+          return 'no_live_agent';
+        case AGENT_ERR_TIMEOUT:
+          return 'agent_timeout';
+        default:
+          return 'agent_unknown_error';
+      }
+    break;
+  }
+}
+
+function infra($_action, $_service = null) {
+  global $redis;
+  global $pdo;
+  switch ($_action) {
+    case 'health':
+      switch ($_service) {
+        case 'redis':
+          try {
+            if ($redis instanceof Redis && $redis->ping()) {
+              $info = $redis->info('server');
+              $ver = is_array($info) && isset($info['redis_version']) ? $info['redis_version'] : '';
+              return array('ok' => true, 'image' => 'redis ' . $ver, 'error' => '');
+            }
+          }
+          catch (RedisException $e) {
+            return array('ok' => false, 'image' => 'redis', 'error' => $e->getMessage());
+          }
+          return array('ok' => false, 'image' => 'redis', 'error' => 'PING returned false');
+        break;
+        case 'mysql':
+          try {
+            if ($pdo instanceof PDO) {
+              $row = $pdo->query('SELECT VERSION() AS v')->fetch(PDO::FETCH_ASSOC);
+              $ver = $row && isset($row['v']) ? $row['v'] : '';
+              return array('ok' => true, 'image' => 'mariadb/mysql ' . $ver, 'error' => '');
+            }
+          }
+          catch (Exception $e) {
+            return array('ok' => false, 'image' => 'mariadb/mysql', 'error' => $e->getMessage());
+          }
+          return array('ok' => false, 'image' => 'mariadb/mysql', 'error' => 'no PDO handle');
+        break;
+        case 'memcached':
+          $sock = @fsockopen('memcached', 11211, $errno, $errstr, 2);
+          if (!$sock) {
+            return array('ok' => false, 'image' => 'memcached', 'error' => $errstr ?: 'connection refused');
+          }
+          stream_set_timeout($sock, 2);
+          fwrite($sock, "version\r\n");
+          $line = fgets($sock, 64);
+          fclose($sock);
+          if (is_string($line) && strpos($line, 'VERSION') === 0) {
+            return array('ok' => true, 'image' => 'memcached ' . trim(substr($line, strlen('VERSION '))), 'error' => '');
+          }
+          return array('ok' => false, 'image' => 'memcached', 'error' => 'no VERSION reply');
+        break;
+      }
+    break;
+    case 'status':
+      $out = array();
+      $defs = array(
+        'redis-mailcow' => 'redis',
+        'mysql-mailcow' => 'mysql',
+        'memcached-mailcow' => 'memcached'
+      );
+      foreach ($defs as $key => $svc) {
+        $h = infra('health', $svc);
+        $out[$key] = array(
+          'Service' => $svc,
+          'State' => array(
+            'Running' => $h['ok'] ? 1 : 0,
+            'NodeCount' => $h['ok'] ? 1 : 0,
+            'StartedAt' => '',
+            'StartedAtHR' => '—',
+            'Error' => $h['error']
+          ),
+          'Config' => array('Image' => $h['image']),
+          'Id' => $svc,
+          'Nodes' => array(),
+          'External' => true
+        );
+      }
+      return $out;
+    break;
+  }
+}

data/web/inc/functions.docker.inc.php

@@ -1,207 +0,0 @@
-<?php
-function docker($action, $service_name = null, $attr1 = null, $attr2 = null, $extra_headers = null) {
-  global $DOCKER_TIMEOUT;
-  global $redis;
-  $curl = curl_init();
-  curl_setopt($curl, CURLOPT_HTTPHEADER,array('Content-Type: application/json' ));
-  // We are using our mail certificates for dockerapi, the names will not match, the certs are trusted anyway
-  curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, 0);
-  curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, 0);
-  switch($action) {
-    case 'get_id':
-      curl_setopt($curl, CURLOPT_URL, 'https://dockerapi:443/containers/json');
-      curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
-      curl_setopt($curl, CURLOPT_POST, 0);
-      curl_setopt($curl, CURLOPT_TIMEOUT, $DOCKER_TIMEOUT);
-      $response = curl_exec($curl);
-      if ($response === false) {
-        $err = curl_error($curl);
-        curl_close($curl);
-        return $err;
-      }
-      else {
-        curl_close($curl);
-        $containers = json_decode($response, true);
-        if (!empty($containers)) {
-          foreach ($containers as $container) {
-            if (isset($container['Config']['Labels']['com.docker.compose.service'])
-              && $container['Config']['Labels']['com.docker.compose.service'] == $service_name
-              && strtolower($container['Config']['Labels']['com.docker.compose.project']) == strtolower(getenv('COMPOSE_PROJECT_NAME'))) {
-              return trim($container['Id']);
-            }
-          }
-        }
-      }
-      return false;
-    break;
-    case 'containers':
-      curl_setopt($curl, CURLOPT_URL, 'https://dockerapi:443/containers/json');
-      curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
-      curl_setopt($curl, CURLOPT_POST, 0);
-      curl_setopt($curl, CURLOPT_TIMEOUT, $DOCKER_TIMEOUT);
-      $response = curl_exec($curl);
-      if ($response === false) {
-        $err = curl_error($curl);
-        curl_close($curl);
-        return $err;
-      }
-      else {
-        curl_close($curl);
-        $containers = json_decode($response, true);
-        if (!empty($containers)) {
-          foreach ($containers as $container) {
-            if (strtolower($container['Config']['Labels']['com.docker.compose.project']) == strtolower(getenv('COMPOSE_PROJECT_NAME'))) {
-              $out[$container['Config']['Labels']['com.docker.compose.service']]['State'] = $container['State'];
-              $out[$container['Config']['Labels']['com.docker.compose.service']]['Config'] = $container['Config'];
-              $out[$container['Config']['Labels']['com.docker.compose.service']]['Id'] = trim($container['Id']);
-            }
-          }
-        }
-        return (!empty($out)) ? $out : false;
-      }
-      return false;
-    break;
-    case 'info':
-      if (empty($service_name)) {
-        curl_setopt($curl, CURLOPT_URL, 'https://dockerapi:443/containers/json?all=true');
-        curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
-        curl_setopt($curl, CURLOPT_POST, 0);
-        curl_setopt($curl, CURLOPT_TIMEOUT, $DOCKER_TIMEOUT);
-      }
-      else {
-        $container_id = docker('get_id', $service_name);
-        if (ctype_xdigit($container_id)) {
-          curl_setopt($curl, CURLOPT_URL, 'https://dockerapi:443/containers/' . $container_id . '/json');
-        }
-        else {
-          return false;
-        }
-      }
-      curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
-      curl_setopt($curl, CURLOPT_POST, 0);
-      curl_setopt($curl, CURLOPT_TIMEOUT, $DOCKER_TIMEOUT);
-      $response = curl_exec($curl);
-      if ($response === false) {
-        $err = curl_error($curl);
-        curl_close($curl);
-        return $err;
-      }
-      else {
-        curl_close($curl);
-        $decoded_response = json_decode($response, true);
-        if (!empty($decoded_response)) {
-          if (empty($service_name)) {
-            foreach ($decoded_response as $container) {
-              if (isset($container['Config']['Labels']['com.docker.compose.project'])
-                && strtolower($container['Config']['Labels']['com.docker.compose.project']) == strtolower(getenv('COMPOSE_PROJECT_NAME'))) {
-                unset($container['Config']['Env']);
-                $out[$container['Config']['Labels']['com.docker.compose.service']]['State'] = $container['State'];
-                $out[$container['Config']['Labels']['com.docker.compose.service']]['Config'] = $container['Config'];
-                $out[$container['Config']['Labels']['com.docker.compose.service']]['Id'] = trim($container['Id']);
-              }
-            }
-          }
-          else {
-            if (isset($decoded_response['Config']['Labels']['com.docker.compose.project']) 
-              && strtolower($decoded_response['Config']['Labels']['com.docker.compose.project']) == strtolower(getenv('COMPOSE_PROJECT_NAME'))) {
-              unset($container['Config']['Env']);
-              $out[$decoded_response['Config']['Labels']['com.docker.compose.service']]['State'] = $decoded_response['State'];
-              $out[$decoded_response['Config']['Labels']['com.docker.compose.service']]['Config'] = $decoded_response['Config'];
-              $out[$decoded_response['Config']['Labels']['com.docker.compose.service']]['Id'] = trim($decoded_response['Id']);
-            }
-          }
-        }
-        if (empty($response)) {
-          return true;
-        }
-        else {
-          return (!empty($out)) ? $out : false;
-        }
-      }
-    break;
-    case 'post':
-      if (!empty($attr1)) {
-        $container_id = docker('get_id', $service_name);
-        if (ctype_xdigit($container_id) && ctype_alnum($attr1)) {
-          curl_setopt($curl, CURLOPT_URL, 'https://dockerapi:443/containers/' . $container_id . '/' . $attr1);
-          curl_setopt($curl, CURLOPT_POST, 1);
-          curl_setopt($curl, CURLOPT_TIMEOUT, $DOCKER_TIMEOUT);
-          if (!empty($attr2)) {
-            curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($attr2));
-          }
-          if (!empty($extra_headers) && is_array($extra_headers)) {
-            curl_setopt($curl, CURLOPT_HTTPHEADER, $extra_headers);
-          }
-          curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
-          $response = curl_exec($curl);
-          if ($response === false) {
-            $err = curl_error($curl);
-            curl_close($curl);
-            return $err;
-          }
-          else {
-            curl_close($curl);
-            if (empty($response)) {
-              return true;
-            }
-            else {
-              return $response;
-            }
-          }
-        }
-      }
-    break;
-    case 'container_stats':
-      if (empty($service_name)){
-        return false;
-      }
-
-      $container_id = $service_name;
-      curl_setopt($curl, CURLOPT_URL, 'https://dockerapi:443/container/' . $container_id . '/stats/update');
-      curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
-      curl_setopt($curl, CURLOPT_POST, 1);
-      curl_setopt($curl, CURLOPT_TIMEOUT, $DOCKER_TIMEOUT);
-      $response = curl_exec($curl);
-      if ($response === false) {
-        $err = curl_error($curl);
-        curl_close($curl);
-        return $err;
-      }
-      else {
-        curl_close($curl);
-        $stats = json_decode($response, true);
-        if (!empty($stats)) return $stats;
-      }
-      return false;
-    break;
-    case 'host_stats':
-      curl_setopt($curl, CURLOPT_URL, 'https://dockerapi:443/host/stats');
-      curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
-      curl_setopt($curl, CURLOPT_POST, 0);
-      curl_setopt($curl, CURLOPT_TIMEOUT, $DOCKER_TIMEOUT);
-      $response = curl_exec($curl);
-      if ($response === false) {
-        $err = curl_error($curl);
-        curl_close($curl);
-        return $err;
-      }
-      else {
-        curl_close($curl);
-        $stats = json_decode($response, true);
-        if (!empty($stats)) return $stats;
-      }
-      return false;
-    break;
-    case 'broadcast':
-      $request = array(
-        "api_call" => "container_post",
-        "container_name" => $service_name,
-        "post_action" => $attr1,
-        "request" => $attr2
-      );
-
-      $redis->publish("MC_CHANNEL", json_encode($request));
-      return true;
-    break;
-  }
-}

data/web/inc/functions.fail2ban.inc.php

@@ -109,7 +109,7 @@ function fail2ban($_action, $_data = null, $_extra = null) {
             return false;
           }
           // Rules will also be recreated on log events, but rules may seem empty for a second in the UI
-          docker('post', 'netfilter-mailcow', 'restart');
+          agent('request', 'netfilter', 'restart', array(), 30);
           $fail_count = 0;
           $regex_result = json_decode($redis->Get('F2B_REGEX'), true);
           while (empty($regex_result) && $fail_count < 10) {
@@ -206,7 +206,7 @@ function fail2ban($_action, $_data = null, $_extra = null) {
                 try {
                   $redis->hSet('F2B_BLACKLIST', $network, 1);
                   $redis->hDel('F2B_WHITELIST', $network, 1);
-                  //$response = docker('post', 'netfilter-mailcow', 'restart');
+                  // netfilter picks up the redis changes
                 }
                 catch (RedisException $e) {
                   $_SESSION['return'][] = array(

data/web/inc/functions.inc.php

@@ -2239,30 +2239,19 @@ function rspamd_ui($action, $data = null) {
         );
         return false;
       }
-      $docker_return = docker('post', 'rspamd-mailcow', 'exec', array('cmd' => 'rspamd', 'task' => 'worker_password', 'raw' => $rspamd_ui_pass), array('Content-Type: application/json'));
-      if ($docker_return_array = json_decode($docker_return, true)) {
-        if ($docker_return_array['type'] == 'success') {
-          $_SESSION['return'][] =  array(
-            'type' => 'success',
-            'log' => array(__FUNCTION__, '*', '*'),
-            'msg' => 'rspamd_ui_pw_set'
-          );
-          return true;
-        }
-        else {
-          $_SESSION['return'][] =  array(
-            'type' => $docker_return_array['type'],
-            'log' => array(__FUNCTION__, '*', '*'),
-            'msg' => $docker_return_array['msg']
-          );
-          return false;
-        }
-      }
-      else {
+      $resp = agent('request_all', 'rspamd', 'exec.set-worker-password', array('password' => $rspamd_ui_pass), 30);
+      if (agent('ok', $resp)) {
+        $_SESSION['return'][] =  array(
+          'type' => 'success',
+          'log' => array(__FUNCTION__, '*', '*'),
+          'msg' => 'rspamd_ui_pw_set'
+        );
+        return true;
+      } else {
         $_SESSION['return'][] =  array(
           'type' => 'danger',
           'log' => array(__FUNCTION__, '*', '*'),
-          'msg' => 'unknown'
+          'msg' => agent('first_error', $resp) ?: 'rspamd: no live agent responded'
         );
         return false;
       }

data/web/inc/functions.mailbox.inc.php

@@ -125,8 +125,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 fwrite($filter_handle, $script_data);
                 fclose($filter_handle);
               }
-              $restart_response = json_decode(docker('post', 'dovecot-mailcow', 'restart'), true);
-              if ($restart_response['type'] == "success") {
+              $restart_response = agent('request', 'dovecot', 'restart', array(), 30);
+              if (agent('ok', $restart_response)) {
                 $_SESSION['return'][] = array(
                   'type' => 'success',
                   'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
@@ -160,8 +160,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 fwrite($filter_handle, $script_data);
                 fclose($filter_handle);
               }
-              $restart_response = json_decode(docker('post', 'dovecot-mailcow', 'restart'), true);
-              if ($restart_response['type'] == "success") {
+              $restart_response = agent('request', 'dovecot', 'restart', array(), 30);
+              if (agent('ok', $restart_response)) {
                 $_SESSION['return'][] = array(
                   'type' => 'success',
                   'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
@@ -669,8 +669,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             }
           }
           if (!empty($restart_sogo)) {
-            $restart_response = json_decode(docker('post', 'sogo-mailcow', 'restart'), true);
-            if ($restart_response['type'] == "success") {
+            $restart_response = agent('request', 'sogo', 'restart', array(), 30);
+            if (agent('ok', $restart_response)) {
               $_SESSION['return'][] = array(
                 'type' => 'success',
                 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
@@ -3553,22 +3553,30 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
 
           // get imap acls
           try {
-            $exec_fields = array(
-              'cmd' => 'doveadm',
-              'task' => 'get_acl',
-              'id' => $old_username
-            );
-            $imap_acls = json_decode(docker('post', 'dovecot-mailcow', 'exec', $exec_fields), true);
+            $acl_agg = agent('request_all', 'dovecot', 'exec.acl-get', array('user' => $old_username), 10);
+            $imap_acls = array();
+            $seen = array();
+            foreach ($acl_agg['responses'] as $r) {
+              if (empty($r['ok'])) continue;
+              foreach ((isset($r['result']['acls']) ? $r['result']['acls'] : array()) as $a) {
+                $key = (isset($a['mailbox']) ? $a['mailbox'] : '') . '|' . (isset($a['identifier']) ? $a['identifier'] : '');
+                if (isset($seen[$key])) continue;
+                $seen[$key] = true;
+                $imap_acls[] = array(
+                  'user'    => $old_username,
+                  'mailbox' => isset($a['mailbox']) ? $a['mailbox'] : '',
+                  'id'      => isset($a['identifier']) ? $a['identifier'] : '',
+                  'rights'  => isset($a['rights']) ? $a['rights'] : '',
+                );
+              }
+            }
             // delete imap acls
             foreach ($imap_acls as $imap_acl) {
-              $exec_fields = array(
-                'cmd' => 'doveadm',
-                'task' => 'delete_acl',
-                'user' => $imap_acl['user'],
-                'mailbox' => $imap_acl['mailbox'],
-                'id' => $imap_acl['id']
-              );
-              docker('post', 'dovecot-mailcow', 'exec', $exec_fields);
+              agent('request_all', 'dovecot', 'exec.acl-delete', array(
+                'user'       => $imap_acl['user'],
+                'mailbox'    => $imap_acl['mailbox'],
+                'identifier' => $imap_acl['id'],
+              ), 10);
             }
           } catch (Exception $e) {
             $_SESSION['return'][] = array(
@@ -3649,41 +3657,27 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           }
 
           // move maildir
-          $exec_fields = array(
-            'cmd' => 'maildir',
-            'task' => 'move',
-            'old_maildir' => $domain . '/' . $old_local_part,
-            'new_maildir' => $domain . '/' . $new_local_part
-          );
-          if (getenv("CLUSTERMODE") == "replication") {
-            // broadcast to each dovecot container
-            docker('broadcast', 'dovecot-mailcow', 'exec', $exec_fields);
-          } else {
-            docker('post', 'dovecot-mailcow', 'exec', $exec_fields);
-          }
+          agent('request_all', 'dovecot', 'exec.maildir-move', array(
+            'from' => $domain . '/' . $old_local_part,
+            'to'   => $domain . '/' . $new_local_part,
+          ), 30);
 
           // rename username in sogo
-          $exec_fields = array(
-            'cmd' => 'sogo',
-            'task' => 'rename_user',
-            'old_username' => $old_username,
-            'new_username' => $new_username
-          );
-          docker('post', 'sogo-mailcow', 'exec', $exec_fields);
+          agent('request', 'sogo', 'exec.rename-user', array(
+            'old' => $old_username,
+            'new' => $new_username,
+          ), 30);
 
           // set imap acls
           foreach ($imap_acls as $imap_acl) {
             $user_id = ($imap_acl['id'] == $old_username) ? $new_username : $imap_acl['id'];
             $user = ($imap_acl['user'] == $old_username) ? $new_username : $imap_acl['user'];
-            $exec_fields = array(
-              'cmd' => 'doveadm',
-              'task' => 'set_acl',
-              'user' => $user,
-              'mailbox' => $imap_acl['mailbox'],
-              'id' => $user_id,
-              'rights' => $imap_acl['rights']
-            );
-            docker('post', 'dovecot-mailcow', 'exec', $exec_fields);
+            agent('request_all', 'dovecot', 'exec.acl-set', array(
+              'user'       => $user,
+              'mailbox'    => $imap_acl['mailbox'],
+              'identifier' => $user_id,
+              'rights'     => $imap_acl['rights'],
+            ), 15);
           }
 
           // create alias
@@ -4553,24 +4547,19 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           else {
             $_data = $_SESSION['mailcow_cc_username'];
           }
-          $exec_fields = array(
-            'cmd' => 'sieve',
-            'task' => 'list',
-            'username' => $_data
-          );
-          $filters = docker('post', 'dovecot-mailcow', 'exec', $exec_fields);
-          $filters = array_filter(preg_split("/(\r\n|\n|\r)/",$filters));
-          foreach ($filters as $filter) {
+          $list_resp = agent('request', 'dovecot', 'exec.sieve-list', array('user' => $_data), 10);
+          if (empty($list_resp['ok'])) {
+            return false;
+          }
+          $scripts = isset($list_resp['result']['scripts']) ? $list_resp['result']['scripts'] : array();
+          foreach ($scripts as $filter) {
             if (preg_match('/.+ ACTIVE/i', $filter)) {
-              $exec_fields = array(
-                'cmd' => 'sieve',
-                'task' => 'print',
-                'script_name' => substr($filter, 0, -7),
-                'username' => $_data
-              );
-              $script = docker('post', 'dovecot-mailcow', 'exec', $exec_fields);
-              // Remove first line
-              return preg_replace('/^.+\n/', '', $script);
+              $print_resp = agent('request', 'dovecot', 'exec.sieve-print', array(
+                'user'   => $_data,
+                'script' => substr($filter, 0, -7),
+              ), 10);
+              if (empty($print_resp['ok'])) return false;
+              return isset($print_resp['result']['body']) ? $print_resp['result']['body'] : '';
             }
           }
           return false;
@@ -5712,13 +5701,12 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               );
               continue;
             }
-            $exec_fields = array('cmd' => 'maildir', 'task' => 'cleanup', 'maildir' => $domain);
-            $maildir_gc = json_decode(docker('post', 'dovecot-mailcow', 'exec', $exec_fields), true);
-            if ($maildir_gc['type'] != 'success') {
+            $maildir_gc = agent('request_all', 'dovecot', 'exec.maildir-cleanup', array('maildir' => $domain), 30);
+            if (!agent('ok', $maildir_gc)) {
               $_SESSION['return'][] = array(
                 'type' => 'warning',
                 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
-                'msg' => 'Could not move mail storage to garbage collector: ' . $maildir_gc['msg']
+                'msg' => 'Could not move mail storage to garbage collector: ' . agent('first_error', $maildir_gc)
               );
             }
             $stmt = $pdo->prepare("DELETE FROM `domain` WHERE `domain` = :domain");
@@ -5967,20 +5955,13 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             $mailbox_details = mailbox('get', 'mailbox_details', $username);
             if (!empty($mailbox_details['domain']) && !empty($mailbox_details['local_part'])) {
               $maildir = $mailbox_details['domain'] . '/' . $mailbox_details['local_part'];
-              $exec_fields = array('cmd' => 'maildir', 'task' => 'cleanup', 'maildir' => $maildir);
-
-              if (getenv("CLUSTERMODE") == "replication") {
-                // broadcast to each dovecot container
-                docker('broadcast', 'dovecot-mailcow', 'exec', $exec_fields);
-              } else {
-                $maildir_gc = json_decode(docker('post', 'dovecot-mailcow', 'exec', $exec_fields), true);
-                if ($maildir_gc['type'] != 'success') {
-                  $_SESSION['return'][] = array(
-                    'type' => 'warning',
-                    'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
-                    'msg' => 'Could not move maildir to garbage collector: ' . $maildir_gc['msg']
-                  );
-                }
+              $maildir_gc = agent('request_all', 'dovecot', 'exec.maildir-cleanup', array('maildir' => $maildir), 30);
+              if (!agent('ok', $maildir_gc)) {
+                $_SESSION['return'][] = array(
+                  'type' => 'warning',
+                  'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+                  'msg' => 'Could not move maildir to garbage collector: ' . agent('first_error', $maildir_gc)
+                );
               }
             }
             else {

data/web/inc/functions.mailq.inc.php

@@ -1,121 +1,138 @@
-<?php
-function mailq($_action, $_data = null) {
-  if ($_SESSION['mailcow_cc_role'] != "admin") {
-    $_SESSION['return'][] = array(
-      'type' => 'danger',
-      'log' => array(__FUNCTION__, $_action, $_data),
-      'msg' => 'access_denied'
-    );
-    return false;
-  }
-  function process_mailq_output($returned_output, $_action, $_data) {
-    if ($returned_output !== NULL) {
-      if ($_action == 'cat') {
-        logger(array('return' => array(
-          array(
-            'type' => 'success',
-            'log' => array(__FUNCTION__, $_action, $_data),
-            'msg' => 'queue_cat_success'
-          )
-        )));
-        return $returned_output;
-      }
-      else {
-        if (isset($returned_output['type']) && $returned_output['type'] == 'danger') {
-          $_SESSION['return'][] = array(
-            'type' => 'danger',
-            'log' => array(__FUNCTION__, $_action, $_data),
-            'msg' => 'Error: ' . $returned_output['msg']
-          );
-        }
-        if (isset($returned_output['type']) && $returned_output['type'] == 'success') {
-          $_SESSION['return'][] = array(
-            'type' => 'success',
-            'log' => array(__FUNCTION__, $_action, $_data),
-            'msg' => 'queue_command_success'
-          );
-        }
-      }
-    }
-    else {
-      $_SESSION['return'][] = array(
-        'type' => 'danger',
-        'log' => array(__FUNCTION__, $_action, $_data),
-        'msg' => 'unknown'
-      );
-    }
-  }
-  if ($_action == 'get') {
-    $mailq_lines = docker('post', 'postfix-mailcow', 'exec', array('cmd' => 'mailq', 'task' => 'list'));
-    $lines = 0;
-    // Hard limit to 10000 items
-    foreach (preg_split("/((\r?\n)|(\r\n?))/", $mailq_lines) as $mailq_item) if ($lines++ < 10000) {
-      if (empty($mailq_item) || $mailq_item == '1') {
-        continue;
-      }
-      $mq_line = json_decode($mailq_item, true);
-      if ($mq_line !== NULL) {
-        $rcpts = array();
-        foreach ($mq_line['recipients'] as $rcpt) {
-          if (isset($rcpt['delay_reason'])) {
-            $rcpts[] = $rcpt['address'] . ' (' . $rcpt['delay_reason'] . ')';
-          }
-          else {
-            $rcpts[] = $rcpt['address'];
-          }
-        }
-        if (!empty($rcpts)) {
-          $mq_line['recipients'] = $rcpts;
-        }
-        $line[] = $mq_line;
-      }
-    }
-    if (!isset($line) || empty($line)) {
-      return '[]';
-    }
-    else {
-      return json_encode($line);
-    }
-  }
-  elseif ($_action == 'delete') {
-    if (!is_array($_data['qid'])) {
-      $qids = array();
-      $qids[] = $_data['qid'];
-    }
-    else {
-      $qids = $_data['qid'];
-    }
-    $docker_return = docker('post', 'postfix-mailcow', 'exec', array('cmd' => 'mailq', 'task' => 'delete', 'items' => $qids));
-    process_mailq_output(json_decode($docker_return, true), $_action, $_data);
-  }
-  elseif ($_action == 'cat') {
-    if (!is_array($_data['qid'])) {
-      $qids = array();
-      $qids[] = $_data['qid'];
-    }
-    else {
-      $qids = $_data['qid'];
-    }
-    $docker_return = docker('post', 'postfix-mailcow', 'exec', array('cmd' => 'mailq', 'task' => 'cat', 'items' => $qids));
-    return process_mailq_output($docker_return, $_action, $_data);
-  }
-  elseif ($_action == 'edit') {
-    if (in_array($_data['action'], array('hold', 'unhold', 'deliver'))) {
-      if (!is_array($_data['qid'])) {
-        $qids = array();
-        $qids[] = $_data['qid'];
-      }
-      else {
-        $qids = $_data['qid'];
-      }
-      if (!empty($qids)) {
-        $docker_return = docker('post', 'postfix-mailcow', 'exec', array('cmd' => 'mailq', 'task' => $_data['action'], 'items' => $qids));
-        process_mailq_output(json_decode($docker_return, true), $_action, $_data);
-      }
-    }
-    if (in_array($_data['action'], array('flush', 'super_delete'))) {
-      $docker_return = docker('post', 'postfix-mailcow', 'exec', array('cmd' => 'mailq', 'task' => $_data['action']));
-      process_mailq_output(json_decode($docker_return, true), $_action, $_data);
-    }
-  }
-}
+<?php
+function mailq($_action, $_data = null) {
+  global $lang;
+  if ($_SESSION['mailcow_cc_role'] != "admin") {
+    $_SESSION['return'][] = array(
+      'type' => 'danger',
+      'log' => array(__FUNCTION__, $_action, $_data),
+      'msg' => 'access_denied'
+    );
+    return false;
+  }
+  switch ($_action) {
+    case 'get':
+      $agg = agent('request_all', 'postfix', 'exec.mailq', array(), 15);
+      $lines = array();
+      foreach ($agg['responses'] as $r) {
+        if (empty($r['ok'])) continue;
+        $queue = isset($r['result']['queue']) ? $r['result']['queue'] : array();
+        foreach ($queue as $entry) {
+          if (is_array($entry)) {
+            $entry['node'] = $r['node'];
+            if (!empty($entry['recipients']) && is_array($entry['recipients'])) {
+              $rcpts = array();
+              foreach ($entry['recipients'] as $rcpt) {
+                $addr = isset($rcpt['address']) ? $rcpt['address'] : '';
+                if (isset($rcpt['delay_reason'])) {
+                  $rcpts[] = $addr . ' (' . $rcpt['delay_reason'] . ')';
+                }
+                else {
+                  $rcpts[] = $addr;
+                }
+              }
+              $entry['recipients'] = $rcpts;
+            }
+            $lines[] = $entry;
+          }
+          if (count($lines) >= 10000) break 2;
+        }
+      }
+      return empty($lines) ? '[]' : json_encode($lines);
+    break;
+    case 'delete':
+      $qids = isset($_data['qid']) && is_array($_data['qid']) ? $_data['qid'] : array($_data['qid']);
+      $ok_count = 0;
+      $failed = 0;
+      foreach ($qids as $qid) {
+        $agg = agent('request_all', 'postfix', 'exec.delete-from-queue', array('queue_id' => $qid), 10);
+        if (agent('ok', $agg)) {
+          $ok_count++;
+        }
+        else {
+          $failed++;
+        }
+      }
+      $ok = ($ok_count > 0 && $failed === 0);
+      $_SESSION['return'][] = array(
+        'type' => $ok ? 'success' : 'danger',
+        'log' => array(__FUNCTION__, $_action, $_data),
+        'msg' => $ok ? 'queue_command_success' : 'queue_command_failed'
+      );
+      return $ok;
+    break;
+    case 'cat':
+      $qids = isset($_data['qid']) && is_array($_data['qid']) ? $_data['qid'] : array($_data['qid']);
+      $body = '';
+      foreach ($qids as $qid) {
+        $agg = agent('request_all', 'postfix', 'exec.cat-queue', array('queue_id' => $qid), 15);
+        foreach ($agg['responses'] as $r) {
+          if (!empty($r['ok']) && !empty($r['result']['body'])) {
+            $body .= $r['result']['body'];
+          }
+        }
+      }
+      if ($body === '') {
+        $_SESSION['return'][] = array(
+          'type' => 'danger',
+          'log' => array(__FUNCTION__, $_action, $_data),
+          'msg' => 'queue_cat_empty'
+        );
+        return null;
+      }
+      $_SESSION['return'][] = array(
+        'type' => 'success',
+        'log' => array(__FUNCTION__, $_action, $_data),
+        'msg' => 'queue_cat_success'
+      );
+      return $body;
+    break;
+    case 'edit':
+      $cmd_map = array(
+        'hold' => 'exec.hold-queue',
+        'unhold' => 'exec.unhold-queue',
+        'deliver' => 'exec.deliver-now'
+      );
+      if (isset($cmd_map[$_data['action']])) {
+        $qids = isset($_data['qid']) && is_array($_data['qid']) ? $_data['qid'] : array($_data['qid']);
+        $ok_count = 0;
+        $failed = 0;
+        foreach ($qids as $qid) {
+          $agg = agent('request_all', 'postfix', $cmd_map[$_data['action']], array('queue_id' => $qid), 10);
+          if (agent('ok', $agg)) {
+            $ok_count++;
+          }
+          else {
+            $failed++;
+          }
+        }
+        $ok = ($ok_count > 0 && $failed === 0);
+        $_SESSION['return'][] = array(
+          'type' => $ok ? 'success' : 'danger',
+          'log' => array(__FUNCTION__, $_action, $_data),
+          'msg' => $ok ? 'queue_command_success' : 'queue_command_failed'
+        );
+        return $ok;
+      }
+      if ($_data['action'] == 'flush') {
+        $agg = agent('request_all', 'postfix', 'exec.flush-queue', array(), 30);
+        $ok = agent('ok', $agg);
+        $_SESSION['return'][] = array(
+          'type' => $ok ? 'success' : 'danger',
+          'log' => array(__FUNCTION__, $_action, $_data),
+          'msg' => $ok ? 'queue_command_success' : 'queue_command_failed'
+        );
+        return $ok;
+      }
+      if ($_data['action'] == 'super_delete') {
+        $agg = agent('request_all', 'postfix', 'exec.super-delete', array(), 30);
+        $ok = agent('ok', $agg);
+        $_SESSION['return'][] = array(
+          'type' => $ok ? 'success' : 'danger',
+          'log' => array(__FUNCTION__, $_action, $_data),
+          'msg' => $ok ? 'queue_command_success' : 'queue_command_failed'
+        );
+        return $ok;
+      }
+    break;
+  }
+}

data/web/inc/prerequisites.inc.php

@@ -105,14 +105,6 @@ http_response_code(500);
 <?php
 exit;
 }
-// Stop when dockerapi is not available
-if (fsockopen("tcp://dockerapi", 443, $errno, $errstr) === false) {
-  http_response_code(500);
-?>
-<center style='font-family:sans-serif;'>Connection to dockerapi container failed.<br /><br />The following error was reported:<br/><?=$errno;?> - <?=$errstr;?></center>
-<?php
-exit;
-}
 
 // OAuth2
 class mailcowPdo extends OAuth2\Storage\Pdo {
@@ -280,7 +272,7 @@ require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.admin.inc.php';
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.app_passwd.inc.php';
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.customize.inc.php';
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.dkim.inc.php';
-require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.docker.inc.php';
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.agent.inc.php';
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.domain_admin.inc.php';
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.fail2ban.inc.php';
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.fwdhost.inc.php';

data/web/js/build/013-mailcow.js

@@ -277,19 +277,19 @@ $(document).ready(function() {
   // trigger container restart
   $('#RestartContainer').on('show.bs.modal', function(e) {
     var container = $(e.relatedTarget).data('container');
-    $('#containerName').text(container);
-    $('#triggerRestartContainer').click(function(){
+    var node = $(e.relatedTarget).data('node') || '';
+    $('#containerName').text(container + (node ? ' / ' + node : ''));
+    $('#triggerRestartContainer').off('click').on('click', function(){
       $(this).prop("disabled",true);
       $(this).html('<div class="spinner-border text-white" role="status"><span class="visually-hidden">Loading...</span></div>');
       $('#statusTriggerRestartContainer').html(lang_footer.restarting_container);
+      var payload = { 'service': container, 'action': 'restart' };
+      if (node) payload.node = node;
       $.ajax({
         method: 'get',
         url: '/inc/ajax/container_ctrl.php',
         timeout: docker_timeout,
-        data: {
-        'service': container,
-        'action': 'restart'
-        }
+        data: payload
       })
       .always( function (data, status) {
         $('#statusTriggerRestartContainer').append(data);

data/web/js/site/dashboard.js

@@ -23,8 +23,6 @@ $(document).ready(function() {
     }
   });
 
-  // set update loop container list
-  containersToUpdate = {};
   // set default ChartJs Font Color
   Chart.defaults.color = '#999';
   // create host cpu and mem charts
@@ -72,7 +70,6 @@ $(document).ready(function() {
       $("#host_show_ip").find(".spinner-border").addClass("d-none");
     });
   });
-  update_container_stats();
 });
 jQuery(function($){
   if (localStorage.getItem("current_page") === null) {
@@ -210,6 +207,11 @@ jQuery(function($){
           data: 'priority',
           defaultContent: ''
         },
+        {
+          title: lang_debug.node,
+          data: 'node',
+          defaultContent: '-'
+        },
         {
           title: lang.message,
           data: 'message',
@@ -692,6 +694,11 @@ jQuery(function($){
           data: 'priority',
           defaultContent: ''
         },
+        {
+          title: lang_debug.node,
+          data: 'node',
+          defaultContent: '-'
+        },
         {
           title: lang.message,
           data: 'message',
@@ -747,6 +754,11 @@ jQuery(function($){
           data: 'priority',
           defaultContent: ''
         },
+        {
+          title: lang_debug.node,
+          data: 'node',
+          defaultContent: '-'
+        },
         {
           title: lang.message,
           data: 'message',
@@ -802,6 +814,11 @@ jQuery(function($){
           data: 'priority',
           defaultContent: ''
         },
+        {
+          title: lang_debug.node,
+          data: 'node',
+          defaultContent: '-'
+        },
         {
           title: lang.message,
           data: 'message',
@@ -862,6 +879,11 @@ jQuery(function($){
           data: 'priority',
           defaultContent: ''
         },
+        {
+          title: lang_debug.node,
+          data: 'node',
+          defaultContent: '-'
+        },
         {
           title: lang.message,
           data: 'message',
@@ -1292,52 +1314,6 @@ jQuery(function($){
 
   // start polling host stats if tab is active
   onVisible("[id^=tab-containers]", () => update_stats());
-  // start polling container stats if collapse is active
-  var containerElements = document.querySelectorAll(".container-details-collapse");
-  for (let i = 0; i < containerElements.length; i++){
-    new IntersectionObserver((entries, observer) => {
-      entries.forEach(entry => {
-        if(entry.intersectionRatio > 0) {
-
-          if (!containerElements[i].classList.contains("show")){
-            var container = containerElements[i].id.replace("Collapse", "");
-            var container_id = containerElements[i].getAttribute("data-id");
-
-            // check if chart exists or needs to be created
-            if (!Chart.getChart(container + "_DiskIOChart"))
-              createReadWriteChart(container + "_DiskIOChart", "Read", "Write");
-            if (!Chart.getChart(container + "_NetIOChart"))
-              createReadWriteChart(container + "_NetIOChart", "Recv", "Sent");
-
-            // add container to polling list
-            containersToUpdate[container] = {
-              id: container_id,
-              state: "idle"
-            }
-
-            // stop polling if collapse is closed
-            containerElements[i].addEventListener('hidden.bs.collapse', function () {
-              var diskIOCtx = Chart.getChart(container + "_DiskIOChart");
-              var netIOCtx = Chart.getChart(container + "_NetIOChart");
-
-              diskIOCtx.data.datasets[0].data = [];
-              diskIOCtx.data.datasets[1].data = [];
-              diskIOCtx.data.labels = [];
-              netIOCtx.data.datasets[0].data = [];
-              netIOCtx.data.datasets[1].data = [];
-              netIOCtx.data.labels = [];
-
-              diskIOCtx.update();
-              netIOCtx.update();
-
-              delete containersToUpdate[container];
-            });
-          }
-
-        }
-      });
-    }).observe(containerElements[i]);
-  }
 });
 
 
@@ -1351,127 +1327,49 @@ function update_stats(timeout=5){
   window.fetch("/api/v1/get/status/host", {method:'GET',cache:'no-cache'}).then(function(response) {
     return response.json();
   }).then(function(data) {
-    if (data){
-      // display table data
-      $("#host_date").text(data.system_time);
-      $("#host_uptime").text(formatUptime(data.uptime));
-      $("#host_cpu_cores").text(data.cpu.cores);
-      $("#host_cpu_usage").text(parseInt(data.cpu.usage).toString() + "%");
-      $("#host_memory_total").text((data.memory.total / (1024 ** 3)).toFixed(2).toString() + "GB");
-      $("#host_memory_usage").text(parseInt(data.memory.usage).toString() + "%");
-      $("#host_architecture").html(data.architecture);
-      // update cpu and mem chart
-      var cpu_chart = Chart.getChart("host_cpu_chart");
-      var mem_chart = Chart.getChart("host_mem_chart");
+    // Wrapped in try/catch so a malformed payload doesn't break the
+    // polling loop forever. We always reschedule from .finally.
+    try {
+      if (data && data.cpu && data.memory){
+        $("#host_date").text(data.system_time || "");
+        $("#host_uptime").text(formatUptime(data.uptime));
+        $("#host_cpu_cores").text(data.cpu.cores);
+        $("#host_cpu_usage").text(parseInt(data.cpu.usage).toString() + "%");
+        $("#host_memory_total").text((data.memory.total / (1024 ** 3)).toFixed(2).toString() + "GB");
+        $("#host_memory_usage").text(parseInt(data.memory.usage).toString() + "%");
+        $("#host_architecture").html(data.architecture);
 
-      cpu_chart.data.labels.push(data.system_time.split(" ")[1]);
-      if (cpu_chart.data.labels.length > 30) cpu_chart.data.labels.shift();
-      mem_chart.data.labels.push(data.system_time.split(" ")[1]);
-      if (mem_chart.data.labels.length > 30) mem_chart.data.labels.shift();
+        var cpu_chart = Chart.getChart("host_cpu_chart");
+        var mem_chart = Chart.getChart("host_mem_chart");
 
-      cpu_chart.data.datasets[0].data.push(data.cpu.usage);
-      if (cpu_chart.data.datasets[0].data.length > 30) cpu_chart.data.datasets[0].data.shift();
-      mem_chart.data.datasets[0].data.push(data.memory.usage);
-      if (mem_chart.data.datasets[0].data.length > 30) mem_chart.data.datasets[0].data.shift();
+        if (cpu_chart && mem_chart && typeof data.system_time === "string") {
+          var label = data.system_time.split(" ")[1] || "";
+          cpu_chart.data.labels.push(label);
+          if (cpu_chart.data.labels.length > 30) cpu_chart.data.labels.shift();
+          mem_chart.data.labels.push(label);
+          if (mem_chart.data.labels.length > 30) mem_chart.data.labels.shift();
 
-      cpu_chart.update();
-      mem_chart.update();
+          cpu_chart.data.datasets[0].data.push(data.cpu.usage);
+          if (cpu_chart.data.datasets[0].data.length > 30) cpu_chart.data.datasets[0].data.shift();
+          mem_chart.data.datasets[0].data.push(data.memory.usage);
+          if (mem_chart.data.datasets[0].data.length > 30) mem_chart.data.datasets[0].data.shift();
+
+          cpu_chart.update();
+          mem_chart.update();
+        }
+      } else {
+        console.warn("update_stats: unexpected host payload", data);
+      }
+    } catch (e) {
+      console.warn("update_stats: render error", e);
     }
-
-    // run again in n seconds
+  }).catch(function(e) {
+    console.warn("update_stats: fetch failed", e);
+  }).finally(function() {
+    // Always reschedule so a transient backend hiccup can't kill the poll loop.
     setTimeout(update_stats, timeout * 1000);
   });
 }
-// update specific container stats - every n (default 5s) seconds
-function update_container_stats(timeout=5){
-
-  if ($('#tab-containers').hasClass('active')) {
-    for (let container in containersToUpdate){
-      container_id = containersToUpdate[container].id;
-      // check if container update stats is already running
-      if (containersToUpdate[container].state == "running")
-        continue;
-      containersToUpdate[container].state = "running";
-
-
-      window.fetch("/api/v1/get/status/container/" + container_id, {method:'GET',cache:'no-cache'}).then(function(response) {
-        return response.json();
-      }).then(function(data) {
-        var diskIOCtx = Chart.getChart(container + "_DiskIOChart");
-        var netIOCtx = Chart.getChart(container + "_NetIOChart");
-
-        prev_stats = null;
-        if (data.length >= 2){
-          prev_stats = data[data.length -2];
-
-          // hide spinners if we collected enough data
-          $('#' + container + "_DiskIOChart").removeClass('d-none');
-          $('#' + container + "_DiskIOChart").prev().addClass('d-none');
-          $('#' + container + "_NetIOChart").removeClass('d-none');
-          $('#' + container + "_NetIOChart").prev().addClass('d-none');
-        }
-
-        data = data[data.length -1];
-
-        if (prev_stats != null){
-          // calc time diff
-          var time_diff = (new Date(data.read) - new Date(prev_stats.read)) / 1000;
-
-          // calc disk io b/s
-          if ('io_service_bytes_recursive' in prev_stats.blkio_stats && prev_stats.blkio_stats.io_service_bytes_recursive !== null){
-            var prev_read_bytes = 0;
-            var prev_write_bytes = 0;
-            for (var i = 0; i < prev_stats.blkio_stats.io_service_bytes_recursive.length; i++){
-              if (prev_stats.blkio_stats.io_service_bytes_recursive[i].op == "read")
-                prev_read_bytes = prev_stats.blkio_stats.io_service_bytes_recursive[i].value;
-              else if (prev_stats.blkio_stats.io_service_bytes_recursive[i].op == "write")
-                prev_write_bytes = prev_stats.blkio_stats.io_service_bytes_recursive[i].value;
-            }
-            var read_bytes = 0;
-            var write_bytes = 0;
-            for (var i = 0; i < data.blkio_stats.io_service_bytes_recursive.length; i++){
-              if (data.blkio_stats.io_service_bytes_recursive[i].op == "read")
-                read_bytes = data.blkio_stats.io_service_bytes_recursive[i].value;
-              else if (data.blkio_stats.io_service_bytes_recursive[i].op == "write")
-                write_bytes = data.blkio_stats.io_service_bytes_recursive[i].value;
-            }
-            var diff_bytes_read = (read_bytes - prev_read_bytes) / time_diff;
-            var diff_bytes_write = (write_bytes - prev_write_bytes) / time_diff;
-          }
-
-          // calc net io b/s
-          if ('networks' in prev_stats){
-            var prev_recv_bytes = 0;
-            var prev_sent_bytes = 0;
-            for (var key in prev_stats.networks){
-              prev_recv_bytes += prev_stats.networks[key].rx_bytes;
-              prev_sent_bytes += prev_stats.networks[key].tx_bytes;
-            }
-            var recv_bytes = 0;
-            var sent_bytes = 0;
-            for (var key in data.networks){
-              recv_bytes += data.networks[key].rx_bytes;
-              sent_bytes += data.networks[key].tx_bytes;
-            }
-            var diff_bytes_recv = (recv_bytes - prev_recv_bytes) / time_diff;
-            var diff_bytes_sent = (sent_bytes - prev_sent_bytes) / time_diff;
-          }
-
-          addReadWriteChart(diskIOCtx, diff_bytes_read, diff_bytes_write, "");
-          addReadWriteChart(netIOCtx, diff_bytes_recv, diff_bytes_sent, "");
-        }
-
-        // run again in n seconds
-        containersToUpdate[container].state = "idle";
-      }).catch(err => {
-        console.log(err);
-      });
-    }
-  }
-
-  // run again in n seconds
-  setTimeout(update_container_stats, timeout * 1000);
-}
 // format hosts uptime seconds to readable string
 function formatUptime(seconds){
   seconds = Number(seconds);

data/web/json_api.php

@@ -1461,42 +1461,66 @@ if (isset($_GET['query'])) {
             if ($_SESSION['mailcow_cc_role'] == "admin") {
               switch ($object) {
                 case "containers":
-                  $containers = (docker('info'));
-                  foreach ($containers as $container => $container_info) {
-                    $container . ' (' . $container_info['Config']['Image'] . ')';
-                    $containerstarttime = ($container_info['State']['StartedAt']);
-                    $containerstate = ($container_info['State']['Status']);
-                    $containerimage = ($container_info['Config']['Image']);
-                    $temp[$container] = array(
+                  $temp = array();
+                  foreach (agent('services') as $svc) {
+                    $nodes = agent('live_nodes', $svc);
+                    $first = $nodes ? $nodes[0] : '';
+                    $meta = $first ? (agent('node_meta', $svc, $first) ?: array()) : array();
+                    $key = $svc . '-mailcow';
+                    $temp[$key] = array(
                       'type' => 'info',
-                      'container' => $container,
-                      'state' => $containerstate,
-                      'started_at' => $containerstarttime,
-                      'image' => $containerimage
+                      'container' => $key,
+                      'state' => $nodes ? 'running' : 'exited',
+                      'node_count' => count($nodes),
+                      'started_at' => isset($meta['started_at']) ? $meta['started_at'] : '',
+                      'image' => isset($meta['image']) ? $meta['image'] : '',
+                      'external' => false
                     );
                   }
+                  foreach (infra('status') as $key => $entry) {
+                    $temp[$key] = array(
+                      'type' => 'info',
+                      'container' => $key,
+                      'state' => $entry['State']['Running'] ? 'running' : 'exited',
+                      'node_count' => $entry['State']['NodeCount'],
+                      'started_at' => '',
+                      'image' => $entry['Config']['Image'],
+                      'error' => $entry['State']['Error'],
+                      'external' => true
+                    );
+                  }
+                  ksort($temp);
                   echo json_encode($temp, JSON_UNESCAPED_SLASHES);
                 break;
                 case "container":
-                  $container_stats = docker('container_stats', $extra);
-                  echo json_encode($container_stats);
+                  $stats = null;
+                  foreach (agent('services') as $svc) {
+                    $s = agent('node_stats', $svc, $extra);
+                    if ($s) {
+                      $stats = $s;
+                      break;
+                    }
+                  }
+                  echo json_encode($stats);
                 break;
                 case "vmail":
-                  $exec_fields_vmail = array('cmd' => 'system', 'task' => 'df', 'dir' => '/var/vmail');
-                  $vmail_df = explode(',', json_decode(docker('post', 'dovecot-mailcow', 'exec', $exec_fields_vmail), true));
+                  $vmail_resp = agent('request', 'dovecot', 'exec.df', array('dir' => '/var/vmail'), 5);
+                  $vmail_df = (!empty($vmail_resp['ok']) && is_string($vmail_resp['result']))
+                    ? explode(',', $vmail_resp['result'])
+                    : array('', '', '', '', '', '/var/vmail');
                   $temp = array(
                     'type' => 'info',
                     'disk' => $vmail_df[0],
                     'used' => $vmail_df[2],
-                    'total'=> $vmail_df[1],
+                    'total' => $vmail_df[1],
                     'used_percent' => $vmail_df[4]
                   );
                   echo json_encode($temp, JSON_UNESCAPED_SLASHES);
                 break;
                 case "host":
-                  if (!$extra){
-                    $stats = docker("host_stats");
-                    echo json_encode($stats);
+                  if (!$extra) {
+                    $host_resp = agent('request', 'host', 'exec.host-stats', array(), 5);
+                    echo json_encode(!empty($host_resp['ok']) ? $host_resp['result'] : null);
                   }
                   else if ($extra == "ip") {
                     // get public ips

data/web/lang/lang.de-de.json

@@ -559,7 +559,11 @@
         "template_exists": "Vorlage %s existiert bereits",
         "template_id_invalid": "Vorlagen-ID %s ungültig",
         "template_name_invalid": "Name der Vorlage ungültig",
-        "required_data_missing": "Die benötigten Daten: %s fehlen"
+        "required_data_missing": "Die benötigten Daten: %s fehlen",
+        "no_live_agent": "Kein aktiver Agent für Service %s",
+        "agent_timeout": "Agent-Timeout",
+        "agent_unknown_error": "Unbekannter Fehler vom Agent",
+        "queue_command_failed": "Queue-Befehl fehlgeschlagen"
     },
     "datatables": {
         "collapse_all": "Alle Einklappen",
@@ -623,7 +627,25 @@
         "no_update_available": "Das System ist auf aktuellem Stand",
         "update_failed": "Es konnte nicht nach einem Update gesucht werden",
         "username": "Benutzername",
-        "wip": "Aktuell noch in Arbeit"
+        "wip": "Aktuell noch in Arbeit",
+        "data_stores": "Datenspeicher",
+        "nodes": "Knoten",
+        "disk_io": "Disk-I/O",
+        "net_io": "Netz-I/O",
+        "replicas_badge": "%d× Replicas",
+        "replicas_title": "Lebende Agent-Replicas",
+        "external_dep_info": "Externe Infrastruktur — Health-Check per Protokoll-Ping",
+        "status_ok": "OK",
+        "status_down": "down",
+        "status_healthy": "verbunden",
+        "status_unreachable": "nicht erreichbar",
+        "unknown": "unbekannt",
+        "restart_all_nodes": "Alle Knoten neu starten",
+        "restart_node": "Diesen Knoten neu starten",
+        "nodes_count": "%d Knoten",
+        "node": "Knoten",
+        "container_unhealthy": "Service nicht gesund",
+        "container_degraded": "Service teilweise gesund"
     },
     "diagnostics": {
         "cname_from_a": "Wert abgeleitet von A/AAAA-Eintrag. Wird unterstützt, sofern der Eintrag auf die korrekte Ressource zeigt.",
@@ -1207,7 +1229,8 @@
         "verified_fido2_login": "FIDO2-Anmeldung verifiziert",
         "verified_totp_login": "TOTP-Anmeldung verifiziert",
         "verified_webauthn_login": "WebAuthn-Anmeldung verifiziert",
-        "verified_yotp_login": "Yubico-OTP-Anmeldung verifiziert"
+        "verified_yotp_login": "Yubico-OTP-Anmeldung verifiziert",
+        "service_restart_ok": "Service erfolgreich neu gestartet"
     },
     "tfa": {
         "authenticators": "Authentikatoren",

data/web/lang/lang.en-gb.json

@@ -559,7 +559,11 @@
         "validity_missing": "Please assign a period of validity",
         "value_missing": "Please provide all values",
         "version_invalid": "Version %s is invalid",
-        "yotp_verification_failed": "Yubico OTP verification failed: %s"
+        "yotp_verification_failed": "Yubico OTP verification failed: %s",
+        "no_live_agent": "No live agent for service %s",
+        "agent_timeout": "Agent timed out",
+        "agent_unknown_error": "Unknown error returned by agent",
+        "queue_command_failed": "Queue command failed"
     },
     "datatables": {
         "collapse_all": "Collapse All",
@@ -623,7 +627,25 @@
         "no_update_available": "The System is on the latest version",
         "update_failed": "Could not check for an Update",
         "username": "Username",
-        "wip": "Currently Work in Progress"
+        "wip": "Currently Work in Progress",
+        "data_stores": "Data stores",
+        "nodes": "Nodes",
+        "disk_io": "Disk I/O",
+        "net_io": "Net I/O",
+        "replicas_badge": "%d× replicas",
+        "replicas_title": "Live agent replicas",
+        "external_dep_info": "External infrastructure dependency — health checked via protocol ping",
+        "status_ok": "ok",
+        "status_down": "down",
+        "status_healthy": "healthy",
+        "status_unreachable": "unreachable",
+        "unknown": "unknown",
+        "restart_all_nodes": "Restart all nodes",
+        "restart_node": "Restart this node",
+        "nodes_count": "%d node(s)",
+        "node": "Node",
+        "container_unhealthy": "Service unhealthy",
+        "container_degraded": "Service degraded"
     },
     "diagnostics": {
         "cname_from_a": "Value derived from A/AAAA record. This is supported as long as the record points to the correct resource.",
@@ -1214,7 +1236,8 @@
         "verified_fido2_login": "Verified FIDO2 login",
         "verified_totp_login": "Verified TOTP login",
         "verified_webauthn_login": "Verified WebAuthn login",
-        "verified_yotp_login": "Verified Yubico OTP login"
+        "verified_yotp_login": "Verified Yubico OTP login",
+        "service_restart_ok": "Service restarted successfully"
     },
     "tfa": {
         "authenticators": "Authenticators",

data/web/templates/dashboard.twig

@@ -162,23 +162,64 @@
             </div>
           </div>
         </div>
-        <!-- container info -->
+        <!-- Infrastructure (data stores) — compact strip at the top -->
+        {% if infra_containers %}
+          <div class="card mb-3 border-0 bg-body-tertiary">
+            <div class="card-body py-2 px-3">
+              <div class="d-flex flex-wrap align-items-center">
+                <span class="text-uppercase text-muted small fw-bold me-3" style="letter-spacing:0.05em">{{ lang.debug.data_stores }}</span>
+                {% for container, info in infra_containers %}
+                  {% set svc = info.Service %}
+                  {% set icon = svc == 'mysql' ? 'bi-database' : (svc == 'redis' ? 'bi-lightning-charge' : 'bi-cpu') %}
+                  <div class="d-flex align-items-center me-4 py-1">
+                    <i class="bi {{ icon }} me-2 fs-5 {{ info.State.Running == 1 ? 'text-success' : 'text-danger' }}"></i>
+                    <div class="d-flex flex-column lh-sm">
+                      <span class="fw-semibold">{{ svc }}</span>
+                      <small class="text-muted" title="{{ info.State.Error }}">
+                        {{ info.Config.Image|default(lang.debug.unknown) }}
+                      </small>
+                    </div>
+                    {% if info.State.Running == 1 %}
+                      <span class="badge bg-success-subtle text-success-emphasis ms-2 fs-7">{{ lang.debug.status_ok }}</span>
+                    {% else %}
+                      <span class="badge bg-danger-subtle text-danger-emphasis ms-2 fs-7" title="{{ info.State.Error }}">{{ lang.debug.status_down }}</span>
+                    {% endif %}
+                  </div>
+                {% endfor %}
+              </div>
+            </div>
+          </div>
+        {% endif %}
+
+        <!-- Service containers (agent-managed) -->
         <div class="card mb-4">
           <div class="card-header fs-5">
             <span>{{ lang.debug.containers_info }}</span>
           </div>
           <div class="card-body p-0">
             <div class="row mx-0">
-              <!-- rest of the containers -->
               {% for container, container_info in containers %}
                   <div class="col-md-6 col-sm-12 p-2">
                     <div class="list-group-item p-0">
                       <div class="d-flex p-2 list-group-header">
                         <div>
                           <span class="fw-bold">{{ container }}</span>
-                          <span class="d-block d-md-inline">({{ container_info.Config.Image }})</span>
-                          <small class="d-block">({{ lang.debug.started_on }} <span class="parse_date">{{ container_info.State.StartedAtHR }}</span>)</small>
-                          {% if container_info.State.Running == 1 %}
+                          <span class="badge fs-7 bg-secondary ms-1" title="{{ lang.debug.replicas_title }}">{{ lang.debug.nodes_count|format(container_info.State.NodeCount) }}</span>
+                          {% if container_info.Config.Image %}
+                            <span class="d-block d-md-inline text-muted">{{ container_info.Config.Image }}</span>
+                          {% endif %}
+                          {% if container_info.State.StartedAtHR and container_info.State.StartedAtHR != '—' %}
+                            <small class="d-block">{{ lang.debug.started_on }} <span class="parse_date">{{ container_info.State.StartedAtHR }}</span></small>
+                          {% endif %}
+                          {% if container_info.State.Running == 1 and container_info.State.Health == 'fail' %}
+                            <span class="badge fs-7 bg-warning text-dark" style="min-width:100px" title="{{ container_info.State.HealthDetail }}">
+                              <i class="bi bi-exclamation-triangle-fill me-1"></i>{{ lang.debug.container_unhealthy }}
+                            </span>
+                          {% elseif container_info.State.Running == 1 and container_info.State.Health == 'degraded' %}
+                            <span class="badge fs-7 bg-warning text-dark" style="min-width:100px" title="{{ container_info.State.UnhealthyCount }}/{{ container_info.State.NodeCount }} unhealthy: {{ container_info.State.HealthDetail }}">
+                              <i class="bi bi-exclamation-circle me-1"></i>{{ lang.debug.container_degraded }} ({{ container_info.State.UnhealthyCount }}/{{ container_info.State.NodeCount }})
+                            </span>
+                          {% elseif container_info.State.Running == 1 %}
                             <span class="badge fs-7 bg-success loader" style="min-width:100px">
                               {{ lang.debug.container_running }}
                               <span class="loader-dot">.</span>
@@ -198,38 +239,49 @@
                           </button>
                         </div>
                       </div>
-                      <div class="collapse p-0 list-group-details container-details-collapse" id="{{ container }}Collapse" data-id="{{ container_info.Id }}">
-                        <div class="row p-2 pt-4">
-                          <div class="mt-4 col-sm-12 col-md-6 d-flex flex-column">
-                            <h6>Disk I/O</h6>
-                            <div class="spinner-border my-4 mx-auto" role="status">
-                              <span class="visually-hidden">Loading...</span>
-                            </div>
-                            <canvas class="d-none" id="{{ container }}_DiskIOChart" width="400" height="200"></canvas>
-                          </div>
-                          <div class="mt-4 col-sm-12 col-md-6 d-flex flex-column">
-                            <h6>Net I/O</h6>
-                            <div class="spinner-border my-4 mx-auto" role="status">
-                              <span class="visually-hidden">Loading...</span>
-                            </div>
-                            <canvas class="d-none" id="{{ container }}_NetIOChart" width="400" height="200"></canvas>
-                          </div>
-                          <div class="col-12 d-flex" style="height: 40px">
+                      <div class="collapse p-0 list-group-details" id="{{ container }}Collapse">
+                        <div class="p-2 pt-3">
+                          <h6 class="mb-2">{{ lang.debug.nodes }}</h6>
+                          {% if container_info.Nodes|length > 0 %}
+                            <ul class="list-group list-group-flush small mb-3">
+                              {% for n in container_info.Nodes %}
+                                <li class="list-group-item d-flex justify-content-between align-items-center py-2 px-2">
+                                  <span class="text-truncate me-2">
+                                    {% if n.Health == 'fail' %}
+                                      <i class="bi bi-circle-fill text-warning me-1" title="{{ n.HealthDetail }}"></i>
+                                    {% elseif n.Health == 'ok' %}
+                                      <i class="bi bi-circle-fill text-success me-1"></i>
+                                    {% else %}
+                                      <i class="bi bi-circle text-muted me-1"></i>
+                                    {% endif %}
+                                    <code title="{{ n.NodeId }}">{{ n.NodeId matches '/^[0-9a-f]{12}$/' ? n.NodeId[:8] : n.NodeId }}</code>
+                                    <small class="text-muted ms-2">
+                                      cpu {{ n.CPUPercent ?: '0.00' }}%
+                                      · mem {{ ((n.MemoryBytes ?: 0) / 1024 / 1024) | round }} MiB
+                                    </small>
+                                    {% if n.Health == 'fail' and n.HealthDetail %}
+                                      <small class="d-block text-warning ms-3">{{ n.HealthDetail }}</small>
+                                    {% endif %}
+                                  </span>
+                                  <a href data-bs-toggle="modal"
+                                    data-container="{{ container }}"
+                                    data-node="{{ n.NodeId }}"
+                                    data-bs-target="#RestartContainer"
+                                    class="btn btn-sm btn-outline-secondary">
+                                    <i class="bi bi-arrow-clockwise"></i> {{ lang.debug.restart_node }}
+                                  </a>
+                                </li>
+                              {% endfor %}
+                            </ul>
+                          {% else %}
+                            <div class="text-muted small mb-3">—</div>
+                          {% endif %}
+                          <div class="d-flex justify-content-end">
                             <a href data-bs-toggle="modal"
                               data-container="{{ container }}"
                               data-bs-target="#RestartContainer"
-                              class="btn btn-sm btn-secondary d-flex align-items-center justify-content-center mb-2 ms-auto"
-                              style="height: 30px;">{{ lang.debug.restart_container }}
-                                <i class="ms-1 bi
-                                {% if container_info.State.Running == 1 %}
-                                bi-record-fill text-success
-                                {% elseif container_info.State %}
-                                bi-record-fill text-danger
-                                {% else %}
-                                default
-                                {% endif %}
-                                "
-                              ></i>
+                              class="btn btn-sm btn-secondary">
+                              <i class="bi bi-arrow-repeat"></i> {{ lang.debug.restart_all_nodes }}
                             </a>
                           </div>
                         </div>