[Agent] Replace dockerapi container with Redis-based control bus

data/web/inc/ajax/container_ctrl.php

@@ -1,59 +1,35 @@
-<?php
-
-// Block requests by checking the 'Sec-Fetch-Dest' header.
-if (isset($_SERVER['HTTP_SEC_FETCH_DEST']) && $_SERVER['HTTP_SEC_FETCH_DEST'] !== 'empty') {
-  header('HTTP/1.1 403 Forbidden');
-  exit;
-}
-
-require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/prerequisites.inc.php';
-if (!isset($_SESSION['mailcow_cc_role']) || $_SESSION['mailcow_cc_role'] != 'admin') {
-	exit();
-}
-
-if (preg_match('/^[a-z\-]{0,}-mailcow/', $_GET['service'])) {
-  if ($_GET['action'] == "start") {
-    header('Content-Type: text/html; charset=utf-8');
-    $retry = 0;
-    while (docker('info', $_GET['service'])['State']['Running'] != 1 && $retry <= 3) {
-      $response = docker('post', $_GET['service'], 'start');
-      $response = json_decode($response, true);
-      $last_response = ($response['type'] == "success") ? '<b><span class="pull-right text-success">OK</span></b>' : '<b><span class="pull-right text-danger">Error: ' . $response['msg'] . '</span></b>';
-      if ($response['type'] == "success") {
-        break;
-      }
-      usleep(1500000);
-      $retry++;
-    }
-    echo (!isset($last_response)) ? '<b><span class="pull-right text-warning">Already running</span></b>' : $last_response;
-  }
-  if ($_GET['action'] == "stop") {
-    header('Content-Type: text/html; charset=utf-8');
-    $retry = 0;
-    while (docker('info', $_GET['service'])['State']['Running'] == 1 && $retry <= 3) {
-      $response = docker('post', $_GET['service'], 'stop');
-      $response = json_decode($response, true);
-      $last_response = ($response['type'] == "success") ? '<b><span class="pull-right text-success">OK</span></b>' : '<b><span class="pull-right text-danger">Error: ' . $response['msg'] . '</span></b>';
-      if ($response['type'] == "success") {
-        break;
-      }
-      usleep(1500000);
-      $retry++;
-    }
-    echo (!isset($last_response)) ? '<b><span class="pull-right text-warning">Not running</span></b>' : $last_response;
-  }
-  if ($_GET['action'] == "restart") {
-    header('Content-Type: text/html; charset=utf-8');
-    $response = docker('post', $_GET['service'], 'restart');
-    $response = json_decode($response, true);
-    $last_response = ($response['type'] == "success") ? '<b><span class="pull-right text-success">OK</span></b>' : '<b><span class="pull-right text-danger">Error: ' . $response['msg'] . '</span></b>';
-    echo (!isset($last_response)) ? '<b><span class="pull-right text-warning">Cannot restart container</span></b>' : $last_response;
-  }
-  if ($_GET['action'] == "logs") {
-    $lines = (empty($_GET['lines']) || !is_numeric($_GET['lines'])) ? 1000 : $_GET['lines'];
-    header('Content-Type: text/plain; charset=utf-8');
-    print_r(preg_split('/\n/', docker('logs', $_GET['service'], $lines)));
-  }
-}
-
-?>
+<?php
+if (isset($_SERVER['HTTP_SEC_FETCH_DEST']) && $_SERVER['HTTP_SEC_FETCH_DEST'] !== 'empty') {
+  header('HTTP/1.1 403 Forbidden');
+  exit;
+}
+
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/prerequisites.inc.php';
+if (!isset($_SESSION['mailcow_cc_role']) || $_SESSION['mailcow_cc_role'] != 'admin') {
+  exit();
+}
+
+if (!preg_match('/^[a-z\-]{0,}-mailcow/', $_GET['service'] ?? '')) {
+  exit();
+}
+
+if (($_GET['action'] ?? '') !== 'restart') {
+  exit();
+}
+
+$service = preg_replace('/-mailcow$/', '', $_GET['service']);
+$node = isset($_GET['node']) ? preg_replace('/[^a-zA-Z0-9._\-]/', '', $_GET['node']) : '';
+
+$args = ($node !== '') ? array('target_node' => $node) : array();
+$resp = agent('request', $service, 'restart', $args, 60);
+header('Content-Type: text/html; charset=utf-8');
+if (agent('ok', $resp)) {
+  echo '<b><span class="pull-right text-success">' . htmlspecialchars($lang['success']['service_restart_ok']) . '</span></b>';
+}
+else {
+  $err_key = agent('error_lang', $resp);
+  $err_msg = isset($lang['danger'][$err_key])
+    ? sprintf($lang['danger'][$err_key], $service)
+    : $lang['danger']['agent_unknown_error'];
+  echo '<b><span class="pull-right text-danger">' . htmlspecialchars($err_msg) . '</span></b>';
+}

data/web/inc/functions.agent.inc.php

@@ -0,0 +1,281 @@
+<?php
+define('AGENT_ERR_NOT_FOUND',   'not_found');
+define('AGENT_ERR_TIMEOUT',     'timeout');
+define('AGENT_ERR_VALIDATION',  'validation');
+define('AGENT_ERR_UNSUPPORTED', 'unsupported_command');
+define('AGENT_ERR_INTERNAL',    'internal');
+
+function agent($_action, $_service = null, $_data = null, $_args = array(), $_timeout = 10) {
+  global $redis;
+  switch ($_action) {
+    case 'request_id':
+      return sprintf('%013d%s', (int)(microtime(true) * 1000), substr(bin2hex(random_bytes(10)), 0, 16));
+    break;
+    case 'services':
+      $list = array(
+        'unbound', 'clamd', 'rspamd', 'php-fpm', 'sogo',
+        'dovecot', 'postfix', 'postfix-tlspol', 'nginx', 'acme',
+        'netfilter', 'watchdog', 'olefy', 'host'
+      );
+      if (preg_match('/^([yY][eE][sS]|[yY])+$/', isset($_ENV['SKIP_CLAMD']) ? $_ENV['SKIP_CLAMD'] : '')) {
+        $list = array_values(array_diff($list, array('clamd')));
+      }
+      if (preg_match('/^([yY][eE][sS]|[yY])+$/', isset($_ENV['SKIP_OLEFY']) ? $_ENV['SKIP_OLEFY'] : '')) {
+        $list = array_values(array_diff($list, array('olefy')));
+      }
+      sort($list);
+      return $list;
+    break;
+    case 'live_nodes':
+      try {
+        $members = $redis->zRangeByScore('mailcow.nodes.' . $_service, (string)(time() - 30), '+inf');
+      }
+      catch (RedisException $e) {
+        return array();
+      }
+      return is_array($members) ? $members : array();
+    break;
+    case 'node_meta':
+      try {
+        $h = $redis->hGetAll('mailcow.node.' . $_service . '.' . $_data);
+      }
+      catch (RedisException $e) {
+        return null;
+      }
+      return $h ?: null;
+    break;
+    case 'node_stats':
+      try {
+        $h = $redis->hGetAll('mailcow.stats.' . $_service . '.' . $_data);
+      }
+      catch (RedisException $e) {
+        return null;
+      }
+      return $h ?: null;
+    break;
+    case 'stats':
+      $out = array();
+      foreach (agent('live_nodes', $_service) as $node_id) {
+        $stats = agent('node_stats', $_service, $node_id);
+        if ($stats) {
+          $out[$node_id] = $stats;
+        }
+      }
+      return $out;
+    break;
+    case 'publish':
+      $env = array(
+        'cmd' => $_data,
+        'request_id' => agent('request_id'),
+        'args' => (object)(is_array($_args) ? $_args : array()),
+        'issued_by' => 'mailcow-php'
+      );
+      try {
+        $redis->publish('mailcow.control.' . $_service, json_encode($env));
+      }
+      catch (RedisException $e) {
+        return false;
+      }
+      return true;
+    break;
+    case 'request':
+      $rid = agent('request_id');
+      $reply_to = 'mailcow.reply.' . $rid;
+      $env = array(
+        'cmd' => $_data,
+        'request_id' => $rid,
+        'args' => (object)(is_array($_args) ? $_args : array()),
+        'reply_to' => $reply_to,
+        'deadline' => gmdate('Y-m-d\TH:i:s\Z', time() + $_timeout),
+        'issued_by' => 'mailcow-php'
+      );
+      try {
+        $subs = $redis->publish('mailcow.control.' . $_service, json_encode($env));
+        if ($subs === 0) {
+          return array('ok' => false, 'result' => null, 'error' => $_service, 'error_code' => AGENT_ERR_NOT_FOUND, 'node' => '', 'duration_ms' => 0);
+        }
+        $popped = $redis->blPop(array($reply_to), $_timeout);
+      }
+      catch (RedisException $e) {
+        return array('ok' => false, 'result' => null, 'error' => $e->getMessage(), 'error_code' => AGENT_ERR_INTERNAL, 'node' => '', 'duration_ms' => 0);
+      }
+      if (!$popped || count($popped) < 2) {
+        return array('ok' => false, 'result' => null, 'error' => '', 'error_code' => AGENT_ERR_TIMEOUT, 'node' => '', 'duration_ms' => 0);
+      }
+      $resp = json_decode($popped[1], true);
+      if (!is_array($resp)) {
+        return array('ok' => false, 'result' => null, 'error' => 'malformed reply', 'error_code' => AGENT_ERR_INTERNAL, 'node' => '', 'duration_ms' => 0);
+      }
+      return array(
+        'ok' => !empty($resp['ok']),
+        'result' => isset($resp['result']) ? $resp['result'] : null,
+        'error' => isset($resp['error']) ? $resp['error'] : '',
+        'error_code' => isset($resp['error_code']) ? $resp['error_code'] : '',
+        'node' => isset($resp['node']) ? $resp['node'] : '',
+        'duration_ms' => isset($resp['duration_ms']) ? $resp['duration_ms'] : 0
+      );
+    break;
+    case 'request_all':
+      $rid = agent('request_id');
+      $reply_to = 'mailcow.reply.' . $rid;
+      $env = array(
+        'cmd' => $_data,
+        'request_id' => $rid,
+        'args' => (object)(is_array($_args) ? $_args : array()),
+        'reply_to' => $reply_to,
+        'deadline' => gmdate('Y-m-d\TH:i:s\Z', time() + $_timeout),
+        'issued_by' => 'mailcow-php'
+      );
+      $expected = max(1, count(agent('live_nodes', $_service)));
+      try {
+        $subs = (int)$redis->publish('mailcow.control.' . $_service, json_encode($env));
+      }
+      catch (RedisException $e) {
+        return array('responses' => array(), 'expected_nodes' => $expected, 'received_nodes' => array(), 'missing_nodes' => array(), 'error' => $e->getMessage());
+      }
+      if ($subs === 0) {
+        return array('responses' => array(), 'expected_nodes' => 0, 'received_nodes' => array(), 'missing_nodes' => array());
+      }
+      $responses = array();
+      $deadline = microtime(true) + $_timeout;
+      for ($i = 0; $i < $subs; $i++) {
+        $remaining = (int)ceil($deadline - microtime(true));
+        if ($remaining <= 0) break;
+        try {
+          $popped = $redis->blPop(array($reply_to), $remaining);
+        }
+        catch (RedisException $e) {
+          break;
+        }
+        if (!$popped || count($popped) < 2) break;
+        $resp = json_decode($popped[1], true);
+        if (is_array($resp)) {
+          $responses[] = array(
+            'ok' => !empty($resp['ok']),
+            'result' => isset($resp['result']) ? $resp['result'] : null,
+            'error' => isset($resp['error']) ? $resp['error'] : '',
+            'error_code' => isset($resp['error_code']) ? $resp['error_code'] : '',
+            'node' => isset($resp['node']) ? $resp['node'] : '',
+            'duration_ms' => isset($resp['duration_ms']) ? $resp['duration_ms'] : 0
+          );
+        }
+      }
+      $received_nodes = array();
+      foreach ($responses as $r) {
+        if (!empty($r['node'])) {
+          $received_nodes[] = $r['node'];
+        }
+      }
+      $live = agent('live_nodes', $_service);
+      return array(
+        'responses' => $responses,
+        'expected_nodes' => $expected,
+        'received_nodes' => array_values(array_unique($received_nodes)),
+        'missing_nodes' => array_values(array_diff($live, $received_nodes))
+      );
+    break;
+    case 'ok':
+      if (isset($_service['responses'])) {
+        foreach ($_service['responses'] as $r) {
+          if (!empty($r['ok'])) return true;
+        }
+        return false;
+      }
+      return !empty($_service['ok']);
+    break;
+    case 'first_error':
+      foreach (isset($_service['responses']) ? $_service['responses'] : array() as $r) {
+        if (empty($r['ok']) && !empty($r['error'])) return $r['error'];
+      }
+      return '';
+    break;
+    case 'error_lang':
+      $code = is_array($_service) && isset($_service['error_code']) ? $_service['error_code'] : '';
+      switch ($code) {
+        case AGENT_ERR_NOT_FOUND:
+          return 'no_live_agent';
+        case AGENT_ERR_TIMEOUT:
+          return 'agent_timeout';
+        default:
+          return 'agent_unknown_error';
+      }
+    break;
+  }
+}
+
+function infra($_action, $_service = null) {
+  global $redis;
+  global $pdo;
+  switch ($_action) {
+    case 'health':
+      switch ($_service) {
+        case 'redis':
+          try {
+            if ($redis instanceof Redis && $redis->ping()) {
+              $info = $redis->info('server');
+              $ver = is_array($info) && isset($info['redis_version']) ? $info['redis_version'] : '';
+              return array('ok' => true, 'image' => 'redis ' . $ver, 'error' => '');
+            }
+          }
+          catch (RedisException $e) {
+            return array('ok' => false, 'image' => 'redis', 'error' => $e->getMessage());
+          }
+          return array('ok' => false, 'image' => 'redis', 'error' => 'PING returned false');
+        break;
+        case 'mysql':
+          try {
+            if ($pdo instanceof PDO) {
+              $row = $pdo->query('SELECT VERSION() AS v')->fetch(PDO::FETCH_ASSOC);
+              $ver = $row && isset($row['v']) ? $row['v'] : '';
+              return array('ok' => true, 'image' => 'mariadb/mysql ' . $ver, 'error' => '');
+            }
+          }
+          catch (Exception $e) {
+            return array('ok' => false, 'image' => 'mariadb/mysql', 'error' => $e->getMessage());
+          }
+          return array('ok' => false, 'image' => 'mariadb/mysql', 'error' => 'no PDO handle');
+        break;
+        case 'memcached':
+          $sock = @fsockopen('memcached', 11211, $errno, $errstr, 2);
+          if (!$sock) {
+            return array('ok' => false, 'image' => 'memcached', 'error' => $errstr ?: 'connection refused');
+          }
+          stream_set_timeout($sock, 2);
+          fwrite($sock, "version\r\n");
+          $line = fgets($sock, 64);
+          fclose($sock);
+          if (is_string($line) && strpos($line, 'VERSION') === 0) {
+            return array('ok' => true, 'image' => 'memcached ' . trim(substr($line, strlen('VERSION '))), 'error' => '');
+          }
+          return array('ok' => false, 'image' => 'memcached', 'error' => 'no VERSION reply');
+        break;
+      }
+    break;
+    case 'status':
+      $out = array();
+      $defs = array(
+        'redis-mailcow' => 'redis',
+        'mysql-mailcow' => 'mysql',
+        'memcached-mailcow' => 'memcached'
+      );
+      foreach ($defs as $key => $svc) {
+        $h = infra('health', $svc);
+        $out[$key] = array(
+          'Service' => $svc,
+          'State' => array(
+            'Running' => $h['ok'] ? 1 : 0,
+            'NodeCount' => $h['ok'] ? 1 : 0,
+            'StartedAt' => '',
+            'StartedAtHR' => '—',
+            'Error' => $h['error']
+          ),
+          'Config' => array('Image' => $h['image']),
+          'Id' => $svc,
+          'Nodes' => array(),
+          'External' => true
+        );
+      }
+      return $out;
+    break;
+  }
+}

data/web/inc/functions.docker.inc.php

@@ -1,207 +0,0 @@
-<?php
-function docker($action, $service_name = null, $attr1 = null, $attr2 = null, $extra_headers = null) {
-  global $DOCKER_TIMEOUT;
-  global $redis;
-  $curl = curl_init();
-  curl_setopt($curl, CURLOPT_HTTPHEADER,array('Content-Type: application/json' ));
-  // We are using our mail certificates for dockerapi, the names will not match, the certs are trusted anyway
-  curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, 0);
-  curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, 0);
-  switch($action) {
-    case 'get_id':
-      curl_setopt($curl, CURLOPT_URL, 'https://dockerapi:443/containers/json');
-      curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
-      curl_setopt($curl, CURLOPT_POST, 0);
-      curl_setopt($curl, CURLOPT_TIMEOUT, $DOCKER_TIMEOUT);
-      $response = curl_exec($curl);
-      if ($response === false) {
-        $err = curl_error($curl);
-        curl_close($curl);
-        return $err;
-      }
-      else {
-        curl_close($curl);
-        $containers = json_decode($response, true);
-        if (!empty($containers)) {
-          foreach ($containers as $container) {
-            if (isset($container['Config']['Labels']['com.docker.compose.service'])
-              && $container['Config']['Labels']['com.docker.compose.service'] == $service_name
-              && strtolower($container['Config']['Labels']['com.docker.compose.project']) == strtolower(getenv('COMPOSE_PROJECT_NAME'))) {
-              return trim($container['Id']);
-            }
-          }
-        }
-      }
-      return false;
-    break;
-    case 'containers':
-      curl_setopt($curl, CURLOPT_URL, 'https://dockerapi:443/containers/json');
-      curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
-      curl_setopt($curl, CURLOPT_POST, 0);
-      curl_setopt($curl, CURLOPT_TIMEOUT, $DOCKER_TIMEOUT);
-      $response = curl_exec($curl);
-      if ($response === false) {
-        $err = curl_error($curl);
-        curl_close($curl);
-        return $err;
-      }
-      else {
-        curl_close($curl);
-        $containers = json_decode($response, true);
-        if (!empty($containers)) {
-          foreach ($containers as $container) {
-            if (strtolower($container['Config']['Labels']['com.docker.compose.project']) == strtolower(getenv('COMPOSE_PROJECT_NAME'))) {
-              $out[$container['Config']['Labels']['com.docker.compose.service']]['State'] = $container['State'];
-              $out[$container['Config']['Labels']['com.docker.compose.service']]['Config'] = $container['Config'];
-              $out[$container['Config']['Labels']['com.docker.compose.service']]['Id'] = trim($container['Id']);
-            }
-          }
-        }
-        return (!empty($out)) ? $out : false;
-      }
-      return false;
-    break;
-    case 'info':
-      if (empty($service_name)) {
-        curl_setopt($curl, CURLOPT_URL, 'https://dockerapi:443/containers/json?all=true');
-        curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
-        curl_setopt($curl, CURLOPT_POST, 0);
-        curl_setopt($curl, CURLOPT_TIMEOUT, $DOCKER_TIMEOUT);
-      }
-      else {
-        $container_id = docker('get_id', $service_name);
-        if (ctype_xdigit($container_id)) {
-          curl_setopt($curl, CURLOPT_URL, 'https://dockerapi:443/containers/' . $container_id . '/json');
-        }
-        else {
-          return false;
-        }
-      }
-      curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
-      curl_setopt($curl, CURLOPT_POST, 0);
-      curl_setopt($curl, CURLOPT_TIMEOUT, $DOCKER_TIMEOUT);
-      $response = curl_exec($curl);
-      if ($response === false) {
-        $err = curl_error($curl);
-        curl_close($curl);
-        return $err;
-      }
-      else {
-        curl_close($curl);
-        $decoded_response = json_decode($response, true);
-        if (!empty($decoded_response)) {
-          if (empty($service_name)) {
-            foreach ($decoded_response as $container) {
-              if (isset($container['Config']['Labels']['com.docker.compose.project'])
-                && strtolower($container['Config']['Labels']['com.docker.compose.project']) == strtolower(getenv('COMPOSE_PROJECT_NAME'))) {
-                unset($container['Config']['Env']);
-                $out[$container['Config']['Labels']['com.docker.compose.service']]['State'] = $container['State'];
-                $out[$container['Config']['Labels']['com.docker.compose.service']]['Config'] = $container['Config'];
-                $out[$container['Config']['Labels']['com.docker.compose.service']]['Id'] = trim($container['Id']);
-              }
-            }
-          }
-          else {
-            if (isset($decoded_response['Config']['Labels']['com.docker.compose.project']) 
-              && strtolower($decoded_response['Config']['Labels']['com.docker.compose.project']) == strtolower(getenv('COMPOSE_PROJECT_NAME'))) {
-              unset($container['Config']['Env']);
-              $out[$decoded_response['Config']['Labels']['com.docker.compose.service']]['State'] = $decoded_response['State'];
-              $out[$decoded_response['Config']['Labels']['com.docker.compose.service']]['Config'] = $decoded_response['Config'];
-              $out[$decoded_response['Config']['Labels']['com.docker.compose.service']]['Id'] = trim($decoded_response['Id']);
-            }
-          }
-        }
-        if (empty($response)) {
-          return true;
-        }
-        else {
-          return (!empty($out)) ? $out : false;
-        }
-      }
-    break;
-    case 'post':
-      if (!empty($attr1)) {
-        $container_id = docker('get_id', $service_name);
-        if (ctype_xdigit($container_id) && ctype_alnum($attr1)) {
-          curl_setopt($curl, CURLOPT_URL, 'https://dockerapi:443/containers/' . $container_id . '/' . $attr1);
-          curl_setopt($curl, CURLOPT_POST, 1);
-          curl_setopt($curl, CURLOPT_TIMEOUT, $DOCKER_TIMEOUT);
-          if (!empty($attr2)) {
-            curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($attr2));
-          }
-          if (!empty($extra_headers) && is_array($extra_headers)) {
-            curl_setopt($curl, CURLOPT_HTTPHEADER, $extra_headers);
-          }
-          curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
-          $response = curl_exec($curl);
-          if ($response === false) {
-            $err = curl_error($curl);
-            curl_close($curl);
-            return $err;
-          }
-          else {
-            curl_close($curl);
-            if (empty($response)) {
-              return true;
-            }
-            else {
-              return $response;
-            }
-          }
-        }
-      }
-    break;
-    case 'container_stats':
-      if (empty($service_name)){
-        return false;
-      }
-
-      $container_id = $service_name;
-      curl_setopt($curl, CURLOPT_URL, 'https://dockerapi:443/container/' . $container_id . '/stats/update');
-      curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
-      curl_setopt($curl, CURLOPT_POST, 1);
-      curl_setopt($curl, CURLOPT_TIMEOUT, $DOCKER_TIMEOUT);
-      $response = curl_exec($curl);
-      if ($response === false) {
-        $err = curl_error($curl);
-        curl_close($curl);
-        return $err;
-      }
-      else {
-        curl_close($curl);
-        $stats = json_decode($response, true);
-        if (!empty($stats)) return $stats;
-      }
-      return false;
-    break;
-    case 'host_stats':
-      curl_setopt($curl, CURLOPT_URL, 'https://dockerapi:443/host/stats');
-      curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
-      curl_setopt($curl, CURLOPT_POST, 0);
-      curl_setopt($curl, CURLOPT_TIMEOUT, $DOCKER_TIMEOUT);
-      $response = curl_exec($curl);
-      if ($response === false) {
-        $err = curl_error($curl);
-        curl_close($curl);
-        return $err;
-      }
-      else {
-        curl_close($curl);
-        $stats = json_decode($response, true);
-        if (!empty($stats)) return $stats;
-      }
-      return false;
-    break;
-    case 'broadcast':
-      $request = array(
-        "api_call" => "container_post",
-        "container_name" => $service_name,
-        "post_action" => $attr1,
-        "request" => $attr2
-      );
-
-      $redis->publish("MC_CHANNEL", json_encode($request));
-      return true;
-    break;
-  }
-}

data/web/inc/functions.fail2ban.inc.php

@@ -109,7 +109,7 @@ function fail2ban($_action, $_data = null, $_extra = null) {
             return false;
           }
           // Rules will also be recreated on log events, but rules may seem empty for a second in the UI
-          docker('post', 'netfilter-mailcow', 'restart');
+          agent('request', 'netfilter', 'restart', array(), 30);
           $fail_count = 0;
           $regex_result = json_decode($redis->Get('F2B_REGEX'), true);
           while (empty($regex_result) && $fail_count < 10) {
@@ -206,7 +206,7 @@ function fail2ban($_action, $_data = null, $_extra = null) {
                 try {
                   $redis->hSet('F2B_BLACKLIST', $network, 1);
                   $redis->hDel('F2B_WHITELIST', $network, 1);
-                  //$response = docker('post', 'netfilter-mailcow', 'restart');
+                  // netfilter picks up the redis changes
                 }
                 catch (RedisException $e) {
                   $_SESSION['return'][] = array(

data/web/inc/functions.inc.php

@@ -2239,30 +2239,19 @@ function rspamd_ui($action, $data = null) {
         );
         return false;
       }
-      $docker_return = docker('post', 'rspamd-mailcow', 'exec', array('cmd' => 'rspamd', 'task' => 'worker_password', 'raw' => $rspamd_ui_pass), array('Content-Type: application/json'));
-      if ($docker_return_array = json_decode($docker_return, true)) {
-        if ($docker_return_array['type'] == 'success') {
-          $_SESSION['return'][] =  array(
-            'type' => 'success',
-            'log' => array(__FUNCTION__, '*', '*'),
-            'msg' => 'rspamd_ui_pw_set'
-          );
-          return true;
-        }
-        else {
-          $_SESSION['return'][] =  array(
-            'type' => $docker_return_array['type'],
-            'log' => array(__FUNCTION__, '*', '*'),
-            'msg' => $docker_return_array['msg']
-          );
-          return false;
-        }
-      }
-      else {
+      $resp = agent('request_all', 'rspamd', 'exec.set-worker-password', array('password' => $rspamd_ui_pass), 30);
+      if (agent('ok', $resp)) {
+        $_SESSION['return'][] =  array(
+          'type' => 'success',
+          'log' => array(__FUNCTION__, '*', '*'),
+          'msg' => 'rspamd_ui_pw_set'
+        );
+        return true;
+      } else {
         $_SESSION['return'][] =  array(
           'type' => 'danger',
           'log' => array(__FUNCTION__, '*', '*'),
-          'msg' => 'unknown'
+          'msg' => agent('first_error', $resp) ?: 'rspamd: no live agent responded'
         );
         return false;
       }

data/web/inc/functions.mailbox.inc.php

@@ -125,8 +125,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 fwrite($filter_handle, $script_data);
                 fclose($filter_handle);
               }
-              $restart_response = json_decode(docker('post', 'dovecot-mailcow', 'restart'), true);
-              if ($restart_response['type'] == "success") {
+              $restart_response = agent('request', 'dovecot', 'restart', array(), 30);
+              if (agent('ok', $restart_response)) {
                 $_SESSION['return'][] = array(
                   'type' => 'success',
                   'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
@@ -160,8 +160,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
                 fwrite($filter_handle, $script_data);
                 fclose($filter_handle);
               }
-              $restart_response = json_decode(docker('post', 'dovecot-mailcow', 'restart'), true);
-              if ($restart_response['type'] == "success") {
+              $restart_response = agent('request', 'dovecot', 'restart', array(), 30);
+              if (agent('ok', $restart_response)) {
                 $_SESSION['return'][] = array(
                   'type' => 'success',
                   'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
@@ -669,8 +669,8 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             }
           }
           if (!empty($restart_sogo)) {
-            $restart_response = json_decode(docker('post', 'sogo-mailcow', 'restart'), true);
-            if ($restart_response['type'] == "success") {
+            $restart_response = agent('request', 'sogo', 'restart', array(), 30);
+            if (agent('ok', $restart_response)) {
               $_SESSION['return'][] = array(
                 'type' => 'success',
                 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
@@ -3553,22 +3553,30 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
 
           // get imap acls
           try {
-            $exec_fields = array(
-              'cmd' => 'doveadm',
-              'task' => 'get_acl',
-              'id' => $old_username
-            );
-            $imap_acls = json_decode(docker('post', 'dovecot-mailcow', 'exec', $exec_fields), true);
+            $acl_agg = agent('request_all', 'dovecot', 'exec.acl-get', array('user' => $old_username), 10);
+            $imap_acls = array();
+            $seen = array();
+            foreach ($acl_agg['responses'] as $r) {
+              if (empty($r['ok'])) continue;
+              foreach ((isset($r['result']['acls']) ? $r['result']['acls'] : array()) as $a) {
+                $key = (isset($a['mailbox']) ? $a['mailbox'] : '') . '|' . (isset($a['identifier']) ? $a['identifier'] : '');
+                if (isset($seen[$key])) continue;
+                $seen[$key] = true;
+                $imap_acls[] = array(
+                  'user'    => $old_username,
+                  'mailbox' => isset($a['mailbox']) ? $a['mailbox'] : '',
+                  'id'      => isset($a['identifier']) ? $a['identifier'] : '',
+                  'rights'  => isset($a['rights']) ? $a['rights'] : '',
+                );
+              }
+            }
             // delete imap acls
             foreach ($imap_acls as $imap_acl) {
-              $exec_fields = array(
-                'cmd' => 'doveadm',
-                'task' => 'delete_acl',
-                'user' => $imap_acl['user'],
-                'mailbox' => $imap_acl['mailbox'],
-                'id' => $imap_acl['id']
-              );
-              docker('post', 'dovecot-mailcow', 'exec', $exec_fields);
+              agent('request_all', 'dovecot', 'exec.acl-delete', array(
+                'user'       => $imap_acl['user'],
+                'mailbox'    => $imap_acl['mailbox'],
+                'identifier' => $imap_acl['id'],
+              ), 10);
             }
           } catch (Exception $e) {
             $_SESSION['return'][] = array(
@@ -3649,41 +3657,27 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           }
 
           // move maildir
-          $exec_fields = array(
-            'cmd' => 'maildir',
-            'task' => 'move',
-            'old_maildir' => $domain . '/' . $old_local_part,
-            'new_maildir' => $domain . '/' . $new_local_part
-          );
-          if (getenv("CLUSTERMODE") == "replication") {
-            // broadcast to each dovecot container
-            docker('broadcast', 'dovecot-mailcow', 'exec', $exec_fields);
-          } else {
-            docker('post', 'dovecot-mailcow', 'exec', $exec_fields);
-          }
+          agent('request_all', 'dovecot', 'exec.maildir-move', array(
+            'from' => $domain . '/' . $old_local_part,
+            'to'   => $domain . '/' . $new_local_part,
+          ), 30);
 
           // rename username in sogo
-          $exec_fields = array(
-            'cmd' => 'sogo',
-            'task' => 'rename_user',
-            'old_username' => $old_username,
-            'new_username' => $new_username
-          );
-          docker('post', 'sogo-mailcow', 'exec', $exec_fields);
+          agent('request', 'sogo', 'exec.rename-user', array(
+            'old' => $old_username,
+            'new' => $new_username,
+          ), 30);
 
           // set imap acls
           foreach ($imap_acls as $imap_acl) {
             $user_id = ($imap_acl['id'] == $old_username) ? $new_username : $imap_acl['id'];
             $user = ($imap_acl['user'] == $old_username) ? $new_username : $imap_acl['user'];
-            $exec_fields = array(
-              'cmd' => 'doveadm',
-              'task' => 'set_acl',
-              'user' => $user,
-              'mailbox' => $imap_acl['mailbox'],
-              'id' => $user_id,
-              'rights' => $imap_acl['rights']
-            );
-            docker('post', 'dovecot-mailcow', 'exec', $exec_fields);
+            agent('request_all', 'dovecot', 'exec.acl-set', array(
+              'user'       => $user,
+              'mailbox'    => $imap_acl['mailbox'],
+              'identifier' => $user_id,
+              'rights'     => $imap_acl['rights'],
+            ), 15);
           }
 
           // create alias
@@ -4553,24 +4547,19 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
           else {
             $_data = $_SESSION['mailcow_cc_username'];
           }
-          $exec_fields = array(
-            'cmd' => 'sieve',
-            'task' => 'list',
-            'username' => $_data
-          );
-          $filters = docker('post', 'dovecot-mailcow', 'exec', $exec_fields);
-          $filters = array_filter(preg_split("/(\r\n|\n|\r)/",$filters));
-          foreach ($filters as $filter) {
+          $list_resp = agent('request', 'dovecot', 'exec.sieve-list', array('user' => $_data), 10);
+          if (empty($list_resp['ok'])) {
+            return false;
+          }
+          $scripts = isset($list_resp['result']['scripts']) ? $list_resp['result']['scripts'] : array();
+          foreach ($scripts as $filter) {
             if (preg_match('/.+ ACTIVE/i', $filter)) {
-              $exec_fields = array(
-                'cmd' => 'sieve',
-                'task' => 'print',
-                'script_name' => substr($filter, 0, -7),
-                'username' => $_data
-              );
-              $script = docker('post', 'dovecot-mailcow', 'exec', $exec_fields);
-              // Remove first line
-              return preg_replace('/^.+\n/', '', $script);
+              $print_resp = agent('request', 'dovecot', 'exec.sieve-print', array(
+                'user'   => $_data,
+                'script' => substr($filter, 0, -7),
+              ), 10);
+              if (empty($print_resp['ok'])) return false;
+              return isset($print_resp['result']['body']) ? $print_resp['result']['body'] : '';
             }
           }
           return false;
@@ -5712,13 +5701,12 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
               );
               continue;
             }
-            $exec_fields = array('cmd' => 'maildir', 'task' => 'cleanup', 'maildir' => $domain);
-            $maildir_gc = json_decode(docker('post', 'dovecot-mailcow', 'exec', $exec_fields), true);
-            if ($maildir_gc['type'] != 'success') {
+            $maildir_gc = agent('request_all', 'dovecot', 'exec.maildir-cleanup', array('maildir' => $domain), 30);
+            if (!agent('ok', $maildir_gc)) {
               $_SESSION['return'][] = array(
                 'type' => 'warning',
                 'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
-                'msg' => 'Could not move mail storage to garbage collector: ' . $maildir_gc['msg']
+                'msg' => 'Could not move mail storage to garbage collector: ' . agent('first_error', $maildir_gc)
               );
             }
             $stmt = $pdo->prepare("DELETE FROM `domain` WHERE `domain` = :domain");
@@ -5967,20 +5955,13 @@ function mailbox($_action, $_type, $_data = null, $_extra = null) {
             $mailbox_details = mailbox('get', 'mailbox_details', $username);
             if (!empty($mailbox_details['domain']) && !empty($mailbox_details['local_part'])) {
               $maildir = $mailbox_details['domain'] . '/' . $mailbox_details['local_part'];
-              $exec_fields = array('cmd' => 'maildir', 'task' => 'cleanup', 'maildir' => $maildir);
-
-              if (getenv("CLUSTERMODE") == "replication") {
-                // broadcast to each dovecot container
-                docker('broadcast', 'dovecot-mailcow', 'exec', $exec_fields);
-              } else {
-                $maildir_gc = json_decode(docker('post', 'dovecot-mailcow', 'exec', $exec_fields), true);
-                if ($maildir_gc['type'] != 'success') {
-                  $_SESSION['return'][] = array(
-                    'type' => 'warning',
-                    'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
-                    'msg' => 'Could not move maildir to garbage collector: ' . $maildir_gc['msg']
-                  );
-                }
+              $maildir_gc = agent('request_all', 'dovecot', 'exec.maildir-cleanup', array('maildir' => $maildir), 30);
+              if (!agent('ok', $maildir_gc)) {
+                $_SESSION['return'][] = array(
+                  'type' => 'warning',
+                  'log' => array(__FUNCTION__, $_action, $_type, $_data_log, $_attr),
+                  'msg' => 'Could not move maildir to garbage collector: ' . agent('first_error', $maildir_gc)
+                );
               }
             }
             else {

data/web/inc/functions.mailq.inc.php

@@ -1,121 +1,138 @@
-<?php
-function mailq($_action, $_data = null) {
-  if ($_SESSION['mailcow_cc_role'] != "admin") {
-    $_SESSION['return'][] = array(
-      'type' => 'danger',
-      'log' => array(__FUNCTION__, $_action, $_data),
-      'msg' => 'access_denied'
-    );
-    return false;
-  }
-  function process_mailq_output($returned_output, $_action, $_data) {
-    if ($returned_output !== NULL) {
-      if ($_action == 'cat') {
-        logger(array('return' => array(
-          array(
-            'type' => 'success',
-            'log' => array(__FUNCTION__, $_action, $_data),
-            'msg' => 'queue_cat_success'
-          )
-        )));
-        return $returned_output;
-      }
-      else {
-        if (isset($returned_output['type']) && $returned_output['type'] == 'danger') {
-          $_SESSION['return'][] = array(
-            'type' => 'danger',
-            'log' => array(__FUNCTION__, $_action, $_data),
-            'msg' => 'Error: ' . $returned_output['msg']
-          );
-        }
-        if (isset($returned_output['type']) && $returned_output['type'] == 'success') {
-          $_SESSION['return'][] = array(
-            'type' => 'success',
-            'log' => array(__FUNCTION__, $_action, $_data),
-            'msg' => 'queue_command_success'
-          );
-        }
-      }
-    }
-    else {
-      $_SESSION['return'][] = array(
-        'type' => 'danger',
-        'log' => array(__FUNCTION__, $_action, $_data),
-        'msg' => 'unknown'
-      );
-    }
-  }
-  if ($_action == 'get') {
-    $mailq_lines = docker('post', 'postfix-mailcow', 'exec', array('cmd' => 'mailq', 'task' => 'list'));
-    $lines = 0;
-    // Hard limit to 10000 items
-    foreach (preg_split("/((\r?\n)|(\r\n?))/", $mailq_lines) as $mailq_item) if ($lines++ < 10000) {
-      if (empty($mailq_item) || $mailq_item == '1') {
-        continue;
-      }
-      $mq_line = json_decode($mailq_item, true);
-      if ($mq_line !== NULL) {
-        $rcpts = array();
-        foreach ($mq_line['recipients'] as $rcpt) {
-          if (isset($rcpt['delay_reason'])) {
-            $rcpts[] = $rcpt['address'] . ' (' . $rcpt['delay_reason'] . ')';
-          }
-          else {
-            $rcpts[] = $rcpt['address'];
-          }
-        }
-        if (!empty($rcpts)) {
-          $mq_line['recipients'] = $rcpts;
-        }
-        $line[] = $mq_line;
-      }
-    }
-    if (!isset($line) || empty($line)) {
-      return '[]';
-    }
-    else {
-      return json_encode($line);
-    }
-  }
-  elseif ($_action == 'delete') {
-    if (!is_array($_data['qid'])) {
-      $qids = array();
-      $qids[] = $_data['qid'];
-    }
-    else {
-      $qids = $_data['qid'];
-    }
-    $docker_return = docker('post', 'postfix-mailcow', 'exec', array('cmd' => 'mailq', 'task' => 'delete', 'items' => $qids));
-    process_mailq_output(json_decode($docker_return, true), $_action, $_data);
-  }
-  elseif ($_action == 'cat') {
-    if (!is_array($_data['qid'])) {
-      $qids = array();
-      $qids[] = $_data['qid'];
-    }
-    else {
-      $qids = $_data['qid'];
-    }
-    $docker_return = docker('post', 'postfix-mailcow', 'exec', array('cmd' => 'mailq', 'task' => 'cat', 'items' => $qids));
-    return process_mailq_output($docker_return, $_action, $_data);
-  }
-  elseif ($_action == 'edit') {
-    if (in_array($_data['action'], array('hold', 'unhold', 'deliver'))) {
-      if (!is_array($_data['qid'])) {
-        $qids = array();
-        $qids[] = $_data['qid'];
-      }
-      else {
-        $qids = $_data['qid'];
-      }
-      if (!empty($qids)) {
-        $docker_return = docker('post', 'postfix-mailcow', 'exec', array('cmd' => 'mailq', 'task' => $_data['action'], 'items' => $qids));
-        process_mailq_output(json_decode($docker_return, true), $_action, $_data);
-      }
-    }
-    if (in_array($_data['action'], array('flush', 'super_delete'))) {
-      $docker_return = docker('post', 'postfix-mailcow', 'exec', array('cmd' => 'mailq', 'task' => $_data['action']));
-      process_mailq_output(json_decode($docker_return, true), $_action, $_data);
-    }
-  }
-}
+<?php
+function mailq($_action, $_data = null) {
+  global $lang;
+  if ($_SESSION['mailcow_cc_role'] != "admin") {
+    $_SESSION['return'][] = array(
+      'type' => 'danger',
+      'log' => array(__FUNCTION__, $_action, $_data),
+      'msg' => 'access_denied'
+    );
+    return false;
+  }
+  switch ($_action) {
+    case 'get':
+      $agg = agent('request_all', 'postfix', 'exec.mailq', array(), 15);
+      $lines = array();
+      foreach ($agg['responses'] as $r) {
+        if (empty($r['ok'])) continue;
+        $queue = isset($r['result']['queue']) ? $r['result']['queue'] : array();
+        foreach ($queue as $entry) {
+          if (is_array($entry)) {
+            $entry['node'] = $r['node'];
+            if (!empty($entry['recipients']) && is_array($entry['recipients'])) {
+              $rcpts = array();
+              foreach ($entry['recipients'] as $rcpt) {
+                $addr = isset($rcpt['address']) ? $rcpt['address'] : '';
+                if (isset($rcpt['delay_reason'])) {
+                  $rcpts[] = $addr . ' (' . $rcpt['delay_reason'] . ')';
+                }
+                else {
+                  $rcpts[] = $addr;
+                }
+              }
+              $entry['recipients'] = $rcpts;
+            }
+            $lines[] = $entry;
+          }
+          if (count($lines) >= 10000) break 2;
+        }
+      }
+      return empty($lines) ? '[]' : json_encode($lines);
+    break;
+    case 'delete':
+      $qids = isset($_data['qid']) && is_array($_data['qid']) ? $_data['qid'] : array($_data['qid']);
+      $ok_count = 0;
+      $failed = 0;
+      foreach ($qids as $qid) {
+        $agg = agent('request_all', 'postfix', 'exec.delete-from-queue', array('queue_id' => $qid), 10);
+        if (agent('ok', $agg)) {
+          $ok_count++;
+        }
+        else {
+          $failed++;
+        }
+      }
+      $ok = ($ok_count > 0 && $failed === 0);
+      $_SESSION['return'][] = array(
+        'type' => $ok ? 'success' : 'danger',
+        'log' => array(__FUNCTION__, $_action, $_data),
+        'msg' => $ok ? 'queue_command_success' : 'queue_command_failed'
+      );
+      return $ok;
+    break;
+    case 'cat':
+      $qids = isset($_data['qid']) && is_array($_data['qid']) ? $_data['qid'] : array($_data['qid']);
+      $body = '';
+      foreach ($qids as $qid) {
+        $agg = agent('request_all', 'postfix', 'exec.cat-queue', array('queue_id' => $qid), 15);
+        foreach ($agg['responses'] as $r) {
+          if (!empty($r['ok']) && !empty($r['result']['body'])) {
+            $body .= $r['result']['body'];
+          }
+        }
+      }
+      if ($body === '') {
+        $_SESSION['return'][] = array(
+          'type' => 'danger',
+          'log' => array(__FUNCTION__, $_action, $_data),
+          'msg' => 'queue_cat_empty'
+        );
+        return null;
+      }
+      $_SESSION['return'][] = array(
+        'type' => 'success',
+        'log' => array(__FUNCTION__, $_action, $_data),
+        'msg' => 'queue_cat_success'
+      );
+      return $body;
+    break;
+    case 'edit':
+      $cmd_map = array(
+        'hold' => 'exec.hold-queue',
+        'unhold' => 'exec.unhold-queue',
+        'deliver' => 'exec.deliver-now'
+      );
+      if (isset($cmd_map[$_data['action']])) {
+        $qids = isset($_data['qid']) && is_array($_data['qid']) ? $_data['qid'] : array($_data['qid']);
+        $ok_count = 0;
+        $failed = 0;
+        foreach ($qids as $qid) {
+          $agg = agent('request_all', 'postfix', $cmd_map[$_data['action']], array('queue_id' => $qid), 10);
+          if (agent('ok', $agg)) {
+            $ok_count++;
+          }
+          else {
+            $failed++;
+          }
+        }
+        $ok = ($ok_count > 0 && $failed === 0);
+        $_SESSION['return'][] = array(
+          'type' => $ok ? 'success' : 'danger',
+          'log' => array(__FUNCTION__, $_action, $_data),
+          'msg' => $ok ? 'queue_command_success' : 'queue_command_failed'
+        );
+        return $ok;
+      }
+      if ($_data['action'] == 'flush') {
+        $agg = agent('request_all', 'postfix', 'exec.flush-queue', array(), 30);
+        $ok = agent('ok', $agg);
+        $_SESSION['return'][] = array(
+          'type' => $ok ? 'success' : 'danger',
+          'log' => array(__FUNCTION__, $_action, $_data),
+          'msg' => $ok ? 'queue_command_success' : 'queue_command_failed'
+        );
+        return $ok;
+      }
+      if ($_data['action'] == 'super_delete') {
+        $agg = agent('request_all', 'postfix', 'exec.super-delete', array(), 30);
+        $ok = agent('ok', $agg);
+        $_SESSION['return'][] = array(
+          'type' => $ok ? 'success' : 'danger',
+          'log' => array(__FUNCTION__, $_action, $_data),
+          'msg' => $ok ? 'queue_command_success' : 'queue_command_failed'
+        );
+        return $ok;
+      }
+    break;
+  }
+}

data/web/inc/prerequisites.inc.php

@@ -105,14 +105,6 @@ http_response_code(500);
 <?php
 exit;
 }
-// Stop when dockerapi is not available
-if (fsockopen("tcp://dockerapi", 443, $errno, $errstr) === false) {
-  http_response_code(500);
-?>
-<center style='font-family:sans-serif;'>Connection to dockerapi container failed.<br /><br />The following error was reported:<br/><?=$errno;?> - <?=$errstr;?></center>
-<?php
-exit;
-}
 
 // OAuth2
 class mailcowPdo extends OAuth2\Storage\Pdo {
@@ -280,7 +272,7 @@ require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.admin.inc.php';
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.app_passwd.inc.php';
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.customize.inc.php';
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.dkim.inc.php';
-require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.docker.inc.php';
+require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.agent.inc.php';
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.domain_admin.inc.php';
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.fail2ban.inc.php';
 require_once $_SERVER['DOCUMENT_ROOT'] . '/inc/functions.fwdhost.inc.php';