[Dovecot][Netfilter] Report failed logins from mailcowauth.php, anchor default rules

data/Dockerfiles/netfilter/modules/BanManager.py

@@ -7,9 +7,6 @@ class BanManager:
   and the allowlist. Every firewall change runs under lock, which also guards the other firewall work of netfilter
   (chains, SNAT), because the backends must not run in parallel."""
 
-  # Seconds failed logins of an address are not counted after the auth backend was unavailable for it
-  EXCUSE_TIME = 60
-
   def __init__(self, r, tables, options, logger, lock, clock=time.time):
     self.r = r
     self.tables = tables
@@ -21,16 +18,6 @@ class BanManager:
     self.banned = set()
     self.denied = {}
     self.allowlist = []
-    self.excused = {}
-
-  def excuse(self, ip):
-    """The auth backend could not check a login of ip: its failed logins are not counted for EXCUSE_TIME"""
-    with self.lock:
-      now = self.clock()
-      if self.excused.get(ip, 0) <= now:
-        self.logger.logWarn('Auth backend unavailable for %s, its failed logins of the next %d seconds are not counted' % (
-          ip, self.EXCUSE_TIME))
-      self.excused[ip] = now + self.EXCUSE_TIME
 
   def attempt(self, ip):
     """Counts a failed attempt of ip and bans its network after max_attempts within retry_window"""
@@ -43,9 +30,6 @@ class BanManager:
       if allowed is not None:
         self.logger.logInfo('Address %s is allowlisted by rule %s' % (ip, allowed))
         return
-      if self.excused.get(ip, 0) > now:
-        self.logger.logInfo('Not counting the failed login of %s, the auth backend was unavailable' % ip)
-        return
       if net in self.banned:
         # log lines of connections from before the ban
         return
@@ -208,8 +192,6 @@ class BanManager:
       for net in [net for net, counter in self.counters.items()
                   if net not in self.banned and counter['ban_counter'] == 0 and now - counter['last_attempt'] > window]:
         del self.counters[net]
-      for ip in [ip for ip, until in self.excused.items() if until <= now]:
-        del self.excused[ip]
 
   def apply(self, nets):
     for net in self.tables.ban(nets):

data/Dockerfiles/netfilter/modules/Netfilter.py

@@ -109,12 +109,6 @@ class Netfilter:
   def handle_line(self, line):
     # a broken line must not end the watcher
     try:
-      excused = Regexes.temporary_failure(line)
-      if excused is not None:
-        ip = Regexes.parse_ip(excused)
-        if self.external(ip):
-          self.bans.excuse(ip)
-        return
       result = self.regexes.match(line)
       if result is None:
         return

data/Dockerfiles/netfilter/modules/Regexes.py

@@ -6,20 +6,41 @@ import re
 class Regexes:
   """Rules for log lines (Redis F2B_REGEX): rule id => regex whose first group captures the client address"""
 
+  # Anchored to the line start: text clients control in other lines (login names, SMTP commands) must not match.
+  # Failed IMAP, POP3, Sieve and SMTP logins are reported by mailcowauth.php (rule 3), not by Dovecot or Postfix.
   DEFAULTS = {
-    1: r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)',
-    2: r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)',
-    3: r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+',
-    4: r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+',
-    5: r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+',
-    6: r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)',
-    7: r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)',
-    8: r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked',
-    9: r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+',
+    1: r'^mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)$',
+    2: r'^Rspamd UI: Invalid password by ([0-9a-f\.:]+)$',
+    3: r'^MAILCOWAUTH: Login failed for user .+ with service \w* from IP ([0-9a-f\.:]+)$',
+    4: r'^warning: non-SMTP command from [^\[\]\s]*\[([0-9a-f\.:]+)\]:.+',
+    5: r'^NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+',
+    6: r'^(?:\[\d+\]: )?SOGo\w* Login from \'(?:[0-9a-f\.:]+, )*([0-9a-f\.:]+)\' for user .+ might not have worked',
+    7: r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+',
   }
-  # Dovecot could not ask the auth backend (identity provider down, mailcowauth.php unreachable). Postfix logs
-  # such a temporary failure like a wrong password, so failed logins of the address are not counted for a while.
-  TEMPORARY_FAILURE = re.compile(r'lua\([^,]+,([0-9a-f\.:]+)(?:,<[^>]+>)?\): passdb-lua: Upstream (?:unavailable|unreachable)')
+  # Former default rules (with any rule id) => rule id of their replacement, None: the rule is removed.
+  # Rules the admin changed are kept.
+  FORMER_DEFAULTS = {
+    r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)': 1,
+    r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)': 2,
+    r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed': 3,
+    r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+': 3,
+    r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+': 4,
+    r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+': 5,
+    r"SOGo.+ Login from '([0-9a-f\.:]+)' for user .+ might not have worked": 6,
+    r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked': 6,
+    r'-login: Disconnected \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),': None,
+    r'-login: Disconnected.+ \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),': None,
+    r'-login: Aborted login \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None,
+    r'-login: Aborted login.+ \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None,
+    r'-login: Aborted login \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None,
+    r'-login: Aborted login.+ \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+': None,
+    r'auth: static\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)': None,
+    r'auth: \w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)': None,
+    r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): Password mismatch \(SHA1 of given password: [a-f0-9]+\)': None,
+    r'\w+\([^,]+,([0-9a-f\.:]+),<[^>]+>\): unknown user \(SHA1 of given password: [a-f0-9]+\)': None,
+  }
+  # Lines of the web UI are not sanitized like syslog lines: a newline in a login name must not hide the address
+  CONTROL_CHARS = re.compile(r'[\x00-\x1f\x7f]')
 
   def __init__(self, logger):
     self.logger = logger
@@ -42,6 +63,10 @@ class Regexes:
     if not isinstance(stored, dict):
       self.logger.logWarn('F2B_REGEX is no JSON object, keeping the current rules')
       return
+    migrated = self.migrate(stored)
+    if migrated != stored:
+      self.store_migrated(r)
+      stored = migrated
 
     rules = []
     for rule_id, regex in sorted(stored.items(), key=lambda item: self.sort_key(item[0])):
@@ -54,6 +79,7 @@ class Regexes:
 
   def match(self, line):
     """(rule id, captured address) of the first rule that matches the line, None if no rule matches"""
+    line = self.CONTROL_CHARS.sub('_', line)
     for rule_id, regex in self.rules:
       result = regex.search(line)
       if result and result.lastindex and result.group(1):
@@ -61,10 +87,35 @@ class Regexes:
     return None
 
   @classmethod
-  def temporary_failure(cls, line):
-    """Address of a login that failed because the auth backend was unavailable, None for other lines"""
-    result = cls.TEMPORARY_FAILURE.search(line)
-    return result.group(1) if result else None
+  def migrate(cls, stored):
+    """The stored rules with former default rules replaced by the current ones (or removed)"""
+    migrated = {}
+    for rule_id, regex in stored.items():
+      if isinstance(regex, str) and regex in cls.FORMER_DEFAULTS:
+        replacement = cls.FORMER_DEFAULTS[regex]
+        if replacement is None:
+          continue
+        regex = cls.DEFAULTS[replacement]
+      migrated[rule_id] = regex
+    return migrated
+
+  def store_migrated(self, r):
+    # WATCH keeps a save of the UI between reading and writing
+    def replace(pipe):
+      current = json.loads(pipe.get('F2B_REGEX') or '{}')
+      if not isinstance(current, dict):
+        return
+      migrated = self.migrate(current)
+      if migrated == current:
+        return
+      pipe.multi()
+      pipe.set('F2B_REGEX', json.dumps(migrated, ensure_ascii=False))
+
+    try:
+      r.transaction(replace, 'F2B_REGEX')
+      self.logger.logInfo('Replaced former default rules in F2B_REGEX')
+    except Exception as ex:
+      self.logger.logWarn('Could not replace former default rules in F2B_REGEX: %s' % ex)
 
   @staticmethod
   def parse_ip(text):