diff --git a/data/Dockerfiles/netfilter/main.py b/data/Dockerfiles/netfilter/main.py index 75768009b..d5c1db502 100644 --- a/data/Dockerfiles/netfilter/main.py +++ b/data/Dockerfiles/netfilter/main.py @@ -1,463 +1,469 @@ -#!/usr/bin/env python3 - -import re -import os -import sys -import time -import atexit -import signal -import ipaddress -from collections import Counter -from random import randint -from threading import Thread -from threading import Lock -import redis -import json -import dns.resolver -import dns.exception -from modules.Logger import Logger -from modules.IPTables import IPTables -from modules.NFTables import NFTables - - -# connect to redis -while True: - try: - redis_slaveof_ip = os.getenv('REDIS_SLAVEOF_IP', '') - redis_slaveof_port = os.getenv('REDIS_SLAVEOF_PORT', '') - if "".__eq__(redis_slaveof_ip): - r = redis.StrictRedis(host=os.getenv('IPV4_NETWORK', '172.22.1') + '.249', decode_responses=True, port=6379, db=0) - else: - r = redis.StrictRedis(host=redis_slaveof_ip, decode_responses=True, port=redis_slaveof_port, db=0) - r.ping() - except Exception as ex: - print('%s - trying again in 3 seconds' % (ex)) - time.sleep(3) - else: - break -pubsub = r.pubsub() - -# rename fail2ban to netfilter -if r.exists('F2B_LOG'): - r.rename('F2B_LOG', 'NETFILTER_LOG') - - -# globals -WHITELIST = [] -BLACKLIST= [] -bans = {} -quit_now = False -exit_code = 0 -lock = Lock() - - -# init Logger -logger = Logger(r) -# init backend -backend = sys.argv[1] -if backend == "nftables": - logger.logInfo('Using NFTables backend') - tables = NFTables("MAILCOW", logger) -else: - logger.logInfo('Using IPTables backend') - tables = IPTables("MAILCOW", logger) - - -def refreshF2boptions(): - global f2boptions - global quit_now - global exit_code - - f2boptions = {} - - if not r.get('F2B_OPTIONS'): - f2boptions['ban_time'] = r.get('F2B_BAN_TIME') - f2boptions['max_ban_time'] = r.get('F2B_MAX_BAN_TIME') - f2boptions['ban_time_increment'] = r.get('F2B_BAN_TIME_INCREMENT') - f2boptions['max_attempts'] = r.get('F2B_MAX_ATTEMPTS') - f2boptions['retry_window'] = r.get('F2B_RETRY_WINDOW') - f2boptions['netban_ipv4'] = r.get('F2B_NETBAN_IPV4') - f2boptions['netban_ipv6'] = r.get('F2B_NETBAN_IPV6') - else: - try: - f2boptions = json.loads(r.get('F2B_OPTIONS')) - except ValueError: - logger.logCrit('Error loading F2B options: F2B_OPTIONS is not json') - quit_now = True - exit_code = 2 - - verifyF2boptions(f2boptions) - r.set('F2B_OPTIONS', json.dumps(f2boptions, ensure_ascii=False)) - -def verifyF2boptions(f2boptions): - verifyF2boption(f2boptions,'ban_time', 1800) - verifyF2boption(f2boptions,'max_ban_time', 10000) - verifyF2boption(f2boptions,'ban_time_increment', True) - verifyF2boption(f2boptions,'max_attempts', 10) - verifyF2boption(f2boptions,'retry_window', 600) - verifyF2boption(f2boptions,'netban_ipv4', 32) - verifyF2boption(f2boptions,'netban_ipv6', 128) - -def verifyF2boption(f2boptions, f2boption, f2bdefault): - f2boptions[f2boption] = f2boptions[f2boption] if f2boption in f2boptions and f2boptions[f2boption] is not None else f2bdefault - -def refreshF2bregex(): - global f2bregex - global quit_now - global exit_code - if not r.get('F2B_REGEX'): - f2bregex = {} - f2bregex[1] = 'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)' - f2bregex[2] = 'Rspamd UI: Invalid password by ([0-9a-f\.:]+)' - f2bregex[3] = 'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+' - f2bregex[4] = 'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+' - f2bregex[5] = 'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+' - f2bregex[6] = '-login: Disconnected.+ \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),' - f2bregex[7] = '-login: Aborted login.+ \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+' - f2bregex[8] = '-login: Aborted login.+ \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+' - f2bregex[9] = 'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked' - f2bregex[10] = '([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+' - r.set('F2B_REGEX', json.dumps(f2bregex, ensure_ascii=False)) - else: - try: - f2bregex = {} - f2bregex = json.loads(r.get('F2B_REGEX')) - except ValueError: - logger.logCrit('Error loading F2B options: F2B_REGEX is not json') - quit_now = True - exit_code = 2 - -def get_ip(address): - ip = ipaddress.ip_address(address) - if type(ip) is ipaddress.IPv6Address and ip.ipv4_mapped: - ip = ip.ipv4_mapped - if ip.is_private or ip.is_loopback: - return False - - return ip - -def ban(address): - global lock - - refreshF2boptions() - BAN_TIME = int(f2boptions['ban_time']) - BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment']) - MAX_ATTEMPTS = int(f2boptions['max_attempts']) - RETRY_WINDOW = int(f2boptions['retry_window']) - NETBAN_IPV4 = '/' + str(f2boptions['netban_ipv4']) - NETBAN_IPV6 = '/' + str(f2boptions['netban_ipv6']) - - ip = get_ip(address) - if not ip: return - address = str(ip) - self_network = ipaddress.ip_network(address) - - with lock: - temp_whitelist = set(WHITELIST) - if temp_whitelist: - for wl_key in temp_whitelist: - wl_net = ipaddress.ip_network(wl_key, False) - if wl_net.overlaps(self_network): - logger.logInfo('Address %s is whitelisted by rule %s' % (self_network, wl_net)) - return - - net = ipaddress.ip_network((address + (NETBAN_IPV4 if type(ip) is ipaddress.IPv4Address else NETBAN_IPV6)), strict=False) - net = str(net) - - if not net in bans: - bans[net] = {'attempts': 0, 'last_attempt': 0, 'ban_counter': 0} - - current_attempt = time.time() - if current_attempt - bans[net]['last_attempt'] > RETRY_WINDOW: - bans[net]['attempts'] = 0 - - bans[net]['attempts'] += 1 - bans[net]['last_attempt'] = current_attempt - - if bans[net]['attempts'] >= MAX_ATTEMPTS: - cur_time = int(round(time.time())) - NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** bans[net]['ban_counter'] - logger.logCrit('Banning %s for %d minutes' % (net, NET_BAN_TIME / 60 )) - if type(ip) is ipaddress.IPv4Address: - with lock: - tables.banIPv4(net) - else: - with lock: - tables.banIPv6(net) - - r.hset('F2B_ACTIVE_BANS', '%s' % net, cur_time + NET_BAN_TIME) - else: - logger.logWarn('%d more attempts in the next %d seconds until %s is banned' % (MAX_ATTEMPTS - bans[net]['attempts'], RETRY_WINDOW, net)) - -def unban(net): - global lock - - if not net in bans: - logger.logInfo('%s is not banned, skipping unban and deleting from queue (if any)' % net) - r.hdel('F2B_QUEUE_UNBAN', '%s' % net) - return - - logger.logInfo('Unbanning %s' % net) - if type(ipaddress.ip_network(net)) is ipaddress.IPv4Network: - with lock: - tables.unbanIPv4(net) - else: - with lock: - tables.unbanIPv6(net) - - r.hdel('F2B_ACTIVE_BANS', '%s' % net) - r.hdel('F2B_QUEUE_UNBAN', '%s' % net) - if net in bans: - bans[net]['attempts'] = 0 - bans[net]['ban_counter'] += 1 - -def permBan(net, unban=False): - global lock - - is_unbanned = False - is_banned = False - if type(ipaddress.ip_network(net, strict=False)) is ipaddress.IPv4Network: - with lock: - if unban: - is_unbanned = tables.unbanIPv4(net) - else: - is_banned = tables.banIPv4(net) - else: - with lock: - if unban: - is_unbanned = tables.unbanIPv6(net) - else: - is_banned = tables.banIPv6(net) - - - if is_unbanned: - r.hdel('F2B_PERM_BANS', '%s' % net) - logger.logCrit('Removed host/network %s from blacklist' % net) - elif is_banned: - r.hset('F2B_PERM_BANS', '%s' % net, int(round(time.time()))) - logger.logCrit('Added host/network %s to blacklist' % net) - -def clear(): - global lock - logger.logInfo('Clearing all bans') - for net in bans.copy(): - unban(net) - with lock: - tables.clearIPv4Table() - tables.clearIPv6Table() - r.delete('F2B_ACTIVE_BANS') - r.delete('F2B_PERM_BANS') - pubsub.unsubscribe() - -def watch(): - logger.logInfo('Watching Redis channel F2B_CHANNEL') - pubsub.subscribe('F2B_CHANNEL') - - global quit_now - global exit_code - - while not quit_now: - try: - for item in pubsub.listen(): - refreshF2bregex() - for rule_id, rule_regex in f2bregex.items(): - if item['data'] and item['type'] == 'message': - try: - result = re.search(rule_regex, item['data']) - except re.error: - result = False - if result: - addr = result.group(1) - ip = ipaddress.ip_address(addr) - if ip.is_private or ip.is_loopback: - continue - logger.logWarn('%s matched rule id %s (%s)' % (addr, rule_id, item['data'])) - ban(addr) - except Exception as ex: - logger.logWarn('Error reading log line from pubsub: %s' % ex) - quit_now = True - exit_code = 2 - -def snat4(snat_target): - global lock - global quit_now - - while not quit_now: - time.sleep(10) - with lock: - tables.snat4(snat_target, os.getenv('IPV4_NETWORK', '172.22.1') + '.0/24') - -def snat6(snat_target): - global lock - global quit_now - - while not quit_now: - time.sleep(10) - with lock: - tables.snat6(snat_target, os.getenv('IPV6_NETWORK', 'fd4d:6169:6c63:6f77::/64')) - -def autopurge(): - while not quit_now: - time.sleep(10) - refreshF2boptions() - BAN_TIME = int(f2boptions['ban_time']) - MAX_BAN_TIME = int(f2boptions['max_ban_time']) - BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment']) - MAX_ATTEMPTS = int(f2boptions['max_attempts']) - QUEUE_UNBAN = r.hgetall('F2B_QUEUE_UNBAN') - if QUEUE_UNBAN: - for net in QUEUE_UNBAN: - unban(str(net)) - for net in bans.copy(): - if bans[net]['attempts'] >= MAX_ATTEMPTS: - NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** bans[net]['ban_counter'] - TIME_SINCE_LAST_ATTEMPT = time.time() - bans[net]['last_attempt'] - if TIME_SINCE_LAST_ATTEMPT > NET_BAN_TIME or TIME_SINCE_LAST_ATTEMPT > MAX_BAN_TIME: - unban(net) - -def mailcowChainOrder(): - global lock - global quit_now - global exit_code - while not quit_now: - time.sleep(10) - with lock: - quit_now, exit_code = tables.checkIPv4ChainOrder() - if quit_now: return - quit_now, exit_code = tables.checkIPv6ChainOrder() - -def isIpNetwork(address): - try: - ipaddress.ip_network(address, False) - except ValueError: - return False - return True - -def genNetworkList(list): - resolver = dns.resolver.Resolver() - hostnames = [] - networks = [] - for key in list: - if isIpNetwork(key): - networks.append(key) - else: - hostnames.append(key) - for hostname in hostnames: - hostname_ips = [] - for rdtype in ['A', 'AAAA']: - try: - answer = resolver.resolve(qname=hostname, rdtype=rdtype, lifetime=3) - except dns.exception.Timeout: - logger.logInfo('Hostname %s timedout on resolve' % hostname) - break - except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): - continue - except dns.exception.DNSException as dnsexception: - logger.logInfo('%s' % dnsexception) - continue - for rdata in answer: - hostname_ips.append(rdata.to_text()) - networks.extend(hostname_ips) - return set(networks) - -def whitelistUpdate(): - global lock - global quit_now - global WHITELIST - while not quit_now: - start_time = time.time() - list = r.hgetall('F2B_WHITELIST') - new_whitelist = [] - if list: - new_whitelist = genNetworkList(list) - with lock: - if Counter(new_whitelist) != Counter(WHITELIST): - WHITELIST = new_whitelist - logger.logInfo('Whitelist was changed, it has %s entries' % len(WHITELIST)) - time.sleep(60.0 - ((time.time() - start_time) % 60.0)) - -def blacklistUpdate(): - global quit_now - global BLACKLIST - while not quit_now: - start_time = time.time() - list = r.hgetall('F2B_BLACKLIST') - new_blacklist = [] - if list: - new_blacklist = genNetworkList(list) - if Counter(new_blacklist) != Counter(BLACKLIST): - addban = set(new_blacklist).difference(BLACKLIST) - delban = set(BLACKLIST).difference(new_blacklist) - BLACKLIST = new_blacklist - logger.logInfo('Blacklist was changed, it has %s entries' % len(BLACKLIST)) - if addban: - for net in addban: - permBan(net=net) - if delban: - for net in delban: - permBan(net=net, unban=True) - time.sleep(60.0 - ((time.time() - start_time) % 60.0)) - -def quit(signum, frame): - global quit_now - quit_now = True - - -if __name__ == '__main__': - # In case a previous session was killed without cleanup - clear() - # Reinit MAILCOW chain - # Is called before threads start, no locking - logger.logInfo("Initializing mailcow netfilter chain") - tables.initChainIPv4() - tables.initChainIPv6() - - watch_thread = Thread(target=watch) - watch_thread.daemon = True - watch_thread.start() - - if os.getenv('SNAT_TO_SOURCE') and os.getenv('SNAT_TO_SOURCE') != 'n': - try: - snat_ip = os.getenv('SNAT_TO_SOURCE') - snat_ipo = ipaddress.ip_address(snat_ip) - if type(snat_ipo) is ipaddress.IPv4Address: - snat4_thread = Thread(target=snat4,args=(snat_ip,)) - snat4_thread.daemon = True - snat4_thread.start() - except ValueError: - print(os.getenv('SNAT_TO_SOURCE') + ' is not a valid IPv4 address') - - if os.getenv('SNAT6_TO_SOURCE') and os.getenv('SNAT6_TO_SOURCE') != 'n': - try: - snat_ip = os.getenv('SNAT6_TO_SOURCE') - snat_ipo = ipaddress.ip_address(snat_ip) - if type(snat_ipo) is ipaddress.IPv6Address: - snat6_thread = Thread(target=snat6,args=(snat_ip,)) - snat6_thread.daemon = True - snat6_thread.start() - except ValueError: - print(os.getenv('SNAT6_TO_SOURCE') + ' is not a valid IPv6 address') - - autopurge_thread = Thread(target=autopurge) - autopurge_thread.daemon = True - autopurge_thread.start() - - mailcowchainwatch_thread = Thread(target=mailcowChainOrder) - mailcowchainwatch_thread.daemon = True - mailcowchainwatch_thread.start() - - blacklistupdate_thread = Thread(target=blacklistUpdate) - blacklistupdate_thread.daemon = True - blacklistupdate_thread.start() - - whitelistupdate_thread = Thread(target=whitelistUpdate) - whitelistupdate_thread.daemon = True - whitelistupdate_thread.start() - - signal.signal(signal.SIGTERM, quit) - atexit.register(clear) - - while not quit_now: - time.sleep(0.5) - - sys.exit(exit_code) +#!/usr/bin/env python3 + +import re +import os +import sys +import time +import atexit +import signal +import ipaddress +from collections import Counter +from random import randint +from threading import Thread +from threading import Lock +import redis +import json +import dns.resolver +import dns.exception +import uuid +from modules.Logger import Logger +from modules.IPTables import IPTables +from modules.NFTables import NFTables + + +# connect to redis +while True: + try: + redis_slaveof_ip = os.getenv('REDIS_SLAVEOF_IP', '') + redis_slaveof_port = os.getenv('REDIS_SLAVEOF_PORT', '') + if "".__eq__(redis_slaveof_ip): + r = redis.StrictRedis(host=os.getenv('IPV4_NETWORK', '172.22.1') + '.249', decode_responses=True, port=6379, db=0) + else: + r = redis.StrictRedis(host=redis_slaveof_ip, decode_responses=True, port=redis_slaveof_port, db=0) + r.ping() + except Exception as ex: + print('%s - trying again in 3 seconds' % (ex)) + time.sleep(3) + else: + break +pubsub = r.pubsub() + +# rename fail2ban to netfilter +if r.exists('F2B_LOG'): + r.rename('F2B_LOG', 'NETFILTER_LOG') + + +# globals +WHITELIST = [] +BLACKLIST= [] +bans = {} +quit_now = False +exit_code = 0 +lock = Lock() + + +# init Logger +logger = Logger(r) +# init backend +backend = sys.argv[1] +if backend == "nftables": + logger.logInfo('Using NFTables backend') + tables = NFTables("MAILCOW", logger) +else: + logger.logInfo('Using IPTables backend') + tables = IPTables("MAILCOW", logger) + + +def refreshF2boptions(): + global f2boptions + global quit_now + global exit_code + + f2boptions = {} + + if not r.get('F2B_OPTIONS'): + f2boptions['ban_time'] = r.get('F2B_BAN_TIME') + f2boptions['max_ban_time'] = r.get('F2B_MAX_BAN_TIME') + f2boptions['ban_time_increment'] = r.get('F2B_BAN_TIME_INCREMENT') + f2boptions['max_attempts'] = r.get('F2B_MAX_ATTEMPTS') + f2boptions['retry_window'] = r.get('F2B_RETRY_WINDOW') + f2boptions['netban_ipv4'] = r.get('F2B_NETBAN_IPV4') + f2boptions['netban_ipv6'] = r.get('F2B_NETBAN_IPV6') + else: + try: + f2boptions = json.loads(r.get('F2B_OPTIONS')) + except ValueError: + logger.logCrit('Error loading F2B options: F2B_OPTIONS is not json') + quit_now = True + exit_code = 2 + + verifyF2boptions(f2boptions) + r.set('F2B_OPTIONS', json.dumps(f2boptions, ensure_ascii=False)) + +def verifyF2boptions(f2boptions): + verifyF2boption(f2boptions,'ban_time', 1800) + verifyF2boption(f2boptions,'max_ban_time', 10000) + verifyF2boption(f2boptions,'ban_time_increment', True) + verifyF2boption(f2boptions,'max_attempts', 10) + verifyF2boption(f2boptions,'retry_window', 600) + verifyF2boption(f2boptions,'netban_ipv4', 32) + verifyF2boption(f2boptions,'netban_ipv6', 128) + verifyF2boption(f2boptions,'banlist_id', str(uuid.uuid4())) + verifyF2boption(f2boptions,'manage_external', 0) + +def verifyF2boption(f2boptions, f2boption, f2bdefault): + f2boptions[f2boption] = f2boptions[f2boption] if f2boption in f2boptions and f2boptions[f2boption] is not None else f2bdefault + +def refreshF2bregex(): + global f2bregex + global quit_now + global exit_code + if not r.get('F2B_REGEX'): + f2bregex = {} + f2bregex[1] = 'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)' + f2bregex[2] = 'Rspamd UI: Invalid password by ([0-9a-f\.:]+)' + f2bregex[3] = 'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+' + f2bregex[4] = 'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+' + f2bregex[5] = 'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+' + f2bregex[6] = '-login: Disconnected.+ \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),' + f2bregex[7] = '-login: Aborted login.+ \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+' + f2bregex[8] = '-login: Aborted login.+ \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+' + f2bregex[9] = 'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked' + f2bregex[10] = '([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+' + r.set('F2B_REGEX', json.dumps(f2bregex, ensure_ascii=False)) + else: + try: + f2bregex = {} + f2bregex = json.loads(r.get('F2B_REGEX')) + except ValueError: + logger.logCrit('Error loading F2B options: F2B_REGEX is not json') + quit_now = True + exit_code = 2 + +def get_ip(address): + ip = ipaddress.ip_address(address) + if type(ip) is ipaddress.IPv6Address and ip.ipv4_mapped: + ip = ip.ipv4_mapped + if ip.is_private or ip.is_loopback: + return False + + return ip + +def ban(address): + global f2boptions + global lock + + refreshF2boptions() + BAN_TIME = int(f2boptions['ban_time']) + BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment']) + MAX_ATTEMPTS = int(f2boptions['max_attempts']) + RETRY_WINDOW = int(f2boptions['retry_window']) + NETBAN_IPV4 = '/' + str(f2boptions['netban_ipv4']) + NETBAN_IPV6 = '/' + str(f2boptions['netban_ipv6']) + + ip = get_ip(address) + if not ip: return + address = str(ip) + self_network = ipaddress.ip_network(address) + + with lock: + temp_whitelist = set(WHITELIST) + if temp_whitelist: + for wl_key in temp_whitelist: + wl_net = ipaddress.ip_network(wl_key, False) + if wl_net.overlaps(self_network): + logger.logInfo('Address %s is whitelisted by rule %s' % (self_network, wl_net)) + return + + net = ipaddress.ip_network((address + (NETBAN_IPV4 if type(ip) is ipaddress.IPv4Address else NETBAN_IPV6)), strict=False) + net = str(net) + + if not net in bans: + bans[net] = {'attempts': 0, 'last_attempt': 0, 'ban_counter': 0} + + current_attempt = time.time() + if current_attempt - bans[net]['last_attempt'] > RETRY_WINDOW: + bans[net]['attempts'] = 0 + + bans[net]['attempts'] += 1 + bans[net]['last_attempt'] = current_attempt + + if bans[net]['attempts'] >= MAX_ATTEMPTS: + cur_time = int(round(time.time())) + NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** bans[net]['ban_counter'] + logger.logCrit('Banning %s for %d minutes' % (net, NET_BAN_TIME / 60 )) + if type(ip) is ipaddress.IPv4Address and int(f2boptions['manage_external']) != 1: + with lock: + tables.banIPv4(net) + elif int(f2boptions['manage_external']) != 1: + with lock: + tables.banIPv6(net) + + r.hset('F2B_ACTIVE_BANS', '%s' % net, cur_time + NET_BAN_TIME) + else: + logger.logWarn('%d more attempts in the next %d seconds until %s is banned' % (MAX_ATTEMPTS - bans[net]['attempts'], RETRY_WINDOW, net)) + +def unban(net): + global lock + + if not net in bans: + logger.logInfo('%s is not banned, skipping unban and deleting from queue (if any)' % net) + r.hdel('F2B_QUEUE_UNBAN', '%s' % net) + return + + logger.logInfo('Unbanning %s' % net) + if type(ipaddress.ip_network(net)) is ipaddress.IPv4Network: + with lock: + tables.unbanIPv4(net) + else: + with lock: + tables.unbanIPv6(net) + + r.hdel('F2B_ACTIVE_BANS', '%s' % net) + r.hdel('F2B_QUEUE_UNBAN', '%s' % net) + if net in bans: + bans[net]['attempts'] = 0 + bans[net]['ban_counter'] += 1 + +def permBan(net, unban=False): + global f2boptions + global lock + + is_unbanned = False + is_banned = False + if type(ipaddress.ip_network(net, strict=False)) is ipaddress.IPv4Network: + with lock: + if unban: + is_unbanned = tables.unbanIPv4(net) + elif int(f2boptions['manage_external']) != 1: + is_banned = tables.banIPv4(net) + else: + with lock: + if unban: + is_unbanned = tables.unbanIPv6(net) + elif int(f2boptions['manage_external']) != 1: + is_banned = tables.banIPv6(net) + + + if is_unbanned: + r.hdel('F2B_PERM_BANS', '%s' % net) + logger.logCrit('Removed host/network %s from blacklist' % net) + elif is_banned: + r.hset('F2B_PERM_BANS', '%s' % net, int(round(time.time()))) + logger.logCrit('Added host/network %s to blacklist' % net) + +def clear(): + global lock + logger.logInfo('Clearing all bans') + for net in bans.copy(): + unban(net) + with lock: + tables.clearIPv4Table() + tables.clearIPv6Table() + r.delete('F2B_ACTIVE_BANS') + r.delete('F2B_PERM_BANS') + pubsub.unsubscribe() + +def watch(): + logger.logInfo('Watching Redis channel F2B_CHANNEL') + pubsub.subscribe('F2B_CHANNEL') + + global quit_now + global exit_code + + while not quit_now: + try: + for item in pubsub.listen(): + refreshF2bregex() + for rule_id, rule_regex in f2bregex.items(): + if item['data'] and item['type'] == 'message': + try: + result = re.search(rule_regex, item['data']) + except re.error: + result = False + if result: + addr = result.group(1) + ip = ipaddress.ip_address(addr) + if ip.is_private or ip.is_loopback: + continue + logger.logWarn('%s matched rule id %s (%s)' % (addr, rule_id, item['data'])) + ban(addr) + except Exception as ex: + logger.logWarn('Error reading log line from pubsub: %s' % ex) + quit_now = True + exit_code = 2 + +def snat4(snat_target): + global lock + global quit_now + + while not quit_now: + time.sleep(10) + with lock: + tables.snat4(snat_target, os.getenv('IPV4_NETWORK', '172.22.1') + '.0/24') + +def snat6(snat_target): + global lock + global quit_now + + while not quit_now: + time.sleep(10) + with lock: + tables.snat6(snat_target, os.getenv('IPV6_NETWORK', 'fd4d:6169:6c63:6f77::/64')) + +def autopurge(): + while not quit_now: + time.sleep(10) + refreshF2boptions() + BAN_TIME = int(f2boptions['ban_time']) + MAX_BAN_TIME = int(f2boptions['max_ban_time']) + BAN_TIME_INCREMENT = bool(f2boptions['ban_time_increment']) + MAX_ATTEMPTS = int(f2boptions['max_attempts']) + QUEUE_UNBAN = r.hgetall('F2B_QUEUE_UNBAN') + if QUEUE_UNBAN: + for net in QUEUE_UNBAN: + unban(str(net)) + for net in bans.copy(): + if bans[net]['attempts'] >= MAX_ATTEMPTS: + NET_BAN_TIME = BAN_TIME if not BAN_TIME_INCREMENT else BAN_TIME * 2 ** bans[net]['ban_counter'] + TIME_SINCE_LAST_ATTEMPT = time.time() - bans[net]['last_attempt'] + if TIME_SINCE_LAST_ATTEMPT > NET_BAN_TIME or TIME_SINCE_LAST_ATTEMPT > MAX_BAN_TIME: + unban(net) + +def mailcowChainOrder(): + global lock + global quit_now + global exit_code + while not quit_now: + time.sleep(10) + with lock: + quit_now, exit_code = tables.checkIPv4ChainOrder() + if quit_now: return + quit_now, exit_code = tables.checkIPv6ChainOrder() + +def isIpNetwork(address): + try: + ipaddress.ip_network(address, False) + except ValueError: + return False + return True + +def genNetworkList(list): + resolver = dns.resolver.Resolver() + hostnames = [] + networks = [] + for key in list: + if isIpNetwork(key): + networks.append(key) + else: + hostnames.append(key) + for hostname in hostnames: + hostname_ips = [] + for rdtype in ['A', 'AAAA']: + try: + answer = resolver.resolve(qname=hostname, rdtype=rdtype, lifetime=3) + except dns.exception.Timeout: + logger.logInfo('Hostname %s timedout on resolve' % hostname) + break + except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer): + continue + except dns.exception.DNSException as dnsexception: + logger.logInfo('%s' % dnsexception) + continue + for rdata in answer: + hostname_ips.append(rdata.to_text()) + networks.extend(hostname_ips) + return set(networks) + +def whitelistUpdate(): + global lock + global quit_now + global WHITELIST + while not quit_now: + start_time = time.time() + list = r.hgetall('F2B_WHITELIST') + new_whitelist = [] + if list: + new_whitelist = genNetworkList(list) + with lock: + if Counter(new_whitelist) != Counter(WHITELIST): + WHITELIST = new_whitelist + logger.logInfo('Whitelist was changed, it has %s entries' % len(WHITELIST)) + time.sleep(60.0 - ((time.time() - start_time) % 60.0)) + +def blacklistUpdate(): + global quit_now + global BLACKLIST + while not quit_now: + start_time = time.time() + list = r.hgetall('F2B_BLACKLIST') + new_blacklist = [] + if list: + new_blacklist = genNetworkList(list) + if Counter(new_blacklist) != Counter(BLACKLIST): + addban = set(new_blacklist).difference(BLACKLIST) + delban = set(BLACKLIST).difference(new_blacklist) + BLACKLIST = new_blacklist + logger.logInfo('Blacklist was changed, it has %s entries' % len(BLACKLIST)) + if addban: + for net in addban: + permBan(net=net) + if delban: + for net in delban: + permBan(net=net, unban=True) + time.sleep(60.0 - ((time.time() - start_time) % 60.0)) + +def quit(signum, frame): + global quit_now + quit_now = True + + +if __name__ == '__main__': + refreshF2boptions() + # In case a previous session was killed without cleanup + clear() + # Reinit MAILCOW chain + # Is called before threads start, no locking + logger.logInfo("Initializing mailcow netfilter chain") + tables.initChainIPv4() + tables.initChainIPv6() + + watch_thread = Thread(target=watch) + watch_thread.daemon = True + watch_thread.start() + + if os.getenv('SNAT_TO_SOURCE') and os.getenv('SNAT_TO_SOURCE') != 'n': + try: + snat_ip = os.getenv('SNAT_TO_SOURCE') + snat_ipo = ipaddress.ip_address(snat_ip) + if type(snat_ipo) is ipaddress.IPv4Address: + snat4_thread = Thread(target=snat4,args=(snat_ip,)) + snat4_thread.daemon = True + snat4_thread.start() + except ValueError: + print(os.getenv('SNAT_TO_SOURCE') + ' is not a valid IPv4 address') + + if os.getenv('SNAT6_TO_SOURCE') and os.getenv('SNAT6_TO_SOURCE') != 'n': + try: + snat_ip = os.getenv('SNAT6_TO_SOURCE') + snat_ipo = ipaddress.ip_address(snat_ip) + if type(snat_ipo) is ipaddress.IPv6Address: + snat6_thread = Thread(target=snat6,args=(snat_ip,)) + snat6_thread.daemon = True + snat6_thread.start() + except ValueError: + print(os.getenv('SNAT6_TO_SOURCE') + ' is not a valid IPv6 address') + + autopurge_thread = Thread(target=autopurge) + autopurge_thread.daemon = True + autopurge_thread.start() + + mailcowchainwatch_thread = Thread(target=mailcowChainOrder) + mailcowchainwatch_thread.daemon = True + mailcowchainwatch_thread.start() + + blacklistupdate_thread = Thread(target=blacklistUpdate) + blacklistupdate_thread.daemon = True + blacklistupdate_thread.start() + + whitelistupdate_thread = Thread(target=whitelistUpdate) + whitelistupdate_thread.daemon = True + whitelistupdate_thread.start() + + signal.signal(signal.SIGTERM, quit) + atexit.register(clear) + + while not quit_now: + time.sleep(0.5) + + sys.exit(exit_code) diff --git a/data/web/admin.php b/data/web/admin.php index ebddb7b91..d0fcbc992 100644 --- a/data/web/admin.php +++ b/data/web/admin.php @@ -85,6 +85,8 @@ $cors_settings = cors('get'); $cors_settings['allowed_origins'] = str_replace(", ", "\n", $cors_settings['allowed_origins']); $cors_settings['allowed_methods'] = explode(", ", $cors_settings['allowed_methods']); +$f2b_data = fail2ban('get'); + $template = 'admin.twig'; $template_data = [ 'tfa_data' => $tfa_data, @@ -101,7 +103,8 @@ $template_data = [ 'domains' => $domains, 'all_domains' => $all_domains, 'mailboxes' => $mailboxes, - 'f2b_data' => fail2ban('get'), + 'f2b_data' => $f2b_data, + 'f2b_banlist_url' => getBaseUrl() . "/api/v1/get/fail2ban/banlist/" . $f2b_data['banlist_id'], 'q_data' => quarantine('settings'), 'qn_data' => quota_notification('get'), 'rsettings_map' => file_get_contents('http://nginx:8081/settings.php'), @@ -113,6 +116,7 @@ $template_data = [ 'password_complexity' => password_complexity('get'), 'show_rspamd_global_filters' => @$_SESSION['show_rspamd_global_filters'], 'cors_settings' => $cors_settings, + 'is_https' => isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on', 'lang_admin' => json_encode($lang['admin']), 'lang_datatables' => json_encode($lang['datatables']) ]; diff --git a/data/web/inc/functions.fail2ban.inc.php b/data/web/inc/functions.fail2ban.inc.php index 2c4aa41d1..5962237fc 100644 --- a/data/web/inc/functions.fail2ban.inc.php +++ b/data/web/inc/functions.fail2ban.inc.php @@ -1,5 +1,5 @@ 128) ? 128 : $netban_ipv6; $f2b_options['max_attempts'] = ($max_attempts < 1) ? 1 : $max_attempts; $f2b_options['retry_window'] = ($retry_window < 1) ? 1 : $retry_window; + $f2b_options['banlist_id'] = $is_now['banlist_id']; + $f2b_options['manage_external'] = ($manage_external > 0) ? 1 : 0; try { $redis->Set('F2B_OPTIONS', json_encode($f2b_options)); $redis->Del('F2B_WHITELIST'); @@ -329,5 +332,71 @@ function fail2ban($_action, $_data = null) { 'msg' => 'f2b_modified' ); break; + case 'banlist': + try { + $f2b_options = json_decode($redis->Get('F2B_OPTIONS'), true); + } + catch (RedisException $e) { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log, $_extra), + 'msg' => array('redis_error', $e) + ); + http_response_code(500); + return false; + } + if (is_array($_extra)) { + $_extra = $_extra[0]; + } + if ($_extra != $f2b_options['banlist_id']){ + http_response_code(404); + return false; + } + + switch ($_data) { + case 'get': + try { + $bl = $redis->hKeys('F2B_BLACKLIST'); + $active_bans = $redis->hKeys('F2B_ACTIVE_BANS'); + } + catch (RedisException $e) { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log, $_extra), + 'msg' => array('redis_error', $e) + ); + http_response_code(500); + return false; + } + $banlist = implode("\n", array_merge($bl, $active_bans)); + return $banlist; + break; + case 'refresh': + if ($_SESSION['mailcow_cc_role'] != "admin") { + return false; + } + + $f2b_options['banlist_id'] = uuid4(); + try { + $redis->Set('F2B_OPTIONS', json_encode($f2b_options)); + } + catch (RedisException $e) { + $_SESSION['return'][] = array( + 'type' => 'danger', + 'log' => array(__FUNCTION__, $_action, $_data_log, $_extra), + 'msg' => array('redis_error', $e) + ); + return false; + } + + $_SESSION['return'][] = array( + 'type' => 'success', + 'log' => array(__FUNCTION__, $_action, $_data_log, $_extra), + 'msg' => 'f2b_banlist_refreshed' + ); + return true; + break; + } + break; } } diff --git a/data/web/inc/functions.inc.php b/data/web/inc/functions.inc.php index 6418945c5..3cff09b95 100644 --- a/data/web/inc/functions.inc.php +++ b/data/web/inc/functions.inc.php @@ -2246,6 +2246,21 @@ function cors($action, $data = null) { break; } } +function getBaseURL() { + $protocol = isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on' ? 'https' : 'http'; + $host = $_SERVER['HTTP_HOST']; + $base_url = $protocol . '://' . $host; + + return $base_url; +} +function uuid4() { + $data = openssl_random_pseudo_bytes(16); + + $data[6] = chr(ord($data[6]) & 0x0f | 0x40); + $data[8] = chr(ord($data[8]) & 0x3f | 0x80); + + return vsprintf('%s%s-%s-%s-%s-%s%s%s', str_split(bin2hex($data), 4)); +} function get_logs($application, $lines = false) { if ($lines === false) { diff --git a/data/web/inc/prerequisites.inc.php b/data/web/inc/prerequisites.inc.php index b3b1cc13d..9c5203e7f 100644 --- a/data/web/inc/prerequisites.inc.php +++ b/data/web/inc/prerequisites.inc.php @@ -70,6 +70,8 @@ try { } } catch (Exception $e) { +// Stop when redis is not available +http_response_code(500); ?>
Connection to Redis failed.

The following error was reported:
getMessage();?>
Connection to database failed.

The following error was reported:
getMessage();?>
Connection to dockerapi container failed.

The following error was reported:
-
getChallenge(); return; + break; + case "fail2ban": + if (!isset($_SESSION['mailcow_cc_role'])){ + switch ($object) { + case 'banlist': + header('Content-Type: text/plain'); + echo fail2ban('banlist', 'get', $extra); + break; + } + } break; } if (isset($_SESSION['mailcow_cc_role'])) { @@ -1324,6 +1334,10 @@ if (isset($_GET['query'])) { break; case "fail2ban": switch ($object) { + case 'banlist': + header('Content-Type: text/plain'); + echo fail2ban('banlist', 'get', $extra); + break; default: $data = fail2ban('get'); process_get_return($data); @@ -1943,7 +1957,14 @@ if (isset($_GET['query'])) { process_edit_return(fwdhost('edit', array_merge(array('fwdhost' => $items), $attr))); break; case "fail2ban": - process_edit_return(fail2ban('edit', array_merge(array('network' => $items), $attr))); + switch ($object) { + case 'banlist': + process_edit_return(fail2ban('banlist', 'refresh', $items)); + break; + default: + process_edit_return(fail2ban('edit', array_merge(array('network' => $items), $attr))); + break; + } break; case "ui_texts": process_edit_return(customize('edit', 'ui_texts', $attr)); diff --git a/data/web/lang/lang.de-de.json b/data/web/lang/lang.de-de.json index 31cc07b90..dc7b5dfac 100644 --- a/data/web/lang/lang.de-de.json +++ b/data/web/lang/lang.de-de.json @@ -148,6 +148,7 @@ "change_logo": "Logo ändern", "configuration": "Konfiguration", "convert_html_to_text": "Konvertiere HTML zu reinem Text", + "copy_to_clipboard": "Text wurde in die Zwischenablage kopiert!", "cors_settings": "CORS Einstellungen", "credentials_transport_warning": "Warnung: Das Hinzufügen einer neuen Regel bewirkt die Aktualisierung der Authentifizierungsdaten aller vorhandenen Einträge mit identischem Next Hop.", "customer_id": "Kunde", @@ -181,6 +182,8 @@ "f2b_blacklist": "Blacklist für Netzwerke und Hosts", "f2b_filter": "Regex-Filter", "f2b_list_info": "Ein Host oder Netzwerk auf der Blacklist wird immer eine Whitelist-Einheit überwiegen. Die Aktualisierung der Liste dauert einige Sekunden.", + "f2b_manage_external": "Fail2Ban extern verwalten", + "f2b_manage_external_info": "Fail2ban wird die Banlist weiterhin pflegen, jedoch werden keine aktiven Regeln zum blockieren gesetzt. Die unten generierte Banlist, kann verwendet werden, um den Datenverkehr extern zu blockieren.", "f2b_max_attempts": "Max. Versuche", "f2b_max_ban_time": "Maximale Bannzeit in Sekunden", "f2b_netban_ipv4": "Netzbereich für IPv4-Banns (8-32)", @@ -1035,6 +1038,7 @@ "domain_removed": "Domain %s wurde entfernt", "dovecot_restart_success": "Dovecot wurde erfolgreich neu gestartet", "eas_reset": "ActiveSync Gerät des Benutzers %s wurde zurückgesetzt", + "f2b_banlist_refreshed": "Banlist ID wurde erfolgreich erneuert.", "f2b_modified": "Änderungen an Fail2ban-Parametern wurden gespeichert", "forwarding_host_added": "Weiterleitungs-Host %s wurde hinzugefügt", "forwarding_host_removed": "Weiterleitungs-Host %s wurde entfernt", diff --git a/data/web/lang/lang.en-gb.json b/data/web/lang/lang.en-gb.json index 723ed62e0..edde80078 100644 --- a/data/web/lang/lang.en-gb.json +++ b/data/web/lang/lang.en-gb.json @@ -154,6 +154,7 @@ "logo_dark_label": "Inverted for dark mode", "configuration": "Configuration", "convert_html_to_text": "Convert HTML to plain text", + "copy_to_clipboard": "Text copied to clipboard!", "cors_settings": "CORS Settings", "credentials_transport_warning": "Warning: Adding a new transport map entry will update the credentials for all entries with a matching next hop column.", "customer_id": "Customer ID", @@ -187,6 +188,8 @@ "f2b_blacklist": "Blacklisted networks/hosts", "f2b_filter": "Regex filters", "f2b_list_info": "A blacklisted host or network will always outweigh a whitelist entity. List updates will take a few seconds to be applied.", + "f2b_manage_external": "Manage Fail2Ban externally", + "f2b_manage_external_info": "Fail2ban will still maintain the banlist, but it will not actively set rules to block traffic. Use the generated banlist below to externally block the traffic.", "f2b_max_attempts": "Max. attempts", "f2b_max_ban_time": "Max. ban time (s)", "f2b_netban_ipv4": "IPv4 subnet size to apply ban on (8-32)", @@ -1046,6 +1049,7 @@ "domain_removed": "Domain %s has been removed", "dovecot_restart_success": "Dovecot was restarted successfully", "eas_reset": "ActiveSync devices for user %s were reset", + "f2b_banlist_refreshed": "Banlist ID has been successfully refreshed.", "f2b_modified": "Changes to Fail2ban parameters have been saved", "forwarding_host_added": "Forwarding host %s has been added", "forwarding_host_removed": "Forwarding host %s has been removed", diff --git a/data/web/templates/admin/tab-config-f2b.twig b/data/web/templates/admin/tab-config-f2b.twig index a353f7fa3..bb4a2e85a 100644 --- a/data/web/templates/admin/tab-config-f2b.twig +++ b/data/web/templates/admin/tab-config-f2b.twig @@ -42,6 +42,13 @@ +
+
+ + +
+

{{ lang.admin.f2b_manage_external_info }}

+

{{ lang.admin.f2b_list_info|raw }}

@@ -90,6 +97,15 @@ {% if not f2b_data.active_bans and not f2b_data.perm_bans %} {{ lang.admin.no_active_bans }} {% endif %} +
+
+ + {% if is_https %} + + {% endif %} + +
+
{% for active_ban in f2b_data.active_bans %}