restructure configuration directories

2025-12-20 05:11:30 +00:00 · 2025-05-22 15:37:15 +02:00
parent dba9675a9b
commit f35def48cb
25 changed files with 113 additions and 124 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,3 @@
-!data/conf/nginx/dynmaps.conf
-!data/conf/nginx/meta_exporter.conf
-!data/conf/nginx/site.conf
 !/**/.gitkeep
 *.iml
 .idea
@@ -8,60 +5,33 @@
 data/assets/ssl-example/*
 data/assets/ssl/*
 data/conf/borgmatic/
-data/conf/clamav/whitelist.ign2
-data/conf/dovecot/acl_anyone
-data/conf/dovecot/dovecot-master.passwd
-data/conf/dovecot/dovecot-master.userdb
-data/conf/dovecot/dovecot.conf
+data/conf/clamav/rendered_configs
 data/conf/dovecot/extra.conf
-data/conf/dovecot/mail_replica.conf
-data/conf/dovecot/global_sieve_*
-data/conf/dovecot/last_login
-data/conf/dovecot/lua
-data/conf/dovecot/mail_plugins*
-data/conf/dovecot/shared_namespace.conf
-data/conf/dovecot/sni.conf
-data/conf/dovecot/sogo-sso.conf
-data/conf/dovecot/sogo_trusted_ip.conf
-data/conf/dovecot/sql
-data/conf/dovecot/conf.d/fts.conf
-data/conf/nextcloud-*.bak
-data/conf/nginx/*.active
-data/conf/nginx/*.bak
-data/conf/nginx/*.conf
-data/conf/nginx/*.custom
+data/conf/dovecot/rendered_configs
+data/conf/nginx/rendered_configs
 data/conf/phpfpm/sogo-sso/sogo-sso.pass
-data/conf/phpfpm/opcache-recommended.ini
-data/conf/phpfpm/other.ini
-data/conf/phpfpm/pools.conf
-data/conf/phpfpm/session_store.ini
-data/conf/phpfpm/upload.ini
+data/conf/phpfpm/rendered_configs
 data/conf/portainer/
-data/conf/postfix/allow_mailcow_local.regexp
-data/conf/postfix/custom_postscreen_whitelist.cidr
-data/conf/postfix/custom_transport.pcre
-data/conf/postfix/main.cf
 data/conf/postfix/extra.cf
-data/conf/postfix/sni.map
-data/conf/postfix/sni.map.db
-data/conf/postfix/sql
-data/conf/postfix/dns_blocklists.cf
-data/conf/postfix/dnsbl_reply.map
+data/conf/postfix/rendered_configs
 data/conf/rspamd/custom/*
 data/conf/rspamd/local.d/*
 data/conf/rspamd/override.d/*
+data/conf/rspamd/rendered_configs
 data/conf/sogo/custom-theme.js
-data/conf/sogo/plist_ldap
-data/conf/sogo/plist_ldap.sh
 data/conf/sogo/sieve.creds
 data/conf/sogo/cron.creds
 data/conf/sogo/custom-fulllogo.svg
 data/conf/sogo/custom-shortlogo.svg
 data/conf/sogo/custom-fulllogo.png
-data/conf/mysql/my.cnf
+data/conf/sogo/rendered_configs
+data/conf/mysql/rendered_configs
 data/gitea/
 data/gogs/
+data/hooks/clamd/*
 data/hooks/dovecot/*
+data/hooks/mariadb/*
+data/hooks/nginx/*
 data/hooks/phpfpm/*
 data/hooks/postfix/*
 data/hooks/rspamd/*
--- a/data/Dockerfiles/bootstrap/modules/BootstrapBase.py
+++ b/data/Dockerfiles/bootstrap/modules/BootstrapBase.py
@@ -30,25 +30,27 @@ class BootstrapBase:
    self.mysql_conn = None
    self.redis_conn = None

-  def render_config(self, config_file):
+  def render_config(self, config_dir):
    """
-    Renders multiple Jinja2 templates based on a JSON config file.
+    Renders multiple Jinja2 templates from a config.json file in a given directory.

-    Each config entry must include:
-      - template (str): the template filename
-      - output (str): absolute path to the output file
+    Args:
+      config_dir (str or Path): Path to the directory containing config.json

-    Optional:
-      - clean_blank_lines (bool): remove empty lines from output
-      - if_not_exists (bool): skip rendering if output file already exists
+    Behavior:
+      - Renders each template defined in config.json
+      - Writes the result to the specified output path
+      - Also copies the rendered file to: <config_dir>/rendered_configs/<relative_output_path>
    """

-    from pathlib import Path
    import json
+    from pathlib import Path
+
+    config_dir = Path(config_dir)
+    config_path = config_dir / "config.json"

-    config_path = Path(config_file)
    if not config_path.exists():
-      print(f"Template config file not found: {config_path}")
+      print(f"config.json not found in: {config_dir}")
      return

    with config_path.open("r") as f:
@@ -76,7 +78,11 @@ class BootstrapBase:
      with output_path.open("w") as f:
        f.write(rendered)

-      print(f"Rendered {template_name} to {output_path}")
+      rendered_copy_path = config_dir / "rendered_configs" / output_path.name
+      rendered_copy_path.parent.mkdir(parents=True, exist_ok=True)
+      self.copy_file(output_path, rendered_copy_path)
+
+      print(f"Rendered {template_name} → {output_path}")

  def prepare_template_vars(self, overwrite_path, extra_vars = None):
    """
--- a/data/Dockerfiles/bootstrap/modules/BootstrapClamd.py
+++ b/data/Dockerfiles/bootstrap/modules/BootstrapClamd.py
@@ -32,8 +32,8 @@ class Bootstrap(BootstrapBase):
    # Setup Jinja2 Environment and load vars
    self.env = Environment(
      loader=FileSystemLoader([
-        '/etc/clamav/custom_templates',
-        '/etc/clamav/config_templates'
+        '/service_config/custom_templates',
+        '/service_config/config_templates'
      ]),
      keep_trailing_newline=True,
      lstrip_blocks=True,
@@ -47,7 +47,7 @@ class Bootstrap(BootstrapBase):
    self.set_timezone()

    print("Render config")
-    self.render_config("/etc/clamav/config.json")
+    self.render_config("/service_config")

    # Fix permissions
    self.set_owner("/var/lib/clamav", "clamav", "clamav", recursive=True)
--- a/data/Dockerfiles/bootstrap/modules/BootstrapDovecot.py
+++ b/data/Dockerfiles/bootstrap/modules/BootstrapDovecot.py
@@ -31,8 +31,8 @@ class Bootstrap(BootstrapBase):
    # Setup Jinja2 Environment and load vars
    self.env = Environment(
      loader=FileSystemLoader([
-        '/etc/dovecot/custom_templates',
-        '/etc/dovecot/config_templates'
+        '/service_config/custom_templates',
+        '/service_config/config_templates'
      ]),
      keep_trailing_newline=True,
      lstrip_blocks=True,
@@ -55,7 +55,7 @@ class Bootstrap(BootstrapBase):
    self.set_timezone()

    print("Render config")
-    self.render_config("/etc/dovecot/config.json")
+    self.render_config("/service_config")

    files = [
      "/etc/dovecot/mail_plugins",
--- a/data/Dockerfiles/bootstrap/modules/BootstrapMysql.py
+++ b/data/Dockerfiles/bootstrap/modules/BootstrapMysql.py
@@ -31,8 +31,8 @@ class Bootstrap(BootstrapBase):
    # Setup Jinja2 Environment and load vars
    self.env = Environment(
      loader=FileSystemLoader([
-        '/etc/mysql/conf.d/custom_templates',
-        '/etc/mysql/conf.d/config_templates'
+        '/service_config/custom_templates',
+        '/service_config/config_templates'
      ]),
      keep_trailing_newline=True,
      lstrip_blocks=True,
@@ -46,7 +46,7 @@ class Bootstrap(BootstrapBase):
    self.set_timezone()

    print("Render config")
-    self.render_config("/etc/mysql/conf.d/config.json")
+    self.render_config("/service_config")

  def start_temporary(self, socket):
    """
--- a/data/Dockerfiles/bootstrap/modules/BootstrapNginx.py
+++ b/data/Dockerfiles/bootstrap/modules/BootstrapNginx.py
@@ -23,8 +23,8 @@ class Bootstrap(BootstrapBase):
    # Setup Jinja2 Environment and load vars
    self.env = Environment(
      loader=FileSystemLoader([
-        '/etc/nginx/conf.d/custom_templates',
-        '/etc/nginx/conf.d/config_templates'
+        '/service_config/custom_templates',
+        '/service_config/config_templates'
      ]),
      keep_trailing_newline=True,
      lstrip_blocks=True,
@@ -41,7 +41,7 @@ class Bootstrap(BootstrapBase):
    self.set_timezone()

    print("Render config")
-    self.render_config("/etc/nginx/conf.d/config.json")
+    self.render_config("/service_config")

  def get_valid_cert_dirs(self):
    ssl_dir = '/etc/ssl/mail/'
--- a/data/Dockerfiles/bootstrap/modules/BootstrapPhpfpm.py
+++ b/data/Dockerfiles/bootstrap/modules/BootstrapPhpfpm.py
@@ -16,8 +16,8 @@ class Bootstrap(BootstrapBase):
    # Setup Jinja2 Environment and load vars
    self.env = Environment(
      loader=FileSystemLoader([
-        '/php-conf/custom_templates',
-        '/php-conf/config_templates'
+        '/service_config/custom_templates',
+        '/service_config/config_templates'
      ]),
      keep_trailing_newline=True,
      lstrip_blocks=True,
@@ -44,7 +44,7 @@ class Bootstrap(BootstrapBase):


    print("Render config")
-    self.render_config("/php-conf/config.json")
+    self.render_config("/service_config")

    self.copy_file("/usr/local/etc/php/conf.d/opcache-recommended.ini", "/php-conf/opcache-recommended.ini")
    self.copy_file("/usr/local/etc/php-fpm.d/z-pools.conf", "/php-conf/pools.conf")
--- a/data/Dockerfiles/bootstrap/modules/BootstrapPostfix.py
+++ b/data/Dockerfiles/bootstrap/modules/BootstrapPostfix.py
@@ -18,15 +18,15 @@ class Bootstrap(BootstrapBase):
    # Setup Jinja2 Environment and load vars
    self.env = Environment(
      loader=FileSystemLoader([
-        '/opt/postfix/conf/custom_templates',
-        '/opt/postfix/conf/config_templates'
+        '/service_config/custom_templates',
+        '/service_config/config_templates'
      ]),
      keep_trailing_newline=True,
      lstrip_blocks=True,
      trim_blocks=True
    )
-    with open("/opt/postfix/conf/extra.cf", "r") as f:
-      extra_config = f.read()
+    extra_config_path = Path("/opt/postfix/conf/extra.cf")
+    extra_config = extra_config_path.read_text() if extra_config_path.exists() else ""
    extra_vars = {
      "VALID_CERT_DIRS": self.get_valid_cert_dirs(),
      "EXTRA_CF": extra_config
@@ -40,7 +40,7 @@ class Bootstrap(BootstrapBase):
    self.set_syslog_redis()

    print("Render config")
-    self.render_config("/opt/postfix/conf/config.json")
+    self.render_config("/service_config")

    # Create SNI Config
    self.run_command(["postmap", "-F", "hash:/opt/postfix/conf/sni.map"])
--- a/data/Dockerfiles/bootstrap/modules/BootstrapRspamd.py
+++ b/data/Dockerfiles/bootstrap/modules/BootstrapRspamd.py
@@ -61,8 +61,8 @@ class Bootstrap(BootstrapBase):
    # Setup Jinja2 Environment and load vars
    self.env = Environment(
      loader=FileSystemLoader([
-        '/etc/rspamd/custom_templates',
-        '/etc/rspamd/config_templates'
+        '/service_config/custom_templates',
+        '/service_config/config_templates'
      ]),
      keep_trailing_newline=True,
      lstrip_blocks=True,
@@ -80,7 +80,7 @@ class Bootstrap(BootstrapBase):
    self.set_timezone()

    print("Render config")
-    self.render_config("/etc/rspamd/config.json")
+    self.render_config("/service_config")

    # Fix missing default global maps, if any
    # These exists in mailcow UI and should not be removed
--- a/data/Dockerfiles/bootstrap/modules/BootstrapSogo.py
+++ b/data/Dockerfiles/bootstrap/modules/BootstrapSogo.py
@@ -28,8 +28,8 @@ class Bootstrap(BootstrapBase):
    # Setup Jinja2 Environment and load vars
    self.env = Environment(
      loader=FileSystemLoader([
-        '/etc/sogo/custom_templates',
-        '/etc/sogo/config_templates'
+        '/service_config/custom_templates',
+        '/service_config/config_templates'
      ]),
      keep_trailing_newline=True,
      lstrip_blocks=True,
@@ -48,7 +48,7 @@ class Bootstrap(BootstrapBase):
    self.set_syslog_redis()

    print("Render config")
-    self.render_config("/etc/sogo/config.json")
+    self.render_config("/service_config")

    print("Fix permissions")
    self.set_owner("/var/lib/sogo", "sogo", "sogo", recursive=True)
--- a/data/Dockerfiles/nginx/docker-entrypoint.sh
+++ b/data/Dockerfiles/nginx/docker-entrypoint.sh
@@ -1,5 +1,13 @@
 #!/bin/sh

+# Run hooks
+for file in /hooks/*; do
+  if [ -x "${file}" ]; then
+    echo "Running hook ${file}"
+    "${file}"
+  fi
+done
+
 python3 -u /bootstrap/main.py
 BOOTSTRAP_EXIT_CODE=$?

--- a/data/conf/clamav/config.json
+++ b/data/conf/clamav/config.json
@@ -3,5 +3,13 @@
    "template": "whitelist.ign2.j2",
    "output": "/var/lib/clamav/whitelist.ign2",
    "clean_blank_lines": true
+  },
+  {
+    "template": "clamd.conf.j2",
+    "output": "/etc/clamav/clamd.conf"
+  },
+  {
+    "template": "freshclam.conf.j2",
+    "output": "/etc/clamav/freshclam.conf"
  }
 ]
--- a/data/conf/clamav/config_templates/clamd.conf.j2
+++ b/data/conf/clamav/config_templates/clamd.conf.j2
--- a/data/conf/clamav/config_templates/freshclam.conf.j2
+++ b/data/conf/clamav/config_templates/freshclam.conf.j2
--- a/data/conf/postfix/config.json
+++ b/data/conf/postfix/config.json
@@ -106,5 +106,25 @@
  {
    "template": "custom_transport.pcre.j2",
    "output":   "/opt/postfix/conf/custom_transport.pcre"
+  },
+  {
+    "template": "allow_mailcow_local.regexp.j2",
+    "output":   "/opt/postfix/conf/allow_mailcow_local.regexp"
+  },
+  {
+    "template": "anonymize_headers.pcre.j2",
+    "output":   "/opt/postfix/conf/anonymize_headers.pcre"
+  },
+  {
+    "template": "local_transport.j2",
+    "output":   "/opt/postfix/conf/local_transport"
+  },
+  {
+    "template": "master.cf.j2",
+    "output":   "/opt/postfix/conf/master.cf"
+  },
+  {
+    "template": "smtp_dsn_filter.j2",
+    "output":   "/opt/postfix/conf/smtp_dsn_filter"
  }
 ]
--- a/data/conf/postfix/config_templates/allow_mailcow_local.regexp.j2
+++ b/data/conf/postfix/config_templates/allow_mailcow_local.regexp.j2
--- a/data/conf/postfix/config_templates/anonymize_headers.pcre.j2
+++ b/data/conf/postfix/config_templates/anonymize_headers.pcre.j2
--- a/data/conf/postfix/config_templates/local_transport.j2
+++ b/data/conf/postfix/config_templates/local_transport.j2
--- a/data/conf/postfix/config_templates/master.cf.j2
+++ b/data/conf/postfix/config_templates/master.cf.j2
--- a/data/conf/postfix/config_templates/smtp_dsn_filter.j2
+++ b/data/conf/postfix/config_templates/smtp_dsn_filter.j2
--- a/data/conf/postfix/dns_reply.map
+++ b/data/conf/postfix/dns_reply.map
--- a/data/conf/sogo/config.json
+++ b/data/conf/sogo/config.json
@@ -6,5 +6,9 @@
  {
    "template": "UIxTopnavToolbar.wox.j2",
    "output":   "/usr/lib/GNUstep/SOGo/Templates/UIxTopnavToolbar.wox"
+  },
+  {
+    "template": "sogo.conf.j2",
+    "output":   "/etc/sogo/sogo.conf"
  }
 ]
--- a/data/conf/sogo/config_templates/sogo.conf.j2
+++ b/data/conf/sogo/config_templates/sogo.conf.j2
--- a/data/conf/sogo/plist_ldap.sh
+++ b/data/conf/sogo/plist_ldap.sh
@@ -1,34 +0,0 @@
-#!/bin/bash
-
-domain="$1"
-gal_status="$2"
-
-echo "
-                <!--
-                <example>
-                    <key>canAuthenticate</key>
-                    <string>YES</string>
-                    <key>id</key>
-                    <string>"${domain}"_ldap</string>
-                    <key>isAddressBook</key>
-                    <string>"${gal_status}"</string>
-                    <key>IDFieldName</key>
-                    <string>mail</string>
-                    <key>UIDFieldName</key>
-                    <string>uid</string>
-                    <key>bindFields</key>
-                    <array>
-                        <string>mail</string>
-                    </array>
-                    <key>type</key>
-                    <string>ldap</string>
-                    <key>bindDN</key>
-                    <string>cn=admin,dc=example,dc=local</string>
-                    <key>bindPassword</key>
-                    <string>password</string>
-                    <key>baseDN</key>
-                    <string>ou=People,dc=example,dc=local</string>
-                    <key>hostname</key>
-                    <string>ldap://1.2.3.4:389</string>
-                </example>
-                -->"
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -24,7 +24,7 @@ services:
      stop_grace_period: 45s
      volumes:
        - ./data/hooks/mariadb:/hooks:z
-        - ./data/conf/mysql/:/etc/mysql/conf.d/:z
+        - ./data/conf/mysql:/service_config:z
        - mysql-vol-1:/var/lib/mysql/
        - mysql-socket-vol-1:/var/run/mysqld/
      environment:
@@ -83,7 +83,7 @@ services:
        - SKIP_CLAMD=${SKIP_CLAMD:-n}
      volumes:
        - ./data/hooks/clamd:/hooks:Z
-        - ./data/conf/clamav/:/etc/clamav/:Z
+        - ./data/conf/clamav:/service_config:Z
        - mysql-socket-vol-1:/var/run/mysqld/
        - clamd-db-vol-1:/var/lib/clamav
      networks:
@@ -111,7 +111,10 @@ services:
        - SPAMHAUS_DQS_KEY=${SPAMHAUS_DQS_KEY:-}
      volumes:
        - ./data/hooks/rspamd:/hooks:Z
-        - ./data/conf/rspamd/config_templates/:/etc/rspamd/config_templates:z
+        - ./data/conf/rspamd/config_templates/:/service_config/config_templates:z
+        - ./data/conf/rspamd/custom_templates/:/service_config/custom_templates:z
+        - ./data/conf/rspamd/rendered_configs/:/service_config/rendered_configs:z
+        - ./data/conf/rspamd/config.json:/service_config/config.json:Z
        - ./data/conf/rspamd/custom/:/etc/rspamd/custom:z
        - ./data/conf/rspamd/override.d/:/etc/rspamd/override.d:Z
        - ./data/conf/rspamd/local.d/:/etc/rspamd/local.d:Z
@@ -119,7 +122,6 @@ services:
        - ./data/conf/rspamd/lua/:/etc/rspamd/lua/:ro,Z
        - ./data/conf/rspamd/rspamd.conf.local:/etc/rspamd/rspamd.conf.local:Z
        - ./data/conf/rspamd/rspamd.conf.override:/etc/rspamd/rspamd.conf.override:Z
-        - ./data/conf/rspamd/config.json:/etc/rspamd/config.json:Z
        - mysql-socket-vol-1:/var/run/mysqld/
        - rspamd-vol-1:/var/lib/rspamd
      restart: always
@@ -152,12 +154,11 @@ services:
        - mysql-socket-vol-1:/var/run/mysqld/
        - ./data/conf/sogo/:/etc/sogo/:z
        - ./data/conf/rspamd/meta_exporter:/meta_exporter:ro,z
-        - ./data/conf/phpfpm:/php-conf:z
+        - ./data/conf/phpfpm:/service_config:z
        - ./data/conf/phpfpm/sogo-sso/:/etc/sogo-sso/:z
        - ./data/conf/dovecot/global_sieve_before:/global_sieve/before:z
        - ./data/conf/dovecot/global_sieve_after:/global_sieve/after:z
        - ./data/assets/templates:/tpls:z
-        - ./data/conf/nginx/:/etc/nginx/conf.d/:z
      dns:
        - ${IPV4_NETWORK:-172.22.1}.254
      environment:
@@ -235,7 +236,9 @@ services:
        - ${IPV4_NETWORK:-172.22.1}.254
      volumes:
        - ./data/hooks/sogo:/hooks:Z
-        - ./data/conf/sogo/:/etc/sogo/:z
+        - ./data/conf/sogo:/service_config:z
+        - ./data/conf/sogo/cron.creds:/etc/sogo/cron.creds:z
+        - ./data/conf/sogo/sieve.creds:/etc/sogo/sieve.creds:z
        - ./data/web/inc/init_db.inc.php:/init_db.inc.php:z
        - ./data/conf/sogo/custom-favicon.ico:/usr/lib/GNUstep/SOGo/WebServerResources/img/sogo.ico:z
        - ./data/conf/sogo/custom-shortlogo.svg:/usr/lib/GNUstep/SOGo/WebServerResources/img/sogo-compact.svg:z
@@ -275,7 +278,8 @@ services:
        - NET_BIND_SERVICE
      volumes:
        - ./data/hooks/dovecot:/hooks:Z
-        - ./data/conf/dovecot:/etc/dovecot:z
+        - ./data/conf/dovecot/auth:/etc/dovecot/auth:z
+        - ./data/conf/dovecot:/service_config:z
        - ./data/assets/ssl:/etc/ssl/mail/:ro,z
        - ./data/conf/sogo/:/etc/sogo/:z
        - ./data/conf/phpfpm/sogo-sso/:/etc/phpfpm/:z
@@ -283,7 +287,7 @@ services:
        - vmail-vol-1:/var/vmail
        - vmail-index-vol-1:/var/vmail_index
        - crypt-vol-1:/mail_crypt/
-        - ./data/conf/rspamd/custom/:/etc/rspamd/custom:z
+        - ./data/conf/rspamd/custom/sa-rules:/etc/rspamd/custom/sa-rules:z
        - ./data/assets/templates:/templates:z
        - rspamd-vol-1:/var/lib/rspamd
        - mysql-socket-vol-1:/var/run/mysqld/
@@ -361,7 +365,9 @@ services:
          condition: service_healthy
      volumes:
        - ./data/hooks/postfix:/hooks:Z
-        - ./data/conf/postfix:/opt/postfix/conf:z
+        - ./data/conf/postfix:/service_config:z
+        - ./data/conf/postfix/postscreen_access.cidr:/opt/postfix/conf/postscreen_access.cidr:z
+        - ./data/conf/postfix/extra.cf:/opt/postfix/conf/extra.cf:z
        - ./data/assets/ssl:/etc/ssl/mail/:ro,z
        - postfix-vol-1:/var/spool/postfix
        - crypt-vol-1:/var/lib/zeyple
@@ -435,10 +441,11 @@ services:
        - NGINX_USE_PROXY_PROTOCOL=${NGINX_USE_PROXY_PROTOCOL:-n}
        - TRUSTED_PROXIES=${TRUSTED_PROXIES:-}
      volumes:
+        - ./data/hooks/nginx:/hooks:Z
        - ./data/web:/web:ro,z
        - ./data/conf/rspamd/dynmaps:/dynmaps:ro,z
        - ./data/assets/ssl/:/etc/ssl/mail/:ro,z
-        - ./data/conf/nginx/:/etc/nginx/conf.d/:z
+        - ./data/conf/nginx:/service_config:z
        - ./data/conf/rspamd/meta_exporter:/meta_exporter:ro,z
        - ./data/conf/dovecot/auth/mailcowauth.php:/mailcowauth/mailcowauth.php:z
        - ./data/web/inc/functions.inc.php:/mailcowauth/functions.inc.php:z