diff --git a/_snippets/common-links.md b/_snippets/common-links.md index 3dc98fe..de1b00f 100644 --- a/_snippets/common-links.md +++ b/_snippets/common-links.md @@ -24,6 +24,7 @@ [jackett]: /recipes/autopirate/jackett/ [jellyfin]: /recipes/jellyfin/ [k8s/mastodon]: /recipes/kubernetes/mastodon/ +[metallb]: /kubernetes/loadbalancer/metallb/ [kavita]: /recipes/kavita/ [keycloak]: /recipes/keycloak/ [komga]: /recipes/komga/ diff --git a/docs/images/metallb-l2-routing.png b/docs/images/metallb-l2-routing.png new file mode 100644 index 0000000..3f94ee1 Binary files /dev/null and b/docs/images/metallb-l2-routing.png differ diff --git a/docs/images/metallb-l3-routing.png b/docs/images/metallb-l3-routing.png new file mode 100644 index 0000000..afbb228 Binary files /dev/null and b/docs/images/metallb-l3-routing.png differ diff --git a/docs/images/metallb-routing.drawio b/docs/images/metallb-routing.drawio new file mode 100644 index 0000000..b682baf --- /dev/null +++ b/docs/images/metallb-routing.drawio @@ -0,0 +1 @@ +7VpZb9s4EP4tfQiw+xBDlxX50c7RFk27wSZAt08FLdEyG1rUUnRs99fvDEXqtJ0U66SXESAWh8PhMd8MP1I68c8X69eS5PP3IqH8xHOS9Yl/ceJ5ruNG8IOSTSkJQ78UpJIlRqkW3LKv1LY00iVLaNFSVEJwxfK2MBZZRmPVkhEpxaqtNhO83WtOUtoT3MaE96UfWaLmpTTyzmr5G8rSue3ZDUdlzYJYZTOTYk4SsWqI/MsT/1wKocqnxfqcclw8uy5lu6sdtdXAJM3UUxpsLoYszO7Gqzcf391+3rjJX1+CU7vMhdrYGdMEFsAUhVRzkYqM8MtaOomX8oGiVRcKUiyzRJccKBX3VMVzUzUn8Xwp6WuSgyAAQW3uWojcaH2hSm2M48lSCWyoFtzU0jVT/6DxwXBoip90cWSLF2vTty5sGoUbKtmCKiqNbCYydUUWjKPSWMZzpgAyBUwfOgYXoqJWuhVLGeN45koBzryhP4Z/sLL4DxWKQSpEyinJWTGIxUJXxIVWvZqVXcBjq5OhN+l2U04aQgTGmyVjhCsuek4zXEpFZKUSlCrNYqEHWc3xNicxy1KoRXPQNqVqR2VCinnlv9L/6PSduDKiwq7KTjCZ+NR979HzKvRD2qACRihhuRxJOVHsoT0OYuI3rfRqiMODQfm3IL60+0D40vT0NoMVWtHp134ocA55BiEPwZujMOZiCdYnK3QrrioKVxIR3kTt/jD4QXHoORUcHqhUdL0fEH0HmgZeYJLKpp3KV3X+dIdGNm/kzsh5Jpd7PZdfjz+c4HRhOR135A3cMBq4A5jWlRf0UFA5E9dHLBVnGT2vthubWM4FF1I38OHvCoc3SSVJGK3rMpEhmGaM8676hY54Ke5pR7kKVuwI/cJgc7omU8pvRMEUE5ArLqZCKfB+rTDmLMUKhWl2Qkwpplnp9QZUu4lIl83ksUtS5OVEZ2yN45jkgqGVywcwVli8m/BYrFOkAQOyKoJBSjPIP/HnGZN0RdBlP3YoHAL6UdCCvhf2sX8W9aEfhs8Efb8H/XcRrm4mkFV5IYcBTKYSnlJ8qmPB2R4L7d3+mAX3QMESQQOFwNmSBr0taTB8rjQY9tMgTNT5m2Sw9++EgvebQ6FFljqZOyE0msW9zA01YRzR6ewwSAqGweNI8sMXRFLUQ9I0zXsAaR0TGvw25qQoWNwGyLccI35QuOjt8/lY+1OJeh9HDZxs411W9mT+bXq4QSLQgKnThql/1oGfmXrZqnlQ7RjyzzqG3I4hs0xdQ4AvsmmoaaJS7Bmw1+kncPaOqzfB9oEbHsoR1HFV+eB/HFf6O/jbDJCKuPCcN3d3N7f6WkPilcMmEQvCshLJ5/2cDkkaogo9XQj+gATAgfM2TiMkC8zYWj9fTjmEp+e8vakqdPNehENOU+0w7jHNLiFdsCTRlwgwBvaVTCueabwFdoeTkyHSYbwMKEzo7GLHzYRsRVsI7aPZ4gBp2j1rH3tG/Sztv+h+7wY96HyEc48+9Bw9fgCPV+60bN/fsjGHW1wePJvL+xzvyPdfiO+fdfj+tmuPl43/syMYvhcYog4YvO8NhmEPC++pIvy6n+LrS89f0cNudKDc3yWpUd/Doz1k+/DR3j+U2RCHDgjUxHMiC3MxH/67xPdOk6Wa4WnOFm0qsC2LnGQtdFjFuNyBkUrIdPqHo+mmY3/+1CYcXPZT46A26zBWtsOio9SwVXIDfWkb5etmRTnAsU5uckF4o+6BSEbgF+gIUQDR4hG9mOS7VFbGkViJp3Bdw6mCUZ8W9rDWbSlkPieZMemVMqRRp4Y5obgiT7aOQaLNTE+OnaquURKMAd9f2J40BSqBp992NrpZCZm0B1bZgrlM7xmYQ5sl0To10G3pTUl8n+rEf9rxuRdEpbu9YGQehtbz2mpCYyEJ3lCfKjh43Ge0MMNjGVPMrk9Xt+HLvXqN4bT0ZlwQ1V2chBU5Jxurjlf48PCKLXIhFckM5jtR8GF8t5sqs7xDlf1xEPg2cCCEy9hpxxNutsdoOkbTTxNNx43hCOVfBMoH3Bj0pVn9giR4JPf/nseo7//iZNQ+hFVvWZuHsG0c/flO5KOdHH1nQu1mBE5n20FZSz4hPO1b91fwfEuzBANFktlMk5fyvriJYO/K19/uNJHtDlzHbXX0JHw/hujOtwJTUrB4IGBNPscwXrFUXfD8lDFQ3kQ2YJ/QGVlydRhoB04H28P+8XPrbbPbfZnyBHBDsf5MsXyxUX/s6V/+Bw==7VtZc9s2EP4tedBM+yANL9HSow47SeOkbuyZ1H3xQCREIgYJBoQO+9d3AYI3bdmp5SiNJjMWsFji2m8/7IJMz55F27ccJeFH5mPaswx/27PnPcsyDXMEP1Jyl0lc184EASe+VioFl+Qe509q6Yr4OK0pCsaoIEld6LE4xp6oyRDnbFNXWzJaHzVBAW4JLj1E29IvxBdhJh1ZJ6X8HSZBmI9suuOsJUK5sl5JGiKfbSoi+7RnzzhjIitF2xmmcvPyfblP/rn+8PmP0fr6r9l0SG9JsnH7WWdnz3mkWALHsfjurodfxHKdTL/9OT21779ur4O3C6EfMdaIrvR+6bWKu3wDE0ZioYwwnPaG8549RWmSWWpJthh6n4YiolA1ZRslQQxlD6aKOQiKXTOg4qM0lE+oComU6fLfOYkCmD0lC/iLKCXYvxGY4pSkIEg3RHghFC5V4cYZ3SSMi5tza5CuAznOLZYK2SxC5IUrjt+iBAQOCJYsFmcoIlSieMK9kAhYAnRszNEKjK+mKpUu2Yp7cjahEIBQa2hP4A9sqvwjFdJBwFhAMUpIOvBYpBq8VKmeLbMhoFgbZGhNm8NkfgLOBXVC6YxRBm1zHy/RisLeTrVhMBd420DyDhiYBTbBqTGLsOAwJUP3Yp24A1dbWbu0Y9oDN5NsShcZW4MT7SRhxUFca6AZAWnPDIoxSvRBQQPwGThvYw/74Mu6CtYOWcBiRE9L6dRb8bVClDQ7Z6vYL/D1OCLK7s4ZS7TWVyzEnbYNWglWxzbeEvG37HwwHOrqtaqO8+p8q8dWlbtK5QJzArskAaBkhwxIHPsTybxy0xMcy60UiBcqTqZSraZqksUaLxPkkRi8ci67g2cDLB5oLBjBLDAvjf4diAfg5Fv1kJ4+ubIJPaLndHsQxxQJsq5P7sW9wG5R8ntJpRu8uG/7B9BkkuKMZhMp9ChbSUreSFvLrZbCDZewr0L5p2RLy9gnLzpGjRTz+KXCiObQaPPhyPjvOOg8mq3dR/Nh0WPNlQ8UQq/Ob/umNKdNaQ8Hej+K0pwWlM8nn3rSncHWhgmRhumOBuYAxj+znBbMC6RK2LKVoCTGsyJdyE/TPITqWTb8O5PTmwYc+QSXbTGLcSPiytXnCgac3eKGci1mlbxDILk4RwtML1hKBGEy3F0wIQCapcJEx8FCOk87Kq64TSsclHW9eKMr2lYBOeana5zF5WaF/qNtINO4AdqkziDAMYDSu1kSjjdImuywqX5v1D5yatRuuW1uPxl1hLrunvxh2PKHDyO55TGTqbLlqgRgwaEUyFLpIEa3g9SJ/Xj0Pxsf4xo+hk7H2W91nP3uC5z9nfN124QJqzc+oxjOkUfwYRr94rbil4NHPf6oU7yP8GjptSgeWlxvhBfLPaLLOXF2o8t2XxFdJy10TT5ftFBTCx0rIZNHUZoSr46a54SWB4ohdfgeaCDYBlcFPF1ZSS57cnynR7iQsUUFu1YDu2YDk3o/sqeql4+NjuyTenblDBsd6b1rdgSgQ3cVNX0Z+fCEnfqEnZHx6LyaC3Tcmj4UshmUzlbY4Pv9b9Tyv/cxoFdixTLeXV1dXKqLbi4voe98FiESZ+ietbkf2Fxei4KhU0bXMnowIC+Th4CLIkntSj9ZLSi4rGW8vygagvKWseL1QH5ix41uM8SNiO+rZBPmQO7RoohcO26OIWlMtTs9FG9XmTsXdYTIOxlkX3xuNqA8btO5/arBwrgFpy+QXanU6oiCfaGgMHGeU9gdp7rbAQNnXzDIqfmYVRxIVjFuZBXDH51VmOYRIYeEENtoIKTjXuKVEdK+dP6IBaLn7SOifPXwf7S7Odqn3ceNqNpq2338SFD/8mZvv3PK2SBNUFwzvPttJT94mHrZ4SyjDB4sfjNUdGrkP7+r7TPkjvb13tcDEt1Lt8UbSpW+srBB3RqPkm21IZvgRLEZjxCttK0RJwh+IVJBAtCX7tDzUPKQykabQzY6RkYOBsUCZt1P83yv+STjSYhi3aWVyWSE1ddBlRQXcVXeRoBZYz2SkS9VtQgOnUF6EOUjqegog4/6XKYyzIZxvz6xoi9Yy+KWQHeyzywG62sA1vQWyLsNFNP3GzaH7Cozt+WMdWGYW1716mOPcSSvyPsC8pTbGKd6eiQmguT709St2PJRvcp0anpLypBobo5P0oSiu1xdvkOAwhsSyc9IUJyFwTm48xPw0+Tq4SiaJI0o2p44jp07jvxmRfmOq0Ps8nQ9etPRm34ab2q6xPFgOEL5J4XyCx4M6o6t+t5lB/n/monTgb6RaSTmxXveatrVFX/vL+1qfyCxk2abPEHxshuqpeRagjZdLWIs3kDpKpRA/ziZ9fLb5sdfLcpVSFwNXRjRA5NOFrDKCQorpSfifxfiG18zLFBKvAGD7bnxEKVsJZrg+il95Ad8/etYdezbT729Nptvfp4AfqiWX8dnL0/K/2Ngn/4L \ No newline at end of file diff --git a/docs/kubernetes/loadbalancer/metallb/index.md b/docs/kubernetes/loadbalancer/metallb/index.md index 386a961..3ecef02 100644 --- a/docs/kubernetes/loadbalancer/metallb/index.md +++ b/docs/kubernetes/loadbalancer/metallb/index.md @@ -21,6 +21,30 @@ MetalLB does two jobs: * [ ] Network firewall/router supporting BGP (*ideal but not required*) +## L3 vs L2 + +MetalLB can be configured to operate in either Layer 2 or Layer 3 mode (below). See my highly accurate and technically appropriate diagrams below to understand the difference: + +### Layer 3 (recommended) + +![MetalLB Layer 3 Routing](/images/metallb-l3-routing.png){ loading=lazy } + +When configuring MetalLB for Layer 3, you define a dedicated subnet to be advertised from your MetalLB pods to your BGP-speaking router/firewall. This subnet **shouldn't** be configured on any nodes, or any of your network equipment. We are taking advantage of [a protocol first designed in 1989](https://www.rfc-editor.org/rfc/rfc1105) to allow MetalLB to tell your router where to send traffic to this new subnet (*it should send it to the Kubernetes nodes, of course, which are on the same network as the router already is*). + +If you need to access your services externally, then perform NAT on your firewall to the external IP assigned to your `LoadBalancerIP` Kubernetes service by MetalLB. + +Use BGP if possible - it's far easier to debug / monitor than Layer 2 (*below*) + +### Layer 2 + +![MetalLB Layer 2 Routing](/images/metallb-l2-routing.png){ loading=lazy } + +Now we are taking advantage of [a protocol first designed in 1982](https://www.rfc-editor.org/rfc/rfc826) to "lie to" other devices on your subnet, telling them that the MAC address for a given IP belongs whichever MetalLB pod has the "leader" role for this virtual IP. + +As above, if you need to access your services externally, then perform NAT on your firewall to the external IP assigned to your `LoadBalancerIP` Kubernetes service by MetalLB. + +Use Layer 2 if your firewall / router can't support BGP. + ## MetalLB Requirements ### Allocations @@ -108,58 +132,118 @@ data: Then work your way through the values you pasted, and change any which are specific to your configuration. I'd recommend changing the following: -* `existingConfigMap: metallb-config`: I prefer to set my MetalLB config independently of the chart config, so I set this to `metallb-config`, which I then define below. * `commonAnnotations`: Anticipating the future use of Reloader to bounce applications when their config changes, I add the `configmap.reloader.stakater.com/reload: "metallb-config"` annotation to all deployed objects, which will instruct Reloader to bounce the daemonset if the ConfigMap changes. -### ConfigMap (for MetalLB) +### Kustomization for CRs (Config) -Finally, it's time to actually configure MetalLB! As discussed above, I prefer to configure the helm chart to apply config from an existing ConfigMap, so that I isolate my application configuration from my chart configuration (*and make tracking changes easier*). In my setup, I'm using BGP against a pair of pfsense[^1] firewalls, so per the [official docs](https://metallb.universe.tf/configuration/), I use the following configuration, saved in my flux repo: +Older versions of MetalLB were configured by a simple ConfigMap, which could be deployed into Kubernetes **alongside** the helmrelease, since a ConfigMap is a standard Kubernetes primitive. -```yaml title="metallb-system/configmap-metallb-config.yaml" -apiVersion: v1 -kind: ConfigMap +Since v0.13 though, MetalLB is [configured exclusively using CRDs](https://metallb.universe.tf/configuration/migration_to_crds/) (*this allows for syntax validation, among other advantages*). This means that the custom resources (*CRs*) have to be applied **after** MetalLB's helm chart has been deployed, since it's the chart which creates the CRD definitions. So we can't deploy the config CRs in the same kustomization as we deploy the helmrelease (*because the CRDs won't exist yet!*) + +The simplest way to solve this chicken-and-egg problem is to create a **second** Kustomization for the MetalLB CRs, and make it depend on the **first** Kustomization (*MetalLB itself*). + +I create this example Kustomization in my flux repo: + +```yaml title="/bootstrap/kustomizations/kustomization-metallb.yaml" +apiVersion: kustomize.toolkit.fluxcd.io/v1beta1 +kind: Kustomization metadata: - namespace: metallb-system - name: metallb-config -data: - config: | - peers: - - peer-address: 192.168.33.2 - peer-asn: 64501 - my-asn: 64500 - - peer-address: 192.168.33.4 - peer-asn: 64501 - my-asn: 64500 - - address-pools: - - name: default - protocol: bgp - avoid-buggy-ips: true - addresses: - - 192.168.32.0/24 + name: config--metallb-system + namespace: flux-system +spec: + interval: 15m + dependsOn: # (1)! + - name: metallb--metallb-system + path: ./metallb-config + prune: true # remove any elements later removed from the above path + timeout: 2m # if not set, this defaults to interval duration, which is 1h + sourceRef: + kind: GitRepository + name: flux-system + validation: server + healthChecks: + - apiVersion: apps/v1 + kind: Deployment + name: metallb-controller + namespace: metallb-system ``` -!!! question "What does that mean?" - In the config referenced above, I define one pool of addresses (`192.168.32.0/24`) which MetalLB is responsible for allocating to my services. MetalLB will then "advertise" these addresses to my firewalls (`192.168.33.2` and `192.168.33.4`), in an eBGP relationship where the firewalls' ASN is `64501` and MetalLB's ASN is `64500`. Provided I'm using my firewalls as my default gateway (*a VIP*), when I try to access one of the `192.168.32.x` IPs from any subnet connected to my firewalls, the traffic will be routed from the firewall to one of the cluster nodes running the pods selected by that service. +1. The `dependsOn` key will prevent Flux from trying to reconcile this Kustomization until the kustomizations it depends on, have successfully reconcilled. -!!! note "Dude, that's too complicated!" - There's an easier way, with some limitations. If you configure MetalLB in L2 mode, all you need to do is to define a range of IPs within your existing node subnet, like this: +### Custom Resources - ```yaml title="metallb-system/configmap-metallb-config.yaml" - apiVersion: v1 - kind: ConfigMap +Finally, it's time to actually configure MetalLB! In my setup, I'm using BGP against a pair of pfsense[^1] firewalls, so per the [official docs](https://metallb.universe.tf/configuration/), I use the following configurations, saved in my flux repo: + +#### IPAddressPool + +```yaml title="/metallb-config/ipaddresspool.yaml" +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: metallb-pool + namespace: metallb-system +spec: + addresses: + - 192.168.32.0/24 +``` + +#### BGPAdvertisment + +```yaml title="/metallb-config/bgpadvertisment.yaml" +apiVersion: metallb.io/v1beta1 +kind: BGPAdvertisement +metadata: + name: metallb-advertisment + namespace: metallb-system +spec: + ipAddressPools: + - metallb-pool # (1)! + aggregationLength: 32 + localPref: 100 + communities: + - 65535:65282 +``` + +1. This must be the same as the name of the `IPAddressPool` defined above + +#### BGPPeer(s) + +You need separate `BGPPeer` resource for every BGP peer, from MetalLB's perspective. Because I use dual pfsense firewalls, I maintain two files, each identifying its peer in its filename, like this: + +```yaml title="/metallb-config/bgppeer-192.168.33.2.yaml" +apiVersion: metallb.io/v1beta2 +kind: BGPPeer +metadata: + name: bgppeer-192.168.33.2 + namespace: metallb-system +spec: + myASN: 64500 + peerASN: 64501 + peerAddress: 192.168.33.2 +``` + +#### Summary + +In the config referenced above, I define one pool of addresses (`192.168.32.0/24`) which MetalLB is responsible for allocating to my services. MetalLB will then "advertise" these addresses to my firewalls (`192.168.33.2` and `192.168.33.4`), in an eBGP relationship where the firewalls' ASN is `64501` and MetalLB's ASN is `64500`. + +Provided I'm using my firewalls as my default gateway (*a VIP*), when I try to access one of the `192.168.32.x` IPs from any subnet connected to my firewalls, the traffic will be routed from the firewall to one of the cluster nodes running the pods selected by that service. + +!!! note "Dude, BGP is too complicated!" + There's an easier way, with some limitations. If you configure MetalLB in L2 mode, all you need to do is to define your `IPAddressPool`, and then an `L2Advertisment`, like this: + + ```yaml title="/metallb-config/l2advertisment.yaml" + apiVersion: metallb.io/v1beta1 + kind: L2Advertisement metadata: + name: my-l2-advertisment namespace: metallb-system - name: metallb-config - data: - config: | - address-pools: - - name: default - protocol: layer2 - addresses: - - 192.168.1.240-192.168.1.250 + spec: + ipAddressPools: + - metallb-pool # (1)! ``` + 1. This must be the same as the name of the `IPAddressPool` defined above, although docs indicate it's optional, and leaving it out will simply use **all** `IPAddressPools`. + ### HelmRelease Lastly, having set the scene above, we define the HelmRelease which will actually deploy MetalLB into the cluster, with the config and extra ConfigMap we defined above. I save this in my flux repo: @@ -174,7 +258,7 @@ spec: chart: spec: chart: metallb - version: 2.x # (1)! + version: 4.x sourceRef: kind: HelmRepository name: bitnami @@ -188,8 +272,6 @@ spec: valuesKey: values.yaml # This is the default, but best to be explicit for clarity ``` -1. This recipe was written when the chart was at version 2, it's now at v4.x, which introduces some breaking changes. Stay tuned for an upcoming refresh! - --8<-- "kubernetes-why-not-config-in-helmrelease.md" ## Deploy MetalLB diff --git a/docs/recent-changes.md b/docs/recent-changes.md index 875c97a..e9f9cc3 100644 --- a/docs/recent-changes.md +++ b/docs/recent-changes.md @@ -12,20 +12,15 @@ Recipe | Description [Mastodon (K8s)][k8s/mastodon] | Kubernetes version of the Mastodon recipe below | *8 Aug 2022* [Mastodon][mastodon] | Federated social network. Think "*twitter but like email*" | *5 Aug 2022* [Kavita][kavita] | "Rocket-fueled" reader for manga/comics/ebooks, able to save reading position across devices/sessions | *27 Jul 2022* -[Authelia][authelia] | Authentication and two factor authorization server with Authelia | *1 Nov 2021* -[Prowlarr][prowlarr] | An indexer manager/proxy built on the popular arr .net/reactjs base stack to integrate with the [AutoPirate][autopirate] friends | *27 Oct 2021* -[Archivebox][archivebox] | Website Archiving service to save websites to view offline | *19 Oct 2021* -[Readarr][readarr] | [Autopirate][autopirate] component to grab and manage eBooks (*think "Sonarr/Radarr for books*") | *18 Oct 2021* ## Recent updates Recipe | Description | Date ----------------------------|------------------------------------------------------------------------------|-------------- +[MetalLB][metallb] | Updated for CRDs required from v0.13, added diagrams explaining L3 vs L2 | *16 Jan 2023* [Nextcloud][nextcloud] | Updated for version 24, improve Redis / cron support | *24 Aug 2022* [Authelia][authelia] | Updated with test services, fixed errors | *27 Jul 2022* [Minio][minio] | Major update to Minio recipe, for new Console UI and Traefik v2 | *22 Oct 2021* -[Traefik Forward Auth][tfa] | Major update for Traefik v2, included instructions for Dex, Google, Keycloak | *29 Jan 2021* -[Autopirate][autopirate] | Updated all components for Traefik v2 labels | *29 Jan 2021* ## Recent reviews