From a5843638349dde2ad0dbd9bac642ec036a0f61c5 Mon Sep 17 00:00:00 2001 From: David Young Date: Wed, 1 May 2019 13:40:29 +1200 Subject: [PATCH] Corrected not-so-tasty gitlab runner recipe --- manuscript/Gemfile | 2 + manuscript/Gemfile.lock | 48 ++++++++ manuscript/generate_preview.py | 8 ++ manuscript/go.sh | 1 + manuscript/htpasswd | 0 manuscript/recipes/gitlab-runner.md | 10 +- manuscript/test.html | 168 ++++++++++++++++++++++++++++ 7 files changed, 232 insertions(+), 5 deletions(-) create mode 100644 manuscript/Gemfile create mode 100644 manuscript/Gemfile.lock create mode 100755 manuscript/generate_preview.py create mode 100644 manuscript/go.sh create mode 100644 manuscript/htpasswd create mode 100644 manuscript/test.html diff --git a/manuscript/Gemfile b/manuscript/Gemfile new file mode 100644 index 0000000..a7c369c --- /dev/null +++ b/manuscript/Gemfile @@ -0,0 +1,2 @@ +source 'https://rubygems.org' +gem 'html-proofer' diff --git a/manuscript/Gemfile.lock b/manuscript/Gemfile.lock new file mode 100644 index 0000000..1d3cc0b --- /dev/null +++ b/manuscript/Gemfile.lock @@ -0,0 +1,48 @@ +GEM + remote: https://rubygems.org/ + specs: + activesupport (5.2.3) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 0.7, < 2) + minitest (~> 5.1) + tzinfo (~> 1.1) + addressable (2.6.0) + public_suffix (>= 2.0.2, < 4.0) + colorize (0.8.1) + concurrent-ruby (1.1.5) + ethon (0.12.0) + ffi (>= 1.3.0) + ffi (1.10.0) + html-proofer (3.10.2) + activesupport (>= 4.2, < 6.0) + addressable (~> 2.3) + colorize (~> 0.8) + mercenary (~> 0.3.2) + nokogiri (~> 1.9) + parallel (~> 1.3) + typhoeus (~> 1.3) + yell (~> 2.0) + i18n (1.6.0) + concurrent-ruby (~> 1.0) + mercenary (0.3.6) + mini_portile2 (2.4.0) + minitest (5.11.3) + nokogiri (1.10.3) + mini_portile2 (~> 2.4.0) + parallel (1.17.0) + public_suffix (3.0.3) + thread_safe (0.3.6) + typhoeus (1.3.1) + ethon (>= 0.9.0) + tzinfo (1.2.5) + thread_safe (~> 0.1) + yell (2.1.0) + +PLATFORMS + ruby + +DEPENDENCIES + html-proofer + +BUNDLED WITH + 2.0.1 diff --git a/manuscript/generate_preview.py b/manuscript/generate_preview.py new file mode 100755 index 0000000..b7614dc --- /dev/null +++ b/manuscript/generate_preview.py @@ -0,0 +1,8 @@ +#!/usr/bin/python + +with open("Book.txt") as f: + print ('echo "Starting build of {book}.epub";' + "pandoc {files} " + + "+smart --table-of-contents --chapters -o {book}.epub;" + 'echo " {book}.epub created."' + ).format(book="Book", files=f.read().replace("\n", " ")) diff --git a/manuscript/go.sh b/manuscript/go.sh new file mode 100644 index 0000000..98cfd2b --- /dev/null +++ b/manuscript/go.sh @@ -0,0 +1 @@ +echo "Starting build of Book.epub";pandoc # This file determines what documents are loaded into the book, and in what sequence. index.md README.md CHANGELOG.md whoami.md sections/ha-docker-swarm.md ha-docker-swarm/design.md ha-docker-swarm/vms.md ha-docker-swarm/shared-storage-ceph.md ha-docker-swarm/shared-storage-gluster.md ha-docker-swarm/keepalived.md ha-docker-swarm/docker-swarm-mode.md ha-docker-swarm/traefik.md ha-docker-swarm/registry.md ha-docker-swarm/duplicity.md sections/chefs-favorites-docker.md recipes/autopirate.md recipes/autopirate/sabnzbd.md recipes/autopirate/nzbget.md recipes/autopirate/rtorrent.md recipes/autopirate/sonarr.md recipes/autopirate/radarr.md recipes/autopirate/mylar.md recipes/autopirate/lazylibrarian.md recipes/autopirate/headphones.md recipes/autopirate/lidarr.md recipes/autopirate/nzbhydra.md recipes/autopirate/nzbhydra2.md recipes/autopirate/ombi.md recipes/autopirate/jackett.md recipes/autopirate/heimdall.md recipes/autopirate/end.md recipes/elkarbackup.md recipes/emby.md recipes/homeassistant.md recipes/homeassistant/ibeacon.md recipes/huginn.md recipes/kanboard.md recipes/miniflux.md recipes/munin.md recipes/nextcloud.md recipes/owntracks.md recipes/phpipam.md recipes/plex.md recipes/privatebin.md recipes/swarmprom.md recipes/turtle-pool.md sections/menu-docker.md recipes/bookstack.md recipes/cryptominer.md recipes/cryptominer/mining-rig.md recipes/cryptominer/amd-gpu.md recipes/cryptominer/nvidia-gpu.md recipes/cryptominer/mining-pool.md recipes/cryptominer/wallet.md recipes/cryptominer/exchange.md recipes/cryptominer/minerhotel.md recipes/cryptominer/monitor.md recipes/cryptominer/profit.md recipes/calibre-web.md recipes/collabora-online.md recipes/ghost.md recipes/gitlab.md recipes/gitlab-runner.md recipes/gollum.md recipes/instapy.md recipes/keycloak.md recipes/openldap.md recipes/mail.md recipes/minio.md recipes/piwik.md recipes/portainer.md recipes/realms.md recipes/tiny-tiny-rss.md recipes/wallabag.md recipes/wekan.md recipes/wetty.md sections/reference.md reference/oauth_proxy.md reference/data_layout.md reference/networks.md reference/containers.md reference/git-docker.md reference/openvpn.md reference/troubleshooting.md +smart --table-of-contents --chapters -o Book.epub;echo " Book.epub created." diff --git a/manuscript/htpasswd b/manuscript/htpasswd new file mode 100644 index 0000000..e69de29 diff --git a/manuscript/recipes/gitlab-runner.md b/manuscript/recipes/gitlab-runner.md index 4913c12..e681173 100644 --- a/manuscript/recipes/gitlab-runner.md +++ b/manuscript/recipes/gitlab-runner.md @@ -33,17 +33,17 @@ Create a docker swarm config file in docker-compose syntax (v3), something like version: '3' services: - 1: + thing1: image: gitlab/gitlab-runner volumes: - - /var/data/gitlab-runner/1:/var/data/gitlab/runners/1 + - /var/data/gitlab/runners/1:/etc/gitlab-runner networks: - internal - 2: + thing2: image: gitlab/gitlab-runner volumes: - - /var/data/gitlab-runner/:/var/data/gitlab/runners/2 + - /var/data/gitlab/runners/2:/etc/gitlab-runner networks: - internal @@ -58,7 +58,7 @@ networks: ### Configure runners -From your GitLab UI, you can retrieve a "token" necessary to register a new runner. To register the runner, you can either create config.toml in each runner's bind-mounted folder (example below), or just "docker exec" into each runner container and execute ```gitlab-container register``` to interactively generate config.toml. +From your GitLab UI, you can retrieve a "token" necessary to register a new runner. To register the runner, you can either create config.toml in each runner's bind-mounted folder (example below), or just "docker exec" into each runner container and execute ```gitlab-runner register``` to interactively generate config.toml. Sample runner config.toml: diff --git a/manuscript/test.html b/manuscript/test.html new file mode 100644 index 0000000..1c93782 --- /dev/null +++ b/manuscript/test.html @@ -0,0 +1,168 @@ +

phpIPAM

+

phpIPAM is an open-source web IP address management application (IPAM). Its goal is to provide light, modern and useful IP address management. It is php-based application with MySQL database backend, using jQuery libraries, ajax and HTML5/CSS3 features.

+
+phpIPAM Screenshot
phpIPAM Screenshot
+
+

phpIPAM fulfils a non-sexy, but important role - It helps you manage your IP address allocation.

+

Why should you care about this?

+

You probably have a home network, with 20-30 IP addresses, for your family devices, your IoT devices, your smart TV, etc. If you want to (a) monitor them, and (b) audit who does what, you care about what IPs they’re assigned by your DHCP server.

+

You could simple keep track of all devices with leases in your DHCP server, but what happens if your (hypothetical?) Ubiquity Edge Router X crashes and burns due to lack of disk space, and you loose track of all your leases? Well, you have to start from scratch, is what!

+

And that HomeAssistant config, which you so carefully compiled, refers to each device by IP/DNS name, so you’d better make sure you recreate it consistently!

+

Enter phpIPAM. A tool designed to help home keeps as well as large organisations keep track of their IP (and VLAN, VRF, and AS number) allocations.

+

Ingredients

+
    +
  1. Docker swarm cluster with persistent shared storage
    2. +
  2. Traefik configured per design
    3. +
  3. DNS entry for the hostname (i.e. “phpipam.your-domain.com”) you intend to use for phpIPAM, pointed to your keepalived IPIP
    4. +
+

Preparation

+

Setup data locations

+

We’ll need several directories to bind-mount into our container, so create them in /var/data/phpipam:

+
mkdir /var/data/phpipam/databases-dump -p
+mkdir /var/data/runtime/phpipam -p
+

Prepare environment

+

Create phpipam.env, and populate with the following variables

+
# Setup for github, phpipam application
+OAUTH2_PROXY_CLIENT_ID=
+OAUTH2_PROXY_CLIENT_SECRET=
+OAUTH2_PROXY_COOKIE_SECRET=
+
+# For MariaDB/MySQL database
+MYSQL_ROOT_PASSWORD=imtoosecretformyshorts
+MYSQL_DATABASE=phpipam
+MYSQL_USER=phpipam
+MYSQL_PASSWORD=secret
+
+# phpIPAM-specific variables
+MYSQL_ENV_MYSQL_USER=phpipam
+MYSQL_ENV_MYSQL_PASSWORD=secret
+MYSQL_ENV_MYSQL_DB=phpipam
+MYSQL_ENV_MYSQL_HOST=db
+
+# For backup
+BACKUP_NUM_KEEP=7
+BACKUP_FREQUENCY=1d
+

Additionally, create phpipam-backup.env, and populate with the following variables:

+
# For MariaDB/MySQL database
+MYSQL_ROOT_PASSWORD=imtoosecretformyshorts
+MYSQL_DATABASE=phpipam
+MYSQL_USER=phpipam
+MYSQL_PASSWORD=secret
+
+# For backup
+BACKUP_NUM_KEEP=7
+BACKUP_FREQUENCY=1d
+

Create nginx.conf

+

I usually protect my stacks using an oauth proxy container in front of the app. This protects me from either accidentally exposing a platform to the world, or having a insecure platform accessed and abused.

+

In the case of phpIPAM, the oauth_proxy creates an additional complexity, since it passes the “Authorization” HTTP header to the phpIPAM container. phpIPAH then examines the header, determines that the provided username (my email address associated with my oauth provider) doesn’t match a local user account, and denies me access without the opportunity to retry.

+

The (dirty) solution I’ve come up with is to insert an Nginx instance in the path between the oauth_proxy and the phpIPAM container itself. Nginx can remove the authorization header, so that phpIPAM can prompt me to login with a web-based form.

+

Create /var/data/phpipam/nginx.conf as follows:

+
upstream app-upstream {
+    server app:80;
+}
+
+server {
+    listen 80;
+    server_name ~.;
+
+    # Just redirect everything to the upstream
+    # Yes, it's embarassing. We are just a mechanism to strip an AUTH header :(
+    location ^~ / {
+        proxy_pass http://app-upstream;
+    proxy_set_header       Authorization "";
+    }
+
+}
+

Setup Docker Swarm

+

Create a docker swarm config file in docker-compose syntax (v3), something like this:

+

!!! tip I share (with my patreon patrons) a private “premix” git repository, which includes necessary docker-compose and env files for all published recipes. This means that patrons can launch any recipe with just a git pull and a docker stack deploy 👍

+
version: '3'
+
+services:
+
+  db:
+    image: mariadb:10
+    env_file: /var/data/config/phpipam/phpipam.env
+    networks:
+      - internal
+    volumes:
+      - /var/data/runtime/phpipam/db:/var/lib/mysql
+
+  proxy:
+    image: funkypenguin/oauth2_proxy
+    env_file: /var/data/config/phpipam/phpipam.env
+    networks:
+      - internal
+      - traefik_public
+    deploy:
+      labels:
+        - traefik.frontend.rule=Host:phpipam.example.com
+        - traefik.docker.network=traefik_public
+        - traefik.port=4180
+    volumes:
+      - /var/data/config/phpipam/authenticated-emails.txt:/authenticated-emails.txt
+    command: |
+      -cookie-secure=false
+      -upstream=http://nginx
+      -redirect-url=https://phpipam.example.com
+      -http-address=http://0.0.0.0:4180
+      -email-domain=example.com
+      -provider=github
+      -authenticated-emails-file=/authenticated-emails.txt
+
+  # Wait, what? Why do we have an oauth_proxy _and_ an nginx frontend for a simple webapp?
+  # Well, it's a long story. Basically, the phpipam container sees the "auth" headers passed by the
+  # oauth_proxy, and decides to use these exclusively to authenticate users. So no web-based login form, just "access denied"
+  # To work around this, we add nginx reverse proxy to the mix. A PITA, but an easy way to solve without altering the PHPIPAM code
+  nginx:
+    image: nginx:latest
+    networks:
+      - internal
+    volumes:
+      - /var/data/phpipam/nginx.conf:/etc/nginx/conf.d/default.conf:ro
+
+  app:
+    image: pierrecdn/phpipam
+    env_file: /var/data/config/phpipam/phpipam.env
+    networks:
+      - internal
+
+  db-backup:
+    image: mariadb:10
+    env_file: /var/data/config/phpipam/phpipam.env
+    volumes:
+      - /var/data/phpipam/database-dump:/dump
+      - /etc/localtime:/etc/localtime:ro
+    entrypoint: |
+      bash -c 'bash -s <<EOF
+      trap "break;exit" SIGHUP SIGINT SIGTERM
+      sleep 2m
+      while /bin/true; do
+        mysqldump -h db --all-databases | gzip -c > /dump/dump_\`date +%d-%m-%Y"_"%H_%M_%S\`.sql.gz
+        (ls -t /dump/dump*.sql.gz|head -n $$BACKUP_NUM_KEEP;ls /dump/dump*.sql.gz)|sort|uniq -u|xargs rm -- {}
+        sleep $$BACKUP_FREQUENCY
+      done
+      EOF'
+    networks:
+      - internal
+
+networks:
+  traefik_public:
+    external: true
+  internal:
+    driver: overlay
+    ipam:
+      config:
+        - subnet: 172.16.47.0/24
+

!!! note Setup unique static subnets for every stack you deploy. This avoids IP/gateway conflicts which can otherwise occur when you’re creating/removing stacks a lot. See my list here.

+

Serving

+

Launch phpIPAM stack

+

Launch the phpIPAM stack by running docker stack deploy phpipam -c <path -to-docker-compose.yml>

+

Log into your new instance at https://YOUR-FQDN, and follow the on-screen prompts to set your first user/password.

+

Chef’s Notes

+
    +
  1. If you wanted to expose the phpIPAM UI directly, you could remove the oauth2_proxy and the nginx services from the design, and move the traefik_public-related labels directly to the phpipam container. You’d also need to add the traefik_public network to the phpipam container.
    2. +
+

Tip your waiter (donate) 👏

+

Did you receive excellent service? Want to make your waiter happy? (..and support development of current and future recipes!) See the support page for (free or paid) ways to say thank you! 👏

+

Your comments? 💬