From 4d7a93c535ceeb720dc1873bfa61531099b611cb Mon Sep 17 00:00:00 2001 From: Jakob Odersky Date: Mon, 22 Oct 2018 04:02:44 -0700 Subject: Initial commit --- Makefile | 5 ++ README.md | 77 +++++++++++++++++ packages/.dockerignore | 3 + packages/.gitignore | 1 + packages/Dockerfile | 25 ++++++ packages/Dockerfile.base | 11 +++ packages/Makefile | 90 ++++++++++++++++++++ packages/crashbox-config/base/20auto-upgrades | 2 + packages/crashbox-config/debian/changelog | 5 ++ packages/crashbox-config/debian/compat | 1 + packages/crashbox-config/debian/control | 38 +++++++++ packages/crashbox-config/debian/copyright | 27 ++++++ .../debian/crashbox-base-config.install | 1 + .../debian/crashbox-base-config.postinst | 42 ++++++++++ .../debian/crashbox-git-config.cron.d | 1 + .../debian/crashbox-git-config.install | 3 + .../debian/crashbox-git-config.postinst | 45 ++++++++++ .../debian/crashbox-ip-config.install | 1 + .../debian/crashbox-ip-config.postinst | 40 +++++++++ .../debian/crashbox-nginx-config.install | 1 + .../debian/crashbox-nginx-config.postinst | 54 ++++++++++++ packages/crashbox-config/debian/rules | 18 ++++ packages/crashbox-config/debian/source/format | 1 + packages/crashbox-config/git/etc/cgitrc.d/crashbox | 63 ++++++++++++++ packages/crashbox-config/git/etc/gh-mirror | 4 + .../git/etc/nginx/sites-enabled/git.conf | 33 ++++++++ packages/crashbox-config/git/usr/bin/gh-mirror | 59 +++++++++++++ packages/crashbox-config/git/usr/bin/gh-mirror-all | 7 ++ .../crashbox-config/git/var/lib/git/www/about.md | 5 ++ .../git/var/lib/git/www/crashbox.png | Bin 0 -> 666 bytes .../git/var/lib/git/www/instagram.png | Bin 0 -> 44502 bytes packages/crashbox-config/ip/ip.conf | 13 +++ .../nginx/etc/nginx/conf.d/ssl.conf | 15 ++++ .../nginx/etc/nginx/sites-available/default.conf | 9 ++ packages/vm/customize.sh | 24 ++++++ terraform/.gitignore | 22 +++++ terraform/main.tf | 57 +++++++++++++ terraform/role/README.md | 4 + terraform/role/main.tf | 91 +++++++++++++++++++++ terraform/stdvps/README.md | 2 + terraform/stdvps/main.tf | 54 ++++++++++++ terraform/tf | 3 + terraform/volume/main.tf | 68 +++++++++++++++ 43 files changed, 1025 insertions(+) create mode 100644 Makefile create mode 100644 README.md create mode 100644 packages/.dockerignore create mode 100644 packages/.gitignore create mode 100644 packages/Dockerfile create mode 100644 packages/Dockerfile.base create mode 100644 packages/Makefile create mode 100644 packages/crashbox-config/base/20auto-upgrades create mode 100644 packages/crashbox-config/debian/changelog create mode 100644 packages/crashbox-config/debian/compat create mode 100644 packages/crashbox-config/debian/control create mode 100644 packages/crashbox-config/debian/copyright create mode 100644 packages/crashbox-config/debian/crashbox-base-config.install create mode 100644 packages/crashbox-config/debian/crashbox-base-config.postinst create mode 100644 packages/crashbox-config/debian/crashbox-git-config.cron.d create mode 100644 packages/crashbox-config/debian/crashbox-git-config.install create mode 100644 packages/crashbox-config/debian/crashbox-git-config.postinst create mode 100644 packages/crashbox-config/debian/crashbox-ip-config.install create mode 100644 packages/crashbox-config/debian/crashbox-ip-config.postinst create mode 100644 packages/crashbox-config/debian/crashbox-nginx-config.install create mode 100644 packages/crashbox-config/debian/crashbox-nginx-config.postinst create mode 100755 packages/crashbox-config/debian/rules create mode 100644 packages/crashbox-config/debian/source/format create mode 100644 packages/crashbox-config/git/etc/cgitrc.d/crashbox create mode 100644 packages/crashbox-config/git/etc/gh-mirror create mode 100644 packages/crashbox-config/git/etc/nginx/sites-enabled/git.conf create mode 100755 packages/crashbox-config/git/usr/bin/gh-mirror create mode 100755 packages/crashbox-config/git/usr/bin/gh-mirror-all create mode 100644 packages/crashbox-config/git/var/lib/git/www/about.md create mode 100644 packages/crashbox-config/git/var/lib/git/www/crashbox.png create mode 100644 packages/crashbox-config/git/var/lib/git/www/instagram.png create mode 100644 packages/crashbox-config/ip/ip.conf create mode 100644 packages/crashbox-config/nginx/etc/nginx/conf.d/ssl.conf create mode 100644 packages/crashbox-config/nginx/etc/nginx/sites-available/default.conf create mode 100755 packages/vm/customize.sh create mode 100644 terraform/.gitignore create mode 100644 terraform/main.tf create mode 100644 terraform/role/README.md create mode 100644 terraform/role/main.tf create mode 100644 terraform/stdvps/README.md create mode 100644 terraform/stdvps/main.tf create mode 100755 terraform/tf create mode 100644 terraform/volume/main.tf diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e52ddd7 --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +all: + $(MAKE) -C packages all + (cd terraform && bash -c "terraform apply -var-file=<(pass infra/terraform)") + +.PHONY: all diff --git a/README.md b/README.md new file mode 100644 index 0000000..214f154 --- /dev/null +++ b/README.md @@ -0,0 +1,77 @@ +## Quasi-Immutable Infrastructure for Small-Scale Deployments + +This project combines [terraform](https://www.terraform.io) with +[debian configuration packages](https://wiki.debian.org/Packaging) to +manage infrastructure automatically. + +It has a very opinionated structure, ideal for managing a small amount +of (personal) servers. + +It is used to manage the crashbox.io services. + +## Overview + +Management of infrastructure revolves around two central concepts: + +1. Provisioning of infrastructure, such as virtual private + servers and DNS entries, with terraform. + +2. Configuration of servers with debian packages. Custom debian + packages integrate easily into the debian ecosystem and provide a + robust way of managing files. + +These two concepts are brought together by *roles* which aggregate DNS +entries and packages. + +**In a nutshell, all infrastructure is configured by assigning sets of +roles to servers. A role will apply a debian configuration package to +a server and create a CNAME to the server's A record.** + +For example, assigning the `ip` role to server `server.crashbox.io` will: + +1. Create the server and A record if it isn't already there. +2. Install the package `crashbox-ip-config` on the server. +3. Create a DNS CNAME, aliasing `ip.crashbox.io` to `server.crashbox.io`. + +In the given example, the ip-config package will ensure a webserver is +installed and configure it to serve an ip address echo website. + +## Structure + +- Provisioning scripts are in `terraform/`. + +- Configuration package sources are in `packages/`. Note that for a + given role ``, the corresponding debian package is + `crasbox--config`. + +## Running + +### Bootstrap + +Before infrastructure configuration can be automated, a couple of +bootstrapping steps need to be performed manually: + +0. Create accounts for the various providers specified in the +terraform configuration. + +1. Provision a storage space for the terraform state file. + +2. Install dependencies for this project: + - make + - debhelper + - debuild + - terraform + - pass + +### Apply + +Run `make` to apply configuration. + +## Note about immutability + +This project uses debian packages for stronger consistency guarantees +when removing packages. Nevertheless, it is recommended to completely +reprovision a server if a role is removed. + +Keeping in mind that the goal of this project is to automate +deployments, regular reprovisions are encouraged. diff --git a/packages/.dockerignore b/packages/.dockerignore new file mode 100644 index 0000000..1c44263 --- /dev/null +++ b/packages/.dockerignore @@ -0,0 +1,3 @@ +target/build/ +target/base.img +target/snapshot.qcow2 diff --git a/packages/.gitignore b/packages/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/packages/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/packages/Dockerfile b/packages/Dockerfile new file mode 100644 index 0000000..c09ce4a --- /dev/null +++ b/packages/Dockerfile @@ -0,0 +1,25 @@ +# Bare image for testing config packages. +# +# This image is configured to use apt-cacher-ng running on the host +# machine, and hence is designed to not require an internet connection +# to build and run (assuming the base image is cached). +# +# To enable access from a running container to the host's +# apt-cacher-ng, the following firewall rule is required: +# +# iptables -A INPUT -i docker0 -p tcp --dport 3142 -j ACCEPT + +FROM crashbox-config-base:latest + +RUN echo 'Acquire::http::Proxy "http://172.17.0.1:3142";' \ + > /etc/apt/apt.conf.d/00aptproxy + +COPY target/archive/ /usr/local/share/archive/ +RUN echo "deb [trusted=yes] file:/usr/local/share/archive ./" \ + > /etc/apt/sources.list.d/local-archive.list +RUN apt-get update \ + -o Dir::Etc::sourcelist="sources.list.d/local-archive.list" \ + -o Dir::Etc::sourceparts="-" \ + -o APT::Get::List-Cleanup="0" + +CMD ["bash"] diff --git a/packages/Dockerfile.base b/packages/Dockerfile.base new file mode 100644 index 0000000..a188b76 --- /dev/null +++ b/packages/Dockerfile.base @@ -0,0 +1,11 @@ +# Base image for the bare image for testing config packages (see Dockerfile). +# Note that building this image requires an internet connection. + +FROM debian:buster + +RUN echo 'deb http://httpredir.debian.org/debian buster main' > /etc/apt/sources.list +RUN echo 'Acquire::ForceIPv4 "true";' > /etc/apt/apt.conf.d/99force-ipv4 +RUN apt-get update +RUN apt-get install curl -y + +CMD ["bash"] diff --git a/packages/Makefile b/packages/Makefile new file mode 100644 index 0000000..e8b72d9 --- /dev/null +++ b/packages/Makefile @@ -0,0 +1,90 @@ +src-packages=crashbox-config + +all: target/archive + +target/build/%: % + mkdir --parent target/build + cp -rf $< $@ + (cd $@ && debuild -us -uc) + +target/build/Packages.gz: $(addprefix target/build/,$(src-packages)) + mkdir --parent target/build + (cd target/build && dpkg-scanpackages . | gzip -9c > Packages.gz) + +target/archive: target/build/Packages.gz + mkdir --parent $@ + cp \ + --target-directory $@ \ + $(wildcard target/build/*.deb) \ + $< + +# Create a base Debian image. +# +# The base image will configure apt to use apt-cacher-ng, which is +# required to be installed on the host. +target/base.img: + mkdir -p target + sudo vmdebootstrap \ + --verbose \ + --owner=$(shell whoami) \ + --size=3G \ + --mirror=http://127.0.0.1:3142/debian \ + --apt-mirror=http://10.0.2.2:3142/debian \ + --configure-apt \ + --distribution=buster \ + --sudo \ + --grub \ + --serial-console \ + --customize=./vm/customize.sh \ + --image $@ + +# Create a copy-on-write snapshot of the base image. +# VMs will use this image to enable quick testing and fast roll-back. +target/snapshot.qcow2: target/base.img + mkdir -p target + rm -f $@ + qemu-img create \ + -f qcow2 \ + -b $(notdir $<) $@ + +# Start a VM for testing config packages +run: target/snapshot.qcow2 target/archive + qemu-system-x86_64 \ + -enable-kvm \ + -machine q35,accel=kvm,kernel-irqchip=split \ + -m 1024 \ + -smp 2 \ + -device intel-iommu,intremap=on \ + -netdev user,\ + hostfwd=tcp::10022-:22,\ + hostfwd=tcp::10080-:80,\ + hostfwd=tcp::10443-:443,\ + id=net0 \ + -device e1000,netdev=net0 \ + -virtfs local,\ + path=target/archive,\ + mount_tag=host0,\ + security_model=mapped,\ + id=host0 \ + -drive format=qcow2,file=$< \ + -nographic \ + -monitor none \ + -serial stdio + +docker-base: + docker build --tag crashbox-config-base:latest -f Dockerfile.base . + +docker: $(archive) + docker build --tag crashbox-config . + docker run --tty --interactive crashbox-config + +clean: + rm -rf target/archive + rm -rf target/build + rm -rf target/snapshot.qcow2 + +dist-clean: + rm -rf target + +.PHONY: all run clean dist-clean +# apt install devscripts build-essential lintian qemu-kvm vmdebootstrap diff --git a/packages/crashbox-config/base/20auto-upgrades b/packages/crashbox-config/base/20auto-upgrades new file mode 100644 index 0000000..8d6d7c8 --- /dev/null +++ b/packages/crashbox-config/base/20auto-upgrades @@ -0,0 +1,2 @@ +APT::Periodic::Update-Package-Lists "1"; +APT::Periodic::Unattended-Upgrade "1"; diff --git a/packages/crashbox-config/debian/changelog b/packages/crashbox-config/debian/changelog new file mode 100644 index 0000000..4991b8d --- /dev/null +++ b/packages/crashbox-config/debian/changelog @@ -0,0 +1,5 @@ +crashbox-config (1) unstable; urgency=medium + + * Initial Release. + + -- Jakob Odersky Tue, 28 Aug 2018 21:47:21 -0700 diff --git a/packages/crashbox-config/debian/compat b/packages/crashbox-config/debian/compat new file mode 100644 index 0000000..b4de394 --- /dev/null +++ b/packages/crashbox-config/debian/compat @@ -0,0 +1 @@ +11 diff --git a/packages/crashbox-config/debian/control b/packages/crashbox-config/debian/control new file mode 100644 index 0000000..aacca52 --- /dev/null +++ b/packages/crashbox-config/debian/control @@ -0,0 +1,38 @@ +Source: crashbox-config +Section: admin +Priority: optional +Maintainer: Jakob Odersky +Build-Depends: debhelper (>= 11) +Standards-Version: 4.1.3 + +Package: crashbox-base-config +Architecture: any +Depends: ${shlibs:Depends}, ${misc:Depends}, apt-listchanges, ca-certificates, curl, jq, openssl, rsync, ufw, unattended-upgrades, wget, sudo +Provides: ${diverted-files} +Conflicts: ${diverted-files} +Description: configuration for base system + Adds local customizations to the base system configuration. + +Package: crashbox-nginx-config +Architecture: any +Depends: ${shlibs:Depends}, ${misc:Depends}, ssl-cert, nginx, crashbox-base-config +Provides: ${diverted-files} +Conflicts: ${diverted-files} +Description: local nginx configuration + Adds local customizations to nginx config + +Package: crashbox-ip-config +Architecture: any +Depends: ${shlibs:Depends}, ${misc:Depends}, crashbox-nginx-config +Provides: ${diverted-files} +Conflicts: ${diverted-files} +Description: what-is-my-ip website + Adds an nginx site that echoes back a remote IP address + +Package: crashbox-git-config +Architecture: any +Depends: ${shlibs:Depends}, ${misc:Depends}, crashbox-nginx-config, cgit, python3-pygments, python3-markdown, git-core, fcgiwrap, adduser +Provides: ${diverted-files} +Conflicts: ${diverted-files} +Description: cgit web interface + Adds an nginx site that serves a CGit instance \ No newline at end of file diff --git a/packages/crashbox-config/debian/copyright b/packages/crashbox-config/debian/copyright new file mode 100644 index 0000000..ac7fbf4 --- /dev/null +++ b/packages/crashbox-config/debian/copyright @@ -0,0 +1,27 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: crashbox-config + +Files: * +Copyright: 2018 Jakob Odersky +License: GPL-3.0+ + +Files: debian/* +Copyright: 2018 Jakob Odersky +License: GPL-3.0+ + +License: GPL-3.0+ + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + . + This package is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + . + You should have received a copy of the GNU General Public License + along with this program. If not, see . + . + On Debian systems, the complete text of the GNU General + Public License version 3 can be found in "/usr/share/common-licenses/GPL-3". \ No newline at end of file diff --git a/packages/crashbox-config/debian/crashbox-base-config.install b/packages/crashbox-config/debian/crashbox-base-config.install new file mode 100644 index 0000000..ef80655 --- /dev/null +++ b/packages/crashbox-config/debian/crashbox-base-config.install @@ -0,0 +1 @@ +base/20auto-upgrades etc/apt/apt.conf.d/ diff --git a/packages/crashbox-config/debian/crashbox-base-config.postinst b/packages/crashbox-config/debian/crashbox-base-config.postinst new file mode 100644 index 0000000..b48f01f --- /dev/null +++ b/packages/crashbox-config/debian/crashbox-base-config.postinst @@ -0,0 +1,42 @@ +#!/bin/sh +# postinst script for crashbox-base-config +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `configure' +# * `abort-upgrade' +# * `abort-remove' `in-favour' +# +# * `abort-remove' +# * `abort-deconfigure' `in-favour' +# `removing' +# +# for details, see https://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + configure) + ufw allow 22/tcp || true + ufw default deny || true + ufw --force enable || true + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/packages/crashbox-config/debian/crashbox-git-config.cron.d b/packages/crashbox-config/debian/crashbox-git-config.cron.d new file mode 100644 index 0000000..d9cadfd --- /dev/null +++ b/packages/crashbox-config/debian/crashbox-git-config.cron.d @@ -0,0 +1 @@ +0 0 * * * git /usr/bin/gh-mirror-all \ No newline at end of file diff --git a/packages/crashbox-config/debian/crashbox-git-config.install b/packages/crashbox-config/debian/crashbox-git-config.install new file mode 100644 index 0000000..a7d3e36 --- /dev/null +++ b/packages/crashbox-config/debian/crashbox-git-config.install @@ -0,0 +1,3 @@ +git/etc/* etc +git/usr/* usr +git/var/* var diff --git a/packages/crashbox-config/debian/crashbox-git-config.postinst b/packages/crashbox-config/debian/crashbox-git-config.postinst new file mode 100644 index 0000000..774869e --- /dev/null +++ b/packages/crashbox-config/debian/crashbox-git-config.postinst @@ -0,0 +1,45 @@ +#!/bin/sh +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `configure' +# * `abort-upgrade' +# * `abort-remove' `in-favour' +# +# * `abort-remove' +# * `abort-deconfigure' `in-favour' +# `removing' +# +# for details, see https://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + configure) + adduser --group --system --home /var/lib/git git + mkdir -p /srv/git + chown -R git:git /srv/git + mkdir -p /var/lib/git/www/ + ln -s /usr/share/cgit/cgit.css /var/lib/git/www/cgit.css + ln -s /usr/share/cgit/robots.txt /var/lib/git/www/robots.txt + deb-systemd-invoke restart nginx + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/packages/crashbox-config/debian/crashbox-ip-config.install b/packages/crashbox-config/debian/crashbox-ip-config.install new file mode 100644 index 0000000..2646928 --- /dev/null +++ b/packages/crashbox-config/debian/crashbox-ip-config.install @@ -0,0 +1 @@ +ip/ip.conf etc/nginx/sites-enabled/ diff --git a/packages/crashbox-config/debian/crashbox-ip-config.postinst b/packages/crashbox-config/debian/crashbox-ip-config.postinst new file mode 100644 index 0000000..90e58d6 --- /dev/null +++ b/packages/crashbox-config/debian/crashbox-ip-config.postinst @@ -0,0 +1,40 @@ +#!/bin/sh +# postinst script for crashbox-ip-config +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `configure' +# * `abort-upgrade' +# * `abort-remove' `in-favour' +# +# * `abort-remove' +# * `abort-deconfigure' `in-favour' +# `removing' +# +# for details, see https://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + configure) + deb-systemd-invoke restart nginx + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/packages/crashbox-config/debian/crashbox-nginx-config.install b/packages/crashbox-config/debian/crashbox-nginx-config.install new file mode 100644 index 0000000..f2ed0d3 --- /dev/null +++ b/packages/crashbox-config/debian/crashbox-nginx-config.install @@ -0,0 +1 @@ +nginx/etc/* etc diff --git a/packages/crashbox-config/debian/crashbox-nginx-config.postinst b/packages/crashbox-config/debian/crashbox-nginx-config.postinst new file mode 100644 index 0000000..7a22244 --- /dev/null +++ b/packages/crashbox-config/debian/crashbox-nginx-config.postinst @@ -0,0 +1,54 @@ +#!/bin/sh +# postinst script for crashbox-nginx-config +# +# see: dh_installdeb(1) + +set -e + +# summary of how this script can be called: +# * `configure' +# * `abort-upgrade' +# * `abort-remove' `in-favour' +# +# * `abort-remove' +# * `abort-deconfigure' `in-favour' +# `removing' +# +# for details, see https://www.debian.org/doc/debian-policy/ or +# the debian-policy package + + +case "$1" in + configure) + ln -f -s /etc/nginx/sites-available/default.conf /etc/nginx/sites-enabled/default + usermod --append --groups ssl-cert www-data + ufw allow 80/tcp + ufw allow 443/tcp + + if [ ! -r /etc/ssl/private/server.key.pem ] \ + || [ ! -r /etc/ssl/server.cert.pem ] \ + || [ ! -r /etc/ssl/issuer.cert.pem ]; then + ln -f -s /etc/ssl/private/ssl-cert-snakeoil.key /etc/ssl/private/server.key.pem + ln -f -s /etc/ssl/certs/ssl-cert-snakeoil.pem /etc/ssl/server.cert.pem + ln -f -s /etc/ssl/certs/ssl-cert-snakeoil.pem /etc/ssl/issuer.cert.pem + echo "WARNING: no certificates found, falling back to snakeoil certificates!" >&2 + fi + + deb-systemd-invoke restart nginx + ;; + + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +# dh_installdeb will replace this with shell code automatically +# generated by other debhelper scripts. + +#DEBHELPER# + +exit 0 diff --git a/packages/crashbox-config/debian/rules b/packages/crashbox-config/debian/rules new file mode 100755 index 0000000..9946432 --- /dev/null +++ b/packages/crashbox-config/debian/rules @@ -0,0 +1,18 @@ +#!/usr/bin/make -f +# See debhelper(7) (uncomment to enable) +# output every command that modifies files on the build system. +#export DH_VERBOSE = 1 + + +# see FEATURE AREAS in dpkg-buildflags(1) +#export DEB_BUILD_MAINT_OPTIONS = hardening=+all + +# see ENVIRONMENT in dpkg-buildflags(1) +# package maintainers to append CFLAGS +#export DEB_CFLAGS_MAINT_APPEND = -Wall -pedantic +# package maintainers to append LDFLAGS +#export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed + + +%: + dh $@ diff --git a/packages/crashbox-config/debian/source/format b/packages/crashbox-config/debian/source/format new file mode 100644 index 0000000..89ae9db --- /dev/null +++ b/packages/crashbox-config/debian/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/packages/crashbox-config/git/etc/cgitrc.d/crashbox b/packages/crashbox-config/git/etc/cgitrc.d/crashbox new file mode 100644 index 0000000..e95ff11 --- /dev/null +++ b/packages/crashbox-config/git/etc/cgitrc.d/crashbox @@ -0,0 +1,63 @@ +# +# cgit config +# see cgitrc(5) for details +# +# https://git.zx2c4.com/cgit/tree/cgitrc.5.txt + +favicon=/crashbox.png +logo=/crashbox.png +root-title=git.crashbox.io +root-desc=Git repositories hosted at crashbox.io +root-readme=/var/lib/git/www/about.md +clone-url=https://git.crashbox.io/$CGIT_REPO_URL + +## List of common mimetypes +mimetype.gif=image/gif +mimetype.html=text/html +mimetype.jpg=image/jpeg +mimetype.jpeg=image/jpeg +mimetype.pdf=application/pdf +mimetype.png=image/png +mimetype.svg=image/svg+xml +mimetype-file=/etc/mime.types + +# Don't show owner on index page +enable-index-owner=0 + +# Enable blame page and create links to it from tree page +enable-blame=1 + +# Enable ASCII art commit history graph on the log pages +enable-commit-graph=1 + +# Show extra links for each repository on the index page +enable-index-links=1 + +# Show number of affected files per commit on the log pages +enable-log-filecount=1 + +# Show number of added/removed lines per commit on the log pages +enable-log-linecount=1 + +# Allow download of tar.gz, tar.bz2 and zip-files +snapshots=tar.gz tar.bz2 zip + +# Highlight code +source-filter=/usr/lib/cgit/filters/syntax-highlighting.py + +# Format "about" files such as markdown readmes +about-filter=/usr/lib/cgit/filters/about-formatting.sh +readme=master:README.md + +# nginx handles negotiating git clones +enable-http-clone=0 + +section-from-path=-1 + +# Remove ".git" suffix in listings +remove-suffix=1 + +# Base URL +virtual-root=/ + +scan-path=/srv/git diff --git a/packages/crashbox-config/git/etc/gh-mirror b/packages/crashbox-config/git/etc/gh-mirror new file mode 100644 index 0000000..4fc987b --- /dev/null +++ b/packages/crashbox-config/git/etc/gh-mirror @@ -0,0 +1,4 @@ +users jodersky /srv/git/mirrors/github/jodersky +orgs project-condor /srv/git/mirrors/github/project-condor +orgs driver-oss /srv/git/mirrors/github/driver-oss +orgs johnandjohn /srv/git/mirrors/github/johnandjohn diff --git a/packages/crashbox-config/git/etc/nginx/sites-enabled/git.conf b/packages/crashbox-config/git/etc/nginx/sites-enabled/git.conf new file mode 100644 index 0000000..7210dbc --- /dev/null +++ b/packages/crashbox-config/git/etc/nginx/sites-enabled/git.conf @@ -0,0 +1,33 @@ +server { + server_name git.*; + listen 80; + listen [::]:80; + listen 443 ssl; + listen [::]:443 ssl; + + root /var/lib/git/www; + + # requests that should to go to git-http-backend + location ~ ^.*/(HEAD|info/refs|objects/info/.*|git-(upload|receive)-pack)$ { + root /srv/git; + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME /usr/lib/git-core/git-http-backend; + fastcgi_param GIT_PROJECT_ROOT /srv/git; + fastcgi_param GIT_HTTP_EXPORT_ALL ""; + fastcgi_param PATH_INFO $uri; + fastcgi_pass unix:/run/fcgiwrap.socket; + } + + location @cgit { + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME /usr/lib/cgit/cgit.cgi; + fastcgi_param CGIT_CONFIG /etc/cgitrc.d/crashbox; + fastcgi_param PATH_INFO $uri; + fastcgi_pass unix:/run/fcgiwrap.socket; + } + + location / { + try_files $uri @cgit; + } + +} diff --git a/packages/crashbox-config/git/usr/bin/gh-mirror b/packages/crashbox-config/git/usr/bin/gh-mirror new file mode 100755 index 0000000..54985cb --- /dev/null +++ b/packages/crashbox-config/git/usr/bin/gh-mirror @@ -0,0 +1,59 @@ +#!/bin/bash +# Mirror repositories from GitHub +# +# Arguments: (users|orgs) +# +# Clones (or updates) all repositories of a GitHub user or +# organization. Repositories are created as children of the given +# output directory. +# +# Example: +# gh-mirror users jodersky mirrors/github/jodersky +# +# This script uses GitHub's API, version 3 +# https://developer.github.com/v3/repos/#list-user-repositories +set -o errexit + +account_type="$1" +account_name="$2" +out_dir="${3:-.}" +mkdir -p "$out_dir" + +if [[ -z $account_type ]] || [[ -z $account_name ]]; then + echo "Usage: (users|orgs) " >&2 + exit 1 +fi + +tmp="$(mktemp /tmp/mirror-XXXXXXXXXXXX)" +url="https://api.github.com/$account_type/$account_name/repos?per_page=100" + +function finish { + echo "An error was encountered." >&2 + echo "curl headers are saved in $tmp" >&2 +} +trap finish ERR + +while [[ ! -z "$url" ]]; do + echo "Fetching $url..." >&2 + + mapfile -t repo_data < <(curl --dump-header "$tmp" "$url" | jq --compact-output '.[]') + url="$(< "$tmp" grep Link | grep -oE "[a-zA-Z0-9:/?=.&_]*>; rel=.next" | cut -d'>' -f1)" + + for repo in "${repo_data[@]}"; do + clone_url="$(echo "$repo" | jq -r .clone_url)" + project="$(basename "$clone_url")" + description=$(echo "$repo" | jq -r .description) + + git_dir="$out_dir/$project" + + if [ -d "$git_dir" ]; then + echo "updating $project" >&2 + git -C "$git_dir" fetch --prune + else + echo "mirroring new $project" >&2 + git clone --mirror "$clone_url" "$git_dir" + fi + echo "$description" > "$git_dir/description" + done +done +rm "$tmp" diff --git a/packages/crashbox-config/git/usr/bin/gh-mirror-all b/packages/crashbox-config/git/usr/bin/gh-mirror-all new file mode 100755 index 0000000..fa9054f --- /dev/null +++ b/packages/crashbox-config/git/usr/bin/gh-mirror-all @@ -0,0 +1,7 @@ +#!/bin/bash +mapfile -t lines < /etc/gh-mirror + +for line in "${lines[@]}"; do + read -r type name dir <<< "$line" + gh-mirror "$type" "$name" "$dir" +done diff --git a/packages/crashbox-config/git/var/lib/git/www/about.md b/packages/crashbox-config/git/var/lib/git/www/about.md new file mode 100644 index 0000000..55e68fa --- /dev/null +++ b/packages/crashbox-config/git/var/lib/git/www/about.md @@ -0,0 +1,5 @@ +Tracking of various git repositories. + +![instagram](instagram.png) + + diff --git a/packages/crashbox-config/git/var/lib/git/www/crashbox.png b/packages/crashbox-config/git/var/lib/git/www/crashbox.png new file mode 100644 index 0000000..632118e Binary files /dev/null and b/packages/crashbox-config/git/var/lib/git/www/crashbox.png differ diff --git a/packages/crashbox-config/git/var/lib/git/www/instagram.png b/packages/crashbox-config/git/var/lib/git/www/instagram.png new file mode 100644 index 0000000..dcaff14 Binary files /dev/null and b/packages/crashbox-config/git/var/lib/git/www/instagram.png differ diff --git a/packages/crashbox-config/ip/ip.conf b/packages/crashbox-config/ip/ip.conf new file mode 100644 index 0000000..2f3ab1e --- /dev/null +++ b/packages/crashbox-config/ip/ip.conf @@ -0,0 +1,13 @@ +# Echo remote IP address +# https://michael.lustfield.net/nginx/simple-ip-echo +server { + server_name ip.*; + listen 80; + listen [::]:80; + listen 443 ssl; + listen [::]:443 ssl; + location = / { + default_type text/plain; + echo $remote_addr; + } +} \ No newline at end of file diff --git a/packages/crashbox-config/nginx/etc/nginx/conf.d/ssl.conf b/packages/crashbox-config/nginx/etc/nginx/conf.d/ssl.conf new file mode 100644 index 0000000..bb96ec7 --- /dev/null +++ b/packages/crashbox-config/nginx/etc/nginx/conf.d/ssl.conf @@ -0,0 +1,15 @@ +# The configuration below can be obtained with the Mozilla SSL +# Configuration Generator at +# https://mozilla.github.io/server-side-tls/ssl-config-generator/ + +ssl_certificate /etc/ssl/server.cert.pem; +ssl_certificate_key /etc/ssl/private/server.key.pem; +ssl_session_timeout 1d; +ssl_session_cache shared:SSL:50m; +ssl_session_tickets off; + +ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256'; + +ssl_stapling on; +ssl_stapling_verify on; +ssl_trusted_certificate /etc/ssl/issuer.cert.pem; diff --git a/packages/crashbox-config/nginx/etc/nginx/sites-available/default.conf b/packages/crashbox-config/nginx/etc/nginx/sites-available/default.conf new file mode 100644 index 0000000..e10725d --- /dev/null +++ b/packages/crashbox-config/nginx/etc/nginx/sites-available/default.conf @@ -0,0 +1,9 @@ +# Default catch-all configuration, applied when no other configuration matches +server { + server_name _; + listen 80 default_server; + listen [::]:80 default_server; + + # close the connection without sending a response + return 444; +} \ No newline at end of file diff --git a/packages/vm/customize.sh b/packages/vm/customize.sh new file mode 100755 index 0000000..d4b2478 --- /dev/null +++ b/packages/vm/customize.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +abort() { + echo "$1" >&2 + exit 1 +} + +rootdir="$1" + +# avoid messing with host system +[[ -n $rootdir ]] || abort "root directory is not set" + +# set up local apt archive +mkdir -p $rootdir/usr/local/share/archive +echo "deb [trusted=yes] file:/usr/local/share/archive ./" > $rootdir/etc/apt/sources.list.d/local.list + +# mount local apt archive from host on startup +echo 9p >> $rootdir/etc/initramfs-tools/modules +echo 9pnet >> $rootdir/etc/initramfs-tools/modules +echo 9pnet_virtio >> $rootdir/etc/initramfs-tools/modules +echo "host0 /usr/local/share/archive 9p trans=virtio,version=9p2000.L 0 0" >> $rootdir/etc/fstab + +# boot immediately +sed --in-place 's/GRUB_TIMEOUT=[0-9]\+/GRUB_TIMEOUT=0/g' $rootdir/etc/default/grub diff --git a/terraform/.gitignore b/terraform/.gitignore new file mode 100644 index 0000000..4aa60fc --- /dev/null +++ b/terraform/.gitignore @@ -0,0 +1,22 @@ +# Local .terraform directories +**/.terraform/* +terraform.d/ + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log + +# Ignore any .tfvars files that are generated automatically for each Terraform run. Most +# .tfvars files are managed as part of configuration and so should be included in +# version control. +# +# example.tfvars + +# Ignore override files as they are usually used to override ressources locally +override.tf +override.tf.json +*_override.tf +*_override.tf.json \ No newline at end of file diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..8a5e599 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,57 @@ +variable "secret_hcloud_token" { + type = "string" +} + +variable "secret_cloudflare_token" { + type = "string" +} + +provider "hcloud" { + token = "${var.secret_hcloud_token}" +} + +provider "cloudflare" { + email = "jakob@odersky.com" + token = "${var.secret_cloudflare_token}" +} + +provider "acme" { + #server_url = "https://acme-staging-v02.api.letsencrypt.org/directory" + server_url = "https://acme-v02.api.letsencrypt.org/directory" +} + +################################################################################ + +# Main ssh key +resource "hcloud_ssh_key" "root" { + name = "root" + public_key = "${file("~/.ssh/id_rsa.pub")}" +} + +module "vps" { + source = "./stdvps" + location = "nbg1" + ssh_key_name = "${hcloud_ssh_key.root.name}" +} + +module "volume" { + source = "./volume" + server_id = "${module.vps.id}" + server_fqdn = "${module.vps.fqdn}" +} + +module "roles" { + source = "./role" + secret_cloudflare_token = "${var.secret_cloudflare_token}" + host = "${module.vps.fqdn}" + id = "${module.volume.server_id}" + roles = ["ip", "git"] +} + +output "vps_address" { + value = "${module.vps.fqdn}" +} + +output "vps_roles" { + value = "${join(" ", module.roles.roles)}" +} diff --git a/terraform/role/README.md b/terraform/role/README.md new file mode 100644 index 0000000..11e2e21 --- /dev/null +++ b/terraform/role/README.md @@ -0,0 +1,4 @@ +# Role-based configuration for standalone hosts. + +Applying a role to a host will install corresponding config packages +and create a role CNAME record to the host. diff --git a/terraform/role/main.tf b/terraform/role/main.tf new file mode 100644 index 0000000..e85fd3b --- /dev/null +++ b/terraform/role/main.tf @@ -0,0 +1,91 @@ +variable "host" { + type = "string" +} + +variable "id" { + type = "string" +} + +variable "roles" { + type = "list" +} + +variable "secret_cloudflare_token" { + type = "string" +} + +resource "tls_private_key" "private_key" { + algorithm = "RSA" +} + +resource "acme_registration" "reg" { + account_key_pem = "${tls_private_key.private_key.private_key_pem}" + email_address = "jakob@odersky.com" +} + +resource "acme_certificate" "certificate" { + account_key_pem = "${acme_registration.reg.account_key_pem}" + common_name = "${var.host}" + subject_alternative_names = "${formatlist("%s.crashbox.io", var.roles)}" + + dns_challenge { + provider = "cloudflare" + + config { + CLOUDFLARE_EMAIL = "jakob@odersky.com" + CLOUDFLARE_API_KEY = "${var.secret_cloudflare_token}" + } + } +} + +resource "cloudflare_record" "role_cname" { + count = "${length(var.roles)}" + + domain = "crashbox.io" + name = "${element(var.roles, count.index)}" + value = "${var.host}" + type = "CNAME" +} + +resource "null_resource" "role_config" { + triggers = { + host_id = "${var.id}" + config_packages = "${join(" ", sort(formatlist("crashbox-%s-config", var.roles)))}" + } + + connection { + host = "${var.host}" + } + + provisioner "file" { + content = "${acme_certificate.certificate.certificate_pem}" + destination = "/etc/ssl/server.cert.pem" + } + + provisioner "file" { + content = "${acme_certificate.certificate.issuer_pem}" + destination = "/etc/ssl/issuer.cert.pem" + } + + provisioner "file" { + content = "${acme_certificate.certificate.private_key_pem}" + destination = "/etc/ssl/private/server.key.pem" + } + + provisioner "file" { + source = "${path.root}/../packages/target/archive" + destination = "/usr/local/share/" + } + + provisioner "remote-exec" { + inline = [ + "echo deb [trusted=yes] file:/usr/local/share/archive ./ > /etc/apt/sources.list.d/local-archive.list", + "apt update --quiet=2", + "apt install --quiet=2 --yes ${null_resource.role_config.triggers.config_packages}", + ] + } +} + +output "roles" { + value = "${var.roles}" +} diff --git a/terraform/stdvps/README.md b/terraform/stdvps/README.md new file mode 100644 index 0000000..316641c --- /dev/null +++ b/terraform/stdvps/README.md @@ -0,0 +1,2 @@ +# stdvps +A basic cloud virtual machine. diff --git a/terraform/stdvps/main.tf b/terraform/stdvps/main.tf new file mode 100644 index 0000000..2328bb1 --- /dev/null +++ b/terraform/stdvps/main.tf @@ -0,0 +1,54 @@ +variable "ssh_key_name" { + type = "string" +} + +variable "location" { + type = "string" +} + +resource "random_id" "server" { + prefix = "peter-" + byte_length = 2 +} + +resource "hcloud_server" "server" { + name = "${random_id.server.hex}.crashbox.io" + image = "debian-9" + server_type = "cx11" + location = "${var.location}" + ssh_keys = ["${var.ssh_key_name}"] +} + +resource "cloudflare_record" "record_a" { + domain = "crashbox.io" + name = "${hcloud_server.server.name}" + value = "${hcloud_server.server.ipv4_address}" + type = "A" +} + +resource "cloudflare_record" "record_aaaa" { + domain = "crashbox.io" + name = "${hcloud_server.server.name}" + value = "${hcloud_server.server.ipv6_address}1" + type = "AAAA" +} + +output "ipv4" { + value = "${hcloud_server.server.ipv4_address}" +} + +output "ipv6" { + value = "${hcloud_server.server.ipv6_address}" +} + +output "fqdn" { + value = "${cloudflare_record.record_aaaa.hostname}" +} + +output "id" { + value = "${hcloud_server.server.id}" +} + +output "name" { + value = "${hcloud_server.server.name}" +} diff --git a/terraform/tf b/terraform/tf new file mode 100755 index 0000000..888e681 --- /dev/null +++ b/terraform/tf @@ -0,0 +1,3 @@ +#!/bin/bash + +exec terraform "$@" -var-file=<(pass infra/terraform) \ No newline at end of file diff --git a/terraform/volume/main.tf b/terraform/volume/main.tf new file mode 100644 index 0000000..828d85d --- /dev/null +++ b/terraform/volume/main.tf @@ -0,0 +1,68 @@ +variable "server_fqdn" { + type = "string" +} + +variable "server_id" { + type = "string" +} + +resource "hcloud_volume" "master" { + name = "master" + size = 50 + server_id = "${var.server_id}" +} + +# this is only run once if the volume id changes +resource "null_resource" "volume_format" { + triggers = { + volume_id = "${hcloud_volume.master.id}" + } + + connection { + host = "${var.server_fqdn}" + } + + provisioner "remote-exec" { + inline = [ + "mkfs.ext4 ${hcloud_volume.master.linux_device}", + ] + } +} + +resource "null_resource" "volume_mount" { + triggers = { + server_id = "${var.server_id}" + volume_id = "${hcloud_volume.master.id}" + } + + connection { + host = "${var.server_fqdn}" + } + + provisioner "file" { + content = <