From 9c14dc02e523fad3423564c2929bc520257d70c0 Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Sat, 6 May 2023 12:56:29 +0200 Subject: [PATCH] Move git module to included files in order to be able to modify the code Signed-off-by: Knut Ahlers --- .gitmodules | 3 - alpine-on-hetzner | 1 - alpine-on-hetzner/.dockerignore | 2 + alpine-on-hetzner/.gitignore | 3 + alpine-on-hetzner/Dockerfile | 40 +++++++ alpine-on-hetzner/LICENSE | 21 ++++ alpine-on-hetzner/README.md | 100 +++++++++++++++++ alpine-on-hetzner/alpine.pkr.hcl | 67 ++++++++++++ alpine-on-hetzner/default.json | 57 ++++++++++ alpine-on-hetzner/entrypoint.sh | 17 +++ alpine-on-hetzner/playbook.yml | 179 +++++++++++++++++++++++++++++++ 11 files changed, 486 insertions(+), 4 deletions(-) delete mode 100644 .gitmodules delete mode 160000 alpine-on-hetzner create mode 100644 alpine-on-hetzner/.dockerignore create mode 100644 alpine-on-hetzner/.gitignore create mode 100644 alpine-on-hetzner/Dockerfile create mode 100644 alpine-on-hetzner/LICENSE create mode 100644 alpine-on-hetzner/README.md create mode 100644 alpine-on-hetzner/alpine.pkr.hcl create mode 100644 alpine-on-hetzner/default.json create mode 100644 alpine-on-hetzner/entrypoint.sh create mode 100644 alpine-on-hetzner/playbook.yml diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index be673c7..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "alpine-on-hetzner"] - path = alpine-on-hetzner - url = https://github.com/MathiasPius/alpine-on-hetzner.git diff --git a/alpine-on-hetzner b/alpine-on-hetzner deleted file mode 160000 index 1d21f3e..0000000 --- a/alpine-on-hetzner +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1d21f3e35c65bcef267c0f98144d3e5d4ab3c4a3 diff --git a/alpine-on-hetzner/.dockerignore b/alpine-on-hetzner/.dockerignore new file mode 100644 index 0000000..e7222f0 --- /dev/null +++ b/alpine-on-hetzner/.dockerignore @@ -0,0 +1,2 @@ +manifests/ +cache/ diff --git a/alpine-on-hetzner/.gitignore b/alpine-on-hetzner/.gitignore new file mode 100644 index 0000000..4ebf2f0 --- /dev/null +++ b/alpine-on-hetzner/.gitignore @@ -0,0 +1,3 @@ +manifests/ +configs/ +cache/ diff --git a/alpine-on-hetzner/Dockerfile b/alpine-on-hetzner/Dockerfile new file mode 100644 index 0000000..9db0eeb --- /dev/null +++ b/alpine-on-hetzner/Dockerfile @@ -0,0 +1,40 @@ +ARG ALPINE_VERSION=3.16.0 +ARG PACKER_VERSION=1.8.0-r3 +ARG ANSIBLE_CORE_VERSION=2.13.0-r0 +ARG JQ_VERSION=1.6-r1 +ARG UID=1000 +ARG GID=1000 + +FROM alpine:$ALPINE_VERSION +ARG PACKER_VERSION +ARG ANSIBLE_CORE_VERSION +ARG JQ_VERSION +ARG UID +ARG GID + +RUN apk add --no-cache \ + ansible-core=$ANSIBLE_CORE_VERSION \ + packer=$PACKER_VERSION \ + jq=$JQ_VERSION + +RUN adduser ansible -u "$UID" -D -h /home/ansible "$GID" + +RUN mkdir -p /configs /manifests /cache \ + && chown ansible /manifests /configs /cache + +USER ansible +WORKDIR /home/ansible +COPY default.json default.json +COPY alpine.pkr.hcl alpine.pkr.hcl +COPY playbook.yml playbook.yml +COPY --chmod=u=rx,og= entrypoint.sh entrypoint.sh + +VOLUME /cache + +ENTRYPOINT ["/bin/sh", "entrypoint.sh"] +CMD ["default.json"] + +LABEL "dev.pius.alpine-on-hetzner.alpine.version"=$ALPINE_VERSION +LABEL "dev.pius.alpine-on-hetzner.pkgs.ansible-core.version"=$ANSIBLE_CORE_VERSION +LABEL "dev.pius.alpine-on-hetzner.pkgs.packer.version"=$PACKER_VERSION +LABEL "dev.pius.alpine-on-hetzner.pkgs.jq.version"=$JQ_VERSION \ No newline at end of file diff --git a/alpine-on-hetzner/LICENSE b/alpine-on-hetzner/LICENSE new file mode 100644 index 0000000..47ad020 --- /dev/null +++ b/alpine-on-hetzner/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Mathias Pius + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/alpine-on-hetzner/README.md b/alpine-on-hetzner/README.md new file mode 100644 index 0000000..9c95ec8 --- /dev/null +++ b/alpine-on-hetzner/README.md @@ -0,0 +1,100 @@ +This folder contains a modified copy of [MathiasPius/alpine-on-hetzner](https://github.com/MathiasPius/alpine-on-hetzner). + +---- + +# alpine-hetzner +Tool for building cloud-init ready Alpine snapshots on Hetzner Cloud. + +You can either run it as a docker container or as a regular packer build (see [entrypoint.sh](/entrypoint.sh) for hints on how), but this latter method is not officially supported. + +# Examples + +### Create an alpine image with the [default](/default.json) configuration +Running this will create an `alpine` snapshot within your Hetzner Cloud project, ready to use for creating new servers. See the [launching a server](#launching-a-server) section for how to test it! +```shell +docker run -it --rm -e "HCLOUD_TOKEN=" ghcr.io/mathiaspius/alpine-on-hetzner:latest +``` + +### Default image, with `doas` installed, and `template.local` as default hostname +Configuration values can be overwritten by creating new configuration file with just the changes you want, and supplying the path as an argument when running it. See [Custom Configuration](#custom-configuration) for technical details on how the values are merged. +```shell +mkdir -p configs +echo '{ + "packages": { "doas": "=6.8.1-r7" }, + "hostname": "template.local" +}' > configs/my-override.json + + +export HCLOUD_TOKEN=myHetznerCloudToken +docker run -it --rm \ + -e "HCLOUD_TOKEN" \ + -v "$(pwd)/configs:/configs" \ + ghcr.io/mathiaspius/alpine-on-hetzner:latest default.json /configs/my-override.json +``` + +There are a number of optional docker mounts you can use: +* `/manifests` contains the output manifests from the run. +* `/cache` used for caching the `apk-tools` package locally between runs. +* `/configs` used for providing [custom configuration files](#custom-configuration) to builds. + +# Custom Configuration +Any command arguments passed to the docker run invocation will be treated as paths to configuration files to merge into a single combined configuration file which is then fed into the packer build. + +The merge is a "deep merge", meaning you can only *add to* or *change* the configuration file not remove from it. If you want to remove a package from the default.json configuration for example you will have to create a copy of it without the package in question and use that as the basis for your build. + +### Adding a custom package to your image +In order to add a custom package like `nginx` for example you can create the following config file `configs/nginx.json` in your local directory: +```json +{ "packages": { "nginx": "" } } +``` +`packages` is a map where the keys are package names and the value is the version selector. The map is passed directly to an `apk add` command, see [this link](https://wiki.alpinelinux.org/wiki/Package_management#Holding_a_specific_package_back) for version-pinning syntax. + +When the container is then run like so: +```shell +docker run -it --rm \ + -e "HCLOUD_TOKEN" \ + -v "$(pwd)/configs:/configs" \ + ghcr.io/mathiaspius/alpine-on-hetzner:latest default.json /configs/nginx.json +``` +The package will be appended to `packages` array, like so, immediately before the packer build runs: +```json +{ + (...) + "packages": { + "openssh": "=8.8_p1-r1", + "syslinux": "=6.04_pre1-r9", + "linux-virt": "=5.15.16-r0", + "cloud-init": "@community=21.4-r0", + "nginx": "" + } +} +``` + +# What's in the finished snapshot? +See the [default.json](/default.json) config for a list of packages that will be installed into the snapshot if run without any arguments. + +[playbook.yml](/playbook.yml) contains the entire ansible playbook used for generating the snapshot. +[alpine.pkr.hcl](/alpine.pkr.hcl) contains the packer build configuration which uses the playbook above via the [Ansible Provisioner](https://www.packer.io/plugins/provisioners/ansible/ansible) + +# How it works +The docker image comes with packer, ansible and jq pre-installed (check labels for versions), and builds the [alpine.pkr.hcl](/alpine.pkr.hcl) build against your Hetzner Cloud project using your provided API key. The Packer build will boot a server in rescue mode, then format and install Alpine Linux onto the primary drive of the server. Once done, the server will be saved as a snapshot and shut down. You can then create Alpine Linux servers using the finished snapshot. + +# Launching a server +Servers built from the snapshot won't be immediately accessible because the root user is locked by default, but can be configured using the Hetzner interface. Use the following cloud-init config to enable root access and select an ssh key when creating the server to allow login: +```yaml +#cloud-config +disable_root: false +users: +- name: root + lock-passwd: false +``` + +# Development +I have a number of ideas I would like to explore: + +- [ ] Re-using or expanding this tool to provision Alpine Linux on dedicated servers, but maintaining the same configuration -interface. I've previously done a less refined version fo this project for dedicated servers [here](https://github.com/MathiasPius/hetzner-zfs-host) +- [ ] Splitting up configuration files so you can mix-and-match a little more. Would also allow optional *hardened* configurations for example which you could opt into for stricter security. +- [ ] Creating configuration files for older versions of Alpine Linux. +- [x] Pipelining alpine-on-hetzner docker image builds and perhaps more importantly.. +- [ ] .. Testing that they work. +- [x] Add more customization abilities to the configuration file. Being able to enable openrc services with a simple array for example would be simple to implement and very useful. diff --git a/alpine-on-hetzner/alpine.pkr.hcl b/alpine-on-hetzner/alpine.pkr.hcl new file mode 100644 index 0000000..633f334 --- /dev/null +++ b/alpine-on-hetzner/alpine.pkr.hcl @@ -0,0 +1,67 @@ +# Please see default.json for default values for these +variable "apk_tools_url" {} +variable "apk_tools_arch" {} +variable "apk_tools_version" {} +variable "apk_tools_checksum" {} + +variable "alpine_version" {} +variable "alpine_mirror" {} +variable "alpine_repositories" {} + +variable "boot_size" {} +variable "root_size" {} +variable "hostname" {} + +variable "packages" {} +variable "services" {} +variable "nameservers" {} +variable "extlinux_modules" {} +variable "kernel_features" {} +variable "kernel_modules" {} +variable "default_kernel_opts" {} +variable "sysctl" {} +variable "chroot_commands" {} + +locals { + timestamp = formatdate("DD-MM-YY.hh-mm-ss", timestamp()) + snapshot_id = sha1(uuidv4()) +} + +source "hcloud" "alpine" { + location = "fsn1" + server_type = "cx11" + image = "ubuntu-20.04" + rescue = "linux64" + ssh_username = "root" +} + +build { + name = "alpine" + + source "source.hcloud.alpine" { + snapshot_name = var.hostname + snapshot_labels = { + "alpine.pius.dev/timestamp" = local.timestamp + "alpine.pius.dev/alpine-version" = var.alpine_version + "alpine.pius.dev/snapshot-id" = local.snapshot_id + } + } + + provisioner "ansible" { + playbook_file = "playbook.yml" + extra_arguments = ["--extra-vars", "@config.json"] + } + + post-processor "manifest" { + output = "/manifests/${build.PackerRunUUID}.json" + strip_path = true + custom_data = merge({ + "alpine.pius.dev/alpine-version": var.alpine_version, + "alpine.pius.dev/packer-run-id": build.PackerRunUUID, + "alpine.pius.dev/snapshot-id": local.snapshot_id + }, zipmap( + formatlist("alpine.pius.dev/%s-version", keys(var.packages)), + values(var.packages) + )) + } +} diff --git a/alpine-on-hetzner/default.json b/alpine-on-hetzner/default.json new file mode 100644 index 0000000..ee64fa4 --- /dev/null +++ b/alpine-on-hetzner/default.json @@ -0,0 +1,57 @@ +{ + "apk_tools_version": "v2.12.9", + "apk_tools_arch": "x86_64", + "apk_tools_url": "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic//{{ apk_tools_version }}/{{ apk_tools_arch }}/apk.static", + "apk_tools_checksum": "sha256:5176da3d4c41f12a08b82809aca8e7e2e383b7930979651b8958eca219815af5", + + "alpine_version": "v3.15", + "alpine_mirror": "http://dl-cdn.alpinelinux.org/alpine", + "alpine_repositories": ["main", "community"], + + "boot_size": "+100m", + "root_size": "0", + + "hostname": "alpine", + + "packages": { + "openssh": "=8.8_p1-r1", + "syslinux": "=6.04_pre1-r9", + "linux-virt": "=5.15.16-r0", + "cloud-init": "@community=21.4-r0" + }, + + "services": { + "devfs": "sysinit", + "dmesg": "sysinit", + "mdev": "sysinit", + "hwdrivers": "sysinit", + + "hwclock": "boot", + "modules": "boot", + "sysctl": "boot", + "hostname": "boot", + "bootmisc": "boot", + "syslog": "boot", + "networking": "boot", + + "mount-ro": "shutdown", + "killprocs": "shutdown", + "savecache": "shutdown", + + "sshd": "default" + }, + + "nameservers": [ + "185.12.64.1", + "185.12.64.2", + "2a01:4ff:ff00::add:1", + "2a01:4ff:ff00::add:2" + ], + + "sysctl": {}, + "extlinux_modules": ["ext4"], + "kernel_features": ["base", "ext4", "keymap", "virtio"], + "kernel_modules": ["ipv6", "af_packet"], + "default_kernel_opts": ["quiet"], + "chroot_commands": [] +} \ No newline at end of file diff --git a/alpine-on-hetzner/entrypoint.sh b/alpine-on-hetzner/entrypoint.sh new file mode 100644 index 0000000..df9ea27 --- /dev/null +++ b/alpine-on-hetzner/entrypoint.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +export PACKER_CACHE_DIR=/cache/.cache +export PACKER_CONFIG_DIR=/cache/.config +export PACKER_ANSIBLE_DIR=/cache/.ansible + +# Combine all the configuration paths passed as arguments. +jq -s 'reduce .[] as $item ({}; . * $item)' "$@" > config.json + +echo "Combined configuration:" +cat config.json + + +echo "Starting Packer Build" +/usr/bin/packer build \ + -var-file="config.json" \ + . \ No newline at end of file diff --git a/alpine-on-hetzner/playbook.yml b/alpine-on-hetzner/playbook.yml new file mode 100644 index 0000000..423617a --- /dev/null +++ b/alpine-on-hetzner/playbook.yml @@ -0,0 +1,179 @@ +--- +- name: prepare environment + hosts: localhost + tasks: + - name: cache apk tools + get_url: + url: "{{ apk_tools_url }}" + dest: /cache/apk.static + checksum: "{{ apk_tools_checksum }}" + +- name: configure alpine + hosts: all + gather_facts: false + vars: + chroot_directory: /mnt + root_device_path: "/dev/sda" + tasks: + - name: deploy apk-tools to rescue system + copy: + src: /cache/apk.static + dest: apk + mode: ug=rwx,o=r + + - name: "zap all partitions on {{ root_device_path }} and create GPT table" + shell: "sgdisk --zap-all {{ root_device_path }}" + + - name: "create boot partition {{ root_device_path }}-boot" + shell: "sgdisk -g -n1:0:{{ boot_size }} -t1:8300 -c1:boot -A1:set:2 {{ root_device_path }}" + + - name: "create root partition {{ root_device_path }}-root" + shell: "sgdisk -g -n2:0:{{ root_size }} -t2:8300 -c2:root {{ root_device_path }}" + + - name: mount-drives + shell: | + umount -R /mnt + + mkfs.ext4 -q -L root /dev/disk/by-partlabel/root + mkfs.ext4 -m 0 -q -L boot /dev/disk/by-partlabel/boot + mount /dev/disk/by-partlabel/root {{ chroot_directory }} + mkdir -p {{ chroot_directory }}/boot + mount /dev/disk/by-partlabel/boot {{ chroot_directory }}/boot + + - name: initialize alpine-base in directory + shell: >- + ./apk -X {{ alpine_mirror }}/{{ alpine_version }}/{{ alpine_repositories[0] }} + -u + --allow-untrusted + --root /{{ chroot_directory }} + --initdb + add alpine-base + + - name: prepare chroot + shell: | + mount --bind /dev {{ chroot_directory }}/dev + mount --bind /proc {{ chroot_directory }}/proc + mount --bind /sys {{ chroot_directory }}/sys + + - name: copy resolv conf from the rescue system to the server + copy: + content: | + {% for nameserver in nameservers %} + nameserver {{ nameserver }} + {% endfor %} + dest: "{{ chroot_directory }}/etc/resolv.conf" + + - name: setup networking + copy: + content: | + auto lo + iface lo inet loopback + auto eth0 + iface eth0 inet dhcp + iface eth0 inet6 auto + dest: "{{ chroot_directory }}/etc/network/interfaces" + + - name: write out hostname file + copy: + dest: "{{ chroot_directory }}/etc/hostname" + content: "{{ hostname }}" + + - name: overwrite hosts file + copy: + dest: "{{ chroot_directory }}/etc/hosts" + content: | + 127.0.0.1 {{ hostname }} localhost localhost.localdomain + ::1 {{ hostname }} localhost localhost.localdomain + ::1 {{ hostname }} localhost ipv6-localhost ipv6-loopback + fe00::0 ipv6-localnet + ff00::0 ipv6-mcastprefix + ff02::1 ipv6-allnodes + ff02::2 ipv6-allrouters + ff02::3 ipv6-allhosts + + - name: define alpine repositories + copy: + dest: "{{ chroot_directory }}/etc/apk/repositories" + content: | + {% for repository in alpine_repositories %} + {% if loop.first %} + {{ alpine_mirror }}/{{ alpine_version }}/{{ repository }} + {% else %} + @{{ repository }} {{ alpine_mirror }}/{{ alpine_version }}/{{ repository }} + {% endif %} + {% endfor %} + + - name: install requisite packages + shell: | + chroot {{ chroot_directory }} apk add --no-cache {{ item.key }}{{ item.value }} + loop: "{{ packages | dict2items }}" + + - name: configure services + shell: | + chroot {{ chroot_directory }} rc-update add {{ item.key }} {{ item.value }} + loop: "{{ services | dict2items }}" + + - name: enable cloud-init + shell: | + chroot {{ chroot_directory }} setup-cloud-init + when: packages["cloud-init"] is defined + + - name: configure fstab + copy: + dest: "{{ chroot_directory }}/etc/fstab" + content: | + {{ root_device_path }}2 / ext4 defaults,noatime 0 0 + {{ root_device_path }}1 /boot ext4 defaults 0 2 + + - name: configure sysctl + copy: + dest: "{{ chroot_directory }}/etc/sysctl.conf" + content: | + {% for setting in sysctl | dict2items %} + {{ setting.key }} = {{ setting.value }} + {% endfor %} + + - name: configure kernel modules + copy: + dest: "{{ chroot_directory }}/etc/modules" + content: | + {% for module in kernel_modules %} + {{ module }} + {% endfor %} + + - name: configure extlinux + copy: + dest: "{{ chroot_directory }}/etc/update-extlinux.conf" + content: | + overwrite=1 + vesa_menu=0 + default_kernel_opts="{{ default_kernel_opts | join(" ") }}" + modules={{ extlinux_modules | join(",") }} + root={{ root_device_path }}2 + verbose=0 + hidden=1 + timeout=1 + default=lts + serial_port= + serial_baud=115200 + xen_opts=dom0_mem=384M + password='' + + - name: configure mkinitfs + copy: + dest: "{{ chroot_directory }}/etc/mkinitfs/mkinitfs.conf" + content: | + features="{{ kernel_features | join(" ") }}" + + - name: install boot + shell: | + chroot {{ chroot_directory }} update-extlinux + chroot {{ chroot_directory }} extlinux -i /boot + chroot {{ chroot_directory }} dd bs=440 conv=notrunc count=1 if=/usr/share/syslinux/gptmbr.bin of={{ root_device_path }} + + - name: execute arbitrary commands + shell: | + chroot {{ chroot_directory }} sh <