From af5c00659a9807bcddcbda0d12038b39da2925b5 Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Sat, 8 Jun 2024 00:38:28 +0200 Subject: [PATCH] Update build tooling Signed-off-by: Knut Ahlers --- Makefile | 2 +- README.md | 9 ++-- repo-urls | 1 + scripts/update-all.sh | 120 +++++++++++++++++++++++++++++++++++++++++ scripts/update-repo.sh | 54 ------------------- 5 files changed, 125 insertions(+), 61 deletions(-) create mode 100644 scripts/update-all.sh delete mode 100755 scripts/update-repo.sh diff --git a/Makefile b/Makefile index 37ff87b..915bf1b 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ list_packages: bash ./scripts/listing.sh >$(REPO_DIR)/packages.txt repo_update: check_tools load_ssh_key - bash -euo pipefail -c 'for repo in $$(grep -v "^#" repo-urls | cut -d "#" -f 1); do script_level=1 ./scripts/update-repo.sh $${repo}; done' + bash ./scripts/update-all.sh sign_database: repo-add -s --key $(REPOKEY) $(DATABASE) diff --git a/README.md b/README.md index 928e4b1..c4496a9 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ This repository contains four essential parts: - The `scripts` folder containing bash scripts to control all actions - The `Makefile` to orchestrate everything. The main functionality is the `maintenance` target (or just `make`) -- The package lists (`aur-packages` and `repo-urls`) +- The package lists in `repo-urls` - The `repo/luzifer.asc` public key It currently relies on my [`luzifer/arch-repo-builder`](https://github.com/luzifer-docker/arch-repo-builder) docker image which does all of the building within a clean environment for each package. @@ -20,9 +20,7 @@ For the initial setup you need to do some steps: - Adjust the `Makefile` as you need different `download` and `upload` targets - Create an empty database `tar -cJf repo/luzifer.db.tar.xz -T /dev/null` (adjust the filename) - Put the public key for your repo into `repo/luzifer.asc` (filename should match the database, makes it easier to find) -- Set up your `aur-packages` and `repo-urls` package lists - - `aur-packages` contains just names of AUR packages (no comments or other stuff!) - - `repo-urls` contains one git repository URL per line (comments allowed) +- Set up your `repo-urls` package list: it contains one git repository URL per line (comments allowed) - Provide a docker daemon and all tools listed in the `check_tools` target of the `Makefile` Afterwards you should be good to just `make` your first build. Depending on the number of packages you selected to be in your repo you might go and fetch dinner while it builds. @@ -34,5 +32,4 @@ The repo should be updated on a regular base by just executing `make` on it. Thi ## Flaws / Remarks / TODOs - The whole build already strongly relies on Archlinux tools so will not run on any other distro -- For `aur-packages` having dynamic `pkgver` calculation the update check will not work properly until the `PKGBUILD` in AUR is updated (those packages can be built manually using `bash ./scripts/update-aur.sh && make do_cleanup upload` -- For `repo-urls` the same applies: for example my `vim-go-tools` package relies on periodic re-builds which are not executed as commits are quite rare. For those packages at the moment a call to `bash ./scripts/update-repo.sh && make do_cleanup upload` is required +- For packages having dynamic `pkgver` calculation the update check will not work properly until the `PKGBUILD` in the repo is updated. You can force a rebuild by removing the corresponding line from the `.repo_cache` diff --git a/repo-urls b/repo-urls index 3c5aeab..5e9e619 100644 --- a/repo-urls +++ b/repo-urls @@ -53,6 +53,7 @@ https://aur.archlinux.org/mozjpeg.git https://aur.archlinux.org/mpd-mpris.git https://aur.archlinux.org/nvm.git https://aur.archlinux.org/obs-cli.git +https://aur.archlinux.org/oda-file-converter.git https://aur.archlinux.org/oh-my-posh-bin.git https://aur.archlinux.org/orca-slicer-bin.git https://aur.archlinux.org/packer-plugin-arm-image.git diff --git a/scripts/update-all.sh b/scripts/update-all.sh new file mode 100644 index 0000000..f0548be --- /dev/null +++ b/scripts/update-all.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +set -euo pipefail + +BUILD_IMAGE="git.luzifer.io/registry/arch-repo-builder" +declare -A ICON=( + ["CHECK"]=$(printf "\e[34m\u3f\e[0m") + ["FAIL"]=$(printf "\e[31m\uf05e\e[0m") + ["RUN"]=$(printf "\e[32m\uf2f1\e[0m") + ["SKIP"]=$(printf "\e[1;30m\uf00c\e[0m") + ["SUCCESS"]=$(printf "\e[32m\uf00c\e[0m") + ["WAIT"]=$(printf "\e[1;30m\uf251\e[0m") +) +REPO_DIR=${REPO_DIR:-$(pwd)/repo} + +cleanup=() +last_line_len=0 + +function cleanup() { + rm -rf "${cleanup[@]}" +} + +function main() { + repo_list=( + $(grep -v "^#" ./repo-urls | cut -d "#" -f 1) + ) + + docker pull -q "${BUILD_IMAGE}" >/dev/null + + for repo in $(grep -v "^#" ./repo-urls | cut -d "#" -f 1); do + update ${repo} + echo + done +} + +function update() { + repo="${1}" + + write_status CHECK ${repo} "Checking build status..." + local last_remote_hash=$(git ls-remote ${repo} master | awk '{print $1}') + + if grep -q "${repo}#${last_remote_hash}" .repo_cache; then + write_status SKIP ${repo} "No changes from last build" + return + fi + + write_status RUN ${repo} "Build running..." + + # Create working dir + local tmpdir="/tmp/aur2repo_$(basename ${repo})" + mkdir -p "${tmpdir}/cfg" + + # Ensure cleanup on script exit + cleanup+=("${tmpdir}") + + write_status RUN ${repo} "Fetching signing key..." + vault read --field=key secret/jenkins/arch-signing >"${tmpdir}/cfg/signing.asc" + + write_status RUN ${repo} "Building package..." + local extra_opts=() + if [[ -f /etc/pacman.d/mirrorlist ]]; then + extra_opts+=(-v "/etc/pacman.d/mirrorlist:/etc/pacman.d/mirrorlist:ro") + fi + + local container=$( + docker run -d \ + -v "${tmpdir}/src:/src" \ + -v "${tmpdir}/cfg:/config" \ + -v "${REPO_DIR}:/repo" \ + -v "$(pwd)/scripts/pacman.conf:/etc/pacman.conf:ro" \ + "${extra_opts[@]}" \ + --ulimit nofile=262144:262144 \ + "${BUILD_IMAGE}" \ + "${repo}" + ) + + local status="running" + while [[ $status == "running" ]]; do + status=$(docker inspect ${container} | jq -r '.[0].State.Status') + local started=$(date -d $(docker inspect ${container} | + jq -r '.[0].State.StartedAt') +%s) + write_status RUN ${repo} "Building package in container $(printf "%.12s" ${container}) for $(($(date +%s) - started))s..." + sleep 1 + done + + local exitcode=$(docker inspect ${container} | jq -r '.[0].State.ExitCode') + if [ $exitcode -gt 0 ]; then + local logfile="/tmp/arch-package-build_$(basename ${repo}).log" + docker logs ${container} 2>&1 >${logfile} + write_status FAIL ${repo} "Build failed (${exitcode}), see logs at ${logfile}" + else + write_status SUCCESS ${repo} "Updating cache entry..." + + grep -v "^${repo}#" .repo_cache >.repo_cache.tmp || true + echo "${repo}#${last_remote_hash}" >>.repo_cache.tmp + mv .repo_cache.tmp .repo_cache + + write_status SUCCESS ${repo} "Build succeeded" + fi + + docker rm ${container} >/dev/null +} + +function write_status() { + local icon="${ICON[$1]}" + local name="${2%%.git}" + local comment="$3" + + # Wipe previous printed line + local wipe=$(echo -en "\r$(printf "%-${last_line_len}s" "")") + + # Assemble new line + local line=$(echo -en "\r${icon} \e[1m${name##*/}\e[0m - ${comment}") + # Count line length for later wipe + last_line_len=$(wc -c <<<"${line}") + # Print line + echo -en "${wipe}${line}" +} + +trap cleanup EXIT +main diff --git a/scripts/update-repo.sh b/scripts/update-repo.sh deleted file mode 100755 index 2d5c865..0000000 --- a/scripts/update-repo.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -set -euo pipefail - -source ./scripts/script_framework.sh - -REPO_DIR=${REPO_DIR:-$(pwd)} - -REPO=${1:-} -[ -z "${REPO}" ] && fail "No repo given as CLI argument" - -step "Checking for changes from last build" -last_remote_hash=$(git ls-remote ${REPO} master | awk '{print $1}') -grep "${REPO}#${last_remote_hash}" .repo_cache && { - warn "Remote has no changes from last build, skipping..." - exit 0 -} || true - -# Create working dir -TMPDIR="/tmp/aur2repo_$(basename ${REPO})" -mkdir -p "${TMPDIR}/cfg" - -# Ensure cleanup on script exit -function cleanup() { - rm -rf "${TMPDIR}" -} -trap cleanup EXIT - -step "Fetching signing key" -vault read --field=key secret/jenkins/arch-signing >"${TMPDIR}/cfg/signing.asc" - -step "Re-fetching Docker image" -docker pull git.luzifer.io/registry/arch-repo-builder - -step "Building package $(basename ${REPO})" -extra_opts=() -if [[ -f /etc/pacman.d/mirrorlist ]]; then - info "Using host system mirrorlist..." - extra_opts+=(-v "/etc/pacman.d/mirrorlist:/etc/pacman.d/mirrorlist:ro") -fi - -docker run --rm -ti \ - -v "${TMPDIR}/src:/src" \ - -v "${TMPDIR}/cfg:/config" \ - -v "${REPO_DIR}:/repo" \ - -v "$(pwd)/scripts/pacman.conf:/etc/pacman.conf:ro" \ - "${extra_opts[@]}" \ - --ulimit nofile=262144:262144 \ - git.luzifer.io/registry/arch-repo-builder \ - "${REPO}" - -step "Updating cache entry" -grep -v "^${REPO}#" .repo_cache >.repo_cache.tmp || true -echo "${REPO}#${last_remote_hash}" >>.repo_cache.tmp -mv .repo_cache.tmp .repo_cache