Update build tooling

Signed-off-by: Knut Ahlers <knut@ahlers.me>

Makefile

@@ -51,7 +51,7 @@ list_packages:
 	bash ./scripts/listing.sh >$(REPO_DIR)/packages.txt
 
 repo_update: check_tools load_ssh_key
-	bash -euo pipefail -c 'for repo in $$(grep -v "^#" repo-urls | cut -d "#" -f 1); do script_level=1 ./scripts/update-repo.sh $${repo}; done'
+	bash ./scripts/update-all.sh
 
 sign_database:
 	repo-add -s --key $(REPOKEY) $(DATABASE)

README.md

@@ -10,7 +10,7 @@ This repository contains four essential parts:
 
 - The `scripts` folder containing bash scripts to control all actions
 - The `Makefile` to orchestrate everything. The main functionality is the `maintenance` target (or just `make`)
-- The package lists (`aur-packages` and `repo-urls`)
+- The package lists in `repo-urls`
 - The `repo/luzifer.asc` public key
 
 It currently relies on my [`luzifer/arch-repo-builder`](https://github.com/luzifer-docker/arch-repo-builder) docker image which does all of the building within a clean environment for each package.
@@ -20,9 +20,7 @@ For the initial setup you need to do some steps:
 - Adjust the `Makefile` as you need different `download` and `upload` targets
 - Create an empty database `tar -cJf repo/luzifer.db.tar.xz -T /dev/null` (adjust the filename)
 - Put the public key for your repo into `repo/luzifer.asc` (filename should match the database, makes it easier to find)
-- Set up your `aur-packages` and `repo-urls` package lists
+- Set up your `repo-urls` package list: it contains one git repository URL per line (comments allowed)
-  - `aur-packages` contains just names of AUR packages (no comments or other stuff!)
-  - `repo-urls` contains one git repository URL per line (comments allowed)
 - Provide a docker daemon and all tools listed in the `check_tools` target of the `Makefile`
 
 Afterwards you should be good to just `make` your first build. Depending on the number of packages you selected to be in your repo you might go and fetch dinner while it builds.
@@ -34,5 +32,4 @@ The repo should be updated on a regular base by just executing `make` on it. Thi
 ## Flaws / Remarks / TODOs
 
 - The whole build already strongly relies on Archlinux tools so will not run on any other distro
-- For `aur-packages` having dynamic `pkgver` calculation the update check will not work properly until the `PKGBUILD` in AUR is updated (those packages can be built manually using `bash ./scripts/update-aur.sh <packagename> && make do_cleanup upload`
+- For packages having dynamic `pkgver` calculation the update check will not work properly until the `PKGBUILD` in the repo is updated. You can force a rebuild by removing the corresponding line from the `.repo_cache`
-- For `repo-urls` the same applies: for example my `vim-go-tools` package relies on periodic re-builds which are not executed as commits are quite rare. For those packages at the moment a call to `bash ./scripts/update-repo.sh <url> && make do_cleanup upload` is required

repo-urls

@@ -53,6 +53,7 @@ https://aur.archlinux.org/mozjpeg.git
 https://aur.archlinux.org/mpd-mpris.git
 https://aur.archlinux.org/nvm.git
 https://aur.archlinux.org/obs-cli.git
+https://aur.archlinux.org/oda-file-converter.git
 https://aur.archlinux.org/oh-my-posh-bin.git
 https://aur.archlinux.org/orca-slicer-bin.git
 https://aur.archlinux.org/packer-plugin-arm-image.git

scripts/update-all.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+BUILD_IMAGE="git.luzifer.io/registry/arch-repo-builder"
+declare -A ICON=(
+  ["CHECK"]=$(printf "\e[34m\u3f\e[0m")
+  ["FAIL"]=$(printf "\e[31m\uf05e\e[0m")
+  ["RUN"]=$(printf "\e[32m\uf2f1\e[0m")
+  ["SKIP"]=$(printf "\e[1;30m\uf00c\e[0m")
+  ["SUCCESS"]=$(printf "\e[32m\uf00c\e[0m")
+  ["WAIT"]=$(printf "\e[1;30m\uf251\e[0m")
+)
+REPO_DIR=${REPO_DIR:-$(pwd)/repo}
+
+cleanup=()
+last_line_len=0
+
+function cleanup() {
+  rm -rf "${cleanup[@]}"
+}
+
+function main() {
+  repo_list=(
+    $(grep -v "^#" ./repo-urls | cut -d "#" -f 1)
+  )
+
+  docker pull -q "${BUILD_IMAGE}" >/dev/null
+
+  for repo in $(grep -v "^#" ./repo-urls | cut -d "#" -f 1); do
+    update ${repo}
+    echo
+  done
+}
+
+function update() {
+  repo="${1}"
+
+  write_status CHECK ${repo} "Checking build status..."
+  local last_remote_hash=$(git ls-remote ${repo} master | awk '{print $1}')
+
+  if grep -q "${repo}#${last_remote_hash}" .repo_cache; then
+    write_status SKIP ${repo} "No changes from last build"
+    return
+  fi
+
+  write_status RUN ${repo} "Build running..."
+
+  # Create working dir
+  local tmpdir="/tmp/aur2repo_$(basename ${repo})"
+  mkdir -p "${tmpdir}/cfg"
+
+  # Ensure cleanup on script exit
+  cleanup+=("${tmpdir}")
+
+  write_status RUN ${repo} "Fetching signing key..."
+  vault read --field=key secret/jenkins/arch-signing >"${tmpdir}/cfg/signing.asc"
+
+  write_status RUN ${repo} "Building package..."
+  local extra_opts=()
+  if [[ -f /etc/pacman.d/mirrorlist ]]; then
+    extra_opts+=(-v "/etc/pacman.d/mirrorlist:/etc/pacman.d/mirrorlist:ro")
+  fi
+
+  local container=$(
+    docker run -d \
+      -v "${tmpdir}/src:/src" \
+      -v "${tmpdir}/cfg:/config" \
+      -v "${REPO_DIR}:/repo" \
+      -v "$(pwd)/scripts/pacman.conf:/etc/pacman.conf:ro" \
+      "${extra_opts[@]}" \
+      --ulimit nofile=262144:262144 \
+      "${BUILD_IMAGE}" \
+      "${repo}"
+  )
+
+  local status="running"
+  while [[ $status == "running" ]]; do
+    status=$(docker inspect ${container} | jq -r '.[0].State.Status')
+    local started=$(date -d $(docker inspect ${container} |
+      jq -r '.[0].State.StartedAt') +%s)
+    write_status RUN ${repo} "Building package in container $(printf "%.12s" ${container}) for $(($(date +%s) - started))s..."
+    sleep 1
+  done
+
+  local exitcode=$(docker inspect ${container} | jq -r '.[0].State.ExitCode')
+  if [ $exitcode -gt 0 ]; then
+    local logfile="/tmp/arch-package-build_$(basename ${repo}).log"
+    docker logs ${container} 2>&1 >${logfile}
+    write_status FAIL ${repo} "Build failed (${exitcode}), see logs at ${logfile}"
+  else
+    write_status SUCCESS ${repo} "Updating cache entry..."
+
+    grep -v "^${repo}#" .repo_cache >.repo_cache.tmp || true
+    echo "${repo}#${last_remote_hash}" >>.repo_cache.tmp
+    mv .repo_cache.tmp .repo_cache
+
+    write_status SUCCESS ${repo} "Build succeeded"
+  fi
+
+  docker rm ${container} >/dev/null
+}
+
+function write_status() {
+  local icon="${ICON[$1]}"
+  local name="${2%%.git}"
+  local comment="$3"
+
+  # Wipe previous printed line
+  local wipe=$(echo -en "\r$(printf "%-${last_line_len}s" "")")
+
+  # Assemble new line
+  local line=$(echo -en "\r${icon} \e[1m${name##*/}\e[0m - ${comment}")
+  # Count line length for later wipe
+  last_line_len=$(wc -c <<<"${line}")
+  # Print line
+  echo -en "${wipe}${line}"
+}
+
+trap cleanup EXIT
+main

scripts/update-repo.sh

@@ -1,54 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-source ./scripts/script_framework.sh
-
-REPO_DIR=${REPO_DIR:-$(pwd)}
-
-REPO=${1:-}
-[ -z "${REPO}" ] && fail "No repo given as CLI argument"
-
-step "Checking for changes from last build"
-last_remote_hash=$(git ls-remote ${REPO} master | awk '{print $1}')
-grep "${REPO}#${last_remote_hash}" .repo_cache && {
-  warn "Remote has no changes from last build, skipping..."
-  exit 0
-} || true
-
-# Create working dir
-TMPDIR="/tmp/aur2repo_$(basename ${REPO})"
-mkdir -p "${TMPDIR}/cfg"
-
-# Ensure cleanup on script exit
-function cleanup() {
-  rm -rf "${TMPDIR}"
-}
-trap cleanup EXIT
-
-step "Fetching signing key"
-vault read --field=key secret/jenkins/arch-signing >"${TMPDIR}/cfg/signing.asc"
-
-step "Re-fetching Docker image"
-docker pull git.luzifer.io/registry/arch-repo-builder
-
-step "Building package $(basename ${REPO})"
-extra_opts=()
-if [[ -f /etc/pacman.d/mirrorlist ]]; then
-  info "Using host system mirrorlist..."
-  extra_opts+=(-v "/etc/pacman.d/mirrorlist:/etc/pacman.d/mirrorlist:ro")
-fi
-
-docker run --rm -ti \
-  -v "${TMPDIR}/src:/src" \
-  -v "${TMPDIR}/cfg:/config" \
-  -v "${REPO_DIR}:/repo" \
-  -v "$(pwd)/scripts/pacman.conf:/etc/pacman.conf:ro" \
-  "${extra_opts[@]}" \
-  --ulimit nofile=262144:262144 \
-  git.luzifer.io/registry/arch-repo-builder \
-  "${REPO}"
-
-step "Updating cache entry"
-grep -v "^${REPO}#" .repo_cache >.repo_cache.tmp || true
-echo "${REPO}#${last_remote_hash}" >>.repo_cache.tmp
-mv .repo_cache.tmp .repo_cache