mirror of
https://github.com/Luzifer/archrepo.git
synced 2024-09-18 18:22:55 +00:00
Update build tooling
Signed-off-by: Knut Ahlers <knut@ahlers.me>
This commit is contained in:
parent
e4d0b57e0e
commit
af5c00659a
SSH Key Fingerprint:
SHA256:/xtE5lCgiRDQr8SLxHMS92ZBlACmATUmF1crK16Ks4E
2
Makefile
2
Makefile
@ -51,7 +51,7 @@ list_packages:
|
||||
bash ./scripts/listing.sh >$(REPO_DIR)/packages.txt
|
||||
|
||||
repo_update: check_tools load_ssh_key
|
||||
bash -euo pipefail -c 'for repo in $$(grep -v "^#" repo-urls | cut -d "#" -f 1); do script_level=1 ./scripts/update-repo.sh $${repo}; done'
|
||||
bash ./scripts/update-all.sh
|
||||
|
||||
sign_database:
|
||||
repo-add -s --key $(REPOKEY) $(DATABASE)
|
||||
|
||||
@ -10,7 +10,7 @@ This repository contains four essential parts:
|
||||
|
||||
- The `scripts` folder containing bash scripts to control all actions
|
||||
- The `Makefile` to orchestrate everything. The main functionality is the `maintenance` target (or just `make`)
|
||||
- The package lists (`aur-packages` and `repo-urls`)
|
||||
- The package lists in `repo-urls`
|
||||
- The `repo/luzifer.asc` public key
|
||||
|
||||
It currently relies on my [`luzifer/arch-repo-builder`](https://github.com/luzifer-docker/arch-repo-builder) docker image which does all of the building within a clean environment for each package.
|
||||
@ -20,9 +20,7 @@ For the initial setup you need to do some steps:
|
||||
- Adjust the `Makefile` as you need different `download` and `upload` targets
|
||||
- Create an empty database `tar -cJf repo/luzifer.db.tar.xz -T /dev/null` (adjust the filename)
|
||||
- Put the public key for your repo into `repo/luzifer.asc` (filename should match the database, makes it easier to find)
|
||||
- Set up your `aur-packages` and `repo-urls` package lists
|
||||
- `aur-packages` contains just names of AUR packages (no comments or other stuff!)
|
||||
- `repo-urls` contains one git repository URL per line (comments allowed)
|
||||
- Set up your `repo-urls` package list: it contains one git repository URL per line (comments allowed)
|
||||
- Provide a docker daemon and all tools listed in the `check_tools` target of the `Makefile`
|
||||
|
||||
Afterwards you should be good to just `make` your first build. Depending on the number of packages you selected to be in your repo you might go and fetch dinner while it builds.
|
||||
@ -34,5 +32,4 @@ The repo should be updated on a regular base by just executing `make` on it. Thi
|
||||
## Flaws / Remarks / TODOs
|
||||
|
||||
- The whole build already strongly relies on Archlinux tools so will not run on any other distro
|
||||
- For `aur-packages` having dynamic `pkgver` calculation the update check will not work properly until the `PKGBUILD` in AUR is updated (those packages can be built manually using `bash ./scripts/update-aur.sh <packagename> && make do_cleanup upload`
|
||||
- For `repo-urls` the same applies: for example my `vim-go-tools` package relies on periodic re-builds which are not executed as commits are quite rare. For those packages at the moment a call to `bash ./scripts/update-repo.sh <url> && make do_cleanup upload` is required
|
||||
- For packages having dynamic `pkgver` calculation the update check will not work properly until the `PKGBUILD` in the repo is updated. You can force a rebuild by removing the corresponding line from the `.repo_cache`
|
||||
|
||||
@ -53,6 +53,7 @@ https://aur.archlinux.org/mozjpeg.git
|
||||
https://aur.archlinux.org/mpd-mpris.git
|
||||
https://aur.archlinux.org/nvm.git
|
||||
https://aur.archlinux.org/obs-cli.git
|
||||
https://aur.archlinux.org/oda-file-converter.git
|
||||
https://aur.archlinux.org/oh-my-posh-bin.git
|
||||
https://aur.archlinux.org/orca-slicer-bin.git
|
||||
https://aur.archlinux.org/packer-plugin-arm-image.git
|
||||
|
||||
120
scripts/update-all.sh
Normal file
120
scripts/update-all.sh
Normal file
@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
BUILD_IMAGE="git.luzifer.io/registry/arch-repo-builder"
|
||||
declare -A ICON=(
|
||||
["CHECK"]=$(printf "\e[34m\u3f\e[0m")
|
||||
["FAIL"]=$(printf "\e[31m\uf05e\e[0m")
|
||||
["RUN"]=$(printf "\e[32m\uf2f1\e[0m")
|
||||
["SKIP"]=$(printf "\e[1;30m\uf00c\e[0m")
|
||||
["SUCCESS"]=$(printf "\e[32m\uf00c\e[0m")
|
||||
["WAIT"]=$(printf "\e[1;30m\uf251\e[0m")
|
||||
)
|
||||
REPO_DIR=${REPO_DIR:-$(pwd)/repo}
|
||||
|
||||
cleanup=()
|
||||
last_line_len=0
|
||||
|
||||
function cleanup() {
|
||||
rm -rf "${cleanup[@]}"
|
||||
}
|
||||
|
||||
function main() {
|
||||
repo_list=(
|
||||
$(grep -v "^#" ./repo-urls | cut -d "#" -f 1)
|
||||
)
|
||||
|
||||
docker pull -q "${BUILD_IMAGE}" >/dev/null
|
||||
|
||||
for repo in $(grep -v "^#" ./repo-urls | cut -d "#" -f 1); do
|
||||
update ${repo}
|
||||
echo
|
||||
done
|
||||
}
|
||||
|
||||
function update() {
|
||||
repo="${1}"
|
||||
|
||||
write_status CHECK ${repo} "Checking build status..."
|
||||
local last_remote_hash=$(git ls-remote ${repo} master | awk '{print $1}')
|
||||
|
||||
if grep -q "${repo}#${last_remote_hash}" .repo_cache; then
|
||||
write_status SKIP ${repo} "No changes from last build"
|
||||
return
|
||||
fi
|
||||
|
||||
write_status RUN ${repo} "Build running..."
|
||||
|
||||
# Create working dir
|
||||
local tmpdir="/tmp/aur2repo_$(basename ${repo})"
|
||||
mkdir -p "${tmpdir}/cfg"
|
||||
|
||||
# Ensure cleanup on script exit
|
||||
cleanup+=("${tmpdir}")
|
||||
|
||||
write_status RUN ${repo} "Fetching signing key..."
|
||||
vault read --field=key secret/jenkins/arch-signing >"${tmpdir}/cfg/signing.asc"
|
||||
|
||||
write_status RUN ${repo} "Building package..."
|
||||
local extra_opts=()
|
||||
if [[ -f /etc/pacman.d/mirrorlist ]]; then
|
||||
extra_opts+=(-v "/etc/pacman.d/mirrorlist:/etc/pacman.d/mirrorlist:ro")
|
||||
fi
|
||||
|
||||
local container=$(
|
||||
docker run -d \
|
||||
-v "${tmpdir}/src:/src" \
|
||||
-v "${tmpdir}/cfg:/config" \
|
||||
-v "${REPO_DIR}:/repo" \
|
||||
-v "$(pwd)/scripts/pacman.conf:/etc/pacman.conf:ro" \
|
||||
"${extra_opts[@]}" \
|
||||
--ulimit nofile=262144:262144 \
|
||||
"${BUILD_IMAGE}" \
|
||||
"${repo}"
|
||||
)
|
||||
|
||||
local status="running"
|
||||
while [[ $status == "running" ]]; do
|
||||
status=$(docker inspect ${container} | jq -r '.[0].State.Status')
|
||||
local started=$(date -d $(docker inspect ${container} |
|
||||
jq -r '.[0].State.StartedAt') +%s)
|
||||
write_status RUN ${repo} "Building package in container $(printf "%.12s" ${container}) for $(($(date +%s) - started))s..."
|
||||
sleep 1
|
||||
done
|
||||
|
||||
local exitcode=$(docker inspect ${container} | jq -r '.[0].State.ExitCode')
|
||||
if [ $exitcode -gt 0 ]; then
|
||||
local logfile="/tmp/arch-package-build_$(basename ${repo}).log"
|
||||
docker logs ${container} 2>&1 >${logfile}
|
||||
write_status FAIL ${repo} "Build failed (${exitcode}), see logs at ${logfile}"
|
||||
else
|
||||
write_status SUCCESS ${repo} "Updating cache entry..."
|
||||
|
||||
grep -v "^${repo}#" .repo_cache >.repo_cache.tmp || true
|
||||
echo "${repo}#${last_remote_hash}" >>.repo_cache.tmp
|
||||
mv .repo_cache.tmp .repo_cache
|
||||
|
||||
write_status SUCCESS ${repo} "Build succeeded"
|
||||
fi
|
||||
|
||||
docker rm ${container} >/dev/null
|
||||
}
|
||||
|
||||
function write_status() {
|
||||
local icon="${ICON[$1]}"
|
||||
local name="${2%%.git}"
|
||||
local comment="$3"
|
||||
|
||||
# Wipe previous printed line
|
||||
local wipe=$(echo -en "\r$(printf "%-${last_line_len}s" "")")
|
||||
|
||||
# Assemble new line
|
||||
local line=$(echo -en "\r${icon} \e[1m${name##*/}\e[0m - ${comment}")
|
||||
# Count line length for later wipe
|
||||
last_line_len=$(wc -c <<<"${line}")
|
||||
# Print line
|
||||
echo -en "${wipe}${line}"
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
main
|
||||
@ -1,54 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
source ./scripts/script_framework.sh
|
||||
|
||||
REPO_DIR=${REPO_DIR:-$(pwd)}
|
||||
|
||||
REPO=${1:-}
|
||||
[ -z "${REPO}" ] && fail "No repo given as CLI argument"
|
||||
|
||||
step "Checking for changes from last build"
|
||||
last_remote_hash=$(git ls-remote ${REPO} master | awk '{print $1}')
|
||||
grep "${REPO}#${last_remote_hash}" .repo_cache && {
|
||||
warn "Remote has no changes from last build, skipping..."
|
||||
exit 0
|
||||
} || true
|
||||
|
||||
# Create working dir
|
||||
TMPDIR="/tmp/aur2repo_$(basename ${REPO})"
|
||||
mkdir -p "${TMPDIR}/cfg"
|
||||
|
||||
# Ensure cleanup on script exit
|
||||
function cleanup() {
|
||||
rm -rf "${TMPDIR}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
step "Fetching signing key"
|
||||
vault read --field=key secret/jenkins/arch-signing >"${TMPDIR}/cfg/signing.asc"
|
||||
|
||||
step "Re-fetching Docker image"
|
||||
docker pull git.luzifer.io/registry/arch-repo-builder
|
||||
|
||||
step "Building package $(basename ${REPO})"
|
||||
extra_opts=()
|
||||
if [[ -f /etc/pacman.d/mirrorlist ]]; then
|
||||
info "Using host system mirrorlist..."
|
||||
extra_opts+=(-v "/etc/pacman.d/mirrorlist:/etc/pacman.d/mirrorlist:ro")
|
||||
fi
|
||||
|
||||
docker run --rm -ti \
|
||||
-v "${TMPDIR}/src:/src" \
|
||||
-v "${TMPDIR}/cfg:/config" \
|
||||
-v "${REPO_DIR}:/repo" \
|
||||
-v "$(pwd)/scripts/pacman.conf:/etc/pacman.conf:ro" \
|
||||
"${extra_opts[@]}" \
|
||||
--ulimit nofile=262144:262144 \
|
||||
git.luzifer.io/registry/arch-repo-builder \
|
||||
"${REPO}"
|
||||
|
||||
step "Updating cache entry"
|
||||
grep -v "^${REPO}#" .repo_cache >.repo_cache.tmp || true
|
||||
echo "${REPO}#${last_remote_hash}" >>.repo_cache.tmp
|
||||
mv .repo_cache.tmp .repo_cache
|
||||
Loading…
Reference in New Issue
Block a user