From ef12f2f7a17b6654afd68dab4fc3f0f0bfc06c33 Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Mon, 30 Sep 2019 01:20:25 +0200 Subject: [PATCH] Initial version --- .gitignore | 6 ++++ Makefile | 62 +++++++++++++++++++++++++++++++++ luzifer.asc | 18 ++++++++++ repo-urls | 5 +++ scripts/check_aur_updates.sh | 66 ++++++++++++++++++++++++++++++++++++ scripts/repoctl.sh | 11 ++++++ scripts/script_framework.sh | 38 +++++++++++++++++++++ scripts/update-aur.sh | 33 ++++++++++++++++++ scripts/update-repo.sh | 33 ++++++++++++++++++ 9 files changed, 272 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 luzifer.asc create mode 100644 repo-urls create mode 100755 scripts/check_aur_updates.sh create mode 100755 scripts/repoctl.sh create mode 100644 scripts/script_framework.sh create mode 100755 scripts/update-aur.sh create mode 100755 scripts/update-repo.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1f972f7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +*.db +*.db.sig +*.files +*.files.sig +*.tar.xz +*.tar.xz.sig diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..25a482f --- /dev/null +++ b/Makefile @@ -0,0 +1,62 @@ +export DATABASE:=$(shell find . -maxdepth 1 -mindepth 1 -name '*.db.tar.xz') +export REPOKEY:=D0391BF9 + + +maintanance: aur_update +#maintanance: repo_update +maintanance: cleanup_repo +maintanance: cleanup_orphan_signatures +maintanance: sign_database +maintanance: upload + +download: + vault2env --key=secret/aws/private -- aws s3 sync \ + --delete \ + --exclude '*.old*' \ + --exclude '.git/*' \ + --acl=public-read \ + s3://arch-luzifer-io/repo/x86_64/ $(CURDIR)/ + +upload: cleanup_files + vault2env --key=secret/aws/private -- aws s3 sync \ + --delete \ + --exclude '*.old*' \ + --exclude '.git/*' \ + --acl=public-read \ + $(CURDIR)/ s3://arch-luzifer-io/repo/x86_64/ + +# Maintenance targets + +aur_update: check_tools check_database + bash -euo pipefail -c 'for pkg in $$(script_level=1 ./scripts/check_aur_updates.sh); do script_level=1 ./scripts/update-aur.sh $${pkg}; done' + +check_database: + test -n '$(DATABASE)' + +check_tools: + @which aws + @which curl + @which docker + @which jq + @which repo-add + @which repoctl + @which vault + @which vault2env + +cleanup_files: + rm -f *.old* scripts/repoctl.toml + +cleanup_orphan_signatures: check_database + bash -euo pipefail -c 'for i in *.sig; do [[ -f $${i//.sig} ]] || rm $${i}; done' + +cleanup_repo: check_tools check_database scripts/repoctl.toml + repoctl update + +repo_update: check_tools check_database + bash -euo pipefail -c 'for repo in $$(grep -v "^#" repo-urls); do script_level=1 ./scripts/update-repo.sh $${repo}; done' + +scripts/repoctl.toml: + ./scripts/repoctl.sh + +sign_database: + repo-add -s --key $(REPOKEY) $(DATABASE) diff --git a/luzifer.asc b/luzifer.asc new file mode 100644 index 0000000..01daea3 --- /dev/null +++ b/luzifer.asc @@ -0,0 +1,18 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQENBFvwZQ0BCADb4fU1yk9YwMVBVaybxtiFLQSoRxHU8SqJtM8INY+fo/RPTqFM +gQ34m4cu3BHRzmg2KkDZ4IH3UyO3pe1WaYnK2DnYJryxvLwTETmf5wtiuDdJhoH6 +6PUr5G7hhtzyvjNgaQ4b0eQQlmF0ArdBgBfeHcOLFPvzCTxZQpq/qr6RK2pfhQNM +8MHRfZ+tXIc/SP/FzmQhlxXwOJ0VJNYdl4R6rczx1b/cvjTWldy/fdJa4dn/DqiI +h/5x8kJid0FcR0hRipxVbUYWm0dHY1T4m/tDkDuhEBqBp4By7qfaNgIyU9nJSfJY +9ehIiNYG89lTlSOP4OqEvpDGZPl9PzHFHlBlABEBAAG0LUtudXQgQWhsZXJzIChQ +YWNrYWdlIFNpZ25pbmcpIDxwa2dAYWhsZXJzLm1lPokBTgQTAQgAOBYhBG9zpPOc +32UuP5RBQghaoiPQORv5BQJb8GUNAhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheA +AAoJEAhaoiPQORv5QpUH/iaSo1T6uArs1j4Sf1OTAxKd46Yfyf3h72gWqJGU+EqZ +YdRwDrmVMN7UgPJ832RuWudUDYa5egXsHulBYyNaV+4bbLM3/8lsuXvJpaBUkhad +mPTbcy1pKlIx9fSaUrkitxTu8BAHuNpF4RwH9Wtj2Etx1bnjj8nZmPp4ebEfUnMd +yrqBTxD+AUIKB1Ix15TJzjG4uhf2lU60t6XQjK9XvRqYWdySfXje91DOT7D7+iJf +MOawzYnKuGLQiecE3BJvgsgE4fPckt5bMItfAOyacQDMW3r0dnxpG/pYl97Nve8M +1mn4n2lLtWidC6UmvYc+kNwMKY9q0iR+uArX0tXeWVA= +=Tc6u +-----END PGP PUBLIC KEY BLOCK----- diff --git a/repo-urls b/repo-urls new file mode 100644 index 0000000..978d314 --- /dev/null +++ b/repo-urls @@ -0,0 +1,5 @@ +# vim-go-tools +https://github.com/luzifer-aur/vim-go-tools.git + +# Arch base setup +https://github.com/luzifer-aur/luzifer.git diff --git a/scripts/check_aur_updates.sh b/scripts/check_aur_updates.sh new file mode 100755 index 0000000..da6fbe6 --- /dev/null +++ b/scripts/check_aur_updates.sh @@ -0,0 +1,66 @@ +#!/bin/bash +set -euo pipefail + +source scripts/script_framework.sh + +join_by() { + local d=$1 + shift + echo -n "$1" + shift + printf "%s" "${@/#/$d}" +} + +declare -A local_versions +declare -A aur_versions + +database=$(find . -maxdepth 1 -mindepth 1 -name '*.db.tar.xz') + +aur_query=("https://aur.archlinux.org/rpc/?v=5&type=info") + +step "Collecting local package versions..." +known_packages=$(tar -tf luzifer.db.tar.xz | grep -v /desc | sed -E 's@^(.*)-([^-]+-[0-9]+)/$@\1 \2@') + +IFS=$'\n' + +for package in ${known_packages}; do + name=$(echo "${package}" | cut -d ' ' -f 1) + version=$(echo "${package}" | cut -d ' ' -f 2) + + local_versions[${name}]=${version} + aur_query+=("arg[]=${name}") +done + +step "Fetching AUR package versions..." +aur_packages=$(curl -sSfL "$(join_by "&" "${aur_query[@]}")" | jq -r '.results | .[] | .Name + " " + .Version') + +step "Collecting AUR package versions..." +for package in ${aur_packages}; do + name=$(echo "${package}" | cut -d ' ' -f 1) + version=$(echo "${package}" | cut -d ' ' -f 2) + + aur_versions[${name}]=${version} +done + +updates=() + +step "Checking for updates..." +for package in "${!local_versions[@]}"; do + local_version="${local_versions[${package}]}" + aur_version="${aur_versions[${package}]:-}" + + [[ -n ${aur_version} ]] || { + error "Package ${package} did not yield a version from AUR (local=${local_version})" + continue + } + + [[ ${local_version} == ${aur_version} ]] || { + warn "Package ${package} needs update (${local_version} => ${aur_version})" + updates+=("${package}") + continue + } + + success "Package ${package} is up-to-date (${local_version})" +done + +echo "${updates[@]}" diff --git a/scripts/repoctl.sh b/scripts/repoctl.sh new file mode 100755 index 0000000..c8261cf --- /dev/null +++ b/scripts/repoctl.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -euo pipefail + +cat -s <scripts/repoctl.toml +repo = "$(find $(pwd) -mindepth 1 -maxdepth 1 -name '*.db.tar.xz')" +backup = false +interactive = false +columnate = false +color = "auto" +quiet = false +EOF diff --git a/scripts/script_framework.sh b/scripts/script_framework.sh new file mode 100644 index 0000000..4fd16f6 --- /dev/null +++ b/scripts/script_framework.sh @@ -0,0 +1,38 @@ +COLOR_RED="\033[0;31m" +COLOR_GREEN="\033[0;32m" +COLOR_CYAN="\033[0;36m" +COLOR_YELLOW="\033[0;33m" +COLOR_PLAIN="\033[0m" + +function check_util() { + which ${1} >/dev/null 2>&1 || fail "Missing ${1} util" +} + +function error() { + echo -e "${COLOR_RED}$@${COLOR_PLAIN}" >&2 +} + +function fail() { + error "$@" + exit 1 +} + +function fatal() { + fail "$@" +} + +function info() { + echo -e "${COLOR_CYAN}$@${COLOR_PLAIN}" >&2 +} + +function step() { + info "[$(date +%H:%M:%S)] $(printf "%${script_level:-0}s" '' | tr ' ' '+')$@" +} + +function success() { + echo -e "${COLOR_GREEN}$@${COLOR_PLAIN}" >&2 +} + +function warn() { + echo -e "${COLOR_YELLOW}$@${COLOR_PLAIN}" >&2 +} diff --git a/scripts/update-aur.sh b/scripts/update-aur.sh new file mode 100755 index 0000000..12718da --- /dev/null +++ b/scripts/update-aur.sh @@ -0,0 +1,33 @@ +#!/bin/bash +set -euo pipefail + +source ./scripts/script_framework.sh + +REPO_DIR=${REPO_DIR:-$(pwd)} + +PACKAGE=${1:-} +[ -z "${PACKAGE}" ] && fail "No package given as CLI argument" + +# Create working dir +TMPDIR="/tmp/aur2repo_${PACKAGE}" +mkdir -p "${TMPDIR}/cfg" + +# Ensure cleanup on script exit +function cleanup() { + rm -rf "${TMPDIR}" +} +trap cleanup EXIT + +step "Fetching signing key" +vault read --field=key secret/jenkins/arch-signing >"${TMPDIR}/cfg/signing.asc" + +step "Re-fetching Docker image" +docker pull luzifer/arch-repo-builder + +step "Building AUR package ${PACKAGE}" +docker run --rm -ti \ + -v "${TMPDIR}/src:/src" \ + -v "${TMPDIR}/cfg:/config" \ + -v "${REPO_DIR}:/repo" \ + luzifer/arch-repo-builder \ + "https://aur.archlinux.org/${PACKAGE}.git" diff --git a/scripts/update-repo.sh b/scripts/update-repo.sh new file mode 100755 index 0000000..e30cdd6 --- /dev/null +++ b/scripts/update-repo.sh @@ -0,0 +1,33 @@ +#!/bin/bash +set -euo pipefail + +source ./scripts/script_framework.sh + +REPO_DIR=${REPO_DIR:-$(pwd)} + +REPO=${1:-} +[ -z "${REPO}" ] && fail "No repo given as CLI argument" + +# Create working dir +TMPDIR="/tmp/aur2repo_$(basename ${REPO})" +mkdir -p "${TMPDIR}/cfg" + +# Ensure cleanup on script exit +function cleanup() { + rm -rf "${TMPDIR}" +} +trap cleanup EXIT + +step "Fetching signing key" +vault read --field=key secret/jenkins/arch-signing >"${TMPDIR}/cfg/signing.asc" + +step "Re-fetching Docker image" +docker pull luzifer/arch-repo-builder + +step "Building package $(basename ${REPO})" +docker run --rm -ti \ + -v "${TMPDIR}/src:/src" \ + -v "${TMPDIR}/cfg:/config" \ + -v "${REPO_DIR}:/repo" \ + luzifer/arch-repo-builder \ + "${REPO}"