From dd631fd3c7dc47caf0ea9858bf8871a508edfc49 Mon Sep 17 00:00:00 2001 From: Markus Fritsche Date: Tue, 19 May 2026 09:24:23 +0200 Subject: [PATCH] ka-build: arch makepkg wrapper + sign + publish (closes #34) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase-1 ka-build per umbrella #21: 1. Read manifest.lock from ka-promote output. Refuse if missing. 2. Verify each PKGBUILD-side patch in marfrit-packages still matches the kernel-agent-side patch by sha256 (manifest.lock is authoritative). 3. ssh-dispatch makepkg --syncdeps --noconfirm --cleanbuild to the manifest's build_host.primary. Native build only — no distcc (feedback_kernel_agent_no_distcc). 4. Pull the resulting *.pkg.tar.zst back; scp to hertz and run /opt/herding/bin/marfrit-publish-arch aarch64 . 5. Append a `build:` block to manifest.lock with built_at, host, per-package b2sum + size. Flags: --dry-run (stop before makepkg), --skip-publish (build only), --packages-repo (override default ~/src/marfrit-packages). Out of scope (separate followups): - Debian .deb path - PKGBUILD template *generation* (current PKGBUILDs are hand-authored; ka-build verifies + stamps, doesn't author) - distcc routing (explicitly NOT in kernel-agent flow) - ka-build --validate-against (apply-check harness) Tests: 6/6 pass (arg parsing, missing manifest.lock, missing PKGBUILD, patch drift via sha256 mismatch, happy-path dry-run on fresnel). Full-build path manually exercisable; CI integration deferred until the sandbox supports mock build-host + mock marfrit-publish-arch. --- README.md | 4 +- bin/ka-build | 199 ++++++++++++++++++++++++++++++++++++ tests/ka-build/run-tests.sh | 113 ++++++++++++++++++++ 3 files changed, 314 insertions(+), 2 deletions(-) create mode 100755 bin/ka-build create mode 100755 tests/ka-build/run-tests.sh diff --git a/README.md b/README.md index 3d89f7d..ee559d0 100644 --- a/README.md +++ b/README.md @@ -264,8 +264,8 @@ build. `ka-promote` (issue #22) replaced the manual step #1 below as of 2026-05- |---|---|---| | `ka-import fresnel-fourier --to board/pinebook-pro` (originally named `ka-promote` in this row) | Authored 3 patches with proper headers/scope tags, pushed to `marfrit/kernel-agent/patches/board/pinebook-pro/` via Gitea contents API as `claude-noether`. | still manual — `ka-import` unimplemented | | `ka-promote fresnel` (new — manifest → cumulative.patch + manifest.lock) | n/a (didn't exist) | **automated 2026-05-18, issue #22** | -| `ka-build fresnel` | On boltzmann: cloned linux v7.0 from kernel.org, ran `makepkg -s --skipchecksums --skippgpcheck` against `marfrit-packages/arch/linux-fresnel-fourier/PKGBUILD`. Native aarch64 (boltzmann is RK3588). One headers-pkg bug discovered (`ln -sr` on missing parent dir) and fixed mid-flight. Repackaged. | still manual — next verb to implement | -| `ka-sign + push` | scp pkgs hertz → `sudo /opt/herding/bin/marfrit-publish-arch aarch64 ` per pkg. Script signs with key `92D5E96D8F63C75E4116AA1FF5C8C4603D0D250C`, runs repo-add, rsyncs to nc. | still manual — folded into `ka-build` | +| `ka-build fresnel` | On boltzmann: cloned linux v7.0 from kernel.org, ran `makepkg -s --skipchecksums --skippgpcheck` against `marfrit-packages/arch/linux-fresnel-fourier/PKGBUILD`. Native aarch64 (boltzmann is RK3588). One headers-pkg bug discovered (`ln -sr` on missing parent dir) and fixed mid-flight. Repackaged. | **automated 2026-05-19, issue #34** — `ka-build ` ssh-dispatches makepkg to `build_host.primary`, verifies kernel-agent patches still match the PKGBUILD-side files (b2sum cross-check from `manifest.lock`), and pulls the resulting `*.pkg.tar.zst` back. | +| `ka-sign + push` | scp pkgs hertz → `sudo /opt/herding/bin/marfrit-publish-arch aarch64 ` per pkg. Script signs with key `92D5E96D8F63C75E4116AA1FF5C8C4603D0D250C`, runs repo-add, rsyncs to nc. | **folded into `ka-build` 2026-05-19** — `ka-build` scp's each pkg to hertz and runs `marfrit-publish-arch` over ssh. `--skip-publish` flag retained for offline builds. | | `ka-install fresnel` (consent-via-action) | `sudo pacman -U /tmp/` over LAN scp (HTTPS to nc was throttled by fresnel's wifi). pacman post-transaction hook updated extlinux. mkinitcpio run manually because the standard hook trigger watches `vmlinuz` not `Image`. | still manual — last verb to implement | | Bar 1..3 verification | SSH heartbeat OK, `pacman -Q linux-fresnel-fourier` = `7.0-1`, post-reboot cluster0 1.704 GHz / cluster1 2.184 GHz confirmed. | folded into `ka-install` | diff --git a/bin/ka-build b/bin/ka-build new file mode 100755 index 0000000..cf90784 --- /dev/null +++ b/bin/ka-build @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +# ka-build — render PKGBUILD from manifest.lock, build native on host, +# sign+publish via marfrit-publish-arch on hertz. +# +# Phase-1 (issue #34): arch makepkg wrapper. Debian path deferred. +# +# Usage: +# ka-build +# ka-build --packages-repo # default: ~/src/marfrit-packages +# ka-build --dry-run # stop after staging, don't makepkg +# ka-build --skip-publish # build only, don't push to hertz +# +# Exit codes: +# 0 success (pkg built + published) +# 2 missing input (manifest.lock, PKGBUILD, ssh target) +# 3 patch drift (resolved.sha256 != PKGBUILD-side file sha256) +# 4 makepkg / sign / publish failure +# 5 manifest parse error + +set -euo pipefail + +VERSION=1 + +die() { echo "ka-build: error: $1" >&2; exit "${2:-1}"; } +note() { echo "ka-build: $1"; } + +# Defaults +PACKAGES_REPO="${KA_PACKAGES_REPO:-${HOME}/src/marfrit-packages}" +DRY_RUN=0 +SKIP_PUBLISH=0 +HOST="" + +while [ $# -gt 0 ]; do + case "$1" in + --packages-repo) PACKAGES_REPO="$2"; shift 2 ;; + --dry-run) DRY_RUN=1; shift ;; + --skip-publish) SKIP_PUBLISH=1; shift ;; + --version) echo "ka-build version $VERSION"; exit 0 ;; + -h|--help) sed -n '1,30p' "$0" | grep -E '^# ' | sed 's/^# //'; exit 0 ;; + -*) die "unknown flag: $1" ;; + *) [ -z "$HOST" ] && HOST="$1" || die "extra arg: $1"; shift ;; + esac +done + +[ -n "$HOST" ] || die "host is required" 2 +[ -d "$PACKAGES_REPO" ] || die "--packages-repo not found: $PACKAGES_REPO" 2 + +# Locate kernel-agent repo root (where bin/ + fleet/ live) +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$script_dir/.." && pwd)" +[ -d "$REPO_ROOT/fleet" ] || die "fleet/ not found relative to $script_dir" 2 + +manifest="$REPO_ROOT/fleet/${HOST}.yaml" +[ -f "$manifest" ] || die "no manifest for host '$HOST': $manifest" 2 + +# Read fields from manifest via python (yaml in bash is masochism) +py_read() { + python3 -c " +import sys, yaml, os +m = yaml.safe_load(open('$manifest')) +keys = '$1'.split('.') +v = m +for k in keys: + if not isinstance(v, dict) or k not in v: sys.exit('missing key: $1') + v = v[k] +print(v) +" +} + +PKG_NAME="$(py_read package.name)" +BASELINE_REF="$(py_read baseline.ref)" +BUILD_HOST="$(py_read build_host.primary)" + +# Locate the most recent ka-promote output +build_dir_root="${KA_BUILD_DIR:-$REPO_ROOT/build}" +promote_out="${build_dir_root}/${HOST}/${BASELINE_REF}" +lock="${promote_out}/manifest.lock" +cumulative="${promote_out}/cumulative.patch" +[ -f "$lock" ] || die "no manifest.lock at $lock — run 'ka-promote $HOST' first" 2 +[ -f "$cumulative" ] || die "no cumulative.patch at $cumulative — run 'ka-promote $HOST' first" 2 + +# Locate the PKGBUILD +pkg_dir="${PACKAGES_REPO}/arch/${PKG_NAME}" +pkgbuild="${pkg_dir}/PKGBUILD" +[ -f "$pkgbuild" ] || die "no PKGBUILD at $pkgbuild (expected from manifest package.name)" 2 + +note "host=$HOST pkg=$PKG_NAME baseline=$BASELINE_REF build_host=$BUILD_HOST" +note "PKGBUILD: $pkgbuild" +note "manifest.lock: $lock" + +# Refuse if PKGBUILD-side patches drifted from kernel-agent patches/. +# manifest.lock.resolved_patches[].sha256 must match PKGBUILD-dir-side +# files of the same basename. (If a patch is in resolved but missing from +# PKGBUILD dir, fail loud — operator needs to sync.) +note "verifying patch consistency between kernel-agent and marfrit-packages..." +drift=0 +while IFS=$'\t' read -r basename expected_sha; do + pkg_side="${pkg_dir}/${basename}" + if [ ! -f "$pkg_side" ]; then + echo " MISSING in PKGBUILD dir: $basename" >&2 + drift=1; continue + fi + actual_sha=$(sha256sum "$pkg_side" | cut -d' ' -f1) + if [ "$actual_sha" != "$expected_sha" ]; then + echo " DRIFT: $basename (expected $expected_sha, got $actual_sha)" >&2 + drift=1 + fi +done < <(python3 -c " +import yaml, sys, os +lk = yaml.safe_load(open('$lock')) +for r in lk['resolved_patches']: + bn = os.path.basename(r['include']) + print(f\"{bn}\t{r['sha256']}\") +") +[ "$drift" -eq 0 ] || die "patches differ between kernel-agent and marfrit-packages — sync first" 3 +note "patches OK ($(python3 -c "import yaml; print(len(yaml.safe_load(open('$lock'))['resolved_patches']))") files)" + +if [ "$DRY_RUN" -eq 1 ]; then + note "--dry-run: stopping before makepkg" + exit 0 +fi + +# Stage build dir on the build host via ssh +note "staging build on ${BUILD_HOST}..." +remote_stage="/tmp/ka-build-${HOST}-$$" +ssh "${BUILD_HOST}" "mkdir -p '$remote_stage'" +rsync -a "${pkg_dir}/" "${BUILD_HOST}:${remote_stage}/" + +# Run makepkg natively +note "running makepkg --syncdeps --noconfirm --cleanbuild on ${BUILD_HOST}..." +ssh "${BUILD_HOST}" "cd '$remote_stage' && makepkg --syncdeps --noconfirm --cleanbuild --skipchecksums" \ + || die "makepkg failed on ${BUILD_HOST}" 4 + +# Fetch built packages +note "fetching .pkg.tar.zst from ${BUILD_HOST}..." +local_out="${promote_out}/pkgs" +mkdir -p "$local_out" +rsync -av "${BUILD_HOST}:${remote_stage}/*.pkg.tar.zst" "$local_out/" 2>&1 | tail -5 + +# Compute b2sums +pkg_b2sum_list=$(cd "$local_out" && for p in *.pkg.tar.zst; do + [ -f "$p" ] || continue + printf '%s %s\n' "$(b2sum "$p" | cut -d' ' -f1)" "$p" +done) +note "built packages:" +echo "$pkg_b2sum_list" | sed 's/^/ /' + +# Publish via hertz marfrit-publish-arch (unless --skip-publish) +if [ "$SKIP_PUBLISH" -eq 0 ]; then + note "publishing to packages.reauktion.de/arch/aarch64/..." + for p in "$local_out"/*.pkg.tar.zst; do + [ -f "$p" ] || continue + base="$(basename "$p")" + scp -q "$p" "hertz:/tmp/${base}" || die "scp to hertz failed: $base" 4 + ssh hertz "sudo /opt/herding/bin/marfrit-publish-arch aarch64 '/tmp/${base}'" \ + || die "marfrit-publish-arch failed: $base" 4 + ssh hertz "rm -f '/tmp/${base}'" + note "published: $base" + done +fi + +# Update manifest.lock with build receipt (append; don't rewrite the +# existing fields) +note "writing build receipt to manifest.lock..." +python3 - <&1) +rc=$? +set -e +if [ "$rc" -eq 2 ] && echo "$out" | grep -q "host is required"; then ok "requires host arg"; else ko "requires host arg" "exit=$rc out=$out"; fi + +# ----- 2. unknown flag ----- +echo "::: unknown flag rejected" +set +e +out=$("$repo_root/bin/ka-build" fresnel --nonsense 2>&1) +rc=$? +set -e +if [ "$rc" -ne 0 ] && echo "$out" | grep -q "unknown flag"; then ok "unknown flag rejected"; else ko "unknown flag rejected" "exit=$rc out=$out"; fi + +# ----- 3. refuses if manifest.lock missing ----- +echo "::: refuses if manifest.lock missing (ka-promote not run)" +set +e +out=$("$repo_root/bin/ka-build" fresnel --dry-run --packages-repo "$packages_repo" 2>&1) +rc=$? +set -e +if [ "$rc" -eq 2 ] && echo "$out" | grep -q "no manifest.lock"; then ok "refuses no-lock"; else ko "refuses no-lock" "exit=$rc out=$out"; fi + +# Now run ka-promote so the rest can proceed +"$repo_root/bin/ka-promote" fresnel >/dev/null + +# ----- 4. refuses if PKGBUILD missing ----- +echo "::: refuses if PKGBUILD missing (--packages-repo wrong)" +set +e +out=$("$repo_root/bin/ka-build" fresnel --dry-run --packages-repo /tmp/non-existent-mp 2>&1) +rc=$? +set -e +if [ "$rc" -eq 2 ]; then ok "refuses bad packages-repo"; else ko "refuses bad packages-repo" "exit=$rc out=$out"; fi + +# ----- 5. happy-path dry-run ----- +echo "::: happy-path dry-run (fresnel, real packages-repo)" +if [ ! -f "$packages_repo/arch/linux-fresnel-fourier/PKGBUILD" ]; then + note "SKIP: $packages_repo/arch/linux-fresnel-fourier/PKGBUILD not present" + results+=("SKIP happy-path dry-run (PKGBUILD missing locally)") +else + set +e + out=$("$repo_root/bin/ka-build" fresnel --dry-run --packages-repo "$packages_repo" 2>&1) + rc=$? + set -e + if [ "$rc" -eq 0 ] && echo "$out" | grep -q "patches OK (6 files)"; then ok "happy-path dry-run"; else ko "happy-path dry-run" "exit=$rc out=$out"; fi +fi + +# ----- 6. patch drift detection ----- +echo "::: patch drift detection (mutate a copied patch, expect exit 3)" +if [ ! -d "$packages_repo/arch/linux-fresnel-fourier" ]; then + note "SKIP: $packages_repo/arch/linux-fresnel-fourier not present" + results+=("SKIP patch drift detection") +else + sandbox=$(mktemp -d -t ka-build-drift.XXXXXX) + cp -r "$packages_repo/arch/linux-fresnel-fourier" "$sandbox/linux-fresnel-fourier" + mkdir -p "$sandbox/arch" + mv "$sandbox/linux-fresnel-fourier" "$sandbox/arch/linux-fresnel-fourier" + # Mutate one patch so its sha256 differs from manifest.lock's recorded sha + echo "drift" >> "$sandbox/arch/linux-fresnel-fourier/0001-arm64-dts-rk3399-pinebook-pro-add-OC-OPP-tables-1704-2184.patch" + set +e + out=$("$repo_root/bin/ka-build" fresnel --dry-run --packages-repo "$sandbox" 2>&1) + rc=$? + set -e + rm -rf "$sandbox" + if [ "$rc" -eq 3 ] && echo "$out" | grep -q "DRIFT:"; then ok "patch drift detection"; else ko "patch drift detection" "exit=$rc out=$out"; fi +fi + +echo +echo "====================" +printf '%s\n' "${results[@]}" +echo "====================" +echo "passed: $pass" +echo "failed: $fail" +[ "$fail" -eq 0 ] || exit 1