ci: add 3-try retry wrappers to network-facing steps

Observed two flakes in 24h — distcc-avahi-aarch64 at 66a7050 and lmcp-any
at 22060e0. Journal for both showed the job starting then failing silently
before any sudo/makepkg line. Theory: transient pacman mirror sync
failures on \"pacman -Syu --needed\" in the bootstrap steps, with the
rsync-to-nc step as secondary exposure (IPv6 prefix rotations from the
Fritz reconnect cron are visible in the runner journal).

Wrap pacman -Syu (5 sites), rsync to nc and hertz (5 sites), and ssh
publish-deb (2 sites) in a tiny 3-try exponential-backoff retry helper
defined per-step. No workflow restructure, no job-graph changes.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-04-19 10:32:24 +02:00
parent 6a97649c1d
commit 9ce6943575
+32 -12
View File
@@ -18,7 +18,9 @@ jobs:
- name: bootstrap runner (idempotent)
run: |
pacman -Syu --noconfirm --needed base-devel git rsync gnupg openssh sudo avahi popt python python-setuptools
set -e
retry() { for i in 1 2 3; do "$@" && return 0; rc=$?; echo "retry $i (exit=$rc)" >&2; sleep $((i*5)); done; return 1; }
retry pacman -Syu --noconfirm --needed base-devel git rsync gnupg openssh sudo avahi popt python python-setuptools
- name: import signing key
env:
@@ -103,8 +105,10 @@ jobs:
- name: publish to aarch64
run: |
set -e
retry() { for i in 1 2 3; do "$@" && return 0; rc=$?; echo "retry $i (exit=$rc)" >&2; sleep $((i*5)); done; return 1; }
cd /tmp/arch-stage
rsync -avL --copy-unsafe-links \
retry rsync -avL --copy-unsafe-links \
-e 'ssh -i /root/.ssh/id_ed25519' \
./ mfritsche@nc.reauktion.de:arch/aarch64/
@@ -125,7 +129,10 @@ jobs:
- uses: actions/checkout@v4
- name: install dpkg
run: pacman -Syu --noconfirm --needed dpkg openssh rsync curl
run: |
set -e
retry() { for i in 1 2 3; do "$@" && return 0; rc=$?; echo "retry $i (exit=$rc)" >&2; sleep $((i*5)); done; return 1; }
retry pacman -Syu --noconfirm --needed dpkg openssh rsync curl
- name: install hertz deploy ssh key
env:
@@ -146,14 +153,15 @@ jobs:
- name: upload + publish to suites
run: |
set -e
retry() { for i in 1 2 3; do "$@" && return 0; rc=$?; echo "retry $i (exit=$rc)" >&2; sleep $((i*5)); done; return 1; }
cd debian/lmcp
DEB=$(ls lmcp_*.deb | head -1)
# Push the .deb into hertz's incoming dir via rrsync.
rsync -av -e 'ssh -i /root/.ssh/id_ed25519_hertz' "$DEB" \
retry rsync -av -e 'ssh -i /root/.ssh/id_ed25519_hertz' "$DEB" \
marfritrepo@hertz.fritz.box:
# Trigger reprepro for each suite.
for suite in bookworm trixie; do
ssh -i /root/.ssh/id_ed25519_hertz marfritrepo@hertz.fritz.box \
retry ssh -i /root/.ssh/id_ed25519_hertz marfritrepo@hertz.fritz.box \
"publish-deb $suite $DEB"
done
@@ -174,7 +182,10 @@ jobs:
- uses: actions/checkout@v4
- name: bootstrap runner (idempotent)
run: pacman -Syu --noconfirm --needed base-devel git rsync gnupg openssh sudo lua lua-socket
run: |
set -e
retry() { for i in 1 2 3; do "$@" && return 0; rc=$?; echo "retry $i (exit=$rc)" >&2; sleep $((i*5)); done; return 1; }
retry pacman -Syu --noconfirm --needed base-devel git rsync gnupg openssh sudo lua lua-socket
- name: import signing key
env:
@@ -221,6 +232,7 @@ jobs:
- name: publish lmcp to both arches
run: |
set -e
retry() { for i in 1 2 3; do "$@" && return 0; rc=$?; echo "retry $i (exit=$rc)" >&2; sleep $((i*5)); done; return 1; }
export GNUPGHOME=/root/.gnupg
printf 'pinentry-mode loopback\npassphrase-file /root/repo_pass\n' > /root/.gnupg/gpg.conf
printf 'allow-loopback-pinentry\n' > /root/.gnupg/gpg-agent.conf
@@ -250,7 +262,7 @@ jobs:
ln -sf marfrit.files.tar.gz marfrit.files
ln -sf marfrit.db.tar.gz.sig marfrit.db.sig
ln -sf marfrit.files.tar.gz.sig marfrit.files.sig
rsync -avL --copy-unsafe-links \
retry rsync -avL --copy-unsafe-links \
-e 'ssh -i /root/.ssh/id_ed25519' \
./ "mfritsche@nc.reauktion.de:arch/$target/"
done
@@ -270,7 +282,10 @@ jobs:
- uses: actions/checkout@v4
- name: bootstrap runner (idempotent)
run: pacman -Syu --noconfirm --needed base-devel git rsync gnupg openssh sudo
run: |
set -e
retry() { for i in 1 2 3; do "$@" && return 0; rc=$?; echo "retry $i (exit=$rc)" >&2; sleep $((i*5)); done; return 1; }
retry pacman -Syu --noconfirm --needed base-devel git rsync gnupg openssh sudo
- name: import signing key
env:
@@ -317,6 +332,7 @@ jobs:
- name: publish claude-his-agent to both arches
run: |
set -e
retry() { for i in 1 2 3; do "$@" && return 0; rc=$?; echo "retry $i (exit=$rc)" >&2; sleep $((i*5)); done; return 1; }
export GNUPGHOME=/root/.gnupg
printf 'pinentry-mode loopback\npassphrase-file /root/repo_pass\n' > /root/.gnupg/gpg.conf
printf 'allow-loopback-pinentry\n' > /root/.gnupg/gpg-agent.conf
@@ -346,7 +362,7 @@ jobs:
ln -sf marfrit.files.tar.gz marfrit.files
ln -sf marfrit.db.tar.gz.sig marfrit.db.sig
ln -sf marfrit.files.tar.gz.sig marfrit.files.sig
rsync -avL --copy-unsafe-links \
retry rsync -avL --copy-unsafe-links \
-e 'ssh -i /root/.ssh/id_ed25519' \
./ "mfritsche@nc.reauktion.de:arch/$target/"
done
@@ -366,7 +382,10 @@ jobs:
- uses: actions/checkout@v4
- name: install dpkg
run: pacman -Syu --noconfirm --needed dpkg openssh rsync curl
run: |
set -e
retry() { for i in 1 2 3; do "$@" && return 0; rc=$?; echo "retry $i (exit=$rc)" >&2; sleep $((i*5)); done; return 1; }
retry pacman -Syu --noconfirm --needed dpkg openssh rsync curl
- name: install hertz deploy ssh key
env:
@@ -387,12 +406,13 @@ jobs:
- name: upload + publish to suites
run: |
set -e
retry() { for i in 1 2 3; do "$@" && return 0; rc=$?; echo "retry $i (exit=$rc)" >&2; sleep $((i*5)); done; return 1; }
cd debian/claude-his-agent
DEB=$(ls claude-his-agent_*.deb | head -1)
rsync -av -e 'ssh -i /root/.ssh/id_ed25519_hertz' "$DEB" \
retry rsync -av -e 'ssh -i /root/.ssh/id_ed25519_hertz' "$DEB" \
marfritrepo@hertz.fritz.box:
for suite in bookworm trixie; do
ssh -i /root/.ssh/id_ed25519_hertz marfritrepo@hertz.fritz.box \
retry ssh -i /root/.ssh/id_ed25519_hertz marfritrepo@hertz.fritz.box \
"publish-deb $suite $DEB"
done