From 34bf78e21f4db7e43f9c8131967b10cf9b59cb59 Mon Sep 17 00:00:00 2001 From: Max042004 Date: Mon, 29 Jun 2026 00:58:20 +0800 Subject: [PATCH] tests: resolve Alpine fixture versions from the live APKINDEX The fixture builder pinned an exact version for every Alpine package and for the minirootfs tarball. Alpine's mirror keeps only the latest build of each package, so a pinned version 404s as soon as a newer build lands upstream -- forcing a manual version bump on every refresh just to keep the test fixtures fetchable (linux-virt alone churned 6.12.85 -> .91 -> .94 across recent weeks). Resolve versions dynamically instead, staying within the bash 3.2 subset the harness now targets (no associative arrays): - PKGS becomes a plain "repo:name" list. resolve_versions() downloads each repo's APKINDEX, flattens it to a cached "name version" table, and fills PKG_RESOLVED with "repo:name:version" tuples -- the same shape the staging loops and pkg_version already consume. - resolve_minirootfs() picks the newest point release the releases listing advertises and writes it back to ALPINE_PATCH so the x86_64 path reuses the same patch. ALPINE_PATCH still pins one when set. - A versions.lock manifest records the resolved set; when it changes the staged rootfs/kernel/initramfs (and x86_64 tree) are rebuilt so a mirror bump actually takes effect instead of reusing stale apks. - Both resolvers fall back to the cache when the mirror is unreachable, preserving warm offline re-runs, and error cleanly when neither the mirror nor a cache is available. Verified on stock /bin/bash 3.2.57: cold build, warm idempotent re-run, INCLUDE_X86_64=1, offline warm fallback, lockfile-driven rebuild, and the offline+empty-cache error path. --- tests/fetch-fixtures.sh | 259 +++++++++++++++++++++++++++++++--------- 1 file changed, 201 insertions(+), 58 deletions(-) diff --git a/tests/fetch-fixtures.sh b/tests/fetch-fixtures.sh index 87a6e36..8fbfa65 100755 --- a/tests/fetch-fixtures.sh +++ b/tests/fetch-fixtures.sh @@ -36,7 +36,10 @@ set -euo pipefail . "$(dirname "$0")/lib/bash-compat.sh" ALPINE_VERSION="${ALPINE_VERSION:-3.21}" -ALPINE_PATCH="${ALPINE_PATCH:-3.21.0}" +# Empty by default -- the exact point release is resolved from the live +# releases listing at fetch time (see resolve_minirootfs), then written back +# here so the x86_64 path reuses the same patch. Set explicitly to pin one. +ALPINE_PATCH="${ALPINE_PATCH:-}" ALPINE_ARCH="${ALPINE_ARCH:-aarch64}" CDN_BASE="https://dl-cdn.alpinelinux.org/alpine/v${ALPINE_VERSION}" @@ -53,60 +56,71 @@ KEYS_DIR="${FIXTURES}/keys" STATICBIN="${FIXTURES}/aarch64-musl/staticbin/bin" INITRAMFS="${FIXTURES}/initramfs.cpio.gz" -# Pinned package versions (Alpine 3.21). When bumping ALPINE_VERSION, -# refresh these by querying the repo's APKINDEX. +# Required packages as "repo:name". Versions are NOT pinned here: Alpine's +# mirror keeps only the latest build of each package, so any hard-coded +# version 404s once a newer build supersedes it. resolve_versions() reads +# each repo's live APKINDEX and fills PKG_RESOLVED with "repo:name:version" +# tuples; the rest of the script and pkg_version go through that. # -# Encoded as "repo:name:version" tuples so bash 3.2 hosts (stock macOS -# /bin/bash) do not need associative arrays. Lookup goes through -# pkg_version below. +# Tuples (not associative arrays) keep this working on bash 3.2 hosts (stock +# macOS /bin/bash) -- see tests/lib/bash-compat.sh. PKGS=( - "main:linux-virt:6.12.91-r0" - "main:busybox-static:1.37.0-r14" - "main:dropbear:2024.86-r0" - "main:zlib:1.3.2-r0" - "main:utmps-libs:0.1.2.3-r2" - "main:skalibs-libs:2.14.3.0-r0" - "main:musl:1.2.5-r11" - "main:musl-dev:1.2.5-r11" - "main:musl-utils:1.2.5-r11" - "main:libgcc:14.2.0-r4" - "main:libcrypto3:3.3.7-r0" - "main:acl-libs:2.3.2-r1" - "main:libattr:2.5.2-r2" - "main:pcre2:10.43-r0" - "main:coreutils:9.5-r2" - "main:coreutils-env:9.5-r2" - "main:coreutils-fmt:9.5-r2" - "main:coreutils-sha512sum:9.5-r2" - "main:bash:5.2.37-r0" - "main:dash:0.5.12-r2" - "main:findutils:4.10.0-r0" - "main:diffutils:3.10-r0" - "main:grep:3.11-r0" - "main:sed:4.9-r2" - "main:gawk:5.3.1-r0" - "main:gmp:6.3.0-r2" - "main:readline:8.2.13-r0" - "main:libncursesw:6.5_p20241006-r3" - "main:ncurses-terminfo-base:6.5_p20241006-r3" - "main:lua5.4:5.4.7-r0" - "main:lua5.4-libs:5.4.7-r0" - "main:luajit:2.1_p20240815-r0" - "main:jq:1.7.1-r0" - "main:oniguruma:6.9.9-r0" - "main:sqlite:3.48.0-r4" - "main:sqlite-libs:3.48.0-r4" - "main:tree:2.2.1-r0" + "main:linux-virt" + "main:busybox-static" + "main:dropbear" + "main:zlib" + "main:utmps-libs" + "main:skalibs-libs" + "main:musl" + "main:musl-dev" + "main:musl-utils" + "main:libgcc" + "main:libcrypto3" + "main:acl-libs" + "main:libattr" + "main:pcre2" + "main:coreutils" + "main:coreutils-env" + "main:coreutils-fmt" + "main:coreutils-sha512sum" + "main:bash" + "main:dash" + "main:findutils" + "main:diffutils" + "main:grep" + "main:sed" + "main:gawk" + "main:gmp" + "main:readline" + "main:libncursesw" + "main:ncurses-terminfo-base" + "main:lua5.4" + "main:lua5.4-libs" + "main:luajit" + "main:jq" + "main:oniguruma" + "main:sqlite" + "main:sqlite-libs" + "main:tree" ) -# Look up a package version by its "repo:name" prefix. Returns the +# "repo:name:version" tuples, populated by resolve_versions() from live +# APKINDEX data. Same shape the staging loops and pkg_version expect. +PKG_RESOLVED=() + +# Set to 1 by main() when the resolved version set differs from versions.lock, +# forcing a rebuild of the staged tree even without FORCE. Global so the +# x86_64 path can honor it too. +REBUILD=0 + +# Look up a resolved package version by its "repo:name" prefix. Returns the # version on stdout and rc=0 on hit, rc=1 (silently) on miss so the # old ${PKGS[key]:-} fallback callers keep working. pkg_version() { local target="$1:" local entry - for entry in "${PKGS[@]}"; do + for entry in ${PKG_RESOLVED[@]+"${PKG_RESOLVED[@]}"}; do case "$entry" in "$target"*) printf '%s\n' "${entry#"$target"}" @@ -132,7 +146,8 @@ STATIC_APPLETS=( cmp diff find sed grep awk ) -MINIROOTFS_TGZ="alpine-minirootfs-${ALPINE_PATCH}-${ALPINE_ARCH}.tar.gz" +# Resolved by resolve_minirootfs() before staging. +MINIROOTFS_TGZ="" c_blue() { @@ -186,6 +201,113 @@ apk_path() echo "${CACHE}/${name}-${version}.apk" } +repo_url() +{ + case "$1" in + main) echo "$MAIN_REPO" ;; + community) echo "$COMMUNITY_REPO" ;; + *) + echo "unknown repo: $1" >&2 + return 1 + ;; + esac +} + +# Resolve the current version of every package in PKGS by parsing each +# referenced repo's APKINDEX, populating PKG_RESOLVED with "repo:name:version" +# tuples. Alpine prunes superseded builds from the mirror, so reading the live +# index is the only reliable way to keep the fixture build from 404-ing on a +# stale pin. Flattened "name version" indexes are cached so warm re-runs work +# offline. +resolve_versions() +{ + # Unique set of repos referenced by PKGS (no associative arrays: track a + # space-padded string for membership tests). + local entry repo repos="" + for entry in "${PKGS[@]}"; do + repo="${entry%%:*}" + case " $repos " in + *" $repo "*) ;; + *) repos="$repos $repo" ;; + esac + done + + local base idxtgz idxfile + for repo in $repos; do + base="$(repo_url "$repo")" + idxtgz="${CACHE}/APKINDEX-${repo}.tar.gz" + idxfile="${CACHE}/APKINDEX-${repo}.versions" + log "resolve versions ($repo)" + # Always try to refresh the index -- tracking the live mirror is the + # point -- but fall back to a cached copy so warm re-runs work offline. + # APKINDEX records are blank-line separated; P: is the package name, + # V: its version. Flatten to "name version" lines. + if curl -fsSL --retry 3 -o "${idxtgz}.partial" "${base}/APKINDEX.tar.gz"; then + mv "${idxtgz}.partial" "$idxtgz" + tar xzOf "$idxtgz" APKINDEX 2> /dev/null | awk ' + /^P:/ { name = substr($0, 3) } + /^V:/ { ver = substr($0, 3) } + /^$/ { if (name != "") print name, ver; name = ""; ver = "" } + END { if (name != "") print name, ver } + ' > "${idxfile}.partial" + mv "${idxfile}.partial" "$idxfile" + else + rm -f "${idxtgz}.partial" + [ -s "$idxfile" ] || { + echo "error: cannot fetch APKINDEX for $repo (mirror unreachable, no cache)" >&2 + exit 1 + } + log "offline: reusing cached APKINDEX ($repo)" + fi + done + + # Resolve each package against its repo's flattened index. + local name version missing=0 + PKG_RESOLVED=() + for entry in "${PKGS[@]}"; do + repo="${entry%%:*}" + name="${entry#*:}" + idxfile="${CACHE}/APKINDEX-${repo}.versions" + version="$(awk -v n="$name" '$1 == n { print $2; exit }' "$idxfile")" + if [ -z "$version" ]; then + echo "error: package ${repo}:${name} not present in APKINDEX" >&2 + missing=1 + continue + fi + PKG_RESOLVED+=("${repo}:${name}:${version}") + done + [ "$missing" = 0 ] || exit 1 +} + +# Resolve the minirootfs point release. Alpine keeps only a handful of recent +# releases under releases/, so honor an explicit ALPINE_PATCH but otherwise +# pick the newest the mirror advertises and write it back to ALPINE_PATCH so +# the x86_64 path reuses the same patch. Sets MINIROOTFS_TGZ. +resolve_minirootfs() +{ + if [ -z "$ALPINE_PATCH" ]; then + log "resolve minirootfs" + local listing="" + listing="$(curl -fsSL --retry 3 "${RELEASES}/" 2> /dev/null || true)" + ALPINE_PATCH="$(printf '%s\n' "$listing" \ + | grep -oE "alpine-minirootfs-[0-9.]+-${ALPINE_ARCH}\.tar\.gz" \ + | sed -E "s/^alpine-minirootfs-([0-9.]+)-${ALPINE_ARCH}\.tar\.gz/\1/" \ + | sort -uV | tail -1 || true)" + if [ -z "$ALPINE_PATCH" ]; then + # Offline fallback: newest minirootfs already in the cache. + ALPINE_PATCH="$(ls "${CACHE}"/alpine-minirootfs-*-"${ALPINE_ARCH}".tar.gz \ + 2> /dev/null \ + | sed -E "s#.*/alpine-minirootfs-([0-9.]+)-${ALPINE_ARCH}\.tar\.gz#\1#" \ + | sort -V | tail -1 || true)" + fi + if [ -z "$ALPINE_PATCH" ]; then + echo "error: no minirootfs tarball found (mirror unreachable, cache empty)" >&2 + exit 1 + fi + fi + MINIROOTFS_TGZ="alpine-minirootfs-${ALPINE_PATCH}-${ALPINE_ARCH}.tar.gz" +} + # Strip Alpine apk metadata (.PKGINFO, .SIGN.*, .pre-install, etc.) when # extracting into a target tree. These are not real files and pollute the # rootfs. @@ -209,9 +331,26 @@ main() { mkdir -p "$CACHE" "$KERNEL_DIR" "$KEYS_DIR" "$STATICBIN" "$ROOTFS" + # Resolve all package versions and the minirootfs name from the live mirror + # before downloading anything. + resolve_versions + resolve_minirootfs + + # If the resolved version set changed since the last run, the staged + # rootfs/kernel/initramfs are built from stale apks and must be rebuilt + # even without an explicit FORCE. + local entry manifest lockfile + manifest="$( { for entry in "${PKG_RESOLVED[@]}"; do + printf '%s\n' "$entry" + done | LC_ALL=C sort; printf 'minirootfs=%s\n' "$MINIROOTFS_TGZ"; } )" + lockfile="${FIXTURES}/versions.lock" + if [ ! -f "$lockfile" ] || [ "$manifest" != "$(cat "$lockfile")" ]; then + REBUILD=1 + fi + # Download all required apk packages. local entry repo name version - for entry in "${PKGS[@]}"; do + for entry in "${PKG_RESOLVED[@]}"; do repo="${entry%%:*}" name="${entry#*:}" name="${name%:*}" @@ -222,7 +361,7 @@ main() fetch "${RELEASES}/${MINIROOTFS_TGZ}" "${CACHE}/${MINIROOTFS_TGZ}" # Stage the rootfs. - if [ "${FORCE:-0}" = "1" ] || [ ! -e "${ROOTFS}/.staged" ]; then + if [ "${FORCE:-0}" = "1" ] || [ "$REBUILD" = 1 ] || [ ! -e "${ROOTFS}/.staged" ]; then log "stage rootfs" rm -rf "$ROOTFS" mkdir -p "$ROOTFS" @@ -231,7 +370,7 @@ main() # Overlay every cached apk except linux-virt (kernel goes elsewhere). # The kernel apk's lib/modules/ tree IS overlayed (needed for modprobe). local entry name version - for entry in "${PKGS[@]}"; do + for entry in "${PKG_RESOLVED[@]}"; do name="${entry#*:}" name="${name%:*}" version="${entry##*:}" @@ -326,7 +465,7 @@ EOF ok "ssh keypair installed" # Extract the kernel from linux-virt. - if [ ! -s "${KERNEL_DIR}/vmlinuz-virt" ] || [ "${FORCE:-0}" = "1" ]; then + if [ ! -s "${KERNEL_DIR}/vmlinuz-virt" ] || [ "${FORCE:-0}" = "1" ] || [ "$REBUILD" = 1 ]; then log "extract kernel" local linux_virt_ver linux_virt_ver="$(pkg_version "main:linux-virt")" @@ -340,7 +479,7 @@ EOF ok "kernel: ${KERNEL_DIR}/vmlinuz-virt" # Build the initramfs archive. - if [ ! -s "$INITRAMFS" ] || [ "$ROOTFS/.staged" -nt "$INITRAMFS" ]; then + if [ ! -s "$INITRAMFS" ] || [ "$REBUILD" = 1 ] || [ "$ROOTFS/.staged" -nt "$INITRAMFS" ]; then log "build initramfs" (cd "$ROOTFS" && find . -print0 | LC_ALL=C sort -z \ | cpio --quiet --null -o -H newc 2> /dev/null | gzip -9) > "$INITRAMFS" @@ -353,7 +492,7 @@ EOF # AND by qemu's guest kernel after a 9p mount, where any absolute host # path would no longer resolve. local dynbin="${FIXTURES}/aarch64-musl/dyn-bin" - if [ ! -d "$dynbin" ] || [ "${FORCE:-0}" = "1" ]; then + if [ ! -d "$dynbin" ] || [ "${FORCE:-0}" = "1" ] || [ "$REBUILD" = 1 ]; then log "stage dyn-bin aggregate" rm -rf "$dynbin" mkdir -p "$dynbin" @@ -373,7 +512,7 @@ EOF ok "dyn-bin: $(find "$dynbin" -maxdepth 1 -type l 2> /dev/null | wc -l | tr -d ' ') entries" # Stage the static-bin tree. - if [ ! -s "${STATICBIN}/busybox" ] || [ "${FORCE:-0}" = "1" ]; then + if [ ! -s "${STATICBIN}/busybox" ] || [ "${FORCE:-0}" = "1" ] || [ "$REBUILD" = 1 ]; then log "stage static-bin tree" local busybox_ver busybox_ver="$(pkg_version "main:busybox-static")" @@ -395,6 +534,10 @@ EOF fetch_x86_64_userspace fi + # Record the resolved version set so the next run can detect a mirror bump + # and rebuild the staged tree instead of silently reusing stale apks. + printf '%s\n' "$manifest" > "$lockfile" + printf '\n%s\n' "$(c_yellow 'Fixtures ready.')" printf 'rootfs/sysroot: %s\n' "$ROOTFS" printf 'kernel: %s\n' "${KERNEL_DIR}/vmlinuz-virt" @@ -426,7 +569,7 @@ fetch_x86_64_userspace() log "x86_64: fetch packages" local entry repo name version x86_url x86_dest - for entry in "${PKGS[@]}"; do + for entry in "${PKG_RESOLVED[@]}"; do repo="${entry%%:*}" name="${entry#*:}" name="${name%:*}" @@ -438,13 +581,13 @@ fetch_x86_64_userspace() done fetch "${x86_releases}/${x86_minirootfs}" "${x86_cache}/${x86_minirootfs}" - if [ "${FORCE:-0}" = "1" ] || [ ! -e "${x86_rootfs}/.staged" ]; then + if [ "${FORCE:-0}" = "1" ] || [ "$REBUILD" = 1 ] || [ ! -e "${x86_rootfs}/.staged" ]; then log "x86_64: stage rootfs" rm -rf "$x86_rootfs" mkdir -p "$x86_rootfs" tar xzf "${x86_cache}/${x86_minirootfs}" -C "$x86_rootfs" 2> /dev/null local stage_entry stage_repo stage_name stage_version - for stage_entry in "${PKGS[@]}"; do + for stage_entry in "${PKG_RESOLVED[@]}"; do stage_repo="${stage_entry%%:*}" stage_name="${stage_entry#*:}" stage_name="${stage_name%:*}" @@ -457,7 +600,7 @@ fetch_x86_64_userspace() fi ok "x86_64 rootfs ($(du -sh "$x86_rootfs" 2> /dev/null | cut -f1))" - if [ ! -s "${x86_staticbin}/busybox" ] || [ "${FORCE:-0}" = "1" ]; then + if [ ! -s "${x86_staticbin}/busybox" ] || [ "${FORCE:-0}" = "1" ] || [ "$REBUILD" = 1 ]; then log "x86_64: stage static-bin tree" local busybox_ver busybox_ver="$(pkg_version "main:busybox-static")" @@ -477,7 +620,7 @@ fetch_x86_64_userspace() ok "x86_64 static-bin: ${x86_staticbin}/busybox + ${#STATIC_APPLETS[@]} applets" if [ ! -d "$x86_dynbin" ] || [ -z "$(ls -A "$x86_dynbin" 2> /dev/null)" ] \ - || [ "${FORCE:-0}" = "1" ]; then + || [ "${FORCE:-0}" = "1" ] || [ "$REBUILD" = 1 ]; then log "x86_64: stage dyn-bin aggregate" rm -rf "$x86_dynbin" mkdir -p "$x86_dynbin"