diff --git a/.github/workflows/build-deploy.yml b/.github/workflows/build-deploy.yml index a380d50..7730942 100644 --- a/.github/workflows/build-deploy.yml +++ b/.github/workflows/build-deploy.yml @@ -316,10 +316,20 @@ jobs: # ca-certificates-mozilla moved from 3.127-1 to 3.128-1 inside it. # # The bootstrap stage is already in the builder from the step above, so - # this exports files rather than installing anything again. A cache miss - # would resolve against newer databases, and gen-evidence still names any - # package it cannot account for, so the check is unchanged either way. - # HISTORY/evidence-race.md. + # this exports files rather than installing anything again. + # + # ⛔ The build args are part of that stage's cache key, so this step is + # handed the same three the step above was. BUILD_DATE in particular + # changes every run, and a step that omits it computes a key the build + # step never wrote, reuses a previous run's bootstrap layer, and exports + # the databases that layer was installed from. The image then holds + # today's packages and the evidence is joined against an older set, which + # gen-evidence reports as installed packages in no database. Handed the + # same args, the key matches and the layer is the one this run built. + # + # A cache miss would resolve against newer databases, and gen-evidence + # still names any package it cannot account for, so the check is + # unchanged either way. HISTORY/evidence-race.md. - name: Export the databases the build resolved against uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: @@ -328,6 +338,8 @@ jobs: target: dbsnapshot build-args: | IMAGE_VERSION=${{ needs.resolve.outputs.version }} + SOURCE_COMMIT=${{ github.sha }} + BUILD_DATE=${{ needs.resolve.outputs.build_date }} outputs: type=local,dest=/tmp/dbsnapshot cache-from: type=gha,scope=${{ matrix.docker_arch }} @@ -585,8 +597,11 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -uo pipefail - scripts/check-schedules-fired - rc=$? + # An exit of 3 is a normal result, not a failure. Under the runner's + # -e it would end this block before the capture below, so the command + # goes in an || list and the status is read from there. + rc=0 + scripts/check-schedules-fired || rc=$? case "$rc" in 0) echo "::notice::every schedule has fired within its own tolerance" ;; 3) diff --git a/.github/workflows/freshness-image-pins.yml b/.github/workflows/freshness-image-pins.yml index a30e6d7..451eb74 100644 --- a/.github/workflows/freshness-image-pins.yml +++ b/.github/workflows/freshness-image-pins.yml @@ -38,9 +38,12 @@ jobs: id: check run: | set -uo pipefail - # 0 every pin current, 3 at least one behind, 1 the check could not run. - scripts/check-image-pins | tee /tmp/pins.txt - rc="${PIPESTATUS[0]}" + # 0 every pin current, 3 at least one behind, 1 the check could not + # run. An exit of 3 is a result, so the pipeline is put in an || + # list: under the runner's -e and pipefail a bare 3 would end the + # block before PIPESTATUS could be read. + rc=0 + scripts/check-image-pins | tee /tmp/pins.txt || rc=$? echo "rc=$rc" >> "$GITHUB_OUTPUT" case "$rc" in 0) echo "::notice::every pinned image is current" ;; @@ -56,8 +59,9 @@ jobs: branch="freshness/image-pins-$(date -u +%Y%m%d)" git switch -c "$branch" echo "branch=$branch" >> "$GITHUB_OUTPUT" - scripts/check-image-pins --apply | tee /tmp/applied.txt - rc="${PIPESTATUS[0]}" + # The apply exits 3 on a bump as well, so it is guarded the same way. + rc=0 + scripts/check-image-pins --apply | tee /tmp/applied.txt || rc=$? if [ "$rc" != "3" ]; then echo "applying the bumps did not report a bump, refusing to open a pull request" >&2 exit 1 diff --git a/.github/workflows/freshness-keyring.yml b/.github/workflows/freshness-keyring.yml index 4799836..2083823 100644 --- a/.github/workflows/freshness-keyring.yml +++ b/.github/workflows/freshness-keyring.yml @@ -68,9 +68,12 @@ jobs: run: | set -uo pipefail # 0 current, 3 behind, 4 gone, 5 the pinned bytes changed, 1 the check - # could not run. - scripts/check-keyring-pin "$PIN" | tee /tmp/check.txt - rc="${PIPESTATUS[0]}" + # could not run. Every one of those is a result, so the pipeline is + # put in an || list: under the runner's -e and pipefail a bare 3, 4 + # or 5 would end the block before PIPESTATUS could be read, and the + # annotation below would never print. + rc=0 + scripts/check-keyring-pin "$PIN" | tee /tmp/check.txt || rc=$? echo "rc=$rc" >> "$GITHUB_OUTPUT" case "$rc" in 0) echo "::notice::${PIN} is current" ;; @@ -108,8 +111,10 @@ jobs: set -uo pipefail branch="freshness/${KEYRING}-$(date -u +%Y%m%d)" git switch -c "$branch" - scripts/check-keyring-pin --apply "$PIN" | tee /tmp/apply.txt - rc="${PIPESTATUS[0]}" + # The apply is a result too, and it exits 3, 4 or 5 rather than 0. + # Same || list as the check step, for the same reason. + rc=0 + scripts/check-keyring-pin --apply "$PIN" | tee /tmp/apply.txt || rc=$? case "$rc" in 3 | 4 | 5) ;; *) @@ -143,16 +148,22 @@ jobs: # installs but does not verify produces an empty root rather than a # build error. ver="$(date -u +%Y.%m.%d)" + # Read once. The dbsnapshot build below has to be handed the same + # value or its cache key is not the one the image build wrote, and it + # exports a previous run's databases instead of this build's. + build_date="$(date -u +%Y-%m-%dT%H:%M:%SZ)" docker build --platform "$PLATFORM" \ --build-arg "IMAGE_VERSION=$ver" \ --build-arg "SOURCE_COMMIT=${{ github.sha }}" \ - --build-arg "BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --build-arg "BUILD_DATE=$build_date" \ -t "freshness-check:$DOCKER_ARCH" . # The databases the build resolved against, so the evidence is not # joined against a set that moved. HISTORY/evidence-race.md. docker build --platform "$PLATFORM" --target dbsnapshot \ --build-arg "IMAGE_VERSION=$ver" \ + --build-arg "SOURCE_COMMIT=${{ github.sha }}" \ + --build-arg "BUILD_DATE=$build_date" \ --output "type=local,dest=/tmp/dbsnapshot" . echo "::group::image suite" @@ -160,7 +171,7 @@ jobs: # docker and podman, and gen-evidence prefers podman when it finds # one. The image was built with docker and lives in docker's store, so # podman tries to pull it from Docker Hub and quay.io and fails. - SOURCE_COMMIT="${{ github.sha }}" BUILD_DATE="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + SOURCE_COMMIT="${{ github.sha }}" BUILD_DATE="$build_date" \ DB_SNAPSHOT=/tmp/dbsnapshot CONTAINER_RUNTIME=docker \ scripts/gen-evidence "$DOCKER_ARCH" "freshness-check:$DOCKER_ARCH" "$PLATFORM" \ "/tmp/evidence-$DOCKER_ARCH.json" diff --git a/.github/workflows/freshness-publish.yml b/.github/workflows/freshness-publish.yml index 5659313..52ad801 100644 --- a/.github/workflows/freshness-publish.yml +++ b/.github/workflows/freshness-publish.yml @@ -65,8 +65,13 @@ jobs: id: recency run: | set -uo pipefail - scripts/check-publish-recency - rc=$? + # A non-zero exit is the answer this check is asked for, so the + # command is put in an || list. Under the runner's -e a bare command + # that exits 3 would end the block before the status could be read, + # the annotation below would never print, and the step outcome would + # be red for the wrong reason. + rc=0 + scripts/check-publish-recency || rc=$? echo "rc=$rc" >> "$GITHUB_OUTPUT" case "$rc" in 0) echo "::notice::the publish is current" ;; @@ -80,8 +85,9 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -uo pipefail - scripts/check-schedules-fired - rc=$? + # Same shape as the recency check above, and for the same reason. + rc=0 + scripts/check-schedules-fired || rc=$? echo "rc=$rc" >> "$GITHUB_OUTPUT" case "$rc" in 0) echo "::notice::every schedule has fired within its own tolerance" ;; diff --git a/Dockerfile b/Dockerfile index 7575f97..5bd5f9b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ # against what that tag resolves to now. A Dockerfile instruction takes no # trailing comment, so the marker sits on its own line directly above. # tag: latest -FROM --platform=$BUILDPLATFORM docker.io/library/archlinux@sha256:b860afd5823683f7ea389ba5f00d812f4fe55f6f286dea329d2abeefa535e309 AS bootstrap +FROM --platform=$BUILDPLATFORM docker.io/library/archlinux@sha256:3c64b538588416593f69a1aebd2f2ec0f2541b29159f32556fa7def2167147a0 AS bootstrap ARG TARGETARCH ARG TARGETVARIANT diff --git a/HISTORY/CONTINUE.md b/HISTORY/CONTINUE.md index f86fa76..833ff26 100644 --- a/HISTORY/CONTINUE.md +++ b/HISTORY/CONTINUE.md @@ -693,6 +693,9 @@ manual cross reference remap. | 50 | The dangling `.tmp/` citations, and a test that stops them coming back | `tests/static/97-scratch-citations.sh` | | 51 | This file moved into the repository, so it is no longer one wiped directory away from being lost | `HISTORY/CONTINUE.md` | | 52 | The first release, `v0.1.0`: 50 assets, eight architectures, six families, verified by downloading three of them and checking them against the published `SHA256SUMS`. It took three runs and the two that failed published nothing | run `33213236059`, `HISTORY/releases.md` | +| 53 | The freshness checks could never act on a result. A run block executes under `-e`, so a check exiting 3, 4 or 5 ended the step before the status was read, the annotation was never printed and the remediation never fired. Seven sites now capture with `cmd \|\| rc=$?`, and a third shape in the pipeline-trap test reports the old one | `HISTORY/freshness-status-capture.md`, `tests/static/25-pipeline-traps.sh` | +| 54 | The dbsnapshot export was never reading this run's bootstrap layer. A stage's build args are its cache key and `BUILD_DATE` changes every run, so the export reused an older day's databases and riscv64 failed `Record the evidence` from 2026-10-05. Both steps are now handed the same args, and the test fails if the two sets drift | `HISTORY/evidence-race.md`, `tests/static/68-evidence-snapshot.sh` | +| 55 | `gen-mirrorlist all` no longer regenerates an architecture the build matrix has disabled, which is what the monthly mirror freshness run failed on for the three PowerPC ports: ArchPOWER's anchor refuses a GitHub runner with 403 | `tests/static/75-architecture-set.sh` | --- diff --git a/HISTORY/evidence-race.md b/HISTORY/evidence-race.md index 82344bc..1975aaf 100644 --- a/HISTORY/evidence-race.md +++ b/HISTORY/evidence-race.md @@ -181,6 +181,11 @@ with `target: dbsnapshot` and `outputs: type=local,dest=/tmp/dbsnapshot`. The bootstrap stage is already in the builder from the step above, so it exports files rather than installing anything again. +⛔ **That last sentence was only true once the two steps were handed the same +build args.** A stage's build args are part of its cache key, and `BUILD_DATE` +changes every run, so the export was not reading the layer the build step had +just written. The measurement and the fix are below. + ## Measurements Built 2026-08-28 on `linux/amd64`, from the tree this document ships in. @@ -265,15 +270,59 @@ Its log shows `CACHED` on every layer of the bootstrap stage and one 15 seconds on the run that failed. Reading `extra.db` locally replaces fetching 8.9 MB, and the export adds a second back. The point is correctness, not speed. +## The arg mismatch, 2026-10-13 + +⚠ **The export was not hitting this build's cache.** On the riscv64 job the +bootstrap stage ran on every image build and reported `CACHED` on every export: + +```bash +for r in 38255111643 38135773513 38047830800; do + jid="$(gh api repos/pkgforge-dev/docker-archlinux/actions/runs/"$r"/jobs \ + --jq '.jobs[] | select(.name=="Build riscv64") | .id')" + gh run view --job "$jid" --log | grep -A1 '\[bootstrap 9/11\]' +done +``` + +| step, every day | line | +| --- | --- | +| image build | `#18 [bootstrap 9/11] RUN < Setting up target directory...` | +| export | `#11 [bootstrap 9/11] CACHED` | + +⛔ **The cause is the build args.** The image build was handed `IMAGE_VERSION`, +`SOURCE_COMMIT` and `BUILD_DATE`; the export was handed `IMAGE_VERSION` alone. +A different arg set is a different cache key, so the export found an entry from +an earlier day and exported those databases. The image held today's packages, +the join had a hole, and the run failed on it: + +``` +gen-evidence: these installed packages are in no repository database: + libsysprof-capture 50.0-6 + lmdb 0.9.35-1 + nftables 1:1.1.7-3 +gen-evidence: /tmp/dbsnapshot is not the snapshot this image was installed from +``` + +⭐ **The fix is to hand both steps the same args.** `Build and push by digest` +and `Export the databases the build resolved against` now pass the same three. +`freshness-keyring.yml` builds an image and an export in one step and had the +same mismatch; it now reads `BUILD_DATE` once so the two builds cannot straddle +a second either. `tests/static/68-evidence-snapshot.sh` assertion 5 fails if the +two arg sets drift apart, which is the one input the tree can check. + +⚠ **A matching key is necessary, not sufficient.** The export still reports +`CACHED`; what changed is that it is now this run's entry. When the build step +misses, the export misses with it, and gen-evidence still names anything it +cannot account for. + ## What this does not fix -⚠ **A cache miss on the export.** The exported databases are the ones the image -was installed from only when the export hits the layer cache the build filled. -Nothing in the tree can check that. What happens on a miss is the old behaviour: -gen-evidence still names every package it cannot account for, and the run fails -rather than passing quietly. The diagnostic distinguishes the two causes, because -a snapshot that has a hole did not come from this build, and telling the reader -that upstream moved would send them to the wrong place. +⚠ **A cache miss with matching args.** The exported databases are the ones the +image was installed from because both steps are handed the same key. If that +key misses, both miss and both resolve at the same moment against the same +mirrors, which is the pre-snapshot behaviour with a much smaller window. What +is not measurable from the tree is whether the build step and the export step +landed on the same second; `tests/static/68-evidence-snapshot.sh` checks the +inputs, not the outcome. ⚠ **The four architectures still resolve at four different times.** Each build job runs its own `pacman -Sy`, so one run is four package sets rather than one. @@ -290,9 +339,10 @@ shorter is not recorded anywhere. | --- | --- | | the Dockerfile has a `dbsnapshot` stage, taking content from the bootstrap stage | `tests/static/68-evidence-snapshot.sh` 1, 2 | | the databases are copied out before the directory is emptied | `68` 3 | -| the build job exports the target, and reads it back from the same path | `68` 4, 5 | -| a `DB_SNAPSHOT` that cannot be used is refused, and named | `68` 6, 7 | -| a usable one is accepted | `68` 8 | +| the build job exports the target, and reads it back from the same path | `68` 4, 6 | +| the export step is handed the same build args as the build step | `68` 5 | +| a `DB_SNAPSHOT` that cannot be used is refused, and named | `68` 7, 8 | +| a usable one is accepted | `68` 9 | | the evidence is refused when a package cannot be accounted for | `scripts/gen-evidence`, and `tests/image/40-evidence.sh` on the file it writes | Every one of those was seen to fail. `HISTORY/tests-seen-to-fail.md`. diff --git a/HISTORY/freshness-status-capture.md b/HISTORY/freshness-status-capture.md new file mode 100644 index 0000000..26ff6b4 --- /dev/null +++ b/HISTORY/freshness-status-capture.md @@ -0,0 +1,83 @@ +# The freshness checks could not act on a result + +⭐ **Measured 2026-10-12.** Every freshness workflow runs a check that exits +non-zero on a normal result. The step read that status on the line after the +command, and the command's exit ended the step before the line ran. + +## What the 2026-10-12 run shows + +`Freshness, port keyring pins`, run `38245829268`, the `archlinux-lcpu` job: + +```bash +gh run view --job 114799312447 --log \ + | grep -E 'result: GONE|Process completed' +``` + +``` +result: GONE, the pinned package no longer answers, this port is broken now +##[error]Process completed with exit code 4. +``` + +⭐ The check ran and answered correctly. The step had two lines after it: + +```bash +scripts/check-keyring-pin "$PIN" | tee /tmp/check.txt +rc="${PIPESTATUS[0]}" +``` + +Neither ran. The `::error` annotation was never printed, and +`steps.check.outputs.rc` was empty, so every +`if: steps.check.outputs.rc != '0'` recovery step was skipped. + +## The mechanism + +GitHub Actions runs every `shell: bash` block as +`bash --noprofile --norc -e -o pipefail`. A block may add `set -u` (and these +do, `set -uo pipefail`) but `-e` stays on. A check that exits 3, 4 or 5 and the +capture of that status therefore cannot sit in that order: the shell leaves at +the check. + +⛔ **The interface is not the defect.** Exiting 3, 4 or 5 is what +`tests/static/95-publish-watchdog.sh` and the freshness workflows were built +around. The step was reading that interface with the one construct `-e` +forbids. + +## What changed + +The command goes in a `||` list, where a non-zero status is the result the list +was written for: + +| file | before | after | +| --- | --- | --- | +| `freshness-keyring.yml` | `cmd \| tee ...; rc="${PIPESTATUS[0]}"` | `rc=0; cmd \| tee ... \|\| rc=$?` | +| `freshness-keyring.yml`, apply | same shape | same fix | +| `freshness-image-pins.yml` | same shape | same fix | +| `freshness-image-pins.yml`, apply | same shape | same fix | +| `freshness-publish.yml`, recency | `cmd; rc=$?` | `rc=0; cmd \|\| rc=$?` | +| `freshness-publish.yml`, schedules | `cmd; rc=$?` | same fix | +| `build-deploy.yml`, watchdog | `cmd; rc=$?` | same fix | + +⭐ `cmd || rc=$?` is already the shape the tests use, and it is not one of the +shapes `tests/static/20-no-swallowed-errors.sh` forbids. `set +e` and `|| true` +would have let the check run and then discarded its answer, which is the defect +that policy exists for. + +## What each red run was trying to say + +| workflow | the result that was lost | +| --- | --- | +| `freshness-keyring.yml` | the loong64 pin had been gone from its mirror since 2026-09-06, and the job no longer applied the bump | +| `freshness-image-pins.yml` | the base image digest had moved, and the job no longer applied it | +| `freshness-publish.yml` | the publish had stopped, but the run said `the recency check could not run` instead of `the newest dated index tag is older than the schedule allows` | +| `build-deploy.yml` | the watchdog produced a bare red mark and no annotation | + +## What is checked by a test + +`tests/static/25-pipeline-traps.sh` gained a third shape. It reports a line that +reads `rc=$?` or `PIPESTATUS` unless the command is guarded by `||`. The one +file it does not read is itself, because the regular expression is its own +source. `HISTORY/tests-seen-to-fail.md`. + +⚠ **What it cannot see** is a step that reads the status through some other +variable name. `rc` is the name every step here uses; a step that invents +another would pass the scan. diff --git a/HISTORY/tests-seen-to-fail.md b/HISTORY/tests-seen-to-fail.md index bdd698d..93a5f0c 100644 --- a/HISTORY/tests-seen-to-fail.md +++ b/HISTORY/tests-seen-to-fail.md @@ -14,8 +14,10 @@ two look identical in a green run. Each row below records a fault that was injected into a scratch copy of the tree, and the assertion that caught it. Measured 2026-08-26, the `45` and `60` rows on 2026-08-27, the `68`, `gen-evidence`, `90` and port keyring rows on -2026-08-28, and the publish watchdog, release asset and signed tag rows on -2026-08-29. Nothing here was injected into the repository itself. +2026-08-28, the publish watchdog, release asset and signed tag rows on +2026-08-29, and the status-capture, dbsnapshot build-arg and gen-mirrorlist +disabled-set rows on 2026-10-13. Nothing here was injected into the repository +itself. ⚠ A quoted failure message is what that run printed. Counts inside one, such as the number of `fail` calls, are the value at the time and are not kept current. @@ -41,6 +43,7 @@ fault, and pointing `REPO_ROOT` at that directory. | `25-pipeline-traps.sh` | `\| head -n "$LIMIT"` in a script that sets pipefail | `not ok 1 - none of the N pipefail scripts uses a pipeline that fails on a normal result`, naming `scripts/faulty:3` | | `25` | the `grep -c` form restored at `scripts/gen-mirrorlist:278` | `not ok 1`, naming the file and line | | `25` | the `grep -c` form restored at `.github/workflows/freshness-mirrors.yml:47` | `not ok 2 - none of the 5 workflows uses a pipeline that fails on a normal result` | +| `25` | a scratch workflow running `scripts/check-publish-recency` and then reading `rc=$?` on the next line, the shape GitHub's `-e` ends before the capture | `not ok 2 - none of the 10 workflows uses a shape that fails on a normal result`, naming `.github/workflows/zz-tmp-trap-test.yml:12: rc=$?` | | `30-signature-checking-on.sh` | `SigLevel = Never` in `rootfs/riscv64/etc/pacman.conf` | `not ok 7 - does not disable signature checking`, `not ok 8 - sets SigLevel = Required` | | `35-publish-targets.sh` | a tag-creating step changed back to `images+=("$HUB_IMAGE")` | `not ok 4 - no tag-creating step names GHCR_IMAGE or HUB_IMAGE directly`, naming both lines | | `35` | `HUB_SCRATCH_IMAGE` set to `pkgforge/archlinux`, the real repository | `not ok 3 - HUB_SCRATCH_IMAGE names a different repository from HUB_IMAGE` | @@ -70,6 +73,7 @@ fault, and pointing `REPO_ROOT` at that directory. | `68` | the copy into `/dbsnapshot` moved below the line that empties the sync directory | `not ok 3 - the databases are copied out before the directory is emptied`, `the copy is at line 76, the delete at line 75` | | `68` | `target: dbsnapshot` deleted from the build workflow | `not ok 4 - the build job builds the dbsnapshot target` | | `68` | `DB_SNAPSHOT` deleted from the evidence step | `not ok 5 - the evidence step is given DB_SNAPSHOT` | +| `68` | `SOURCE_COMMIT` and `BUILD_DATE` deleted from the dbsnapshot export step's `build-args`, leaving only `IMAGE_VERSION` | `not ok 5 - the dbsnapshot export is handed the same build args as the build`, `the build is handed: IMAGE_VERSION SOURCE_COMMIT BUILD_DATE`, `the export is handed: IMAGE_VERSION` | | `68` | the whole `DB_SNAPSHOT` validation block removed from `scripts/gen-evidence` | `not ok 6` and `not ok 7`, both reporting the runtime error instead of a refusal. ⭐ Assertion 8, the control, stays green, because without the validation the run reaches the image and dies there, so nothing names DB_SNAPSHOT at all. A control that flipped here would be measuring the wrong thing | | `90-package-lists.sh` | `archlinux-lcpu-keyring` deleted from `bootstrap/loong64/etc/bootstrap-packages.txt` | `not ok 11 - bootstrap/loong64 names archlinux-lcpu-keyring`, naming the pin that serves that architecture | | `90` | every pin's `arch` line changed to `sparc64`, so no pin matches any shipped `pacman.conf` | `not ok 11 - every pinned keyring was matched to an architecture that ships it`, `no pacman.conf Architecture matched any pin's arch list, so nothing above was checked`. ⭐ The floor is the whole point: the pairing is derived, and a derivation that matches nothing would otherwise report no failure and assert nothing | @@ -91,6 +95,7 @@ fault, and pointing `REPO_ROOT` at that directory. | `75` | the usage string cut to `gen-mirrorlist ARCH` | `not ok 17 - the gen-mirrorlist usage string names the architectures it takes` | | `75` | every `expect_alias_count` call commented out in `60-tag-families.sh` | `not ok 23 - 60-tag-families.sh checks an alias count per architecture` | | `75` | `chmod -x scripts/tag-names` | `not ok 16 - scripts/tag-names is executable`. ⚠ Injected in the container. MSYS reports every file executable, so this fault cannot be made on Windows at all | +| `75` | the `build_matrix_disabled` call deleted from the `all` branch of `gen-mirrorlist` | `not ok 18 - gen-mirrorlist all reads the disabled set from the build matrix` | | `80-docs-claims.sh` | the usage block the README shipped before, where `!#` is not a comment | `not ok 2 - README.md bash block 9 parses` | | `80` | the badge pointed back at `github.com/pkgforge/docker-archlinux` | `not ok 3 - README does not link to github.com/pkgforge/docker-archlinux` | | `80` | every mention of GHCR removed from the README | `not ok 4 - README names the publish target ghcr.io/pkgforge-dev/archlinux` | diff --git a/bootstrap/keyrings/archlinux-lcpu.pin b/bootstrap/keyrings/archlinux-lcpu.pin index f79e6ea..9bd0b97 100644 --- a/bootstrap/keyrings/archlinux-lcpu.pin +++ b/bootstrap/keyrings/archlinux-lcpu.pin @@ -42,8 +42,8 @@ keyring = archlinux-lcpu arch = loong64 mirror = https://loongarchlinux.lcpu.dev/loongarch/archlinux/core/os/loong64 -package = archlinux-lcpu-keyring-20241126-1-any.pkg.tar.zst -sha256 = cfdd65dceddbc5824df091787bd8546143beae9adb12f8ecccc25fbb67baa303 +package = archlinux-lcpu-keyring-20260906-1-any.pkg.tar.zst +sha256 = 09ab1bfa4e4e5c06863f3ce21e5bc8a734c05f88eacbb58ee7de13603cddf42f trusted = 49E74F2DF61CFF4217AB8EB737EFC8DBFB4CC627 2029-02-24 trusted = 5CDE9ADCBD04454BAF547A1C40D0304E1192746B 2026-04-24 trusted = 60922C5D40F6297BC1616C270A8F993ECADF8CE7 2026-07-14 diff --git a/scripts/gen-mirrorlist b/scripts/gen-mirrorlist index 50b6118..8e596af 100755 --- a/scripts/gen-mirrorlist +++ b/scripts/gen-mirrorlist @@ -415,12 +415,37 @@ generate() { echo " wrote $(awk '/^Server/ { n++ } END { print n + 0 }' "$any_dest") servers to rootfs/$dir/etc/pacman.d/mirrorlist-any" } +# build_matrix_disabled -> the docker_arch of every entry the build matrix +# carries commented out +# +# ⛔ A port whose build is disabled is not regenerated either. ArchPOWER's +# origin refuses a GitHub runner with 403, so `all` cannot generate the three +# PowerPC lists from CI at all, and the files on disk are the last ones written +# from a network that could reach it. Regenerating a subset is deliberate: the +# explicit `scripts/gen-mirrorlist ppc` still works, it just is not part of +# `all`. The static suite skips a disabled architecture for the same reason. +build_matrix_disabled() { + awk ' + /^[[:space:]]*#[[:space:]]*-[[:space:]]+docker_arch:/ { + s = $0 + sub(/^[^:]*:[[:space:]]*/, "", s) + sub(/[[:space:]]*$/, "", s) + print s + } + ' "$REPO_ROOT/.github/workflows/build-deploy.yml" +} + case "${1:-}" in "") die "usage: scripts/gen-mirrorlist " ;; all) + disabled="$(build_matrix_disabled | tr -d '\r')" for d in amd64 arm64 armv7 loong64 riscv64 ppc ppc64 ppc64le; do + if printf '%s\n' "$disabled" | grep -qxF "$d"; then + echo "gen-mirrorlist: skipping $d, its build is disabled in the matrix" >&2 + continue + fi generate "$d" done ;; diff --git a/tests/static/25-pipeline-traps.sh b/tests/static/25-pipeline-traps.sh index c5e9164..956855e 100755 --- a/tests/static/25-pipeline-traps.sh +++ b/tests/static/25-pipeline-traps.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash # # No script that sets pipefail may use a pipeline shape that fails on a normal -# result. +# result, and no run block may read a status the shell never reached. # -# Two shapes, both measured rather than reasoned about: +# Three shapes, all measured rather than reasoned about: # # seq 1 2000000 | head -n 40 | tail -1 rc 141. head closes the pipe, the # producer takes SIGPIPE, and pipefail @@ -11,17 +11,25 @@ # grep -c '^x' /dev/null rc 1. A count of zero is a failure # exit, so under set -e an assignment # from it stops the script. +# cmd; rc=$? the assignment is never reached. +# GitHub Actions runs a run block under +# -e, so cmd exiting 3 ends the block +# on that line. The capture, the case +# that explains the status, and the +# output an if: reads are all skipped. # -# Both kill a script with no message at all, and both do it on an input that is +# All kill a script with no message at all, and all do it on an input that is # perfectly normal. The second is worse: it stops the script before it can reach # the error path written to handle exactly that case. That is what # scripts/gen-mirrorlist did, where a count of zero servers stopped the function -# three lines above the die that would have explained it. +# three lines above the die that would have explained it. The third is what +# limited every freshness workflow to exit 0 or a bare failure, so a stale pin +# was reported as a check that could not run and the remediation never fired. # # ⚠ This is a text scan, so it reads what a line looks like rather than what the # shell would do with it. Comments are stripped first. A pattern inside a # diagnostic string is indistinguishable from a command here, so no diagnostic -# in this repository is written using either shape. +# in this repository is written using any of the three shapes. set -euo pipefail # shellcheck source-path=SCRIPTDIR/.. # shellcheck source=lib/harness.sh @@ -38,6 +46,15 @@ early_close="[|][[:space:]]*head([[:space:]]|$)" # that happens to contain the letter c. zero_is_failure="grep[[:space:]]+(-[a-zA-Z]*[[:space:]]+)*-[a-zA-Z]*c[a-zA-Z]*([[:space:]]|$)" +# The status read that under -e cannot run. +# +# ⛔ A non-zero exit is a result here, not a failure: 3 is behind, 4 is gone, 5 +# is changed bytes. The command has to go in an `||` list, where a non-zero +# status is what the list was written for, so `cmd || rc=$?` is the supported +# shape and is excluded. The variable is spelled `rc` exactly, so an unrelated +# `unset_rc=$?` is not read as one. +status_capture="(^|[^A-Za-z0-9_])rc=[^|]*[$][?]|PIPESTATUS" + # strip_comments FILE -> the file with # comments removed, line numbers kept strip_comments() { awk '{ sub(/(^|[[:space:]])#.*$/, ""); print }' "$1" @@ -46,8 +63,8 @@ strip_comments() { # scan FILE LABEL -> prints "path:line: text" for each trapped line scan() { local file="$1" rel="$2" - strip_comments "$file" | awk -v rel="$rel" -v a="$early_close" -v b="$zero_is_failure" ' - $0 ~ a || $0 ~ b { + strip_comments "$file" | awk -v rel="$rel" -v a="$early_close" -v b="$zero_is_failure" -v c="$status_capture" ' + $0 ~ a || $0 ~ b || ($0 ~ c && $0 !~ /\|\|/) { line = $0 sub(/^[[:space:]]+/, "", line) printf "%s:%d: %s\n", rel, NR, line @@ -65,6 +82,12 @@ for d in scripts tests bootstrap examples; do # Only files that actually turn pipefail on. A `set` command enabling it, # not the word appearing in prose. grep -qE '^[[:space:]]*set[[:space:]][^#]*pipefail' "$f" || continue + # ⛔ This file is skipped. The status shape is defined here as a regular + # expression, so the definition line carries the very text the scan looks + # for and a file cannot police its own source. Every other file is read. + if [ "$f" = "$REPO_ROOT/tests/static/25-pipeline-traps.sh" ]; then + continue + fi shell_files="$shell_files$f " done <<< "$(find "$REPO_ROOT/$d" -type f \( -name '*.sh' -o -perm -u+x \) | sort)" @@ -85,13 +108,15 @@ if [ "$scanned" -eq 0 ]; then "none found, so this test asserted nothing" \ "reproduce: grep -rlE '^[[:space:]]*set[[:space:]][^#]*pipefail' scripts tests" elif [ -z "$trapped" ]; then - ok "none of the $scanned pipefail scripts uses a pipeline that fails on a normal result" + ok "none of the $scanned pipefail scripts uses a shape that fails on a normal result" else - fail "none of the $scanned pipefail scripts uses a pipeline that fails on a normal result" \ + fail "none of the $scanned pipefail scripts uses a shape that fails on a normal result" \ "head closing the pipe makes the producer take SIGPIPE, which pipefail reports as 141" \ "counting with a grep flag exits 1 on zero, which under set -e stops the script silently" \ + "reading \$? after a bare command is a line the shell never reaches, because that command's exit already stopped it" \ "count lines with: awk '/pattern/ { n++ } END { print n + 0 }' FILE" \ "limit output with: awk -v n=N 'NR <= n' instead of head -n N" \ + "read a status with: cmd || rc=\$? , which is why that shape is not reported" \ "reproduce: bash tests/run.sh static, which names each file and line" while IFS= read -r t; do [ -n "$t" ] && diag "$t" @@ -122,11 +147,13 @@ if [ "$wf_scanned" -eq 0 ]; then "no .yml found under .github/workflows" \ "reproduce: ls .github/workflows" elif [ -z "$wf_trapped" ]; then - ok "none of the $wf_scanned workflows uses a pipeline that fails on a normal result" + ok "none of the $wf_scanned workflows uses a shape that fails on a normal result" else - fail "none of the $wf_scanned workflows uses a pipeline that fails on a normal result" \ + fail "none of the $wf_scanned workflows uses a shape that fails on a normal result" \ "run blocks execute under -eo pipefail even when the block does not set it" \ + "a status read after a bare command is a line the shell never reaches, so the annotation and the outputs an if: reads are skipped" \ "count lines with: awk '/pattern/ { n++ } END { print n + 0 }' FILE" \ + "read a status with: cmd || rc=\$? , which is why that shape is not reported" \ "reproduce: bash tests/run.sh static, which names each file and line" while IFS= read -r t; do [ -n "$t" ] && diag "$t" diff --git a/tests/static/68-evidence-snapshot.sh b/tests/static/68-evidence-snapshot.sh index 48cd185..6e7d4d2 100755 --- a/tests/static/68-evidence-snapshot.sh +++ b/tests/static/68-evidence-snapshot.sh @@ -21,6 +21,12 @@ # the exported databases are the ones the image was installed from. Nothing in # the tree can. gen-evidence still names any package it cannot account for, so # a miss that mattered fails the run rather than passing quietly. +# +# ⭐ The one thing that can be checked is the input both builds share: whether +# they are handed the same cache key, because a differing set of build args is +# what produced a stale export in the first place. Section 2 asserts the export +# step gets the same build args as the build step, so a drift there fails here +# rather than as an unexplained evidence error on one architecture. set -euo pipefail # shellcheck source-path=SCRIPTDIR/.. # shellcheck source=lib/harness.sh @@ -105,6 +111,40 @@ else "reproduce: grep -n dbsnapshot .github/workflows/build-deploy.yml" fi +#---------------------------------------------------------------------------# +# ⛔ The two steps have to be handed the same build args. They are the cache +# key of the bootstrap stage, and BUILD_DATE changes every run, so the export +# step omitting it computes a key the build step never wrote, reuses a previous +# run's layer, and exports the databases that layer was installed from. The +# image holds today's packages, the evidence is joined against an older set, +# and gen-evidence reports installed packages that are in no database. +#---------------------------------------------------------------------------# +step_args() { # -> the build-arg names under it, one per line + awk -v step="$1" ' + index($0, step) { f = 1 } + f && index($0, "build-args: |") { g = 1; next } + f && g && /^[[:space:]]+[A-Z_]+=/ { sub(/=.*/, ""); gsub(/[[:space:]]/, ""); print; next } + f && g { exit } + ' "$WF" +} + +build_args="$(step_args "name: Build and push by digest")" +snap_args="$(step_args "name: Export the databases the build resolved against")" + +if [ -z "$build_args" ]; then + fail "the dbsnapshot export is handed the same build args as the build" \ + "no build-args block found under 'name: Build and push by digest'" \ + "reproduce: grep -n -A5 'build-args' .github/workflows/build-deploy.yml" +elif [ "$build_args" = "$snap_args" ]; then + ok "the dbsnapshot export is handed the same build args as the build: $(printf '%s' "$build_args" | tr '\n' ' ')" +else + fail "the dbsnapshot export is handed the same build args as the build" \ + "the build is handed: $(printf '%s' "$build_args" | tr '\n' ' ')" \ + "the export is handed: $(printf '%s' "$snap_args" | tr '\n' ' ')" \ + "a different set is a different cache key, so the export reads a previous run's databases" \ + "reproduce: grep -n -A5 'build-args' .github/workflows/build-deploy.yml" +fi + dest="$(awk 'index($0, "outputs: type=local,dest=") { sub(/.*dest=/, "") gsub(/[[:space:]]/, "") diff --git a/tests/static/75-architecture-set.sh b/tests/static/75-architecture-set.sh index 16dce30..e335082 100755 --- a/tests/static/75-architecture-set.sh +++ b/tests/static/75-architecture-set.sh @@ -376,6 +376,23 @@ else "reproduce: grep -n 'usage: scripts/gen-mirrorlist' scripts/gen-mirrorlist" fi fi + + # `all` must skip an architecture the matrix has disabled. The PowerPC + # anchors refuse a GitHub runner, so regenerating them from CI fails before + # anything is written. + # + # ⚠ Checked as the call in the all branch rather than by running it. A run + # would probe every mirror and rewrite the lists, which this suite must not + # do. Naming the disabled set is the part that goes missing when the skip is + # removed, so that is the part asserted. + all_branch="$(awk '/^[[:space:]]*all\)[[:space:]]*$/ { f = 1 } f && /^[[:space:]]*;;/ { exit } f { print }' "$GM")" + if printf '%s\n' "$all_branch" | grep -q 'build_matrix_disabled'; then + ok "gen-mirrorlist all reads the disabled set from the build matrix" + else + fail "gen-mirrorlist all reads the disabled set from the build matrix" \ + "the all branch does not call build_matrix_disabled, so it regenerates a port whose build is disabled" \ + "reproduce: sed -n '/^ all)/,/;;/p' scripts/gen-mirrorlist" + fi fi #---------------------------------------------------------------------------#