diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index fdfa770..eb6aa08 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -64,6 +64,43 @@ on: schedule: - cron: '0 6 * * 0' +# One build.yml run per branch at a time (#4290). +# +# Without this, two pushes to one branch run in full parallel. Both read +# `fc.revision` off the channel tag before either has pushed, so both miss the +# reuse check and both build, and whichever finishes LAST owns the tag — so a +# slower older build can leave `:dev` carrying content older than the commit +# that moved it. Family rule 146 says a rolling channel refreshes itself; that +# is the case where it quietly does not. +# +# `cancel-in-progress: false` — QUEUE, never cancel. Cancelling could kill +# sign-extension mid-AMO-upload, leaving the version registered at AMO with no +# cached asset: exactly the stuck state the rollback trap in that job exists to +# prevent, reached through a different door. AMO will not release a burned +# version, so that state is unrecoverable rather than merely annoying. Waiting +# a few minutes is the cheaper end of that trade by a wide margin. +# +# Keyed on `github.ref`, so `dev` and `main` never block each other. Not on +# BUILD_REF: the group is evaluated before any job starts and cannot read the +# `env` context (the same restriction that makes a job-level `if:` unable to +# see it — see build-web's `outputs.candidate` note). The one consequence is +# that a scheduled refresh, whose ref is the default branch, shares dev's +# queue while publishing main's channel. It only ever waits, and it fires +# 06:00 Sunday precisely because nothing else is running then. +# +# UNVERIFIED AT THE TIME OF WRITING, and this file has been burned by exactly +# that before: the `format()` note below records `true == 'true'` evaluating +# FALSE on run 5270 with no symptom whatsoever — every lane green, the feature +# simply not happening. A `concurrency:` key this Gitea ignored would look +# identical: runs still overlapping, nothing red. So this is a belt, and the +# digest-pinned repoint in each build job is the braces — that one makes the +# :c- correctness property hold whether or not this key is honoured. +# Confirm by pushing twice in quick succession and reading the RUN LIST for a +# queued second run, never by reading this comment. +concurrency: + group: build-${{ github.ref }} + cancel-in-progress: false + # Which branch a run BUILDS, as opposed to which one triggered it. # # They are the same thing on every trigger but `schedule`. Forgejo registers a @@ -873,6 +910,10 @@ jobs: ls -la frontend/public/extension/ - name: Build and push web image + # `id:` so the repoint step below can read `outputs.digest` — the + # manifest THIS run published, as opposed to whatever the channel tag + # happens to name by the time that step runs (#4290). + id: build if: steps.reuse.outputs.hit != 'true' # Read by buildx out of the ENVIRONMENT, not passed as a build-arg — # it normalises the image config's `created` field and the history @@ -1002,10 +1043,42 @@ jobs: - name: Write the remaining tags from the published image env: IMAGE: git.fabledsword.com/bvandeusen/fabledcurator - SOURCE: ${{ steps.reuse.outputs.channel_ref }} + CHANNEL_REF: ${{ steps.reuse.outputs.channel_ref }} + # Empty when no build ran this job (a reuse hit, or the step's `if:` + # skipped it). Non-empty means THIS run pushed that manifest. + BUILT_DIGEST: ${{ steps.build.outputs.digest }} TAGS: ${{ steps.tag.outputs.tags }} run: | set -euf + # WHAT WE COPY FROM, which is not what we EXCLUDE (#4290). + # + # This step used to copy from the channel tag by NAME. Nothing + # serialises builds — there is no `concurrency:` key anywhere in + # .forgejo/workflows/ — so two pushes to one branch run in full + # parallel, both miss the reuse check, and both build. If the OLDER + # one finishes last it wins the channel tag; and then its repoint + # step, reading that tag by name, wrote :c- from whatever the + # other run had just published. An immutable rollback tag (rule 145) + # naming a different commit's bytes, wrong from birth — and + # immutability then guarantees nobody ever corrects it. Nothing goes + # red; it surfaces the day someone needs to roll back. + # + # So when this job built, copy from the DIGEST it pushed. Correct + # whatever a concurrent run does to the tag, and it does not depend + # on the runner honouring a `concurrency:` key — which this file has + # already been burned by once (the `format()` note at the top: an + # expression that evaluated false with no symptom at all). + # + # On a reuse hit there is no digest, and the channel tag is still the + # right source: "hit" MEANS that tag already carries this commit's + # fc.revision, which the reuse step verified by reading it. + if [ -n "${BUILT_DIGEST:-}" ]; then + SOURCE="$IMAGE@$BUILT_DIGEST" + echo "repoint: copying the digest this run published: $SOURCE" + else + SOURCE="$CHANNEL_REF" + echo "repoint: no build this run (reuse hit) — copying from $SOURCE" + fi # The source tag is EXCLUDED from the targets, and that is load- # bearing rather than an optimisation. # @@ -1031,18 +1104,25 @@ jobs: ARGS="" IFS=, for t in $TAGS; do - [ "$t" = "$SOURCE" ] && continue + # Keyed on CHANNEL_REF, never on SOURCE. SOURCE may now be a digest + # ref, which never equals a tag string — testing against it would + # stop excluding the channel tag, imagetools would index-wrap it, + # and `.Image.Config.Labels` would stop resolving through it. That + # kills the reuse label permanently (see the note just below). + [ "$t" = "$CHANNEL_REF" ] && continue ARGS="$ARGS -t $t" done unset IFS # # This is also the whole of the scheduled refresh's tag handling - # (#3154): a refresh's tag list is the channel tag alone, so SOURCE - # is the only entry, it gets excluded, and this step correctly does - # nothing. No `if:` on the step and no schedule special-case — - # excluding the source was already the right rule. + # (#3154): a refresh's tag list is the channel tag alone, so + # CHANNEL_REF is the only entry, it gets excluded, and this step + # correctly does nothing. No `if:` on the step and no schedule + # special-case — excluding the channel tag was already the right + # rule. (A refresh builds to :refresh-candidate, so BUILT_DIGEST is + # set here and simply goes unused; `promote` moves :latest later.) if [ -z "$ARGS" ]; then - echo "repoint: $SOURCE is the only tag for this channel and" + echo "repoint: $CHANNEL_REF is the only tag for this channel and" echo "repoint: already holds this revision — nothing to write." exit 0 fi @@ -1591,6 +1671,10 @@ jobs: fi - name: Build and push ml image + # `id:` so the repoint step below can read `outputs.digest` — the + # manifest THIS run published, as opposed to whatever the channel tag + # happens to name by the time that step runs (#4290). + id: build if: steps.reuse.outputs.hit != 'true' # Read by buildx out of the ENVIRONMENT, not passed as a build-arg — # it normalises the image config's `created` field and the history @@ -1714,10 +1798,42 @@ jobs: - name: Write the remaining tags from the published image env: IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-ml - SOURCE: ${{ steps.reuse.outputs.channel_ref }} + CHANNEL_REF: ${{ steps.reuse.outputs.channel_ref }} + # Empty when no build ran this job (a reuse hit, or the step's `if:` + # skipped it). Non-empty means THIS run pushed that manifest. + BUILT_DIGEST: ${{ steps.build.outputs.digest }} TAGS: ${{ steps.tag.outputs.tags }} run: | set -euf + # WHAT WE COPY FROM, which is not what we EXCLUDE (#4290). + # + # This step used to copy from the channel tag by NAME. Nothing + # serialises builds — there is no `concurrency:` key anywhere in + # .forgejo/workflows/ — so two pushes to one branch run in full + # parallel, both miss the reuse check, and both build. If the OLDER + # one finishes last it wins the channel tag; and then its repoint + # step, reading that tag by name, wrote :c- from whatever the + # other run had just published. An immutable rollback tag (rule 145) + # naming a different commit's bytes, wrong from birth — and + # immutability then guarantees nobody ever corrects it. Nothing goes + # red; it surfaces the day someone needs to roll back. + # + # So when this job built, copy from the DIGEST it pushed. Correct + # whatever a concurrent run does to the tag, and it does not depend + # on the runner honouring a `concurrency:` key — which this file has + # already been burned by once (the `format()` note at the top: an + # expression that evaluated false with no symptom at all). + # + # On a reuse hit there is no digest, and the channel tag is still the + # right source: "hit" MEANS that tag already carries this commit's + # fc.revision, which the reuse step verified by reading it. + if [ -n "${BUILT_DIGEST:-}" ]; then + SOURCE="$IMAGE@$BUILT_DIGEST" + echo "repoint: copying the digest this run published: $SOURCE" + else + SOURCE="$CHANNEL_REF" + echo "repoint: no build this run (reuse hit) — copying from $SOURCE" + fi # The source tag is EXCLUDED from the targets, and that is load- # bearing rather than an optimisation. # @@ -1743,12 +1859,17 @@ jobs: ARGS="" IFS=, for t in $TAGS; do - [ "$t" = "$SOURCE" ] && continue + # Keyed on CHANNEL_REF, never on SOURCE. SOURCE may now be a digest + # ref, which never equals a tag string — testing against it would + # stop excluding the channel tag, imagetools would index-wrap it, + # and `.Image.Config.Labels` would stop resolving through it. That + # kills the reuse label permanently (see the note just below). + [ "$t" = "$CHANNEL_REF" ] && continue ARGS="$ARGS -t $t" done unset IFS if [ -z "$ARGS" ]; then - echo "repoint: $SOURCE is the only tag for this channel and" + echo "repoint: $CHANNEL_REF is the only tag for this channel and" echo "repoint: already holds this revision — nothing to write." exit 0 fi @@ -2019,6 +2140,10 @@ jobs: fi - name: Build and push agent image + # `id:` so the repoint step below can read `outputs.digest` — the + # manifest THIS run published, as opposed to whatever the channel tag + # happens to name by the time that step runs (#4290). + id: build if: steps.reuse.outputs.hit != 'true' # Read by buildx out of the ENVIRONMENT, not passed as a build-arg — # it normalises the image config's `created` field and the history @@ -2142,10 +2267,42 @@ jobs: - name: Write the remaining tags from the published image env: IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-agent - SOURCE: ${{ steps.reuse.outputs.channel_ref }} + CHANNEL_REF: ${{ steps.reuse.outputs.channel_ref }} + # Empty when no build ran this job (a reuse hit, or the step's `if:` + # skipped it). Non-empty means THIS run pushed that manifest. + BUILT_DIGEST: ${{ steps.build.outputs.digest }} TAGS: ${{ steps.tag.outputs.tags }} run: | set -euf + # WHAT WE COPY FROM, which is not what we EXCLUDE (#4290). + # + # This step used to copy from the channel tag by NAME. Nothing + # serialises builds — there is no `concurrency:` key anywhere in + # .forgejo/workflows/ — so two pushes to one branch run in full + # parallel, both miss the reuse check, and both build. If the OLDER + # one finishes last it wins the channel tag; and then its repoint + # step, reading that tag by name, wrote :c- from whatever the + # other run had just published. An immutable rollback tag (rule 145) + # naming a different commit's bytes, wrong from birth — and + # immutability then guarantees nobody ever corrects it. Nothing goes + # red; it surfaces the day someone needs to roll back. + # + # So when this job built, copy from the DIGEST it pushed. Correct + # whatever a concurrent run does to the tag, and it does not depend + # on the runner honouring a `concurrency:` key — which this file has + # already been burned by once (the `format()` note at the top: an + # expression that evaluated false with no symptom at all). + # + # On a reuse hit there is no digest, and the channel tag is still the + # right source: "hit" MEANS that tag already carries this commit's + # fc.revision, which the reuse step verified by reading it. + if [ -n "${BUILT_DIGEST:-}" ]; then + SOURCE="$IMAGE@$BUILT_DIGEST" + echo "repoint: copying the digest this run published: $SOURCE" + else + SOURCE="$CHANNEL_REF" + echo "repoint: no build this run (reuse hit) — copying from $SOURCE" + fi # The source tag is EXCLUDED from the targets, and that is load- # bearing rather than an optimisation. # @@ -2171,12 +2328,17 @@ jobs: ARGS="" IFS=, for t in $TAGS; do - [ "$t" = "$SOURCE" ] && continue + # Keyed on CHANNEL_REF, never on SOURCE. SOURCE may now be a digest + # ref, which never equals a tag string — testing against it would + # stop excluding the channel tag, imagetools would index-wrap it, + # and `.Image.Config.Labels` would stop resolving through it. That + # kills the reuse label permanently (see the note just below). + [ "$t" = "$CHANNEL_REF" ] && continue ARGS="$ARGS -t $t" done unset IFS if [ -z "$ARGS" ]; then - echo "repoint: $SOURCE is the only tag for this channel and" + echo "repoint: $CHANNEL_REF is the only tag for this channel and" echo "repoint: already holds this revision — nothing to write." exit 0 fi