.forgejo/workflows/build.yml

@@ -64,6 +64,43 @@ on:
   schedule:
     - cron: '0 6 * * 0'
 
+# One build.yml run per branch at a time (#4290).
+#
+# Without this, two pushes to one branch run in full parallel. Both read
+# `fc.revision` off the channel tag before either has pushed, so both miss the
+# reuse check and both build, and whichever finishes LAST owns the tag — so a
+# slower older build can leave `:dev` carrying content older than the commit
+# that moved it. Family rule 146 says a rolling channel refreshes itself; that
+# is the case where it quietly does not.
+#
+# `cancel-in-progress: false` — QUEUE, never cancel. Cancelling could kill
+# sign-extension mid-AMO-upload, leaving the version registered at AMO with no
+# cached asset: exactly the stuck state the rollback trap in that job exists to
+# prevent, reached through a different door. AMO will not release a burned
+# version, so that state is unrecoverable rather than merely annoying. Waiting
+# a few minutes is the cheaper end of that trade by a wide margin.
+#
+# Keyed on `github.ref`, so `dev` and `main` never block each other. Not on
+# BUILD_REF: the group is evaluated before any job starts and cannot read the
+# `env` context (the same restriction that makes a job-level `if:` unable to
+# see it — see build-web's `outputs.candidate` note). The one consequence is
+# that a scheduled refresh, whose ref is the default branch, shares dev's
+# queue while publishing main's channel. It only ever waits, and it fires
+# 06:00 Sunday precisely because nothing else is running then.
+#
+# UNVERIFIED AT THE TIME OF WRITING, and this file has been burned by exactly
+# that before: the `format()` note below records `true == 'true'` evaluating
+# FALSE on run 5270 with no symptom whatsoever — every lane green, the feature
+# simply not happening. A `concurrency:` key this Gitea ignored would look
+# identical: runs still overlapping, nothing red. So this is a belt, and the
+# digest-pinned repoint in each build job is the braces — that one makes the
+# :c-<sha> correctness property hold whether or not this key is honoured.
+# Confirm by pushing twice in quick succession and reading the RUN LIST for a
+# queued second run, never by reading this comment.
+concurrency:
+  group: build-${{ github.ref }}
+  cancel-in-progress: false
+
 # Which branch a run BUILDS, as opposed to which one triggered it.
 #
 # They are the same thing on every trigger but `schedule`. Forgejo registers a
@@ -873,6 +910,10 @@ jobs:
           ls -la frontend/public/extension/
 
       - name: Build and push web image
+        # `id:` so the repoint step below can read `outputs.digest` — the
+        # manifest THIS run published, as opposed to whatever the channel tag
+        # happens to name by the time that step runs (#4290).
+        id: build
         if: steps.reuse.outputs.hit != 'true'
         # Read by buildx out of the ENVIRONMENT, not passed as a build-arg —
         # it normalises the image config's `created` field and the history
@@ -1002,10 +1043,42 @@ jobs:
       - name: Write the remaining tags from the published image
         env:
           IMAGE: git.fabledsword.com/bvandeusen/fabledcurator
-          SOURCE: ${{ steps.reuse.outputs.channel_ref }}
+          CHANNEL_REF: ${{ steps.reuse.outputs.channel_ref }}
+          # Empty when no build ran this job (a reuse hit, or the step's `if:`
+          # skipped it). Non-empty means THIS run pushed that manifest.
+          BUILT_DIGEST: ${{ steps.build.outputs.digest }}
           TAGS: ${{ steps.tag.outputs.tags }}
         run: |
           set -euf
+          # WHAT WE COPY FROM, which is not what we EXCLUDE (#4290).
+          #
+          # This step used to copy from the channel tag by NAME. Nothing
+          # serialises builds — there is no `concurrency:` key anywhere in
+          # .forgejo/workflows/ — so two pushes to one branch run in full
+          # parallel, both miss the reuse check, and both build. If the OLDER
+          # one finishes last it wins the channel tag; and then its repoint
+          # step, reading that tag by name, wrote :c-<sha> from whatever the
+          # other run had just published. An immutable rollback tag (rule 145)
+          # naming a different commit's bytes, wrong from birth — and
+          # immutability then guarantees nobody ever corrects it. Nothing goes
+          # red; it surfaces the day someone needs to roll back.
+          #
+          # So when this job built, copy from the DIGEST it pushed. Correct
+          # whatever a concurrent run does to the tag, and it does not depend
+          # on the runner honouring a `concurrency:` key — which this file has
+          # already been burned by once (the `format()` note at the top: an
+          # expression that evaluated false with no symptom at all).
+          #
+          # On a reuse hit there is no digest, and the channel tag is still the
+          # right source: "hit" MEANS that tag already carries this commit's
+          # fc.revision, which the reuse step verified by reading it.
+          if [ -n "${BUILT_DIGEST:-}" ]; then
+            SOURCE="$IMAGE@$BUILT_DIGEST"
+            echo "repoint: copying the digest this run published: $SOURCE"
+          else
+            SOURCE="$CHANNEL_REF"
+            echo "repoint: no build this run (reuse hit) — copying from $SOURCE"
+          fi
           # The source tag is EXCLUDED from the targets, and that is load-
           # bearing rather than an optimisation.
           #
@@ -1031,18 +1104,25 @@ jobs:
           ARGS=""
           IFS=,
           for t in $TAGS; do
-            [ "$t" = "$SOURCE" ] && continue
+            # Keyed on CHANNEL_REF, never on SOURCE. SOURCE may now be a digest
+            # ref, which never equals a tag string — testing against it would
+            # stop excluding the channel tag, imagetools would index-wrap it,
+            # and `.Image.Config.Labels` would stop resolving through it. That
+            # kills the reuse label permanently (see the note just below).
+            [ "$t" = "$CHANNEL_REF" ] && continue
             ARGS="$ARGS -t $t"
           done
           unset IFS
           #
           # This is also the whole of the scheduled refresh's tag handling
-          # (#3154): a refresh's tag list is the channel tag alone, so SOURCE
-          # is the only entry, it gets excluded, and this step correctly does
-          # nothing. No `if:` on the step and no schedule special-case —
-          # excluding the source was already the right rule.
+          # (#3154): a refresh's tag list is the channel tag alone, so
+          # CHANNEL_REF is the only entry, it gets excluded, and this step
+          # correctly does nothing. No `if:` on the step and no schedule
+          # special-case — excluding the channel tag was already the right
+          # rule. (A refresh builds to :refresh-candidate, so BUILT_DIGEST is
+          # set here and simply goes unused; `promote` moves :latest later.)
           if [ -z "$ARGS" ]; then
-            echo "repoint: $SOURCE is the only tag for this channel and"
+            echo "repoint: $CHANNEL_REF is the only tag for this channel and"
             echo "repoint: already holds this revision — nothing to write."
             exit 0
           fi
@@ -1591,6 +1671,10 @@ jobs:
           fi
 
       - name: Build and push ml image
+        # `id:` so the repoint step below can read `outputs.digest` — the
+        # manifest THIS run published, as opposed to whatever the channel tag
+        # happens to name by the time that step runs (#4290).
+        id: build
         if: steps.reuse.outputs.hit != 'true'
         # Read by buildx out of the ENVIRONMENT, not passed as a build-arg —
         # it normalises the image config's `created` field and the history
@@ -1714,10 +1798,42 @@ jobs:
       - name: Write the remaining tags from the published image
         env:
           IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-ml
-          SOURCE: ${{ steps.reuse.outputs.channel_ref }}
+          CHANNEL_REF: ${{ steps.reuse.outputs.channel_ref }}
+          # Empty when no build ran this job (a reuse hit, or the step's `if:`
+          # skipped it). Non-empty means THIS run pushed that manifest.
+          BUILT_DIGEST: ${{ steps.build.outputs.digest }}
           TAGS: ${{ steps.tag.outputs.tags }}
         run: |
           set -euf
+          # WHAT WE COPY FROM, which is not what we EXCLUDE (#4290).
+          #
+          # This step used to copy from the channel tag by NAME. Nothing
+          # serialises builds — there is no `concurrency:` key anywhere in
+          # .forgejo/workflows/ — so two pushes to one branch run in full
+          # parallel, both miss the reuse check, and both build. If the OLDER
+          # one finishes last it wins the channel tag; and then its repoint
+          # step, reading that tag by name, wrote :c-<sha> from whatever the
+          # other run had just published. An immutable rollback tag (rule 145)
+          # naming a different commit's bytes, wrong from birth — and
+          # immutability then guarantees nobody ever corrects it. Nothing goes
+          # red; it surfaces the day someone needs to roll back.
+          #
+          # So when this job built, copy from the DIGEST it pushed. Correct
+          # whatever a concurrent run does to the tag, and it does not depend
+          # on the runner honouring a `concurrency:` key — which this file has
+          # already been burned by once (the `format()` note at the top: an
+          # expression that evaluated false with no symptom at all).
+          #
+          # On a reuse hit there is no digest, and the channel tag is still the
+          # right source: "hit" MEANS that tag already carries this commit's
+          # fc.revision, which the reuse step verified by reading it.
+          if [ -n "${BUILT_DIGEST:-}" ]; then
+            SOURCE="$IMAGE@$BUILT_DIGEST"
+            echo "repoint: copying the digest this run published: $SOURCE"
+          else
+            SOURCE="$CHANNEL_REF"
+            echo "repoint: no build this run (reuse hit) — copying from $SOURCE"
+          fi
           # The source tag is EXCLUDED from the targets, and that is load-
           # bearing rather than an optimisation.
           #
@@ -1743,12 +1859,17 @@ jobs:
           ARGS=""
           IFS=,
           for t in $TAGS; do
-            [ "$t" = "$SOURCE" ] && continue
+            # Keyed on CHANNEL_REF, never on SOURCE. SOURCE may now be a digest
+            # ref, which never equals a tag string — testing against it would
+            # stop excluding the channel tag, imagetools would index-wrap it,
+            # and `.Image.Config.Labels` would stop resolving through it. That
+            # kills the reuse label permanently (see the note just below).
+            [ "$t" = "$CHANNEL_REF" ] && continue
             ARGS="$ARGS -t $t"
           done
           unset IFS
           if [ -z "$ARGS" ]; then
-            echo "repoint: $SOURCE is the only tag for this channel and"
+            echo "repoint: $CHANNEL_REF is the only tag for this channel and"
             echo "repoint: already holds this revision — nothing to write."
             exit 0
           fi
@@ -2019,6 +2140,10 @@ jobs:
           fi
 
       - name: Build and push agent image
+        # `id:` so the repoint step below can read `outputs.digest` — the
+        # manifest THIS run published, as opposed to whatever the channel tag
+        # happens to name by the time that step runs (#4290).
+        id: build
         if: steps.reuse.outputs.hit != 'true'
         # Read by buildx out of the ENVIRONMENT, not passed as a build-arg —
         # it normalises the image config's `created` field and the history
@@ -2142,10 +2267,42 @@ jobs:
       - name: Write the remaining tags from the published image
         env:
           IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-agent
-          SOURCE: ${{ steps.reuse.outputs.channel_ref }}
+          CHANNEL_REF: ${{ steps.reuse.outputs.channel_ref }}
+          # Empty when no build ran this job (a reuse hit, or the step's `if:`
+          # skipped it). Non-empty means THIS run pushed that manifest.
+          BUILT_DIGEST: ${{ steps.build.outputs.digest }}
           TAGS: ${{ steps.tag.outputs.tags }}
         run: |
           set -euf
+          # WHAT WE COPY FROM, which is not what we EXCLUDE (#4290).
+          #
+          # This step used to copy from the channel tag by NAME. Nothing
+          # serialises builds — there is no `concurrency:` key anywhere in
+          # .forgejo/workflows/ — so two pushes to one branch run in full
+          # parallel, both miss the reuse check, and both build. If the OLDER
+          # one finishes last it wins the channel tag; and then its repoint
+          # step, reading that tag by name, wrote :c-<sha> from whatever the
+          # other run had just published. An immutable rollback tag (rule 145)
+          # naming a different commit's bytes, wrong from birth — and
+          # immutability then guarantees nobody ever corrects it. Nothing goes
+          # red; it surfaces the day someone needs to roll back.
+          #
+          # So when this job built, copy from the DIGEST it pushed. Correct
+          # whatever a concurrent run does to the tag, and it does not depend
+          # on the runner honouring a `concurrency:` key — which this file has
+          # already been burned by once (the `format()` note at the top: an
+          # expression that evaluated false with no symptom at all).
+          #
+          # On a reuse hit there is no digest, and the channel tag is still the
+          # right source: "hit" MEANS that tag already carries this commit's
+          # fc.revision, which the reuse step verified by reading it.
+          if [ -n "${BUILT_DIGEST:-}" ]; then
+            SOURCE="$IMAGE@$BUILT_DIGEST"
+            echo "repoint: copying the digest this run published: $SOURCE"
+          else
+            SOURCE="$CHANNEL_REF"
+            echo "repoint: no build this run (reuse hit) — copying from $SOURCE"
+          fi
           # The source tag is EXCLUDED from the targets, and that is load-
           # bearing rather than an optimisation.
           #
@@ -2171,12 +2328,17 @@ jobs:
           ARGS=""
           IFS=,
           for t in $TAGS; do
-            [ "$t" = "$SOURCE" ] && continue
+            # Keyed on CHANNEL_REF, never on SOURCE. SOURCE may now be a digest
+            # ref, which never equals a tag string — testing against it would
+            # stop excluding the channel tag, imagetools would index-wrap it,
+            # and `.Image.Config.Labels` would stop resolving through it. That
+            # kills the reuse label permanently (see the note just below).
+            [ "$t" = "$CHANNEL_REF" ] && continue
             ARGS="$ARGS -t $t"
           done
           unset IFS
           if [ -z "$ARGS" ]; then
-            echo "repoint: $SOURCE is the only tag for this channel and"
+            echo "repoint: $CHANNEL_REF is the only tag for this channel and"
             echo "repoint: already holds this revision — nothing to write."
             exit 0
           fi