From 2587421f5bc0ecdbd9b5de3a8ea3139b0e4f6f42 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 24 Sep 2026 16:51:29 -0400 Subject: [PATCH] ci: a push publishes through a candidate tag and promotes only after the smoke (4310) On a push, build-web and build-agent wrote :dev / :latest straight from the build, and :c- right after it. smoke-web then booted the image, so it could detect a broken image but not stop one reaching the tag deployments follow. Rule 164's verify_with puts the check between build and push. Both image jobs now build to :-candidate (the refresh keeps :refresh-candidate). Their repoint step acts only on a reuse hit. promote needs build-web, build-agent and smoke-web on every trigger, and writes each built image's full tag list (channel, plus :c- on main) by manifest PUT from that job's digest, reading each tag back. A failed or skipped smoke leaves every tag on the last build that worked. The refresh path now also promotes by digest rather than by the candidate tag's name. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR --- .forgejo/workflows/build.yml | 192 ++++++++++++++++++++++------------- 1 file changed, 124 insertions(+), 68 deletions(-) diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index db5e219..3a6d09c 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -850,6 +850,9 @@ jobs: # reuse hit — deliberately NOT folded into `digest`, which the :c- # repoint reads and which must keep meaning "what this run built" (#4290). published_digest: ${{ steps.reuse.outputs.published_digest }} + # Every tag this commit should end up under (channel, plus :c- on + # main). `promote` writes them from `digest` once the smoke has passed. + tags: ${{ steps.tag.outputs.tags }} # A plain `needs` — no `always()`. That expression existed to let a # SKIPPED sign-extension through on a tag push while still blocking a # FAILED one. With no tag trigger, sign-extension always runs, so the @@ -1108,11 +1111,13 @@ jobs: # WHERE THE BUILD PUBLISHES, which is not always the channel — and # whether the channel then has to be written separately. # - # On a push the build writes the channel tag directly: the bytes came - # from a commit, and a commit is the thing CI tests. Nothing to hold - # it behind. + # Every build writes a CANDIDATE tag, never the channel (#4310): + # :dev-candidate / :latest-candidate on a push, :refresh-candidate on + # the refresh. `promote` moves the channel only after smoke-web has + # booted the bytes. The lanes prove the SOURCE; only the smoke proves + # the BYTES, and the two are not the same claim. # - # On the scheduled refresh it writes a CANDIDATE tag instead. A + # The refresh is the case that made this obvious. A # refresh rebuilds against freshly resolved base images, and the web # image's runtime is a line of UNPINNED Debian packages (ffmpeg, # libjpeg62-turbo, libpq5, megatools…) re-resolved on every build. @@ -1142,7 +1147,10 @@ jobs: echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT" echo "promote=true" >> "$GITHUB_OUTPUT" else - echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT" + # A push builds to a per-channel candidate too (#4310). The channel + # tag moves in `promote`, after smoke-web has booted these bytes — + # so a broken image never reaches the tag deployments follow. + echo "build_ref=$IMAGE:$T-candidate" >> "$GITHUB_OUTPUT" echo "promote=false" >> "$GITHUB_OUTPUT" fi @@ -1418,6 +1426,16 @@ jobs: TAGS: ${{ steps.tag.outputs.tags }} run: | set -euf + # A BUILD publishes nothing from here (#4310). Its bytes sit on the + # candidate tag until smoke-web has booted them; `promote` then + # writes the channel tag AND :c- from this run's digest. Writing + # :c- here would publish an immutable rollback tag for bytes + # that might then fail the smoke. + if [ -n "${BUILT_DIGEST:-}" ]; then + echo "repoint: built $BUILT_DIGEST this run — promote publishes it" + echo "repoint: after the smoke; nothing to write here." + exit 0 + fi # WHAT WE COPY FROM, which is not what we EXCLUDE (#4290). # # This step used to copy from the channel tag by NAME. Nothing @@ -1511,14 +1529,9 @@ jobs: # source tree, and not a static inspection: `ffmpeg -version` exiting 0 would # pass while a codec removal broke every thumbnail in the library. # - # Refresh-only. On a push the bytes came from a commit, and a commit is what - # the lanes above already test — and since 2026-09-23 they gate the build, so - # those bytes could not exist without them having passed. - # - # Reports a verdict; it does not yet gate the promote (milestone 362 step 4). - # Landing the gate and the thing it gates in one change would mean the first - # time anyone saw this job run would also be the first time it could stop a - # publish. + # It gates `promote` on every trigger (#4310). The lanes above prove the + # source; they cannot see the bytes, and a Dockerfile or base change breaks + # the bytes without touching the source. smoke-web: needs: [build-web] # Every run that actually BUILT something, not just the weekly refresh. @@ -1530,13 +1543,10 @@ jobs: # and were smoked when they were built. Re-smoking them would burn two # minutes to re-learn a fact. # - # HONEST LIMIT, and it is the reason #4299 exists: on a push this runs - # AFTER build-web has written the channel tag, so it detects rather than - # gates. Rule 164's verify_with asks for the check BETWEEN build and push. - # Closing that needs the push path to adopt the candidate-then-promote - # shape the refresh already has — per-channel candidate tags, promote - # learning its channel, and the :c- repoint moving after the gate. - # That is a redesign of the production publish path and is its own task. + # This GATES the publish on every trigger (#4310): builds land on a + # candidate tag, and `promote` needs this job, so :dev / :latest and a + # built :c- move only after it passes. Rule 164's verify_with — the + # check BETWEEN build and push. # # NO `if:` — this job always runs (#4323). It used to be gated on the # build having published something, which skipped it on a reuse hit. That @@ -1954,88 +1964,113 @@ jobs: # smoked it, and it should not be read as if it were. promote: needs: [build-web, build-agent, smoke-web] - # Only a refresh publishes through a candidate; a push writes its channel - # tag directly from the build. Reads the same reuse-step decision the build - # took, via a job output — a job's `if:` cannot see the `env` context. - if: needs.build-web.outputs.candidate == 'true' + # THE PUBLISH (#4310). Both image jobs build to a candidate tag — a push to + # :dev-candidate / :latest-candidate, the weekly refresh to + # :refresh-candidate — and nothing names those bytes under a tag anybody + # deploys until this job runs. It runs only when every job in `needs` + # SUCCEEDED (a job-level `if:` without a status function implies + # success()), so a failed smoke — or a skipped one, which is not the same + # as a passing one (run 5290) — leaves :dev / :latest on the last build + # that worked. Rule 164's verify_with: the check sits BETWEEN build and + # push. + # + # Per image, from the DIGEST that image's job built (#4290): a tag can + # move between the build and this job, a digest cannot. An image whose job + # hit reuse built nothing and has nothing to promote — its build job + # already wrote its :c- from the channel tag, which "hit" proved + # carries this commit. + if: github.event_name != 'pull_request' runs-on: python-ci container: image: git.fabledsword.com/bvandeusen/ci-python:3.14 steps: - - name: Point the channel tags at the smoked candidates + - name: Point this commit's tags at the smoked builds env: TOKEN: ${{ secrets.RELEASE_TOKEN }} ACTOR: ${{ github.actor }} + WEB_DIGEST: ${{ needs.build-web.outputs.digest }} + WEB_TAGS: ${{ needs.build-web.outputs.tags }} + AGENT_DIGEST: ${{ needs.build-agent.outputs.digest }} + AGENT_TAGS: ${{ needs.build-agent.outputs.tags }} run: | set -eu - # `latest` is not a guess: a refresh always builds `main` (BUILD_REF), - # and the "must have checked out main" guard in every build job fails - # the run if that did not hold. So the channel is main's. - TAG=latest FAILED="" + MOVED=0 - for NAME in fabledcurator fabledcurator-agent; do + promote() { + NAME="$1"; DIGEST="$2"; TAGS="$3" REPO="bvandeusen/$NAME" - echo "promote: $REPO" + if [ -z "$DIGEST" ]; then + echo "promote: $NAME — no build this run (reuse hit); nothing to publish" + return 0 + fi + echo "promote: $NAME $DIGEST" # Registry auth is its own token exchange — `docker login` # authenticates the docker client, not curl. Deadline on every call # (rule 156): a registry that stops answering must fail this step, - # not hang the weekly refresh until the job times out. + # not hang the run until the job times out. BEARER=$(curl -fsS --max-time 30 -u "$ACTOR:$TOKEN" \ "https://git.fabledsword.com/v2/token?scope=repository:$REPO:pull,push&service=git.fabledsword.com" \ | python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])') # Ask for the IMAGE manifest media types only. Offering the index - # types too would let the registry hand back an index if one ever - # existed at this tag, and we would faithfully copy the thing this - # whole approach exists to avoid creating. + # types too would let the registry hand back an index, and we would + # faithfully copy the thing this approach exists to avoid creating. ACCEPT='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' CT=$(curl -fsS --max-time 60 -o manifest.json -D headers.txt \ -H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \ - "https://git.fabledsword.com/v2/$REPO/manifests/refresh-candidate" \ + "https://git.fabledsword.com/v2/$REPO/manifests/$DIGEST" \ && tr -d '\r' < headers.txt | awk -F': ' '/^[Cc]ontent-[Tt]ype:/{print $2}') test -n "$CT" - SRC=$(tr -d '\r' < headers.txt | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}') - echo "promote: candidate $SRC ($CT)" # NOT `imagetools create`. That wraps its source in an INDEX, and # `.Image.Config.Labels` does not resolve through one — the # fc.revision the reuse check reads off the channel tag would come # back empty, every later push would miss and rebuild, and nothing # would go red (#3183, run 4751). A manifest PUT is what "make this - # tag name that image" means at the registry: same bytes, same media - # type, same digest, no layer transfer. - curl -fsS --max-time 120 -X PUT \ - -H "Authorization: Bearer $BEARER" -H "Content-Type: $CT" \ - --data-binary @manifest.json \ - "https://git.fabledsword.com/v2/$REPO/manifests/$TAG" + # tag name that image" means at the registry: same bytes, same + # media type, same digest, no layer transfer. (It also makes a + # built :c- a plain image rather than the index the old + # repoint left.) + IFS=, + for REF in $TAGS; do + TAG="${REF##*:}" + curl -fsS --max-time 120 -X PUT \ + -H "Authorization: Bearer $BEARER" -H "Content-Type: $CT" \ + --data-binary @manifest.json \ + "https://git.fabledsword.com/v2/$REPO/manifests/$TAG" - # Read it back. A PUT that returned 2xx but landed something else is - # exactly the silent-and-plausible failure this pipeline keeps - # producing, and the check costs one request. - NOW=$(curl -fsS --max-time 30 -o /dev/null -D - \ - -H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \ - "https://git.fabledsword.com/v2/$REPO/manifests/$TAG" \ - | tr -d '\r' | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}') - if [ "$NOW" != "$SRC" ]; then - echo "promote: FAILED — $NAME:$TAG is $NOW, expected $SRC" >&2 - FAILED="$FAILED $NAME" - continue - fi - echo "promote: $NAME:$TAG now names $NOW" - done + # Read it back. A PUT that returned 2xx but landed something else + # is exactly the silent-and-plausible failure this pipeline keeps + # producing, and the check costs one request. + NOW=$(curl -fsS --max-time 30 -o /dev/null -D - \ + -H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \ + "https://git.fabledsword.com/v2/$REPO/manifests/$TAG" \ + | tr -d '\r' | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}') + if [ "$NOW" != "$DIGEST" ]; then + echo "promote: FAILED — $NAME:$TAG is $NOW, expected $DIGEST" >&2 + FAILED="$FAILED $NAME:$TAG" + continue + fi + echo "promote: $NAME:$TAG now names $NOW" + MOVED=$((MOVED + 1)) + done + unset IFS + } + + promote fabledcurator "$WEB_DIGEST" "$WEB_TAGS" + promote fabledcurator-agent "$AGENT_DIGEST" "$AGENT_TAGS" if [ -n "$FAILED" ]; then echo "" >&2 echo "promote: FAILED for:$FAILED" >&2 - echo "promote: the channel tags are now INCONSISTENT — some images" >&2 - echo "promote: moved and some did not. Re-run this refresh; the" >&2 - echo "promote: candidates are still published and the promote is" >&2 - echo "promote: idempotent." >&2 + echo "promote: the tags are now INCONSISTENT — some moved and some" >&2 + echo "promote: did not. Re-run this workflow; the builds are still" >&2 + echo "promote: published by digest and the promote is idempotent." >&2 exit 1 fi - echo "promote: both channel tags moved" + echo "promote: $MOVED tag(s) written" build-agent: # THE GATE (2026-09-23). Every lane above must have PASSED before this job # exists at all — so a red suite does not produce an image, let alone push @@ -2049,6 +2084,12 @@ jobs: # where only a FAILED gate blocks would have published unverified images # while reporting success. needs: [lint, extension-version, backend-lint-and-test, frontend-build, integration] + # What `promote` needs to publish this image once the smoke has passed: + # the manifest this run built (empty on a reuse hit) and every tag it + # belongs under. Same meaning as build-web's outputs of the same names. + outputs: + digest: ${{ steps.build.outputs.digest }} + tags: ${{ steps.tag.outputs.tags }} # A pull_request run is the lanes and nothing else. This is the ONLY thing # separating "validate a Renovate bump" from "publish a Renovate bump", so # it is stated on each publishing job rather than inferred from a `needs` @@ -2240,11 +2281,13 @@ jobs: # WHERE THE BUILD PUBLISHES, which is not always the channel — and # whether the channel then has to be written separately. # - # On a push the build writes the channel tag directly: the bytes came - # from a commit, and a commit is the thing CI tests. Nothing to hold - # it behind. + # Every build writes a CANDIDATE tag, never the channel (#4310): + # :dev-candidate / :latest-candidate on a push, :refresh-candidate on + # the refresh. `promote` moves the channel only after smoke-web has + # booted the bytes. The lanes prove the SOURCE; only the smoke proves + # the BYTES, and the two are not the same claim. # - # On the scheduled refresh it writes a CANDIDATE tag instead. A + # The refresh is the case that made this obvious. A # refresh rebuilds against freshly resolved base images, and the web # image's runtime is a line of UNPINNED Debian packages (ffmpeg, # libjpeg62-turbo, libpq5, megatools…) re-resolved on every build. @@ -2273,7 +2316,10 @@ jobs: if [ "${IS_REFRESH:-}" = "true" ]; then echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT" else - echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT" + # A push builds to a per-channel candidate too (#4310). The channel + # tag moves in `promote`, after smoke-web has booted these bytes — + # so a broken image never reaches the tag deployments follow. + echo "build_ref=$IMAGE:$T-candidate" >> "$GITHUB_OUTPUT" fi # Compare VALUES, never exit codes. Measured on buildx v0.36.1 @@ -2464,6 +2510,16 @@ jobs: TAGS: ${{ steps.tag.outputs.tags }} run: | set -euf + # A BUILD publishes nothing from here (#4310). Its bytes sit on the + # candidate tag until smoke-web has booted them; `promote` then + # writes the channel tag AND :c- from this run's digest. Writing + # :c- here would publish an immutable rollback tag for bytes + # that might then fail the smoke. + if [ -n "${BUILT_DIGEST:-}" ]; then + echo "repoint: built $BUILT_DIGEST this run — promote publishes it" + echo "repoint: after the smoke; nothing to write here." + exit 0 + fi # WHAT WE COPY FROM, which is not what we EXCLUDE (#4290). # # This step used to copy from the channel tag by NAME. Nothing