diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index c6bd756..cd895f2 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -651,6 +651,41 @@ jobs: if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi echo "channel_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT" + # WHERE THE BUILD PUBLISHES, which is not always the channel — and + # whether the channel then has to be written separately. + # + # On a push the build writes the channel tag directly: the bytes came + # from a commit, and a commit is the thing CI tests. Nothing to hold + # it behind. + # + # On the scheduled refresh it writes a CANDIDATE tag instead. A + # refresh rebuilds against freshly resolved base images, and the web + # image's runtime is a line of UNPINNED Debian packages (ffmpeg, + # libjpeg62-turbo, libpq5, megatools…) re-resolved on every build. + # Nothing in ci.yml can see that: its lanes run on ci-python:3.14 and + # install requirements.txt, and a base bump changes neither. So + # refreshed bytes have to be proven before :latest names them, and + # proving needs a moment between "built" and "published" to occupy. + # This is that moment; :latest goes on naming the build that works + # until something says otherwise. + # + # `:refresh-candidate` is one moving ref per image, overwritten in + # place, holding a build nobody is told to pull — the shape rule 145 + # already allows for :buildcache, not the per-build tag family that + # milestone 318 withdrew. + # + # Both values are decided HERE, beside `hit`, for the reason the + # force/schedule branch below gives: one step decides what this job + # does. A promote condition derived independently could disagree with + # the tag the build actually wrote. + if [ "${EVENT:-}" = "schedule" ]; then + echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT" + echo "promote=true" >> "$GITHUB_OUTPUT" + else + echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT" + echo "promote=false" >> "$GITHUB_OUTPUT" + fi + # Compare VALUES, never exit codes. Measured on buildx v0.36.1 # (run 4732): a missing key returns an empty string and exits 0, so # branching on the exit code would read "no label yet" as success. @@ -817,7 +852,7 @@ jobs: # out of a local image store a registry-direct build never filled — # #3190, which cost `main` its :c- on 2026-08-29 while :latest # published perfectly well. - tags: ${{ steps.reuse.outputs.channel_ref }} + tags: ${{ steps.reuse.outputs.build_ref }} # The reuse key. Read back off the channel tag on the next push to # decide whether that push needs to build at all, so this is not # decoration — an unstamped image is one that will always rebuild. @@ -864,6 +899,77 @@ jobs: FC_CHANNEL=${{ steps.tag.outputs.channel }} FC_VERSION=${{ steps.reuse.outputs.version }} + # Point the channel tag at the candidate the refresh just built. + # + # Unconditional TODAY, so this milestone never leaves the refresh in a + # state where it builds and publishes nothing. Step 4 wraps it in the + # smoke suite's verdict; until then the scheduled path behaves exactly + # as it did, just via two operations instead of one. + # + # NOT `imagetools create`. That wraps its source in an INDEX, and an + # indexed channel tag is the one thing this pipeline cannot survive: + # `.Image.Config.Labels` does not resolve through an index, so the + # fc.revision the reuse check reads off the channel tag would come back + # empty, every subsequent push would miss and rebuild, and nothing would + # go red. That is #3183, observed on run 4751 — reuse worked exactly once + # and the only symptom was the bill. The repoint step below excludes its + # own source tag for precisely this reason; a promote that re-introduced + # the wrap through a different door would undo that care. + # + # A manifest PUT is what "make this tag name that image" means at the + # registry level: the same bytes under the same media type, so the digest + # is identical, the media type is preserved, and no layer moves. + - name: Promote the refresh candidate to the channel + if: steps.reuse.outputs.promote == 'true' + env: + IMAGE: git.fabledsword.com/bvandeusen/fabledcurator + CHANNEL_REF: ${{ steps.reuse.outputs.channel_ref }} + TOKEN: ${{ secrets.RELEASE_TOKEN }} + ACTOR: ${{ github.actor }} + run: | + set -eu + REPO=${IMAGE#git.fabledsword.com/} + TAG=${CHANNEL_REF##*:} + + # Registry auth is its own token exchange — the `docker login` above + # authenticates the docker client, not curl. Deadline on every call + # (rule 156): a registry that stops answering must fail this step, + # not hang the weekly refresh until the job times out. + BEARER=$(curl -fsS --max-time 30 -u "$ACTOR:$TOKEN" \ + "https://git.fabledsword.com/v2/token?scope=repository:$REPO:pull,push&service=git.fabledsword.com" \ + | python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])') + + # Ask for the image manifest media types ONLY. Offering the index + # types too would let the registry hand back an index if one ever + # existed at this tag, and we would faithfully copy the thing we are + # trying not to create. + ACCEPT='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' + CT=$(curl -fsS --max-time 60 -o manifest.json -D headers.txt \ + -H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \ + "https://git.fabledsword.com/v2/$REPO/manifests/refresh-candidate" \ + && tr -d '\r' < headers.txt | awk -F': ' '/^[Cc]ontent-[Tt]ype:/{print $2}') + test -n "$CT" + SRC_DIGEST=$(tr -d '\r' < headers.txt | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}') + echo "promote: candidate is $SRC_DIGEST ($CT)" + + curl -fsS --max-time 120 -X PUT \ + -H "Authorization: Bearer $BEARER" -H "Content-Type: $CT" \ + --data-binary @manifest.json \ + "https://git.fabledsword.com/v2/$REPO/manifests/$TAG" + + # Read it back. A PUT that returned 2xx but landed something else is + # exactly the silent-and-plausible failure this pipeline keeps + # producing, and the check costs one request. + NOW=$(curl -fsS --max-time 30 -o /dev/null -D - \ + -H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \ + "https://git.fabledsword.com/v2/$REPO/manifests/$TAG" \ + | tr -d '\r' | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}') + if [ "$NOW" != "$SRC_DIGEST" ]; then + echo "promote: $IMAGE:$TAG is $NOW, expected $SRC_DIGEST" >&2 + exit 1 + fi + echo "promote: $IMAGE:$TAG now names $NOW" + # Every tag but the channel's own is written HERE, registry-side, # whether or not a build ran. Each -t becomes another reference to the # SAME manifest the channel tag holds, so :c- is byte-identical to @@ -1121,6 +1227,41 @@ jobs: if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi echo "channel_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT" + # WHERE THE BUILD PUBLISHES, which is not always the channel — and + # whether the channel then has to be written separately. + # + # On a push the build writes the channel tag directly: the bytes came + # from a commit, and a commit is the thing CI tests. Nothing to hold + # it behind. + # + # On the scheduled refresh it writes a CANDIDATE tag instead. A + # refresh rebuilds against freshly resolved base images, and the web + # image's runtime is a line of UNPINNED Debian packages (ffmpeg, + # libjpeg62-turbo, libpq5, megatools…) re-resolved on every build. + # Nothing in ci.yml can see that: its lanes run on ci-python:3.14 and + # install requirements.txt, and a base bump changes neither. So + # refreshed bytes have to be proven before :latest names them, and + # proving needs a moment between "built" and "published" to occupy. + # This is that moment; :latest goes on naming the build that works + # until something says otherwise. + # + # `:refresh-candidate` is one moving ref per image, overwritten in + # place, holding a build nobody is told to pull — the shape rule 145 + # already allows for :buildcache, not the per-build tag family that + # milestone 318 withdrew. + # + # Both values are decided HERE, beside `hit`, for the reason the + # force/schedule branch below gives: one step decides what this job + # does. A promote condition derived independently could disagree with + # the tag the build actually wrote. + if [ "${EVENT:-}" = "schedule" ]; then + echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT" + echo "promote=true" >> "$GITHUB_OUTPUT" + else + echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT" + echo "promote=false" >> "$GITHUB_OUTPUT" + fi + # Compare VALUES, never exit codes. Measured on buildx v0.36.1 # (run 4732): a missing key returns an empty string and exits 0, so # branching on the exit code would read "no label yet" as success. @@ -1211,7 +1352,7 @@ jobs: # out of a local image store a registry-direct build never filled — # #3190, which cost `main` its :c- on 2026-08-29 while :latest # published perfectly well. - tags: ${{ steps.reuse.outputs.channel_ref }} + tags: ${{ steps.reuse.outputs.build_ref }} # The reuse key. Read back off the channel tag on the next push to # decide whether that push needs to build at all, so this is not # decoration — an unstamped image is one that will always rebuild. @@ -1252,6 +1393,77 @@ jobs: cache-from: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator-ml:buildcache cache-to: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator-ml:buildcache,mode=max + # Point the channel tag at the candidate the refresh just built. + # + # Unconditional TODAY, so this milestone never leaves the refresh in a + # state where it builds and publishes nothing. Step 4 wraps it in the + # smoke suite's verdict; until then the scheduled path behaves exactly + # as it did, just via two operations instead of one. + # + # NOT `imagetools create`. That wraps its source in an INDEX, and an + # indexed channel tag is the one thing this pipeline cannot survive: + # `.Image.Config.Labels` does not resolve through an index, so the + # fc.revision the reuse check reads off the channel tag would come back + # empty, every subsequent push would miss and rebuild, and nothing would + # go red. That is #3183, observed on run 4751 — reuse worked exactly once + # and the only symptom was the bill. The repoint step below excludes its + # own source tag for precisely this reason; a promote that re-introduced + # the wrap through a different door would undo that care. + # + # A manifest PUT is what "make this tag name that image" means at the + # registry level: the same bytes under the same media type, so the digest + # is identical, the media type is preserved, and no layer moves. + - name: Promote the refresh candidate to the channel + if: steps.reuse.outputs.promote == 'true' + env: + IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-ml + CHANNEL_REF: ${{ steps.reuse.outputs.channel_ref }} + TOKEN: ${{ secrets.RELEASE_TOKEN }} + ACTOR: ${{ github.actor }} + run: | + set -eu + REPO=${IMAGE#git.fabledsword.com/} + TAG=${CHANNEL_REF##*:} + + # Registry auth is its own token exchange — the `docker login` above + # authenticates the docker client, not curl. Deadline on every call + # (rule 156): a registry that stops answering must fail this step, + # not hang the weekly refresh until the job times out. + BEARER=$(curl -fsS --max-time 30 -u "$ACTOR:$TOKEN" \ + "https://git.fabledsword.com/v2/token?scope=repository:$REPO:pull,push&service=git.fabledsword.com" \ + | python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])') + + # Ask for the image manifest media types ONLY. Offering the index + # types too would let the registry hand back an index if one ever + # existed at this tag, and we would faithfully copy the thing we are + # trying not to create. + ACCEPT='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' + CT=$(curl -fsS --max-time 60 -o manifest.json -D headers.txt \ + -H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \ + "https://git.fabledsword.com/v2/$REPO/manifests/refresh-candidate" \ + && tr -d '\r' < headers.txt | awk -F': ' '/^[Cc]ontent-[Tt]ype:/{print $2}') + test -n "$CT" + SRC_DIGEST=$(tr -d '\r' < headers.txt | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}') + echo "promote: candidate is $SRC_DIGEST ($CT)" + + curl -fsS --max-time 120 -X PUT \ + -H "Authorization: Bearer $BEARER" -H "Content-Type: $CT" \ + --data-binary @manifest.json \ + "https://git.fabledsword.com/v2/$REPO/manifests/$TAG" + + # Read it back. A PUT that returned 2xx but landed something else is + # exactly the silent-and-plausible failure this pipeline keeps + # producing, and the check costs one request. + NOW=$(curl -fsS --max-time 30 -o /dev/null -D - \ + -H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \ + "https://git.fabledsword.com/v2/$REPO/manifests/$TAG" \ + | tr -d '\r' | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}') + if [ "$NOW" != "$SRC_DIGEST" ]; then + echo "promote: $IMAGE:$TAG is $NOW, expected $SRC_DIGEST" >&2 + exit 1 + fi + echo "promote: $IMAGE:$TAG now names $NOW" + # Every tag but the channel's own is written HERE, registry-side, # whether or not a build ran. Each -t becomes another reference to the # SAME manifest the channel tag holds, so :c- is byte-identical to @@ -1501,6 +1713,41 @@ jobs: if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi echo "channel_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT" + # WHERE THE BUILD PUBLISHES, which is not always the channel — and + # whether the channel then has to be written separately. + # + # On a push the build writes the channel tag directly: the bytes came + # from a commit, and a commit is the thing CI tests. Nothing to hold + # it behind. + # + # On the scheduled refresh it writes a CANDIDATE tag instead. A + # refresh rebuilds against freshly resolved base images, and the web + # image's runtime is a line of UNPINNED Debian packages (ffmpeg, + # libjpeg62-turbo, libpq5, megatools…) re-resolved on every build. + # Nothing in ci.yml can see that: its lanes run on ci-python:3.14 and + # install requirements.txt, and a base bump changes neither. So + # refreshed bytes have to be proven before :latest names them, and + # proving needs a moment between "built" and "published" to occupy. + # This is that moment; :latest goes on naming the build that works + # until something says otherwise. + # + # `:refresh-candidate` is one moving ref per image, overwritten in + # place, holding a build nobody is told to pull — the shape rule 145 + # already allows for :buildcache, not the per-build tag family that + # milestone 318 withdrew. + # + # Both values are decided HERE, beside `hit`, for the reason the + # force/schedule branch below gives: one step decides what this job + # does. A promote condition derived independently could disagree with + # the tag the build actually wrote. + if [ "${EVENT:-}" = "schedule" ]; then + echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT" + echo "promote=true" >> "$GITHUB_OUTPUT" + else + echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT" + echo "promote=false" >> "$GITHUB_OUTPUT" + fi + # Compare VALUES, never exit codes. Measured on buildx v0.36.1 # (run 4732): a missing key returns an empty string and exits 0, so # branching on the exit code would read "no label yet" as success. @@ -1591,7 +1838,7 @@ jobs: # out of a local image store a registry-direct build never filled — # #3190, which cost `main` its :c- on 2026-08-29 while :latest # published perfectly well. - tags: ${{ steps.reuse.outputs.channel_ref }} + tags: ${{ steps.reuse.outputs.build_ref }} # The reuse key. Read back off the channel tag on the next push to # decide whether that push needs to build at all, so this is not # decoration — an unstamped image is one that will always rebuild. @@ -1632,6 +1879,77 @@ jobs: cache-from: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator-agent:buildcache cache-to: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator-agent:buildcache,mode=max + # Point the channel tag at the candidate the refresh just built. + # + # Unconditional TODAY, so this milestone never leaves the refresh in a + # state where it builds and publishes nothing. Step 4 wraps it in the + # smoke suite's verdict; until then the scheduled path behaves exactly + # as it did, just via two operations instead of one. + # + # NOT `imagetools create`. That wraps its source in an INDEX, and an + # indexed channel tag is the one thing this pipeline cannot survive: + # `.Image.Config.Labels` does not resolve through an index, so the + # fc.revision the reuse check reads off the channel tag would come back + # empty, every subsequent push would miss and rebuild, and nothing would + # go red. That is #3183, observed on run 4751 — reuse worked exactly once + # and the only symptom was the bill. The repoint step below excludes its + # own source tag for precisely this reason; a promote that re-introduced + # the wrap through a different door would undo that care. + # + # A manifest PUT is what "make this tag name that image" means at the + # registry level: the same bytes under the same media type, so the digest + # is identical, the media type is preserved, and no layer moves. + - name: Promote the refresh candidate to the channel + if: steps.reuse.outputs.promote == 'true' + env: + IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-agent + CHANNEL_REF: ${{ steps.reuse.outputs.channel_ref }} + TOKEN: ${{ secrets.RELEASE_TOKEN }} + ACTOR: ${{ github.actor }} + run: | + set -eu + REPO=${IMAGE#git.fabledsword.com/} + TAG=${CHANNEL_REF##*:} + + # Registry auth is its own token exchange — the `docker login` above + # authenticates the docker client, not curl. Deadline on every call + # (rule 156): a registry that stops answering must fail this step, + # not hang the weekly refresh until the job times out. + BEARER=$(curl -fsS --max-time 30 -u "$ACTOR:$TOKEN" \ + "https://git.fabledsword.com/v2/token?scope=repository:$REPO:pull,push&service=git.fabledsword.com" \ + | python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])') + + # Ask for the image manifest media types ONLY. Offering the index + # types too would let the registry hand back an index if one ever + # existed at this tag, and we would faithfully copy the thing we are + # trying not to create. + ACCEPT='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' + CT=$(curl -fsS --max-time 60 -o manifest.json -D headers.txt \ + -H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \ + "https://git.fabledsword.com/v2/$REPO/manifests/refresh-candidate" \ + && tr -d '\r' < headers.txt | awk -F': ' '/^[Cc]ontent-[Tt]ype:/{print $2}') + test -n "$CT" + SRC_DIGEST=$(tr -d '\r' < headers.txt | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}') + echo "promote: candidate is $SRC_DIGEST ($CT)" + + curl -fsS --max-time 120 -X PUT \ + -H "Authorization: Bearer $BEARER" -H "Content-Type: $CT" \ + --data-binary @manifest.json \ + "https://git.fabledsword.com/v2/$REPO/manifests/$TAG" + + # Read it back. A PUT that returned 2xx but landed something else is + # exactly the silent-and-plausible failure this pipeline keeps + # producing, and the check costs one request. + NOW=$(curl -fsS --max-time 30 -o /dev/null -D - \ + -H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \ + "https://git.fabledsword.com/v2/$REPO/manifests/$TAG" \ + | tr -d '\r' | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}') + if [ "$NOW" != "$SRC_DIGEST" ]; then + echo "promote: $IMAGE:$TAG is $NOW, expected $SRC_DIGEST" >&2 + exit 1 + fi + echo "promote: $IMAGE:$TAG now names $NOW" + # Every tag but the channel's own is written HERE, registry-side, # whether or not a build ran. Each -t becomes another reference to the # SAME manifest the channel tag holds, so :c- is byte-identical to