Make a digest mean something again, and give the refresh somewhere to stand #247

Merged
bvandeusen merged 4 commits from dev into main 2026-09-02 14:56:12 -04:00
Showing only changes of commit b590d25f8f - Show all commits
+321 -3
View File
@@ -651,6 +651,41 @@ jobs:
if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi
echo "channel_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
# WHERE THE BUILD PUBLISHES, which is not always the channel — and
# whether the channel then has to be written separately.
#
# On a push the build writes the channel tag directly: the bytes came
# from a commit, and a commit is the thing CI tests. Nothing to hold
# it behind.
#
# On the scheduled refresh it writes a CANDIDATE tag instead. A
# refresh rebuilds against freshly resolved base images, and the web
# image's runtime is a line of UNPINNED Debian packages (ffmpeg,
# libjpeg62-turbo, libpq5, megatools…) re-resolved on every build.
# Nothing in ci.yml can see that: its lanes run on ci-python:3.14 and
# install requirements.txt, and a base bump changes neither. So
# refreshed bytes have to be proven before :latest names them, and
# proving needs a moment between "built" and "published" to occupy.
# This is that moment; :latest goes on naming the build that works
# until something says otherwise.
#
# `:refresh-candidate` is one moving ref per image, overwritten in
# place, holding a build nobody is told to pull — the shape rule 145
# already allows for :buildcache, not the per-build tag family that
# milestone 318 withdrew.
#
# Both values are decided HERE, beside `hit`, for the reason the
# force/schedule branch below gives: one step decides what this job
# does. A promote condition derived independently could disagree with
# the tag the build actually wrote.
if [ "${EVENT:-}" = "schedule" ]; then
echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT"
echo "promote=true" >> "$GITHUB_OUTPUT"
else
echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
echo "promote=false" >> "$GITHUB_OUTPUT"
fi
# Compare VALUES, never exit codes. Measured on buildx v0.36.1
# (run 4732): a missing key returns an empty string and exits 0, so
# branching on the exit code would read "no label yet" as success.
@@ -817,7 +852,7 @@ jobs:
# out of a local image store a registry-direct build never filled —
# #3190, which cost `main` its :c-<sha> on 2026-08-29 while :latest
# published perfectly well.
tags: ${{ steps.reuse.outputs.channel_ref }}
tags: ${{ steps.reuse.outputs.build_ref }}
# The reuse key. Read back off the channel tag on the next push to
# decide whether that push needs to build at all, so this is not
# decoration — an unstamped image is one that will always rebuild.
@@ -864,6 +899,77 @@ jobs:
FC_CHANNEL=${{ steps.tag.outputs.channel }}
FC_VERSION=${{ steps.reuse.outputs.version }}
# Point the channel tag at the candidate the refresh just built.
#
# Unconditional TODAY, so this milestone never leaves the refresh in a
# state where it builds and publishes nothing. Step 4 wraps it in the
# smoke suite's verdict; until then the scheduled path behaves exactly
# as it did, just via two operations instead of one.
#
# NOT `imagetools create`. That wraps its source in an INDEX, and an
# indexed channel tag is the one thing this pipeline cannot survive:
# `.Image.Config.Labels` does not resolve through an index, so the
# fc.revision the reuse check reads off the channel tag would come back
# empty, every subsequent push would miss and rebuild, and nothing would
# go red. That is #3183, observed on run 4751 — reuse worked exactly once
# and the only symptom was the bill. The repoint step below excludes its
# own source tag for precisely this reason; a promote that re-introduced
# the wrap through a different door would undo that care.
#
# A manifest PUT is what "make this tag name that image" means at the
# registry level: the same bytes under the same media type, so the digest
# is identical, the media type is preserved, and no layer moves.
- name: Promote the refresh candidate to the channel
if: steps.reuse.outputs.promote == 'true'
env:
IMAGE: git.fabledsword.com/bvandeusen/fabledcurator
CHANNEL_REF: ${{ steps.reuse.outputs.channel_ref }}
TOKEN: ${{ secrets.RELEASE_TOKEN }}
ACTOR: ${{ github.actor }}
run: |
set -eu
REPO=${IMAGE#git.fabledsword.com/}
TAG=${CHANNEL_REF##*:}
# Registry auth is its own token exchange — the `docker login` above
# authenticates the docker client, not curl. Deadline on every call
# (rule 156): a registry that stops answering must fail this step,
# not hang the weekly refresh until the job times out.
BEARER=$(curl -fsS --max-time 30 -u "$ACTOR:$TOKEN" \
"https://git.fabledsword.com/v2/token?scope=repository:$REPO:pull,push&service=git.fabledsword.com" \
| python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])')
# Ask for the image manifest media types ONLY. Offering the index
# types too would let the registry hand back an index if one ever
# existed at this tag, and we would faithfully copy the thing we are
# trying not to create.
ACCEPT='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json'
CT=$(curl -fsS --max-time 60 -o manifest.json -D headers.txt \
-H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
"https://git.fabledsword.com/v2/$REPO/manifests/refresh-candidate" \
&& tr -d '\r' < headers.txt | awk -F': ' '/^[Cc]ontent-[Tt]ype:/{print $2}')
test -n "$CT"
SRC_DIGEST=$(tr -d '\r' < headers.txt | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
echo "promote: candidate is $SRC_DIGEST ($CT)"
curl -fsS --max-time 120 -X PUT \
-H "Authorization: Bearer $BEARER" -H "Content-Type: $CT" \
--data-binary @manifest.json \
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG"
# Read it back. A PUT that returned 2xx but landed something else is
# exactly the silent-and-plausible failure this pipeline keeps
# producing, and the check costs one request.
NOW=$(curl -fsS --max-time 30 -o /dev/null -D - \
-H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG" \
| tr -d '\r' | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
if [ "$NOW" != "$SRC_DIGEST" ]; then
echo "promote: $IMAGE:$TAG is $NOW, expected $SRC_DIGEST" >&2
exit 1
fi
echo "promote: $IMAGE:$TAG now names $NOW"
# Every tag but the channel's own is written HERE, registry-side,
# whether or not a build ran. Each -t becomes another reference to the
# SAME manifest the channel tag holds, so :c-<sha> is byte-identical to
@@ -1121,6 +1227,41 @@ jobs:
if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi
echo "channel_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
# WHERE THE BUILD PUBLISHES, which is not always the channel — and
# whether the channel then has to be written separately.
#
# On a push the build writes the channel tag directly: the bytes came
# from a commit, and a commit is the thing CI tests. Nothing to hold
# it behind.
#
# On the scheduled refresh it writes a CANDIDATE tag instead. A
# refresh rebuilds against freshly resolved base images, and the web
# image's runtime is a line of UNPINNED Debian packages (ffmpeg,
# libjpeg62-turbo, libpq5, megatools…) re-resolved on every build.
# Nothing in ci.yml can see that: its lanes run on ci-python:3.14 and
# install requirements.txt, and a base bump changes neither. So
# refreshed bytes have to be proven before :latest names them, and
# proving needs a moment between "built" and "published" to occupy.
# This is that moment; :latest goes on naming the build that works
# until something says otherwise.
#
# `:refresh-candidate` is one moving ref per image, overwritten in
# place, holding a build nobody is told to pull — the shape rule 145
# already allows for :buildcache, not the per-build tag family that
# milestone 318 withdrew.
#
# Both values are decided HERE, beside `hit`, for the reason the
# force/schedule branch below gives: one step decides what this job
# does. A promote condition derived independently could disagree with
# the tag the build actually wrote.
if [ "${EVENT:-}" = "schedule" ]; then
echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT"
echo "promote=true" >> "$GITHUB_OUTPUT"
else
echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
echo "promote=false" >> "$GITHUB_OUTPUT"
fi
# Compare VALUES, never exit codes. Measured on buildx v0.36.1
# (run 4732): a missing key returns an empty string and exits 0, so
# branching on the exit code would read "no label yet" as success.
@@ -1211,7 +1352,7 @@ jobs:
# out of a local image store a registry-direct build never filled —
# #3190, which cost `main` its :c-<sha> on 2026-08-29 while :latest
# published perfectly well.
tags: ${{ steps.reuse.outputs.channel_ref }}
tags: ${{ steps.reuse.outputs.build_ref }}
# The reuse key. Read back off the channel tag on the next push to
# decide whether that push needs to build at all, so this is not
# decoration — an unstamped image is one that will always rebuild.
@@ -1252,6 +1393,77 @@ jobs:
cache-from: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator-ml:buildcache
cache-to: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator-ml:buildcache,mode=max
# Point the channel tag at the candidate the refresh just built.
#
# Unconditional TODAY, so this milestone never leaves the refresh in a
# state where it builds and publishes nothing. Step 4 wraps it in the
# smoke suite's verdict; until then the scheduled path behaves exactly
# as it did, just via two operations instead of one.
#
# NOT `imagetools create`. That wraps its source in an INDEX, and an
# indexed channel tag is the one thing this pipeline cannot survive:
# `.Image.Config.Labels` does not resolve through an index, so the
# fc.revision the reuse check reads off the channel tag would come back
# empty, every subsequent push would miss and rebuild, and nothing would
# go red. That is #3183, observed on run 4751 — reuse worked exactly once
# and the only symptom was the bill. The repoint step below excludes its
# own source tag for precisely this reason; a promote that re-introduced
# the wrap through a different door would undo that care.
#
# A manifest PUT is what "make this tag name that image" means at the
# registry level: the same bytes under the same media type, so the digest
# is identical, the media type is preserved, and no layer moves.
- name: Promote the refresh candidate to the channel
if: steps.reuse.outputs.promote == 'true'
env:
IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-ml
CHANNEL_REF: ${{ steps.reuse.outputs.channel_ref }}
TOKEN: ${{ secrets.RELEASE_TOKEN }}
ACTOR: ${{ github.actor }}
run: |
set -eu
REPO=${IMAGE#git.fabledsword.com/}
TAG=${CHANNEL_REF##*:}
# Registry auth is its own token exchange — the `docker login` above
# authenticates the docker client, not curl. Deadline on every call
# (rule 156): a registry that stops answering must fail this step,
# not hang the weekly refresh until the job times out.
BEARER=$(curl -fsS --max-time 30 -u "$ACTOR:$TOKEN" \
"https://git.fabledsword.com/v2/token?scope=repository:$REPO:pull,push&service=git.fabledsword.com" \
| python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])')
# Ask for the image manifest media types ONLY. Offering the index
# types too would let the registry hand back an index if one ever
# existed at this tag, and we would faithfully copy the thing we are
# trying not to create.
ACCEPT='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json'
CT=$(curl -fsS --max-time 60 -o manifest.json -D headers.txt \
-H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
"https://git.fabledsword.com/v2/$REPO/manifests/refresh-candidate" \
&& tr -d '\r' < headers.txt | awk -F': ' '/^[Cc]ontent-[Tt]ype:/{print $2}')
test -n "$CT"
SRC_DIGEST=$(tr -d '\r' < headers.txt | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
echo "promote: candidate is $SRC_DIGEST ($CT)"
curl -fsS --max-time 120 -X PUT \
-H "Authorization: Bearer $BEARER" -H "Content-Type: $CT" \
--data-binary @manifest.json \
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG"
# Read it back. A PUT that returned 2xx but landed something else is
# exactly the silent-and-plausible failure this pipeline keeps
# producing, and the check costs one request.
NOW=$(curl -fsS --max-time 30 -o /dev/null -D - \
-H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG" \
| tr -d '\r' | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
if [ "$NOW" != "$SRC_DIGEST" ]; then
echo "promote: $IMAGE:$TAG is $NOW, expected $SRC_DIGEST" >&2
exit 1
fi
echo "promote: $IMAGE:$TAG now names $NOW"
# Every tag but the channel's own is written HERE, registry-side,
# whether or not a build ran. Each -t becomes another reference to the
# SAME manifest the channel tag holds, so :c-<sha> is byte-identical to
@@ -1501,6 +1713,41 @@ jobs:
if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi
echo "channel_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
# WHERE THE BUILD PUBLISHES, which is not always the channel — and
# whether the channel then has to be written separately.
#
# On a push the build writes the channel tag directly: the bytes came
# from a commit, and a commit is the thing CI tests. Nothing to hold
# it behind.
#
# On the scheduled refresh it writes a CANDIDATE tag instead. A
# refresh rebuilds against freshly resolved base images, and the web
# image's runtime is a line of UNPINNED Debian packages (ffmpeg,
# libjpeg62-turbo, libpq5, megatools…) re-resolved on every build.
# Nothing in ci.yml can see that: its lanes run on ci-python:3.14 and
# install requirements.txt, and a base bump changes neither. So
# refreshed bytes have to be proven before :latest names them, and
# proving needs a moment between "built" and "published" to occupy.
# This is that moment; :latest goes on naming the build that works
# until something says otherwise.
#
# `:refresh-candidate` is one moving ref per image, overwritten in
# place, holding a build nobody is told to pull — the shape rule 145
# already allows for :buildcache, not the per-build tag family that
# milestone 318 withdrew.
#
# Both values are decided HERE, beside `hit`, for the reason the
# force/schedule branch below gives: one step decides what this job
# does. A promote condition derived independently could disagree with
# the tag the build actually wrote.
if [ "${EVENT:-}" = "schedule" ]; then
echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT"
echo "promote=true" >> "$GITHUB_OUTPUT"
else
echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
echo "promote=false" >> "$GITHUB_OUTPUT"
fi
# Compare VALUES, never exit codes. Measured on buildx v0.36.1
# (run 4732): a missing key returns an empty string and exits 0, so
# branching on the exit code would read "no label yet" as success.
@@ -1591,7 +1838,7 @@ jobs:
# out of a local image store a registry-direct build never filled —
# #3190, which cost `main` its :c-<sha> on 2026-08-29 while :latest
# published perfectly well.
tags: ${{ steps.reuse.outputs.channel_ref }}
tags: ${{ steps.reuse.outputs.build_ref }}
# The reuse key. Read back off the channel tag on the next push to
# decide whether that push needs to build at all, so this is not
# decoration — an unstamped image is one that will always rebuild.
@@ -1632,6 +1879,77 @@ jobs:
cache-from: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator-agent:buildcache
cache-to: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator-agent:buildcache,mode=max
# Point the channel tag at the candidate the refresh just built.
#
# Unconditional TODAY, so this milestone never leaves the refresh in a
# state where it builds and publishes nothing. Step 4 wraps it in the
# smoke suite's verdict; until then the scheduled path behaves exactly
# as it did, just via two operations instead of one.
#
# NOT `imagetools create`. That wraps its source in an INDEX, and an
# indexed channel tag is the one thing this pipeline cannot survive:
# `.Image.Config.Labels` does not resolve through an index, so the
# fc.revision the reuse check reads off the channel tag would come back
# empty, every subsequent push would miss and rebuild, and nothing would
# go red. That is #3183, observed on run 4751 — reuse worked exactly once
# and the only symptom was the bill. The repoint step below excludes its
# own source tag for precisely this reason; a promote that re-introduced
# the wrap through a different door would undo that care.
#
# A manifest PUT is what "make this tag name that image" means at the
# registry level: the same bytes under the same media type, so the digest
# is identical, the media type is preserved, and no layer moves.
- name: Promote the refresh candidate to the channel
if: steps.reuse.outputs.promote == 'true'
env:
IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-agent
CHANNEL_REF: ${{ steps.reuse.outputs.channel_ref }}
TOKEN: ${{ secrets.RELEASE_TOKEN }}
ACTOR: ${{ github.actor }}
run: |
set -eu
REPO=${IMAGE#git.fabledsword.com/}
TAG=${CHANNEL_REF##*:}
# Registry auth is its own token exchange — the `docker login` above
# authenticates the docker client, not curl. Deadline on every call
# (rule 156): a registry that stops answering must fail this step,
# not hang the weekly refresh until the job times out.
BEARER=$(curl -fsS --max-time 30 -u "$ACTOR:$TOKEN" \
"https://git.fabledsword.com/v2/token?scope=repository:$REPO:pull,push&service=git.fabledsword.com" \
| python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])')
# Ask for the image manifest media types ONLY. Offering the index
# types too would let the registry hand back an index if one ever
# existed at this tag, and we would faithfully copy the thing we are
# trying not to create.
ACCEPT='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json'
CT=$(curl -fsS --max-time 60 -o manifest.json -D headers.txt \
-H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
"https://git.fabledsword.com/v2/$REPO/manifests/refresh-candidate" \
&& tr -d '\r' < headers.txt | awk -F': ' '/^[Cc]ontent-[Tt]ype:/{print $2}')
test -n "$CT"
SRC_DIGEST=$(tr -d '\r' < headers.txt | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
echo "promote: candidate is $SRC_DIGEST ($CT)"
curl -fsS --max-time 120 -X PUT \
-H "Authorization: Bearer $BEARER" -H "Content-Type: $CT" \
--data-binary @manifest.json \
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG"
# Read it back. A PUT that returned 2xx but landed something else is
# exactly the silent-and-plausible failure this pipeline keeps
# producing, and the check costs one request.
NOW=$(curl -fsS --max-time 30 -o /dev/null -D - \
-H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG" \
| tr -d '\r' | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
if [ "$NOW" != "$SRC_DIGEST" ]; then
echo "promote: $IMAGE:$TAG is $NOW, expected $SRC_DIGEST" >&2
exit 1
fi
echo "promote: $IMAGE:$TAG now names $NOW"
# Every tag but the channel's own is written HERE, registry-side,
# whether or not a build ran. Each -t becomes another reference to the
# SAME manifest the channel tag holds, so :c-<sha> is byte-identical to