diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index c8a84f6..47228b7 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -14,13 +14,16 @@ on: # with a stale `:dev` ml or agent is a worse trap than no dev channel at # all, since the mismatch only shows up as a runtime failure. branches: [main, dev] - # Tag-push triggers an immutable per-version image build (e.g. - # `:v26.05.26.5`) — gives a real rollback story alongside the floating - # `:main` / `:latest`. Layer reuse keeps the registry-storage cost - # negligible per tag. Doesn't overlap with the push-to-main build (that - # one publishes `:main` + `:latest`; the tag-push build publishes only - # `:`). - tags: ['v*'] + # + # NO tag trigger (milestone 318 step 2). A `v*` tag names a commit `main` + # already built and published; rebuilding it produces the same source under + # the same names and RE-PUSHES `:c-`, which rule 145 forbids even when + # the bytes match — image configs carry timestamps, so "same source" does + # not mean "same manifest". The release build was publishing nothing new + # and violating an immutability rule to do it. + # + # Releases still happen (rule 148, on explicit request per rule 2). They + # produce a changelog, not an image. # Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes: # - write:package, read:package (for docker push to git.fabledsword.com) @@ -54,10 +57,11 @@ jobs: # have hit the existing ext-1.0.11 cache and bundled MAIN's stale XPI into # `:dev` — a dev channel confidently serving old code. # - # Tags stay excluded: the tag path deliberately skips signing and polls for - # the release instead (see build-web's race note, 2026-05-27). + # Unconditional since milestone 318 step 2: main and dev are now the only + # triggers, so the branch gate that used to exclude tag pushes matched + # everything. A condition that is always true reads as if some path avoids + # it, which is worse than no condition. sign-extension: - if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' runs-on: python-ci container: image: git.fabledsword.com/bvandeusen/ci-python:3.14 @@ -119,10 +123,9 @@ jobs: run: | set -u A=extension - T=$(sh scripts/artifacts.sh tag "$A" 2>&1 || echo UNAVAILABLE) V=$(sh scripts/artifacts.sh version "$A" 2>&1 || echo UNAVAILABLE) R=$(sh scripts/artifacts.sh revision "$A" 2>&1 || echo UNAVAILABLE) - echo "derived: artifact=$A tag=$T version=$V revision=$R sha=$GITHUB_SHA" + echo "derived: artifact=$A version=$V revision=$R sha=$GITHUB_SHA" - name: Guard — the derived version must never go backwards env: @@ -323,12 +326,12 @@ jobs: # to. Same source of truth; no double-store. build-web: + # A plain `needs` — no `always()`. That expression existed to let a + # SKIPPED sign-extension through on a tag push while still blocking a + # FAILED one. With no tag trigger, sign-extension always runs, so the + # default behaviour is exactly what we want: a failed sign skips build-web + # rather than shipping an image without its XPI. needs: [sign-extension] - # sign-extension runs on main and dev, and is skipped on a tag push (which - # polls for the release instead). Either is fine to build on; a FAILED sign - # is not — this condition lets success and skipped through, so a failure - # skips build-web rather than shipping an image without the XPI. - if: always() && (needs.sign-extension.result == 'success' || needs.sign-extension.result == 'skipped') runs-on: python-ci container: image: git.fabledsword.com/bvandeusen/ci-python:3.14 @@ -365,71 +368,44 @@ jobs: run: | set -u A=web - T=$(sh scripts/artifacts.sh tag "$A" 2>&1 || echo UNAVAILABLE) V=$(sh scripts/artifacts.sh version "$A" 2>&1 || echo UNAVAILABLE) R=$(sh scripts/artifacts.sh revision "$A" 2>&1 || echo UNAVAILABLE) - echo "derived: artifact=$A tag=$T version=$V revision=$R sha=$GITHUB_SHA" + echo "derived: artifact=$A version=$V revision=$R sha=$GITHUB_SHA" - name: Determine tag id: tag run: | - # Three trigger shapes: - # refs/tags/v… → tag-push: opt-in milestone label (vYY.MM.DD, - # plus `.N` when the day already carries a tag — - # family rule 148, amended 2026-08-24 after a - # same-day tag was retargeted and a release - # deleted to make room, note 2813). - # Publish ONLY the immutable version tag; - # don't touch :latest (the main-push build - # for the merge commit already did that). - # refs/heads/main → push to main: publish :main + :latest - # (floating) AND :c- (immutable - # per-commit rollback substrate, per family - # release-posture rule "Tags are milestones, - # not gates — commit-SHA images are the - # rollback unit"). Rollback to any commit - # becomes `docker pull …:c-` without a - # release ceremony. - # refs/heads/dev → push to dev: publish :dev, the rolling test - # channel (family rule 146). Rolling means it may - # carry newer contents than the :c- of the - # same commit; it never writes :c- itself, - # because that is the rollback unit (rule 145). + # Two trigger shapes, and between them they publish three tags: + # main → :latest (production, moving — rule 147: main IS production) + # :c- (immutable, the rollback unit — rule 145) + # dev → :dev (the rolling test channel — rule 146) + # + # That is the whole list. No :, and no :main — rule 145, + # narrowed 2026-08-28 once it was verified that nothing pins: + # "a third name for the same thing is upkeep for a model we do not + # run." The date tag published between milestone 313 step 3 and + # milestone 318 was exactly that; :main was a second moving name for + # whatever :latest already pointed at. + # + # `dev` gets no :c- deliberately. On a channel whose entire + # contract is that it moves, a per-push immutable tag is a rollback + # target nobody has ever pulled, accumulating forever. The accepted + # cost: on dev there is no rollback but the previous :dev, which is + # gone — recovery is revert-on-git plus a CI cycle. + # + # Reinstating : is a real decision, not a default. It earns + # its place when something genuinely pins: a second instance held on + # a known-good build, or a deliberately frozen window. Tag at the + # moment you decide to freeze; no back-catalogue is needed. + # # POSIX-safe substring (the runner shell is dash/BusyBox sh, not # bash — `${var:0:7}` errors with "Bad substitution"; cut works - # everywhere). Operator-flagged 2026-06-01 after first :c- + # everywhere). Operator-flagged 2026-06-01 after the first :c- # main-push build failed at this step. SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7) - # The pinnable tag (milestone 313 step 3): YYYY.M.D of the commit - # THIS artifact's shipped files last changed in. Day precision is - # deliberate — same-day work is not something worth pinning, so a - # second main build the same day replaces the first rather than - # accumulating a tag nobody would roll back to. - # - # Derived per artifact, so an image whose files did not change keeps - # the tag it already had: the agent reads 2026.7.17 today while web - # reads 2026.8.27 — and the reuse step below turns that into a - # skipped build rather than a rebuild of bytes that already exist. - # `channel` is baked into the image as FC_CHANNEL and reported by - # /api/extension/manifest (milestone 271 step 7). A tag-push counts as - # `main`: a vYY.MM.DD tag is cut from main, so that image is a - # main-channel artifact wearing an immutable name. - if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then - TAG_NAME="${GITHUB_REF#refs/tags/}" - echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:${TAG_NAME}" >> "$GITHUB_OUTPUT" - echo "channel=main" >> "$GITHUB_OUTPUT" - elif [ "${GITHUB_REF##*/}" = "main" ]; then - CALVER=$(sh scripts/artifacts.sh tag web) - # Guarded, and computed only on this path. There is no `set -e` in - # this step, so a failed derivation would otherwise leave CALVER - # empty and publish the tag `fabledcurator:` — an invalid - # name, from a green step. An empty pin must never reach the - # registry. - if [ -z "$CALVER" ]; then - echo "ERROR: could not derive a web version tag" >&2 - exit 1 - fi - echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:main,git.fabledsword.com/bvandeusen/fabledcurator:latest,git.fabledsword.com/bvandeusen/fabledcurator:c-${SHORT_SHA},git.fabledsword.com/bvandeusen/fabledcurator:${CALVER}" >> "$GITHUB_OUTPUT" + # Mirrors build-web's tag list; see the comment there. + if [ "${GITHUB_REF##*/}" = "main" ]; then + echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:latest,git.fabledsword.com/bvandeusen/fabledcurator:c-${SHORT_SHA}" >> "$GITHUB_OUTPUT" echo "channel=main" >> "$GITHUB_OUTPUT" else echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:dev" >> "$GITHUB_OUTPUT" @@ -462,12 +438,21 @@ jobs: run: echo "$TOKEN" | docker login git.fabledsword.com -u "$ACTOR" --password-stdin # --- reuse-if-published (milestone 313, step 4) ---------------------- - # The identity tag names this artifact's CONTENT — r-, the - # commit its shipped files last changed in, plus the channel for images - # that bake one in. If the registry already carries it, the bytes this - # job would produce are already published and the build is pure waste: - # the channel and date tags get repointed at the existing manifest - # instead, registry-side, in seconds. + # Does the image the channel tag already points at carry THIS commit's + # revision? If so the bytes this job would produce are already published + # and the build is pure waste: the remaining tags get repointed at that + # existing manifest instead, registry-side, in seconds. + # + # Keyed on an `fc.revision` LABEL rather than on a tag of its own + # (milestone 318 step 3). A tag would be a name minted per build that one + # thing reads — what rule 145 narrowed against — and would be prunable + # under the registry's keep_pattern (#3157), silently expiring the cache. + # A label rides inside a tag that has to exist anyway. + # + # An image with no such label reads as a miss and rebuilds. That is the + # migration, not a fault: labels cannot be backfilled, since the reuse + # path copies a manifest and config labels are not manifest annotations. + # Each artifact pays one rebuild, once. # # This is what stops a push that touched only `agent/` from rebuilding # web and ml, and a merge to main from rebuilding what dev already built. @@ -493,54 +478,71 @@ jobs: IMAGE: git.fabledsword.com/bvandeusen/fabledcurator CHANNEL: ${{ steps.tag.outputs.channel }} TAGS: ${{ steps.tag.outputs.tags }} - IS_TAG_PUSH: ${{ startsWith(github.ref, 'refs/tags/') }} run: | set -eu - ID=$(sh scripts/artifacts.sh identity web "$CHANNEL") - echo "identity=$ID" >> "$GITHUB_OUTPUT" + DERIVED=$(sh scripts/artifacts.sh revision web) + echo "revision=$DERIVED" >> "$GITHUB_OUTPUT" + echo "build_tags=$TAGS" >> "$GITHUB_OUTPUT" - # A tag-push builds a revision that main already published, so it - # must NOT claim the identity: image configs are not bit-reproducible - # (embedded timestamps), so re-pushing r- would point an - # immutable tag at fresh bytes — rule 145's exact prohibition. It - # publishes only its own :v… label and otherwise reuses. - if [ "$IS_TAG_PUSH" = "true" ]; then - echo "build_tags=$TAGS" >> "$GITHUB_OUTPUT" - else - echo "build_tags=$TAGS,$IMAGE:$ID" >> "$GITHUB_OUTPUT" + # The moving tag for this channel. Which tag we ask IS the channel — + # that is why the revision needs no -main/-dev qualifier any more. + if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi + echo "channel_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT" + + # Compare VALUES, never exit codes. Measured on buildx v0.36.1 + # (run 4732): a missing key returns an empty string and exits 0, so + # branching on the exit code would read "no label yet" as success. + # An unreachable tag also lands here as empty via the `|| echo`. + # Empty never equals a 12-char revision, so every uncertain case + # falls through to a build — the safe direction, with no special + # casing for it. + # + # Read the SPECIFIC key. The map also carries whatever the base image + # set, and `org.opencontainers.image.version` sits right beside ours + # looking like a plausible answer (it reads 24.04 on the agent). + PUBLISHED=$(docker buildx imagetools inspect "$IMAGE:$T" \ + --format '{{ index .Image.Config.Labels "fc.revision" }}' \ + 2>/dev/null || echo "") + echo "reuse: $IMAGE:$T carries fc.revision=${PUBLISHED:-}; derived=$DERIVED" + if [ -z "$PUBLISHED" ] && docker buildx imagetools inspect "$IMAGE:$T" >/dev/null 2>&1; then + # The tag resolves but carries no readable label. Expected exactly + # once per artifact, during the migration onto labels. If it recurs + # every push, something is rewriting the channel tag as a manifest + # index — see the repoint step's note. + echo "reuse: NOTE $IMAGE:$T exists but has no readable fc.revision." + echo "reuse: NOTE Fine once, while migrating. Every push means the" + echo "reuse: NOTE tag is being index-wrapped and reuse is dead." fi - if docker buildx imagetools inspect "$IMAGE:$ID" >/dev/null 2>&1; then + if [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then echo "hit=true" >> "$GITHUB_OUTPUT" - echo "reuse: $IMAGE:$ID is already published — skipping the build" + echo "reuse: already published — skipping the build" else echo "hit=false" >> "$GITHUB_OUTPUT" - echo "reuse: $IMAGE:$ID is not published — building" + echo "reuse: not published — building" fi - name: Download signed XPI from Forgejo release asset - # Fires on every trigger shape. dev and main each bundle the XPI their - # own sign-extension just published — that is the whole point of the - # channel work (milestone 271 step 6): the dev image carries the - # extension being developed, rather than requiring a merge to try it. - # Tag-push builds re-package the same source as the preceding main-push - # build but with an immutable version tag — they need the XPI too, - # otherwise the versioned image ships without the signed extension. + # dev and main each bundle the XPI their own sign-extension just + # published — the point of the channel work (milestone 271 step 6): the + # dev image carries the extension being developed, rather than + # requiring a merge to try it. # - # Tag-push vs main-push race (operator-flagged 2026-05-27 after - # v26.05.27.0 hit it): a release cut fires BOTH workflows almost - # simultaneously. Main-push runs sign-extension (1-5min AMO round - # trip) before publishing the ext- release; tag-push - # skips sign-extension (gated to main) and races straight to - # this download step. Tag-push lost every time. Fix: poll the - # ext- release endpoint with a sleep+retry loop (30s - # for up to 10min total) before giving up. Main-push's signing - # eventually wins and tag-push picks the release up on a later - # iteration. - # Gated on the reuse miss as well: if the image is already published it - # already contains its XPI, so this would download (and on a tag-push, - # poll up to 10 minutes for) a file nothing then reads. - if: steps.reuse.outputs.hit != 'true' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) + # The 10-minute polling loop that used to live here is gone with the + # tag trigger (milestone 318 step 2). It existed for one shape only: a + # release cut fired the tag build and the main build together, the tag + # build skipped sign-extension and raced straight here, and it lost + # every time (operator-flagged 2026-05-27 after v26.05.27.0). Polling + # was the fix for a build that should not have been running. + # + # sign-extension is a `needs` dependency and it succeeded, so the + # release exists. A single fetch is correct, and a 404 now means a real + # disagreement about the derived version rather than a race — which is + # exactly what should fail loudly instead of being slept through. + # + # Still gated on the reuse miss: a published image already contains its + # XPI, so this would fetch a file nothing then reads. + if: steps.reuse.outputs.hit != 'true' env: TOKEN: ${{ secrets.RELEASE_TOKEN }} run: | @@ -552,25 +554,21 @@ jobs: # didn't, this download 404s and the build fails loudly instead of # shipping a stale XPI. VERSION=$(sh extension/scripts/packaging.sh version) - # Poll for the ext- release. main-push's sign-extension - # step (AMO round-trip, 1-5min) needs to finish + upload before - # tag-push can fetch. 30s * 20 = up to 10min wait, then hard-fail. - for attempt in $(seq 1 20); do - STATUS=$(curl -s -o release.json -w "%{http_code}" \ - -H "Authorization: token $TOKEN" \ - "https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" || echo 000) - if [ "$STATUS" = "200" ]; then - echo "Found ext-$VERSION release on attempt $attempt" - break - fi - if [ "$attempt" = "20" ]; then - echo "ERROR: ext-$VERSION release not available after 10min of polling" - echo "Last HTTP status: $STATUS" - exit 1 - fi - echo "Attempt $attempt: ext-$VERSION not yet published (HTTP $STATUS); sleeping 30s" - sleep 30 - done + # One fetch, no retry. sign-extension ran to success in this same + # workflow and published ext-$VERSION; both jobs derive $VERSION from + # the same commit, so they agree by construction. A 404 here means + # they did NOT agree, and sleeping on that would only delay the + # report. + STATUS=$(curl -s -o release.json -w "%{http_code}" \ + -H "Authorization: token $TOKEN" \ + "https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" || echo 000) + if [ "$STATUS" != "200" ]; then + echo "ERROR: ext-$VERSION release not found (HTTP $STATUS)." + echo "sign-extension succeeded in this run, so it published some" + echo "other version — the two jobs derived different values for one" + echo "commit. Check that both checked out with fetch-depth: 0." + exit 1 + fi # Extract the .xpi asset's browser_download_url (Forgejo's # /releases/assets/ endpoint returns ASSET METADATA, not # the binary blob — operator-flagged 2026-05-26: my prior @@ -608,6 +606,11 @@ jobs: file: Dockerfile push: true tags: ${{ steps.reuse.outputs.build_tags }} + # The reuse key. Read back off the channel tag on the next push to + # decide whether that push needs to build at all, so this is not + # decoration — an unstamped image is one that will always rebuild. + labels: | + fc.revision=${{ steps.reuse.outputs.revision }} # Only the web image carries a channel: it is the one that serves # /api/extension/manifest. The ml and agent images have nothing to # report it to. @@ -616,8 +619,8 @@ jobs: # Registry-side manifest copy: no layer transfer, no local daemon, no # rebuild. Each -t becomes another reference to the SAME manifest the - # identity tag holds, so :latest and the date pin are byte-identical to - # what was published rather than a lookalike rebuild. + # channel tag already holds, so :c- and the date pin are + # byte-identical to what is published rather than a lookalike rebuild. # # Runs on EVERY reuse, which is what keeps family rule 146 true: a # rolling channel refreshes itself, so skipping a build must never mean @@ -627,19 +630,46 @@ jobs: if: steps.reuse.outputs.hit == 'true' env: IMAGE: git.fabledsword.com/bvandeusen/fabledcurator - IDENTITY: ${{ steps.reuse.outputs.identity }} + SOURCE: ${{ steps.reuse.outputs.channel_ref }} TAGS: ${{ steps.tag.outputs.tags }} run: | set -euf + # The source tag is EXCLUDED from the targets, and that is load- + # bearing rather than an optimisation. + # + # `imagetools create` wraps the source manifest in an INDEX. Point it + # at the channel tag with that same tag as a target and the tag stops + # being a plain image — after which `.Image.Config.Labels` no longer + # resolves through it and the fc.revision label reads as absent. The + # next push then misses and rebuilds, so reuse worked exactly once + # and every subsequent push paid full price. Observed on run 4751: + # ml:dev reported fc.revision= one push after run 4749 had read + # a7e626a67a79 off it. Nothing failed; the savings just evaporated. + # + # Excluding the source means the channel tag is only ever written by + # a real build, so it stays a plain image and stays readable. On dev + # that leaves nothing to do — :dev already points at the right + # content, which is what the hit established. On main it leaves + # :c-, which rule 145 requires of every main push whether or not + # a build ran. + # # steps.tag emits ONE comma-separated list, because that is the shape # docker/build-push-action takes; imagetools wants a -t per ref. ARGS="" IFS=, - for t in $TAGS; do ARGS="$ARGS -t $t"; done + for t in $TAGS; do + [ "$t" = "$SOURCE" ] && continue + ARGS="$ARGS -t $t" + done unset IFS + if [ -z "$ARGS" ]; then + echo "repoint: $SOURCE already carries this revision and is the" + echo "repoint: only tag for this channel — nothing to write." + exit 0 + fi # shellcheck disable=SC2086 - docker buildx imagetools create $ARGS "$IMAGE:$IDENTITY" - echo "repointed to $IMAGE:$IDENTITY: $TAGS" + docker buildx imagetools create $ARGS "$SOURCE" + echo "repointed from $SOURCE:$ARGS" build-ml: runs-on: python-ci @@ -679,49 +709,22 @@ jobs: run: | set -u A=ml - T=$(sh scripts/artifacts.sh tag "$A" 2>&1 || echo UNAVAILABLE) V=$(sh scripts/artifacts.sh version "$A" 2>&1 || echo UNAVAILABLE) R=$(sh scripts/artifacts.sh revision "$A" 2>&1 || echo UNAVAILABLE) - echo "derived: artifact=$A tag=$T version=$V revision=$R sha=$GITHUB_SHA" + echo "derived: artifact=$A version=$V revision=$R sha=$GITHUB_SHA" - name: Determine tag id: tag run: | - # Mirrors build-web's three-shape logic (tag-push / main-push / - # safety-net dev) including the per-commit :c- tag - # on main-push per the family release-posture rule. The -ml - # image follows the same release cadence as the web image. + # Mirrors build-web's tag list; see the comment there. # POSIX-safe substring (the runner shell is dash/BusyBox sh, not # bash — `${var:0:7}` errors with "Bad substitution"; cut works # everywhere). Operator-flagged 2026-06-01 after first :c- # main-push build failed at this step. SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7) - # The pinnable tag (milestone 313 step 3): YYYY.M.D of the commit - # THIS artifact's shipped files last changed in. Day precision is - # deliberate — same-day work is not something worth pinning, so a - # second main build the same day replaces the first rather than - # accumulating a tag nobody would roll back to. - # - # Derived per artifact, so an image whose files did not change keeps - # the tag it already had: the agent reads 2026.7.17 today while web - # reads 2026.8.27 — and the reuse step below turns that into a - # skipped build rather than a rebuild of bytes that already exist. - if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then - TAG_NAME="${GITHUB_REF#refs/tags/}" - echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:${TAG_NAME}" >> "$GITHUB_OUTPUT" - echo "channel=main" >> "$GITHUB_OUTPUT" - elif [ "${GITHUB_REF##*/}" = "main" ]; then - CALVER=$(sh scripts/artifacts.sh tag ml) - # Guarded, and computed only on this path. There is no `set -e` in - # this step, so a failed derivation would otherwise leave CALVER - # empty and publish the tag `fabledcurator-ml:` — an invalid - # name, from a green step. An empty pin must never reach the - # registry. - if [ -z "$CALVER" ]; then - echo "ERROR: could not derive a ml version tag" >&2 - exit 1 - fi - echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:main,git.fabledsword.com/bvandeusen/fabledcurator-ml:latest,git.fabledsword.com/bvandeusen/fabledcurator-ml:c-${SHORT_SHA},git.fabledsword.com/bvandeusen/fabledcurator-ml:${CALVER}" >> "$GITHUB_OUTPUT" + # Mirrors build-web's tag list; see the comment there. + if [ "${GITHUB_REF##*/}" = "main" ]; then + echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:latest,git.fabledsword.com/bvandeusen/fabledcurator-ml:c-${SHORT_SHA}" >> "$GITHUB_OUTPUT" echo "channel=main" >> "$GITHUB_OUTPUT" else echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:dev" >> "$GITHUB_OUTPUT" @@ -737,12 +740,21 @@ jobs: run: echo "$TOKEN" | docker login git.fabledsword.com -u "$ACTOR" --password-stdin # --- reuse-if-published (milestone 313, step 4) ---------------------- - # The identity tag names this artifact's CONTENT — r-, the - # commit its shipped files last changed in, plus the channel for images - # that bake one in. If the registry already carries it, the bytes this - # job would produce are already published and the build is pure waste: - # the channel and date tags get repointed at the existing manifest - # instead, registry-side, in seconds. + # Does the image the channel tag already points at carry THIS commit's + # revision? If so the bytes this job would produce are already published + # and the build is pure waste: the remaining tags get repointed at that + # existing manifest instead, registry-side, in seconds. + # + # Keyed on an `fc.revision` LABEL rather than on a tag of its own + # (milestone 318 step 3). A tag would be a name minted per build that one + # thing reads — what rule 145 narrowed against — and would be prunable + # under the registry's keep_pattern (#3157), silently expiring the cache. + # A label rides inside a tag that has to exist anyway. + # + # An image with no such label reads as a miss and rebuilds. That is the + # migration, not a fault: labels cannot be backfilled, since the reuse + # path copies a manifest and config labels are not manifest annotations. + # Each artifact pays one rebuild, once. # # This is what stops a push that touched only `agent/` from rebuilding # web and ml, and a merge to main from rebuilding what dev already built. @@ -768,29 +780,48 @@ jobs: IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-ml CHANNEL: ${{ steps.tag.outputs.channel }} TAGS: ${{ steps.tag.outputs.tags }} - IS_TAG_PUSH: ${{ startsWith(github.ref, 'refs/tags/') }} run: | set -eu - ID=$(sh scripts/artifacts.sh identity ml "$CHANNEL") - echo "identity=$ID" >> "$GITHUB_OUTPUT" + DERIVED=$(sh scripts/artifacts.sh revision ml) + echo "revision=$DERIVED" >> "$GITHUB_OUTPUT" + echo "build_tags=$TAGS" >> "$GITHUB_OUTPUT" - # A tag-push builds a revision that main already published, so it - # must NOT claim the identity: image configs are not bit-reproducible - # (embedded timestamps), so re-pushing r- would point an - # immutable tag at fresh bytes — rule 145's exact prohibition. It - # publishes only its own :v… label and otherwise reuses. - if [ "$IS_TAG_PUSH" = "true" ]; then - echo "build_tags=$TAGS" >> "$GITHUB_OUTPUT" - else - echo "build_tags=$TAGS,$IMAGE:$ID" >> "$GITHUB_OUTPUT" + # The moving tag for this channel. Which tag we ask IS the channel — + # that is why the revision needs no -main/-dev qualifier any more. + if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi + echo "channel_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT" + + # Compare VALUES, never exit codes. Measured on buildx v0.36.1 + # (run 4732): a missing key returns an empty string and exits 0, so + # branching on the exit code would read "no label yet" as success. + # An unreachable tag also lands here as empty via the `|| echo`. + # Empty never equals a 12-char revision, so every uncertain case + # falls through to a build — the safe direction, with no special + # casing for it. + # + # Read the SPECIFIC key. The map also carries whatever the base image + # set, and `org.opencontainers.image.version` sits right beside ours + # looking like a plausible answer (it reads 24.04 on the agent). + PUBLISHED=$(docker buildx imagetools inspect "$IMAGE:$T" \ + --format '{{ index .Image.Config.Labels "fc.revision" }}' \ + 2>/dev/null || echo "") + echo "reuse: $IMAGE:$T carries fc.revision=${PUBLISHED:-}; derived=$DERIVED" + if [ -z "$PUBLISHED" ] && docker buildx imagetools inspect "$IMAGE:$T" >/dev/null 2>&1; then + # The tag resolves but carries no readable label. Expected exactly + # once per artifact, during the migration onto labels. If it recurs + # every push, something is rewriting the channel tag as a manifest + # index — see the repoint step's note. + echo "reuse: NOTE $IMAGE:$T exists but has no readable fc.revision." + echo "reuse: NOTE Fine once, while migrating. Every push means the" + echo "reuse: NOTE tag is being index-wrapped and reuse is dead." fi - if docker buildx imagetools inspect "$IMAGE:$ID" >/dev/null 2>&1; then + if [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then echo "hit=true" >> "$GITHUB_OUTPUT" - echo "reuse: $IMAGE:$ID is already published — skipping the build" + echo "reuse: already published — skipping the build" else echo "hit=false" >> "$GITHUB_OUTPUT" - echo "reuse: $IMAGE:$ID is not published — building" + echo "reuse: not published — building" fi - name: Build and push ml image @@ -801,11 +832,16 @@ jobs: file: Dockerfile.ml push: true tags: ${{ steps.reuse.outputs.build_tags }} + # The reuse key. Read back off the channel tag on the next push to + # decide whether that push needs to build at all, so this is not + # decoration — an unstamped image is one that will always rebuild. + labels: | + fc.revision=${{ steps.reuse.outputs.revision }} # Registry-side manifest copy: no layer transfer, no local daemon, no # rebuild. Each -t becomes another reference to the SAME manifest the - # identity tag holds, so :latest and the date pin are byte-identical to - # what was published rather than a lookalike rebuild. + # channel tag already holds, so :c- and the date pin are + # byte-identical to what is published rather than a lookalike rebuild. # # Runs on EVERY reuse, which is what keeps family rule 146 true: a # rolling channel refreshes itself, so skipping a build must never mean @@ -815,19 +851,46 @@ jobs: if: steps.reuse.outputs.hit == 'true' env: IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-ml - IDENTITY: ${{ steps.reuse.outputs.identity }} + SOURCE: ${{ steps.reuse.outputs.channel_ref }} TAGS: ${{ steps.tag.outputs.tags }} run: | set -euf + # The source tag is EXCLUDED from the targets, and that is load- + # bearing rather than an optimisation. + # + # `imagetools create` wraps the source manifest in an INDEX. Point it + # at the channel tag with that same tag as a target and the tag stops + # being a plain image — after which `.Image.Config.Labels` no longer + # resolves through it and the fc.revision label reads as absent. The + # next push then misses and rebuilds, so reuse worked exactly once + # and every subsequent push paid full price. Observed on run 4751: + # ml:dev reported fc.revision= one push after run 4749 had read + # a7e626a67a79 off it. Nothing failed; the savings just evaporated. + # + # Excluding the source means the channel tag is only ever written by + # a real build, so it stays a plain image and stays readable. On dev + # that leaves nothing to do — :dev already points at the right + # content, which is what the hit established. On main it leaves + # :c-, which rule 145 requires of every main push whether or not + # a build ran. + # # steps.tag emits ONE comma-separated list, because that is the shape # docker/build-push-action takes; imagetools wants a -t per ref. ARGS="" IFS=, - for t in $TAGS; do ARGS="$ARGS -t $t"; done + for t in $TAGS; do + [ "$t" = "$SOURCE" ] && continue + ARGS="$ARGS -t $t" + done unset IFS + if [ -z "$ARGS" ]; then + echo "repoint: $SOURCE already carries this revision and is the" + echo "repoint: only tag for this channel — nothing to write." + exit 0 + fi # shellcheck disable=SC2086 - docker buildx imagetools create $ARGS "$IMAGE:$IDENTITY" - echo "repointed to $IMAGE:$IDENTITY: $TAGS" + docker buildx imagetools create $ARGS "$SOURCE" + echo "repointed from $SOURCE:$ARGS" # The desktop GPU agent (#114) — published so the operator pulls + runs it on # the GPU machine instead of building locally. Independent of web/ml (its own @@ -870,41 +933,17 @@ jobs: run: | set -u A=agent - T=$(sh scripts/artifacts.sh tag "$A" 2>&1 || echo UNAVAILABLE) V=$(sh scripts/artifacts.sh version "$A" 2>&1 || echo UNAVAILABLE) R=$(sh scripts/artifacts.sh revision "$A" 2>&1 || echo UNAVAILABLE) - echo "derived: artifact=$A tag=$T version=$V revision=$R sha=$GITHUB_SHA" + echo "derived: artifact=$A version=$V revision=$R sha=$GITHUB_SHA" - name: Determine tag id: tag run: | SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7) - # The pinnable tag (milestone 313 step 3): YYYY.M.D of the commit - # THIS artifact's shipped files last changed in. Day precision is - # deliberate — same-day work is not something worth pinning, so a - # second main build the same day replaces the first rather than - # accumulating a tag nobody would roll back to. - # - # Derived per artifact, so an image whose files did not change keeps - # the tag it already had: the agent reads 2026.7.17 today while web - # reads 2026.8.27 — and the reuse step below turns that into a - # skipped build rather than a rebuild of bytes that already exist. - if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then - TAG_NAME="${GITHUB_REF#refs/tags/}" - echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-agent:${TAG_NAME}" >> "$GITHUB_OUTPUT" - echo "channel=main" >> "$GITHUB_OUTPUT" - elif [ "${GITHUB_REF##*/}" = "main" ]; then - CALVER=$(sh scripts/artifacts.sh tag agent) - # Guarded, and computed only on this path. There is no `set -e` in - # this step, so a failed derivation would otherwise leave CALVER - # empty and publish the tag `fabledcurator-agent:` — an invalid - # name, from a green step. An empty pin must never reach the - # registry. - if [ -z "$CALVER" ]; then - echo "ERROR: could not derive a agent version tag" >&2 - exit 1 - fi - echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-agent:main,git.fabledsword.com/bvandeusen/fabledcurator-agent:latest,git.fabledsword.com/bvandeusen/fabledcurator-agent:c-${SHORT_SHA},git.fabledsword.com/bvandeusen/fabledcurator-agent:${CALVER}" >> "$GITHUB_OUTPUT" + # Mirrors build-web's tag list; see the comment there. + if [ "${GITHUB_REF##*/}" = "main" ]; then + echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-agent:latest,git.fabledsword.com/bvandeusen/fabledcurator-agent:c-${SHORT_SHA}" >> "$GITHUB_OUTPUT" echo "channel=main" >> "$GITHUB_OUTPUT" else echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-agent:dev" >> "$GITHUB_OUTPUT" @@ -920,12 +959,21 @@ jobs: run: echo "$TOKEN" | docker login git.fabledsword.com -u "$ACTOR" --password-stdin # --- reuse-if-published (milestone 313, step 4) ---------------------- - # The identity tag names this artifact's CONTENT — r-, the - # commit its shipped files last changed in, plus the channel for images - # that bake one in. If the registry already carries it, the bytes this - # job would produce are already published and the build is pure waste: - # the channel and date tags get repointed at the existing manifest - # instead, registry-side, in seconds. + # Does the image the channel tag already points at carry THIS commit's + # revision? If so the bytes this job would produce are already published + # and the build is pure waste: the remaining tags get repointed at that + # existing manifest instead, registry-side, in seconds. + # + # Keyed on an `fc.revision` LABEL rather than on a tag of its own + # (milestone 318 step 3). A tag would be a name minted per build that one + # thing reads — what rule 145 narrowed against — and would be prunable + # under the registry's keep_pattern (#3157), silently expiring the cache. + # A label rides inside a tag that has to exist anyway. + # + # An image with no such label reads as a miss and rebuilds. That is the + # migration, not a fault: labels cannot be backfilled, since the reuse + # path copies a manifest and config labels are not manifest annotations. + # Each artifact pays one rebuild, once. # # This is what stops a push that touched only `agent/` from rebuilding # web and ml, and a merge to main from rebuilding what dev already built. @@ -951,29 +999,48 @@ jobs: IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-agent CHANNEL: ${{ steps.tag.outputs.channel }} TAGS: ${{ steps.tag.outputs.tags }} - IS_TAG_PUSH: ${{ startsWith(github.ref, 'refs/tags/') }} run: | set -eu - ID=$(sh scripts/artifacts.sh identity agent "$CHANNEL") - echo "identity=$ID" >> "$GITHUB_OUTPUT" + DERIVED=$(sh scripts/artifacts.sh revision agent) + echo "revision=$DERIVED" >> "$GITHUB_OUTPUT" + echo "build_tags=$TAGS" >> "$GITHUB_OUTPUT" - # A tag-push builds a revision that main already published, so it - # must NOT claim the identity: image configs are not bit-reproducible - # (embedded timestamps), so re-pushing r- would point an - # immutable tag at fresh bytes — rule 145's exact prohibition. It - # publishes only its own :v… label and otherwise reuses. - if [ "$IS_TAG_PUSH" = "true" ]; then - echo "build_tags=$TAGS" >> "$GITHUB_OUTPUT" - else - echo "build_tags=$TAGS,$IMAGE:$ID" >> "$GITHUB_OUTPUT" + # The moving tag for this channel. Which tag we ask IS the channel — + # that is why the revision needs no -main/-dev qualifier any more. + if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi + echo "channel_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT" + + # Compare VALUES, never exit codes. Measured on buildx v0.36.1 + # (run 4732): a missing key returns an empty string and exits 0, so + # branching on the exit code would read "no label yet" as success. + # An unreachable tag also lands here as empty via the `|| echo`. + # Empty never equals a 12-char revision, so every uncertain case + # falls through to a build — the safe direction, with no special + # casing for it. + # + # Read the SPECIFIC key. The map also carries whatever the base image + # set, and `org.opencontainers.image.version` sits right beside ours + # looking like a plausible answer (it reads 24.04 on the agent). + PUBLISHED=$(docker buildx imagetools inspect "$IMAGE:$T" \ + --format '{{ index .Image.Config.Labels "fc.revision" }}' \ + 2>/dev/null || echo "") + echo "reuse: $IMAGE:$T carries fc.revision=${PUBLISHED:-}; derived=$DERIVED" + if [ -z "$PUBLISHED" ] && docker buildx imagetools inspect "$IMAGE:$T" >/dev/null 2>&1; then + # The tag resolves but carries no readable label. Expected exactly + # once per artifact, during the migration onto labels. If it recurs + # every push, something is rewriting the channel tag as a manifest + # index — see the repoint step's note. + echo "reuse: NOTE $IMAGE:$T exists but has no readable fc.revision." + echo "reuse: NOTE Fine once, while migrating. Every push means the" + echo "reuse: NOTE tag is being index-wrapped and reuse is dead." fi - if docker buildx imagetools inspect "$IMAGE:$ID" >/dev/null 2>&1; then + if [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then echo "hit=true" >> "$GITHUB_OUTPUT" - echo "reuse: $IMAGE:$ID is already published — skipping the build" + echo "reuse: already published — skipping the build" else echo "hit=false" >> "$GITHUB_OUTPUT" - echo "reuse: $IMAGE:$ID is not published — building" + echo "reuse: not published — building" fi - name: Build and push agent image @@ -984,11 +1051,16 @@ jobs: file: agent/Dockerfile push: true tags: ${{ steps.reuse.outputs.build_tags }} + # The reuse key. Read back off the channel tag on the next push to + # decide whether that push needs to build at all, so this is not + # decoration — an unstamped image is one that will always rebuild. + labels: | + fc.revision=${{ steps.reuse.outputs.revision }} # Registry-side manifest copy: no layer transfer, no local daemon, no # rebuild. Each -t becomes another reference to the SAME manifest the - # identity tag holds, so :latest and the date pin are byte-identical to - # what was published rather than a lookalike rebuild. + # channel tag already holds, so :c- and the date pin are + # byte-identical to what is published rather than a lookalike rebuild. # # Runs on EVERY reuse, which is what keeps family rule 146 true: a # rolling channel refreshes itself, so skipping a build must never mean @@ -998,16 +1070,43 @@ jobs: if: steps.reuse.outputs.hit == 'true' env: IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-agent - IDENTITY: ${{ steps.reuse.outputs.identity }} + SOURCE: ${{ steps.reuse.outputs.channel_ref }} TAGS: ${{ steps.tag.outputs.tags }} run: | set -euf + # The source tag is EXCLUDED from the targets, and that is load- + # bearing rather than an optimisation. + # + # `imagetools create` wraps the source manifest in an INDEX. Point it + # at the channel tag with that same tag as a target and the tag stops + # being a plain image — after which `.Image.Config.Labels` no longer + # resolves through it and the fc.revision label reads as absent. The + # next push then misses and rebuilds, so reuse worked exactly once + # and every subsequent push paid full price. Observed on run 4751: + # ml:dev reported fc.revision= one push after run 4749 had read + # a7e626a67a79 off it. Nothing failed; the savings just evaporated. + # + # Excluding the source means the channel tag is only ever written by + # a real build, so it stays a plain image and stays readable. On dev + # that leaves nothing to do — :dev already points at the right + # content, which is what the hit established. On main it leaves + # :c-, which rule 145 requires of every main push whether or not + # a build ran. + # # steps.tag emits ONE comma-separated list, because that is the shape # docker/build-push-action takes; imagetools wants a -t per ref. ARGS="" IFS=, - for t in $TAGS; do ARGS="$ARGS -t $t"; done + for t in $TAGS; do + [ "$t" = "$SOURCE" ] && continue + ARGS="$ARGS -t $t" + done unset IFS + if [ -z "$ARGS" ]; then + echo "repoint: $SOURCE already carries this revision and is the" + echo "repoint: only tag for this channel — nothing to write." + exit 0 + fi # shellcheck disable=SC2086 - docker buildx imagetools create $ARGS "$IMAGE:$IDENTITY" - echo "repointed to $IMAGE:$IDENTITY: $TAGS" + docker buildx imagetools create $ARGS "$SOURCE" + echo "repointed from $SOURCE:$ARGS" diff --git a/scripts/artifacts.sh b/scripts/artifacts.sh index a1ea0d5..0b19cb2 100755 --- a/scripts/artifacts.sh +++ b/scripts/artifacts.sh @@ -56,25 +56,9 @@ ML_PATHS='Dockerfile.ml requirements-ml.txt requirements.txt backend alembic ale # it: this is deliberately NOT `agent/`. AGENT_PATHS='agent/Dockerfile agent/requirements.txt agent/fc_agent' -# Which artifacts bake the BUILD CHANNEL into the image, and therefore cannot -# share a content identity across channels. The web image takes FC_CHANNEL as -# a build-arg and reports it from /api/extension/manifest (milestone 271 step -# 7), so `main` and `dev` builds of one revision are genuinely different -# images — reusing the dev one on main would ship an instance that names -# itself `dev` forever. -# -# ml and agent take no build-args at all: one revision, one image, and a merge -# to main can reuse exactly what dev already built. That is not a detail, it is -# most of what step 4 saves — merges would otherwise rebuild the agent's CUDA -# image to produce bytes that already exist. -# -# Extend this list if a second artifact ever gains a build-arg; -# tests/test_artifact_identity.py reads the Dockerfiles and fails if it drifts. -CHANNELLED='web' usage() { - echo "usage: artifacts.sh {paths|revision|version|tag} {web|ml|agent|extension}" >&2 - echo " artifacts.sh identity {web|ml|agent} [channel]" >&2 + echo "usage: artifacts.sh {paths|revision|version} {web|ml|agent|extension}" >&2 exit 2 } @@ -124,9 +108,18 @@ strip0() { } # The IDENTITY of an artifact's content: the commit its shipped files last -# changed in. This — not the tag — is what decides whether a build can be -# skipped, because the published tag is only day-precise and two different -# builds can share it. +# changed in. This is what decides whether a build can be skipped. +# +# It is published as the `fc.revision` LABEL on the image itself, and read +# back off the moving channel tag — not as a tag of its own (milestone 318 +# step 3). A tag would be a name minted per build that only one thing reads, +# which is what rule 145 narrowed against; it would also be prunable under the +# registry's keep_pattern (#3157), so the cache would silently expire. +# +# A published image with no such label reads as a MISS and rebuilds. That is +# the migration path, not a fault: `imagetools create` copies a manifest and +# config labels are not manifest annotations, so the reuse path cannot stamp +# one and there is nothing to backfill. Each artifact pays one rebuild, once. cmd_revision() { echo "$(newest "$1")" | cut -d' ' -f2 | cut -c1-12 } @@ -144,63 +137,10 @@ cmd_version() { "$(strip0 "$(fmt "$sha" %H%M)")" } -# The PUBLISHED IMAGE TAG: day precision, YYYY.M.D. Deliberately coarser than -# the ordering key, per the operator 2026-08-28 — same-day work is not -# something worth pinning, so a second build the same day replaces the first -# rather than accumulating a tag nobody would roll back to. Safe only because -# skip decisions key on cmd_revision, never on this. -cmd_tag() { - sha=$(echo "$(newest "$1")" | cut -d' ' -f2) - printf '%s.%s.%s\n' \ - "$(fmt "$sha" %Y)" \ - "$(strip0 "$(fmt "$sha" %m)")" \ - "$(strip0 "$(fmt "$sha" %d)")" -} - -# The CONTENT IDENTITY of a published image: an immutable tag naming exactly -# what a build of this commit would produce. build.yml asks the registry for it -# and, on a hit, skips the build entirely and repoints the channel and date -# tags at the manifest that is already there (milestone 313 step 4). -# -# It is deliberately NOT either of the other two values: -# * the date tag is day-precise and last-one-wins, so two different builds -# share it — it cannot answer "is this content published?". -# * the commit sha moves on every push, so it would never hit, which is the -# redundant rebuild this exists to remove. -# -# The revision does both jobs: it is content-unique AND stable across pushes -# that did not touch the artifact. -cmd_identity() { - _art=$1 - _chan=${2:-} - case "$_art" in - web|ml|agent) ;; - extension) - echo "artifacts.sh: the extension is cached as an ext- Forgejo release, not an image tag — use \`version\`" >&2 - exit 2 ;; - *) usage ;; - esac - for _c in $CHANNELLED; do - if [ "$_art" = "$_c" ]; then - # Refused rather than defaulted: an unqualified identity for a - # channelled artifact would let a dev image be reused as the main one. - if [ -z "$_chan" ]; then - echo "artifacts.sh: $_art bakes the channel into the image — identity needs one" >&2 - exit 2 - fi - printf 'r-%s-%s\n' "$(cmd_revision "$_art")" "$_chan" - return - fi - done - printf 'r-%s\n' "$(cmd_revision "$_art")" -} - [ $# -ge 2 ] || usage case "$1" in paths) cmd_paths "$2" ;; revision) cmd_revision "$2" ;; version) cmd_version "$2" ;; - tag) cmd_tag "$2" ;; - identity) cmd_identity "$2" "${3:-}" ;; *) usage ;; esac diff --git a/tests/test_artifact_identity.py b/tests/test_artifact_identity.py index b65ff17..e234ab2 100644 --- a/tests/test_artifact_identity.py +++ b/tests/test_artifact_identity.py @@ -1,20 +1,29 @@ -"""`artifacts.sh identity` is what decides whether a build gets skipped. +"""`artifacts.sh revision` is what decides whether a build gets skipped. -Milestone 313 step 4: build.yml asks the registry for `:` and, -on a hit, publishes NO new bytes — it repoints the channel and date tags at the -manifest already there. So the identity has to be a true name for the content. -Both ways of getting it wrong are silent at build time and only surface in -production: +Milestone 318 step 3: each image carries its revision as an `fc.revision` +label, and build.yml reads that label back off the moving channel tag. Equal +to the derived revision means the bytes this push would produce are already +published, so the build is skipped. -* **too coarse** — two genuinely different images share an identity, so the - second one never gets built and its tags point at the first one's bytes. The - live case is FC_CHANNEL: a `dev` and a `main` build of one revision differ, - and collapsing them ships an instance that reports the wrong channel forever. -* **too fine** — the identity moves when the content did not, nothing ever - hits, and step 4 buys nothing. A commit sha would do exactly this. +That makes the revision load-bearing in a way a version string is not — it is +compared for equality against a value stamped into a real published artifact. +Both ways of getting it wrong are silent: -The Dockerfiles are read here rather than trusted, because the coarse direction -appears the moment someone adds a build-arg without touching `CHANNELLED`. +* **it does not identify the content** — a revision that moves when the source + did not (a HEAD-derived value, say) never matches, nothing is ever skipped, + and the mechanism quietly buys nothing while every lane stays green. +* **it identifies the wrong content** — a revision that holds still when the + source DID change matches a stale label, the build is skipped, and the + channel serves bytes that do not correspond to the commit. This is the + dangerous direction, and it is what `test_artifact_paths.py` guards from the + other side by pinning the path sets. + +This module owns the narrower claim: whatever the path sets say, the revision +is genuinely the commit those paths last changed in. + +The identity-TAG tests this file used to hold are gone with the tag. There is +no longer a `CHANNELLED` list to drift (the channel is which tag you inspect), +and no `identity` subcommand to refuse an unqualified call. """ from __future__ import annotations @@ -26,115 +35,75 @@ import pytest ROOT = Path(__file__).resolve().parent.parent -# Only image artifacts have an identity — the extension is cached as an -# ext- Forgejo release, not a registry tag. -IMAGE_ARTIFACTS = { - "web": "Dockerfile", - "ml": "Dockerfile.ml", - "agent": "agent/Dockerfile", -} +ARTIFACTS = ("web", "ml", "agent", "extension") -CHANNELS = ("main", "dev") - -# docker's own tag grammar: [A-Za-z0-9_][A-Za-z0-9._-]{0,127} -_TAG = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9._-]{0,127}$") - -# `ARG FC_CHANNEL` in a Dockerfile means build.yml passes a per-channel value -# in, so the channel is part of what the image IS. -_ARG_CHANNEL = re.compile(r"^\s*ARG\s+FC_CHANNEL\b", re.MULTILINE) +# 12 hex chars — the prefix build.yml stamps and compares. +_REVISION = re.compile(r"^[0-9a-f]{12}$") -def identity(artifact: str, channel: str | None = None) -> subprocess.CompletedProcess: - cmd = ["sh", str(ROOT / "scripts" / "artifacts.sh"), "identity", artifact] - if channel is not None: - cmd.append(channel) - return subprocess.run(cmd, capture_output=True, text=True, cwd=ROOT) - - -def ok(artifact: str, channel: str | None = None) -> str: - proc = identity(artifact, channel) - assert proc.returncode == 0, f"identity {artifact} {channel}: {proc.stderr}" - return proc.stdout.strip() - - -def bakes_the_channel(artifact: str) -> bool: - return bool(_ARG_CHANNEL.search((ROOT / IMAGE_ARTIFACTS[artifact]).read_text())) - - -@pytest.mark.parametrize("artifact", sorted(IMAGE_ARTIFACTS)) -def test_channel_dependence_matches_the_dockerfile(artifact): - """The coarse direction, caught at its source. - - Whether the channel belongs in the identity is not a preference — it is - dictated by whether the Dockerfile takes it as a build-arg. Adding an - `ARG FC_CHANNEL` to another image without adding it to `CHANNELLED` would - make its dev and main builds collide, and nothing else would notice. - """ - per_channel = {c: ok(artifact, c) for c in CHANNELS} - differs = len(set(per_channel.values())) > 1 - - if bakes_the_channel(artifact): - assert differs, ( - f"{IMAGE_ARTIFACTS[artifact]} declares ARG FC_CHANNEL, so a dev " - f"build and a main build of one revision are different images — " - f"but both derive the identity {per_channel['main']!r}. The main " - f"build would reuse the dev image and report the wrong channel. " - f"Add {artifact!r} to CHANNELLED in scripts/artifacts.sh." - ) - else: - assert not differs, ( - f"{IMAGE_ARTIFACTS[artifact]} takes no channel build-arg, so one " - f"revision is one image and a merge to main should reuse what dev " - f"already built — but the identity differs per channel " - f"({per_channel}), so every merge rebuilds it for nothing. Remove " - f"{artifact!r} from CHANNELLED in scripts/artifacts.sh." - ) - - -@pytest.mark.parametrize("artifact", sorted(IMAGE_ARTIFACTS)) -def test_identity_tracks_the_artifacts_own_revision(artifact): - """The fine direction: the identity must be the revision, not the push. - - `revision` is the commit this artifact's shipped files last changed in, so - it holds still across pushes that did not touch it. Anything derived from - HEAD instead would move every push and never hit the registry. - """ - rev = subprocess.run( - ["sh", str(ROOT / "scripts" / "artifacts.sh"), "revision", artifact], +# Everything here goes through artifacts.sh rather than importing a sibling +# test module. That is the interface build.yml actually calls, so the tests +# exercise the contract instead of a Python re-implementation of it — and no +# other test module in this repo imports another, so a cross-test import would +# be a new convention introduced for no gain. +def artifacts(*args: str) -> str: + return subprocess.run( + ["sh", str(ROOT / "scripts" / "artifacts.sh"), *args], capture_output=True, text=True, check=True, cwd=ROOT, - ).stdout.strip() - value = ok(artifact, "main") - assert rev and rev in value, ( - f"identity {value!r} does not contain the {artifact} revision {rev!r}" + ).stdout + + +def revision(artifact: str) -> str: + return artifacts("revision", artifact).strip() + + +def newest_by_commit_time(artifact: str) -> str: + """The full SHA of the newest commit touching this artifact's shipped set. + + Ordered by committer TIME, matching what artifacts.sh means. Deliberately + not `git log -1`: git's default order is reverse-chronological only within + topological constraints, so on a merged history it can name a different + commit than the newest timestamp does. They agree on this repo today, and + a test that silently depends on them continuing to agree would be a flake + waiting for the branch shape that separates them. + """ + paths = artifacts("paths", artifact).split() + log = subprocess.run( + ["git", "log", "--format=%ct %H", "HEAD", "--", *paths], + capture_output=True, text=True, check=True, cwd=ROOT, + ).stdout.split("\n") + commits = [line.split(" ", 1) for line in log if line.strip()] + assert commits, ( + f"no commit in this history touches the {artifact} path set — the " + f"derivation has nothing to stand on" + ) + return max(commits, key=lambda c: int(c[0]))[1] + + +@pytest.mark.parametrize("artifact", ARTIFACTS) +def test_revision_is_the_commit_its_own_shipped_files_last_changed_in(artifact): + """The claim the whole skip decision rests on. + + Computed from git rather than asked of the script, so it fails if the + derivation ever stops meaning what it says — switching to HEAD, to a build + clock, or to a path set it did not actually use. Each of those still + produces a plausible 12-hex value, which is why this is worth asserting + rather than eyeballing. + """ + expected = newest_by_commit_time(artifact) + got = revision(artifact) + assert expected.startswith(got), ( + f"{artifact} derives {got!r}, but the newest commit touching its " + f"shipped files is {expected[:12]!r}. The label stamped into the image " + f"would not identify its own content." ) -@pytest.mark.parametrize("artifact", sorted(IMAGE_ARTIFACTS)) -def test_identity_is_a_legal_docker_tag(artifact): - """It is pushed as a tag, so an illegal one fails at the registry — after - the build has already run.""" - for channel in CHANNELS: - value = ok(artifact, channel) - assert _TAG.match(value), f"{value!r} is not a valid docker tag" - - -def test_a_channelled_artifact_refuses_an_unqualified_identity(): - """Refusing beats defaulting. If `identity web` quietly returned the - unqualified `r-`, a workflow that forgot to pass the channel would - publish one image under a name both channels then reuse — the exact - collision the CHANNELLED list exists to prevent, reintroduced by an - omission rather than by an edit.""" - proc = identity("web") - assert proc.returncode != 0, ( - "identity web returned a value with no channel: " - f"{proc.stdout.strip()!r}" - ) - - -def test_the_extension_has_no_image_identity(): - """It is cached as an ext- release asset, and its cache key is the - version. Answering with a plausible image tag would invite a second, - divergent cache.""" - proc = identity("extension", "main") - assert proc.returncode != 0 - assert "ext-" in proc.stderr +@pytest.mark.parametrize("artifact", ARTIFACTS) +def test_revision_is_a_legal_label_value_and_is_stable(artifact): + """It is stamped as a docker label and compared for string equality, so a + stray newline or a varying value breaks the comparison rather than the + build — the mechanism would simply stop hitting, silently.""" + first = revision(artifact) + assert _REVISION.match(first), f"{first!r} is not a 12-char hex revision" + assert first == revision(artifact), "revision is not stable across calls"