diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index cd61c3e..c8a84f6 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -122,7 +122,7 @@ jobs: T=$(sh scripts/artifacts.sh tag "$A" 2>&1 || echo UNAVAILABLE) V=$(sh scripts/artifacts.sh version "$A" 2>&1 || echo UNAVAILABLE) R=$(sh scripts/artifacts.sh revision "$A" 2>&1 || echo UNAVAILABLE) - echo "shadow: artifact=$A tag=$T version=$V revision=$R sha=$GITHUB_SHA" + echo "derived: artifact=$A tag=$T version=$V revision=$R sha=$GITHUB_SHA" - name: Guard — the derived version must never go backwards env: @@ -341,11 +341,19 @@ jobs: # that exists perfectly well under its real name. fetch-depth: 0 - # --- shadow mode (milestone 313, step 2) ----------------------------- - # Informational ONLY. Nothing reads this and it must never fail the - # build — no `set -e`, and every derivation falls back to UNAVAILABLE. + # --- derived values, one line (milestone 313) ------------------------ + # These stopped being shadow output at step 3: `tag` is published on + # main and `revision` decides whether the build below runs at all. This + # step prints all three anyway, because the load-bearing steps each + # print only the one they use, and on dev the date tag is computed + # nowhere else. When a build is skipped or a pin looks wrong, this is + # the line that says what the commit derived. # - # What to watch across pushes, because this is what step 3 will trust: + # Still diagnostic, so it still must not fail the build — no `set -e`, + # and every derivation falls back to UNAVAILABLE. A broken echo must + # never be the reason an image does not ship. + # + # What it should say: # * a push touching only agent/ moves the agent and leaves web and ml # STILL. If web moves, its path set is too wide. # * a push touching only docs moves nothing. @@ -353,14 +361,162 @@ jobs: # web bakes in the XPI. If web does not move, its set is too narrow # — the direction that serves stale bytes on a pin. # * dev and main derive the same values for the same source. - - name: Shadow — derived artifact version (informational) + - name: Report the derived artifact version run: | set -u A=web T=$(sh scripts/artifacts.sh tag "$A" 2>&1 || echo UNAVAILABLE) V=$(sh scripts/artifacts.sh version "$A" 2>&1 || echo UNAVAILABLE) R=$(sh scripts/artifacts.sh revision "$A" 2>&1 || echo UNAVAILABLE) - echo "shadow: artifact=$A tag=$T version=$V revision=$R sha=$GITHUB_SHA" + echo "derived: artifact=$A tag=$T version=$V revision=$R sha=$GITHUB_SHA" + + - name: Determine tag + id: tag + run: | + # Three trigger shapes: + # refs/tags/v… → tag-push: opt-in milestone label (vYY.MM.DD, + # plus `.N` when the day already carries a tag — + # family rule 148, amended 2026-08-24 after a + # same-day tag was retargeted and a release + # deleted to make room, note 2813). + # Publish ONLY the immutable version tag; + # don't touch :latest (the main-push build + # for the merge commit already did that). + # refs/heads/main → push to main: publish :main + :latest + # (floating) AND :c- (immutable + # per-commit rollback substrate, per family + # release-posture rule "Tags are milestones, + # not gates — commit-SHA images are the + # rollback unit"). Rollback to any commit + # becomes `docker pull …:c-` without a + # release ceremony. + # refs/heads/dev → push to dev: publish :dev, the rolling test + # channel (family rule 146). Rolling means it may + # carry newer contents than the :c- of the + # same commit; it never writes :c- itself, + # because that is the rollback unit (rule 145). + # POSIX-safe substring (the runner shell is dash/BusyBox sh, not + # bash — `${var:0:7}` errors with "Bad substitution"; cut works + # everywhere). Operator-flagged 2026-06-01 after first :c- + # main-push build failed at this step. + SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7) + # The pinnable tag (milestone 313 step 3): YYYY.M.D of the commit + # THIS artifact's shipped files last changed in. Day precision is + # deliberate — same-day work is not something worth pinning, so a + # second main build the same day replaces the first rather than + # accumulating a tag nobody would roll back to. + # + # Derived per artifact, so an image whose files did not change keeps + # the tag it already had: the agent reads 2026.7.17 today while web + # reads 2026.8.27 — and the reuse step below turns that into a + # skipped build rather than a rebuild of bytes that already exist. + # `channel` is baked into the image as FC_CHANNEL and reported by + # /api/extension/manifest (milestone 271 step 7). A tag-push counts as + # `main`: a vYY.MM.DD tag is cut from main, so that image is a + # main-channel artifact wearing an immutable name. + if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then + TAG_NAME="${GITHUB_REF#refs/tags/}" + echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:${TAG_NAME}" >> "$GITHUB_OUTPUT" + echo "channel=main" >> "$GITHUB_OUTPUT" + elif [ "${GITHUB_REF##*/}" = "main" ]; then + CALVER=$(sh scripts/artifacts.sh tag web) + # Guarded, and computed only on this path. There is no `set -e` in + # this step, so a failed derivation would otherwise leave CALVER + # empty and publish the tag `fabledcurator:` — an invalid + # name, from a green step. An empty pin must never reach the + # registry. + if [ -z "$CALVER" ]; then + echo "ERROR: could not derive a web version tag" >&2 + exit 1 + fi + echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:main,git.fabledsword.com/bvandeusen/fabledcurator:latest,git.fabledsword.com/bvandeusen/fabledcurator:c-${SHORT_SHA},git.fabledsword.com/bvandeusen/fabledcurator:${CALVER}" >> "$GITHUB_OUTPUT" + echo "channel=main" >> "$GITHUB_OUTPUT" + else + echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:dev" >> "$GITHUB_OUTPUT" + echo "channel=dev" >> "$GITHUB_OUTPUT" + fi + + # A shell step, not docker/login-action@v3, because the action's shared + # cache races itself (#3118). act_runner caches a remote action under one + # /root/.cache/act/ per runner, and build-web, build-ml and + # build-agent all start in the same second and all want this same action. + # One job re-clones the directory — which empties and repopulates it — + # while another is walking it to copy into its container, and the walker + # lstat()s a file that has just vanished. It failed twice on 2026-08-27, + # naming a DIFFERENT missing file each time (`eslint.config.mjs`, then + # `jest.config.ts`), which is what rules out a corrupt cache and points at + # a race. The loser dies with MODULE_NOT_FOUND on dist/index.js before the + # action runs at all, so the secret is never even reached. + # + # Nothing is lost by dropping it: logging in is one command, the docker + # CLI is already in the CI image (ci-requirements.md), and the same + # reasoning as family rule 5 applies — a marketplace action buys nothing + # when the tool is baked into the image the workflow already selected. + # + # Password on stdin, never as an argument: an argument lands in the + # process table and draws docker's own deprecation warning. + - name: Login to Forgejo registry + env: + TOKEN: ${{ secrets.RELEASE_TOKEN }} + ACTOR: ${{ github.actor }} + run: echo "$TOKEN" | docker login git.fabledsword.com -u "$ACTOR" --password-stdin + + # --- reuse-if-published (milestone 313, step 4) ---------------------- + # The identity tag names this artifact's CONTENT — r-, the + # commit its shipped files last changed in, plus the channel for images + # that bake one in. If the registry already carries it, the bytes this + # job would produce are already published and the build is pure waste: + # the channel and date tags get repointed at the existing manifest + # instead, registry-side, in seconds. + # + # This is what stops a push that touched only `agent/` from rebuilding + # web and ml, and a merge to main from rebuilding what dev already built. + # + # The failure direction is deliberate. An inspect that errors for ANY + # reason — network, auth, a registry hiccup — reads as a miss and the + # build runs. Only a genuine 200 skips one, so there is no path here + # that skips a build that was actually needed; the worst case is paying + # for a build we could have avoided. + # + # BASE-IMAGE FRESHNESS, decided rather than left implicit: an artifact + # whose source stops moving stops picking up base-image updates under + # its pinned tag. That is what a pin MEANS — a date tag has to keep + # serving the bytes it served (fabledcurator:2026.7.17 still + # resolves to July's image), or it is not a pin — and family rule + # 145 already says where the refresh goes instead: a rebuild with + # different contents publishes only the MOVING tag, never the immutable + # one. A scheduled channel-only refresh is tracked separately (#3154); + # it does not belong in the push path. + - name: Is this content already published? + id: reuse + env: + IMAGE: git.fabledsword.com/bvandeusen/fabledcurator + CHANNEL: ${{ steps.tag.outputs.channel }} + TAGS: ${{ steps.tag.outputs.tags }} + IS_TAG_PUSH: ${{ startsWith(github.ref, 'refs/tags/') }} + run: | + set -eu + ID=$(sh scripts/artifacts.sh identity web "$CHANNEL") + echo "identity=$ID" >> "$GITHUB_OUTPUT" + + # A tag-push builds a revision that main already published, so it + # must NOT claim the identity: image configs are not bit-reproducible + # (embedded timestamps), so re-pushing r- would point an + # immutable tag at fresh bytes — rule 145's exact prohibition. It + # publishes only its own :v… label and otherwise reuses. + if [ "$IS_TAG_PUSH" = "true" ]; then + echo "build_tags=$TAGS" >> "$GITHUB_OUTPUT" + else + echo "build_tags=$TAGS,$IMAGE:$ID" >> "$GITHUB_OUTPUT" + fi + + if docker buildx imagetools inspect "$IMAGE:$ID" >/dev/null 2>&1; then + echo "hit=true" >> "$GITHUB_OUTPUT" + echo "reuse: $IMAGE:$ID is already published — skipping the build" + else + echo "hit=false" >> "$GITHUB_OUTPUT" + echo "reuse: $IMAGE:$ID is not published — building" + fi - name: Download signed XPI from Forgejo release asset # Fires on every trigger shape. dev and main each bundle the XPI their @@ -381,7 +537,10 @@ jobs: # for up to 10min total) before giving up. Main-push's signing # eventually wins and tag-push picks the release up on a later # iteration. - if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/') + # Gated on the reuse miss as well: if the image is already published it + # already contains its XPI, so this would download (and on a tag-push, + # poll up to 10 minutes for) a file nothing then reads. + if: steps.reuse.outputs.hit != 'true' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || startsWith(github.ref, 'refs/tags/')) env: TOKEN: ${{ secrets.RELEASE_TOKEN }} run: | @@ -441,109 +600,47 @@ jobs: cp "$DEST" "frontend/public/extension/fabledcurator-latest.xpi" ls -la frontend/public/extension/ - - name: Determine tag - id: tag - run: | - # Three trigger shapes: - # refs/tags/v… → tag-push: opt-in milestone label (vYY.MM.DD, - # plus `.N` when the day already carries a tag — - # family rule 148, amended 2026-08-24 after a - # same-day tag was retargeted and a release - # deleted to make room, note 2813). - # Publish ONLY the immutable version tag; - # don't touch :latest (the main-push build - # for the merge commit already did that). - # refs/heads/main → push to main: publish :main + :latest - # (floating) AND :c- (immutable - # per-commit rollback substrate, per family - # release-posture rule "Tags are milestones, - # not gates — commit-SHA images are the - # rollback unit"). Rollback to any commit - # becomes `docker pull …:c-` without a - # release ceremony. - # refs/heads/dev → push to dev: publish :dev, the rolling test - # channel (family rule 146). Rolling means it may - # carry newer contents than the :c- of the - # same commit; it never writes :c- itself, - # because that is the rollback unit (rule 145). - # POSIX-safe substring (the runner shell is dash/BusyBox sh, not - # bash — `${var:0:7}` errors with "Bad substitution"; cut works - # everywhere). Operator-flagged 2026-06-01 after first :c- - # main-push build failed at this step. - SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7) - # The pinnable tag (milestone 313 step 3): YYYY.M.D of the commit - # THIS artifact's shipped files last changed in. Day precision is - # deliberate — same-day work is not something worth pinning, so a - # second main build the same day replaces the first rather than - # accumulating a tag nobody would roll back to. - # - # Derived per artifact, so an image whose files did not change keeps - # the tag it already had: the agent reads 2026.7.17 today while web - # reads 2026.8.27. Step 4 uses that to stop rebuilding it at all. - # `channel` is baked into the image as FC_CHANNEL and reported by - # /api/extension/manifest (milestone 271 step 7). A tag-push counts as - # `main`: a vYY.MM.DD tag is cut from main, so that image is a - # main-channel artifact wearing an immutable name. - if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then - TAG_NAME="${GITHUB_REF#refs/tags/}" - echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:${TAG_NAME}" >> "$GITHUB_OUTPUT" - echo "channel=main" >> "$GITHUB_OUTPUT" - elif [ "${GITHUB_REF##*/}" = "main" ]; then - CALVER=$(sh scripts/artifacts.sh tag web) - # Guarded, and computed only on this path. There is no `set -e` in - # this step, so a failed derivation would otherwise leave CALVER - # empty and publish the tag `fabledcurator:` — an invalid - # name, from a green step. An empty pin must never reach the - # registry. - if [ -z "$CALVER" ]; then - echo "ERROR: could not derive a web version tag" >&2 - exit 1 - fi - echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:main,git.fabledsword.com/bvandeusen/fabledcurator:latest,git.fabledsword.com/bvandeusen/fabledcurator:c-${SHORT_SHA},git.fabledsword.com/bvandeusen/fabledcurator:${CALVER}" >> "$GITHUB_OUTPUT" - echo "channel=main" >> "$GITHUB_OUTPUT" - else - echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:dev" >> "$GITHUB_OUTPUT" - echo "channel=dev" >> "$GITHUB_OUTPUT" - fi - - # A shell step, not docker/login-action@v3, because the action's shared - # cache races itself (#3118). act_runner caches a remote action under one - # /root/.cache/act/ per runner, and build-web, build-ml and - # build-agent all start in the same second and all want this same action. - # One job re-clones the directory — which empties and repopulates it — - # while another is walking it to copy into its container, and the walker - # lstat()s a file that has just vanished. It failed twice on 2026-08-27, - # naming a DIFFERENT missing file each time (`eslint.config.mjs`, then - # `jest.config.ts`), which is what rules out a corrupt cache and points at - # a race. The loser dies with MODULE_NOT_FOUND on dist/index.js before the - # action runs at all, so the secret is never even reached. - # - # Nothing is lost by dropping it: logging in is one command, the docker - # CLI is already in the CI image (ci-requirements.md), and the same - # reasoning as family rule 5 applies — a marketplace action buys nothing - # when the tool is baked into the image the workflow already selected. - # - # Password on stdin, never as an argument: an argument lands in the - # process table and draws docker's own deprecation warning. - - name: Login to Forgejo registry - env: - TOKEN: ${{ secrets.RELEASE_TOKEN }} - ACTOR: ${{ github.actor }} - run: echo "$TOKEN" | docker login git.fabledsword.com -u "$ACTOR" --password-stdin - - name: Build and push web image + if: steps.reuse.outputs.hit != 'true' uses: docker/build-push-action@v5 with: context: . file: Dockerfile push: true - tags: ${{ steps.tag.outputs.tags }} + tags: ${{ steps.reuse.outputs.build_tags }} # Only the web image carries a channel: it is the one that serves # /api/extension/manifest. The ml and agent images have nothing to # report it to. build-args: | FC_CHANNEL=${{ steps.tag.outputs.channel }} + # Registry-side manifest copy: no layer transfer, no local daemon, no + # rebuild. Each -t becomes another reference to the SAME manifest the + # identity tag holds, so :latest and the date pin are byte-identical to + # what was published rather than a lookalike rebuild. + # + # Runs on EVERY reuse, which is what keeps family rule 146 true: a + # rolling channel refreshes itself, so skipping a build must never mean + # leaving :dev or :latest pointing at something older than the commit + # that was just pushed. + - name: Repoint the tags at the published image (reuse) + if: steps.reuse.outputs.hit == 'true' + env: + IMAGE: git.fabledsword.com/bvandeusen/fabledcurator + IDENTITY: ${{ steps.reuse.outputs.identity }} + TAGS: ${{ steps.tag.outputs.tags }} + run: | + set -euf + # steps.tag emits ONE comma-separated list, because that is the shape + # docker/build-push-action takes; imagetools wants a -t per ref. + ARGS="" + IFS=, + for t in $TAGS; do ARGS="$ARGS -t $t"; done + unset IFS + # shellcheck disable=SC2086 + docker buildx imagetools create $ARGS "$IMAGE:$IDENTITY" + echo "repointed to $IMAGE:$IDENTITY: $TAGS" + build-ml: runs-on: python-ci container: @@ -558,11 +655,19 @@ jobs: # the build would otherwise notice. fetch-depth: 0 - # --- shadow mode (milestone 313, step 2) ----------------------------- - # Informational ONLY. Nothing reads this and it must never fail the - # build — no `set -e`, and every derivation falls back to UNAVAILABLE. + # --- derived values, one line (milestone 313) ------------------------ + # These stopped being shadow output at step 3: `tag` is published on + # main and `revision` decides whether the build below runs at all. This + # step prints all three anyway, because the load-bearing steps each + # print only the one they use, and on dev the date tag is computed + # nowhere else. When a build is skipped or a pin looks wrong, this is + # the line that says what the commit derived. # - # What to watch across pushes, because this is what step 3 will trust: + # Still diagnostic, so it still must not fail the build — no `set -e`, + # and every derivation falls back to UNAVAILABLE. A broken echo must + # never be the reason an image does not ship. + # + # What it should say: # * a push touching only agent/ moves the agent and leaves web and ml # STILL. If web moves, its path set is too wide. # * a push touching only docs moves nothing. @@ -570,14 +675,14 @@ jobs: # web bakes in the XPI. If web does not move, its set is too narrow # — the direction that serves stale bytes on a pin. # * dev and main derive the same values for the same source. - - name: Shadow — derived artifact version (informational) + - name: Report the derived artifact version run: | set -u A=ml T=$(sh scripts/artifacts.sh tag "$A" 2>&1 || echo UNAVAILABLE) V=$(sh scripts/artifacts.sh version "$A" 2>&1 || echo UNAVAILABLE) R=$(sh scripts/artifacts.sh revision "$A" 2>&1 || echo UNAVAILABLE) - echo "shadow: artifact=$A tag=$T version=$V revision=$R sha=$GITHUB_SHA" + echo "derived: artifact=$A tag=$T version=$V revision=$R sha=$GITHUB_SHA" - name: Determine tag id: tag @@ -599,10 +704,12 @@ jobs: # # Derived per artifact, so an image whose files did not change keeps # the tag it already had: the agent reads 2026.7.17 today while web - # reads 2026.8.27. Step 4 uses that to stop rebuilding it at all. + # reads 2026.8.27 — and the reuse step below turns that into a + # skipped build rather than a rebuild of bytes that already exist. if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then TAG_NAME="${GITHUB_REF#refs/tags/}" echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:${TAG_NAME}" >> "$GITHUB_OUTPUT" + echo "channel=main" >> "$GITHUB_OUTPUT" elif [ "${GITHUB_REF##*/}" = "main" ]; then CALVER=$(sh scripts/artifacts.sh tag ml) # Guarded, and computed only on this path. There is no `set -e` in @@ -615,8 +722,10 @@ jobs: exit 1 fi echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:main,git.fabledsword.com/bvandeusen/fabledcurator-ml:latest,git.fabledsword.com/bvandeusen/fabledcurator-ml:c-${SHORT_SHA},git.fabledsword.com/bvandeusen/fabledcurator-ml:${CALVER}" >> "$GITHUB_OUTPUT" + echo "channel=main" >> "$GITHUB_OUTPUT" else echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:dev" >> "$GITHUB_OUTPUT" + echo "channel=dev" >> "$GITHUB_OUTPUT" fi # Shell step rather than docker/login-action — see build-web's note on @@ -627,13 +736,98 @@ jobs: ACTOR: ${{ github.actor }} run: echo "$TOKEN" | docker login git.fabledsword.com -u "$ACTOR" --password-stdin + # --- reuse-if-published (milestone 313, step 4) ---------------------- + # The identity tag names this artifact's CONTENT — r-, the + # commit its shipped files last changed in, plus the channel for images + # that bake one in. If the registry already carries it, the bytes this + # job would produce are already published and the build is pure waste: + # the channel and date tags get repointed at the existing manifest + # instead, registry-side, in seconds. + # + # This is what stops a push that touched only `agent/` from rebuilding + # web and ml, and a merge to main from rebuilding what dev already built. + # + # The failure direction is deliberate. An inspect that errors for ANY + # reason — network, auth, a registry hiccup — reads as a miss and the + # build runs. Only a genuine 200 skips one, so there is no path here + # that skips a build that was actually needed; the worst case is paying + # for a build we could have avoided. + # + # BASE-IMAGE FRESHNESS, decided rather than left implicit: an artifact + # whose source stops moving stops picking up base-image updates under + # its pinned tag. That is what a pin MEANS — a date tag has to keep + # serving the bytes it served (fabledcurator-ml:2026.7.17 still + # resolves to July's image), or it is not a pin — and family rule + # 145 already says where the refresh goes instead: a rebuild with + # different contents publishes only the MOVING tag, never the immutable + # one. A scheduled channel-only refresh is tracked separately (#3154); + # it does not belong in the push path. + - name: Is this content already published? + id: reuse + env: + IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-ml + CHANNEL: ${{ steps.tag.outputs.channel }} + TAGS: ${{ steps.tag.outputs.tags }} + IS_TAG_PUSH: ${{ startsWith(github.ref, 'refs/tags/') }} + run: | + set -eu + ID=$(sh scripts/artifacts.sh identity ml "$CHANNEL") + echo "identity=$ID" >> "$GITHUB_OUTPUT" + + # A tag-push builds a revision that main already published, so it + # must NOT claim the identity: image configs are not bit-reproducible + # (embedded timestamps), so re-pushing r- would point an + # immutable tag at fresh bytes — rule 145's exact prohibition. It + # publishes only its own :v… label and otherwise reuses. + if [ "$IS_TAG_PUSH" = "true" ]; then + echo "build_tags=$TAGS" >> "$GITHUB_OUTPUT" + else + echo "build_tags=$TAGS,$IMAGE:$ID" >> "$GITHUB_OUTPUT" + fi + + if docker buildx imagetools inspect "$IMAGE:$ID" >/dev/null 2>&1; then + echo "hit=true" >> "$GITHUB_OUTPUT" + echo "reuse: $IMAGE:$ID is already published — skipping the build" + else + echo "hit=false" >> "$GITHUB_OUTPUT" + echo "reuse: $IMAGE:$ID is not published — building" + fi + - name: Build and push ml image + if: steps.reuse.outputs.hit != 'true' uses: docker/build-push-action@v5 with: context: . file: Dockerfile.ml push: true - tags: ${{ steps.tag.outputs.tags }} + tags: ${{ steps.reuse.outputs.build_tags }} + + # Registry-side manifest copy: no layer transfer, no local daemon, no + # rebuild. Each -t becomes another reference to the SAME manifest the + # identity tag holds, so :latest and the date pin are byte-identical to + # what was published rather than a lookalike rebuild. + # + # Runs on EVERY reuse, which is what keeps family rule 146 true: a + # rolling channel refreshes itself, so skipping a build must never mean + # leaving :dev or :latest pointing at something older than the commit + # that was just pushed. + - name: Repoint the tags at the published image (reuse) + if: steps.reuse.outputs.hit == 'true' + env: + IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-ml + IDENTITY: ${{ steps.reuse.outputs.identity }} + TAGS: ${{ steps.tag.outputs.tags }} + run: | + set -euf + # steps.tag emits ONE comma-separated list, because that is the shape + # docker/build-push-action takes; imagetools wants a -t per ref. + ARGS="" + IFS=, + for t in $TAGS; do ARGS="$ARGS -t $t"; done + unset IFS + # shellcheck disable=SC2086 + docker buildx imagetools create $ARGS "$IMAGE:$IDENTITY" + echo "repointed to $IMAGE:$IDENTITY: $TAGS" # The desktop GPU agent (#114) — published so the operator pulls + runs it on # the GPU machine instead of building locally. Independent of web/ml (its own @@ -652,11 +846,19 @@ jobs: # the build would otherwise notice. fetch-depth: 0 - # --- shadow mode (milestone 313, step 2) ----------------------------- - # Informational ONLY. Nothing reads this and it must never fail the - # build — no `set -e`, and every derivation falls back to UNAVAILABLE. + # --- derived values, one line (milestone 313) ------------------------ + # These stopped being shadow output at step 3: `tag` is published on + # main and `revision` decides whether the build below runs at all. This + # step prints all three anyway, because the load-bearing steps each + # print only the one they use, and on dev the date tag is computed + # nowhere else. When a build is skipped or a pin looks wrong, this is + # the line that says what the commit derived. # - # What to watch across pushes, because this is what step 3 will trust: + # Still diagnostic, so it still must not fail the build — no `set -e`, + # and every derivation falls back to UNAVAILABLE. A broken echo must + # never be the reason an image does not ship. + # + # What it should say: # * a push touching only agent/ moves the agent and leaves web and ml # STILL. If web moves, its path set is too wide. # * a push touching only docs moves nothing. @@ -664,14 +866,14 @@ jobs: # web bakes in the XPI. If web does not move, its set is too narrow # — the direction that serves stale bytes on a pin. # * dev and main derive the same values for the same source. - - name: Shadow — derived artifact version (informational) + - name: Report the derived artifact version run: | set -u A=agent T=$(sh scripts/artifacts.sh tag "$A" 2>&1 || echo UNAVAILABLE) V=$(sh scripts/artifacts.sh version "$A" 2>&1 || echo UNAVAILABLE) R=$(sh scripts/artifacts.sh revision "$A" 2>&1 || echo UNAVAILABLE) - echo "shadow: artifact=$A tag=$T version=$V revision=$R sha=$GITHUB_SHA" + echo "derived: artifact=$A tag=$T version=$V revision=$R sha=$GITHUB_SHA" - name: Determine tag id: tag @@ -685,10 +887,12 @@ jobs: # # Derived per artifact, so an image whose files did not change keeps # the tag it already had: the agent reads 2026.7.17 today while web - # reads 2026.8.27. Step 4 uses that to stop rebuilding it at all. + # reads 2026.8.27 — and the reuse step below turns that into a + # skipped build rather than a rebuild of bytes that already exist. if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then TAG_NAME="${GITHUB_REF#refs/tags/}" echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-agent:${TAG_NAME}" >> "$GITHUB_OUTPUT" + echo "channel=main" >> "$GITHUB_OUTPUT" elif [ "${GITHUB_REF##*/}" = "main" ]; then CALVER=$(sh scripts/artifacts.sh tag agent) # Guarded, and computed only on this path. There is no `set -e` in @@ -701,8 +905,10 @@ jobs: exit 1 fi echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-agent:main,git.fabledsword.com/bvandeusen/fabledcurator-agent:latest,git.fabledsword.com/bvandeusen/fabledcurator-agent:c-${SHORT_SHA},git.fabledsword.com/bvandeusen/fabledcurator-agent:${CALVER}" >> "$GITHUB_OUTPUT" + echo "channel=main" >> "$GITHUB_OUTPUT" else echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-agent:dev" >> "$GITHUB_OUTPUT" + echo "channel=dev" >> "$GITHUB_OUTPUT" fi # Shell step rather than docker/login-action — see build-web's note on @@ -713,10 +919,95 @@ jobs: ACTOR: ${{ github.actor }} run: echo "$TOKEN" | docker login git.fabledsword.com -u "$ACTOR" --password-stdin + # --- reuse-if-published (milestone 313, step 4) ---------------------- + # The identity tag names this artifact's CONTENT — r-, the + # commit its shipped files last changed in, plus the channel for images + # that bake one in. If the registry already carries it, the bytes this + # job would produce are already published and the build is pure waste: + # the channel and date tags get repointed at the existing manifest + # instead, registry-side, in seconds. + # + # This is what stops a push that touched only `agent/` from rebuilding + # web and ml, and a merge to main from rebuilding what dev already built. + # + # The failure direction is deliberate. An inspect that errors for ANY + # reason — network, auth, a registry hiccup — reads as a miss and the + # build runs. Only a genuine 200 skips one, so there is no path here + # that skips a build that was actually needed; the worst case is paying + # for a build we could have avoided. + # + # BASE-IMAGE FRESHNESS, decided rather than left implicit: an artifact + # whose source stops moving stops picking up base-image updates under + # its pinned tag. That is what a pin MEANS — a date tag has to keep + # serving the bytes it served (fabledcurator-agent:2026.7.17 still + # resolves to July's image), or it is not a pin — and family rule + # 145 already says where the refresh goes instead: a rebuild with + # different contents publishes only the MOVING tag, never the immutable + # one. A scheduled channel-only refresh is tracked separately (#3154); + # it does not belong in the push path. + - name: Is this content already published? + id: reuse + env: + IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-agent + CHANNEL: ${{ steps.tag.outputs.channel }} + TAGS: ${{ steps.tag.outputs.tags }} + IS_TAG_PUSH: ${{ startsWith(github.ref, 'refs/tags/') }} + run: | + set -eu + ID=$(sh scripts/artifacts.sh identity agent "$CHANNEL") + echo "identity=$ID" >> "$GITHUB_OUTPUT" + + # A tag-push builds a revision that main already published, so it + # must NOT claim the identity: image configs are not bit-reproducible + # (embedded timestamps), so re-pushing r- would point an + # immutable tag at fresh bytes — rule 145's exact prohibition. It + # publishes only its own :v… label and otherwise reuses. + if [ "$IS_TAG_PUSH" = "true" ]; then + echo "build_tags=$TAGS" >> "$GITHUB_OUTPUT" + else + echo "build_tags=$TAGS,$IMAGE:$ID" >> "$GITHUB_OUTPUT" + fi + + if docker buildx imagetools inspect "$IMAGE:$ID" >/dev/null 2>&1; then + echo "hit=true" >> "$GITHUB_OUTPUT" + echo "reuse: $IMAGE:$ID is already published — skipping the build" + else + echo "hit=false" >> "$GITHUB_OUTPUT" + echo "reuse: $IMAGE:$ID is not published — building" + fi + - name: Build and push agent image + if: steps.reuse.outputs.hit != 'true' uses: docker/build-push-action@v5 with: context: agent file: agent/Dockerfile push: true - tags: ${{ steps.tag.outputs.tags }} + tags: ${{ steps.reuse.outputs.build_tags }} + + # Registry-side manifest copy: no layer transfer, no local daemon, no + # rebuild. Each -t becomes another reference to the SAME manifest the + # identity tag holds, so :latest and the date pin are byte-identical to + # what was published rather than a lookalike rebuild. + # + # Runs on EVERY reuse, which is what keeps family rule 146 true: a + # rolling channel refreshes itself, so skipping a build must never mean + # leaving :dev or :latest pointing at something older than the commit + # that was just pushed. + - name: Repoint the tags at the published image (reuse) + if: steps.reuse.outputs.hit == 'true' + env: + IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-agent + IDENTITY: ${{ steps.reuse.outputs.identity }} + TAGS: ${{ steps.tag.outputs.tags }} + run: | + set -euf + # steps.tag emits ONE comma-separated list, because that is the shape + # docker/build-push-action takes; imagetools wants a -t per ref. + ARGS="" + IFS=, + for t in $TAGS; do ARGS="$ARGS -t $t"; done + unset IFS + # shellcheck disable=SC2086 + docker buildx imagetools create $ARGS "$IMAGE:$IDENTITY" + echo "repointed to $IMAGE:$IDENTITY: $TAGS" diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 17a6c9f..e7f71c8 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -115,6 +115,13 @@ jobs: SECRET_KEY: ci_unit_test_placeholder steps: - uses: actions/checkout@v4 + with: + # Full history for tests/test_artifact_identity.py, which derives + # each artifact's revision to check the identity scheme. On a + # depth-1 clone that derivation either fails or returns the tip sha + # — so the lane would go green while asserting nothing, which is + # the one outcome worse than a red one. + fetch-depth: 0 # Cache step removed 2026-05-26: act_runner's cache backend has been # broken on this homelab runner since 2026-05-15 (first as request- diff --git a/scripts/artifacts.sh b/scripts/artifacts.sh index 948706a..a1ea0d5 100755 --- a/scripts/artifacts.sh +++ b/scripts/artifacts.sh @@ -56,8 +56,25 @@ ML_PATHS='Dockerfile.ml requirements-ml.txt requirements.txt backend alembic ale # it: this is deliberately NOT `agent/`. AGENT_PATHS='agent/Dockerfile agent/requirements.txt agent/fc_agent' +# Which artifacts bake the BUILD CHANNEL into the image, and therefore cannot +# share a content identity across channels. The web image takes FC_CHANNEL as +# a build-arg and reports it from /api/extension/manifest (milestone 271 step +# 7), so `main` and `dev` builds of one revision are genuinely different +# images — reusing the dev one on main would ship an instance that names +# itself `dev` forever. +# +# ml and agent take no build-args at all: one revision, one image, and a merge +# to main can reuse exactly what dev already built. That is not a detail, it is +# most of what step 4 saves — merges would otherwise rebuild the agent's CUDA +# image to produce bytes that already exist. +# +# Extend this list if a second artifact ever gains a build-arg; +# tests/test_artifact_identity.py reads the Dockerfiles and fails if it drifts. +CHANNELLED='web' + usage() { echo "usage: artifacts.sh {paths|revision|version|tag} {web|ml|agent|extension}" >&2 + echo " artifacts.sh identity {web|ml|agent} [channel]" >&2 exit 2 } @@ -140,11 +157,50 @@ cmd_tag() { "$(strip0 "$(fmt "$sha" %d)")" } +# The CONTENT IDENTITY of a published image: an immutable tag naming exactly +# what a build of this commit would produce. build.yml asks the registry for it +# and, on a hit, skips the build entirely and repoints the channel and date +# tags at the manifest that is already there (milestone 313 step 4). +# +# It is deliberately NOT either of the other two values: +# * the date tag is day-precise and last-one-wins, so two different builds +# share it — it cannot answer "is this content published?". +# * the commit sha moves on every push, so it would never hit, which is the +# redundant rebuild this exists to remove. +# +# The revision does both jobs: it is content-unique AND stable across pushes +# that did not touch the artifact. +cmd_identity() { + _art=$1 + _chan=${2:-} + case "$_art" in + web|ml|agent) ;; + extension) + echo "artifacts.sh: the extension is cached as an ext- Forgejo release, not an image tag — use \`version\`" >&2 + exit 2 ;; + *) usage ;; + esac + for _c in $CHANNELLED; do + if [ "$_art" = "$_c" ]; then + # Refused rather than defaulted: an unqualified identity for a + # channelled artifact would let a dev image be reused as the main one. + if [ -z "$_chan" ]; then + echo "artifacts.sh: $_art bakes the channel into the image — identity needs one" >&2 + exit 2 + fi + printf 'r-%s-%s\n' "$(cmd_revision "$_art")" "$_chan" + return + fi + done + printf 'r-%s\n' "$(cmd_revision "$_art")" +} + [ $# -ge 2 ] || usage case "$1" in paths) cmd_paths "$2" ;; revision) cmd_revision "$2" ;; version) cmd_version "$2" ;; tag) cmd_tag "$2" ;; + identity) cmd_identity "$2" "${3:-}" ;; *) usage ;; esac diff --git a/tests/test_artifact_identity.py b/tests/test_artifact_identity.py new file mode 100644 index 0000000..b65ff17 --- /dev/null +++ b/tests/test_artifact_identity.py @@ -0,0 +1,140 @@ +"""`artifacts.sh identity` is what decides whether a build gets skipped. + +Milestone 313 step 4: build.yml asks the registry for `:` and, +on a hit, publishes NO new bytes — it repoints the channel and date tags at the +manifest already there. So the identity has to be a true name for the content. +Both ways of getting it wrong are silent at build time and only surface in +production: + +* **too coarse** — two genuinely different images share an identity, so the + second one never gets built and its tags point at the first one's bytes. The + live case is FC_CHANNEL: a `dev` and a `main` build of one revision differ, + and collapsing them ships an instance that reports the wrong channel forever. +* **too fine** — the identity moves when the content did not, nothing ever + hits, and step 4 buys nothing. A commit sha would do exactly this. + +The Dockerfiles are read here rather than trusted, because the coarse direction +appears the moment someone adds a build-arg without touching `CHANNELLED`. +""" +from __future__ import annotations + +import re +import subprocess +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent + +# Only image artifacts have an identity — the extension is cached as an +# ext- Forgejo release, not a registry tag. +IMAGE_ARTIFACTS = { + "web": "Dockerfile", + "ml": "Dockerfile.ml", + "agent": "agent/Dockerfile", +} + +CHANNELS = ("main", "dev") + +# docker's own tag grammar: [A-Za-z0-9_][A-Za-z0-9._-]{0,127} +_TAG = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9._-]{0,127}$") + +# `ARG FC_CHANNEL` in a Dockerfile means build.yml passes a per-channel value +# in, so the channel is part of what the image IS. +_ARG_CHANNEL = re.compile(r"^\s*ARG\s+FC_CHANNEL\b", re.MULTILINE) + + +def identity(artifact: str, channel: str | None = None) -> subprocess.CompletedProcess: + cmd = ["sh", str(ROOT / "scripts" / "artifacts.sh"), "identity", artifact] + if channel is not None: + cmd.append(channel) + return subprocess.run(cmd, capture_output=True, text=True, cwd=ROOT) + + +def ok(artifact: str, channel: str | None = None) -> str: + proc = identity(artifact, channel) + assert proc.returncode == 0, f"identity {artifact} {channel}: {proc.stderr}" + return proc.stdout.strip() + + +def bakes_the_channel(artifact: str) -> bool: + return bool(_ARG_CHANNEL.search((ROOT / IMAGE_ARTIFACTS[artifact]).read_text())) + + +@pytest.mark.parametrize("artifact", sorted(IMAGE_ARTIFACTS)) +def test_channel_dependence_matches_the_dockerfile(artifact): + """The coarse direction, caught at its source. + + Whether the channel belongs in the identity is not a preference — it is + dictated by whether the Dockerfile takes it as a build-arg. Adding an + `ARG FC_CHANNEL` to another image without adding it to `CHANNELLED` would + make its dev and main builds collide, and nothing else would notice. + """ + per_channel = {c: ok(artifact, c) for c in CHANNELS} + differs = len(set(per_channel.values())) > 1 + + if bakes_the_channel(artifact): + assert differs, ( + f"{IMAGE_ARTIFACTS[artifact]} declares ARG FC_CHANNEL, so a dev " + f"build and a main build of one revision are different images — " + f"but both derive the identity {per_channel['main']!r}. The main " + f"build would reuse the dev image and report the wrong channel. " + f"Add {artifact!r} to CHANNELLED in scripts/artifacts.sh." + ) + else: + assert not differs, ( + f"{IMAGE_ARTIFACTS[artifact]} takes no channel build-arg, so one " + f"revision is one image and a merge to main should reuse what dev " + f"already built — but the identity differs per channel " + f"({per_channel}), so every merge rebuilds it for nothing. Remove " + f"{artifact!r} from CHANNELLED in scripts/artifacts.sh." + ) + + +@pytest.mark.parametrize("artifact", sorted(IMAGE_ARTIFACTS)) +def test_identity_tracks_the_artifacts_own_revision(artifact): + """The fine direction: the identity must be the revision, not the push. + + `revision` is the commit this artifact's shipped files last changed in, so + it holds still across pushes that did not touch it. Anything derived from + HEAD instead would move every push and never hit the registry. + """ + rev = subprocess.run( + ["sh", str(ROOT / "scripts" / "artifacts.sh"), "revision", artifact], + capture_output=True, text=True, check=True, cwd=ROOT, + ).stdout.strip() + value = ok(artifact, "main") + assert rev and rev in value, ( + f"identity {value!r} does not contain the {artifact} revision {rev!r}" + ) + + +@pytest.mark.parametrize("artifact", sorted(IMAGE_ARTIFACTS)) +def test_identity_is_a_legal_docker_tag(artifact): + """It is pushed as a tag, so an illegal one fails at the registry — after + the build has already run.""" + for channel in CHANNELS: + value = ok(artifact, channel) + assert _TAG.match(value), f"{value!r} is not a valid docker tag" + + +def test_a_channelled_artifact_refuses_an_unqualified_identity(): + """Refusing beats defaulting. If `identity web` quietly returned the + unqualified `r-`, a workflow that forgot to pass the channel would + publish one image under a name both channels then reuse — the exact + collision the CHANNELLED list exists to prevent, reintroduced by an + omission rather than by an edit.""" + proc = identity("web") + assert proc.returncode != 0, ( + "identity web returned a value with no channel: " + f"{proc.stdout.strip()!r}" + ) + + +def test_the_extension_has_no_image_identity(): + """It is cached as an ext- release asset, and its cache key is the + version. Answering with a plausible image tag would invite a second, + divergent cache.""" + proc = identity("extension", "main") + assert proc.returncode != 0 + assert "ext-" in proc.stderr