diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 42a9a72..eb4299b 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -25,6 +25,26 @@ on: # Releases still happen (rule 148, on explicit request per rule 2). They # produce a changelog, not an image. + # The escape hatch for the one thing skip-if-exists makes untestable: a + # build that WOULD be skipped. `agent/` has not changed since 2026-07-17, so + # every push since has correctly declined to build it — which also means the + # agent build path has not run in six weeks and cannot be exercised on + # demand. #3190 lives on exactly that path. + # + # Editing build.yml does not force one either, and that is deliberate: the + # workflow is not shipped bytes, so it is in no artifact's path set. Putting + # it in one would re-version every artifact for a comment change. + # + # ONE input, not one per artifact. Forcing all three is cheap once the + # registry cache is warm (#3114), and three booleans is an interface nobody + # remembers the meaning of. + workflow_dispatch: + inputs: + force_build: + description: 'Rebuild every image even if the published revision matches' + type: boolean + default: false + # Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes: # - write:package, read:package (for docker push to git.fabledsword.com) # - write:release (for ext- release asset cache) @@ -443,6 +463,23 @@ jobs: ACTOR: ${{ github.actor }} run: echo "$TOKEN" | docker login git.fabledsword.com -u "$ACTOR" --password-stdin + # A REAL buildx builder, not the default `docker` driver (#3114, #3190). + # + # The default driver builds through the local dockerd. It cannot export a + # registry cache at all — which is why the agent rebuilds a ~6.3 GB CUDA + # + torch image from scratch whenever the runner's local cache is cold, + # measured at 9m26s against 7s warm. It is also #3190's leading suspect: + # after a registry-direct push it resolves image metadata against a local + # store the push never filled, and reports `No such image` on an image + # that published perfectly well three seconds earlier. + # + # These jobs run INSIDE a container against a mounted docker socket, so + # the buildkit container this starts is a SIBLING of the job container, + # not a child. That works over the socket mount; it had never been tried + # here before milestone 326 step 1. + - name: Set up buildx + uses: docker/setup-buildx-action@v3 + # --- reuse-if-published (milestone 313, step 4) ---------------------- # Does the image the channel tag already points at carry THIS commit's # revision? If so the bytes this job would produce are already published @@ -481,6 +518,12 @@ jobs: env: IMAGE: git.fabledsword.com/bvandeusen/fabledcurator CHANNEL: ${{ steps.tag.outputs.channel }} + # Empty on a push; the string "true" only from a workflow_dispatch + # that asked for it. `github.event.inputs` rather than the `inputs` + # context — release.yml already uses that form, and it is the one + # this runner is known to evaluate. Read through env rather than + # interpolated into the run block, same rule as release.yml's TAG. + FORCE: ${{ github.event.inputs.force_build }} run: | set -eu DERIVED=$(sh scripts/artifacts.sh revision web) @@ -520,7 +563,14 @@ jobs: echo "reuse: NOTE tag is being index-wrapped and reuse is dead." fi - if [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then + # FORCE is checked here rather than in the build step's `if:`, so + # that one decision drives everything downstream. The repoint step + # keys off `hit` too, and a force that bypassed only the build would + # leave the two disagreeing about what just happened. + if [ "${FORCE:-false}" = "true" ]; then + echo "hit=false" >> "$GITHUB_OUTPUT" + echo "reuse: force_build set — building regardless" + elif [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then echo "hit=true" >> "$GITHUB_OUTPUT" echo "reuse: already published — skipping the build" else @@ -623,6 +673,40 @@ jobs: # decoration — an unstamped image is one that will always rebuild. labels: | fc.revision=${{ steps.reuse.outputs.revision }} + # LOAD-BEARING, not a preference. On the default docker driver these + # were no-ops; on the docker-container driver above, + # build-push-action@v5 defaults provenance to TRUE when pushing. + # Provenance attaches an attestation manifest, which makes the pushed + # tag a manifest INDEX — and `.Image.Config.Labels` does not resolve + # through an index. + # + # The label directly above IS the reuse key. Wrap the channel tag in + # an index and the next push reads fc.revision=, misses, and + # rebuilds. Then so does the one after that, forever. Nothing fails, + # nothing goes red, and the only symptom is the bill. That is #3183 + # arriving through a different door, and note #3127 §4 records the + # same shape for `platforms:`. + provenance: false + sbom: false + # The ONLY cache this driver can have. `docker-container` gets a + # FRESH buildkit instance per job, so unlike the default docker + # driver it has no local layer store to fall back on — measured on + # run 4896, the first builds after the driver change: web 3m44s + # (was 2m23s), ml 3m49s (was 3m20s), agent 11m12s (was 9m26s). The + # driver change ALONE is a regression; this is the other half of it. + # + # mode=max so intermediate stages cache too. web's frontend-builder + # stage and the agent's two ~150s pip layers are the whole cost, and + # they are exactly what a min-mode cache would drop. + # + # A `:buildcache` tag is NOT the withdrawn tag scheme coming back. + # Rule 145 narrowed against names NOTHING reads; this one is read by + # every build that runs, is one moving ref per image rather than one + # per build, holds cache blobs rather than a shippable artifact, and + # is overwritten in place rather than accumulating. It is closer to + # :dev than to the :2026.8.28 tags milestone 318 deleted. (#3114.) + cache-from: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator:buildcache + cache-to: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator:buildcache,mode=max # Only the web image carries these: it is the one with a UI and an # HTTP surface to report them on. The ml and agent images have # nothing to tell. @@ -776,6 +860,23 @@ jobs: ACTOR: ${{ github.actor }} run: echo "$TOKEN" | docker login git.fabledsword.com -u "$ACTOR" --password-stdin + # A REAL buildx builder, not the default `docker` driver (#3114, #3190). + # + # The default driver builds through the local dockerd. It cannot export a + # registry cache at all — which is why the agent rebuilds a ~6.3 GB CUDA + # + torch image from scratch whenever the runner's local cache is cold, + # measured at 9m26s against 7s warm. It is also #3190's leading suspect: + # after a registry-direct push it resolves image metadata against a local + # store the push never filled, and reports `No such image` on an image + # that published perfectly well three seconds earlier. + # + # These jobs run INSIDE a container against a mounted docker socket, so + # the buildkit container this starts is a SIBLING of the job container, + # not a child. That works over the socket mount; it had never been tried + # here before milestone 326 step 1. + - name: Set up buildx + uses: docker/setup-buildx-action@v3 + # --- reuse-if-published (milestone 313, step 4) ---------------------- # Does the image the channel tag already points at carry THIS commit's # revision? If so the bytes this job would produce are already published @@ -814,6 +915,12 @@ jobs: env: IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-ml CHANNEL: ${{ steps.tag.outputs.channel }} + # Empty on a push; the string "true" only from a workflow_dispatch + # that asked for it. `github.event.inputs` rather than the `inputs` + # context — release.yml already uses that form, and it is the one + # this runner is known to evaluate. Read through env rather than + # interpolated into the run block, same rule as release.yml's TAG. + FORCE: ${{ github.event.inputs.force_build }} run: | set -eu DERIVED=$(sh scripts/artifacts.sh revision ml) @@ -849,7 +956,14 @@ jobs: echo "reuse: NOTE tag is being index-wrapped and reuse is dead." fi - if [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then + # FORCE is checked here rather than in the build step's `if:`, so + # that one decision drives everything downstream. The repoint step + # keys off `hit` too, and a force that bypassed only the build would + # leave the two disagreeing about what just happened. + if [ "${FORCE:-false}" = "true" ]; then + echo "hit=false" >> "$GITHUB_OUTPUT" + echo "reuse: force_build set — building regardless" + elif [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then echo "hit=true" >> "$GITHUB_OUTPUT" echo "reuse: already published — skipping the build" else @@ -876,6 +990,40 @@ jobs: # decoration — an unstamped image is one that will always rebuild. labels: | fc.revision=${{ steps.reuse.outputs.revision }} + # LOAD-BEARING, not a preference. On the default docker driver these + # were no-ops; on the docker-container driver above, + # build-push-action@v5 defaults provenance to TRUE when pushing. + # Provenance attaches an attestation manifest, which makes the pushed + # tag a manifest INDEX — and `.Image.Config.Labels` does not resolve + # through an index. + # + # The label directly above IS the reuse key. Wrap the channel tag in + # an index and the next push reads fc.revision=, misses, and + # rebuilds. Then so does the one after that, forever. Nothing fails, + # nothing goes red, and the only symptom is the bill. That is #3183 + # arriving through a different door, and note #3127 §4 records the + # same shape for `platforms:`. + provenance: false + sbom: false + # The ONLY cache this driver can have. `docker-container` gets a + # FRESH buildkit instance per job, so unlike the default docker + # driver it has no local layer store to fall back on — measured on + # run 4896, the first builds after the driver change: web 3m44s + # (was 2m23s), ml 3m49s (was 3m20s), agent 11m12s (was 9m26s). The + # driver change ALONE is a regression; this is the other half of it. + # + # mode=max so intermediate stages cache too. web's frontend-builder + # stage and the agent's two ~150s pip layers are the whole cost, and + # they are exactly what a min-mode cache would drop. + # + # A `:buildcache` tag is NOT the withdrawn tag scheme coming back. + # Rule 145 narrowed against names NOTHING reads; this one is read by + # every build that runs, is one moving ref per image rather than one + # per build, holds cache blobs rather than a shippable artifact, and + # is overwritten in place rather than accumulating. It is closer to + # :dev than to the :2026.8.28 tags milestone 318 deleted. (#3114.) + cache-from: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator-ml:buildcache + cache-to: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator-ml:buildcache,mode=max # Every tag but the channel's own is written HERE, registry-side, # whether or not a build ran. Each -t becomes another reference to the @@ -1021,6 +1169,23 @@ jobs: ACTOR: ${{ github.actor }} run: echo "$TOKEN" | docker login git.fabledsword.com -u "$ACTOR" --password-stdin + # A REAL buildx builder, not the default `docker` driver (#3114, #3190). + # + # The default driver builds through the local dockerd. It cannot export a + # registry cache at all — which is why the agent rebuilds a ~6.3 GB CUDA + # + torch image from scratch whenever the runner's local cache is cold, + # measured at 9m26s against 7s warm. It is also #3190's leading suspect: + # after a registry-direct push it resolves image metadata against a local + # store the push never filled, and reports `No such image` on an image + # that published perfectly well three seconds earlier. + # + # These jobs run INSIDE a container against a mounted docker socket, so + # the buildkit container this starts is a SIBLING of the job container, + # not a child. That works over the socket mount; it had never been tried + # here before milestone 326 step 1. + - name: Set up buildx + uses: docker/setup-buildx-action@v3 + # --- reuse-if-published (milestone 313, step 4) ---------------------- # Does the image the channel tag already points at carry THIS commit's # revision? If so the bytes this job would produce are already published @@ -1059,6 +1224,12 @@ jobs: env: IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-agent CHANNEL: ${{ steps.tag.outputs.channel }} + # Empty on a push; the string "true" only from a workflow_dispatch + # that asked for it. `github.event.inputs` rather than the `inputs` + # context — release.yml already uses that form, and it is the one + # this runner is known to evaluate. Read through env rather than + # interpolated into the run block, same rule as release.yml's TAG. + FORCE: ${{ github.event.inputs.force_build }} run: | set -eu DERIVED=$(sh scripts/artifacts.sh revision agent) @@ -1094,7 +1265,14 @@ jobs: echo "reuse: NOTE tag is being index-wrapped and reuse is dead." fi - if [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then + # FORCE is checked here rather than in the build step's `if:`, so + # that one decision drives everything downstream. The repoint step + # keys off `hit` too, and a force that bypassed only the build would + # leave the two disagreeing about what just happened. + if [ "${FORCE:-false}" = "true" ]; then + echo "hit=false" >> "$GITHUB_OUTPUT" + echo "reuse: force_build set — building regardless" + elif [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then echo "hit=true" >> "$GITHUB_OUTPUT" echo "reuse: already published — skipping the build" else @@ -1121,6 +1299,40 @@ jobs: # decoration — an unstamped image is one that will always rebuild. labels: | fc.revision=${{ steps.reuse.outputs.revision }} + # LOAD-BEARING, not a preference. On the default docker driver these + # were no-ops; on the docker-container driver above, + # build-push-action@v5 defaults provenance to TRUE when pushing. + # Provenance attaches an attestation manifest, which makes the pushed + # tag a manifest INDEX — and `.Image.Config.Labels` does not resolve + # through an index. + # + # The label directly above IS the reuse key. Wrap the channel tag in + # an index and the next push reads fc.revision=, misses, and + # rebuilds. Then so does the one after that, forever. Nothing fails, + # nothing goes red, and the only symptom is the bill. That is #3183 + # arriving through a different door, and note #3127 §4 records the + # same shape for `platforms:`. + provenance: false + sbom: false + # The ONLY cache this driver can have. `docker-container` gets a + # FRESH buildkit instance per job, so unlike the default docker + # driver it has no local layer store to fall back on — measured on + # run 4896, the first builds after the driver change: web 3m44s + # (was 2m23s), ml 3m49s (was 3m20s), agent 11m12s (was 9m26s). The + # driver change ALONE is a regression; this is the other half of it. + # + # mode=max so intermediate stages cache too. web's frontend-builder + # stage and the agent's two ~150s pip layers are the whole cost, and + # they are exactly what a min-mode cache would drop. + # + # A `:buildcache` tag is NOT the withdrawn tag scheme coming back. + # Rule 145 narrowed against names NOTHING reads; this one is read by + # every build that runs, is one moving ref per image rather than one + # per build, holds cache blobs rather than a shippable artifact, and + # is overwritten in place rather than accumulating. It is closer to + # :dev than to the :2026.8.28 tags milestone 318 deleted. (#3114.) + cache-from: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator-agent:buildcache + cache-to: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator-agent:buildcache,mode=max # Every tag but the channel's own is written HERE, registry-side, # whether or not a build ran. Each -t becomes another reference to the diff --git a/ci-requirements.md b/ci-requirements.md index 2744b40..340e277 100644 --- a/ci-requirements.md +++ b/ci-requirements.md @@ -133,6 +133,40 @@ per `docs/process.md`'s "add deps to the image when used by >1 project". already ran, so both paths now share one proven route. The cost: `:c-` is an index rather than a plain image, so `fc.revision` does not resolve through it — nothing reads it there, and the index names the same manifest. +- **The image builds run on a `docker-container` buildx builder, and + `provenance`/`sbom` are explicitly OFF** (milestone 326 step 1). The builder + is what makes a registry layer cache possible at all — the default `docker` + driver cannot export one (#3114) — and it is #3190's leading suspect, since + it is the driver that resolves image metadata against a local store a + registry-direct push never fills. **The attestation flags are load-bearing, + not tidiness:** on the container driver `build-push-action@v5` defaults + `provenance` to true when pushing, an attestation manifest makes the pushed + tag a manifest INDEX, and config labels do not resolve through an index — so + leaving them on would make every push read `fc.revision=`, miss, and + rebuild forever with every lane green. Same failure as #3183, different door. + These jobs run inside a container against a mounted docker socket, so the + buildkit container is a sibling rather than a child. +- **All three images import and export a registry layer cache** + (`:buildcache`, `mode=max`). This is not an optimisation bolted onto + the driver change — it is the other half of it. The `docker-container` + driver gets a fresh buildkit instance per job and therefore has **no local + layer store at all**, where the old `docker` driver at least reused whatever + the runner's dockerd happened to hold. Measured on run 4896, the first builds + after the driver moved: web 3m44s (was 2m23s), ml 3m49s (was 3m20s), agent + 11m12s (was 9m26s) — every one slower. A `:buildcache` tag is read by every + build that runs, is one moving ref per image, holds cache blobs rather than a + shippable artifact, and is overwritten in place, so it is not a return of the + per-version tags milestone 318 withdrew (#3114). +- **`build.yml` accepts a `workflow_dispatch` with `force_build`**, which + bypasses the reuse check for all three images. It exists because + skip-if-exists makes its own build path untestable: `agent/` has not changed + since 2026-07-17, so the agent build has not run in six weeks and cannot be + exercised on demand — and #3190 lives on exactly that path. Editing + `build.yml` does not force a build either, deliberately: the workflow is not + shipped bytes and is in no artifact's path set. The flag is read through + `github.event.inputs` into an env var rather than interpolated into a run + block, and it is checked inside the reuse step so that one decision drives + both the build and the repoint. - **`FC_CHANNEL` and `FC_VERSION` are build args, not runtime settings.** `build.yml` passes them to the web image only — the ml and agent images have nothing to report them to. `/api/health` returns both, the foot of Settings