diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 7bd8066..c6bd756 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -638,6 +638,14 @@ jobs: # adds no variability the reuse check would have to account for. echo "version=$(sh scripts/artifacts.sh version web)" >> "$GITHUB_OUTPUT" + # The build clock, pinned to the same commit (#3265). Without it + # buildkit stamps the image config with the wall clock of the build, + # so identical layers republish under a new config blob and the + # channel tag gets a new manifest digest for no reason. Derived from + # `newest()` like revision and version, so all three name one commit + # and cannot drift apart. + echo "epoch=$(sh scripts/artifacts.sh epoch web)" >> "$GITHUB_OUTPUT" + # The moving tag for this channel. Which tag we ask IS the channel — # that is why the revision needs no -main/-dev qualifier any more. if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi @@ -764,6 +772,12 @@ jobs: - name: Build and push web image if: steps.reuse.outputs.hit != 'true' + # Read by buildx out of the ENVIRONMENT, not passed as a build-arg — + # it normalises the image config's `created` field and the history + # timestamps rather than being consumed by the Dockerfile. See #3265 + # and the reuse step's `epoch` output. + env: + SOURCE_DATE_EPOCH: ${{ steps.reuse.outputs.epoch }} uses: docker/build-push-action@v5 with: context: . @@ -776,20 +790,17 @@ jobs: # invalidates, and the image genuinely rebuilds. # # MEASURED on the first real fire, run 4934 (#3265): when the base - # did NOT move, the build is ~13s and every content step reports - # CACHED — but the channel tag STILL gets a new manifest digest. - # buildkit mints a fresh image config each run, so identical layers - # are republished under a new config blob. All three images moved - # that way on 2026-08-30 with nothing whatsoever changed in them. + # did NOT move, the build was ~13s with every content step CACHED — + # and the channel tag STILL got a new manifest digest, because + # buildkit stamps a fresh image config per run and republishes the + # identical layers under it. All three images moved that way on + # 2026-08-30 with nothing whatsoever changed in them. # - # So a refresh currently rewrites :latest every Sunday whether or - # not there is anything new in it, and :c- is handed a new - # manifest to diverge from on the same cadence. Layers are shared, - # so the storage cost is a config blob; the cost that matters is - # that a digest change no longer MEANS anything. Tracked in #3265 — - # the likely fix is a deterministic SOURCE_DATE_EPOCH, which would - # make "same source, same bytes" true and turn the no-op case into - # a genuine no-op. + # SOURCE_DATE_EPOCH (below) is the fix: pinned to the commit the + # content came from, the config is byte-identical across runs, so + # the manifest digest is too and the push is a registry no-op. A + # digest change means the content changed again, which is the only + # thing a digest is any use for. # # What `pull` does NOT catch either: a Debian package update inside # the `apt-get install` layer while the base tag itself stands @@ -1097,6 +1108,14 @@ jobs: DERIVED=$(sh scripts/artifacts.sh revision ml) echo "revision=$DERIVED" >> "$GITHUB_OUTPUT" + # The build clock, pinned to the same commit (#3265). Without it + # buildkit stamps the image config with the wall clock of the build, + # so identical layers republish under a new config blob and the + # channel tag gets a new manifest digest for no reason. Derived from + # `newest()` like revision and version, so all three name one commit + # and cannot drift apart. + echo "epoch=$(sh scripts/artifacts.sh epoch ml)" >> "$GITHUB_OUTPUT" + # The moving tag for this channel. Which tag we ask IS the channel — # that is why the revision needs no -main/-dev qualifier any more. if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi @@ -1147,6 +1166,12 @@ jobs: - name: Build and push ml image if: steps.reuse.outputs.hit != 'true' + # Read by buildx out of the ENVIRONMENT, not passed as a build-arg — + # it normalises the image config's `created` field and the history + # timestamps rather than being consumed by the Dockerfile. See #3265 + # and the reuse step's `epoch` output. + env: + SOURCE_DATE_EPOCH: ${{ steps.reuse.outputs.epoch }} uses: docker/build-push-action@v5 with: context: . @@ -1159,20 +1184,17 @@ jobs: # invalidates, and the image genuinely rebuilds. # # MEASURED on the first real fire, run 4934 (#3265): when the base - # did NOT move, the build is ~13s and every content step reports - # CACHED — but the channel tag STILL gets a new manifest digest. - # buildkit mints a fresh image config each run, so identical layers - # are republished under a new config blob. All three images moved - # that way on 2026-08-30 with nothing whatsoever changed in them. + # did NOT move, the build was ~13s with every content step CACHED — + # and the channel tag STILL got a new manifest digest, because + # buildkit stamps a fresh image config per run and republishes the + # identical layers under it. All three images moved that way on + # 2026-08-30 with nothing whatsoever changed in them. # - # So a refresh currently rewrites :latest every Sunday whether or - # not there is anything new in it, and :c- is handed a new - # manifest to diverge from on the same cadence. Layers are shared, - # so the storage cost is a config blob; the cost that matters is - # that a digest change no longer MEANS anything. Tracked in #3265 — - # the likely fix is a deterministic SOURCE_DATE_EPOCH, which would - # make "same source, same bytes" true and turn the no-op case into - # a genuine no-op. + # SOURCE_DATE_EPOCH (below) is the fix: pinned to the commit the + # content came from, the config is byte-identical across runs, so + # the manifest digest is too and the push is a registry no-op. A + # digest change means the content changed again, which is the only + # thing a digest is any use for. # # What `pull` does NOT catch either: a Debian package update inside # the `apt-get install` layer while the base tag itself stands @@ -1466,6 +1488,14 @@ jobs: DERIVED=$(sh scripts/artifacts.sh revision agent) echo "revision=$DERIVED" >> "$GITHUB_OUTPUT" + # The build clock, pinned to the same commit (#3265). Without it + # buildkit stamps the image config with the wall clock of the build, + # so identical layers republish under a new config blob and the + # channel tag gets a new manifest digest for no reason. Derived from + # `newest()` like revision and version, so all three name one commit + # and cannot drift apart. + echo "epoch=$(sh scripts/artifacts.sh epoch agent)" >> "$GITHUB_OUTPUT" + # The moving tag for this channel. Which tag we ask IS the channel — # that is why the revision needs no -main/-dev qualifier any more. if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi @@ -1516,6 +1546,12 @@ jobs: - name: Build and push agent image if: steps.reuse.outputs.hit != 'true' + # Read by buildx out of the ENVIRONMENT, not passed as a build-arg — + # it normalises the image config's `created` field and the history + # timestamps rather than being consumed by the Dockerfile. See #3265 + # and the reuse step's `epoch` output. + env: + SOURCE_DATE_EPOCH: ${{ steps.reuse.outputs.epoch }} uses: docker/build-push-action@v5 with: context: agent @@ -1528,20 +1564,17 @@ jobs: # invalidates, and the image genuinely rebuilds. # # MEASURED on the first real fire, run 4934 (#3265): when the base - # did NOT move, the build is ~13s and every content step reports - # CACHED — but the channel tag STILL gets a new manifest digest. - # buildkit mints a fresh image config each run, so identical layers - # are republished under a new config blob. All three images moved - # that way on 2026-08-30 with nothing whatsoever changed in them. + # did NOT move, the build was ~13s with every content step CACHED — + # and the channel tag STILL got a new manifest digest, because + # buildkit stamps a fresh image config per run and republishes the + # identical layers under it. All three images moved that way on + # 2026-08-30 with nothing whatsoever changed in them. # - # So a refresh currently rewrites :latest every Sunday whether or - # not there is anything new in it, and :c- is handed a new - # manifest to diverge from on the same cadence. Layers are shared, - # so the storage cost is a config blob; the cost that matters is - # that a digest change no longer MEANS anything. Tracked in #3265 — - # the likely fix is a deterministic SOURCE_DATE_EPOCH, which would - # make "same source, same bytes" true and turn the no-op case into - # a genuine no-op. + # SOURCE_DATE_EPOCH (below) is the fix: pinned to the commit the + # content came from, the config is byte-identical across runs, so + # the manifest digest is too and the push is a registry no-op. A + # digest change means the content changed again, which is the only + # thing a digest is any use for. # # What `pull` does NOT catch either: a Debian package update inside # the `apt-get install` layer while the base tag itself stands diff --git a/ci-requirements.md b/ci-requirements.md index ccd3081..8b05574 100644 --- a/ci-requirements.md +++ b/ci-requirements.md @@ -198,14 +198,21 @@ per `docs/process.md`'s "add deps to the image when used by >1 project". refresh from being undone. - **`pull: true` on the scheduled path only** is the mechanism: a moved base tag changes the `FROM` layer's cache key and everything above it rebuilds. - **It does not currently make the unmoved case free.** Measured on the first - real fire (run 4934, 2026-08-30): every content step reported `CACHED` and - the bases resolved to unchanged digests, yet all three `:latest` tags got a - NEW manifest digest, because buildkit mints a fresh image config per run and - republishes identical layers under it. So `:latest` is rewritten weekly - whether or not anything changed, and `:c-` is handed a new manifest to - diverge from on the same cadence — a digest change stops meaning anything. - Tracked as #3265; the likely fix is a deterministic `SOURCE_DATE_EPOCH`. + It did not always make the unmoved case free. Measured on the first real + fire (run 4934, 2026-08-30): every content step reported `CACHED` and the + bases resolved to unchanged digests, yet all three `:latest` tags got a NEW + manifest digest, because buildkit stamps a fresh image config per run and + republishes identical layers under it — so `:latest` was rewritten weekly + whether or not anything changed, and a digest change stopped meaning + anything (#3265). + - **`SOURCE_DATE_EPOCH` is what makes it free.** Set on each build step from + `artifacts.sh epoch ` — the unix timestamp of the same commit + `revision` and `version` name, so all three are views of one `newest()` + lookup and cannot drift into disagreeing. With the config's `created` field + and history timestamps pinned to the content rather than to the wall clock, + identical source produces an identical manifest digest and the push is a + registry no-op. That restores the property the whole scheme rests on: a + channel tag's digest changes when, and only when, its content does. Separately not caught: a Debian package update inside the `apt-get install` layer while the base tag stands still — a lag rather than a hole, since the official python/cuda images rebuild with those updates baked in. diff --git a/scripts/artifacts.sh b/scripts/artifacts.sh index 5f86a79..e93b9eb 100755 --- a/scripts/artifacts.sh +++ b/scripts/artifacts.sh @@ -90,7 +90,7 @@ DERIVER='scripts/artifacts.sh' usage() { - echo "usage: artifacts.sh {paths|revision|version} {web|ml|agent|extension}" >&2 + echo "usage: artifacts.sh {paths|revision|version|epoch} {web|ml|agent|extension}" >&2 exit 2 } @@ -149,6 +149,26 @@ cmd_revision() { echo "$(newest "$1")" | cut -d' ' -f2 | cut -c1-12 } +# The BUILD CLOCK: the same commit's unix timestamp, for SOURCE_DATE_EPOCH. +# +# buildkit stamps the image config's `created` field and every history entry +# with the wall clock of the build unless this is set, so two builds of +# identical source produce different config blobs and therefore different +# manifest digests. That is #3265: the weekly refresh republished all three +# `:latest` tags on 2026-08-30 with every content step CACHED and the bases +# resolved to unchanged digests — nothing was different, and the digest moved +# anyway. A digest that changes on a calendar cannot also mean "the content +# changed", which is the only thing anyone wants it for. +# +# It is the same commit `revision` and `version` name — deliberately, and this +# is the point of routing it through `newest()` rather than taking git's word +# separately. Three values derived from three lookups can disagree; three +# views of one lookup cannot. Note #3127 §2 is the record of what a second +# clock costs. +cmd_epoch() { + echo "$(newest "$1")" | cut -d' ' -f1 +} + # The VERSION: `YYYY.MM.DD.HHMM`, zero-padded, UTC. One shape across the whole # family (note #3127 §1, rule 148) — the number an instance reports about # itself, and, with a `v` in front, the release tag naming the same build. @@ -197,5 +217,6 @@ case "$1" in paths) cmd_paths "$2" ;; revision) cmd_revision "$2" ;; version) cmd_version "$2" ;; + epoch) cmd_epoch "$2" ;; *) usage ;; esac diff --git a/tests/test_artifact_identity.py b/tests/test_artifact_identity.py index fe614bf..a0beab3 100644 --- a/tests/test_artifact_identity.py +++ b/tests/test_artifact_identity.py @@ -234,3 +234,56 @@ def test_version_and_revision_describe_the_same_commit(artifact): f"in AMO_UNPADDED may differ here." ) assert sha.startswith(revision(artifact)) + + +@pytest.mark.parametrize("artifact", ARTIFACTS) +def test_epoch_is_the_same_commit_the_version_names(artifact): + """The build clock and the version must be one lookup, not two. + + `epoch` feeds SOURCE_DATE_EPOCH, which decides the image config's bytes and + therefore the manifest digest; `version` is what the instance reports about + itself. If they could name different commits, an image would be stamped + reproducibly against one commit while claiming to be another — and both + values would look perfectly well-formed, exactly like the divergence the + test above guards. + + They cannot, because `cmd_epoch` and `cmd_version` are two fields of one + `newest()` result. This pins that they stay that way: a future refactor + that gave epoch its own `git log` would pass every other test here. + """ + epoch = artifacts("epoch", artifact).strip() + assert epoch.isdigit(), f"{artifact} epoch is {epoch!r}, not a unix timestamp" + + sha = newest_by_commit_time(artifact) + committed = subprocess.run( + ["git", "show", "-s", "--format=%ct", sha], + capture_output=True, text=True, check=True, cwd=ROOT, + ).stdout.strip() + assert epoch == committed, ( + f"{artifact} derives epoch {epoch}, but its newest shipped commit " + f"{sha[:12]} was committed at {committed}. SOURCE_DATE_EPOCH would " + f"pin the image config to a commit the version does not name." + ) + + # And the two renderings must agree, which is the property that actually + # matters at build time: same commit in, same digest and same reported + # version out. + rendered = subprocess.run( + ["git", "show", "-s", "--format=%cd", "--date=format-local:%Y.%m.%d.%H%M", sha], + capture_output=True, text=True, check=True, cwd=ROOT, + env={"TZ": "UTC", "PATH": os.environ.get("PATH", "")}, + ).stdout.strip() + assert segments(artifacts("version", artifact).strip()) == segments(rendered) + + +def test_epoch_is_stable_across_calls(): + """SOURCE_DATE_EPOCH's entire job is to be the same on the next build. + + A value that moved between two invocations on one unchanged checkout would + reintroduce #3265 through the very mechanism meant to close it, and the + symptom would be indistinguishable: a digest that changes for no reason. + """ + for artifact in ARTIFACTS: + first = artifacts("epoch", artifact).strip() + second = artifacts("epoch", artifact).strip() + assert first == second, f"{artifact} epoch moved: {first} then {second}"