.forgejo/workflows/build.yml

@@ -638,6 +638,14 @@ jobs:
           # adds no variability the reuse check would have to account for.
           echo "version=$(sh scripts/artifacts.sh version web)" >> "$GITHUB_OUTPUT"
 
+          # The build clock, pinned to the same commit (#3265). Without it
+          # buildkit stamps the image config with the wall clock of the build,
+          # so identical layers republish under a new config blob and the
+          # channel tag gets a new manifest digest for no reason. Derived from
+          # `newest()` like revision and version, so all three name one commit
+          # and cannot drift apart.
+          echo "epoch=$(sh scripts/artifacts.sh epoch web)" >> "$GITHUB_OUTPUT"
+
           # The moving tag for this channel. Which tag we ask IS the channel —
           # that is why the revision needs no -main/-dev qualifier any more.
           if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi
@@ -764,6 +772,12 @@ jobs:
 
       - name: Build and push web image
         if: steps.reuse.outputs.hit != 'true'
+        # Read by buildx out of the ENVIRONMENT, not passed as a build-arg —
+        # it normalises the image config's `created` field and the history
+        # timestamps rather than being consumed by the Dockerfile. See #3265
+        # and the reuse step's `epoch` output.
+        env:
+          SOURCE_DATE_EPOCH: ${{ steps.reuse.outputs.epoch }}
         uses: docker/build-push-action@v5
         with:
           context: .
@@ -776,20 +790,17 @@ jobs:
           # invalidates, and the image genuinely rebuilds.
           #
           # MEASURED on the first real fire, run 4934 (#3265): when the base
-          # did NOT move, the build is ~13s and every content step reports
-          # CACHED — but the channel tag STILL gets a new manifest digest.
-          # buildkit mints a fresh image config each run, so identical layers
-          # are republished under a new config blob. All three images moved
-          # that way on 2026-08-30 with nothing whatsoever changed in them.
+          # did NOT move, the build was ~13s with every content step CACHED —
+          # and the channel tag STILL got a new manifest digest, because
+          # buildkit stamps a fresh image config per run and republishes the
+          # identical layers under it. All three images moved that way on
+          # 2026-08-30 with nothing whatsoever changed in them.
           #
-          # So a refresh currently rewrites :latest every Sunday whether or
-          # not there is anything new in it, and :c-<sha> is handed a new
-          # manifest to diverge from on the same cadence. Layers are shared,
-          # so the storage cost is a config blob; the cost that matters is
-          # that a digest change no longer MEANS anything. Tracked in #3265 —
-          # the likely fix is a deterministic SOURCE_DATE_EPOCH, which would
-          # make "same source, same bytes" true and turn the no-op case into
-          # a genuine no-op.
+          # SOURCE_DATE_EPOCH (below) is the fix: pinned to the commit the
+          # content came from, the config is byte-identical across runs, so
+          # the manifest digest is too and the push is a registry no-op. A
+          # digest change means the content changed again, which is the only
+          # thing a digest is any use for.
           #
           # What `pull` does NOT catch either: a Debian package update inside
           # the `apt-get install` layer while the base tag itself stands
@@ -1097,6 +1108,14 @@ jobs:
           DERIVED=$(sh scripts/artifacts.sh revision ml)
           echo "revision=$DERIVED" >> "$GITHUB_OUTPUT"
 
+          # The build clock, pinned to the same commit (#3265). Without it
+          # buildkit stamps the image config with the wall clock of the build,
+          # so identical layers republish under a new config blob and the
+          # channel tag gets a new manifest digest for no reason. Derived from
+          # `newest()` like revision and version, so all three name one commit
+          # and cannot drift apart.
+          echo "epoch=$(sh scripts/artifacts.sh epoch ml)" >> "$GITHUB_OUTPUT"
+
           # The moving tag for this channel. Which tag we ask IS the channel —
           # that is why the revision needs no -main/-dev qualifier any more.
           if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi
@@ -1147,6 +1166,12 @@ jobs:
 
       - name: Build and push ml image
         if: steps.reuse.outputs.hit != 'true'
+        # Read by buildx out of the ENVIRONMENT, not passed as a build-arg —
+        # it normalises the image config's `created` field and the history
+        # timestamps rather than being consumed by the Dockerfile. See #3265
+        # and the reuse step's `epoch` output.
+        env:
+          SOURCE_DATE_EPOCH: ${{ steps.reuse.outputs.epoch }}
         uses: docker/build-push-action@v5
         with:
           context: .
@@ -1159,20 +1184,17 @@ jobs:
           # invalidates, and the image genuinely rebuilds.
           #
           # MEASURED on the first real fire, run 4934 (#3265): when the base
-          # did NOT move, the build is ~13s and every content step reports
-          # CACHED — but the channel tag STILL gets a new manifest digest.
-          # buildkit mints a fresh image config each run, so identical layers
-          # are republished under a new config blob. All three images moved
-          # that way on 2026-08-30 with nothing whatsoever changed in them.
+          # did NOT move, the build was ~13s with every content step CACHED —
+          # and the channel tag STILL got a new manifest digest, because
+          # buildkit stamps a fresh image config per run and republishes the
+          # identical layers under it. All three images moved that way on
+          # 2026-08-30 with nothing whatsoever changed in them.
           #
-          # So a refresh currently rewrites :latest every Sunday whether or
-          # not there is anything new in it, and :c-<sha> is handed a new
-          # manifest to diverge from on the same cadence. Layers are shared,
-          # so the storage cost is a config blob; the cost that matters is
-          # that a digest change no longer MEANS anything. Tracked in #3265 —
-          # the likely fix is a deterministic SOURCE_DATE_EPOCH, which would
-          # make "same source, same bytes" true and turn the no-op case into
-          # a genuine no-op.
+          # SOURCE_DATE_EPOCH (below) is the fix: pinned to the commit the
+          # content came from, the config is byte-identical across runs, so
+          # the manifest digest is too and the push is a registry no-op. A
+          # digest change means the content changed again, which is the only
+          # thing a digest is any use for.
           #
           # What `pull` does NOT catch either: a Debian package update inside
           # the `apt-get install` layer while the base tag itself stands
@@ -1466,6 +1488,14 @@ jobs:
           DERIVED=$(sh scripts/artifacts.sh revision agent)
           echo "revision=$DERIVED" >> "$GITHUB_OUTPUT"
 
+          # The build clock, pinned to the same commit (#3265). Without it
+          # buildkit stamps the image config with the wall clock of the build,
+          # so identical layers republish under a new config blob and the
+          # channel tag gets a new manifest digest for no reason. Derived from
+          # `newest()` like revision and version, so all three name one commit
+          # and cannot drift apart.
+          echo "epoch=$(sh scripts/artifacts.sh epoch agent)" >> "$GITHUB_OUTPUT"
+
           # The moving tag for this channel. Which tag we ask IS the channel —
           # that is why the revision needs no -main/-dev qualifier any more.
           if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi
@@ -1516,6 +1546,12 @@ jobs:
 
       - name: Build and push agent image
         if: steps.reuse.outputs.hit != 'true'
+        # Read by buildx out of the ENVIRONMENT, not passed as a build-arg —
+        # it normalises the image config's `created` field and the history
+        # timestamps rather than being consumed by the Dockerfile. See #3265
+        # and the reuse step's `epoch` output.
+        env:
+          SOURCE_DATE_EPOCH: ${{ steps.reuse.outputs.epoch }}
         uses: docker/build-push-action@v5
         with:
           context: agent
@@ -1528,20 +1564,17 @@ jobs:
           # invalidates, and the image genuinely rebuilds.
           #
           # MEASURED on the first real fire, run 4934 (#3265): when the base
-          # did NOT move, the build is ~13s and every content step reports
-          # CACHED — but the channel tag STILL gets a new manifest digest.
-          # buildkit mints a fresh image config each run, so identical layers
-          # are republished under a new config blob. All three images moved
-          # that way on 2026-08-30 with nothing whatsoever changed in them.
+          # did NOT move, the build was ~13s with every content step CACHED —
+          # and the channel tag STILL got a new manifest digest, because
+          # buildkit stamps a fresh image config per run and republishes the
+          # identical layers under it. All three images moved that way on
+          # 2026-08-30 with nothing whatsoever changed in them.
           #
-          # So a refresh currently rewrites :latest every Sunday whether or
-          # not there is anything new in it, and :c-<sha> is handed a new
-          # manifest to diverge from on the same cadence. Layers are shared,
-          # so the storage cost is a config blob; the cost that matters is
-          # that a digest change no longer MEANS anything. Tracked in #3265 —
-          # the likely fix is a deterministic SOURCE_DATE_EPOCH, which would
-          # make "same source, same bytes" true and turn the no-op case into
-          # a genuine no-op.
+          # SOURCE_DATE_EPOCH (below) is the fix: pinned to the commit the
+          # content came from, the config is byte-identical across runs, so
+          # the manifest digest is too and the push is a registry no-op. A
+          # digest change means the content changed again, which is the only
+          # thing a digest is any use for.
           #
           # What `pull` does NOT catch either: a Debian package update inside
           # the `apt-get install` layer while the base tag itself stands

ci-requirements.md

@@ -198,14 +198,21 @@ per `docs/process.md`'s "add deps to the image when used by >1 project".
     refresh from being undone.
   - **`pull: true` on the scheduled path only** is the mechanism: a moved base
     tag changes the `FROM` layer's cache key and everything above it rebuilds.
-    **It does not currently make the unmoved case free.** Measured on the first
-    real fire (run 4934, 2026-08-30): every content step reported `CACHED` and
-    the bases resolved to unchanged digests, yet all three `:latest` tags got a
-    NEW manifest digest, because buildkit mints a fresh image config per run and
-    republishes identical layers under it. So `:latest` is rewritten weekly
-    whether or not anything changed, and `:c-<sha>` is handed a new manifest to
-    diverge from on the same cadence — a digest change stops meaning anything.
-    Tracked as #3265; the likely fix is a deterministic `SOURCE_DATE_EPOCH`.
+    It did not always make the unmoved case free. Measured on the first real
+    fire (run 4934, 2026-08-30): every content step reported `CACHED` and the
+    bases resolved to unchanged digests, yet all three `:latest` tags got a NEW
+    manifest digest, because buildkit stamps a fresh image config per run and
+    republishes identical layers under it — so `:latest` was rewritten weekly
+    whether or not anything changed, and a digest change stopped meaning
+    anything (#3265).
+  - **`SOURCE_DATE_EPOCH` is what makes it free.** Set on each build step from
+    `artifacts.sh epoch <artifact>` — the unix timestamp of the same commit
+    `revision` and `version` name, so all three are views of one `newest()`
+    lookup and cannot drift into disagreeing. With the config's `created` field
+    and history timestamps pinned to the content rather than to the wall clock,
+    identical source produces an identical manifest digest and the push is a
+    registry no-op. That restores the property the whole scheme rests on: a
+    channel tag's digest changes when, and only when, its content does.
     Separately not caught: a Debian package update inside the `apt-get install`
     layer while the base tag stands still — a lag rather than a hole, since the
     official python/cuda images rebuild with those updates baked in.

scripts/artifacts.sh

@@ -90,7 +90,7 @@ DERIVER='scripts/artifacts.sh'
 
 
 usage() {
-  echo "usage: artifacts.sh {paths|revision|version} {web|ml|agent|extension}" >&2
+  echo "usage: artifacts.sh {paths|revision|version|epoch} {web|ml|agent|extension}" >&2
   exit 2
 }
 
@@ -149,6 +149,26 @@ cmd_revision() {
   echo "$(newest "$1")" | cut -d' ' -f2 | cut -c1-12
 }
 
+# The BUILD CLOCK: the same commit's unix timestamp, for SOURCE_DATE_EPOCH.
+#
+# buildkit stamps the image config's `created` field and every history entry
+# with the wall clock of the build unless this is set, so two builds of
+# identical source produce different config blobs and therefore different
+# manifest digests. That is #3265: the weekly refresh republished all three
+# `:latest` tags on 2026-08-30 with every content step CACHED and the bases
+# resolved to unchanged digests — nothing was different, and the digest moved
+# anyway. A digest that changes on a calendar cannot also mean "the content
+# changed", which is the only thing anyone wants it for.
+#
+# It is the same commit `revision` and `version` name — deliberately, and this
+# is the point of routing it through `newest()` rather than taking git's word
+# separately. Three values derived from three lookups can disagree; three
+# views of one lookup cannot. Note #3127 §2 is the record of what a second
+# clock costs.
+cmd_epoch() {
+  echo "$(newest "$1")" | cut -d' ' -f1
+}
+
 # The VERSION: `YYYY.MM.DD.HHMM`, zero-padded, UTC. One shape across the whole
 # family (note #3127 §1, rule 148) — the number an instance reports about
 # itself, and, with a `v` in front, the release tag naming the same build.
@@ -197,5 +217,6 @@ case "$1" in
   paths)    cmd_paths "$2" ;;
   revision) cmd_revision "$2" ;;
   version)  cmd_version "$2" ;;
+  epoch)    cmd_epoch "$2" ;;
   *)        usage ;;
 esac

tests/test_artifact_identity.py

@@ -234,3 +234,56 @@ def test_version_and_revision_describe_the_same_commit(artifact):
             f"in AMO_UNPADDED may differ here."
         )
     assert sha.startswith(revision(artifact))
+
+
+@pytest.mark.parametrize("artifact", ARTIFACTS)
+def test_epoch_is_the_same_commit_the_version_names(artifact):
+    """The build clock and the version must be one lookup, not two.
+
+    `epoch` feeds SOURCE_DATE_EPOCH, which decides the image config's bytes and
+    therefore the manifest digest; `version` is what the instance reports about
+    itself. If they could name different commits, an image would be stamped
+    reproducibly against one commit while claiming to be another — and both
+    values would look perfectly well-formed, exactly like the divergence the
+    test above guards.
+
+    They cannot, because `cmd_epoch` and `cmd_version` are two fields of one
+    `newest()` result. This pins that they stay that way: a future refactor
+    that gave epoch its own `git log` would pass every other test here.
+    """
+    epoch = artifacts("epoch", artifact).strip()
+    assert epoch.isdigit(), f"{artifact} epoch is {epoch!r}, not a unix timestamp"
+
+    sha = newest_by_commit_time(artifact)
+    committed = subprocess.run(
+        ["git", "show", "-s", "--format=%ct", sha],
+        capture_output=True, text=True, check=True, cwd=ROOT,
+    ).stdout.strip()
+    assert epoch == committed, (
+        f"{artifact} derives epoch {epoch}, but its newest shipped commit "
+        f"{sha[:12]} was committed at {committed}. SOURCE_DATE_EPOCH would "
+        f"pin the image config to a commit the version does not name."
+    )
+
+    # And the two renderings must agree, which is the property that actually
+    # matters at build time: same commit in, same digest and same reported
+    # version out.
+    rendered = subprocess.run(
+        ["git", "show", "-s", "--format=%cd", "--date=format-local:%Y.%m.%d.%H%M", sha],
+        capture_output=True, text=True, check=True, cwd=ROOT,
+        env={"TZ": "UTC", "PATH": os.environ.get("PATH", "")},
+    ).stdout.strip()
+    assert segments(artifacts("version", artifact).strip()) == segments(rendered)
+
+
+def test_epoch_is_stable_across_calls():
+    """SOURCE_DATE_EPOCH's entire job is to be the same on the next build.
+
+    A value that moved between two invocations on one unchanged checkout would
+    reintroduce #3265 through the very mechanism meant to close it, and the
+    symptom would be indistinguishable: a digest that changes for no reason.
+    """
+    for artifact in ARTIFACTS:
+        first = artifacts("epoch", artifact).strip()
+        second = artifacts("epoch", artifact).strip()
+        assert first == second, f"{artifact} epoch moved: {first} then {second}"