diff --git a/scripts/artifacts.sh b/scripts/artifacts.sh new file mode 100755 index 0000000..948706a --- /dev/null +++ b/scripts/artifacts.sh @@ -0,0 +1,150 @@ +#!/bin/sh +# Single definition of WHAT EACH PUBLISHED ARTIFACT IS BUILT FROM, and the +# version derived from it. Milestone 313; generalises the shape +# extension/scripts/packaging.sh established for the extension alone. +# +# Four artifacts, four independent versions. An artifact whose shipped files +# did not change keeps its version and does not rebuild — that is the whole +# point, and it is why each path set must match its Dockerfile rather than +# being a plausible guess. Getting a set wrong is quiet in BOTH directions: +# +# too narrow -> a pin serves stale bytes, because the version did not move +# when the content did. This is the dangerous one. +# too wide -> the artifact re-versions and rebuilds for a change it does +# not ship. Merely wasteful. +# +# tests/test_artifact_paths.py asserts every COPY source in each Dockerfile is +# covered here, so adding a COPY without updating this file fails CI. +# +# POSIX sh only — CI's run shell is busybox on some paths. +# +# -f (no pathname expansion) is load-bearing for the whole script: the lists +# below are iterated with deliberate word-splitting, and without it the shell +# would glob `frontend/test/**` against the working tree and silently narrow +# the pattern. Callers substituting the output need their own `set -f` too; +# the two guards protect different expansions. +set -euf + +ROOT=$(git rev-parse --show-toplevel) + +# --- what each artifact ships ------------------------------------------------ +# +# Each set includes its own Dockerfile and requirements: changing a base image +# or a pin changes the artifact just as surely as changing a source file. +# +# web (Dockerfile, context `.`) — the runtime stage copies backend/, alembic/, +# alembic.ini, entrypoint.sh and requirements.txt; the frontend-builder stage +# copies frontend/ and the runtime takes its `dist` output. +# +# frontend/test is excluded: `npm run build` is vite, which builds from src/, +# index.html and public/ and never reads test/. It lands in the builder layer +# but not in `dist`, so it cannot reach the shipped image. +# +# The web image ALSO bundles the signed XPI (build.yml downloads it into +# frontend/public/extension/ before the docker build), so an extension change +# changes the web image. The extension's packaged set is appended in cmd_paths +# rather than restated — one definition, per #2397. +WEB_PATHS='Dockerfile requirements.txt backend alembic alembic.ini entrypoint.sh frontend :(exclude)frontend/test :(exclude)frontend/test/**' + +# ml (Dockerfile.ml, context `.`) — no frontend, no extension. Note it copies +# BOTH requirements-ml.txt and requirements.txt. +ML_PATHS='Dockerfile.ml requirements-ml.txt requirements.txt backend alembic alembic.ini entrypoint.sh' + +# agent (agent/Dockerfile, context `agent`) — copies requirements.txt and +# fc_agent only. agent/README.md, agent/docker-compose.yml and agent/ruff.toml +# live in the directory but never reach the image, so they must not re-version +# it: this is deliberately NOT `agent/`. +AGENT_PATHS='agent/Dockerfile agent/requirements.txt agent/fc_agent' + +usage() { + echo "usage: artifacts.sh {paths|revision|version|tag} {web|ml|agent|extension}" >&2 + exit 2 +} + +# The extension's packaged set, read from its own definition rather than +# copied. packaging.sh emits `:(exclude)extension/...` entries, so the bare +# `extension` include has to come with them. +ext_paths() { + echo "extension $(sh "$ROOT/extension/scripts/packaging.sh" pathspec)" +} + +cmd_paths() { + case "$1" in + web) echo "$WEB_PATHS $(ext_paths)" ;; + ml) echo "$ML_PATHS" ;; + agent) echo "$AGENT_PATHS" ;; + extension) ext_paths ;; + *) usage ;; + esac +} + +# " " of the newest commit touching this artifact's shipped set. +# Unquoted on purpose: the pathspec must word-split into separate args. +# Globbing is already off script-wide. +newest() { + # shellcheck disable=SC2046 + set -- "$(cd "$ROOT" && git log --format='%ct %H' HEAD -- $(cmd_paths "$1") \ + | sort -n | tail -1)" + if [ -z "$1" ]; then + echo "artifacts.sh: no commit touches this artifact's shipped files" >&2 + exit 1 + fi + echo "$1" +} + +# Formatted through git rather than date(1): busybox date does not reliably +# accept `-d @`, and git's own --date=format-local is available wherever +# git is. TZ=UTC so the value does not depend on the runner's timezone. +fmt() { + (cd "$ROOT" && TZ=UTC git show -s --format=%cd --date="format-local:$2" "$1") +} + +# Leading zeros stripped so every segment is a plain integer — some version +# validators reject `08`, and a leading zero buys nothing. `0000` (midnight) +# must survive as `0`, not as the empty string. +strip0() { + printf '%s' "$1" | sed -e 's/^0*//' -e 's/^$/0/' +} + +# The IDENTITY of an artifact's content: the commit its shipped files last +# changed in. This — not the tag — is what decides whether a build can be +# skipped, because the published tag is only day-precise and two different +# builds can share it. +cmd_revision() { + echo "$(newest "$1")" | cut -d' ' -f2 | cut -c1-12 +} + +# The ORDERING KEY: full precision, YYYY.M.D.HHMM. Used by the extension, +# where the value is what Firefox compares to decide whether an update exists +# — two same-day builds MUST be distinguishable or the second never reaches +# anyone. +cmd_version() { + sha=$(echo "$(newest "$1")" | cut -d' ' -f2) + printf '%s.%s.%s.%s\n' \ + "$(fmt "$sha" %Y)" \ + "$(strip0 "$(fmt "$sha" %m)")" \ + "$(strip0 "$(fmt "$sha" %d)")" \ + "$(strip0 "$(fmt "$sha" %H%M)")" +} + +# The PUBLISHED IMAGE TAG: day precision, YYYY.M.D. Deliberately coarser than +# the ordering key, per the operator 2026-08-28 — same-day work is not +# something worth pinning, so a second build the same day replaces the first +# rather than accumulating a tag nobody would roll back to. Safe only because +# skip decisions key on cmd_revision, never on this. +cmd_tag() { + sha=$(echo "$(newest "$1")" | cut -d' ' -f2) + printf '%s.%s.%s\n' \ + "$(fmt "$sha" %Y)" \ + "$(strip0 "$(fmt "$sha" %m)")" \ + "$(strip0 "$(fmt "$sha" %d)")" +} + +[ $# -ge 2 ] || usage +case "$1" in + paths) cmd_paths "$2" ;; + revision) cmd_revision "$2" ;; + version) cmd_version "$2" ;; + tag) cmd_tag "$2" ;; + *) usage ;; +esac diff --git a/tests/test_artifact_paths.py b/tests/test_artifact_paths.py new file mode 100644 index 0000000..c226f5b --- /dev/null +++ b/tests/test_artifact_paths.py @@ -0,0 +1,138 @@ +"""`scripts/artifacts.sh` path sets must match what the Dockerfiles copy. + +Each published artifact's version derives from the newest commit touching its +own shipped file set (milestone 313). The whole scheme rests on those sets +being right, and both ways of being wrong are silent: + +* **too narrow** — a file ships but is not in the set, so the version does not + move when the content does, and a pin serves stale bytes. This is the + dangerous direction and the one this module exists for. +* **too wide** — a file is in the set but never reaches the image, so the + artifact re-versions and rebuilds for a change it does not ship. + +Nothing else notices either. The version still derives, CI still goes green, +and the mismatch only surfaces as "I pinned that build and got the wrong +bytes". So the Dockerfiles are read here and compared against the declaration. +""" +from __future__ import annotations + +import re +import subprocess +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent + +# artifact -> (dockerfile, build context relative to the repo root) +ARTIFACTS = { + "web": ("Dockerfile", ""), + "ml": ("Dockerfile.ml", ""), + "agent": ("agent/Dockerfile", "agent"), +} + +# COPY --from= copies from an earlier build stage, not from the build +# context, so its source is not a repo path and cannot be in a path set. +_COPY = re.compile(r"^\s*COPY\s+(?!--from=)(?P.+)$", re.MULTILINE) + + +def declared_paths(artifact: str) -> list[str]: + out = subprocess.run( + ["sh", str(ROOT / "scripts" / "artifacts.sh"), "paths", artifact], + capture_output=True, text=True, check=True, cwd=ROOT, + ).stdout + return out.split() + + +def includes(artifact: str) -> list[str]: + """The set minus its `:(exclude)…` entries.""" + return [p for p in declared_paths(artifact) if not p.startswith(":(exclude)")] + + +def copy_sources(dockerfile: str, context: str) -> list[str]: + """Repo-relative sources of every context COPY in a Dockerfile.""" + text = (ROOT / dockerfile).read_text() + sources: list[str] = [] + for m in _COPY.finditer(text): + args = m.group("args").split() + # Last arg is the destination; everything before it is a source. + for src in args[:-1]: + # `frontend/package-lock.json*` — the glob is an optional-file + # idiom; the directory it sits in is what matters for coverage. + src = src.rstrip("*") + sources.append(f"{context}/{src}" if context else src) + return sources + + +def covered_by(path: str, include: str) -> bool: + """`path` ships if an include names it or one of its ancestors.""" + path = path.rstrip("/").lstrip("./") + include = include.rstrip("/") + return path == include or path.startswith(include + "/") + + +@pytest.mark.parametrize("artifact", sorted(ARTIFACTS)) +def test_every_copied_path_is_in_the_artifacts_path_set(artifact): + """The too-narrow direction — the one that serves stale bytes on a pin.""" + dockerfile, context = ARTIFACTS[artifact] + inc = includes(artifact) + for src in copy_sources(dockerfile, context): + assert any(covered_by(src, i) for i in inc), ( + f"{dockerfile} copies {src!r} into the {artifact} image, but no " + f"include in scripts/artifacts.sh covers it. The {artifact} " + f"version will not move when that file changes, so a pinned build " + f"will serve stale bytes. Add it to the path set.\n" + f" declared includes: {inc}" + ) + + +@pytest.mark.parametrize("artifact", sorted(ARTIFACTS)) +def test_the_dockerfile_itself_is_in_the_path_set(artifact): + """Changing a base image or a RUN changes the artifact as surely as + changing a source file, so each set must include its own Dockerfile.""" + dockerfile, _ = ARTIFACTS[artifact] + assert any(covered_by(dockerfile, i) for i in includes(artifact)), ( + f"{dockerfile} is not in the {artifact} path set — a base-image bump " + f"would not move the version." + ) + + +def test_the_web_image_versions_on_an_extension_change(): + """The web image bundles the signed XPI, so the extension's packaged files + are part of what it ships. Miss this and `:latest` serves a NEW extension + under an unchanged web version — a pin that quietly disagrees with itself. + """ + inc = includes("web") + assert any(covered_by("extension/background/background.js", i) for i in inc), ( + "the web path set does not cover the extension's packaged files, but " + "build.yml downloads the signed XPI into frontend/public/extension/ " + "before the docker build" + ) + + +@pytest.mark.parametrize( + "artifact, path", + [ + # Deliberate exclusions — the too-wide direction. Each of these lives + # beside shipped code but never reaches an image, and including it + # would re-version the artifact for a change it does not carry. + ("agent", "agent/README.md"), + ("agent", "agent/ruff.toml"), + ("agent", "agent/docker-compose.yml"), + # vite builds from src/, index.html and public/; it never reads test/, + # so a frontend test change cannot reach `dist`. + ("web", "frontend/test/gallery.spec.js"), + ], +) +def test_files_that_never_reach_an_image_do_not_version_it(artifact, path): + paths = declared_paths(artifact) + excluded = [p[len(":(exclude)"):] for p in paths if p.startswith(":(exclude)")] + inc = [p for p in paths if not p.startswith(":(exclude)")] + + included = any(covered_by(path, i) for i in inc) + exempted = any(covered_by(path, e.rstrip("*").rstrip("/")) for e in excluded) + assert not included or exempted, ( + f"{path} is in the {artifact} path set but is not copied into the " + f"image — it would re-version and rebuild {artifact} for a change it " + f"does not ship." + )