Merge pull request 'A gate for the weekly refresh, and the lever that makes it testable' (#248) from dev into main

.forgejo/workflows/build.yml

@@ -89,12 +89,30 @@ on:
 # makes the milestone-362 gate verifiable at all: a gate has to be watched
 # rejecting something before anyone can believe it is wired up.
 #
-# Note this is a STRING comparison, not a boolean. Forgejo delivers
+# The input is normalised through `format()` before it is compared, and that
-# workflow_dispatch inputs as strings, so `inputs.refresh` is 'true'/'false'
+# is not defensive styling — the direct comparison is WRONG and fails silently.
-# and `&&` on it would treat the string 'false' as truthy.
+#
+# `type: boolean` delivers a real boolean, and GitHub expression semantics cast
+# operands to numbers when their types differ: `true == 'true'` compares 1
+# against NaN and is FALSE. Measured on run 5270, whose own log says it —
+#
+#   expression '(github.event_name == 'schedule'
+#                || github.event.inputs.refresh == 'true') && 'true' || 'false''
+#     evaluated to '%!t(string=false)'
+#   trigger: raw inputs refresh='true'
+#
+# — the input arrived as `true` and the expression still said false. The run
+# then went green with every step skipped, because a refresh that evaluates
+# false behaves exactly like an ordinary push. That is the whole hazard: the
+# failure has no symptom.
+#
+# `force_build` never hit this because it never compares in an expression. It
+# passes the raw value into an env var and tests it in the shell, where
+# everything is already a string. `format('{0}', x)` buys the same thing here,
+# where a step-level `if:` needs the answer before any shell runs.
 env:
-  IS_REFRESH: ${{ (github.event_name == 'schedule' || github.event.inputs.refresh == 'true') && 'true' || 'false' }}
+  IS_REFRESH: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'true' || 'false' }}
-  BUILD_REF: ${{ (github.event_name == 'schedule' || github.event.inputs.refresh == 'true') && 'main' || github.ref }}
+  BUILD_REF: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'main' || github.ref }}
 
 # Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes:
 #   - write:package, read:package  (for docker push to git.fabledsword.com)
@@ -491,8 +509,18 @@ jobs:
       #     the PREVIOUS XPI while the freshly signed one is orphaned (#3156).
       #   * dev and main derive the same values for the same source.
       - name: Report the derived artifact version
+        env:
+          # Diagnostic for the trigger normalisation. `refresh` is reported RAW
+          # as well as normalised, because the two disagreeing is the whole
+          # failure mode: a dispatch input whose type does not compare the way
+          # the expression assumes evaluates to false silently, and the only
+          # symptom is a refresh that quietly behaves like an ordinary push.
+          RAW_REFRESH: ${{ github.event.inputs.refresh }}
+          RAW_FORCE: ${{ github.event.inputs.force_build }}
         run: |
           set -u
+          echo "trigger: event=$GITHUB_EVENT_NAME IS_REFRESH='${IS_REFRESH:-<unset>}' BUILD_REF='${BUILD_REF:-<unset>}'"
+          echo "trigger: raw inputs refresh='${RAW_REFRESH:-<unset>}' force_build='${RAW_FORCE:-<unset>}'"
           A=web
           V=$(sh scripts/artifacts.sh version "$A" 2>&1 || echo UNAVAILABLE)
           R=$(sh scripts/artifacts.sh revision "$A" 2>&1 || echo UNAVAILABLE)
@@ -1074,6 +1102,151 @@ jobs:
           docker buildx imagetools create $ARGS "$SOURCE"
           echo "repointed from $SOURCE:$ARGS"
 
+  # Does the image a refresh just built still work?
+  #
+  # This is the gate the base refresh never had. `ci.yml` cannot be it: its
+  # lanes run on ci-python:3.14 and install requirements.txt, and a base bump
+  # changes neither — all five stay green through a refresh that breaks the
+  # product. What a refresh re-resolves is the Dockerfile's apt layer (ffmpeg,
+  # unar, libpq5, postgresql-client, zstd, megatools, libjpeg62-turbo,
+  # libwebp7, libpng16-16), unpinned, every build.
+  #
+  # So this runs the CANDIDATE IMAGE, against real Postgres and Redis. Not the
+  # source tree, and not a static inspection: `ffmpeg -version` exiting 0 would
+  # pass while a codec removal broke every thumbnail in the library.
+  #
+  # Refresh-only. On a push the bytes came from a commit, and a commit is what
+  # ci.yml already tests.
+  #
+  # Reports a verdict; it does not yet gate the promote (milestone 362 step 4).
+  # Landing the gate and the thing it gates in one change would mean the first
+  # time anyone saw this job run would also be the first time it could stop a
+  # publish.
+  smoke-web:
+    if: env.IS_REFRESH == 'true'
+    needs: [build-web]
+    runs-on: python-ci
+    container:
+      image: git.fabledsword.com/bvandeusen/ci-python:3.14
+    env:
+      DB_USER: fabledcurator
+      DB_PASSWORD: ci_smoke
+      DB_PORT: "5432"
+      DB_NAME: fabledcurator_smoke
+      SECRET_KEY: ci_smoke_placeholder
+      IMAGE: git.fabledsword.com/bvandeusen/fabledcurator
+    services:
+      postgres:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_USER: fabledcurator
+          POSTGRES_PASSWORD: ci_smoke
+          POSTGRES_DB: fabledcurator_smoke
+        options: >-
+          --health-cmd "pg_isready -U fabledcurator"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+      redis:
+        image: redis:7-alpine
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # The same ref the image was built from, so the smoke script matches
+          # the code inside the candidate.
+          ref: ${{ env.BUILD_REF }}
+
+      - name: Smoke the candidate image
+        env:
+          TOKEN: ${{ secrets.RELEASE_TOKEN }}
+          ACTOR: ${{ github.actor }}
+        run: |
+          set -eux
+          # Service discovery mirrors ci.yml's integration lane: these jobs run
+          # in a container against a mounted docker socket, so the services are
+          # SIBLINGS reachable by IP, not by hostname.
+          PG=$(docker ps --filter "name=smoke" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
+          RD=$(docker ps --filter "name=smoke" --filter "ancestor=redis:7-alpine" -q | head -n1)
+          test -n "$PG" && test -n "$RD"
+          PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
+          RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
+          test -n "$PG_IP" && test -n "$RD_IP"
+
+          # Socket probe in python, not bash's /dev/tcp — these steps run under
+          # `sh -e`, where that path does not exist. Same fix and reasoning as
+          # ci.yml's integration job; see the comment there.
+          pg_ready=""
+          for i in $(seq 1 60); do
+            if python -c "import socket,sys; s=socket.socket(); s.settimeout(2); sys.exit(0 if s.connect_ex(('$PG_IP', 5432)) == 0 else 1)"; then
+              pg_ready=1
+              break
+            fi
+            sleep 2
+          done
+          if [ -z "$pg_ready" ]; then
+            echo "postgres at $PG_IP:5432 did not accept a connection within 120s"
+            exit 1
+          fi
+
+          echo "$TOKEN" | docker login git.fabledsword.com -u "$ACTOR" --password-stdin
+          CANDIDATE="$IMAGE:refresh-candidate"
+          docker pull "$CANDIDATE"
+
+          ENVOPTS="-e DB_USER=$DB_USER -e DB_PASSWORD=$DB_PASSWORD -e DB_HOST=$PG_IP"
+          ENVOPTS="$ENVOPTS -e DB_PORT=5432 -e DB_NAME=$DB_NAME -e SECRET_KEY=$SECRET_KEY"
+          ENVOPTS="$ENVOPTS -e CELERY_BROKER_URL=redis://$RD_IP:6379/0"
+          ENVOPTS="$ENVOPTS -e CELERY_RESULT_BACKEND=redis://$RD_IP:6379/0"
+
+          # 1. The schema builds from empty, using the image's OWN libpq and
+          #    psycopg. This is the same call entrypoint.sh makes before it
+          #    serves anything, so a failure here is a failure to boot.
+          echo "smoke: alembic upgrade head"
+          docker run --rm $ENVOPTS "$CANDIDATE" alembic upgrade head
+
+          # 2. The apt layer's binaries and the app's own thumbnail path, run
+          #    inside the image. Piped over stdin rather than bind-mounted: the
+          #    workspace is a docker VOLUME belonging to this job's container,
+          #    so a host bind of $PWD would not resolve for a sibling.
+          echo "smoke: image-internal checks"
+          docker run --rm -i $ENVOPTS "$CANDIDATE" shell -c 'python3 -' < scripts/smoke_image.py
+
+          # 3. It actually serves. `docker run -d` then poll the container's own
+          #    IP — no port publishing, because the job container reaches
+          #    siblings directly and a published port would collide with
+          #    whatever else the runner is hosting.
+          echo "smoke: web boots and answers /api/health"
+          CID=$(docker run -d $ENVOPTS "$CANDIDATE" web)
+          # Clean up the container however this ends, and dump its log ONLY
+          # on failure — a boot that never answers must fail with the reason
+          # visible rather than as a bare timeout (rule 156), while a green run
+          # has nothing to say. `exit $rc` preserves the real status, which a
+          # trap that ends on a successful `docker rm` would otherwise mask.
+          trap 'rc=$?; [ $rc -eq 0 ] || docker logs "$CID" 2>&1 | tail -40; docker rm -f "$CID" >/dev/null 2>&1 || true; exit $rc' EXIT
+          WEB_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$CID")
+          test -n "$WEB_IP"
+          healthy=""
+          for i in $(seq 1 60); do
+            if curl -fsS --max-time 5 "http://$WEB_IP:8080/api/health" >/dev/null 2>&1; then
+              healthy=1
+              break
+            fi
+            sleep 2
+          done
+          if [ -z "$healthy" ]; then
+            echo "smoke: FAILED — web did not answer /api/health within 120s." >&2
+            echo "smoke: entrypoint runs alembic BEFORE serving, and step 1" >&2
+            echo "smoke: passed, so look at hypercorn and the python base." >&2
+            exit 1
+          fi
+          curl -fsS --max-time 5 "http://$WEB_IP:8080/api/health"
+          echo
+          echo "smoke: all checks passed against $CANDIDATE"
+
   build-ml:
     runs-on: python-ci
     container:
@@ -1127,8 +1300,18 @@ jobs:
       #     the PREVIOUS XPI while the freshly signed one is orphaned (#3156).
       #   * dev and main derive the same values for the same source.
       - name: Report the derived artifact version
+        env:
+          # Diagnostic for the trigger normalisation. `refresh` is reported RAW
+          # as well as normalised, because the two disagreeing is the whole
+          # failure mode: a dispatch input whose type does not compare the way
+          # the expression assumes evaluates to false silently, and the only
+          # symptom is a refresh that quietly behaves like an ordinary push.
+          RAW_REFRESH: ${{ github.event.inputs.refresh }}
+          RAW_FORCE: ${{ github.event.inputs.force_build }}
         run: |
           set -u
+          echo "trigger: event=$GITHUB_EVENT_NAME IS_REFRESH='${IS_REFRESH:-<unset>}' BUILD_REF='${BUILD_REF:-<unset>}'"
+          echo "trigger: raw inputs refresh='${RAW_REFRESH:-<unset>}' force_build='${RAW_FORCE:-<unset>}'"
           A=ml
           V=$(sh scripts/artifacts.sh version "$A" 2>&1 || echo UNAVAILABLE)
           R=$(sh scripts/artifacts.sh revision "$A" 2>&1 || echo UNAVAILABLE)
@@ -1617,8 +1800,18 @@ jobs:
       #     the PREVIOUS XPI while the freshly signed one is orphaned (#3156).
       #   * dev and main derive the same values for the same source.
       - name: Report the derived artifact version
+        env:
+          # Diagnostic for the trigger normalisation. `refresh` is reported RAW
+          # as well as normalised, because the two disagreeing is the whole
+          # failure mode: a dispatch input whose type does not compare the way
+          # the expression assumes evaluates to false silently, and the only
+          # symptom is a refresh that quietly behaves like an ordinary push.
+          RAW_REFRESH: ${{ github.event.inputs.refresh }}
+          RAW_FORCE: ${{ github.event.inputs.force_build }}
         run: |
           set -u
+          echo "trigger: event=$GITHUB_EVENT_NAME IS_REFRESH='${IS_REFRESH:-<unset>}' BUILD_REF='${BUILD_REF:-<unset>}'"
+          echo "trigger: raw inputs refresh='${RAW_REFRESH:-<unset>}' force_build='${RAW_FORCE:-<unset>}'"
           A=agent
           V=$(sh scripts/artifacts.sh version "$A" 2>&1 || echo UNAVAILABLE)
           R=$(sh scripts/artifacts.sh revision "$A" 2>&1 || echo UNAVAILABLE)

scripts/smoke_image.py

@@ -0,0 +1,155 @@
+"""Prove a freshly built image can still do the things its OS packages provide.
+
+Run INSIDE the image, not against the source tree. That distinction is the
+entire reason this file exists.
+
+`ci.yml`'s lanes run on `ci-python:3.14` and install `requirements.txt`. A base
+refresh changes neither, so all five lanes stay green through a base bump that
+breaks the product. What a refresh actually re-resolves is this, from the
+Dockerfile:
+
+    RUN apt-get update && apt-get install -y --no-install-recommends \
+            ffmpeg unar libpq5 postgresql-client zstd megatools \
+            libjpeg62-turbo libwebp7 libpng16-16 ca-certificates
+
+Unpinned, every build. Nothing else in this repo looks at it.
+
+So the checks below run the APPLICATION'S OWN code — `Thumbnailer`, which needs
+no database and no app context — against whatever Pillow and ffmpeg have
+become. `ffmpeg -version` exiting 0 would pass while a codec removal or an
+soname bump broke every thumbnail in the library; producing a thumbnail would
+not.
+
+Every failure names the package it implicates. This fires on a Sunday,
+unattended, about a change nobody made deliberately — "assertion failed" a week
+later teaches nobody anything.
+
+Usage:  docker run --rm -i <image> shell -c 'python3 -' < scripts/smoke_image.py
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+
+try:
+    from PIL import Image
+
+    from backend.app.services.thumbnailer import Thumbnailer
+except Exception as exc:  # noqa: BLE001 — a smoke test reports, it never raises
+    print(f"smoke: FAILED — could not import the thumbnail path at all: {exc}")
+    print("  Implicates Pillow or its shared libraries (libjpeg62-turbo,")
+    print("  libpng16-16, libwebp7), or the python base image itself.")
+    raise SystemExit(1) from exc
+
+
+# Binary → what stops working without it. Listed individually because
+# `--no-install-recommends` means any one of them can vanish on its own when a
+# dependency chain higher up changes.
+REQUIRED_BINARIES = {
+    "ffmpeg": "video thumbnails and transcoding (Dockerfile: ffmpeg)",
+    "unar": "archive import — cbz/zip/rar members (Dockerfile: unar)",
+    "pg_dump": "database backup (Dockerfile: postgresql-client)",
+    "zstd": "backup compression, pg_dump | tar --zstd (Dockerfile: zstd)",
+    "megatools": "mega.nz public-link downloads, #830 (Dockerfile: megatools)",
+}
+
+
+def check_jpeg(thumbs: Thumbnailer, src: Path) -> None:
+    path = src / "flat.jpg"
+    Image.new("RGB", (900, 400), (30, 90, 160)).save(path, "JPEG")
+    result = thumbs.generate_image_thumbnail(path, "a" * 64)
+    assert result.mime == "image/jpeg", f"mime was {result.mime}"
+    assert result.path.stat().st_size > 0, "no bytes written"
+    # Re-open it. A file that writes but cannot be read back is the shape a
+    # half-broken codec produces, and size alone would not catch it.
+    with Image.open(result.path) as im:
+        im.load()
+
+
+def check_png_alpha(thumbs: Thumbnailer, src: Path) -> None:
+    path = src / "alpha.png"
+    Image.new("RGBA", (400, 900), (200, 40, 40, 128)).save(path, "PNG")
+    result = thumbs.generate_image_thumbnail(path, "b" * 64)
+    assert result.mime == "image/png", f"mime was {result.mime}"
+    with Image.open(result.path) as im:
+        im.load()
+        assert im.mode in ("RGBA", "LA", "P"), f"alpha lost, mode={im.mode}"
+
+
+def check_webp(thumbs: Thumbnailer, src: Path) -> None:
+    path = src / "sample.webp"
+    Image.new("RGB", (500, 500), (10, 140, 70)).save(path, "WEBP")
+    result = thumbs.generate_image_thumbnail(path, "c" * 64)
+    assert result.path.stat().st_size > 0, "no bytes written"
+
+
+def check_video(thumbs: Thumbnailer, src: Path) -> None:
+    # Synthesised rather than committed as a fixture: a checked-in video is a
+    # binary blob nobody can review, and lavfi ships with every ffmpeg build.
+    #
+    # 3 seconds, not 2. The seek lands at max(1.0, duration * 0.05) = 1.0s, and
+    # a clip barely longer than its own seek is how #1231 produced zero frames.
+    # This check exists to exercise ffmpeg, not to re-litigate that edge.
+    clip = src / "clip.mp4"
+    subprocess.run(
+        ["ffmpeg", "-nostdin", "-f", "lavfi", "-i", "testsrc=size=640x360:rate=10",
+         "-t", "3", "-pix_fmt", "yuv420p", "-y", str(clip)],
+        check=True, capture_output=True, timeout=120,
+    )
+    result = thumbs.generate_video_thumbnail(clip, "d" * 64, duration_seconds=3.0)
+    assert result.path.stat().st_size > 0, "no bytes written"
+    with Image.open(result.path) as im:
+        im.load()
+
+
+CHECKS = (
+    ("JPEG thumbnail", "libjpeg62-turbo / Pillow", check_jpeg),
+    ("PNG thumbnail (alpha)", "libpng16-16 / Pillow", check_png_alpha),
+    ("WebP decode", "libwebp7 / Pillow", check_webp),
+    ("video thumbnail", "ffmpeg", check_video),
+)
+
+
+def main() -> int:
+    failures: list[str] = []
+
+    print("smoke: binaries the apt layer provides")
+    for binary, purpose in REQUIRED_BINARIES.items():
+        if shutil.which(binary) is None:
+            print(f"  FAIL  {binary}: not on PATH")
+            failures.append(f"{binary} — {purpose}")
+        else:
+            print(f"  ok    {binary}")
+
+    print("smoke: the application's own thumbnail path, against this image's libraries")
+    with tempfile.TemporaryDirectory() as tmp:
+        root = Path(tmp)
+        src = root / "src"
+        src.mkdir()
+        thumbs = Thumbnailer(root)
+        for name, implicates, fn in CHECKS:
+            try:
+                fn(thumbs, src)
+                print(f"  ok    {name}")
+            except Exception as exc:  # noqa: BLE001 — report every check, then fail once
+                print(f"  FAIL  {name}: {exc}")
+                failures.append(f"{name} — {implicates}")
+
+    if failures:
+        print(f"\nsmoke: FAILED — {len(failures)} check(s)")
+        for failure in failures:
+            print(f"  - {failure}")
+        print("\nThis image was built against freshly resolved base layers. The")
+        print("named packages are where to look: compare this build's apt versions")
+        print("against the previous :latest before assuming the app changed.")
+        return 1
+
+    print("\nsmoke: all checks passed")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())