diff --git a/.forgejo/workflows/baseline.yml b/.forgejo/workflows/baseline.yml index 56248fd..f56f4ba 100644 --- a/.forgejo/workflows/baseline.yml +++ b/.forgejo/workflows/baseline.yml @@ -79,7 +79,7 @@ jobs: - name: Resolve the Postgres service and install deps run: | set -eux - # Same service-IP dance as ci.yml's integration job; see the long + # Same service-IP dance as build.yml's integration job; see the long # comment there for why the job name must stay separator-free. PG=$(docker ps --filter "name=compare" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) test -n "$PG" @@ -89,7 +89,7 @@ jobs: echo "DB_HOST=$PG_IP" >> "$GITHUB_ENV" # Socket probe in python, not bash's /dev/tcp — these steps run under # `sh -e`, where that path does not exist. Same fix and same reasoning - # as ci.yml's integration job; see the comment there. + # as build.yml's integration job; see the comment there. pg_ready="" for i in $(seq 1 60); do if python -c "import socket,sys; s=socket.socket(); s.settimeout(2); sys.exit(0 if s.connect_ex(('$PG_IP', 5432)) == 0 else 1)"; then diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index eb6aa08..2dab36a 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -1,4 +1,4 @@ -name: Build images +name: CI and images on: push: @@ -10,10 +10,25 @@ on: # pressure to merge in order to try something does not come from # carelessness; it comes from `:dev` being unable to carry the build. # - # All three images build on dev, deliberately: a `:dev` web image paired - # with a stale `:dev` ml or agent is a worse trap than no dev channel at - # all, since the mismatch only shows up as a runtime failure. + # BOTH images build on dev, deliberately: a `:dev` web image paired + # with a stale `:dev` agent is a worse trap than no dev channel at all, + # since the mismatch only shows up as a runtime failure. (It was three + # until #4311 — `fabledcurator-ml` was the same bytes as `fabledcurator` + # under a second name, and the stack that needed the second name is being + # collapsed onto the consolidated image.) branches: [main, dev] + + # Renovate opens PRs from `renovate/*` branches into `dev`. Those branches + # never push to dev/main, so the push trigger above gives them NO pre-merge + # CI — a bump could only be validated after it was already merged. Base + # `dev` only: it deliberately does NOT fire on dev→main PRs, which rely on + # the dev push run, so no duplicate runs. FC has no fork PRs (single-operator + # Forgejo repo), so secrets-on-PR is not a concern. + # + # Nothing PUBLISHES on this trigger — see the `if:` on the three publishing + # jobs below. A PR run is the lanes and nothing else. + pull_request: + branches: [dev] # # NO tag trigger (milestone 318 step 2). A `v*` tag names a commit `main` # already built and published; rebuilding it produces the same source under @@ -26,17 +41,24 @@ on: # produce a changelog, not an image. # The escape hatch for the one thing skip-if-exists makes untestable: a - # build that WOULD be skipped. `agent/` has not changed since 2026-07-17, so - # every push since has correctly declined to build it — which also means the - # agent build path has not run in six weeks and cannot be exercised on - # demand. #3190 lives on exactly that path. + # build that WOULD be skipped. An artifact whose source is quiet has its + # build path correctly declined on every push — which also means that path + # is not exercised, and cannot be exercised on demand, for as long as the + # quiet lasts. #3190 lives on exactly such a path. + # + # This used to name the agent and a date ("unchanged since 2026-07-17"). It + # was already false when read on 2026-09-24 — the agent had changed the day + # before — and a rationale resting on a stale fact reads as settled + # reasoning forever after (lesson #4383). Which artifact is quiet is a + # question for `git log`, not for a comment; the hatch exists because ANY of + # them can be. # # Editing build.yml does not force one either, and that is deliberate: the # workflow is not shipped bytes, so it is in no artifact's path set. Putting # it in one would re-version every artifact for a comment change. # - # ONE input, not one per artifact. Forcing all three is cheap once the - # registry cache is warm (#3114), and three booleans is an interface nobody + # ONE input, not one per artifact. Forcing both is cheap once the registry + # cache is warm (#3114), and a boolean per artifact is an interface nobody # remembers the meaning of. workflow_dispatch: inputs: @@ -52,11 +74,18 @@ on: # The base-image refresh (milestone 326 step 4, #3154). # # Skip-if-exists is keyed on OUR source, so an artifact whose source stops - # moving stops picking up base-image updates. `agent/` last changed - # 2026-07-17; every push since has correctly declined to rebuild it, which - # also means it will serve that day's `nvidia/cuda` layers forever. Nothing - # is wrong until it has been unchanged for months, which is precisely why - # this is a calendar trigger and not a condition on the push path. + # moving stops picking up base-image updates — it will serve the base layers + # of its last build forever. The agent is the standing example because its + # base is the heaviest (`nvidia/cuda`, ~6.3 GB) and its source the quietest: + # 48 of 1111 commits touch it, about 4% (#3114). + # + # Deliberately no date here. Nothing is wrong until an artifact has been + # unchanged for months, and "how long has it been" is a `git log` question + # whose answer in a comment is wrong the next time anyone commits — which is + # exactly how the previous version of this paragraph came to claim the agent + # had not moved since 2026-07-17 while it had moved the day before. That is + # precisely why this is a CALENDAR trigger and not a condition on the push + # path: the calendar cannot be wrong about how much time has passed. # # Weekly, Sunday 06:00 UTC. Away from CI-runner's Monday security sweep so # the two are never diagnosing each other, and on the quietest day so a @@ -113,6 +142,16 @@ concurrency: # Deriving it per job invites the two halves to disagree: sign-extension would # derive dev's extension version while build-web bundled main's, and the # release download would 404 on a version that exists perfectly well. +# +# On every other trigger it is the COMMIT that fired (`github.sha`), never the +# branch name. A branch is re-resolved by each job's checkout when that job +# starts, so a push landing mid-run moved the later jobs onto the new tip: run +# 7499 signed 423275a's extension, then build-web checked out 83e1382 (pushed +# while 7499 ran), derived a version nobody had signed, and 404'd on the +# download (#4427). The guard failed closed that time; a job without one would +# have published a commit the run's own lanes never tested — the lanes check +# out `github.sha` by default, so pinning here makes the publish build exactly +# what they passed. # IS THIS A BASE REFRESH? Asked in five places and previously spelled five # ways — `github.event_name == 'schedule'` in an `if:`, `$GITHUB_EVENT_NAME` in # one shell, an `EVENT:` env passed into another, and a bare expression on @@ -149,7 +188,16 @@ concurrency: # where a step-level `if:` needs the answer before any shell runs. env: IS_REFRESH: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'true' || 'false' }} - BUILD_REF: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'main' || github.ref }} + BUILD_REF: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'main' || github.sha }} +# What the six LANES check out. Empty — the checkout default, the triggering +# commit (or a PR's merge ref) — on every trigger but the refresh, where it is +# `main`: the refresh publishes main, so the gate has to test main (#4430). It +# is not BUILD_REF itself because a pull_request run's `github.sha` is a merge +# commit the default checkout reaches through its ref, not by sha. Each job +# still resolves `main` when it starts, so a merge to main during the ~5 min of +# a Sunday-06:00 refresh could put the lanes and the build one commit apart; +# the build jobs' own guards assert the branch, not the commit. + LANE_REF: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'main' || '' }} # Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes: # - write:package, read:package (for docker push to git.fabledsword.com) @@ -158,6 +206,379 @@ env: # The injected GITHUB_TOKEN cannot be used — it lacks write:package. jobs: + # --------------------------------------------------------------------------- + # THE LANES. Merged in from `ci.yml`, which is deleted, 2026-09-23. + # + # They were a separate workflow on the same push trigger, which meant the + # build could not see their verdict and published regardless. Run 7348 is the + # worked example: the unit lane went red on `2f8f0bc` and `build-web` pushed + # `:dev` anyway, in the same minute. Operator: *"tighten the gate so :dev + # can't publish on red tests"*, then *"I don't want failing builds to publish + # anywhere going forward."* + # + # Two workflows cannot express that. A `needs:` edge only exists inside one + # graph — so the lanes and the publish are one graph now, and the gate is the + # `needs:` on the three publishing jobs rather than anything new. + # + # The cost, accepted knowingly: this workflow QUEUES per branch and never + # cancels (see `concurrency:` above), so on two pushes in quick succession + # the second push's lint feedback waits out the first run's build. That is + # the price of the edge, and it is the right way round — a slower red is + # better than a fast red that ships. + # --------------------------------------------------------------------------- + + # Fast-fail lint lane. ruff is pre-installed in the ci-python image, so + # this runs with NO dependency install and surfaces the most common bounce + # class (lint: I001 / UP037 / ASYNC109 / W293 …) in seconds — instead of + # after the backend job's ~30-60s wheel install. ruff is static analysis, + # so no DB/secret env is needed. + lint: + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ env.LANE_REF }} + - name: Ruff lint + # agent/ included so the GPU-agent is linted before its image is built + # (build.yml only `docker build`s it — this is where it gets checked). + # scripts/ likewise: release_notes.py runs only on a tag push, so a + # syntax or import error there would otherwise surface at the one + # moment nobody wants to debug a workflow. + run: ruff check backend/ tests/ alembic/ agent/ scripts/ + - name: Agent syntax check + # The agent's runtime deps (torch/transformers/ultralytics) aren't in the + # CI image, so we can't import it — but compileall parses every module, + # catching syntax errors before the image build. + run: python -m compileall -q agent/fc_agent + + # The extension version is DERIVED, not hand-maintained (milestone 271 step + # 4): build.yml computes it from the commit TIME of the newest packaged + # extension change and stamps it into manifest.json / package.json at build + # time. The guard that used to live here — "packaged files changed but nobody + # bumped the version" — was therefore checking a fact that had stopped + # existing. Worse than useless: it would have failed this lane on every real + # extension change, demanding a bump that decides nothing. Retired 2026-08-27 + # rather than left running beside the new mechanism (rule 22). + # + # Two things are still worth asserting, and this is the only lane that can: + # the extension-test lane runs on node:24-slim, which is exactly why + # version.spec.js sticks to packaging.sh's git-free subcommands. + # 1. the derivation actually resolves on this commit + # 2. the derived string is one AMO will accept, checked against Mozilla's + # own published grammar rather than a loose "digits and dots" + # + # The MAJOR.MINOR-agreement check that used to be (2) is gone with milestone + # 318 step 8: the committed version no longer seeds anything, so there is no + # hand-set part left for the two files to disagree about. + # + # Deliberately NOT checked here: that the derived value beats what has already + # been signed. That guard belongs in build.yml, where it compares against the + # real ext-* releases. Comparing against origin/main here would be wrong — + # dev legitimately derives a LOWER value whenever main is ahead on the + # extension, and a lane that fails for being behind is a lane people learn to + # ignore. + extension-version: + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ env.LANE_REF }} + # The derivation needs real history: a depth-1 clone sees one commit + # and produces a wrong, too-low value RATHER THAN FAILING. Checking + # that here is half the point of the lane. + fetch-depth: 0 + - name: Extension version derives cleanly + run: | + set -eu + # busybox sh on the act_runner — no bashisms (family rule). + VERSION=$(sh extension/scripts/packaging.sh version) + echo "derived: $VERSION" + + # Mozilla's published grammar for AMO, transcribed verbatim from + # MDN's manifest.json/version page: + # + # ^(0|[1-9][0-9]{0,8})([.](0|[1-9][0-9]{0,8})){0,3}$ + # + # Not the looser `^[0-9]+(\.[0-9]+)*$` this lane used to carry. That + # one passes `2026.08.29.0201`, which AMO REJECTS — a segment must be + # the single digit 0 or start 1-9 — and it also passes five segments, + # where AMO allows four. Both would surface as a failed sign with the + # version already burned: AMO 409s on re-signing, so a rejected value + # cannot be reclaimed and cannot be reused. This lane is the cheap + # place to find out. (#3138, milestone 318 step 8.) + if ! echo "$VERSION" | grep -qE '^(0|[1-9][0-9]{0,8})(\.(0|[1-9][0-9]{0,8})){0,3}$'; then + echo "ERROR: derived version '$VERSION' is not a version AMO accepts." + echo "AMO's grammar: ^(0|[1-9][0-9]{0,8})([.](0|[1-9][0-9]{0,8})){0,3}$" + echo "Most likely cause: a zero-padded segment (08, 0201). The rest" + echo "of the family pads; the extension must not — see packaging.sh." + exit 1 + fi + + # ...and the shape this project actually derives. AMO would happily + # take `1.0.3500147` too, so the grammar check alone would not notice + # a regression to the pre-318 shape — which orders BELOW everything + # signed since, and is unrecoverable once Firefox has the higher one. + if ! echo "$VERSION" | grep -qE '^20[0-9][0-9]\.[0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,4}$'; then + echo "ERROR: derived version '$VERSION' is not YYYY.M.D.HHMM." + echo "Rule 148's CalVer is what build.yml signs; the old" + echo "1.0. shape would order below every ext-2026.* release." + exit 1 + fi + echo "OK: derived version $VERSION" + + backend-lint-and-test: + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + env: + # DB_PASSWORD and SECRET_KEY are required by config.py at import time + # even though unit tests don't actually touch the DB or use the secret. + DB_PASSWORD: ci_unit_test_placeholder + SECRET_KEY: ci_unit_test_placeholder + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ env.LANE_REF }} + # Full history for tests/test_artifact_identity.py, which derives + # each artifact's revision to check the identity scheme. On a + # depth-1 clone that derivation either fails or returns the tip sha + # — so the lane would go green while asserting nothing, which is + # the one outcome worse than a red one. + fetch-depth: 0 + + # Cache step removed 2026-05-26: act_runner's cache backend has been + # broken on this homelab runner since 2026-05-15 (first as request- + # timeout warnings, then as hard "Cannot find module .../dist/restore/ + # index.js" failures that tank the whole job). The cache step targeted + # ~/.cache/pip but the install below uses `uv pip install` primarily, + # whose own cache lives at ~/.cache/uv — so the cache step's real + # benefit was marginal even when working. Cost of removal: ~30s of + # wheel downloads per job. Future re-enable: mount ~/.cache/uv as a + # docker volume at the runner level (skips actions/cache entirely), + # or fix the runner-side cache backend (clear /var/run/act/actions/*, + # pin act_runner version, etc.). + + - name: Install Python deps + # ruff is pre-installed in the ci-python image (see CI-Runner/CI-python/ + # Dockerfile's RUFF_VERSION). Per FabledRulebook ci-runners.md, toolchain + # versions live on the runner image, not here. + # uv: 5-10x faster wheel resolve than pip for cold caches. + # Falls back to pip install on uv-missing runners (older images). + run: | + if command -v uv >/dev/null 2>&1; then + uv pip install --system -r requirements.txt pytest pytest-asyncio + else + pip install -r requirements.txt pytest pytest-asyncio + fi + + # Ruff moved to the dedicated fast `lint` job above (fails in seconds, + # no dep install). This job is now unit tests only. + - name: Pytest (unit only — integration runs in the integration job) + run: pytest tests/ -v -m "not integration" + + frontend-build: + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ env.LANE_REF }} + # No package-lock.json is tracked yet (we don't run npm locally per + # feedback-no-local-runs). Using `npm install` instead of `npm ci`. + # If we want strict lockfile-based reproducibility later, commit a + # package-lock.json and flip this back to `npm ci`. + - run: npm install --no-audit --no-fund + # No type-check step: the frontend is pure JS (no .ts files, no JSDoc), + # so a type-checker has nothing to do. The vue-tsc devDep + its `check` + # script were dropped 2026-07-11 rather than bumped to v3. If we add + # TS/JSDoc later, re-add a tsconfig.json + vue-tsc + a type-check step. + - run: npm run test:unit + - run: npm run build + + # The extension's lane: web-ext lint, the vitest suite, and the check that + # asks web-ext what it ACTUALLY packaged. Moved in from `extension.yml` + # (deleted) at milestone 429 — the same move `ci.yml` made on 2026-09-23 and + # for the same reason: a separate workflow cannot gate this one, so a red + # extension suite still let `sign-extension` sign and `build-web` ship the + # XPI. It is a lane in THE GATE now. Its old path filter is gone with it; a + # filter here would make the lane skip, and a skipped lane blocks the publish. + # + # The vitest suite is also the JS half of the #3093 artist-pattern mirror — + # the Python half runs in backend-lint-and-test — so until this move one half + # of that guard could fail without stopping anything. + extension-test: + runs-on: python-ci + container: + image: node:24-bookworm-slim + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ env.LANE_REF }} + # Not --no-save: vitest and web-ext are both real devDependencies now, + # and the suite needs vitest resolvable from node_modules. + - name: Install dev dependencies + run: cd extension && npm install --no-audit --no-fund + - name: Lint + run: cd extension && npm run lint + # Pure-logic specs over lib/url.js and lib/platforms.js plus manifest / + # package version-consistency checks. No browser, no network. + - name: Unit tests + run: cd extension && npm run test:unit + + # Everything else about packaging is asserted against our own declaration + # of what ships. This is the only check that asks web-ext what it ACTUALLY + # put in the archive. Until now that was an unverified assumption about + # glob semantics — and a fragile one: `test/**` reaches web-ext intact + # only because callers `set -f` first, so losing that quoting would + # silently start shipping dev files with no other signal. + - name: Verify XPI contents + run: | + set -eu + command -v unzip >/dev/null 2>&1 || { apt-get update -qq && apt-get install -y -qq unzip; } + cd extension + npm run build + ZIP=$(ls web-ext-artifacts/*.zip | head -1) + echo "=== packaged entries in $ZIP ===" + unzip -Z1 "$ZIP" | sort + echo "=== end ===" + ENTRIES=$(unzip -Z1 "$ZIP") + fail=0 + # Must NOT ship: repo infrastructure with no business in a user's browser. + for pat in 'test/' 'scripts/' 'vitest.config.js' 'package.json' 'package-lock.json' 'README.md' 'node_modules/' 'web-ext-artifacts/'; do + if echo "$ENTRIES" | grep -q "^$pat"; then + echo "ERROR: '$pat' was packaged into the XPI but must not be" + fail=1 + fi + done + # Must ship: if an exclusion pattern ever over-matches, the extension + # breaks at runtime rather than at build time, so assert presence too. + for req in 'manifest.json' 'lib/url.js' 'lib/api.js' 'lib/platforms.js' 'lib/cookies.js' 'lib/chip.js' 'lib/popup-format.js'; do + if ! echo "$ENTRIES" | grep -q "^$req$"; then + echo "ERROR: '$req' is missing from the XPI" + fail=1 + fi + done + for dir in 'background/' 'popup/' 'options/' 'content/' 'icons/'; do + if ! echo "$ENTRIES" | grep -q "^$dir"; then + echo "ERROR: nothing from '$dir' was packaged" + fail=1 + fi + done + [ "$fail" -eq 0 ] || exit 1 + echo "XPI contents verified." + + # Single integration job — collapsed from a 3-way shard split on 2026-06-04. + # The shards existed to parallelize ~8.5min of integration tests; once the + # throwaway Postgres runs with fsync OFF (the durability step below) the whole + # suite runs in ~45s, so the split only triplicated the ~2min fixed overhead + # (container + `uv pip install` + `alembic upgrade head`) and burned 3 of 6 + # runner slots for no wall-clock gain. One job now: spin up once, install + # once, migrate once, run every integration test. + # + # The docker-ps filter scopes to THIS job's own Postgres/Redis service + # containers by job name. act_runner strips underscores from job names when + # labelling containers (`int_api` matched nothing on 2026-05-25), so the name + # stays separator-free (`integration`). The step prints `docker ps -a` first + # so a future naming-convention shift surfaces in the log without a + # guess-and-push cycle. + # + # Pre-baking requirements.txt into ci-python:3.14 is intentionally NOT done — + # per ci-requirements.md, FC is the only Python consumer of that image and the + # CI-Runner "add deps to image when used by >1 project" rule keeps it per-job. + integration: + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + env: + DB_USER: fabledcurator + DB_PASSWORD: ci_integration + DB_PORT: "5432" + DB_NAME: fabledcurator_test + SECRET_KEY: ci_integration_placeholder + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_USER: fabledcurator + POSTGRES_PASSWORD: ci_integration + POSTGRES_DB: fabledcurator_test + options: >- + --health-cmd "pg_isready -U fabledcurator" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + redis: + image: redis:8-alpine + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ env.LANE_REF }} + - name: Integration suite (resolve service IPs, migrate, test) + run: | + set -eux + echo "=== container landscape (diagnostic for filter scoping) ===" + docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}' + echo "=== end landscape ===" + PG=$(docker ps --filter "name=integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) + RD=$(docker ps --filter "name=integration" --filter "ancestor=redis:8-alpine" -q | head -n1) + test -n "$PG" && test -n "$RD" + PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") + RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD") + test -n "$PG_IP" && test -n "$RD_IP" + export DB_HOST="$PG_IP" + export CELERY_BROKER_URL="redis://$RD_IP:6379/0" + export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0" + # These steps run under `sh -e`, not bash, so bash's /dev/tcp magic + # path does not exist here — the probe this loop used to run could + # never succeed and simply burned the full 120s on every run, green + # or red, then continued without having established anything. Python + # is in the image and needs no installed package for a socket + # connect, so it is the probe. Exhausting the budget is now a named + # failure rather than a silent fall-through (rule 156): if Postgres + # is genuinely not up, that is what the log should say, instead of + # whatever the first query happens to raise two minutes later. + pg_ready="" + for i in $(seq 1 60); do + if python -c "import socket,sys; s=socket.socket(); s.settimeout(2); sys.exit(0 if s.connect_ex(('$PG_IP', 5432)) == 0 else 1)"; then + pg_ready=1 + break + fi + sleep 2 + done + if [ -z "$pg_ready" ]; then + echo "postgres at $PG_IP:5432 did not accept a connection within 120s" + exit 1 + fi + if command -v uv >/dev/null 2>&1; then + uv pip install --system -r requirements.txt pytest pytest-asyncio + else + pip install -r requirements.txt pytest pytest-asyncio + fi + # Relax durability on the throwaway CI Postgres so the per-test + # TRUNCATE's commit-fsync — the integration teardown's dominant cost + # (~1.5-2s/test, which collapsed the suite from ~13min to ~45s) — is + # skipped. fsync/full_page_writes are sighup GUCs and synchronous_commit + # is user-context, so ALTER SYSTEM + pg_reload_conf() applies them with + # NO restart. Ephemeral DB ⇒ fsync-off is safe. Non-fatal so a perms + # surprise can't red the job; fabledcurator is the postgres image's + # bootstrap superuser. + python -c "import os,psycopg; c=psycopg.connect(host=os.environ['DB_HOST'],port=5432,user=os.environ['DB_USER'],password=os.environ['DB_PASSWORD'],dbname=os.environ['DB_NAME'],autocommit=True); [c.execute(q) for q in ('ALTER SYSTEM SET fsync=off','ALTER SYSTEM SET synchronous_commit=off','ALTER SYSTEM SET full_page_writes=off','SELECT pg_reload_conf()')]; c.close()" || echo 'WARN: durability GUC relax failed (continuing)' + alembic upgrade head + pytest tests/ -v -m integration --durations=15 + # Sign-or-fetch-from-cache: signs the extension via AMO if no ext- # Forgejo release exists yet, otherwise downloads the cached signed XPI. # Result is uploaded as an Actions artifact for build-web to consume. @@ -188,6 +609,24 @@ jobs: # everything. A condition that is always true reads as if some path avoids # it, which is worse than no condition. sign-extension: + # THE GATE (2026-09-23). Every lane above must have PASSED before this job + # exists at all — so a red suite does not produce an image, let alone push + # one. Nothing here is a new mechanism: it is the same `needs:` edge that + # has gated `promote` since milestone 362 step 4, and it carries that + # step's hardest-won property unchanged — **not running is not the same as + # passing.** `needs` treats a SKIPPED dependency as unsatisfied, so a lane + # that silently skips itself blocks the publish exactly as a failing one + # does. Run 5290 is why that is worth saying out loud: `smoke-web` skipped + # itself through a job-level `if:` that could not read `env`, and a design + # where only a FAILED gate blocks would have published unverified images + # while reporting success. + needs: [lint, extension-version, backend-lint-and-test, frontend-build, extension-test, + integration] + # A pull_request run is the lanes and nothing else. This is the ONLY thing + # separating "validate a Renovate bump" from "publish a Renovate bump", so + # it is stated on each publishing job rather than inferred from a `needs` + # chain that a later edit could quietly break. + if: github.event_name != 'pull_request' runs-on: python-ci container: image: git.fabledsword.com/bvandeusen/ci-python:3.14 @@ -195,8 +634,8 @@ jobs: - uses: actions/checkout@v4 with: # Not the triggering ref — see the `env:` block at the top. On a - # scheduled refresh this is `main`; on everything else it is the ref - # that fired, so this is a no-op on every ordinary path. + # scheduled refresh this is `main`; on everything else it is the + # commit that fired, the same one the lanes above tested. ref: ${{ env.BUILD_REF }} # Full history is load-bearing, not a convenience: the version this # job signs is derived from the commit TIME of the newest packaged @@ -240,7 +679,7 @@ jobs: # Unpadded, and only here: AMO's grammar rejects a leading zero, so the # extension renders rule 148's numbers without the family's padding # (milestone 318 step 8). Same value, one character narrower per segment; - # ci.yml's extension-version lane checks the string against Mozilla's + # the extension-version lane above checks the string against Mozilla's # published regex before this job ever calls AMO. # # The committed "version" in manifest.json / package.json decides NOTHING @@ -274,8 +713,8 @@ jobs: # build — no `set -e`, and every derivation falls back to UNAVAILABLE. # # What to watch across pushes, because this is what step 3 will trust: - # * a push touching only agent/ moves the agent and leaves web and ml - # STILL. If web moves, its path set is too wide. + # * a push touching only agent/ moves the agent and leaves web STILL. + # If web moves, its path set is too wide. # * a push touching only docs moves nothing. # * a push touching the extension moves the extension AND web, since # web bakes in the XPI. If web does not move, its set is too narrow: @@ -388,7 +827,8 @@ jobs: # web-ext signs whatever manifest.json says, so the derived value has to # reach the tree before signing. package.json is written too: the two are - # required to agree (ci.yml's guard), and a local `npm run build` reads + # required to agree (the extension-version lane's guard), and a local + # `npm run build` reads # it. Working tree only — never committed, per the note on the derive # step. - name: Stamp the derived version into manifest.json + package.json @@ -502,12 +942,31 @@ jobs: # to know — a candidate was published — rather than restating why. outputs: candidate: ${{ steps.reuse.outputs.promote }} + # The manifest THIS run pushed, empty on a reuse hit. smoke-web addresses + # it by digest rather than by tag: a tag can move between the build and + # the smoke, and then the check reports on bytes nobody built here. + digest: ${{ steps.build.outputs.digest }} + # What the channel tag already names. This is what smoke-web checks on a + # reuse hit — deliberately NOT folded into `digest`, which the :c- + # repoint reads and which must keep meaning "what this run built" (#4290). + published_digest: ${{ steps.reuse.outputs.published_digest }} + # Every tag this commit should end up under (channel, plus :c- on + # main). `promote` writes them from `digest` once the smoke has passed. + tags: ${{ steps.tag.outputs.tags }} # A plain `needs` — no `always()`. That expression existed to let a # SKIPPED sign-extension through on a tag push while still blocking a # FAILED one. With no tag trigger, sign-extension always runs, so the # default behaviour is exactly what we want: a failed sign skips build-web # rather than shipping an image without its XPI. - needs: [sign-extension] + needs: [sign-extension, lint, extension-version, backend-lint-and-test, + frontend-build, extension-test, integration] + # The lanes in that list are THE GATE (2026-09-23) — see sign-extension's + # copy of this comment for why, including the property that a SKIPPED lane + # blocks as firmly as a failing one. They are repeated here rather than + # inherited through `sign-extension`: this job is what pushes the channel + # tag, and the one place the gate must be legible is the place that + # publishes. + if: github.event_name != 'pull_request' runs-on: python-ci container: image: git.fabledsword.com/bvandeusen/ci-python:3.14 @@ -515,8 +974,8 @@ jobs: - uses: actions/checkout@v4 with: # Not the triggering ref — see the `env:` block at the top. On a - # scheduled refresh this is `main`; on everything else it is the ref - # that fired, so this is a no-op on every ordinary path. + # scheduled refresh this is `main`; on everything else it is the + # commit that fired, the same one the lanes above tested. ref: ${{ env.BUILD_REF }} # Full history: this job RE-DERIVES the extension version rather than # being handed it, and a depth-1 clone derives a wrong, too-low value @@ -550,8 +1009,8 @@ jobs: # never be the reason an image does not ship. # # What it should say: - # * a push touching only agent/ moves the agent and leaves web and ml - # STILL. If web moves, its path set is too wide. + # * a push touching only agent/ moves the agent and leaves web STILL. + # If web moves, its path set is too wide. # * a push touching only docs moves nothing. # * a push touching the extension moves the extension AND web, since # web bakes in the XPI. If web does not move, its set is too narrow: @@ -698,7 +1157,7 @@ jobs: # Each artifact pays one rebuild, once. # # This is what stops a push that touched only `agent/` from rebuilding - # web and ml, and a merge to main from rebuilding what dev already built. + # web, and a merge to main from rebuilding what dev already built. # # The failure direction is deliberate. An inspect that errors for ANY # reason — network, auth, a registry hiccup — reads as a miss and the @@ -752,15 +1211,17 @@ jobs: # WHERE THE BUILD PUBLISHES, which is not always the channel — and # whether the channel then has to be written separately. # - # On a push the build writes the channel tag directly: the bytes came - # from a commit, and a commit is the thing CI tests. Nothing to hold - # it behind. + # Every build writes a CANDIDATE tag, never the channel (#4310): + # :dev-candidate / :latest-candidate on a push, :refresh-candidate on + # the refresh. `promote` moves the channel only after smoke-web has + # booted the bytes. The lanes prove the SOURCE; only the smoke proves + # the BYTES, and the two are not the same claim. # - # On the scheduled refresh it writes a CANDIDATE tag instead. A + # The refresh is the case that made this obvious. A # refresh rebuilds against freshly resolved base images, and the web # image's runtime is a line of UNPINNED Debian packages (ffmpeg, # libjpeg62-turbo, libpq5, megatools…) re-resolved on every build. - # Nothing in ci.yml can see that: its lanes run on ci-python:3.14 and + # No verification LANE can see that: they run on ci-python:3.14 and # install requirements.txt, and a base bump changes neither. So # refreshed bytes have to be proven before :latest names them, and # proving needs a moment between "built" and "published" to occupy. @@ -779,14 +1240,17 @@ jobs: # # build-web additionally exposes this as `outputs.candidate`, which is # what gates the `promote` job — a job's `if:` cannot read `env`, and - # one flag is enough because all three derive it from the same - # IS_REFRESH. ml and agent do not re-emit it; a second copy nothing + # one flag is enough because both jobs derive it from the same + # IS_REFRESH. build-agent does not re-emit it; a second copy nothing # reads is the kind of thing that later reads as load-bearing. if [ "${IS_REFRESH:-}" = "true" ]; then echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT" echo "promote=true" >> "$GITHUB_OUTPUT" else - echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT" + # A push builds to a per-channel candidate too (#4310). The channel + # tag moves in `promote`, after smoke-web has booted these bytes — + # so a broken image never reaches the tag deployments follow. + echo "build_ref=$IMAGE:$T-candidate" >> "$GITHUB_OUTPUT" echo "promote=false" >> "$GITHUB_OUTPUT" fi @@ -804,6 +1268,16 @@ jobs: PUBLISHED=$(docker buildx imagetools inspect "$IMAGE:$T" \ --format '{{ index .Image.Config.Labels "fc.revision" }}' \ 2>/dev/null || echo "") + + # The manifest the channel tag names RIGHT NOW. Exported so the + # smoke has something to check on a reuse hit, when this job builds + # nothing and emits no digest of its own (#4323). Resolved here + # because this step is already resolving the tag to read its label — + # one lookup, one answer, rather than a second one that could name a + # different manifest if anything moved the tag in between. + PUBLISHED_DIGEST=$(docker buildx imagetools inspect "$IMAGE:$T" \ + --format '{{ .Manifest.Digest }}' 2>/dev/null || echo "") + echo "published_digest=$PUBLISHED_DIGEST" >> "$GITHUB_OUTPUT" echo "reuse: $IMAGE:$T carries fc.revision=${PUBLISHED:-}; derived=$DERIVED" if [ -z "$PUBLISHED" ] && docker buildx imagetools inspect "$IMAGE:$T" >/dev/null 2>&1; then # The tag resolves but carries no readable label. Expected exactly @@ -1000,9 +1474,11 @@ jobs: # :dev than to the :2026.8.28 tags milestone 318 deleted. (#3114.) cache-from: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator:buildcache cache-to: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator:buildcache,mode=max - # Only the web image carries these: it is the one with a UI and an - # HTTP surface to report them on. The ml and agent images have - # nothing to tell. + # The agent carries its own copy of these (plus FC_REVISION) since + # 2026-09-24. This used to say the agent "has nothing to tell" — true + # of the ml image, which no longer exists (#4311), and never true of + # the agent, which has a control page and a /status endpoint and was + # reporting a hand-written literal on both. build-args: | FC_CHANNEL=${{ steps.tag.outputs.channel }} FC_VERSION=${{ steps.reuse.outputs.version }} @@ -1050,6 +1526,16 @@ jobs: TAGS: ${{ steps.tag.outputs.tags }} run: | set -euf + # A BUILD publishes nothing from here (#4310). Its bytes sit on the + # candidate tag until smoke-web has booted them; `promote` then + # writes the channel tag AND :c- from this run's digest. Writing + # :c- here would publish an immutable rollback tag for bytes + # that might then fail the smoke. + if [ -n "${BUILT_DIGEST:-}" ]; then + echo "repoint: built $BUILT_DIGEST this run — promote publishes it" + echo "repoint: after the smoke; nothing to write here." + exit 0 + fi # WHAT WE COPY FROM, which is not what we EXCLUDE (#4290). # # This step used to copy from the channel tag by NAME. Nothing @@ -1132,9 +1618,9 @@ jobs: # Does the image a refresh just built still work? # - # This is the gate the base refresh never had. `ci.yml` cannot be it: its - # lanes run on ci-python:3.14 and install requirements.txt, and a base bump - # changes neither — all five stay green through a refresh that breaks the + # This is the gate the base refresh never had. The five verification lanes + # cannot be it: they run on ci-python:3.14 and install requirements.txt, and + # a base bump changes neither — all five stay green through a refresh that breaks the # product. What a refresh re-resolves is the Dockerfile's apt layer (ffmpeg, # unar, libpq5, postgresql-client, zstd, megatools, libjpeg62-turbo, # libwebp7, libpng16-16), unpinned, every build. @@ -1143,16 +1629,47 @@ jobs: # source tree, and not a static inspection: `ffmpeg -version` exiting 0 would # pass while a codec removal broke every thumbnail in the library. # - # Refresh-only. On a push the bytes came from a commit, and a commit is what - # ci.yml already tests. - # - # Reports a verdict; it does not yet gate the promote (milestone 362 step 4). - # Landing the gate and the thing it gates in one change would mean the first - # time anyone saw this job run would also be the first time it could stop a - # publish. + # It gates `promote` on every trigger (#4310). The lanes above prove the + # source; they cannot see the bytes, and a Dockerfile or base change breaks + # the bytes without touching the source. smoke-web: needs: [build-web] - if: needs.build-web.outputs.candidate == 'true' + # Every run that actually BUILT something, not just the weekly refresh. + # The egress property (rule 164) is broken by a code or Dockerfile change, + # which is a push — checking it only on the refresh would test it on the + # one trigger that changes no source. + # + # A reuse hit is skipped deliberately: those bytes are already published + # and were smoked when they were built. Re-smoking them would burn two + # minutes to re-learn a fact. + # + # This GATES the publish on every trigger (#4310): builds land on a + # candidate tag, and `promote` needs this job, so :dev / :latest and a + # built :c- move only after it passes. Rule 164's verify_with — the + # check BETWEEN build and push. + # + # NO `if:` — this job always runs (#4323). It used to be gated on the + # build having published something, which skipped it on a reuse hit. That + # sounds like an optimisation and is a hole: this workflow file is in no + # artifact's path set (correctly — editing it changes no shipped byte), so + # a commit that touches ONLY the smoke moves no revision, hits reuse, + # emits no digest, and skips the smoke. The one commit whose purpose is + # changing this check was the one commit that could not run it. + # + # That is not hypothetical twice over. 5ca1058 added the egress sandbox + # and went green three times with this job SKIPPED. 7175ace fixed the + # bug that hid behind those greens (#4319) and needed a manual + # force_build to exercise at all. Both relied on someone remembering. + # + # It is also where the other historical failure lived: on run 5290 this + # expression read `env`, which a job `if:` cannot see, so it evaluated + # empty and skipped silently. Two skips, one expression. The expression + # goes. + # + # The cost is one pull and boot on a reuse-hit push, re-smoking bytes + # that were smoked when they were built. That is the price of a harness + # that tests itself, and it overlaps the lanes above, so little wall-clock + # moves. Not running is not the same as passing. runs-on: python-ci container: image: git.fabledsword.com/bvandeusen/ci-python:3.14 @@ -1176,7 +1693,7 @@ jobs: --health-timeout 5s --health-retries 10 redis: - image: redis:7-alpine + image: redis:8-alpine options: >- --health-cmd "redis-cli ping" --health-interval 10s @@ -1193,13 +1710,16 @@ jobs: env: TOKEN: ${{ secrets.RELEASE_TOKEN }} ACTOR: ${{ github.actor }} + BUILT_DIGEST: ${{ needs.build-web.outputs.digest }} + PUBLISHED_DIGEST: ${{ needs.build-web.outputs.published_digest }} + IS_CANDIDATE: ${{ needs.build-web.outputs.candidate }} run: | set -eux - # Service discovery mirrors ci.yml's integration lane: these jobs run + # Service discovery mirrors the integration lane above: these jobs run # in a container against a mounted docker socket, so the services are # SIBLINGS reachable by IP, not by hostname. PG=$(docker ps --filter "name=smoke" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) - RD=$(docker ps --filter "name=smoke" --filter "ancestor=redis:7-alpine" -q | head -n1) + RD=$(docker ps --filter "name=smoke" --filter "ancestor=redis:8-alpine" -q | head -n1) test -n "$PG" && test -n "$RD" PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD") @@ -1207,7 +1727,7 @@ jobs: # Socket probe in python, not bash's /dev/tcp — these steps run under # `sh -e`, where that path does not exist. Same fix and reasoning as - # ci.yml's integration job; see the comment there. + # the integration lane above; see the comment there. pg_ready="" for i in $(seq 1 60); do if python -c "import socket,sys; s=socket.socket(); s.settimeout(2); sys.exit(0 if s.connect_ex(('$PG_IP', 5432)) == 0 else 1)"; then @@ -1222,10 +1742,124 @@ jobs: fi echo "$TOKEN" | docker login git.fabledsword.com -u "$ACTOR" --password-stdin - CANDIDATE="$IMAGE:refresh-candidate" + # WHAT GETS SMOKED, in order of preference — always a DIGEST, never + # a tag: a tag can move between the build and this job, and then the + # check reports on bytes nobody here decided to ship (#4290). + # + # 1. what this run built, when it built; + # 2. what the channel tag already names, on a reuse hit — the + # digest the reuse step resolved while reading its label. This + # is the case that makes the job able to verify a change to + # ITSELF (#4323), since such a change rebuilds nothing. + # + # A refresh always lands in (1); the candidate tag stays only as the + # last resort for one, and says so rather than being a silent else. + DIGEST="${BUILT_DIGEST:-}" + [ -n "$DIGEST" ] || DIGEST="${PUBLISHED_DIGEST:-}" + if [ -n "$DIGEST" ]; then + CANDIDATE="$IMAGE@$DIGEST" + elif [ "${IS_CANDIDATE:-}" = "true" ]; then + CANDIDATE="$IMAGE:refresh-candidate" + else + # Nothing built and nothing published. There is no artifact this + # job could honestly report on, so it fails rather than passing + # quietly — a guard with nothing to check is not a passing guard. + echo "smoke: FAILED — no image to smoke. build-web neither built" >&2 + echo "smoke: one nor resolved a published digest, so there is" >&2 + echo "smoke: nothing here to verify." >&2 + exit 1 + fi docker pull "$CANDIDATE" - ENVOPTS="-e DB_USER=$DB_USER -e DB_PASSWORD=$DB_PASSWORD -e DB_HOST=$PG_IP" + # --- EGRESS BLOCKED from here (rule 164) --------------------------- + # + # Rule 164 requires a deployed instance to start and serve its full + # UI with NO outbound internet, and says to verify it by removing the + # network rather than by reading the code. Until now this job proved + # the image WORKS; it never proved it works OFFLINE, because every + # container below ran on the runner's default network with the + # internet one hop away. + # + # That gap became load-bearing at milestone 422 step 6. The ML role + # used to run `download_models` before celery started — a boot that + # reached HuggingFace for ~3.5GB — and that fetch moved to a task + # enqueued when the lane is enabled. This check is what proves it + # actually moved, rather than proving it on the machine that built it + # where the model is already cached. + # + # `--internal` is the mechanism rule 164's own verify_with names, and + # `--network none` is explicitly the WRONG check here: it would only + # prove the app fails without a database, which proves nothing about + # egress. An internal network blocks the default route while leaving + # container-to-container traffic and embedded DNS intact, so Postgres + # and Redis stay reachable and nothing else is. + # + # The service containers are SIBLINGS created by the runner, so they + # are attached to the internal network rather than created on it. + # They keep their original network too — that is fine, since what + # must be offline is the APP container, and it is created with only + # this network. + # Named for the RUN, not for `$$`. The shell's pid is deterministic + # in this runner — every execution of this step got 157 — so `$$` + # produced one shared name, and the second run died on "network with + # name smoke-noegress-157 already exists". A pid is unique among + # LIVE processes, which is not the same as unique over time, and in + # a fresh container it is neither. + # The date fallback matters: if the runner does not set these, + # a literal default would put every run back on one shared name + # — the bug, with different letters. + NET="smoke-noegress-${GITHUB_RUN_ID:-$(date +%s)}-${GITHUB_RUN_ATTEMPT:-0}" + + # Sweep anything an earlier run left behind. Needed because the + # cleanup below used to be destroyed before it could fire (see the + # trap note), so every execution leaked its network. `rm` on one + # still in use fails, and `|| true` keeps that harmless — so a + # concurrent run's network survives this. + docker network ls --filter name=^smoke-noegress- -q \ + | while read -r stale; do + docker network rm "$stale" >/dev/null 2>&1 || true + done + + docker network create --internal "$NET" + + # ONE exit trap, for everything. `trap ... EXIT` REPLACES the + # previous handler rather than adding to it, so the network's own + # trap used to be silently discarded the moment the container's was + # installed further down — and the network was never removed. That + # is invisible in a passing run and only ever surfaces on the NEXT + # one, as a name collision. + # + # CID is empty until the app container exists, so this is safe to + # arm now and still covers a failure before that point. + CID="" + CID_ALL="" + cleanup() { + rc=$? + for c in $CID $CID_ALL; do + # The log ONLY on failure — a boot that never answered must fail + # with the reason visible rather than as a bare timeout (rule + # 156), while a green run has nothing to say. + [ $rc -eq 0 ] || docker logs "$c" 2>&1 | tail -40 + docker rm -f "$c" >/dev/null 2>&1 || true + done + docker network rm "$NET" >/dev/null 2>&1 || true + # Preserve the real status, which a trap ending on a successful + # `docker rm` would otherwise mask. + exit $rc + } + trap cleanup EXIT + + docker network connect "$NET" "$PG" + docker network connect "$NET" "$RD" + # Re-read the addresses ON THIS NETWORK. The IPs discovered above + # belong to the runner's default bridge and are not routable from a + # container that is only on the internal one. + PG_IP=$(docker inspect -f "{{(index .NetworkSettings.Networks \"$NET\").IPAddress}}" "$PG") + RD_IP=$(docker inspect -f "{{(index .NetworkSettings.Networks \"$NET\").IPAddress}}" "$RD") + test -n "$PG_IP" && test -n "$RD_IP" + + ENVOPTS="--network $NET" + ENVOPTS="$ENVOPTS -e DB_USER=$DB_USER -e DB_PASSWORD=$DB_PASSWORD -e DB_HOST=$PG_IP" ENVOPTS="$ENVOPTS -e DB_PORT=5432 -e DB_NAME=$DB_NAME -e SECRET_KEY=$SECRET_KEY" ENVOPTS="$ENVOPTS -e CELERY_BROKER_URL=redis://$RD_IP:6379/0" ENVOPTS="$ENVOPTS -e CELERY_RESULT_BACKEND=redis://$RD_IP:6379/0" @@ -1238,6 +1872,22 @@ jobs: # user-facing file mentions this variable at all. ENVOPTS="$ENVOPTS -e CURATOR_BOOTSTRAP_NEW_KEY=1" + # 0. PROVE the network is actually blocking egress. Without this the + # rest is theatre: if `--internal` silently stopped working, or + # the app container picked up a second network, every check below + # would pass with the internet available and report an offline + # boot that never happened. A guard that cannot fail is not a + # guard (rule 167). + echo "smoke: confirming the sandbox has no route out" + if docker run --rm --network "$NET" "$CANDIDATE" shell -c \ + 'python3 -c "import socket,sys; s=socket.socket(); s.settimeout(5); sys.exit(0 if s.connect_ex((\"1.1.1.1\", 443)) == 0 else 1)"'; then + echo "smoke: FAILED — the sandbox reached 1.1.1.1:443." >&2 + echo "smoke: the network is NOT internal, so nothing below would" >&2 + echo "smoke: have tested the offline property (rule 164)." >&2 + exit 1 + fi + echo "smoke: no route out, as required" + # 1. The schema builds from empty, using the image's OWN libpq and # psycopg. This is the same call entrypoint.sh makes before it # serves anything, so a failure here is a failure to boot. @@ -1256,18 +1906,41 @@ jobs: # siblings directly and a published port would collide with # whatever else the runner is hosting. echo "smoke: web boots and answers /api/health" + # Assigning CID is all that is needed — the single EXIT trap armed + # beside the network already covers the container, and it checks CID + # for emptiness precisely so it can be installed before this line. + # Installing a second trap here is what used to discard the first. CID=$(docker run -d $ENVOPTS "$CANDIDATE" web) - # Clean up the container however this ends, and dump its log ONLY - # on failure — a boot that never answers must fail with the reason - # visible rather than as a bare timeout (rule 156), while a green run - # has nothing to say. `exit $rc` preserves the real status, which a - # trap that ends on a successful `docker rm` would otherwise mask. - trap 'rc=$?; [ $rc -eq 0 ] || docker logs "$CID" 2>&1 | tail -40; docker rm -f "$CID" >/dev/null 2>&1 || true; exit $rc' EXIT WEB_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$CID") test -n "$WEB_IP" + + # The probe runs INSIDE the sandbox, like every check above it. + # + # It has to. Docker gives an `--internal` network isolation rules + # that DROP traffic entering it from any other interface, and this + # job's own container sits on the runner's default bridge — so a + # curl from here to $WEB_IP is discarded before it arrives. Because + # the packets are dropped rather than refused, every attempt burns + # the full --max-time and the job reports a web container that + # "never answered" while the app is running perfectly. That is how + # this read on its first real execution (run 7282): a false failure + # blaming the application for the harness's own blind spot. + # + # Steps 0-2 were already right by accident — each runs a container + # ON $NET. Only this one reached in from outside, and it was the + # only one that could not work. + # + # Same shape as the egress probe above: the image's own python3 over + # `shell -c`, since the runtime stage ships no curl. + probe() { + docker run --rm --network "$NET" "$CANDIDATE" shell -c \ + "python3 -c \"import urllib.request; urllib.request.urlopen('http://$WEB_IP:8080/api/health', timeout=5)\"" \ + >/dev/null 2>&1 + } + healthy="" for i in $(seq 1 60); do - if curl -fsS --max-time 5 "http://$WEB_IP:8080/api/health" >/dev/null 2>&1; then + if probe; then healthy=1 break fi @@ -1293,10 +1966,77 @@ jobs: echo "smoke: python base rather than at startup." >&2 exit 1 fi - curl -fsS --max-time 5 "http://$WEB_IP:8080/api/health" - echo + # Print what it actually answered — from inside, for the same reason. + docker run --rm --network "$NET" "$CANDIDATE" shell -c \ + "python3 -c \"import urllib.request; print(urllib.request.urlopen('http://$WEB_IP:8080/api/health', timeout=5).read().decode())\"" - echo "smoke: all checks passed against $CANDIDATE" + # 4. THE DEFAULT ROLE. Everything above boots `web` explicitly, so + # until now nothing had ever started the shape the single- + # container layout actually runs — supervisord bringing up + # hypercorn plus one celery process per lane. The milestone is + # named for that shape and CI had never once executed it. + # + # Deliberately NO command: this is `docker run ` with + # nothing after it, so it checks the Dockerfile's CMD and the + # entrypoint's default together with the role itself. An adopter + # who writes no `command:` gets exactly this. + # + # The image's own `healthcheck` is the right assertion, and it + # was itself never run: for the `all` role it passes only when + # hypercorn answers AND every lane in the table is answering. A web-only check would go + # green with every worker dead, which is the failure mode + # consolidation creates. + # + # Cheap because ml ships at 0 slots and disabled, so nothing + # loads a model — the lane answers `inspect` with its consumers + # cancelled, which is what "healthy" means for a disabled lane. + echo "smoke: the default role boots every lane under supervisord" + CID_ALL=$(docker run -d $ENVOPTS "$CANDIDATE") + lanes_up="" + for i in $(seq 1 60); do + if docker exec "$CID_ALL" python -m backend.app.scripts.healthcheck; then + lanes_up=1 + break + fi + if [ "$(docker inspect -f '{{.State.Running}}' "$CID_ALL" 2>/dev/null)" != "true" ]; then + echo "smoke: FAILED — the all-role container exited during boot." >&2 + exit 1 + fi + sleep 3 + done + if [ -z "$lanes_up" ]; then + echo "smoke: FAILED — supervisord came up but the composite" >&2 + echo "smoke: healthcheck never passed: either hypercorn did not" >&2 + echo "smoke: answer or a lane is not consuming. Its log follows." >&2 + exit 1 + fi + echo "smoke: every lane answered" + + # PID 1 is an init, and the IMAGE provides it — no `init: true` in + # whatever runs this. Read from /proc rather than `ps`, which the + # runtime stage does not install. + PID1=$(docker exec "$CID_ALL" cat /proc/1/comm) + echo "smoke: pid 1 is $PID1" + if [ "$PID1" != "tini" ]; then + echo "smoke: FAILED — pid 1 is '$PID1', not an init." >&2 + echo "smoke: orphaned gallery-dl/ffmpeg/pg_dump processes would" >&2 + echo "smoke: accumulate as zombies, and the image would be back" >&2 + echo "smoke: to needing init:true from every deployment." >&2 + exit 1 + fi + # Say WHICH processes supervisord is running, so a lane that is + # merely restart-looping is visible rather than inferred. + # + # NOT `|| true` any more. Behind that, this printed + # "Error: .ini file does not include supervisorctl section" for a + # whole run and passed — supervisord was fine and supervisorctl + # could not reach it, which is the tool an operator debugging a lane + # inside the one container reaches for first. A diagnostic allowed + # to fail silently is a diagnostic that stops being true without + # telling anyone. + docker exec "$CID_ALL" supervisorctl -c "${SUPERVISOR_CONF:-/tmp/supervisord.conf}" status + + echo "smoke: all checks passed against $CANDIDATE, with egress blocked" # Move the channel tags — the whole point of the gate. # @@ -1311,576 +2051,151 @@ jobs: # a FAILED gate blocks would have published unverified images while reporting # success. Not running is not the same as passing. # - # All three images promote TOGETHER, or none do. They are one stack: build.yml - # already refuses to publish a :dev web image beside a stale :dev ml, because - # the mismatch only shows up as a runtime failure. A refresh that published ml - # and withheld web would be that same trap, arrived at through the gate. + # BOTH images promote TOGETHER, or neither does. They are one stack: + # build.yml already refuses to publish a :dev web image beside a stale :dev + # agent, because the mismatch only shows up as a runtime failure. A refresh + # that published the agent and withheld web would be that same trap, arrived + # at through the gate. # # The gate covers the web image only (milestone 362 step 3 scoped it there), - # so ml and agent are being held to web's verdict rather than their own. That - # is deliberate and it is the conservative direction — they ship together, so - # the weakest evidence should govern all three — but it is not the same as - # having smoked them, and it should not be read as if it were. + # so the agent is being held to web's verdict rather than its own. That is + # deliberate and it is the conservative direction — they ship together, so + # the weakest evidence should govern both — but it is not the same as having + # smoked it, and it should not be read as if it were. promote: - needs: [build-web, build-ml, build-agent, smoke-web] - # Only a refresh publishes through a candidate; a push writes its channel - # tag directly from the build. Reads the same reuse-step decision the build - # took, via a job output — a job's `if:` cannot see the `env` context. - if: needs.build-web.outputs.candidate == 'true' + needs: [build-web, build-agent, smoke-web] + # THE PUBLISH (#4310). Both image jobs build to a candidate tag — a push to + # :dev-candidate / :latest-candidate, the weekly refresh to + # :refresh-candidate — and nothing names those bytes under a tag anybody + # deploys until this job runs. It runs only when every job in `needs` + # SUCCEEDED (a job-level `if:` without a status function implies + # success()), so a failed smoke — or a skipped one, which is not the same + # as a passing one (run 5290) — leaves :dev / :latest on the last build + # that worked. Rule 164's verify_with: the check sits BETWEEN build and + # push. + # + # Per image, from the DIGEST that image's job built (#4290): a tag can + # move between the build and this job, a digest cannot. An image whose job + # hit reuse built nothing and has nothing to promote — its build job + # already wrote its :c- from the channel tag, which "hit" proved + # carries this commit. + if: github.event_name != 'pull_request' runs-on: python-ci container: image: git.fabledsword.com/bvandeusen/ci-python:3.14 steps: - - name: Point the channel tags at the smoked candidates + - name: Point this commit's tags at the smoked builds env: TOKEN: ${{ secrets.RELEASE_TOKEN }} ACTOR: ${{ github.actor }} + WEB_DIGEST: ${{ needs.build-web.outputs.digest }} + WEB_TAGS: ${{ needs.build-web.outputs.tags }} + AGENT_DIGEST: ${{ needs.build-agent.outputs.digest }} + AGENT_TAGS: ${{ needs.build-agent.outputs.tags }} run: | set -eu - # `latest` is not a guess: a refresh always builds `main` (BUILD_REF), - # and the "must have checked out main" guard in every build job fails - # the run if that did not hold. So the channel is main's. - TAG=latest FAILED="" + MOVED=0 - for NAME in fabledcurator fabledcurator-ml fabledcurator-agent; do + promote() { + NAME="$1"; DIGEST="$2"; TAGS="$3" REPO="bvandeusen/$NAME" - echo "promote: $REPO" + if [ -z "$DIGEST" ]; then + echo "promote: $NAME — no build this run (reuse hit); nothing to publish" + return 0 + fi + echo "promote: $NAME $DIGEST" # Registry auth is its own token exchange — `docker login` # authenticates the docker client, not curl. Deadline on every call # (rule 156): a registry that stops answering must fail this step, - # not hang the weekly refresh until the job times out. + # not hang the run until the job times out. BEARER=$(curl -fsS --max-time 30 -u "$ACTOR:$TOKEN" \ "https://git.fabledsword.com/v2/token?scope=repository:$REPO:pull,push&service=git.fabledsword.com" \ | python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])') # Ask for the IMAGE manifest media types only. Offering the index - # types too would let the registry hand back an index if one ever - # existed at this tag, and we would faithfully copy the thing this - # whole approach exists to avoid creating. + # types too would let the registry hand back an index, and we would + # faithfully copy the thing this approach exists to avoid creating. ACCEPT='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' CT=$(curl -fsS --max-time 60 -o manifest.json -D headers.txt \ -H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \ - "https://git.fabledsword.com/v2/$REPO/manifests/refresh-candidate" \ + "https://git.fabledsword.com/v2/$REPO/manifests/$DIGEST" \ && tr -d '\r' < headers.txt | awk -F': ' '/^[Cc]ontent-[Tt]ype:/{print $2}') test -n "$CT" - SRC=$(tr -d '\r' < headers.txt | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}') - echo "promote: candidate $SRC ($CT)" # NOT `imagetools create`. That wraps its source in an INDEX, and # `.Image.Config.Labels` does not resolve through one — the # fc.revision the reuse check reads off the channel tag would come # back empty, every later push would miss and rebuild, and nothing # would go red (#3183, run 4751). A manifest PUT is what "make this - # tag name that image" means at the registry: same bytes, same media - # type, same digest, no layer transfer. - curl -fsS --max-time 120 -X PUT \ - -H "Authorization: Bearer $BEARER" -H "Content-Type: $CT" \ - --data-binary @manifest.json \ - "https://git.fabledsword.com/v2/$REPO/manifests/$TAG" + # tag name that image" means at the registry: same bytes, same + # media type, same digest, no layer transfer. (It also makes a + # built :c- a plain image rather than the index the old + # repoint left.) + IFS=, + for REF in $TAGS; do + TAG="${REF##*:}" + curl -fsS --max-time 120 -X PUT \ + -H "Authorization: Bearer $BEARER" -H "Content-Type: $CT" \ + --data-binary @manifest.json \ + "https://git.fabledsword.com/v2/$REPO/manifests/$TAG" - # Read it back. A PUT that returned 2xx but landed something else is - # exactly the silent-and-plausible failure this pipeline keeps - # producing, and the check costs one request. - NOW=$(curl -fsS --max-time 30 -o /dev/null -D - \ - -H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \ - "https://git.fabledsword.com/v2/$REPO/manifests/$TAG" \ - | tr -d '\r' | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}') - if [ "$NOW" != "$SRC" ]; then - echo "promote: FAILED — $NAME:$TAG is $NOW, expected $SRC" >&2 - FAILED="$FAILED $NAME" - continue - fi - echo "promote: $NAME:$TAG now names $NOW" - done + # Read it back. A PUT that returned 2xx but landed something else + # is exactly the silent-and-plausible failure this pipeline keeps + # producing, and the check costs one request. + NOW=$(curl -fsS --max-time 30 -o /dev/null -D - \ + -H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \ + "https://git.fabledsword.com/v2/$REPO/manifests/$TAG" \ + | tr -d '\r' | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}') + if [ "$NOW" != "$DIGEST" ]; then + echo "promote: FAILED — $NAME:$TAG is $NOW, expected $DIGEST" >&2 + FAILED="$FAILED $NAME:$TAG" + continue + fi + echo "promote: $NAME:$TAG now names $NOW" + MOVED=$((MOVED + 1)) + done + unset IFS + } + + promote fabledcurator "$WEB_DIGEST" "$WEB_TAGS" + promote fabledcurator-agent "$AGENT_DIGEST" "$AGENT_TAGS" if [ -n "$FAILED" ]; then echo "" >&2 echo "promote: FAILED for:$FAILED" >&2 - echo "promote: the channel tags are now INCONSISTENT — some images" >&2 - echo "promote: moved and some did not. Re-run this refresh; the" >&2 - echo "promote: candidates are still published and the promote is" >&2 - echo "promote: idempotent." >&2 + echo "promote: the tags are now INCONSISTENT — some moved and some" >&2 + echo "promote: did not. Re-run this workflow; the builds are still" >&2 + echo "promote: published by digest and the promote is idempotent." >&2 exit 1 fi - echo "promote: all three channel tags moved" - - build-ml: - runs-on: python-ci - container: - image: git.fabledsword.com/bvandeusen/ci-python:3.14 - steps: - - uses: actions/checkout@v4 - with: - # Not the triggering ref — see the `env:` block at the top. On a - # scheduled refresh this is `main`; on everything else it is the ref - # that fired, so this is a no-op on every ordinary path. - ref: ${{ env.BUILD_REF }} - # Full history: this job derives its artifact's version from the - # commit its shipped files last changed in (milestone 313). A - # depth-1 clone cannot see that commit — it either derives a wrong, - # too-low value or finds nothing at all, and neither is a failure - # the build would otherwise notice. - fetch-depth: 0 - - # See sign-extension's copy for why this guard exists. - - name: Guard — a scheduled run must have checked out main - if: env.IS_REFRESH == 'true' - run: | - set -eu - BRANCH=$(git rev-parse --abbrev-ref HEAD) - echo "schedule: HEAD is $BRANCH ($(git rev-parse --short HEAD))" - if [ "$BRANCH" != "main" ]; then - echo "schedule: expected main, got '$BRANCH'." >&2 - echo "schedule: BUILD_REF was not honoured by the runner." >&2 - echo "schedule: refusing to publish a channel tag from it." >&2 - exit 1 - fi - - # --- derived values, one line (milestone 313) ------------------------ - # These stopped being shadow output at step 3. `revision` decides - # whether the build below runs at all and `version` is what the image - # reports about itself; the load-bearing steps each print only the one - # they use, so this is the only place the pair appears together. When a - # build is skipped, this is the line that says what the commit derived. - # - # Still diagnostic, so it still must not fail the build — no `set -e`, - # and every derivation falls back to UNAVAILABLE. A broken echo must - # never be the reason an image does not ship. - # - # What it should say: - # * a push touching only agent/ moves the agent and leaves web and ml - # STILL. If web moves, its path set is too wide. - # * a push touching only docs moves nothing. - # * a push touching the extension moves the extension AND web, since - # web bakes in the XPI. If web does not move, its set is too narrow: - # the reuse check hits, and the channel serves a web image bundling - # the PREVIOUS XPI while the freshly signed one is orphaned (#3156). - # * dev and main derive the same values for the same source. - - name: Report the derived artifact version - env: - # Diagnostic for the trigger normalisation. `refresh` is reported RAW - # as well as normalised, because the two disagreeing is the whole - # failure mode: a dispatch input whose type does not compare the way - # the expression assumes evaluates to false silently, and the only - # symptom is a refresh that quietly behaves like an ordinary push. - RAW_REFRESH: ${{ github.event.inputs.refresh }} - RAW_FORCE: ${{ github.event.inputs.force_build }} - run: | - set -u - echo "trigger: event=$GITHUB_EVENT_NAME IS_REFRESH='${IS_REFRESH:-}' BUILD_REF='${BUILD_REF:-}'" - echo "trigger: raw inputs refresh='${RAW_REFRESH:-}' force_build='${RAW_FORCE:-}'" - A=ml - V=$(sh scripts/artifacts.sh version "$A" 2>&1 || echo UNAVAILABLE) - R=$(sh scripts/artifacts.sh revision "$A" 2>&1 || echo UNAVAILABLE) - echo "derived: artifact=$A version=$V revision=$R sha=$GITHUB_SHA" - - - name: Determine tag - id: tag - run: | - # Mirrors build-web's tag list; see the comment there. - # POSIX-safe substring (the runner shell is dash/BusyBox sh, not - # bash — `${var:0:7}` errors with "Bad substitution"; cut works - # everywhere). Operator-flagged 2026-06-01 after first :c- - # main-push build failed at this step. - SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7) - # Mirrors build-web's tag list and its schedule handling; see - # the comments there. - if [ "${IS_REFRESH:-}" = "true" ]; then - echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:latest" >> "$GITHUB_OUTPUT" - echo "channel=main" >> "$GITHUB_OUTPUT" - elif [ "${GITHUB_REF##*/}" = "main" ]; then - echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:latest,git.fabledsword.com/bvandeusen/fabledcurator-ml:c-${SHORT_SHA}" >> "$GITHUB_OUTPUT" - echo "channel=main" >> "$GITHUB_OUTPUT" - else - echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:dev" >> "$GITHUB_OUTPUT" - echo "channel=dev" >> "$GITHUB_OUTPUT" - fi - - # Shell step rather than docker/login-action — see build-web's note on - # the shared action-cache race (#3118). - - name: Login to Forgejo registry - env: - TOKEN: ${{ secrets.RELEASE_TOKEN }} - ACTOR: ${{ github.actor }} - run: echo "$TOKEN" | docker login git.fabledsword.com -u "$ACTOR" --password-stdin - - # A REAL buildx builder, not the default `docker` driver (#3114, #3190). - # - # The default driver builds through the local dockerd. It cannot export a - # registry cache at all — which is why the agent rebuilds a ~6.3 GB CUDA - # + torch image from scratch whenever the runner's local cache is cold, - # measured at 9m26s against 7s warm. It is also #3190's leading suspect: - # after a registry-direct push it resolves image metadata against a local - # store the push never filled, and reports `No such image` on an image - # that published perfectly well three seconds earlier. - # - # These jobs run INSIDE a container against a mounted docker socket, so - # the buildkit container this starts is a SIBLING of the job container, - # not a child. That works over the socket mount; it had never been tried - # here before milestone 326 step 1. - - name: Set up buildx - uses: docker/setup-buildx-action@v3 - - # --- reuse-if-published (milestone 313, step 4) ---------------------- - # Does the image the channel tag already points at carry THIS commit's - # revision? If so the bytes this job would produce are already published - # and the build is pure waste: the remaining tags get repointed at that - # existing manifest instead, registry-side, in seconds. - # - # Keyed on an `fc.revision` LABEL rather than on a tag of its own - # (milestone 318 step 3). A tag would be a name minted per build that one - # thing reads — what rule 145 narrowed against — and would be prunable - # under the registry's keep_pattern (#3157), silently expiring the cache. - # A label rides inside a tag that has to exist anyway. - # - # An image with no such label reads as a miss and rebuilds. That is the - # migration, not a fault: labels cannot be backfilled, since the reuse - # path copies a manifest and config labels are not manifest annotations. - # Each artifact pays one rebuild, once. - # - # This is what stops a push that touched only `agent/` from rebuilding - # web and ml, and a merge to main from rebuilding what dev already built. - # - # The failure direction is deliberate. An inspect that errors for ANY - # reason — network, auth, a registry hiccup — reads as a miss and the - # build runs. Only a genuine 200 skips one, so there is no path here - # that skips a build that was actually needed; the worst case is paying - # for a build we could have avoided. - # - # BASE-IMAGE FRESHNESS: an artifact whose source stops moving stops - # picking up base-image updates. Milestone 318 removed the argument this - # used to need rather than answering it — with no version tags there is - # no immutable name a refresh could contradict, and rule 145 already - # allows a rebuild with different contents to republish a MOVING tag. - # So a refresh is just a build. A scheduled channel-only one is tracked - # separately (#3154); it does not belong in the push path. - - name: Is this content already published? - id: reuse - env: - IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-ml - CHANNEL: ${{ steps.tag.outputs.channel }} - # Empty on a push; the string "true" only from a workflow_dispatch - # that asked for it. `github.event.inputs` rather than the `inputs` - # context — release.yml already uses that form, and it is the one - # this runner is known to evaluate. Read through env rather than - # interpolated into the run block, same rule as release.yml's TAG. - FORCE: ${{ github.event.inputs.force_build }} - # A scheduled refresh has to bypass reuse by construction: it - # rebuilds the SAME source, so fc.revision always matches and the - # check would skip every refresh there has ever been. - run: | - set -eu - DERIVED=$(sh scripts/artifacts.sh revision ml) - echo "revision=$DERIVED" >> "$GITHUB_OUTPUT" - - # The build clock, pinned to the same commit (#3265). Without it - # buildkit stamps the image config with the wall clock of the build, - # so identical layers republish under a new config blob and the - # channel tag gets a new manifest digest for no reason. Derived from - # `newest()` like revision and version, so all three name one commit - # and cannot drift apart. - echo "epoch=$(sh scripts/artifacts.sh epoch ml)" >> "$GITHUB_OUTPUT" - - # The moving tag for this channel. Which tag we ask IS the channel — - # that is why the revision needs no -main/-dev qualifier any more. - if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi - echo "channel_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT" - - # WHERE THE BUILD PUBLISHES, which is not always the channel — and - # whether the channel then has to be written separately. - # - # On a push the build writes the channel tag directly: the bytes came - # from a commit, and a commit is the thing CI tests. Nothing to hold - # it behind. - # - # On the scheduled refresh it writes a CANDIDATE tag instead. A - # refresh rebuilds against freshly resolved base images, and the web - # image's runtime is a line of UNPINNED Debian packages (ffmpeg, - # libjpeg62-turbo, libpq5, megatools…) re-resolved on every build. - # Nothing in ci.yml can see that: its lanes run on ci-python:3.14 and - # install requirements.txt, and a base bump changes neither. So - # refreshed bytes have to be proven before :latest names them, and - # proving needs a moment between "built" and "published" to occupy. - # This is that moment; :latest goes on naming the build that works - # until something says otherwise. - # - # `:refresh-candidate` is one moving ref per image, overwritten in - # place, holding a build nobody is told to pull — the shape rule 145 - # already allows for :buildcache, not the per-build tag family that - # milestone 318 withdrew. - # - # Decided HERE, beside `hit`, for the reason the force/schedule - # branch below gives: one step decides what this job does. A - # condition derived independently could disagree with the tag the - # build actually wrote. - # - # build-web additionally exposes this as `outputs.candidate`, which is - # what gates the `promote` job — a job's `if:` cannot read `env`, and - # one flag is enough because all three derive it from the same - # IS_REFRESH. ml and agent do not re-emit it; a second copy nothing - # reads is the kind of thing that later reads as load-bearing. - if [ "${IS_REFRESH:-}" = "true" ]; then - echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT" - else - echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT" - fi - - # Compare VALUES, never exit codes. Measured on buildx v0.36.1 - # (run 4732): a missing key returns an empty string and exits 0, so - # branching on the exit code would read "no label yet" as success. - # An unreachable tag also lands here as empty via the `|| echo`. - # Empty never equals a 12-char revision, so every uncertain case - # falls through to a build — the safe direction, with no special - # casing for it. - # - # Read the SPECIFIC key. The map also carries whatever the base image - # set, and `org.opencontainers.image.version` sits right beside ours - # looking like a plausible answer (it reads 24.04 on the agent). - PUBLISHED=$(docker buildx imagetools inspect "$IMAGE:$T" \ - --format '{{ index .Image.Config.Labels "fc.revision" }}' \ - 2>/dev/null || echo "") - echo "reuse: $IMAGE:$T carries fc.revision=${PUBLISHED:-}; derived=$DERIVED" - if [ -z "$PUBLISHED" ] && docker buildx imagetools inspect "$IMAGE:$T" >/dev/null 2>&1; then - # The tag resolves but carries no readable label. Expected exactly - # once per artifact, during the migration onto labels. If it recurs - # every push, something is rewriting the channel tag as a manifest - # index — see the repoint step's note. - echo "reuse: NOTE $IMAGE:$T exists but has no readable fc.revision." - echo "reuse: NOTE Fine once, while migrating. Every push means the" - echo "reuse: NOTE tag is being index-wrapped and reuse is dead." - fi - - # FORCE is checked here rather than in the build step's `if:`, so - # that one decision drives everything downstream. The repoint step - # keys off `hit` too, and a force that bypassed only the build would - # leave the two disagreeing about what just happened. - if [ "${FORCE:-false}" = "true" ]; then - echo "hit=false" >> "$GITHUB_OUTPUT" - echo "reuse: force_build set — building regardless" - elif [ "${IS_REFRESH:-}" = "true" ]; then - echo "hit=false" >> "$GITHUB_OUTPUT" - echo "reuse: scheduled base refresh — building regardless" - elif [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then - echo "hit=true" >> "$GITHUB_OUTPUT" - echo "reuse: already published — skipping the build" - else - echo "hit=false" >> "$GITHUB_OUTPUT" - echo "reuse: not published — building" - fi - - - name: Build and push ml image - # `id:` so the repoint step below can read `outputs.digest` — the - # manifest THIS run published, as opposed to whatever the channel tag - # happens to name by the time that step runs (#4290). - id: build - if: steps.reuse.outputs.hit != 'true' - # Read by buildx out of the ENVIRONMENT, not passed as a build-arg — - # it normalises the image config's `created` field and the history - # timestamps rather than being consumed by the Dockerfile. See #3265 - # and the reuse step's `epoch` output. - env: - SOURCE_DATE_EPOCH: ${{ steps.reuse.outputs.epoch }} - uses: docker/build-push-action@v5 - with: - context: . - file: Dockerfile.ml - push: true - # Re-resolve the FROM references against the registry instead of - # trusting whatever digest the cache was built against. This is the - # whole mechanism of the scheduled refresh (#3154): if the base tag - # moved, the FROM layer's cache key changes, every layer above it - # invalidates, and the image genuinely rebuilds. - # - # MEASURED on the first real fire, run 4934 (#3265): when the base - # did NOT move, the build was ~13s with every content step CACHED — - # and the channel tag STILL got a new manifest digest, because - # buildkit stamps a fresh image config per run and republishes the - # identical layers under it. All three images moved that way on - # 2026-08-30 with nothing whatsoever changed in them. - # - # SOURCE_DATE_EPOCH (below) is the fix: pinned to the commit the - # content came from, the config is byte-identical across runs, so - # the manifest digest is too and the push is a registry no-op. A - # digest change means the content changed again, which is the only - # thing a digest is any use for. - # - # What `pull` does NOT catch either: a Debian package update inside - # the `apt-get install` layer while the base tag itself stands - # still. The official python/cuda images rebuild with those updates - # baked in, so this is a lag rather than a hole; closing it needs - # `no-cache: true`, which is a much larger version of the same - # churn #3265 is about. - # - # Only on the schedule. An ordinary push wants the cached base. - pull: ${{ env.IS_REFRESH == 'true' }} - # ONE tag, the channel's. Every other tag is written by the step - # below, registry-side. buildx here pushes the first tag to the - # registry and then re-pushes the rest through the DOCKER driver, - # out of a local image store a registry-direct build never filled — - # #3190, which cost `main` its :c- on 2026-08-29 while :latest - # published perfectly well. - tags: ${{ steps.reuse.outputs.build_ref }} - # The reuse key. Read back off the channel tag on the next push to - # decide whether that push needs to build at all, so this is not - # decoration — an unstamped image is one that will always rebuild. - labels: | - fc.revision=${{ steps.reuse.outputs.revision }} - # LOAD-BEARING, not a preference. On the default docker driver these - # were no-ops; on the docker-container driver above, - # build-push-action@v5 defaults provenance to TRUE when pushing. - # Provenance attaches an attestation manifest, which makes the pushed - # tag a manifest INDEX — and `.Image.Config.Labels` does not resolve - # through an index. - # - # The label directly above IS the reuse key. Wrap the channel tag in - # an index and the next push reads fc.revision=, misses, and - # rebuilds. Then so does the one after that, forever. Nothing fails, - # nothing goes red, and the only symptom is the bill. That is #3183 - # arriving through a different door, and note #3127 §4 records the - # same shape for `platforms:`. - provenance: false - sbom: false - # The ONLY cache this driver can have. `docker-container` gets a - # FRESH buildkit instance per job, so unlike the default docker - # driver it has no local layer store to fall back on — measured on - # run 4896, the first builds after the driver change: web 3m44s - # (was 2m23s), ml 3m49s (was 3m20s), agent 11m12s (was 9m26s). The - # driver change ALONE is a regression; this is the other half of it. - # - # mode=max so intermediate stages cache too. web's frontend-builder - # stage and the agent's two ~150s pip layers are the whole cost, and - # they are exactly what a min-mode cache would drop. - # - # A `:buildcache` tag is NOT the withdrawn tag scheme coming back. - # Rule 145 narrowed against names NOTHING reads; this one is read by - # every build that runs, is one moving ref per image rather than one - # per build, holds cache blobs rather than a shippable artifact, and - # is overwritten in place rather than accumulating. It is closer to - # :dev than to the :2026.8.28 tags milestone 318 deleted. (#3114.) - cache-from: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator-ml:buildcache - cache-to: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator-ml:buildcache,mode=max - - # Every tag but the channel's own is written HERE, registry-side, - # whether or not a build ran. Each -t becomes another reference to the - # SAME manifest the channel tag holds, so :c- is byte-identical to - # what is published rather than a lookalike rebuild. - # - # Owning the build path too is #3190's fix, not a tidy-up: - # - # #27 pushing …/fabledcurator:latest DONE 15.8s - # #28 pushing …/fabledcurator:c-0e15c44 with docker - # #28 ERROR: tag does not exist: …:c-0e15c44 - # - # Intermittent — build-ml made the identical two-tag push seconds later - # and succeeded — and worse than it looks. `:latest` had already - # published, so production was correct while the immutable rollback tag - # rule 145 requires of every main push simply did not exist. Nothing but - # the red job would ever have noticed: a missing :c- has no - # consumer that fails, so it surfaces when somebody needs to roll back. - # - # `imagetools create` is a registry-side manifest copy — no layer - # transfer, no local daemon, nothing that can be absent. The reuse case - # has always gone this way, so this puts the build case on the code that - # was already proven rather than on a second path. - # - # Running on every path also keeps family rule 146 true: a rolling - # channel refreshes itself, so skipping a build must never leave :dev or - # :latest pointing at something older than the commit just pushed. - # - # The cost, accepted knowingly: `imagetools create` wraps its source in - # an index, so :c- becomes an index and fc.revision does not - # resolve through it. Nothing reads that label off :c- — the reuse - # check only ever inspects the CHANNEL tag — and the index names the - # same manifest, so a pull is byte-identical. The reuse path already - # produced :c- this way; this only makes it uniform. - - name: Write the remaining tags from the published image - env: - IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-ml - CHANNEL_REF: ${{ steps.reuse.outputs.channel_ref }} - # Empty when no build ran this job (a reuse hit, or the step's `if:` - # skipped it). Non-empty means THIS run pushed that manifest. - BUILT_DIGEST: ${{ steps.build.outputs.digest }} - TAGS: ${{ steps.tag.outputs.tags }} - run: | - set -euf - # WHAT WE COPY FROM, which is not what we EXCLUDE (#4290). - # - # This step used to copy from the channel tag by NAME. Nothing - # serialises builds — there is no `concurrency:` key anywhere in - # .forgejo/workflows/ — so two pushes to one branch run in full - # parallel, both miss the reuse check, and both build. If the OLDER - # one finishes last it wins the channel tag; and then its repoint - # step, reading that tag by name, wrote :c- from whatever the - # other run had just published. An immutable rollback tag (rule 145) - # naming a different commit's bytes, wrong from birth — and - # immutability then guarantees nobody ever corrects it. Nothing goes - # red; it surfaces the day someone needs to roll back. - # - # So when this job built, copy from the DIGEST it pushed. Correct - # whatever a concurrent run does to the tag, and it does not depend - # on the runner honouring a `concurrency:` key — which this file has - # already been burned by once (the `format()` note at the top: an - # expression that evaluated false with no symptom at all). - # - # On a reuse hit there is no digest, and the channel tag is still the - # right source: "hit" MEANS that tag already carries this commit's - # fc.revision, which the reuse step verified by reading it. - if [ -n "${BUILT_DIGEST:-}" ]; then - SOURCE="$IMAGE@$BUILT_DIGEST" - echo "repoint: copying the digest this run published: $SOURCE" - else - SOURCE="$CHANNEL_REF" - echo "repoint: no build this run (reuse hit) — copying from $SOURCE" - fi - # The source tag is EXCLUDED from the targets, and that is load- - # bearing rather than an optimisation. - # - # `imagetools create` wraps the source manifest in an INDEX. Point it - # at the channel tag with that same tag as a target and the tag stops - # being a plain image — after which `.Image.Config.Labels` no longer - # resolves through it and the fc.revision label reads as absent. The - # next push then misses and rebuilds, so reuse worked exactly once - # and every subsequent push paid full price. Observed on run 4751: - # ml:dev reported fc.revision= one push after run 4749 had read - # a7e626a67a79 off it. Nothing failed; the savings just evaporated. - # - # Excluding the source means the channel tag is only ever written - # by a real build, so it stays a plain image and stays readable. - # On dev that leaves nothing to do either way: the build pushed :dev - # itself, or the hit established it was already right. On main it - # leaves :c-, which rule 145 requires of every main push whether - # or not a build ran. - # - # steps.tag emits ONE comma-separated list; imagetools wants a -t per - # ref. (That list used to feed docker/build-push-action directly — - # which is exactly what #3190 made unsafe.) - ARGS="" - IFS=, - for t in $TAGS; do - # Keyed on CHANNEL_REF, never on SOURCE. SOURCE may now be a digest - # ref, which never equals a tag string — testing against it would - # stop excluding the channel tag, imagetools would index-wrap it, - # and `.Image.Config.Labels` would stop resolving through it. That - # kills the reuse label permanently (see the note just below). - [ "$t" = "$CHANNEL_REF" ] && continue - ARGS="$ARGS -t $t" - done - unset IFS - if [ -z "$ARGS" ]; then - echo "repoint: $CHANNEL_REF is the only tag for this channel and" - echo "repoint: already holds this revision — nothing to write." - exit 0 - fi - # shellcheck disable=SC2086 - docker buildx imagetools create $ARGS "$SOURCE" - echo "repointed from $SOURCE:$ARGS" - - # The desktop GPU agent (#114) — published so the operator pulls + runs it on - # the GPU machine instead of building locally. Independent of web/ml (its own - # CUDA + onnxruntime-gpu image, context = agent/). Same tag cadence. + echo "promote: $MOVED tag(s) written" build-agent: + # THE GATE (2026-09-23). Every lane above must have PASSED before this job + # exists at all — so a red suite does not produce an image, let alone push + # one. Nothing here is a new mechanism: it is the same `needs:` edge that + # has gated `promote` since milestone 362 step 4, and it carries that + # step's hardest-won property unchanged — **not running is not the same as + # passing.** `needs` treats a SKIPPED dependency as unsatisfied, so a lane + # that silently skips itself blocks the publish exactly as a failing one + # does. Run 5290 is why that is worth saying out loud: `smoke-web` skipped + # itself through a job-level `if:` that could not read `env`, and a design + # where only a FAILED gate blocks would have published unverified images + # while reporting success. + needs: [lint, extension-version, backend-lint-and-test, frontend-build, extension-test, + integration] + # What `promote` needs to publish this image once the smoke has passed: + # the manifest this run built (empty on a reuse hit) and every tag it + # belongs under. Same meaning as build-web's outputs of the same names. + outputs: + digest: ${{ steps.build.outputs.digest }} + tags: ${{ steps.tag.outputs.tags }} + # A pull_request run is the lanes and nothing else. This is the ONLY thing + # separating "validate a Renovate bump" from "publish a Renovate bump", so + # it is stated on each publishing job rather than inferred from a `needs` + # chain that a later edit could quietly break. + if: github.event_name != 'pull_request' runs-on: python-ci container: image: git.fabledsword.com/bvandeusen/ci-python:3.14 @@ -1888,8 +2203,8 @@ jobs: - uses: actions/checkout@v4 with: # Not the triggering ref — see the `env:` block at the top. On a - # scheduled refresh this is `main`; on everything else it is the ref - # that fired, so this is a no-op on every ordinary path. + # scheduled refresh this is `main`; on everything else it is the + # commit that fired, the same one the lanes above tested. ref: ${{ env.BUILD_REF }} # Full history: this job derives its artifact's version from the # commit its shipped files last changed in (milestone 313). A @@ -1924,8 +2239,8 @@ jobs: # never be the reason an image does not ship. # # What it should say: - # * a push touching only agent/ moves the agent and leaves web and ml - # STILL. If web moves, its path set is too wide. + # * a push touching only agent/ moves the agent and leaves web STILL. + # If web moves, its path set is too wide. # * a push touching only docs moves nothing. # * a push touching the extension moves the extension AND web, since # web bakes in the XPI. If web does not move, its set is too narrow: @@ -2010,7 +2325,7 @@ jobs: # Each artifact pays one rebuild, once. # # This is what stops a push that touched only `agent/` from rebuilding - # web and ml, and a merge to main from rebuilding what dev already built. + # web, and a merge to main from rebuilding what dev already built. # # The failure direction is deliberate. An inspect that errors for ANY # reason — network, auth, a registry hiccup — reads as a miss and the @@ -2043,6 +2358,13 @@ jobs: set -eu DERIVED=$(sh scripts/artifacts.sh revision agent) echo "revision=$DERIVED" >> "$GITHUB_OUTPUT" + # Baked into the agent image as FC_VERSION and reported by its + # control page and /status. A pure function of the revision — same + # commit, same string — so it adds no variability the reuse check + # would have to account for. Added 2026-09-24: the agent's version + # used to be a hand-written literal in app.py that nobody bumped, so + # the September image reported the same string as the July one. + echo "version=$(sh scripts/artifacts.sh version agent)" >> "$GITHUB_OUTPUT" # The build clock, pinned to the same commit (#3265). Without it # buildkit stamps the image config with the wall clock of the build, @@ -2060,15 +2382,17 @@ jobs: # WHERE THE BUILD PUBLISHES, which is not always the channel — and # whether the channel then has to be written separately. # - # On a push the build writes the channel tag directly: the bytes came - # from a commit, and a commit is the thing CI tests. Nothing to hold - # it behind. + # Every build writes a CANDIDATE tag, never the channel (#4310): + # :dev-candidate / :latest-candidate on a push, :refresh-candidate on + # the refresh. `promote` moves the channel only after smoke-web has + # booted the bytes. The lanes prove the SOURCE; only the smoke proves + # the BYTES, and the two are not the same claim. # - # On the scheduled refresh it writes a CANDIDATE tag instead. A + # The refresh is the case that made this obvious. A # refresh rebuilds against freshly resolved base images, and the web # image's runtime is a line of UNPINNED Debian packages (ffmpeg, # libjpeg62-turbo, libpq5, megatools…) re-resolved on every build. - # Nothing in ci.yml can see that: its lanes run on ci-python:3.14 and + # No verification LANE can see that: they run on ci-python:3.14 and # install requirements.txt, and a base bump changes neither. So # refreshed bytes have to be proven before :latest names them, and # proving needs a moment between "built" and "published" to occupy. @@ -2087,13 +2411,16 @@ jobs: # # build-web additionally exposes this as `outputs.candidate`, which is # what gates the `promote` job — a job's `if:` cannot read `env`, and - # one flag is enough because all three derive it from the same - # IS_REFRESH. ml and agent do not re-emit it; a second copy nothing + # one flag is enough because both jobs derive it from the same + # IS_REFRESH. build-agent does not re-emit it; a second copy nothing # reads is the kind of thing that later reads as load-bearing. if [ "${IS_REFRESH:-}" = "true" ]; then echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT" else - echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT" + # A push builds to a per-channel candidate too (#4310). The channel + # tag moves in `promote`, after smoke-web has booted these bytes — + # so a broken image never reaches the tag deployments follow. + echo "build_ref=$IMAGE:$T-candidate" >> "$GITHUB_OUTPUT" fi # Compare VALUES, never exit codes. Measured on buildx v0.36.1 @@ -2196,6 +2523,16 @@ jobs: # decoration — an unstamped image is one that will always rebuild. labels: | fc.revision=${{ steps.reuse.outputs.revision }} + # What the agent reports about ITSELF, as opposed to the label above, + # which is what the registry reports about it. Three values kept + # apart (rule 149). FC_REVISION is the same string as the label, so + # the image and the registry cannot disagree about which commit this + # is — a build whose self-report names a different commit from the + # tag it was published under is unrollbackable in practice. + build-args: | + FC_CHANNEL=${{ steps.tag.outputs.channel }} + FC_VERSION=${{ steps.reuse.outputs.version }} + FC_REVISION=${{ steps.reuse.outputs.revision }} # LOAD-BEARING, not a preference. On the default docker driver these # were no-ops; on the docker-container driver above, # build-push-action@v5 defaults provenance to TRUE when pushing. @@ -2274,6 +2611,16 @@ jobs: TAGS: ${{ steps.tag.outputs.tags }} run: | set -euf + # A BUILD publishes nothing from here (#4310). Its bytes sit on the + # candidate tag until smoke-web has booted them; `promote` then + # writes the channel tag AND :c- from this run's digest. Writing + # :c- here would publish an immutable rollback tag for bytes + # that might then fail the smoke. + if [ -n "${BUILT_DIGEST:-}" ]; then + echo "repoint: built $BUILT_DIGEST this run — promote publishes it" + echo "repoint: after the smoke; nothing to write here." + exit 0 + fi # WHAT WE COPY FROM, which is not what we EXCLUDE (#4290). # # This step used to copy from the channel tag by NAME. Nothing diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml deleted file mode 100644 index fb71947..0000000 --- a/.forgejo/workflows/ci.yml +++ /dev/null @@ -1,294 +0,0 @@ -name: CI - -# CI lanes per FabledRulebook/forgejo.md "CI philosophy": -# - lint: ruff only, no dep install — fast-fail for the common lint bounce. -# - extension-version: the derived version resolves and is a shape AMO takes. -# - backend-lint-and-test: `pytest -m "not integration"`, no service containers. -# - frontend-build: vitest unit + vite build. -# - integration: pgvector + redis service containers; alembic + `pytest -m integration`. - -on: - push: - branches: [dev, main] - # Renovate opens PRs from `renovate/*` branches into `dev`. Those branches - # never push to dev/main, so the push trigger above gives them NO pre-merge - # CI — a bump could only be validated after it was already merged. This - # pull_request trigger (base `dev` only) validates Renovate PRs before merge. - # It deliberately does NOT fire on dev→main PRs (base `main`), which still - # rely on the dev push run — so no duplicate runs. FC has no fork PRs - # (single-operator Forgejo repo), so secrets-on-PR is not a concern. - pull_request: - branches: [dev] - -jobs: - # Fast-fail lint lane. ruff is pre-installed in the ci-python image, so - # this runs with NO dependency install and surfaces the most common bounce - # class (lint: I001 / UP037 / ASYNC109 / W293 …) in seconds — instead of - # after the backend job's ~30-60s wheel install. ruff is static analysis, - # so no DB/secret env is needed. - lint: - runs-on: python-ci - container: - image: git.fabledsword.com/bvandeusen/ci-python:3.14 - steps: - - uses: actions/checkout@v4 - - name: Ruff lint - # agent/ included so the GPU-agent is linted before its image is built - # (build.yml only `docker build`s it — this is where it gets checked). - # scripts/ likewise: release_notes.py runs only on a tag push, so a - # syntax or import error there would otherwise surface at the one - # moment nobody wants to debug a workflow. - run: ruff check backend/ tests/ alembic/ agent/ scripts/ - - name: Agent syntax check - # The agent's runtime deps (torch/transformers/ultralytics) aren't in the - # CI image, so we can't import it — but compileall parses every module, - # catching syntax errors before the image build. - run: python -m compileall -q agent/fc_agent - - # The extension version is DERIVED, not hand-maintained (milestone 271 step - # 4): build.yml computes it from the commit TIME of the newest packaged - # extension change and stamps it into manifest.json / package.json at build - # time. The guard that used to live here — "packaged files changed but nobody - # bumped the version" — was therefore checking a fact that had stopped - # existing. Worse than useless: it would have failed this lane on every real - # extension change, demanding a bump that decides nothing. Retired 2026-08-27 - # rather than left running beside the new mechanism (rule 22). - # - # Two things are still worth asserting, and this is the only lane that can: - # the extension.yml suite runs on node:24-slim, which is exactly why - # version.spec.js sticks to packaging.sh's git-free subcommands. - # 1. the derivation actually resolves on this commit - # 2. the derived string is one AMO will accept, checked against Mozilla's - # own published grammar rather than a loose "digits and dots" - # - # The MAJOR.MINOR-agreement check that used to be (2) is gone with milestone - # 318 step 8: the committed version no longer seeds anything, so there is no - # hand-set part left for the two files to disagree about. - # - # Deliberately NOT checked here: that the derived value beats what has already - # been signed. That guard belongs in build.yml, where it compares against the - # real ext-* releases. Comparing against origin/main here would be wrong — - # dev legitimately derives a LOWER value whenever main is ahead on the - # extension, and a lane that fails for being behind is a lane people learn to - # ignore. - extension-version: - runs-on: python-ci - container: - image: git.fabledsword.com/bvandeusen/ci-python:3.14 - steps: - - uses: actions/checkout@v4 - with: - # The derivation needs real history: a depth-1 clone sees one commit - # and produces a wrong, too-low value RATHER THAN FAILING. Checking - # that here is half the point of the lane. - fetch-depth: 0 - - name: Extension version derives cleanly - run: | - set -eu - # busybox sh on the act_runner — no bashisms (family rule). - VERSION=$(sh extension/scripts/packaging.sh version) - echo "derived: $VERSION" - - # Mozilla's published grammar for AMO, transcribed verbatim from - # MDN's manifest.json/version page: - # - # ^(0|[1-9][0-9]{0,8})([.](0|[1-9][0-9]{0,8})){0,3}$ - # - # Not the looser `^[0-9]+(\.[0-9]+)*$` this lane used to carry. That - # one passes `2026.08.29.0201`, which AMO REJECTS — a segment must be - # the single digit 0 or start 1-9 — and it also passes five segments, - # where AMO allows four. Both would surface as a failed sign with the - # version already burned: AMO 409s on re-signing, so a rejected value - # cannot be reclaimed and cannot be reused. This lane is the cheap - # place to find out. (#3138, milestone 318 step 8.) - if ! echo "$VERSION" | grep -qE '^(0|[1-9][0-9]{0,8})(\.(0|[1-9][0-9]{0,8})){0,3}$'; then - echo "ERROR: derived version '$VERSION' is not a version AMO accepts." - echo "AMO's grammar: ^(0|[1-9][0-9]{0,8})([.](0|[1-9][0-9]{0,8})){0,3}$" - echo "Most likely cause: a zero-padded segment (08, 0201). The rest" - echo "of the family pads; the extension must not — see packaging.sh." - exit 1 - fi - - # ...and the shape this project actually derives. AMO would happily - # take `1.0.3500147` too, so the grammar check alone would not notice - # a regression to the pre-318 shape — which orders BELOW everything - # signed since, and is unrecoverable once Firefox has the higher one. - if ! echo "$VERSION" | grep -qE '^20[0-9][0-9]\.[0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,4}$'; then - echo "ERROR: derived version '$VERSION' is not YYYY.M.D.HHMM." - echo "Rule 148's CalVer is what build.yml signs; the old" - echo "1.0. shape would order below every ext-2026.* release." - exit 1 - fi - echo "OK: derived version $VERSION" - - backend-lint-and-test: - runs-on: python-ci - container: - image: git.fabledsword.com/bvandeusen/ci-python:3.14 - env: - # DB_PASSWORD and SECRET_KEY are required by config.py at import time - # even though unit tests don't actually touch the DB or use the secret. - DB_PASSWORD: ci_unit_test_placeholder - SECRET_KEY: ci_unit_test_placeholder - steps: - - uses: actions/checkout@v4 - with: - # Full history for tests/test_artifact_identity.py, which derives - # each artifact's revision to check the identity scheme. On a - # depth-1 clone that derivation either fails or returns the tip sha - # — so the lane would go green while asserting nothing, which is - # the one outcome worse than a red one. - fetch-depth: 0 - - # Cache step removed 2026-05-26: act_runner's cache backend has been - # broken on this homelab runner since 2026-05-15 (first as request- - # timeout warnings, then as hard "Cannot find module .../dist/restore/ - # index.js" failures that tank the whole job). The cache step targeted - # ~/.cache/pip but the install below uses `uv pip install` primarily, - # whose own cache lives at ~/.cache/uv — so the cache step's real - # benefit was marginal even when working. Cost of removal: ~30s of - # wheel downloads per job. Future re-enable: mount ~/.cache/uv as a - # docker volume at the runner level (skips actions/cache entirely), - # or fix the runner-side cache backend (clear /var/run/act/actions/*, - # pin act_runner version, etc.). - - - name: Install Python deps - # ruff is pre-installed in the ci-python image (see CI-Runner/CI-python/ - # Dockerfile's RUFF_VERSION). Per FabledRulebook ci-runners.md, toolchain - # versions live on the runner image, not here. - # uv: 5-10x faster wheel resolve than pip for cold caches. - # Falls back to pip install on uv-missing runners (older images). - run: | - if command -v uv >/dev/null 2>&1; then - uv pip install --system -r requirements.txt pytest pytest-asyncio - else - pip install -r requirements.txt pytest pytest-asyncio - fi - - # Ruff moved to the dedicated fast `lint` job above (fails in seconds, - # no dep install). This job is now unit tests only. - - name: Pytest (unit only — integration runs in the integration job) - run: pytest tests/ -v -m "not integration" - - frontend-build: - runs-on: python-ci - container: - image: git.fabledsword.com/bvandeusen/ci-python:3.14 - defaults: - run: - working-directory: frontend - steps: - - uses: actions/checkout@v4 - # No package-lock.json is tracked yet (we don't run npm locally per - # feedback-no-local-runs). Using `npm install` instead of `npm ci`. - # If we want strict lockfile-based reproducibility later, commit a - # package-lock.json and flip this back to `npm ci`. - - run: npm install --no-audit --no-fund - # No type-check step: the frontend is pure JS (no .ts files, no JSDoc), - # so a type-checker has nothing to do. The vue-tsc devDep + its `check` - # script were dropped 2026-07-11 rather than bumped to v3. If we add - # TS/JSDoc later, re-add a tsconfig.json + vue-tsc + a type-check step. - - run: npm run test:unit - - run: npm run build - - # Single integration job — collapsed from a 3-way shard split on 2026-06-04. - # The shards existed to parallelize ~8.5min of integration tests; once the - # throwaway Postgres runs with fsync OFF (the durability step below) the whole - # suite runs in ~45s, so the split only triplicated the ~2min fixed overhead - # (container + `uv pip install` + `alembic upgrade head`) and burned 3 of 6 - # runner slots for no wall-clock gain. One job now: spin up once, install - # once, migrate once, run every integration test. - # - # The docker-ps filter scopes to THIS job's own Postgres/Redis service - # containers by job name. act_runner strips underscores from job names when - # labelling containers (`int_api` matched nothing on 2026-05-25), so the name - # stays separator-free (`integration`). The step prints `docker ps -a` first - # so a future naming-convention shift surfaces in the log without a - # guess-and-push cycle. - # - # Pre-baking requirements.txt into ci-python:3.14 is intentionally NOT done — - # per ci-requirements.md, FC is the only Python consumer of that image and the - # CI-Runner "add deps to image when used by >1 project" rule keeps it per-job. - integration: - runs-on: python-ci - container: - image: git.fabledsword.com/bvandeusen/ci-python:3.14 - env: - DB_USER: fabledcurator - DB_PASSWORD: ci_integration - DB_PORT: "5432" - DB_NAME: fabledcurator_test - SECRET_KEY: ci_integration_placeholder - services: - postgres: - image: pgvector/pgvector:pg16 - env: - POSTGRES_USER: fabledcurator - POSTGRES_PASSWORD: ci_integration - POSTGRES_DB: fabledcurator_test - options: >- - --health-cmd "pg_isready -U fabledcurator" - --health-interval 10s - --health-timeout 5s - --health-retries 10 - redis: - image: redis:7-alpine - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 10 - steps: - - uses: actions/checkout@v4 - - name: Integration suite (resolve service IPs, migrate, test) - run: | - set -eux - echo "=== container landscape (diagnostic for filter scoping) ===" - docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}' - echo "=== end landscape ===" - PG=$(docker ps --filter "name=integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) - RD=$(docker ps --filter "name=integration" --filter "ancestor=redis:7-alpine" -q | head -n1) - test -n "$PG" && test -n "$RD" - PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") - RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD") - test -n "$PG_IP" && test -n "$RD_IP" - export DB_HOST="$PG_IP" - export CELERY_BROKER_URL="redis://$RD_IP:6379/0" - export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0" - # These steps run under `sh -e`, not bash, so bash's /dev/tcp magic - # path does not exist here — the probe this loop used to run could - # never succeed and simply burned the full 120s on every run, green - # or red, then continued without having established anything. Python - # is in the image and needs no installed package for a socket - # connect, so it is the probe. Exhausting the budget is now a named - # failure rather than a silent fall-through (rule 156): if Postgres - # is genuinely not up, that is what the log should say, instead of - # whatever the first query happens to raise two minutes later. - pg_ready="" - for i in $(seq 1 60); do - if python -c "import socket,sys; s=socket.socket(); s.settimeout(2); sys.exit(0 if s.connect_ex(('$PG_IP', 5432)) == 0 else 1)"; then - pg_ready=1 - break - fi - sleep 2 - done - if [ -z "$pg_ready" ]; then - echo "postgres at $PG_IP:5432 did not accept a connection within 120s" - exit 1 - fi - if command -v uv >/dev/null 2>&1; then - uv pip install --system -r requirements.txt pytest pytest-asyncio - else - pip install -r requirements.txt pytest pytest-asyncio - fi - # Relax durability on the throwaway CI Postgres so the per-test - # TRUNCATE's commit-fsync — the integration teardown's dominant cost - # (~1.5-2s/test, which collapsed the suite from ~13min to ~45s) — is - # skipped. fsync/full_page_writes are sighup GUCs and synchronous_commit - # is user-context, so ALTER SYSTEM + pg_reload_conf() applies them with - # NO restart. Ephemeral DB ⇒ fsync-off is safe. Non-fatal so a perms - # surprise can't red the job; fabledcurator is the postgres image's - # bootstrap superuser. - python -c "import os,psycopg; c=psycopg.connect(host=os.environ['DB_HOST'],port=5432,user=os.environ['DB_USER'],password=os.environ['DB_PASSWORD'],dbname=os.environ['DB_NAME'],autocommit=True); [c.execute(q) for q in ('ALTER SYSTEM SET fsync=off','ALTER SYSTEM SET synchronous_commit=off','ALTER SYSTEM SET full_page_writes=off','SELECT pg_reload_conf()')]; c.close()" || echo 'WARN: durability GUC relax failed (continuing)' - alembic upgrade head - pytest tests/ -v -m integration --durations=15 diff --git a/.forgejo/workflows/extension.yml b/.forgejo/workflows/extension.yml deleted file mode 100644 index f29cad9..0000000 --- a/.forgejo/workflows/extension.yml +++ /dev/null @@ -1,87 +0,0 @@ -name: extension -# Lint + unit tests. The sign-and-publish dance moved into build.yml's -# `sign-extension` job (2026-05-25) — `:latest` now always bundles the XPI -# because sign-extension runs as a build-web dependency in the SAME workflow, -# eliminating the prior race between build.yml and a separate extension.yml. -# Signed XPIs are cached in Forgejo Release Assets named `ext-`. -on: - push: - branches: [dev, main] - paths: - - 'extension/**' - - '.forgejo/workflows/extension.yml' - # test/version.spec.js asserts things ABOUT the other two workflows — - # that neither inlines the packaged-file set, and that build.yml derives - # the shipped version rather than reading it out of the repo. A - # workflow-only edit can therefore break this suite, so it has to trigger - # it. build.yml joined the list at milestone 271 step 5, when the spec - # started asserting against it. - - '.forgejo/workflows/ci.yml' - - '.forgejo/workflows/build.yml' - pull_request: - branches: [main] - paths: - - 'extension/**' - - '.forgejo/workflows/ci.yml' - - '.forgejo/workflows/build.yml' - workflow_dispatch: - -jobs: - lint: - runs-on: python-ci - container: - image: node:24-bookworm-slim - steps: - - uses: actions/checkout@v4 - # Not --no-save: vitest and web-ext are both real devDependencies now, - # and the suite needs vitest resolvable from node_modules. - - name: Install dev dependencies - run: cd extension && npm install --no-audit --no-fund - - name: Lint - run: cd extension && npm run lint - # Pure-logic specs over lib/url.js and lib/platforms.js plus manifest / - # package version-consistency checks. No browser, no network. - - name: Unit tests - run: cd extension && npm run test:unit - - # Everything else about packaging is asserted against our own declaration - # of what ships. This is the only check that asks web-ext what it ACTUALLY - # put in the archive. Until now that was an unverified assumption about - # glob semantics — and a fragile one: `test/**` reaches web-ext intact - # only because callers `set -f` first, so losing that quoting would - # silently start shipping dev files with no other signal. - - name: Verify XPI contents - run: | - set -eu - command -v unzip >/dev/null 2>&1 || { apt-get update -qq && apt-get install -y -qq unzip; } - cd extension - npm run build - ZIP=$(ls web-ext-artifacts/*.zip | head -1) - echo "=== packaged entries in $ZIP ===" - unzip -Z1 "$ZIP" | sort - echo "=== end ===" - ENTRIES=$(unzip -Z1 "$ZIP") - fail=0 - # Must NOT ship: repo infrastructure with no business in a user's browser. - for pat in 'test/' 'scripts/' 'vitest.config.js' 'package.json' 'package-lock.json' 'README.md' 'node_modules/' 'web-ext-artifacts/'; do - if echo "$ENTRIES" | grep -q "^$pat"; then - echo "ERROR: '$pat' was packaged into the XPI but must not be" - fail=1 - fi - done - # Must ship: if an exclusion pattern ever over-matches, the extension - # breaks at runtime rather than at build time, so assert presence too. - for req in 'manifest.json' 'lib/url.js' 'lib/api.js' 'lib/platforms.js' 'lib/cookies.js'; do - if ! echo "$ENTRIES" | grep -q "^$req$"; then - echo "ERROR: '$req' is missing from the XPI" - fail=1 - fi - done - for dir in 'background/' 'popup/' 'options/' 'content/' 'icons/'; do - if ! echo "$ENTRIES" | grep -q "^$dir"; then - echo "ERROR: nothing from '$dir' was packaged" - fail=1 - fi - done - [ "$fail" -eq 0 ] || exit 1 - echo "XPI contents verified." diff --git a/Dockerfile b/Dockerfile index f0e184d..36eb7f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ postgresql-client \ zstd \ megatools \ + # PID 1 for every role. See the ENTRYPOINT note at the foot of this file: + # without it the image needs `init: true` in whatever runs it, which is a + # deployment remembering a flag for the image to behave correctly. + tini \ libjpeg62-turbo \ libwebp7 \ libpng16-16 \ @@ -36,9 +40,59 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /app -COPY requirements.txt ./ +COPY requirements.txt requirements-ml.txt ./ RUN pip install -r requirements.txt +# --- ML, merged from Dockerfile.ml (milestone 422 step 6) -------------------- +# +# ONE image now serves every lane. It was two because the ML lane ran in its +# own container; with the single-container layout (step 5) running every lane +# in one process tree, a second image would mean the `ml` lane could never be +# enabled from the UI — there would be no worker in this container to enable. +# +# THE COST, MEASURED from run 7273 rather than guessed — and it is far +# smaller than the estimate this comment first carried, which said "everyone +# pulls ~4GB": +# +# torch 2.12.1+cpu wheel 192.3 MB +# torchvision 0.27.1+cpu 1.8 MB +# transformers / onnxruntime / opencv / sklearn and friends (opencv and +# onnxruntime since dropped, #1451 — nothing here imported them) +# 62.0, 35.3, 23.6, 16.7, 12.3, 9.2, 6.9 MB +# largest newly-pushed layer 222.07 MB +# +# So the ML code adds a few hundred MB to the pull, not gigabytes. The CPU +# index is what makes that true: the default PyPI torch wheel bundles the +# NVIDIA CUDA runtime and is ~2GB on its own. +# +# The GIGABYTES are in the MODEL — ~3.5GB of SigLIP weights — and those are +# NOT in this image. They arrive only when the operator enables the lane, +# which is what lets rule 164 permit a runtime fetch at all ("optional and +# clearly off"). That also settles the trade this step was asked to weigh: +# baking the weights in would add ~3.5GB to every pull for a feature many +# adopters never enable, against ~350MB for the code that makes the switch +# available. Off-by-default wins by an order of magnitude, which was NOT +# obvious before measuring — the estimate had the two costs within 15% of +# each other. +# +# `--index-url`, not `--extra-index-url`: the latter would let pip resolve a +# +cu wheel anyway, and the whole saving above depends on it not doing that. +# +# CPU-only torch from the PyTorch CPU index. Nothing here uses a GPU — the +# GPU agent is a separate service with its own image. +RUN pip install --index-url https://download.pytorch.org/whl/cpu \ + "torch>=2.14" "torchvision>=0.29" +RUN pip install -r requirements-ml.txt + +# Where the model lands. Deliberately NOT a VOLUME instruction: that mints an +# anonymous volume when nobody mounts one, which survives `docker rm` and +# accumulates 3.5GB copies nobody can find. The compose files mount it +# explicitly instead, so an unmounted run simply re-downloads — visible, and +# recoverable. +ENV HF_HOME=/models/.huggingface \ + TRANSFORMERS_CACHE=/models/.huggingface \ + ML_MODEL_DIR=/models + COPY backend/ ./backend/ COPY alembic/ ./alembic/ COPY alembic.ini ./ @@ -72,5 +126,51 @@ ENV FC_VERSION=${FC_VERSION} EXPOSE 8080 -ENTRYPOINT ["./entrypoint.sh"] -CMD ["web"] +# ONE healthcheck for every role, because the image knows which role it is +# running and a deployment should not have to repeat it. `healthcheck` reads +# the role entrypoint.sh recorded and asks the right question: HTTP for web, +# a self-addressed celery ping for a worker lane, both-for-every-lane for the +# consolidated `all`. +# +# start-period covers the SLOWEST role, which is `all`: alembic, then +# hypercorn, then four celery workers registering with the broker. A web-only +# container is ready long before this; the cost of the shared number is that +# a broken one takes a little longer to be called broken. +# +# A service may still declare its own healthcheck and docker will prefer it — +# the escape hatch for a deployment that wants something different. +HEALTHCHECK --interval=30s --timeout=15s --start-period=90s --retries=3 \ + CMD ["python", "-m", "backend.app.scripts.healthcheck"] + +# tini is PID 1, and the image brings its own rather than asking the +# deployment for one. +# +# PID 1 carries a duty no other process has: every orphaned process in the +# container reparents to it and must be reaped, or it stays a zombie holding +# a PID slot. This app makes orphans in normal operation — six service +# modules shell out (gallery-dl, ffmpeg, pg_dump, the external fetchers) and +# celery's prefork pool forks children that spawn them. +# +# Whatever the role, something that is not an init ends up as PID 1: +# supervisord for `all`, hypercorn for `web`, celery for a worker. The fix +# was `init: true` in the compose/stack file, which is out of the norm and +# put correct process handling in the hands of whoever deploys the image — +# the same mistake as declaring the healthcheck per service. A flag that is +# silently dropped (an older Swarm, a `docker run` without it) costs reaping +# with no signal at all. +# +# So the image owns it. `docker run ` is correct on its own, and +# nothing downstream has to know. The smoke asserts /proc/1/comm is tini. +ENTRYPOINT ["/usr/bin/tini", "--", "./entrypoint.sh"] +# The DEFAULT is the whole application, not one lane of it. +# +# `docker run fabledcurator` with no command starts hypercorn plus every +# worker lane under supervisord — the shape an adopter wants and the shape the +# consolidated stack runs. It was `web`, which meant the single-container +# layout only worked if you knew to ask for it by name, and a compose file +# that forgot `command:` got a web server with nothing processing its queues: +# a gallery that loads, accepts an import, and never finishes one. +# +# The multi-service stack is unaffected — every service there names its role +# explicitly, which is exactly what makes it the multi-service stack. +CMD ["all"] diff --git a/Dockerfile.ml b/Dockerfile.ml deleted file mode 100644 index 13efe30..0000000 --- a/Dockerfile.ml +++ /dev/null @@ -1,43 +0,0 @@ -# syntax=docker/dockerfile:1.25 - -FROM python:3.14-slim -ENV PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 \ - PIP_NO_CACHE_DIR=1 \ - PIP_DISABLE_PIP_VERSION_CHECK=1 \ - HF_HOME=/models/.huggingface \ - TRANSFORMERS_CACHE=/models/.huggingface \ - ML_MODEL_DIR=/models - -RUN apt-get update && apt-get install -y --no-install-recommends \ - ffmpeg \ - libpq5 \ - libjpeg62-turbo \ - libwebp7 \ - libpng16-16 \ - libgl1 \ - libglib2.0-0 \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /app - -COPY requirements-ml.txt requirements.txt ./ -# CPU-only torch: the default PyPI wheel bundles the CUDA runtime (~5.6GB -# layer); this pipeline never uses a GPU. --index-url (not --extra-index-url) -# guarantees only +cpu wheels are considered, so no nvidia-*-cu12 deps. -RUN pip install --index-url https://download.pytorch.org/whl/cpu \ - "torch>=2.12,<3.0" "torchvision>=0.27,<0.28" -RUN pip install -r requirements-ml.txt - -COPY backend/ ./backend/ -COPY alembic/ ./alembic/ -COPY alembic.ini ./ -COPY entrypoint.sh ./ -RUN chmod +x entrypoint.sh - -# Models self-heal into /models on first start (FC-2 implements this) -VOLUME ["/models"] - -ENTRYPOINT ["./entrypoint.sh"] -CMD ["ml-worker"] diff --git a/README.md b/README.md index 8c94e40..ed78b17 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,8 @@ through afterwards. - **ML tagging.** Runs image models in-container to suggest tags, group characters, find near-duplicates and power similarity search. Suggestions are reviewable — it proposes, you confirm, and it learns which proposals you keep - rejecting. + rejecting. It ships switched off: turn it on under Settings → System when + you want it, and it fetches its model weights then. - **Deduplication and provenance.** Everything that arrives is hashed and deduplicated by content, metadata sidecars are read wherever the source writes them, and every file keeps a record of where it came from. @@ -119,9 +120,12 @@ needs every credential entered again by hand. A few other things are worth knowing about the first few minutes: -- **The ML worker downloads its model weights on first boot**, several GB from - HuggingFace into `./models`. Until that finishes, tagging is queued rather - than broken. It is idempotent — a restart resumes rather than refetches. +- **ML tagging starts switched off, and nothing is downloaded at boot.** Give + the ML lane a slot under **Settings → System** and it fetches its model + weights then — several GB from HuggingFace into `./models`, shown as a job + under **Settings → Activity** that you can watch and retry. Until it + finishes, tagging is queued rather than broken, and the fetch only takes + what is missing, so turning the lane off and on again does not refetch. - **The gallery starts empty**, and that is the expected state. Add a creator under **Subscriptions** and it fills as posts come down. - **If you already have a library on disk**, there is no screen that imports @@ -245,9 +249,9 @@ reasoning is note #3127 §5). Rolling back is `docker pull …:c-`. Each artifact still has a version, derived rather than chosen: the commit time of the newest change to that artifact's *own* shipped files, as -`YYYY.MM.DD.HHMM` UTC (rule 148). Four artifacts, four independent versions — -a push touching only `agent/` re-versions the agent and leaves web and ml -alone, and CI skips the builds whose content did not move. +`YYYY.MM.DD.HHMM` UTC (rule 148). Three artifacts, three independent versions +— a push touching only `agent/` re-versions the agent and leaves web and the +extension alone, and CI skips the builds whose content did not move. Because no registry name carries it, the running instance's own report is the only answer to "which build is this?". The foot of Settings shows @@ -260,22 +264,32 @@ commits since the previous tag; it builds no image. ## What's in here -Five deployable pieces, built by `.forgejo/workflows/build.yml`: +Four deployable pieces, built by `.forgejo/workflows/build.yml`: | Piece | Built from | Image | Role | | --- | --- | --- | --- | -| **Web / workers** | `Dockerfile` | `fabledcurator` | Quart API + the built Vue SPA in one image. `entrypoint.sh` picks the role: `web`, `worker`, `scheduler`. The `maintenance-long` service is a second `worker` pinned to the long-running maintenance queue. | -| **ML worker** | `Dockerfile.ml` | `fabledcurator-ml` | Same app, plus `requirements-ml.txt` — tagging and embedding models that run in-container. | +| **Web / workers** | `Dockerfile` | `fabledcurator` | Quart API + the built Vue SPA in one image. `entrypoint.sh` picks the role: `web`, `worker`, `scheduler`, `ml-worker`, or `all` (every lane under supervisord, the single-container layout). The `maintenance-long` service is a second `worker` pinned to the long-running maintenance queue. | | **GPU agent** | `agent/Dockerfile` | `fabledcurator-agent` | Optional desktop-GPU worker (`agent/`). Leases jobs over **HTTP only** — never touches the database or Redis. See `agent/README.md`. | | **Firefox extension** | `extension/` | signed XPI | MV3 extension: pushes platform session cookies into FC and adds a creator as a Source in one click. AMO-signed on both `dev` and `main` (one signature per extension change, shared by the two channels), bundled into that channel's web image and served from Settings → Maintenance. See `extension/README.md`. | | **Data** | — | `pgvector/pgvector:pg16`, `redis:7-alpine` | Postgres with pgvector for embeddings; Redis as the Celery broker. | ## CI / Forgejo setup -Four workflows: `ci.yml` (lint, extension-version check, backend unit tests, -frontend build, integration), `extension.yml` (extension lint, vitest, XPI -content verification), `build.yml` (sign + publish), and `release.yml`, which -runs only on a `v*` tag and publishes a changelog without building anything. +Two workflows that matter here: `build.yml` (the six verification lanes — lint, +extension-version check, backend unit tests, frontend build, extension lint + +vitest + XPI content check, integration — and then sign + publish), and +`release.yml`, which runs only on a `v*` tag and publishes a changelog without +building anything. The extension lane was its own `extension.yml` until +milestone 429, which let a red extension suite sign and ship the XPI anyway. + +**The lanes and the publish are one workflow on purpose.** They were two +(`ci.yml` and `build.yml`) until 2026-09-23, on the same push trigger, which +meant the build could not see the tests' verdict and published whatever it +built — a red unit lane and a fresh `:dev` image, in the same minute. A +`needs:` edge only exists inside one workflow graph, so the two are one graph +and the gate is that edge: a lane that fails, **or that merely skips**, leaves +the publishing jobs unrun. Pull-request runs (Renovate bumps into `dev`) are +the lanes and nothing else. **The toolchain each job runs in is its `container.image`, not its `runs-on` label.** `runs-on: python-ci` only schedules the job onto a runner; every job diff --git a/agent/Dockerfile b/agent/Dockerfile index 2f79e0c..2d0323f 100644 --- a/agent/Dockerfile +++ b/agent/Dockerfile @@ -1,10 +1,21 @@ # FabledCurator GPU agent — runs on the desktop with the GPU. -# CUDA 12.9 + cuDNN 9 runtime so onnxruntime-gpu can use the card (it needs -# cuDNN 9 — the plain -runtime image lacks it: "libcudnn.so.9: cannot open -# shared object file"); ffmpeg for video frames. Ubuntu 24.04 → Python 3.12. -# Stays on the CUDA-12 / cuDNN-9 line the default onnxruntime-gpu + torch are -# built against (CUDA 13 has only nascent ONNX Runtime support). -FROM nvidia/cuda:12.9.2-cudnn-runtime-ubuntu24.04 +# +# The `base` flavour, not `cudnn-runtime`: CUDA and cuDNN arrive as the +# `nvidia-*` pip packages torch and onnxruntime-gpu depend on, so the base only +# has to hand the container the driver (it sets NVIDIA_VISIBLE_DEVICES / +# NVIDIA_DRIVER_CAPABILITIES for the Container Toolkit). Until #1451 this was +# `12.9.2-cudnn-runtime` under a `torch==2.6.0+cu124` — and requirements.txt then +# REPLACED that torch with PyPI's CUDA-13 build (ultralytics pulls torchvision, +# which pulls its matching torch), beside a CUDA-13 onnxruntime-gpu. The image +# ran CUDA 13 on a CUDA-12 base, carrying ~3 GB of base libraries and a ~3 GB +# torch nothing loaded: 10 GB compressed. +# +# 13.0 because that is the line both wheels are built for (torch's cu130 index, +# onnxruntime-gpu's `nvidia-cuda-runtime~=13.0`). Needs an NVIDIA driver that +# supports CUDA 13 (580+); fc_agent/accel.py logs at startup whether torch and +# onnxruntime actually got the GPU, since both fall back to the CPU silently. +# ffmpeg for video frames. Ubuntu 24.04 → Python 3.12. +FROM nvidia/cuda:13.0.3-base-ubuntu24.04 # PIP_BREAK_SYSTEM_PACKAGES: Ubuntu 24.04 marks its system Python as externally # managed (PEP 668), so a global `pip install` errors without this. It's a @@ -16,10 +27,12 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* WORKDIR /app -# torch from the CUDA-12.4 wheel index; its wheels bundle their own CUDA + cuDNN -# so they run on the 12.9 base and coexist with onnxruntime-gpu. Installed first -# + separately so the GPU build of torch is deterministic and layer-cached. -RUN pip3 install --no-cache-dir torch==2.6.0 --index-url https://download.pytorch.org/whl/cu124 +# torch AND torchvision from the cu130 index, together and first. Installing +# torch alone is what let the next step swap it out: ultralytics needs +# torchvision, PyPI's torchvision pins its own torch, and pip replaced ours to +# match. With both present, requirements.txt finds them satisfied. +RUN pip3 install --no-cache-dir --index-url https://download.pytorch.org/whl/cu130 \ + torch torchvision COPY requirements.txt . RUN pip3 install --no-cache-dir -r requirements.txt COPY fc_agent ./fc_agent @@ -27,6 +40,23 @@ COPY fc_agent ./fc_agent # imgutils ONNX models + the transformers SigLIP weights both cache here; mount # a volume to persist them across restarts (the SigLIP download is ~3.5 GB once). ENV HF_HOME=/models + +# Declared LAST on purpose, exactly as the web Dockerfile does: an ARG/ENV +# invalidates every layer below it, and these are the only values that differ +# between builds of otherwise identical source. Any earlier and the ~6.3 GB +# CUDA + torch layers could never be shared between the dev and main builds of +# one commit — which is the cost #3114 measured at 9m26s cold. +# +# Three values, never folded together (rule 149) — the NAME a person reads, the +# CHANNEL it came from, and the REVISION that identifies the content. See +# fc_agent/build_info.py; CI derives all three from scripts/artifacts.sh. +ARG FC_CHANNEL="" +ENV FC_CHANNEL=${FC_CHANNEL} +ARG FC_VERSION="" +ENV FC_VERSION=${FC_VERSION} +ARG FC_REVISION="" +ENV FC_REVISION=${FC_REVISION} + EXPOSE 8770 # The control UI; the worker is started from it (or POST /start). diff --git a/agent/README.md b/agent/README.md index ac2d7b2..7d51eee 100644 --- a/agent/README.md +++ b/agent/README.md @@ -15,13 +15,28 @@ sudo pacman -S nvidia-container-toolkit sudo nvidia-ctk runtime configure --runtime=docker sudo systemctl restart docker # verify: -docker run --rm --gpus all nvidia/cuda:12.4.1-base-ubuntu22.04 nvidia-smi +docker run --rm --gpus all nvidia/cuda:13.0.3-base-ubuntu24.04 nvidia-smi +# the header's CUDA version must be 13.0 or later (driver 580+) +``` + +### After a driver update: regenerate the CDI spec +If the agent's first log lines say `accel: torch is NOT on the GPU` or report +`cudaGetDeviceCount: unknown error (999)` while `nvidia-smi` still works, the +toolkit's saved device list (`/etc/cdi/nvidia.yaml`) is out of date. The +`nvidia-uvm` device number changes between driver versions, and a spec +generated before the update hands the container a device node that no longer +exists (2026-09-24: host `511,0`, container `235,0`). Compare +`ls -l /dev/nvidia-uvm` on the host with the same inside the container, then: +```sh +sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml +# if your toolkit ships it, this keeps it current on every driver update: +sudo systemctl enable --now nvidia-cdi-refresh.path ``` ## 1. Get a token In FC: **Settings → Tagging → GPU agent → Generate token** (or Rotate). Copy it. -## 2. Pull (CI publishes it alongside the web/ml images) +## 2. Pull (CI publishes it alongside the web image) ```sh docker pull git.fabledsword.com/bvandeusen/fabledcurator-agent:latest ``` diff --git a/agent/fc_agent/accel.py b/agent/fc_agent/accel.py new file mode 100644 index 0000000..601bcad --- /dev/null +++ b/agent/fc_agent/accel.py @@ -0,0 +1,124 @@ +"""Which accelerator each runtime actually got — reported once, at startup. + +The agent has two GPU runtimes and both fall back to the CPU without raising: +torch when the driver is too old for its CUDA build, and onnxruntime (the imgutils +detector + CCIP models) when its CUDA provider cannot load its libraries. A +fallback shows up only as slower work, and nothing reported it. On 2026-09-24 the +image turned out to be running a CUDA-13 torch and onnxruntime on a CUDA-12 base +(#1451), and whether the ONNX half was on the GPU could not be answered from +anything the agent had ever logged. + +Also the fix for the likeliest way the ONNX half misses: onnxruntime-gpu's CUDA +provider finds libcudart/cuBLAS/cuDNN only on the loader path, and in this image +they live in the `nvidia-*` pip packages torch installs. `preload_dlls()` (ORT +1.21+) loads them from there, so the provider resolves them by soname. + +Stdlib-only at import, so the unit suite can import it — torch and onnxruntime +are imported inside the functions. +""" + +from __future__ import annotations + +import ctypes +import importlib +import logging +from pathlib import Path + +log = logging.getLogger("fc_agent.accel") + +# Filled by report(); /status carries it so the page can show it too. +LAST: dict = {} + + +def torch_status(imp=importlib.import_module) -> dict: + try: + torch = imp("torch") + except Exception as e: + return {"device": "unavailable", "error": str(e)} + out = {"version": torch.__version__, "cuda_build": torch.version.cuda} + if torch.cuda.is_available(): + out["device"] = "cuda" + out["gpu"] = torch.cuda.get_device_name(0) + else: + out["device"] = "cpu" + return out + + +def onnx_status(imp=importlib.import_module, load=ctypes.CDLL) -> dict: + try: + ort = imp("onnxruntime") + except Exception as e: + return {"device": "unavailable", "error": str(e)} + out = {"version": ort.__version__, "providers": list(ort.get_available_providers())} + if "CUDAExecutionProvider" not in out["providers"]: + out["device"] = "cpu" + return out + preload = getattr(ort, "preload_dlls", None) + if preload is not None: + try: + preload() + except Exception as e: + out["preload_error"] = str(e) + # "Available" only means the build HAS the provider. Loading its library is + # what resolves libcudart/cuBLAS/cuDNN — the step that fails when they are + # missing, and the one a session would otherwise fail silently on. + capi = Path(ort.__file__).parent / "capi" + try: + load(str(capi / "libonnxruntime_providers_shared.so"), mode=ctypes.RTLD_GLOBAL) + load(str(capi / "libonnxruntime_providers_cuda.so")) + except OSError as e: + out["device"] = "cpu" + out["error"] = str(e) + return out + # Loading proves the libraries resolve, NOT that a GPU can be used: on + # 2026-09-24 this reported "onnx on GPU" beside torch failing cuInit with + # "CUDA unknown error" (a driver update awaiting a reboot). Asking the CUDA + # runtime for a device initialises the driver the provider would use. + error = _cuda_device_error(load) + out["device"] = "cpu" if error else "cuda" + if error: + out["error"] = error + return out + + +def _cuda_device_error(load=ctypes.CDLL) -> str | None: + """None when the CUDA runtime can reach a device, else why it cannot.""" + try: + cudart = load("libcudart.so.13") + except OSError as e: + return str(e) + count = ctypes.c_int(0) + rc = cudart.cudaGetDeviceCount(ctypes.byref(count)) + if rc != 0: + cudart.cudaGetErrorString.restype = ctypes.c_char_p + return f"cudaGetDeviceCount: {cudart.cudaGetErrorString(rc).decode()} ({rc})" + return None if count.value > 0 else "no CUDA device visible" + + +def summary() -> dict | None: + """The report as FabledCurator stores it: each runtime's device, and why + when it is not the GPU. Sent on every lease and heartbeat, so the System + view can call a running agent that fell back to the CPU "degraded" rather + than "running" — the 2026-09-24 fallback went unseen for weeks because + only this agent's own log said so. None before report() has run.""" + if not LAST: + return None + out = {} + for name, s in LAST.items(): + entry = {"device": s.get("device")} + if s.get("error"): + entry["error"] = str(s["error"])[:200] + out[name] = entry + return out + + +def report() -> dict: + """Check both runtimes, log the result, and keep it for /status.""" + LAST.clear() + LAST.update(torch=torch_status(), onnx=onnx_status()) + for name, s in LAST.items(): + if s.get("device") == "cuda": + log.info("accel: %s on GPU (%s)", name, s) + else: + log.warning("accel: %s is NOT on the GPU — work runs on the CPU (%s)", name, s) + return dict(LAST) diff --git a/agent/fc_agent/app.py b/agent/fc_agent/app.py index 1338d28..cc8a040 100644 --- a/agent/fc_agent/app.py +++ b/agent/fc_agent/app.py @@ -11,17 +11,22 @@ import logging from fastapi import FastAPI, Request from fastapi.responses import HTMLResponse, JSONResponse -from . import logbuf +from . import accel, logbuf +from .build_info import FC_CHANNEL, FC_REVISION, FC_VERSION, build_id, display_version from .config import Config from .gpu import read_gpu from .worker import Worker log = logging.getLogger("fc_agent.app") -# Bump on every agent change. The page embeds this and /status reports it; the UI -# warns to reload when they differ — so a stale browser-cached page can't be -# mistaken for "the new image didn't deploy". (Belt-and-braces with no-store.) -VERSION = "2026-07-17.1 · idle model-unload: after ~5 min idle the GPU models release their VRAM and reload on the next job (env IDLE_UNLOAD_SECONDS, 0=off) · sleep mode sheds to one downloader" +# DERIVED at image build time, not hand-maintained — see build_info. This was a +# literal an author was asked to bump on every agent change, and the September +# image printed the same "2026-07-17.1" as the July one, so the surface meant to +# answer "did my pull work?" answered the same either way. +# +# Two values with two jobs, kept apart (rule 149): the page SHOWS the version +# and COMPARES the build id. /status reports both, plus the raw fields, so a +# reader never has to take a formatted string apart to get at one of them. logbuf.install() cfg = Config.from_env() @@ -42,6 +47,9 @@ async def _no_store(request, call_next): @app.on_event("startup") def _maybe_autostart() -> None: + # Before the worker: the report also preloads the CUDA libraries the ONNX + # models need, and it says in the log which runtimes landed on the GPU. + accel.report() # With AUTO_START set, a container restart (host reboot, or `restart: # unless-stopped` after a crash) resumes the worker on its own — the slots # then ride out a still-down curator via lease backoff. Lets the agent @@ -52,7 +60,14 @@ def _maybe_autostart() -> None: @app.get("/", response_class=HTMLResponse) def index() -> str: - return _PAGE.replace("__BUILD__", VERSION) + # Two substitutions, not one: `__VERSION__` is what a person reads in the + # meta line, `__BUILD_ID__` is what the script compares against /status to + # notice the page is a cached copy from a previous build. + return ( + _PAGE + .replace("__VERSION__", display_version()) + .replace("__BUILD_ID__", build_id()) + ) @app.post("/start") @@ -117,7 +132,15 @@ def status(): s["fc_url"] = cfg.fc_url s["configured"] = bool(cfg.token) s["queue"] = worker.latest_queue() - s["build"] = VERSION + # `build` is the comparison token the page checks — see build_info. + # `version`/`channel`/`revision` ride BESIDE it rather than inside it, so a + # reader wanting the version never has to parse it back out of something + # else. Absent rather than empty when the image carries no stamp. + s["build"] = build_id() + s["version"] = FC_VERSION or None + s["channel"] = FC_CHANNEL or None + s["revision"] = FC_REVISION or None + s["accel"] = accel.LAST or None return JSONResponse(s) @@ -169,7 +192,11 @@ _PAGE = """ width:30px;height:32px;font:700 16px system-ui;cursor:pointer} .step:hover{border-color:var(--acc)} #conc,#bw{width:3.4rem;height:32px;text-align:center;font:700 16px system-ui;background:#11151a; - color:var(--fg);border:1px solid var(--bd);border-radius:8px} + color:var(--fg);border:1px solid var(--bd);border-radius:8px;appearance:textfield;-moz-appearance:textfield} + /* The browser's own spin arrows, hidden: the − / + beside each field are the + control, styled like the rest of the page (operator, 2026-09-24). */ + #conc::-webkit-inner-spin-button,#conc::-webkit-outer-spin-button, + #bw::-webkit-inner-spin-button,#bw::-webkit-outer-spin-button{-webkit-appearance:none;margin:0} .unit{color:var(--mut);font-size:12px;font-weight:600} .hint{color:var(--mut);font-size:12px;margin-top:12px} .tiles{display:grid;grid-template-columns:repeat(6,1fr);gap:8px;margin-bottom:16px} @@ -203,11 +230,12 @@ _PAGE = """
FabledCurator GPU agent
—
-

Server — · token — · build __BUILD__

+

Server — · token — · build __VERSION__

+ @@ -225,7 +253,9 @@ _PAGE = """
+ + MB/s
@@ -262,7 +292,7 @@ _PAGE = """ + diff --git a/extension/popup/popup.js b/extension/popup/popup.js index 7becdba..1fe7410 100644 --- a/extension/popup/popup.js +++ b/extension/popup/popup.js @@ -76,7 +76,7 @@ function updateConnectionDot(connected) { async function checkForUpdate() { try { const r = await browser.runtime.sendMessage({ type: 'CHECK_UPDATE' }); - if (r && r.updateAvailable && r.xpiUrl) showUpdateBanner(r); + if (r && r.updateAvailable && r.installPageUrl) showUpdateBanner(r); } catch { /* non-fatal */ } } @@ -86,10 +86,14 @@ function showUpdateBanner(r) { // exactly as it did before the field existed. const channel = r.channel ? ` (${r.channel})` : ''; document.getElementById('update-text').textContent = - `Update available${channel} — v${r.latestVersion} (installed v${r.currentVersion})`; - // Opening the signed XPI triggers Firefox's native install prompt. + `Update available${channel} — v${r.latestVersion} (installed v${r.currentVersion}). ` + + 'Opens FabledCurator — click “Install Firefox extension” there.'; + // Opens FC's install card rather than the XPI: Firefox only installs an + // add-on from a user click on a web page, never from a tab an extension + // opened on the .xpi itself. document.getElementById('update-btn').addEventListener('click', () => { - browser.tabs.create({ url: r.xpiUrl }); + browser.tabs.create({ url: r.installPageUrl }); + window.close(); }); document.getElementById('update-banner').classList.remove('hidden'); } @@ -159,7 +163,11 @@ async function exportPlatformCookies(key, card) { try { const r = await browser.runtime.sendMessage({ type: 'EXPORT_COOKIES', platform: key }); if (r.error) showError(r.error); - else { + else if (key === 'discord') { + const m = tokenExportMessage(r.verify); + showStatusMessage(m.text, m.kind); + await loadPlatformStatus(); + } else { const n = r.cookieCount ?? null; const verifiedSuffix = r.verified ? ' (verified ✓)' : ''; const msg = n !== null @@ -205,7 +213,10 @@ async function loadSources() { c.appendChild(mutedNote('No sources yet.')); return; } - for (const src of r.sources) c.appendChild(createSourceRow(src)); + // Grouped by artist so a creator's Patreon and Discord sit together. + const sorted = [...r.sources].sort((a, b) => + (a.artist_name || '').localeCompare(b.artist_name || '') || a.id - b.id); + for (const src of sorted) c.appendChild(createSourceRow(src)); } function createSourceRow(src) { @@ -215,11 +226,16 @@ function createSourceRow(src) { info.className = 'info'; const name = document.createElement('div'); name.className = 'name'; - name.textContent = `${src.platform} · #${src.id}`; + const platformName = PLATFORMS[src.platform]?.name || src.platform; + name.textContent = `${src.artist_name || `Source #${src.id}`} · ${platformName}`; + const state = sourceStatus(src); + const st = document.createElement('div'); + st.className = `status ${state.kind}`; + st.textContent = state.text; const url = document.createElement('div'); url.className = 'url'; url.textContent = src.url; - info.appendChild(name); info.appendChild(url); + info.appendChild(name); info.appendChild(st); info.appendChild(url); const play = document.createElement('button'); play.className = 'play'; play.textContent = '▶'; @@ -229,7 +245,7 @@ function createSourceRow(src) { const r = await browser.runtime.sendMessage({ type: 'CHECK_SOURCE', sourceId: src.id }); play.disabled = false; if (r.error) showError(r.error); - else showSuccess(`Triggered check for source #${src.id}`); + else showSuccess(`Check queued for ${src.artist_name || `source #${src.id}`} (${platformName})`); }); row.appendChild(info); row.appendChild(play); return row; diff --git a/extension/test/artist-url-samples.json b/extension/test/artist-url-samples.json index 712edf4..ceb793d 100644 --- a/extension/test/artist-url-samples.json +++ b/extension/test/artist-url-samples.json @@ -134,5 +134,42 @@ { "url": "https://www.hentai-foundry.com/pictures/popular", "why": "gallery listing, not a user" }, { "url": "https://www.hentai-foundry.com/", "why": "site root" } ] + }, + + "discord": { + "match": [ + { + "url": "https://discord.com/channels/111111111111111111/222222222222222222", + "slug": "111111111111111111/222222222222222222", + "why": "a channel: the slug is server/channel -- a Discord URL names a place, not a creator, so the artist is chosen in the Add panel" + }, + { + "url": "https://discord.com/channels/111111111111111111", + "slug": "111111111111111111", + "why": "a whole server" + }, + { + "url": "https://discord.com/channels/111111111111111111/222222222222222222/333333333333333333", + "slug": "111111111111111111/222222222222222222", + "why": "a message jump link still names its channel" + }, + { + "url": "https://ptb.discord.com/channels/111111111111111111/222222222222222222", + "slug": "111111111111111111/222222222222222222", + "why": "the ptb and canary clients serve the same channels" + }, + { + "url": "https://discord.com/channels/111111111111111111/222222222222222222/", + "slug": "111111111111111111/222222222222222222", + "why": "trailing slash is tolerated" + } + ], + "no_match": [ + { "url": "https://discord.com/channels/@me", "why": "the DM list is not a source" }, + { "url": "https://discord.com/channels/@me/222222222222222222", "why": "a DM is not a source" }, + { "url": "https://discord.com/app", "why": "the app shell, no server open" }, + { "url": "https://discord.com/channels/111111111111111111/222222222222222222/threads/444444444444444444", "why": "thread links are left to the manual Add form" }, + { "url": "https://discord.com/servers/111111111111111111", "why": "a server-discovery page, not a channel" } + ] } } diff --git a/extension/test/chip.spec.js b/extension/test/chip.spec.js new file mode 100644 index 0000000..2edfa43 --- /dev/null +++ b/extension/test/chip.spec.js @@ -0,0 +1,200 @@ +import { describe, it, expect } from 'vitest' +import { loadLib } from './helpers/loadLib.js' + +const { chipState, chipLabel, panelDefaults, addRequest, renameOffer } = loadLib('chip.js', [ + 'chipState', + 'chipLabel', + 'panelDefaults', + 'addRequest', + 'renameOffer' +]) + +const discord = (extra = {}) => ({ + platform: 'discord', + slug: '111/222', + discord: { + server_id: '111', + channel_id: '222', + server_name: 'Studio', + channel_name: 'drops', + server_url: 'https://discord.com/channels/111', + channel_url: 'https://discord.com/channels/111/222' + }, + ...extra +}) + +describe('chip state and label', () => { + it('keeps the one-click wording on creator platforms', () => { + expect(chipLabel({ state: 'new', platform: 'patreon' }, 'Patreon')).toBe('+ Add to FabledCurator') + expect( + chipLabel({ state: 'artist_match', platform: 'patreon', artist: { name: 'Atole' } }, 'Patreon') + ).toBe('+ Add Patreon source to Atole') + expect(chipLabel({ state: 'source_match', platform: 'patreon' }, 'Patreon')).toBe( + '✓ In FabledCurator · Patreon' + ) + }) + + it('offers the channel by name on Discord, whatever the suggestion', () => { + expect(chipLabel(discord({ state: 'new' }), 'Discord')).toBe('+ Add #drops to FabledCurator') + expect(chipLabel(discord({ state: 'artist_match', artist: { name: 'A' } }), 'Discord')).toBe( + '+ Add #drops to FabledCurator' + ) + }) + + it('says "this channel" when the token could not read the name', () => { + const p = discord({ state: 'new' }) + p.discord.channel_name = null + expect(chipLabel(p, 'Discord')).toBe('+ Add this channel to FabledCurator') + }) + + it('says whose source a Discord channel already is, and when the server covers it', () => { + const artist = { name: 'Tamada', slug: 'tamada' } + expect(chipLabel(discord({ state: 'source_match', artist, covered_by_server: false }), 'Discord')).toBe( + '✓ In FabledCurator · Tamada' + ) + expect(chipLabel(discord({ state: 'source_match', artist, covered_by_server: true }), 'Discord')).toBe( + '✓ Whole server in FabledCurator · Tamada' + ) + }) + + it('falls back to the generic add when the probe failed', () => { + expect(chipState({ error: 'x' })).toBe('new') + expect(chipLabel({ error: 'x' }, '')).toBe('+ Add to FabledCurator') + expect(chipState({ state: 'source_match' })).toBe('source-match') + }) +}) + +describe('Add panel on Discord', () => { + it('opens on the channel with the suggested artist preselected', () => { + const d = panelDefaults(discord({ state: 'artist_match', artist: { id: 7, name: 'Tamada' } })) + expect(d.scope).toBe('channel') + expect(d.artist).toEqual({ id: 7, name: 'Tamada' }) + expect(d.artistName).toBe('Tamada') + }) + + it('proposes a new artist named after the server when nothing is suggested', () => { + const d = panelDefaults(discord({ state: 'new' })) + expect(d.artist).toBe(null) + expect(d.artistName).toBe('Studio') + }) + + it('sends a picked artist by id', () => { + const choice = panelDefaults(discord({ state: 'artist_match', artist: { id: 7, name: 'Tamada' } })) + expect(addRequest(choice)).toEqual({ + url: 'https://discord.com/channels/111/222', + artistId: 7 + }) + }) + + it('sends a typed name once the picked artist has been edited away', () => { + const choice = panelDefaults(discord({ state: 'artist_match', artist: { id: 7, name: 'Tamada' } })) + choice.artistName = 'Tamada Alt' + expect(addRequest(choice)).toEqual({ + url: 'https://discord.com/channels/111/222', + artistName: 'Tamada Alt' + }) + }) + + it('adds the whole server when the operator picks it', () => { + const choice = panelDefaults(discord({ state: 'new' })) + choice.scope = 'server' + expect(addRequest(choice).url).toBe('https://discord.com/channels/111') + }) + + it('has nothing to send without an artist', () => { + const choice = panelDefaults(discord({ state: 'new' })) + choice.artistName = ' ' + expect(addRequest(choice)).toBe(null) + }) +}) + +const { squashName, exactArtistMatch, inlineCompletion } = loadLib('chip.js', [ + 'squashName', + 'exactArtistMatch', + 'inlineCompletion' +]) + +describe('artist matching for the Add panel', () => { + const results = [ + { id: 1, name: 'TamadaHeijun' }, + { id: 2, name: 'Tamago' }, + { id: 3, name: 'Sabu Art' } + ] + + it('treats spacing, case and punctuation as the same name', () => { + expect(squashName('Tamada Heijun')).toBe('tamadaheijun') + expect(squashName('sabu_art!')).toBe('sabuart') + expect(squashName('玉田 平順')).toBe('玉田平順') + }) + + it('finds the result that is the query, spacing aside', () => { + expect(exactArtistMatch('tamada heijun', results)).toEqual({ id: 1, name: 'TamadaHeijun' }) + expect(exactArtistMatch('Tama', results)).toBe(null) + expect(exactArtistMatch(' ', results)).toBe(null) + }) + + it('autofills the first name that extends what was typed', () => { + expect(inlineCompletion('tamad', results)).toEqual({ id: 1, name: 'TamadaHeijun' }) + expect(inlineCompletion('Sab', results).name).toBe('Sabu Art') + // Nothing to add once the name is complete, or when nothing extends it. + expect(inlineCompletion('Sabu Art', results)).toBe(null) + expect(inlineCompletion('heijun', results)).toBe(null) + expect(inlineCompletion('', results)).toBe(null) + }) +}) + +describe('Add panel on Patreon and SubscribeStar', () => { + const PAGE = 'https://www.patreon.com/cw/tamadaheijun' + const patreon = (extra = {}) => ({ platform: 'patreon', slug: 'tamadaheijun', ...extra }) + + it('opens on the Patreon display name as a new artist', () => { + const c = panelDefaults(patreon({ state: 'new', display_name: 'Tamada Heijun' }), PAGE) + expect(c.scope).toBe('page') + expect(c.artistName).toBe('Tamada Heijun') + expect(addRequest(c)).toEqual({ url: PAGE, artistName: 'Tamada Heijun' }) + }) + + it('leaves an untouched URL handle to the server to resolve', () => { + const c = panelDefaults(patreon({ state: 'new' }), PAGE) + expect(c.artistName).toBe('tamadaheijun') + expect(addRequest(c)).toEqual({ url: PAGE }) + c.artistName = 'Someone Else' + expect(addRequest(c)).toEqual({ url: PAGE, artistName: 'Someone Else' }) + }) + + it('offers the Patreon name when joining an artist known by another name', () => { + const c = panelDefaults(patreon({ state: 'new', display_name: 'Tamada Heijun' }), PAGE) + c.artist = { id: 4, name: 'tamada' } + c.artistName = 'tamada' + expect(renameOffer(c)).toEqual({ from: 'tamada', to: 'Tamada Heijun' }) + expect(addRequest(c)).toEqual({ url: PAGE, artistId: 4, usePlatformName: true }) + c.adoptPlatformName = false + expect(addRequest(c)).toEqual({ url: PAGE, artistId: 4 }) + }) + + it('offers no rename when the names agree or the name was not read', () => { + const same = panelDefaults(patreon({ state: 'new', display_name: 'Tamada Heijun' }), PAGE) + same.artist = { id: 4, name: 'Tamada Heijun' } + expect(renameOffer(same)).toBe(null) + const unread = panelDefaults(patreon({ state: 'new' }), PAGE) + unread.artist = { id: 4, name: 'tamada' } + expect(renameOffer(unread)).toBe(null) + }) + + it('never renames from SubscribeStar or Discord — Patreon is the canon', () => { + const ss = panelDefaults( + { platform: 'subscribestar', slug: 'tamada', state: 'new', display_name: 'SS Tamada' }, + 'https://subscribestar.adult/tamada' + ) + ss.artist = { id: 4, name: 'Tamada Heijun' } + ss.artistName = 'Tamada Heijun' + expect(renameOffer(ss)).toBe(null) + expect(addRequest(ss)).toEqual({ url: 'https://subscribestar.adult/tamada', artistId: 4 }) + }) + + it('preselects the artist the URL already names', () => { + const c = panelDefaults(patreon({ state: 'artist_match', artist: { id: 9, name: 'Tamada' } }), PAGE) + expect(c.artist).toEqual({ id: 9, name: 'Tamada' }) + expect(c.artistName).toBe('Tamada') + }) +}) diff --git a/extension/test/platforms.spec.js b/extension/test/platforms.spec.js index bee61ab..367419c 100644 --- a/extension/test/platforms.spec.js +++ b/extension/test/platforms.spec.js @@ -7,10 +7,14 @@ import { loadLib } from './helpers/loadLib.js' const EXT_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), '..') const manifest = JSON.parse(readFileSync(path.join(EXT_DIR, 'manifest.json'), 'utf8')) -const { getPlatformFromUrl, isArtistPage, PLATFORMS, PLATFORM_ARTIST_PATTERNS } = loadLib( - 'platforms.js', - ['getPlatformFromUrl', 'isArtistPage', 'PLATFORMS', 'PLATFORM_ARTIST_PATTERNS'] -) +const { getPlatformFromUrl, isArtistPage, parseDiscordUrl, PLATFORMS, PLATFORM_ARTIST_PATTERNS } = + loadLib('platforms.js', [ + 'getPlatformFromUrl', + 'isArtistPage', + 'parseDiscordUrl', + 'PLATFORMS', + 'PLATFORM_ARTIST_PATTERNS' + ]) describe('getPlatformFromUrl', () => { it('identifies each platform from a domain URL', () => { @@ -88,8 +92,11 @@ describe('isArtistPage', () => { ) }) - it('returns false for a platform with no artist pattern (discord)', () => { + it('matches Discord server and channel pages, not DMs (milestone 429)', () => { + expect(isArtistPage('https://discord.com/channels/111/222', 'discord')).toBe(true) + expect(isArtistPage('https://discord.com/channels/111', 'discord')).toBe(true) expect(isArtistPage('https://discord.com/channels/@me', 'discord')).toBe(false) + expect(isArtistPage('https://discord.com/channels/@me/222', 'discord')).toBe(false) }) it('returns false for an unknown platform key', () => { @@ -100,8 +107,8 @@ describe('isArtistPage', () => { describe('platform table integrity', () => { it('gives every artist pattern a corresponding platform entry', () => { // A pattern keyed to a platform that no longer exists is dead code that - // silently never fires; the reverse (a platform with no pattern) is the - // legitimate discord case, so only this direction is an error. + // silently never fires; the reverse (a platform with no pattern) would be + // a platform the button never offers, a product choice rather than an error. for (const key of Object.keys(PLATFORM_ARTIST_PATTERNS)) { expect(Object.keys(PLATFORMS)).toContain(key) } @@ -125,7 +132,8 @@ describe('platform table integrity', () => { const samples = { patreon: 'https://www.patreon.com/cw/Atole', subscribestar: 'https://subscribestar.adult/someone', - hentaifoundry: 'https://www.hentai-foundry.com/user/someone' + hentaifoundry: 'https://www.hentai-foundry.com/user/someone', + discord: 'https://ptb.discord.com/channels/111/222' } for (const [key, url] of Object.entries(samples)) { expect(isArtistPage(url, key), `${key} artist pattern`).toBe(true) @@ -152,7 +160,7 @@ describe('manifest.json agrees with the platform table', () => { ) expect(owner, `no platform claims content-script match "${m}"`).toBeTruthy() // The content script exists to draw the Add-as-source button, so a - // platform with no artist pattern (discord) has no business here. + // platform with no artist pattern has no business here. expect( PLATFORM_ARTIST_PATTERNS[owner[0]], `"${m}" injects for ${owner[0]}, which has no artist pattern` @@ -232,9 +240,8 @@ describe('the JS<->Py artist-pattern mirror (#3093)', () => { it('has samples for every platform that has an artist pattern', () => { // The guard's own coverage check: without it, deleting a platform's - // samples would make this block pass by testing less. Discord is - // deliberately in neither — it is channel-based, with no creator page to - // put a button on, so it has no artist pattern on either side. + // samples would make this block pass by testing less. Discord joined at + // milestone 429 — its slug is server/channel and the artist is chosen. expect(Object.keys(samples).sort()).toEqual(Object.keys(PLATFORM_ARTIST_PATTERNS).sort()) }) @@ -247,3 +254,26 @@ describe('the JS<->Py artist-pattern mirror (#3093)', () => { } }) }) + +describe('parseDiscordUrl', () => { + it('reads the server and channel ids the Add panel offers', () => { + expect(parseDiscordUrl('https://discord.com/channels/111/222')).toEqual({ + serverId: '111', + channelId: '222' + }) + expect(parseDiscordUrl('https://discord.com/channels/111/222/333')).toEqual({ + serverId: '111', + channelId: '222' + }) + expect(parseDiscordUrl('https://discord.com/channels/111')).toEqual({ + serverId: '111', + channelId: null + }) + }) + + it('returns null for anything the artist pattern rejects', () => { + expect(parseDiscordUrl('https://discord.com/channels/@me/222')).toBe(null) + expect(parseDiscordUrl('https://discord.com/app')).toBe(null) + expect(parseDiscordUrl('')).toBe(null) + }) +}) diff --git a/extension/test/popup-format.spec.js b/extension/test/popup-format.spec.js new file mode 100644 index 0000000..ceee042 --- /dev/null +++ b/extension/test/popup-format.spec.js @@ -0,0 +1,53 @@ +import { describe, it, expect } from 'vitest' +import { loadLib } from './helpers/loadLib.js' + +const { sourceStatus, relativeTime, tokenExportMessage } = loadLib('popup-format.js', [ + 'sourceStatus', + 'relativeTime', + 'tokenExportMessage' +]) + +const NOW = Date.parse('2026-09-25T12:00:00Z') +const src = (extra = {}) => ({ enabled: true, last_error: null, backfill_state: null, ...extra }) + +describe('sourceStatus', () => { + it('puts an error first, trimmed to its first line', () => { + const s = sourceStatus(src({ last_error: 'Discord rejected the token\nstack…', backfill_state: 'running' }), NOW) + expect(s).toEqual({ text: 'Error — Discord rejected the token', kind: 'error' }) + }) + + it('shows a running backfill and its progress', () => { + expect(sourceStatus(src({ backfill_state: 'running', backfill_chunks: 0 }), NOW).text).toBe('Backfill queued') + expect(sourceStatus(src({ backfill_state: 'running', backfill_chunks: 3 }), NOW).text).toBe( + 'Backfilling — 3 chunks done' + ) + }) + + it('says when a source was last checked, or that it never was', () => { + expect(sourceStatus(src({ last_checked_at: '2026-09-25T11:55:00Z' }), NOW).text).toBe('Checked 5m ago') + expect(sourceStatus(src({ last_checked_at: null }), NOW).text).toBe('Not checked yet') + }) + + it('shows a disabled source as disabled, whatever else it carries', () => { + expect(sourceStatus(src({ enabled: false, last_error: 'x' }), NOW).text).toBe('Disabled') + }) +}) + +describe('relativeTime', () => { + it('uses the web UI formatRelative buckets', () => { + expect(relativeTime('2026-09-25T11:59:18Z', NOW)).toBe('42s ago') + expect(relativeTime('2026-09-25T09:00:00Z', NOW)).toBe('3h ago') + expect(relativeTime('2026-09-23T12:00:00Z', NOW)).toBe('2d ago') + expect(relativeTime('garbage', NOW)).toBe('Never') + }) +}) + +describe('tokenExportMessage', () => { + it('distinguishes verified, rejected and untested tokens', () => { + expect(tokenExportMessage({ valid: true, reason: 'Token valid (me)' }).kind).toBe('success') + expect(tokenExportMessage({ valid: false, reason: 'Discord rejected the token' }).kind).toBe('error') + const untested = tokenExportMessage({ valid: null, reason: 'No enabled source' }) + expect(untested.kind).toBe('warning') + expect(untested.text).toContain('No enabled source') + }) +}) diff --git a/extension/test/version.spec.js b/extension/test/version.spec.js index b84a7e1..a8f945e 100644 --- a/extension/test/version.spec.js +++ b/extension/test/version.spec.js @@ -10,7 +10,7 @@ const readText = (...seg) => readFileSync(path.join(EXT_DIR, ...seg), 'utf8') // Only the git-free subcommands are exercised here: `version` shells out to // git, and the extension lane runs on node:24-bookworm-slim which may not ship -// it. That one is covered where git is guaranteed — ci.yml's extension-version +// it. That one is covered where git is guaranteed — build.yml's extension-version // lane and build.yml both run on ci-python. const packaging = (cmd) => execFileSync('sh', [path.join(EXT_DIR, 'scripts', 'packaging.sh'), cmd], { @@ -39,7 +39,7 @@ describe('packaging.sh — the single definition of what ships', () => { it('emits glob patterns literally, never expanded against the working tree', () => { // The script iterates its lists with deliberate word-splitting, so it must // run with pathname expansion off. Without that, invoking it from a cwd - // where test/ exists (exactly how ci.yml and vitest call it) expands + // where test/ exists (exactly how build.yml and vitest call it) expands // `test/**` into the individual spec files, and the pathspec silently stops // covering anything added later. const pathspec = packaging('pathspec') @@ -116,7 +116,7 @@ describe('consumers delegate rather than keeping their own copy', () => { ) it('no workflow hardcodes the packaged-file set', () => { - // ci.yml used to substitute `packaging.sh pathspec` directly, for the + // build.yml used to substitute `packaging.sh pathspec` directly, for the // manual-bump guard that milestone 271 step 5 retired. Nothing inlines the // set today, and nothing should start to: a literal :(exclude)extension/... // in a workflow means someone bypassed the shared definition, which is @@ -157,7 +157,7 @@ describe('extension version', () => { // // It is still asserted, for one reason: `npm run build` locally packages // whatever is committed, so a value AMO would reject turns a local build - // into a confusing failure with no CI signal ahead of it. ci.yml checks + // into a confusing failure with no CI signal ahead of it. build.yml checks // the same grammar against the DERIVED value, which is the one AMO sees. for (const file of ['manifest.json', 'package.json']) { expect(read(file).version, `${file} version is not AMO-shaped`).toMatch(AMO) diff --git a/frontend/src/components/TopNav.vue b/frontend/src/components/TopNav.vue index 31a9645..e1249ca 100644 --- a/frontend/src/components/TopNav.vue +++ b/frontend/src/components/TopNav.vue @@ -153,6 +153,14 @@ const health = computed(() => { label: (worst?.detail || 'A part has stopped') + suffix, } } + // Running but slow — a GPU agent that fell back to the CPU (#4410). Worth + // the amber dot: nothing else anywhere says so. + if (overall === 'degraded') { + return { + icon: 'mdi-speedometer-slow', color: 'warning', + label: (worst?.detail || 'A part is running degraded') + suffix, + } + } if (overall === 'stale') { return { icon: 'mdi-alert', color: 'warning', diff --git a/frontend/src/components/artist/ArtistHeader.vue b/frontend/src/components/artist/ArtistHeader.vue index 2c63210..992622a 100644 --- a/frontend/src/components/artist/ArtistHeader.vue +++ b/frontend/src/components/artist/ArtistHeader.vue @@ -103,9 +103,9 @@ const stats = computed(() => { padding: 0.5rem 1rem; background: linear-gradient( to bottom, - rgba(20, 23, 26, 0.92) 0%, - rgba(20, 23, 26, 0.65) 60%, - rgba(20, 23, 26, 0) 100% + rgba(var(--fc-chrome-rgb), 0.92) 0%, + rgba(var(--fc-chrome-rgb), 0.65) 60%, + rgba(var(--fc-chrome-rgb), 0) 100% ); backdrop-filter: blur(2px); -webkit-backdrop-filter: blur(2px); diff --git a/frontend/src/components/gallery/GalleryFilterBar.vue b/frontend/src/components/gallery/GalleryFilterBar.vue index 0c5dc39..7217b2b 100644 --- a/frontend/src/components/gallery/GalleryFilterBar.vue +++ b/frontend/src/components/gallery/GalleryFilterBar.vue @@ -329,7 +329,7 @@ function pushFilter(mutate) { opaque than the bar/nav so the controls stay legible. */ .fc-filterbar-wrap :deep(.v-field), .fc-filterbar-wrap :deep(.v-btn-group) { - background-color: rgba(20, 23, 26, 0.72); + background-color: rgba(var(--v-theme-background), 0.72); } /* Media toggle (All / Images / Videos) as ONE cohesive segmented control. FC's global VBtn { rounded: 'pill' } default made Vuetify 4 pill-round each diff --git a/frontend/src/components/gallery/HiddenReviewStrip.vue b/frontend/src/components/gallery/HiddenReviewStrip.vue index 8a63786..5ff8384 100644 --- a/frontend/src/components/gallery/HiddenReviewStrip.vue +++ b/frontend/src/components/gallery/HiddenReviewStrip.vue @@ -2,14 +2,14 @@
mdi-alert-outline - {{ items.length }} auto-tagged {{ items.length === 1 ? 'image' : 'images' }} - may be real content — review + {{ items.length }} {{ items.length === 1 ? 'auto-tag' : 'auto-tags' }} to check
@@ -21,22 +21,23 @@ class="fc-review-card__thumb" loading="lazy" >
+
{{ question(it) }}
- also looks like {{ it.conflict_name || 'content' }} + also {{ Math.round(it.conflict_score * 100) }}% + {{ it.conflict_name || 'content' }}
-
{{ tagLine(it) }}
+ >Is {{ withArticle(it) }} + >Is not {{ withArticle(it) }}
@@ -55,11 +56,18 @@ const items = ref([]) const busy = ref([]) function keyOf(it) { return `${it.image_id}:${it.tag_id}` } -// Chrome flags hide the image (keep-hidden / un-hide); process flags leave it -// visible and just tagged (keep-tag / remove-tag). Same endpoints, different words. -function tagLine(it) { return (it.mode === 'process' ? 'auto-tagged ' : 'hidden as ') + it.tag_name } -function keepLabel(it) { return it.mode === 'process' ? 'Keep tag' : 'Keep hidden' } -function removeLabel(it) { return it.mode === 'process' ? 'Remove tag' : 'Un-hide' } +// The card asks whether the image IS the auto-applied system tag, and the buttons +// answer that (operator, #4424: "is a " / "is not a "). "Is" keeps the +// tag ('keep'); "Is not" removes it, un-hiding a chrome image ('unhide'). The +// content tag it also scored on is the reason it was flagged, not the question. +function noun(it) { return it.tag_name === 'wip' ? 'WIP' : it.tag_name } +function withArticle(it) { return (/^[aeiou]/i.test(noun(it)) ? 'an ' : 'a ') + noun(it) } +function question(it) { return `Is this ${withArticle(it)}?` } +function reasonTitle(it) { + const hidden = it.mode === 'process' ? '' : ' It is hidden from the gallery until you answer.' + return `Auto-tagged “${it.tag_name}”, but it also scored ${Math.round(it.conflict_score * 100)}% ` + + `on “${it.conflict_name || 'a content tag'}”, so it may be finished art.${hidden}` +} async function load() { // Fetched unconditionally on mount — the strip prompts for pending misfires @@ -77,14 +85,11 @@ async function resolve(it, action) { await api.post(`/api/gallery/hidden-review/${it.image_id}/${it.tag_id}/${action}`) items.value = items.value.filter((x) => keyOf(x) !== k) if (action === 'unhide') { - const verb = it.mode === 'process' ? 'Removed' : 'Un-hidden' - toast({ text: `${verb} — “${it.tag_name}” removed; it'll train the head`, type: 'success' }) + const shown = it.mode === 'process' ? '' : ', back in the gallery' + toast({ text: `Not ${withArticle(it)} — “${it.tag_name}” removed${shown}; the tagger learns from it`, type: 'success' }) } } catch (e) { - toast({ - text: `Could not ${action === 'keep' ? 'keep hidden' : 'un-hide'}: ${e.message}`, - type: 'error', - }) + toast({ text: `Could not save your answer: ${e.message}`, type: 'error' }) } finally { busy.value = busy.value.filter((x) => x !== k) } @@ -112,7 +117,7 @@ onMounted(load) display: flex; gap: 10px; overflow-x: auto; padding-bottom: 4px; } .fc-review-card { - flex: 0 0 auto; width: 150px; + flex: 0 0 auto; width: 170px; display: flex; flex-direction: column; border: 1px solid rgb(var(--v-theme-surface-light)); border-radius: 6px; overflow: hidden; @@ -123,16 +128,16 @@ onMounted(load) background: rgb(var(--v-theme-surface-light)); } .fc-review-card__body { padding: 6px 8px; } -.fc-review-card__conflict { - font-size: 11px; color: rgb(var(--v-theme-on-surface)); +.fc-review-card__question { + font-size: 12px; font-weight: 600; color: rgb(var(--v-theme-on-surface)); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } -.fc-review-card__conflict strong { color: rgb(var(--v-theme-warning)); } -.fc-review-card__tag { +.fc-review-card__reason { font-size: 10px; color: rgb(var(--v-theme-on-surface-variant)); margin: 1px 0 6px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.fc-review-card__reason strong { color: rgb(var(--v-theme-warning)); font-weight: 600; } .fc-review-card__acts { display: flex; gap: 4px; } .fc-review-btn { flex: 1; font-size: 11px; padding: 3px 4px; border-radius: 4px; diff --git a/frontend/src/components/modal/ImageViewer.vue b/frontend/src/components/modal/ImageViewer.vue index 1ff8d69..f3db735 100644 --- a/frontend/src/components/modal/ImageViewer.vue +++ b/frontend/src/components/modal/ImageViewer.vue @@ -213,7 +213,7 @@ function nextFrame() { --fc-side-w: 320px; /* Obsidian haze (#14171A = 20,23,26) — same palette as TopNav, mid-opacity + blur so the page behind shows through faintly. */ - background: rgba(20, 23, 26, 0.65); + background: rgba(var(--v-theme-background), 0.65); backdrop-filter: blur(8px); -webkit-backdrop-filter: blur(8px); outline: none; @@ -221,7 +221,7 @@ function nextFrame() { } .fc-viewer__close, .fc-viewer__nav { position: absolute; top: 50%; - background: rgba(20, 23, 26, 0.7); + background: rgba(var(--v-theme-background), 0.7); color: rgb(var(--v-theme-parchment, 232 228 216)); border: 1px solid rgb(var(--v-theme-surface-light)); border-radius: 50%; @@ -359,7 +359,7 @@ function nextFrame() { z-index: 1; /* Opaque obsidian so the scrolling panel never bleeds through the haze behind the pinned image. */ - background: rgb(20, 23, 26); + background: rgb(var(--v-theme-background)); } .fc-viewer__side { width: 100%; diff --git a/frontend/src/components/posts/PostCard.vue b/frontend/src/components/posts/PostCard.vue index 8a8cfdc..d8a7981 100644 --- a/frontend/src/components/posts/PostCard.vue +++ b/frontend/src/components/posts/PostCard.vue @@ -20,7 +20,7 @@ class="fc-post-card__synthetic" :title="synthesisTitle" > - grouped by FabledCurator + {{ synthesisChip }} · {{ totalImages }} image{{ totalImages === 1 ? '' : 's' }} + + + · +{{ refImages.length }} from Discord + @@ -66,9 +72,16 @@ + +
+
+ Discord · {{ shortDate(t.date) }}{{ t.role === 'variant' ? ' · variant' : '' }} +
+

{{ t.text }}

+
+ +
@@ -191,6 +235,7 @@ import { RouterLink } from 'vue-router' import { useModalStore } from '../../stores/modal.js' import { usePostsStore } from '../../stores/posts.js' import { toPlainText } from '../../utils/htmlSanitize.js' +import { toast } from '../../utils/toast.js' import PostSeriesMenu from './PostSeriesMenu.vue' import PostTranslationControl from './PostTranslationControl.vue' @@ -207,8 +252,16 @@ const modal = useModalStore() const detail = ref(null) const attachments = computed(() => props.post.attachments || []) -const images = computed(() => props.post.thumbnails || []) -const totalImages = computed(() => images.value.length + (props.post.thumbnails_more || 0)) +// #4402. A teaser's card also shows what it points at: the linked drop's +// images and the piece's other variants, each tagged with the post it really +// belongs to. They follow the post's own images so the teaser keeps its hero, +// and the post's own capped list stays a PREFIX of everything shown — which is +// what lets the "+N" tile and the modal playlist keep indexing correctly. +const unified = computed(() => props.post.unified || null) +const ownImages = computed(() => props.post.thumbnails || []) +const refImages = computed(() => unified.value?.thumbnails || []) +const images = computed(() => [...ownImages.value, ...refImages.value]) +const totalImages = computed(() => ownImages.value.length + (props.post.thumbnails_more || 0)) const plainTitle = computed(() => toPlainText(props.post.post_title)) // #388 E2. Non-null `synthesized_by` means FC authored this row by grouping a @@ -217,11 +270,18 @@ const plainTitle = computed(() => toPlainText(props.post.post_title)) // post dict by hand, must degrade to "not synthetic" rather than throw. const synthesized = computed(() => Boolean(props.post.synthesized_by)) const messageCount = computed(() => props.post.synthesis?.message_count ?? 0) +// A drop of one message stays a synthetic post (teaser matching only looks at +// drops), but there is nothing grouped in it — #4390: "Grouped from 1 Discord +// message" described a wrapper, not a grouping. Say what it is instead. const synthesisTitle = computed(() => { const n = messageCount.value if (!n) return 'Grouped from Discord' - return `Grouped from ${n} Discord message${n === 1 ? '' : 's'}` + if (n === 1) return 'Discord message' + return `Grouped from ${n} Discord messages` }) +const synthesisChip = computed(() => + messageCount.value === 1 ? 'from Discord' : 'grouped by FabledCurator' +) const hero = computed(() => images.value[0]) @@ -273,6 +333,44 @@ const grewAt = computed(() => (synthesized.value ? props.post.last_grew_at : nul // never beside the artwork. Defaults to [] so a post dict from before the // feature (or composed by hand) renders without a link rather than throwing. const associations = computed(() => props.post.associations || []) +// The teaser side of a link is drawn by the unified block when there is one; +// only a link it does not cover (the drop's "announced by", or a feed payload +// from before #4402) keeps the plain text link. +const plainLinks = computed(() => + unified.value + ? associations.value.filter((a) => a.role !== 'announces') + : associations.value, +) + +function linkLabel (l) { + if (l.linked_by === 'fc') { + return l.token + ? `Linked by FabledCurator — matched on “${l.token}”` + : 'Linked by FabledCurator' + } + return 'Linked to its Discord drop' +} + +function refTitle (t) { + return t.role === 'variant' ? 'a variant from Discord' : 'from the Discord drop' +} + +function shortDate (iso) { + return new Date(iso).toLocaleDateString() +} + +const unlinking = ref(null) +async function undoLink (l) { + unlinking.value = l.association_id + try { + await postsStore.unlink(props.post.id, l.association_id) + toast({ text: 'Unlinked — the Discord drop returns to the feed on the next load', type: 'success' }) + } catch (e) { + toast({ text: `Unlink failed: ${e.message}`, type: 'error' }) + } finally { + unlinking.value = null + } +} const grewRelative = computed(() => (grewAt.value ? relativeFrom(grewAt.value) : '')) const absoluteDate = computed(() => new Date(sortDateIso.value).toLocaleString()) function relativeFrom (iso) { @@ -298,7 +396,8 @@ async function fullImageIds () { detail.value = await postsStore.getPostFull(props.post.id) } catch { /* fall back to the capped feed list */ } } - return (detail.value?.thumbnails || images.value).map((t) => t.image_id) + const own = detail.value?.thumbnails || ownImages.value + return [...own, ...refImages.value].map((t) => t.image_id) } async function openModal (imageId) { @@ -447,6 +546,54 @@ function formatBytes (n) { .fc-post-card__grew { color: rgb(var(--v-theme-accent)); } .fc-post-card__assoc { margin-top: 8px; } + +/* #4402 — the unified block. Quiet by design: it explains the references in + the rail, it does not compete with them. */ +.fc-post-card__unified { + margin-top: 10px; + padding-left: 10px; + border-left: 2px solid rgba(var(--v-theme-accent), 0.5); +} +.fc-post-card__unified-head { + display: flex; align-items: center; flex-wrap: wrap; gap: 6px; + font-size: 0.8125rem; + color: rgb(var(--v-theme-accent)); +} +.fc-post-card__undo { + padding: 0; border: 0; background: none; cursor: pointer; + font-size: 0.75rem; font-weight: 600; + color: rgb(var(--v-theme-on-surface-variant)); +} +.fc-post-card__undo:hover { color: rgb(var(--v-theme-accent)); text-decoration: underline; } +.fc-post-card__undo:disabled { cursor: default; text-decoration: none; } +.fc-post-card__unified-text { margin-top: 6px; } +.fc-post-card__unified-meta { + font-size: 0.7rem; font-weight: 600; text-transform: uppercase; + letter-spacing: 0.04em; + color: rgb(var(--v-theme-on-surface-variant)); +} +.fc-post-card__unified-body { + margin: 2px 0 0; + font-size: 0.85rem; line-height: 1.45; + white-space: pre-wrap; + color: rgb(var(--v-theme-on-surface)); + display: -webkit-box; + -webkit-line-clamp: 3; + -webkit-box-orient: vertical; + overflow: hidden; +} +.fc-post-card__ref-meta { color: rgb(var(--v-theme-accent)); } + +/* A referenced tile carries a corner badge: the images are Discord's, shown + here, and the card must not pass them off as the teaser's own. */ +.fc-post-card__rail-cell { position: relative; } +.fc-post-card__rail-cell--ref { outline: 1px solid rgba(var(--v-theme-accent), 0.55); outline-offset: -1px; } +.fc-post-card__ref-badge { + position: absolute; top: 4px; right: 4px; + padding: 2px; border-radius: 4px; + background: rgba(var(--v-theme-surface), 0.85); + color: rgb(var(--v-theme-accent)); +} .fc-post-card__assoc-link { display: inline-flex; align-items: center; diff --git a/frontend/src/components/settings/ArchiveReextractCard.vue b/frontend/src/components/settings/ArchiveReextractCard.vue index 897a71e..a69a083 100644 --- a/frontend/src/components/settings/ArchiveReextractCard.vue +++ b/frontend/src/components/settings/ArchiveReextractCard.vue @@ -17,7 +17,7 @@ mdi-folder-zip-outline Re-extract archives now Queued ✓ - + diff --git a/frontend/src/components/settings/BrowserExtensionCard.vue b/frontend/src/components/settings/BrowserExtensionCard.vue index 50b6a6a..0b96c0e 100644 --- a/frontend/src/components/settings/BrowserExtensionCard.vue +++ b/frontend/src/components/settings/BrowserExtensionCard.vue @@ -49,16 +49,20 @@ sometimes triggered nothing instead of the install dialog (operator-flagged 2026-05-26). No `download` attribute — that would force a save dialog instead of install. --> + Install Firefox extension Download XPI diff --git a/frontend/src/components/settings/ImportFiltersForm.vue b/frontend/src/components/settings/ImportFiltersForm.vue index 903342f..176def1 100644 --- a/frontend/src/components/settings/ImportFiltersForm.vue +++ b/frontend/src/components/settings/ImportFiltersForm.vue @@ -103,17 +103,6 @@ the Explore browse. Applies to new imports; run the scan below to catch posts already in your library.
- -
- Extends the above to softer cues (sketch, doodle, - scribble). These stay visible and never train - the tagging model — a daily audit flags any that actually look like finished - art for review. Off by default. -
store.settings, (s) => { if (s) Object.assign(local, s) }, { immediate: true }) diff --git a/frontend/src/components/settings/MissingFileRepairCard.vue b/frontend/src/components/settings/MissingFileRepairCard.vue index 301c320..e416dd5 100644 --- a/frontend/src/components/settings/MissingFileRepairCard.vue +++ b/frontend/src/components/settings/MissingFileRepairCard.vue @@ -18,7 +18,7 @@ mdi-file-remove-outline Repair missing-file records Queued ✓ - + diff --git a/frontend/src/components/settings/PostAssociationsCard.vue b/frontend/src/components/settings/PostAssociationsCard.vue index 141d789..0e9f996 100644 --- a/frontend/src/components/settings/PostAssociationsCard.vue +++ b/frontend/src/components/settings/PostAssociationsCard.vue @@ -6,9 +6,10 @@ >
Some creators post a cropped fragment on Patreon to say the real thing has - landed in their Discord. FC proposes those pairs; nothing is linked until - you accept one. A wrong link would tell you two different pieces are the - same, so this stays a suggestion. + landed in their Discord. When the creator's own file name settles it — + the same working name on both posts and nowhere else in their library — + FC links the pair for you. Anything less certain waits here, because a + wrong link would tell you two different pieces are the same.
+ +
+ Off, every pair waits for you here — including the ones there is nothing + left to decide about. +
+
- A pair always needs two reasons — being close in time - and the post itself mentioning Discord. Neither is enough alone at any - setting at or above 0.55, which is what stops a busy posting day from - producing false pairs. + A pair needs two reasons — close in time, the post + mentioning Discord or the server, or a marker the creator uses on + both and almost nowhere else. None is enough alone at any setting at + or above 0.45, which is what stops a busy posting day from producing + false pairs. A shared working name is weighed separately: it says the + two are the same piece rather than that they happened near each + other.
@@ -46,6 +61,47 @@ + +
On the teaser's card
+
+ A linked teaser shows the Discord drop it announced — its images and its + message — and the other versions of the same piece: the wips, alts and + censor passes posted under the same working name. They stay where they + landed in the feed; the card only shows them together. +
+ + + +
+ A drop this close to its teaser is the same release shown twice, so + only the teaser's card stays in the feed. A drop further away keeps + its own card. 0 hides nothing. +
+ + + +
+ How far either side of the teaser to look for the rest of the piece. + Measured on real drops: a piece's versions span up to about six + weeks, while unrelated pieces that happen to share a name are years + apart. 0 shows the drop alone. +
+ + +
{{ store.proposals.length }} waiting for review @@ -57,8 +113,9 @@
- Nothing proposed. That is the expected state most of the time — pairs only - appear when a post both lands near a drop and says it is about Discord. + Nothing waiting. That is the expected state most of the time — a pair the + file names settle is linked without asking, and everything else needs a + post to both land near a drop and point at Discord.
+ · shared marker {{ Math.round(p.signals.marker * 100) }}% + +
+ +
+ both call it + {{ p.signals.identity_token }} + ({{ Math.round((p.signals.identity ?? 0) * 100) }}% — shared with + another post of theirs, so not conclusive on its own)
Link @@ -98,12 +168,18 @@ import SettingNumberField from '../common/SettingNumberField.vue' const store = usePostAssociationsStore() const enabled = ref(true) +const auto = ref(true) const threshold = ref(0.6) const windowHours = ref(24) +const foldHours = ref(24) +const familyDays = ref(60) watch(() => store.enabled, (v) => { enabled.value = v }, { immediate: true }) +watch(() => store.auto, (v) => { auto.value = v }, { immediate: true }) watch(() => store.threshold, (v) => { threshold.value = v }, { immediate: true }) watch(() => store.windowHours, (v) => { windowHours.value = v }, { immediate: true }) +watch(() => store.foldHours, (v) => { foldHours.value = v }, { immediate: true }) +watch(() => store.familyDays, (v) => { familyDays.value = v }, { immediate: true }) onMounted(async () => { // Both swallow their own failures: a settings read that fails should not diff --git a/frontend/src/components/settings/QueuesTable.vue b/frontend/src/components/settings/QueuesTable.vue index 6ccc372..e75a729 100644 --- a/frontend/src/components/settings/QueuesTable.vue +++ b/frontend/src/components/settings/QueuesTable.vue @@ -41,7 +41,7 @@ const props = defineProps({ const QUEUE_NAMES = [ 'default', 'import', 'thumbnail', 'ml', - 'download', 'scan', 'maintenance', + 'download', 'scan', 'maintenance', 'maintenance_long', ] function formatDepth(name) { diff --git a/frontend/src/components/settings/SystemActivityTab.vue b/frontend/src/components/settings/SystemActivityTab.vue index 8470352..5e7f091 100644 --- a/frontend/src/components/settings/SystemActivityTab.vue +++ b/frontend/src/components/settings/SystemActivityTab.vue @@ -198,6 +198,16 @@ const filterErrorType = ref(null) const filterTask = ref(null) // server-side task-name search (All activity) const failureSearch = ref('') // client-side search over loaded failures +// This filters HISTORY — task_run.queue — which is why it stays a written-out +// list rather than being derived from the lanes endpoint like the other queue +// lists were in milestone 422. Two reasons, and the second is the real one: +// a derived list is empty whenever that endpoint is down, and more importantly +// it would HIDE the filter for any queue that has rows but no longer has a +// lane serving it, which is exactly when someone is looking. +// +// It had drifted regardless — `maintenance_long` was missing, so activity on +// the long-maintenance lane could not be filtered for at all despite four task +// routes pointing there. Added. const queueOptions = [ { title: 'All queues', value: null }, { title: 'import', value: 'import' }, @@ -206,6 +216,7 @@ const queueOptions = [ { title: 'download', value: 'download' }, { title: 'scan', value: 'scan' }, { title: 'maintenance', value: 'maintenance' }, + { title: 'maintenance_long', value: 'maintenance_long' }, { title: 'default', value: 'default' }, ] const statusOptions = [ diff --git a/frontend/src/components/settings/SystemHealthTab.vue b/frontend/src/components/settings/SystemHealthTab.vue index c1853aa..e1b30c4 100644 --- a/frontend/src/components/settings/SystemHealthTab.vue +++ b/frontend/src/components/settings/SystemHealthTab.vue @@ -1,8 +1,22 @@ diff --git a/frontend/src/components/subscriptions/SettingsTab.vue b/frontend/src/components/subscriptions/SettingsTab.vue index 16bb78b..c94d496 100644 --- a/frontend/src/components/subscriptions/SettingsTab.vue +++ b/frontend/src/components/subscriptions/SettingsTab.vue @@ -153,6 +153,24 @@ + + + +
+ A routine check keeps looking back this far after it runs out of + new posts, so a creator who edits an older post to add a file is + still picked up. New attachments download; the post text is + re-read from the same page, costing nothing extra. 0 turns this + off. Default 30. +
+ + {{ importStore.settingsError }} @@ -209,6 +227,7 @@ const dl = reactive({ download_schedule_default_seconds: 28800, download_event_retention_days: 90, download_failure_warning_threshold: 5, + download_revisit_days: 30, extdl_mega_enabled: true, extdl_gdrive_enabled: true, extdl_mediafire_enabled: true, diff --git a/frontend/src/components/subscriptions/SourceActions.vue b/frontend/src/components/subscriptions/SourceActions.vue index e026aed..3512b70 100644 --- a/frontend/src/components/subscriptions/SourceActions.vue +++ b/frontend/src/components/subscriptions/SourceActions.vue @@ -45,8 +45,9 @@ > Recapture post text & links - Re-grab every post's body + external links and localize inline images - already on disk — without re-downloading media + Re-grab every post's body + external links, localize inline images + already on disk, and import any downloaded file that never reached the + library — without re-downloading media @@ -75,10 +76,10 @@ const running = computed(() => props.source.backfill_state === 'running') const recovering = computed(() => !!props.source.backfill_bypass_seen) const recapturing = computed(() => !!props.source.backfill_recapture) // Recover / recapture are native-ingester features (ledger-bypass re-walk and -// post-text re-grab), available to every native platform — not just Patreon. -// Mirrors backend download_backends.NATIVE_INGESTER_PLATFORMS. -const NATIVE_PLATFORMS = ['patreon', 'subscribestar'] -const isNative = computed(() => NATIVE_PLATFORMS.includes(props.source.platform)) +// post-text re-grab), available on every native platform. The backend says +// which those are (`native_ingester`); a copied list here went stale when +// Discord moved over. +const isNative = computed(() => !!props.source.native_ingester)