FabledCurator/.forgejo/workflows/ci.yml

name: CI

# CI lanes per FabledRulebook/forgejo.md "CI philosophy":
#   - lint: ruff only, no dep install — fast-fail for the common lint bounce.
#   - extension-version: guards the extension publish path (see the job).
#   - backend-lint-and-test: `pytest -m "not integration"`, no service containers.
#   - frontend-build: vitest unit + vite build.
#   - integration: pgvector + redis service containers; alembic + `pytest -m integration`.

on:
  push:
    branches: [dev, main]
  # Renovate opens PRs from `renovate/*` branches into `dev`. Those branches
  # never push to dev/main, so the push trigger above gives them NO pre-merge
  # CI — a bump could only be validated after it was already merged. This
  # pull_request trigger (base `dev` only) validates Renovate PRs before merge.
  # It deliberately does NOT fire on dev→main PRs (base `main`), which still
  # rely on the dev push run — so no duplicate runs. FC has no fork PRs
  # (single-operator Forgejo repo), so secrets-on-PR is not a concern.
  pull_request:
    branches: [dev]

jobs:
  # Fast-fail lint lane. ruff is pre-installed in the ci-python image, so
  # this runs with NO dependency install and surfaces the most common bounce
  # class (lint: I001 / UP037 / ASYNC109 / W293 …) in seconds — instead of
  # after the backend job's ~30-60s wheel install. ruff is static analysis,
  # so no DB/secret env is needed.
  lint:
    runs-on: python-ci
    container:
      image: git.fabledsword.com/bvandeusen/ci-python:3.14
    steps:
      - uses: actions/checkout@v4
      - name: Ruff lint
        # agent/ included so the GPU-agent is linted before its image is built
        # (build.yml only `docker build`s it — this is where it gets checked).
        run: ruff check backend/ tests/ alembic/ agent/
      - name: Agent syntax check
        # The agent's runtime deps (torch/transformers/ultralytics) aren't in the
        # CI image, so we can't import it — but compileall parses every module,
        # catching syntax errors before the image build.
        run: python -m compileall -q agent/fc_agent

  # Guards the extension publish path, which has no self-correcting behavior.
  #
  # build.yml's sign-extension job keys its AMO-signing cache purely on the
  # version string in extension/package.json: if an `ext-<version>` Forgejo
  # release already carries an XPI, signing is SKIPPED and that old signed XPI
  # is what build-web bakes into `:latest`. Nothing in that path inspects
  # whether extension/ actually changed — so a forgotten version bump ships a
  # stale extension on a fully green build, silently. (AMO can't help: it 409s
  # on re-signing a version, which is exactly why the cache exists.)
  #
  # This job makes that case loud, on the dev push, instead of invisible at
  # merge-to-main. It is pure git + text work — no deps, no services.
  extension-version:
    runs-on: python-ci
    container:
      image: git.fabledsword.com/bvandeusen/ci-python:3.14
    steps:
      - uses: actions/checkout@v4
        with:
          # Full history: the check diffs against the push's `before` SHA (or
          # the PR base), which a depth-1 clone wouldn't contain.
          fetch-depth: 0
      - name: Extension version guard
        env:
          BEFORE: ${{ github.event.before }}
          PR_BASE: ${{ github.event.pull_request.base.sha }}
        run: |
          set -eu
          # busybox sh on the act_runner — no bashisms (family rule).
          ver() { grep -E '"version"' "$1" | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/'; }
          PKG=$(ver extension/package.json)
          MAN=$(ver extension/manifest.json)
          test -n "$PKG" || { echo "ERROR: no version found in extension/package.json"; exit 1; }
          test -n "$MAN" || { echo "ERROR: no version found in extension/manifest.json"; exit 1; }

          # --- shadow mode (milestone #271, step 2) -------------------------
          # Informational ONLY: nothing below reads DERIVED, and this must never
          # fail the job. `web-ext sign` is one-shot per version (AMO 409s on a
          # repeat), so a wrong formula would burn a real version number that
          # can't be reclaimed. Logging it against real pushes first is the only
          # way to validate it at zero cost.
          # Placed before every early-exit path so it reports on all runs.
          #
          # The formula CHANGED on 2026-08-27 (commit count -> commit time, per
          # rule 149), so observations logged before that date describe the old
          # one and prove nothing about this. The window restarts here. Unlike
          # build.yml's copy this runs on dev too, so both channels' numbers are
          # visible — which is the pair that has to stay ordered.
          DERIVED=$(sh extension/scripts/packaging.sh version 2>&1 || echo "UNAVAILABLE")
          echo "shadow: manual=$PKG derived=$DERIVED"
          # -----------------------------------------------------------------

          # (1) Unconditional: the two version strings must agree. `web-ext sign`
          # reads manifest.json (package.json sits in --ignore-files and isn't
          # even inside the XPI), so AMO signs MAN and Firefox installs MAN.
          # build.yml keys its cache, release tag, XPI filename — and therefore
          # the version /api/extension/manifest reports to the update prompt —
          # on PKG. Divergence either hard-fails at AMO or ships a mislabelled
          # XPI whose update prompt lies about what's installed.
          if [ "$MAN" != "$PKG" ]; then
            echo "ERROR: extension version mismatch."
            echo "  extension/manifest.json = $MAN  <- what AMO signs / Firefox installs"
            echo "  extension/package.json  = $PKG  <- what CI caches, names, and reports"
            echo "Set both to the same value."
            exit 1
          fi

          # (2) If the SHIPPED extension changed, the version must have moved.
          #
          # Compare against MAIN, not against the previous push. The publish
          # decision is made at merge-to-main against whatever ext-<version>
          # already exists, so "differs from main" is the question that matters.
          # Diffing against the previous dev push instead would demand a fresh
          # bump on every iteration — push, tweak the extension again, and CI
          # would insist on a second bump that buys nothing, inflating the
          # version for no reason. On a main push there is no "main to compare
          # to" yet, so fall back to that push's own before-SHA.
          if [ "${GITHUB_REF##*/}" = "main" ]; then
            BASE="${BEFORE:-}"
          else
            BASE=$(git rev-parse --verify -q origin/main 2>/dev/null || git rev-parse --verify -q main 2>/dev/null || echo "")
            # PR base is the fallback when main isn't in the clone at all.
            [ -n "$BASE" ] || BASE="${PR_BASE:-}"
          fi
          case "$BASE" in
            ''|0000000000000000000000000000000000000000)
              echo "No usable base ref (no main in clone / first push) — skipping the bump check."
              echo "OK: extension version $PKG"
              exit 0
              ;;
          esac
          if ! git cat-file -e "$BASE^{commit}" 2>/dev/null; then
            echo "Base commit $BASE not in this clone — skipping the bump check."
            echo "OK: extension version $PKG"
            exit 0
          fi
          # The exclusion list is NOT written out here — it comes from
          # extension/scripts/packaging.sh, the one definition of what ships,
          # shared with web-ext's --ignore-files and the derived-version patch
          # count. Three hand-kept copies of that fact is how #2397 happened.
          #
          # `set -f` is required around the substitution: without it the shell
          # globs `test/**` against the working tree and silently narrows it.
          set -f
          CHANGED=$(git diff --name-only "$BASE" HEAD -- extension/ $(sh extension/scripts/packaging.sh pathspec))
          set +f
          if [ -z "$CHANGED" ]; then
            echo "No packaged extension files changed since $BASE — nothing to guard."
            echo "OK: extension version $PKG"
            exit 0
          fi
          echo "Packaged extension files changed since $BASE:"
          echo "$CHANGED" | sed 's/^/  /'
          PKG_OLD=$(git show "$BASE:extension/package.json" 2>/dev/null | grep -E '"version"' | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')
          if [ -z "$PKG_OLD" ]; then
            echo "Could not read the base version — skipping the bump check."
            echo "OK: extension version $PKG"
            exit 0
          fi
          if [ "$PKG_OLD" = "$PKG" ]; then
            echo "ERROR: packaged extension files changed but the version is still $PKG."
            echo "build.yml would find the existing ext-$PKG release, skip AMO signing,"
            echo "and bake the OLD signed XPI into :latest — a green build shipping stale code."
            echo "Bump the version in BOTH extension/package.json and extension/manifest.json."
            exit 1
          fi
          echo "OK: extension version $PKG_OLD -> $PKG"

  backend-lint-and-test:
    runs-on: python-ci
    container:
      image: git.fabledsword.com/bvandeusen/ci-python:3.14
    env:
      # DB_PASSWORD and SECRET_KEY are required by config.py at import time
      # even though unit tests don't actually touch the DB or use the secret.
      DB_PASSWORD: ci_unit_test_placeholder
      SECRET_KEY: ci_unit_test_placeholder
    steps:
      - uses: actions/checkout@v4

      # Cache step removed 2026-05-26: act_runner's cache backend has been
      # broken on this homelab runner since 2026-05-15 (first as request-
      # timeout warnings, then as hard "Cannot find module .../dist/restore/
      # index.js" failures that tank the whole job). The cache step targeted
      # ~/.cache/pip but the install below uses `uv pip install` primarily,
      # whose own cache lives at ~/.cache/uv — so the cache step's real
      # benefit was marginal even when working. Cost of removal: ~30s of
      # wheel downloads per job. Future re-enable: mount ~/.cache/uv as a
      # docker volume at the runner level (skips actions/cache entirely),
      # or fix the runner-side cache backend (clear /var/run/act/actions/*,
      # pin act_runner version, etc.).

      - name: Install Python deps
        # ruff is pre-installed in the ci-python image (see CI-Runner/CI-python/
        # Dockerfile's RUFF_VERSION). Per FabledRulebook ci-runners.md, toolchain
        # versions live on the runner image, not here.
        # uv: 5-10x faster wheel resolve than pip for cold caches.
        # Falls back to pip install on uv-missing runners (older images).
        run: |
          if command -v uv >/dev/null 2>&1; then
            uv pip install --system -r requirements.txt pytest pytest-asyncio
          else
            pip install -r requirements.txt pytest pytest-asyncio
          fi

      # Ruff moved to the dedicated fast `lint` job above (fails in seconds,
      # no dep install). This job is now unit tests only.
      - name: Pytest (unit only — integration runs in the integration job)
        run: pytest tests/ -v -m "not integration"

  frontend-build:
    runs-on: python-ci
    container:
      image: git.fabledsword.com/bvandeusen/ci-python:3.14
    defaults:
      run:
        working-directory: frontend
    steps:
      - uses: actions/checkout@v4
      # No package-lock.json is tracked yet (we don't run npm locally per
      # feedback-no-local-runs). Using `npm install` instead of `npm ci`.
      # If we want strict lockfile-based reproducibility later, commit a
      # package-lock.json and flip this back to `npm ci`.
      - run: npm install --no-audit --no-fund
      # No type-check step: the frontend is pure JS (no .ts files, no JSDoc),
      # so a type-checker has nothing to do. The vue-tsc devDep + its `check`
      # script were dropped 2026-07-11 rather than bumped to v3. If we add
      # TS/JSDoc later, re-add a tsconfig.json + vue-tsc + a type-check step.
      - run: npm run test:unit
      - run: npm run build

  # Single integration job — collapsed from a 3-way shard split on 2026-06-04.
  # The shards existed to parallelize ~8.5min of integration tests; once the
  # throwaway Postgres runs with fsync OFF (the durability step below) the whole
  # suite runs in ~45s, so the split only triplicated the ~2min fixed overhead
  # (container + `uv pip install` + `alembic upgrade head`) and burned 3 of 6
  # runner slots for no wall-clock gain. One job now: spin up once, install
  # once, migrate once, run every integration test.
  #
  # The docker-ps filter scopes to THIS job's own Postgres/Redis service
  # containers by job name. act_runner strips underscores from job names when
  # labelling containers (`int_api` matched nothing on 2026-05-25), so the name
  # stays separator-free (`integration`). The step prints `docker ps -a` first
  # so a future naming-convention shift surfaces in the log without a
  # guess-and-push cycle.
  #
  # Pre-baking requirements.txt into ci-python:3.14 is intentionally NOT done —
  # per ci-requirements.md, FC is the only Python consumer of that image and the
  # CI-Runner "add deps to image when used by >1 project" rule keeps it per-job.
  integration:
    runs-on: python-ci
    container:
      image: git.fabledsword.com/bvandeusen/ci-python:3.14
    env:
      DB_USER: fabledcurator
      DB_PASSWORD: ci_integration
      DB_PORT: "5432"
      DB_NAME: fabledcurator_test
      SECRET_KEY: ci_integration_placeholder
    services:
      postgres:
        image: pgvector/pgvector:pg16
        env:
          POSTGRES_USER: fabledcurator
          POSTGRES_PASSWORD: ci_integration
          POSTGRES_DB: fabledcurator_test
        options: >-
          --health-cmd "pg_isready -U fabledcurator"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 10
      redis:
        image: redis:7-alpine
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 10
    steps:
      - uses: actions/checkout@v4
      - name: Integration suite (resolve service IPs, migrate, test)
        run: |
          set -eux
          echo "=== container landscape (diagnostic for filter scoping) ==="
          docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
          echo "=== end landscape ==="
          PG=$(docker ps --filter "name=integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
          RD=$(docker ps --filter "name=integration" --filter "ancestor=redis:7-alpine" -q | head -n1)
          test -n "$PG" && test -n "$RD"
          PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
          RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
          test -n "$PG_IP" && test -n "$RD_IP"
          export DB_HOST="$PG_IP"
          export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
          export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
          for i in $(seq 1 60); do
            (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
            sleep 2
          done
          if command -v uv >/dev/null 2>&1; then
            uv pip install --system -r requirements.txt pytest pytest-asyncio
          else
            pip install -r requirements.txt pytest pytest-asyncio
          fi
          # Relax durability on the throwaway CI Postgres so the per-test
          # TRUNCATE's commit-fsync — the integration teardown's dominant cost
          # (~1.5-2s/test, which collapsed the suite from ~13min to ~45s) — is
          # skipped. fsync/full_page_writes are sighup GUCs and synchronous_commit
          # is user-context, so ALTER SYSTEM + pg_reload_conf() applies them with
          # NO restart. Ephemeral DB ⇒ fsync-off is safe. Non-fatal so a perms
          # surprise can't red the job; fabledcurator is the postgres image's
          # bootstrap superuser.
          python -c "import os,psycopg; c=psycopg.connect(host=os.environ['DB_HOST'],port=5432,user=os.environ['DB_USER'],password=os.environ['DB_PASSWORD'],dbname=os.environ['DB_NAME'],autocommit=True); [c.execute(q) for q in ('ALTER SYSTEM SET fsync=off','ALTER SYSTEM SET synchronous_commit=off','ALTER SYSTEM SET full_page_writes=off','SELECT pg_reload_conf()')]; c.close()" || echo 'WARN: durability GUC relax failed (continuing)'
          alembic upgrade head
          pytest tests/ -v -m integration --durations=15