287 lines
11 KiB
YAML
287 lines
11 KiB
YAML
name: CI
|
|
|
|
# CI lanes per FabledRulebook/forgejo.md "CI philosophy":
|
|
# - backend-lint-and-test: ruff + `pytest -m "not integration"`, no service containers.
|
|
# - frontend-build: vitest unit + vite build.
|
|
# - integration: pgvector + redis service containers; alembic + `pytest -m integration`.
|
|
|
|
on:
|
|
push:
|
|
branches: [dev, main]
|
|
pull_request:
|
|
branches: [main]
|
|
|
|
jobs:
|
|
backend-lint-and-test:
|
|
runs-on: python-ci
|
|
container:
|
|
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
|
env:
|
|
# DB_PASSWORD and SECRET_KEY are required by config.py at import time
|
|
# even though unit tests don't actually touch the DB or use the secret.
|
|
DB_PASSWORD: ci_unit_test_placeholder
|
|
SECRET_KEY: ci_unit_test_placeholder
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Cache pip wheels
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: ~/.cache/pip
|
|
key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
|
|
restore-keys: |
|
|
pip-${{ runner.os }}-py314-
|
|
|
|
- name: Install Python deps
|
|
# ruff is pre-installed in the ci-python image (see CI-Runner/CI-python/
|
|
# Dockerfile's RUFF_VERSION). Per FabledRulebook ci-runners.md, toolchain
|
|
# versions live on the runner image, not here.
|
|
# uv: 5-10x faster wheel resolve than pip for cold caches.
|
|
# Falls back to pip install on uv-missing runners (older images).
|
|
run: |
|
|
if command -v uv >/dev/null 2>&1; then
|
|
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
|
else
|
|
pip install -r requirements.txt pytest pytest-asyncio
|
|
fi
|
|
|
|
- name: Ruff lint
|
|
run: ruff check backend/ tests/ alembic/
|
|
|
|
- name: Pytest (unit only — integration runs in the integration job)
|
|
run: pytest tests/ -v -m "not integration"
|
|
|
|
frontend-build:
|
|
runs-on: python-ci
|
|
container:
|
|
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
|
defaults:
|
|
run:
|
|
working-directory: frontend
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
# No package-lock.json is tracked yet (we don't run npm locally per
|
|
# feedback-no-local-runs). Using `npm install` instead of `npm ci`.
|
|
# If we want strict lockfile-based reproducibility later, commit a
|
|
# package-lock.json and flip this back to `npm ci`.
|
|
- run: npm install --no-audit --no-fund
|
|
# `npm run check` (vue-tsc --noEmit) skipped: the frontend is pure JS
|
|
# with no .ts files and no JSDoc annotations, so vue-tsc has nothing
|
|
# to type-check. Re-enable once we add a tsconfig.json and either
|
|
# convert to TS or add JSDoc.
|
|
- run: npm run test:unit
|
|
- run: npm run build
|
|
|
|
# Integration suite split into THREE parallel shards (2026-05-25, runner
|
|
# capacity bumped 2→6). Each shard gets its own Postgres + Redis service
|
|
# set and runs alembic + a disjoint subset of integration tests. Shards
|
|
# share no DB state, so the autouse TRUNCATE fixture in tests/conftest.py
|
|
# stays single-threaded per shard but multiple shards run in parallel
|
|
# wall-clock. Approximate split — rebalance once --durations=15 output
|
|
# reveals which shard is the long pole.
|
|
#
|
|
# Each shard's docker-ps filter uses its own unique job name to scope
|
|
# service-container resolution (act_runner names containers
|
|
# `...JOB-<job-name>-...`); shard names must stay unique across the
|
|
# workflow for this to work.
|
|
#
|
|
# Pre-baking requirements.txt into ci-python:3.14 is intentionally NOT
|
|
# done — per ci-requirements.md, FC is the only Python consumer of that
|
|
# image and the CI-Runner project's "add deps to image when used by >1
|
|
# project" rule keeps the install per-job.
|
|
|
|
int_api:
|
|
runs-on: python-ci
|
|
container:
|
|
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
|
env:
|
|
DB_USER: fabledcurator
|
|
DB_PASSWORD: ci_integration
|
|
DB_PORT: "5432"
|
|
DB_NAME: fabledcurator_test
|
|
SECRET_KEY: ci_integration_placeholder
|
|
services:
|
|
postgres:
|
|
image: pgvector/pgvector:pg16
|
|
env:
|
|
POSTGRES_USER: fabledcurator
|
|
POSTGRES_PASSWORD: ci_integration
|
|
POSTGRES_DB: fabledcurator_test
|
|
options: >-
|
|
--health-cmd "pg_isready -U fabledcurator"
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 10
|
|
redis:
|
|
image: redis:7-alpine
|
|
options: >-
|
|
--health-cmd "redis-cli ping"
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 10
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Cache pip wheels
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: ~/.cache/pip
|
|
key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
|
|
restore-keys: |
|
|
pip-${{ runner.os }}-py314-
|
|
- name: API integration shard (resolve service IPs, migrate, test)
|
|
run: |
|
|
set -eux
|
|
PG=$(docker ps --filter "name=JOB-int_api" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
|
RD=$(docker ps --filter "name=JOB-int_api" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
|
test -n "$PG" && test -n "$RD"
|
|
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
|
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
|
test -n "$PG_IP" && test -n "$RD_IP"
|
|
export DB_HOST="$PG_IP"
|
|
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
|
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
|
for i in $(seq 1 60); do
|
|
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
|
sleep 2
|
|
done
|
|
if command -v uv >/dev/null 2>&1; then
|
|
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
|
else
|
|
pip install -r requirements.txt pytest pytest-asyncio
|
|
fi
|
|
alembic upgrade head
|
|
pytest tests/test_api_*.py -v -m integration --durations=15
|
|
|
|
int_imp:
|
|
runs-on: python-ci
|
|
container:
|
|
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
|
env:
|
|
DB_USER: fabledcurator
|
|
DB_PASSWORD: ci_integration
|
|
DB_PORT: "5432"
|
|
DB_NAME: fabledcurator_test
|
|
SECRET_KEY: ci_integration_placeholder
|
|
services:
|
|
postgres:
|
|
image: pgvector/pgvector:pg16
|
|
env:
|
|
POSTGRES_USER: fabledcurator
|
|
POSTGRES_PASSWORD: ci_integration
|
|
POSTGRES_DB: fabledcurator_test
|
|
options: >-
|
|
--health-cmd "pg_isready -U fabledcurator"
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 10
|
|
redis:
|
|
image: redis:7-alpine
|
|
options: >-
|
|
--health-cmd "redis-cli ping"
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 10
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Cache pip wheels
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: ~/.cache/pip
|
|
key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
|
|
restore-keys: |
|
|
pip-${{ runner.os }}-py314-
|
|
- name: Importer integration shard (resolve service IPs, migrate, test)
|
|
run: |
|
|
set -eux
|
|
PG=$(docker ps --filter "name=JOB-int_imp" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
|
RD=$(docker ps --filter "name=JOB-int_imp" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
|
test -n "$PG" && test -n "$RD"
|
|
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
|
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
|
test -n "$PG_IP" && test -n "$RD_IP"
|
|
export DB_HOST="$PG_IP"
|
|
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
|
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
|
for i in $(seq 1 60); do
|
|
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
|
sleep 2
|
|
done
|
|
if command -v uv >/dev/null 2>&1; then
|
|
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
|
else
|
|
pip install -r requirements.txt pytest pytest-asyncio
|
|
fi
|
|
alembic upgrade head
|
|
pytest tests/test_importer*.py tests/test_import_*.py tests/test_migration_*.py tests/test_phash_*.py tests/test_sidecar_*.py tests/test_scan_*.py tests/test_archive_extractor.py tests/test_backfill_phash.py -v -m integration --durations=15
|
|
|
|
int_core:
|
|
runs-on: python-ci
|
|
container:
|
|
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
|
env:
|
|
DB_USER: fabledcurator
|
|
DB_PASSWORD: ci_integration
|
|
DB_PORT: "5432"
|
|
DB_NAME: fabledcurator_test
|
|
SECRET_KEY: ci_integration_placeholder
|
|
services:
|
|
postgres:
|
|
image: pgvector/pgvector:pg16
|
|
env:
|
|
POSTGRES_USER: fabledcurator
|
|
POSTGRES_PASSWORD: ci_integration
|
|
POSTGRES_DB: fabledcurator_test
|
|
options: >-
|
|
--health-cmd "pg_isready -U fabledcurator"
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 10
|
|
redis:
|
|
image: redis:7-alpine
|
|
options: >-
|
|
--health-cmd "redis-cli ping"
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 10
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Cache pip wheels
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: ~/.cache/pip
|
|
key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
|
|
restore-keys: |
|
|
pip-${{ runner.os }}-py314-
|
|
- name: Core integration shard (everything not api / importer / migration / phash / sidecar / scan / archive / backfill)
|
|
run: |
|
|
set -eux
|
|
PG=$(docker ps --filter "name=JOB-int_core" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
|
RD=$(docker ps --filter "name=JOB-int_core" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
|
test -n "$PG" && test -n "$RD"
|
|
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
|
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
|
test -n "$PG_IP" && test -n "$RD_IP"
|
|
export DB_HOST="$PG_IP"
|
|
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
|
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
|
for i in $(seq 1 60); do
|
|
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
|
sleep 2
|
|
done
|
|
if command -v uv >/dev/null 2>&1; then
|
|
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
|
else
|
|
pip install -r requirements.txt pytest pytest-asyncio
|
|
fi
|
|
alembic upgrade head
|
|
pytest tests/ -v -m integration --durations=15 \
|
|
--ignore-glob='tests/test_api_*.py' \
|
|
--ignore-glob='tests/test_importer*.py' \
|
|
--ignore-glob='tests/test_import_*.py' \
|
|
--ignore-glob='tests/test_migration_*.py' \
|
|
--ignore-glob='tests/test_phash_*.py' \
|
|
--ignore-glob='tests/test_sidecar_*.py' \
|
|
--ignore-glob='tests/test_scan_*.py' \
|
|
--ignore-glob='tests/test_archive_extractor.py' \
|
|
--ignore-glob='tests/test_backfill_phash.py'
|