diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 6402f01..daf65c2 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -72,17 +72,25 @@ jobs: - run: npm run test:unit - run: npm run build - integration: - # This act_runner (swarm-runner v0.6.1) puts service containers on the - # default bridge with NO service-name DNS, and publishing fixed host - # ports collides with the operator's running docker-compose dev stack on - # the same shared daemon. Workaround: publish NO host ports, and reach - # each service by its bridge IP — discovered at runtime via the mounted - # docker socket (the ci-python image ships /usr/bin/docker). Default-bridge - # containers can talk by IP (only embedded DNS is missing), so IP - # addressing is reliable here. Everything runs in ONE step so resolved - # values don't depend on cross-step env passing. Pattern documented in - # FabledRulebook/forgejo.md "CI philosophy". + # Integration suite split into THREE parallel shards (2026-05-25, runner + # capacity bumped 2→6). Each shard gets its own Postgres + Redis service + # set and runs alembic + a disjoint subset of integration tests. Shards + # share no DB state, so the autouse TRUNCATE fixture in tests/conftest.py + # stays single-threaded per shard but multiple shards run in parallel + # wall-clock. Approximate split — rebalance once --durations=15 output + # reveals which shard is the long pole. + # + # Each shard's docker-ps filter uses its own unique job name to scope + # service-container resolution (act_runner names containers + # `...JOB--...`); shard names must stay unique across the + # workflow for this to work. + # + # Pre-baking requirements.txt into ci-python:3.14 is intentionally NOT + # done — per ci-requirements.md, FC is the only Python consumer of that + # image and the CI-Runner project's "add deps to image when used by >1 + # project" rule keeps the install per-job. + + int_api: runs-on: python-ci container: image: git.fabledsword.com/bvandeusen/ci-python:3.14 @@ -113,7 +121,6 @@ jobs: --health-retries 10 steps: - uses: actions/checkout@v4 - - name: Cache pip wheels uses: actions/cache@v4 with: @@ -121,15 +128,11 @@ jobs: key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }} restore-keys: | pip-${{ runner.os }}-py314- - - - name: Integration suite (resolve service IPs, migrate, test) + - name: API integration shard (resolve service IPs, migrate, test) run: | set -eux - # Scope to THIS job's service containers (act_runner names them - # ...JOB-integration...); the operator's compose stack uses the - # same images but different names, so it won't match. - PG=$(docker ps --filter "name=JOB-integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) - RD=$(docker ps --filter "name=JOB-integration" --filter "ancestor=redis:7-alpine" -q | head -n1) + PG=$(docker ps --filter "name=JOB-int_api" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) + RD=$(docker ps --filter "name=JOB-int_api" --filter "ancestor=redis:7-alpine" -q | head -n1) test -n "$PG" && test -n "$RD" PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD") @@ -137,16 +140,147 @@ jobs: export DB_HOST="$PG_IP" export CELERY_BROKER_URL="redis://$RD_IP:6379/0" export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0" - # Wait for Postgres to accept TCP (bash /dev/tcp; no extra tools). for i in $(seq 1 60); do (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break sleep 2 done - # uv when available (5-10x faster wheel resolve); fall back to pip. if command -v uv >/dev/null 2>&1; then uv pip install --system -r requirements.txt pytest pytest-asyncio else pip install -r requirements.txt pytest pytest-asyncio fi alembic upgrade head - pytest tests/ -v -m integration --durations=25 + pytest tests/test_api_*.py -v -m integration --durations=15 + + int_imp: + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + env: + DB_USER: fabledcurator + DB_PASSWORD: ci_integration + DB_PORT: "5432" + DB_NAME: fabledcurator_test + SECRET_KEY: ci_integration_placeholder + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_USER: fabledcurator + POSTGRES_PASSWORD: ci_integration + POSTGRES_DB: fabledcurator_test + options: >- + --health-cmd "pg_isready -U fabledcurator" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + redis: + image: redis:7-alpine + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v4 + - name: Cache pip wheels + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }} + restore-keys: | + pip-${{ runner.os }}-py314- + - name: Importer integration shard (resolve service IPs, migrate, test) + run: | + set -eux + PG=$(docker ps --filter "name=JOB-int_imp" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) + RD=$(docker ps --filter "name=JOB-int_imp" --filter "ancestor=redis:7-alpine" -q | head -n1) + test -n "$PG" && test -n "$RD" + PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") + RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD") + test -n "$PG_IP" && test -n "$RD_IP" + export DB_HOST="$PG_IP" + export CELERY_BROKER_URL="redis://$RD_IP:6379/0" + export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0" + for i in $(seq 1 60); do + (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break + sleep 2 + done + if command -v uv >/dev/null 2>&1; then + uv pip install --system -r requirements.txt pytest pytest-asyncio + else + pip install -r requirements.txt pytest pytest-asyncio + fi + alembic upgrade head + pytest tests/test_importer*.py tests/test_import_*.py tests/test_migration_*.py tests/test_phash_*.py tests/test_sidecar_*.py tests/test_scan_*.py tests/test_archive_extractor.py tests/test_backfill_phash.py -v -m integration --durations=15 + + int_core: + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + env: + DB_USER: fabledcurator + DB_PASSWORD: ci_integration + DB_PORT: "5432" + DB_NAME: fabledcurator_test + SECRET_KEY: ci_integration_placeholder + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_USER: fabledcurator + POSTGRES_PASSWORD: ci_integration + POSTGRES_DB: fabledcurator_test + options: >- + --health-cmd "pg_isready -U fabledcurator" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + redis: + image: redis:7-alpine + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v4 + - name: Cache pip wheels + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }} + restore-keys: | + pip-${{ runner.os }}-py314- + - name: Core integration shard (everything not api / importer / migration / phash / sidecar / scan / archive / backfill) + run: | + set -eux + PG=$(docker ps --filter "name=JOB-int_core" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) + RD=$(docker ps --filter "name=JOB-int_core" --filter "ancestor=redis:7-alpine" -q | head -n1) + test -n "$PG" && test -n "$RD" + PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") + RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD") + test -n "$PG_IP" && test -n "$RD_IP" + export DB_HOST="$PG_IP" + export CELERY_BROKER_URL="redis://$RD_IP:6379/0" + export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0" + for i in $(seq 1 60); do + (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break + sleep 2 + done + if command -v uv >/dev/null 2>&1; then + uv pip install --system -r requirements.txt pytest pytest-asyncio + else + pip install -r requirements.txt pytest pytest-asyncio + fi + alembic upgrade head + pytest tests/ -v -m integration --durations=15 \ + --ignore-glob='tests/test_api_*.py' \ + --ignore-glob='tests/test_importer*.py' \ + --ignore-glob='tests/test_import_*.py' \ + --ignore-glob='tests/test_migration_*.py' \ + --ignore-glob='tests/test_phash_*.py' \ + --ignore-glob='tests/test_sidecar_*.py' \ + --ignore-glob='tests/test_scan_*.py' \ + --ignore-glob='tests/test_archive_extractor.py' \ + --ignore-glob='tests/test_backfill_phash.py'