diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index da95322..9a31d7c 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -92,28 +92,25 @@ jobs: - run: npm run test:unit - run: npm run build - # Integration suite split into THREE parallel shards (2026-05-25, runner - # capacity bumped 2→6). Each shard gets its own Postgres + Redis service - # set and runs alembic + a disjoint subset of integration tests. Shards - # share no DB state, so the autouse TRUNCATE fixture in tests/conftest.py - # stays single-threaded per shard but multiple shards run in parallel - # wall-clock. Approximate split — rebalance once --durations=15 output - # reveals which shard is the long pole. + # Single integration job — collapsed from a 3-way shard split on 2026-06-04. + # The shards existed to parallelize ~8.5min of integration tests; once the + # throwaway Postgres runs with fsync OFF (the durability step below) the whole + # suite runs in ~45s, so the split only triplicated the ~2min fixed overhead + # (container + `uv pip install` + `alembic upgrade head`) and burned 3 of 6 + # runner slots for no wall-clock gain. One job now: spin up once, install + # once, migrate once, run every integration test. # - # Each shard's docker-ps filter uses its own unique job name to scope - # service-container resolution. act_runner appears to strip underscores - # from job names when building container labels — `int_api` yielded - # zero matches on 2026-05-25 — so shards use no-separator names - # (`intapi`, `intimp`, `intcore`) instead. Each step prints - # `docker ps -a` first so a future naming-convention shift surfaces in - # the log without another guess-and-push cycle. + # The docker-ps filter scopes to THIS job's own Postgres/Redis service + # containers by job name. act_runner strips underscores from job names when + # labelling containers (`int_api` matched nothing on 2026-05-25), so the name + # stays separator-free (`integration`). The step prints `docker ps -a` first + # so a future naming-convention shift surfaces in the log without a + # guess-and-push cycle. # - # Pre-baking requirements.txt into ci-python:3.14 is intentionally NOT - # done — per ci-requirements.md, FC is the only Python consumer of that - # image and the CI-Runner project's "add deps to image when used by >1 - # project" rule keeps the install per-job. - - intapi: + # Pre-baking requirements.txt into ci-python:3.14 is intentionally NOT done — + # per ci-requirements.md, FC is the only Python consumer of that image and the + # CI-Runner "add deps to image when used by >1 project" rule keeps it per-job. + integration: runs-on: python-ci container: image: git.fabledsword.com/bvandeusen/ci-python:3.14 @@ -144,14 +141,14 @@ jobs: --health-retries 10 steps: - uses: actions/checkout@v4 - - name: API integration shard (resolve service IPs, migrate, test) + - name: Integration suite (resolve service IPs, migrate, test) run: | set -eux echo "=== container landscape (diagnostic for filter scoping) ===" docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}' echo "=== end landscape ===" - PG=$(docker ps --filter "name=intapi" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) - RD=$(docker ps --filter "name=intapi" --filter "ancestor=redis:7-alpine" -q | head -n1) + PG=$(docker ps --filter "name=integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) + RD=$(docker ps --filter "name=integration" --filter "ancestor=redis:7-alpine" -q | head -n1) test -n "$PG" && test -n "$RD" PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD") @@ -170,155 +167,12 @@ jobs: fi # Relax durability on the throwaway CI Postgres so the per-test # TRUNCATE's commit-fsync — the integration teardown's dominant cost - # (~1.5-2s/test per --durations, unchanged by pooling the teardown - # connection) — is skipped. fsync/full_page_writes are sighup GUCs and - # synchronous_commit is user-context, so ALTER SYSTEM + pg_reload_conf() - # applies them with NO restart. Ephemeral DB ⇒ fsync-off is safe. - # Non-fatal so a perms surprise can't red the shard; fabledcurator is - # the postgres image's bootstrap superuser. + # (~1.5-2s/test, which collapsed the suite from ~13min to ~45s) — is + # skipped. fsync/full_page_writes are sighup GUCs and synchronous_commit + # is user-context, so ALTER SYSTEM + pg_reload_conf() applies them with + # NO restart. Ephemeral DB ⇒ fsync-off is safe. Non-fatal so a perms + # surprise can't red the job; fabledcurator is the postgres image's + # bootstrap superuser. python -c "import os,psycopg; c=psycopg.connect(host=os.environ['DB_HOST'],port=5432,user=os.environ['DB_USER'],password=os.environ['DB_PASSWORD'],dbname=os.environ['DB_NAME'],autocommit=True); [c.execute(q) for q in ('ALTER SYSTEM SET fsync=off','ALTER SYSTEM SET synchronous_commit=off','ALTER SYSTEM SET full_page_writes=off','SELECT pg_reload_conf()')]; c.close()" || echo 'WARN: durability GUC relax failed (continuing)' alembic upgrade head - pytest tests/test_api_*.py -v -m integration --durations=15 - - intimp: - runs-on: python-ci - container: - image: git.fabledsword.com/bvandeusen/ci-python:3.14 - env: - DB_USER: fabledcurator - DB_PASSWORD: ci_integration - DB_PORT: "5432" - DB_NAME: fabledcurator_test - SECRET_KEY: ci_integration_placeholder - services: - postgres: - image: pgvector/pgvector:pg16 - env: - POSTGRES_USER: fabledcurator - POSTGRES_PASSWORD: ci_integration - POSTGRES_DB: fabledcurator_test - options: >- - --health-cmd "pg_isready -U fabledcurator" - --health-interval 10s - --health-timeout 5s - --health-retries 10 - redis: - image: redis:7-alpine - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 10 - steps: - - uses: actions/checkout@v4 - - name: Importer integration shard (resolve service IPs, migrate, test) - run: | - set -eux - echo "=== container landscape (diagnostic for filter scoping) ===" - docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}' - echo "=== end landscape ===" - PG=$(docker ps --filter "name=intimp" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) - RD=$(docker ps --filter "name=intimp" --filter "ancestor=redis:7-alpine" -q | head -n1) - test -n "$PG" && test -n "$RD" - PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") - RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD") - test -n "$PG_IP" && test -n "$RD_IP" - export DB_HOST="$PG_IP" - export CELERY_BROKER_URL="redis://$RD_IP:6379/0" - export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0" - for i in $(seq 1 60); do - (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break - sleep 2 - done - if command -v uv >/dev/null 2>&1; then - uv pip install --system -r requirements.txt pytest pytest-asyncio - else - pip install -r requirements.txt pytest pytest-asyncio - fi - # Relax durability on the throwaway CI Postgres so the per-test - # TRUNCATE's commit-fsync — the integration teardown's dominant cost - # (~1.5-2s/test per --durations, unchanged by pooling the teardown - # connection) — is skipped. fsync/full_page_writes are sighup GUCs and - # synchronous_commit is user-context, so ALTER SYSTEM + pg_reload_conf() - # applies them with NO restart. Ephemeral DB ⇒ fsync-off is safe. - # Non-fatal so a perms surprise can't red the shard; fabledcurator is - # the postgres image's bootstrap superuser. - python -c "import os,psycopg; c=psycopg.connect(host=os.environ['DB_HOST'],port=5432,user=os.environ['DB_USER'],password=os.environ['DB_PASSWORD'],dbname=os.environ['DB_NAME'],autocommit=True); [c.execute(q) for q in ('ALTER SYSTEM SET fsync=off','ALTER SYSTEM SET synchronous_commit=off','ALTER SYSTEM SET full_page_writes=off','SELECT pg_reload_conf()')]; c.close()" || echo 'WARN: durability GUC relax failed (continuing)' - alembic upgrade head - pytest tests/test_importer*.py tests/test_import_*.py tests/test_migration_*.py tests/test_phash_*.py tests/test_sidecar_*.py tests/test_scan_*.py tests/test_archive_extractor.py tests/test_backfill_phash.py -v -m integration --durations=15 - - intcore: - runs-on: python-ci - container: - image: git.fabledsword.com/bvandeusen/ci-python:3.14 - env: - DB_USER: fabledcurator - DB_PASSWORD: ci_integration - DB_PORT: "5432" - DB_NAME: fabledcurator_test - SECRET_KEY: ci_integration_placeholder - services: - postgres: - image: pgvector/pgvector:pg16 - env: - POSTGRES_USER: fabledcurator - POSTGRES_PASSWORD: ci_integration - POSTGRES_DB: fabledcurator_test - options: >- - --health-cmd "pg_isready -U fabledcurator" - --health-interval 10s - --health-timeout 5s - --health-retries 10 - redis: - image: redis:7-alpine - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 10 - steps: - - uses: actions/checkout@v4 - - name: Core integration shard (everything not api / importer / migration / phash / sidecar / scan / archive / backfill) - run: | - set -eux - echo "=== container landscape (diagnostic for filter scoping) ===" - docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}' - echo "=== end landscape ===" - PG=$(docker ps --filter "name=intcore" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) - RD=$(docker ps --filter "name=intcore" --filter "ancestor=redis:7-alpine" -q | head -n1) - test -n "$PG" && test -n "$RD" - PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") - RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD") - test -n "$PG_IP" && test -n "$RD_IP" - export DB_HOST="$PG_IP" - export CELERY_BROKER_URL="redis://$RD_IP:6379/0" - export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0" - for i in $(seq 1 60); do - (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break - sleep 2 - done - if command -v uv >/dev/null 2>&1; then - uv pip install --system -r requirements.txt pytest pytest-asyncio - else - pip install -r requirements.txt pytest pytest-asyncio - fi - # Relax durability on the throwaway CI Postgres so the per-test - # TRUNCATE's commit-fsync — the integration teardown's dominant cost - # (~1.5-2s/test per --durations, unchanged by pooling the teardown - # connection) — is skipped. fsync/full_page_writes are sighup GUCs and - # synchronous_commit is user-context, so ALTER SYSTEM + pg_reload_conf() - # applies them with NO restart. Ephemeral DB ⇒ fsync-off is safe. - # Non-fatal so a perms surprise can't red the shard; fabledcurator is - # the postgres image's bootstrap superuser. - python -c "import os,psycopg; c=psycopg.connect(host=os.environ['DB_HOST'],port=5432,user=os.environ['DB_USER'],password=os.environ['DB_PASSWORD'],dbname=os.environ['DB_NAME'],autocommit=True); [c.execute(q) for q in ('ALTER SYSTEM SET fsync=off','ALTER SYSTEM SET synchronous_commit=off','ALTER SYSTEM SET full_page_writes=off','SELECT pg_reload_conf()')]; c.close()" || echo 'WARN: durability GUC relax failed (continuing)' - alembic upgrade head - pytest tests/ -v -m integration --durations=15 \ - --ignore-glob='tests/test_api_*.py' \ - --ignore-glob='tests/test_importer*.py' \ - --ignore-glob='tests/test_import_*.py' \ - --ignore-glob='tests/test_migration_*.py' \ - --ignore-glob='tests/test_phash_*.py' \ - --ignore-glob='tests/test_sidecar_*.py' \ - --ignore-glob='tests/test_scan_*.py' \ - --ignore-glob='tests/test_archive_extractor.py' \ - --ignore-glob='tests/test_backfill_phash.py' + pytest tests/ -v -m integration --durations=15