perf(ci): shard integration suite into 3 parallel jobs (int_api, int_imp, int_core) — newly feasible after act_runner capacity 2→6 — Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
+156
-22
@@ -72,17 +72,25 @@ jobs:
|
||||
- run: npm run test:unit
|
||||
- run: npm run build
|
||||
|
||||
integration:
|
||||
# This act_runner (swarm-runner v0.6.1) puts service containers on the
|
||||
# default bridge with NO service-name DNS, and publishing fixed host
|
||||
# ports collides with the operator's running docker-compose dev stack on
|
||||
# the same shared daemon. Workaround: publish NO host ports, and reach
|
||||
# each service by its bridge IP — discovered at runtime via the mounted
|
||||
# docker socket (the ci-python image ships /usr/bin/docker). Default-bridge
|
||||
# containers can talk by IP (only embedded DNS is missing), so IP
|
||||
# addressing is reliable here. Everything runs in ONE step so resolved
|
||||
# values don't depend on cross-step env passing. Pattern documented in
|
||||
# FabledRulebook/forgejo.md "CI philosophy".
|
||||
# Integration suite split into THREE parallel shards (2026-05-25, runner
|
||||
# capacity bumped 2→6). Each shard gets its own Postgres + Redis service
|
||||
# set and runs alembic + a disjoint subset of integration tests. Shards
|
||||
# share no DB state, so the autouse TRUNCATE fixture in tests/conftest.py
|
||||
# stays single-threaded per shard but multiple shards run in parallel
|
||||
# wall-clock. Approximate split — rebalance once --durations=15 output
|
||||
# reveals which shard is the long pole.
|
||||
#
|
||||
# Each shard's docker-ps filter uses its own unique job name to scope
|
||||
# service-container resolution (act_runner names containers
|
||||
# `...JOB-<job-name>-...`); shard names must stay unique across the
|
||||
# workflow for this to work.
|
||||
#
|
||||
# Pre-baking requirements.txt into ci-python:3.14 is intentionally NOT
|
||||
# done — per ci-requirements.md, FC is the only Python consumer of that
|
||||
# image and the CI-Runner project's "add deps to image when used by >1
|
||||
# project" rule keeps the install per-job.
|
||||
|
||||
int_api:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
@@ -113,7 +121,6 @@ jobs:
|
||||
--health-retries 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Cache pip wheels
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
@@ -121,15 +128,11 @@ jobs:
|
||||
key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
|
||||
restore-keys: |
|
||||
pip-${{ runner.os }}-py314-
|
||||
|
||||
- name: Integration suite (resolve service IPs, migrate, test)
|
||||
- name: API integration shard (resolve service IPs, migrate, test)
|
||||
run: |
|
||||
set -eux
|
||||
# Scope to THIS job's service containers (act_runner names them
|
||||
# ...JOB-integration...); the operator's compose stack uses the
|
||||
# same images but different names, so it won't match.
|
||||
PG=$(docker ps --filter "name=JOB-integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=JOB-integration" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||
PG=$(docker ps --filter "name=JOB-int_api" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=JOB-int_api" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||
test -n "$PG" && test -n "$RD"
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||
@@ -137,16 +140,147 @@ jobs:
|
||||
export DB_HOST="$PG_IP"
|
||||
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
||||
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
||||
# Wait for Postgres to accept TCP (bash /dev/tcp; no extra tools).
|
||||
for i in $(seq 1 60); do
|
||||
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
||||
sleep 2
|
||||
done
|
||||
# uv when available (5-10x faster wheel resolve); fall back to pip.
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
||||
else
|
||||
pip install -r requirements.txt pytest pytest-asyncio
|
||||
fi
|
||||
alembic upgrade head
|
||||
pytest tests/ -v -m integration --durations=25
|
||||
pytest tests/test_api_*.py -v -m integration --durations=15
|
||||
|
||||
int_imp:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
env:
|
||||
DB_USER: fabledcurator
|
||||
DB_PASSWORD: ci_integration
|
||||
DB_PORT: "5432"
|
||||
DB_NAME: fabledcurator_test
|
||||
SECRET_KEY: ci_integration_placeholder
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
env:
|
||||
POSTGRES_USER: fabledcurator
|
||||
POSTGRES_PASSWORD: ci_integration
|
||||
POSTGRES_DB: fabledcurator_test
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U fabledcurator"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Cache pip wheels
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
|
||||
restore-keys: |
|
||||
pip-${{ runner.os }}-py314-
|
||||
- name: Importer integration shard (resolve service IPs, migrate, test)
|
||||
run: |
|
||||
set -eux
|
||||
PG=$(docker ps --filter "name=JOB-int_imp" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=JOB-int_imp" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||
test -n "$PG" && test -n "$RD"
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||
test -n "$PG_IP" && test -n "$RD_IP"
|
||||
export DB_HOST="$PG_IP"
|
||||
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
||||
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
||||
for i in $(seq 1 60); do
|
||||
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
||||
sleep 2
|
||||
done
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
||||
else
|
||||
pip install -r requirements.txt pytest pytest-asyncio
|
||||
fi
|
||||
alembic upgrade head
|
||||
pytest tests/test_importer*.py tests/test_import_*.py tests/test_migration_*.py tests/test_phash_*.py tests/test_sidecar_*.py tests/test_scan_*.py tests/test_archive_extractor.py tests/test_backfill_phash.py -v -m integration --durations=15
|
||||
|
||||
int_core:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
env:
|
||||
DB_USER: fabledcurator
|
||||
DB_PASSWORD: ci_integration
|
||||
DB_PORT: "5432"
|
||||
DB_NAME: fabledcurator_test
|
||||
SECRET_KEY: ci_integration_placeholder
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
env:
|
||||
POSTGRES_USER: fabledcurator
|
||||
POSTGRES_PASSWORD: ci_integration
|
||||
POSTGRES_DB: fabledcurator_test
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U fabledcurator"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Cache pip wheels
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
|
||||
restore-keys: |
|
||||
pip-${{ runner.os }}-py314-
|
||||
- name: Core integration shard (everything not api / importer / migration / phash / sidecar / scan / archive / backfill)
|
||||
run: |
|
||||
set -eux
|
||||
PG=$(docker ps --filter "name=JOB-int_core" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=JOB-int_core" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||
test -n "$PG" && test -n "$RD"
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||
test -n "$PG_IP" && test -n "$RD_IP"
|
||||
export DB_HOST="$PG_IP"
|
||||
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
||||
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
||||
for i in $(seq 1 60); do
|
||||
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
||||
sleep 2
|
||||
done
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
||||
else
|
||||
pip install -r requirements.txt pytest pytest-asyncio
|
||||
fi
|
||||
alembic upgrade head
|
||||
pytest tests/ -v -m integration --durations=15 \
|
||||
--ignore-glob='tests/test_api_*.py' \
|
||||
--ignore-glob='tests/test_importer*.py' \
|
||||
--ignore-glob='tests/test_import_*.py' \
|
||||
--ignore-glob='tests/test_migration_*.py' \
|
||||
--ignore-glob='tests/test_phash_*.py' \
|
||||
--ignore-glob='tests/test_sidecar_*.py' \
|
||||
--ignore-glob='tests/test_scan_*.py' \
|
||||
--ignore-glob='tests/test_archive_extractor.py' \
|
||||
--ignore-glob='tests/test_backfill_phash.py'
|
||||
|
||||
Reference in New Issue
Block a user