From 6a25db4b8bcce6aa8ce1e69d2c7a76bba29a62f9 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 4 Jun 2026 07:45:21 -0400 Subject: [PATCH] perf(ci): relax Postgres durability in integration shards (fsync off) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Option 1 (pooling the teardown connection) left teardowns at ~1.5-2s/test, so the cost is the per-test TRUNCATE's commit forcing an fsync, not the connect handshake. Each shard now ALTER SYSTEM SETs fsync/synchronous_commit/ full_page_writes off + pg_reload_conf() right after deps install, before alembic — sighup/user-context GUCs apply with no restart. The DB is ephemeral (rebuilt per run) so fsync-off is safe; the step is non-fatal so a perms surprise can't red a shard. Speeds up every test's commit (setup inserts + the teardown TRUNCATE), stacking on the pooled engine from the prior commit. Co-Authored-By: Claude Opus 4.8 (1M context) --- .forgejo/workflows/ci.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index cb247cf..da95322 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -168,6 +168,15 @@ jobs: else pip install -r requirements.txt pytest pytest-asyncio fi + # Relax durability on the throwaway CI Postgres so the per-test + # TRUNCATE's commit-fsync — the integration teardown's dominant cost + # (~1.5-2s/test per --durations, unchanged by pooling the teardown + # connection) — is skipped. fsync/full_page_writes are sighup GUCs and + # synchronous_commit is user-context, so ALTER SYSTEM + pg_reload_conf() + # applies them with NO restart. Ephemeral DB ⇒ fsync-off is safe. + # Non-fatal so a perms surprise can't red the shard; fabledcurator is + # the postgres image's bootstrap superuser. + python -c "import os,psycopg; c=psycopg.connect(host=os.environ['DB_HOST'],port=5432,user=os.environ['DB_USER'],password=os.environ['DB_PASSWORD'],dbname=os.environ['DB_NAME'],autocommit=True); [c.execute(q) for q in ('ALTER SYSTEM SET fsync=off','ALTER SYSTEM SET synchronous_commit=off','ALTER SYSTEM SET full_page_writes=off','SELECT pg_reload_conf()')]; c.close()" || echo 'WARN: durability GUC relax failed (continuing)' alembic upgrade head pytest tests/test_api_*.py -v -m integration --durations=15 @@ -226,6 +235,15 @@ jobs: else pip install -r requirements.txt pytest pytest-asyncio fi + # Relax durability on the throwaway CI Postgres so the per-test + # TRUNCATE's commit-fsync — the integration teardown's dominant cost + # (~1.5-2s/test per --durations, unchanged by pooling the teardown + # connection) — is skipped. fsync/full_page_writes are sighup GUCs and + # synchronous_commit is user-context, so ALTER SYSTEM + pg_reload_conf() + # applies them with NO restart. Ephemeral DB ⇒ fsync-off is safe. + # Non-fatal so a perms surprise can't red the shard; fabledcurator is + # the postgres image's bootstrap superuser. + python -c "import os,psycopg; c=psycopg.connect(host=os.environ['DB_HOST'],port=5432,user=os.environ['DB_USER'],password=os.environ['DB_PASSWORD'],dbname=os.environ['DB_NAME'],autocommit=True); [c.execute(q) for q in ('ALTER SYSTEM SET fsync=off','ALTER SYSTEM SET synchronous_commit=off','ALTER SYSTEM SET full_page_writes=off','SELECT pg_reload_conf()')]; c.close()" || echo 'WARN: durability GUC relax failed (continuing)' alembic upgrade head pytest tests/test_importer*.py tests/test_import_*.py tests/test_migration_*.py tests/test_phash_*.py tests/test_sidecar_*.py tests/test_scan_*.py tests/test_archive_extractor.py tests/test_backfill_phash.py -v -m integration --durations=15 @@ -284,6 +302,15 @@ jobs: else pip install -r requirements.txt pytest pytest-asyncio fi + # Relax durability on the throwaway CI Postgres so the per-test + # TRUNCATE's commit-fsync — the integration teardown's dominant cost + # (~1.5-2s/test per --durations, unchanged by pooling the teardown + # connection) — is skipped. fsync/full_page_writes are sighup GUCs and + # synchronous_commit is user-context, so ALTER SYSTEM + pg_reload_conf() + # applies them with NO restart. Ephemeral DB ⇒ fsync-off is safe. + # Non-fatal so a perms surprise can't red the shard; fabledcurator is + # the postgres image's bootstrap superuser. + python -c "import os,psycopg; c=psycopg.connect(host=os.environ['DB_HOST'],port=5432,user=os.environ['DB_USER'],password=os.environ['DB_PASSWORD'],dbname=os.environ['DB_NAME'],autocommit=True); [c.execute(q) for q in ('ALTER SYSTEM SET fsync=off','ALTER SYSTEM SET synchronous_commit=off','ALTER SYSTEM SET full_page_writes=off','SELECT pg_reload_conf()')]; c.close()" || echo 'WARN: durability GUC relax failed (continuing)' alembic upgrade head pytest tests/ -v -m integration --durations=15 \ --ignore-glob='tests/test_api_*.py' \