From 1803a09306d22fb089517bde3377ad7bcba6500a Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 26 May 2026 16:54:05 -0400 Subject: [PATCH 1/2] =?UTF-8?q?ci(workflow):=20remove=20the=204=20Cache=20?= =?UTF-8?q?pip=20wheels=20steps=20entirely=20=E2=80=94=20act=5Frunner's=20?= =?UTF-8?q?cache=20backend=20has=20been=20broken=20for=2011+=20days=20and?= =?UTF-8?q?=20the=20cached=20path=20(~/.cache/pip)=20wasn't=20even=20the?= =?UTF-8?q?=20primary=20install=20tool's=20cache=20anyway=20(uv=20uses=20~?= =?UTF-8?q?/.cache/uv).=20Net=20cost=20~30s/job=20of=20wheel=20downloads.?= =?UTF-8?q?=20Long-term:=20mount=20~/.cache/uv=20as=20a=20docker=20volume?= =?UTF-8?q?=20at=20the=20runner=20level=20(skips=20actions/cache=20entirel?= =?UTF-8?q?y)=20or=20fix=20the=20runner-side=20cache=20backend.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- .forgejo/workflows/ci.yml | 75 ++++++--------------------------------- 1 file changed, 11 insertions(+), 64 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 904662e..46db216 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -24,22 +24,17 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Cache pip wheels - # continue-on-error: act_runner's cache backend has been broken on - # this homelab setup since 2026-05-15 — bolt.db + cache/ dir exist - # but the action's JS bundle now fails to load at all - # ("Cannot find module .../dist/restore/index.js"), hard-failing - # the whole job. The install step that follows handles cold caches - # natively (uv pip / pip install both work without it), so net - # cost of disabling the cache here is ~30s of wheel downloads per - # job. Re-enable once the runner-side cache backend is fixed. - continue-on-error: true - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }} - restore-keys: | - pip-${{ runner.os }}-py314- + # Cache step removed 2026-05-26: act_runner's cache backend has been + # broken on this homelab runner since 2026-05-15 (first as request- + # timeout warnings, then as hard "Cannot find module .../dist/restore/ + # index.js" failures that tank the whole job). The cache step targeted + # ~/.cache/pip but the install below uses `uv pip install` primarily, + # whose own cache lives at ~/.cache/uv — so the cache step's real + # benefit was marginal even when working. Cost of removal: ~30s of + # wheel downloads per job. Future re-enable: mount ~/.cache/uv as a + # docker volume at the runner level (skips actions/cache entirely), + # or fix the runner-side cache backend (clear /var/run/act/actions/*, + # pin act_runner version, etc.). - name: Install Python deps # ruff is pre-installed in the ci-python image (see CI-Runner/CI-python/ @@ -133,22 +128,6 @@ jobs: --health-retries 10 steps: - uses: actions/checkout@v4 - - name: Cache pip wheels - # continue-on-error: act_runner's cache backend has been broken on - # this homelab setup since 2026-05-15 — bolt.db + cache/ dir exist - # but the action's JS bundle now fails to load at all - # ("Cannot find module .../dist/restore/index.js"), hard-failing - # the whole job. The install step that follows handles cold caches - # natively (uv pip / pip install both work without it), so net - # cost of disabling the cache here is ~30s of wheel downloads per - # job. Re-enable once the runner-side cache backend is fixed. - continue-on-error: true - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }} - restore-keys: | - pip-${{ runner.os }}-py314- - name: API integration shard (resolve service IPs, migrate, test) run: | set -eux @@ -207,22 +186,6 @@ jobs: --health-retries 10 steps: - uses: actions/checkout@v4 - - name: Cache pip wheels - # continue-on-error: act_runner's cache backend has been broken on - # this homelab setup since 2026-05-15 — bolt.db + cache/ dir exist - # but the action's JS bundle now fails to load at all - # ("Cannot find module .../dist/restore/index.js"), hard-failing - # the whole job. The install step that follows handles cold caches - # natively (uv pip / pip install both work without it), so net - # cost of disabling the cache here is ~30s of wheel downloads per - # job. Re-enable once the runner-side cache backend is fixed. - continue-on-error: true - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }} - restore-keys: | - pip-${{ runner.os }}-py314- - name: Importer integration shard (resolve service IPs, migrate, test) run: | set -eux @@ -281,22 +244,6 @@ jobs: --health-retries 10 steps: - uses: actions/checkout@v4 - - name: Cache pip wheels - # continue-on-error: act_runner's cache backend has been broken on - # this homelab setup since 2026-05-15 — bolt.db + cache/ dir exist - # but the action's JS bundle now fails to load at all - # ("Cannot find module .../dist/restore/index.js"), hard-failing - # the whole job. The install step that follows handles cold caches - # natively (uv pip / pip install both work without it), so net - # cost of disabling the cache here is ~30s of wheel downloads per - # job. Re-enable once the runner-side cache backend is fixed. - continue-on-error: true - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }} - restore-keys: | - pip-${{ runner.os }}-py314- - name: Core integration shard (everything not api / importer / migration / phash / sidecar / scan / archive / backfill) run: | set -eux From 7a64730bd298906d345d8fd7c57dca4d8cccf5d7 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 26 May 2026 17:52:29 -0400 Subject: [PATCH 2/2] =?UTF-8?q?fix(migration-0022):=20pre-merge=20ALL=20du?= =?UTF-8?q?plicate-external=5Fpost=5Fid=20Posts=20across=20the=20(canonica?= =?UTF-8?q?l+others)=20group,=20not=20just=20canonical-vs-others=20?= =?UTF-8?q?=E2=80=94=20operator's=20v26.05.26.2=20deploy=20still=20tripped?= =?UTF-8?q?=20uq=5Fpost=5Fsource=5Fexternal=5Fid=20because=20two=20non-can?= =?UTF-8?q?onical=20Sources=20both=20had=20Posts=20with=20epid=3D6166997.?= =?UTF-8?q?=20Bulk=20UPDATE=20moved=20the=20first=20cleanly=20then=20colli?= =?UTF-8?q?ded=20on=20the=20second.=20New=20pre-merge=20groups=20all=20Pos?= =?UTF-8?q?ts=20in=20the=20(artist,=20platform)=20by=20external=5Fpost=5Fi?= =?UTF-8?q?d;=20for=20any=20group=20with=20count>1,=20picks=20the=20keep?= =?UTF-8?q?=20(prefer=20one=20under=20canonical;=20else=20lowest=20id)=20a?= =?UTF-8?q?nd=20merges=20the=20rest=20before=20the=20bulk=20reparent.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- .../0022_source_per_artist_platform.py | 105 ++++++++++-------- 1 file changed, 60 insertions(+), 45 deletions(-) diff --git a/alembic/versions/0022_source_per_artist_platform.py b/alembic/versions/0022_source_per_artist_platform.py index 2b2853e..f85b96a 100644 --- a/alembic/versions/0022_source_per_artist_platform.py +++ b/alembic/versions/0022_source_per_artist_platform.py @@ -83,55 +83,70 @@ def upgrade() -> None: if not other_ids: continue - # STEP 2: PRE-merge colliding Posts BEFORE the bulk reparent. - # Find pairs (keep, drop) where: - # keep = a Post under the canonical source - # drop = a Post under one of the non-canonical sources whose - # external_post_id equals keep.external_post_id - # If we let the bulk UPDATE try to repoint drop onto canonical, - # uq_post_source_external_id fires row-by-row and aborts the - # whole migration. - colliding = conn.execute( + # STEP 2: PRE-merge ALL Posts with duplicate external_post_id + # across the entire (canonical + others) group, BEFORE the bulk + # reparent. Two cases must both be handled: + # (A) canonical has Post X with epid=N; an "other" source has + # Post Y with epid=N → after bulk UPDATE, (canonical, N) + # collides with itself. + # (B) two different "other" sources each have a Post with + # epid=N; canonical has none → after bulk UPDATE, both + # are repointed to (canonical, N) and the second collides. + # The earlier version of this migration only handled (A); the + # operator's deploy 2026-05-26 tripped (B) at line 139. + # Fix: group ALL Posts in the (artist, platform) by epid; for + # any group with count>1, pick the keep (prefer one already + # under canonical; else lowest id) and merge the rest into it. + all_posts = conn.execute( text(""" - SELECT keep.id AS keep_id, drop_.id AS drop_id - FROM post AS keep - JOIN post AS drop_ - ON drop_.external_post_id = keep.external_post_id - AND drop_.id != keep.id - WHERE keep.source_id = :canonical - AND drop_.source_id = ANY(:others) + SELECT external_post_id, id, source_id + FROM post + WHERE source_id = :canonical OR source_id = ANY(:others) + ORDER BY external_post_id, id """), {"canonical": canonical_id, "others": other_ids}, ).fetchall() - for keep_id, drop_id in colliding: - conn.execute( - text(""" - UPDATE image_provenance SET post_id = :keep - WHERE post_id = :drop_ - """), - {"keep": keep_id, "drop_": drop_id}, - ) - conn.execute( - text(""" - UPDATE image_record SET primary_post_id = :keep - WHERE primary_post_id = :drop_ - """), - {"keep": keep_id, "drop_": drop_id}, - ) - # Repointed provenance may now collide on - # uq_image_provenance_image_post (alembic 0021). Dedupe: - # keep min(id) per (image_record_id, post_id). - conn.execute(text(""" - DELETE FROM image_provenance ip1 - USING image_provenance ip2 - WHERE ip1.image_record_id = ip2.image_record_id - AND ip1.post_id = ip2.post_id - AND ip1.id > ip2.id - """)) - conn.execute( - text("DELETE FROM post WHERE id = :drop_"), - {"drop_": drop_id}, - ) + by_epid: dict = {} + for epid, post_id, src_id in all_posts: + by_epid.setdefault(epid, []).append((post_id, src_id)) + for epid, posts in by_epid.items(): + if len(posts) <= 1: + continue + # Prefer a Post already under canonical as the keep. + canonical_posts = [p for p in posts if p[1] == canonical_id] + if canonical_posts: + keep_id = canonical_posts[0][0] + else: + keep_id = posts[0][0] # already sorted by id ASC + drop_ids = [p[0] for p in posts if p[0] != keep_id] + for drop_id in drop_ids: + conn.execute( + text(""" + UPDATE image_provenance SET post_id = :keep + WHERE post_id = :drop_ + """), + {"keep": keep_id, "drop_": drop_id}, + ) + conn.execute( + text(""" + UPDATE image_record SET primary_post_id = :keep + WHERE primary_post_id = :drop_ + """), + {"keep": keep_id, "drop_": drop_id}, + ) + # Repointed provenance may now collide on + # uq_image_provenance_image_post (alembic 0021). Dedupe. + conn.execute(text(""" + DELETE FROM image_provenance ip1 + USING image_provenance ip2 + WHERE ip1.image_record_id = ip2.image_record_id + AND ip1.post_id = ip2.post_id + AND ip1.id > ip2.id + """)) + conn.execute( + text("DELETE FROM post WHERE id = :drop_"), + {"drop_": drop_id}, + ) # STEP 3: Bulk reparent the remaining Posts off the other # Sources. After step 2, no collisions on