bvandeusen/FabledCurator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 33 Wiki Activity

Merge pull request 'v26.05.26.3 — hotfix: migration 0022 pre-merge across ENTIRE (canonical+others) group' (#25) from dev into main

Browse Source
This commit was merged in pull request #25.
This commit is contained in:
bvandeusen
2026-05-26 17:52:59 -04:00
parent 8c36dd28b0 7a64730bd2
commit c14338cbce
2 changed files with 71 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.forgejo/workflows/ci.yml
+11 -64
View File
@@ -24,22 +24,17 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Cache pip wheels
# continue-on-error: act_runner's cache backend has been broken on
# this homelab setup since 2026-05-15 — bolt.db + cache/ dir exist
# but the action's JS bundle now fails to load at all
# ("Cannot find module .../dist/restore/index.js"), hard-failing
# the whole job. The install step that follows handles cold caches
# natively (uv pip / pip install both work without it), so net
# cost of disabling the cache here is ~30s of wheel downloads per
# job. Re-enable once the runner-side cache backend is fixed.
continue-on-error: true
uses: actions/cache@v4
with:
path: ~/.cache/pip
key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
restore-keys: |
pip-${{ runner.os }}-py314-
# Cache step removed 2026-05-26: act_runner's cache backend has been
# broken on this homelab runner since 2026-05-15 (first as request-
# timeout warnings, then as hard "Cannot find module .../dist/restore/
# index.js" failures that tank the whole job). The cache step targeted
# ~/.cache/pip but the install below uses `uv pip install` primarily,
# whose own cache lives at ~/.cache/uv — so the cache step's real
# benefit was marginal even when working. Cost of removal: ~30s of
# wheel downloads per job. Future re-enable: mount ~/.cache/uv as a
# docker volume at the runner level (skips actions/cache entirely),
# or fix the runner-side cache backend (clear /var/run/act/actions/*,
# pin act_runner version, etc.).
- name: Install Python deps
# ruff is pre-installed in the ci-python image (see CI-Runner/CI-python/
@@ -133,22 +128,6 @@ jobs:
--health-retries 10
steps:
- uses: actions/checkout@v4
- name: Cache pip wheels
# continue-on-error: act_runner's cache backend has been broken on
# this homelab setup since 2026-05-15 — bolt.db + cache/ dir exist
# but the action's JS bundle now fails to load at all
# ("Cannot find module .../dist/restore/index.js"), hard-failing
# the whole job. The install step that follows handles cold caches
# natively (uv pip / pip install both work without it), so net
# cost of disabling the cache here is ~30s of wheel downloads per
# job. Re-enable once the runner-side cache backend is fixed.
continue-on-error: true
uses: actions/cache@v4
with:
path: ~/.cache/pip
key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
restore-keys: |
pip-${{ runner.os }}-py314-
- name: API integration shard (resolve service IPs, migrate, test)
run: |
set -eux
@@ -207,22 +186,6 @@ jobs:
--health-retries 10
steps:
- uses: actions/checkout@v4
- name: Cache pip wheels
# continue-on-error: act_runner's cache backend has been broken on
# this homelab setup since 2026-05-15 — bolt.db + cache/ dir exist
# but the action's JS bundle now fails to load at all
# ("Cannot find module .../dist/restore/index.js"), hard-failing
# the whole job. The install step that follows handles cold caches
# natively (uv pip / pip install both work without it), so net
# cost of disabling the cache here is ~30s of wheel downloads per
# job. Re-enable once the runner-side cache backend is fixed.
continue-on-error: true
uses: actions/cache@v4
with:
path: ~/.cache/pip
key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
restore-keys: |
pip-${{ runner.os }}-py314-
- name: Importer integration shard (resolve service IPs, migrate, test)
run: |
set -eux
@@ -281,22 +244,6 @@ jobs:
--health-retries 10
steps:
- uses: actions/checkout@v4
- name: Cache pip wheels
# continue-on-error: act_runner's cache backend has been broken on
# this homelab setup since 2026-05-15 — bolt.db + cache/ dir exist
# but the action's JS bundle now fails to load at all
# ("Cannot find module .../dist/restore/index.js"), hard-failing
# the whole job. The install step that follows handles cold caches
# natively (uv pip / pip install both work without it), so net
# cost of disabling the cache here is ~30s of wheel downloads per
# job. Re-enable once the runner-side cache backend is fixed.
continue-on-error: true
uses: actions/cache@v4
with:
path: ~/.cache/pip
key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
restore-keys: |
pip-${{ runner.os }}-py314-
- name: Core integration shard (everything not api / importer / migration / phash / sidecar / scan / archive / backfill)
run: |
set -eux
alembic/versions/0022_source_per_artist_platform.py
+60 -45
View File
@@ -83,55 +83,70 @@ def upgrade() -> None:
if not other_ids:
continue
# STEP 2: PRE-merge colliding Posts BEFORE the bulk reparent.
# Find pairs (keep, drop) where:
# keep = a Post under the canonical source
# drop = a Post under one of the non-canonical sources whose
# external_post_id equals keep.external_post_id
# If we let the bulk UPDATE try to repoint drop onto canonical,
# uq_post_source_external_id fires row-by-row and aborts the
# whole migration.
colliding = conn.execute(
# STEP 2: PRE-merge ALL Posts with duplicate external_post_id
# across the entire (canonical + others) group, BEFORE the bulk
# reparent. Two cases must both be handled:
# (A) canonical has Post X with epid=N; an "other" source has
# Post Y with epid=N → after bulk UPDATE, (canonical, N)
# collides with itself.
# (B) two different "other" sources each have a Post with
# epid=N; canonical has none → after bulk UPDATE, both
# are repointed to (canonical, N) and the second collides.
# The earlier version of this migration only handled (A); the
# operator's deploy 2026-05-26 tripped (B) at line 139.
# Fix: group ALL Posts in the (artist, platform) by epid; for
# any group with count>1, pick the keep (prefer one already
# under canonical; else lowest id) and merge the rest into it.
all_posts = conn.execute(
text("""
SELECT keep.id AS keep_id, drop_.id AS drop_id
FROM post AS keep
JOIN post AS drop_
ON drop_.external_post_id = keep.external_post_id
AND drop_.id != keep.id
WHERE keep.source_id = :canonical
AND drop_.source_id = ANY(:others)
SELECT external_post_id, id, source_id
FROM post
WHERE source_id = :canonical OR source_id = ANY(:others)
ORDER BY external_post_id, id
"""),
{"canonical": canonical_id, "others": other_ids},
).fetchall()
for keep_id, drop_id in colliding:
conn.execute(
text("""
UPDATE image_provenance SET post_id = :keep
WHERE post_id = :drop_
"""),
{"keep": keep_id, "drop_": drop_id},
)
conn.execute(
text("""
UPDATE image_record SET primary_post_id = :keep
WHERE primary_post_id = :drop_
"""),
{"keep": keep_id, "drop_": drop_id},
)
# Repointed provenance may now collide on
# uq_image_provenance_image_post (alembic 0021). Dedupe:
# keep min(id) per (image_record_id, post_id).
conn.execute(text("""
DELETE FROM image_provenance ip1
USING image_provenance ip2
WHERE ip1.image_record_id = ip2.image_record_id
AND ip1.post_id = ip2.post_id
AND ip1.id > ip2.id
"""))
conn.execute(
text("DELETE FROM post WHERE id = :drop_"),
{"drop_": drop_id},
)
by_epid: dict = {}
for epid, post_id, src_id in all_posts:
by_epid.setdefault(epid, []).append((post_id, src_id))
for epid, posts in by_epid.items():
if len(posts) <= 1:
continue
# Prefer a Post already under canonical as the keep.
canonical_posts = [p for p in posts if p[1] == canonical_id]
if canonical_posts:
keep_id = canonical_posts[0][0]
else:
keep_id = posts[0][0] # already sorted by id ASC
drop_ids = [p[0] for p in posts if p[0] != keep_id]
for drop_id in drop_ids:
conn.execute(
text("""
UPDATE image_provenance SET post_id = :keep
WHERE post_id = :drop_
"""),
{"keep": keep_id, "drop_": drop_id},
)
conn.execute(
text("""
UPDATE image_record SET primary_post_id = :keep
WHERE primary_post_id = :drop_
"""),
{"keep": keep_id, "drop_": drop_id},
)
# Repointed provenance may now collide on
# uq_image_provenance_image_post (alembic 0021). Dedupe.
conn.execute(text("""
DELETE FROM image_provenance ip1
USING image_provenance ip2
WHERE ip1.image_record_id = ip2.image_record_id
AND ip1.post_id = ip2.post_id
AND ip1.id > ip2.id
"""))
conn.execute(
text("DELETE FROM post WHERE id = :drop_"),
{"drop_": drop_id},
)
# STEP 3: Bulk reparent the remaining Posts off the other
# Sources. After step 2, no collisions on