From b7b313cc05e90eb7deccad4e2b8ed79d54bd0f82 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Wed, 27 May 2026 15:38:18 -0400 Subject: [PATCH] fix(alembic 0025): include HF + Discord post_url backfill (no longer 'deferred to deep-scan') MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator-flagged: the claim that 'a future deep-scan via the new parser will fix HF and Discord post_url' was conditional on the operator actually running a deep-scan, which they might not do for ages. Until then HF posts stay at post_url=NULL (HF sidecars have no `url` field) and Discord posts stay pointing at cdn.discordapp.com/attachments/... (the file URL, not the message permalink). The migration was already opening sidecar files for SubscribeStar. Generalizing the loop to also handle HF and Discord is a tiny addition that closes the gap without operator intervention. Per-platform Part 1 logic now: subscribestar — read sidecar.post_id, overwrite external_post_id + post_url with the derived /posts/ permalink. hentaifoundry — read sidecar.user + .index, overwrite post_url with /pictures/user//. external_post_id (= index) unchanged. discord — read sidecar.server_id + .channel_id + .message_id, overwrite post_url with the discord.com/channels/.../ triple. external_post_id (= message_id) unchanged. Part 2 (SubscribeStar fragment merge) and Part 3 (pure-SQL Pixiv post_url backfill) unchanged. Posts whose related ImageRecord paths don't resolve on disk (orphan filesystem state) are reported per-platform in the migration output — those still need a future deep-scan, but the in-DB-with-on-disk-files common case is now fully covered by the migration alone. --- .../0025_fix_subscribestar_post_ids.py | 159 +++++++++++------- 1 file changed, 98 insertions(+), 61 deletions(-) diff --git a/alembic/versions/0025_fix_subscribestar_post_ids.py b/alembic/versions/0025_fix_subscribestar_post_ids.py index fb4fa49..2baa72b 100644 --- a/alembic/versions/0025_fix_subscribestar_post_ids.py +++ b/alembic/versions/0025_fix_subscribestar_post_ids.py @@ -1,46 +1,48 @@ -"""subscribestar: correct external_post_id + post_url, merge fragmented Posts +"""sidecar-audit followup: correct external_post_id + post_url across all platforms Revision ID: 0025 Revises: 0024 Create Date: 2026-05-27 -Closes the operator-flagged 2026-05-27 SubscribeStar data-fragmentation -issue: gallery-dl writes the per-attachment id in `id` and the actual -post id in `post_id`. FC's sidecar parser had `id` winning the -external_post_id fallback chain, so every multi-image SubscribeStar post -was fragmented into N Post rows (one per attachment) in the database. +Closes the operator-flagged 2026-05-27 sidecar audit findings. Three +data-correctness bugs across non-Patreon platforms had been silently +corrupting Posts since FC-3 shipped; the parser fix (sidecar.py, same +commit) addresses new imports. This migration cleans up existing rows. -The parser fix (sidecar.py, same commit) reorders the chain so `post_id` -wins. This migration cleans up the existing fragmented data. +Per-platform actions: -Strategy per SubscribeStar Post in the database: - 1. Find a related ImageRecord via ImageProvenance to get an on-disk - path. Walk to its sidecar JSON (gallery-dl's NN_ numbering prefix - convention is honored). - 2. Read the sidecar; if it has `post_id`, that's the correct - external_post_id. - 3. Update Post.external_post_id (= post_id) and Post.post_url (= - derived https://www.subscribestar.com/posts/). + subscribestar — gallery-dl wrote the per-attachment id in `id` and + the actual post id in `post_id`. FC's parser picked `id`, so every + multi-image SubscribeStar post was fragmented into N Post rows. + 1. For each SubscribeStar Post, read its sidecar (via the related + ImageRecord's on-disk path), pull `post_id`, overwrite + external_post_id and post_url. + 2. Merge groups of Posts under one source that now share an + external_post_id (fragments of the same actual post). Same + ImageProvenance pre-delete + repoint dance as alembic 0022. -After all rows are updated, every group of Posts under the same source -that share the same NEW external_post_id is a duplicate set — -fragments of the same actual post. Merge them: pick canonical (lowest -id), repoint ImageProvenance + ImageRecord.primary_post_id + -PostAttachment to canonical, dedupe ImageProvenance against alembic -0021's UNIQUE, delete drop Posts. + hentaifoundry — sidecars have NO `url` field; `src` is the image + URL. FC's parser stored post_url=NULL. Read each HF Post's sidecar + for `user` + `index`, derive the canonical /pictures/user// + permalink. external_post_id (= `index`) was already correct. -Also: pure-SQL backfill of Pixiv post_url (Pixiv sidecars also write a -file URL in `url`; the post permalink is /artworks/ and FC's parser -now derives it). Pixiv external_post_id is already correct (gallery-dl's -`id` matches the URL pattern) so no per-row sidecar IO needed. + discord — gallery-dl wrote the CDN attachment URL in `url`. FC's + parser stored that as post_url. Read each Discord Post's sidecar + for the server/channel/message triple, derive the proper + discord.com/channels/.../ permalink. external_post_id (= + `message_id`) was already correct. -HF + Discord post_url backfills are skipped here — HF needs the user -field (not stored on Post directly), Discord needs server/channel -triple. Both will eventually be corrected by a deep-scan that -re-applies sidecars via the new parser. + pixiv — pure-SQL backfill: replace any `i.pximg.net`-style URL on + Post.post_url with the derived `/artworks/` permalink. Pixiv + external_post_id (= `id`) was already correct; no sidecar IO + needed. -Idempotent: re-running on already-corrected data is a no-op (the post_id -in sidecar matches what's already in DB). +Idempotent: re-running on already-corrected data is a no-op (skips +rows whose derived value matches what's already stored). + +Posts whose related ImageRecord paths don't resolve on disk (orphaned +filesystem state) are skipped with a count in the migration output — +those will be picked up by a future deep-scan. """ from __future__ import annotations @@ -89,45 +91,79 @@ def _find_sidecar(media_path: Path) -> Path | None: return None +def _str_id(v) -> str | None: + """str() a JSON scalar id; reject bool (JSON booleans are ints in + Python's eyes but they aren't valid sidecar ids).""" + if isinstance(v, bool): + return None + if isinstance(v, (str, int)) and str(v).strip(): + return str(v).strip() + return None + + +def _str_field(v) -> str | None: + if isinstance(v, str) and v.strip(): + return v.strip() + return None + + def upgrade() -> None: conn = op.get_bind() - # ── PART 1: SubscribeStar post_id correction ───────────────────── - ss_posts = conn.execute(text(""" - SELECT p.id, p.external_post_id, p.source_id, p.post_url + # ── PART 1: Per-platform corrections requiring filesystem IO ───── + # SubscribeStar, HentaiFoundry, Discord all need fields from the + # sidecar to construct the right post_url. We walk each Post's + # related ImageRecord.path to find the sidecar, read it, derive, + # and update. + targets = conn.execute(text(""" + SELECT p.id, p.external_post_id, p.post_url, s.platform FROM post p JOIN source s ON s.id = p.source_id - WHERE s.platform = 'subscribestar' + WHERE s.platform IN ('subscribestar', 'hentaifoundry', 'discord') """)).fetchall() - sidecars_read = 0 - posts_updated = 0 - posts_missing_sidecar = 0 - for post_row in ss_posts: + stats: dict[str, dict[str, int]] = { + plat: {"read": 0, "updated": 0, "no_sidecar": 0} + for plat in ("subscribestar", "hentaifoundry", "discord") + } + for post_row in targets: + plat = post_row.platform path = _first_attachment_path(conn, post_row.id) if not path: - posts_missing_sidecar += 1 + stats[plat]["no_sidecar"] += 1 continue sidecar = _find_sidecar(Path(path)) if sidecar is None: - posts_missing_sidecar += 1 + stats[plat]["no_sidecar"] += 1 continue try: data = json.loads(sidecar.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): - posts_missing_sidecar += 1 + stats[plat]["no_sidecar"] += 1 continue - sidecars_read += 1 - post_id = data.get("post_id") - if not isinstance(post_id, (str, int)) or isinstance(post_id, bool): - continue - post_id_str = str(post_id).strip() - if not post_id_str: - continue - new_url = f"https://www.subscribestar.com/posts/{post_id_str}" - # Idempotent: skip if already correct. - if (post_row.external_post_id == post_id_str - and post_row.post_url == new_url): + stats[plat]["read"] += 1 + + new_epid = post_row.external_post_id + new_url = None + if plat == "subscribestar": + pid = _str_id(data.get("post_id")) + if pid: + new_epid = pid + new_url = f"https://www.subscribestar.com/posts/{pid}" + elif plat == "hentaifoundry": + user = _str_field(data.get("user")) or _str_field(data.get("artist")) + idx = _str_id(data.get("index")) + if user and idx: + new_url = f"https://www.hentai-foundry.com/pictures/user/{user}/{idx}" + elif plat == "discord": + sid = _str_id(data.get("server_id")) + cid = _str_id(data.get("channel_id")) + mid = _str_id(data.get("message_id")) + if sid and cid and mid: + new_url = f"https://discord.com/channels/{sid}/{cid}/{mid}" + + # Idempotent: skip if nothing changed. + if new_epid == post_row.external_post_id and new_url == post_row.post_url: continue conn.execute( text(""" @@ -135,15 +171,16 @@ def upgrade() -> None: SET external_post_id = :epid, post_url = :url WHERE id = :id """), - {"epid": post_id_str, "url": new_url, "id": post_row.id}, + {"epid": new_epid, "url": new_url, "id": post_row.id}, ) - posts_updated += 1 + stats[plat]["updated"] += 1 - print( - f"0025: subscribestar — read {sidecars_read} sidecars, " - f"updated {posts_updated} Posts, " - f"{posts_missing_sidecar} Posts had no resolvable sidecar" - ) + for plat, s in stats.items(): + print( + f"0025: {plat} — read {s['read']} sidecars, " + f"updated {s['updated']} Posts, " + f"{s['no_sidecar']} Posts had no resolvable sidecar" + ) # ── PART 2: Merge SubscribeStar fragments now sharing epid ─────── # After Part 1, each group of Posts under one source with the SAME