Files
FabledCurator/tests/test_sidecar_util.py
T
bvandeusen bd3f996582 fix(sidecar): correct external_post_id + post_url derivation for non-Patreon platforms
Audit of one sample sidecar per platform on the operator's
/mnt/Data/Patreon/ archive surfaced three parser bugs that have been
silently corrupting non-Patreon Posts since FC-3 shipped:

1. SubscribeStar `id` vs `post_id` confusion. gallery-dl puts the
   per-attachment id in `id` (e.g. 711509) and the actual post id in
   `post_id` (e.g. 360360). FC's external_post_id chain had `id`
   winning, so every multi-image SubscribeStar post was fragmented into
   N Post rows in the database. Reorder the chain to
   `("post_id", "id", "index", "message_id")` — Patreon/Pixiv (no
   `post_id`), HF (uses `index`), Discord (uses `message_id`) all
   unaffected.

2. Discord `message` field not captured. Discord posts put the body in
   `message`, not `content`. Append it to the description fallback chain
   `("content", "description", "caption", "message")`.

3. post_url is the file URL on SubscribeStar/Pixiv/HF/Discord. New
   `_derive_post_url(platform, data)` helper synthesizes proper
   permalinks from per-platform fields:
     subscribestar → https://www.subscribestar.com/posts/<post_id>
     pixiv         → https://www.pixiv.net/artworks/<id>
     hentaifoundry → https://www.hentai-foundry.com/pictures/user/<user>/<index>
     discord       → https://discord.com/channels/<server>/<channel>/<message>
   Patreon's bare `url` IS a real permalink and is used as-is. For the
   four file-URL platforms, the bare `url` is NEVER trusted: derive or
   return None rather than persist a CDN URL.

Tests:
- `test_parse_core_fields_and_id_priority` flipped to assert post_id
  wins over id.
- New `test_parse_id_used_when_no_post_id` covers the Patreon real
  shape.
- New `test_parse_message_used_as_description_fallback` covers Discord
  bodies.
- Five new tests cover per-platform post_url derivation
  (SubscribeStar/Pixiv/HF/Discord/Patreon-untouched + missing-fields →
  None).

Cleanup migration alembic 0025_fix_subscribestar_post_ids:
- For each SubscribeStar Post: find a related ImageRecord.path, walk to
  its sidecar JSON, read `post_id`, overwrite Post.external_post_id +
  post_url with the corrected values.
- After all updates, every group of Posts under one source sharing the
  same NEW external_post_id is a fragment-set — merge to a canonical
  row using the same ImageProvenance pre-delete + repoint dance as
  alembic 0022 (banked pattern).
- Pure-SQL backfill of Pixiv post_url: replace any `i.pximg.net`-shape
  url with the derived `/artworks/<id>` permalink.
- HF and Discord post_url backfills skipped — HF would need the `user`
  field (not stored on Post), Discord needs server/channel triple.
  Both will be corrected by a deep-scan re-applying sidecars through
  the new parser.

Idempotent: re-running on already-corrected data is a no-op.
2026-05-27 15:35:25 -04:00

252 lines
9.3 KiB
Python

from backend.app.utils.sidecar import SidecarData, find_sidecar, parse_sidecar
def test_find_sidecar_stem_json(tmp_path):
media = tmp_path / "photo.jpg"
media.write_bytes(b"x")
sc = tmp_path / "photo.json"
sc.write_text("{}")
assert find_sidecar(media) == sc
def test_find_sidecar_full_name_json(tmp_path):
media = tmp_path / "photo.jpg"
media.write_bytes(b"x")
sc = tmp_path / "photo.jpg.json"
sc.write_text("{}")
assert find_sidecar(media) == sc
def test_find_sidecar_none(tmp_path):
media = tmp_path / "photo.jpg"
media.write_bytes(b"x")
assert find_sidecar(media) is None
def test_find_sidecar_gallerydl_numbered_prefix(tmp_path):
"""gallery-dl prefixes media filenames with NN_ for in-post ordering
(e.g. `01_HOLLOW-ICHIGO.png`) but writes the post-level sidecar under
the attachment stem WITHOUT the prefix (`HOLLOW-ICHIGO.json`).
Confirmed against real Patreon downloads 2026-05-26 — operator's deep
scan produced 24 refresh calls but 0 Posts because the unprefixed
sidecar was invisible to find_sidecar."""
media = tmp_path / "01_HOLLOW-ICHIGO.png"
media.write_bytes(b"x")
sc = tmp_path / "HOLLOW-ICHIGO.json"
sc.write_text("{}")
assert find_sidecar(media) == sc
def test_find_sidecar_multidigit_prefix(tmp_path):
"""Numbering prefix can be wider than 2 digits (`001_...`); the strip
handles any \\d+_ form."""
media = tmp_path / "001_mirko-sketch.png"
media.write_bytes(b"x")
sc = tmp_path / "mirko-sketch.json"
sc.write_text("{}")
assert find_sidecar(media) == sc
def test_find_sidecar_prefers_attachment_level_over_post_level(tmp_path):
"""If BOTH a per-attachment sidecar and a post-level sidecar exist,
the attachment-level one wins (it's more specific)."""
media = tmp_path / "01_image.png"
media.write_bytes(b"x")
per_attachment = tmp_path / "01_image.json"
per_attachment.write_text('{"specific": true}')
post_level = tmp_path / "image.json"
post_level.write_text('{"specific": false}')
assert find_sidecar(media) == per_attachment
def test_find_sidecar_no_underscore_not_treated_as_prefix(tmp_path):
"""`01.png` (just digits, no underscore-separated stem) shouldn't
match. The regex requires NN_<something>."""
media = tmp_path / "01.png"
media.write_bytes(b"x")
(tmp_path / ".json").write_text("{}") # would be matched only if buggy
assert find_sidecar(media) is None
def test_parse_empty_dict_all_none():
sd = parse_sidecar({})
assert isinstance(sd, SidecarData)
assert (sd.platform, sd.external_post_id, sd.post_url, sd.post_title,
sd.description, sd.attachment_count, sd.post_date) == (
None, None, None, None, None, None, None)
assert sd.raw == {}
def test_parse_core_fields_and_id_priority():
"""`post_id` MUST win over `id` (SubscribeStar duplicate-post fix).
Patreon sidecars in the wild don't expose post_id; this test uses
`category=patreon` but synthetically sets both fields to pin the
parser's precedence."""
sd = parse_sidecar({
"category": "patreon",
"id": 12345, "post_id": 999,
"url": "https://patreon.com/posts/12345",
"title": " Hello ",
"content": "<p>body</p>",
"page_count": 4,
"published_at": "2023-08-01T04:20:02Z",
})
assert sd.platform == "patreon"
assert sd.external_post_id == "999" # 'post_id' wins over 'id'
assert sd.post_url == "https://patreon.com/posts/12345"
assert sd.post_title == "Hello"
assert sd.description == "<p>body</p>"
assert sd.attachment_count == 4
assert sd.post_date.year == 2023 and sd.post_date.tzinfo is not None
def test_parse_id_used_when_no_post_id():
"""Without post_id (Patreon/Pixiv/Discord real shape), `id` wins."""
sd = parse_sidecar({"id": 12345, "url": "https://example.test/p/1"})
assert sd.external_post_id == "12345"
def test_parse_description_precedence_and_images_count():
sd = parse_sidecar({"description": "d", "caption": "c",
"images": [1, 2, 3]})
assert sd.description == "d" # content>description>caption>message
assert sd.attachment_count == 3 # len(images) fallback
def test_parse_message_used_as_description_fallback():
"""Discord posts have `message` not `content`; FC must surface it."""
sd = parse_sidecar({"category": "discord", "message": "hello channel"})
assert sd.description == "hello channel"
def test_parse_date_epoch_and_unparseable_and_naive():
assert parse_sidecar({"timestamp": 1690857602}).post_date.tzinfo is not None
assert parse_sidecar({"date": "not-a-date"}).post_date is None
naive = parse_sidecar({"date": "2023-08-01T00:00:00"}).post_date
assert naive is not None and naive.utcoffset().total_seconds() == 0
def test_parse_title_derived_from_content_when_empty():
"""SubscribeStar gallery-dl writes `title: ""` and puts the leading
sentence in `content` HTML. When `title` is empty, synthesize the
post title from the content body's first non-empty text line."""
sd = parse_sidecar({
"title": "",
"content": "\n<div>Lets say hello to you guys with my Belle <br><br><br>\n</div>\n",
})
assert sd.post_title == "Lets say hello to you guys with my Belle"
assert sd.description == (
"<div>Lets say hello to you guys with my Belle <br><br><br>\n</div>"
)
def test_parse_title_derived_truncates_long_content():
long = "x" * 200
sd = parse_sidecar({"title": "", "content": long})
assert sd.post_title is not None
assert len(sd.post_title) <= 120
assert sd.post_title.endswith("…")
def test_parse_title_explicit_wins_over_content_fallback():
"""If `title` is non-empty, the fallback never runs."""
sd = parse_sidecar({"title": "Real Title", "content": "<p>body line</p>"})
assert sd.post_title == "Real Title"
def test_parse_title_no_fallback_when_no_content():
sd = parse_sidecar({"title": ""})
assert sd.post_title is None
def test_parse_subscribestar_post_url_derived_and_post_id_wins():
"""SubscribeStar gallery-dl puts the per-attachment id in `id` and
the actual post id in `post_id`. The bare `url` is the file URL —
must be ignored and a derived permalink used instead."""
sd = parse_sidecar({
"category": "subscribestar",
"id": 711509, "post_id": 360360,
"url": "/post_uploads?payload=opaque",
"title": "",
"content": "<div>hello</div>",
})
assert sd.external_post_id == "360360"
assert sd.post_url == "https://www.subscribestar.com/posts/360360"
def test_parse_pixiv_post_url_derived():
"""Pixiv's `url` is the image URL (i.pximg.net); must be replaced
with the post permalink under /artworks/<id>."""
sd = parse_sidecar({
"category": "pixiv",
"id": 140466853,
"url": "https://i.pximg.net/img-original/img/2026/01/28/10/28/24/140466853_p0.jpg",
"title": "Nerissa x Jailbird",
})
assert sd.external_post_id == "140466853"
assert sd.post_url == "https://www.pixiv.net/artworks/140466853"
def test_parse_hentaifoundry_post_url_derived():
"""HF sidecars omit `url` entirely and use `index`+`user` for the
post's natural key. Synthesize the canonical /pictures/user/<u>/<i>
permalink."""
sd = parse_sidecar({
"category": "hentaifoundry",
"index": 1182595,
"user": "HolyMeh",
"title": "Annigosa",
})
assert sd.external_post_id == "1182595"
assert sd.post_url == "https://www.hentai-foundry.com/pictures/user/HolyMeh/1182595"
def test_parse_discord_post_url_derived_and_message_id_wins():
"""Discord posts use `message_id` for the post key and the
server/channel/message triple for the permalink."""
sd = parse_sidecar({
"category": "discord",
"message_id": "1195924119762505818",
"channel_id": "968315530597498880",
"server_id": "771088957849075793",
"message": "channel body text",
"url": "https://cdn.discordapp.com/attachments/file.png",
})
assert sd.external_post_id == "1195924119762505818"
assert sd.post_url == (
"https://discord.com/channels/771088957849075793/"
"968315530597498880/1195924119762505818"
)
assert sd.description == "channel body text"
def test_parse_patreon_post_url_kept_as_is():
"""Patreon's bare `url` IS a real permalink — must not be replaced."""
sd = parse_sidecar({
"category": "patreon",
"id": 47074733,
"url": "https://www.patreon.com/posts/barbara-genshin-47074733",
})
assert sd.post_url == "https://www.patreon.com/posts/barbara-genshin-47074733"
def test_parse_derived_url_returns_none_when_fields_missing():
"""If the per-platform fields needed to derive the URL are missing,
return None rather than fall back to the file `url`."""
sd = parse_sidecar({
"category": "subscribestar",
"url": "/post_uploads?payload=opaque",
# no post_id
})
assert sd.post_url is None
def test_parse_ignores_non_str_junk():
sd = parse_sidecar({"category": 5, "title": 7, "page_count": "x",
"id": True})
assert sd.platform is None and sd.post_title is None
assert sd.attachment_count is None
assert sd.external_post_id is None # bool is not a valid id
assert sd.raw == {"category": 5, "title": 7, "page_count": "x",
"id": True}