bd3f996582
Audit of one sample sidecar per platform on the operator's
/mnt/Data/Patreon/ archive surfaced three parser bugs that have been
silently corrupting non-Patreon Posts since FC-3 shipped:
1. SubscribeStar `id` vs `post_id` confusion. gallery-dl puts the
per-attachment id in `id` (e.g. 711509) and the actual post id in
`post_id` (e.g. 360360). FC's external_post_id chain had `id`
winning, so every multi-image SubscribeStar post was fragmented into
N Post rows in the database. Reorder the chain to
`("post_id", "id", "index", "message_id")` — Patreon/Pixiv (no
`post_id`), HF (uses `index`), Discord (uses `message_id`) all
unaffected.
2. Discord `message` field not captured. Discord posts put the body in
`message`, not `content`. Append it to the description fallback chain
`("content", "description", "caption", "message")`.
3. post_url is the file URL on SubscribeStar/Pixiv/HF/Discord. New
`_derive_post_url(platform, data)` helper synthesizes proper
permalinks from per-platform fields:
subscribestar → https://www.subscribestar.com/posts/<post_id>
pixiv → https://www.pixiv.net/artworks/<id>
hentaifoundry → https://www.hentai-foundry.com/pictures/user/<user>/<index>
discord → https://discord.com/channels/<server>/<channel>/<message>
Patreon's bare `url` IS a real permalink and is used as-is. For the
four file-URL platforms, the bare `url` is NEVER trusted: derive or
return None rather than persist a CDN URL.
Tests:
- `test_parse_core_fields_and_id_priority` flipped to assert post_id
wins over id.
- New `test_parse_id_used_when_no_post_id` covers the Patreon real
shape.
- New `test_parse_message_used_as_description_fallback` covers Discord
bodies.
- Five new tests cover per-platform post_url derivation
(SubscribeStar/Pixiv/HF/Discord/Patreon-untouched + missing-fields →
None).
Cleanup migration alembic 0025_fix_subscribestar_post_ids:
- For each SubscribeStar Post: find a related ImageRecord.path, walk to
its sidecar JSON, read `post_id`, overwrite Post.external_post_id +
post_url with the corrected values.
- After all updates, every group of Posts under one source sharing the
same NEW external_post_id is a fragment-set — merge to a canonical
row using the same ImageProvenance pre-delete + repoint dance as
alembic 0022 (banked pattern).
- Pure-SQL backfill of Pixiv post_url: replace any `i.pximg.net`-shape
url with the derived `/artworks/<id>` permalink.
- HF and Discord post_url backfills skipped — HF would need the `user`
field (not stored on Post), Discord needs server/channel triple.
Both will be corrected by a deep-scan re-applying sidecars through
the new parser.
Idempotent: re-running on already-corrected data is a no-op.
252 lines
9.3 KiB
Python
252 lines
9.3 KiB
Python
from backend.app.utils.sidecar import SidecarData, find_sidecar, parse_sidecar
|
|
|
|
|
|
def test_find_sidecar_stem_json(tmp_path):
|
|
media = tmp_path / "photo.jpg"
|
|
media.write_bytes(b"x")
|
|
sc = tmp_path / "photo.json"
|
|
sc.write_text("{}")
|
|
assert find_sidecar(media) == sc
|
|
|
|
|
|
def test_find_sidecar_full_name_json(tmp_path):
|
|
media = tmp_path / "photo.jpg"
|
|
media.write_bytes(b"x")
|
|
sc = tmp_path / "photo.jpg.json"
|
|
sc.write_text("{}")
|
|
assert find_sidecar(media) == sc
|
|
|
|
|
|
def test_find_sidecar_none(tmp_path):
|
|
media = tmp_path / "photo.jpg"
|
|
media.write_bytes(b"x")
|
|
assert find_sidecar(media) is None
|
|
|
|
|
|
def test_find_sidecar_gallerydl_numbered_prefix(tmp_path):
|
|
"""gallery-dl prefixes media filenames with NN_ for in-post ordering
|
|
(e.g. `01_HOLLOW-ICHIGO.png`) but writes the post-level sidecar under
|
|
the attachment stem WITHOUT the prefix (`HOLLOW-ICHIGO.json`).
|
|
Confirmed against real Patreon downloads 2026-05-26 — operator's deep
|
|
scan produced 24 refresh calls but 0 Posts because the unprefixed
|
|
sidecar was invisible to find_sidecar."""
|
|
media = tmp_path / "01_HOLLOW-ICHIGO.png"
|
|
media.write_bytes(b"x")
|
|
sc = tmp_path / "HOLLOW-ICHIGO.json"
|
|
sc.write_text("{}")
|
|
assert find_sidecar(media) == sc
|
|
|
|
|
|
def test_find_sidecar_multidigit_prefix(tmp_path):
|
|
"""Numbering prefix can be wider than 2 digits (`001_...`); the strip
|
|
handles any \\d+_ form."""
|
|
media = tmp_path / "001_mirko-sketch.png"
|
|
media.write_bytes(b"x")
|
|
sc = tmp_path / "mirko-sketch.json"
|
|
sc.write_text("{}")
|
|
assert find_sidecar(media) == sc
|
|
|
|
|
|
def test_find_sidecar_prefers_attachment_level_over_post_level(tmp_path):
|
|
"""If BOTH a per-attachment sidecar and a post-level sidecar exist,
|
|
the attachment-level one wins (it's more specific)."""
|
|
media = tmp_path / "01_image.png"
|
|
media.write_bytes(b"x")
|
|
per_attachment = tmp_path / "01_image.json"
|
|
per_attachment.write_text('{"specific": true}')
|
|
post_level = tmp_path / "image.json"
|
|
post_level.write_text('{"specific": false}')
|
|
assert find_sidecar(media) == per_attachment
|
|
|
|
|
|
def test_find_sidecar_no_underscore_not_treated_as_prefix(tmp_path):
|
|
"""`01.png` (just digits, no underscore-separated stem) shouldn't
|
|
match. The regex requires NN_<something>."""
|
|
media = tmp_path / "01.png"
|
|
media.write_bytes(b"x")
|
|
(tmp_path / ".json").write_text("{}") # would be matched only if buggy
|
|
assert find_sidecar(media) is None
|
|
|
|
|
|
def test_parse_empty_dict_all_none():
|
|
sd = parse_sidecar({})
|
|
assert isinstance(sd, SidecarData)
|
|
assert (sd.platform, sd.external_post_id, sd.post_url, sd.post_title,
|
|
sd.description, sd.attachment_count, sd.post_date) == (
|
|
None, None, None, None, None, None, None)
|
|
assert sd.raw == {}
|
|
|
|
|
|
def test_parse_core_fields_and_id_priority():
|
|
"""`post_id` MUST win over `id` (SubscribeStar duplicate-post fix).
|
|
Patreon sidecars in the wild don't expose post_id; this test uses
|
|
`category=patreon` but synthetically sets both fields to pin the
|
|
parser's precedence."""
|
|
sd = parse_sidecar({
|
|
"category": "patreon",
|
|
"id": 12345, "post_id": 999,
|
|
"url": "https://patreon.com/posts/12345",
|
|
"title": " Hello ",
|
|
"content": "<p>body</p>",
|
|
"page_count": 4,
|
|
"published_at": "2023-08-01T04:20:02Z",
|
|
})
|
|
assert sd.platform == "patreon"
|
|
assert sd.external_post_id == "999" # 'post_id' wins over 'id'
|
|
assert sd.post_url == "https://patreon.com/posts/12345"
|
|
assert sd.post_title == "Hello"
|
|
assert sd.description == "<p>body</p>"
|
|
assert sd.attachment_count == 4
|
|
assert sd.post_date.year == 2023 and sd.post_date.tzinfo is not None
|
|
|
|
|
|
def test_parse_id_used_when_no_post_id():
|
|
"""Without post_id (Patreon/Pixiv/Discord real shape), `id` wins."""
|
|
sd = parse_sidecar({"id": 12345, "url": "https://example.test/p/1"})
|
|
assert sd.external_post_id == "12345"
|
|
|
|
|
|
def test_parse_description_precedence_and_images_count():
|
|
sd = parse_sidecar({"description": "d", "caption": "c",
|
|
"images": [1, 2, 3]})
|
|
assert sd.description == "d" # content>description>caption>message
|
|
assert sd.attachment_count == 3 # len(images) fallback
|
|
|
|
|
|
def test_parse_message_used_as_description_fallback():
|
|
"""Discord posts have `message` not `content`; FC must surface it."""
|
|
sd = parse_sidecar({"category": "discord", "message": "hello channel"})
|
|
assert sd.description == "hello channel"
|
|
|
|
|
|
def test_parse_date_epoch_and_unparseable_and_naive():
|
|
assert parse_sidecar({"timestamp": 1690857602}).post_date.tzinfo is not None
|
|
assert parse_sidecar({"date": "not-a-date"}).post_date is None
|
|
naive = parse_sidecar({"date": "2023-08-01T00:00:00"}).post_date
|
|
assert naive is not None and naive.utcoffset().total_seconds() == 0
|
|
|
|
|
|
def test_parse_title_derived_from_content_when_empty():
|
|
"""SubscribeStar gallery-dl writes `title: ""` and puts the leading
|
|
sentence in `content` HTML. When `title` is empty, synthesize the
|
|
post title from the content body's first non-empty text line."""
|
|
sd = parse_sidecar({
|
|
"title": "",
|
|
"content": "\n<div>Lets say hello to you guys with my Belle <br><br><br>\n</div>\n",
|
|
})
|
|
assert sd.post_title == "Lets say hello to you guys with my Belle"
|
|
assert sd.description == (
|
|
"<div>Lets say hello to you guys with my Belle <br><br><br>\n</div>"
|
|
)
|
|
|
|
|
|
def test_parse_title_derived_truncates_long_content():
|
|
long = "x" * 200
|
|
sd = parse_sidecar({"title": "", "content": long})
|
|
assert sd.post_title is not None
|
|
assert len(sd.post_title) <= 120
|
|
assert sd.post_title.endswith("…")
|
|
|
|
|
|
def test_parse_title_explicit_wins_over_content_fallback():
|
|
"""If `title` is non-empty, the fallback never runs."""
|
|
sd = parse_sidecar({"title": "Real Title", "content": "<p>body line</p>"})
|
|
assert sd.post_title == "Real Title"
|
|
|
|
|
|
def test_parse_title_no_fallback_when_no_content():
|
|
sd = parse_sidecar({"title": ""})
|
|
assert sd.post_title is None
|
|
|
|
|
|
def test_parse_subscribestar_post_url_derived_and_post_id_wins():
|
|
"""SubscribeStar gallery-dl puts the per-attachment id in `id` and
|
|
the actual post id in `post_id`. The bare `url` is the file URL —
|
|
must be ignored and a derived permalink used instead."""
|
|
sd = parse_sidecar({
|
|
"category": "subscribestar",
|
|
"id": 711509, "post_id": 360360,
|
|
"url": "/post_uploads?payload=opaque",
|
|
"title": "",
|
|
"content": "<div>hello</div>",
|
|
})
|
|
assert sd.external_post_id == "360360"
|
|
assert sd.post_url == "https://www.subscribestar.com/posts/360360"
|
|
|
|
|
|
def test_parse_pixiv_post_url_derived():
|
|
"""Pixiv's `url` is the image URL (i.pximg.net); must be replaced
|
|
with the post permalink under /artworks/<id>."""
|
|
sd = parse_sidecar({
|
|
"category": "pixiv",
|
|
"id": 140466853,
|
|
"url": "https://i.pximg.net/img-original/img/2026/01/28/10/28/24/140466853_p0.jpg",
|
|
"title": "Nerissa x Jailbird",
|
|
})
|
|
assert sd.external_post_id == "140466853"
|
|
assert sd.post_url == "https://www.pixiv.net/artworks/140466853"
|
|
|
|
|
|
def test_parse_hentaifoundry_post_url_derived():
|
|
"""HF sidecars omit `url` entirely and use `index`+`user` for the
|
|
post's natural key. Synthesize the canonical /pictures/user/<u>/<i>
|
|
permalink."""
|
|
sd = parse_sidecar({
|
|
"category": "hentaifoundry",
|
|
"index": 1182595,
|
|
"user": "HolyMeh",
|
|
"title": "Annigosa",
|
|
})
|
|
assert sd.external_post_id == "1182595"
|
|
assert sd.post_url == "https://www.hentai-foundry.com/pictures/user/HolyMeh/1182595"
|
|
|
|
|
|
def test_parse_discord_post_url_derived_and_message_id_wins():
|
|
"""Discord posts use `message_id` for the post key and the
|
|
server/channel/message triple for the permalink."""
|
|
sd = parse_sidecar({
|
|
"category": "discord",
|
|
"message_id": "1195924119762505818",
|
|
"channel_id": "968315530597498880",
|
|
"server_id": "771088957849075793",
|
|
"message": "channel body text",
|
|
"url": "https://cdn.discordapp.com/attachments/file.png",
|
|
})
|
|
assert sd.external_post_id == "1195924119762505818"
|
|
assert sd.post_url == (
|
|
"https://discord.com/channels/771088957849075793/"
|
|
"968315530597498880/1195924119762505818"
|
|
)
|
|
assert sd.description == "channel body text"
|
|
|
|
|
|
def test_parse_patreon_post_url_kept_as_is():
|
|
"""Patreon's bare `url` IS a real permalink — must not be replaced."""
|
|
sd = parse_sidecar({
|
|
"category": "patreon",
|
|
"id": 47074733,
|
|
"url": "https://www.patreon.com/posts/barbara-genshin-47074733",
|
|
})
|
|
assert sd.post_url == "https://www.patreon.com/posts/barbara-genshin-47074733"
|
|
|
|
|
|
def test_parse_derived_url_returns_none_when_fields_missing():
|
|
"""If the per-platform fields needed to derive the URL are missing,
|
|
return None rather than fall back to the file `url`."""
|
|
sd = parse_sidecar({
|
|
"category": "subscribestar",
|
|
"url": "/post_uploads?payload=opaque",
|
|
# no post_id
|
|
})
|
|
assert sd.post_url is None
|
|
|
|
|
|
def test_parse_ignores_non_str_junk():
|
|
sd = parse_sidecar({"category": 5, "title": 7, "page_count": "x",
|
|
"id": True})
|
|
assert sd.platform is None and sd.post_title is None
|
|
assert sd.attachment_count is None
|
|
assert sd.external_post_id is None # bool is not a valid id
|
|
assert sd.raw == {"category": 5, "title": 7, "page_count": "x",
|
|
"id": True}
|