"""Minimal gallery-dl sidecar parsing (one-time filesystem-import aid). No per-platform branching: a small common key set with fallbacks; the full JSON is kept in raw so anything unmapped is recoverable later. """ import re from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path @dataclass(frozen=True) class SidecarData: platform: str | None external_post_id: str | None post_url: str | None post_title: str | None description: str | None attachment_count: int | None post_date: datetime | None raw: dict # gallery-dl prefixes media filenames with `NN_` for in-post ordering # (`01_HOLLOW-ICHIGO.png`, `02_HOLOW ICHIGO.zip`) but writes the sidecar # under the attachment's stem WITHOUT that ordering prefix # (`HOLLOW-ICHIGO.json`). Strip the prefix when looking for sidecars. # Confirmed against real Patreon downloads 2026-05-26 — without this # strip, every gallery-dl post-level sidecar was invisible to FC since # FC-3 shipped (24 deep-refresh calls produced 0 Posts in operator's DB). _NUMBERING_PREFIX = re.compile(r"^\d+_(.+)$") def find_sidecar(media: Path) -> Path | None: # Attachment-level sidecars (image.jpg.json, image.json). for cand in (media.with_suffix(".json"), Path(str(media) + ".json")): if cand.is_file(): return cand # gallery-dl post-numbered convention: strip the `NN_` prefix from # the stem and look for that.json in the same directory. m = _NUMBERING_PREFIX.match(media.stem) if m: cand = media.parent / f"{m.group(1)}.json" if cand.is_file(): return cand return None def _first_str(data: dict, keys: tuple[str, ...]) -> str | None: for k in keys: v = data.get(k) if isinstance(v, str) and v.strip(): return v.strip() return None # Strip HTML tags + collapse whitespace + take the first non-empty line. # Used to derive a display title from a body when the platform doesn't # expose a separate title field (subscribestar posts always write # `title: ""` and put the leading sentence inside `content` as HTML). # Truncated to 120 chars with an ellipsis if longer — long enough to be # meaningful in a feed, short enough to fit a row. _TAG_RE = re.compile(r"<[^>]+>") _WS_RE = re.compile(r"\s+") def _first_line_text(body: str, limit: int = 120) -> str | None: if not body: return None text = _TAG_RE.sub(" ", body) text = text.replace("\xa0", " ") # Split on hard line breaks first; the body-stripped HTML often # collapses to one logical line, in which case the first sentence # split is the next-best heuristic. for line in text.splitlines(): line = _WS_RE.sub(" ", line).strip() if line: if len(line) > limit: return line[: limit - 1].rstrip() + "…" return line return None def _parse_date(v) -> datetime | None: if isinstance(v, bool): return None if isinstance(v, (int, float)): try: return datetime.fromtimestamp(v, tz=UTC) except (OverflowError, OSError, ValueError): return None if isinstance(v, str): s = v.strip() if not s: return None if s.endswith("Z"): s = s[:-1] + "+00:00" try: dt = datetime.fromisoformat(s) except ValueError: return None return dt if dt.tzinfo else dt.replace(tzinfo=UTC) return None def parse_sidecar(data: dict) -> SidecarData: if not isinstance(data, dict): data = {} cat = data.get("category") platform = cat if isinstance(cat, str) and cat.strip() else None # external_post_id lookup order: post_id MUST come before id. # SubscribeStar gallery-dl writes the per-attachment id in `id` # (e.g. 711509) and the actual post id in `post_id` (e.g. 360360); # picking `id` first fragments every multi-image subscribestar post # into N distinct Post rows in FC. Patreon/Pixiv have no `post_id` # so `id` still wins for them; HF uses `index`, Discord uses # `message_id` — all reached via the remaining chain entries. # Operator-flagged 2026-05-27 during the sidecar audit. external_post_id = None for k in ("post_id", "id", "index", "message_id"): v = data.get(k) if isinstance(v, bool): continue if isinstance(v, (str, int)) and str(v).strip(): external_post_id = str(v) break pc = data.get("page_count") if isinstance(pc, bool): attachment_count = None elif isinstance(pc, int) and pc > 0: attachment_count = pc elif isinstance(data.get("images"), list): attachment_count = len(data["images"]) else: attachment_count = None post_date = None for k in ("published_at", "created_at", "date", "create_date", "timestamp"): if k in data: post_date = _parse_date(data[k]) if post_date is not None: break # `message` is Discord gallery-dl's body field (no `content`); added # 2026-05-27 to the description fallback chain. description = _first_str( data, ("content", "description", "caption", "message"), ) # SubscribeStar posts always write `title: ""` and put the leading # sentence inside `content` (confirmed against the operator's # /mnt/Data/Patreon/Cheunart/subscribestar/ dump 2026-05-27). When # no explicit title is present, synthesize one from the description # body's first non-empty line. Patreon retains its explicit titles # because they're non-empty and short-circuit the fallback. post_title = _first_str(data, ("title",)) if post_title is None and description: post_title = _first_line_text(description) # post_url derivation: SubscribeStar/Pixiv/HF/Discord put the FILE # download URL in `url`, not a post permalink. Synthesize the # permalink from per-platform fields when possible. Patreon's `url` # IS a permalink and is used as-is. For the four file-URL platforms, # the bare `url` is NEVER trusted — derive or return None rather # than persist a CDN URL in post.post_url. if platform in _DERIVED_URL_PLATFORMS: post_url = _derive_post_url(platform, data) else: post_url = _first_str(data, ("url", "post_url")) return SidecarData( platform=platform, external_post_id=external_post_id, post_url=post_url, post_title=post_title, description=description, attachment_count=attachment_count, post_date=post_date, raw=data, ) _DERIVED_URL_PLATFORMS = frozenset({ "subscribestar", "pixiv", "hentaifoundry", "discord", }) def _derive_post_url(platform: str, data: dict) -> str | None: """Synthesize the post-permalink URL from per-platform metadata. gallery-dl writes the file-download URL in `url` for these four platforms; we need a real permalink for the PostCard "open original" button. Returns None if the platform-specific fields are missing (rare in well-formed sidecars but defensive). """ if platform == "subscribestar": pid = data.get("post_id") if isinstance(pid, (str, int)) and str(pid).strip(): return f"https://www.subscribestar.com/posts/{pid}" elif platform == "pixiv": pid = data.get("id") if isinstance(pid, (str, int)) and not isinstance(pid, bool) and str(pid).strip(): return f"https://www.pixiv.net/artworks/{pid}" elif platform == "hentaifoundry": user = _first_str(data, ("user", "artist")) idx = data.get("index") if user and isinstance(idx, (str, int)) and not isinstance(idx, bool) and str(idx).strip(): return f"https://www.hentai-foundry.com/pictures/user/{user}/{idx}" elif platform == "discord": sid = data.get("server_id") cid = data.get("channel_id") mid = data.get("message_id") if all(isinstance(v, (str, int)) and not isinstance(v, bool) and str(v).strip() for v in (sid, cid, mid)): return f"https://discord.com/channels/{sid}/{cid}/{mid}" return None