diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 407dc24..2bf6248 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -173,24 +173,51 @@ jobs: # same source code as the preceding main-push build but with an # immutable version tag — they need the XPI too, otherwise the # versioned image ships without the signed extension. + # + # Tag-push vs main-push race (operator-flagged 2026-05-27 after + # v26.05.27.0 hit it): a release cut fires BOTH workflows almost + # simultaneously. Main-push runs sign-extension (1-5min AMO round + # trip) before publishing the ext- release; tag-push + # skips sign-extension (gated to main) and races straight to + # this download step. Tag-push lost every time. Fix: poll the + # ext- release endpoint with a sleep+retry loop (30s + # for up to 10min total) before giving up. Main-push's signing + # eventually wins and tag-push picks the release up on a later + # iteration. if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') env: TOKEN: ${{ secrets.RELEASE_TOKEN }} run: | set -eux VERSION=$(grep -E '"version"' extension/package.json | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/') - # Look up the ext- release; extract the .xpi asset's - # browser_download_url (Forgejo's /releases/assets/ endpoint - # returns ASSET METADATA, not the binary blob — operator-flagged - # 2026-05-26: my prior code curl'd the metadata endpoint without - # -f and wrote the resulting 404-page-not-found text into - # fabledcurator-*.xpi, which Firefox then rejected as "corrupt"). - # browser_download_url is the canonical binary endpoint and is - # also publicly accessible (no token needed) but we pass the - # token anyway for symmetry with private-repo support. - curl -sf -H "Authorization: token $TOKEN" \ - "https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" \ - -o release.json + # Poll for the ext- release. main-push's sign-extension + # step (AMO round-trip, 1-5min) needs to finish + upload before + # tag-push can fetch. 30s * 20 = up to 10min wait, then hard-fail. + for attempt in $(seq 1 20); do + STATUS=$(curl -s -o release.json -w "%{http_code}" \ + -H "Authorization: token $TOKEN" \ + "https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" || echo 000) + if [ "$STATUS" = "200" ]; then + echo "Found ext-$VERSION release on attempt $attempt" + break + fi + if [ "$attempt" = "20" ]; then + echo "ERROR: ext-$VERSION release not available after 10min of polling" + echo "Last HTTP status: $STATUS" + exit 1 + fi + echo "Attempt $attempt: ext-$VERSION not yet published (HTTP $STATUS); sleeping 30s" + sleep 30 + done + # Extract the .xpi asset's browser_download_url (Forgejo's + # /releases/assets/ endpoint returns ASSET METADATA, not + # the binary blob — operator-flagged 2026-05-26: my prior + # code curl'd the metadata endpoint without -f and wrote the + # resulting 404-page-not-found text into fabledcurator-*.xpi, + # which Firefox then rejected as "corrupt"). + # browser_download_url is the canonical binary endpoint and + # is also publicly accessible (no token needed) but we pass + # the token anyway for symmetry with private-repo support. DOWNLOAD_URL=$(python3 -c "import json; r=json.load(open('release.json')); xpis=[a for a in r.get('assets', []) if a.get('name','').endswith('.xpi')]; print(xpis[0]['browser_download_url'])") test -n "$DOWNLOAD_URL" echo "Downloading XPI from: $DOWNLOAD_URL" diff --git a/alembic/versions/0024_backfill_post_title_from_description.py b/alembic/versions/0024_backfill_post_title_from_description.py new file mode 100644 index 0000000..2b1385c --- /dev/null +++ b/alembic/versions/0024_backfill_post_title_from_description.py @@ -0,0 +1,80 @@ +"""backfill post.post_title from description first-line — 2026-05-27 + +Revision ID: 0024 +Revises: 0023 +Create Date: 2026-05-27 + +SubscribeStar gallery-dl always writes `title: ""` and embeds the leading +sentence inside `content` HTML. FC's sidecar parser was leaving +post_title NULL for every SubscribeStar post since FC-3 shipped. The +parser fix (sidecar._first_line_text fallback) now synthesizes a title +at parse time; this migration applies the same logic retroactively to +existing rows. + +Operator-flagged 2026-05-27 after inspecting +/mnt/Data/Patreon/Cheunart/subscribestar/ sidecars. + +Idempotent: only touches rows where post_title IS NULL or empty AND +description IS NOT NULL. Re-running the migration is a no-op. +""" +from __future__ import annotations + +import re +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import text + +revision: str = "0024" +down_revision: Union[str, None] = "0023" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +_TAG_RE = re.compile(r"<[^>]+>") +_WS_RE = re.compile(r"\s+") + + +def _first_line_text(body: str, limit: int = 120) -> str | None: + """Mirror of sidecar._first_line_text. Kept inline so the migration + doesn't carry a runtime import dependency from app code that may + have moved by the time the migration is replayed years from now.""" + if not body: + return None + text_ = _TAG_RE.sub(" ", body) + text_ = text_.replace("\xa0", " ") + for line in text_.splitlines(): + line = _WS_RE.sub(" ", line).strip() + if line: + if len(line) > limit: + return line[: limit - 1].rstrip() + "…" + return line + return None + + +def upgrade() -> None: + bind = op.get_bind() + rows = bind.execute( + text( + "SELECT id, description FROM post " + "WHERE (post_title IS NULL OR post_title = '') " + "AND description IS NOT NULL AND description <> ''" + ) + ).fetchall() + updated = 0 + for row in rows: + derived = _first_line_text(row.description) + if not derived: + continue + bind.execute( + text("UPDATE post SET post_title = :t WHERE id = :id"), + {"t": derived, "id": row.id}, + ) + updated += 1 + print(f"0024: backfilled post_title on {updated} row(s)") + + +def downgrade() -> None: + # No safe restore — we can't tell which post_titles were derived vs + # genuinely present. Leave the column alone on rollback. + pass diff --git a/alembic/versions/0025_fix_subscribestar_post_ids.py b/alembic/versions/0025_fix_subscribestar_post_ids.py new file mode 100644 index 0000000..b44f430 --- /dev/null +++ b/alembic/versions/0025_fix_subscribestar_post_ids.py @@ -0,0 +1,288 @@ +"""sidecar-audit followup: correct external_post_id + post_url across all platforms + +Revision ID: 0025 +Revises: 0024 +Create Date: 2026-05-27 + +Closes the operator-flagged 2026-05-27 sidecar audit findings. Three +data-correctness bugs across non-Patreon platforms had been silently +corrupting Posts since FC-3 shipped; the parser fix (sidecar.py, same +commit) addresses new imports. This migration cleans up existing rows. + +Per-platform actions: + + subscribestar — gallery-dl wrote the per-attachment id in `id` and + the actual post id in `post_id`. FC's parser picked `id`, so every + multi-image SubscribeStar post was fragmented into N Post rows. + 1. For each SubscribeStar Post, read its sidecar (via the related + ImageRecord's on-disk path), pull `post_id`, overwrite + external_post_id and post_url. + 2. Merge groups of Posts under one source that now share an + external_post_id (fragments of the same actual post). Same + ImageProvenance pre-delete + repoint dance as alembic 0022. + + hentaifoundry — sidecars have NO `url` field; `src` is the image + URL. FC's parser stored post_url=NULL. Read each HF Post's sidecar + for `user` + `index`, derive the canonical /pictures/user// + permalink. external_post_id (= `index`) was already correct. + + discord — gallery-dl wrote the CDN attachment URL in `url`. FC's + parser stored that as post_url. Read each Discord Post's sidecar + for the server/channel/message triple, derive the proper + discord.com/channels/.../ permalink. external_post_id (= + `message_id`) was already correct. + + pixiv — pure-SQL backfill: replace any `i.pximg.net`-style URL on + Post.post_url with the derived `/artworks/` permalink. Pixiv + external_post_id (= `id`) was already correct; no sidecar IO + needed. + +Idempotent: re-running on already-corrected data is a no-op (skips +rows whose derived value matches what's already stored). + +Posts whose related ImageRecord paths don't resolve on disk (orphaned +filesystem state) are skipped with a count in the migration output — +those will be picked up by a future deep-scan. +""" +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import text + +revision: str = "0025" +down_revision: Union[str, None] = "0024" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +# Mirror of sidecar._NUMBERING_PREFIX. Kept inline so the migration is +# self-contained (the operator's banked rule: +# reference_postgres_enum_swap_drop_checks.md says migrations shouldn't +# import from runtime app code). +_NUMBERING_PREFIX = re.compile(r"^\d+_(.+)$") + + +def _find_sidecar(media_path: Path) -> Path | None: + """gallery-dl writes the sidecar under the unprefixed stem + (`HOLLOW-ICHIGO.json`) while the media file gets a NN_ ordering + prefix (`01_HOLLOW-ICHIGO.png`). Try in order: + 1. .json next to the media + 2. .json next to the media (full-name variant) + 3. strip the NN_ prefix from the stem, then .json + """ + if not media_path: + return None + cand = media_path.with_suffix(".json") + if cand.is_file(): + return cand + cand = media_path.parent / f"{media_path.name}.json" + if cand.is_file(): + return cand + m = _NUMBERING_PREFIX.match(media_path.stem) + if m: + cand = media_path.parent / f"{m.group(1)}.json" + if cand.is_file(): + return cand + return None + + +def _str_id(v) -> str | None: + """str() a JSON scalar id; reject bool (JSON booleans are ints in + Python's eyes but they aren't valid sidecar ids).""" + if isinstance(v, bool): + return None + if isinstance(v, (str, int)) and str(v).strip(): + return str(v).strip() + return None + + +def _str_field(v) -> str | None: + if isinstance(v, str) and v.strip(): + return v.strip() + return None + + +def upgrade() -> None: + conn = op.get_bind() + + # ── PART 1: Per-platform corrections requiring filesystem IO ───── + # SubscribeStar, HentaiFoundry, Discord all need fields from the + # sidecar to construct the right post_url. We walk each Post's + # related ImageRecord.path to find the sidecar, read it, derive, + # and update. + targets = conn.execute(text(""" + SELECT p.id, p.external_post_id, p.post_url, s.platform + FROM post p + JOIN source s ON s.id = p.source_id + WHERE s.platform IN ('subscribestar', 'hentaifoundry', 'discord') + """)).fetchall() + + stats: dict[str, dict[str, int]] = { + plat: {"read": 0, "updated": 0, "no_sidecar": 0} + for plat in ("subscribestar", "hentaifoundry", "discord") + } + for post_row in targets: + plat = post_row.platform + path = _first_attachment_path(conn, post_row.id) + if not path: + stats[plat]["no_sidecar"] += 1 + continue + sidecar = _find_sidecar(Path(path)) + if sidecar is None: + stats[plat]["no_sidecar"] += 1 + continue + try: + data = json.loads(sidecar.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + stats[plat]["no_sidecar"] += 1 + continue + stats[plat]["read"] += 1 + + new_epid = post_row.external_post_id + new_url = None + if plat == "subscribestar": + pid = _str_id(data.get("post_id")) + if pid: + new_epid = pid + new_url = f"https://www.subscribestar.com/posts/{pid}" + elif plat == "hentaifoundry": + user = _str_field(data.get("user")) or _str_field(data.get("artist")) + idx = _str_id(data.get("index")) + if user and idx: + new_url = f"https://www.hentai-foundry.com/pictures/user/{user}/{idx}" + elif plat == "discord": + sid = _str_id(data.get("server_id")) + cid = _str_id(data.get("channel_id")) + mid = _str_id(data.get("message_id")) + if sid and cid and mid: + new_url = f"https://discord.com/channels/{sid}/{cid}/{mid}" + + # Idempotent: skip if nothing changed. + if new_epid == post_row.external_post_id and new_url == post_row.post_url: + continue + conn.execute( + text(""" + UPDATE post + SET external_post_id = :epid, post_url = :url + WHERE id = :id + """), + {"epid": new_epid, "url": new_url, "id": post_row.id}, + ) + stats[plat]["updated"] += 1 + + for plat, s in stats.items(): + print( + f"0025: {plat} — read {s['read']} sidecars, " + f"updated {s['updated']} Posts, " + f"{s['no_sidecar']} Posts had no resolvable sidecar" + ) + + # ── PART 2: Merge SubscribeStar fragments now sharing epid ─────── + # After Part 1, each group of Posts under one source with the SAME + # new external_post_id is a fragment-set of the same actual post. + # Merge to one canonical row. Pre-handle the same ImageProvenance + # collision pattern as alembic 0022 (uq_image_provenance_image_post). + fragment_groups = conn.execute(text(""" + SELECT p.source_id, p.external_post_id, + ARRAY_AGG(p.id ORDER BY p.id ASC) AS post_ids + FROM post p + JOIN source s ON s.id = p.source_id + WHERE s.platform = 'subscribestar' + AND p.external_post_id IS NOT NULL + GROUP BY p.source_id, p.external_post_id + HAVING COUNT(*) > 1 + """)).fetchall() + + merged = 0 + for grp in fragment_groups: + post_ids = list(grp.post_ids) + keep_id, *drop_ids = post_ids + for drop_id in drop_ids: + # Pre-DELETE colliding ImageProvenance under drop_ that + # already exist under keep (alembic 0022 banked the pattern). + conn.execute( + text(""" + DELETE FROM image_provenance + WHERE post_id = :drop_ + AND image_record_id IN ( + SELECT image_record_id FROM image_provenance + WHERE post_id = :keep + ) + """), + {"keep": keep_id, "drop_": drop_id}, + ) + conn.execute( + text(""" + UPDATE image_provenance SET post_id = :keep + WHERE post_id = :drop_ + """), + {"keep": keep_id, "drop_": drop_id}, + ) + conn.execute( + text(""" + UPDATE image_record SET primary_post_id = :keep + WHERE primary_post_id = :drop_ + """), + {"keep": keep_id, "drop_": drop_id}, + ) + conn.execute( + text(""" + UPDATE post_attachment SET post_id = :keep + WHERE post_id = :drop_ + """), + {"keep": keep_id, "drop_": drop_id}, + ) + conn.execute( + text("DELETE FROM post WHERE id = :drop_"), + {"drop_": drop_id}, + ) + merged += 1 + print(f"0025: subscribestar — merged {merged} duplicate Post fragments") + + # ── PART 3: Pixiv post_url backfill (pure SQL) ─────────────────── + # Pixiv's external_post_id is already correct (gallery-dl's `id` is + # the post id). Only post_url needs derivation: replace anything + # under i.pximg.net (the file URL) with the /artworks/ permalink. + pixiv_updated = conn.execute(text(""" + UPDATE post p + SET post_url = 'https://www.pixiv.net/artworks/' || p.external_post_id + FROM source s + WHERE p.source_id = s.id + AND s.platform = 'pixiv' + AND p.external_post_id IS NOT NULL + AND (p.post_url IS NULL + OR p.post_url LIKE 'https://i.pximg.net/%' + OR p.post_url LIKE 'http://i.pximg.net/%') + """)).rowcount + print(f"0025: pixiv — backfilled post_url on {pixiv_updated} Posts") + + +def _first_attachment_path(conn, post_id: int) -> str | None: + """Return any ImageRecord.path attached to this post (via + ImageProvenance). Lowest-id row keeps the migration deterministic + so re-running on the same DB picks the same sidecar.""" + row = conn.execute( + text(""" + SELECT ir.path + FROM image_provenance ip + JOIN image_record ir ON ir.id = ip.image_record_id + WHERE ip.post_id = :pid + ORDER BY ip.id ASC + LIMIT 1 + """), + {"pid": post_id}, + ).first() + return row[0] if row else None + + +def downgrade() -> None: + # Lossy: external_post_id values were overwritten with the correct + # post_id; original per-attachment ids weren't preserved. Post-merge + # also deleted drop rows. No safe restore. To roll back the schema + # invariant, fork from 0024 and re-run sidecar imports. + pass diff --git a/backend/app/api/downloads.py b/backend/app/api/downloads.py index cac9015..e7a86c9 100644 --- a/backend/app/api/downloads.py +++ b/backend/app/api/downloads.py @@ -5,8 +5,10 @@ status/source/artist. Returns slim records. Detail view: full DownloadEvent including the metadata JSONB. """ +from datetime import UTC, datetime, timedelta + from quart import Blueprint, jsonify, request -from sqlalchemy import select +from sqlalchemy import func, select from ..extensions import get_session from ..models import Artist, DownloadEvent, Source @@ -95,6 +97,35 @@ async def list_downloads(): return jsonify([_list_record(e, s, a) for e, s, a in rows]) +@downloads_bp.route("/stats", methods=["GET"]) +async def downloads_stats(): + """Status-grouped count over download_event for the dashboard stat chips. + + `?window_hours=` (default 24) bounds by `started_at`. The full set of + statuses is always present in the response (zero for missing) so the + UI doesn't have to fill in defaults. + """ + try: + window_hours = int(request.args.get("window_hours", "24")) + except ValueError: + return jsonify({"error": "invalid_window_hours"}), 400 + if window_hours < 1 or window_hours > 24 * 365: + return jsonify({"error": "invalid_window_hours"}), 400 + + since = datetime.now(UTC) - timedelta(hours=window_hours) + out = {"pending": 0, "running": 0, "ok": 0, "error": 0, "skipped": 0} + async with get_session() as session: + stmt = ( + select(DownloadEvent.status, func.count()) + .where(DownloadEvent.started_at >= since) + .group_by(DownloadEvent.status) + ) + for status, n in (await session.execute(stmt)).all(): + if status in out: + out[status] = int(n) + return jsonify(out) + + @downloads_bp.route("/", methods=["GET"]) async def get_download(event_id: int): async with get_session() as session: diff --git a/backend/app/utils/sidecar.py b/backend/app/utils/sidecar.py index 8df78b4..13ced7c 100644 --- a/backend/app/utils/sidecar.py +++ b/backend/app/utils/sidecar.py @@ -55,6 +55,33 @@ def _first_str(data: dict, keys: tuple[str, ...]) -> str | None: return None +# Strip HTML tags + collapse whitespace + take the first non-empty line. +# Used to derive a display title from a body when the platform doesn't +# expose a separate title field (subscribestar posts always write +# `title: ""` and put the leading sentence inside `content` as HTML). +# Truncated to 120 chars with an ellipsis if longer — long enough to be +# meaningful in a feed, short enough to fit a row. +_TAG_RE = re.compile(r"<[^>]+>") +_WS_RE = re.compile(r"\s+") + + +def _first_line_text(body: str, limit: int = 120) -> str | None: + if not body: + return None + text = _TAG_RE.sub(" ", body) + text = text.replace("\xa0", " ") + # Split on hard line breaks first; the body-stripped HTML often + # collapses to one logical line, in which case the first sentence + # split is the next-best heuristic. + for line in text.splitlines(): + line = _WS_RE.sub(" ", line).strip() + if line: + if len(line) > limit: + return line[: limit - 1].rstrip() + "…" + return line + return None + + def _parse_date(v) -> datetime | None: if isinstance(v, bool): return None @@ -84,8 +111,16 @@ def parse_sidecar(data: dict) -> SidecarData: cat = data.get("category") platform = cat if isinstance(cat, str) and cat.strip() else None + # external_post_id lookup order: post_id MUST come before id. + # SubscribeStar gallery-dl writes the per-attachment id in `id` + # (e.g. 711509) and the actual post id in `post_id` (e.g. 360360); + # picking `id` first fragments every multi-image subscribestar post + # into N distinct Post rows in FC. Patreon/Pixiv have no `post_id` + # so `id` still wins for them; HF uses `index`, Discord uses + # `message_id` — all reached via the remaining chain entries. + # Operator-flagged 2026-05-27 during the sidecar audit. external_post_id = None - for k in ("id", "post_id", "index", "message_id"): + for k in ("post_id", "id", "index", "message_id"): v = data.get(k) if isinstance(v, bool): continue @@ -111,13 +146,76 @@ def parse_sidecar(data: dict) -> SidecarData: if post_date is not None: break + # `message` is Discord gallery-dl's body field (no `content`); added + # 2026-05-27 to the description fallback chain. + description = _first_str( + data, ("content", "description", "caption", "message"), + ) + + # SubscribeStar posts always write `title: ""` and put the leading + # sentence inside `content` (confirmed against the operator's + # /mnt/Data/Patreon/Cheunart/subscribestar/ dump 2026-05-27). When + # no explicit title is present, synthesize one from the description + # body's first non-empty line. Patreon retains its explicit titles + # because they're non-empty and short-circuit the fallback. + post_title = _first_str(data, ("title",)) + if post_title is None and description: + post_title = _first_line_text(description) + + # post_url derivation: SubscribeStar/Pixiv/HF/Discord put the FILE + # download URL in `url`, not a post permalink. Synthesize the + # permalink from per-platform fields when possible. Patreon's `url` + # IS a permalink and is used as-is. For the four file-URL platforms, + # the bare `url` is NEVER trusted — derive or return None rather + # than persist a CDN URL in post.post_url. + if platform in _DERIVED_URL_PLATFORMS: + post_url = _derive_post_url(platform, data) + else: + post_url = _first_str(data, ("url", "post_url")) + return SidecarData( platform=platform, external_post_id=external_post_id, - post_url=_first_str(data, ("url", "post_url")), - post_title=_first_str(data, ("title",)), - description=_first_str(data, ("content", "description", "caption")), + post_url=post_url, + post_title=post_title, + description=description, attachment_count=attachment_count, post_date=post_date, raw=data, ) + + +_DERIVED_URL_PLATFORMS = frozenset({ + "subscribestar", "pixiv", "hentaifoundry", "discord", +}) + + +def _derive_post_url(platform: str, data: dict) -> str | None: + """Synthesize the post-permalink URL from per-platform metadata. + + gallery-dl writes the file-download URL in `url` for these four + platforms; we need a real permalink for the PostCard "open original" + button. Returns None if the platform-specific fields are missing + (rare in well-formed sidecars but defensive). + """ + if platform == "subscribestar": + pid = data.get("post_id") + if isinstance(pid, (str, int)) and str(pid).strip(): + return f"https://www.subscribestar.com/posts/{pid}" + elif platform == "pixiv": + pid = data.get("id") + if isinstance(pid, (str, int)) and not isinstance(pid, bool) and str(pid).strip(): + return f"https://www.pixiv.net/artworks/{pid}" + elif platform == "hentaifoundry": + user = _first_str(data, ("user", "artist")) + idx = data.get("index") + if user and isinstance(idx, (str, int)) and not isinstance(idx, bool) and str(idx).strip(): + return f"https://www.hentai-foundry.com/pictures/user/{user}/{idx}" + elif platform == "discord": + sid = data.get("server_id") + cid = data.get("channel_id") + mid = data.get("message_id") + if all(isinstance(v, (str, int)) and not isinstance(v, bool) and str(v).strip() + for v in (sid, cid, mid)): + return f"https://discord.com/channels/{sid}/{cid}/{mid}" + return None diff --git a/frontend/src/App.vue b/frontend/src/App.vue index 38a3ccf..5cfbdda 100644 --- a/frontend/src/App.vue +++ b/frontend/src/App.vue @@ -4,7 +4,6 @@ - @@ -14,7 +13,6 @@ import { onMounted, ref } from 'vue' import AppShell from './components/AppShell.vue' import AppSnackbar from './components/AppSnackbar.vue' import ImageViewer from './components/modal/ImageViewer.vue' -import PostModal from './components/posts/PostModal.vue' import { useModalStore } from './stores/modal.js' const modal = useModalStore() diff --git a/frontend/src/components/posts/PostCard.vue b/frontend/src/components/posts/PostCard.vue index c80a0ab..65194a0 100644 --- a/frontend/src/components/posts/PostCard.vue +++ b/frontend/src/components/posts/PostCard.vue @@ -1,8 +1,8 @@