From 0978fbac666f37c18262e3a33d03856161f59cc4 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 26 May 2026 00:27:43 -0400 Subject: [PATCH 1/2] =?UTF-8?q?fix(sidecar):=20strip=20gallery-dl=20'NN=5F?= =?UTF-8?q?'=20numbering=20prefix=20when=20locating=20sidecars=20=E2=80=94?= =?UTF-8?q?=20fixes=20'deep=20scan=20refresh=20count=20high=20but=200=20Po?= =?UTF-8?q?sts=20created'=20(operator-flagged=202026-05-26)=20=E2=80=94=20?= =?UTF-8?q?Co-Authored-By:=20Claude=20Opus=204.7=20(1M=20context)=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/utils/sidecar.py | 19 +++++++++++++++ tests/test_sidecar_util.py | 45 ++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/backend/app/utils/sidecar.py b/backend/app/utils/sidecar.py index ae35b28..8df78b4 100644 --- a/backend/app/utils/sidecar.py +++ b/backend/app/utils/sidecar.py @@ -4,6 +4,7 @@ No per-platform branching: a small common key set with fallbacks; the full JSON is kept in raw so anything unmapped is recoverable later. """ +import re from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path @@ -21,10 +22,28 @@ class SidecarData: raw: dict +# gallery-dl prefixes media filenames with `NN_` for in-post ordering +# (`01_HOLLOW-ICHIGO.png`, `02_HOLOW ICHIGO.zip`) but writes the sidecar +# under the attachment's stem WITHOUT that ordering prefix +# (`HOLLOW-ICHIGO.json`). Strip the prefix when looking for sidecars. +# Confirmed against real Patreon downloads 2026-05-26 — without this +# strip, every gallery-dl post-level sidecar was invisible to FC since +# FC-3 shipped (24 deep-refresh calls produced 0 Posts in operator's DB). +_NUMBERING_PREFIX = re.compile(r"^\d+_(.+)$") + + def find_sidecar(media: Path) -> Path | None: + # Attachment-level sidecars (image.jpg.json, image.json). for cand in (media.with_suffix(".json"), Path(str(media) + ".json")): if cand.is_file(): return cand + # gallery-dl post-numbered convention: strip the `NN_` prefix from + # the stem and look for that.json in the same directory. + m = _NUMBERING_PREFIX.match(media.stem) + if m: + cand = media.parent / f"{m.group(1)}.json" + if cand.is_file(): + return cand return None diff --git a/tests/test_sidecar_util.py b/tests/test_sidecar_util.py index 0fd3e40..3abff31 100644 --- a/tests/test_sidecar_util.py +++ b/tests/test_sidecar_util.py @@ -23,6 +23,51 @@ def test_find_sidecar_none(tmp_path): assert find_sidecar(media) is None +def test_find_sidecar_gallerydl_numbered_prefix(tmp_path): + """gallery-dl prefixes media filenames with NN_ for in-post ordering + (e.g. `01_HOLLOW-ICHIGO.png`) but writes the post-level sidecar under + the attachment stem WITHOUT the prefix (`HOLLOW-ICHIGO.json`). + Confirmed against real Patreon downloads 2026-05-26 — operator's deep + scan produced 24 refresh calls but 0 Posts because the unprefixed + sidecar was invisible to find_sidecar.""" + media = tmp_path / "01_HOLLOW-ICHIGO.png" + media.write_bytes(b"x") + sc = tmp_path / "HOLLOW-ICHIGO.json" + sc.write_text("{}") + assert find_sidecar(media) == sc + + +def test_find_sidecar_multidigit_prefix(tmp_path): + """Numbering prefix can be wider than 2 digits (`001_...`); the strip + handles any \\d+_ form.""" + media = tmp_path / "001_mirko-sketch.png" + media.write_bytes(b"x") + sc = tmp_path / "mirko-sketch.json" + sc.write_text("{}") + assert find_sidecar(media) == sc + + +def test_find_sidecar_prefers_attachment_level_over_post_level(tmp_path): + """If BOTH a per-attachment sidecar and a post-level sidecar exist, + the attachment-level one wins (it's more specific).""" + media = tmp_path / "01_image.png" + media.write_bytes(b"x") + per_attachment = tmp_path / "01_image.json" + per_attachment.write_text('{"specific": true}') + post_level = tmp_path / "image.json" + post_level.write_text('{"specific": false}') + assert find_sidecar(media) == per_attachment + + +def test_find_sidecar_no_underscore_not_treated_as_prefix(tmp_path): + """`01.png` (just digits, no underscore-separated stem) shouldn't + match. The regex requires NN_.""" + media = tmp_path / "01.png" + media.write_bytes(b"x") + (tmp_path / ".json").write_text("{}") # would be matched only if buggy + assert find_sidecar(media) is None + + def test_parse_empty_dict_all_none(): sd = parse_sidecar({}) assert isinstance(sd, SidecarData) From 06a2f60c082033478db66738f2d1f0d6502283e3 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Tue, 26 May 2026 00:42:35 -0400 Subject: [PATCH 2/2] =?UTF-8?q?fix(ext-ci):=20use=20browser=5Fdownload=5Fu?= =?UTF-8?q?rl=20not=20/releases/assets/=20+=20add=20-f=20to=20curl=20+?= =?UTF-8?q?=20magic-byte=20sanity=20check=20(operator-flagged=202026-05-26?= =?UTF-8?q?:=20prior=20build=20silently=20wrote=20'404=20page=20not=20foun?= =?UTF-8?q?d'=20into=20the=20XPI=20file,=20Firefox=20rejected=20as=20corru?= =?UTF-8?q?pt)=20=E2=80=94=20Co-Authored-By:=20Claude=20Opus=204.7=20(1M?= =?UTF-8?q?=20context)=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .forgejo/workflows/build.yml | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 898419b..79b6b85 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -165,16 +165,35 @@ jobs: run: | set -eux VERSION=$(grep -E '"version"' extension/package.json | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/') - # Look up the ext- release + its .xpi asset id. + # Look up the ext- release; extract the .xpi asset's + # browser_download_url (Forgejo's /releases/assets/ endpoint + # returns ASSET METADATA, not the binary blob — operator-flagged + # 2026-05-26: my prior code curl'd the metadata endpoint without + # -f and wrote the resulting 404-page-not-found text into + # fabledcurator-*.xpi, which Firefox then rejected as "corrupt"). + # browser_download_url is the canonical binary endpoint and is + # also publicly accessible (no token needed) but we pass the + # token anyway for symmetry with private-repo support. curl -sf -H "Authorization: token $TOKEN" \ "https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" \ -o release.json - ASSET_ID=$(python3 -c "import json; r=json.load(open('release.json')); xpis=[a for a in r.get('assets', []) if a.get('name','').endswith('.xpi')]; print(xpis[0]['id'])") - test -n "$ASSET_ID" + DOWNLOAD_URL=$(python3 -c "import json; r=json.load(open('release.json')); xpis=[a for a in r.get('assets', []) if a.get('name','').endswith('.xpi')]; print(xpis[0]['browser_download_url'])") + test -n "$DOWNLOAD_URL" + echo "Downloading XPI from: $DOWNLOAD_URL" mkdir -p frontend/public/extension DEST="frontend/public/extension/fabledcurator-$VERSION.xpi" - curl -sL -H "Authorization: token $TOKEN" -o "$DEST" \ - "https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/assets/$ASSET_ID" + # -f = fail on HTTP error (prevents silent corruption like the + # 2026-05-26 incident); -L = follow redirects. + curl -sfL -H "Authorization: token $TOKEN" -o "$DEST" "$DOWNLOAD_URL" + # Sanity check: the binary should start with the ZIP magic (PK\x03\x04). + # If it's anything else, the next docker build will ship a corrupt XPI. + MAGIC=$(head -c 2 "$DEST" | od -An -c | tr -d ' \n') + if [ "$MAGIC" != "PK" ]; then + echo "ERROR: downloaded XPI does not start with ZIP magic 'PK' (got '$MAGIC')" + echo "File contents preview:" + head -c 200 "$DEST" + exit 1 + fi cp "$DEST" "frontend/public/extension/fabledcurator-latest.xpi" ls -la frontend/public/extension/