Merge pull request 'fix(ext-ci): use browser_download_url + curl -f + ZIP magic check (XPI silently corrupt)' (#20) from dev into main

.forgejo/workflows/build.yml

@@ -49,9 +49,13 @@ jobs:
             -H "Authorization: token $TOKEN" \
             "https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" || echo 000)
           echo "Tag lookup HTTP status: $STATUS"
+          # JSON parsing via python (ci-python:3.14 has stdlib json; jq is
+          # not in the image and adding it per ci-requirements.md is not
+          # warranted for a single consumer — operator-flagged 2026-05-26
+          # after a sign job failed with `jq: not found`).
           if [ "$STATUS" = "200" ]; then
-            ASSET_ID=$(jq -r '.assets[]? | select(.name | test("\\.xpi$")) | .id' release.json | head -1)
-            if [ -n "$ASSET_ID" ] && [ "$ASSET_ID" != "null" ]; then
+            ASSET_ID=$(python3 -c "import json; r=json.load(open('release.json')); xpis=[a for a in r.get('assets', []) if a.get('name','').endswith('.xpi')]; print(xpis[0]['id'] if xpis else '')")
+            if [ -n "$ASSET_ID" ]; then
               echo "cached=true" >> "$GITHUB_OUTPUT"
               echo "asset_id=$ASSET_ID" >> "$GITHUB_OUTPUT"
               echo "Cached XPI exists at ext-$VERSION (asset id $ASSET_ID); skipping AMO sign"
@@ -64,17 +68,11 @@ jobs:
             echo "No release named ext-$VERSION; will sign via AMO and upload"
           fi
 
-      - name: Download cached signed XPI (cache hit)
-        if: steps.cache.outputs.cached == 'true'
-        env:
-          TOKEN: ${{ secrets.RELEASE_TOKEN }}
-        run: |
-          set -eu
-          mkdir -p extension/web-ext-artifacts
-          curl -sL -H "Authorization: token $TOKEN" \
-            -o "extension/web-ext-artifacts/fabledcurator-${{ steps.extver.outputs.version }}.xpi" \
-            "https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/assets/${{ steps.cache.outputs.asset_id }}"
-          ls -la extension/web-ext-artifacts/
+      # No "download cached XPI in sign-extension" step: build-web
+      # fetches directly from the Forgejo ext-<version> release asset
+      # (removed 2026-05-26 alongside the actions/upload-artifact
+      # removal — sign-extension's job is just to ensure the cache
+      # exists on Forgejo; the build-web side reads it independently).
 
       - name: Sign via AMO (cache miss)
         if: steps.cache.outputs.cached != 'true'
@@ -98,29 +96,57 @@ jobs:
           SIGNED=$(ls extension/web-ext-artifacts/*.xpi | head -1)
           XPI="extension/web-ext-artifacts/fabledcurator-$VERSION.xpi"
           cp "$SIGNED" "$XPI"
-          # Find-or-create the ext-<version> release.
+          # Find-or-create the ext-<version> release. Track whether WE
+          # created it so an upload failure below can roll back (don't
+          # leave an empty release tombstone that the next run's
+          # cache-check mistakes for a partial-failure state).
           STATUS=$(curl -s -o release.json -w "%{http_code}" \
             -H "Authorization: token $TOKEN" \
             "https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" || echo 000)
-          if [ "$STATUS" != "200" ]; then
+          if [ "$STATUS" = "200" ]; then
+            CREATED_BY_US=false
+          else
             curl -s -X POST -H "Authorization: token $TOKEN" -H "Content-Type: application/json" \
               -d "{\"tag_name\":\"ext-$VERSION\",\"name\":\"Extension $VERSION (signed XPI cache)\",\"body\":\"Internal cache for the signed XPI consumed by build.yml's build-web job. Not a user-facing FC release.\",\"target_commitish\":\"main\"}" \
               -o release.json \
               "https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases"
+            CREATED_BY_US=true
           fi
-          RELEASE_ID=$(jq -r '.id' release.json)
-          test -n "$RELEASE_ID" && test "$RELEASE_ID" != "null"
-          curl -s -X POST -H "Authorization: token $TOKEN" \
+          RELEASE_ID=$(python3 -c "import json; print(json.load(open('release.json'))['id'])")
+          test -n "$RELEASE_ID"
+          # Rollback-on-failure: if the asset upload fails AND we just
+          # created the release in this run, delete it. Prevents an empty
+          # ext-<version> release from poisoning the next workflow run
+          # (operator-flagged 2026-05-26 — without rollback the next run
+          # saw 'release exists, no asset → cache miss → sign' which AMO
+          # then rejected with 409 'Version already exists').
+          rollback_if_we_created() {
+            if [ "$CREATED_BY_US" = "true" ]; then
+              echo "Rolling back: deleting just-created release $RELEASE_ID"
+              curl -s -X DELETE -H "Authorization: token $TOKEN" \
+                "https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/$RELEASE_ID" || true
+              curl -s -X DELETE -H "Authorization: token $TOKEN" \
+                "https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/tags/ext-$VERSION" || true
+            fi
+          }
+          trap 'rollback_if_we_created' EXIT
+          HTTP_CODE=$(curl -s -X POST -H "Authorization: token $TOKEN" \
             -F "attachment=@$XPI" \
-            "https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/$RELEASE_ID/assets?name=fabledcurator-$VERSION.xpi"
+            -o /dev/null -w "%{http_code}" \
+            "https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/$RELEASE_ID/assets?name=fabledcurator-$VERSION.xpi")
+          if [ "$HTTP_CODE" != "201" ] && [ "$HTTP_CODE" != "200" ]; then
+            echo "Asset upload failed with HTTP $HTTP_CODE"
+            exit 1
+          fi
+          # Upload succeeded — clear the rollback trap.
+          trap - EXIT
           echo "Uploaded fabledcurator-$VERSION.xpi to ext-$VERSION release"
 
-      - name: Upload XPI as Actions artifact (handoff to build-web)
-        uses: actions/upload-artifact@v4
-        with:
-          name: signed-extension
-          path: extension/web-ext-artifacts/fabledcurator-*.xpi
-          retention-days: 1
+      # No actions/upload-artifact step: Forgejo Actions (and our
+       # act_runner) doesn't support upload-artifact@v4+ (GHES limitation
+       # surfaced 2026-05-26). Instead build-web reads the signed XPI
+       # straight from the ext-<version> Forgejo release we just uploaded
+       # to. Same source of truth; no double-store.
 
   build-web:
     needs: [sign-extension]
@@ -132,22 +158,43 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Download signed extension (main only)
-        if: github.ref == 'refs/heads/main'
-        uses: actions/download-artifact@v4
-        with:
-          name: signed-extension
-          path: extension/web-ext-artifacts/
-
-      - name: Place signed XPI in build context (main only)
+      - name: Download signed XPI from Forgejo release asset (main only)
         if: github.ref == 'refs/heads/main'
+        env:
+          TOKEN: ${{ secrets.RELEASE_TOKEN }}
         run: |
           set -eux
           VERSION=$(grep -E '"version"' extension/package.json | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')
-          XPI=$(ls extension/web-ext-artifacts/*.xpi | head -1)
+          # Look up the ext-<version> release; extract the .xpi asset's
+          # browser_download_url (Forgejo's /releases/assets/<id> endpoint
+          # returns ASSET METADATA, not the binary blob — operator-flagged
+          # 2026-05-26: my prior code curl'd the metadata endpoint without
+          # -f and wrote the resulting 404-page-not-found text into
+          # fabledcurator-*.xpi, which Firefox then rejected as "corrupt").
+          # browser_download_url is the canonical binary endpoint and is
+          # also publicly accessible (no token needed) but we pass the
+          # token anyway for symmetry with private-repo support.
+          curl -sf -H "Authorization: token $TOKEN" \
+            "https://git.fabledsword.com/api/v1/repos/bvandeusen/FabledCurator/releases/tags/ext-$VERSION" \
+            -o release.json
+          DOWNLOAD_URL=$(python3 -c "import json; r=json.load(open('release.json')); xpis=[a for a in r.get('assets', []) if a.get('name','').endswith('.xpi')]; print(xpis[0]['browser_download_url'])")
+          test -n "$DOWNLOAD_URL"
+          echo "Downloading XPI from: $DOWNLOAD_URL"
           mkdir -p frontend/public/extension
-          cp "$XPI" "frontend/public/extension/fabledcurator-$VERSION.xpi"
-          cp "$XPI" "frontend/public/extension/fabledcurator-latest.xpi"
+          DEST="frontend/public/extension/fabledcurator-$VERSION.xpi"
+          # -f = fail on HTTP error (prevents silent corruption like the
+          # 2026-05-26 incident); -L = follow redirects.
+          curl -sfL -H "Authorization: token $TOKEN" -o "$DEST" "$DOWNLOAD_URL"
+          # Sanity check: the binary should start with the ZIP magic (PK\x03\x04).
+          # If it's anything else, the next docker build will ship a corrupt XPI.
+          MAGIC=$(head -c 2 "$DEST" | od -An -c | tr -d ' \n')
+          if [ "$MAGIC" != "PK" ]; then
+            echo "ERROR: downloaded XPI does not start with ZIP magic 'PK' (got '$MAGIC')"
+            echo "File contents preview:"
+            head -c 200 "$DEST"
+            exit 1
+          fi
+          cp "$DEST" "frontend/public/extension/fabledcurator-latest.xpi"
           ls -la frontend/public/extension/
 
       - name: Determine tag

backend/app/utils/sidecar.py

@@ -4,6 +4,7 @@ No per-platform branching: a small common key set with fallbacks; the
 full JSON is kept in raw so anything unmapped is recoverable later.
 """
 
+import re
 from dataclasses import dataclass
 from datetime import UTC, datetime
 from pathlib import Path
@@ -21,10 +22,28 @@ class SidecarData:
     raw: dict
 
 
+# gallery-dl prefixes media filenames with `NN_` for in-post ordering
+# (`01_HOLLOW-ICHIGO.png`, `02_HOLOW ICHIGO.zip`) but writes the sidecar
+# under the attachment's stem WITHOUT that ordering prefix
+# (`HOLLOW-ICHIGO.json`). Strip the prefix when looking for sidecars.
+# Confirmed against real Patreon downloads 2026-05-26 — without this
+# strip, every gallery-dl post-level sidecar was invisible to FC since
+# FC-3 shipped (24 deep-refresh calls produced 0 Posts in operator's DB).
+_NUMBERING_PREFIX = re.compile(r"^\d+_(.+)$")
+
+
 def find_sidecar(media: Path) -> Path | None:
+    # Attachment-level sidecars (image.jpg.json, image.json).
     for cand in (media.with_suffix(".json"), Path(str(media) + ".json")):
         if cand.is_file():
             return cand
+    # gallery-dl post-numbered convention: strip the `NN_` prefix from
+    # the stem and look for that.json in the same directory.
+    m = _NUMBERING_PREFIX.match(media.stem)
+    if m:
+        cand = media.parent / f"{m.group(1)}.json"
+        if cand.is_file():
+            return cand
     return None
 
 

extension/manifest.json

@@ -1,7 +1,7 @@
 {
   "manifest_version": 3,
   "name": "FabledCurator",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "description": "Export cookies from supported platforms to FabledCurator and add creators as sources in one click.",
 
   "browser_specific_settings": {

extension/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fabledcurator-extension",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "private": true,
   "description": "Firefox extension for FabledCurator",
   "scripts": {

tests/test_sidecar_util.py

@@ -23,6 +23,51 @@ def test_find_sidecar_none(tmp_path):
     assert find_sidecar(media) is None
 
 
+def test_find_sidecar_gallerydl_numbered_prefix(tmp_path):
+    """gallery-dl prefixes media filenames with NN_ for in-post ordering
+    (e.g. `01_HOLLOW-ICHIGO.png`) but writes the post-level sidecar under
+    the attachment stem WITHOUT the prefix (`HOLLOW-ICHIGO.json`).
+    Confirmed against real Patreon downloads 2026-05-26 — operator's deep
+    scan produced 24 refresh calls but 0 Posts because the unprefixed
+    sidecar was invisible to find_sidecar."""
+    media = tmp_path / "01_HOLLOW-ICHIGO.png"
+    media.write_bytes(b"x")
+    sc = tmp_path / "HOLLOW-ICHIGO.json"
+    sc.write_text("{}")
+    assert find_sidecar(media) == sc
+
+
+def test_find_sidecar_multidigit_prefix(tmp_path):
+    """Numbering prefix can be wider than 2 digits (`001_...`); the strip
+    handles any \\d+_ form."""
+    media = tmp_path / "001_mirko-sketch.png"
+    media.write_bytes(b"x")
+    sc = tmp_path / "mirko-sketch.json"
+    sc.write_text("{}")
+    assert find_sidecar(media) == sc
+
+
+def test_find_sidecar_prefers_attachment_level_over_post_level(tmp_path):
+    """If BOTH a per-attachment sidecar and a post-level sidecar exist,
+    the attachment-level one wins (it's more specific)."""
+    media = tmp_path / "01_image.png"
+    media.write_bytes(b"x")
+    per_attachment = tmp_path / "01_image.json"
+    per_attachment.write_text('{"specific": true}')
+    post_level = tmp_path / "image.json"
+    post_level.write_text('{"specific": false}')
+    assert find_sidecar(media) == per_attachment
+
+
+def test_find_sidecar_no_underscore_not_treated_as_prefix(tmp_path):
+    """`01.png` (just digits, no underscore-separated stem) shouldn't
+    match. The regex requires NN_<something>."""
+    media = tmp_path / "01.png"
+    media.write_bytes(b"x")
+    (tmp_path / ".json").write_text("{}")  # would be matched only if buggy
+    assert find_sidecar(media) is None
+
+
 def test_parse_empty_dict_all_none():
     sd = parse_sidecar({})
     assert isinstance(sd, SidecarData)