Merge pull request 'Release v26.05.25.4 — importer ext sanitize fix, CI shard split, BrowserExtensionCard on Overview' (#15) from dev into main

2026-05-25 21:11:50 -04:00
parent ac55d0e8d8 ac39509a74
commit b214460fdb
6 changed files with 259 additions and 37 deletions
@@ -72,17 +72,28 @@ jobs:
      - run: npm run test:unit
      - run: npm run build

-  integration:
-    # This act_runner (swarm-runner v0.6.1) puts service containers on the
-    # default bridge with NO service-name DNS, and publishing fixed host
-    # ports collides with the operator's running docker-compose dev stack on
-    # the same shared daemon. Workaround: publish NO host ports, and reach
-    # each service by its bridge IP — discovered at runtime via the mounted
-    # docker socket (the ci-python image ships /usr/bin/docker). Default-bridge
-    # containers can talk by IP (only embedded DNS is missing), so IP
-    # addressing is reliable here. Everything runs in ONE step so resolved
-    # values don't depend on cross-step env passing. Pattern documented in
-    # FabledRulebook/forgejo.md "CI philosophy".
+  # Integration suite split into THREE parallel shards (2026-05-25, runner
+  # capacity bumped 2→6). Each shard gets its own Postgres + Redis service
+  # set and runs alembic + a disjoint subset of integration tests. Shards
+  # share no DB state, so the autouse TRUNCATE fixture in tests/conftest.py
+  # stays single-threaded per shard but multiple shards run in parallel
+  # wall-clock. Approximate split — rebalance once --durations=15 output
+  # reveals which shard is the long pole.
+  #
+  # Each shard's docker-ps filter uses its own unique job name to scope
+  # service-container resolution. act_runner appears to strip underscores
+  # from job names when building container labels — `int_api` yielded
+  # zero matches on 2026-05-25 — so shards use no-separator names
+  # (`intapi`, `intimp`, `intcore`) instead. Each step prints
+  # `docker ps -a` first so a future naming-convention shift surfaces in
+  # the log without another guess-and-push cycle.
+  #
+  # Pre-baking requirements.txt into ci-python:3.14 is intentionally NOT
+  # done — per ci-requirements.md, FC is the only Python consumer of that
+  # image and the CI-Runner project's "add deps to image when used by >1
+  # project" rule keeps the install per-job.
+
+  intapi:
    runs-on: python-ci
    container:
      image: git.fabledsword.com/bvandeusen/ci-python:3.14
@@ -113,7 +124,6 @@ jobs:
          --health-retries 10
    steps:
      - uses: actions/checkout@v4
-
      - name: Cache pip wheels
        uses: actions/cache@v4
        with:
@@ -121,15 +131,14 @@ jobs:
          key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
          restore-keys: |
            pip-${{ runner.os }}-py314-
-
-      - name: Integration suite (resolve service IPs, migrate, test)
+      - name: API integration shard (resolve service IPs, migrate, test)
        run: |
          set -eux
-          # Scope to THIS job's service containers (act_runner names them
-          # ...JOB-integration...); the operator's compose stack uses the
-          # same images but different names, so it won't match.
-          PG=$(docker ps --filter "name=JOB-integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
-          RD=$(docker ps --filter "name=JOB-integration" --filter "ancestor=redis:7-alpine" -q | head -n1)
+          echo "=== container landscape (diagnostic for filter scoping) ==="
+          docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
+          echo "=== end landscape ==="
+          PG=$(docker ps --filter "name=intapi" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
+          RD=$(docker ps --filter "name=intapi" --filter "ancestor=redis:7-alpine" -q | head -n1)
          test -n "$PG" && test -n "$RD"
          PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
          RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
@@ -137,16 +146,153 @@ jobs:
          export DB_HOST="$PG_IP"
          export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
          export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
-          # Wait for Postgres to accept TCP (bash /dev/tcp; no extra tools).
          for i in $(seq 1 60); do
            (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
            sleep 2
          done
-          # uv when available (5-10x faster wheel resolve); fall back to pip.
          if command -v uv >/dev/null 2>&1; then
            uv pip install --system -r requirements.txt pytest pytest-asyncio
          else
            pip install -r requirements.txt pytest pytest-asyncio
          fi
          alembic upgrade head
-          pytest tests/ -v -m integration --durations=25
+          pytest tests/test_api_*.py -v -m integration --durations=15
+
+  intimp:
+    runs-on: python-ci
+    container:
+      image: git.fabledsword.com/bvandeusen/ci-python:3.14
+    env:
+      DB_USER: fabledcurator
+      DB_PASSWORD: ci_integration
+      DB_PORT: "5432"
+      DB_NAME: fabledcurator_test
+      SECRET_KEY: ci_integration_placeholder
+    services:
+      postgres:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_USER: fabledcurator
+          POSTGRES_PASSWORD: ci_integration
+          POSTGRES_DB: fabledcurator_test
+        options: >-
+          --health-cmd "pg_isready -U fabledcurator"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+      redis:
+        image: redis:7-alpine
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+    steps:
+      - uses: actions/checkout@v4
+      - name: Cache pip wheels
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            pip-${{ runner.os }}-py314-
+      - name: Importer integration shard (resolve service IPs, migrate, test)
+        run: |
+          set -eux
+          echo "=== container landscape (diagnostic for filter scoping) ==="
+          docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
+          echo "=== end landscape ==="
+          PG=$(docker ps --filter "name=intimp" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
+          RD=$(docker ps --filter "name=intimp" --filter "ancestor=redis:7-alpine" -q | head -n1)
+          test -n "$PG" && test -n "$RD"
+          PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
+          RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
+          test -n "$PG_IP" && test -n "$RD_IP"
+          export DB_HOST="$PG_IP"
+          export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
+          export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
+          for i in $(seq 1 60); do
+            (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
+            sleep 2
+          done
+          if command -v uv >/dev/null 2>&1; then
+            uv pip install --system -r requirements.txt pytest pytest-asyncio
+          else
+            pip install -r requirements.txt pytest pytest-asyncio
+          fi
+          alembic upgrade head
+          pytest tests/test_importer*.py tests/test_import_*.py tests/test_migration_*.py tests/test_phash_*.py tests/test_sidecar_*.py tests/test_scan_*.py tests/test_archive_extractor.py tests/test_backfill_phash.py -v -m integration --durations=15
+
+  intcore:
+    runs-on: python-ci
+    container:
+      image: git.fabledsword.com/bvandeusen/ci-python:3.14
+    env:
+      DB_USER: fabledcurator
+      DB_PASSWORD: ci_integration
+      DB_PORT: "5432"
+      DB_NAME: fabledcurator_test
+      SECRET_KEY: ci_integration_placeholder
+    services:
+      postgres:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_USER: fabledcurator
+          POSTGRES_PASSWORD: ci_integration
+          POSTGRES_DB: fabledcurator_test
+        options: >-
+          --health-cmd "pg_isready -U fabledcurator"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+      redis:
+        image: redis:7-alpine
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+    steps:
+      - uses: actions/checkout@v4
+      - name: Cache pip wheels
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-${{ runner.os }}-py314-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            pip-${{ runner.os }}-py314-
+      - name: Core integration shard (everything not api / importer / migration / phash / sidecar / scan / archive / backfill)
+        run: |
+          set -eux
+          echo "=== container landscape (diagnostic for filter scoping) ==="
+          docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
+          echo "=== end landscape ==="
+          PG=$(docker ps --filter "name=intcore" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
+          RD=$(docker ps --filter "name=intcore" --filter "ancestor=redis:7-alpine" -q | head -n1)
+          test -n "$PG" && test -n "$RD"
+          PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
+          RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
+          test -n "$PG_IP" && test -n "$RD_IP"
+          export DB_HOST="$PG_IP"
+          export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
+          export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
+          for i in $(seq 1 60); do
+            (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
+            sleep 2
+          done
+          if command -v uv >/dev/null 2>&1; then
+            uv pip install --system -r requirements.txt pytest pytest-asyncio
+          else
+            pip install -r requirements.txt pytest pytest-asyncio
+          fi
+          alembic upgrade head
+          pytest tests/ -v -m integration --durations=15 \
+            --ignore-glob='tests/test_api_*.py' \
+            --ignore-glob='tests/test_importer*.py' \
+            --ignore-glob='tests/test_import_*.py' \
+            --ignore-glob='tests/test_migration_*.py' \
+            --ignore-glob='tests/test_phash_*.py' \
+            --ignore-glob='tests/test_sidecar_*.py' \
+            --ignore-glob='tests/test_scan_*.py' \
+            --ignore-glob='tests/test_archive_extractor.py' \
+            --ignore-glob='tests/test_backfill_phash.py'
@@ -2,7 +2,14 @@ name: extension
 on:
  push:
    branches: [dev, main]
-    paths: ['extension/**']
+    # Self-retriggering: include the workflow file itself so edits to
+    # this workflow (filename-glob fixes, env tweaks, etc.) run the
+    # sign-and-publish dance on next push without needing a manual
+    # Forgejo dispatch. Side-effect commits (`ext: publish signed XPI`)
+    # only touch frontend/public/extension/ so they DON'T re-trigger.
+    paths:
+      - 'extension/**'
+      - '.forgejo/workflows/extension.yml'
  workflow_dispatch:

 jobs:
@@ -38,34 +45,52 @@ jobs:
          WEB_EXT_API_SECRET: ${{ secrets.MOZILLA_AMO_JWT_SECRET }}
      - name: Commit signed XPI to frontend/public/extension/
        run: |
-          set -e
+          set -ex
          # AMO renames signed XPIs using its internal addon-id-safe-string
-          # (e.g. 997017ca3e104e30a75a-1.0.1.xpi), NOT our gecko ID. The
-          # original 'fabledcurator-*' glob never matched and the step
-          # exited 1 even on a successful sign (operator-flagged
-          # 2026-05-25). Match any *.xpi in the artifacts dir — fresh
-          # CI runner means there's exactly one — and rename it on copy
-          # so the FC server's whitelist (backend/app/frontend.py expects
-          # 'fabledcurator-*.xpi') keeps working.
+          # (e.g. 997017ca3e104e30a75a-1.0.1.xpi), NOT our gecko ID. Match
+          # any *.xpi in the artifacts dir — fresh CI runner means there's
+          # exactly one — and rename it on copy so the FC server's
+          # whitelist (backend/app/frontend.py expects 'fabledcurator-*.xpi')
+          # keeps working.
+          # Diagnostic instrumentation 2026-05-25: a prior run (id 1731)
+          # reported success without producing an `ext: publish signed XPI`
+          # commit on main. Tracing the cwd, artifacts dir, version
+          # extraction, staging state, and push response so the next run's
+          # log explains the gap.
+          echo "=== cwd ==="
+          pwd
+          echo "=== ref / branch ==="
+          echo "ref: ${GITHUB_REF:-unset}  sha: ${GITHUB_SHA:-unset}"
+          git log --oneline -3 || true
+          echo "=== artifacts dir ==="
+          ls -la extension/web-ext-artifacts/ 2>&1 || echo "(dir missing)"
          XPI=$(ls extension/web-ext-artifacts/*.xpi 2>/dev/null | head -1)
+          echo "XPI=$XPI"
          if [ -z "$XPI" ]; then
            echo "No XPI produced by web-ext sign — exiting"
-            ls -la extension/web-ext-artifacts/ || true
            exit 1
          fi
          VERSION=$(grep -E '"version"' extension/package.json | head -1 | sed -E 's/.*"version"[[:space:]]*:[[:space:]]*"([^"]+)".*/\1/')
+          echo "VERSION=$VERSION"
          DEST="frontend/public/extension/fabledcurator-${VERSION}.xpi"
+          echo "DEST=$DEST"
          mkdir -p frontend/public/extension
          # Wipe any prior versions so the directory doesn't grow each release.
          rm -f frontend/public/extension/fabledcurator-*.xpi
          cp "$XPI" "$DEST"
          cp "$XPI" "frontend/public/extension/fabledcurator-latest.xpi"
+          echo "=== frontend/public/extension/ after copy ==="
+          ls -la frontend/public/extension/
          git config user.name  "FC extension CI"
          git config user.email "noreply@fabledsword.com"
          git add frontend/public/extension/
+          echo "=== git status after add ==="
+          git status --short
+          echo "=== diff --cached --stat ==="
+          git diff --cached --stat || true
          if git diff --cached --quiet; then
-            echo "No changes to commit"
+            echo "No changes to commit (frontend/public/extension/ matches HEAD)"
          else
            git commit -m "ext: publish signed XPI fabledcurator-${VERSION}.xpi"
-            git push origin HEAD:main
+            git push -v origin HEAD:main
          fi
@@ -71,6 +71,31 @@ def is_video(path: Path) -> bool:
    return path.suffix.lower() in VIDEO_EXTS


+def _safe_ext(path: Path) -> str:
+    """Conservatively extract a file extension for PostAttachment.ext
+    (varchar(32)).
+
+    gallery-dl produces some filenames with URL-encoded query-string
+    artifacts embedded into the basename (e.g.
+    `79507046_media_..._https___www.patreon.com_media-u_Z0FBQUFBQm5q...`).
+    `Path.suffix` finds the LAST dot and returns everything after, which
+    in those cases yields a 50+ char "extension" of mostly base64-ish
+    junk. That blows the column. Operator-flagged 2026-05-25.
+
+    Real extensions are short and alphanumeric. We accept anything ≤ 16
+    chars where every post-dot character is alphanumeric; anything else
+    means the input wasn't a real extension and we return the empty
+    string. ext is nullable-ish (empty string still satisfies NOT NULL)
+    and consumers should treat "" as "no known extension".
+    """
+    suffix = path.suffix.lower()
+    if not suffix or len(suffix) > 16:
+        return ""
+    if not all(c.isalnum() for c in suffix[1:]):
+        return ""
+    return suffix
+
+
 def _mime_for(path: Path) -> str:
    suffix = path.suffix.lower()
    image_mimes = {
@@ -209,7 +234,7 @@ class Importer:
                sha256=sha,
                path=stored,
                original_filename=source.name,
-                ext=source.suffix.lower(),
+                ext=_safe_ext(source),
                mime=_mime_for(source),
                size_bytes=source.stat().st_size,
            ))
@@ -14,7 +14,6 @@
    <AliasTable class="mt-4" />
    <BackupCard class="mt-6" />
    <TagMaintenanceCard class="mt-6" />
-    <BrowserExtensionCard class="mt-6" />
    <LegacyMigrationCard class="mt-6" />
  </div>
 </template>
@@ -27,7 +26,6 @@ import AllowlistTable from './AllowlistTable.vue'
 import AliasTable from './AliasTable.vue'
 import BackupCard from './BackupCard.vue'
 import TagMaintenanceCard from './TagMaintenanceCard.vue'
-import BrowserExtensionCard from './BrowserExtensionCard.vue'
 import LegacyMigrationCard from './LegacyMigrationCard.vue'
 </script>

@@ -31,6 +31,11 @@
          {{ system.stats.tasks.pending + system.stats.tasks.queued }} import task(s) pending.
          <v-btn variant="text" size="small" @click="tab = 'import'">Go to Import tab</v-btn>
        </v-alert>
+        <!-- Browser-extension install/download lives on Overview (moved
+             from Maintenance 2026-05-25). Overview is the discovery
+             surface for "things to set up"; Maintenance is for
+             housekeeping of already-set-up systems. -->
+        <BrowserExtensionCard class="mt-6" />
      </v-window-item>

      <v-window-item value="activity">
@@ -62,6 +67,7 @@ import { useImportStore } from '../stores/import.js'
 import SystemStatsCards from '../components/settings/SystemStatsCards.vue'
 import SystemActivitySummary from '../components/settings/SystemActivitySummary.vue'
 import SystemActivityTab from '../components/settings/SystemActivityTab.vue'
+import BrowserExtensionCard from '../components/settings/BrowserExtensionCard.vue'
 import ImportTriggerPanel from '../components/settings/ImportTriggerPanel.vue'
 import ImportFiltersForm from '../components/settings/ImportFiltersForm.vue'
 import ImportTaskList from '../components/settings/ImportTaskList.vue'
@@ -59,3 +59,25 @@ def test_json_sidecar_is_not_attached(importer, import_layout):
        select(func.count()).select_from(PostAttachment)
    ).scalar_one()
    assert n == 0
+
+
+def test_mangled_filename_extension_is_sanitized(importer, import_layout):
+    """gallery-dl sometimes URL-encodes a query string into the basename
+    (`...https___www.patreon.com_media-u_Z0F...`). Python's Path.suffix
+    returns 50+ chars of base64-ish junk for those, which blows the
+    PostAttachment.ext varchar(32) column. Operator-flagged 2026-05-25.
+    The importer should record an empty ext rather than crash."""
+    import_root, _ = import_layout
+    f = (
+        import_root / "Alice"
+        / "79507046_media_https___www.patreon.com_media-u_Z0FBQUFBQm5q"
+    )
+    f.parent.mkdir(parents=True, exist_ok=True)
+    f.write_bytes(b"binary blob")
+    r = importer.import_one(f)
+    assert r.status == "attached"
+    att = importer.session.execute(select(PostAttachment)).scalar_one()
+    # Junk "extension" -> stored as empty string (not the 50-char garbage).
+    assert att.ext == ""
+    # original_filename is Text-typed so the full name survives intact.
+    assert att.original_filename.endswith("_Z0FBQUFBQm5q")